diff --git a/.all-contributorsrc b/.all-contributorsrc new file mode 100644 index 0000000..be5ae52 --- /dev/null +++ b/.all-contributorsrc @@ -0,0 +1,283 @@ +{ + "files": [ + "README.md" + ], + "imageSize": 100, + "commit": false, + "contributors": [ + { + "login": "joohoi", + "name": "Joona Hoikkala", + "avatar_url": "https://avatars.githubusercontent.com/u/5235109?v=4", + "profile": "https://io.fi", + "contributions": [ + "doc" + ] + }, + { + "login": "jsav0", + "name": "J Savage", + "avatar_url": "https://avatars.githubusercontent.com/u/20546041?v=4", + "profile": "https://github.com/jsav0", + "contributions": [ + "infra", + "doc" + ] + }, + { + "login": "TGotwig", + "name": "Thomas Gotwig", + "avatar_url": "https://avatars.githubusercontent.com/u/30773779?v=4", + "profile": "http://www.tgotwig.dev", + "contributions": [ + "infra", + "doc" + ] + }, + { + "login": "spikecodes", + "name": "Spike", + "avatar_url": "https://avatars.githubusercontent.com/u/19519553?v=4", + "profile": "https://github.com/spikecodes", + "contributions": [ + "infra", + "doc" + ] + }, + { + "login": "evanrichter", + "name": "Evan Richter", + "avatar_url": "https://avatars.githubusercontent.com/u/330292?v=4", + "profile": "https://github.com/evanrichter", + "contributions": [ + "code", + "doc" + ] + }, + { + "login": "mzpqnxow", + "name": "AG", + "avatar_url": "https://avatars.githubusercontent.com/u/8016228?v=4", + "profile": "https://github.com/mzpqnxow", + "contributions": [ + "ideas", + "doc" + ] + }, + { + "login": "n-thumann", + "name": "Nicolas Thumann", + "avatar_url": "https://avatars.githubusercontent.com/u/46975855?v=4", + "profile": "https://n-thumann.de/", + "contributions": [ + "code", + "doc" + ] + }, + { + "login": "tomtastic", + "name": "Tom Matthews", + "avatar_url": "https://avatars.githubusercontent.com/u/302127?v=4", + "profile": "https://github.com/tomtastic", + "contributions": [ + "doc" + ] + }, + { + "login": "bsysop", + "name": "bsysop", + "avatar_url": "https://avatars.githubusercontent.com/u/9998303?v=4", + "profile": "https://github.com/bsysop", + "contributions": [ + "doc" + ] + }, + { + "login": "bpsizemore", + "name": "Brian Sizemore", + "avatar_url": "https://avatars.githubusercontent.com/u/11645898?v=4", + "profile": "http://bpsizemore.me", + "contributions": [ + "code" + ] + }, + { + "login": "noraj", + "name": "Alexandre ZANNI", + "avatar_url": "https://avatars.githubusercontent.com/u/16578570?v=4", + "profile": "https://pwn.by/noraj", + "contributions": [ + "infra", + "doc" + ] + }, + { + "login": "craig", + "name": "Craig", + "avatar_url": "https://avatars.githubusercontent.com/u/99729?v=4", + "profile": "https://github.com/craig", + "contributions": [ + "infra" + ] + }, + { + "login": "EONRaider", + "name": "EONRaider", + "avatar_url": "https://avatars.githubusercontent.com/u/15611424?v=4", + "profile": "https://www.reddit.com/u/EONRaider", + "contributions": [ + "infra" + ] + }, + { + "login": "wtwver", + "name": "wtwver", + "avatar_url": "https://avatars.githubusercontent.com/u/53866088?v=4", + "profile": "https://github.com/wtwver", + "contributions": [ + "infra" + ] + }, + { + "login": "Tib3rius", + "name": "Tib3rius", + "avatar_url": "https://avatars.githubusercontent.com/u/48113936?v=4", + "profile": "https://tib3rius.com", + "contributions": [ + "bug" + ] + }, + { + "login": "0xdf", + "name": "0xdf", + "avatar_url": "https://avatars.githubusercontent.com/u/1489045?v=4", + "profile": "https://github.com/0xdf", + "contributions": [ + "bug" + ] + }, + { + "login": "secure-77", + "name": "secure-77", + "avatar_url": "https://avatars.githubusercontent.com/u/31564517?v=4", + "profile": "http://secure77.de", + "contributions": [ + "bug" + ] + }, + { + "login": "sbrun", + "name": "Sophie Brun", + "avatar_url": "https://avatars.githubusercontent.com/u/7712154?v=4", + "profile": "https://github.com/sbrun", + "contributions": [ + "infra" + ] + }, + { + "login": "black-A", + "name": "black-A", + "avatar_url": "https://avatars.githubusercontent.com/u/30686803?v=4", + "profile": "https://github.com/black-A", + "contributions": [ + "ideas" + ] + }, + { + "login": "dinosn", + "name": "Nicolas Krassas", + "avatar_url": "https://avatars.githubusercontent.com/u/3851678?v=4", + "profile": "https://github.com/dinosn", + "contributions": [ + "ideas" + ] + }, + { + "login": "N0ur5", + "name": "N0ur5", + "avatar_url": "https://avatars.githubusercontent.com/u/24260009?v=4", + "profile": "https://github.com/N0ur5", + "contributions": [ + "ideas" + ] + }, + { + "login": "moscowchill", + "name": "mchill", + "avatar_url": "https://avatars.githubusercontent.com/u/72578879?v=4", + "profile": "https://github.com/moscowchill", + "contributions": [ + "bug" + ] + }, + { + "login": "BitThr3at", + "name": "Naman", + "avatar_url": "https://avatars.githubusercontent.com/u/45028933?v=4", + "profile": "http://BitThr3at.github.io", + "contributions": [ + "bug" + ] + }, + { + "login": "sicks3c", + "name": "Ayoub Elaich", + "avatar_url": "https://avatars.githubusercontent.com/u/32225186?v=4", + "profile": "https://github.com/Sicks3c", + "contributions": [ + "bug" + ] + }, + { + "login": "HenryHoggard", + "name": "Henry", + "avatar_url": "https://avatars.githubusercontent.com/u/1208121?v=4", + "profile": "https://github.com/HenryHoggard", + "contributions": [ + "bug" + ] + }, + { + "login": "SleepiPanda", + "name": "SleepiPanda", + "avatar_url": "https://avatars.githubusercontent.com/u/6428561?v=4", + "profile": "https://github.com/SleepiPanda", + "contributions": [ + "bug" + ] + }, + { + "login": "uBadRequest", + "name": "Bad Requests", + "avatar_url": "https://avatars.githubusercontent.com/u/47282747?v=4", + "profile": "https://github.com/uBadRequest", + "contributions": [ + "bug" + ] + }, + { + "login": "dnaka91", + "name": "Dominik Nakamura", + "avatar_url": "https://avatars.githubusercontent.com/u/36804488?v=4", + "profile": "https://home.dnaka91.rocks", + "contributions": [ + "infra" + ] + }, + { + "login": "hunter0x8", + "name": "Muhammad Ahsan", + "avatar_url": "https://avatars.githubusercontent.com/u/46222314?v=4", + "profile": "https://github.com/hunter0x8", + "contributions": [ + "bug" + ] + } + ], + "contributorsPerLine": 7, + "projectName": "feroxbuster", + "projectOwner": "epi052", + "repoType": "github", + "repoHost": "https://github.com", + "skipCi": true +} diff --git a/.github/stale.yml b/.github/stale.yml index cd6e1b8..07ec901 100644 --- a/.github/stale.yml +++ b/.github/stale.yml @@ -6,6 +6,7 @@ daysUntilClose: 7 exemptLabels: - pinned - security + - confirmed # Label to use when marking an issue as stale staleLabel: stale # Comment to post when marking an issue as stale. Set to `false` to disable diff --git a/.github/workflows/cicd-to-dockerhub.yml b/.github/workflows/cicd-to-dockerhub.yml index a7cfbf9..863c508 100644 --- a/.github/workflows/cicd-to-dockerhub.yml +++ b/.github/workflows/cicd-to-dockerhub.yml @@ -3,8 +3,6 @@ name: ci-to-dockerhub on: push: branches: [ main ] - pull_request: - branches: [ main ] jobs: build: diff --git a/.gitignore b/.gitignore index a6b9e1e..cfcc7f9 100644 --- a/.gitignore +++ b/.gitignore @@ -23,7 +23,7 @@ lcov_cobertura.py .dockerignore # state file created during tests -ferox-http* +ferox-*.state # python stuff cuz reasons Pipfile* diff --git a/Cargo.lock b/Cargo.lock index 1bd9abb..09f4307 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -596,7 +596,7 @@ dependencies = [ [[package]] name = "feroxbuster" -version = "2.3.4" +version = "2.4.0" dependencies = [ "anyhow", "assert_cmd", diff --git a/Cargo.toml b/Cargo.toml index 964590e..5f32675 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "feroxbuster" -version = "2.3.4" +version = "2.4.0" authors = ["Ben 'epi' Risher "] license = "MIT" edition = "2018" diff --git a/README.md b/README.md index 27812dd..1ce73bf 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,14 @@ + + + + +

![demo](img/demo.gif) @@ -164,3 +172,59 @@ cat targets | ./feroxbuster --stdin --silent -s 200 301 302 --redirects -x js | For realsies, there used to be over 1300 lines in this README, but it's all been moved to the [new documentation site](https://epi052.github.io/feroxbuster-docs/docs/). Go check it out!

✨🎉👉 DOCUMENTATION 👈🎉✨

+ +## Contributors ✨ + +Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/docs/en/emoji-key)): + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

Joona Hoikkala
📖
J Savage
🚇 📖
Thomas Gotwig
🚇 📖
Spike
🚇 📖
Evan Richter
💻 📖
AG
🤔 📖
Nicolas Thumann
💻 📖

Tom Matthews
📖
bsysop
📖
Brian Sizemore
💻
Alexandre ZANNI
🚇 📖
Craig
🚇
EONRaider
🚇
wtwver
🚇

Tib3rius
🐛
0xdf
🐛
secure-77
🐛
Sophie Brun
🚇
black-A
🤔
Nicolas Krassas
🤔
N0ur5
🤔

mchill
🐛
Naman
🐛
Ayoub Elaich
🐛
Henry
🐛
SleepiPanda
🐛
Bad Requests
🐛
Dominik Nakamura
🚇

Muhammad Ahsan
🐛
+ + + + + + +This project follows the [all-contributors](https://github.com/all-contributors/all-contributors) specification. Contributions of any kind welcome! \ No newline at end of file diff --git a/ferox-config.toml.example b/ferox-config.toml.example index 7bf75fd..9392c08 100644 --- a/ferox-config.toml.example +++ b/ferox-config.toml.example @@ -27,6 +27,7 @@ # output = "/targets/ellingson_mineral_company/gibson.txt" # debug_log = "/var/log/find-the-derp.log" # user_agent = "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:47.0) Gecko/20100101 Firefox/47.0" +# random_agent = false # redirects = true # insecure = true # extensions = ["php", "html"] diff --git a/shell_completions/_feroxbuster b/shell_completions/_feroxbuster index bb0a87f..3f586ce 100644 --- a/shell_completions/_feroxbuster +++ b/shell_completions/_feroxbuster @@ -72,14 +72,16 @@ _feroxbuster() { '--json[Emit JSON logs to --output and --debug-log instead of normal text]' \ '-D[Don'\''t auto-filter wildcard responses]' \ '--dont-filter[Don'\''t auto-filter wildcard responses]' \ +'-A[Use a random User-Agent]' \ +'--random-agent[Use a random User-Agent]' \ '-r[Follow redirects]' \ '--redirects[Follow redirects]' \ '-k[Disables TLS certificate validation]' \ '--insecure[Disables TLS certificate validation]' \ '-n[Do not scan recursively]' \ '--no-recursion[Do not scan recursively]' \ -'(-x --extensions)-f[Append / to each request]' \ -'(-x --extensions)--add-slash[Append / to each request]' \ +'-f[Append / to each request]' \ +'--add-slash[Append / to each request]' \ '(-u --url)--stdin[Read url(s) from STDIN]' \ '-e[Extract links from response body (html, javascript, etc...); make new requests based on findings (default: false)]' \ '--extract-links[Extract links from response body (html, javascript, etc...); make new requests based on findings (default: false)]' \ diff --git a/shell_completions/_feroxbuster.ps1 b/shell_completions/_feroxbuster.ps1 index 9968c7c..87d987e 100644 --- a/shell_completions/_feroxbuster.ps1 +++ b/shell_completions/_feroxbuster.ps1 @@ -77,6 +77,8 @@ Register-ArgumentCompleter -Native -CommandName 'feroxbuster' -ScriptBlock { [CompletionResult]::new('--json', 'json', [CompletionResultType]::ParameterName, 'Emit JSON logs to --output and --debug-log instead of normal text') [CompletionResult]::new('-D', 'D', [CompletionResultType]::ParameterName, 'Don''t auto-filter wildcard responses') [CompletionResult]::new('--dont-filter', 'dont-filter', [CompletionResultType]::ParameterName, 'Don''t auto-filter wildcard responses') + [CompletionResult]::new('-A', 'A', [CompletionResultType]::ParameterName, 'Use a random User-Agent') + [CompletionResult]::new('--random-agent', 'random-agent', [CompletionResultType]::ParameterName, 'Use a random User-Agent') [CompletionResult]::new('-r', 'r', [CompletionResultType]::ParameterName, 'Follow redirects') [CompletionResult]::new('--redirects', 'redirects', [CompletionResultType]::ParameterName, 'Follow redirects') [CompletionResult]::new('-k', 'k', [CompletionResultType]::ParameterName, 'Disables TLS certificate validation') diff --git a/shell_completions/feroxbuster.bash b/shell_completions/feroxbuster.bash index 1a23ade..5e1016e 100644 --- a/shell_completions/feroxbuster.bash +++ b/shell_completions/feroxbuster.bash @@ -20,7 +20,7 @@ _feroxbuster() { case "${cmd}" in feroxbuster) - opts=" -v -q -D -r -k -n -f -e -h -V -w -u -t -d -T -p -P -R -s -o -a -x -H -Q -S -X -W -N -C -L --verbosity --silent --quiet --auto-tune --auto-bail --json --dont-filter --redirects --insecure --no-recursion --add-slash --stdin --extract-links --help --version --wordlist --url --threads --depth --timeout --proxy --replay-proxy --replay-codes --status-codes --output --resume-from --debug-log --user-agent --extensions --dont-scan --headers --query --filter-size --filter-regex --filter-words --filter-lines --filter-status --filter-similar-to --scan-limit --parallel --rate-limit --time-limit " + opts=" -v -q -D -A -r -k -n -f -e -h -V -w -u -t -d -T -p -P -R -s -o -a -x -H -Q -S -X -W -N -C -L --verbosity --silent --quiet --auto-tune --auto-bail --json --dont-filter --random-agent --redirects --insecure --no-recursion --add-slash --stdin --extract-links --help --version --wordlist --url --threads --depth --timeout --proxy --replay-proxy --replay-codes --status-codes --output --resume-from --debug-log --user-agent --extensions --dont-scan --headers --query --filter-size --filter-regex --filter-words --filter-lines --filter-status --filter-similar-to --scan-limit --parallel --rate-limit --time-limit " if [[ ${cur} == -* || ${COMP_CWORD} -eq 1 ]] ; then COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) return 0 diff --git a/shell_completions/feroxbuster.fish b/shell_completions/feroxbuster.fish index a827fa3..5e2f837 100644 --- a/shell_completions/feroxbuster.fish +++ b/shell_completions/feroxbuster.fish @@ -32,6 +32,7 @@ complete -c feroxbuster -n "__fish_use_subcommand" -l auto-tune -d 'Automaticall complete -c feroxbuster -n "__fish_use_subcommand" -l auto-bail -d 'Automatically stop scanning when an excessive amount of errors are encountered' complete -c feroxbuster -n "__fish_use_subcommand" -l json -d 'Emit JSON logs to --output and --debug-log instead of normal text' complete -c feroxbuster -n "__fish_use_subcommand" -s D -l dont-filter -d 'Don\'t auto-filter wildcard responses' +complete -c feroxbuster -n "__fish_use_subcommand" -s A -l random-agent -d 'Use a random User-Agent' complete -c feroxbuster -n "__fish_use_subcommand" -s r -l redirects -d 'Follow redirects' complete -c feroxbuster -n "__fish_use_subcommand" -s k -l insecure -d 'Disables TLS certificate validation' complete -c feroxbuster -n "__fish_use_subcommand" -s n -l no-recursion -d 'Do not scan recursively' diff --git a/src/banner/container.rs b/src/banner/container.rs index cdada98..7acef2b 100644 --- a/src/banner/container.rs +++ b/src/banner/container.rs @@ -50,6 +50,9 @@ pub struct Banner { /// represents Configuration.user_agent user_agent: BannerEntry, + /// represents Configuration.random_agent + random_agent: BannerEntry, + /// represents Configuration.config config: BannerEntry, @@ -288,6 +291,7 @@ impl Banner { let wordlist = BannerEntry::new("📖", "Wordlist", &config.wordlist); let timeout = BannerEntry::new("💥", "Timeout (secs)", &config.timeout.to_string()); let user_agent = BannerEntry::new("🦡", "User-Agent", &config.user_agent); + let random_agent = BannerEntry::new("🦡", "User-Agent", "Random"); let extract_links = BannerEntry::new("🔎", "Extract Links", &config.extract_links.to_string()); let json = BannerEntry::new("🧔", "JSON Output", &config.json.to_string()); @@ -316,6 +320,7 @@ impl Banner { filter_status, timeout, user_agent, + random_agent, auto_bail, auto_tune, proxy, @@ -449,7 +454,12 @@ by Ben "epi" Risher {} ver: {}"#, } writeln!(&mut writer, "{}", self.timeout)?; - writeln!(&mut writer, "{}", self.user_agent)?; + + if config.random_agent { + writeln!(&mut writer, "{}", self.random_agent)?; + } else { + writeln!(&mut writer, "{}", self.user_agent)?; + } // followed by the maybe printed or variably displayed values if !config.config.is_empty() { diff --git a/src/config/container.rs b/src/config/container.rs index a3b96e0..a0ff33e 100644 --- a/src/config/container.rs +++ b/src/config/container.rs @@ -155,6 +155,10 @@ pub struct Configuration { #[serde(default = "user_agent")] pub user_agent: String, + /// Use random User-Agent + #[serde(default)] + pub random_agent: bool, + /// Follow redirects #[serde(default)] pub redirects: bool, @@ -299,6 +303,7 @@ impl Default for Configuration { redirects: false, no_recursion: false, extract_links: false, + random_agent: false, save_state: true, proxy: String::new(), config: String::new(), @@ -349,6 +354,7 @@ impl Configuration { /// - **auto_bail**: `false` /// - **save_state**: `true` /// - **user_agent**: `feroxbuster/VERSION` + /// - **random_agent**: `false` /// - **insecure**: `false` (don't be insecure, i.e. don't allow invalid certs) /// - **extensions**: `None` /// - **url_denylist**: `None` @@ -693,6 +699,10 @@ impl Configuration { update_config_if_present!(&mut config.user_agent, args, "user_agent", String); update_config_if_present!(&mut config.timeout, args, "timeout", u64); + if args.is_present("random_agent") { + config.random_agent = true; + } + if args.is_present("redirects") { config.redirects = true; } @@ -874,6 +884,7 @@ impl Configuration { update_if_not_default!(&mut conf.timeout, new.timeout, timeout()); update_if_not_default!(&mut conf.user_agent, new.user_agent, user_agent()); + update_if_not_default!(&mut conf.random_agent, new.random_agent, false); update_if_not_default!(&mut conf.threads, new.threads, threads()); update_if_not_default!(&mut conf.depth, new.depth, depth()); update_if_not_default!(&mut conf.wordlist, new.wordlist, wordlist()); diff --git a/src/config/tests.rs b/src/config/tests.rs index fe6f104..79d35cd 100644 --- a/src/config/tests.rs +++ b/src/config/tests.rs @@ -84,6 +84,7 @@ fn default_configuration() { assert!(!config.auto_bail); assert_eq!(config.requester_policy, RequesterPolicy::Default); assert!(!config.no_recursion); + assert!(!config.random_agent); assert!(!config.json); assert!(config.save_state); assert!(!config.stdin); @@ -400,6 +401,12 @@ fn config_reads_queries() { assert_eq!(config.queries, queries); } +#[test] +fn config_default_not_random_agent() { + let config = setup_config_test(); + assert!(!config.random_agent); +} + #[test] #[should_panic] /// test that an error message is printed and panic is called when report_and_exit is called diff --git a/src/event_handlers/outputs.rs b/src/event_handlers/outputs.rs index d94cb90..7d70b7e 100644 --- a/src/event_handlers/outputs.rs +++ b/src/event_handlers/outputs.rs @@ -215,6 +215,7 @@ impl TermOutHandler { self.config.replay_client.as_ref().unwrap(), resp.url(), self.config.output_level, + &self.config, tx_stats.clone(), ) .await diff --git a/src/extractor/container.rs b/src/extractor/container.rs index 9dae754..24392cd 100644 --- a/src/extractor/container.rs +++ b/src/extractor/container.rs @@ -404,6 +404,7 @@ impl<'a> Extractor<'a> { &client, &url, self.handles.config.output_level, + &self.handles.config, self.handles.stats.tx.clone(), ) .await?; diff --git a/src/extractor/tests.rs b/src/extractor/tests.rs index 477eb01..94b4d2f 100644 --- a/src/extractor/tests.rs +++ b/src/extractor/tests.rs @@ -211,16 +211,23 @@ async fn extractor_get_links_with_absolute_url_that_differs_from_target_domain() let mock = srv.mock(|when, then| { when.method(GET).path("/some-path"); then.status(200).body( - "\"http://defintely.not.a.thing.probably.com/homepage/assets/img/icons/handshake.svg\"", + "\"http://definitely.not.a.thing.probably.com/homepage/assets/img/icons/handshake.svg\"", ); }); let client = Client::new(); let url = Url::parse(&srv.url("/some-path")).unwrap(); + let config = Configuration::new().unwrap(); - let response = make_request(&client, &url, OutputLevel::Default, tx_stats.clone()) - .await - .unwrap(); + let response = make_request( + &client, + &url, + OutputLevel::Default, + &config, + tx_stats.clone(), + ) + .await + .unwrap(); let (handles, _rx) = Handles::for_testing(None, None); let handles = Arc::new(handles); diff --git a/src/heuristics.rs b/src/heuristics.rs index 026e699..ec7b5c0 100644 --- a/src/heuristics.rs +++ b/src/heuristics.rs @@ -156,7 +156,15 @@ impl HeuristicTests { log::trace!("enter: make_wildcard_request({}, {})", target, length); let unique_str = self.unique_string(length); - let nonexistent_url = target.format(&unique_str, None)?; + + // To take care of slash when needed + let slash = if self.handles.config.add_slash { + Some("/") + } else { + None + }; + + let nonexistent_url = target.format(&unique_str, slash)?; let response = logged_request(&nonexistent_url.to_owned(), self.handles.clone()).await?; diff --git a/src/lib.rs b/src/lib.rs index c187471..fc8ead6 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -91,6 +91,21 @@ pub const DEFAULT_STATUS_CODES: [StatusCode; 10] = [ /// /// Expected location is in the same directory as the feroxbuster binary. pub const DEFAULT_CONFIG_NAME: &str = "ferox-config.toml"; +/// User agents to select from when random agent is being used +pub const USER_AGENTS: [&str; 12] = [ + "Mozilla/5.0 (Linux; Android 8.0.0; SM-G960F Build/R16NW) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.84 Mobile Safari/537.36", + "Mozilla/5.0 (iPhone; CPU iPhone OS 12_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.0 Mobile/15E148 Safari/604.1", + "Mozilla/5.0 (Windows Phone 10.0; Android 6.0.1; Microsoft; RM-1152) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Mobile Safari/537.36 Edge/15.15254", + "Mozilla/5.0 (Linux; Android 7.0; Pixel C Build/NRD90M; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/52.0.2743.98 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.246", + "Mozilla/5.0 (X11; CrOS x86_64 8172.45.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.64 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_2) AppleWebKit/601.3.9 (KHTML, like Gecko) Version/9.0.2 Safari/601.3.9", + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.111 Safari/537.36", + "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:15.0) Gecko/20100101 Firefox/15.0.1", + "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", + "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)", + "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)", +]; #[cfg(test)] mod tests { diff --git a/src/parser.rs b/src/parser.rs index 00ae712..8b56ede 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -192,6 +192,15 @@ pub fn initialize() -> App<'static, 'static> { "Sets the User-Agent (default: feroxbuster/VERSION)" ), ) + .arg( + Arg::with_name("random_agent") + .short("A") + .long("random-agent") + .takes_value(false) + .help( + "Use a random User-Agent" + ), + ) .arg( Arg::with_name("redirects") .short("r") @@ -265,7 +274,6 @@ pub fn initialize() -> App<'static, 'static> { .short("f") .long("add-slash") .takes_value(false) - .conflicts_with("extensions") .help("Append / to each request") ) .arg( @@ -458,7 +466,7 @@ mod tests { use super::*; #[test] - /// initalize parser, expect a clap::App returned + /// initialize parser, expect a clap::App returned fn parser_initialize_gives_defaults() { let app = initialize(); assert_eq!(app.get_name(), "feroxbuster"); diff --git a/src/scan_manager/tests.rs b/src/scan_manager/tests.rs index d59f278..613e94e 100644 --- a/src/scan_manager/tests.rs +++ b/src/scan_manager/tests.rs @@ -378,12 +378,79 @@ fn feroxstates_feroxserialize_implementation() { assert!(expected_strs.eval(&ferox_state.as_str())); let json_state = ferox_state.as_json().unwrap(); - let expected = format!( - r#"{{"scans":[{{"id":"{}","url":"https://spiritanimal.com","scan_type":"Directory","status":"NotStarted","num_requests":0}}],"config":{{"type":"configuration","wordlist":"/usr/share/seclists/Discovery/Web-Content/raft-medium-directories.txt","config":"","proxy":"","replay_proxy":"","target_url":"","status_codes":[200,204,301,302,307,308,401,403,405,500],"replay_codes":[200,204,301,302,307,308,401,403,405,500],"filter_status":[],"threads":50,"timeout":7,"verbosity":0,"silent":false,"quiet":false,"auto_bail":false,"auto_tune":false,"json":false,"output":"","debug_log":"","user_agent":"feroxbuster/{}","redirects":false,"insecure":false,"extensions":[],"headers":{{}},"queries":[],"no_recursion":false,"extract_links":false,"add_slash":false,"stdin":false,"depth":4,"scan_limit":0,"parallel":0,"rate_limit":0,"filter_size":[],"filter_line_count":[],"filter_word_count":[],"filter_regex":[],"dont_filter":false,"resumed":false,"resume_from":"","save_state":false,"time_limit":"","filter_similar":[],"url_denylist":[],"regex_denylist":[]}},"responses":[{{"type":"response","url":"https://nerdcore.com/css","path":"/css","wildcard":true,"status":301,"content_length":173,"line_count":10,"word_count":16,"headers":{{"server":"nginx/1.16.1"}}}}]"#, - saved_id, VERSION - ); - println!("{}\n{}", expected, json_state); - assert!(predicates::str::contains(expected).eval(&json_state)); + for expected in [ + r#""scans""#, + &format!(r#""id":"{}""#, saved_id), + r#""url":"https://spiritanimal.com""#, + r#""scan_type":"Directory""#, + r#""status":"NotStarted""#, + r#""num_requests":0"#, + r#""config""#, + r#""type":"configuration""#, + r#""wordlist":"/usr/share/seclists/Discovery/Web-Content/raft-medium-directories.txt""#, + r#""config""#, + r#""proxy":"""#, + r#""replay_proxy":"""#, + r#""target_url":"""#, + r#""status_codes":[200,204,301,302,307,308,401,403,405,500]"#, + r#""replay_codes":[200,204,301,302,307,308,401,403,405,500]"#, + r#""filter_status":[]"#, + r#""threads":50"#, + r#""timeout":7"#, + r#""verbosity":0"#, + r#""silent":false"#, + r#""quiet":false"#, + r#""auto_bail":false"#, + r#""auto_tune":false"#, + r#""json":false"#, + r#""output":"""#, + r#""debug_log":"""#, + &format!(r#""user_agent":"feroxbuster/{}""#, VERSION), + r#""random_agent":false"#, + r#""redirects":false"#, + r#""insecure":false"#, + r#""extensions":[]"#, + r#""headers""#, + r#""queries":[]"#, + r#""no_recursion":false"#, + r#""extract_links":false"#, + r#""add_slash":false"#, + r#""stdin":false"#, + r#""depth":4"#, + r#""scan_limit":0"#, + r#""parallel":0"#, + r#""rate_limit":0"#, + r#""filter_size":[]"#, + r#""filter_line_count":[]"#, + r#""filter_word_count":[]"#, + r#""filter_regex":[]"#, + r#""dont_filter":false"#, + r#""resumed":false"#, + r#""resume_from":"""#, + r#""save_state":false"#, + r#""time_limit":"""#, + r#""filter_similar":[]"#, + r#""url_denylist":[]"#, + r#""responses""#, + r#""type":"response""#, + r#""url":"https://nerdcore.com/css""#, + r#""path":"/css""#, + r#""wildcard":true"#, + r#""status":301"#, + r#""content_length":173"#, + r#""line_count":10"#, + r#""word_count":16"#, + r#""headers""#, + r#""server":"nginx/1.16.1"#, + ] + .iter() + { + assert!( + predicates::str::contains(*expected).eval(&json_state), + "{}", + expected + ) + } } #[should_panic] diff --git a/src/url.rs b/src/url.rs index d771fb6..aea222f 100644 --- a/src/url.rs +++ b/src/url.rs @@ -42,7 +42,13 @@ impl FeroxUrl { let mut urls = vec![]; - match self.format(word, None) { + let slash = if self.handles.config.add_slash { + Some("/") + } else { + None + }; + + match self.format(word, slash) { // default request, i.e. no extension Ok(url) => urls.push(url), Err(_) => self.handles.stats.send(AddError(UrlFormat))?, @@ -55,7 +61,6 @@ impl FeroxUrl { Err(_) => self.handles.stats.send(AddError(UrlFormat))?, } } - log::trace!("exit: formatted_urls -> {:?}", urls); Ok(urls) } @@ -97,13 +102,25 @@ impl FeroxUrl { self.target.to_string() }; - // extensions and slashes are mutually exclusive cases - let word = if extension.is_some() { - format!("{}.{}", word, extension.unwrap()) - } else if self.handles.config.add_slash && !word.ends_with('/') { - // -f used, and word doesn't already end with a / - format!("{}/", word) - } else if word.starts_with("//") { + // As of version 2.3.4, extensions and trailing slashes are no longer mutually exclusive. + // Trailing slashes are now treated as just another extension, which is pretty clever. + // + // In addition to the change above, @cortantief ID'd a bug here that incorrectly handled + // 2 leading forward slashes when extensions were used. This block addresses the bugfix. + let mut word = if let Some(ext) = extension { + // We handle the special case of forward slash + // That allow us to treat it as an extension with a particular format + if ext == "/" { + format!("{}/", word) + } else { + format!("{}.{}", word, ext) + } + } else { + String::from(word) + }; + + // We check separately if the current word begins with 2 forward slashes + if word.starts_with("//") { // bug ID'd by @Sicks3c, when a wordlist contains words that begin with 2 forward slashes // i.e. //1_40_0/static/js, it gets joined onto the base url in a surprising way // ex: https://localhost/ + //1_40_0/static/js -> https://1_40_0/static/js @@ -111,9 +128,7 @@ impl FeroxUrl { // and simply removes prefixed forward slashes if there are two of them. Additionally, // trim_start_matches will trim the pattern until it's gone, so even if there are more than // 2 /'s, they'll still be trimmed - word.trim_start_matches('/').to_string() - } else { - String::from(word) + word = word.trim_start_matches('/').to_string(); }; let base_url = Url::parse(&url)?; @@ -451,6 +466,20 @@ mod tests { ); } + #[test] + /// word with two prepended slashes and extensions doesn't discard the entire domain + fn format_url_word_with_two_prepended_slashes_and_extensions() { + let handles = Arc::new(Handles::for_testing(None, None).0); + let url = FeroxUrl::from_string("http://localhost", handles); + for ext in ["rocks", "fun"] { + let to_check = format!("http://localhost/upload/ferox.{}", ext); + assert_eq!( + url.format("//upload/ferox", Some(ext)).unwrap(), + reqwest::Url::parse(&to_check[..]).unwrap() + ); + } + } + #[test] /// word that is a fully formed url, should return an error fn format_url_word_that_is_a_url() { @@ -460,4 +489,33 @@ mod tests { assert!(formatted.is_err()); } + + #[test] + /// sending url + word with both an extension and add-slash should get back + /// two urls, one with '/' appended to the word, and the other with the extension + /// appended + fn formatted_urls_with_postslash_and_extensions() { + let config = Configuration { + add_slash: true, + extensions: vec!["rocks".to_string(), "fun".to_string()], + ..Default::default() + }; + let handles = Arc::new(Handles::for_testing(None, Some(Arc::new(config))).0); + let url = FeroxUrl::from_string("http://localhost", handles); + match url.formatted_urls("ferox") { + Ok(urls) => { + // 3 = One for the main word + slash and for the two extensions + assert_eq!(urls.len(), 3); + assert_eq!( + urls, + [ + Url::parse("http://localhost/ferox/").unwrap(), + Url::parse("http://localhost/ferox.rocks").unwrap(), + Url::parse("http://localhost/ferox.fun").unwrap(), + ] + ) + } + Err(err) => panic!("{}", err.to_string()), + } + } } diff --git a/src/utils.rs b/src/utils.rs index 82c2b17..07d787d 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -14,6 +14,7 @@ use std::{ }; use tokio::sync::mpsc::UnboundedSender; +use crate::config::Configuration; use crate::{ config::OutputLevel, event_handlers::{ @@ -24,8 +25,12 @@ use crate::{ send_command, statistics::StatError::{Connection, Other, Redirection, Request, Timeout}, traits::FeroxSerialize, + USER_AGENTS, }; +/// simple counter for grabbing 'random' user agents +static mut USER_AGENT_CTR: usize = 0; + /// Given the path to a file, open the file in append mode (create it if it doesn't exist) and /// return a reference to the buffered file pub fn open_file(filename: &str) -> Result> { @@ -95,7 +100,7 @@ pub async fn logged_request(url: &Url, handles: Arc) -> Result, ) -> Result { log::trace!( @@ -131,7 +137,20 @@ pub async fn make_request( tx_stats ); - match client.get(url.to_owned()).send().await { + let mut request = client.get(url.to_owned()); + + if config.random_agent { + let index = unsafe { + USER_AGENT_CTR += 1; + USER_AGENT_CTR % USER_AGENTS.len() + }; + + let user_agent = USER_AGENTS[index]; + + request = request.header("User-Agent", user_agent); + } + + match request.send().await { Err(e) => { log::trace!("exit: make_request -> {}", e); diff --git a/tests/test_banner.rs b/tests/test_banner.rs index 21c01ff..410da62 100644 --- a/tests/test_banner.rs +++ b/tests/test_banner.rs @@ -147,6 +147,31 @@ fn banner_prints_denied_urls() { ); } +#[test] +/// test allows non-existent wordlist to trigger the banner printing to stderr +/// expect to see all mandatory prints + multiple headers +fn banner_prints_random_agent() { + Command::cargo_bin("feroxbuster") + .unwrap() + .arg("--url") + .arg("http://localhost") + .arg("--random-agent") + .assert() + .success() + .stderr( + predicate::str::contains("─┬─") + .and(predicate::str::contains("Target Url")) + .and(predicate::str::contains("http://localhost")) + .and(predicate::str::contains("Threads")) + .and(predicate::str::contains("Wordlist")) + .and(predicate::str::contains("Status Codes")) + .and(predicate::str::contains("Timeout (secs)")) + .and(predicate::str::contains("User-Agent")) + .and(predicate::str::contains("Random")) + .and(predicate::str::contains("─┴─")), + ); +} + #[test] /// test allows non-existent wordlist to trigger the banner printing to stderr /// expect to see all mandatory prints + multiple size filters