Merge pull request #357 from dsaxton/random-agent

Implement random user agent flag

.gitignore

@@ -23,7 +23,7 @@ lcov_cobertura.py
 .dockerignore
 
 # state file created during tests
-ferox-http*
+ferox-*.state
 
 # python stuff cuz reasons
 Pipfile*

Cargo.lock

@@ -596,7 +596,7 @@ dependencies = [
 
 [[package]]
 name = "feroxbuster"
-version = "2.3.3"
+version = "2.4.0"
 dependencies = [
  "anyhow",
  "assert_cmd",

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "feroxbuster"
-version = "2.3.3"
+version = "2.4.0"
 authors = ["Ben 'epi' Risher <epibar052@gmail.com>"]
 license = "MIT"
 edition = "2018"

ferox-config.toml.example

@@ -27,6 +27,7 @@
 # output = "/targets/ellingson_mineral_company/gibson.txt"
 # debug_log = "/var/log/find-the-derp.log"
 # user_agent = "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:47.0) Gecko/20100101 Firefox/47.0"
+# random_agent = false
 # redirects = true
 # insecure = true
 # extensions = ["php", "html"]

shell_completions/_feroxbuster

@@ -72,6 +72,8 @@ _feroxbuster() {
 '--json[Emit JSON logs to --output and --debug-log instead of normal text]' \
 '-D[Don'\''t auto-filter wildcard responses]' \
 '--dont-filter[Don'\''t auto-filter wildcard responses]' \
+'-A[Use a random User-Agent]' \
+'--random-agent[Use a random User-Agent]' \
 '-r[Follow redirects]' \
 '--redirects[Follow redirects]' \
 '-k[Disables TLS certificate validation]' \

shell_completions/_feroxbuster.ps1

@@ -77,6 +77,8 @@ Register-ArgumentCompleter -Native -CommandName 'feroxbuster' -ScriptBlock {
             [CompletionResult]::new('--json', 'json', [CompletionResultType]::ParameterName, 'Emit JSON logs to --output and --debug-log instead of normal text')
             [CompletionResult]::new('-D', 'D', [CompletionResultType]::ParameterName, 'Don''t auto-filter wildcard responses')
             [CompletionResult]::new('--dont-filter', 'dont-filter', [CompletionResultType]::ParameterName, 'Don''t auto-filter wildcard responses')
+            [CompletionResult]::new('-A', 'A', [CompletionResultType]::ParameterName, 'Use a random User-Agent')
+            [CompletionResult]::new('--random-agent', 'random-agent', [CompletionResultType]::ParameterName, 'Use a random User-Agent')
             [CompletionResult]::new('-r', 'r', [CompletionResultType]::ParameterName, 'Follow redirects')
             [CompletionResult]::new('--redirects', 'redirects', [CompletionResultType]::ParameterName, 'Follow redirects')
             [CompletionResult]::new('-k', 'k', [CompletionResultType]::ParameterName, 'Disables TLS certificate validation')

shell_completions/feroxbuster.bash

@@ -20,7 +20,7 @@ _feroxbuster() {
 
     case "${cmd}" in
         feroxbuster)
-            opts=" -v -q -D -r -k -n -f -e -h -V -w -u -t -d -T -p -P -R -s -o -a -x -H -Q -S -X -W -N -C -L  --verbosity --silent --quiet --auto-tune --auto-bail --json --dont-filter --redirects --insecure --no-recursion --add-slash --stdin --extract-links --help --version --wordlist --url --threads --depth --timeout --proxy --replay-proxy --replay-codes --status-codes --output --resume-from --debug-log --user-agent --extensions --dont-scan --headers --query --filter-size --filter-regex --filter-words --filter-lines --filter-status --filter-similar-to --scan-limit --parallel --rate-limit --time-limit  "
+            opts=" -v -q -D -A -r -k -n -f -e -h -V -w -u -t -d -T -p -P -R -s -o -a -x -H -Q -S -X -W -N -C -L  --verbosity --silent --quiet --auto-tune --auto-bail --json --dont-filter --random-agent --redirects --insecure --no-recursion --add-slash --stdin --extract-links --help --version --wordlist --url --threads --depth --timeout --proxy --replay-proxy --replay-codes --status-codes --output --resume-from --debug-log --user-agent --extensions --dont-scan --headers --query --filter-size --filter-regex --filter-words --filter-lines --filter-status --filter-similar-to --scan-limit --parallel --rate-limit --time-limit  "
             if [[ ${cur} == -* || ${COMP_CWORD} -eq 1 ]] ; then
                 COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
                 return 0

shell_completions/feroxbuster.fish

@@ -32,6 +32,7 @@ complete -c feroxbuster -n "__fish_use_subcommand" -l auto-tune -d 'Automaticall
 complete -c feroxbuster -n "__fish_use_subcommand" -l auto-bail -d 'Automatically stop scanning when an excessive amount of errors are encountered'
 complete -c feroxbuster -n "__fish_use_subcommand" -l json -d 'Emit JSON logs to --output and --debug-log instead of normal text'
 complete -c feroxbuster -n "__fish_use_subcommand" -s D -l dont-filter -d 'Don\'t auto-filter wildcard responses'
+complete -c feroxbuster -n "__fish_use_subcommand" -s A -l random-agent -d 'Use a random User-Agent'
 complete -c feroxbuster -n "__fish_use_subcommand" -s r -l redirects -d 'Follow redirects'
 complete -c feroxbuster -n "__fish_use_subcommand" -s k -l insecure -d 'Disables TLS certificate validation'
 complete -c feroxbuster -n "__fish_use_subcommand" -s n -l no-recursion -d 'Do not scan recursively'

src/banner/container.rs

@@ -50,6 +50,9 @@ pub struct Banner {
     /// represents Configuration.user_agent
     user_agent: BannerEntry,
 
+    /// represents Configuration.random_agent
+    random_agent: BannerEntry,
+
     /// represents Configuration.config
     config: BannerEntry,
 
@@ -276,6 +279,7 @@ impl Banner {
         let wordlist = BannerEntry::new("📖", "Wordlist", &config.wordlist);
         let timeout = BannerEntry::new("💥", "Timeout (secs)", &config.timeout.to_string());
         let user_agent = BannerEntry::new("🦡", "User-Agent", &config.user_agent);
+        let random_agent = BannerEntry::new("🦡", "User-Agent", "Random");
         let extract_links =
             BannerEntry::new("🔎", "Extract Links", &config.extract_links.to_string());
         let json = BannerEntry::new("🧔", "JSON Output", &config.json.to_string());
@@ -304,6 +308,7 @@ impl Banner {
             filter_status,
             timeout,
             user_agent,
+            random_agent,
             auto_bail,
             auto_tune,
             proxy,
@@ -437,7 +442,12 @@ by Ben "epi" Risher {}                 ver: {}"#,
         }
 
         writeln!(&mut writer, "{}", self.timeout)?;
-        writeln!(&mut writer, "{}", self.user_agent)?;
+
+        if config.random_agent {
+            writeln!(&mut writer, "{}", self.random_agent)?;
+        } else {
+            writeln!(&mut writer, "{}", self.user_agent)?;
+        }
 
         // followed by the maybe printed or variably displayed values
         if !config.config.is_empty() {

src/config/container.rs

@@ -154,6 +154,10 @@ pub struct Configuration {
     #[serde(default = "user_agent")]
     pub user_agent: String,
 
+    /// Use random User-Agent
+    #[serde(default)]
+    pub random_agent: bool,
+
     /// Follow redirects
     #[serde(default)]
     pub redirects: bool,
@@ -295,6 +299,7 @@ impl Default for Configuration {
             redirects: false,
             no_recursion: false,
             extract_links: false,
+            random_agent: false,
             save_state: true,
             proxy: String::new(),
             config: String::new(),
@@ -344,6 +349,7 @@ impl Configuration {
     /// - **auto_bail**: `false`
     /// - **save_state**: `true`
     /// - **user_agent**: `feroxbuster/VERSION`
+    /// - **random_agent**: `false`
     /// - **insecure**: `false` (don't be insecure, i.e. don't allow invalid certs)
     /// - **extensions**: `None`
     /// - **url_denylist**: `None`
@@ -647,6 +653,10 @@ impl Configuration {
         update_config_if_present!(&mut config.user_agent, args, "user_agent", String);
         update_config_if_present!(&mut config.timeout, args, "timeout", u64);
 
+        if args.is_present("random_agent") {
+            config.random_agent = true;
+        }
+
         if args.is_present("redirects") {
             config.redirects = true;
         }
@@ -824,6 +834,7 @@ impl Configuration {
 
         update_if_not_default!(&mut conf.timeout, new.timeout, timeout());
         update_if_not_default!(&mut conf.user_agent, new.user_agent, user_agent());
+        update_if_not_default!(&mut conf.random_agent, new.random_agent, false);
         update_if_not_default!(&mut conf.threads, new.threads, threads());
         update_if_not_default!(&mut conf.depth, new.depth, depth());
         update_if_not_default!(&mut conf.wordlist, new.wordlist, wordlist());

src/config/tests.rs

@@ -81,6 +81,7 @@ fn default_configuration() {
     assert!(!config.auto_bail);
     assert_eq!(config.requester_policy, RequesterPolicy::Default);
     assert!(!config.no_recursion);
+    assert!(!config.random_agent);
     assert!(!config.json);
     assert!(config.save_state);
     assert!(!config.stdin);
@@ -383,6 +384,12 @@ fn config_reads_queries() {
     assert_eq!(config.queries, queries);
 }
 
+#[test]
+fn config_default_not_random_agent() {
+    let config = setup_config_test();
+    assert!(!config.random_agent);
+}
+
 #[test]
 #[should_panic]
 /// test that an error message is printed and panic is called when report_and_exit is called

src/event_handlers/outputs.rs

@@ -215,6 +215,7 @@ impl TermOutHandler {
                             self.config.replay_client.as_ref().unwrap(),
                             resp.url(),
                             self.config.output_level,
+                            &self.config,
                             tx_stats.clone(),
                         )
                         .await

src/extractor/container.rs

@@ -402,6 +402,7 @@ impl<'a> Extractor<'a> {
             &client,
             &url,
             self.handles.config.output_level,
+            &self.handles.config,
             self.handles.stats.tx.clone(),
         )
         .await?;

src/extractor/tests.rs

@@ -211,16 +211,23 @@ async fn extractor_get_links_with_absolute_url_that_differs_from_target_domain()
     let mock = srv.mock(|when, then| {
         when.method(GET).path("/some-path");
         then.status(200).body(
-            "\"http://defintely.not.a.thing.probably.com/homepage/assets/img/icons/handshake.svg\"",
+            "\"http://definitely.not.a.thing.probably.com/homepage/assets/img/icons/handshake.svg\"",
         );
     });
 
     let client = Client::new();
     let url = Url::parse(&srv.url("/some-path")).unwrap();
+    let config = Configuration::new().unwrap();
 
-    let response = make_request(&client, &url, OutputLevel::Default, tx_stats.clone())
-        .await
-        .unwrap();
+    let response = make_request(
+        &client,
+        &url,
+        OutputLevel::Default,
+        &config,
+        tx_stats.clone(),
+    )
+    .await
+    .unwrap();
     let (handles, _rx) = Handles::for_testing(None, None);
 
     let handles = Arc::new(handles);

src/lib.rs

@@ -91,6 +91,21 @@ pub const DEFAULT_STATUS_CODES: [StatusCode; 10] = [
 ///
 /// Expected location is in the same directory as the feroxbuster binary.
 pub const DEFAULT_CONFIG_NAME: &str = "ferox-config.toml";
+/// User agents to select from when random agent is being used
+pub const USER_AGENTS: [&str; 12] = [
+    "Mozilla/5.0 (Linux; Android 8.0.0; SM-G960F Build/R16NW) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.84 Mobile Safari/537.36",
+    "Mozilla/5.0 (iPhone; CPU iPhone OS 12_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.0 Mobile/15E148 Safari/604.1",
+    "Mozilla/5.0 (Windows Phone 10.0; Android 6.0.1; Microsoft; RM-1152) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Mobile Safari/537.36 Edge/15.15254",
+    "Mozilla/5.0 (Linux; Android 7.0; Pixel C Build/NRD90M; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/52.0.2743.98 Safari/537.36",
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.246",
+    "Mozilla/5.0 (X11; CrOS x86_64 8172.45.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.64 Safari/537.36",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_2) AppleWebKit/601.3.9 (KHTML, like Gecko) Version/9.0.2 Safari/601.3.9",
+    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.111 Safari/537.36",
+    "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:15.0) Gecko/20100101 Firefox/15.0.1",
+    "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
+    "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)",
+    "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)",
+];
 
 #[cfg(test)]
 mod tests {

src/parser.rs

@@ -192,6 +192,15 @@ pub fn initialize() -> App<'static, 'static> {
                     "Sets the User-Agent (default: feroxbuster/VERSION)"
                 ),
         )
+        .arg(
+            Arg::with_name("random_agent")
+                .short("A")
+                .long("random-agent")
+                .takes_value(false)
+                .help(
+                    "Use a random User-Agent"
+                ),
+        )
         .arg(
             Arg::with_name("redirects")
                 .short("r")

src/scan_manager/tests.rs

@@ -378,12 +378,79 @@ fn feroxstates_feroxserialize_implementation() {
     assert!(expected_strs.eval(&ferox_state.as_str()));
 
     let json_state = ferox_state.as_json().unwrap();
-    let expected = format!(
-        r#"{{"scans":[{{"id":"{}","url":"https://spiritanimal.com","scan_type":"Directory","status":"NotStarted","num_requests":0}}],"config":{{"type":"configuration","wordlist":"/usr/share/seclists/Discovery/Web-Content/raft-medium-directories.txt","config":"","proxy":"","replay_proxy":"","target_url":"","status_codes":[200,204,301,302,307,308,401,403,405,500],"replay_codes":[200,204,301,302,307,308,401,403,405,500],"filter_status":[],"threads":50,"timeout":7,"verbosity":0,"silent":false,"quiet":false,"auto_bail":false,"auto_tune":false,"json":false,"output":"","debug_log":"","user_agent":"feroxbuster/{}","redirects":false,"insecure":false,"extensions":[],"headers":{{}},"queries":[],"no_recursion":false,"extract_links":false,"add_slash":false,"stdin":false,"depth":4,"scan_limit":0,"parallel":0,"rate_limit":0,"filter_size":[],"filter_line_count":[],"filter_word_count":[],"filter_regex":[],"dont_filter":false,"resumed":false,"resume_from":"","save_state":false,"time_limit":"","filter_similar":[],"url_denylist":[]}},"responses":[{{"type":"response","url":"https://nerdcore.com/css","path":"/css","wildcard":true,"status":301,"content_length":173,"line_count":10,"word_count":16,"headers":{{"server":"nginx/1.16.1"}}}}]"#,
-        saved_id, VERSION
-    );
-    println!("{}\n{}", expected, json_state);
-    assert!(predicates::str::contains(expected).eval(&json_state));
+    for expected in [
+        r#""scans""#,
+        &format!(r#""id":"{}""#, saved_id),
+        r#""url":"https://spiritanimal.com""#,
+        r#""scan_type":"Directory""#,
+        r#""status":"NotStarted""#,
+        r#""num_requests":0"#,
+        r#""config""#,
+        r#""type":"configuration""#,
+        r#""wordlist":"/usr/share/seclists/Discovery/Web-Content/raft-medium-directories.txt""#,
+        r#""config""#,
+        r#""proxy":"""#,
+        r#""replay_proxy":"""#,
+        r#""target_url":"""#,
+        r#""status_codes":[200,204,301,302,307,308,401,403,405,500]"#,
+        r#""replay_codes":[200,204,301,302,307,308,401,403,405,500]"#,
+        r#""filter_status":[]"#,
+        r#""threads":50"#,
+        r#""timeout":7"#,
+        r#""verbosity":0"#,
+        r#""silent":false"#,
+        r#""quiet":false"#,
+        r#""auto_bail":false"#,
+        r#""auto_tune":false"#,
+        r#""json":false"#,
+        r#""output":"""#,
+        r#""debug_log":"""#,
+        &format!(r#""user_agent":"feroxbuster/{}""#, VERSION),
+        r#""random_agent":false"#,
+        r#""redirects":false"#,
+        r#""insecure":false"#,
+        r#""extensions":[]"#,
+        r#""headers""#,
+        r#""queries":[]"#,
+        r#""no_recursion":false"#,
+        r#""extract_links":false"#,
+        r#""add_slash":false"#,
+        r#""stdin":false"#,
+        r#""depth":4"#,
+        r#""scan_limit":0"#,
+        r#""parallel":0"#,
+        r#""rate_limit":0"#,
+        r#""filter_size":[]"#,
+        r#""filter_line_count":[]"#,
+        r#""filter_word_count":[]"#,
+        r#""filter_regex":[]"#,
+        r#""dont_filter":false"#,
+        r#""resumed":false"#,
+        r#""resume_from":"""#,
+        r#""save_state":false"#,
+        r#""time_limit":"""#,
+        r#""filter_similar":[]"#,
+        r#""url_denylist":[]"#,
+        r#""responses""#,
+        r#""type":"response""#,
+        r#""url":"https://nerdcore.com/css""#,
+        r#""path":"/css""#,
+        r#""wildcard":true"#,
+        r#""status":301"#,
+        r#""content_length":173"#,
+        r#""line_count":10"#,
+        r#""word_count":16"#,
+        r#""headers""#,
+        r#""server":"nginx/1.16.1"#,
+    ]
+    .iter()
+    {
+        assert!(
+            predicates::str::contains(*expected).eval(&json_state),
+            "{}",
+            expected
+        )
+    }
 }
 
 #[should_panic]

src/utils.rs

@@ -13,6 +13,7 @@ use std::{
 };
 use tokio::sync::mpsc::UnboundedSender;
 
+use crate::config::Configuration;
 use crate::{
     config::OutputLevel,
     event_handlers::{
@@ -23,8 +24,12 @@ use crate::{
     send_command,
     statistics::StatError::{Connection, Other, Redirection, Request, Timeout},
     traits::FeroxSerialize,
+    USER_AGENTS,
 };
 
+/// simple counter for grabbing 'random' user agents
+static mut USER_AGENT_CTR: usize = 0;
+
 /// Given the path to a file, open the file in append mode (create it if it doesn't exist) and
 /// return a reference to the buffered file
 pub fn open_file(filename: &str) -> Result<BufWriter<fs::File>> {
@@ -94,7 +99,7 @@ pub async fn logged_request(url: &Url, handles: Arc<Handles>) -> Result<Response
     let level = handles.config.output_level;
     let tx_stats = handles.stats.tx.clone();
 
-    let response = make_request(client, url, level, tx_stats).await;
+    let response = make_request(client, url, level, &handles.config, tx_stats).await;
 
     let scans = handles.ferox_scans()?;
 
@@ -121,6 +126,7 @@ pub async fn make_request(
     client: &Client,
     url: &Url,
     output_level: OutputLevel,
+    config: &Configuration,
     tx_stats: UnboundedSender<Command>,
 ) -> Result<Response> {
     log::trace!(
@@ -130,7 +136,20 @@ pub async fn make_request(
         tx_stats
     );
 
-    match client.get(url.to_owned()).send().await {
+    let mut request = client.get(url.to_owned());
+
+    if config.random_agent {
+        let index = unsafe {
+            USER_AGENT_CTR += 1;
+            USER_AGENT_CTR % USER_AGENTS.len()
+        };
+
+        let user_agent = USER_AGENTS[index];
+
+        request = request.header("User-Agent", user_agent);
+    }
+
+    match request.send().await {
         Err(e) => {
             log::trace!("exit: make_request -> {}", e);
 

tests/test_banner.rs

@@ -143,6 +143,31 @@ fn banner_prints_denied_urls() {
         );
 }
 
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + multiple headers
+fn banner_prints_random_agent() {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("--random-agent")
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Random"))
+                .and(predicate::str::contains("─┴─")),
+        );
+}
+
 #[test]
 /// test allows non-existent wordlist to trigger the banner printing to stderr
 /// expect to see all mandatory prints + multiple size filters