Merge pull request #72 from epi052/FEATURE-add-link-extraction--add-cli-option

added -e|--extract-links to parser/banner/config 🕵

src/banner.rs

@@ -77,10 +77,12 @@ by Ben "epi" Risher {}                  ver: {}"#,
         "{}",
         format_banner_entry!("\u{1F680}", "Threads", config.threads)
     ); // 🚀
+
     eprintln!(
         "{}",
         format_banner_entry!("\u{1f4d6}", "Wordlist", config.wordlist)
     ); // 📖
+
     eprintln!(
         "{}",
         format_banner_entry!(
@@ -89,10 +91,12 @@ by Ben "epi" Risher {}                  ver: {}"#,
             format!("[{}]", codes.join(", "))
         )
     ); // 🆗
+
     eprintln!(
         "{}",
         format_banner_entry!("\u{1f4a5}", "Timeout (secs)", config.timeout)
     ); // 💥
+
     eprintln!(
         "{}",
         format_banner_entry!("\u{1F9a1}", "User-Agent", config.useragent)
@@ -131,6 +135,13 @@ by Ben "epi" Risher {}                  ver: {}"#,
         }
     }
 
+    if config.extract_links {
+        eprintln!(
+            "{}",
+            format_banner_entry!("\u{1F50E}", "Extract Links", config.extract_links)
+        ); // 🔎
+    }
+
     if !config.queries.is_empty() {
         for query in &config.queries {
             eprintln!(

src/config.rs

@@ -107,6 +107,10 @@ pub struct Configuration {
     #[serde(default)]
     pub norecursion: bool,
 
+    /// Extract links from html/javscript
+    #[serde(default)]
+    pub extract_links: bool,
+
     /// Append / to each request
     #[serde(default)]
     pub addslash: bool,
@@ -182,8 +186,9 @@ impl Default for Configuration {
             verbosity: 0,
             addslash: false,
             insecure: false,
-            norecursion: false,
             redirects: false,
+            norecursion: false,
+            extract_links: false,
             proxy: String::new(),
             config: String::new(),
             output: String::new(),
@@ -206,6 +211,7 @@ impl Configuration {
     ///
     /// - **timeout**: `5` seconds
     /// - **redirects**: `false`
+    /// - **extract-links**: `false`
     /// - **wordlist**: [`DEFAULT_WORDLIST`](constant.DEFAULT_WORDLIST.html)
     /// - **config**: `None`
     /// - **threads**: `50`
@@ -390,6 +396,10 @@ impl Configuration {
             config.addslash = args.is_present("addslash");
         }
 
+        if args.is_present("extract_links") {
+            config.extract_links = args.is_present("extract_links");
+        }
+
         if args.is_present("stdin") {
             config.stdin = args.is_present("stdin");
         } else {
@@ -514,6 +524,7 @@ impl Configuration {
         settings.useragent = settings_to_merge.useragent;
         settings.redirects = settings_to_merge.redirects;
         settings.insecure = settings_to_merge.insecure;
+        settings.extract_links = settings_to_merge.extract_links;
         settings.extensions = settings_to_merge.extensions;
         settings.headers = settings_to_merge.headers;
         settings.queries = settings_to_merge.queries;
@@ -574,6 +585,7 @@ mod tests {
             addslash = true
             stdin = true
             dontfilter = true
+            extract_links = true
             depth = 1
             sizefilters = [4120]
         "#;
@@ -602,6 +614,7 @@ mod tests {
         assert_eq!(config.stdin, false);
         assert_eq!(config.addslash, false);
         assert_eq!(config.redirects, false);
+        assert_eq!(config.extract_links, false);
         assert_eq!(config.insecure, false);
         assert_eq!(config.queries, Vec::new());
         assert_eq!(config.extensions, Vec::<String>::new());
@@ -714,6 +727,13 @@ mod tests {
         assert_eq!(config.addslash, true);
     }
 
+    #[test]
+    /// parse the test config and see that the value parsed is correct
+    fn config_reads_extract_links() {
+        let config = setup_config_test();
+        assert_eq!(config.extract_links, true);
+    }
+
     #[test]
     /// parse the test config and see that the value parsed is correct
     fn config_reads_extensions() {

src/parser.rs

@@ -195,6 +195,13 @@ pub fn initialize() -> App<'static, 'static> {
                     "Filter out messages of a particular size (ex: -S 5120 -S 4927,1970)",
                 ),
         )
+        .arg(
+            Arg::with_name("extract_links")
+                .short("e")
+                .long("extract-links")
+                .takes_value(false)
+                .help("Extract links from html and javascript files; make new requests based on findings (default: false)")
+        )
 
         .after_help(r#"NOTE:
     Options that take multiple values are very flexible.  Consider the following ways of specifying
@@ -225,6 +232,9 @@ EXAMPLES:
     Pass auth token via query parameter
         ./feroxbuster -u http://127.1 --query token=0123456789ABCDEF
 
+    Find links in javascript/html and make additional requests based on results
+        ./feroxbuster -u http://127.1 --extract-links
+
     Ludicrous speed... go!
         ./feroxbuster -u http://127.1 -t 200
     "#)

tests/test_banner.rs

@@ -536,3 +536,30 @@ fn banner_doesnt_print() -> Result<(), Box<dyn std::error::Error>> {
         ));
     Ok(())
 }
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + extract-links
+fn banner_prints_extract_links() -> Result<(), Box<dyn std::error::Error>> {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("-e")
+        .assert()
+        .failure()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Extract Links"))
+                .and(predicate::str::contains("true"))
+                .and(predicate::str::contains("─┴─")),
+        );
+    Ok(())
+}