fixed bug in dynamic wildcards; reorded 404-like id strat

src/config/utils.rs

@@ -46,10 +46,11 @@ pub(super) fn threads() -> usize {
 
 /// default status codes
 pub(super) fn status_codes() -> Vec<u16> {
-    DEFAULT_STATUS_CODES
-        .iter()
-        .map(|code| code.as_u16())
-        .collect()
+    // DEFAULT_STATUS_CODES
+    //     .iter()
+    //     .map(|code| code.as_u16())
+    //     .collect()
+    Vec::new()
 }
 
 /// default HTTP Method

src/event_handlers/outputs.rs

@@ -265,7 +265,7 @@ impl TermOutHandler {
 
             if should_process_response {
                 // print to stdout
-                log::warn!("{:?}", self.handles.as_ref().unwrap().filters);
+                log::warn!("{:?}", self.handles.as_ref().unwrap().filters.data);
                 ferox_print(&resp.as_str(), &PROGRESS_PRINTER);
 
                 send_command!(tx_stats, AddToUsizeField(ResourcesDiscovered, 1));

src/filters/similarity.rs

@@ -27,7 +27,10 @@ impl FeroxFilter for SimilarityFilter {
         }
 
         // couldn't hash the response, don't filter
-        log::warn!("Could not hash body from {}", response.as_str());
+        log::warn!(
+            "Could not compare similarity of body from {}; returning not-similar",
+            response.url().as_str()
+        );
         false
     }
 

src/heuristics.rs

@@ -156,17 +156,6 @@ impl HeuristicTests {
                 .make_wildcard_request(&ferox_url, method.as_str(), data, 1)
                 .await?;
 
-            // use crate::SIMILARITY_THRESHOLD;
-            // use fuzzyhash::FuzzyHash;
-
-            // let filter = SimilarityFilter {
-            //     hash: FuzzyHash::new(ferox_response.text()).to_string(),
-            //     threshold: SIMILARITY_THRESHOLD,
-            //     original_url: target_url.to_string(),
-            // };
-            // self.handles.filters.send(Command::AddFilter(Box::new(filter)))?;
-            // return Ok(1);
-
             // found a wildcard response
             let mut wildcard = WildcardFilter::new(self.handles.config.dont_filter);
 
@@ -197,7 +186,9 @@ impl HeuristicTests {
                 // reflected in the response along with some static content; aka custom 404
                 let url_len = ferox_url.path_length()?;
 
-                wildcard.dynamic = wc_length - url_len;
+                log::warn!("{:?}", dbg!(url_len, wc_length, wc2_length));
+                log::warn!("{:?}", ferox_url);
+                wildcard.dynamic = wc_length - UUID_LENGTH;
 
                 print_dont_filter_message(
                     WildcardType::Dynamic(&wildcard),
@@ -452,18 +443,18 @@ impl HeuristicTests {
 
     /// given a target's base url, attempt to automatically detect its 404 response
     /// pattern, and then set a filter that will exclude all but the first result
-    pub async fn detect_404_response(&self, target_url: &str) -> Result<()> {
+    pub async fn detect_404_response(&self, target_url: &str) -> Result<bool> {
         log::trace!("enter: detect_404_response");
 
         if self.handles.config.dont_filter {
             log::trace!("exit: detect_404_response -> dont_filter is true");
-            return Ok(());
+            return Ok(false);
         }
 
         let mut responses = Vec::with_capacity(3);
 
         for prefix in ["", ".htaccess", "admin"] {
-            let path = format!("/{prefix}{}", self.unique_string(1));
+            let path = format!("{prefix}{}", self.unique_string(1));
             let ferox_url = FeroxUrl::from_string(target_url, self.handles.clone());
             let request = ferox_url.format(&path, None)?;
             let response =
@@ -490,13 +481,6 @@ impl HeuristicTests {
         let word_count = responses[0].word_count();
         let line_count = responses[0].line_count();
 
-        if responses
-            .iter()
-            .all(|resp| resp.content_length() == content_length)
-        {
-            size_sentry = false;
-        }
-
         for response in &responses[1..] {
             if response.content_length() != content_length {
                 size_sentry = false;
@@ -511,39 +495,21 @@ impl HeuristicTests {
             }
         }
 
-        if size_sentry {
-            self.handles
-                .filters
-                .send(Command::AddFilter(Box::new(SizeFilter { content_length })))?;
-            ferox_print(responses[0].url().as_str(), &PROGRESS_PRINTER);
-            ferox_print(
-                &format!("Detected 404-like response; filtering requests of size {content_length}"),
-                &PROGRESS_PRINTER,
-            );
+        let command = if size_sentry {
+            Command::AddFilter(Box::new(SizeFilter { content_length }))
         } else if word_sentry {
-            self.handles
-                .filters
-                .send(Command::AddFilter(Box::new(WordsFilter { word_count })))?;
-            ferox_print(responses[0].url().as_str(), &PROGRESS_PRINTER);
-
-            ferox_print(
-                &format!("Detected 404-like response; filtering requests with {word_count} words"),
-                &PROGRESS_PRINTER,
-            );
+            Command::AddFilter(Box::new(WordsFilter { word_count }))
         } else if line_sentry {
-            self.handles
-                .filters
-                .send(Command::AddFilter(Box::new(LinesFilter { line_count })))?;
-            ferox_print(responses[0].url().as_str(), &PROGRESS_PRINTER);
+            Command::AddFilter(Box::new(LinesFilter { line_count }))
+        } else {
+            log::trace!("exit: detect_404_response -> no filter added");
+            return Ok(false);
+        };
 
-            ferox_print(
-                &format!("Detected 404-like response; filtering requests with {line_count} lines"),
-                &PROGRESS_PRINTER,
-            );
-        }
+        self.handles.filters.send(command)?;
 
         log::trace!("exit: detect_404_response");
-        Ok(())
+        Ok(size_sentry || word_sentry || line_sentry)
     }
 }
 

src/scanner/ferox_scanner.rs

@@ -182,6 +182,7 @@ impl FeroxScanner {
                     Err(e) => {
                         log::warn!("error awaiting a response: {}", e);
                         self.handles.stats.send(AddError(Other)).unwrap_or_default();
+                        std::process::exit(1);
                     }
                 }
             });
@@ -289,22 +290,19 @@ impl FeroxScanner {
 
                     ferox_scan.finish()?;
 
-                    return Ok(());  // nothing left to do if we found a dir listing
+                    return Ok(()); // nothing left to do if we found a dir listing
                 }
             }
 
-            // on error, we'll have 0, same for --dont-filter
-            // anything higher than 0 indicates a wildcard was found
-            let num_reqs_made = test.wildcard(&self.target_url).await.unwrap_or_default();
-            
-            progress_bar.inc(num_reqs_made);
+            let detected = test.detect_404_response(&self.target_url).await?;
+            log::info!("404 response detected: {}", detected);
 
-            if num_reqs_made == 0 {
-                // no dir listing and no wildcard, go ahead and see if we can
-                // identify their 404
-                test.detect_404_response(&self.target_url).await?;
+            if !detected {
+                // on error, we'll have 0, same for --dont-filter
+                // anything higher than 0 indicates a wildcard was found
+                let num_reqs_made = test.wildcard(&self.target_url).await.unwrap_or_default();
+                progress_bar.inc(num_reqs_made);
             }
-
         }
 
         // Arc clones to be passed around to the various scans

src/utils.rs

@@ -196,6 +196,8 @@ pub async fn make_request(
         request = request.header("User-Agent", user_agent);
     }
 
+    log::warn!("{:?}", url.path());
+
     match request.send().await {
         Err(e) => {
             log::trace!("exit: make_request -> {}", e);
@@ -240,7 +242,7 @@ pub async fn make_request(
             }
 
             log::warn!("Error while making request: {}", e);
-            bail!("{}", e)
+            std::process::exit(1);
         }
         Ok(resp) => {
             log::trace!("exit: make_request -> {:?}", resp);