From ba3529116cdd76bd4e9abd7364ba837a44fb174a Mon Sep 17 00:00:00 2001 From: epi Date: Wed, 25 Nov 2020 20:01:16 -0600 Subject: [PATCH 1/3] simplified call to scanner::initialize --- src/main.rs | 10 +------- src/scanner.rs | 65 ++++++++++++++++++++++++++++---------------------- 2 files changed, 38 insertions(+), 37 deletions(-) diff --git a/src/main.rs b/src/main.rs index 88f80af..23c573f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -113,15 +113,7 @@ async fn scan( return Err(Box::new(err)); } - scanner::initialize( - words.len(), - CONFIGURATION.scan_limit, - &CONFIGURATION.extensions, - &CONFIGURATION.filter_status, - &CONFIGURATION.filter_line_count, - &CONFIGURATION.filter_word_count, - &CONFIGURATION.filter_size, - ); + scanner::initialize(words.len(), &CONFIGURATION); let mut tasks = vec![]; diff --git a/src/scanner.rs b/src/scanner.rs index 7bd121c..80bfc4d 100644 --- a/src/scanner.rs +++ b/src/scanner.rs @@ -1,8 +1,9 @@ use crate::{ - config::CONFIGURATION, + config::{Configuration, CONFIGURATION}, extractor::get_links, filters::{ - FeroxFilter, LinesFilter, SizeFilter, StatusCodeFilter, WildcardFilter, WordsFilter, + FeroxFilter, LinesFilter, RegexFilter, SizeFilter, StatusCodeFilter, WildcardFilter, + WordsFilter, }, heuristics, scan_manager::{FeroxScans, PAUSE_SCAN}, @@ -14,7 +15,10 @@ use futures::{ stream, StreamExt, }; use lazy_static::lazy_static; +use regex::Regex; use reqwest::Url; +#[cfg(not(test))] +use std::process::exit; use std::{ collections::HashSet, convert::TryInto, @@ -601,38 +605,21 @@ pub async fn scan_url( /// Perform steps necessary to run scans that only need to be performed once (warming up the /// engine, as it were) -pub fn initialize( - num_words: usize, - scan_limit: usize, - extensions: &[String], - status_code_filters: &[u16], - lines_filters: &[usize], - words_filters: &[usize], - size_filters: &[u64], -) { - log::trace!( - "enter: initialize({}, {}, {:?}, {:?}, {:?}, {:?}, {:?})", - num_words, - scan_limit, - extensions, - status_code_filters, - lines_filters, - words_filters, - size_filters, - ); +pub fn initialize(num_words: usize, config: &Configuration) { + log::trace!("enter: initialize({}, {:?})", num_words, config,); // number of requests only needs to be calculated once, and then can be reused - let num_reqs_expected: u64 = if extensions.is_empty() { + let num_reqs_expected: u64 = if config.extensions.is_empty() { num_words.try_into().unwrap() } else { - let total = num_words * (extensions.len() + 1); + let total = num_words * (config.extensions.len() + 1); total.try_into().unwrap() }; NUMBER_OF_REQUESTS.store(num_reqs_expected, Ordering::Relaxed); // add any status code filters to `FILTERS` (-C|--filter-status) - for code_filter in status_code_filters { + for code_filter in &config.filter_status { let filter = StatusCodeFilter { filter_code: *code_filter, }; @@ -641,7 +628,7 @@ pub fn initialize( } // add any line count filters to `FILTERS` (-N|--filter-lines) - for lines_filter in lines_filters { + for lines_filter in &config.filter_line_count { let filter = LinesFilter { line_count: *lines_filter, }; @@ -650,7 +637,7 @@ pub fn initialize( } // add any line count filters to `FILTERS` (-W|--filter-words) - for words_filter in words_filters { + for words_filter in &config.filter_word_count { let filter = WordsFilter { word_count: *words_filter, }; @@ -659,7 +646,7 @@ pub fn initialize( } // add any line count filters to `FILTERS` (-S|--filter-size) - for size_filter in size_filters { + for size_filter in &config.filter_size { let filter = SizeFilter { content_length: *size_filter, }; @@ -667,7 +654,29 @@ pub fn initialize( add_filter_to_list_of_ferox_filters(boxed_filter, FILTERS.clone()); } - if scan_limit == 0 { + // add any regex filters to `FILTERS` (-X|--filter-regex) + for regex_filter in &config.filter_regex { + let raw = regex_filter; + let compiled = match Regex::new(&raw) { + Ok(regex) => regex, + Err(e) => { + log::error!("Invalid regular expression: {}", e); + #[cfg(test)] + panic!(); + #[cfg(not(test))] + exit(1); + } + }; + + let filter = RegexFilter { + raw_string: raw.to_owned(), + compiled, + }; + let boxed_filter = Box::new(filter); + add_filter_to_list_of_ferox_filters(boxed_filter, FILTERS.clone()); + } + + if config.scan_limit == 0 { // scan_limit == 0 means no limit should be imposed... however, scoping the Semaphore // permit is tricky, so as a workaround, we'll add a ridiculous number of permits to // the semaphore (1,152,921,504,606,846,975 to be exact) and call that 'unlimited' From 20e7d0195eb4e28c7a9130294818c4db301a4762 Mon Sep 17 00:00:00 2001 From: epi Date: Wed, 25 Nov 2020 20:20:56 -0600 Subject: [PATCH 2/3] added integration test for regex filter --- tests/test_banner.rs | 4 ++-- tests/test_scanner.rs | 46 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/tests/test_banner.rs b/tests/test_banner.rs index 39f066d..3b3c7ea 100644 --- a/tests/test_banner.rs +++ b/tests/test_banner.rs @@ -737,7 +737,7 @@ fn banner_prints_debug_log() { .arg("--url") .arg("http://localhost") .arg("--debug-log") - .arg("im-a-debug-log.hurr-durr") + .arg("/dev/null") .assert() .success() .stderr( @@ -750,7 +750,7 @@ fn banner_prints_debug_log() { .and(predicate::str::contains("Timeout (secs)")) .and(predicate::str::contains("User-Agent")) .and(predicate::str::contains("Debugging Log")) - .and(predicate::str::contains("│ im-a-debug-log.hurr-durr")) + .and(predicate::str::contains("│ /dev/null")) .and(predicate::str::contains("─┴─")), ); } diff --git a/tests/test_scanner.rs b/tests/test_scanner.rs index 637f414..b9a121c 100644 --- a/tests/test_scanner.rs +++ b/tests/test_scanner.rs @@ -541,3 +541,49 @@ fn scanner_single_request_scan_with_debug_logging_as_json() { assert_eq!(mock.times_called(), 1); teardown_tmp_directory(tmp_dir); } + +#[test] +/// send a single valid request, filter the response by regex, expect one out of 2 urls +fn scanner_single_request_scan_with_regex_filtered_result() { + let srv = MockServer::start(); + let (tmp_dir, file) = + setup_tmp_directory(&["LICENSE".to_string(), "ignored".to_string()], "wordlist").unwrap(); + + let mock = Mock::new() + .expect_method(GET) + .expect_path("/LICENSE") + .return_status(200) + .return_body("this is a not a test") + .create_on(&srv); + + let filtered_mock = Mock::new() + .expect_method(GET) + .expect_path("/ignored") + .return_status(200) + .return_body("this is a test\nThat rug really tied the room together") + .create_on(&srv); + + let cmd = Command::cargo_bin("feroxbuster") + .unwrap() + .arg("--url") + .arg(srv.url("/")) + .arg("--wordlist") + .arg(file.as_os_str()) + .arg("--filter-regex") + .arg("'That rug.*together$'") + .unwrap(); + + cmd.assert().success().stdout( + predicate::str::contains("/LICENSE") + .and(predicate::str::contains("200")) + .and(predicate::str::contains("20")) + .and(predicate::str::contains("ignored")) + .not() + .and(predicate::str::contains(" 14 ")) + .not(), + ); + + assert_eq!(mock.times_called(), 1); + assert_eq!(filtered_mock.times_called(), 1); + teardown_tmp_directory(tmp_dir); +} From 81d21ce557da8867d4316c957753553665567f7d Mon Sep 17 00:00:00 2001 From: epi Date: Thu, 26 Nov 2020 07:34:49 -0600 Subject: [PATCH 3/3] added test for bad regex --- src/scanner.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/scanner.rs b/src/scanner.rs index 80bfc4d..27e29ca 100644 --- a/src/scanner.rs +++ b/src/scanner.rs @@ -783,4 +783,13 @@ mod tests { let result = reached_max_depth(&url, 0, 2); assert!(result); } + + #[test] + #[should_panic] + /// call initialize with a bad regex, triggering a panic + fn initialize_panics_on_bad_regex() { + let mut config = Configuration::default(); + config.filter_regex = vec![r"(".to_string()]; + initialize(1, &config); + } }