From a9dde3f7e10b63904bee3274044fec9828812957 Mon Sep 17 00:00:00 2001 From: epi Date: Tue, 4 Oct 2022 05:45:34 -0500 Subject: [PATCH 1/4] normalized directory scan input + search in feroxscans --- src/scan_manager/scan_container.rs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/scan_manager/scan_container.rs b/src/scan_manager/scan_container.rs index 84f0c34..ce521ef 100644 --- a/src/scan_manager/scan_container.rs +++ b/src/scan_manager/scan_container.rs @@ -213,8 +213,10 @@ impl FeroxScans { /// on the given URL pub fn contains(&self, url: &str) -> bool { if let Ok(scans) = self.scans.read() { + let normalized = format!("{}/", url.trim_end_matches('/')); + for scan in scans.iter() { - if scan.url == url { + if scan.url == normalized { return true; } } @@ -225,8 +227,10 @@ impl FeroxScans { /// Find and return a `FeroxScan` based on the given URL pub fn get_scan_by_url(&self, url: &str) -> Option> { if let Ok(guard) = self.scans.read() { + let normalized = format!("{}/", url.trim_end_matches('/')); + for scan in guard.iter() { - if scan.url == url { + if scan.url == normalized { return Some(scan.clone()); } } @@ -589,7 +593,8 @@ impl FeroxScans { /// /// Also return a reference to the new `FeroxScan` pub fn add_directory_scan(&self, url: &str, scan_order: ScanOrder) -> (bool, Arc) { - self.add_scan(url, ScanType::Directory, scan_order) + let normalized = format!("{}/", url.trim_end_matches('/')); + self.add_scan(&normalized, ScanType::Directory, scan_order) } /// Given a url, create a new `FeroxScan` and add it to `FeroxScans` as a File Scan From fd5b1f6f25ccc62a0a82a40f2c59a21635e691be Mon Sep 17 00:00:00 2001 From: epi Date: Tue, 4 Oct 2022 07:07:24 -0500 Subject: [PATCH 2/4] refined the fix; updated tests and serialization --- src/scan_manager/scan.rs | 14 +++++++++++++- src/scan_manager/scan_container.rs | 4 ++-- src/scan_manager/tests.rs | 6 ++++-- tests/test_scan_manager.rs | 8 +++++--- 4 files changed, 24 insertions(+), 8 deletions(-) diff --git a/src/scan_manager/scan.rs b/src/scan_manager/scan.rs index ba8f4f6..6d416b2 100644 --- a/src/scan_manager/scan.rs +++ b/src/scan_manager/scan.rs @@ -32,6 +32,9 @@ pub struct FeroxScan { /// The URL that to be scanned pub(super) url: String, + /// A url used solely for comparison to other URLs + pub(super) normalized_url: String, + /// The type of scan pub scan_type: ScanType, @@ -79,6 +82,7 @@ impl Default for FeroxScan { num_requests: 0, scan_order: ScanOrder::Latest, url: String::new(), + normalized_url: String::new(), progress_bar: Mutex::new(None), scan_type: ScanType::File, output_level: Default::default(), @@ -191,6 +195,7 @@ impl FeroxScan { ) -> Arc { Arc::new(Self { url: url.to_string(), + normalized_url: format!("{}/", url.trim_end_matches('/')), scan_type, scan_order, num_requests, @@ -332,10 +337,11 @@ impl Serialize for FeroxScan { where S: Serializer, { - let mut state = serializer.serialize_struct("FeroxScan", 4)?; + let mut state = serializer.serialize_struct("FeroxScan", 6)?; state.serialize_field("id", &self.id)?; state.serialize_field("url", &self.url)?; + state.serialize_field("normalized_url", &self.normalized_url)?; state.serialize_field("scan_type", &self.scan_type)?; state.serialize_field("status", &self.status)?; state.serialize_field("num_requests", &self.num_requests)?; @@ -387,6 +393,11 @@ impl<'de> Deserialize<'de> for FeroxScan { scan.url = url.to_string(); } } + "normalized_url" => { + if let Some(normalized_url) = value.as_str() { + scan.normalized_url = normalized_url.to_string(); + } + } "num_requests" => { if let Some(num_requests) = value.as_u64() { scan.num_requests = num_requests; @@ -480,6 +491,7 @@ mod tests { let scan = FeroxScan { id: "".to_string(), url: "".to_string(), + normalized_url: String::from("/"), scan_type: ScanType::Directory, scan_order: ScanOrder::Initial, num_requests: 0, diff --git a/src/scan_manager/scan_container.rs b/src/scan_manager/scan_container.rs index ce521ef..33b23a3 100644 --- a/src/scan_manager/scan_container.rs +++ b/src/scan_manager/scan_container.rs @@ -216,7 +216,7 @@ impl FeroxScans { let normalized = format!("{}/", url.trim_end_matches('/')); for scan in scans.iter() { - if scan.url == normalized { + if scan.normalized_url == normalized { return true; } } @@ -230,7 +230,7 @@ impl FeroxScans { let normalized = format!("{}/", url.trim_end_matches('/')); for scan in guard.iter() { - if scan.url == normalized { + if scan.normalized_url == normalized { return Some(scan.clone()); } } diff --git a/src/scan_manager/tests.rs b/src/scan_manager/tests.rs index c9a0c00..eaa3f7d 100644 --- a/src/scan_manager/tests.rs +++ b/src/scan_manager/tests.rs @@ -277,7 +277,7 @@ fn ferox_scan_serialize() { None, ); let fs_json = format!( - r#"{{"id":"{}","url":"https://spiritanimal.com","scan_type":"Directory","status":"NotStarted","num_requests":0}}"#, + r#"{{"id":"{}","url":"https://spiritanimal.com","normalized_url":"https://spiritanimal.com/","scan_type":"Directory","status":"NotStarted","num_requests":0}}"#, fs.id ); assert_eq!(fs_json, serde_json::to_string(&*fs).unwrap()); @@ -296,7 +296,7 @@ fn ferox_scans_serialize() { ); let ferox_scans = FeroxScans::default(); let ferox_scans_json = format!( - r#"[{{"id":"{}","url":"https://spiritanimal.com","scan_type":"Directory","status":"NotStarted","num_requests":0}}]"#, + r#"[{{"id":"{}","url":"https://spiritanimal.com","normalized_url":"https://spiritanimal.com/","scan_type":"Directory","status":"NotStarted","num_requests":0}}]"#, ferox_scan.id ); ferox_scans.scans.write().unwrap().push(ferox_scan); @@ -556,6 +556,7 @@ fn feroxscan_display() { let scan = FeroxScan { id: "".to_string(), url: String::from("http://localhost"), + normalized_url: String::from("http://localhost/"), scan_order: ScanOrder::Latest, scan_type: Default::default(), num_requests: 0, @@ -600,6 +601,7 @@ async fn ferox_scan_abort() { let scan = FeroxScan { id: "".to_string(), url: String::from("http://localhost"), + normalized_url: String::from("http://localhost/"), scan_order: ScanOrder::Latest, scan_type: Default::default(), num_requests: 0, diff --git a/tests/test_scan_manager.rs b/tests/test_scan_manager.rs index ce41745..067684c 100644 --- a/tests/test_scan_manager.rs +++ b/tests/test_scan_manager.rs @@ -20,11 +20,13 @@ fn resume_scan_works() { // localhost:PORT/ <- complete // localhost:PORT/js <- will get scanned with /css and /stuff let complete_scan = format!( - r#"{{"id":"057016a14769414aac9a7a62707598cb","url":"{}","scan_type":"Directory","status":"Complete"}}"#, - srv.url("/") + r#"{{"id":"057016a14769414aac9a7a62707598cb","url":"{}","normalized_url":"{}","scan_type":"Directory","status":"Complete"}}"#, + srv.url("/"), + srv.url("/"), ); let incomplete_scan = format!( - r#"{{"id":"400b2323a16f43468a04ffcbbeba34c6","url":"{}","scan_type":"Directory","status":"NotStarted"}}"#, + r#"{{"id":"400b2323a16f43468a04ffcbbeba34c6","url":"{}","normalized_url":"{}/","scan_type":"Directory","status":"NotStarted"}}"#, + srv.url("/js"), srv.url("/js") ); let scans = format!(r#""scans":[{},{}]"#, complete_scan, incomplete_scan); From 04e3ad69cc4b2db9d0c3c2ace4f6585f2280b9c3 Mon Sep 17 00:00:00 2001 From: epi Date: Tue, 4 Oct 2022 07:09:40 -0500 Subject: [PATCH 3/4] allowing a test build to happen --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c1363ff..2c67ac9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -7,7 +7,7 @@ jobs: env: IN_PIPELINE: true runs-on: ${{ matrix.os }} - if: github.ref == 'refs/heads/main' + # if: github.ref == 'refs/heads/main' strategy: matrix: type: [ubuntu-x64, ubuntu-x86, armv7, aarch64] From 1f60e06247d42f4bc9c50b27f53401aaa3777795 Mon Sep 17 00:00:00 2001 From: epi Date: Wed, 5 Oct 2022 05:30:00 -0500 Subject: [PATCH 4/4] turned off builds for all but main --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2c67ac9..c1363ff 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -7,7 +7,7 @@ jobs: env: IN_PIPELINE: true runs-on: ${{ matrix.os }} - # if: github.ref == 'refs/heads/main' + if: github.ref == 'refs/heads/main' strategy: matrix: type: [ubuntu-x64, ubuntu-x86, armv7, aarch64]