Merge pull request #68 from epi052/67-duplicate-scans-occurring

fixed duplicate directory scans

.github/workflows/build.yml

@@ -59,18 +59,40 @@ jobs:
           name: feroxbuster_amd64.deb
           path: ./target/x86_64-unknown-linux-musl/debian/*
 
-  build-rest:
+  build-macos:
+    runs-on: macos-latest
+    if: github.ref == 'refs/heads/master'
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+          target: x86_64-apple-darwin
+          override: true
+      - uses: actions-rs/cargo@v1
+        with:
+          use-cross: true
+          command: build
+          args: --release --target=x86_64-apple-darwin
+      - name: Build tar.gz for homebrew installs
+        run: |
+          tar czf x86_64-macos-feroxbuster.tar.gz -C target/x86_64-apple-darwin/release feroxbuster
+      - uses: actions/upload-artifact@v2
+        with:
+          name: x86_64-macos-feroxbuster
+          path: target/x86_64-apple-darwin/release/feroxbuster
+      - uses: actions/upload-artifact@v2
+        with:
+          name: x86_64-macos-feroxbuster.tar.gz
+          path: x86_64-macos-feroxbuster.tar.gz
+
+  build-windows:
     runs-on: ${{ matrix.os }}
     if: github.ref == 'refs/heads/master'
     strategy:
       matrix:
-        type: [windows-x64, windows-x86, macos]
+        type: [windows-x64, windows-x86]
         include:
-          - type: macos
-            os: macos-latest
-            target: x86_64-apple-darwin
-            name: x86_64-macos-feroxbuster
-            path: target/x86_64-apple-darwin/release/feroxbuster
           - type: windows-x64
             os: windows-latest
             target: x86_64-pc-windows-msvc
@@ -97,3 +119,4 @@ jobs:
         with:
           name: ${{ matrix.name }}
           path: ${{ matrix.path }}
+

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "feroxbuster"
-version = "1.0.1"
+version = "1.0.3"
 authors = ["Ben 'epi' Risher <epibar052@gmail.com>"]
 license = "MIT"
 edition = "2018"
@@ -49,4 +49,4 @@ conf-files = ["/etc/feroxbuster/ferox-config.toml"]
 assets = [
     ["target/release/feroxbuster", "/usr/bin/", "755"],
     ["ferox-config.toml.example", "/etc/feroxbuster/ferox-config.toml", "644"],
-]
+]

Dockerfile

@@ -0,0 +1,12 @@
+FROM alpine:latest
+LABEL maintainer="wfnintr@null.net"
+
+# download default wordlists 
+RUN apk add --no-cache --virtual .depends subversion && \
+	svn export https://github.com/danielmiessler/SecLists/trunk/Discovery/Web-Content /usr/share/seclists/Discovery/Web-Content && \
+	apk del .depends
+
+# install latest release
+RUN wget https://github.com/epi052/feroxbuster/releases/latest/download/x86_64-linux-feroxbuster.zip -qO feroxbuster.zip && unzip -d /usr/local/bin/ feroxbuster.zip feroxbuster && rm feroxbuster.zip && chmod +x /usr/local/bin/feroxbuster
+
+ENTRYPOINT ["feroxbuster"]

README.md

@@ -62,8 +62,10 @@ This attack is also known as Predictable Resource Location, File Enumeration, Di
 - [Downloads](#-downloads)
 - [Installation](#-installation)
     - [Download a Release](#download-a-release)
+    - [Homebrew on MacOS and Linux](#homebrew-on-macos-and-linux)
     - [Cargo Install](#cargo-install)
     - [apt Install](#apt-install)
+    - [Docker Install](#docker-install)
 - [Configuration](#-configuration)
     - [Default Values](#default-values)
     - [ferox-config.toml](#ferox-configtoml)
@@ -90,6 +92,24 @@ Releases for multiple architectures can be found in the [Releases](https://githu
 - Windows x86
 - Windows x86_64
 
+### Homebrew on MacOS and Linux
+
+Installable by Homebrew throughout own formulas:
+
+🍏 [MacOS](https://github.com/TGotwig/homebrew-feroxbuster/blob/main/feroxbuster.rb)
+
+```shell
+brew tap tgotwig/feroxbuster
+brew install feroxbuster
+```
+
+🐧 [Linux](https://github.com/TGotwig/homebrew-linux-feroxbuster/blob/main/feroxbuster.rb)
+
+```shell
+brew tap tgotwig/linux-feroxbuster
+brew install feroxbuster
+```
+
 ### Cargo Install
 
 `feroxbuster` is published on crates.io, making it easy to install if you already have rust installed on your system.
@@ -106,6 +126,61 @@ Head to the [Releases](https://github.com/epi052/feroxbuster/releases) section a
 sudo apt install ./feroxbuster_amd64.deb
 ```
 
+### Docker Install
+
+> The following steps assume you have docker installed / setup
+
+First, clone the repository.
+
+```
+git clone https://github.com/epi052/feroxbuster.git
+cd feroxbuster
+```
+
+Next, build the image.
+
+```
+sudo docker build -t feroxbuster .
+```
+
+After that, you should be able to use `docker run` to perform scans with `feroxbuster`.
+
+#### Basic usage
+
+```
+sudo docker run --init -it feroxbuster -u http://example.com -x js,html
+```
+
+#### Piping from stdin and proxying all requests through socks5 proxy
+
+```
+cat targets.txt | sudo docker run --net=host --init -i feroxbuster --stdin -x js,html --proxy socks5://127.0.0.1:9050
+```
+
+#### Mount a volume to pass in `ferox-config.toml`
+
+You've got some options available if you want to pass in a config file.  [`ferox-buster.toml`](#ferox-configtoml) can live in multiple locations and still be valid, so it's up to you how you'd like to pass it in.  Below are a few valid examples:
+
+```
+sudo docker run --init -v $(pwd)/ferox-config.toml:/etc/feroxbuster/ferox-config.toml -it feroxbuster -u http://example.com
+```
+
+```
+sudo docker run --init -v ~/.config/feroxbuster:/root/.config/feroxbuster -it feroxbuster -u http://example.com
+```
+
+Note: If you are on a SELinux enforced system, you will need to pass the `:Z` attribute also.
+
+```
+docker run --init -v (pwd)/ferox-config.toml:/etc/feroxbuster/ferox-config.toml:Z -it feroxbuster -u http://example.com
+```
+
+#### Define an alias for simplicity
+
+```
+alias feroxbuster="sudo docker run --init -v ~/.config/feroxbuster:/root/.config/feroxbuster -i feroxbuster"
+```
+
 ## ⚙️ Configuration
 ### Default Values
 Configuration begins with with the following built-in default values baked into the binary:

src/scanner.rs

@@ -6,12 +6,13 @@ use crate::utils::{
 use crate::{heuristics, progress};
 use futures::future::{BoxFuture, FutureExt};
 use futures::{stream, StreamExt};
+use lazy_static::lazy_static;
 use reqwest::{Response, Url};
 use std::collections::HashSet;
 use std::convert::TryInto;
 use std::ops::Deref;
 use std::sync::atomic::{AtomicUsize, Ordering};
-use std::sync::Arc;
+use std::sync::{Arc, RwLock};
 use tokio::fs;
 use tokio::io::{self, AsyncWriteExt};
 use tokio::sync::mpsc::{self, UnboundedReceiver, UnboundedSender};
@@ -19,6 +20,11 @@ use tokio::task::JoinHandle;
 
 static CALL_COUNT: AtomicUsize = AtomicUsize::new(0);
 
+lazy_static! {
+    /// Global configuration state
+    static ref SCANNED_URLS: RwLock<HashSet<String>> = RwLock::new(HashSet::new());
+}
+
 /// Spawn a single consumer task (sc side of mpsc)
 ///
 /// The consumer simply receives responses and writes them to the given output file if they meet
@@ -119,6 +125,43 @@ async fn spawn_terminal_reporter(mut report_channel: UnboundedReceiver<Response>
     log::trace!("exit: spawn_terminal_reporter");
 }
 
+/// Adds the given url to `SCANNED_URLS`
+///
+/// If `SCANNED_URLS` did not already contain the url, return true; otherwise return false
+fn add_url_to_list_of_scanned_urls(resp: &str, scanned_urls: &RwLock<HashSet<String>>) -> bool {
+    log::trace!(
+        "enter: add_url_to_list_of_scanned_urls({}, {:?})",
+        resp,
+        scanned_urls
+    );
+
+    match scanned_urls.write() {
+        // check new url against what's already been scanned
+        Ok(mut urls) => {
+            let normalized_url = if resp.ends_with('/') {
+                // append a / to the list of 'seen' urls, this is to prevent the case where
+                // 3xx and 2xx duplicate eachother
+                resp.to_string()
+            } else {
+                format!("{}/", resp)
+            };
+
+            // If the set did not contain resp, true is returned.
+            // If the set did contain resp, false is returned.
+            let response = urls.insert(normalized_url);
+
+            log::trace!("exit: add_url_to_list_of_scanned_urls -> {}", response);
+            response
+        }
+        Err(e) => {
+            // poisoned lock
+            log::error!("Set of scanned urls poisoned: {}", e);
+            log::trace!("exit: add_url_to_list_of_scanned_urls -> false");
+            false
+        }
+    }
+}
+
 /// Spawn a single consumer task (sc side of mpsc)
 ///
 /// The consumer simply receives Urls and scans them
@@ -137,6 +180,13 @@ fn spawn_recursion_handler(
     let boxed_future = async move {
         let mut scans = vec![];
         while let Some(resp) = recursion_channel.recv().await {
+            let unknown = add_url_to_list_of_scanned_urls(&resp, &SCANNED_URLS);
+
+            if !unknown {
+                // not unknown, i.e. we've seen the url before and don't need to scan again
+                continue;
+            }
+
             log::info!("received {} on recursion channel", resp);
             let clonedresp = resp.clone();
             let clonedlist = wordlist.clone();
@@ -437,6 +487,10 @@ pub async fn scan_url(target_url: &str, wordlist: Arc<HashSet<String>>, base_dep
         // join can only be called once, otherwise it causes the thread to panic
         tokio::task::spawn_blocking(move || PROGRESS_BAR.join().unwrap());
         CALL_COUNT.fetch_add(1, Ordering::Relaxed);
+
+        // this protection around join also allows us to add the first scanned url to SCANNED_URLS
+        // from within the scan_url function instead of the recursion handler
+        add_url_to_list_of_scanned_urls(&target_url, &SCANNED_URLS);
     }
 
     let wildcard_bar = progress_bar.clone();
@@ -583,4 +637,39 @@ mod tests {
             assert_eq!(urls, expected[i]);
         }
     }
+
+    #[test]
+    /// add an unknown url to the hashset, expect true
+    fn add_url_to_list_of_scanned_urls_with_unknown_url() {
+        let urls = RwLock::new(HashSet::<String>::new());
+        let url = "http://unknown_url";
+        assert_eq!(add_url_to_list_of_scanned_urls(url, &urls), true);
+    }
+
+    #[test]
+    /// add a known url to the hashset, with a trailing slash, expect false
+    fn add_url_to_list_of_scanned_urls_with_known_url() {
+        let urls = RwLock::new(HashSet::<String>::new());
+        let url = "http://unknown_url/";
+
+        assert_eq!(urls.write().unwrap().insert(url.to_string()), true);
+
+        assert_eq!(add_url_to_list_of_scanned_urls(url, &urls), false);
+    }
+
+    #[test]
+    /// add a known url to the hashset, without a trailing slash, expect false
+    fn add_url_to_list_of_scanned_urls_with_known_url_without_slash() {
+        let urls = RwLock::new(HashSet::<String>::new());
+        let url = "http://unknown_url";
+
+        assert_eq!(
+            urls.write()
+                .unwrap()
+                .insert("http://unknown_url/".to_string()),
+            true
+        );
+
+        assert_eq!(add_url_to_list_of_scanned_urls(url, &urls), false);
+    }
 }

src/utils.rs

@@ -223,7 +223,12 @@ pub async fn make_request(client: &Client, url: &Url) -> FeroxResult<Response> {
         }
         Err(e) => {
             log::trace!("exit: make_request -> {}", e);
-            log::error!("Error while making request: {}", e);
+            if e.to_string().contains("operation timed out") {
+                // only warn for timeouts, while actual errors are still left as errors
+                log::warn!("Error while making request: {}", e);
+            } else {
+                log::error!("Error while making request: {}", e);
+            }
             Err(Box::new(e))
         }
     }