Merge pull request #234 from epi052/233-ordered-wordlist

changed wordlist read so that ordering is maintained

Cargo.lock

@@ -473,9 +473,9 @@ dependencies = [
 
 [[package]]
 name = "ctrlc"
-version = "3.1.7"
+version = "3.1.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b57a92e9749e10f25a171adcebfafe72991d45e7ec2dcb853e8f83d9dafaeb08"
+checksum = "c15b8ec3b5755a188c141c1f6a98e76de31b936209bf066b647979e2a84764a9"
 dependencies = [
  "nix",
  "winapi",
@@ -633,7 +633,7 @@ dependencies = [
 
 [[package]]
 name = "feroxbuster"
-version = "2.2.0"
+version = "2.2.2"
 dependencies = [
  "anyhow",
  "assert_cmd",
@@ -724,9 +724,9 @@ dependencies = [
 
 [[package]]
 name = "futures"
-version = "0.3.12"
+version = "0.3.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "da9052a1a50244d8d5aa9bf55cbc2fb6f357c86cc52e46c62ed390a7180cf150"
+checksum = "7f55667319111d593ba876406af7c409c0ebb44dc4be6132a783ccf163ea14c1"
 dependencies = [
  "futures-channel",
  "futures-core",
@@ -739,9 +739,9 @@ dependencies = [
 
 [[package]]
 name = "futures-channel"
-version = "0.3.12"
+version = "0.3.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f2d31b7ec7efab6eefc7c57233bb10b847986139d88cc2f5a02a1ae6871a1846"
+checksum = "8c2dd2df839b57db9ab69c2c9d8f3e8c81984781937fe2807dc6dcf3b2ad2939"
 dependencies = [
  "futures-core",
  "futures-sink",
@@ -749,15 +749,15 @@ dependencies = [
 
 [[package]]
 name = "futures-core"
-version = "0.3.12"
+version = "0.3.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "79e5145dde8da7d1b3892dad07a9c98fc04bc39892b1ecc9692cf53e2b780a65"
+checksum = "15496a72fabf0e62bdc3df11a59a3787429221dd0710ba8ef163d6f7a9112c94"
 
 [[package]]
 name = "futures-executor"
-version = "0.3.12"
+version = "0.3.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e9e59fdc009a4b3096bf94f740a0f2424c082521f20a9b08c5c07c48d90fd9b9"
+checksum = "891a4b7b96d84d5940084b2a37632dd65deeae662c114ceaa2c879629c9c0ad1"
 dependencies = [
  "futures-core",
  "futures-task",
@@ -766,9 +766,9 @@ dependencies = [
 
 [[package]]
 name = "futures-io"
-version = "0.3.12"
+version = "0.3.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "28be053525281ad8259d47e4de5de657b25e7bac113458555bb4b70bc6870500"
+checksum = "d71c2c65c57704c32f5241c1223167c2c3294fd34ac020c807ddbe6db287ba59"
 
 [[package]]
 name = "futures-lite"
@@ -787,9 +787,9 @@ dependencies = [
 
 [[package]]
 name = "futures-macro"
-version = "0.3.12"
+version = "0.3.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c287d25add322d9f9abdcdc5927ca398917996600182178774032e9f8258fedd"
+checksum = "ea405816a5139fb39af82c2beb921d52143f556038378d6db21183a5c37fbfb7"
 dependencies = [
  "proc-macro-hack",
  "proc-macro2",
@@ -799,24 +799,21 @@ dependencies = [
 
 [[package]]
 name = "futures-sink"
-version = "0.3.12"
+version = "0.3.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "caf5c69029bda2e743fddd0582d1083951d65cc9539aebf8812f36c3491342d6"
+checksum = "85754d98985841b7d4f5e8e6fbfa4a4ac847916893ec511a2917ccd8525b8bb3"
 
 [[package]]
 name = "futures-task"
-version = "0.3.12"
+version = "0.3.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "13de07eb8ea81ae445aca7b69f5f7bf15d7bf4912d8ca37d6645c77ae8a58d86"
-dependencies = [
- "once_cell",
-]
+checksum = "fa189ef211c15ee602667a6fcfe1c1fd9e07d42250d2156382820fba33c9df80"
 
 [[package]]
 name = "futures-util"
-version = "0.3.12"
+version = "0.3.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "632a8cd0f2a4b3fdea1657f08bde063848c3bd00f9bbf6e256b8be78802e624b"
+checksum = "1812c7ab8aedf8d6f2701a43e1243acdbcc2b36ab26e2ad421eb99ac963d96d1"
 dependencies = [
  "futures-channel",
  "futures-core",
@@ -1298,13 +1295,13 @@ checksum = "e4a24736216ec316047a1fc4252e27dabb04218aa4a3f37c6e7ddbf1f9782b54"
 
 [[package]]
 name = "nix"
-version = "0.18.0"
+version = "0.20.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "83450fe6a6142ddd95fb064b746083fc4ef1705fe81f64a64e1d4b39f54a1055"
+checksum = "fa9b4819da1bc61c0ea48b63b7bc8604064dd43013e7cc325df098d49cd7c18a"
 dependencies = [
  "bitflags",
  "cc",
- "cfg-if 0.1.10",
+ "cfg-if 1.0.0",
  "libc",
 ]
 
@@ -1718,9 +1715,9 @@ dependencies = [
 
 [[package]]
 name = "reqwest"
-version = "0.11.0"
+version = "0.11.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fd281b1030aa675fb90aa994d07187645bb3c8fc756ca766e7c3070b439de9de"
+checksum = "0460542b551950620a3648c6aa23318ac6b3cd779114bd873209e6e8b5eb1c34"
 dependencies = [
  "base64",
  "bytes",
@@ -1753,9 +1750,9 @@ dependencies = [
 
 [[package]]
 name = "rlimit"
-version = "0.5.3"
+version = "0.5.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6e7148757b4951f04391d2b301b2e3597d504c4d2434212d542b73c1a6b3f847"
+checksum = "81a9ed03edbed449d6897c2092c71ab5f7b5fb80f6f0b1a3ed6d40a6f9fc0720"
 dependencies = [
  "libc",
 ]
@@ -1839,9 +1836,9 @@ dependencies = [
 
 [[package]]
 name = "serde_json"
-version = "1.0.62"
+version = "1.0.64"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ea1c6153794552ea7cf7cf63b1231a25de00ec90db326ba6264440fa08e31486"
+checksum = "799e97dc9fdae36a5c8b8f2cae9ce2ee9fdce2058c57a93e6099d919fd982f79"
 dependencies = [
  "itoa",
  "ryu",

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "feroxbuster"
-version = "2.2.0"
+version = "2.2.2"
 authors = ["Ben 'epi' Risher <epibar052@gmail.com>"]
 license = "MIT"
 edition = "2018"
@@ -21,17 +21,17 @@ regex = "1"
 lazy_static = "1.4"
 
 [dependencies]
-futures = { version = "0.3"}
+futures = { version = "0.3.13"}
 tokio = { version = "1.2.0", features = ["full"] }
 tokio-util = {version = "0.6.3", features = ["codec"]}
 log = "0.4"
 env_logger = "0.8.3"
-reqwest = { version = "0.11", features = ["socks"] }
+reqwest = { version = "0.11.1", features = ["socks"] }
 clap = "2.33"
 lazy_static = "1.4"
 toml = "0.5"
 serde = { version = "1.0", features = ["derive", "rc"] }
-serde_json = "1.0.62"
+serde_json = "1.0.64"
 uuid = { version = "0.8", features = ["v4"] }
 indicatif = "0.15"
 console = "0.14"
@@ -39,8 +39,8 @@ openssl = { version = "0.10", features = ["vendored"] }
 dirs = "3.0"
 regex = "1"
 crossterm = "0.19"
-rlimit = "0.5"
-ctrlc = "3.1"
+rlimit = "0.5.4"
+ctrlc = "3.1.8"
 fuzzyhash = "0.2.1"
 anyhow = "1.0"
 leaky-bucket = "0.10.0"
@@ -63,4 +63,7 @@ conf-files = ["/etc/feroxbuster/ferox-config.toml"]
 assets = [
     ["target/release/feroxbuster", "/usr/bin/", "755"],
     ["ferox-config.toml.example", "/etc/feroxbuster/ferox-config.toml", "644"],
+    ["shell_completions/feroxbuster.bash", "/usr/share/bash-completion/completions/feroxbuster.bash", "644"],
+    ["shell_completions/feroxbuster.fish", "/usr/share/fish/completions/feroxbuster.fish", "644"],
+    ["shell_completions/_feroxbuster", "/usr/share/zsh/vendor-completions/_feroxbuster", "644"],
 ]

Makefile

@@ -6,12 +6,16 @@ datarootdir = $(prefix)/share
 datadir = $(datarootdir)
 example_config = ferox-config.toml.example
 config_file = ferox-config.toml
+completion_dir = shell_completions
+completion_prefix = $(completion_dir)/$(BIN)
 
+BIN=feroxbuster
 SHR_SOURCES = $(shell find src -type f -wholename '*src/*.rs') Cargo.toml Cargo.lock
 
 RELEASE = debug
 DEBUG ?= 0
-ifeq (0,$(DEBUG))
+
+ifeq (0, $(DEBUG))
 	ARGS = --release
 	RELEASE = release
 endif
@@ -23,54 +27,47 @@ endif
 
 TARGET = target/$(RELEASE)
 
-.PHONY: all clean distclean install uninstall update
-
-BIN=feroxbuster
-DESKTOP=$(APPID).desktop
+.PHONY: all clean install uninstall
 
 all: cli
-
 cli: $(TARGET)/$(BIN) $(TARGET)/$(BIN).1.gz $(SHR_SOURCES)
+install: all install-cli
 
 clean:
 	cargo clean
 
-distclean: clean
-	rm -rf .cargo vendor Cargo.lock vendor.tar
-
 vendor: vendor.tar
 
 vendor.tar:
-	mkdir -p .cargo
-	cargo vendor | head -n -1 > .cargo/config
-	echo 'directory = "vendor"' >> .cargo/config
+	cargo vendor
 	tar pcf vendor.tar vendor
 	rm -rf vendor
 
 install-cli: cli
-	install -Dm 0755 "$(TARGET)/$(BIN)" "$(DESTDIR)$(bindir)/$(BIN)"
+	install -Dm 0644 "$(completion_prefix).bash" "$(DESTDIR)/usr/share/bash-completion/completions/$(BIN).bash"
+	install -Dm 0644 "$(completion_prefix).fish" "$(DESTDIR)/usr/share/fish/completions/$(BIN).fish"
+	install -Dm 0644 "$(completion_dir)/_$(BIN)" "$(DESTDIR)/usr/share/zsh/vendor-completions/_$(BIN)"
+	install -sDm 0755 "$(TARGET)/$(BIN)" "$(DESTDIR)$(bindir)/$(BIN)"
 	install -Dm 0644 "$(TARGET)/$(BIN).1.gz" "$(DESTDIR)$(datadir)/man/man1/$(BIN).1.gz"
-	install -Dm 0644 "$(example_config)" "/etc/$(BIN)/$(config_File)"
+	install -Dm 0644 "$(example_config)" "$(DESTDIR)/etc/$(BIN)/$(config_file)"
 
-install: all install-cli
-
-uninstall-cli:
+uninstall:
 	rm -f "$(DESTDIR)$(bindir)/$(BIN)"
 	rm -f "$(DESTDIR)$(datadir)/man/man1/$(BIN).1.gz"
-	rm -rf "/etc/$(BIN)/"
-
-uninstall: uninstall-cli
-
-update:
-	cargo update
+	rm -rf "$(DESTDIR)/etc/$(BIN)/"
+	rm -f "$(DESTDIR)/usr/share/bash-completion/completions/$(BIN).bash"
+	rm -f "$(DESTDIR)/usr/share/zsh/vendor-completions/_$(BIN)"
+	rm -f "$(DESTDIR)/usr/share/fish/completions/$(BIN).fish"
 
 extract:
-ifeq ($(VENDORED),1)
+ifeq (1, $(VENDORED))
 	tar pxf vendor.tar
 endif
 
 $(TARGET)/$(BIN): extract
-	cargo build --manifest-path Cargo.toml $(ARGS)
+	mkdir -p .cargo
+	cp debian/cargo.config .cargo/config.toml
+	cargo build $(ARGS)
 
 $(TARGET)/$(BIN).1.gz: $(TARGET)/$(BIN)
 	help2man --no-info $< | gzip -c > $@.partial

README.md

@@ -70,6 +70,7 @@ Enumeration.
     - [Snap Install](#snap-install)
     - [Homebrew on MacOS and Linux](#homebrew-on-macos-and-linux)
     - [Cargo Install](#cargo-install)
+    - [Kali Install](#kali-install)
     - [apt Install](#apt-install)
     - [AUR Install](#aur-install)
     - [Docker Install](#docker-install)
@@ -194,6 +195,16 @@ brew install feroxbuster
 cargo install feroxbuster
 ```
 
+### Kali Install
+
+🥳 `feroxbuster` was recently added to the official Kali Linux repos 🥳 
+
+If you're using kali, this is the preferred install method. Installing from the repos adds a [**ferox-config.toml**](#ferox-config.toml) in `/etc/feroxbuster/`, adds command completion for bash, fish, and zsh, includes a man page entry, and installs `feroxbuster` itself. 
+
+```
+sudo apt update && sudo apt install -y feroxbuster
+```
+
 ### apt Install
 
 Download `feroxbuster_amd64.deb` from the [Releases](https://github.com/epi052/feroxbuster/releases) section. After
@@ -918,12 +929,12 @@ cat large-target-list | ./feroxbuster --stdin --parallel 10 --extract-links --au
 
 Resuling Process List (illustrative):
 ```
-\_ target/debug/feroxbuster --stdin --parallel 10
-    \_ target/debug/feroxbuster --silent --extract-links --auto-bail -u https://target-one
-    \_ target/debug/feroxbuster --silent --extract-links --auto-bail -u https://target-two
-    \_ target/debug/feroxbuster --silent --extract-links --auto-bail -u https://target-three
-    \_ ...
-    \_ target/debug/feroxbuster --silent --extract-links --auto-bail -u https://target-ten
+feroxbuster --stdin --parallel 10
+ \_ feroxbuster --silent --extract-links --auto-bail -u https://target-one
+ \_ feroxbuster --silent --extract-links --auto-bail -u https://target-two
+ \_ feroxbuster --silent --extract-links --auto-bail -u https://target-three
+ \_ ...
+ \_ feroxbuster --silent --extract-links --auto-bail -u https://target-ten
 ```
 
 ## 🧐 Comparison w/ Similar Tools
@@ -949,7 +960,7 @@ few of the use-cases in which feroxbuster may be a better fit:
 | fast                                                                         | ✔ | ✔ | ✔ |
 | allows recursion                                                             | ✔ |   | ✔ |
 | can specify query parameters                                                 | ✔ |   | ✔ |
-| SOCKS proxy support                                                          | ✔ |   |   |
+| SOCKS proxy support                                                          | ✔ |   | ✔ |
 | multiple target scan (via stdin or multiple -u)                              | ✔ |   | ✔ |
 | configuration file for default value override                                | ✔ |   | ✔ |
 | can accept urls via STDIN as part of a pipeline                              | ✔ |   | ✔ |

src/banner/container.rs

@@ -171,7 +171,7 @@ impl Banner {
             code_filters.push(status_colorizer(&code.to_string()))
         }
         let filter_status = BannerEntry::new(
-            "🗑",
+            "💢",
             "Status Code Filters",
             &format!("[{}]", code_filters.join(", ")),
         );

src/event_handlers/command.rs

@@ -1,4 +1,3 @@
-use std::collections::HashSet;
 use std::sync::Arc;
 
 use reqwest::StatusCode;
@@ -53,7 +52,7 @@ pub enum Command {
     TryRecursion(Box<FeroxResponse>),
 
     /// Send a pointer to the wordlist to the recursion handler
-    UpdateWordlist(Arc<HashSet<String>>),
+    UpdateWordlist(Arc<Vec<String>>),
 
     /// Instruct the ScanHandler to join on all known scans, use sender to notify main when done
     JoinTasks(Sender<bool>),

src/event_handlers/scans.rs

@@ -1,4 +1,3 @@
-use std::collections::HashSet;
 use std::sync::Arc;
 
 use anyhow::{bail, Result};
@@ -54,7 +53,7 @@ pub struct ScanHandler {
     receiver: CommandReceiver,
 
     /// wordlist (re)used for each scan
-    wordlist: std::sync::Mutex<Option<Arc<HashSet<String>>>>,
+    wordlist: std::sync::Mutex<Option<Arc<Vec<String>>>>,
 
     /// group of scans that need to be joined
     tasks: Vec<Arc<FeroxScan>>,
@@ -105,7 +104,7 @@ impl ScanHandler {
     }
 
     /// Set the wordlist
-    fn wordlist(&self, wordlist: Arc<HashSet<String>>) {
+    fn wordlist(&self, wordlist: Arc<Vec<String>>) {
         if let Ok(mut guard) = self.wordlist.lock() {
             if guard.is_none() {
                 let _ = std::mem::replace(&mut *guard, Some(wordlist));
@@ -175,7 +174,7 @@ impl ScanHandler {
     }
 
     /// Helper to easily get the (locked) underlying wordlist
-    pub fn get_wordlist(&self) -> Result<Arc<HashSet<String>>> {
+    pub fn get_wordlist(&self) -> Result<Arc<Vec<String>>> {
         if let Ok(guard) = self.wordlist.lock().as_ref() {
             if let Some(list) = guard.as_ref() {
                 return Ok(list.clone());

src/main.rs

@@ -1,5 +1,4 @@
 use std::{
-    collections::HashSet,
     env::args,
     fs::File,
     io::{stderr, BufRead, BufReader},
@@ -41,14 +40,14 @@ lazy_static! {
 }
 
 /// Create a HashSet of Strings from the given wordlist then stores it inside an Arc
-fn get_unique_words_from_wordlist(path: &str) -> Result<Arc<HashSet<String>>> {
+fn get_unique_words_from_wordlist(path: &str) -> Result<Arc<Vec<String>>> {
     log::trace!("enter: get_unique_words_from_wordlist({})", path);
 
     let file = File::open(&path).with_context(|| format!("Could not open {}", path))?;
 
     let reader = BufReader::new(file);
 
-    let mut words = HashSet::new();
+    let mut words = Vec::new();
 
     for line in reader.lines() {
         let result = match line {
@@ -60,7 +59,7 @@ fn get_unique_words_from_wordlist(path: &str) -> Result<Arc<HashSet<String>>> {
             continue;
         }
 
-        words.insert(result);
+        words.push(result);
     }
 
     log::trace!(
@@ -78,11 +77,7 @@ async fn scan(targets: Vec<String>, handles: Arc<Handles>) -> Result<()> {
     // so that will allow for cheap/safe sharing of a single wordlist across multi-target scans
     // as well as additional directories found as part of recursion
 
-    let words = {
-        let words_handles = handles.clone();
-        tokio::spawn(async move { get_unique_words_from_wordlist(&words_handles.config.wordlist) })
-            .await??
-    };
+    let words = get_unique_words_from_wordlist(&handles.config.wordlist)?;
 
     if words.len() == 0 {
         bail!("Did not find any words in {}", handles.config.wordlist);

src/scanner/ferox_scanner.rs

@@ -1,4 +1,4 @@
-use std::{collections::HashSet, ops::Deref, sync::atomic::Ordering, sync::Arc, time::Instant};
+use std::{ops::Deref, sync::atomic::Ordering, sync::Arc, time::Instant};
 
 use anyhow::{bail, Result};
 use futures::{stream, StreamExt};
@@ -40,7 +40,7 @@ pub struct FeroxScanner {
     order: ScanOrder,
 
     /// wordlist that's already been read from disk
-    wordlist: Arc<HashSet<String>>,
+    wordlist: Arc<Vec<String>>,
 
     /// limiter that restricts the number of active FeroxScanners  
     scan_limiter: Arc<Semaphore>,
@@ -52,7 +52,7 @@ impl FeroxScanner {
     pub fn new(
         target_url: &str,
         order: ScanOrder,
-        wordlist: Arc<HashSet<String>>,
+        wordlist: Arc<Vec<String>>,
         scan_limiter: Arc<Semaphore>,
         handles: Arc<Handles>,
     ) -> Self {