fixed build script when on cd pipeline

added small check for help param to always print and exit

.github/FUNDING.yml

@@ -0,0 +1,4 @@
+# These are supported funding model platforms
+
+github: [epi052]
+ko_fi: epi052

.github/pull_request_template.md

@@ -11,7 +11,7 @@ Long form explanations of most of the items below can be found in the [CONTRIBUT
 
 ## Static analysis checks
 - [ ] All rust files are formatted using `cargo fmt`
-- [ ] All `clippy` checks pass when running `cargo clippy --all-targets --all-features -- -D warnings -A clippy::deref_addrof -A clippy::mutex-atomic`
+- [ ] All `clippy` checks pass when running `cargo clippy --all-targets --all-features -- -D warnings -A clippy::mutex-atomic`
 - [ ] All existing tests pass
 
 ## Documentation

.github/workflows/build.yml

@@ -4,6 +4,8 @@ on: [push]
 
 jobs:
   build-nix:
+    env:
+      IN_PIPELINE: true
     runs-on: ${{ matrix.os }}
     if: github.ref == 'refs/heads/main'
     strategy:
@@ -38,6 +40,7 @@ jobs:
       - uses: actions/checkout@v2
       - name: Install System Dependencies
         run: |
+          env
           sudo apt-get update
           sudo apt-get install -y --no-install-recommends libssl-dev pkg-config gcc-arm-linux-gnueabihf gcc-aarch64-linux-gnu
       - uses: actions-rs/toolchain@v1
@@ -84,6 +87,8 @@ jobs:
           path: ./target/x86_64-unknown-linux-musl/debian/*
 
   build-macos:
+    env:
+      IN_PIPELINE: true
     runs-on: macos-latest
     if: github.ref == 'refs/heads/main'
     steps:
@@ -114,6 +119,8 @@ jobs:
           path: x86_64-macos-feroxbuster.tar.gz
 
   build-windows:
+    env:
+      IN_PIPELINE: true
     runs-on: ${{ matrix.os }}
     if: github.ref == 'refs/heads/main'
     strategy:
@@ -146,4 +153,3 @@ jobs:
         with:
           name: ${{ matrix.name }}
           path: ${{ matrix.path }}
-

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "feroxbuster"
-version = "2.2.3"
+version = "2.2.5"
 authors = ["Ben 'epi' Risher <epibar052@gmail.com>"]
 license = "MIT"
 edition = "2018"
@@ -19,14 +19,15 @@ maintenance = { status = "actively-developed" }
 clap = "2.33"
 regex = "1"
 lazy_static = "1.4"
+dirs = "3.0"
 
 [dependencies]
-futures = { version = "0.3.13"}
-tokio = { version = "1.2.0", features = ["full"] }
-tokio-util = {version = "0.6.3", features = ["codec"]}
+futures = { version = "0.3.14"}
+tokio = { version = "1.6", features = ["full"] }
+tokio-util = {version = "0.6.6", features = ["codec"]}
 log = "0.4"
-env_logger = "0.8.3"
-reqwest = { version = "0.11.1", features = ["socks"] }
+env_logger = "0.8"
+reqwest = { version = "0.11", features = ["socks"] }
 clap = "2.33"
 lazy_static = "1.4"
 toml = "0.5"
@@ -38,18 +39,18 @@ console = "0.14"
 openssl = { version = "0.10", features = ["vendored"] }
 dirs = "3.0"
 regex = "1"
-crossterm = "0.19"
+crossterm = "0.20"
 rlimit = "0.5.4"
-ctrlc = "3.1.8"
+ctrlc = "3.1.9"
 fuzzyhash = "0.2.1"
 anyhow = "1.0"
 leaky-bucket = "0.10.0"
 
 [dev-dependencies]
 tempfile = "3.1"
-httpmock = "0.5.2"
-assert_cmd = "1.0.3"
-predicates = "1.0.7"
+httpmock = "0.5.8"
+assert_cmd = "1.0"
+predicates = "1.0.8"
 
 [profile.release]
 lto = true

Makefile

@@ -27,12 +27,17 @@ endif
 
 TARGET = target/$(RELEASE)
 
-.PHONY: all clean install uninstall
+.PHONY: all clean install uninstall test update
 
 all: cli
 cli: $(TARGET)/$(BIN) $(TARGET)/$(BIN).1.gz $(SHR_SOURCES)
 install: all install-cli
 
+verify:
+	cargo fmt
+	cargo clippy --all-targets --all-features -- -D warnings -A clippy::mutex-atomic
+	cargo test
+
 clean:
 	cargo clean
 

README.md

@@ -225,6 +225,14 @@ Install `feroxbuster-git` on Arch Linux with your AUR helper of choice:
 yay -S feroxbuster-git
 ```
 
+### BlackArch install
+
+Install `feroxbuster` on BlackArch Linux:
+
+```
+pacman -S feroxbuster
+```
+
 ### Docker Install
 
 > The following steps assume you have docker installed / setup
@@ -828,10 +836,11 @@ Below is an example of the Scan Cancel Menu™.
 Using the menu is pretty simple:
 - Press `ENTER` to view the menu
 - Choose a scan to cancel by entering its scan index (`1`)
-  - more than one scan can be selected by using a comma-separated list (`1,2,3` ... etc)
+  - more than one scan can be selected by using a comma-separated list of indexes and/or ranges (`1-4,8,9-13` ... etc)
 - Confirm selections, after which all non-cancelled scans will resume
+  - To skip confirmation, simply add a `-f` somewhere in your input (`3-5 -f`)
 
-Here is a short demonstration of cancelling two in-progress scans found via recursion.
+Here is a short demonstration of force cancelling a range of scans followed by a single scan with interactive prompt.
 
 ![cancel-scan](img/cancel-scan.gif)
 

build.rs

@@ -1,4 +1,7 @@
+use std::fs::{copy, create_dir_all, OpenOptions};
+use std::io::{Read, Seek, SeekFrom, Write};
 extern crate clap;
+extern crate dirs;
 
 use clap::Shell;
 
@@ -20,4 +23,64 @@ fn main() {
     for shell in &shells {
         app.gen_completions("feroxbuster", *shell, outdir);
     }
+
+    // 0xdf pointed out an oddity when tab-completing options that expect file paths, the fix we
+    // landed on was to add -o plusdirs to the bash completion script. The following code aims to
+    // automate that fix and have it present in all future builds
+    let mut contents = String::new();
+
+    let mut bash_file = OpenOptions::new()
+        .read(true)
+        .write(true)
+        .open(format!("{}/feroxbuster.bash", outdir))
+        .expect("Couldn't open bash completion script");
+
+    bash_file
+        .read_to_string(&mut contents)
+        .expect("Couldn't read bash completion script");
+
+    contents = contents.replace("default feroxbuster", "default -o plusdirs feroxbuster");
+
+    bash_file
+        .seek(SeekFrom::Start(0))
+        .expect("Couldn't seek to position 0 in bash completion script");
+
+    bash_file
+        .write_all(contents.as_bytes())
+        .expect("Couldn't write updated bash completion script to disk");
+
+    // hunter0x8 let me know that when installing via cargo, it would be nice if we dropped a
+    // config file during the build process. The following code will place an example config in
+    // the user's configuration directory
+    //   - linux: $XDG_CONFIG_HOME or $HOME/.config
+    //   - macOS: $HOME/Library/Application Support
+    //   - windows: {FOLDERID_RoamingAppData}
+    let mut config_dir = dirs::config_dir().expect("Couldn't resolve user's config directory");
+    config_dir = config_dir.join("feroxbuster"); // $HOME/.config/feroxbuster
+
+    if !config_dir.exists() {
+        // recursively create the feroxbuster directory and all of its parent components if
+        // they are missing
+        if !config_dir.exists() {
+            // recursively create the feroxbuster directory and all of its parent components if
+            // they are missing
+            if create_dir_all(&config_dir).is_err() {
+                // only copy the config file when we're not running in the CI/CD pipeline
+                // which fails with permission denied
+                eprintln!("Couldn't create one or more directories needed to copy the config file");
+                return;
+            }
+        }
+    }
+
+    // hard-coding config name here to not rely on the crate we're building, if DEFAULT_CONFIG_NAME
+    // ever changes, this will need to be updated
+    let config_file = config_dir.join("ferox-config.toml");
+
+    if !config_file.exists() {
+        // config file doesn't exist, add it to the config directory
+        if copy("ferox-config.toml.example", config_file).is_err() {
+            eprintln!("Couldn't copy example config into config directory");
+        }
+    }
 }

install-nix.sh

@@ -3,59 +3,63 @@
 BASE_URL=https://github.com/epi052/feroxbuster/releases/latest/download
 
 MAC_ZIP=x86_64-macos-feroxbuster.zip
-MAC_URL="${BASE_URL}/${MAC_ZIP}"
+MAC_URL="$BASE_URL/$MAC_ZIP"
 
 LIN32_ZIP=x86-linux-feroxbuster.zip
-LIN32_URL="${BASE_URL}/${LIN32_ZIP}"
+LIN32_URL="$BASE_URL/$LIN32_ZIP"
 
 LIN64_ZIP=x86_64-linux-feroxbuster.zip
-LIN64_URL="${BASE_URL}/${LIN64_ZIP}"
+LIN64_URL="$BASE_URL/$LIN64_ZIP"
 
 EMOJI_URL=https://gist.github.com/epi052/8196b550ea51d0907ad4b93751b1b57d/raw/6112c9f32ae07922983fdc549c54fd3fb9a38e4c/NotoColorEmoji.ttf
 
 echo "[+] Installing feroxbuster!"
 
-if [[ "$(uname)" == "Darwin" ]]; then
-    echo "[=] Found MacOS, downloading from ${MAC_URL}"
-
-    curl -sLO "${MAC_URL}"
-    unzip -o "${MAC_ZIP}" > /dev/null
-    rm "${MAC_ZIP}"
-elif [[ "$(expr substr $(uname -s) 1 5)" == "Linux" ]]; then
-    if [[ $(getconf LONG_BIT) == 32 ]]; then
-        echo "[=] Found 32-bit Linux, downloading from ${LIN32_URL}"
-
-        curl -sLO "${LIN32_URL}"
-        unzip -o "${LIN32_ZIP}" > /dev/null
-        rm "${LIN32_ZIP}"
-    else
-        echo "[=] Found 64-bit Linux, downloading from ${LIN64_URL}"
-
-        curl -sLO "${LIN64_URL}"
-        unzip -o "${LIN64_ZIP}" > /dev/null
-        rm "${LIN64_ZIP}"
-    fi
-
-    if [[ -e ~/.fonts/NotoColorEmoji.ttf ]]; then
-       echo "[=] Found Noto Emoji Font, skipping install" 
-    else 
-        echo "[=] Installing Noto Emoji Font"
-        mkdir -p ~/.fonts
-        pushd ~/.fonts 2>&1 >/dev/null
-
-        curl -sLO "${EMOJI_URL}"
-
-        fc-cache -f -v >/dev/null
-
-        popd 2>&1 >/dev/null
-        echo "[+] Noto Emoji Font installed"
-    fi
+which unzip &>/dev/null
+if [ "$?" = "0" ]; then
+  echo "[+] unzip found"
+else
+  echo "[ ] unzip not found, exiting. "
+  exit -1
 fi
 
+if [[ "$(uname)" == "Darwin" ]]; then
+  echo "[=] Found MacOS, downloading from $MAC_URL"
+
+  curl -sLO "$MAC_URL"
+  unzip -o "$MAC_ZIP" >/dev/null
+  rm "$MAC_ZIP"
+elif [[ "$(expr substr $(uname -s) 1 5)" == "Linux" ]]; then
+  if [[ $(getconf LONG_BIT) == 32 ]]; then
+    echo "[=] Found 32-bit Linux, downloading from $LIN32_URL"
+
+    curl -sLO "$LIN32_URL"
+    unzip -o "$LIN32_ZIP" >/dev/null
+    rm "$LIN32_ZIP"
+  else
+    echo "[=] Found 64-bit Linux, downloading from $LIN64_URL"
+
+    curl -sLO "$LIN64_URL"
+    unzip -o "$LIN64_ZIP" >/dev/null
+    rm "$LIN64_ZIP"
+  fi
+
+  if [[ -e ~/.fonts/NotoColorEmoji.ttf ]]; then
+    echo "[=] Found Noto Emoji Font, skipping install"
+  else
+    echo "[=] Installing Noto Emoji Font"
+    mkdir -p ~/.fonts
+    pushd ~/.fonts 2>&1 >/dev/null
+
+    curl -sLO "$EMOJI_URL"
+
+    fc-cache -f -v >/dev/null
+
+    popd 2>&1 >/dev/null
+    echo "[+] Noto Emoji Font installed"
+  fi
+fi
 
 chmod +x ./feroxbuster
 
 echo "[+] Installed feroxbuster version $(./feroxbuster -V)"
-
-
-

shell_completions/feroxbuster.bash

@@ -222,4 +222,4 @@ _feroxbuster() {
     esac
 }
 
-complete -F _feroxbuster -o bashdefault -o default feroxbuster
+complete -F _feroxbuster -o bashdefault -o default -o plusdirs feroxbuster

src/config/tests.rs

@@ -364,9 +364,10 @@ fn config_reads_headers() {
 /// parse the test config and see that the values parsed are correct
 fn config_reads_queries() {
     let config = setup_config_test();
-    let mut queries = vec![];
-    queries.push(("name".to_string(), "value".to_string()));
-    queries.push(("rick".to_string(), "astley".to_string()));
+    let queries = vec![
+        ("name".to_string(), "value".to_string()),
+        ("rick".to_string(), "astley".to_string()),
+    ];
     assert_eq!(config.queries, queries);
 }
 

src/event_handlers/outputs.rs

@@ -139,8 +139,8 @@ impl TermOutHandler {
         Self {
             receiver,
             tx_file,
-            config,
             file_task,
+            config,
         }
     }
 

src/parser.rs

@@ -1,6 +1,8 @@
 use clap::{App, Arg, ArgGroup};
 use lazy_static::lazy_static;
 use regex::Regex;
+use std::env;
+use std::process;
 
 lazy_static! {
     /// Regex used to validate values passed to --time-limit
@@ -16,7 +18,7 @@ lazy_static! {
 
 /// Create and return an instance of [clap::App](https://docs.rs/clap/latest/clap/struct.App.html), i.e. the Command Line Interface's configuration
 pub fn initialize() -> App<'static, 'static> {
-    App::new("feroxbuster")
+    let mut app = App::new("feroxbuster")
         .version(env!("CARGO_PKG_VERSION"))
         .author("Ben 'epi' Risher (@epi052)")
         .about("A fast, simple, recursive content discovery tool written in Rust")
@@ -410,7 +412,20 @@ EXAMPLES:
 
     Ludicrous speed... go!
         ./feroxbuster -u http://127.1 -t 200
-    "#)
+    "#);
+
+    for arg in env::args() {
+        // secure-77 noticed that when an incorrect flag/option is used, the short help message is printed
+        // which is fine, but if you add -h|--help, it still errors out on the bad flag/option,
+        // never showing the full help message. This code addresses that behavior
+        if arg == "--help" || arg == "-h" {
+            app.print_long_help().unwrap();
+            println!(); // just a newline to mirror original --help output
+            process::exit(1);
+        }
+    }
+
+    app
 }
 
 /// Validate that a string is formatted as a number followed by s, m, h, or d (10d, 30s, etc...)

src/scan_manager/menu.rs

@@ -31,9 +31,10 @@ impl Menu {
         let separator = "─".to_string();
 
         let instructions = format!(
-            "Enter a {} list of indexes to {} (ex: 2,3)",
+            "Enter a {} list of indexes/ranges to {} ({}: 1-4,8,9-13)",
             style("comma-separated").yellow(),
             style("cancel").red(),
+            style("ex").cyan(),
         );
 
         let name = format!(
@@ -43,14 +44,22 @@ impl Menu {
             "💀"
         );
 
+        let force_msg = format!(
+            "Add {} to {} confirmation ({}: 3-5 -f)",
+            style("-f").yellow(),
+            style("skip").yellow(),
+            style("ex").cyan(),
+        );
+
         let longest = measure_text_width(&instructions).max(measure_text_width(&name));
 
         let border = separator.repeat(longest);
 
         let padded_name = pad_str(&name, longest, Alignment::Center, None);
+        let padded_force = pad_str(&force_msg, longest, Alignment::Center, None);
 
         let header = format!("{}\n{}\n{}", border, padded_name, border);
-        let footer = format!("{}\n{}\n{}", border, instructions, border);
+        let footer = format!("{}\n{}\n{}\n{}", border, instructions, padded_force, border);
 
         Self {
             separator,
@@ -93,23 +102,71 @@ impl Menu {
         self.term.write_line(msg).unwrap_or_default();
     }
 
-    /// split a string into vec of usizes
-    pub(super) fn split_to_nums(&self, line: &str) -> Vec<usize> {
-        line.split(',')
-            .map(|s| {
-                s.trim().to_string().parse::<usize>().unwrap_or_else(|e| {
-                    self.println(&format!("Found non-numeric input: {}", e));
-                    0
-                })
+    /// Helper for parsing a usize from a str
+    fn str_to_usize(&self, value: &str) -> usize {
+        if value.is_empty() {
+            return 0;
+        }
+
+        value
+            .trim()
+            .to_string()
+            .parse::<usize>()
+            .unwrap_or_else(|e| {
+                self.println(&format!("Found non-numeric input: {}: {:?}", e, value));
+                0
             })
-            .filter(|m| *m != 0)
-            .collect()
+    }
+
+    /// split a comma delimited string into vec of usizes
+    pub(super) fn split_to_nums(&self, line: &str) -> Vec<usize> {
+        let mut nums = Vec::new();
+        let values = line.split(',');
+
+        for mut value in values {
+            value = value.trim();
+
+            if value.contains('-') {
+                // range of two values, needs further processing
+
+                let range: Vec<usize> = value
+                    .split('-')
+                    .map(|s| self.str_to_usize(s))
+                    .filter(|m| *m != 0)
+                    .collect();
+
+                if range.len() != 2 {
+                    // expecting [1, 4] or similar, if a 0 was used, we'd be left with a vec of size 1
+                    self.println(&format!("Found invalid range of scans: {}", value));
+                    continue;
+                }
+
+                (range[0]..=range[1]).for_each(|n| {
+                    // iterate from lower to upper bound and add all interim values, skipping
+                    // any already known
+                    if !nums.contains(&n) {
+                        nums.push(n)
+                    }
+                });
+            } else {
+                let value = self.str_to_usize(value);
+
+                if value != 0 && !nums.contains(&value) {
+                    // the zeroth scan is always skipped, skip already known values
+                    nums.push(value);
+                }
+            }
+        }
+
+        nums
     }
 
     /// get comma-separated list of scan indexes from the user
-    pub(super) fn get_scans_from_user(&self) -> Option<Vec<usize>> {
+    pub(super) fn get_scans_from_user(&self) -> Option<(Vec<usize>, bool)> {
         if let Ok(line) = self.term.read_line() {
-            Some(self.split_to_nums(&line))
+            let force = line.contains("-f");
+            let line = line.replace("-f", "");
+            Some((self.split_to_nums(&line), force))
         } else {
             None
         }

src/scan_manager/scan_container.rs

@@ -252,7 +252,7 @@ impl FeroxScans {
     }
 
     /// Given a list of indexes, cancel their associated FeroxScans
-    async fn cancel_scans(&self, indexes: Vec<usize>) -> usize {
+    async fn cancel_scans(&self, indexes: Vec<usize>, force: bool) -> usize {
         let menu_pause_duration = Duration::from_millis(SLEEP_DURATION);
 
         let mut num_cancelled = 0_usize;
@@ -273,7 +273,11 @@ impl FeroxScans {
                 Err(..) => continue,
             };
 
-            let input = self.menu.confirm_cancellation(&selected.url);
+            let input = if force {
+                'y'
+            } else {
+                self.menu.confirm_cancellation(&selected.url)
+            };
 
             if input == 'y' || input == '\n' {
                 self.menu.println(&format!("Stopping {}...", selected.url));
@@ -305,8 +309,8 @@ impl FeroxScans {
 
         let mut num_cancelled = 0_usize;
 
-        if let Some(input) = self.menu.get_scans_from_user() {
-            num_cancelled += self.cancel_scans(input).await;
+        if let Some((input, force)) = self.menu.get_scans_from_user() {
+            num_cancelled += self.cancel_scans(input, force).await;
         };
 
         self.menu.clear_screen();

src/scan_manager/state.rs

@@ -47,7 +47,7 @@ impl FeroxSerialize for FeroxState {
 
     /// Simple call to produce a JSON string using the given FeroxState
     fn as_json(&self) -> Result<String> {
-        Ok(serde_json::to_string(&self)
-            .with_context(|| fmt_err("Could not convert scan's running state to JSON"))?)
+        serde_json::to_string(&self)
+            .with_context(|| fmt_err("Could not convert scan's running state to JSON"))
     }
 }

src/scan_manager/tests.rs

@@ -521,9 +521,13 @@ fn menu_print_header_and_footer() {
 fn split_to_nums_is_correct() {
     let menu = Menu::new();
 
-    let nums = menu.split_to_nums("1, 3,      4");
+    let nums = menu.split_to_nums("1, 3,      4, 7 -     12, 10-10, 10-11, 9-12, 12-6, -1, 4-");
 
-    assert_eq!(nums, vec![1, 3, 4]);
+    assert_eq!(nums, vec![1, 3, 4, 7, 8, 9, 10, 11, 12]);
+    assert_eq!(menu.split_to_nums("9-12"), vec![9, 10, 11, 12]);
+    assert!(menu.split_to_nums("-12").is_empty());
+    assert!(menu.split_to_nums("12-").is_empty());
+    assert!(menu.split_to_nums("\n").is_empty());
 }
 
 #[test]

src/scan_manager/utils.rs

@@ -41,7 +41,7 @@ pub async fn start_max_time_thread(handles: Arc<Handles>) {
         log::trace!("exit: start_max_time_thread");
 
         #[cfg(test)]
-        panic!(handles);
+        panic!("{:?}", handles);
         #[cfg(not(test))]
         let _ = TermInputHandler::sigint_handler(handles.clone());
     }

tests/test_parser.rs

@@ -0,0 +1,46 @@
+use assert_cmd::Command;
+use predicates::prelude::*;
+
+#[test]
+/// specify an incorrect param (-fc) with --help after it on the command line
+/// old behavior printed
+/// error: Found argument '-c' which wasn't expected, or isn't valid in this context
+///
+/// USAGE:
+///     feroxbuster --add-slash --url <URL>...
+///
+/// For more information try --help
+///
+/// the new behavior we expect to see is to print the long form help message, of which
+/// Ludicrous speed... go! is near the bottom of that output, so we can test for that
+fn parser_incorrect_param_with_tack_tack_help() {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("-fc")
+        .arg("--help")
+        .assert()
+        .failure()
+        .stdout(predicate::str::contains("Ludicrous speed... go!"));
+}
+
+#[test]
+/// specify an incorrect param (-fc) with --help after it on the command line
+/// old behavior printed
+/// error: Found argument '-c' which wasn't expected, or isn't valid in this context
+///
+/// USAGE:
+///     feroxbuster --add-slash --url <URL>...
+///
+/// For more information try --help
+///
+/// the new behavior we expect to see is to print the long form help message, of which
+/// Ludicrous speed... go! is near the bottom of that output, so we can test for that
+fn parser_incorrect_param_with_tack_h() {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("-fc")
+        .arg("-h")
+        .assert()
+        .failure()
+        .stdout(predicate::str::contains("Ludicrous speed... go!"));
+}