h/feroxbuster
1
Fork 0
mirror of https://github.com/epi052/feroxbuster.git synced 2026-06-03 06:41:12 -03:00
Code Issues Packages Projects Releases Wiki Activity

parallel time limit enforced individually instead of main thread (#1072)

Browse Source 
* parallel time limit enforced individually instead of main thread
* added ability to select silent/quiet when using --parallel
* fixed robots.txt error -> hang
* build pipeline back to main
* fixed up tests
* removed retries from nextest
* clippy
This commit is contained in:
epi
2024-02-28 06:59:43 -05:00
committed by GitHub
parent 423889b142
commit 9ff0253deb
14 changed files with 955 additions and 633 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
.github/workflows/build.yml vendored
View File

@@ -102,22 +102,28 @@ jobs:
name: x86_64-linux-debug-feroxbuster
path: target/x86_64-unknown-linux-musl/debug/feroxbuster
# build-deb:
# needs: [build-nix]
# runs-on: ubuntu-latest
# steps:
# - uses: actions/checkout@master
# - name: Install cargo-deb
# run: cargo install -f cargo-deb
# - name: Install musl toolchain
# run: rustup target add x86_64-unknown-linux-musl
# - name: Deb Build
# run: cargo deb --target=x86_64-unknown-linux-musl
# - name: Upload Deb Artifact
# uses: actions/upload-artifact@v2
# with:
# name: feroxbuster_amd64.deb
# path: ./target/x86_64-unknown-linux-musl/debian/*
build-deb:
needs: [build-nix]
runs-on: ubuntu-latest
env:
IN_PIPELINE: true
steps:
- uses: actions/checkout@master
- name: Install cargo-deb
run: cargo install -f cargo-deb
- uses: awalsh128/cache-apt-pkgs-action@v1
with:
packages: musl-tools # provides musl-gcc
version: 1.0
- name: Install musl toolchain
run: rustup target add x86_64-unknown-linux-musl
- name: Deb Build
run: cargo deb --target=x86_64-unknown-linux-musl
- name: Upload Deb Artifact
uses: actions/upload-artifact@v2
with:
name: feroxbuster_amd64.deb
path: ./target/x86_64-unknown-linux-musl/debian/*
build-macos:
env:

2
.github/workflows/check.yml vendored
View File

@@ -23,7 +23,7 @@ jobs:
uses: actions-rs/cargo@v1
with:
command: nextest
args: run --all-features --all-targets --retries 10
args: run --all-features --all-targets
fmt:
name: Rust fmt

1354
Cargo.lock generated
View File

File diff suppressed because it is too large Load Diff

4
Makefile.toml
View File

@@ -11,7 +11,7 @@ rm ferox-*.state
# dependency management
[tasks.upgrade-deps]
command = "cargo"
args = ["upgrade", "--to-lockfile", "--exclude", "indicatif", "self_update"]
args = ["upgrade", "--exclude", "indicatif, self_update"]
[tasks.update]
command = "cargo"
@@ -28,5 +28,5 @@ cargo clippy --all-targets --all-features -- -D warnings
[tasks.test]
clear = true
script = """
cargo nextest run --all-features --all-targets --retries 10
cargo nextest run --all-features --all-targets
"""

2
shell_completions/_feroxbuster
View File

@@ -62,7 +62,7 @@ _feroxbuster() {
'--depth=[Maximum recursion depth, a depth of 0 is infinite recursion (default\: 4)]:RECURSION_DEPTH: ' \
'-L+[Limit total number of concurrent scans (default\: 0, i.e. no limit)]:SCAN_LIMIT: ' \
'--scan-limit=[Limit total number of concurrent scans (default\: 0, i.e. no limit)]:SCAN_LIMIT: ' \
'--parallel=[Run parallel feroxbuster instances (one child process per url passed via stdin)]:PARALLEL_SCANS: ' \
'(-v --verbosity)--parallel=[Run parallel feroxbuster instances (one child process per url passed via stdin)]:PARALLEL_SCANS: ' \
'(--auto-tune)--rate-limit=[Limit number of requests per second (per directory) (default\: 0, i.e. no limit)]:RATE_LIMIT: ' \
'--time-limit=[Limit total run time of all scans (ex\: --time-limit 10m)]:TIME_SPEC: ' \
'-w+[Path or URL of the wordlist]:FILE:_files' \

105
shell_completions/feroxbuster.bash
View File

@@ -34,7 +34,18 @@ _feroxbuster() {
return 0
;;
--resume-from)
local oldifs
if [[ -v IFS ]]; then
oldifs="$IFS"
fi
IFS=$'\n'
COMPREPLY=($(compgen -f "${cur}"))
if [[ -v oldifs ]]; then
IFS="$oldifs"
fi
if [[ "${BASH_VERSINFO[0]}" -ge 4 ]]; then
compopt -o filenames
fi
return 0
;;
--proxy)
@@ -178,15 +189,48 @@ _feroxbuster() {
return 0
;;
--server-certs)
local oldifs
if [[ -v IFS ]]; then
oldifs="$IFS"
fi
IFS=$'\n'
COMPREPLY=($(compgen -f "${cur}"))
if [[ -v oldifs ]]; then
IFS="$oldifs"
fi
if [[ "${BASH_VERSINFO[0]}" -ge 4 ]]; then
compopt -o filenames
fi
return 0
;;
--client-cert)
local oldifs
if [[ -v IFS ]]; then
oldifs="$IFS"
fi
IFS=$'\n'
COMPREPLY=($(compgen -f "${cur}"))
if [[ -v oldifs ]]; then
IFS="$oldifs"
fi
if [[ "${BASH_VERSINFO[0]}" -ge 4 ]]; then
compopt -o filenames
fi
return 0
;;
--client-key)
local oldifs
if [[ -v IFS ]]; then
oldifs="$IFS"
fi
IFS=$'\n'
COMPREPLY=($(compgen -f "${cur}"))
if [[ -v oldifs ]]; then
IFS="$oldifs"
fi
if [[ "${BASH_VERSINFO[0]}" -ge 4 ]]; then
compopt -o filenames
fi
return 0
;;
--threads)
@@ -226,11 +270,33 @@ _feroxbuster() {
return 0
;;
--wordlist)
local oldifs
if [[ -v IFS ]]; then
oldifs="$IFS"
fi
IFS=$'\n'
COMPREPLY=($(compgen -f "${cur}"))
if [[ -v oldifs ]]; then
IFS="$oldifs"
fi
if [[ "${BASH_VERSINFO[0]}" -ge 4 ]]; then
compopt -o filenames
fi
return 0
;;
-w)
local oldifs
if [[ -v IFS ]]; then
oldifs="$IFS"
fi
IFS=$'\n'
COMPREPLY=($(compgen -f "${cur}"))
if [[ -v oldifs ]]; then
IFS="$oldifs"
fi
if [[ "${BASH_VERSINFO[0]}" -ge 4 ]]; then
compopt -o filenames
fi
return 0
;;
--collect-backups)
@@ -250,15 +316,48 @@ _feroxbuster() {
return 0
;;
--output)
local oldifs
if [[ -v IFS ]]; then
oldifs="$IFS"
fi
IFS=$'\n'
COMPREPLY=($(compgen -f "${cur}"))
if [[ -v oldifs ]]; then
IFS="$oldifs"
fi
if [[ "${BASH_VERSINFO[0]}" -ge 4 ]]; then
compopt -o filenames
fi
return 0
;;
-o)
local oldifs
if [[ -v IFS ]]; then
oldifs="$IFS"
fi
IFS=$'\n'
COMPREPLY=($(compgen -f "${cur}"))
if [[ -v oldifs ]]; then
IFS="$oldifs"
fi
if [[ "${BASH_VERSINFO[0]}" -ge 4 ]]; then
compopt -o filenames
fi
return 0
;;
--debug-log)
local oldifs
if [[ -v IFS ]]; then
oldifs="$IFS"
fi
IFS=$'\n'
COMPREPLY=($(compgen -f "${cur}"))
if [[ -v oldifs ]]; then
IFS="$oldifs"
fi
if [[ "${BASH_VERSINFO[0]}" -ge 4 ]]; then
compopt -o filenames
fi
return 0
;;
*)
@@ -271,4 +370,8 @@ _feroxbuster() {
esac
}
complete -F _feroxbuster -o bashdefault -o default -o plusdirs feroxbuster
if [[ "${BASH_VERSINFO[0]}" -eq 4 && "${BASH_VERSINFO[1]}" -ge 4 || "${BASH_VERSINFO[0]}" -gt 4 ]]; then
complete -F _feroxbuster -o nosort -o bashdefault -o default -o plusdirs feroxbuster
else
complete -F _feroxbuster -o bashdefault -o default -o plusdirs feroxbuster
fi

6
src/config/container.rs
View File

@@ -21,7 +21,7 @@ use std::{
collections::HashMap,
env::{current_dir, current_exe},
fs::read_to_string,
path::PathBuf,
path::{Path, PathBuf},
};
/// macro helper to abstract away repetitive configuration updates
@@ -576,9 +576,7 @@ impl Configuration {
// - current directory
// merge a config found at /etc/feroxbuster/ferox-config.toml
let config_file = PathBuf::new()
.join("/etc/feroxbuster")
.join(DEFAULT_CONFIG_NAME);
let config_file = Path::new("/etc/feroxbuster").join(DEFAULT_CONFIG_NAME);
Self::parse_and_merge_config(config_file, config)?;
// merge a config found at ~/.config/feroxbuster/ferox-config.toml

2
src/event_handlers/statistics.rs
View File

@@ -132,7 +132,7 @@ impl StatsHandler {
self.bar.finish();
log::debug!("{:#?}", *self.stats);
log::info!("{:#?}", *self.stats);
log::trace!("exit: start");
Ok(())
}

45
src/main.rs
View File

@@ -8,7 +8,7 @@ use std::{
io::{stderr, BufRead, BufReader},
ops::Index,
path::Path,
process::{exit, Command},
process::{exit, Command, Stdio},
sync::{atomic::Ordering, Arc},
};
@@ -325,9 +325,15 @@ async fn wrapped_main(config: Arc<Configuration>) -> Result<()> {
// create new Tasks object, each of these handles is one that will be joined on later
let tasks = Tasks::new(out_task, stats_task, filters_task, scan_task);
if !config.time_limit.is_empty() {
if !config.time_limit.is_empty() && config.parallel == 0 {
// --time-limit value not an empty string, need to kick off the thread that enforces
// the limit
//
// if --parallel is used, this branch won't execute in the main process, but will in the
// children. This is because --parallel is stripped from the children's command line
// arguments, so, when spawned, they won't have --parallel, the parallel value will be set
// to the default of 0, and will hit this branch. This makes it so that the time limit
// is enforced on each individual child process, instead of the main process
let time_handles = handles.clone();
tokio::spawn(async move { scan_manager::start_max_time_thread(time_handles).await });
}
@@ -370,8 +376,7 @@ async fn wrapped_main(config: Arc<Configuration>) -> Result<()> {
let invocation = args();
let para_regex =
Regex::new("--stdin|-q|--quiet|--silent|--verbosity|-v|-vv|-vvv|-vvvv").unwrap();
let para_regex = Regex::new("--stdin").unwrap();
// remove stdin since only the original process will process targets
// remove quiet and silent so we can force silent later to normalize output
@@ -379,8 +384,6 @@ async fn wrapped_main(config: Arc<Configuration>) -> Result<()> {
.filter(|s| !para_regex.is_match(s))
.collect::<Vec<String>>();
original.push("--silent".to_string()); // only output modifier allowed
// we need remove --parallel from command line so we don't hit this branch over and over
// but we must remove --parallel N manually; the filter above never sees --parallel and the
// value passed to it at the same time, so can't filter them out in one pass
@@ -455,16 +458,32 @@ async fn wrapped_main(config: Arc<Configuration>) -> Result<()> {
log::debug!("parallel exec: {} {}", bin, args.join(" "));
tokio::task::spawn_blocking(move || {
let result = Command::new(bin)
tokio::task::spawn(async move {
let mut output = Command::new(bin)
.args(&args)
.stdout(Stdio::piped())
.spawn()
.expect("failed to spawn a child process")
.wait()
.expect("child process errored during execution");
.expect("failed to spawn a child process");
let stdout = output.stdout.take().unwrap();
let mut bufread = BufReader::new(stdout);
// output for a single line is a minimum of 51 bytes, so we'll start with that
// + a little wiggle room, and grow as needed
let mut buf: String = String::with_capacity(128);
while let Ok(n) = bufread.read_line(&mut buf) {
if n > 0 {
let trimmed = buf.trim();
if !trimmed.is_empty() {
println!("{}", trimmed);
}
buf.clear();
} else {
break;
}
}
drop(permit);
result
});
}
@@ -646,7 +665,7 @@ fn main() -> Result<()> {
// print the banner to stderr
let std_stderr = stderr(); // std::io::stderr
let banner = Banner::new(&targets, &config);
if !config.quiet && !config.silent {
if (!config.quiet && !config.silent) || config.parallel != 0 {
banner.print_to(std_stderr, config).unwrap();
}
}

7
src/parser.rs
View File

@@ -485,8 +485,10 @@ pub fn initialize() -> Command {
Arg::new("parallel")
.long("parallel")
.value_name("PARALLEL_SCANS")
.conflicts_with("verbosity")
.num_args(1)
.requires("stdin")
.requires("output_limiters")
.help_heading("Scan settings")
.help("Run parallel feroxbuster instances (one child process per url passed via stdin)")
)
@@ -647,6 +649,11 @@ pub fn initialize() -> Command {
.args(["debug_log", "output", "silent"])
.multiple(true),
)
.group(
ArgGroup::new("output_limiters")
.args(["quiet", "silent"])
.multiple(false),
)
.arg(
Arg::new("update_app")
.short('U')

6
src/scanner/ferox_scanner.rs
View File

@@ -214,9 +214,9 @@ impl FeroxScanner {
.url(&self.target_url)
.handles(self.handles.clone())
.build()?;
let result = extractor.extract().await?;
extraction_tasks.push(extractor.request_links(result).await?)
if let Ok(result) = extractor.extract().await {
extraction_tasks.push(extractor.request_links(result).await?)
}
}
let scanned_urls = self.handles.ferox_scans()?;

1
tests/test_banner.rs
View File

@@ -1146,6 +1146,7 @@ fn banner_prints_parallel() {
Command::cargo_bin("feroxbuster")
.unwrap()
.arg("--stdin")
.arg("--quiet")
.arg("--parallel")
.arg("4316")
.arg("--wordlist")

4
tests/test_main.rs
View File

@@ -108,10 +108,11 @@ fn main_parallel_spawns_children() -> Result<(), Box<dyn std::error::Error>> {
Command::cargo_bin("feroxbuster")
.unwrap()
.env("RUST_LOG", "trace")
.arg("--stdin")
.arg("--parallel")
.arg("2")
.arg("-vvvv")
.arg("--quiet")
.arg("--debug-log")
.arg(outfile.as_os_str())
.arg("--wordlist")
@@ -172,6 +173,7 @@ fn main_parallel_creates_output_directory() -> Result<(), Box<dyn std::error::Er
Command::cargo_bin("feroxbuster")
.unwrap()
.arg("--stdin")
.arg("--quiet")
.arg("--parallel")
.arg("2")
.arg("--output")

12
tests/test_policies.rs
View File

@@ -37,7 +37,7 @@ fn auto_bail_cancels_scan_with_timeouts() {
let error_mock = srv.mock(|when, then| {
when.method(GET)
.path_matches(Regex::new("/[a-zA-Z]{6}error[a-zA-Z]{6}").unwrap());
then.delay(Duration::new(3, 0))
then.delay(Duration::new(2, 5000))
.status(200)
.body("verboten, nerd");
});
@@ -57,7 +57,7 @@ fn auto_bail_cancels_scan_with_timeouts() {
Command::cargo_bin("feroxbuster")
.unwrap()
.arg("--url")
.arg(srv.url("/"))
.arg(&srv.url("/"))
.arg("--wordlist")
.arg(file.as_os_str())
.arg("--auto-bail")
@@ -65,10 +65,10 @@ fn auto_bail_cancels_scan_with_timeouts() {
.arg("--timeout")
.arg("2")
.arg("--threads")
.arg("4")
.arg("8")
.arg("--debug-log")
.arg(logfile.as_os_str())
.arg("-vvvv")
.arg("-vv")
.arg("--json")
.assert()
.success();
@@ -146,7 +146,7 @@ fn auto_bail_cancels_scan_with_403s() {
.arg("4")
.arg("--debug-log")
.arg(logfile.as_os_str())
.arg("-vvvv")
.arg("-vv")
.arg("--json")
.assert()
.success();
@@ -228,7 +228,7 @@ fn auto_bail_cancels_scan_with_429s() {
.arg("4")
.arg("--debug-log")
.arg(logfile.as_os_str())
.arg("-vvvv")
.arg("-vvv")
.arg("--json")
.assert()
.success();