Compare commits

..

38 Commits

Author SHA1 Message Date
epi
0ea798e70e Merge pull request #150 from epi052/allow-cli-override-resume-from
allow CLI args to work w/ --resume-from
2020-12-09 08:40:49 -06:00
epi
3caa8d2ceb comment lint and corrections 2020-12-09 07:52:33 -06:00
epi
bd836c8b55 fixed test with hardcoded ferox version in expected msg 2020-12-08 20:49:38 -06:00
epi
f3bf05ab9b fixed replay proxy issue 2020-12-08 20:37:07 -06:00
epi
ef0b5d3780 Update pull_request_template.md 2020-12-08 15:37:20 -06:00
epi
ab5fbeb6ed feature appears to be working; tests passing 2020-12-08 15:35:11 -06:00
epi
6c779bd4c1 added shell completion scripts and build.rs; closes #146 2020-12-04 07:08:49 -06:00
epi
fbb964a893 added emoji font to Dockerfile 2020-12-04 06:21:07 -06:00
epi
4f1f63671e Merge pull request #147 from epi052/144-resume-scan
added 1.9 images to repo
2020-12-03 19:42:06 -06:00
epi
5578e8db5c added 1.9 images to repo 2020-12-03 19:41:17 -06:00
epi
5a93907d74 Merge pull request #145 from epi052/144-resume-scan
add ability to resume scans
2020-12-03 19:37:47 -06:00
epi
1d4403b497 CI still doesnt like the new addr_of stuff, reverted 2020-12-03 07:09:40 -06:00
epi
6939884a95 removed addr_of suppression from clippy 2020-12-02 20:30:30 -06:00
epi
509f09165a added documentation for 1.9.0; added save_state to example config 2020-12-02 20:10:34 -06:00
epi
40d8e1b76a added integration test for --resume-from 2020-12-01 17:00:09 -06:00
epi
da1c085f4a added integration test for --resume-from 2020-12-01 16:59:30 -06:00
epi
53281c0921 added more tests for scan_manager 2020-12-01 07:54:31 -06:00
epi
b9cf9b5558 added more tests for scan_manager 2020-12-01 07:31:57 -06:00
epi
295500a746 added more tests for scan_manager 2020-12-01 07:04:17 -06:00
epi
b1f77d202d added test for progress 2020-12-01 06:07:38 -06:00
epi
5a29f5fbb1 added progress test 2020-11-30 20:44:57 -06:00
epi
1d6e4374c0 simplified test, removed possible fail condition 2020-11-30 18:47:33 -06:00
epi
eaa7d1c790 added test for ferox response; fixed bug found in status code deserialization 2020-11-30 18:45:04 -06:00
epi
f29cd16616 added a few more tests 2020-11-29 20:14:42 -06:00
epi
1279ad6e68 updated json test 2020-11-29 18:24:44 -06:00
epi
8d4ba43cbe added deserialize test for FeroxScan 2020-11-29 17:40:34 -06:00
epi
d2562a5e0a resume appears to be fully implemented, just need tests 2020-11-29 10:12:53 -06:00
epi
a1d67afb72 resume appears to be fully implemented, just need tests 2020-11-29 10:12:38 -06:00
epi
fd61b8506b json can be used with both output files at the same time 2020-11-28 12:14:28 -06:00
epi
75babad426 made resume-from mutually exclusive with all other settings; json now requires one of the output files 2020-11-28 12:11:30 -06:00
epi
2b64030c0c all three types can be deserialized from state file 2020-11-28 09:29:09 -06:00
epi
26fcf457e6 added serialization/deserialization of a few different types 2020-11-28 07:27:58 -06:00
epi
26bf1e482d added logic for tracking responses 2020-11-28 07:25:40 -06:00
epi
107eac7e25 added --resume-from option to the parser 2020-11-28 07:15:47 -06:00
epi
e2b442ab0b added logic to kickoff ctrlc handler in main 2020-11-28 07:11:57 -06:00
epi
b822a5d862 added client config logic to resume_scan call branch 2020-11-28 07:11:02 -06:00
epi
dc4e41305e added ctrlc crate 2020-11-28 07:09:24 -06:00
epi
fdfb4cff64 bumped version to 1.9.0 2020-11-27 06:42:19 -06:00
26 changed files with 1588 additions and 229 deletions

View File

@@ -4,7 +4,7 @@ Long form explanations of most of the items below can be found in the [CONTRIBUT
## Branching checklist
- [ ] There is an issue associated with your PR (bug, feature, etc.. if not, create one)
- [ ] Your PR description references the associated issue (i.e. fixes #123)
- [ ] Your PR description references the associated issue (i.e. fixes #123456)
- [ ] Code is in its own branch
- [ ] Branch name is related to the PR contents
- [ ] PR targets master

View File

@@ -61,4 +61,4 @@ jobs:
- uses: actions-rs/cargo@v1
with:
command: clippy
args: --all-targets --all-features -- -D warnings -A clippy::unnecessary_unwrap -A clippy::deref_addrof
args: --all-targets --all-features -- -D warnings -A clippy::deref_addrof

3
.gitignore vendored
View File

@@ -22,3 +22,6 @@ img/**
# scripts to check code coverage using nightly compiler
check-coverage.sh
lcov_cobertura.py
# dockerignore file that makes it so i can work on the docker config without copying a 4GB manifest or w/e it is
.dockerignore

View File

@@ -1,6 +1,6 @@
[package]
name = "feroxbuster"
version = "1.8.0"
version = "1.9.1"
authors = ["Ben 'epi' Risher <epibar052@gmail.com>"]
license = "MIT"
edition = "2018"
@@ -10,10 +10,14 @@ description = "A fast, simple, recursive content discovery tool."
categories = ["command-line-utilities"]
keywords = ["pentest", "enumeration", "url-bruteforce", "content-discovery", "web"]
exclude = [".github/*", "img/*", "check-coverage.sh"]
build = "build.rs"
[badges]
maintenance = { status = "actively-developed" }
[build-dependencies]
clap = "2"
[dependencies]
futures = { version = "0.3"}
tokio = { version = "0.2", features = ["full"] }
@@ -34,6 +38,7 @@ dirs = "3.0"
regex = "1"
crossterm = "0.18"
rlimit = "0.5"
ctrlc = "3.1"
[dev-dependencies]
tempfile = "3.1"

View File

@@ -1,8 +1,10 @@
FROM alpine:latest
LABEL maintainer="wfnintr@null.net"
RUN sed -i -e 's/v[[:digit:]]\..*\//edge\//g' /etc/apk/repositories && apk upgrade --update-cache --available
# download default wordlists
RUN apk add --no-cache --virtual .depends subversion && \
RUN apk add --no-cache --virtual .depends subversion font-noto-emoji && \
svn export https://github.com/danielmiessler/SecLists/trunk/Discovery/Web-Content /usr/share/seclists/Discovery/Web-Content && \
apk del .depends

165
README.md
View File

@@ -73,19 +73,20 @@ This attack is also known as Predictable Resource Location, File Enumeration, Di
- [ferox-config.toml](#ferox-configtoml)
- [Command Line Parsing](#command-line-parsing)
- [Example Usage](#-example-usage)
- [Pause and Resume Scans (new in `v1.4.0`)](#pause-and-resume-scans-new-in-v140)
- [Multiple Values](#multiple-values)
- [Extract Links from Response Body (new in `v1.1.0`)](#extract-links-from-response-body-new-in-v110)
- [Include Headers](#include-headers)
- [IPv6, Non-recursive scan with INFO logging enabled](#ipv6-non-recursive-scan-with-info-level-logging-enabled)
- [Read urls from STDIN; pipe only resulting urls out to another tool](#read-urls-from-stdin-pipe-only-resulting-urls-out-to-another-tool)
- [Proxy traffic through Burp](#proxy-traffic-through-burp)
- [Proxy traffic through a SOCKS proxy](#proxy-traffic-through-a-socks-proxy)
- [Pass auth token via query parameter](#pass-auth-token-via-query-parameter)
- [Extract Links from Response Body (new in `v1.1.0`)](#extract-links-from-response-body-new-in-v110)
- [Limit Total Number of Concurrent Scans (new in `v1.2.0`)](#limit-total-number-of-concurrent-scans-new-in-v120)
- [Filter Response by Status Code (new in `v1.3.0`)](#filter-response-by-status-code--new-in-v130)
- [Filter Response Using a Regular Expression (new in `v1.8.0`)](#filter-response-using-a-regular-expression-new-in-v180)
- [Pause an Active Scan (new in `v1.4.0`)](#pause-an-active-scan-new-in-v140)
- [Replay Responses to a Proxy based on Status Code (new in `v1.5.0`)](#replay-responses-to-a-proxy-based-on-status-code-new-in-v150)
- [Filter Response Using a Regular Expression (new in `v1.8.0`)](#filter-response-using-a-regular-expression-new-in-v180)
- [Stop and Resume Scans (save scan's state to disk) (new in `v1.9.0`)](#stop-and-resume-scans---resume-from-file-new-in-v190)
- [Comparison w/ Similar Tools](#-comparison-w-similar-tools)
- [Common Problems/Issues (FAQ)](#-common-problemsissues-faq)
- [No file descriptors available](#no-file-descriptors-available)
@@ -257,6 +258,7 @@ Configuration begins with with the following built-in default values baked into
- recursion depth: `4`
- auto-filter wildcards - `true`
- output: `stdout`
- save_state: `true` (create a state file in cwd when `Ctrl+C` is received)
### Threads and Connection Limits At A High-Level
@@ -348,6 +350,7 @@ A pre-made configuration file with examples of all available settings can be fou
# filter_word_count = [993]
# filter_line_count = [35, 36]
# queries = [["name","value"], ["rick", "astley"]]
# save_state = false
# headers can be specified on multiple lines or as an inline table
#
@@ -404,6 +407,8 @@ OPTIONS:
-codes value)
-P, --replay-proxy <REPLAY_PROXY> Send only unfiltered requests through a Replay Proxy, instead of all
requests
--resume-from <STATE_FILE> State file from which to resume a partially complete scan (ex. --resume-from
ferox-1606586780.state)
-L, --scan-limit <SCAN_LIMIT> Limit total number of concurrent scans (default: 0, i.e. no limit)
-s, --status-codes <STATUS_CODE>... Status Codes to include (allow list) (default: 200 204 301 302 307 308 401
403 405)
@@ -416,12 +421,6 @@ OPTIONS:
## 🧰 Example Usage
### Pause and Resume Scans (new in `v1.4.0`)
Scans can be paused and resumed by pressing the ENTER key (shown below)
![pause-resume-demo](img/pause-resume-demo.gif)
### Multiple Values
Options that take multiple values are very flexible. Consider the following ways of specifying extensions:
@@ -440,6 +439,36 @@ All of the methods above (multiple flags, space separated, comma separated, etc.
./feroxbuster -u http://127.1 -H Accept:application/json "Authorization: Bearer {token}"
```
### IPv6, non-recursive scan with INFO-level logging enabled
```
./feroxbuster -u http://[::1] --no-recursion -vv
```
### Read urls from STDIN; pipe only resulting urls out to another tool
```
cat targets | ./feroxbuster --stdin --quiet -s 200 301 302 --redirects -x js | fff -s 200 -o js-files
```
### Proxy traffic through Burp
```
./feroxbuster -u http://127.1 --insecure --proxy http://127.0.0.1:8080
```
### Proxy traffic through a SOCKS proxy
```
./feroxbuster -u http://127.1 --proxy socks5://127.0.0.1:9050
```
### Pass auth token via query parameter
```
./feroxbuster -u http://127.1 --query token=0123456789ABCDEF
```
### Extract Links from Response Body (New in `v1.1.0`)
Search through the body of valid responses (html, javascript, etc...) for additional endpoints to scan. This turns
@@ -470,37 +499,6 @@ With `--extract-links`
![extract-scan-cmp-normal](img/extract-scan-cmp-normal.gif)
### IPv6, non-recursive scan with INFO-level logging enabled
```
./feroxbuster -u http://[::1] --no-recursion -vv
```
### Read urls from STDIN; pipe only resulting urls out to another tool
```
cat targets | ./feroxbuster --stdin --quiet -s 200 301 302 --redirects -x js | fff -s 200 -o js-files
```
### Proxy traffic through Burp
```
./feroxbuster -u http://127.1 --insecure --proxy http://127.0.0.1:8080
```
### Proxy traffic through a SOCKS proxy
```
./feroxbuster -u http://127.1 --proxy socks5://127.0.0.1:9050
```
### Pass auth token via query parameter
```
./feroxbuster -u http://127.1 --query token=0123456789ABCDEF
```
### Limit Total Number of Concurrent Scans (new in `v1.2.0`)
Limit the number of scans permitted to run at any given time. Recursion will still identify new directories, but newly
@@ -523,18 +521,11 @@ each one is checked against a list of known filters and either displayed or not
./feroxbuster -u http://127.1 --filter-status 301
```
### Filter Response Using a Regular Expression (new in `v1.8.0`)
### Pause an Active Scan (new in `v1.4.0`)
Version 1.3.0 included an overhaul to the filtering system which will allow for a wide array of filters to be added
with minimal effort. The latest addition is a Regular Expression Filter. As responses come back from the scanned server,
the **body** of the response is checked against the filter's regular expression. If the expression is found in the
body, then that response is filtered out.
Scans can be paused and resumed by pressing the ENTER key (shown below)
**NOTE: Using regular expressions to filter large responses or many regular expressions may negatively impact performance.**
```
./feroxbuster -u http://127.1 --filter-regex '[aA]ccess [dD]enied.?' --output results.txt --json
```
![pause-resume-demo](img/pause-resume-demo.gif)
### Replay Responses to a Proxy based on Status Code (new in `v1.5.0`)
@@ -550,6 +541,80 @@ Of note: this means that for every response that matches your replay criteria, y
![replay-proxy-demo](img/replay-proxy-demo.gif)
### Filter Response Using a Regular Expression (new in `v1.8.0`)
Version 1.3.0 included an overhaul to the filtering system which will allow for a wide array of filters to be added
with minimal effort. The latest addition is a Regular Expression Filter. As responses come back from the scanned server,
the **body** of the response is checked against the filter's regular expression. If the expression is found in the
body, then that response is filtered out.
**NOTE: Using regular expressions to filter large responses or many regular expressions may negatively impact performance.**
```
./feroxbuster -u http://127.1 --filter-regex '[aA]ccess [dD]enied.?' --output results.txt --json
```
### Stop and Resume Scans (`--resume-from FILE`) (new in `v1.9.0`)
Version 1.9.0 adds a few features that allow for completely stopping a scan, and resuming that same scan from a file on disk.
A simple `Ctrl+C` during a scan will create a file that contains information about the scan that was cancelled.
![save-state](img/save-state.png)
```json
// example snippet of state file
{
"scans":[
{
"id":"057016a14769414aac9a7a62707598cb",
"url":"https://localhost.com",
"scan_type":"Directory",
"complete":true
},
{
"id":"400b2323a16f43468a04ffcbbeba34c6",
"url":"https://localhost.com/css",
"scan_type":"Directory",
"complete":false
}
],
"config":{
"wordlist":"/wordlists/seclists/Discovery/Web-Content/common.txt",
"...":"..."
},
"responses":[
{
"type":"response",
"url":"https://localhost.com/Login",
"path":"/Login",
"wildcard":false,
"status":302,
"content_length":0,
"line_count":0,
"word_count":0,
"headers":{
"content-length":"0",
"server":"nginx/1.16.1"
}
}
]
},
```
Based on the example image above, the same scan can be resumed by using `feroxbuster --resume-from ferox-http_localhost-1606947491.state`. Directories that were already complete are not rescanned, however partially complete scans are started from the beginning.
![resumed-scan](img/resumed-scan.gif)
In order to prevent state file creation when `Ctrl+C` is pressed, you can simply add the entry below to your `ferox-config.toml`.
```toml
# ferox-config.toml
save_state = false
```
## 🧐 Comparison w/ Similar Tools
There are quite a few similar tools for forced browsing/content discovery. Burp Suite Pro, Dirb, Dirbuster, etc...

17
build.rs Normal file
View File

@@ -0,0 +1,17 @@
extern crate clap;
use clap::Shell;
include!("src/parser.rs");
fn main() {
let outdir = "shell_completions";
let mut app = initialize();
let shells: [Shell; 4] = [Shell::Bash, Shell::Fish, Shell::Zsh, Shell::PowerShell];
for shell in &shells {
app.gen_completions("feroxbuster", *shell, outdir);
}
}

View File

@@ -36,6 +36,7 @@
# filter_word_count = [993]
# filter_line_count = [35, 36]
# queries = [["name","value"], ["rick", "astley"]]
# save_state = false
# headers can be specified on multiple lines or as an inline table
#

BIN
img/resumed-scan.gif Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 269 KiB

BIN
img/save-state.png Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 104 KiB

View File

@@ -0,0 +1,92 @@
#compdef feroxbuster
_feroxbuster() {
typeset -A opt_args
local ret=1
local context curcontext="$curcontext" state line
_arguments -s -S -C \
'-w+[Path to the wordlist]' \
'--wordlist+[Path to the wordlist]' \
'*-u+[The target URL(s) (required, unless --stdin used)]' \
'*--url+[The target URL(s) (required, unless --stdin used)]' \
'-t+[Number of concurrent threads (default: 50)]' \
'--threads+[Number of concurrent threads (default: 50)]' \
'-d+[Maximum recursion depth, a depth of 0 is infinite recursion (default: 4)]' \
'--depth+[Maximum recursion depth, a depth of 0 is infinite recursion (default: 4)]' \
'-T+[Number of seconds before a request times out (default: 7)]' \
'--timeout+[Number of seconds before a request times out (default: 7)]' \
'-p+[Proxy to use for requests (ex: http(s)://host:port, socks5://host:port)]' \
'--proxy+[Proxy to use for requests (ex: http(s)://host:port, socks5://host:port)]' \
'-P+[Send only unfiltered requests through a Replay Proxy, instead of all requests]' \
'--replay-proxy+[Send only unfiltered requests through a Replay Proxy, instead of all requests]' \
'*-R+[Status Codes to send through a Replay Proxy when found (default: --status-codes value)]' \
'*--replay-codes+[Status Codes to send through a Replay Proxy when found (default: --status-codes value)]' \
'*-s+[Status Codes to include (allow list) (default: 200 204 301 302 307 308 401 403 405)]' \
'*--status-codes+[Status Codes to include (allow list) (default: 200 204 301 302 307 308 401 403 405)]' \
'-o+[Output file to write results to (use w/ --json for JSON entries)]' \
'--output+[Output file to write results to (use w/ --json for JSON entries)]' \
'(-u --url)--resume-from+[State file from which to resume a partially complete scan (ex. --resume-from ferox-1606586780.state)]' \
'--debug-log+[Output file to write log entries (use w/ --json for JSON entries)]' \
'-a+[Sets the User-Agent (default: feroxbuster/VERSION)]' \
'--user-agent+[Sets the User-Agent (default: feroxbuster/VERSION)]' \
'*-x+[File extension(s) to search for (ex: -x php -x pdf js)]' \
'*--extensions+[File extension(s) to search for (ex: -x php -x pdf js)]' \
'*-H+[Specify HTTP headers (ex: -H Header:val '\''stuff: things'\'')]' \
'*--headers+[Specify HTTP headers (ex: -H Header:val '\''stuff: things'\'')]' \
'*-Q+[Specify URL query parameters (ex: -Q token=stuff -Q secret=key)]' \
'*--query+[Specify URL query parameters (ex: -Q token=stuff -Q secret=key)]' \
'*-S+[Filter out messages of a particular size (ex: -S 5120 -S 4927,1970)]' \
'*--filter-size+[Filter out messages of a particular size (ex: -S 5120 -S 4927,1970)]' \
'*-X+[Filter out messages via regular expression matching on the response'\''s body (ex: -X '\''^ignore me$'\'')]' \
'*--filter-regex+[Filter out messages via regular expression matching on the response'\''s body (ex: -X '\''^ignore me$'\'')]' \
'*-W+[Filter out messages of a particular word count (ex: -W 312 -W 91,82)]' \
'*--filter-words+[Filter out messages of a particular word count (ex: -W 312 -W 91,82)]' \
'*-N+[Filter out messages of a particular line count (ex: -N 20 -N 31,30)]' \
'*--filter-lines+[Filter out messages of a particular line count (ex: -N 20 -N 31,30)]' \
'*-C+[Filter out status codes (deny list) (ex: -C 200 -C 401)]' \
'*--filter-status+[Filter out status codes (deny list) (ex: -C 200 -C 401)]' \
'-L+[Limit total number of concurrent scans (default: 0, i.e. no limit)]' \
'--scan-limit+[Limit total number of concurrent scans (default: 0, i.e. no limit)]' \
'*-v[Increase verbosity level (use -vv or more for greater effect. \[CAUTION\] 4 -v'\''s is probably too much)]' \
'*--verbosity[Increase verbosity level (use -vv or more for greater effect. \[CAUTION\] 4 -v'\''s is probably too much)]' \
'-q[Only print URLs; Don'\''t print status codes, response size, running config, etc...]' \
'--quiet[Only print URLs; Don'\''t print status codes, response size, running config, etc...]' \
'--json[Emit JSON logs to --output and --debug-log instead of normal text]' \
'-D[Don'\''t auto-filter wildcard responses]' \
'--dont-filter[Don'\''t auto-filter wildcard responses]' \
'-r[Follow redirects]' \
'--redirects[Follow redirects]' \
'-k[Disables TLS certificate validation]' \
'--insecure[Disables TLS certificate validation]' \
'-n[Do not scan recursively]' \
'--no-recursion[Do not scan recursively]' \
'(-x --extensions)-f[Append / to each request]' \
'(-x --extensions)--add-slash[Append / to each request]' \
'(-u --url)--stdin[Read url(s) from STDIN]' \
'-e[Extract links from response body (html, javascript, etc...); make new requests based on findings (default: false)]' \
'--extract-links[Extract links from response body (html, javascript, etc...); make new requests based on findings (default: false)]' \
'-h[Prints help information]' \
'--help[Prints help information]' \
'-V[Prints version information]' \
'--version[Prints version information]' \
&& ret=0
}
(( $+functions[_feroxbuster_commands] )) ||
_feroxbuster_commands() {
local commands; commands=(
)
_describe -t commands 'feroxbuster commands' commands "$@"
}
(( $+functions[_feroxbuster_commands] )) ||
_feroxbuster_commands() {
local commands; commands=(
)
_describe -t commands 'feroxbuster commands' commands "$@"
}
_feroxbuster "$@"

View File

@@ -0,0 +1,38 @@
@('feroxbuster', './feroxbuster') | %{
Register-ArgumentCompleter -Native -CommandName $_ -ScriptBlock {
param($wordToComplete, $commandAst, $cursorPosition)
$command = '_feroxbuster'
$commandAst.CommandElements |
Select-Object -Skip 1 |
%{
switch ($_.ToString()) {
'feroxbuster' {
$command += '_feroxbuster'
break
}
default {
break
}
}
}
$completions = @()
switch ($command) {
'_feroxbuster' {
$completions = @('-v', '-q', '-D', '-r', '-k', '-n', '-f', '-e', '-h', '-V', '-w', '-u', '-t', '-d', '-T', '-p', '-P', '-R', '-s', '-o', '-a', '-x', '-H', '-Q', '-S', '-X', '-W', '-N', '-C', '-L', '--verbosity', '--quiet', '--json', '--dont-filter', '--redirects', '--insecure', '--no-recursion', '--add-slash', '--stdin', '--extract-links', '--help', '--version', '--wordlist', '--url', '--threads', '--depth', '--timeout', '--proxy', '--replay-proxy', '--replay-codes', '--status-codes', '--output', '--resume-from', '--debug-log', '--user-agent', '--extensions', '--headers', '--query', '--filter-size', '--filter-regex', '--filter-words', '--filter-lines', '--filter-status', '--scan-limit')
}
}
$completions |
?{ $_ -like "$wordToComplete*" } |
Sort-Object |
%{ New-Object System.Management.Automation.CompletionResult $_, $_, 'ParameterValue', $_ }
}
}

View File

@@ -0,0 +1,212 @@
_feroxbuster() {
local i cur prev opts cmds
COMPREPLY=()
cur="${COMP_WORDS[COMP_CWORD]}"
prev="${COMP_WORDS[COMP_CWORD-1]}"
cmd=""
opts=""
for i in ${COMP_WORDS[@]}
do
case "${i}" in
feroxbuster)
cmd="feroxbuster"
;;
feroxbuster)
cmd+="__feroxbuster"
;;
*)
;;
esac
done
case "${cmd}" in
feroxbuster)
opts=" -v -q -D -r -k -n -f -e -h -V -w -u -t -d -T -p -P -R -s -o -a -x -H -Q -S -X -W -N -C -L --verbosity --quiet --json --dont-filter --redirects --insecure --no-recursion --add-slash --stdin --extract-links --help --version --wordlist --url --threads --depth --timeout --proxy --replay-proxy --replay-codes --status-codes --output --resume-from --debug-log --user-agent --extensions --headers --query --filter-size --filter-regex --filter-words --filter-lines --filter-status --scan-limit "
if [[ ${cur} == -* || ${COMP_CWORD} -eq 1 ]] ; then
COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) )
return 0
fi
case "${prev}" in
--wordlist)
COMPREPLY=("<FILE>")
return 0
;;
-w)
COMPREPLY=("<FILE>")
return 0
;;
--url)
COMPREPLY=("<URL>...")
return 0
;;
-u)
COMPREPLY=("<URL>...")
return 0
;;
--threads)
COMPREPLY=("<THREADS>")
return 0
;;
-t)
COMPREPLY=("<THREADS>")
return 0
;;
--depth)
COMPREPLY=("<RECURSION_DEPTH>")
return 0
;;
-d)
COMPREPLY=("<RECURSION_DEPTH>")
return 0
;;
--timeout)
COMPREPLY=("<SECONDS>")
return 0
;;
-T)
COMPREPLY=("<SECONDS>")
return 0
;;
--proxy)
COMPREPLY=("<PROXY>")
return 0
;;
-p)
COMPREPLY=("<PROXY>")
return 0
;;
--replay-proxy)
COMPREPLY=("<REPLAY_PROXY>")
return 0
;;
-P)
COMPREPLY=("<REPLAY_PROXY>")
return 0
;;
--replay-codes)
COMPREPLY=("<REPLAY_CODE>...")
return 0
;;
-R)
COMPREPLY=("<REPLAY_CODE>...")
return 0
;;
--status-codes)
COMPREPLY=("<STATUS_CODE>...")
return 0
;;
-s)
COMPREPLY=("<STATUS_CODE>...")
return 0
;;
--output)
COMPREPLY=("<FILE>")
return 0
;;
-o)
COMPREPLY=("<FILE>")
return 0
;;
--resume-from)
COMPREPLY=("<STATE_FILE>")
return 0
;;
--debug-log)
COMPREPLY=("<FILE>")
return 0
;;
--user-agent)
COMPREPLY=("<USER_AGENT>")
return 0
;;
-a)
COMPREPLY=("<USER_AGENT>")
return 0
;;
--extensions)
COMPREPLY=("<FILE_EXTENSION>...")
return 0
;;
-x)
COMPREPLY=("<FILE_EXTENSION>...")
return 0
;;
--headers)
COMPREPLY=("<HEADER>...")
return 0
;;
-H)
COMPREPLY=("<HEADER>...")
return 0
;;
--query)
COMPREPLY=("<QUERY>...")
return 0
;;
-Q)
COMPREPLY=("<QUERY>...")
return 0
;;
--filter-size)
COMPREPLY=("<SIZE>...")
return 0
;;
-S)
COMPREPLY=("<SIZE>...")
return 0
;;
--filter-regex)
COMPREPLY=("<REGEX>...")
return 0
;;
-X)
COMPREPLY=("<REGEX>...")
return 0
;;
--filter-words)
COMPREPLY=("<WORDS>...")
return 0
;;
-W)
COMPREPLY=("<WORDS>...")
return 0
;;
--filter-lines)
COMPREPLY=("<LINES>...")
return 0
;;
-N)
COMPREPLY=("<LINES>...")
return 0
;;
--filter-status)
COMPREPLY=("<STATUS_CODE>...")
return 0
;;
-C)
COMPREPLY=("<STATUS_CODE>...")
return 0
;;
--scan-limit)
COMPREPLY=("<SCAN_LIMIT>")
return 0
;;
-L)
COMPREPLY=("<SCAN_LIMIT>")
return 0
;;
*)
COMPREPLY=()
;;
esac
COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) )
return 0
;;
esac
}
complete -F _feroxbuster -o bashdefault -o default feroxbuster

View File

@@ -0,0 +1,47 @@
function __fish_using_command
set cmd (commandline -opc)
if [ (count $cmd) -eq (count $argv) ]
for i in (seq (count $argv))
if [ $cmd[$i] != $argv[$i] ]
return 1
end
end
return 0
end
return 1
end
complete -c feroxbuster -n "__fish_using_command feroxbuster" -s w -l wordlist -d 'Path to the wordlist'
complete -c feroxbuster -n "__fish_using_command feroxbuster" -s u -l url -d 'The target URL(s) (required, unless --stdin used)'
complete -c feroxbuster -n "__fish_using_command feroxbuster" -s t -l threads -d 'Number of concurrent threads (default: 50)'
complete -c feroxbuster -n "__fish_using_command feroxbuster" -s d -l depth -d 'Maximum recursion depth, a depth of 0 is infinite recursion (default: 4)'
complete -c feroxbuster -n "__fish_using_command feroxbuster" -s T -l timeout -d 'Number of seconds before a request times out (default: 7)'
complete -c feroxbuster -n "__fish_using_command feroxbuster" -s p -l proxy -d 'Proxy to use for requests (ex: http(s)://host:port, socks5://host:port)'
complete -c feroxbuster -n "__fish_using_command feroxbuster" -s P -l replay-proxy -d 'Send only unfiltered requests through a Replay Proxy, instead of all requests'
complete -c feroxbuster -n "__fish_using_command feroxbuster" -s R -l replay-codes -d 'Status Codes to send through a Replay Proxy when found (default: --status-codes value)'
complete -c feroxbuster -n "__fish_using_command feroxbuster" -s s -l status-codes -d 'Status Codes to include (allow list) (default: 200 204 301 302 307 308 401 403 405)'
complete -c feroxbuster -n "__fish_using_command feroxbuster" -s o -l output -d 'Output file to write results to (use w/ --json for JSON entries)'
complete -c feroxbuster -n "__fish_using_command feroxbuster" -l resume-from -d 'State file from which to resume a partially complete scan (ex. --resume-from ferox-1606586780.state)'
complete -c feroxbuster -n "__fish_using_command feroxbuster" -l debug-log -d 'Output file to write log entries (use w/ --json for JSON entries)'
complete -c feroxbuster -n "__fish_using_command feroxbuster" -s a -l user-agent -d 'Sets the User-Agent (default: feroxbuster/VERSION)'
complete -c feroxbuster -n "__fish_using_command feroxbuster" -s x -l extensions -d 'File extension(s) to search for (ex: -x php -x pdf js)'
complete -c feroxbuster -n "__fish_using_command feroxbuster" -s H -l headers -d 'Specify HTTP headers (ex: -H Header:val \'stuff: things\')'
complete -c feroxbuster -n "__fish_using_command feroxbuster" -s Q -l query -d 'Specify URL query parameters (ex: -Q token=stuff -Q secret=key)'
complete -c feroxbuster -n "__fish_using_command feroxbuster" -s S -l filter-size -d 'Filter out messages of a particular size (ex: -S 5120 -S 4927,1970)'
complete -c feroxbuster -n "__fish_using_command feroxbuster" -s X -l filter-regex -d 'Filter out messages via regular expression matching on the response\'s body (ex: -X \'^ignore me$\')'
complete -c feroxbuster -n "__fish_using_command feroxbuster" -s W -l filter-words -d 'Filter out messages of a particular word count (ex: -W 312 -W 91,82)'
complete -c feroxbuster -n "__fish_using_command feroxbuster" -s N -l filter-lines -d 'Filter out messages of a particular line count (ex: -N 20 -N 31,30)'
complete -c feroxbuster -n "__fish_using_command feroxbuster" -s C -l filter-status -d 'Filter out status codes (deny list) (ex: -C 200 -C 401)'
complete -c feroxbuster -n "__fish_using_command feroxbuster" -s L -l scan-limit -d 'Limit total number of concurrent scans (default: 0, i.e. no limit)'
complete -c feroxbuster -n "__fish_using_command feroxbuster" -s v -l verbosity -d 'Increase verbosity level (use -vv or more for greater effect. [CAUTION] 4 -v\'s is probably too much)'
complete -c feroxbuster -n "__fish_using_command feroxbuster" -s q -l quiet -d 'Only print URLs; Don\'t print status codes, response size, running config, etc...'
complete -c feroxbuster -n "__fish_using_command feroxbuster" -l json -d 'Emit JSON logs to --output and --debug-log instead of normal text'
complete -c feroxbuster -n "__fish_using_command feroxbuster" -s D -l dont-filter -d 'Don\'t auto-filter wildcard responses'
complete -c feroxbuster -n "__fish_using_command feroxbuster" -s r -l redirects -d 'Follow redirects'
complete -c feroxbuster -n "__fish_using_command feroxbuster" -s k -l insecure -d 'Disables TLS certificate validation'
complete -c feroxbuster -n "__fish_using_command feroxbuster" -s n -l no-recursion -d 'Do not scan recursively'
complete -c feroxbuster -n "__fish_using_command feroxbuster" -s f -l add-slash -d 'Append / to each request'
complete -c feroxbuster -n "__fish_using_command feroxbuster" -l stdin -d 'Read url(s) from STDIN'
complete -c feroxbuster -n "__fish_using_command feroxbuster" -s e -l extract-links -d 'Extract links from response body (html, javascript, etc...); make new requests based on findings (default: false)'
complete -c feroxbuster -n "__fish_using_command feroxbuster" -s h -l help -d 'Prints help information'
complete -c feroxbuster -n "__fish_using_command feroxbuster" -s V -l version -d 'Prints version information'

View File

@@ -1,7 +1,8 @@
use crate::scan_manager::resume_scan;
use crate::utils::{module_colorizer, status_colorizer};
use crate::{client, parser, progress};
use crate::{FeroxSerialize, DEFAULT_CONFIG_NAME, DEFAULT_STATUS_CODES, DEFAULT_WORDLIST, VERSION};
use clap::value_t;
use clap::{value_t, ArgMatches};
use indicatif::{MultiProgress, ProgressBar, ProgressDrawTarget};
use lazy_static::lazy_static;
use reqwest::{Client, StatusCode};
@@ -21,7 +22,33 @@ lazy_static! {
pub static ref PROGRESS_BAR: MultiProgress = MultiProgress::with_draw_target(ProgressDrawTarget::stdout());
/// Global progress bar that is only used for printing messages that don't jack up other bars
pub static ref PROGRESS_PRINTER: ProgressBar = progress::add_bar("", 0, true);
pub static ref PROGRESS_PRINTER: ProgressBar = progress::add_bar("", 0, true, false);
}
/// macro helper to abstract away repetitive configuration updates
macro_rules! update_config_if_present {
($c:expr, $m:ident, $v:expr, $t:ty) => {
match value_t!($m, $v, $t) {
Ok(value) => *$c = value, // Update value
Err(clap::Error {
kind: clap::ErrorKind::ArgumentNotFound,
message: _,
info: _,
}) => {
// Do nothing if argument not found
}
Err(e) => e.exit(), // Exit with error on parse error
}
};
}
/// macro helper to abstract away repetitive if not default: update checks
macro_rules! update_if_not_default {
($old:expr, $new:expr, $default:expr) => {
if $new != $default {
*$old = $new;
}
};
}
/// simple helper to clean up some code reuse below; panics under test / exits in prod
@@ -80,7 +107,7 @@ pub struct Configuration {
pub status_codes: Vec<u16>,
/// Status Codes to replay to the Replay Proxy (default: whatever is passed to --status-code)
#[serde(default)]
#[serde(default = "status_codes")]
pub replay_codes: Vec<u16>,
/// Status Codes to filter out (deny list)
@@ -191,9 +218,19 @@ pub struct Configuration {
/// Don't auto-filter wildcard responses
#[serde(default)]
pub dont_filter: bool,
/// Scan started from a state file, not from CLI args
#[serde(default)]
pub resumed: bool,
/// Whether or not a scan's current state should be saved when user presses Ctrl+C
///
/// Not configurable from CLI; can only be set from a config file
#[serde(default = "save_state")]
pub save_state: bool,
}
// functions timeout, threads, status_codes, user_agent, wordlist, and depth are used to provide
// functions timeout, threads, status_codes, user_agent, wordlist, save_state, and depth are used to provide
// defaults in the event that a ferox-config.toml is found but one or more of the values below
// aren't listed in the config. This way, we get the correct defaults upon Deserialization
@@ -207,6 +244,11 @@ fn timeout() -> u64 {
7
}
/// default save_state value
fn save_state() -> bool {
true
}
/// default threads value
fn threads() -> usize {
50
@@ -256,6 +298,7 @@ impl Default for Configuration {
replay_client,
dont_filter: false,
quiet: false,
resumed: false,
stdin: false,
json: false,
verbosity: 0,
@@ -265,6 +308,7 @@ impl Default for Configuration {
redirects: false,
no_recursion: false,
extract_links: false,
save_state: true,
proxy: String::new(),
config: String::new(),
output: String::new(),
@@ -304,6 +348,7 @@ impl Configuration {
/// - **output**: `None` (print to stdout)
/// - **debug_log**: `None`
/// - **quiet**: `false`
/// - **save_state**: `true`
/// - **user_agent**: `feroxbuster/VERSION`
/// - **insecure**: `false` (don't be insecure, i.e. don't allow invalid certs)
/// - **extensions**: `None`
@@ -346,10 +391,60 @@ impl Configuration {
return Configuration::default();
}
let args = parser::initialize().get_matches();
// Get the default configuration, this is what will apply if nothing
// else is specified.
let mut config = Configuration::default();
// read in all config files
Self::parse_config_files(&mut config);
// read in the user provided options, this produces a separate instance of Configuration
// in order to allow for potentially merging into a --resume-from Configuration
let cli_config = Self::parse_cli_args(&args);
// --resume-from used, need to first read the Configuration from disk, and then
// merge the cli_config into the resumed config
if let Some(filename) = args.value_of("resume_from") {
// when resuming a scan, instead of normal configuration loading, we just
// load the config from disk by calling resume_scan
let mut previous_config = resume_scan(filename);
// if any other arguments were passed on the command line, the theory is that the
// user meant to modify the previously cancelled/saved scan in some way that we
// should take into account
Self::merge_config(&mut previous_config, cli_config);
// the resumed flag isn't printed in the banner and really has no business being
// serialized or included in much of the usual config logic; simply setting it to true
// here and being done with it
previous_config.resumed = true;
// if the user used --stdin, we already have all the scans started (or complete), we
// need to flip stdin to false so that the 'read from stdin' logic doesn't fire (if
// not flipped to false, the program hangs waiting for input from stdin again)
previous_config.stdin = false;
// clients aren't serialized, have to remake them from the previous config
Self::try_rebuild_clients(&mut previous_config);
return previous_config;
}
// if we've gotten to this point in the code, --resume-from was not used, so we need to
// merge the cli options into the config file options and return the result
Self::merge_config(&mut config, cli_config);
// rebuild clients is the last step in either code branch
Self::try_rebuild_clients(&mut config);
config
}
/// Parse all possible versions of the ferox-config.toml file, adhering to the order of
/// precedence outlined above
fn parse_config_files(mut config: &mut Self) {
// Next, we parse the ferox-config.toml file, if present and set the values
// therein to overwrite our default values. Deserialized defaults are specified
// in the Configuration struct so that we don't change anything that isn't
@@ -391,24 +486,12 @@ impl Configuration {
let config_file = cwd.join(DEFAULT_CONFIG_NAME);
Self::parse_and_merge_config(config_file, &mut config);
}
}
let args = parser::initialize().get_matches();
macro_rules! update_config_if_present {
($c:expr, $m:ident, $v:expr, $t:ty) => {
match value_t!($m, $v, $t) {
Ok(value) => *$c = value, // Update value
Err(clap::Error {
kind: clap::ErrorKind::ArgumentNotFound,
message: _,
info: _,
}) => {
// Do nothing if argument not found
}
Err(e) => e.exit(), // Exit with error on parse error
}
};
}
/// Given a set of ArgMatches read from the CLI, update and return the default Configuration
/// settings
fn parse_cli_args(args: &ArgMatches) -> Self {
let mut config = Configuration::default();
update_config_if_present!(&mut config.threads, args, "threads", usize);
update_config_if_present!(&mut config.depth, args, "depth", usize);
@@ -522,8 +605,8 @@ impl Configuration {
if args.is_present("stdin") {
config.stdin = true;
} else {
config.target_url = String::from(args.value_of("url").unwrap());
} else if let Some(url) = args.value_of("url") {
config.target_url = String::from(url);
}
////
@@ -569,50 +652,53 @@ impl Configuration {
}
}
// this if statement determines if we've gotten a Client configuration change from
// either the config file or command line arguments; if we have, we need to rebuild
// the client and store it in the config struct
if !config.proxy.is_empty()
|| config.timeout != timeout()
|| config.user_agent != user_agent()
|| config.redirects
|| config.insecure
|| !config.headers.is_empty()
config
}
/// this function determines if we've gotten a Client configuration change from
/// either the config file or command line arguments; if we have, we need to rebuild
/// the client and store it in the config struct
fn try_rebuild_clients(configuration: &mut Configuration) {
if !configuration.proxy.is_empty()
|| configuration.timeout != timeout()
|| configuration.user_agent != user_agent()
|| configuration.redirects
|| configuration.insecure
|| !configuration.headers.is_empty()
|| configuration.resumed
{
if config.proxy.is_empty() {
config.client = client::initialize(
config.timeout,
&config.user_agent,
config.redirects,
config.insecure,
&config.headers,
if configuration.proxy.is_empty() {
configuration.client = client::initialize(
configuration.timeout,
&configuration.user_agent,
configuration.redirects,
configuration.insecure,
&configuration.headers,
None,
)
} else {
config.client = client::initialize(
config.timeout,
&config.user_agent,
config.redirects,
config.insecure,
&config.headers,
Some(&config.proxy),
configuration.client = client::initialize(
configuration.timeout,
&configuration.user_agent,
configuration.redirects,
configuration.insecure,
&configuration.headers,
Some(&configuration.proxy),
)
}
}
if !config.replay_proxy.is_empty() {
if !configuration.replay_proxy.is_empty() {
// only set replay_client when replay_proxy is set
config.replay_client = Some(client::initialize(
config.timeout,
&config.user_agent,
config.redirects,
config.insecure,
&config.headers,
Some(&config.replay_proxy),
configuration.replay_client = Some(client::initialize(
configuration.timeout,
&configuration.user_agent,
configuration.redirects,
configuration.insecure,
&configuration.headers,
Some(&configuration.replay_proxy),
));
}
config
}
/// Given a configuration file's location and an instance of `Configuration`, read in
@@ -636,37 +722,63 @@ impl Configuration {
}
/// Given two Configurations, overwrite `settings` with the fields found in `settings_to_merge`
fn merge_config(settings: &mut Self, settings_to_merge: Self) {
settings.threads = settings_to_merge.threads;
settings.wordlist = settings_to_merge.wordlist;
settings.status_codes = settings_to_merge.status_codes;
settings.proxy = settings_to_merge.proxy;
settings.timeout = settings_to_merge.timeout;
settings.verbosity = settings_to_merge.verbosity;
settings.quiet = settings_to_merge.quiet;
settings.output = settings_to_merge.output;
settings.user_agent = settings_to_merge.user_agent;
settings.redirects = settings_to_merge.redirects;
settings.insecure = settings_to_merge.insecure;
settings.extract_links = settings_to_merge.extract_links;
settings.extensions = settings_to_merge.extensions;
settings.headers = settings_to_merge.headers;
settings.queries = settings_to_merge.queries;
settings.no_recursion = settings_to_merge.no_recursion;
settings.add_slash = settings_to_merge.add_slash;
settings.stdin = settings_to_merge.stdin;
settings.depth = settings_to_merge.depth;
settings.filter_size = settings_to_merge.filter_size;
settings.filter_regex = settings_to_merge.filter_regex;
settings.filter_word_count = settings_to_merge.filter_word_count;
settings.filter_line_count = settings_to_merge.filter_line_count;
settings.filter_status = settings_to_merge.filter_status;
settings.dont_filter = settings_to_merge.dont_filter;
settings.scan_limit = settings_to_merge.scan_limit;
settings.replay_proxy = settings_to_merge.replay_proxy;
settings.replay_codes = settings_to_merge.replay_codes;
settings.debug_log = settings_to_merge.debug_log;
settings.json = settings_to_merge.json;
fn merge_config(conf: &mut Self, new: Self) {
// does not include the following Configuration fields, as they don't make sense here
// - kind
// - client
// - replay_client
// - resumed
// - config
update_if_not_default!(&mut conf.target_url, new.target_url, "");
update_if_not_default!(&mut conf.proxy, new.proxy, "");
update_if_not_default!(&mut conf.verbosity, new.verbosity, 0);
update_if_not_default!(&mut conf.quiet, new.quiet, false);
update_if_not_default!(&mut conf.output, new.output, "");
update_if_not_default!(&mut conf.redirects, new.redirects, false);
update_if_not_default!(&mut conf.insecure, new.insecure, false);
update_if_not_default!(&mut conf.extract_links, new.extract_links, false);
update_if_not_default!(&mut conf.extensions, new.extensions, Vec::<String>::new());
update_if_not_default!(&mut conf.headers, new.headers, HashMap::new());
update_if_not_default!(&mut conf.queries, new.queries, Vec::new());
update_if_not_default!(&mut conf.no_recursion, new.no_recursion, false);
update_if_not_default!(&mut conf.add_slash, new.add_slash, false);
update_if_not_default!(&mut conf.stdin, new.stdin, false);
update_if_not_default!(&mut conf.filter_size, new.filter_size, Vec::<u64>::new());
update_if_not_default!(
&mut conf.filter_regex,
new.filter_regex,
Vec::<String>::new()
);
update_if_not_default!(
&mut conf.filter_word_count,
new.filter_word_count,
Vec::<usize>::new()
);
update_if_not_default!(
&mut conf.filter_line_count,
new.filter_line_count,
Vec::<usize>::new()
);
update_if_not_default!(
&mut conf.filter_status,
new.filter_status,
Vec::<u16>::new()
);
update_if_not_default!(&mut conf.dont_filter, new.dont_filter, false);
update_if_not_default!(&mut conf.scan_limit, new.scan_limit, 0);
update_if_not_default!(&mut conf.replay_proxy, new.replay_proxy, "");
update_if_not_default!(&mut conf.debug_log, new.debug_log, "");
update_if_not_default!(&mut conf.json, new.json, false);
update_if_not_default!(&mut conf.timeout, new.timeout, timeout());
update_if_not_default!(&mut conf.user_agent, new.user_agent, user_agent());
update_if_not_default!(&mut conf.threads, new.threads, threads());
update_if_not_default!(&mut conf.depth, new.depth, depth());
update_if_not_default!(&mut conf.wordlist, new.wordlist, wordlist());
update_if_not_default!(&mut conf.status_codes, new.status_codes, status_codes());
// status_codes() is the default for replay_codes, if they're not provided
update_if_not_default!(&mut conf.replay_codes, new.replay_codes, status_codes());
update_if_not_default!(&mut conf.save_state, new.save_state, save_state());
}
/// If present, read in `DEFAULT_CONFIG_NAME` and deserialize the specified values
@@ -765,6 +877,7 @@ mod tests {
dont_filter = true
extract_links = true
json = true
save_state = false
depth = 1
filter_size = [4120]
filter_regex = ["^ignore me$"]
@@ -800,6 +913,7 @@ mod tests {
assert_eq!(config.dont_filter, false);
assert_eq!(config.no_recursion, false);
assert_eq!(config.json, false);
assert_eq!(config.save_state, true);
assert_eq!(config.stdin, false);
assert_eq!(config.add_slash, false);
assert_eq!(config.redirects, false);
@@ -1004,6 +1118,13 @@ mod tests {
assert_eq!(config.filter_status, vec![201]);
}
#[test]
/// parse the test config and see that the value parsed is correct
fn config_reads_save_state() {
let config = setup_config_test();
assert_eq!(config.save_state, false);
}
#[test]
/// parse the test config and see that the values parsed are correct
fn config_reads_headers() {

View File

@@ -360,6 +360,8 @@ mod tests {
wildcard: true,
url: Url::parse("http://localhost").unwrap(),
content_length: 100,
word_count: 50,
line_count: 25,
headers: reqwest::header::HeaderMap::new(),
status: reqwest::StatusCode::OK,
};
@@ -380,6 +382,8 @@ mod tests {
wildcard: true,
url: Url::parse("http://localhost/stuff").unwrap(),
content_length: 100,
word_count: 50,
line_count: 25,
headers: reqwest::header::HeaderMap::new(),
status: reqwest::StatusCode::OK,
};
@@ -400,6 +404,8 @@ mod tests {
wildcard: false,
url: Url::parse("http://localhost/stuff").unwrap(),
content_length: 100,
word_count: 50,
line_count: 25,
headers: reqwest::header::HeaderMap::new(),
status: reqwest::StatusCode::OK,
};

View File

@@ -14,9 +14,13 @@ pub mod utils;
use crate::utils::{get_url_path_length, status_colorizer};
use console::{style, Color};
use reqwest::header::{HeaderName, HeaderValue};
use reqwest::{header::HeaderMap, Response, StatusCode, Url};
use serde::{ser::SerializeStruct, Deserialize, Serialize, Serializer};
use serde::{ser::SerializeStruct, Deserialize, Deserializer, Serialize, Serializer};
use serde_json::Value;
use std::collections::HashMap;
use std::convert::{TryFrom, TryInto};
use std::str::FromStr;
use std::{error, fmt};
use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender};
@@ -112,6 +116,12 @@ pub struct FeroxResponse {
/// The content-length of this response, if known
content_length: u64,
/// The number of lines contained in the body of this response, if known
line_count: usize,
/// The number of words contained in the body of this response, if known
word_count: usize,
/// The `Headers` of this `FeroxResponse`
headers: HeaderMap,
@@ -194,15 +204,12 @@ impl FeroxResponse {
/// Returns line count of the response text.
pub fn line_count(&self) -> usize {
self.text().lines().count()
self.line_count
}
/// Returns word count of the response text.
pub fn word_count(&self) -> usize {
self.text()
.lines()
.map(|s| s.split_whitespace().count())
.sum()
self.word_count
}
/// Create a new `FeroxResponse` from the given `Response`
@@ -228,18 +235,23 @@ impl FeroxResponse {
String::new()
};
let line_count = text.lines().count();
let word_count = text.lines().map(|s| s.split_whitespace().count()).sum();
FeroxResponse {
url,
status,
content_length,
text,
headers,
line_count,
word_count,
wildcard: false,
}
}
}
/// Implement FeroxSerialize for FeroxResponse
/// Implement FeroxSerialusize::from(ize for FeroxRespons)e
impl FeroxSerialize for FeroxResponse {
/// Simple wrapper around create_report_string
fn as_str(&self) -> String {
@@ -359,18 +371,99 @@ impl Serialize for FeroxResponse {
state.serialize_field("wildcard", &self.wildcard)?;
state.serialize_field("status", &self.status.as_u16())?;
state.serialize_field("content_length", &self.content_length)?;
state.serialize_field("line_count", &self.line_count())?;
state.serialize_field("word_count", &self.word_count())?;
state.serialize_field("line_count", &self.line_count)?;
state.serialize_field("word_count", &self.word_count)?;
state.serialize_field("headers", &headers)?;
state.end()
}
}
/// Deserialize implementation for FeroxResponse
impl<'de> Deserialize<'de> for FeroxResponse {
/// Deserialize a FeroxResponse from a serde_json::Value
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
let mut response = Self {
url: Url::parse("http://localhost").unwrap(),
status: StatusCode::OK,
text: String::new(),
content_length: 0,
headers: HeaderMap::new(),
wildcard: false,
line_count: 0,
word_count: 0,
};
let map: HashMap<String, Value> = HashMap::deserialize(deserializer)?;
for (key, value) in &map {
match key.as_str() {
"url" => {
if let Some(url) = value.as_str() {
if let Ok(parsed) = Url::parse(url) {
response.url = parsed;
}
}
}
"status" => {
if let Some(num) = value.as_u64() {
if let Ok(smaller) = u16::try_from(num) {
if let Ok(status) = StatusCode::from_u16(smaller) {
response.status = status;
}
}
}
}
"content_length" => {
if let Some(num) = value.as_u64() {
response.content_length = num;
}
}
"line_count" => {
if let Some(num) = value.as_u64() {
response.line_count = num.try_into().unwrap_or_default();
}
}
"word_count" => {
if let Some(num) = value.as_u64() {
response.word_count = num.try_into().unwrap_or_default();
}
}
"headers" => {
let mut headers = HeaderMap::<HeaderValue>::default();
if let Some(map_headers) = value.as_object() {
for (h_key, h_value) in map_headers {
let h_value_str = h_value.as_str().unwrap_or("");
let h_name = HeaderName::from_str(h_key)
.unwrap_or_else(|_| HeaderName::from_str("Unknown").unwrap());
let h_value_parsed = HeaderValue::from_str(h_value_str)
.unwrap_or_else(|_| HeaderValue::from_str("Unknown").unwrap());
headers.insert(h_name, h_value_parsed);
}
}
response.headers = headers;
}
"wildcard" => {
if let Some(result) = value.as_bool() {
response.wildcard = result;
}
}
_ => {}
}
}
Ok(response)
}
}
#[derive(Serialize, Deserialize, Default)]
/// Representation of a log entry, can be represented as a human readable string or JSON
pub struct FeroxMessage {
// todo probably move to lib
#[serde(rename = "type")]
/// Name of this type of struct, used for serialization, i.e. `{"type":"log"}`
kind: String,

View File

@@ -1,16 +1,18 @@
use crossterm::event::{self, Event, KeyCode};
use feroxbuster::progress::add_bar;
use feroxbuster::{
banner,
config::{CONFIGURATION, PROGRESS_BAR, PROGRESS_PRINTER},
heuristics, logger, reporter,
scan_manager::PAUSE_SCAN,
scanner::{self, scan_url},
scan_manager::{self, PAUSE_SCAN},
scanner::{self, scan_url, RESPONSES, SCANNED_URLS},
utils::{ferox_print, get_current_depth, module_colorizer, status_colorizer},
FeroxError, FeroxResponse, FeroxResult, SLEEP_DURATION, VERSION,
FeroxError, FeroxResponse, FeroxResult, FeroxSerialize, SLEEP_DURATION, VERSION,
};
#[cfg(not(target_os = "windows"))]
use feroxbuster::{utils::set_open_file_limit, DEFAULT_OPEN_FILE_LIMIT};
use futures::StreamExt;
use std::convert::TryInto;
use std::{
collections::HashSet,
fs::File,
@@ -115,6 +117,31 @@ async fn scan(
scanner::initialize(words.len(), &CONFIGURATION);
if CONFIGURATION.resumed {
if let Ok(scans) = SCANNED_URLS.scans.lock() {
for scan in scans.iter() {
if let Ok(locked_scan) = scan.lock() {
if locked_scan.complete {
// these scans are complete, and just need to be shown to the user
let pb = add_bar(
&locked_scan.url,
words.len().try_into().unwrap_or_default(),
false,
true,
);
pb.finish();
}
}
}
}
if let Ok(responses) = RESPONSES.responses.read() {
for response in responses.iter() {
PROGRESS_PRINTER.println(response.as_str());
}
}
}
let mut tasks = vec![];
for target in targets {
@@ -152,6 +179,22 @@ async fn get_targets() -> FeroxResult<Vec<String>> {
while let Some(line) = reader.next().await {
targets.push(line?);
}
} else if CONFIGURATION.resumed {
// resume-from can't be used with --url, and --stdin is marked false for every resumed
// scan, making it mutually exclusive from either of the other two options
if let Ok(scans) = SCANNED_URLS.scans.lock() {
for scan in scans.iter() {
// SCANNED_URLS gets deserialized scans added to it at program start if --resume-from
// is used, so scans that aren't marked complete still need to be scanned
if let Ok(locked_scan) = scan.lock() {
if locked_scan.complete {
// this one's already done, ignore it
continue;
}
targets.push(locked_scan.url.to_owned());
}
}
}
} else {
targets.push(CONFIGURATION.target_url.clone());
}
@@ -298,6 +341,11 @@ fn main() {
// setup logging based on the number of -v's used
logger::initialize(CONFIGURATION.verbosity);
if CONFIGURATION.save_state {
// start the ctrl+c handler
scan_manager::initialize();
}
// this function uses rlimit, which is not supported on windows
#[cfg(not(target_os = "windows"))]
set_open_file_limit(DEFAULT_OPEN_FILE_LIMIT);

View File

@@ -1,10 +1,9 @@
use crate::VERSION;
use clap::{App, Arg};
use clap::{App, Arg, ArgGroup};
/// Create and return an instance of [clap::App](https://docs.rs/clap/latest/clap/struct.App.html), i.e. the Command Line Interface's configuration
pub fn initialize() -> App<'static, 'static> {
App::new("feroxbuster")
.version(VERSION)
.version(env!("CARGO_PKG_VERSION"))
.author("Ben 'epi' Risher (@epi052)")
.about("A fast, simple, recursive content discovery tool written in Rust")
.arg(
@@ -19,7 +18,7 @@ pub fn initialize() -> App<'static, 'static> {
Arg::with_name("url")
.short("u")
.long("url")
.required_unless("stdin")
.required_unless_one(&["stdin", "resume_from"])
.value_name("URL")
.multiple(true)
.use_delimiter(true)
@@ -113,6 +112,7 @@ pub fn initialize() -> App<'static, 'static> {
Arg::with_name("json")
.long("json")
.takes_value(false)
.requires("output_files")
.help("Emit JSON logs to --output and --debug-log instead of normal text")
)
.arg(
@@ -130,6 +130,14 @@ pub fn initialize() -> App<'static, 'static> {
.help("Output file to write results to (use w/ --json for JSON entries)")
.takes_value(true),
)
.arg(
Arg::with_name("resume_from")
.long("resume-from")
.value_name("STATE_FILE")
.help("State file from which to resume a partially complete scan (ex. --resume-from ferox-1606586780.state)")
.conflicts_with("url")
.takes_value(true),
)
.arg(
Arg::with_name("debug_log")
.long("debug-log")
@@ -294,6 +302,10 @@ pub fn initialize() -> App<'static, 'static> {
.takes_value(true)
.help("Limit total number of concurrent scans (default: 0, i.e. no limit)")
)
.group(ArgGroup::with_name("output_files")
.args(&["debug_log", "output"])
.multiple(true)
)
.after_help(r#"NOTE:
Options that take multiple values are very flexible. Consider the following ways of specifying
extensions:

View File

@@ -3,9 +3,16 @@ use indicatif::{ProgressBar, ProgressStyle};
/// Add an [indicatif::ProgressBar](https://docs.rs/indicatif/latest/indicatif/struct.ProgressBar.html)
/// to the global [PROGRESS_BAR](../config/struct.PROGRESS_BAR.html)
pub fn add_bar(prefix: &str, length: u64, hidden: bool) -> ProgressBar {
pub fn add_bar(prefix: &str, length: u64, hidden: bool, hide_per_sec: bool) -> ProgressBar {
let style = if hidden || CONFIGURATION.quiet {
ProgressStyle::default_bar().template("")
} else if hide_per_sec {
ProgressStyle::default_bar()
.template(&format!(
"[{{bar:.cyan/blue}}] - {{elapsed:<4}} {{pos:>7}}/{{len:7}} {:7} {{prefix}}",
"-"
))
.progress_chars("#>-")
} else {
ProgressStyle::default_bar()
.template("[{bar:.cyan/blue}] - {elapsed:<4} {pos:>7}/{len:7} {per_sec:7} {prefix}")
@@ -20,3 +27,24 @@ pub fn add_bar(prefix: &str, length: u64, hidden: bool) -> ProgressBar {
progress_bar
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
/// hit all code branches for add_bar
fn add_bar_with_all_configurations() {
let p1 = add_bar("prefix", 2, true, false); // hidden
let p2 = add_bar("prefix", 2, false, true); // no per second field
let p3 = add_bar("prefix", 2, false, false); // normal bar
p1.finish();
p2.finish();
p3.finish();
assert!(p1.is_finished());
assert!(p2.is_finished());
assert!(p3.is_finished());
}
}

View File

@@ -1,5 +1,6 @@
use crate::{
config::{CONFIGURATION, PROGRESS_PRINTER},
scanner::RESPONSES,
utils::{ferox_print, make_request, open_file},
FeroxChannel, FeroxResponse, FeroxSerialize,
};
@@ -96,7 +97,11 @@ async fn spawn_terminal_reporter(
while let Some(resp) = resp_chan.recv().await {
log::trace!("received {} on reporting channel", resp.url());
if CONFIGURATION.status_codes.contains(&resp.status().as_u16()) {
let contains_sentry = CONFIGURATION.status_codes.contains(&resp.status().as_u16());
let unknown_sentry = !RESPONSES.contains(&resp); // !contains == unknown
let should_process_response = contains_sentry && unknown_sentry;
if should_process_response {
// print to stdout
ferox_print(&resp.as_str(), &PROGRESS_PRINTER);
@@ -114,9 +119,7 @@ async fn spawn_terminal_reporter(
}
log::trace!("report complete: {}", resp.url());
if CONFIGURATION.replay_client.is_some()
&& CONFIGURATION.replay_codes.contains(&resp.status().as_u16())
{
if CONFIGURATION.replay_client.is_some() && should_process_response {
// replay proxy specified/client created and this response's status code is one that
// should be replayed
match make_request(CONFIGURATION.replay_client.as_ref().unwrap(), &resp.url()).await {
@@ -126,6 +129,13 @@ async fn spawn_terminal_reporter(
}
}
}
if should_process_response {
// add response to RESPONSES for serialization in case of ctrl+c
// placed all by its lonesome like this so that RESPONSES can take ownership
// of the FeroxResponse
RESPONSES.insert(resp);
}
}
log::trace!("exit: spawn_terminal_reporter");
}

View File

@@ -1,11 +1,28 @@
use crate::{config::PROGRESS_PRINTER, progress, scanner::NUMBER_OF_REQUESTS, SLEEP_DURATION};
use crate::config::Configuration;
use crate::reporter::safe_file_write;
use crate::utils::open_file;
use crate::{
config::{CONFIGURATION, PROGRESS_PRINTER},
progress,
scanner::{NUMBER_OF_REQUESTS, RESPONSES, SCANNED_URLS},
FeroxResponse, FeroxSerialize, SLEEP_DURATION,
};
use console::style;
use indicatif::{ProgressBar, ProgressStyle};
use lazy_static::lazy_static;
use serde::{
ser::{SerializeSeq, SerializeStruct},
Deserialize, Deserializer, Serialize, Serializer,
};
use serde_json::Value;
use std::collections::HashMap;
use std::{
cmp::PartialEq,
fmt,
fs::File,
io::BufReader,
sync::{Arc, Mutex, RwLock},
time::{SystemTime, UNIX_EPOCH},
};
use std::{
io::{stderr, Write},
@@ -28,12 +45,20 @@ static INTERACTIVE_BARRIER: AtomicUsize = AtomicUsize::new(0);
pub static PAUSE_SCAN: AtomicBool = AtomicBool::new(false);
/// Simple enum used to flag a `FeroxScan` as likely a directory or file
#[derive(Debug)]
#[derive(Debug, Serialize, Deserialize)]
pub enum ScanType {
File,
Directory,
}
/// Default implementation for ScanType
impl Default for ScanType {
/// Return ScanType::File as default
fn default() -> Self {
Self::File
}
}
/// Struct to hold scan-related state
///
/// The purpose of this container is to open up the pathway to aborting currently running tasks and
@@ -59,17 +84,8 @@ pub struct FeroxScan {
pub progress_bar: Option<ProgressBar>,
}
/// Implementation of FeroxScan
impl FeroxScan {
/// Stop a currently running scan
pub fn abort(&self) {
self.stop_progress_bar();
if let Some(_task) = &self.task {
// task.abort(); todo uncomment once upgraded to tokio 0.3 (issue #107)
}
}
/// Default implementation for FeroxScan
impl Default for FeroxScan {
/// Create a default FeroxScan, populates ID with a new UUID
fn default() -> Self {
let new_id = Uuid::new_v4().to_simple().to_string();
@@ -83,6 +99,18 @@ impl FeroxScan {
scan_type: ScanType::File,
}
}
}
/// Implementation of FeroxScan
impl FeroxScan {
/// Stop a currently running scan
pub fn abort(&self) {
self.stop_progress_bar();
if let Some(_task) = &self.task {
// task.abort(); todo uncomment once upgraded to tokio 0.3 (issue #107)
}
}
/// Simple helper to call .finish on the scan's progress bar
fn stop_progress_bar(&self) {
@@ -97,7 +125,7 @@ impl FeroxScan {
pb.clone()
} else {
let num_requests = NUMBER_OF_REQUESTS.load(Ordering::Relaxed);
let pb = progress::add_bar(&self.url, num_requests, false);
let pb = progress::add_bar(&self.url, num_requests, false, false);
pb.reset_elapsed();
@@ -145,6 +173,69 @@ impl PartialEq for FeroxScan {
}
}
/// Serialize implementation for FeroxScan
impl Serialize for FeroxScan {
/// Function that handles serialization of a FeroxScan
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
let mut state = serializer.serialize_struct("FeroxScan", 4)?;
state.serialize_field("id", &self.id)?;
state.serialize_field("url", &self.url)?;
state.serialize_field("scan_type", &self.scan_type)?;
state.serialize_field("complete", &self.complete)?;
state.end()
}
}
/// Deserialize implementation for FeroxScan
impl<'de> Deserialize<'de> for FeroxScan {
/// Deserialize a FeroxScan from a serde_json::Value
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
let mut scan = Self::default();
let map: HashMap<String, Value> = HashMap::deserialize(deserializer)?;
for (key, value) in &map {
match key.as_str() {
"id" => {
if let Some(id) = value.as_str() {
scan.id = id.to_string();
}
}
"scan_type" => {
if let Some(scan_type) = value.as_str() {
scan.scan_type = match scan_type {
"File" => ScanType::File,
"Directory" => ScanType::Directory,
_ => ScanType::File,
}
}
}
"complete" => {
if let Some(complete) = value.as_bool() {
scan.complete = complete;
}
}
"url" => {
if let Some(url) = value.as_str() {
scan.url = url.to_string();
}
}
_ => {}
}
}
Ok(scan)
}
}
/// Container around a locked hashset of `FeroxScan`s, adds wrappers for insertion and searching
#[derive(Debug, Default)]
pub struct FeroxScans {
@@ -152,6 +243,31 @@ pub struct FeroxScans {
pub scans: Mutex<Vec<Arc<Mutex<FeroxScan>>>>,
}
/// Serialize implementation for FeroxScans
impl Serialize for FeroxScans {
/// Function that handles serialization of FeroxScans
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
if let Ok(scans) = self.scans.lock() {
let mut seq = serializer.serialize_seq(Some(scans.len()))?;
for scan in scans.iter() {
if let Ok(unlocked) = scan.lock() {
seq.serialize_element(&*unlocked)?;
}
}
seq.end()
} else {
// if for some reason we can't unlock the mutex, just write an empty list
let seq = serializer.serialize_seq(Some(0))?;
seq.end()
}
}
}
/// Implementation of `FeroxScans`
impl FeroxScans {
/// Add a `FeroxScan` to the internal container
@@ -324,8 +440,12 @@ impl FeroxScans {
fn add_scan(&self, url: &str, scan_type: ScanType) -> (bool, Arc<Mutex<FeroxScan>>) {
let bar = match scan_type {
ScanType::Directory => {
let progress_bar =
progress::add_bar(&url, NUMBER_OF_REQUESTS.load(Ordering::Relaxed), false);
let progress_bar = progress::add_bar(
&url,
NUMBER_OF_REQUESTS.load(Ordering::Relaxed),
false,
false,
);
progress_bar.reset_elapsed();
@@ -382,9 +502,212 @@ fn get_single_spinner() -> ProgressBar {
spinner
}
/// Container around a locked vector of `FeroxResponse`s, adds wrappers for insertion and search
#[derive(Debug, Default)]
pub struct FeroxResponses {
/// Internal structure: locked hashset of `FeroxScan`s
pub responses: Arc<RwLock<Vec<FeroxResponse>>>,
}
/// Serialize implementation for FeroxResponses
impl Serialize for FeroxResponses {
/// Function that handles serialization of FeroxResponses
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
if let Ok(responses) = self.responses.read() {
let mut seq = serializer.serialize_seq(Some(responses.len()))?;
for response in responses.iter() {
seq.serialize_element(response)?;
}
seq.end()
} else {
// if for some reason we can't unlock the mutex, just write an empty list
let seq = serializer.serialize_seq(Some(0))?;
seq.end()
}
}
}
/// Implementation of `FeroxResponses`
impl FeroxResponses {
/// Add a `FeroxResponse` to the internal container
pub fn insert(&self, response: FeroxResponse) {
match self.responses.write() {
Ok(mut responses) => {
responses.push(response);
}
Err(e) => {
log::error!("FeroxResponses' container's mutex is poisoned: {}", e);
}
}
}
/// Simple check for whether or not a FeroxResponse is contained within the inner container
pub fn contains(&self, other: &FeroxResponse) -> bool {
match self.responses.read() {
Ok(responses) => {
for response in responses.iter() {
if response.url == other.url {
return true;
}
}
}
Err(e) => {
log::error!("FeroxResponses' container's mutex is poisoned: {}", e);
}
}
false
}
}
/// Data container for (de)?serialization of multiple items
#[derive(Serialize, Debug)]
pub struct FeroxState {
/// Known scans
scans: &'static FeroxScans,
/// Current running config
config: &'static Configuration,
/// Known responses
responses: &'static FeroxResponses,
}
/// FeroxSerialize implementation for FeroxState
impl FeroxSerialize for FeroxState {
/// Simply return debug format of FeroxState to satisfy as_str
fn as_str(&self) -> String {
format!("{:?}", self)
}
/// Simple call to produce a JSON string using the given FeroxState
fn as_json(&self) -> String {
serde_json::to_string(&self).unwrap_or_default()
}
}
/// Initialize the ctrl+c handler that saves scan state to disk
pub fn initialize() {
log::trace!("enter: initialize");
let result = ctrlc::set_handler(move || {
let ts = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap()
.as_secs();
let slug = if !CONFIGURATION.target_url.is_empty() {
// target url populated
CONFIGURATION
.target_url
.replace("://", "_")
.replace("/", "_")
.replace(".", "_")
} else {
// stdin used
"stdin".to_string()
};
let filename = format!("ferox-{}-{}.state", slug, ts);
let warning = format!(
"🚨 Caught {} 🚨 saving scan state to {} ...",
style("ctrl+c").yellow(),
filename
);
PROGRESS_PRINTER.println(warning);
let state = FeroxState {
config: &CONFIGURATION,
scans: &SCANNED_URLS,
responses: &RESPONSES,
};
let state_file = open_file(&filename);
if let Some(buffered_file) = state_file {
safe_file_write(&state, buffered_file, true);
}
std::process::exit(1);
});
if result.is_err() {
log::error!("Could not set Ctrl+c handler");
std::process::exit(1);
}
log::trace!("exit: initialize");
}
/// Primary logic used to load a Configuration from disk and populate the appropriate data
/// structures
pub fn resume_scan(filename: &str) -> Configuration {
log::trace!("enter: resume_scan({})", filename);
let file = File::open(filename).unwrap_or_else(|e| {
log::error!("{}", e);
log::error!("Could not open state file, exiting");
std::process::exit(1);
});
let reader = BufReader::new(file);
let state: serde_json::Value = serde_json::from_reader(reader).unwrap();
let conf = state.get("config").unwrap_or_else(|| {
log::error!("Could not load configuration from state file, exiting");
std::process::exit(1);
});
let config = serde_json::from_value(conf.clone()).unwrap_or_else(|e| {
log::error!("{}", e);
log::error!("Could not deserialize configuration found in state file, exiting");
std::process::exit(1);
});
if let Some(responses) = state.get("responses") {
if let Some(arr_responses) = responses.as_array() {
for response in arr_responses {
if let Ok(deser_resp) = serde_json::from_value(response.clone()) {
RESPONSES.insert(deser_resp);
}
}
}
}
if let Some(scans) = state.get("scans") {
if let Some(arr_scans) = scans.as_array() {
for scan in arr_scans {
let deser_scan: FeroxScan =
serde_json::from_value(scan.clone()).unwrap_or_default();
// need to determine if it's complete and based on that create a progress bar
// populate it accordingly based on completion
SCANNED_URLS.insert(Arc::new(Mutex::new(deser_scan)));
}
}
}
log::trace!("exit: resume_scan -> {:?}", config);
config
}
#[cfg(test)]
mod tests {
use super::*;
use crate::VERSION;
use predicates::prelude::*;
#[test]
/// test that ScanType's default is File
fn default_scantype_is_file() {
match ScanType::default() {
ScanType::File => {}
ScanType::Directory => panic!(),
}
}
#[test]
/// test that get_single_spinner returns the correct spinner
@@ -528,4 +851,142 @@ mod tests {
assert!(scan.progress_bar.is_some()); // new pb created
assert!(!pb.is_finished()) // not finished
}
#[test]
/// given a JSON entry representing a FeroxScan, test that it deserializes into the proper type
/// with the right attributes
fn ferox_scan_deserialize() {
let fs_json = r#"{"id":"057016a14769414aac9a7a62707598cb","url":"https://spiritanimal.com","scan_type":"Directory","complete":true}"#;
let fs_json_two = r#"{"id":"057016a14769414aac9a7a62707598cb","url":"https://spiritanimal.com","scan_type":"Not Correct","complete":true}"#;
let fs: FeroxScan = serde_json::from_str(fs_json).unwrap();
let fs_two: FeroxScan = serde_json::from_str(fs_json_two).unwrap();
assert_eq!(fs.url, "https://spiritanimal.com");
match fs.scan_type {
ScanType::Directory => {}
ScanType::File => {
panic!();
}
}
match fs_two.scan_type {
ScanType::Directory => {
panic!();
}
ScanType::File => {}
}
match fs.progress_bar {
None => {}
Some(_) => {
panic!();
}
}
assert_eq!(fs.complete, true);
assert_eq!(fs.id, "057016a14769414aac9a7a62707598cb");
}
#[test]
/// given a FeroxScan, test that it serializes into the proper JSON entry
fn ferox_scan_serialize() {
let fs = FeroxScan::new("https://spiritanimal.com", ScanType::Directory, None);
let fs_json = format!(
r#"{{"id":"{}","url":"https://spiritanimal.com","scan_type":"Directory","complete":false}}"#,
fs.lock().unwrap().id
);
assert_eq!(
fs_json,
serde_json::to_string(&*fs.lock().unwrap()).unwrap()
);
}
#[test]
/// given a FeroxScans, test that it serializes into the proper JSON entry
fn ferox_scans_serialize() {
let ferox_scan = FeroxScan::new("https://spiritanimal.com", ScanType::Directory, None);
let ferox_scans = FeroxScans::default();
let ferox_scans_json = format!(
r#"[{{"id":"{}","url":"https://spiritanimal.com","scan_type":"Directory","complete":false}}]"#,
ferox_scan.lock().unwrap().id
);
ferox_scans.scans.lock().unwrap().push(ferox_scan);
assert_eq!(
ferox_scans_json,
serde_json::to_string(&ferox_scans).unwrap()
);
}
#[test]
/// given a FeroxResponses, test that it serializes into the proper JSON entry
fn ferox_responses_serialize() {
let json_response = r#"{"type":"response","url":"https://nerdcore.com/css","path":"/css","wildcard":true,"status":301,"content_length":173,"line_count":10,"word_count":16,"headers":{"server":"nginx/1.16.1"}}"#;
let response: FeroxResponse = serde_json::from_str(json_response).unwrap();
let responses = FeroxResponses::default();
responses.insert(response);
// responses has a response now
// serialized should be a list of responses
let expected = format!("[{}]", json_response);
let serialized = serde_json::to_string(&responses).unwrap();
assert_eq!(expected, serialized);
}
#[test]
/// given a FeroxResponse, test that it serializes into the proper JSON entry
fn ferox_response_serialize_and_deserialize() {
// deserialize
let json_response = r#"{"type":"response","url":"https://nerdcore.com/css","path":"/css","wildcard":true,"status":301,"content_length":173,"line_count":10,"word_count":16,"headers":{"server":"nginx/1.16.1"}}"#;
let response: FeroxResponse = serde_json::from_str(json_response).unwrap();
assert_eq!(response.url.as_str(), "https://nerdcore.com/css");
assert_eq!(response.url.path(), "/css");
assert_eq!(response.wildcard, true);
assert_eq!(response.status.as_u16(), 301);
assert_eq!(response.content_length, 173);
assert_eq!(response.line_count, 10);
assert_eq!(response.word_count, 16);
assert_eq!(response.headers.get("server").unwrap(), "nginx/1.16.1");
// serialize, however, this can fail when headers are out of order
let new_json = serde_json::to_string(&response).unwrap();
assert_eq!(json_response, new_json);
}
#[test]
/// test FeroxSerialize implementation of FeroxState
fn feroxstates_feroxserialize_implementation() {
let ferox_scan = FeroxScan::new("https://spiritanimal.com", ScanType::Directory, None);
let saved_id = ferox_scan.lock().unwrap().id.clone();
SCANNED_URLS.insert(ferox_scan);
let json_response = r#"{"type":"response","url":"https://nerdcore.com/css","path":"/css","wildcard":true,"status":301,"content_length":173,"line_count":10,"word_count":16,"headers":{"server":"nginx/1.16.1"}}"#;
let response: FeroxResponse = serde_json::from_str(json_response).unwrap();
RESPONSES.insert(response);
let ferox_state = FeroxState {
scans: &SCANNED_URLS,
responses: &RESPONSES,
config: &CONFIGURATION,
};
let expected_strs = predicates::str::contains("scans: FeroxScans").and(
predicate::str::contains("config: Configuration")
.and(predicate::str::contains("responses: FeroxResponses"))
.and(predicate::str::contains("nerdcore.com"))
.and(predicate::str::contains("/css"))
.and(predicate::str::contains("https://spiritanimal.com")),
);
assert!(expected_strs.eval(&ferox_state.as_str()));
let json_state = ferox_state.as_json();
let expected = format!(
r#"{{"scans":[{{"id":"{}","url":"https://spiritanimal.com","scan_type":"Directory","complete":false}}],"config":{{"type":"configuration","wordlist":"/usr/share/seclists/Discovery/Web-Content/raft-medium-directories.txt","config":"","proxy":"","replay_proxy":"","target_url":"","status_codes":[200,204,301,302,307,308,401,403,405],"replay_codes":[200,204,301,302,307,308,401,403,405],"filter_status":[],"threads":50,"timeout":7,"verbosity":0,"quiet":false,"json":false,"output":"","debug_log":"","user_agent":"feroxbuster/{}","redirects":false,"insecure":false,"extensions":[],"headers":{{}},"queries":[],"no_recursion":false,"extract_links":false,"add_slash":false,"stdin":false,"depth":4,"scan_limit":0,"filter_size":[],"filter_line_count":[],"filter_word_count":[],"filter_regex":[],"dont_filter":false,"resumed":false,"save_state":true}},"responses":[{{"type":"response","url":"https://nerdcore.com/css","path":"/css","wildcard":true,"status":301,"content_length":173,"line_count":10,"word_count":16,"headers":{{"server":"nginx/1.16.1"}}}}]}}"#,
saved_id, VERSION
);
assert!(predicates::str::similar(expected).eval(&json_state));
}
}

View File

@@ -6,7 +6,7 @@ use crate::{
WordsFilter,
},
heuristics,
scan_manager::{FeroxScans, PAUSE_SCAN},
scan_manager::{FeroxResponses, FeroxScans, PAUSE_SCAN},
utils::{format_url, get_current_depth, make_request},
FeroxChannel, FeroxResponse,
};
@@ -47,6 +47,9 @@ lazy_static! {
/// Vector of implementors of the FeroxFilter trait
static ref FILTERS: Arc<RwLock<Vec<Box<dyn FeroxFilter>>>> = Arc::new(RwLock::new(Vec::<Box<dyn FeroxFilter>>::new()));
/// Vector of FeroxResponse objects
pub static ref RESPONSES: FeroxResponses = FeroxResponses::default();
/// Bounded semaphore used as a barrier to limit concurrent scans
static ref SCAN_LIMITER: Semaphore = Semaphore::new(CONFIGURATION.scan_limit);
}

View File

@@ -711,6 +711,8 @@ fn banner_prints_json() {
.arg("--url")
.arg("http://localhost")
.arg("--json")
.arg("--output")
.arg("/dev/null")
.assert()
.success()
.stderr(

View File

@@ -0,0 +1,93 @@
mod utils;
use assert_cmd::Command;
use httpmock::Method::GET;
use httpmock::{Mock, MockServer};
use predicates::prelude::*;
use utils::{setup_tmp_directory, teardown_tmp_directory};
#[test]
/// pass a known serialized scan with 1 scan complete and 1 not. expect the incomplete scan to
/// start and the complete to not start. expect the responses, scans, and configuration structures
/// to be populated based off the contents of the given state file
fn resume_scan_works() {
let srv = MockServer::start();
let (tmp_dir, file) =
setup_tmp_directory(&["css".to_string(), "stuff".to_string()], "wordlist").unwrap();
// localhost:PORT/ <- complete
// localhost:PORT/js <- will get scanned with /css and /stuff
let complete_scan = format!(
r#"{{"id":"057016a14769414aac9a7a62707598cb","url":"{}","scan_type":"Directory","complete":true}}"#,
srv.url("/")
);
let incomplete_scan = format!(
r#"{{"id":"400b2323a16f43468a04ffcbbeba34c6","url":"{}","scan_type":"Directory","complete":false}}"#,
srv.url("/js")
);
let scans = format!(r#""scans":[{},{}]"#, complete_scan, incomplete_scan);
let config = format!(
r#""config": {{"type":"configuration","wordlist":"{}","config":"","proxy":"","replay_proxy":"","target_url":"{}","status_codes":[200,204,301,302,307,308,401,403,405],"replay_codes":[200,204,301,302,307,308,401,403,405],"filter_status":[],"threads":50,"timeout":7,"verbosity":0,"quiet":false,"json":false,"output":"","debug_log":"","user_agent":"feroxbuster/1.9.0","redirects":false,"insecure":false,"extensions":[],"headers":{{}},"queries":[],"no_recursion":false,"extract_links":false,"add_slash":false,"stdin":false,"depth":2,"scan_limit":1,"filter_size":[],"filter_line_count":[],"filter_word_count":[],"filter_regex":[],"dont_filter":false}}"#,
file.to_string_lossy(),
srv.url("/")
);
// // localhost:PORT/js/css has already been seen, expect not to be scanned
let response = format!(
r#"{{"type":"response","url":"{}","path":"/js/css","wildcard":true,"status":301,"content_length":173,"line_count":10,"word_count":16,"headers":{{"server":"nginx/1.16.1"}}}}"#,
srv.url("/js/css")
);
let responses = format!(r#""responses":[{}]"#, response);
// not scanned because /js is not complete, and /js/stuff response is not known
let not_scanned_yet = Mock::new()
.expect_method(GET)
.expect_path("/js/stuff")
.return_status(200)
.return_body("i expect to be scanned")
.create_on(&srv);
// will get scanned because /js is not complete, but because response of /js/css is known, the
// response will not be in stdout
let already_scanned = Mock::new()
.expect_method(GET)
.expect_path("/js/css")
.return_status(200)
.create_on(&srv);
// already scanned because scan on / is complete
let also_already_scanned = Mock::new()
.expect_method(GET)
.expect_path("/css")
.return_status(200)
.return_body("two words")
.create_on(&srv);
let state_file_contents = format!("{{{},{},{}}}", scans, config, responses);
let (tmp_dir2, state_file) = setup_tmp_directory(&[state_file_contents], "state-file").unwrap();
Command::cargo_bin("feroxbuster")
.unwrap()
.arg("--resume-from")
.arg(state_file.as_os_str())
.assert()
.success()
.stdout(
predicate::str::contains("/js/stuff")
.and(predicate::str::contains("22c"))
.and(predicate::str::contains("5w"))
.and(predicate::str::contains("/js/css"))
.not()
.and(predicate::str::contains("2w"))
.not()
.and(predicate::str::contains("9c"))
.not(),
);
teardown_tmp_directory(tmp_dir);
teardown_tmp_directory(tmp_dir2);
assert_eq!(already_scanned.times_called(), 1);
assert_eq!(also_already_scanned.times_called(), 0);
assert_eq!(not_scanned_yet.times_called(), 1);
}

View File

@@ -364,6 +364,55 @@ fn scanner_single_request_returns_301_without_location_header(
Ok(())
}
#[test]
/// send a single valid request, expect a 200 response that then gets routed to the replay
/// proxy
fn scanner_single_request_replayed_to_proxy() -> Result<(), Box<dyn std::error::Error>> {
let srv = MockServer::start();
let proxy = MockServer::start();
let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
let mock = Mock::new()
.expect_method(GET)
.expect_path("/LICENSE")
.return_status(200)
.return_body("this is a test")
.create_on(&srv);
let mock_two = Mock::new()
.expect_method(GET)
.expect_path("/LICENSE")
.return_status(200)
.return_body("this is a test")
.create_on(&proxy);
let cmd = Command::cargo_bin("feroxbuster")
.unwrap()
.arg("--url")
.arg(srv.url("/"))
.arg("--wordlist")
.arg(file.as_os_str())
.arg("--replay-proxy")
.arg(format!("http://{}", proxy.address().to_string()))
.arg("--replay-codes")
.arg("200")
.unwrap();
cmd.assert()
.success()
.stdout(
predicate::str::contains("/LICENSE")
.and(predicate::str::contains("200"))
.and(predicate::str::contains("14c")),
)
.stderr(predicate::str::contains("Replay Proxy Codes"));
assert_eq!(mock.times_called(), 1);
assert_eq!(mock_two.times_called(), 1);
teardown_tmp_directory(tmp_dir);
Ok(())
}
#[test]
/// send a single valid request, filter the size of the response, expect one out of 2 urls
fn scanner_single_request_scan_with_filtered_result() -> Result<(), Box<dyn std::error::Error>> {
@@ -412,55 +461,6 @@ fn scanner_single_request_scan_with_filtered_result() -> Result<(), Box<dyn std:
Ok(())
}
#[test]
/// send a single valid request, expect a 200 response that then gets routed to the replay
/// proxy
fn scanner_single_request_replayed_to_proxy() -> Result<(), Box<dyn std::error::Error>> {
let srv = MockServer::start();
let proxy = MockServer::start();
let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
let mock = Mock::new()
.expect_method(GET)
.expect_path("/LICENSE")
.return_status(200)
.return_body("this is a test")
.create_on(&srv);
let mock_two = Mock::new()
.expect_method(GET)
.expect_path("/LICENSE")
.return_status(200)
.return_body("this is a test")
.create_on(&proxy);
let cmd = Command::cargo_bin("feroxbuster")
.unwrap()
.arg("--url")
.arg(srv.url("/"))
.arg("--wordlist")
.arg(file.as_os_str())
.arg("--replay-proxy")
.arg(format!("http://{}", proxy.address().to_string()))
.arg("--replay-codes")
.arg("200")
.unwrap();
cmd.assert()
.success()
.stdout(
predicate::str::contains("/LICENSE")
.and(predicate::str::contains("200"))
.and(predicate::str::contains("14")),
)
.stderr(predicate::str::contains("Replay Proxy Codes"));
assert_eq!(mock.times_called(), 1);
assert_eq!(mock_two.times_called(), 1);
teardown_tmp_directory(tmp_dir);
Ok(())
}
#[test]
/// send a single valid request, get a response, and write the logging messages to disk
fn scanner_single_request_scan_with_debug_logging() {