Merge pull request #150 from epi052/allow-cli-override-resume-from

allow CLI args to work w/ --resume-from

.github/pull_request_template.md

@@ -4,7 +4,7 @@ Long form explanations of most of the items below can be found in the [CONTRIBUT
 
 ## Branching checklist
 - [ ] There is an issue associated with your PR (bug, feature, etc.. if not, create one)
-- [ ] Your PR description references the associated issue (i.e. fixes #123)
+- [ ] Your PR description references the associated issue (i.e. fixes #123456)
 - [ ] Code is in its own branch
 - [ ] Branch name is related to the PR contents
 - [ ] PR targets master

.gitignore

@@ -22,3 +22,6 @@ img/**
 # scripts to check code coverage using nightly compiler
 check-coverage.sh
 lcov_cobertura.py
+
+# dockerignore file that makes it so i can work on the docker config without copying a 4GB manifest or w/e it is
+.dockerignore

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "feroxbuster"
-version = "1.9.0"
+version = "1.9.1"
 authors = ["Ben 'epi' Risher <epibar052@gmail.com>"]
 license = "MIT"
 edition = "2018"
@@ -10,10 +10,14 @@ description = "A fast, simple, recursive content discovery tool."
 categories = ["command-line-utilities"]
 keywords = ["pentest", "enumeration", "url-bruteforce", "content-discovery", "web"]
 exclude = [".github/*", "img/*", "check-coverage.sh"]
+build = "build.rs"
 
 [badges]
 maintenance = { status = "actively-developed" }
 
+[build-dependencies]
+clap = "2"
+
 [dependencies]
 futures = { version = "0.3"}
 tokio = { version = "0.2", features = ["full"] }

Dockerfile

@@ -1,8 +1,10 @@
 FROM alpine:latest
 LABEL maintainer="wfnintr@null.net"
 
+RUN sed -i -e 's/v[[:digit:]]\..*\//edge\//g' /etc/apk/repositories && apk upgrade --update-cache --available
+
 # download default wordlists 
-RUN apk add --no-cache --virtual .depends subversion && \
+RUN apk add --no-cache --virtual .depends subversion font-noto-emoji && \
 	svn export https://github.com/danielmiessler/SecLists/trunk/Discovery/Web-Content /usr/share/seclists/Discovery/Web-Content && \
 	apk del .depends
 

build.rs

@@ -0,0 +1,17 @@
+extern crate clap;
+
+use clap::Shell;
+
+include!("src/parser.rs");
+
+fn main() {
+    let outdir = "shell_completions";
+
+    let mut app = initialize();
+
+    let shells: [Shell; 4] = [Shell::Bash, Shell::Fish, Shell::Zsh, Shell::PowerShell];
+
+    for shell in &shells {
+        app.gen_completions("feroxbuster", *shell, outdir);
+    }
+}

shell_completions/_feroxbuster

@@ -0,0 +1,92 @@
+#compdef feroxbuster
+
+_feroxbuster() {
+    typeset -A opt_args
+    local ret=1
+
+    local context curcontext="$curcontext" state line
+    _arguments -s -S -C \
+'-w+[Path to the wordlist]' \
+'--wordlist+[Path to the wordlist]' \
+'*-u+[The target URL(s) (required, unless --stdin used)]' \
+'*--url+[The target URL(s) (required, unless --stdin used)]' \
+'-t+[Number of concurrent threads (default: 50)]' \
+'--threads+[Number of concurrent threads (default: 50)]' \
+'-d+[Maximum recursion depth, a depth of 0 is infinite recursion (default: 4)]' \
+'--depth+[Maximum recursion depth, a depth of 0 is infinite recursion (default: 4)]' \
+'-T+[Number of seconds before a request times out (default: 7)]' \
+'--timeout+[Number of seconds before a request times out (default: 7)]' \
+'-p+[Proxy to use for requests (ex: http(s)://host:port, socks5://host:port)]' \
+'--proxy+[Proxy to use for requests (ex: http(s)://host:port, socks5://host:port)]' \
+'-P+[Send only unfiltered requests through a Replay Proxy, instead of all requests]' \
+'--replay-proxy+[Send only unfiltered requests through a Replay Proxy, instead of all requests]' \
+'*-R+[Status Codes to send through a Replay Proxy when found (default: --status-codes value)]' \
+'*--replay-codes+[Status Codes to send through a Replay Proxy when found (default: --status-codes value)]' \
+'*-s+[Status Codes to include (allow list) (default: 200 204 301 302 307 308 401 403 405)]' \
+'*--status-codes+[Status Codes to include (allow list) (default: 200 204 301 302 307 308 401 403 405)]' \
+'-o+[Output file to write results to (use w/ --json for JSON entries)]' \
+'--output+[Output file to write results to (use w/ --json for JSON entries)]' \
+'(-u --url)--resume-from+[State file from which to resume a partially complete scan (ex. --resume-from ferox-1606586780.state)]' \
+'--debug-log+[Output file to write log entries (use w/ --json for JSON entries)]' \
+'-a+[Sets the User-Agent (default: feroxbuster/VERSION)]' \
+'--user-agent+[Sets the User-Agent (default: feroxbuster/VERSION)]' \
+'*-x+[File extension(s) to search for (ex: -x php -x pdf js)]' \
+'*--extensions+[File extension(s) to search for (ex: -x php -x pdf js)]' \
+'*-H+[Specify HTTP headers (ex: -H Header:val '\''stuff: things'\'')]' \
+'*--headers+[Specify HTTP headers (ex: -H Header:val '\''stuff: things'\'')]' \
+'*-Q+[Specify URL query parameters (ex: -Q token=stuff -Q secret=key)]' \
+'*--query+[Specify URL query parameters (ex: -Q token=stuff -Q secret=key)]' \
+'*-S+[Filter out messages of a particular size (ex: -S 5120 -S 4927,1970)]' \
+'*--filter-size+[Filter out messages of a particular size (ex: -S 5120 -S 4927,1970)]' \
+'*-X+[Filter out messages via regular expression matching on the response'\''s body (ex: -X '\''^ignore me$'\'')]' \
+'*--filter-regex+[Filter out messages via regular expression matching on the response'\''s body (ex: -X '\''^ignore me$'\'')]' \
+'*-W+[Filter out messages of a particular word count (ex: -W 312 -W 91,82)]' \
+'*--filter-words+[Filter out messages of a particular word count (ex: -W 312 -W 91,82)]' \
+'*-N+[Filter out messages of a particular line count (ex: -N 20 -N 31,30)]' \
+'*--filter-lines+[Filter out messages of a particular line count (ex: -N 20 -N 31,30)]' \
+'*-C+[Filter out status codes (deny list) (ex: -C 200 -C 401)]' \
+'*--filter-status+[Filter out status codes (deny list) (ex: -C 200 -C 401)]' \
+'-L+[Limit total number of concurrent scans (default: 0, i.e. no limit)]' \
+'--scan-limit+[Limit total number of concurrent scans (default: 0, i.e. no limit)]' \
+'*-v[Increase verbosity level (use -vv or more for greater effect. \[CAUTION\] 4 -v'\''s is probably too much)]' \
+'*--verbosity[Increase verbosity level (use -vv or more for greater effect. \[CAUTION\] 4 -v'\''s is probably too much)]' \
+'-q[Only print URLs; Don'\''t print status codes, response size, running config, etc...]' \
+'--quiet[Only print URLs; Don'\''t print status codes, response size, running config, etc...]' \
+'--json[Emit JSON logs to --output and --debug-log instead of normal text]' \
+'-D[Don'\''t auto-filter wildcard responses]' \
+'--dont-filter[Don'\''t auto-filter wildcard responses]' \
+'-r[Follow redirects]' \
+'--redirects[Follow redirects]' \
+'-k[Disables TLS certificate validation]' \
+'--insecure[Disables TLS certificate validation]' \
+'-n[Do not scan recursively]' \
+'--no-recursion[Do not scan recursively]' \
+'(-x --extensions)-f[Append / to each request]' \
+'(-x --extensions)--add-slash[Append / to each request]' \
+'(-u --url)--stdin[Read url(s) from STDIN]' \
+'-e[Extract links from response body (html, javascript, etc...); make new requests based on findings (default: false)]' \
+'--extract-links[Extract links from response body (html, javascript, etc...); make new requests based on findings (default: false)]' \
+'-h[Prints help information]' \
+'--help[Prints help information]' \
+'-V[Prints version information]' \
+'--version[Prints version information]' \
+&& ret=0
+    
+}
+
+(( $+functions[_feroxbuster_commands] )) ||
+_feroxbuster_commands() {
+    local commands; commands=(
+        
+    )
+    _describe -t commands 'feroxbuster commands' commands "$@"
+}
+(( $+functions[_feroxbuster_commands] )) ||
+_feroxbuster_commands() {
+    local commands; commands=(
+        
+    )
+    _describe -t commands 'feroxbuster commands' commands "$@"
+}
+
+_feroxbuster "$@"

shell_completions/_feroxbuster.ps1

@@ -0,0 +1,38 @@
+
+@('feroxbuster', './feroxbuster') | %{
+    Register-ArgumentCompleter -Native -CommandName $_ -ScriptBlock {
+        param($wordToComplete, $commandAst, $cursorPosition)
+
+        $command = '_feroxbuster'
+        $commandAst.CommandElements |
+            Select-Object -Skip 1 |
+            %{
+                switch ($_.ToString()) {
+
+                    'feroxbuster' {
+                        $command += '_feroxbuster'
+                        break
+                    }
+
+                    default { 
+                        break
+                    }
+                }
+            }
+
+        $completions = @()
+
+        switch ($command) {
+
+            '_feroxbuster' {
+                $completions = @('-v', '-q', '-D', '-r', '-k', '-n', '-f', '-e', '-h', '-V', '-w', '-u', '-t', '-d', '-T', '-p', '-P', '-R', '-s', '-o', '-a', '-x', '-H', '-Q', '-S', '-X', '-W', '-N', '-C', '-L', '--verbosity', '--quiet', '--json', '--dont-filter', '--redirects', '--insecure', '--no-recursion', '--add-slash', '--stdin', '--extract-links', '--help', '--version', '--wordlist', '--url', '--threads', '--depth', '--timeout', '--proxy', '--replay-proxy', '--replay-codes', '--status-codes', '--output', '--resume-from', '--debug-log', '--user-agent', '--extensions', '--headers', '--query', '--filter-size', '--filter-regex', '--filter-words', '--filter-lines', '--filter-status', '--scan-limit')
+            }
+
+        }
+
+        $completions |
+            ?{ $_ -like "$wordToComplete*" } |
+            Sort-Object |
+            %{ New-Object System.Management.Automation.CompletionResult $_, $_, 'ParameterValue', $_ }
+    }
+}

shell_completions/feroxbuster.bash-completion

@@ -0,0 +1,212 @@
+_feroxbuster() {
+    local i cur prev opts cmds
+    COMPREPLY=()
+    cur="${COMP_WORDS[COMP_CWORD]}"
+    prev="${COMP_WORDS[COMP_CWORD-1]}"
+    cmd=""
+    opts=""
+
+    for i in ${COMP_WORDS[@]}
+    do
+        case "${i}" in
+            feroxbuster)
+                cmd="feroxbuster"
+                ;;
+            
+            feroxbuster)
+                cmd+="__feroxbuster"
+                ;;
+            *)
+                ;;
+        esac
+    done
+
+    case "${cmd}" in
+        feroxbuster)
+            opts=" -v -q -D -r -k -n -f -e -h -V -w -u -t -d -T -p -P -R -s -o -a -x -H -Q -S -X -W -N -C -L  --verbosity --quiet --json --dont-filter --redirects --insecure --no-recursion --add-slash --stdin --extract-links --help --version --wordlist --url --threads --depth --timeout --proxy --replay-proxy --replay-codes --status-codes --output --resume-from --debug-log --user-agent --extensions --headers --query --filter-size --filter-regex --filter-words --filter-lines --filter-status --scan-limit  "
+            if [[ ${cur} == -* || ${COMP_CWORD} -eq 1 ]] ; then
+                COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) )
+                return 0
+            fi
+            case "${prev}" in
+                
+                --wordlist)
+                    COMPREPLY=("<FILE>")
+                    return 0
+                    ;;
+                    -w)
+                    COMPREPLY=("<FILE>")
+                    return 0
+                    ;;
+                --url)
+                    COMPREPLY=("<URL>...")
+                    return 0
+                    ;;
+                    -u)
+                    COMPREPLY=("<URL>...")
+                    return 0
+                    ;;
+                --threads)
+                    COMPREPLY=("<THREADS>")
+                    return 0
+                    ;;
+                    -t)
+                    COMPREPLY=("<THREADS>")
+                    return 0
+                    ;;
+                --depth)
+                    COMPREPLY=("<RECURSION_DEPTH>")
+                    return 0
+                    ;;
+                    -d)
+                    COMPREPLY=("<RECURSION_DEPTH>")
+                    return 0
+                    ;;
+                --timeout)
+                    COMPREPLY=("<SECONDS>")
+                    return 0
+                    ;;
+                    -T)
+                    COMPREPLY=("<SECONDS>")
+                    return 0
+                    ;;
+                --proxy)
+                    COMPREPLY=("<PROXY>")
+                    return 0
+                    ;;
+                    -p)
+                    COMPREPLY=("<PROXY>")
+                    return 0
+                    ;;
+                --replay-proxy)
+                    COMPREPLY=("<REPLAY_PROXY>")
+                    return 0
+                    ;;
+                    -P)
+                    COMPREPLY=("<REPLAY_PROXY>")
+                    return 0
+                    ;;
+                --replay-codes)
+                    COMPREPLY=("<REPLAY_CODE>...")
+                    return 0
+                    ;;
+                    -R)
+                    COMPREPLY=("<REPLAY_CODE>...")
+                    return 0
+                    ;;
+                --status-codes)
+                    COMPREPLY=("<STATUS_CODE>...")
+                    return 0
+                    ;;
+                    -s)
+                    COMPREPLY=("<STATUS_CODE>...")
+                    return 0
+                    ;;
+                --output)
+                    COMPREPLY=("<FILE>")
+                    return 0
+                    ;;
+                    -o)
+                    COMPREPLY=("<FILE>")
+                    return 0
+                    ;;
+                --resume-from)
+                    COMPREPLY=("<STATE_FILE>")
+                    return 0
+                    ;;
+                --debug-log)
+                    COMPREPLY=("<FILE>")
+                    return 0
+                    ;;
+                --user-agent)
+                    COMPREPLY=("<USER_AGENT>")
+                    return 0
+                    ;;
+                    -a)
+                    COMPREPLY=("<USER_AGENT>")
+                    return 0
+                    ;;
+                --extensions)
+                    COMPREPLY=("<FILE_EXTENSION>...")
+                    return 0
+                    ;;
+                    -x)
+                    COMPREPLY=("<FILE_EXTENSION>...")
+                    return 0
+                    ;;
+                --headers)
+                    COMPREPLY=("<HEADER>...")
+                    return 0
+                    ;;
+                    -H)
+                    COMPREPLY=("<HEADER>...")
+                    return 0
+                    ;;
+                --query)
+                    COMPREPLY=("<QUERY>...")
+                    return 0
+                    ;;
+                    -Q)
+                    COMPREPLY=("<QUERY>...")
+                    return 0
+                    ;;
+                --filter-size)
+                    COMPREPLY=("<SIZE>...")
+                    return 0
+                    ;;
+                    -S)
+                    COMPREPLY=("<SIZE>...")
+                    return 0
+                    ;;
+                --filter-regex)
+                    COMPREPLY=("<REGEX>...")
+                    return 0
+                    ;;
+                    -X)
+                    COMPREPLY=("<REGEX>...")
+                    return 0
+                    ;;
+                --filter-words)
+                    COMPREPLY=("<WORDS>...")
+                    return 0
+                    ;;
+                    -W)
+                    COMPREPLY=("<WORDS>...")
+                    return 0
+                    ;;
+                --filter-lines)
+                    COMPREPLY=("<LINES>...")
+                    return 0
+                    ;;
+                    -N)
+                    COMPREPLY=("<LINES>...")
+                    return 0
+                    ;;
+                --filter-status)
+                    COMPREPLY=("<STATUS_CODE>...")
+                    return 0
+                    ;;
+                    -C)
+                    COMPREPLY=("<STATUS_CODE>...")
+                    return 0
+                    ;;
+                --scan-limit)
+                    COMPREPLY=("<SCAN_LIMIT>")
+                    return 0
+                    ;;
+                    -L)
+                    COMPREPLY=("<SCAN_LIMIT>")
+                    return 0
+                    ;;
+                *)
+                    COMPREPLY=()
+                    ;;
+            esac
+            COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) )
+            return 0
+            ;;
+        
+    esac
+}
+
+complete -F _feroxbuster -o bashdefault -o default feroxbuster

shell_completions/feroxbuster.fish

@@ -0,0 +1,47 @@
+function __fish_using_command
+    set cmd (commandline -opc)
+    if [ (count $cmd) -eq (count $argv) ]
+        for i in (seq (count $argv))
+            if [ $cmd[$i] != $argv[$i] ]
+                return 1
+            end
+        end
+        return 0
+    end
+    return 1
+end
+
+complete -c feroxbuster -n "__fish_using_command feroxbuster" -s w -l wordlist -d 'Path to the wordlist'
+complete -c feroxbuster -n "__fish_using_command feroxbuster" -s u -l url -d 'The target URL(s) (required, unless --stdin used)'
+complete -c feroxbuster -n "__fish_using_command feroxbuster" -s t -l threads -d 'Number of concurrent threads (default: 50)'
+complete -c feroxbuster -n "__fish_using_command feroxbuster" -s d -l depth -d 'Maximum recursion depth, a depth of 0 is infinite recursion (default: 4)'
+complete -c feroxbuster -n "__fish_using_command feroxbuster" -s T -l timeout -d 'Number of seconds before a request times out (default: 7)'
+complete -c feroxbuster -n "__fish_using_command feroxbuster" -s p -l proxy -d 'Proxy to use for requests (ex: http(s)://host:port, socks5://host:port)'
+complete -c feroxbuster -n "__fish_using_command feroxbuster" -s P -l replay-proxy -d 'Send only unfiltered requests through a Replay Proxy, instead of all requests'
+complete -c feroxbuster -n "__fish_using_command feroxbuster" -s R -l replay-codes -d 'Status Codes to send through a Replay Proxy when found (default: --status-codes value)'
+complete -c feroxbuster -n "__fish_using_command feroxbuster" -s s -l status-codes -d 'Status Codes to include (allow list) (default: 200 204 301 302 307 308 401 403 405)'
+complete -c feroxbuster -n "__fish_using_command feroxbuster" -s o -l output -d 'Output file to write results to (use w/ --json for JSON entries)'
+complete -c feroxbuster -n "__fish_using_command feroxbuster" -l resume-from -d 'State file from which to resume a partially complete scan (ex. --resume-from ferox-1606586780.state)'
+complete -c feroxbuster -n "__fish_using_command feroxbuster" -l debug-log -d 'Output file to write log entries (use w/ --json for JSON entries)'
+complete -c feroxbuster -n "__fish_using_command feroxbuster" -s a -l user-agent -d 'Sets the User-Agent (default: feroxbuster/VERSION)'
+complete -c feroxbuster -n "__fish_using_command feroxbuster" -s x -l extensions -d 'File extension(s) to search for (ex: -x php -x pdf js)'
+complete -c feroxbuster -n "__fish_using_command feroxbuster" -s H -l headers -d 'Specify HTTP headers (ex: -H Header:val \'stuff: things\')'
+complete -c feroxbuster -n "__fish_using_command feroxbuster" -s Q -l query -d 'Specify URL query parameters (ex: -Q token=stuff -Q secret=key)'
+complete -c feroxbuster -n "__fish_using_command feroxbuster" -s S -l filter-size -d 'Filter out messages of a particular size (ex: -S 5120 -S 4927,1970)'
+complete -c feroxbuster -n "__fish_using_command feroxbuster" -s X -l filter-regex -d 'Filter out messages via regular expression matching on the response\'s body (ex: -X \'^ignore me$\')'
+complete -c feroxbuster -n "__fish_using_command feroxbuster" -s W -l filter-words -d 'Filter out messages of a particular word count (ex: -W 312 -W 91,82)'
+complete -c feroxbuster -n "__fish_using_command feroxbuster" -s N -l filter-lines -d 'Filter out messages of a particular line count (ex: -N 20 -N 31,30)'
+complete -c feroxbuster -n "__fish_using_command feroxbuster" -s C -l filter-status -d 'Filter out status codes (deny list) (ex: -C 200 -C 401)'
+complete -c feroxbuster -n "__fish_using_command feroxbuster" -s L -l scan-limit -d 'Limit total number of concurrent scans (default: 0, i.e. no limit)'
+complete -c feroxbuster -n "__fish_using_command feroxbuster" -s v -l verbosity -d 'Increase verbosity level (use -vv or more for greater effect. [CAUTION] 4 -v\'s is probably too much)'
+complete -c feroxbuster -n "__fish_using_command feroxbuster" -s q -l quiet -d 'Only print URLs; Don\'t print status codes, response size, running config, etc...'
+complete -c feroxbuster -n "__fish_using_command feroxbuster" -l json -d 'Emit JSON logs to --output and --debug-log instead of normal text'
+complete -c feroxbuster -n "__fish_using_command feroxbuster" -s D -l dont-filter -d 'Don\'t auto-filter wildcard responses'
+complete -c feroxbuster -n "__fish_using_command feroxbuster" -s r -l redirects -d 'Follow redirects'
+complete -c feroxbuster -n "__fish_using_command feroxbuster" -s k -l insecure -d 'Disables TLS certificate validation'
+complete -c feroxbuster -n "__fish_using_command feroxbuster" -s n -l no-recursion -d 'Do not scan recursively'
+complete -c feroxbuster -n "__fish_using_command feroxbuster" -s f -l add-slash -d 'Append / to each request'
+complete -c feroxbuster -n "__fish_using_command feroxbuster" -l stdin -d 'Read url(s) from STDIN'
+complete -c feroxbuster -n "__fish_using_command feroxbuster" -s e -l extract-links -d 'Extract links from response body (html, javascript, etc...); make new requests based on findings (default: false)'
+complete -c feroxbuster -n "__fish_using_command feroxbuster" -s h -l help -d 'Prints help information'
+complete -c feroxbuster -n "__fish_using_command feroxbuster" -s V -l version -d 'Prints version information'

src/config.rs

@@ -2,7 +2,7 @@ use crate::scan_manager::resume_scan;
 use crate::utils::{module_colorizer, status_colorizer};
 use crate::{client, parser, progress};
 use crate::{FeroxSerialize, DEFAULT_CONFIG_NAME, DEFAULT_STATUS_CODES, DEFAULT_WORDLIST, VERSION};
-use clap::value_t;
+use clap::{value_t, ArgMatches};
 use indicatif::{MultiProgress, ProgressBar, ProgressDrawTarget};
 use lazy_static::lazy_static;
 use reqwest::{Client, StatusCode};
@@ -25,6 +25,32 @@ lazy_static! {
     pub static ref PROGRESS_PRINTER: ProgressBar = progress::add_bar("", 0, true, false);
 }
 
+/// macro helper to abstract away repetitive configuration updates
+macro_rules! update_config_if_present {
+    ($c:expr, $m:ident, $v:expr, $t:ty) => {
+        match value_t!($m, $v, $t) {
+            Ok(value) => *$c = value, // Update value
+            Err(clap::Error {
+                kind: clap::ErrorKind::ArgumentNotFound,
+                message: _,
+                info: _,
+            }) => {
+                // Do nothing if argument not found
+            }
+            Err(e) => e.exit(), // Exit with error on parse error
+        }
+    };
+}
+
+/// macro helper to abstract away repetitive if not default: update checks
+macro_rules! update_if_not_default {
+    ($old:expr, $new:expr, $default:expr) => {
+        if $new != $default {
+            *$old = $new;
+        }
+    };
+}
+
 /// simple helper to clean up some code reuse below; panics under test / exits in prod
 fn report_and_exit(err: &str) -> ! {
     eprintln!(
@@ -81,7 +107,7 @@ pub struct Configuration {
     pub status_codes: Vec<u16>,
 
     /// Status Codes to replay to the Replay Proxy (default: whatever is passed to --status-code)
-    #[serde(default)]
+    #[serde(default = "status_codes")]
     pub replay_codes: Vec<u16>,
 
     /// Status Codes to filter out (deny list)
@@ -367,11 +393,29 @@ impl Configuration {
 
         let args = parser::initialize().get_matches();
 
+        // Get the default configuration, this is what will apply if nothing
+        // else is specified.
+        let mut config = Configuration::default();
+
+        // read in all config files
+        Self::parse_config_files(&mut config);
+
+        // read in the user provided options, this produces a separate instance of Configuration
+        // in order to allow for potentially merging into a --resume-from Configuration
+        let cli_config = Self::parse_cli_args(&args);
+
+        // --resume-from used, need to first read the Configuration from disk, and then
+        // merge the cli_config into the resumed config
         if let Some(filename) = args.value_of("resume_from") {
             // when resuming a scan, instead of normal configuration loading, we just
             // load the config from disk by calling resume_scan
             let mut previous_config = resume_scan(filename);
 
+            // if any other arguments were passed on the command line, the theory is that the
+            // user meant to modify the previously cancelled/saved scan in some way that we
+            // should take into account
+            Self::merge_config(&mut previous_config, cli_config);
+
             // the resumed flag isn't printed in the banner and really has no business being
             // serialized or included in much of the usual config logic; simply setting it to true
             // here and being done with it
@@ -388,10 +432,19 @@ impl Configuration {
             return previous_config;
         }
 
-        // Get the default configuration, this is what will apply if nothing
-        // else is specified.
-        let mut config = Configuration::default();
+        // if we've gotten to this point in the code, --resume-from was not used, so we need to
+        // merge the cli options into the config file options and return the result
+        Self::merge_config(&mut config, cli_config);
 
+        // rebuild clients is the last step in either code branch
+        Self::try_rebuild_clients(&mut config);
+
+        config
+    }
+
+    /// Parse all possible versions of the ferox-config.toml file, adhering to the order of
+    /// precedence outlined above
+    fn parse_config_files(mut config: &mut Self) {
         // Next, we parse the ferox-config.toml file, if present and set the values
         // therein to overwrite our default values. Deserialized defaults are specified
         // in the Configuration struct so that we don't change anything that isn't
@@ -433,22 +486,12 @@ impl Configuration {
             let config_file = cwd.join(DEFAULT_CONFIG_NAME);
             Self::parse_and_merge_config(config_file, &mut config);
         }
+    }
 
-        macro_rules! update_config_if_present {
-            ($c:expr, $m:ident, $v:expr, $t:ty) => {
-                match value_t!($m, $v, $t) {
-                    Ok(value) => *$c = value, // Update value
-                    Err(clap::Error {
-                        kind: clap::ErrorKind::ArgumentNotFound,
-                        message: _,
-                        info: _,
-                    }) => {
-                        // Do nothing if argument not found
-                    }
-                    Err(e) => e.exit(), // Exit with error on parse error
-                }
-            };
-        }
+    /// Given a set of ArgMatches read from the CLI, update and return the default Configuration
+    /// settings
+    fn parse_cli_args(args: &ArgMatches) -> Self {
+        let mut config = Configuration::default();
 
         update_config_if_present!(&mut config.threads, args, "threads", usize);
         update_config_if_present!(&mut config.depth, args, "depth", usize);
@@ -562,8 +605,8 @@ impl Configuration {
 
         if args.is_present("stdin") {
             config.stdin = true;
-        } else {
-            config.target_url = String::from(args.value_of("url").unwrap());
+        } else if let Some(url) = args.value_of("url") {
+            config.target_url = String::from(url);
         }
 
         ////
@@ -609,8 +652,6 @@ impl Configuration {
             }
         }
 
-        Self::try_rebuild_clients(&mut config);
-
         config
     }
 
@@ -681,38 +722,63 @@ impl Configuration {
     }
 
     /// Given two Configurations, overwrite `settings` with the fields found in `settings_to_merge`
-    fn merge_config(settings: &mut Self, settings_to_merge: Self) {
-        settings.threads = settings_to_merge.threads;
-        settings.wordlist = settings_to_merge.wordlist;
-        settings.status_codes = settings_to_merge.status_codes;
-        settings.proxy = settings_to_merge.proxy;
-        settings.timeout = settings_to_merge.timeout;
-        settings.verbosity = settings_to_merge.verbosity;
-        settings.quiet = settings_to_merge.quiet;
-        settings.output = settings_to_merge.output;
-        settings.user_agent = settings_to_merge.user_agent;
-        settings.redirects = settings_to_merge.redirects;
-        settings.insecure = settings_to_merge.insecure;
-        settings.extract_links = settings_to_merge.extract_links;
-        settings.extensions = settings_to_merge.extensions;
-        settings.headers = settings_to_merge.headers;
-        settings.queries = settings_to_merge.queries;
-        settings.no_recursion = settings_to_merge.no_recursion;
-        settings.add_slash = settings_to_merge.add_slash;
-        settings.stdin = settings_to_merge.stdin;
-        settings.depth = settings_to_merge.depth;
-        settings.filter_size = settings_to_merge.filter_size;
-        settings.filter_regex = settings_to_merge.filter_regex;
-        settings.filter_word_count = settings_to_merge.filter_word_count;
-        settings.filter_line_count = settings_to_merge.filter_line_count;
-        settings.filter_status = settings_to_merge.filter_status;
-        settings.dont_filter = settings_to_merge.dont_filter;
-        settings.scan_limit = settings_to_merge.scan_limit;
-        settings.replay_proxy = settings_to_merge.replay_proxy;
-        settings.replay_codes = settings_to_merge.replay_codes;
-        settings.save_state = settings_to_merge.save_state;
-        settings.debug_log = settings_to_merge.debug_log;
-        settings.json = settings_to_merge.json;
+    fn merge_config(conf: &mut Self, new: Self) {
+        // does not include the following Configuration fields, as they don't make sense here
+        //  - kind
+        //  - client
+        //  - replay_client
+        //  - resumed
+        //  - config
+        update_if_not_default!(&mut conf.target_url, new.target_url, "");
+        update_if_not_default!(&mut conf.proxy, new.proxy, "");
+        update_if_not_default!(&mut conf.verbosity, new.verbosity, 0);
+        update_if_not_default!(&mut conf.quiet, new.quiet, false);
+        update_if_not_default!(&mut conf.output, new.output, "");
+        update_if_not_default!(&mut conf.redirects, new.redirects, false);
+        update_if_not_default!(&mut conf.insecure, new.insecure, false);
+        update_if_not_default!(&mut conf.extract_links, new.extract_links, false);
+        update_if_not_default!(&mut conf.extensions, new.extensions, Vec::<String>::new());
+        update_if_not_default!(&mut conf.headers, new.headers, HashMap::new());
+        update_if_not_default!(&mut conf.queries, new.queries, Vec::new());
+        update_if_not_default!(&mut conf.no_recursion, new.no_recursion, false);
+        update_if_not_default!(&mut conf.add_slash, new.add_slash, false);
+        update_if_not_default!(&mut conf.stdin, new.stdin, false);
+        update_if_not_default!(&mut conf.filter_size, new.filter_size, Vec::<u64>::new());
+        update_if_not_default!(
+            &mut conf.filter_regex,
+            new.filter_regex,
+            Vec::<String>::new()
+        );
+        update_if_not_default!(
+            &mut conf.filter_word_count,
+            new.filter_word_count,
+            Vec::<usize>::new()
+        );
+        update_if_not_default!(
+            &mut conf.filter_line_count,
+            new.filter_line_count,
+            Vec::<usize>::new()
+        );
+        update_if_not_default!(
+            &mut conf.filter_status,
+            new.filter_status,
+            Vec::<u16>::new()
+        );
+        update_if_not_default!(&mut conf.dont_filter, new.dont_filter, false);
+        update_if_not_default!(&mut conf.scan_limit, new.scan_limit, 0);
+        update_if_not_default!(&mut conf.replay_proxy, new.replay_proxy, "");
+        update_if_not_default!(&mut conf.debug_log, new.debug_log, "");
+        update_if_not_default!(&mut conf.json, new.json, false);
+
+        update_if_not_default!(&mut conf.timeout, new.timeout, timeout());
+        update_if_not_default!(&mut conf.user_agent, new.user_agent, user_agent());
+        update_if_not_default!(&mut conf.threads, new.threads, threads());
+        update_if_not_default!(&mut conf.depth, new.depth, depth());
+        update_if_not_default!(&mut conf.wordlist, new.wordlist, wordlist());
+        update_if_not_default!(&mut conf.status_codes, new.status_codes, status_codes());
+        // status_codes() is the default for replay_codes, if they're not provided
+        update_if_not_default!(&mut conf.replay_codes, new.replay_codes, status_codes());
+        update_if_not_default!(&mut conf.save_state, new.save_state, save_state());
     }
 
     /// If present, read in `DEFAULT_CONFIG_NAME` and deserialize the specified values

src/main.rs

@@ -180,8 +180,8 @@ async fn get_targets() -> FeroxResult<Vec<String>> {
             targets.push(line?);
         }
     } else if CONFIGURATION.resumed {
-        // resume-from can't be used with any other flag, making it mutually exclusive from either
-        // of the other two options
+        // resume-from can't be used with --url, and --stdin is marked false for every resumed
+        // scan, making it mutually exclusive from either of the other two options
         if let Ok(scans) = SCANNED_URLS.scans.lock() {
             for scan in scans.iter() {
                 // SCANNED_URLS gets deserialized scans added to it at program start if --resume-from

src/parser.rs

@@ -1,10 +1,9 @@
-use crate::VERSION;
 use clap::{App, Arg, ArgGroup};
 
 /// Create and return an instance of [clap::App](https://docs.rs/clap/latest/clap/struct.App.html), i.e. the Command Line Interface's configuration
 pub fn initialize() -> App<'static, 'static> {
     App::new("feroxbuster")
-        .version(VERSION)
+        .version(env!("CARGO_PKG_VERSION"))
         .author("Ben 'epi' Risher (@epi052)")
         .about("A fast, simple, recursive content discovery tool written in Rust")
         .arg(
@@ -136,7 +135,7 @@ pub fn initialize() -> App<'static, 'static> {
                 .long("resume-from")
                 .value_name("STATE_FILE")
                 .help("State file from which to resume a partially complete scan (ex. --resume-from ferox-1606586780.state)")
-                .conflicts_with_all(&["wordlist", "url", "threads", "depth", "timeout", "verbosity", "proxy", "replay_proxy", "replay_codes", "status_codes", "quiet", "json", "dont_filter", "output", "debug_log", "user_agent", "redirects", "insecure", "extensions", "headers", "queries", "no_recursion", "add_slash", "stdin", "filter_size", "filter_regex", "filter_words", "filter_lines", "filter_status", "extract_links", "scan_limit"])
+                .conflicts_with("url")
                 .takes_value(true),
         )
         .arg(

src/scan_manager.rs

@@ -668,7 +668,6 @@ pub fn resume_scan(filename: &str) -> Configuration {
         std::process::exit(1);
     });
 
-    // let scans: FeroxScans = serde_json::from_value(state.get("scans").unwrap().clone()).unwrap();
     if let Some(responses) = state.get("responses") {
         if let Some(arr_responses) = responses.as_array() {
             for response in arr_responses {
@@ -698,6 +697,7 @@ pub fn resume_scan(filename: &str) -> Configuration {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::VERSION;
     use predicates::prelude::*;
 
     #[test]
@@ -983,8 +983,8 @@ mod tests {
 
         let json_state = ferox_state.as_json();
         let expected = format!(
-            r#"{{"scans":[{{"id":"{}","url":"https://spiritanimal.com","scan_type":"Directory","complete":false}}],"config":{{"type":"configuration","wordlist":"/usr/share/seclists/Discovery/Web-Content/raft-medium-directories.txt","config":"","proxy":"","replay_proxy":"","target_url":"","status_codes":[200,204,301,302,307,308,401,403,405],"replay_codes":[200,204,301,302,307,308,401,403,405],"filter_status":[],"threads":50,"timeout":7,"verbosity":0,"quiet":false,"json":false,"output":"","debug_log":"","user_agent":"feroxbuster/1.9.0","redirects":false,"insecure":false,"extensions":[],"headers":{{}},"queries":[],"no_recursion":false,"extract_links":false,"add_slash":false,"stdin":false,"depth":4,"scan_limit":0,"filter_size":[],"filter_line_count":[],"filter_word_count":[],"filter_regex":[],"dont_filter":false,"resumed":false,"save_state":true}},"responses":[{{"type":"response","url":"https://nerdcore.com/css","path":"/css","wildcard":true,"status":301,"content_length":173,"line_count":10,"word_count":16,"headers":{{"server":"nginx/1.16.1"}}}}]}}"#,
-            saved_id
+            r#"{{"scans":[{{"id":"{}","url":"https://spiritanimal.com","scan_type":"Directory","complete":false}}],"config":{{"type":"configuration","wordlist":"/usr/share/seclists/Discovery/Web-Content/raft-medium-directories.txt","config":"","proxy":"","replay_proxy":"","target_url":"","status_codes":[200,204,301,302,307,308,401,403,405],"replay_codes":[200,204,301,302,307,308,401,403,405],"filter_status":[],"threads":50,"timeout":7,"verbosity":0,"quiet":false,"json":false,"output":"","debug_log":"","user_agent":"feroxbuster/{}","redirects":false,"insecure":false,"extensions":[],"headers":{{}},"queries":[],"no_recursion":false,"extract_links":false,"add_slash":false,"stdin":false,"depth":4,"scan_limit":0,"filter_size":[],"filter_line_count":[],"filter_word_count":[],"filter_regex":[],"dont_filter":false,"resumed":false,"save_state":true}},"responses":[{{"type":"response","url":"https://nerdcore.com/css","path":"/css","wildcard":true,"status":301,"content_length":173,"line_count":10,"word_count":16,"headers":{{"server":"nginx/1.16.1"}}}}]}}"#,
+            saved_id, VERSION
         );
 
         assert!(predicates::str::similar(expected).eval(&json_state));

tests/test_scanner.rs

@@ -364,6 +364,55 @@ fn scanner_single_request_returns_301_without_location_header(
     Ok(())
 }
 
+#[test]
+/// send a single valid request, expect a 200 response that then gets routed to the replay
+/// proxy
+fn scanner_single_request_replayed_to_proxy() -> Result<(), Box<dyn std::error::Error>> {
+    let srv = MockServer::start();
+    let proxy = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
+
+    let mock = Mock::new()
+        .expect_method(GET)
+        .expect_path("/LICENSE")
+        .return_status(200)
+        .return_body("this is a test")
+        .create_on(&srv);
+
+    let mock_two = Mock::new()
+        .expect_method(GET)
+        .expect_path("/LICENSE")
+        .return_status(200)
+        .return_body("this is a test")
+        .create_on(&proxy);
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--replay-proxy")
+        .arg(format!("http://{}", proxy.address().to_string()))
+        .arg("--replay-codes")
+        .arg("200")
+        .unwrap();
+
+    cmd.assert()
+        .success()
+        .stdout(
+            predicate::str::contains("/LICENSE")
+                .and(predicate::str::contains("200"))
+                .and(predicate::str::contains("14c")),
+        )
+        .stderr(predicate::str::contains("Replay Proxy Codes"));
+
+    assert_eq!(mock.times_called(), 1);
+    assert_eq!(mock_two.times_called(), 1);
+    teardown_tmp_directory(tmp_dir);
+    Ok(())
+}
+
 #[test]
 /// send a single valid request, filter the size of the response, expect one out of 2 urls
 fn scanner_single_request_scan_with_filtered_result() -> Result<(), Box<dyn std::error::Error>> {
@@ -412,55 +461,6 @@ fn scanner_single_request_scan_with_filtered_result() -> Result<(), Box<dyn std:
     Ok(())
 }
 
-#[test]
-/// send a single valid request, expect a 200 response that then gets routed to the replay
-/// proxy
-fn scanner_single_request_replayed_to_proxy() -> Result<(), Box<dyn std::error::Error>> {
-    let srv = MockServer::start();
-    let proxy = MockServer::start();
-    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
-
-    let mock = Mock::new()
-        .expect_method(GET)
-        .expect_path("/LICENSE")
-        .return_status(200)
-        .return_body("this is a test")
-        .create_on(&srv);
-
-    let mock_two = Mock::new()
-        .expect_method(GET)
-        .expect_path("/LICENSE")
-        .return_status(200)
-        .return_body("this is a test")
-        .create_on(&proxy);
-
-    let cmd = Command::cargo_bin("feroxbuster")
-        .unwrap()
-        .arg("--url")
-        .arg(srv.url("/"))
-        .arg("--wordlist")
-        .arg(file.as_os_str())
-        .arg("--replay-proxy")
-        .arg(format!("http://{}", proxy.address().to_string()))
-        .arg("--replay-codes")
-        .arg("200")
-        .unwrap();
-
-    cmd.assert()
-        .success()
-        .stdout(
-            predicate::str::contains("/LICENSE")
-                .and(predicate::str::contains("200"))
-                .and(predicate::str::contains("14")),
-        )
-        .stderr(predicate::str::contains("Replay Proxy Codes"));
-
-    assert_eq!(mock.times_called(), 1);
-    assert_eq!(mock_two.times_called(), 1);
-    teardown_tmp_directory(tmp_dir);
-    Ok(())
-}
-
 #[test]
 /// send a single valid request, get a response, and write the logging messages to disk
 fn scanner_single_request_scan_with_debug_logging() {