Merge pull request #222 from epi052/213-add-parallel-option

add --parallel option
updated readme with parallel option
2026-05-22 03:51:12 -03:00 · 2021-02-18 11:37:36 -06:00 · 2021-02-18 11:26:15 -06:00 · 2021-02-18 11:25:55 -06:00 · 2021-02-18 11:01:11 -06:00 · 2021-02-18 09:42:39 -06:00
51 changed files with 9344 additions and 501 deletions
--- a/.github/pull_request_template.md
+++ b/.github/pull_request_template.md
@@ -7,11 +7,11 @@ Long form explanations of most of the items below can be found in the [CONTRIBUT
 - [ ] Your PR description references the associated issue (i.e. fixes #123456)
 - [ ] Code is in its own branch
 - [ ] Branch name is related to the PR contents
- [ ] PR targets master
+- [ ] PR targets main

 ## Static analysis checks
 - [ ] All rust files are formatted using `cargo fmt`
- [ ] All `clippy` checks pass when running `cargo clippy --all-targets --all-features -- -D warnings -A clippy::deref_addrof`
+- [ ] All `clippy` checks pass when running `cargo clippy --all-targets --all-features -- -D warnings -A clippy::deref_addrof -A clippy::mutex-atomic`
 - [ ] All existing tests pass

 ## Documentation
--- a/.github/workflows/check.yml
+++ b/.github/workflows/check.yml
@@ -61,4 +61,4 @@ jobs:
      - uses: actions-rs/cargo@v1
        with:
          command: clippy
-          args: --all-targets --all-features -- -D warnings -A clippy::deref_addrof
+          args: --all-targets --all-features -- -D warnings -A clippy::deref_addrof -A clippy::mutex-atomic
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -633,7 +633,7 @@ dependencies = [

 [[package]]
 name = "feroxbuster"
-version = "2.0.2"
+version = "2.2.0"
 dependencies = [
 "anyhow",
 "assert_cmd",
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "feroxbuster"
-version = "2.0.2"
+version = "2.2.0"
 authors = ["Ben 'epi' Risher <epibar052@gmail.com>"]
 license = "MIT"
 edition = "2018"
@@ -22,16 +22,16 @@ lazy_static = "1.4"

 [dependencies]
 futures = { version = "0.3"}
-tokio = { version = "1.0", features = ["full"] }
+tokio = { version = "1.2.0", features = ["full"] }
 tokio-util = {version = "0.6.3", features = ["codec"]}
 log = "0.4"
-env_logger = "0.8"
+env_logger = "0.8.3"
 reqwest = { version = "0.11", features = ["socks"] }
 clap = "2.33"
 lazy_static = "1.4"
 toml = "0.5"
 serde = { version = "1.0", features = ["derive", "rc"] }
-serde_json = "1.0"
+serde_json = "1.0.62"
 uuid = { version = "0.8", features = ["v4"] }
 indicatif = "0.15"
 console = "0.14"
--- a/README.md
+++ b/README.md
@@ -8,7 +8,7 @@

 <p align="center">
  <a href="https://github.com/epi052/feroxbuster/actions?query=workflow%3A%22CI+Pipeline%22">
-    <img src="https://img.shields.io/github/workflow/status/epi052/feroxbuster/CI%20Pipeline/master?logo=github">
+    <img src="https://img.shields.io/github/workflow/status/epi052/feroxbuster/CI%20Pipeline/main?logo=github">
  </a>

  <a href="https://github.com/epi052/feroxbuster/releases">
@@ -104,6 +104,8 @@ Enumeration.
    - [Cancel a Recursive Scan Interactively (new in `v1.12.0`)](#cancel-a-recursive-scan-interactively-new-in-v1120)
    - [Limit Number of Requests per Second (Rate Limiting) (new in `v2.0.0`)](#limit-number-of-requests-per-second-rate-limiting-new-in-v200)
    - [Silence all Output or Be Kinda Quiet (new in `v2.0.0`)](#silence-all-output-or-be-kinda-quiet-new-in-v200)
+    - [Auto-tune or Auto-bail from Scans (new in `v2.1.0`)](#auto-tune-or-auto-bail-from-scans-new-in-v210)
+    - [Run Scans in Parallel (new in `v2.2.0`)](#run-scans-in-parallel-new-in-v220)
 - [Comparison w/ Similar Tools](#-comparison-w-similar-tools)
 - [Common Problems/Issues (FAQ)](#-common-problemsissues-faq)
    - [No file descriptors available](#no-file-descriptors-available)
@@ -369,7 +371,10 @@ A pre-made configuration file with examples of all available settings can be fou
 # status_codes = [200, 500]
 # filter_status = [301]
 # threads = 1
+# parallel = 2
 # timeout = 5
+# auto_tune = true
+# auto_bail = true
 # proxy = "http://127.0.0.1:8080"
 # replay_proxy = "http://127.0.0.1:8081"
 # replay_codes = [200, 302]
@@ -425,6 +430,8 @@ USAGE:

 FLAGS:
    -f, --add-slash        Append / to each request
+        --auto-bail        Automatically stop scanning when an excessive amount of errors are encountered
+        --auto-tune        Automatically lower scan rate when an excessive amount of errors are encountered
    -D, --dont-filter      Don't auto-filter wildcard responses
    -e, --extract-links    Extract links from response body (html, javascript, etc...); make new requests based on
                           findings (default: false)
@@ -458,6 +465,9 @@ OPTIONS:
    -W, --filter-words <WORDS>...                 Filter out messages of a particular word count (ex: -W 312 -W 91,82)
    -H, --headers <HEADER>...                     Specify HTTP headers (ex: -H Header:val 'stuff: things')
    -o, --output <FILE>                           Output file to write results to (use w/ --json for JSON entries)
+        --parallel <PARALLEL_SCANS>
+            Run parallel feroxbuster instances (one child process per url passed via stdin)
+
    -p, --proxy <PROXY>
            Proxy to use for requests (ex: http(s)://host:port, socks5(h)://host:port)

@@ -484,7 +494,6 @@ OPTIONS:
    -u, --url <URL>...                            The target URL(s) (required, unless --stdin used)
    -a, --user-agent <USER_AGENT>                 Sets the User-Agent (default: feroxbuster/VERSION)
    -w, --wordlist <FILE>                         Path to the wordlist
-
 ```

 ## 📊 Scan's Display Explained
@@ -871,6 +880,52 @@ Scanning: https://localhost.com/homepage
 Scanning: https://localhost.com/api
 ```

+### Auto-tune or Auto-bail from scans (new in `v2.1.0`)
+
+Version 2.1.0 introduces the `--auto-tune` and `--auto-bail` flags. You can think of these flags as Policies. Both actions (tuning and bailing) are triggered by the same criteria (below).  Policies are only enforced after at least 50 requests have been made (or # of threads, if that's > 50).
+
+Policy Enforcement Criteria:
+  - number of general errors (timeouts, etc) is higher than half the number of threads (or at least 25 if threads are lower) (per directory scanned)
+  - 90% of responses are `403|Forbidden` (per directory scanned)
+  - 30% of requests are `429|Too Many Requests` (per directory scanned)
+
+> both demo gifs below use --timeout to overload a single-threaded python web server and elicit timeouts
+
+#### --auto-tune
+
+The AutoTune policy enforces a rate limit on individual directory scans when one of the criteria above is met.  The rate limit self-adjusts every (`timeout / 2`) seconds. If the number of errors have increased during that time, the allowed rate of requests is lowered.  On the other hand, if the number of errors hasn't moved, the allowed rate of requests is increased.  If no additional errors are found after a certain number of checks, the rate limit will be removed completely. 
+
+![auto-tune](img/auto-tune-demo.gif)
+
+#### --auto-bail
+
+The AutoBail policy aborts individual directory scans when one of the criteria above is met.  They just stop getting scanned, no muss, no fuss. 
+
+![auto-bail](img/auto-bail-demo.gif)
+
+### Run Scans in Parallel (new in `v2.2.0`)
+
+Version 2.2.0 introduces the `--parallel` option.  If you're one of those people who use `feroxbuster` to scan 100s of hosts at a time, this is the option for you! `--parallel` spawns a child process per target passed in over stdin (recursive directories are still async within each child).
+
+The number of parallel scans is limited to whatever you pass to `--parallel`. When one child finishes its scan, the next child will be spawned.  
+
+Unfortunately, using `--parallel` limits terminal output such that only discovered URLs are shown. No amount of `-v`'s will help you here. I imagine this isn't too big of a deal, as folks that need `--parallel` probably aren't sitting there watching the output... 🙃
+
+Example Command:
+```
+cat large-target-list | ./feroxbuster --stdin --parallel 10 --extract-links --auto-bail
+```
+
+Resuling Process List (illustrative):
+```
+\_ target/debug/feroxbuster --stdin --parallel 10
+    \_ target/debug/feroxbuster --silent --extract-links --auto-bail -u https://target-one
+    \_ target/debug/feroxbuster --silent --extract-links --auto-bail -u https://target-two
+    \_ target/debug/feroxbuster --silent --extract-links --auto-bail -u https://target-three
+    \_ ...
+    \_ target/debug/feroxbuster --silent --extract-links --auto-bail -u https://target-ten
+```
+
 ## 🧐 Comparison w/ Similar Tools

 There are quite a few similar tools for forced browsing/content discovery. Burp Suite Pro, Dirb, Dirbuster, etc...
@@ -918,6 +973,9 @@ few of the use-cases in which feroxbuster may be a better fit:
 | cancel a recursive scan interactively (`v1.12.0`)                            | ✔ |   |   |
 | limit number of requests per second (`v2.0.0`)                               | ✔ | ✔ | ✔ |
 | hide progress bars or be silent (or some variation) (`v2.0.0`)               | ✔ | ✔ | ✔ |
+| automatically tune scans based on errors/403s/429s  (`v2.1.0`)               | ✔ |   |   |
+| automatically stop scans based on errors/403s/429s  (`v2.1.0`)               | ✔ |   | ✔ |
+| run scans in parallel (1 process per target) (`v2.2.0`)                      | ✔ |   |   |
 | **huge** number of other options                                             |   |   | ✔ |

 Of note, there's another written-in-rust content discovery tool, [rustbuster](https://github.com/phra/rustbuster). I
--- a/ferox-config.toml.example
+++ b/ferox-config.toml.example
@@ -16,10 +16,13 @@
 # replay_proxy = "http://127.0.0.1:8081"
 # replay_codes = [200, 302]
 # verbosity = 1
+# parallel = 8
 # scan_limit = 6
 # rate_limit = 250
 # quiet = true
 # silent = true
+# auto_tune = true
+# auto_bail = true
 # json = true
 # output = "/targets/ellingson_mineral_company/gibson.txt"
 # debug_log = "/var/log/find-the-derp.log"
--- a/img/auto-bail-demo.gif
+++ b/img/auto-bail-demo.gif
--- a/img/auto-tune-demo.gif
+++ b/img/auto-tune-demo.gif
--- a/shell_completions/_feroxbuster
+++ b/shell_completions/_feroxbuster
@@ -58,13 +58,16 @@ _feroxbuster() {
 '*--filter-similar-to=[Filter out pages that are similar to the given page (ex. --filter-similar-to http://site.xyz/soft404)]' \
 '-L+[Limit total number of concurrent scans (default: 0, i.e. no limit)]' \
 '--scan-limit=[Limit total number of concurrent scans (default: 0, i.e. no limit)]' \
-'--rate-limit=[Limit number of requests per second (per directory) (default: 0, i.e. no limit)]' \
+'--parallel=[Run parallel feroxbuster instances (one child process per url passed via stdin)]' \
+'(--auto-tune)--rate-limit=[Limit number of requests per second (per directory) (default: 0, i.e. no limit)]' \
 '--time-limit=[Limit total run time of all scans (ex: --time-limit 10m)]' \
 '(--silent)*-v[Increase verbosity level (use -vv or more for greater effect. \[CAUTION\] 4 -v'\''s is probably too much)]' \
 '(--silent)*--verbosity[Increase verbosity level (use -vv or more for greater effect. \[CAUTION\] 4 -v'\''s is probably too much)]' \
 '(-q --quiet)--silent[Only print URLs + turn off logging (good for piping a list of urls to other commands)]' \
 '-q[Hide progress bars and banner (good for tmux windows w/ notifications)]' \
 '--quiet[Hide progress bars and banner (good for tmux windows w/ notifications)]' \
+'(--auto-bail)--auto-tune[Automatically lower scan rate when an excessive amount of errors are encountered]' \
+'--auto-bail[Automatically stop scanning when an excessive amount of errors are encountered]' \
 '--json[Emit JSON logs to --output and --debug-log instead of normal text]' \
 '-D[Don'\''t auto-filter wildcard responses]' \
 '--dont-filter[Don'\''t auto-filter wildcard responses]' \
--- a/shell_completions/_feroxbuster.ps1
+++ b/shell_completions/_feroxbuster.ps1
@@ -63,6 +63,7 @@ Register-ArgumentCompleter -Native -CommandName 'feroxbuster' -ScriptBlock {
            [CompletionResult]::new('--filter-similar-to', 'filter-similar-to', [CompletionResultType]::ParameterName, 'Filter out pages that are similar to the given page (ex. --filter-similar-to http://site.xyz/soft404)')
            [CompletionResult]::new('-L', 'L', [CompletionResultType]::ParameterName, 'Limit total number of concurrent scans (default: 0, i.e. no limit)')
            [CompletionResult]::new('--scan-limit', 'scan-limit', [CompletionResultType]::ParameterName, 'Limit total number of concurrent scans (default: 0, i.e. no limit)')
+            [CompletionResult]::new('--parallel', 'parallel', [CompletionResultType]::ParameterName, 'Run parallel feroxbuster instances (one child process per url passed via stdin)')
            [CompletionResult]::new('--rate-limit', 'rate-limit', [CompletionResultType]::ParameterName, 'Limit number of requests per second (per directory) (default: 0, i.e. no limit)')
            [CompletionResult]::new('--time-limit', 'time-limit', [CompletionResultType]::ParameterName, 'Limit total run time of all scans (ex: --time-limit 10m)')
            [CompletionResult]::new('-v', 'v', [CompletionResultType]::ParameterName, 'Increase verbosity level (use -vv or more for greater effect. [CAUTION] 4 -v''s is probably too much)')
@@ -70,6 +71,8 @@ Register-ArgumentCompleter -Native -CommandName 'feroxbuster' -ScriptBlock {
            [CompletionResult]::new('--silent', 'silent', [CompletionResultType]::ParameterName, 'Only print URLs + turn off logging (good for piping a list of urls to other commands)')
            [CompletionResult]::new('-q', 'q', [CompletionResultType]::ParameterName, 'Hide progress bars and banner (good for tmux windows w/ notifications)')
            [CompletionResult]::new('--quiet', 'quiet', [CompletionResultType]::ParameterName, 'Hide progress bars and banner (good for tmux windows w/ notifications)')
+            [CompletionResult]::new('--auto-tune', 'auto-tune', [CompletionResultType]::ParameterName, 'Automatically lower scan rate when an excessive amount of errors are encountered')
+            [CompletionResult]::new('--auto-bail', 'auto-bail', [CompletionResultType]::ParameterName, 'Automatically stop scanning when an excessive amount of errors are encountered')
            [CompletionResult]::new('--json', 'json', [CompletionResultType]::ParameterName, 'Emit JSON logs to --output and --debug-log instead of normal text')
            [CompletionResult]::new('-D', 'D', [CompletionResultType]::ParameterName, 'Don''t auto-filter wildcard responses')
            [CompletionResult]::new('--dont-filter', 'dont-filter', [CompletionResultType]::ParameterName, 'Don''t auto-filter wildcard responses')
--- a/shell_completions/feroxbuster.bash
+++ b/shell_completions/feroxbuster.bash
@@ -20,7 +20,7 @@ _feroxbuster() {

    case "${cmd}" in
        feroxbuster)
-            opts=" -v -q -D -r -k -n -f -e -h -V -w -u -t -d -T -p -P -R -s -o -a -x -H -Q -S -X -W -N -C -L  --verbosity --silent --quiet --json --dont-filter --redirects --insecure --no-recursion --add-slash --stdin --extract-links --help --version --wordlist --url --threads --depth --timeout --proxy --replay-proxy --replay-codes --status-codes --output --resume-from --debug-log --user-agent --extensions --headers --query --filter-size --filter-regex --filter-words --filter-lines --filter-status --filter-similar-to --scan-limit --rate-limit --time-limit  "
+            opts=" -v -q -D -r -k -n -f -e -h -V -w -u -t -d -T -p -P -R -s -o -a -x -H -Q -S -X -W -N -C -L  --verbosity --silent --quiet --auto-tune --auto-bail --json --dont-filter --redirects --insecure --no-recursion --add-slash --stdin --extract-links --help --version --wordlist --url --threads --depth --timeout --proxy --replay-proxy --replay-codes --status-codes --output --resume-from --debug-log --user-agent --extensions --headers --query --filter-size --filter-regex --filter-words --filter-lines --filter-status --filter-similar-to --scan-limit --parallel --rate-limit --time-limit  "
            if [[ ${cur} == -* || ${COMP_CWORD} -eq 1 ]] ; then
                COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
                return 0
@@ -199,6 +199,10 @@ _feroxbuster() {
                    COMPREPLY=($(compgen -f "${cur}"))
                    return 0
                    ;;
+                --parallel)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
                --rate-limit)
                    COMPREPLY=($(compgen -f "${cur}"))
                    return 0
--- a/shell_completions/feroxbuster.fish
+++ b/shell_completions/feroxbuster.fish
@@ -21,11 +21,14 @@ complete -c feroxbuster -n "__fish_use_subcommand" -s N -l filter-lines -d 'Filt
 complete -c feroxbuster -n "__fish_use_subcommand" -s C -l filter-status -d 'Filter out status codes (deny list) (ex: -C 200 -C 401)'
 complete -c feroxbuster -n "__fish_use_subcommand" -l filter-similar-to -d 'Filter out pages that are similar to the given page (ex. --filter-similar-to http://site.xyz/soft404)'
 complete -c feroxbuster -n "__fish_use_subcommand" -s L -l scan-limit -d 'Limit total number of concurrent scans (default: 0, i.e. no limit)'
+complete -c feroxbuster -n "__fish_use_subcommand" -l parallel -d 'Run parallel feroxbuster instances (one child process per url passed via stdin)'
 complete -c feroxbuster -n "__fish_use_subcommand" -l rate-limit -d 'Limit number of requests per second (per directory) (default: 0, i.e. no limit)'
 complete -c feroxbuster -n "__fish_use_subcommand" -l time-limit -d 'Limit total run time of all scans (ex: --time-limit 10m)'
 complete -c feroxbuster -n "__fish_use_subcommand" -s v -l verbosity -d 'Increase verbosity level (use -vv or more for greater effect. [CAUTION] 4 -v\'s is probably too much)'
 complete -c feroxbuster -n "__fish_use_subcommand" -l silent -d 'Only print URLs + turn off logging (good for piping a list of urls to other commands)'
 complete -c feroxbuster -n "__fish_use_subcommand" -s q -l quiet -d 'Hide progress bars and banner (good for tmux windows w/ notifications)'
+complete -c feroxbuster -n "__fish_use_subcommand" -l auto-tune -d 'Automatically lower scan rate when an excessive amount of errors are encountered'
+complete -c feroxbuster -n "__fish_use_subcommand" -l auto-bail -d 'Automatically stop scanning when an excessive amount of errors are encountered'
 complete -c feroxbuster -n "__fish_use_subcommand" -l json -d 'Emit JSON logs to --output and --debug-log instead of normal text'
 complete -c feroxbuster -n "__fish_use_subcommand" -s D -l dont-filter -d 'Don\'t auto-filter wildcard responses'
 complete -c feroxbuster -n "__fish_use_subcommand" -s r -l redirects -d 'Follow redirects'
--- a/src/banner/container.rs
+++ b/src/banner/container.rs
@@ -1,8 +1,8 @@
 use super::entry::BannerEntry;
-use crate::event_handlers::Handles;
 use crate::{
    config::Configuration,
-    utils::{make_request, status_colorizer},
+    event_handlers::Handles,
+    utils::{logged_request, status_colorizer},
    VERSION,
 };
 use anyhow::{bail, Result};
@@ -125,6 +125,15 @@ pub struct Banner {
    /// represents Configuration.rate_limit
    rate_limit: BannerEntry,

+    /// represents Configuration.parallel
+    parallel: BannerEntry,
+
+    /// represents Configuration.auto_tune
+    auto_tune: BannerEntry,
+
+    /// represents Configuration.auto_bail
+    auto_bail: BannerEntry,
+
    /// current version of feroxbuster
    pub(super) version: String,

@@ -251,6 +260,8 @@ impl Banner {
        );

        let replay_proxy = BannerEntry::new("🎥", "Replay Proxy", &config.replay_proxy);
+        let auto_tune = BannerEntry::new("🎶", "Auto Tune", &config.auto_tune.to_string());
+        let auto_bail = BannerEntry::new("🪣", "Auto Bail", &config.auto_bail.to_string());
        let cfg = BannerEntry::new("💉", "Config File", &config.config);
        let proxy = BannerEntry::new("💎", "Proxy", &config.proxy);
        let threads = BannerEntry::new("🚀", "Threads", &config.threads.to_string());
@@ -273,6 +284,7 @@ impl Banner {
            BannerEntry::new("🤪", "Filter Wildcards", &(!config.dont_filter).to_string());
        let add_slash = BannerEntry::new("🪓", "Add Slash", &config.add_slash.to_string());
        let time_limit = BannerEntry::new("🕖", "Time Limit", &config.time_limit);
+        let parallel = BannerEntry::new("🛤", "Parallel Scans", &config.parallel.to_string());
        let rate_limit =
            BannerEntry::new("🚧", "Requests per Second", &config.rate_limit.to_string());

@@ -284,6 +296,8 @@ impl Banner {
            filter_status,
            timeout,
            user_agent,
+            auto_bail,
+            auto_tune,
            proxy,
            replay_codes,
            replay_proxy,
@@ -294,6 +308,7 @@ impl Banner {
            filter_line_count,
            filter_regex,
            extract_links,
+            parallel,
            json,
            queries,
            output,
@@ -354,15 +369,8 @@ by Ben "epi" Risher {}                 ver: {}"#,

        let api_url = Url::parse(url)?;

-        let response = make_request(
-            &handles.config.client,
-            &api_url,
-            handles.config.output_level,
-            handles.stats.tx.clone(),
-        )
-        .await?;
-
-        let body = response.text().await?;
+        let result = logged_request(&api_url, handles.clone()).await?;
+        let body = result.text().await?;

        let json_response: Value = serde_json::from_str(&body)?;

@@ -486,6 +494,13 @@ by Ben "epi" Risher {}                 ver: {}"#,
            writeln!(&mut writer, "{}", self.insecure)?;
        }

+        if config.auto_bail {
+            writeln!(&mut writer, "{}", self.auto_bail)?;
+        }
+        if config.auto_tune {
+            writeln!(&mut writer, "{}", self.auto_tune)?;
+        }
+
        if config.redirects {
            writeln!(&mut writer, "{}", self.redirects)?;
        }
@@ -508,6 +523,10 @@ by Ben "epi" Risher {}                 ver: {}"#,
            writeln!(&mut writer, "{}", self.scan_limit)?;
        }

+        if config.parallel > 0 {
+            writeln!(&mut writer, "{}", self.parallel)?;
+        }
+
        if config.rate_limit > 0 {
            writeln!(&mut writer, "{}", self.rate_limit)?;
        }
--- a/src/banner/tests.rs
+++ b/src/banner/tests.rs
@@ -1,6 +1,6 @@
 use super::container::UpdateStatus;
 use super::*;
-use crate::{config::Configuration, event_handlers::Handles};
+use crate::{config::Configuration, event_handlers::Handles, scan_manager::FeroxScans};
 use httpmock::Method::GET;
 use httpmock::MockServer;
 use std::{io::stderr, sync::Arc, time::Duration};
@@ -73,8 +73,9 @@ async fn banner_needs_update_returns_up_to_date() {
        when.method(GET).path("/latest");
        then.status(200).body("{\"tag_name\":\"v1.1.0\"}");
    });
+    let scans = Arc::new(FeroxScans::default());

-    let handles = Arc::new(Handles::for_testing(None, None).0);
+    let handles = Arc::new(Handles::for_testing(Some(scans), None).0);

    let mut banner = Banner::new(&[srv.url("")], &Configuration::new().unwrap());
    banner.version = String::from("1.1.0");
@@ -95,7 +96,9 @@ async fn banner_needs_update_returns_out_of_date() {
        then.status(200).body("{\"tag_name\":\"v1.1.0\"}");
    });

-    let handles = Arc::new(Handles::for_testing(None, None).0);
+    let scans = Arc::new(FeroxScans::default());
+
+    let handles = Arc::new(Handles::for_testing(Some(scans), None).0);

    let mut banner = Banner::new(&[srv.url("")], &Configuration::new().unwrap());
    banner.version = String::from("1.0.1");
--- a/src/config/container.rs
+++ b/src/config/container.rs
@@ -1,8 +1,9 @@
 use super::utils::{
    depth, report_and_exit, save_state, serialized_type, status_codes, threads, timeout,
-    user_agent, wordlist, OutputLevel,
+    user_agent, wordlist, OutputLevel, RequesterPolicy,
 };
 use crate::config::determine_output_level;
+use crate::config::utils::determine_requester_policy;
 use crate::{
    client, parser, scan_manager::resume_scan, traits::FeroxSerialize, utils::fmt_err,
    DEFAULT_CONFIG_NAME,
@@ -124,6 +125,18 @@ pub struct Configuration {
    #[serde(skip)]
    pub output_level: OutputLevel,

+    /// automatically bail at certain error thresholds
+    #[serde(default)]
+    pub auto_bail: bool,
+
+    /// automatically try to lower request rate in order to reduce errors
+    #[serde(default)]
+    pub auto_tune: bool,
+
+    /// more easily differentiate between the three requester policies
+    #[serde(skip)]
+    pub requester_policy: RequesterPolicy,
+
    /// Store log output as NDJSON
    #[serde(default)]
    pub json: bool,
@@ -185,6 +198,10 @@ pub struct Configuration {
    #[serde(default)]
    pub scan_limit: usize,

+    /// Number of parallel scans permitted; a limit of 0 means no limit is imposed
+    #[serde(default)]
+    pub parallel: usize,
+
    /// Number of requests per second permitted (per directory); a limit of 0 means no limit is imposed
    #[serde(default)]
    pub rate_limit: usize,
@@ -245,6 +262,7 @@ impl Default for Configuration {
        let replay_codes = status_codes.clone();
        let kind = serialized_type();
        let output_level = OutputLevel::Default;
+        let requester_policy = RequesterPolicy::Default;

        Configuration {
            kind,
@@ -254,7 +272,10 @@ impl Default for Configuration {
            replay_codes,
            status_codes,
            replay_client,
+            requester_policy,
            dont_filter: false,
+            auto_bail: false,
+            auto_tune: false,
            silent: false,
            quiet: false,
            output_level,
@@ -263,6 +284,7 @@ impl Default for Configuration {
            json: false,
            verbosity: 0,
            scan_limit: 0,
+            parallel: 0,
            rate_limit: 0,
            add_slash: false,
            insecure: false,
@@ -313,6 +335,8 @@ impl Configuration {
    /// - **debug_log**: `None`
    /// - **quiet**: `false`
    /// - **silent**: `false`
+    /// - **auto_tune**: `false`
+    /// - **auto_bail**: `false`
    /// - **save_state**: `true`
    /// - **user_agent**: `feroxbuster/VERSION`
    /// - **insecure**: `false` (don't be insecure, i.e. don't allow invalid certs)
@@ -331,7 +355,8 @@ impl Configuration {
    /// - **dont_filter**: `false` (auto filter wildcard responses)
    /// - **depth**: `4` (maximum recursion depth)
    /// - **scan_limit**: `0` (no limit on concurrent scans imposed)
-    /// - **rate_limit**: `0` (no limit on concurrent scans imposed)
+    /// - **parallel**: `0` (no limit on parallel scans imposed)
+    /// - **rate_limit**: `0` (no limit on requests per second imposed)
    /// - **time_limit**: `None` (no limit on length of scan imposed)
    /// - **replay_proxy**: `None` (no limit on concurrent scans imposed)
    /// - **replay_codes**: [`DEFAULT_RESPONSE_CODES`](constant.DEFAULT_RESPONSE_CODES.html)
@@ -467,6 +492,7 @@ impl Configuration {
        update_config_if_present!(&mut config.threads, args, "threads", usize);
        update_config_if_present!(&mut config.depth, args, "depth", usize);
        update_config_if_present!(&mut config.scan_limit, args, "scan_limit", usize);
+        update_config_if_present!(&mut config.parallel, args, "parallel", usize);
        update_config_if_present!(&mut config.rate_limit, args, "rate_limit", usize);
        update_config_if_present!(&mut config.wordlist, args, "wordlist", String);
        update_config_if_present!(&mut config.output, args, "output", String);
@@ -561,6 +587,16 @@ impl Configuration {
            config.output_level = OutputLevel::Quiet;
        }

+        if args.is_present("auto_tune") {
+            config.auto_tune = true;
+            config.requester_policy = RequesterPolicy::AutoTune;
+        }
+
+        if args.is_present("auto_bail") {
+            config.auto_bail = true;
+            config.requester_policy = RequesterPolicy::AutoBail;
+        }
+
        if args.is_present("dont_filter") {
            config.dont_filter = true;
        }
@@ -721,8 +757,11 @@ impl Configuration {
        update_if_not_default!(&mut conf.verbosity, new.verbosity, 0);
        update_if_not_default!(&mut conf.silent, new.silent, false);
        update_if_not_default!(&mut conf.quiet, new.quiet, false);
-        // use updated quiet/silent values to determin output level
+        update_if_not_default!(&mut conf.auto_bail, new.auto_bail, false);
+        update_if_not_default!(&mut conf.auto_tune, new.auto_tune, false);
+        // use updated quiet/silent values to determine output level; same for requester policy
        conf.output_level = determine_output_level(conf.quiet, conf.silent);
+        conf.requester_policy = determine_requester_policy(conf.auto_tune, conf.auto_bail);
        update_if_not_default!(&mut conf.output, new.output, "");
        update_if_not_default!(&mut conf.redirects, new.redirects, false);
        update_if_not_default!(&mut conf.insecure, new.insecure, false);
@@ -761,6 +800,7 @@ impl Configuration {
        );
        update_if_not_default!(&mut conf.dont_filter, new.dont_filter, false);
        update_if_not_default!(&mut conf.scan_limit, new.scan_limit, 0);
+        update_if_not_default!(&mut conf.parallel, new.parallel, 0);
        update_if_not_default!(&mut conf.rate_limit, new.rate_limit, 0);
        update_if_not_default!(&mut conf.replay_proxy, new.replay_proxy, "");
        update_if_not_default!(&mut conf.debug_log, new.debug_log, "");
--- a/src/config/mod.rs
+++ b/src/config/mod.rs
@@ -6,4 +6,4 @@ mod utils;
 mod tests;

 pub use self::container::Configuration;
-pub use self::utils::{determine_output_level, OutputLevel};
+pub use self::utils::{determine_output_level, OutputLevel, RequesterPolicy};
--- a/src/config/tests.rs
+++ b/src/config/tests.rs
@@ -16,8 +16,11 @@ fn setup_config_test() -> Configuration {
            replay_proxy = "http://127.0.0.1:8081"
            quiet = true
            silent = true
+            auto_tune = true
+            auto_bail = true
            verbosity = 1
            scan_limit = 6
+            parallel = 14
            rate_limit = 250
            time_limit = "10m"
            output = "/some/otherpath"
@@ -71,7 +74,11 @@ fn default_configuration() {
    assert_eq!(config.scan_limit, 0);
    assert_eq!(config.silent, false);
    assert_eq!(config.quiet, false);
+    assert_eq!(config.output_level, OutputLevel::Default);
    assert_eq!(config.dont_filter, false);
+    assert_eq!(config.auto_tune, false);
+    assert_eq!(config.auto_bail, false);
+    assert_eq!(config.requester_policy, RequesterPolicy::Default);
    assert_eq!(config.no_recursion, false);
    assert_eq!(config.json, false);
    assert_eq!(config.save_state, true);
@@ -140,6 +147,13 @@ fn config_reads_scan_limit() {
    assert_eq!(config.scan_limit, 6);
 }

+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_parallel() {
+    let config = setup_config_test();
+    assert_eq!(config.parallel, 14);
+}
+
 #[test]
 /// parse the test config and see that the value parsed is correct
 fn config_reads_rate_limit() {
@@ -189,6 +203,20 @@ fn config_reads_json() {
    assert_eq!(config.json, true);
 }

+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_auto_bail() {
+    let config = setup_config_test();
+    assert_eq!(config.auto_bail, true);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_auto_tune() {
+    let config = setup_config_test();
+    assert_eq!(config.auto_tune, true);
+}
+
 #[test]
 /// parse the test config and see that the value parsed is correct
 fn config_reads_verbosity() {
--- a/src/config/utils.rs
+++ b/src/config/utils.rs
@@ -102,6 +102,41 @@ pub fn determine_output_level(quiet: bool, silent: bool) -> OutputLevel {
    }
 }

+/// represents actions the Requester should take in certain situations
+#[derive(Debug, PartialEq, Copy, Clone)]
+pub enum RequesterPolicy {
+    /// automatically try to lower request rate in order to reduce errors
+    AutoTune,
+
+    /// automatically bail at certain error thresholds
+    AutoBail,
+
+    /// just let that junk run super natural
+    Default,
+}
+
+/// default implementation for RequesterPolicy
+impl Default for RequesterPolicy {
+    /// Default as default
+    fn default() -> Self {
+        Self::Default
+    }
+}
+
+/// given the current settings for quiet and silent, determine output_level (DRY helper)
+pub fn determine_requester_policy(auto_tune: bool, auto_bail: bool) -> RequesterPolicy {
+    if auto_tune && auto_bail {
+        // user COULD have both as true in config file, take the more aggressive of the two
+        RequesterPolicy::AutoBail
+    } else if auto_tune {
+        RequesterPolicy::AutoTune
+    } else if auto_bail {
+        RequesterPolicy::AutoBail
+    } else {
+        RequesterPolicy::Default
+    }
+}
+
 #[cfg(test)]
 mod tests {
    use super::*;
@@ -122,6 +157,22 @@ mod tests {
        assert_eq!(level, OutputLevel::Quiet);
    }

+    #[test]
+    /// test determine_requester_policy returns higher of the two levels if both given values are true
+    fn determine_requester_policy_returns_correct_results() {
+        let mut level = determine_requester_policy(true, true);
+        assert_eq!(level, RequesterPolicy::AutoBail);
+
+        level = determine_requester_policy(false, true);
+        assert_eq!(level, RequesterPolicy::AutoBail);
+
+        level = determine_requester_policy(false, false);
+        assert_eq!(level, RequesterPolicy::Default);
+
+        level = determine_requester_policy(true, false);
+        assert_eq!(level, RequesterPolicy::AutoTune);
+    }
+
    #[test]
    #[should_panic]
    /// report_and_exit should panic/exit when called
--- a/src/event_handlers/command.rs
+++ b/src/event_handlers/command.rs
@@ -25,11 +25,14 @@ pub enum Command {
    /// Create the progress bar (`BarType::Total`) that is updated from the stats thread
    CreateBar,

-    /// Update a `Stats` field that corresponds to the given `StatField` by the given `usize` value
-    UpdateUsizeField(StatField, usize),
+    /// Add to a `Stats` field that corresponds to the given `StatField` by the given `usize` value
+    AddToUsizeField(StatField, usize),
+
+    /// Subtract from a `Stats` field that corresponds to the given `StatField` by the given `usize` value
+    SubtractFromUsizeField(StatField, usize),

    /// Update a `Stats` field that corresponds to the given `StatField` by the given `f64` value
-    UpdateF64Field(StatField, f64),
+    AddToF64Field(StatField, f64),

    /// Save a `Stats` object to disk using `reporter::get_cached_file_handle`
    Save,
--- a/src/event_handlers/outputs.rs
+++ b/src/event_handlers/outputs.rs
@@ -1,4 +1,4 @@
-use super::Command::UpdateUsizeField;
+use super::Command::AddToUsizeField;
 use super::*;

 use anyhow::{Context, Result};
@@ -195,7 +195,7 @@ impl TermOutHandler {
                        // print to stdout
                        ferox_print(&resp.as_str(), &PROGRESS_PRINTER);

-                        send_command!(tx_stats, UpdateUsizeField(ResourcesDiscovered, 1));
+                        send_command!(tx_stats, AddToUsizeField(ResourcesDiscovered, 1));

                        if self.file_task.is_some() {
                            // -o used, need to send the report to be written out to disk
@@ -210,7 +210,7 @@ impl TermOutHandler {

                    if self.config.replay_client.is_some() && should_process_response {
                        // replay proxy specified/client created and this response's status code is one that
-                        // should be replayed
+                        // should be replayed; not using logged_request due to replay proxy client
                        make_request(
                            self.config.replay_client.as_ref().unwrap(),
                            &resp.url(),
--- a/src/event_handlers/scans.rs
+++ b/src/event_handlers/scans.rs
@@ -10,11 +10,12 @@ use crate::{
    scan_manager::{FeroxScan, FeroxScans, ScanOrder},
    scanner::FeroxScanner,
    statistics::StatField::TotalScans,
-    CommandReceiver, CommandSender, FeroxChannel, Joiner,
+    CommandReceiver, CommandSender, FeroxChannel, Joiner, SLEEP_DURATION,
 };

-use super::command::Command::UpdateUsizeField;
+use super::command::Command::AddToUsizeField;
 use super::*;
+use tokio::time::Duration;

 #[derive(Debug)]
 /// Container for recursion transmitter and FeroxScans object
@@ -153,9 +154,7 @@ impl ScanHandler {

                    tokio::spawn(async move {
                        while ferox_scans.has_active_scans() {
-                            for scan in ferox_scans.get_active_scans() {
-                                scan.join().await;
-                            }
+                            tokio::time::sleep(Duration::from_millis(SLEEP_DURATION + 250)).await;
                        }
                        limiter_clone.close();
                        sender.send(true).expect("oneshot channel failed");
@@ -231,7 +230,7 @@ impl ScanHandler {
                }
            });

-            self.handles.stats.send(UpdateUsizeField(TotalScans, 1))?;
+            self.handles.stats.send(AddToUsizeField(TotalScans, 1))?;

            scan.set_task(task).await?;

--- a/src/event_handlers/statistics.rs
+++ b/src/event_handlers/statistics.rs
@@ -89,6 +89,7 @@ impl StatsHandler {
                }
                Command::AddStatus(status) => {
                    self.stats.add_status_code(status);
+
                    self.increment_bar();
                }
                Command::AddRequest => {
@@ -99,14 +100,21 @@ impl StatsHandler {
                    self.stats
                        .save(start.elapsed().as_secs_f64(), output_file)?;
                }
-                Command::UpdateUsizeField(field, value) => {
+                Command::AddToUsizeField(field, value) => {
                    self.stats.update_usize_field(field, value);

                    if matches!(field, StatField::TotalScans) {
                        self.bar.set_length(self.stats.total_expected() as u64);
                    }
                }
-                Command::UpdateF64Field(field, value) => self.stats.update_f64_field(field, value),
+                Command::SubtractFromUsizeField(field, value) => {
+                    self.stats.subtract_from_usize_field(field, value);
+
+                    if matches!(field, StatField::TotalExpected) {
+                        self.bar.set_length(self.stats.total_expected() as u64);
+                    }
+                }
+                Command::AddToF64Field(field, value) => self.stats.update_f64_field(field, value),
                Command::CreateBar => {
                    self.bar = add_bar("", self.stats.total_expected() as u64, BarType::Total);
                }
--- a/src/extractor/container.rs
+++ b/src/extractor/container.rs
@@ -3,7 +3,7 @@ use crate::{
    client,
    event_handlers::{
        Command,
-        Command::{AddError, UpdateUsizeField},
+        Command::{AddError, AddToUsizeField},
        Handles,
    },
    scan_manager::ScanOrder,
@@ -12,7 +12,7 @@ use crate::{
        StatField::{LinksExtracted, TotalExpected},
    },
    url::FeroxUrl,
-    utils::make_request,
+    utils::{logged_request, make_request},
 };
 use anyhow::{bail, Context, Result};
 use reqwest::{StatusCode, Url};
@@ -303,13 +303,7 @@ impl<'a> Extractor<'a> {
        }

        // make the request and store the response
-        let new_response = make_request(
-            &self.handles.config.client,
-            &new_url,
-            self.handles.config.output_level,
-            self.handles.stats.tx.clone(),
-        )
-        .await?;
+        let new_response = logged_request(&new_url, self.handles.clone()).await?;

        let new_ferox_response =
            FeroxResponse::from(new_response, true, self.handles.config.output_level).await;
@@ -384,6 +378,7 @@ impl<'a> Extractor<'a> {
        let mut url = Url::parse(&self.url)?;
        url.set_path("/robots.txt"); // overwrite existing path with /robots.txt

+        // purposefully not using logged_request here due to using the special client
        let response = make_request(
            &client,
            &url,
@@ -391,6 +386,7 @@ impl<'a> Extractor<'a> {
            self.handles.stats.tx.clone(),
        )
        .await?;
+
        let ferox_response =
            FeroxResponse::from(response, true, self.handles.config.output_level).await;

@@ -404,10 +400,10 @@ impl<'a> Extractor<'a> {

        self.handles
            .stats
-            .send(UpdateUsizeField(LinksExtracted, num_links))?;
+            .send(AddToUsizeField(LinksExtracted, num_links))?;
        self.handles
            .stats
-            .send(UpdateUsizeField(TotalExpected, num_links * multiplier))?;
+            .send(AddToUsizeField(TotalExpected, num_links * multiplier))?;

        Ok(())
    }
--- a/src/filters/container.rs
+++ b/src/filters/container.rs
@@ -4,7 +4,7 @@ use anyhow::Result;

 use crate::response::FeroxResponse;
 use crate::{
-    event_handlers::Command::UpdateUsizeField, statistics::StatField::WildcardsFiltered,
+    event_handlers::Command::AddToUsizeField, statistics::StatField::WildcardsFiltered,
    CommandSender,
 };

@@ -44,7 +44,7 @@ impl FeroxFilters {
                if filter.should_filter_response(&response) {
                    if filter.as_any().downcast_ref::<WildcardFilter>().is_some() {
                        tx_stats
-                            .send(UpdateUsizeField(WildcardsFiltered, 1))
+                            .send(AddToUsizeField(WildcardsFiltered, 1))
                            .unwrap_or_default();
                    }
                    return true;
--- a/src/filters/init.rs
+++ b/src/filters/init.rs
@@ -5,7 +5,7 @@ use crate::{
    event_handlers::Handles,
    response::FeroxResponse,
    skip_fail,
-    utils::{fmt_err, make_request},
+    utils::{fmt_err, logged_request},
    Command::AddFilter,
    SIMILARITY_THRESHOLD,
 };
@@ -72,15 +72,7 @@ pub async fn initialize(handles: Arc<Handles>) -> Result<()> {
        let url = skip_fail!(Url::parse(&similarity_filter));

        // attempt to request the given url
-        let resp = skip_fail!(
-            make_request(
-                &handles.config.client,
-                &url,
-                handles.config.output_level,
-                handles.stats.tx.clone()
-            )
-            .await
-        );
+        let resp = skip_fail!(logged_request(&url, handles.clone()).await);

        // if successful, create a filter based on the response's body
        let fr = FeroxResponse::from(resp, true, handles.config.output_level).await;
--- a/src/heuristics.rs
+++ b/src/heuristics.rs
@@ -12,7 +12,7 @@ use crate::{
    response::FeroxResponse,
    skip_fail,
    url::FeroxUrl,
-    utils::{ferox_print, fmt_err, make_request, status_colorizer},
+    utils::{ferox_print, fmt_err, logged_request, status_colorizer},
 };

 /// length of a standard UUID, used when determining wildcard responses
@@ -158,13 +158,7 @@ impl HeuristicTests {
        let unique_str = self.unique_string(length);
        let nonexistent_url = target.format(&unique_str, None)?;

-        let response = make_request(
-            &self.handles.config.client,
-            &nonexistent_url.to_owned(),
-            self.handles.config.output_level,
-            self.handles.stats.tx.clone(),
-        )
-        .await?;
+        let response = logged_request(&nonexistent_url.to_owned(), self.handles.clone()).await?;

        if self
            .handles
@@ -215,13 +209,8 @@ impl HeuristicTests {
        for target_url in target_urls {
            let url = FeroxUrl::from_string(&target_url, self.handles.clone());
            let request = skip_fail!(url.format("", None));
-            let result = make_request(
-                &self.handles.config.client,
-                &request,
-                self.handles.config.output_level,
-                self.handles.stats.tx.clone(),
-            )
-            .await;
+
+            let result = logged_request(&request, self.handles.clone()).await;

            match result {
                Ok(_) => {
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -57,7 +57,10 @@ pub const DEFAULT_WORDLIST: &str =
    "/usr/share/seclists/Discovery/Web-Content/raft-medium-directories.txt";

 /// Number of milliseconds to wait between polls of `PAUSE_SCAN` when user pauses a scan
-pub(crate) static SLEEP_DURATION: u64 = 500;
+pub(crate) const SLEEP_DURATION: u64 = 500;
+
+/// The percentage of requests as errors it takes to be deemed too high
+pub const HIGH_ERROR_RATIO: f64 = 0.90;

 /// Default list of status codes to report
 ///
--- a/src/main.rs
+++ b/src/main.rs
@@ -1,13 +1,19 @@
 use std::{
    collections::HashSet,
+    env::args,
    fs::File,
    io::{stderr, BufRead, BufReader},
+    ops::Index,
+    process::Command,
    sync::{atomic::Ordering, Arc},
 };

 use anyhow::{bail, Context, Result};
 use futures::StreamExt;
-use tokio::{io, sync::oneshot};
+use tokio::{
+    io,
+    sync::{oneshot, Semaphore},
+};
 use tokio_util::codec::{FramedRead, LinesCodec};

 use feroxbuster::{
@@ -26,6 +32,13 @@ use feroxbuster::{
 };
 #[cfg(not(target_os = "windows"))]
 use feroxbuster::{utils::set_open_file_limit, DEFAULT_OPEN_FILE_LIMIT};
+use lazy_static::lazy_static;
+use regex::Regex;
+
+lazy_static! {
+    /// Limits the number of parallel scans active at any given time when using --parallel
+    static ref PARALLEL_LIMITER: Semaphore = Semaphore::new(0);
+}

 /// Create a HashSet of Strings from the given wordlist then stores it inside an Arc
 fn get_unique_words_from_wordlist(path: &str) -> Result<Arc<HashSet<String>>> {
@@ -226,6 +239,72 @@ async fn wrapped_main(config: Arc<Configuration>) -> Result<()> {
        }
    };

+    // --parallel branch
+    if config.parallel > 0 {
+        log::trace!("enter: parallel branch");
+
+        PARALLEL_LIMITER.add_permits(config.parallel);
+
+        let invocation = args();
+
+        let para_regex =
+            Regex::new("--stdin|-q|--quiet|--silent|--verbosity|-v|-vv|-vvv|-vvvv").unwrap();
+
+        // remove stdin since only the original process will process targets
+        // remove quiet and silent so we can force silent later to normalize output
+        let mut original = invocation
+            .filter(|s| !para_regex.is_match(s))
+            .collect::<Vec<String>>();
+
+        original.push("--silent".to_string()); // only output modifier allowed
+
+        // we need remove --parallel from command line so we don't hit this branch over and over
+        // but we must remove --parallel N manually; the filter above never sees --parallel and the
+        // value passed to it at the same time, so can't filter them out in one pass
+
+        // unwrap is fine, as it has to be in the args for us to be in this code branch
+        let parallel_index = original.iter().position(|s| *s == "--parallel").unwrap();
+
+        // remove --parallel
+        original.remove(parallel_index);
+
+        // remove N passed to --parallel (it's the same index again since everything shifts
+        // from removing --parallel)
+        original.remove(parallel_index);
+
+        // unvalidated targets fresh from stdin, just spawn children and let them do all checks
+        for target in targets {
+            // add the current target to the provided command
+            let mut cloned = original.clone();
+            cloned.push("-u".to_string());
+            cloned.push(target);
+
+            let bin = cloned.index(0).to_owned(); // user's path to feroxbuster
+            let args = cloned.index(1..).to_vec(); // and args
+
+            let permit = PARALLEL_LIMITER.acquire().await?;
+
+            log::debug!("parallel exec: {} {}", bin, args.join(" "));
+
+            tokio::task::spawn_blocking(move || {
+                let result = Command::new(bin)
+                    .args(&args)
+                    .spawn()
+                    .expect("failed to spawn a child process")
+                    .wait()
+                    .expect("child process errored during execution");
+
+                drop(permit);
+                result
+            });
+        }
+
+        clean_up(handles, tasks).await?;
+
+        log::trace!("exit: parallel branch && wrapped main");
+        return Ok(());
+    }
+
    if matches!(config.output_level, OutputLevel::Default) {
        // only print banner if output level is default (no banner on --quiet|--silent)
        let std_stderr = stderr(); // std::io::stderr
@@ -246,7 +325,7 @@ async fn wrapped_main(config: Arc<Configuration>) -> Result<()> {

        // The TermOutHandler spawns a FileOutHandler, so errors in the FileOutHandler never bubble
        // up due to the TermOutHandler never awaiting the result of FileOutHandler::start (that's
-        // done later here in main). Ping checks that the tx/rx connection to the file handler works
+        // done later here in main). sync checks that the tx/rx connection to the file handler works
        if send_to_file && handles.output.sync(send_to_file).await.is_err() {
            // output file specified and file handler could not initialize
            clean_up(handles, tasks).await?;
--- a/src/message.rs
+++ b/src/message.rs
@@ -1,9 +1,9 @@
 use anyhow::Context;
 use console::{style, Color};
+use serde::{Deserialize, Serialize};

 use crate::traits::FeroxSerialize;
 use crate::utils::fmt_err;
-use serde::{Deserialize, Serialize};

 #[derive(Serialize, Deserialize, Default)]
 /// Representation of a log entry, can be represented as a human readable string or JSON
@@ -118,4 +118,31 @@ mod tests {
        assert_eq!(json.level, message.level);
        assert_eq!(json.kind, message.kind);
    }
+
+    #[test]
+    /// test defaults for coverage
+    fn message_defaults() {
+        let msg = FeroxMessage::default();
+        assert_eq!(msg.level, String::new());
+        assert_eq!(msg.kind, String::new());
+        assert_eq!(msg.message, String::new());
+        assert_eq!(msg.module, String::new());
+        assert!(msg.time_offset < 0.1);
+    }
+
+    #[test]
+    /// ensure WILDCARD messages serialize to WLD and anything not known to UNK
+    fn message_as_str_edges() {
+        let mut msg = FeroxMessage {
+            message: "message".to_string(),
+            module: "utils".to_string(),
+            time_offset: 1.0,
+            level: "WILDCARD".to_string(),
+            kind: "log".to_string(),
+        };
+        assert!(console::strip_ansi_codes(&msg.as_str()).starts_with("WLD"));
+
+        msg.level = "UNKNOWN".to_string();
+        assert!(console::strip_ansi_codes(&msg.as_str()).starts_with("UNK"));
+    }
 }
--- a/src/parser.rs
+++ b/src/parser.rs
@@ -130,6 +130,19 @@ pub fn initialize() -> App<'static, 'static> {
                .takes_value(false)
                .help("Hide progress bars and banner (good for tmux windows w/ notifications)")
        )
+        .arg(
+            Arg::with_name("auto_tune")
+                .long("auto-tune")
+                .takes_value(false)
+                .conflicts_with("auto_bail")
+                .help("Automatically lower scan rate when an excessive amount of errors are encountered")
+        )
+        .arg(
+            Arg::with_name("auto_bail")
+                .long("auto-bail")
+                .takes_value(false)
+                .help("Automatically stop scanning when an excessive amount of errors are encountered")
+        )
        .arg(
            Arg::with_name("json")
                .long("json")
@@ -335,11 +348,20 @@ pub fn initialize() -> App<'static, 'static> {
                .takes_value(true)
                .help("Limit total number of concurrent scans (default: 0, i.e. no limit)")
        )
+        .arg(
+            Arg::with_name("parallel")
+                .long("parallel")
+                .value_name("PARALLEL_SCANS")
+                .takes_value(true)
+                .requires("stdin")
+                .help("Run parallel feroxbuster instances (one child process per url passed via stdin)")
+        )
        .arg(
            Arg::with_name("rate_limit")
                .long("rate-limit")
                .value_name("RATE_LIMIT")
                .takes_value(true)
+                .conflicts_with("auto_tune")
                .help("Limit number of requests per second (per directory) (default: 0, i.e. no limit)")
        )
        .arg(
--- a/src/scan_manager/scan.rs
+++ b/src/scan_manager/scan.rs
@@ -2,6 +2,7 @@ use super::*;
 use crate::{
    config::OutputLevel,
    progress::{add_bar, BarType},
+    scanner::PolicyTrigger,
 };
 use anyhow::Result;
 use console::style;
@@ -12,8 +13,10 @@ use std::{
    collections::HashMap,
    fmt,
    sync::{Arc, Mutex},
+    time::Instant,
 };

+use std::sync::atomic::{AtomicUsize, Ordering};
 use tokio::{sync, task::JoinHandle};
 use uuid::Uuid;

@@ -49,6 +52,18 @@ pub struct FeroxScan {

    /// whether or not the user passed --silent|--quiet on the command line
    pub(super) output_level: OutputLevel,
+
+    /// tracker for overall number of 403s seen by the FeroxScan instance
+    pub(super) status_403s: AtomicUsize,
+
+    /// tracker for overall number of 429s seen by the FeroxScan instance
+    pub(super) status_429s: AtomicUsize,
+
+    /// tracker for total number of errors encountered by the FeroxScan instance
+    pub(super) errors: AtomicUsize,
+
+    /// tracker for the time at which this scan was started
+    pub(super) start_time: Instant,
 }

 /// Default implementation for FeroxScan
@@ -67,6 +82,10 @@ impl Default for FeroxScan {
            progress_bar: Mutex::new(None),
            scan_type: ScanType::File,
            output_level: Default::default(),
+            errors: Default::default(),
+            status_429s: Default::default(),
+            status_403s: Default::default(),
+            start_time: Instant::now(),
        }
    }
 }
@@ -75,16 +94,22 @@ impl Default for FeroxScan {
 impl FeroxScan {
    /// Stop a currently running scan
    pub async fn abort(&self) -> Result<()> {
-        let mut guard = self.task.lock().await;
+        log::trace!("enter: abort");

-        if guard.is_some() {
-            if let Some(task) = std::mem::replace(&mut *guard, None) {
-                task.abort();
-                self.set_status(ScanStatus::Cancelled)?;
-                self.stop_progress_bar();
+        match self.task.try_lock() {
+            Ok(mut guard) => {
+                if let Some(task) = std::mem::replace(&mut *guard, None) {
+                    log::trace!("aborting {:?}", self);
+                    task.abort();
+                    self.set_status(ScanStatus::Cancelled)?;
+                    self.stop_progress_bar();
+                }
+            }
+            Err(e) => {
+                log::warn!("Could not acquire lock to abort scan (we're already waiting for its results): {:?} {}", self, e);
            }
        }
-
+        log::trace!("exit: abort");
        Ok(())
    }

@@ -134,6 +159,7 @@ impl FeroxScan {
                    pb.reset_elapsed();

                    let _ = std::mem::replace(&mut *guard, Some(pb.clone()));
+
                    pb
                }
            }
@@ -217,6 +243,61 @@ impl FeroxScan {

        log::trace!("exit join({:?})", self);
    }
+    /// increment the value in question by 1
+    pub(crate) fn add_403(&self) {
+        self.status_403s.fetch_add(1, Ordering::Relaxed);
+    }
+
+    /// increment the value in question by 1
+    pub(crate) fn add_429(&self) {
+        self.status_429s.fetch_add(1, Ordering::Relaxed);
+    }
+
+    /// increment the value in question by 1
+    pub(crate) fn add_error(&self) {
+        self.errors.fetch_add(1, Ordering::Relaxed);
+    }
+
+    /// simple wrapper to call the appropriate getter based on the given PolicyTrigger
+    pub fn num_errors(&self, trigger: PolicyTrigger) -> usize {
+        match trigger {
+            PolicyTrigger::Status403 => self.status_403s(),
+            PolicyTrigger::Status429 => self.status_429s(),
+            PolicyTrigger::Errors => self.errors(),
+        }
+    }
+
+    /// return the number of errors seen by this scan
+    fn errors(&self) -> usize {
+        self.errors.load(Ordering::Relaxed)
+    }
+
+    /// return the number of 403s seen by this scan
+    fn status_403s(&self) -> usize {
+        self.status_403s.load(Ordering::Relaxed)
+    }
+
+    /// return the number of 429s seen by this scan
+    fn status_429s(&self) -> usize {
+        self.status_429s.load(Ordering::Relaxed)
+    }
+
+    /// return the number of requests per second performed by this scan's scanner
+    pub fn requests_per_second(&self) -> u64 {
+        if !self.is_active() {
+            return 0;
+        }
+
+        let reqs = self.requests();
+        let seconds = self.start_time.elapsed().as_secs();
+
+        reqs.checked_div(seconds).unwrap_or(0)
+    }
+
+    /// return the number of requests performed by this scan's scanner
+    pub fn requests(&self) -> u64 {
+        self.progress_bar().position()
+    }
 }

 /// Display implementation
@@ -360,3 +441,68 @@ impl Default for ScanStatus {
        Self::NotStarted
    }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::thread::sleep;
+    use tokio::time::Duration;
+
+    #[test]
+    /// ensure that num_errors returns the correct values for the given PolicyTrigger
+    ///
+    /// covers tests for add_[403,429,error] and the related getters in addition to num_errors
+    fn num_errors_returns_correct_values() {
+        let scan = FeroxScan::new(
+            "http://localhost",
+            ScanType::Directory,
+            ScanOrder::Latest,
+            1000,
+            OutputLevel::Default,
+            None,
+        );
+
+        scan.add_error();
+        scan.add_403();
+        scan.add_403();
+        scan.add_429();
+        scan.add_429();
+        scan.add_429();
+
+        assert_eq!(scan.num_errors(PolicyTrigger::Errors), 1);
+        assert_eq!(scan.num_errors(PolicyTrigger::Status403), 2);
+        assert_eq!(scan.num_errors(PolicyTrigger::Status429), 3);
+    }
+
+    #[test]
+    /// ensure that requests_per_second returns the correct values
+    fn requests_per_second_returns_correct_values() {
+        let scan = FeroxScan {
+            id: "".to_string(),
+            url: "".to_string(),
+            scan_type: ScanType::Directory,
+            scan_order: ScanOrder::Initial,
+            num_requests: 0,
+            status: Mutex::new(ScanStatus::Running),
+            task: Default::default(),
+            progress_bar: Mutex::new(None),
+            output_level: Default::default(),
+            status_403s: Default::default(),
+            status_429s: Default::default(),
+            errors: Default::default(),
+            start_time: Instant::now(),
+        };
+
+        let pb = scan.progress_bar();
+        pb.set_position(100);
+
+        sleep(Duration::new(1, 0));
+
+        let req_sec = scan.requests_per_second();
+
+        assert_eq!(req_sec, 100);
+
+        scan.finish().unwrap();
+        assert_eq!(scan.requests_per_second(), 0);
+    }
+}
--- a/src/scan_manager/scan_container.rs
+++ b/src/scan_manager/scan_container.rs
@@ -9,6 +9,7 @@ use crate::{
    SLEEP_DURATION,
 };
 use anyhow::Result;
+use reqwest::StatusCode;
 use serde::{ser::SerializeSeq, Serialize, Serializer};
 use std::{
    convert::TryInto,
@@ -161,6 +162,63 @@ impl FeroxScans {
        None
    }

+    pub(super) fn get_base_scan_by_url(&self, url: &str) -> Option<Arc<FeroxScan>> {
+        log::trace!("enter: get_sub_paths_from_path({})", url);
+
+        // rmatch_indices returns tuples in index, match form, i.e. (10, "/")
+        // with the furthest-right match in the first position in the vector
+        let matches: Vec<_> = url.rmatch_indices('/').collect();
+
+        // iterate from the furthest right matching index and check the given url from the
+        // start to the furthest-right '/' character. compare that slice to the urls associated
+        // with directory scans and return the first match, since it should be the 'deepest'
+        // match.
+        // Example:
+        //   url: http://shmocalhost/src/release/examples/stuff.php
+        //   scans:
+        //      http://shmocalhost/src/statistics
+        //      http://shmocalhost/src/banner
+        //      http://shmocalhost/src/release
+        //      http://shmocalhost/src/release/examples
+        //
+        //  returns: http://shmocalhost/src/release/examples
+        if let Ok(guard) = self.scans.read() {
+            for (idx, _) in &matches {
+                for scan in guard.iter() {
+                    let slice = url.index(0..*idx);
+                    if slice == scan.url || format!("{}/", slice).as_str() == scan.url {
+                        log::trace!("enter: get_sub_paths_from_path -> {}", scan);
+                        return Some(scan.clone());
+                    }
+                }
+            }
+        }
+
+        log::trace!("enter: get_sub_paths_from_path -> None");
+        None
+    }
+    /// add one to either 403 or 429 tracker in the scan related to the given url
+    pub fn increment_status_code(&self, url: &str, code: StatusCode) {
+        if let Some(scan) = self.get_base_scan_by_url(url) {
+            match code {
+                StatusCode::TOO_MANY_REQUESTS => {
+                    scan.add_429();
+                }
+                StatusCode::FORBIDDEN => {
+                    scan.add_403();
+                }
+                _ => {}
+            }
+        }
+    }
+
+    /// add one to either 403 or 429 tracker in the scan related to the given url
+    pub fn increment_error(&self, url: &str) {
+        if let Some(scan) = self.get_base_scan_by_url(url) {
+            scan.add_error();
+        }
+    }
+
    /// Print all FeroxScans of type Directory
    ///
    /// Example:
@@ -194,9 +252,11 @@ impl FeroxScans {
    }

    /// Given a list of indexes, cancel their associated FeroxScans
-    async fn cancel_scans(&self, indexes: Vec<usize>) {
+    async fn cancel_scans(&self, indexes: Vec<usize>) -> usize {
        let menu_pause_duration = Duration::from_millis(SLEEP_DURATION);

+        let mut num_cancelled = 0_usize;
+
        for num in indexes {
            let selected = match self.scans.read() {
                Ok(u_scans) => {
@@ -217,32 +277,42 @@ impl FeroxScans {

            if input == 'y' || input == '\n' {
                self.menu.println(&format!("Stopping {}...", selected.url));
+
                selected
                    .abort()
                    .await
                    .unwrap_or_else(|e| log::warn!("Could not cancel task: {}", e));
+
+                let pb = selected.progress_bar();
+                num_cancelled += pb.length() as usize - pb.position() as usize
            } else {
                self.menu.println("Ok, doing nothing...");
            }

            sleep(menu_pause_duration);
        }
+
+        num_cancelled
    }

    /// CLI menu that allows for interactive cancellation of recursed-into directories
-    async fn interactive_menu(&self) {
+    async fn interactive_menu(&self) -> usize {
        self.menu.hide_progress_bars();
        self.menu.clear_screen();
        self.menu.print_header();
        self.display_scans().await;
        self.menu.print_footer();

+        let mut num_cancelled = 0_usize;
+
        if let Some(input) = self.menu.get_scans_from_user() {
-            self.cancel_scans(input).await
+            num_cancelled += self.cancel_scans(input).await;
        };

        self.menu.clear_screen();
        self.menu.show_progress_bars();
+
+        num_cancelled
    }

    /// prints all known responses that the scanner has already seen
@@ -290,18 +360,19 @@ impl FeroxScans {
    ///
    /// When the value stored in `PAUSE_SCAN` becomes `false`, the function returns, exiting the busy
    /// loop
-    pub async fn pause(&self, get_user_input: bool) {
+    pub async fn pause(&self, get_user_input: bool) -> usize {
        // function uses tokio::time, not std

        // local testing showed a pretty slow increase (less than linear) in CPU usage as # of
        // concurrent scans rose when SLEEP_DURATION was set to 500, using that as the default for now
        let mut interval = time::interval(time::Duration::from_millis(SLEEP_DURATION));
+        let mut num_cancelled = 0_usize;

        if INTERACTIVE_BARRIER.load(Ordering::Relaxed) == 0 {
            INTERACTIVE_BARRIER.fetch_add(1, Ordering::Relaxed);

            if get_user_input {
-                self.interactive_menu().await;
+                num_cancelled += self.interactive_menu().await;
                PAUSE_SCAN.store(false, Ordering::Relaxed);
                self.print_known_responses();
            }
@@ -318,8 +389,8 @@ impl FeroxScans {
                    INTERACTIVE_BARRIER.fetch_sub(1, Ordering::Relaxed);
                }

-                log::trace!("exit: pause_scan");
-                return;
+                log::trace!("exit: pause_scan -> {}", num_cancelled);
+                return num_cancelled;
            }
        }
    }
--- a/src/scan_manager/tests.rs
+++ b/src/scan_manager/tests.rs
@@ -12,6 +12,7 @@ use indicatif::ProgressBar;
 use predicates::prelude::*;
 use std::sync::{atomic::Ordering, Arc};
 use std::thread::sleep;
+use std::time::Instant;
 use tokio::time::{self, Duration};

 #[test]
@@ -382,7 +383,7 @@ fn feroxstates_feroxserialize_implementation() {

    let json_state = ferox_state.as_json().unwrap();
    let expected = format!(
-        r#"{{"scans":[{{"id":"{}","url":"https://spiritanimal.com","scan_type":"Directory","status":"NotStarted","num_requests":0}}],"config":{{"type":"configuration","wordlist":"/usr/share/seclists/Discovery/Web-Content/raft-medium-directories.txt","config":"","proxy":"","replay_proxy":"","target_url":"","status_codes":[200,204,301,302,307,308,401,403,405],"replay_codes":[200,204,301,302,307,308,401,403,405],"filter_status":[],"threads":50,"timeout":7,"verbosity":0,"silent":false,"quiet":false,"json":false,"output":"","debug_log":"","user_agent":"feroxbuster/{}","redirects":false,"insecure":false,"extensions":[],"headers":{{}},"queries":[],"no_recursion":false,"extract_links":false,"add_slash":false,"stdin":false,"depth":4,"scan_limit":0,"rate_limit":0,"filter_size":[],"filter_line_count":[],"filter_word_count":[],"filter_regex":[],"dont_filter":false,"resumed":false,"resume_from":"","save_state":false,"time_limit":"","filter_similar":[]}},"responses":[{{"type":"response","url":"https://nerdcore.com/css","path":"/css","wildcard":true,"status":301,"content_length":173,"line_count":10,"word_count":16,"headers":{{"server":"nginx/1.16.1"}}}}]"#,
+        r#"{{"scans":[{{"id":"{}","url":"https://spiritanimal.com","scan_type":"Directory","status":"NotStarted","num_requests":0}}],"config":{{"type":"configuration","wordlist":"/usr/share/seclists/Discovery/Web-Content/raft-medium-directories.txt","config":"","proxy":"","replay_proxy":"","target_url":"","status_codes":[200,204,301,302,307,308,401,403,405],"replay_codes":[200,204,301,302,307,308,401,403,405],"filter_status":[],"threads":50,"timeout":7,"verbosity":0,"silent":false,"quiet":false,"auto_bail":false,"auto_tune":false,"json":false,"output":"","debug_log":"","user_agent":"feroxbuster/{}","redirects":false,"insecure":false,"extensions":[],"headers":{{}},"queries":[],"no_recursion":false,"extract_links":false,"add_slash":false,"stdin":false,"depth":4,"scan_limit":0,"parallel":0,"rate_limit":0,"filter_size":[],"filter_line_count":[],"filter_word_count":[],"filter_regex":[],"dont_filter":false,"resumed":false,"resume_from":"","save_state":false,"time_limit":"","filter_similar":[]}},"responses":[{{"type":"response","url":"https://nerdcore.com/css","path":"/css","wildcard":true,"status":301,"content_length":173,"line_count":10,"word_count":16,"headers":{{"server":"nginx/1.16.1"}}}}]"#,
        saved_id, VERSION
    );
    println!("{}\n{}", expected, json_state);
@@ -437,10 +438,14 @@ fn feroxscan_display() {
        scan_order: ScanOrder::Latest,
        scan_type: Default::default(),
        num_requests: 0,
+        start_time: Instant::now(),
        output_level: OutputLevel::Default,
+        status_403s: Default::default(),
+        status_429s: Default::default(),
        status: Default::default(),
        task: tokio::sync::Mutex::new(None),
        progress_bar: std::sync::Mutex::new(None),
+        errors: Default::default(),
    };

    let not_started = format!("{}", scan);
@@ -477,12 +482,16 @@ async fn ferox_scan_abort() {
        scan_order: ScanOrder::Latest,
        scan_type: Default::default(),
        num_requests: 0,
+        start_time: Instant::now(),
        output_level: OutputLevel::Default,
+        status_403s: Default::default(),
+        status_429s: Default::default(),
        status: std::sync::Mutex::new(ScanStatus::Running),
        task: tokio::sync::Mutex::new(Some(tokio::spawn(async move {
            sleep(Duration::from_millis(SLEEP_DURATION * 2));
        }))),
        progress_bar: std::sync::Mutex::new(None),
+        errors: Default::default(),
    };

    scan.abort().await.unwrap();
@@ -516,3 +525,70 @@ fn split_to_nums_is_correct() {

    assert_eq!(nums, vec![1, 3, 4]);
 }
+
+#[test]
+/// given a deep url, find the correct scan
+fn get_base_scan_by_url_finds_correct_scan() {
+    let urls = FeroxScans::default();
+    let url = "http://localhost";
+    let url1 = "http://localhost/stuff";
+    let url2 = "http://shlocalhost/stuff/things";
+    let url3 = "http://shlocalhost/stuff/things/mostuff";
+    let (_, scan) = urls.add_scan(url, ScanType::Directory, ScanOrder::Latest);
+    let (_, scan1) = urls.add_scan(url1, ScanType::Directory, ScanOrder::Latest);
+    let (_, scan2) = urls.add_scan(url2, ScanType::Directory, ScanOrder::Latest);
+    let (_, scan3) = urls.add_scan(url3, ScanType::Directory, ScanOrder::Latest);
+
+    assert_eq!(
+        urls.get_base_scan_by_url("http://localhost/things.php")
+            .unwrap()
+            .id,
+        scan.id
+    );
+    assert_eq!(
+        urls.get_base_scan_by_url("http://localhost/stuff/things.php")
+            .unwrap()
+            .id,
+        scan1.id
+    );
+    assert_eq!(
+        urls.get_base_scan_by_url("http://shlocalhost/stuff/things/mostuff.php")
+            .unwrap()
+            .id,
+        scan2.id
+    );
+    assert_eq!(
+        urls.get_base_scan_by_url("http://shlocalhost/stuff/things/mostuff/mothings.php")
+            .unwrap()
+            .id,
+        scan3.id
+    );
+}
+
+#[test]
+/// given a shallow url without a trailing slash, find the correct scan
+fn get_base_scan_by_url_finds_correct_scan_without_trailing_slash() {
+    let urls = FeroxScans::default();
+    let url = "http://localhost";
+    let (_, scan) = urls.add_scan(url, ScanType::Directory, ScanOrder::Latest);
+    assert_eq!(
+        urls.get_base_scan_by_url("http://localhost/BKPMiherrortBPKcw")
+            .unwrap()
+            .id,
+        scan.id
+    );
+}
+
+#[test]
+/// given a shallow url with a trailing slash, find the correct scan
+fn get_base_scan_by_url_finds_correct_scan_with_trailing_slash() {
+    let urls = FeroxScans::default();
+    let url = "http://127.0.0.1:41971/";
+    let (_, scan) = urls.add_scan(url, ScanType::Directory, ScanOrder::Latest);
+    assert_eq!(
+        urls.get_base_scan_by_url("http://127.0.0.1:41971/BKPMiherrortBPKcw")
+            .unwrap()
+            .id,
+        scan.id
+    );
+}
--- a/src/scanner.rs
+++ b/src/scanner.rs
@@ -1,352 +0,0 @@
-use std::{
-    cmp::max, collections::HashSet, convert::TryInto, ops::Deref, sync::atomic::Ordering,
-    sync::Arc, time::Instant,
-};
-
-use anyhow::{bail, Result};
-use futures::{stream, StreamExt};
-use lazy_static::lazy_static;
-use leaky_bucket::LeakyBucket;
-use tokio::sync::{oneshot, Semaphore};
-
-use crate::{
-    event_handlers::{
-        Command::{self, AddError, UpdateF64Field, UpdateUsizeField},
-        Handles,
-    },
-    extractor::{
-        ExtractionTarget::{ResponseBody, RobotsTxt},
-        ExtractorBuilder,
-    },
-    heuristics,
-    response::FeroxResponse,
-    scan_manager::{FeroxResponses, ScanOrder, ScanStatus, PAUSE_SCAN},
-    statistics::{
-        StatError::Other,
-        StatField::{DirScanTimes, ExpectedPerScan},
-    },
-    url::FeroxUrl,
-    utils::{fmt_err, make_request},
-};
-use tokio::time::Duration;
-
-lazy_static! {
-    /// Vector of FeroxResponse objects
-    pub static ref RESPONSES: FeroxResponses = FeroxResponses::default();
-    // todo consider removing this
-}
-
-/// Makes multiple requests based on the presence of extensions
-struct Requester {
-    /// handles to handlers and config
-    handles: Arc<Handles>,
-
-    /// url that will be scanned
-    target_url: String,
-
-    /// limits requests per second if present
-    rate_limiter: Option<LeakyBucket>,
-}
-
-/// Requester implementation
-impl Requester {
-    /// given a FeroxScanner, create a Requester
-    pub fn from(scanner: &FeroxScanner) -> Result<Self> {
-        let limit = scanner.handles.config.rate_limit;
-        let refill = max(limit / 10, 1); // minimum of 1 per second
-        let tokens = max(limit / 2, 1);
-        let interval = if refill == 1 { 1000 } else { 100 }; // 1 second if refill is 1
-
-        let rate_limiter = if limit > 0 {
-            let bucket = LeakyBucket::builder()
-                .refill_interval(Duration::from_millis(interval)) // add tokens every 0.1s
-                .refill_amount(refill) // ex: 100 req/s -> 10 tokens per 0.1s
-                .tokens(tokens) // reduce initial burst, 2 is arbitrary, but felt good
-                .max(limit)
-                .build()?;
-            Some(bucket)
-        } else {
-            None
-        };
-
-        Ok(Self {
-            rate_limiter,
-            handles: scanner.handles.clone(),
-            target_url: scanner.target_url.to_owned(),
-        })
-    }
-
-    /// limit the number of requests per second
-    pub async fn limit(&self) -> Result<()> {
-        self.rate_limiter.as_ref().unwrap().acquire_one().await?;
-        Ok(())
-    }
-
-    /// Wrapper for [make_request](fn.make_request.html)
-    ///
-    /// Attempts recursion when appropriate and sends Responses to the output handler for processing
-    async fn request(&self, word: &str) -> Result<()> {
-        log::trace!("enter: request({})", word);
-
-        let urls =
-            FeroxUrl::from_string(&self.target_url, self.handles.clone()).formatted_urls(word)?;
-
-        for url in urls {
-            if self.rate_limiter.is_some() {
-                // found a rate limiter, limit that junk!
-                if let Err(e) = self.limit().await {
-                    log::warn!("Could not rate limit scan: {}", e);
-                    self.handles.stats.send(AddError(Other)).unwrap_or_default();
-                }
-            }
-
-            let response = make_request(
-                &self.handles.config.client,
-                &url,
-                self.handles.config.output_level,
-                self.handles.stats.tx.clone(),
-            )
-            .await?;
-
-            // response came back without error, convert it to FeroxResponse
-            let ferox_response =
-                FeroxResponse::from(response, true, self.handles.config.output_level).await;
-
-            // do recursion if appropriate
-            if !self.handles.config.no_recursion {
-                self.handles
-                    .send_scan_command(Command::TryRecursion(Box::new(ferox_response.clone())))?;
-                let (tx, rx) = oneshot::channel::<bool>();
-                self.handles.send_scan_command(Command::Sync(tx))?;
-                rx.await?;
-            }
-
-            // purposefully doing recursion before filtering. the thought process is that
-            // even though this particular url is filtered, subsequent urls may not
-            if self
-                .handles
-                .filters
-                .data
-                .should_filter_response(&ferox_response, self.handles.stats.tx.clone())
-            {
-                continue;
-            }
-
-            if self.handles.config.extract_links && !ferox_response.status().is_redirection() {
-                let extractor = ExtractorBuilder::default()
-                    .target(ResponseBody)
-                    .response(&ferox_response)
-                    .handles(self.handles.clone())
-                    .build()?;
-
-                extractor.extract().await?;
-            }
-
-            // everything else should be reported
-            if let Err(e) = ferox_response.send_report(self.handles.output.tx.clone()) {
-                log::warn!("Could not send FeroxResponse to output handler: {}", e);
-            }
-        }
-
-        log::trace!("exit: request");
-        Ok(())
-    }
-}
-
-/// handles the main muscle movement of scanning a url
-pub struct FeroxScanner {
-    /// handles to handlers and config
-    handles: Arc<Handles>,
-
-    /// url that will be scanned
-    target_url: String,
-
-    /// whether or not this scanner is targeting an initial target specified by the user or one
-    /// found via recursion
-    order: ScanOrder,
-
-    /// wordlist that's already been read from disk
-    wordlist: Arc<HashSet<String>>,
-
-    /// limiter that restricts the number of active FeroxScanners  
-    scan_limiter: Arc<Semaphore>,
-}
-
-/// FeroxScanner implementation
-impl FeroxScanner {
-    /// create a new FeroxScanner
-    pub fn new(
-        target_url: &str,
-        order: ScanOrder,
-        wordlist: Arc<HashSet<String>>,
-        scan_limiter: Arc<Semaphore>,
-        handles: Arc<Handles>,
-    ) -> Self {
-        Self {
-            order,
-            handles,
-            wordlist,
-            scan_limiter,
-            target_url: target_url.to_string(),
-        }
-    }
-
-    /// Scan a given url using a given wordlist
-    ///
-    /// This is the primary entrypoint for the scanner
-    pub async fn scan_url(&self) -> Result<()> {
-        log::trace!("enter: scan_url");
-        log::info!("Starting scan against: {}", self.target_url);
-
-        let scan_timer = Instant::now();
-
-        if matches!(self.order, ScanOrder::Initial) && self.handles.config.extract_links {
-            // only grab robots.txt on the initial scan_url calls. all fresh dirs will be passed
-            // to try_recursion
-            let extractor = ExtractorBuilder::default()
-                .url(&self.target_url)
-                .handles(self.handles.clone())
-                .target(RobotsTxt)
-                .build()?;
-
-            let _ = extractor.extract().await;
-        }
-
-        let scanned_urls = self.handles.ferox_scans()?;
-
-        let ferox_scan = match scanned_urls.get_scan_by_url(&self.target_url) {
-            Some(scan) => {
-                scan.set_status(ScanStatus::Running)?;
-                scan
-            }
-            None => {
-                let msg = format!(
-                    "Could not find FeroxScan associated with {}; this shouldn't happen... exiting",
-                    self.target_url
-                );
-                bail!(fmt_err(&msg))
-            }
-        };
-
-        let progress_bar = ferox_scan.progress_bar();
-
-        // When acquire is called and the semaphore has remaining permits, the function immediately
-        // returns a permit. However, if no remaining permits are available, acquire (asynchronously)
-        // waits until an outstanding permit is dropped, at which point, the freed permit is assigned
-        // to the caller.
-        let _permit = self.scan_limiter.acquire().await;
-
-        // Arc clones to be passed around to the various scans
-        let looping_words = self.wordlist.clone();
-
-        {
-            let test = heuristics::HeuristicTests::new(self.handles.clone());
-            if let Ok(num_reqs) = test.wildcard(&self.target_url).await {
-                progress_bar.inc(num_reqs);
-            }
-        }
-
-        let requester = Arc::new(Requester::from(self)?);
-        let increment_len = (self.handles.config.extensions.len() + 1) as u64;
-
-        // producer tasks (mp of mpsc); responsible for making requests
-        let producers = stream::iter(looping_words.deref().to_owned())
-            .map(|word| {
-                let pb = progress_bar.clone(); // progress bar is an Arc around internal state
-                let scanned_urls_clone = scanned_urls.clone();
-                let requester_clone = requester.clone();
-                (
-                    tokio::spawn(async move {
-                        if PAUSE_SCAN.load(Ordering::Acquire) {
-                            // for every word in the wordlist, check to see if PAUSE_SCAN is set to true
-                            // when true; enter a busy loop that only exits by setting PAUSE_SCAN back
-                            // to false
-                            scanned_urls_clone.pause(true).await;
-                        }
-                        requester_clone.request(&word).await
-                    }),
-                    pb,
-                )
-            })
-            .for_each_concurrent(self.handles.config.threads, |(resp, bar)| async move {
-                match resp.await {
-                    Ok(_) => {
-                        bar.inc(increment_len);
-                    }
-                    Err(e) => {
-                        log::warn!("error awaiting a response: {}", e);
-                        self.handles.stats.send(AddError(Other)).unwrap_or_default();
-                    }
-                }
-            });
-
-        // await tx tasks
-        log::trace!("awaiting scan producers");
-        producers.await;
-        log::trace!("done awaiting scan producers");
-
-        self.handles.stats.send(UpdateF64Field(
-            DirScanTimes,
-            scan_timer.elapsed().as_secs_f64(),
-        ))?;
-
-        ferox_scan.finish()?;
-
-        log::trace!("exit: scan_url");
-
-        Ok(())
-    }
-}
-
-/// Perform steps necessary to run scans that only need to be performed once (warming up the
-/// engine, as it were)
-pub async fn initialize(num_words: usize, handles: Arc<Handles>) -> Result<()> {
-    log::trace!("enter: initialize({}, {:?})", num_words, handles);
-
-    // number of requests only needs to be calculated once, and then can be reused
-    let num_reqs_expected: u64 = if handles.config.extensions.is_empty() {
-        num_words.try_into()?
-    } else {
-        let total = num_words * (handles.config.extensions.len() + 1);
-        total.try_into()?
-    };
-
-    {
-        // no real reason to keep the arc around beyond this call
-        let scans = handles.ferox_scans()?;
-        scans.set_bar_length(num_reqs_expected);
-    }
-
-    // tell Stats object about the number of expected requests
-    handles.stats.send(UpdateUsizeField(
-        ExpectedPerScan,
-        num_reqs_expected as usize,
-    ))?;
-
-    log::trace!("exit: initialize");
-    Ok(())
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use crate::config::OutputLevel;
-    use crate::scan_manager::FeroxScans;
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
-    #[should_panic]
-    /// try to hit struct field coverage of FileOutHandler
-    async fn get_scan_by_url_bails_on_unfound_url() {
-        let sem = Semaphore::new(10);
-        let urls = FeroxScans::new(OutputLevel::Default);
-
-        let scanner = FeroxScanner::new(
-            "http://localhost",
-            ScanOrder::Initial,
-            Arc::new(Default::default()),
-            Arc::new(sem),
-            Arc::new(Handles::for_testing(Some(Arc::new(urls)), None).0),
-        );
-        scanner.scan_url().await.unwrap();
-    }
-}
--- a/src/scanner/ferox_scanner.rs
+++ b/src/scanner/ferox_scanner.rs
@@ -0,0 +1,185 @@
+use std::{collections::HashSet, ops::Deref, sync::atomic::Ordering, sync::Arc, time::Instant};
+
+use anyhow::{bail, Result};
+use futures::{stream, StreamExt};
+use lazy_static::lazy_static;
+use tokio::sync::Semaphore;
+
+use crate::{
+    event_handlers::{
+        Command::{AddError, AddToF64Field, SubtractFromUsizeField},
+        Handles,
+    },
+    extractor::{ExtractionTarget::RobotsTxt, ExtractorBuilder},
+    heuristics,
+    scan_manager::{FeroxResponses, ScanOrder, ScanStatus, PAUSE_SCAN},
+    statistics::{
+        StatError::Other,
+        StatField::{DirScanTimes, TotalExpected},
+    },
+    utils::fmt_err,
+};
+
+use super::requester::Requester;
+
+lazy_static! {
+    /// Vector of FeroxResponse objects
+    pub static ref RESPONSES: FeroxResponses = FeroxResponses::default();
+    // todo consider removing this
+}
+/// handles the main muscle movement of scanning a url
+pub struct FeroxScanner {
+    /// handles to handlers and config
+    pub(super) handles: Arc<Handles>,
+
+    /// url that will be scanned
+    pub(super) target_url: String,
+
+    /// whether or not this scanner is targeting an initial target specified by the user or one
+    /// found via recursion
+    order: ScanOrder,
+
+    /// wordlist that's already been read from disk
+    wordlist: Arc<HashSet<String>>,
+
+    /// limiter that restricts the number of active FeroxScanners  
+    scan_limiter: Arc<Semaphore>,
+}
+
+/// FeroxScanner implementation
+impl FeroxScanner {
+    /// create a new FeroxScanner
+    pub fn new(
+        target_url: &str,
+        order: ScanOrder,
+        wordlist: Arc<HashSet<String>>,
+        scan_limiter: Arc<Semaphore>,
+        handles: Arc<Handles>,
+    ) -> Self {
+        Self {
+            order,
+            handles,
+            wordlist,
+            scan_limiter,
+            target_url: target_url.to_string(),
+        }
+    }
+
+    /// Scan a given url using a given wordlist
+    ///
+    /// This is the primary entrypoint for the scanner
+    pub async fn scan_url(&self) -> Result<()> {
+        log::trace!("enter: scan_url");
+        log::info!("Starting scan against: {}", self.target_url);
+
+        let scan_timer = Instant::now();
+
+        if matches!(self.order, ScanOrder::Initial) && self.handles.config.extract_links {
+            // only grab robots.txt on the initial scan_url calls. all fresh dirs will be passed
+            // to try_recursion
+            let extractor = ExtractorBuilder::default()
+                .url(&self.target_url)
+                .handles(self.handles.clone())
+                .target(RobotsTxt)
+                .build()?;
+
+            let _ = extractor.extract().await;
+        }
+
+        let scanned_urls = self.handles.ferox_scans()?;
+
+        let ferox_scan = match scanned_urls.get_scan_by_url(&self.target_url) {
+            Some(scan) => {
+                scan.set_status(ScanStatus::Running)?;
+                scan
+            }
+            None => {
+                let msg = format!(
+                    "Could not find FeroxScan associated with {}; this shouldn't happen... exiting",
+                    self.target_url
+                );
+                bail!(fmt_err(&msg))
+            }
+        };
+
+        let progress_bar = ferox_scan.progress_bar();
+
+        // When acquire is called and the semaphore has remaining permits, the function immediately
+        // returns a permit. However, if no remaining permits are available, acquire (asynchronously)
+        // waits until an outstanding permit is dropped, at which point, the freed permit is assigned
+        // to the caller.
+        let _permit = self.scan_limiter.acquire().await;
+
+        // Arc clones to be passed around to the various scans
+        let looping_words = self.wordlist.clone();
+
+        {
+            let test = heuristics::HeuristicTests::new(self.handles.clone());
+            if let Ok(num_reqs) = test.wildcard(&self.target_url).await {
+                progress_bar.inc(num_reqs);
+            }
+        }
+
+        let requester = Arc::new(Requester::from(self, ferox_scan.clone())?);
+        let increment_len = (self.handles.config.extensions.len() + 1) as u64;
+
+        // producer tasks (mp of mpsc); responsible for making requests
+        let producers = stream::iter(looping_words.deref().to_owned())
+            .map(|word| {
+                let pb = progress_bar.clone(); // progress bar is an Arc around internal state
+                let scanned_urls_clone = scanned_urls.clone();
+                let requester_clone = requester.clone();
+                let handles_clone = self.handles.clone();
+                (
+                    tokio::spawn(async move {
+                        if PAUSE_SCAN.load(Ordering::Acquire) {
+                            // for every word in the wordlist, check to see if PAUSE_SCAN is set to true
+                            // when true; enter a busy loop that only exits by setting PAUSE_SCAN back
+                            // to false
+                            let num_cancelled = scanned_urls_clone.pause(true).await;
+                            if num_cancelled > 0 {
+                                handles_clone
+                                    .stats
+                                    .send(SubtractFromUsizeField(TotalExpected, num_cancelled))
+                                    .unwrap_or_else(|e| {
+                                        log::warn!("Could not update overall scan bar: {}", e)
+                                    });
+                            }
+                        }
+                        requester_clone
+                            .request(&word)
+                            .await
+                            .unwrap_or_else(|e| log::warn!("Requester encountered an error: {}", e))
+                    }),
+                    pb,
+                )
+            })
+            .for_each_concurrent(self.handles.config.threads, |(resp, bar)| async move {
+                match resp.await {
+                    Ok(_) => {
+                        bar.inc(increment_len);
+                    }
+                    Err(e) => {
+                        log::warn!("error awaiting a response: {}", e);
+                        self.handles.stats.send(AddError(Other)).unwrap_or_default();
+                    }
+                }
+            });
+
+        // await tx tasks
+        log::trace!("awaiting scan producers");
+        producers.await;
+        log::trace!("done awaiting scan producers");
+
+        self.handles.stats.send(AddToF64Field(
+            DirScanTimes,
+            scan_timer.elapsed().as_secs_f64(),
+        ))?;
+
+        ferox_scan.finish()?;
+
+        log::trace!("exit: scan_url");
+
+        Ok(())
+    }
+}
--- a/src/scanner/init.rs
+++ b/src/scanner/init.rs
@@ -0,0 +1,34 @@
+use crate::{
+    event_handlers::{Command::AddToUsizeField, Handles},
+    statistics::StatField::ExpectedPerScan,
+};
+use anyhow::Result;
+use std::{convert::TryInto, sync::Arc};
+
+/// Perform steps necessary to run scans that only need to be performed once (warming up the
+/// engine, as it were)
+pub async fn initialize(num_words: usize, handles: Arc<Handles>) -> Result<()> {
+    log::trace!("enter: initialize({}, {:?})", num_words, handles);
+
+    // number of requests only needs to be calculated once, and then can be reused
+    let num_reqs_expected: u64 = if handles.config.extensions.is_empty() {
+        num_words.try_into()?
+    } else {
+        let total = num_words * (handles.config.extensions.len() + 1);
+        total.try_into()?
+    };
+
+    {
+        // no real reason to keep the arc around beyond this call
+        let scans = handles.ferox_scans()?;
+        scans.set_bar_length(num_reqs_expected);
+    }
+
+    // tell Stats object about the number of expected requests
+    handles
+        .stats
+        .send(AddToUsizeField(ExpectedPerScan, num_reqs_expected as usize))?;
+
+    log::trace!("exit: initialize");
+    Ok(())
+}
--- a/src/scanner/limit_heap.rs
+++ b/src/scanner/limit_heap.rs
@@ -0,0 +1,171 @@
+use std::fmt::{Debug, Formatter, Result};
+
+/// bespoke variation on an array-backed max-heap
+///
+/// 255 possible values generated from the initial requests/second
+///
+/// when no additional errors are encountered, the left child is taken (increasing req/sec)
+/// if errors have increased since the last interval, the right child is taken (decreasing req/sec)
+///
+/// formula for each child:
+/// - left: (|parent - current|) / 2 + current
+/// - right: current - ((|parent - current|) / 2)
+pub(super) struct LimitHeap {
+    /// backing array, 255 nodes == height of 7 ( 2^(h+1) -1 nodes )
+    pub(super) inner: [i32; 255],
+
+    /// original # of requests / second
+    pub(super) original: i32,
+
+    /// current position w/in the backing array
+    pub(super) current: usize,
+}
+
+/// default implementation of a LimitHeap
+impl Default for LimitHeap {
+    /// zero-initialize the backing array
+    fn default() -> Self {
+        Self {
+            inner: [0; 255],
+            original: 0,
+            current: 0,
+        }
+    }
+}
+
+/// Debug implementation of a LimitHeap
+impl Debug for LimitHeap {
+    /// return debug representation that conforms to <32 elements in array
+    fn fmt(&self, f: &mut Formatter<'_>) -> Result {
+        let msg = format!(
+            "LimitHeap {{ original: {}, current: {}, inner: [{}...] }}",
+            self.original, self.current, self.inner[0]
+        );
+        write!(f, "{}", msg)
+    }
+}
+
+/// implementation of a LimitHeap
+impl LimitHeap {
+    /// move to right child, return node's index from which the move was requested
+    pub(super) fn move_right(&mut self) -> usize {
+        if self.has_children() {
+            let tmp = self.current;
+            self.current = self.current * 2 + 2;
+            return tmp;
+        }
+        self.current
+    }
+
+    /// move to left child, return node's index from which the move was requested
+    pub(super) fn move_left(&mut self) -> usize {
+        if self.has_children() {
+            let tmp = self.current;
+            self.current = self.current * 2 + 1;
+            return tmp;
+        }
+        self.current
+    }
+
+    /// move to parent, return node's index from which the move was requested
+    pub(super) fn move_up(&mut self) -> usize {
+        if self.has_parent() {
+            let tmp = self.current;
+            self.current = (self.current - 1) / 2;
+            return tmp;
+        }
+        self.current
+    }
+
+    /// move directly to the given index
+    pub(super) fn move_to(&mut self, index: usize) {
+        self.current = index;
+    }
+
+    /// get the current node's value
+    pub(super) fn value(&self) -> i32 {
+        self.inner[self.current]
+    }
+
+    /// set the current node's value
+    pub(super) fn set_value(&mut self, value: i32) {
+        self.inner[self.current] = value;
+    }
+
+    /// check that this node has a parent (true for all except root)
+    pub(super) fn has_parent(&self) -> bool {
+        self.current > 0
+    }
+
+    /// get node's parent's value or self.original if at the root
+    pub(super) fn parent_value(&mut self) -> i32 {
+        if self.has_parent() {
+            let current = self.move_up();
+            let val = self.value();
+            self.move_to(current);
+            return val;
+        }
+        self.original
+    }
+
+    /// check if the current node has children
+    pub(super) fn has_children(&self) -> bool {
+        // inner structure is a complete tree, just check for the right child
+        self.current * 2 + 2 <= self.inner.len()
+    }
+
+    /// get current node's right child's value
+    fn right_child_value(&mut self) -> i32 {
+        let tmp = self.move_right();
+        let val = self.value();
+        self.move_to(tmp);
+        val
+    }
+
+    /// set current node's left child's value
+    fn set_left_child(&mut self) {
+        let parent = self.parent_value();
+        let current = self.value();
+        let value = ((parent - current).abs() / 2) + current;
+
+        self.move_left();
+        self.set_value(value);
+        self.move_up();
+    }
+
+    /// set current node's right child's value
+    fn set_right_child(&mut self) {
+        let parent = self.parent_value();
+        let current = self.value();
+        let value = current - ((parent - current).abs() / 2);
+
+        self.move_right();
+        self.set_value(value);
+        self.move_up();
+    }
+
+    /// iterate over the backing array, filling in each child's value based on the original value
+    pub(super) fn build(&mut self) {
+        // ex: original is 400
+        // arr[0] == 200
+        // arr[1] (left child) == 300
+        // arr[2] (right child) == 100
+        let root = self.original / 2;
+
+        self.inner[0] = root; // set root node to half of the original value
+        self.inner[1] = ((self.original - root).abs() / 2) + root;
+        self.inner[2] = root - ((self.original - root).abs() / 2);
+
+        // start with index 1 and fill in each child below that node
+        for i in 1..self.inner.len() {
+            self.move_to(i);
+
+            if self.has_children() && self.right_child_value() == 0 {
+                // this node has an unset child since the rchild is 0
+                self.set_left_child();
+                self.set_right_child();
+            }
+        }
+        self.move_to(0); // reset current index to the root of the tree
+    }
+}
--- a/src/scanner/mod.rs
+++ b/src/scanner/mod.rs
@@ -0,0 +1,12 @@
+mod ferox_scanner;
+mod utils;
+mod init;
+#[cfg(test)]
+mod tests;
+mod limit_heap;
+mod policy_data;
+mod requester;
+
+pub use self::ferox_scanner::{FeroxScanner, RESPONSES};
+pub use self::init::initialize;
+pub use self::utils::PolicyTrigger;
--- a/src/scanner/policy_data.rs
+++ b/src/scanner/policy_data.rs
@@ -0,0 +1,309 @@
+use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
+
+use crate::{atomic_load, atomic_store, config::RequesterPolicy};
+
+use super::limit_heap::LimitHeap;
+
+/// data regarding policy and metadata about last enforced trigger etc...
+#[derive(Default, Debug)]
+pub struct PolicyData {
+    /// how to handle exceptional cases such as too many errors / 403s / 429s etc
+    pub(super) policy: RequesterPolicy,
+
+    /// whether or not we're in the middle of a cooldown period
+    pub(super) cooling_down: AtomicBool,
+
+    /// length of time to pause tuning after making an adjustment
+    pub(super) wait_time: u64,
+
+    /// rate limit (at last interval)
+    limit: AtomicUsize,
+
+    /// number of errors (at last interval)
+    pub(super) errors: AtomicUsize,
+
+    /// whether or not the owning Requester should remove the rate_limiter, happens when a scan
+    /// has been limited and moves back up to the point of its original scan speed
+    pub(super) remove_limit: AtomicBool,
+
+    /// heap of values used for adjusting # of requests/second
+    pub(super) heap: std::sync::RwLock<LimitHeap>,
+}
+
+/// implementation of PolicyData
+impl PolicyData {
+    /// given a RequesterPolicy, create a new PolicyData
+    pub fn new(policy: RequesterPolicy, timeout: u64) -> Self {
+        // can use this as a tweak for how aggressively adjustments should be made when tuning
+        let wait_time = ((timeout as f64 / 2.0) * 1000.0) as u64;
+
+        Self {
+            policy,
+            wait_time,
+            ..Default::default()
+        }
+    }
+
+    /// setter for requests / second; populates the underlying heap with values from req/sec seed
+    pub(super) fn set_reqs_sec(&self, reqs_sec: usize) {
+        if let Ok(mut guard) = self.heap.write() {
+            guard.original = reqs_sec as i32;
+            guard.build();
+            self.set_limit(guard.inner[0] as usize); // set limit to 1/2 of current request rate
+        }
+    }
+
+    /// setter for errors
+    pub(super) fn set_errors(&self, errors: usize) {
+        atomic_store!(self.errors, errors);
+    }
+
+    /// setter for limit
+    fn set_limit(&self, limit: usize) {
+        atomic_store!(self.limit, limit);
+    }
+
+    /// getter for limit
+    pub(super) fn get_limit(&self) -> usize {
+        atomic_load!(self.limit)
+    }
+
+    /// adjust the rate of requests per second up (increase rate)
+    pub(super) fn adjust_up(&self, streak_counter: &usize) {
+        if let Ok(mut heap) = self.heap.try_write() {
+            if *streak_counter > 2 {
+                // streak of 3 upward moves in a row, traverse the tree upward instead of to a
+                // higher-valued branch lower in the tree
+                let current = heap.value();
+                heap.move_up();
+                heap.move_up();
+                if current > heap.value() {
+                    // the tree's structure makes it so that sometimes 2 moves up results in a
+                    // value greater than the current node's and other times we need to move 3 up
+                    // to arrive at a greater value
+                    if heap.has_parent() && heap.parent_value() > current {
+                        // all nodes except 0th node (root)
+                        heap.move_up();
+                    } else if !heap.has_parent() {
+                        // been here enough that we can try resuming the scan to its original
+                        // speed (no limiting at all)
+                        atomic_store!(self.remove_limit, true);
+                    }
+                }
+                self.set_limit(heap.value() as usize);
+            } else if heap.has_children() {
+                // streak not at 3, just check that we can move down, and do so
+                heap.move_left();
+                self.set_limit(heap.value() as usize);
+            } else {
+                // tree bottomed out, need to move back up the tree a bit
+                let current = heap.value();
+                heap.move_up();
+                heap.move_up();
+
+                if current > heap.value() {
+                    heap.move_up();
+                }
+
+                self.set_limit(heap.value() as usize);
+            }
+        }
+    }
+
+    /// adjust the rate of requests per second down (decrease rate)
+    pub(super) fn adjust_down(&self) {
+        if let Ok(mut heap) = self.heap.try_write() {
+            if heap.has_children() {
+                heap.move_right();
+                self.set_limit(heap.value() as usize);
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    /// PolicyData builds and sets correct values for the inner heap when set_reqs_sec is called
+    fn set_reqs_sec_builds_heap_and_sets_initial_value() {
+        let pd = PolicyData::new(RequesterPolicy::AutoBail, 7);
+        assert_eq!(pd.wait_time, 3500);
+        pd.set_reqs_sec(400);
+        assert_eq!(pd.get_limit(), 200);
+        assert_eq!(pd.heap.read().unwrap().original, 400);
+        assert_eq!(pd.heap.read().unwrap().current, 0);
+        assert_eq!(pd.heap.read().unwrap().inner[0], 200);
+        assert_eq!(pd.heap.read().unwrap().inner[1], 300);
+        assert_eq!(pd.heap.read().unwrap().inner[2], 100);
+    }
+
+    #[test]
+    /// PolicyData setters/getters tests for code coverage / sanity
+    fn policy_data_getters_and_setters() {
+        let pd = PolicyData::new(RequesterPolicy::AutoBail, 7);
+        pd.set_errors(20);
+        assert_eq!(pd.errors.load(Ordering::Relaxed), 20);
+        pd.set_limit(200);
+        assert_eq!(pd.get_limit(), 200);
+    }
+
+    #[test]
+    /// PolicyData adjust_down sets the limit to the correct value
+    fn policy_data_adjust_down_simple() {
+        let pd = PolicyData::new(RequesterPolicy::AutoBail, 7);
+        pd.set_reqs_sec(400);
+        assert_eq!(pd.get_limit(), 200);
+        pd.adjust_down();
+        assert_eq!(pd.get_limit(), 100);
+    }
+
+    #[test]
+    /// PolicyData adjust_down sets the limit to the correct value when no child nodes are present
+    fn policy_data_adjust_down_no_children() {
+        let pd = PolicyData::new(RequesterPolicy::AutoBail, 7);
+        pd.set_reqs_sec(400);
+        assert_eq!(pd.get_limit(), 200);
+        let mut guard = pd.heap.write().unwrap();
+        guard.move_to(250);
+        guard.set_value(27);
+        pd.set_limit(guard.value() as usize);
+        drop(guard);
+
+        pd.adjust_down();
+        assert_eq!(pd.get_limit(), 27);
+    }
+
+    #[test]
+    /// PolicyData adjust_up sets the limit to the correct value
+    fn policy_data_adjust_up_simple() {
+        let pd = PolicyData::new(RequesterPolicy::AutoBail, 7);
+        pd.set_reqs_sec(400);
+        assert_eq!(pd.get_limit(), 200);
+        pd.adjust_up(&0);
+        assert_eq!(pd.get_limit(), 300);
+    }
+
+    #[test]
+    /// PolicyData adjust_up sets the limit to the correct value
+    fn policy_data_adjust_up_with_streak_and_2_moves() {
+        // original: 400
+        // [200, 300, 100, 350, 250, 150, 50, 375, 325, 275, 225, 175, 125, 75, 25, ...]
+        let pd = PolicyData::new(RequesterPolicy::AutoBail, 7);
+        pd.set_reqs_sec(400);
+        assert_eq!(pd.get_limit(), 200);
+
+        // 2 moves
+        pd.heap.write().unwrap().move_to(9);
+        assert_eq!(pd.heap.read().unwrap().value(), 275);
+        pd.adjust_up(&3);
+        assert_eq!(pd.heap.read().unwrap().value(), 300);
+        assert_eq!(pd.limit.load(Ordering::Relaxed), 300);
+        assert_eq!(pd.remove_limit.load(Ordering::Relaxed), false);
+    }
+
+    #[test]
+    /// PolicyData adjust_up sets the limit to the correct value
+    fn policy_data_adjust_up_with_streak_and_2_moves_to_arrive_at_root() {
+        // original: 400
+        // [200, 300, 100, 350, 250, 150, 50, 375, 325, 275, 225, 175, 125, 75, 25, ...]
+        let pd = PolicyData::new(RequesterPolicy::AutoBail, 7);
+        pd.set_reqs_sec(400);
+        assert_eq!(pd.get_limit(), 200);
+
+        pd.heap.write().unwrap().move_to(4);
+        assert_eq!(pd.heap.read().unwrap().value(), 250);
+        pd.adjust_up(&3);
+        assert_eq!(pd.heap.read().unwrap().value(), 200);
+        assert_eq!(pd.limit.load(Ordering::Relaxed), 200);
+        assert_eq!(pd.remove_limit.load(Ordering::Relaxed), true);
+    }
+
+    #[test]
+    /// PolicyData adjust_up sets the limit to the correct value
+    fn policy_data_adjust_up_with_streak_and_2_moves_to_find_less_than_current() {
+        // original: 400
+        // [200, 300, 100, 350, 250, 150, 50, 375, 325, 275, 225, 175, 125, 75, 25, ...]
+        let pd = PolicyData::new(RequesterPolicy::AutoBail, 7);
+        pd.set_reqs_sec(400);
+        assert_eq!(pd.get_limit(), 200);
+
+        pd.heap.write().unwrap().move_to(15);
+        assert_eq!(pd.heap.read().unwrap().value(), 387);
+        pd.adjust_up(&3);
+        assert_eq!(pd.heap.read().unwrap().value(), 350);
+        assert_eq!(pd.limit.load(Ordering::Relaxed), 350);
+        assert_eq!(pd.remove_limit.load(Ordering::Relaxed), false);
+    }
+
+    #[test]
+    /// PolicyData adjust_up sets the limit to the correct value
+    fn policy_data_adjust_up_with_streak_and_3_moves() {
+        // original: 400
+        // [200, 300, 100, 350, 250, 150, 50, 375, 325, 275, 225, 175, 125, 75, 25, ...]
+        let pd = PolicyData::new(RequesterPolicy::AutoBail, 7);
+        pd.set_reqs_sec(400);
+        assert_eq!(pd.get_limit(), 200);
+
+        pd.heap.write().unwrap().move_to(19);
+        assert_eq!(pd.heap.read().unwrap().value(), 287);
+        pd.adjust_up(&3);
+        assert_eq!(pd.heap.read().unwrap().value(), 300);
+        assert_eq!(pd.limit.load(Ordering::Relaxed), 300);
+        assert_eq!(pd.remove_limit.load(Ordering::Relaxed), false);
+    }
+
+    #[test]
+    /// PolicyData adjust_up sets the limit to the correct value
+    fn policy_data_adjust_up_with_no_children_2_moves() {
+        // original: 400
+        // [200, 300, 100, 350, 250, 150, 50, 375, 325, 275, 225, 175, 125, 75, 25, ...]
+        let pd = PolicyData::new(RequesterPolicy::AutoBail, 7);
+        pd.set_reqs_sec(400);
+        assert_eq!(pd.get_limit(), 200);
+
+        pd.heap.write().unwrap().move_to(241);
+
+        assert_eq!(pd.heap.read().unwrap().value(), 41);
+        pd.adjust_up(&0);
+        assert_eq!(pd.heap.read().unwrap().value(), 43);
+        assert_eq!(pd.limit.load(Ordering::Relaxed), 43);
+        assert_eq!(pd.remove_limit.load(Ordering::Relaxed), false);
+    }
+
+    #[test]
+    /// PolicyData adjust_up sets the limit to the correct value
+    fn policy_data_adjust_up_with_no_children_3_moves() {
+        // original: 400
+        // [200, 300, 100, 350, 250, 150, 50, 375, 325, 275, 225, 175, 125, 75, 25, ...]
+        let pd = PolicyData::new(RequesterPolicy::AutoBail, 7);
+        pd.set_reqs_sec(400);
+        assert_eq!(pd.get_limit(), 200);
+
+        pd.heap.write().unwrap().move_to(240);
+
+        assert_eq!(pd.heap.read().unwrap().value(), 45);
+        pd.adjust_up(&0);
+        assert_eq!(pd.heap.read().unwrap().value(), 37);
+        assert_eq!(pd.limit.load(Ordering::Relaxed), 37);
+        assert_eq!(pd.remove_limit.load(Ordering::Relaxed), false);
+    }
+
+    #[test]
+    /// hit some of the out of the way corners of limitheap for coverage
+    fn increase_limit_heap_coverage_by_hitting_edge_cases() {
+        let pd = PolicyData::new(RequesterPolicy::AutoBail, 7);
+        pd.set_reqs_sec(400);
+
+        println!("{:?}", pd.heap.read().unwrap()); // debug derivation
+
+        pd.heap.write().unwrap().move_to(240);
+        assert_eq!(pd.heap.write().unwrap().move_right(), 240);
+        assert_eq!(pd.heap.write().unwrap().move_left(), 240);
+
+        pd.heap.write().unwrap().move_to(0);
+        assert_eq!(pd.heap.write().unwrap().move_up(), 0);
+        assert_eq!(pd.heap.write().unwrap().parent_value(), 400);
+    }
+}
--- a/src/scanner/requester.rs
+++ b/src/scanner/requester.rs
--- a/src/scanner/tests.rs
+++ b/src/scanner/tests.rs
@@ -0,0 +1,28 @@
+use std::sync::Arc;
+
+use tokio::sync::Semaphore;
+
+use crate::{
+    config::OutputLevel,
+    event_handlers::Handles,
+    scan_manager::{FeroxScans, ScanOrder},
+};
+
+use super::*;
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+#[should_panic]
+/// try to hit struct field coverage of FileOutHandler
+async fn get_scan_by_url_bails_on_unfound_url() {
+    let sem = Semaphore::new(10);
+    let urls = FeroxScans::new(OutputLevel::Default);
+
+    let scanner = FeroxScanner::new(
+        "http://localhost",
+        ScanOrder::Initial,
+        Arc::new(Default::default()),
+        Arc::new(sem),
+        Arc::new(Handles::for_testing(Some(Arc::new(urls)), None).0),
+    );
+    scanner.scan_url().await.unwrap();
+}
--- a/src/scanner/utils.rs
+++ b/src/scanner/utils.rs
@@ -0,0 +1,12 @@
+#[derive(Copy, Clone, PartialEq, Debug)]
+/// represents different situations where different criteria can trigger auto-tune/bail behavior
+pub enum PolicyTrigger {
+    /// excessive 403 trigger
+    Status403,
+
+    /// excessive 429 trigger
+    Status429,
+
+    /// excessive general errors
+    Errors,
+}
--- a/src/statistics/container.rs
+++ b/src/statistics/container.rs
@@ -29,7 +29,7 @@ pub struct Stats {
    timeouts: AtomicUsize,

    /// tracker for total number of requests sent by the client
-    requests: AtomicUsize,
+    pub(crate) requests: AtomicUsize,

    /// tracker for total number of requests expected to send if the scan runs to completion
    ///
@@ -42,7 +42,7 @@ pub struct Stats {
    total_expected: AtomicUsize,

    /// tracker for total number of errors encountered by the client
-    errors: AtomicUsize,
+    pub(crate) errors: AtomicUsize,

    /// tracker for overall number of 2xx status codes seen by the client
    successes: AtomicUsize,
@@ -58,7 +58,7 @@ pub struct Stats {

    /// tracker for number of scans performed, this directly equates to number of directories
    /// recursed into and affects the total number of expected requests
-    total_scans: AtomicUsize,
+    pub(crate) total_scans: AtomicUsize,

    /// tracker for initial number of requested targets
    initial_targets: AtomicUsize,
@@ -80,10 +80,10 @@ pub struct Stats {
    status_401s: AtomicUsize,

    /// tracker for overall number of 403s seen by the client
-    status_403s: AtomicUsize,
+    pub(crate) status_403s: AtomicUsize,

    /// tracker for overall number of 429s seen by the client
-    status_429s: AtomicUsize,
+    pub(crate) status_429s: AtomicUsize,

    /// tracker for overall number of 500s seen by the client
    status_500s: AtomicUsize,
@@ -176,6 +176,16 @@ impl Stats {
        atomic_load!(self.errors)
    }

+    /// public getter for status_403s
+    pub fn status_403s(&self) -> usize {
+        atomic_load!(self.status_403s)
+    }
+
+    /// public getter for status_429s
+    pub fn status_429s(&self) -> usize {
+        atomic_load!(self.status_429s)
+    }
+
    /// public getter for total_expected
    pub fn total_expected(&self) -> usize {
        atomic_load!(self.total_expected)
@@ -222,10 +232,6 @@ impl Stats {
            StatError::Timeout => {
                atomic_increment!(self.timeouts);
            }
-            StatError::Status403 => {
-                atomic_increment!(self.status_403s);
-                atomic_increment!(self.client_errors);
-            }
            StatError::UrlFormat => {
                atomic_increment!(self.url_format_errors);
            }
@@ -238,9 +244,7 @@ impl Stats {
            StatError::Request => {
                atomic_increment!(self.request_errors);
            }
-            StatError::Other => {
-                atomic_increment!(self.errors);
-            }
+            _ => {} // no need to hit Other as we always increment self.errors anyway
        }
    }

@@ -248,7 +252,7 @@ impl Stats {
    ///
    /// Implies incrementing:
    ///     - requests
-    ///     - status_403s (when code is 403)
+    ///     - appropriate status_* codes
    ///     - errors (when code is [45]xx)
    pub fn add_status_code(&self, status: StatusCode) {
        self.add_request();
@@ -264,9 +268,6 @@ impl Stats {
        }

        match status {
-            StatusCode::FORBIDDEN => {
-                atomic_increment!(self.status_403s);
-            }
            StatusCode::OK => {
                atomic_increment!(self.status_200s);
            }
@@ -279,6 +280,9 @@ impl Stats {
            StatusCode::UNAUTHORIZED => {
                atomic_increment!(self.status_401s);
            }
+            StatusCode::FORBIDDEN => {
+                atomic_increment!(self.status_403s);
+            }
            StatusCode::TOO_MANY_REQUESTS => {
                atomic_increment!(self.status_429s);
            }
@@ -307,6 +311,13 @@ impl Stats {
        }
    }

+    /// subtract a value from the given field
+    pub fn subtract_from_usize_field(&self, field: StatField, value: usize) {
+        if let StatField::TotalExpected = field {
+            self.total_expected.fetch_sub(value, Ordering::Relaxed);
+        }
+    }
+
    /// Update a `Stats` field of type usize
    pub fn update_usize_field(&self, field: StatField, value: usize) {
        match field {
@@ -435,30 +446,6 @@ mod tests {
        Ok(())
    }

-    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
-    /// when sent StatCommand::AddRequest, stats object should reflect the change
-    ///
-    /// incrementing a 403 (tracked in status_403s) should also increment:
-    ///     - errors
-    ///     - requests
-    ///     - client_errors
-    async fn statistics_handler_increments_403() {
-        let (task, handle) = setup_stats_test();
-
-        let err = Command::AddError(StatError::Status403);
-        let err2 = Command::AddError(StatError::Status403);
-
-        handle.tx.send(err).unwrap_or_default();
-        handle.tx.send(err2).unwrap_or_default();
-
-        teardown_stats_test(handle.tx.clone(), task).await;
-
-        assert_eq!(handle.data.errors.load(Ordering::Relaxed), 2);
-        assert_eq!(handle.data.requests.load(Ordering::Relaxed), 2);
-        assert_eq!(handle.data.status_403s.load(Ordering::Relaxed), 2);
-        assert_eq!(handle.data.client_errors.load(Ordering::Relaxed), 2);
-    }
-
    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
    /// when sent StatCommand::AddRequest, stats object should reflect the change
    ///
@@ -567,7 +554,7 @@ mod tests {

        stats.merge_from(tfile.path().to_str().unwrap()).unwrap();

-        // as of 1.11.1; all Stats fields are accounted for whether they're updated in merge_from
+        // as of 2.1.0; all Stats fields are accounted for whether they're updated in merge_from
        // or not
        assert_eq!(atomic_load!(stats.timeouts), 1);
        assert_eq!(atomic_load!(stats.requests), 9207);
@@ -617,4 +604,22 @@ mod tests {
        stats.update_runtime(20.2);
        assert!((stats.total_runtime.lock().unwrap()[0] - 20.2).abs() < f64::EPSILON);
    }
+
+    #[test]
+    /// ensure status_403s returns the correct value
+    fn status_403s_returns_correct_value() {
+        let config = Configuration::new().unwrap();
+        let stats = Stats::new(config.extensions.len(), config.json);
+        stats.status_403s.store(12, Ordering::Relaxed);
+        assert_eq!(stats.status_403s(), 12);
+    }
+
+    #[test]
+    /// ensure status_403s returns the correct value
+    fn status_429s_returns_correct_value() {
+        let config = Configuration::new().unwrap();
+        let stats = Stats::new(config.extensions.len(), config.json);
+        stats.status_429s.store(141, Ordering::Relaxed);
+        assert_eq!(stats.status_429s(), 141);
+    }
 }
--- a/src/statistics/error.rs
+++ b/src/statistics/error.rs
@@ -1,9 +1,6 @@
 #[derive(Debug, Copy, Clone)]
 /// Enum variants used to inform the `StatCommand` protocol what `Stats` fields should be updated
 pub enum StatError {
-    /// Represents a 403 response code
-    Status403,
-
    /// Represents a timeout error
    Timeout,

--- a/src/statistics/macros.rs
+++ b/src/statistics/macros.rs
@@ -20,4 +20,18 @@ macro_rules! atomic_load {
    ($metric:expr) => {
        $metric.load(Ordering::Relaxed);
    };
+    ($metric:expr, $ordering:expr) => {
+        $metric.load($ordering);
+    };
+}
+
+/// Wrapper around `Atomic*.store` to save me from writing Ordering::Relaxed a bajillion times
+#[macro_export]
+macro_rules! atomic_store {
+    ($metric:expr, $value:expr) => {
+        $metric.store($value, Ordering::Relaxed);
+    };
+    ($metric:expr, $value:expr, $ordering:expr) => {
+        $metric.store($value, $ordering);
+    };
 }
--- a/src/utils.rs
+++ b/src/utils.rs
@@ -1,18 +1,22 @@
 use anyhow::{bail, Context, Result};
 use console::{strip_ansi_codes, style, user_attended};
 use indicatif::ProgressBar;
-use reqwest::{Client, Response, Url};
+use reqwest::{Client, Response, StatusCode, Url};
 #[cfg(not(target_os = "windows"))]
 use rlimit::{getrlimit, setrlimit, Resource, Rlim};
 use std::{
    fs,
    io::{self, BufWriter, Write},
+    sync::Arc,
 };
 use tokio::sync::mpsc::UnboundedSender;

 use crate::{
    config::OutputLevel,
-    event_handlers::Command::{self, AddError, AddStatus},
+    event_handlers::{
+        Command::{self, AddError, AddStatus},
+        Handles,
+    },
    progress::PROGRESS_PRINTER,
    send_command,
    statistics::StatError::{Connection, Other, Redirection, Request, Timeout},
@@ -81,6 +85,35 @@ pub fn ferox_print(msg: &str, bar: &ProgressBar) {
    }
 }

+/// wrapper for make_request used to pass error/response codes to FeroxScans for per-scan stats
+/// tracking of information related to auto-tune/bail
+pub async fn logged_request(url: &Url, handles: Arc<Handles>) -> Result<Response> {
+    let client = &handles.config.client;
+    let level = handles.config.output_level;
+    let tx_stats = handles.stats.tx.clone();
+
+    let response = make_request(client, url, level, tx_stats).await;
+
+    let scans = handles.ferox_scans()?;
+
+    match response {
+        Ok(resp) => {
+            match resp.status() {
+                StatusCode::TOO_MANY_REQUESTS | StatusCode::FORBIDDEN => {
+                    scans.increment_status_code(url.as_str(), resp.status());
+                }
+                _ => {}
+            }
+            Ok(resp)
+        }
+        Err(e) => {
+            log::warn!("err: {:?}", e);
+            scans.increment_error(url.as_str());
+            bail!(e)
+        }
+    }
+}
+
 /// Initiate request to the given `Url` using `Client`
 pub async fn make_request(
    client: &Client,
--- a/tests/policy-test-words.shuffled
+++ b/tests/policy-test-words.shuffled
--- a/tests/test_banner.rs
+++ b/tests/test_banner.rs
@@ -849,6 +849,58 @@ fn banner_prints_rate_limit() {
        );
 }

+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + auto tune
+fn banner_prints_auto_tune() {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("--auto-tune")
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Auto Tune"))
+                .and(predicate::str::contains("│ true"))
+                .and(predicate::str::contains("─┴─")),
+        );
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + auto bail
+fn banner_prints_auto_bail() {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("--auto-bail")
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Auto Bail"))
+                .and(predicate::str::contains("│ true"))
+                .and(predicate::str::contains("─┴─")),
+        );
+}
+
 #[test]
 /// test allows non-existent wordlist to trigger the banner printing to stderr
 /// expect to see no banner output
@@ -896,3 +948,27 @@ fn banner_doesnt_print_when_quiet() {
                .and(predicate::str::contains("User-Agent").not()),
        );
 }
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see nothing as --parallel forces --silent to be true
+fn banner_prints_parallel() {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--stdin")
+        .arg("--parallel")
+        .arg("4316")
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .not()
+                .and(predicate::str::contains("Target Url").not())
+                .and(predicate::str::contains("Parallel Scans").not())
+                .and(predicate::str::contains("Threads").not())
+                .and(predicate::str::contains("Wordlist").not())
+                .and(predicate::str::contains("Status Codes").not())
+                .and(predicate::str::contains("Timeout (secs)").not())
+                .and(predicate::str::contains("User-Agent").not()),
+        );
+}
--- a/tests/test_main.rs
+++ b/tests/test_main.rs
@@ -1,8 +1,9 @@
 mod utils;
 use assert_cmd::Command;
 use httpmock::Method::GET;
-use httpmock::MockServer;
+use httpmock::{MockServer, Regex};
 use predicates::prelude::*;
+use std::fs::read_to_string;
 use utils::{setup_tmp_directory, teardown_tmp_directory};

 #[test]
@@ -89,3 +90,66 @@ fn main_use_empty_stdin_targets() -> Result<(), Box<dyn std::error::Error>> {

    Ok(())
 }
+
+#[test]
+/// send three targets over stdin, expect parallel to spawn children and each child config to show
+/// up in the output file
+fn main_parallel_spawns_children() -> Result<(), Box<dyn std::error::Error>> {
+    let t1 = MockServer::start();
+    let t2 = MockServer::start();
+    let t3 = MockServer::start();
+
+    let words = [
+        String::from("LICENSE"),
+        String::from("stuff"),
+        String::from("things"),
+        String::from("mostuff"),
+        String::from("mothings"),
+    ];
+    let (word_tmp_dir, wordlist) = setup_tmp_directory(&words, "wordlist")?;
+    let (output_dir, outfile) = setup_tmp_directory(&[], "output-file")?;
+    let (tgt_tmp_dir, targets) =
+        setup_tmp_directory(&[t1.url("/"), t2.url("/"), t3.url("/")], "targets")?;
+
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--stdin")
+        .arg("--parallel")
+        .arg("2")
+        .arg("-vvvv")
+        .arg("--debug-log")
+        .arg(outfile.as_os_str())
+        .arg("--wordlist")
+        .arg(wordlist.as_os_str())
+        .pipe_stdin(targets)
+        .unwrap()
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("Could not connect to any target provided")
+                .and(predicate::str::contains("Target Url"))
+                .not(), // no target url found
+        );
+
+    let contents = read_to_string(outfile).unwrap();
+    println!("contents: {}", contents);
+
+    assert!(contents.contains("parallel branch && wrapped main")); // exits parallel branch
+
+    // DBG      0.007 feroxbuster parallel exec: target/debug/feroxbuster
+    //   --debug-log /tmp/.tmpAjRts6/output-file --wordlist /tmp/.tmpS4CKKq/wordlist
+    //   --silent -u http://127.0.0.1:41979/
+    let r1 = Regex::new(&format!("parallel exec:.*-u {}", t1.url("/"))).unwrap();
+    let r2 = Regex::new(&format!("parallel exec:.*-u {}", t2.url("/"))).unwrap();
+    let r3 = Regex::new(&format!("parallel exec:.*-u {}", t3.url("/"))).unwrap();
+
+    assert!(r1.is_match(&contents)); // all 3 were spawned
+    assert!(r2.is_match(&contents));
+    assert!(r3.is_match(&contents));
+
+    teardown_tmp_directory(word_tmp_dir);
+    teardown_tmp_directory(tgt_tmp_dir);
+    teardown_tmp_directory(output_dir);
+
+    Ok(())
+}
--- a/tests/test_policies.rs
+++ b/tests/test_policies.rs
@@ -0,0 +1,432 @@
+mod utils;
+use assert_cmd::prelude::*;
+use httpmock::Method::GET;
+use httpmock::MockServer;
+use regex::Regex;
+use std::fs::{read_to_string, write};
+use std::path::Path;
+use std::process::Command;
+use std::time::Instant;
+use tokio::time::Duration;
+use utils::{setup_tmp_directory, teardown_tmp_directory};
+
+// tests/policy-test-error-words is a wordlist with the following attributes:
+// - 60 errors per error category (error, 403, 429)
+// - 1000 words tagged as normal for noise/padding
+// - each error string is 6_RANDOM_ASCII{error,status403,status429,normal}6_RANDOM_ASCII
+// examples:
+// - BKPMiherrortBPKcw
+// - lTjbLpstatus403fZQaFD
+// - ZhGBHGstatus429SIUZvI
+// - ufzEXWnormalOLhbLM
+// these words will be used along with pattern matching to trigger different policies
+
+#[test]
+/// --auto-bail should cancel a scan with spurious errors  
+fn auto_bail_cancels_scan_with_timeouts() {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["ignored".to_string()], "wordlist").unwrap();
+    let (log_dir, logfile) = setup_tmp_directory(&[], "debug-log").unwrap();
+
+    let policy_words = read_to_string(Path::new("tests/policy-test-words.shuffled")).unwrap();
+
+    write(&file, policy_words).unwrap();
+
+    assert_eq!(file.metadata().unwrap().len(), 117720); // sanity check on wordlist size
+
+    let error_mock = srv.mock(|when, then| {
+        when.method(GET)
+            .path_matches(Regex::new("/[a-zA-Z]{6}error[a-zA-Z]{6}").unwrap());
+        then.delay(Duration::new(3, 0))
+            .status(200)
+            .body("verboten, nerd");
+    });
+
+    let other_errors_mock = srv.mock(|when, then| {
+        when.method(GET)
+            .path_matches(Regex::new("/[a-zA-Z]{6}(status429|status403)[a-zA-Z]{6}").unwrap());
+        then.status(200).body("other errors are a 200");
+    });
+
+    let normal_reqs_mock = srv.mock(|when, then| {
+        when.method(GET)
+            .path_matches(Regex::new("/[a-zA-Z]{6}normal[a-zA-Z]{6}").unwrap());
+        then.status(200).body("any normal request is a 200");
+    });
+
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--auto-bail")
+        .arg("--dont-filter")
+        .arg("--timeout")
+        .arg("2")
+        .arg("--threads")
+        .arg("4")
+        .arg("--debug-log")
+        .arg(logfile.as_os_str())
+        .arg("-vvvv")
+        .arg("--json")
+        .assert()
+        .success();
+
+    let debug_log = read_to_string(logfile).unwrap();
+
+    // read debug log to get the number of errors enforced
+    for line in debug_log.lines() {
+        let log: serde_json::Value = serde_json::from_str(&line).unwrap_or_default();
+        if let Some(message) = log.get("message") {
+            let str_msg = message.as_str().unwrap_or_default().to_string();
+
+            if str_msg.starts_with("Stats") {
+                let re = Regex::new("total_expected: ([0-9]+),").unwrap();
+                assert!(re.is_match(&str_msg));
+                let total_expected = re
+                    .captures(&str_msg)
+                    .unwrap()
+                    .get(1)
+                    .map_or("", |m| m.as_str())
+                    .parse::<usize>()
+                    .unwrap();
+
+                println!("expected: {}", total_expected);
+                // without bailing, should be 6180; after bail decreases significantly
+                assert!(total_expected < 5000);
+            }
+        }
+    }
+
+    teardown_tmp_directory(tmp_dir);
+    teardown_tmp_directory(log_dir);
+
+    assert!(normal_reqs_mock.hits() < 6000); // not all requests should make it
+    assert!(error_mock.hits() >= 25); // need at least 25 to trigger the policy
+    assert!(other_errors_mock.hits() <= 120); // may or may not see all other error requests
+}
+
+#[test]
+/// --auto-bail should cancel a scan with spurious 403s
+fn auto_bail_cancels_scan_with_403s() {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["ignored".to_string()], "wordlist").unwrap();
+    let (log_dir, logfile) = setup_tmp_directory(&[], "debug-log").unwrap();
+
+    let policy_words = read_to_string(Path::new("tests/policy-test-words.shuffled")).unwrap();
+
+    write(&file, policy_words).unwrap();
+
+    assert_eq!(file.metadata().unwrap().len(), 117720); // sanity check on wordlist size
+
+    let error_mock = srv.mock(|when, then| {
+        when.method(GET).path_matches(
+            Regex::new("/[a-zA-Z]{6}(error|status429|status403)[a-zA-Z]{6}").unwrap(),
+        );
+        then.status(200).body("other errors are still a 200");
+    });
+
+    let normal_reqs_mock = srv.mock(|when, then| {
+        when.method(GET)
+            .path_matches(Regex::new("/[a-zA-Z]{6}normal[a-zA-Z]{6}").unwrap());
+        then.status(403)
+            .body("these guys need to be 403 in order to trigger 90% threshold");
+    });
+
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--auto-bail")
+        .arg("--dont-filter")
+        .arg("--threads")
+        .arg("4")
+        .arg("--debug-log")
+        .arg(logfile.as_os_str())
+        .arg("-vvvv")
+        .arg("--json")
+        .assert()
+        .success();
+
+    println!("log filesize: {}", logfile.metadata().unwrap().len());
+    let debug_log = read_to_string(logfile).unwrap();
+
+    // read debug log to get the number of errors enforced
+    for line in debug_log.lines() {
+        let log: serde_json::Value = serde_json::from_str(&line).unwrap_or_default();
+        if let Some(message) = log.get("message") {
+            let str_msg = message.as_str().unwrap_or_default().to_string();
+
+            if str_msg.starts_with("Stats") {
+                println!("{}", str_msg);
+                let re = Regex::new("total_expected: ([0-9]+),").unwrap();
+                assert!(re.is_match(&str_msg));
+                let total_expected = re
+                    .captures(&str_msg)
+                    .unwrap()
+                    .get(1)
+                    .map_or("", |m| m.as_str())
+                    .parse::<usize>()
+                    .unwrap();
+                println!("total_expected: {}", total_expected);
+                assert!(total_expected < 5000);
+            }
+        }
+    }
+
+    teardown_tmp_directory(tmp_dir);
+    teardown_tmp_directory(log_dir);
+
+    assert!(normal_reqs_mock.hits() + error_mock.hits() > 25); // must have at least 50 reqs fly
+
+    // expect much less in the way of requests for this one, 90% is measured against requests made,
+    // not requests expected, so 90% can be reached very quickly. for the same reason, the
+    // num_enforced can be less than 50
+    assert!(normal_reqs_mock.hits() < 500);
+    assert!(error_mock.hits() <= 180); // may or may not see all other error requests
+}
+
+#[test]
+/// --auto-bail should cancel a scan with spurious 429s
+fn auto_bail_cancels_scan_with_429s() {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["ignored".to_string()], "wordlist").unwrap();
+    let (log_dir, logfile) = setup_tmp_directory(&[], "debug-log").unwrap();
+
+    let policy_words = read_to_string(Path::new("tests/policy-test-words.shuffled")).unwrap();
+
+    write(&file, policy_words).unwrap();
+
+    assert_eq!(file.metadata().unwrap().len(), 117720); // sanity check on wordlist size
+
+    let error_mock = srv.mock(|when, then| {
+        when.method(GET).path_matches(
+            Regex::new("/[a-zA-Z]{6}(error|status429|status403)[a-zA-Z]{6}").unwrap(),
+        );
+        then.status(200).body("other errors are still a 200");
+    });
+
+    let normal_reqs_mock = srv.mock(|when, then| {
+        when.method(GET)
+            .path_matches(Regex::new("/[a-zA-Z]{6}normal[a-zA-Z]{6}").unwrap());
+        then.status(429)
+            .body("these guys need to be 403 in order to trigger 90% threshold");
+    });
+
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--auto-bail")
+        .arg("--dont-filter")
+        .arg("--threads")
+        .arg("4")
+        .arg("--debug-log")
+        .arg(logfile.as_os_str())
+        .arg("-vvvv")
+        .arg("--json")
+        .assert()
+        .success();
+
+    println!("log filesize: {}", logfile.metadata().unwrap().len());
+    let debug_log = read_to_string(logfile).unwrap();
+
+    // read debug log to get the number of errors enforced
+    for line in debug_log.lines() {
+        let log: serde_json::Value = serde_json::from_str(&line).unwrap_or_default();
+        if let Some(message) = log.get("message") {
+            let str_msg = message.as_str().unwrap_or_default().to_string();
+
+            if str_msg.starts_with("Stats") {
+                println!("{}", str_msg);
+                let re = Regex::new("total_expected: ([0-9]+),").unwrap();
+                assert!(re.is_match(&str_msg));
+                let total_expected = re
+                    .captures(&str_msg)
+                    .unwrap()
+                    .get(1)
+                    .map_or("", |m| m.as_str())
+                    .parse::<usize>()
+                    .unwrap();
+                println!("total_expected: {}", total_expected);
+                assert!(total_expected < 5000);
+            }
+        }
+    }
+
+    teardown_tmp_directory(tmp_dir);
+    teardown_tmp_directory(log_dir);
+
+    assert!(normal_reqs_mock.hits() + error_mock.hits() > 25); // must have at least 50 reqs fly
+
+    // expect much less in the way of requests for this one, 90% is measured against requests made,
+    // not requests expected, so 90% can be reached very quickly. for the same reason, the
+    // num_enforced can be less than 50
+    assert!(normal_reqs_mock.hits() < 500);
+    assert!(error_mock.hits() <= 180); // may or may not see all other error requests
+}
+
+#[test]
+/// --auto-tune should slow a scan with spurious 429s
+fn auto_tune_slows_scan_with_429s() {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["ignored".to_string()], "wordlist").unwrap();
+
+    let policy_words = read_to_string(Path::new("tests/policy-test-words.shuffled")).unwrap();
+
+    write(&file, policy_words).unwrap();
+
+    assert_eq!(file.metadata().unwrap().len(), 117720); // sanity check on wordlist size
+
+    let error_mock = srv.mock(|when, then| {
+        when.method(GET).path_matches(
+            Regex::new("/[a-zA-Z]{6}(error|status429|status403)[a-zA-Z]{6}").unwrap(),
+        );
+        then.status(200).body("other errors are still a 200");
+    });
+
+    let normal_reqs_mock = srv.mock(|when, then| {
+        when.method(GET)
+            .path_matches(Regex::new("/[a-zA-Z]{6}normal[a-zA-Z]{6}").unwrap());
+        then.status(429)
+            .body("these guys need to be 429 in order to trigger 30% threshold");
+    });
+
+    let start = Instant::now();
+
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--auto-tune")
+        .arg("--dont-filter")
+        .arg("--time-limit")
+        .arg("7s")
+        .arg("--threads")
+        .arg("4")
+        .assert()
+        .failure();
+
+    teardown_tmp_directory(tmp_dir);
+
+    assert!(normal_reqs_mock.hits() + error_mock.hits() > 25); // must have at least 50 reqs fly
+
+    println!("elapsed: {}", start.elapsed().as_millis()); // 3523ms without tuning
+    assert!(normal_reqs_mock.hits() < 500);
+    assert!(error_mock.hits() <= 180); // may or may not see all other error requests
+    assert!(start.elapsed().as_millis() >= 7000); // scan should hit time limit due to limiting
+}
+
+#[test]
+/// --auto-tune should slow a scan with spurious 403s
+fn auto_tune_slows_scan_with_403s() {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["ignored".to_string()], "wordlist").unwrap();
+
+    let policy_words = read_to_string(Path::new("tests/policy-test-words.shuffled")).unwrap();
+
+    write(&file, policy_words).unwrap();
+
+    assert_eq!(file.metadata().unwrap().len(), 117720); // sanity check on wordlist size
+
+    let error_mock = srv.mock(|when, then| {
+        when.method(GET).path_matches(
+            Regex::new("/[a-zA-Z]{6}(error|status429|status403)[a-zA-Z]{6}").unwrap(),
+        );
+        then.status(200).body("other errors are still a 200");
+    });
+
+    let normal_reqs_mock = srv.mock(|when, then| {
+        when.method(GET)
+            .path_matches(Regex::new("/[a-zA-Z]{6}normal[a-zA-Z]{6}").unwrap());
+        then.status(403)
+            .body("these guys need to be 403 in order to trigger 90% threshold");
+    });
+
+    let start = Instant::now();
+
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--auto-tune")
+        .arg("--dont-filter")
+        .arg("--time-limit")
+        .arg("7s")
+        .arg("--threads")
+        .arg("4")
+        .assert()
+        .failure();
+
+    teardown_tmp_directory(tmp_dir);
+
+    assert!(normal_reqs_mock.hits() + error_mock.hits() > 25); // must have at least 50 reqs fly
+
+    println!("elapsed: {}", start.elapsed().as_millis()); // 3523ms without tuning
+    assert!(normal_reqs_mock.hits() < 500);
+    assert!(error_mock.hits() <= 180); // may or may not see all other error requests
+    assert!(start.elapsed().as_millis() >= 7000); // scan should hit time limit due to limiting
+}
+
+#[test]
+/// --auto-tune should slow a scan with spurious errors
+fn auto_tune_slows_scan_with_general_errors() {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["ignored".to_string()], "wordlist").unwrap();
+
+    let policy_words = read_to_string(Path::new("tests/policy-test-words.shuffled")).unwrap();
+
+    write(&file, policy_words).unwrap();
+
+    assert_eq!(file.metadata().unwrap().len(), 117720); // sanity check on wordlist size
+
+    let error_mock = srv.mock(|when, then| {
+        when.method(GET).path_matches(
+            Regex::new("/[a-zA-Z]{6}(error|status429|status403)[a-zA-Z]{6}").unwrap(),
+        );
+        then.status(200).body("other errors are still a 200");
+    });
+
+    let normal_reqs_mock = srv.mock(|when, then| {
+        when.method(GET)
+            .path_matches(Regex::new("/[a-zA-Z]{6}normal[a-zA-Z]{6}").unwrap());
+        then.status(200)
+            .body("these guys need to be 429 in order to trigger 30% threshold")
+            .delay(Duration::new(3, 0));
+    });
+
+    let start = Instant::now();
+
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--auto-tune")
+        .arg("--dont-filter")
+        .arg("--time-limit")
+        .arg("7s")
+        .arg("--threads")
+        .arg("4")
+        .arg("--timeout")
+        .arg("2")
+        .assert()
+        .failure();
+
+    teardown_tmp_directory(tmp_dir);
+
+    println!("elapsed: {}", start.elapsed().as_millis()); // 3523ms without tuning
+    assert!(normal_reqs_mock.hits() < 500);
+    assert!(error_mock.hits() <= 180); // may or may not see all other error requests
+    assert!(start.elapsed().as_millis() >= 7000); // scan should hit time limit due to limiting
+}
--- a/tests/test_scanner.rs
+++ b/tests/test_scanner.rs
@@ -496,7 +496,10 @@ fn scanner_single_request_scan_with_debug_logging_as_json() {
    assert!(contents.contains("\"level\":\"DEBUG\""));
    assert!(contents.contains("\"level\":\"INFO\""));
    assert!(contents.contains("time_offset"));
-    assert!(contents.contains("\"module\":\"feroxbuster::scanner\""));
+    assert!(contents.contains("exit: main"));
+    assert!(contents.contains(&srv.url("/LICENSE")));
+    assert!(contents.contains("\"module\":\"feroxbuster::response\""));
+    assert!(contents.contains("\"module\":\"feroxbuster::url\""));
    assert!(contents.contains("\"module\":\"feroxbuster::event_handlers::inputs\""));
    assert!(contents.contains("exit: start_enter_handler"));
    assert!(contents.contains("All scans complete!"));
Author	SHA1	Message	Date
epi	030b588448	Merge pull request #222 from epi052/213-add-parallel-option add --parallel option	2021-02-18 11:37:36 -06:00
epi	4ee143968e	updated readme with parallel option	2021-02-18 11:26:15 -06:00
epi	834d681bb9	updated readme with parallel option	2021-02-18 11:25:55 -06:00
epi	fc35bb6764	improved parallel testing	2021-02-18 11:01:11 -06:00
epi	13222bfc7b	added Debug impl for LimitHeap	2021-02-18 09:42:39 -06:00
epi	8e2b08ce90	bumped version to 2.2.0	2021-02-18 09:22:55 -06:00
epi	24a44ff253	Merge branch 'main' into 213-add-parallel-option	2021-02-18 09:21:50 -06:00
epi	9e0118fd30	Merge pull request #221 from epi052/123-auto-tune-or-bail added --auto-tune and --auto-bail	2021-02-18 09:16:07 -06:00
epi	3325af2331	updated branch monitoring for CI builds from master to main	2021-02-18 09:15:13 -06:00
epi	ec102a8093	updated readme	2021-02-18 09:11:57 -06:00
epi	9d72109023	added integration tests for auto-tune policy	2021-02-18 08:15:50 -06:00
epi	f1d6f3d8cb	broke utils out into separate files	2021-02-18 07:48:06 -06:00
epi	1e01be712a	added another message test; fixed clippy	2021-02-18 07:13:18 -06:00
epi	1a0c914819	added tests for feroxmessage	2021-02-18 07:09:06 -06:00
epi	19d3f46428	unit tests for scanner/utils are complete	2021-02-18 07:01:36 -06:00
epi	6e2e3ff97f	added tests for adjust_limit on the Requester	2021-02-17 20:46:00 -06:00
epi	303eed03d7	finished up tests for limitheap	2021-02-17 17:07:26 -06:00
epi	a0754d2e3a	finished policy data tests	2021-02-17 17:01:47 -06:00
epi	3d4417d84b	added some tests for policy data	2021-02-17 15:34:52 -06:00
epi	6f5de57115	added test for requests_per_second	2021-02-17 15:04:05 -06:00
epi	7e72d52e4a	removed GetRuntime dead code	2021-02-17 14:36:10 -06:00
epi	7010b00b00	added tests for stats container	2021-02-17 14:34:46 -06:00
epi	3de31f0393	removed all enforced_ dead code	2021-02-17 13:27:50 -06:00
epi	06fe34f291	fixed all existing tests	2021-02-17 12:59:38 -06:00
epi	d78dbb76b1	removed todos related to tuning	2021-02-17 12:23:49 -06:00
epi	a09493b845	Merge branch 'main' into 123-auto-tune-or-bail	2021-02-17 08:30:58 -06:00
epi	3cb5a9b8fa	incremental save	2021-02-17 07:03:04 -06:00
epi	7ad8915d96	updated lockfile	2021-02-15 08:16:14 -06:00
epi	23ec79d897	Merge branch 'master' into 123-auto-tune-or-bail	2021-02-15 08:13:14 -06:00
epi	c4f072e159	incremental save before branch swap	2021-02-15 07:37:50 -06:00
epi	4019c31f9d	auto-tune and rate-limit are mutually exclusive	2021-02-15 07:00:25 -06:00
epi	5cb5541eda	changed memory ordering of feroxscan errors	2021-02-15 06:58:14 -06:00
epi	71084979f3	remvoed clippy ci errors	2021-02-15 06:48:05 -06:00
epi	96527a1419	fixed clippy error from pipeline	2021-02-15 06:35:50 -06:00
epi	4e0a85e64f	removed lint	2021-02-13 20:11:02 -06:00
epi	ed5e1d86cd	bumped env_logger to 0.8.3	2021-02-13 20:08:03 -06:00
epi	d8b15da016	added autobail tests for 403/429s	2021-02-13 20:05:33 -06:00
epi	54e290106d	added test for autobail; fixed lock contention bug	2021-02-13 19:44:19 -06:00
epi	161f8f0aed	added a few tests to scanner/utils	2021-02-13 06:05:16 -06:00
epi	c9e2d302be	autobail mostly complete	2021-02-13 05:43:06 -06:00
epi	bd4f6024c6	another test fix	2021-02-12 07:00:42 -06:00
epi	15de46da7b	fixed existing tests	2021-02-12 06:26:45 -06:00
epi	4e3b8701a2	incremental save	2021-02-11 20:20:40 -06:00
epi	dabcedcf23	added test for get_base_scan_by_url	2021-02-10 05:39:11 -06:00
epi	52a2a1f961	errors incrementing per-scan properly	2021-02-09 20:17:28 -06:00
epi	0345e03e6a	added test for --parallel	2021-02-08 06:44:00 -06:00
epi	873539ac92	fixed up existing tests	2021-02-08 06:16:23 -06:00
epi	9c85f90faf	bumped version to 2.1.0; bumped tokio & serde_json to new versions	2021-02-08 06:02:36 -06:00
epi	1643643e77	more nitpickery in main	2021-02-08 05:58:34 -06:00
epi	a7e4cc914b	updated example config	2021-02-08 05:51:53 -06:00
epi	6daa2a230a	reverted ci change	2021-02-08 05:49:45 -06:00
epi	5486e3c95f	cleaned up main	2021-02-08 05:48:43 -06:00
epi	204aa5e226	implemented --parallel logic; banner/config logic/tests added	2021-02-07 14:35:38 -06:00
epi	e2dd01fb95	incremental save	2021-02-06 20:23:27 -06:00
epi	0ebbd89778	updated example config with new options	2021-02-05 05:20:02 -06:00
epi	c8c2f7b4c8	added banner entries for auto-tune/bail	2021-02-04 20:45:34 -06:00
epi	ac75c01fed	added banner entries and tests for auto[bail,tune]	2021-02-04 20:32:12 -06:00
epi	a823c6040a	added auto-tune and auto-bail to config	2021-02-04 20:24:02 -06:00
epi	05589f3988	broke scanner into sub module	2021-02-04 19:20:31 -06:00
epi	5b8b3f148b	bumped version to 2.1.0	2021-02-04 17:14:37 -06:00