diff --git a/src/event_handlers/outputs.rs b/src/event_handlers/outputs.rs index d94cb90..7d70b7e 100644 --- a/src/event_handlers/outputs.rs +++ b/src/event_handlers/outputs.rs @@ -215,6 +215,7 @@ impl TermOutHandler { self.config.replay_client.as_ref().unwrap(), resp.url(), self.config.output_level, + &self.config, tx_stats.clone(), ) .await diff --git a/src/extractor/container.rs b/src/extractor/container.rs index d86ab60..4e84ea4 100644 --- a/src/extractor/container.rs +++ b/src/extractor/container.rs @@ -402,6 +402,7 @@ impl<'a> Extractor<'a> { &client, &url, self.handles.config.output_level, + &self.handles.config, self.handles.stats.tx.clone(), ) .await?; diff --git a/src/utils.rs b/src/utils.rs index a58662a..85ee145 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -5,7 +5,9 @@ use reqwest::{Client, Response, StatusCode, Url}; #[cfg(not(target_os = "windows"))] use rlimit::{getrlimit, setrlimit, Resource}; use std::{ + thread_local, fs, + cell::RefCell, io::{self, BufWriter, Write}, sync::Arc, time::Duration, @@ -21,9 +23,17 @@ use crate::{ }, progress::PROGRESS_PRINTER, send_command, + USER_AGENTS, statistics::StatError::{Connection, Other, Redirection, Request, Timeout}, traits::FeroxSerialize, }; +use crate::config::Configuration; + +thread_local! { + /// simple counter for grabbing 'random' user agents + static USER_AGENT_CTR: RefCell = RefCell::new(0); +} + /// Given the path to a file, open the file in append mode (create it if it doesn't exist) and /// return a reference to the buffered file @@ -94,7 +104,7 @@ pub async fn logged_request(url: &Url, handles: Arc) -> Result, ) -> Result { log::trace!( @@ -130,7 +141,18 @@ pub async fn make_request( tx_stats ); - match client.get(url.to_owned()).send().await { + let mut request = client.get(url.to_owned()); + + if config.random_agent { + let user_agent = USER_AGENT_CTR.with(|ua_ctr| { + let mut inner = ua_ctr.borrow_mut(); + *inner += 1; + USER_AGENTS[*inner % USER_AGENTS.len()] + }); + request = request.header("User-Agent", user_agent); + } + + match request.send().await { Err(e) => { log::trace!("exit: make_request -> {}", e);