diff --git a/src/scanner/ferox_scanner.rs b/src/scanner/ferox_scanner.rs index b610125..32790e6 100644 --- a/src/scanner/ferox_scanner.rs +++ b/src/scanner/ferox_scanner.rs @@ -279,9 +279,10 @@ impl FeroxScanner { if self.handles.config.collect_words { let new_words = TF_IDF.read().unwrap().all_words(); + let new_words_len = new_words.len(); let cur_length = progress_bar.length(); - let new_length = cur_length + new_words.len() as u64; + let new_length = cur_length + new_words_len as u64; progress_bar.set_length(new_length); @@ -290,6 +291,12 @@ impl FeroxScanner { .send(AddToUsizeField(TotalExpected, new_words.len())) .unwrap_or_default(); + log::info!( + "requesting {} collected words: {:?}...", + new_words_len, + &new_words[..new_words_len.min(3) as usize] + ); + self.stream_requests( Arc::new(new_words), progress_bar.clone(), diff --git a/src/scanner/requester.rs b/src/scanner/requester.rs index 2de81b6..2d01d97 100644 --- a/src/scanner/requester.rs +++ b/src/scanner/requester.rs @@ -405,7 +405,6 @@ impl Requester { } if self.handles.config.collect_words { - // todo think about before/after filtering, similar to recursion if let Ok(mut guard) = TF_IDF.write() { let doc = Document::from_html(ferox_response.text()); guard.add_document(doc);