added info log for collect words

src/scanner/ferox_scanner.rs

@@ -279,9 +279,10 @@ impl FeroxScanner {
 
         if self.handles.config.collect_words {
             let new_words = TF_IDF.read().unwrap().all_words();
+            let new_words_len = new_words.len();
 
             let cur_length = progress_bar.length();
-            let new_length = cur_length + new_words.len() as u64;
+            let new_length = cur_length + new_words_len as u64;
 
             progress_bar.set_length(new_length);
 
@@ -290,6 +291,12 @@ impl FeroxScanner {
                 .send(AddToUsizeField(TotalExpected, new_words.len()))
                 .unwrap_or_default();
 
+            log::info!(
+                "requesting {} collected words: {:?}...",
+                new_words_len,
+                &new_words[..new_words_len.min(3) as usize]
+            );
+
             self.stream_requests(
                 Arc::new(new_words),
                 progress_bar.clone(),

src/scanner/requester.rs

@@ -405,7 +405,6 @@ impl Requester {
                 }
 
                 if self.handles.config.collect_words {
-                    // todo think about before/after filtering, similar to recursion
                     if let Ok(mut guard) = TF_IDF.write() {
                         let doc = Document::from_html(ferox_response.text());
                         guard.add_document(doc);