diff --git a/src/reader.rs b/src/reader.rs
index 786b04cfd..7e6a65752 100644
--- a/src/reader.rs
+++ b/src/reader.rs
@@ -2982,7 +2982,7 @@ fn handle_readline_command(&mut self, c: ReadlineCmd) {
 
                 let was_active_before = self.history_search.active();
 
-                if self.history_search.is_at_present() {
+                if self.history_search.is_at_present() && mode != self.history_search.mode() {
                     let el = &self.data.command_line;
                     if mode == SearchMode::Token {
                         // Searching by token.
@@ -3331,15 +3331,13 @@ fn handle_readline_command(&mut self, c: ReadlineCmd) {
                         },
                         false,
                     );
-                } else {
+                } else if self.history_search.active() {
                     if up {
                         self.history_search.go_to_oldest();
                     } else {
                         self.history_search.go_to_present();
                     }
-                    if self.history_search.active() {
-                        self.update_command_line_from_history_search();
-                    }
+                    self.update_command_line_from_history_search();
                 }
             }
             rl::UpLine | rl::DownLine => {
diff --git a/src/reader_history_search.rs b/src/reader_history_search.rs
index 1637a1e58..7c7acec2d 100644
--- a/src/reader_history_search.rs
+++ b/src/reader_history_search.rs
@@ -79,6 +79,9 @@ pub fn by_line(&self) -> bool {
     pub fn by_prefix(&self) -> bool {
         self.mode == SearchMode::Prefix
     }
+    pub fn mode(&self) -> SearchMode {
+        self.mode
+    }
 
     /// Move the history search in the given direction `dir`.
     pub fn move_in_direction(&mut self, dir: SearchDirection) -> bool {