From ab2cb03189b21363ad769a99aaba1f93969ba4f0 Mon Sep 17 00:00:00 2001
From: sgrj <seb.jambor@gmail.com>
Date: Sun, 27 Sep 2020 15:34:34 +0200
Subject: [PATCH] Consistency-fix for word motions (#7354)

* change word motion test to include start cursor in specification

* add test case for bug

* fix bug
---
 src/fish_tests.cpp | 26 +++++++++++++++-----------
 src/tokenizer.cpp  |  5 ++++-
 2 files changed, 19 insertions(+), 12 deletions(-)

diff --git a/src/fish_tests.cpp b/src/fish_tests.cpp
index 8ec7ef2c9..b8644e7bf 100644
--- a/src/fish_tests.cpp
+++ b/src/fish_tests.cpp
@@ -2479,12 +2479,13 @@ static void test_1_word_motion(word_motion_t motion, move_word_style_t style,
 
     size_t idx, end;
     if (motion == word_motion_left) {
-        idx = command.size();
+        idx = *std::max_element(stops.begin(), stops.end());
         end = 0;
     } else {
-        idx = 0;
+        idx = *std::min_element(stops.begin(), stops.end());
         end = command.size();
     }
+    stops.erase(idx);
 
     move_word_state_machine_t sm(style);
     while (idx != end) {
@@ -2521,22 +2522,25 @@ static void test_1_word_motion(word_motion_t motion, move_word_style_t style,
 /// Test word motion (forward-word, etc.). Carets represent cursor stops.
 static void test_word_motion() {
     say(L"Testing word motion");
-    test_1_word_motion(word_motion_left, move_word_style_punctuation, L"^echo ^hello_^world.^txt");
-    test_1_word_motion(word_motion_right, move_word_style_punctuation, L"echo^ hello^_world^.txt^");
+    test_1_word_motion(word_motion_left, move_word_style_punctuation, L"^echo ^hello_^world.^txt^");
+    test_1_word_motion(word_motion_right, move_word_style_punctuation, L"^echo^ hello^_world^.txt^");
 
     test_1_word_motion(word_motion_left, move_word_style_punctuation,
-                       L"echo ^foo_^foo_^foo/^/^/^/^/^    ");
+                       L"echo ^foo_^foo_^foo/^/^/^/^/^    ^");
     test_1_word_motion(word_motion_right, move_word_style_punctuation,
-                       L"echo^ foo^_foo^_foo^/^/^/^/^/    ^");
+                       L"^echo^ foo^_foo^_foo^/^/^/^/^/    ^");
 
-    test_1_word_motion(word_motion_left, move_word_style_path_components, L"^/^foo/^bar/^baz/");
-    test_1_word_motion(word_motion_left, move_word_style_path_components, L"^echo ^--foo ^--bar");
+    test_1_word_motion(word_motion_left, move_word_style_path_components, L"^/^foo/^bar/^baz/^");
+    test_1_word_motion(word_motion_left, move_word_style_path_components, L"^echo ^--foo ^--bar^");
     test_1_word_motion(word_motion_left, move_word_style_path_components,
-                       L"^echo ^hi ^> /^dev/^null");
+                       L"^echo ^hi ^> /^dev/^null^");
 
     test_1_word_motion(word_motion_left, move_word_style_path_components,
-                       L"^echo /^foo/^bar{^aaa,^bbb,^ccc}^bak/");
-    test_1_word_motion(word_motion_right, move_word_style_punctuation, L"a ^bcd^");
+                       L"^echo /^foo/^bar{^aaa,^bbb,^ccc}^bak/^");
+
+    test_1_word_motion(word_motion_right, move_word_style_punctuation, L"^a^ bcd^");
+    test_1_word_motion(word_motion_right, move_word_style_punctuation, L"a^b^ cde^");
+    test_1_word_motion(word_motion_right, move_word_style_punctuation, L"^ab^ cde^");
 }
 
 /// Test is_potential_path.
diff --git a/src/tokenizer.cpp b/src/tokenizer.cpp
index 161cbf9ec..298def8e9 100644
--- a/src/tokenizer.cpp
+++ b/src/tokenizer.cpp
@@ -687,8 +687,11 @@ bool move_word_state_machine_t::consume_char_punctuation(wchar_t c) {
                 consumed = true;
                 if (iswspace(c)) {
                     state = s_whitespace;
+                } else if (iswalnum(c)) {
+                    state = s_alphanumeric;
                 } else {
-                    // Don't allow switching type (ws->nonws) after non-whitespace.
+                    // Don't allow switching type (ws->nonws) after non-whitespace and
+                    // non-alphanumeric.
                     state = s_rest;
                 }
                 break;