Correctly highlight input following a tokenizer error

src/fish_tests.cpp

@@ -4626,6 +4626,8 @@ static void test_highlighting() {
     highlight_tests.push_back({
         {L"false", highlight_role_t::command},
         {L"|&", highlight_role_t::error},
+        {L"true", highlight_role_t::command},
+        {L"stuff", highlight_role_t::param},
     });
 
     auto &vars = parser_t::principal_parser().vars();

src/parse_tree.cpp

@@ -1052,10 +1052,9 @@ bool parse_tree_from_string(const wcstring &str, parse_tree_flags_t parse_flags,
     // Construct the tokenizer.
     tok_flags_t tok_options = 0;
     if (parse_flags & parse_flag_include_comments) tok_options |= TOK_SHOW_COMMENTS;
-
     if (parse_flags & parse_flag_accept_incomplete_tokens) tok_options |= TOK_ACCEPT_UNFINISHED;
-
     if (parse_flags & parse_flag_show_blank_lines) tok_options |= TOK_SHOW_BLANK_LINES;
+    if (parse_flags & parse_flag_continue_after_error) tok_options |= TOK_CONTINUE_AFTER_ERROR;
 
     tokenizer_t tok(str.c_str(), tok_options);
 

src/tokenizer.cpp

@@ -65,7 +65,14 @@ tok_t tokenizer_t::call_error(tokenizer_error_t error_type, const wchar_t *token
     assert(error_loc >= token_start && "Invalid error location");
     assert(this->buff >= token_start && "Invalid buff location");
 
-    this->has_next = false;
+    // If continue_after_error is set and we have a real token length, then skip past it.
+    // Otherwise give up.
+    if (token_length.has_value() && continue_after_error) {
+        assert(this->buff < error_loc + *token_length && "Unable to continue past error");
+        this->buff = error_loc + *token_length;
+    } else {
+        this->has_next = false;
+    }
 
     tok_t result{token_type_t::error};
     result.error = error_type;
@@ -82,6 +89,7 @@ tokenizer_t::tokenizer_t(const wchar_t *start, tok_flags_t flags) : buff(start),
     this->accept_unfinished = static_cast<bool>(flags & TOK_ACCEPT_UNFINISHED);
     this->show_comments = static_cast<bool>(flags & TOK_SHOW_COMMENTS);
     this->show_blank_lines = static_cast<bool>(flags & TOK_SHOW_BLANK_LINES);
+    this->continue_after_error = static_cast<bool>(flags & TOK_CONTINUE_AFTER_ERROR);
 }
 
 tok_t::tok_t(token_type_t type) : type(type) {}

src/tokenizer.h

@@ -41,6 +41,9 @@ enum class redirection_mode_t {
 /// the tokenizer to return each of them as a separate END.
 #define TOK_SHOW_BLANK_LINES 4
 
+/// Make an effort to continue after an error.
+#define TOK_CONTINUE_AFTER_ERROR 8
+
 typedef unsigned int tok_flags_t;
 
 enum class tokenizer_error_t {
@@ -104,6 +107,8 @@ class tokenizer_t {
     bool show_comments{false};
     /// Whether all blank lines are returned.
     bool show_blank_lines{false};
+    /// Whether to attempt to continue after an error.
+    bool continue_after_error{false};
     /// Whether to continue the previous line after the comment.
     bool continue_line_after_comment{false};