From eeac3333df6b30bf5663beaf116bf607cc44f37b Mon Sep 17 00:00:00 2001
From: ridiculousfish <corydoras@ridiculousfish.com>
Date: Sun, 27 Oct 2019 16:08:49 -0700
Subject: [PATCH] Correctly highlight input following a tokenizer error

---
 src/fish_tests.cpp |  2 ++
 src/parse_tree.cpp |  3 +--
 src/tokenizer.cpp  | 10 +++++++++-
 src/tokenizer.h    |  5 +++++
 4 files changed, 17 insertions(+), 3 deletions(-)

diff --git a/src/fish_tests.cpp b/src/fish_tests.cpp
index 3b0526a53..d53eb3cb1 100644
--- a/src/fish_tests.cpp
+++ b/src/fish_tests.cpp
@@ -4626,6 +4626,8 @@ static void test_highlighting() {
     highlight_tests.push_back({
         {L"false", highlight_role_t::command},
         {L"|&", highlight_role_t::error},
+        {L"true", highlight_role_t::command},
+        {L"stuff", highlight_role_t::param},
     });
 
     auto &vars = parser_t::principal_parser().vars();
diff --git a/src/parse_tree.cpp b/src/parse_tree.cpp
index 269b36d60..a1070ee49 100644
--- a/src/parse_tree.cpp
+++ b/src/parse_tree.cpp
@@ -1052,10 +1052,9 @@ bool parse_tree_from_string(const wcstring &str, parse_tree_flags_t parse_flags,
     // Construct the tokenizer.
     tok_flags_t tok_options = 0;
     if (parse_flags & parse_flag_include_comments) tok_options |= TOK_SHOW_COMMENTS;
-
     if (parse_flags & parse_flag_accept_incomplete_tokens) tok_options |= TOK_ACCEPT_UNFINISHED;
-
     if (parse_flags & parse_flag_show_blank_lines) tok_options |= TOK_SHOW_BLANK_LINES;
+    if (parse_flags & parse_flag_continue_after_error) tok_options |= TOK_CONTINUE_AFTER_ERROR;
 
     tokenizer_t tok(str.c_str(), tok_options);
 
diff --git a/src/tokenizer.cpp b/src/tokenizer.cpp
index 39d54fc81..712d12ee0 100644
--- a/src/tokenizer.cpp
+++ b/src/tokenizer.cpp
@@ -65,7 +65,14 @@ tok_t tokenizer_t::call_error(tokenizer_error_t error_type, const wchar_t *token
     assert(error_loc >= token_start && "Invalid error location");
     assert(this->buff >= token_start && "Invalid buff location");
 
-    this->has_next = false;
+    // If continue_after_error is set and we have a real token length, then skip past it.
+    // Otherwise give up.
+    if (token_length.has_value() && continue_after_error) {
+        assert(this->buff < error_loc + *token_length && "Unable to continue past error");
+        this->buff = error_loc + *token_length;
+    } else {
+        this->has_next = false;
+    }
 
     tok_t result{token_type_t::error};
     result.error = error_type;
@@ -82,6 +89,7 @@ tokenizer_t::tokenizer_t(const wchar_t *start, tok_flags_t flags) : buff(start),
     this->accept_unfinished = static_cast<bool>(flags & TOK_ACCEPT_UNFINISHED);
     this->show_comments = static_cast<bool>(flags & TOK_SHOW_COMMENTS);
     this->show_blank_lines = static_cast<bool>(flags & TOK_SHOW_BLANK_LINES);
+    this->continue_after_error = static_cast<bool>(flags & TOK_CONTINUE_AFTER_ERROR);
 }
 
 tok_t::tok_t(token_type_t type) : type(type) {}
diff --git a/src/tokenizer.h b/src/tokenizer.h
index 1e97b893c..964e9afd6 100644
--- a/src/tokenizer.h
+++ b/src/tokenizer.h
@@ -41,6 +41,9 @@ enum class redirection_mode_t {
 /// the tokenizer to return each of them as a separate END.
 #define TOK_SHOW_BLANK_LINES 4
 
+/// Make an effort to continue after an error.
+#define TOK_CONTINUE_AFTER_ERROR 8
+
 typedef unsigned int tok_flags_t;
 
 enum class tokenizer_error_t {
@@ -104,6 +107,8 @@ class tokenizer_t {
     bool show_comments{false};
     /// Whether all blank lines are returned.
     bool show_blank_lines{false};
+    /// Whether to attempt to continue after an error.
+    bool continue_after_error{false};
     /// Whether to continue the previous line after the comment.
     bool continue_line_after_comment{false};