From b5739ddacfef26d99858db0f37a814f6fb39781d Mon Sep 17 00:00:00 2001
From: Johannes Altmanninger <aclopte@gmail.com>
Date: Sat, 4 Dec 2021 14:41:54 +0100
Subject: [PATCH] Report sub-token error locations again

This fixes a regression about where we report errors:

	echo error(here
	old: ^
	   fixed: ^

Commit 0c22f67bd (Remove the old parser bits, 2020-07-02) removed
uses of "error_offset_within_token" so we always report errors at
token start. Add it back, hopefully restoring the 3.1.2 behavior.

Note that for cases like

	echo "$("

we report "unbalanced quotes" because we treat the $( as double
quote.  Giving a better error seems hard because of the ambguity -
we don't know if quote is meant to be inside or outside the command
substitution.
---
 src/ast.cpp                             | 10 ++++++++++
 tests/checks/syntax-error-location.fish | 10 ++++++++++
 2 files changed, 20 insertions(+)

diff --git a/src/ast.cpp b/src/ast.cpp
index 6e8c8357f..106b35939 100644
--- a/src/ast.cpp
+++ b/src/ast.cpp
@@ -182,6 +182,16 @@ class token_stream_t {
 
         assert(token.length <= SOURCE_OFFSET_INVALID);
         result.source_length = static_cast<source_offset_t>(token.length);
+
+        if (token.error != tokenizer_error_t::none) {
+            auto subtoken_offset = static_cast<source_offset_t>(token.error_offset_within_token);
+            // Skip invalid tokens that have a zero length, especially if they are at EOF.
+            if (subtoken_offset < result.source_length) {
+                result.source_start += subtoken_offset;
+                result.source_length -= subtoken_offset;
+            }
+        }
+
         return result;
     }
 
diff --git a/tests/checks/syntax-error-location.fish b/tests/checks/syntax-error-location.fish
index 22e281170..bf5c9a9ce 100644
--- a/tests/checks/syntax-error-location.fish
+++ b/tests/checks/syntax-error-location.fish
@@ -22,3 +22,13 @@ echo '
 # CHECK: <fish: Command substitutions not allowed>
 # CHECK: <(true one)>
 # CHECK: <^>
+
+$fish -c 'echo "unfinished "(subshell' 2>| string replace -r '.*' '<$0>'
+# CHECK: <fish: Unexpected end of string, expecting ')'>
+# CHECK: <echo "unfinished "(subshell>
+# CHECK: <                  ^>
+
+$fish -c 'echo "unfinished "$(subshell' 2>| string replace -r '.*' '<$0>'
+# CHECK: <fish: Unexpected end of string, expecting ')'>
+# CHECK: <echo "unfinished "$(subshell>
+# CHECK: <                   ^>