diff --git a/parse_constants.h b/parse_constants.h
index e3eebbf5d..a322073e7 100644
--- a/parse_constants.h
+++ b/parse_constants.h
@@ -188,14 +188,13 @@ typedef unsigned int parser_test_error_bits_t;
 */
 #define INVALID_CASE_ERR_MSG _( L"'case' builtin not inside of switch block")
 
-/**
-   Error when using loop control builtins (break or continue) outside of loop
-*/
-#define INVALID_LOOP_ERR_MSG _( L"Loop control command while not inside of loop" )
+/** Error when using break outside of loop */
+#define INVALID_BREAK_ERR_MSG _( L"break command while not inside of loop" )
 
-/**
-   Error when using return builtin outside of function definition
-*/
+/** Error when using continue outside of loop */
+#define INVALID_CONTINUE_ERR_MSG _( L"continue command while not inside of loop" )
+
+/** Error when using return builtin outside of function definition */
 #define INVALID_RETURN_ERR_MSG _( L"'return' builtin command outside of function definition" )
 
 /**
diff --git a/parse_tree.cpp b/parse_tree.cpp
index ad83a0d60..81b6c1633 100644
--- a/parse_tree.cpp
+++ b/parse_tree.cpp
@@ -12,10 +12,10 @@ static bool production_is_empty(const production_t *production)
 }
 
 /** Returns a string description of this parse error */
-wcstring parse_error_t::describe(const wcstring &src) const
+wcstring parse_error_t::describe(const wcstring &src, bool skip_caret) const
 {
     wcstring result = text;
-    if (source_start < src.size() && source_start + source_length <= src.size())
+    if (! skip_caret && source_start < src.size() && source_start + source_length <= src.size())
     {
         // Locate the beginning of this line of source
         size_t line_start = 0;
diff --git a/parse_tree.h b/parse_tree.h
index 8a0b3eedd..bdc622d93 100644
--- a/parse_tree.h
+++ b/parse_tree.h
@@ -33,8 +33,8 @@ struct parse_error_t
     size_t source_start;
     size_t source_length;
 
-    /** Return a string describing the error, suitable for presentation to the user */
-    wcstring describe(const wcstring &src) const;
+    /** Return a string describing the error, suitable for presentation to the user. If skip_caret is false, the offending line with a caret is printed as well */
+    wcstring describe(const wcstring &src, bool skip_caret = false) const;
 };
 typedef std::vector<parse_error_t> parse_error_list_t;
 
diff --git a/parse_util.cpp b/parse_util.cpp
index f95679591..ff7a022c2 100644
--- a/parse_util.cpp
+++ b/parse_util.cpp
@@ -1099,7 +1099,7 @@ parser_test_error_bits_t parse_util_detect_errors(const wcstring &buff_src, pars
                         
                         if (! found_loop && ! first_argument_is_help(node_tree, node, buff_src))
                         {
-                            errored = append_syntax_error(&parse_errors, node, INVALID_LOOP_ERR_MSG);
+                            errored = append_syntax_error(&parse_errors, node, (command == L"break" ? INVALID_BREAK_ERR_MSG : INVALID_CONTINUE_ERR_MSG));
                         }
                     }
                 }
diff --git a/parser.cpp b/parser.cpp
index ee6948a85..59d0f51bc 100644
--- a/parser.cpp
+++ b/parser.cpp
@@ -2922,10 +2922,13 @@ void parser_t::get_backtrace(const wcstring &src, const parse_error_list_t &erro
         }
         else
         {
-            append_format(*output, L"fish: %ls:", _(L"Error:"));
+            output->append(L"fish: ");
         }
         
-        output->append(err.describe(src));
+        // Don't include the caret if we're interactive, this is the first line of text, and our source is at its beginning, because then it's obvious
+        bool skip_caret = (get_is_interactive() && which_line == 1 && err.source_start == 0);
+        
+        output->append(err.describe(src, skip_caret));
         output->push_back(L'\n');
         
         this->stack_trace(current_block, *output);
diff --git a/reader.cpp b/reader.cpp
index 87c014642..1506ae3e9 100644
--- a/reader.cpp
+++ b/reader.cpp
@@ -2478,22 +2478,15 @@ int reader_shell_test(const wchar_t *b)
 
     if (res & PARSER_TEST_ERROR)
     {
-        const int tmp[1] = {0};
-        const int tmp2[1] = {0};
-        const wcstring empty;
-
-        s_write(&data->screen,
-                empty,
-                empty,
-                empty,
-                0,
-                tmp,
-                tmp2,
-                0);
-        
         wcstring sb;
         parser_t::principal_parser().get_backtrace(bstr, errors, &sb);
-        fwprintf(stderr, L"%ls", sb.c_str());
+        
+        // ensure we end with a newline. Also add an initial newline, because it's likely the user just hit enter and so there's junk on the current line
+        if (! string_suffixes_string(L"\n", sb))
+        {
+            sb.push_back(L'\n');
+        }
+        fwprintf(stderr, L"\n%ls", sb.c_str());
     }
     return res;
 }