diff --git a/tests/checks/eval.fish b/tests/checks/eval.fish
new file mode 100644
index 000000000..1ee6b4c1f
--- /dev/null
+++ b/tests/checks/eval.fish
@@ -0,0 +1,46 @@
+# RUN: %fish %s
+# Regression test for issue #4443
+eval set -l previously_undefined foo
+echo $previously_undefined
+# CHECK: foo
+
+# Test redirection
+eval "echo you can\\'t see this 1>&2" 2>/dev/null
+
+# Test return statuses
+false; eval true; echo $status
+# CHECK: 0
+false; eval false; echo $status
+# CHECK: 1
+
+# Test return status in case of parsing error
+false; eval "("; echo $status
+# CHECK: 1
+# CHECKERR: tests/checks/eval.fish (line {{\d+}}): Unexpected end of string, expecting ')'
+# CHECKERR: (
+# CHECKERR: ^
+false; eval '""'; echo $status
+# CHECK: 1
+# CHECKERR: tests/checks/eval.fish (line {{\d+}}): The expanded command was empty.
+# CHECKERR: ""
+# CHECKERR: ^
+
+function empty
+end
+
+# Regression tests for issue #5692
+false; eval;
+echo blank eval: $status # 0
+# CHECK: blank eval: 0
+
+false; eval "";
+echo empty arg eval: $status # 0
+# CHECK: empty arg eval: 0
+
+false; eval empty;
+echo empty function eval $status # 0
+# CHECK: empty function eval 0
+
+false; eval "begin; end;";
+echo empty block eval: $status # 0
+# CHECK: empty block eval: 0
diff --git a/tests/eval.err b/tests/eval.err
deleted file mode 100644
index 3e02f001c..000000000
--- a/tests/eval.err
+++ /dev/null
@@ -1,13 +0,0 @@
-
-####################
-# Testing eval builtin
-fish: Unexpected end of string, expecting ')'
-(
-^
-
-fish: The expanded command was empty.
-""
-^
-
-####################
-# Testing eval with empty functions, blocks, and arguments
diff --git a/tests/eval.in b/tests/eval.in
deleted file mode 100644
index 3008803d4..000000000
--- a/tests/eval.in
+++ /dev/null
@@ -1,33 +0,0 @@
-logmsg "Testing eval builtin"
-
-# Regression test for issue #4443
-eval set -l previously_undefined foo
-echo $previously_undefined
-
-# Test redirection
-eval "echo you can\\'t see this 1>&2" 2>/dev/null
-
-# Test return statuses
-false; eval true; echo $status # 0
-false; eval false; echo $status # 1
-
-# Test return status in case of parsing error
-false; eval "("; echo $status # 1
-false; eval '""'; echo $status # 1
-
-function empty
-end
-
-# Regression tests for issue #5692
-logmsg "Testing eval with empty functions, blocks, and arguments"
-false; eval;
-echo blank eval: $status # 0
-
-false; eval "";
-echo empty arg eval: $status # 0
-
-false; eval empty;
-echo empty function eval $status # 0
-
-false; eval "begin; end;";
-echo empty block eval: $status # 0
diff --git a/tests/eval.out b/tests/eval.out
deleted file mode 100644
index 6cba274e2..000000000
--- a/tests/eval.out
+++ /dev/null
@@ -1,15 +0,0 @@
-
-####################
-# Testing eval builtin
-foo
-0
-1
-1
-1
-
-####################
-# Testing eval with empty functions, blocks, and arguments
-blank eval: 0
-empty arg eval: 0
-empty function eval 0
-empty block eval: 0