From 1f4d16cb07d62d5e2057ce2fa67d2c20aa8e31de Mon Sep 17 00:00:00 2001
From: Mahmoud Al-Qudsi <mqudsi@neosmart.net>
Date: Sun, 28 Mar 2021 13:59:14 -0500
Subject: [PATCH] Add support for bubbling up skipped tests to cmake

This prevents tests that were skipped (e.g. because of a missing
REQUIRES) from being reported as successes in the CTest overall run
results list.
---
 cmake/Tests.cmake |  7 +++++++
 tests/test.fish   | 33 +++++++++++++++++++++++----------
 2 files changed, 30 insertions(+), 10 deletions(-)

diff --git a/cmake/Tests.cmake b/cmake/Tests.cmake
index bc170dbd7..a86fcbd3c 100644
--- a/cmake/Tests.cmake
+++ b/cmake/Tests.cmake
@@ -8,6 +8,10 @@ if(NOT CTEST_PARALLEL_LEVEL)
   set(CTEST_PARALLEL_LEVEL ${CORES})
 endif()
 
+# We will use 125 as a reserved exit code to indicate that a test has been skipped, i.e. it did not
+# pass but it should not be considered a failed test run, either.
+set(SKIP_RETURN_CODE 125)
+
 # Even though we are using CMake's ctest for testing, we still define our own `make test` target
 # rather than use its default for many reasons:
 #  * CMake doesn't run tests in-proc or even add each tests as an individual node in the ninja
@@ -121,6 +125,7 @@ foreach(LTEST ${LOW_LEVEL_TESTS})
     COMMAND ${CMAKE_BINARY_DIR}/fish_tests ${LTEST}
     WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
   )
+  set_tests_properties(${LTEST} PROPERTIES SKIP_RETURN_CODE ${SKIP_RETURN_CODE})
   add_test_target("${LTEST}")
 endforeach(LTEST)
 
@@ -133,6 +138,7 @@ foreach(CHECK ${FISH_CHECKS})
                ${CMAKE_CURRENT_BINARY_DIR}/tests/test.fish ${CHECK}
     WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/tests
   )
+  set_tests_properties(${CHECK_NAME} PROPERTIES SKIP_RETURN_CODE ${SKIP_RETURN_CODE})
   add_test_target("${CHECK_NAME}")
 endforeach(CHECK)
 
@@ -144,5 +150,6 @@ foreach(PEXPECT ${PEXPECTS})
       ${CMAKE_CURRENT_BINARY_DIR}/tests/interactive.fish ${PEXPECT}
     WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/tests
   )
+  set_tests_properties(${PEXPECT} PROPERTIES SKIP_RETURN_CODE ${SKIP_RETURN_CODE})
   add_test_target("${PEXPECT}")
 endforeach(PEXPECT)
diff --git a/tests/test.fish b/tests/test.fish
index 5814f8413..0d699a622 100644
--- a/tests/test.fish
+++ b/tests/test.fish
@@ -19,27 +19,40 @@ else
     set files_to_test checks/*.fish
 end
 
-say -o cyan "Testing high level script functionality"
+# Be less verbose when running tests one-by-one
+if test (count $files_to_test) -gt 1
+    say -o cyan "Testing high level script functionality"
+end
 
 set -g python (__fish_anypython)
 
 # Test littlecheck files.
-set littlecheck_files (string match '*.fish' -- $files_to_test)
-if set -q littlecheck_files[1]
+set -l skipped 0
+set -l failed 0
+if set -q files_to_test[1]
     $python -S ../littlecheck.py \
         --progress \
         -s fish=../test/root/bin/fish \
         -s fish_test_helper=../test/root/bin/fish_test_helper \
-        $littlecheck_files
-    set -l littlecheck_failures $status
-    set failed (math $failed + $littlecheck_failures)
+        $files_to_test
+
+    set -l littlecheck_status $status
+    if test "$littlecheck_status" -eq 125
+        # 125 indicates that all tests executed were skipped.
+        set skipped (count $files_to_test)
+    else
+        # The return code indicates the number of tests that failed
+        set failed $littlecheck_status
+    end
 end
 
-if test $failed -eq 0
-    say green "All high level script tests completed successfully"
+if test $failed -eq 0 && test $skipped -gt 0
+    test (count $files_to_test) -gt 1 && say blue (count $files_to_test)" tests skipped"
+    exit 125
+else if test $failed -eq 0
+    test (count $files_to_test) -gt 1 && say green "All high level script tests completed successfully"
     exit 0
 else
-    set plural (test $failed -eq 1; or echo s)
-    say red "$failed test$plural failed"
+    test (count $files_to_test) -gt 1 && say red "$failed tests failed"
     exit 1
 end