diff --git a/cmake/Tests.cmake b/cmake/Tests.cmake
index 9ce49213a..95650d46e 100644
--- a/cmake/Tests.cmake
+++ b/cmake/Tests.cmake
@@ -114,7 +114,8 @@ foreach(CHECK ${FISH_CHECKS})
   get_filename_component(CHECK_NAME ${CHECK} NAME)
   get_filename_component(CHECK ${CHECK} NAME_WE)
   add_test(NAME ${CHECK_NAME}
-    COMMAND ${CMAKE_CURRENT_BINARY_DIR}/tests/test_driver.py --cachedir ${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CURRENT_BINARY_DIR}
+    # Don't use --cachedir here because that races when running these in parallel
+    COMMAND ${CMAKE_CURRENT_BINARY_DIR}/tests/test_driver.py ${CMAKE_CURRENT_BINARY_DIR}
                 checks/${CHECK}.fish
     WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/tests
   )
@@ -127,7 +128,7 @@ FILE(GLOB PEXPECTS CONFIGURE_DEPENDS ${CMAKE_SOURCE_DIR}/tests/pexpects/*.py)
 foreach(PEXPECT ${PEXPECTS})
   get_filename_component(PEXPECT ${PEXPECT} NAME)
   add_test(NAME ${PEXPECT}
-    COMMAND ${CMAKE_CURRENT_BINARY_DIR}/tests/test_driver.py --cachedir ${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CURRENT_BINARY_DIR}
+    COMMAND ${CMAKE_CURRENT_BINARY_DIR}/tests/test_driver.py ${CMAKE_CURRENT_BINARY_DIR}
                 pexpects/${PEXPECT}
     WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/tests
   )