From 2ebe3134cfe55ba66c1dcaf138c67781c98e800a Mon Sep 17 00:00:00 2001 From: Daniel Rainer Date: Sat, 7 Jun 2025 01:43:01 +0200 Subject: [PATCH 1/3] Extract function for running tests This is done to prepare for running tests concurrently. Align output and prevent flushing stdout between test name and result. --- tests/test_driver.py | 208 ++++++++++++++++++++++++++----------------- 1 file changed, 124 insertions(+), 84 deletions(-) diff --git a/tests/test_driver.py b/tests/test_driver.py index b2f34396a..0c20b0178 100755 --- a/tests/test_driver.py +++ b/tests/test_driver.py @@ -1,12 +1,14 @@ #!/usr/bin/env python3 import argparse -import os +from dataclasses import dataclass from datetime import datetime +import os from pathlib import Path import shutil import subprocess import sys import tempfile +from typing import Optional import littlecheck @@ -158,91 +160,33 @@ def main(): if not PEXPECT and any(x.endswith(".py") for (x, _) in files): print(f"{RED}Skipping pexpect tests because pexpect is not installed{RESET}") + longest_test_name_length = max([len(arg) for _, arg in files]) + max_expected_digits_duration = 5 + + def print_result(arg, result, color, duration=None, suffix=None): + duration_str = ( + "" + if duration is None + else f" {str(duration_ms).rjust(max_expected_digits_duration)} ms" + ) + suffix_str = "" if suffix is None else f"\n{suffix}" + print( + f"{arg.ljust(longest_test_name_length)} {color}{result}{RESET} {duration_str}{suffix_str}" + ) + for f, arg in files: - if not f.endswith(".fish") and not f.endswith(".py"): - print(f"Not a valid test file: {arg}") - failcount += 1 - continue + match run_test(f, arg, script_path, args, def_subs, lconfig, fishdir): + case TestSkip(arg): + skipcount += 1 + print_result(arg, "SKIPPED", BLUE) + case TestFail(arg, duration_ms, error_message): + failcount += 1 + failed += [arg] + print_result(arg, "FAILED", RED, duration_ms, error_message) + case TestPass(arg, duration_ms): + passcount += 1 + print_result(arg, "PASSED", GREEN, duration_ms) - starttime = datetime.now() - with tempfile.TemporaryDirectory(prefix="fishtest-") as home: - makeenv(script_path, home, args.cachedir) - os.chdir(home) - if f.endswith(".fish"): - subs = def_subs.copy() - subs.update({"s": f, "fish_test_helper": home + "/fish_test_helper"}) - - # littlecheck - print(f"{arg}..", end="", flush=True) - ret = littlecheck.check_path( - f, subs, lconfig, lambda x: print(x.message()) - ) - endtime = datetime.now() - duration_ms = round((endtime - starttime).total_seconds() * 1000) - if ret is littlecheck.SKIP: - print(f"{BLUE}SKIPPED{RESET}") - skipcount += 1 - elif ret: - print(f"{GREEN}PASS{RESET} ({duration_ms} ms)") - passcount += 1 - else: - print(f"{RED}FAIL{RESET} ({duration_ms} ms)") - failcount += 1 - failed += [arg] - print(f"Tmpdir is {home}") - elif f.endswith(".py"): - # environ for py files has a few changes. - pyenviron = os.environ.copy() - pyenviron.update( - { - "PYTHONPATH": str(script_path), - "fish": str(fishdir / "fish"), - "fish_key_reader": str(fishdir / "fish_key_reader"), - "fish_indent": str(fishdir / "fish_indent"), - "TERM": "dumb", - "FISH_FORCE_COLOR": "1" if sys.stdout.isatty() else "0", - } - ) - print(f"{arg}..", end="", flush=True) - if not PEXPECT: - print(f"{BLUE}SKIPPED{RESET}") - skipcount += 1 - continue - try: - proc = subprocess.run( - ["python3", f], - capture_output=True, - env=pyenviron, - # Timeout of 120 seconds, about 10 times what any of these takes - timeout=120, - ) - except subprocess.TimeoutExpired as e: - print(f"{RED}FAILED due to timeout{RESET}") - if e.output: - print(e.output.decode("utf-8")) - if e.stderr: - print(e.stderr.decode("utf-8")) - failcount += 1 - failed += [arg] - continue - - endtime = datetime.now() - duration_ms = round((endtime - starttime).total_seconds() * 1000) - if proc.returncode == 0: - print(f"{GREEN}PASS{RESET} ({duration_ms} ms)") - passcount += 1 - elif proc.returncode == 127: - print(f"{BLUE}SKIPPED{RESET}") - skipcount += 1 - else: - print(f"{RED}FAILED{RESET} ({duration_ms} ms)") - if proc.stdout: - print(proc.stdout.decode("utf-8")) - if proc.stderr: - print(proc.stderr.decode("utf-8")) - failcount += 1 - failed += [arg] - print(f"Tmpdir is {home}") if passcount + failcount + skipcount > 1: print(f"{passcount} / {passcount + failcount} passed ({skipcount} skipped)") if failcount: @@ -253,6 +197,102 @@ def main(): return 1 if failcount else 0 +@dataclass +class TestSkip: + arg: str + + +@dataclass +class TestFail: + arg: str + duration_ms: Optional[int] + error_message: Optional[str] + + +@dataclass +class TestPass: + arg: str + duration_ms: int + + +TestResult = TestSkip | TestFail | TestPass + + +def run_test(path, arg, script_path, args, def_subs, lconfig, fishdir) -> TestResult: + if not path.endswith(".fish") and not path.endswith(".py"): + return TestFail(arg, None, f"Not a valid test file: {arg}") + + starttime = datetime.now() + with tempfile.TemporaryDirectory(prefix="fishtest-") as home: + makeenv(script_path, home, args.cachedir) + os.chdir(home) + if path.endswith(".fish"): + subs = def_subs.copy() + subs.update({"s": path, "fish_test_helper": home + "/fish_test_helper"}) + + # littlecheck + ret = littlecheck.check_path( + path, subs, lconfig, lambda x: print(x.message()) + ) + endtime = datetime.now() + duration_ms = round((endtime - starttime).total_seconds() * 1000) + if ret is littlecheck.SKIP: + return TestSkip(arg) + elif ret: + return TestPass(arg, duration_ms) + else: + return TestFail(arg, duration_ms, f"Tmpdir is {home}") + elif path.endswith(".py"): + # environ for py files has a few changes. + pyenviron = os.environ.copy() + pyenviron.update( + { + "PYTHONPATH": str(script_path), + "fish": str(fishdir / "fish"), + "fish_key_reader": str(fishdir / "fish_key_reader"), + "fish_indent": str(fishdir / "fish_indent"), + "TERM": "dumb", + "FISH_FORCE_COLOR": "1" if sys.stdout.isatty() else "0", + } + ) + if not PEXPECT: + return TestSkip(arg) + try: + proc = subprocess.run( + ["python3", path], + capture_output=True, + env=pyenviron, + # Timeout of 120 seconds, about 10 times what any of these takes + timeout=120, + ) + except subprocess.TimeoutExpired as e: + error_message = f"{RED}FAILED due to timeout{RESET}" + if e.output: + error_message += e.output.decode("utf-8") + if e.stderr: + error_message += e.stderr.decode("utf-8") + return TestFail(arg, None, error_message) + + endtime = datetime.now() + duration_ms = round((endtime - starttime).total_seconds() * 1000) + if proc.returncode == 0: + return TestPass(arg, duration_ms) + elif proc.returncode == 127: + return TestSkip(arg) + else: + error_message = "" + if proc.stdout: + error_message += proc.stdout.decode("utf-8") + if proc.stderr: + error_message += proc.stderr.decode("utf-8") + error_message += f"Tmpdir is {home}" + return TestFail(arg, duration_ms, error_message) + else: + return TestFail( + arg, None, "Error in test driver. This should be unreachable." + ) + + if __name__ == "__main__": try: ret = main() From df097b114c61f0c3be298aa825c83b1f85bc2151 Mon Sep 17 00:00:00 2001 From: Daniel Rainer Date: Sat, 7 Jun 2025 19:30:44 +0200 Subject: [PATCH 2/3] Put test tmpdirs under common root tmpdir This is done to prepare for running the tests in parallel. With this approach the root tmpdir can be created before any test starts, each test can create its home dir under the root tmpdir, and when all tests are done the root tmpdir can be deleted. Deletion of per-test dirs is more difficult in an async context. --- tests/test_driver.py | 136 ++++++++++++++++++++++--------------------- 1 file changed, 69 insertions(+), 67 deletions(-) diff --git a/tests/test_driver.py b/tests/test_driver.py index 0c20b0178..49b928e76 100755 --- a/tests/test_driver.py +++ b/tests/test_driver.py @@ -174,8 +174,10 @@ def main(): f"{arg.ljust(longest_test_name_length)} {color}{result}{RESET} {duration_str}{suffix_str}" ) + tmp_root = tempfile.mkdtemp(prefix="fishtest-root-") + for f, arg in files: - match run_test(f, arg, script_path, args, def_subs, lconfig, fishdir): + match run_test(tmp_root, f, arg, script_path, args, def_subs, lconfig, fishdir): case TestSkip(arg): skipcount += 1 print_result(arg, "SKIPPED", BLUE) @@ -187,6 +189,8 @@ def main(): passcount += 1 print_result(arg, "PASSED", GREEN, duration_ms) + shutil.rmtree(tmp_root) + if passcount + failcount + skipcount > 1: print(f"{passcount} / {passcount + failcount} passed ({skipcount} skipped)") if failcount: @@ -218,79 +222,77 @@ class TestPass: TestResult = TestSkip | TestFail | TestPass -def run_test(path, arg, script_path, args, def_subs, lconfig, fishdir) -> TestResult: +def run_test( + tmp_root, path, arg, script_path, args, def_subs, lconfig, fishdir +) -> TestResult: if not path.endswith(".fish") and not path.endswith(".py"): return TestFail(arg, None, f"Not a valid test file: {arg}") starttime = datetime.now() - with tempfile.TemporaryDirectory(prefix="fishtest-") as home: - makeenv(script_path, home, args.cachedir) - os.chdir(home) - if path.endswith(".fish"): - subs = def_subs.copy() - subs.update({"s": path, "fish_test_helper": home + "/fish_test_helper"}) + home = tempfile.mkdtemp(prefix="fishtest-", dir=tmp_root) + makeenv(script_path, home, args.cachedir) + os.chdir(home) + if path.endswith(".fish"): + subs = def_subs.copy() + subs.update({"s": path, "fish_test_helper": home + "/fish_test_helper"}) - # littlecheck - ret = littlecheck.check_path( - path, subs, lconfig, lambda x: print(x.message()) - ) - endtime = datetime.now() - duration_ms = round((endtime - starttime).total_seconds() * 1000) - if ret is littlecheck.SKIP: - return TestSkip(arg) - elif ret: - return TestPass(arg, duration_ms) - else: - return TestFail(arg, duration_ms, f"Tmpdir is {home}") - elif path.endswith(".py"): - # environ for py files has a few changes. - pyenviron = os.environ.copy() - pyenviron.update( - { - "PYTHONPATH": str(script_path), - "fish": str(fishdir / "fish"), - "fish_key_reader": str(fishdir / "fish_key_reader"), - "fish_indent": str(fishdir / "fish_indent"), - "TERM": "dumb", - "FISH_FORCE_COLOR": "1" if sys.stdout.isatty() else "0", - } - ) - if not PEXPECT: - return TestSkip(arg) - try: - proc = subprocess.run( - ["python3", path], - capture_output=True, - env=pyenviron, - # Timeout of 120 seconds, about 10 times what any of these takes - timeout=120, - ) - except subprocess.TimeoutExpired as e: - error_message = f"{RED}FAILED due to timeout{RESET}" - if e.output: - error_message += e.output.decode("utf-8") - if e.stderr: - error_message += e.stderr.decode("utf-8") - return TestFail(arg, None, error_message) - - endtime = datetime.now() - duration_ms = round((endtime - starttime).total_seconds() * 1000) - if proc.returncode == 0: - return TestPass(arg, duration_ms) - elif proc.returncode == 127: - return TestSkip(arg) - else: - error_message = "" - if proc.stdout: - error_message += proc.stdout.decode("utf-8") - if proc.stderr: - error_message += proc.stderr.decode("utf-8") - error_message += f"Tmpdir is {home}" - return TestFail(arg, duration_ms, error_message) + # littlecheck + ret = littlecheck.check_path(path, subs, lconfig, lambda x: print(x.message())) + endtime = datetime.now() + duration_ms = round((endtime - starttime).total_seconds() * 1000) + if ret is littlecheck.SKIP: + return TestSkip(arg) + elif ret: + return TestPass(arg, duration_ms) else: - return TestFail( - arg, None, "Error in test driver. This should be unreachable." + return TestFail(arg, duration_ms, f"Tmpdir is {home}") + elif path.endswith(".py"): + # environ for py files has a few changes. + pyenviron = os.environ.copy() + pyenviron.update( + { + "PYTHONPATH": str(script_path), + "fish": str(fishdir / "fish"), + "fish_key_reader": str(fishdir / "fish_key_reader"), + "fish_indent": str(fishdir / "fish_indent"), + "TERM": "dumb", + "FISH_FORCE_COLOR": "1" if sys.stdout.isatty() else "0", + } + ) + if not PEXPECT: + return TestSkip(arg) + try: + proc = subprocess.run( + ["python3", path], + capture_output=True, + env=pyenviron, + # Timeout of 120 seconds, about 10 times what any of these takes + timeout=120, ) + except subprocess.TimeoutExpired as e: + error_message = f"{RED}FAILED due to timeout{RESET}" + if e.output: + error_message += e.output.decode("utf-8") + if e.stderr: + error_message += e.stderr.decode("utf-8") + return TestFail(arg, None, error_message) + + endtime = datetime.now() + duration_ms = round((endtime - starttime).total_seconds() * 1000) + if proc.returncode == 0: + return TestPass(arg, duration_ms) + elif proc.returncode == 127: + return TestSkip(arg) + else: + error_message = "" + if proc.stdout: + error_message += proc.stdout.decode("utf-8") + if proc.stderr: + error_message += proc.stderr.decode("utf-8") + error_message += f"Tmpdir is {home}" + return TestFail(arg, duration_ms, error_message) + else: + return TestFail(arg, None, "Error in test driver. This should be unreachable.") if __name__ == "__main__": From 4721ffe512c326d26820510ead067e405cf225c8 Mon Sep 17 00:00:00 2001 From: Daniel Rainer Date: Sat, 7 Jun 2025 18:54:05 +0200 Subject: [PATCH 3/3] Remove unused variables --- tests/test_driver.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_driver.py b/tests/test_driver.py index 49b928e76..4a02c05bc 100755 --- a/tests/test_driver.py +++ b/tests/test_driver.py @@ -50,7 +50,7 @@ def makeenv(script_path, home, test_helper_path): if test_helper_path: thp = Path(test_helper_path) if not os.path.exists(thp / "fish_test_helper"): - comp = subprocess.run( + subprocess.run( [ "cc", script_path / "fish_test_helper.c", @@ -60,7 +60,7 @@ def makeenv(script_path, home, test_helper_path): ) shutil.copy(thp / "fish_test_helper", home + "/fish_test_helper") else: - comp = subprocess.run( + subprocess.run( [ "cc", script_path / "fish_test_helper.c",