From 9ec44dd62b1b593e01feaf95eb485d26fcdd50ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Ho=C5=99e=C5=88ovsk=C3=BD?= Date: Sat, 4 Jul 2026 16:35:50 +0200 Subject: [PATCH] catch_discover_tests uses tempfile to retrieve JSON from the binary This allows it to deal with badly behaved code, where 3rd party dependencies write into stdout during global construction. Closes #3162 Closes #3166 --- extras/Catch.cmake | 4 +- extras/CatchAddTests.cmake | 47 ++++++++++++++++++- .../DiscoverTests/VerifyRegistration.py | 38 ++++++++++----- .../DiscoverTests/register-tests.cpp | 18 +++++++ 4 files changed, 92 insertions(+), 15 deletions(-) diff --git a/extras/Catch.cmake b/extras/Catch.cmake index fcdebd60..657bb724 100644 --- a/extras/Catch.cmake +++ b/extras/Catch.cmake @@ -44,7 +44,7 @@ same as the Catch name; see also ``TEST_PREFIX`` and ``TEST_SUFFIX``. ``catch_discover_tests`` sets up a post-build command on the test executable that generates the list of tests by parsing the output from running the test - with the ``--list-test-names-only`` argument. This ensures that the full + with the ``--list-tests --reporter json`` argument. This ensures that the full list of tests is obtained. Since test discovery occurs at build time, it is not necessary to re-run CMake when the list of tests changes. However, it requires that :prop_tgt:`CROSSCOMPILING_EMULATOR` is properly set @@ -67,7 +67,7 @@ same as the Catch name; see also ``TEST_PREFIX`` and ``TEST_SUFFIX``. ``TEST_SPEC arg1...`` Specifies test cases, wildcarded test cases, tags and tag expressions to - pass to the Catch executable with the ``--list-test-names-only`` argument. + pass to the Catch executable when listing the tests. ``EXTRA_ARGS arg1...`` Any extra arguments to pass on the command line to each test case. diff --git a/extras/CatchAddTests.cmake b/extras/CatchAddTests.cmake index 4c27f479..be3d1c8c 100644 --- a/extras/CatchAddTests.cmake +++ b/extras/CatchAddTests.cmake @@ -16,6 +16,42 @@ function(add_command NAME) set(script "${script}${NAME}(${_args})\n" PARENT_SCOPE) endfunction() +# Generates random filename in the temp folder. +# Temp folder is retrieved by checking env vars from various platforms. +function(make_temp_file_path OUT_VARIABLE FALLBACK_PATH) + set(TEMP_DIR "") + set(ENV_VARS + # From XDG base dir specification + XDG_RUNTIME_DIR + # From POSIX standard + TMPDIR + # From Windows + TMP + TEMP + ) + + foreach(var ${ENV_VARS}) + message(NOTICE "Checking ${var} env var") + if(DEFINED ENV{${var}} AND NOT "$ENV{${var}}" STREQUAL "") + set(TEMP_DIR "$ENV{${var}}") + break() + endif() + endforeach() + + # If all checks fail, we use the fallback path + if(TEMP_DIR STREQUAL "") + set(TEMP_DIR "${FALLBACK_PATH}") + endif() + + file(TO_CMAKE_PATH "${TEMP_DIR}" TEMP_DIR) + + # Generate the random file name + string(RANDOM LENGTH 8 RAND_ID) + set(FINAL_TEMP_PATH "${TEMP_DIR}/Catch2-test-listing.${RAND_ID}.json") + + set(${OUT_VARIABLE} "${FINAL_TEMP_PATH}" PARENT_SCOPE) +endfunction() + function(catch_discover_tests_impl) cmake_parse_arguments( @@ -72,8 +108,13 @@ function(catch_discover_tests_impl) set(ENV{DYLD_FRAMEWORK_PATH} "${paths}") endif() + make_temp_file_path(listing_output_path "${_TEST_WORKING_DIR}") + execute_process( - COMMAND ${_TEST_EXECUTOR} "${_TEST_EXECUTABLE}" ${spec} --list-tests --reporter json + COMMAND ${_TEST_EXECUTOR} "${_TEST_EXECUTABLE}" ${spec} + --list-tests + --reporter json + --out "${listing_output_path}" OUTPUT_VARIABLE listing_output RESULT_VARIABLE result WORKING_DIRECTORY "${_TEST_WORKING_DIR}" @@ -86,6 +127,10 @@ function(catch_discover_tests_impl) ) endif() + # Read the JSON output back from the output file (and then get rid of the file) + file(READ ${listing_output_path} listing_output) + file(REMOVE ${listing_output_path}) + # Prepare reporter if(reporter) set(reporter_arg "--reporter ${reporter}") diff --git a/tests/TestScripts/DiscoverTests/VerifyRegistration.py b/tests/TestScripts/DiscoverTests/VerifyRegistration.py index 7d9862cf..c281a403 100644 --- a/tests/TestScripts/DiscoverTests/VerifyRegistration.py +++ b/tests/TestScripts/DiscoverTests/VerifyRegistration.py @@ -12,6 +12,7 @@ import subprocess import sys import re import json +import tempfile from collections import namedtuple from typing import List @@ -72,14 +73,19 @@ def get_test_names(build_path: str) -> List[TestInfo]: config_path = "Debug" if os.name == 'nt' else "" full_path = os.path.join(build_path, config_path, 'tests') - - cmd = [full_path, '--reporter', 'json', '--list-tests'] - result = subprocess.run(cmd, - capture_output = True, - check = True, - text = True) - - test_listing = json.loads(result.stdout) + with tempfile.TemporaryDirectory() as tmpdir: + fname = f'{tmpdir}/listing-output.json' + cmd = [full_path, + '--list-tests', + '--reporter', 'json', + '--out', fname + ] + result = subprocess.run(cmd, + capture_output = False, + check = True, + text = True) + with open(fname, mode='r', encoding='utf-8') as file: + test_listing = json.load(file) assert test_listing['version'] == 1 @@ -96,10 +102,18 @@ def get_ctest_listing(build_path): os.chdir(build_path) cmd = ['ctest', '-C', 'debug', '--show-only=json-v1'] - result = subprocess.run(cmd, - capture_output = True, - check = True, - text = True) + try: + result = subprocess.run(cmd, + capture_output = True, + check = True, + text = True) + except subprocess.CalledProcessError as err: + print('Error when getting output from CTest') + print(f'cmd: {err.cmd}') + print(f'stderr: {err.stderr}') + print(f'stdout: {err.stdout}') + exit(4) + os.chdir(old_path) return result.stdout diff --git a/tests/TestScripts/DiscoverTests/register-tests.cpp b/tests/TestScripts/DiscoverTests/register-tests.cpp index be533ab6..8b071196 100644 --- a/tests/TestScripts/DiscoverTests/register-tests.cpp +++ b/tests/TestScripts/DiscoverTests/register-tests.cpp @@ -8,6 +8,24 @@ #include +#include +#include + +namespace { + +struct PrintsWhenConstructed { + PrintsWhenConstructed() { + std::cout << "Hello\n"; + std::cerr << "Holla\n"; + std::fprintf(stdout, "Hullo\n"); + std::fprintf(stderr, "Hillo\n"); + } +}; + +static PrintsWhenConstructed instance; + +} + TEST_CASE("@Script[C:\\EPM1A]=x;\"SCALA_ZERO:\"", "[script regressions]"){} TEST_CASE("Some test") {} TEST_CASE( "Let's have a test case with a long name. Longer. No, even longer. "