From 8ed8c431c601c7b303dbfe7eb412a63f1641b812 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Ho=C5=99e=C5=88ovsk=C3=BD?= Date: Sun, 21 Sep 2025 11:19:18 +0200 Subject: [PATCH] Support Bazel's TEST_PREMATURE_EXIT_FILE and add it to CLI as well This tells Catch2 to create an empty file at specified path before the tests start, and delete it after the tests finish. This allows callers to catch cases where the test binary silently exits before finishing (e.g. via call to `exit(0)` inside the code under test), by looking whether the file still exists. Closes #3020 --- docs/ci-and-misc.md | 3 + docs/command-line.md | 16 ++++ src/catch2/catch_config.cpp | 9 +++ src/catch2/catch_config.hpp | 4 + src/catch2/catch_session.cpp | 56 ++++++++++++- src/catch2/internal/catch_commandline.cpp | 3 + tests/ExtraTests/CMakeLists.txt | 44 +++++++++++ tests/ExtraTests/X40-QuickExit.cpp | 28 +++++++ tests/TestScripts/testBazelExitGuardFile.py | 88 +++++++++++++++++++++ 9 files changed, 250 insertions(+), 1 deletion(-) create mode 100644 tests/ExtraTests/X40-QuickExit.cpp create mode 100755 tests/TestScripts/testBazelExitGuardFile.py diff --git a/docs/ci-and-misc.md b/docs/ci-and-misc.md index 49bbd989..d2ef3fd7 100644 --- a/docs/ci-and-misc.md +++ b/docs/ci-and-misc.md @@ -66,11 +66,14 @@ test execution. Specifically it understands * JUnit output path via `XML_OUTPUT_FILE` * Test filtering via `TESTBRIDGE_TEST_ONLY` * Test sharding via `TEST_SHARD_INDEX`, `TEST_TOTAL_SHARDS`, and `TEST_SHARD_STATUS_FILE` + * Creating a file to signal premature test exit via `TEST_PREMATURE_EXIT_FILE` > Support for `XML_OUTPUT_FILE` was [introduced](https://github.com/catchorg/Catch2/pull/2399) in Catch2 3.0.1 > Support for `TESTBRIDGE_TEST_ONLY` and sharding was introduced in Catch2 3.2.0 +> Support for `TEST_PREMATURE_EXIT_FILE` was introduced in Catch2 X.Y.Z + This integration is enabled via either a [compile time configuration option](configuration.md#bazel-support), or via `BAZEL_TEST` environment variable set to "1". diff --git a/docs/command-line.md b/docs/command-line.md index 73b94f6c..651efd01 100644 --- a/docs/command-line.md +++ b/docs/command-line.md @@ -32,6 +32,7 @@ [Test Sharding](#test-sharding)
[Allow running the binary without tests](#allow-running-the-binary-without-tests)
[Output verbosity](#output-verbosity)
+[Create file to guard against silent early termination](#create-file-to-guard-against-silent-early-termination)
Catch works quite nicely without any command line options at all - but for those times when you want greater control the following options are available. Click one of the following links to take you straight to that option - or scroll on to browse the available options. @@ -649,6 +650,21 @@ ignored. Verbosity defaults to _normal_. +## Create file to guard against silent early termination +
--premature-exit-guard-file <path>
+ +> Introduced in Catch2 X.Y.Z + +Tells Catch2 to create an empty file at specified path before the tests +start, and delete it after the tests finish. If the file is present after +the process stops, it can be assumed that the testing binary exited +prematurely, e.g. due to the OOM killer. + +All directories in the path must already exist. If this option is used +and Catch2 cannot create the file (e.g. the location is not writable), +the test run will fail. + + --- [Home](Readme.md#top) diff --git a/src/catch2/catch_config.cpp b/src/catch2/catch_config.cpp index 352c1f42..978bcfd7 100644 --- a/src/catch2/catch_config.cpp +++ b/src/catch2/catch_config.cpp @@ -119,6 +119,8 @@ namespace Catch { m_data.reporterSpecifications.push_back( std::move( *parsed ) ); } + // Reading bazel env vars can change some parts of the config data, + // so we have to process the bazel env before acting on the config. if ( enableBazelEnvSupport() ) { readBazelEnvVars(); } @@ -183,6 +185,8 @@ namespace Catch { bool Config::showHelp() const { return m_data.showHelp; } + std::string const& Config::getExitGuardFilePath() const { return m_data.prematureExitGuardFilePath; } + // IConfig interface bool Config::allowThrows() const { return !m_data.noThrow; } StringRef Config::name() const { return m_data.name.empty() ? m_data.processName : m_data.name; } @@ -244,6 +248,11 @@ namespace Catch { m_data.shardCount = bazelShardOptions->shardCount; } } + + const auto bazelExitGuardFile = Detail::getEnv( "TEST_PREMATURE_EXIT_FILE" ); + if (bazelExitGuardFile) { + m_data.prematureExitGuardFilePath = bazelExitGuardFile; + } } } // end namespace Catch diff --git a/src/catch2/catch_config.hpp b/src/catch2/catch_config.hpp index b7b1315e..cdf286ad 100644 --- a/src/catch2/catch_config.hpp +++ b/src/catch2/catch_config.hpp @@ -87,6 +87,8 @@ namespace Catch { std::vector testsOrTags; std::vector sectionsToRun; + + std::string prematureExitGuardFilePath; }; @@ -114,6 +116,8 @@ namespace Catch { bool showHelp() const; + std::string const& getExitGuardFilePath() const; + // IConfig interface bool allowThrows() const override; StringRef name() const override; diff --git a/src/catch2/catch_session.cpp b/src/catch2/catch_session.cpp index 7542447b..042b7f4d 100644 --- a/src/catch2/catch_session.cpp +++ b/src/catch2/catch_session.cpp @@ -26,6 +26,8 @@ #include #include +#include +#include #include #include #include @@ -140,6 +142,50 @@ namespace Catch { } } + // Creates empty file at path. The path must be writable, we do not + // try to create directories in path because that's hard in C++14. + void setUpGuardFile( std::string const& guardFilePath ) { + if ( !guardFilePath.empty() ) { +#if defined( _MSC_VER ) + std::FILE* file = nullptr; + if ( fopen_s( &file, guardFilePath.c_str(), "w" ) ) { + char msgBuffer[100]; + const auto err = errno; + std::string errMsg; + if ( !strerror_s( msgBuffer, err ) ) { + errMsg = msgBuffer; + } else { + errMsg = "Could not translate errno to a string"; + } + +#else + std::FILE* file = std::fopen( guardFilePath.c_str(), "w" ); + if ( !file ) { + const auto err = errno; + const char* errMsg = std::strerror( err ); +#endif + + CATCH_RUNTIME_ERROR( "Could not open the exit guard file '" + << guardFilePath << "' because '" + << errMsg << "' (" << err << ')' ); + } + const int ret = std::fclose( file ); + CATCH_ENFORCE( + ret == 0, + "Error when closing the exit guard file: " << ret ); + } + } + + // Removes file at path. Assuming we created it in setUpGuardFile. + void tearDownGuardFile( std::string const& guardFilePath ) { + if ( !guardFilePath.empty() ) { + const int ret = std::remove( guardFilePath.c_str() ); + CATCH_ENFORCE( + ret == 0, + "Error when removing the exit guard file: " << ret ); + } + } + } // anon namespace Session::Session() { @@ -258,6 +304,7 @@ namespace Catch { static_cast(std::getchar()); } int exitCode = runInternal(); + if( ( m_configData.waitForKeypress & WaitForKeypress::BeforeExit ) != 0 ) { Catch::cout() << "...waiting for enter/ return before exiting, with code: " << exitCode << '\n' << std::flush; static_cast(std::getchar()); @@ -298,6 +345,10 @@ namespace Catch { CATCH_TRY { config(); // Force config to be constructed + // We need to retrieve potential Bazel config with the full Config + // constructor, so we have to create the guard file after it is created. + setUpGuardFile( m_config->getExitGuardFilePath() ); + seedRng( *m_config ); if (m_configData.filenamesAsTags) { @@ -327,9 +378,12 @@ namespace Catch { TestGroup tests { CATCH_MOVE(reporter), m_config.get() }; auto const totals = tests.execute(); + // If we got here, running the tests finished normally-enough. + // They might've failed, but that would've been reported elsewhere. + tearDownGuardFile( m_config->getExitGuardFilePath() ); + if ( tests.hadUnmatchedTestSpecs() && m_config->warnAboutUnmatchedTestSpecs() ) { - // UnmatchedTestSpecExitCode return UnmatchedTestSpecExitCode; } diff --git a/src/catch2/internal/catch_commandline.cpp b/src/catch2/internal/catch_commandline.cpp index 212f1774..db903507 100644 --- a/src/catch2/internal/catch_commandline.cpp +++ b/src/catch2/internal/catch_commandline.cpp @@ -305,6 +305,9 @@ namespace Catch { | Opt( config.allowZeroTests ) ["--allow-running-no-tests"] ( "Treat 'No tests run' as a success" ) + | Opt( config.prematureExitGuardFilePath, "path" ) + ["--premature-exit-guard-file"] + ( "create a file before running tests and delete it during clean exit" ) | Arg( config.testsOrTags, "test name|pattern|tags" ) ( "which test or tests to use" ); diff --git a/tests/ExtraTests/CMakeLists.txt b/tests/ExtraTests/CMakeLists.txt index bb7292d6..28f545c5 100644 --- a/tests/ExtraTests/CMakeLists.txt +++ b/tests/ExtraTests/CMakeLists.txt @@ -429,6 +429,50 @@ set_tests_properties(Reporters::CrashInJunitReporter LABELS "uses-signals" ) + +add_executable(QuickExitInTest ${TESTS_DIR}/X40-QuickExit.cpp) +target_link_libraries(QuickExitInTest PRIVATE Catch2::Catch2WithMain) +add_test( + NAME BazelEnv::TEST_PREMATURE_EXIT_FILE + COMMAND + Python3::Interpreter + "${CATCH_DIR}/tests/TestScripts/testBazelExitGuardFile.py" + $ + "${CMAKE_CURRENT_BINARY_DIR}" + "bazel" +) +set_tests_properties(BazelEnv::TEST_PREMATURE_EXIT_FILE + PROPERTIES + LABELS "uses-python" +) +add_test( + NAME PrematureExitGuardFileCanBeUsedFromCLI::CheckAfterCrash + COMMAND + Python3::Interpreter + "${CATCH_DIR}/tests/TestScripts/testBazelExitGuardFile.py" + $ + "${CMAKE_CURRENT_BINARY_DIR}" + "cli" +) +set_tests_properties(PrematureExitGuardFileCanBeUsedFromCLI::CheckAfterCrash + PROPERTIES + LABELS "uses-python" +) +add_test( + NAME PrematureExitGuardFileCanBeUsedFromCLI::CheckWithoutCrash + COMMAND + Python3::Interpreter + "${CATCH_DIR}/tests/TestScripts/testBazelExitGuardFile.py" + $ + "${CMAKE_CURRENT_BINARY_DIR}" + "no-crash" +) +set_tests_properties(PrematureExitGuardFileCanBeUsedFromCLI::CheckWithoutCrash + PROPERTIES + LABELS "uses-python" +) + + add_executable(AssertionStartingEventGoesBeforeAssertionIsEvaluated X20-AssertionStartingEventGoesBeforeAssertionIsEvaluated.cpp ) diff --git a/tests/ExtraTests/X40-QuickExit.cpp b/tests/ExtraTests/X40-QuickExit.cpp new file mode 100644 index 00000000..3afb7b58 --- /dev/null +++ b/tests/ExtraTests/X40-QuickExit.cpp @@ -0,0 +1,28 @@ + +// Copyright Catch2 Authors +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE.txt or copy at +// https://www.boost.org/LICENSE_1_0.txt) + +// SPDX-License-Identifier: BSL-1.0 + +/**\file + * Call ~~quick_exit~~ inside a test. + * + * This is used to test whether Catch2 properly creates the crash guard + * file based on provided arguments. + */ + +#include + +#include + +TEST_CASE("quick_exit", "[quick_exit]") { + // Return 0 as fake "successful" exit, while there should be a guard + // file created and kept. + std::exit(0); + // We cannot use quick_exit because libstdc++ on older MacOS versions didn't support it yet. + // std::quick_exit(0); +} + +TEST_CASE("pass") {} diff --git a/tests/TestScripts/testBazelExitGuardFile.py b/tests/TestScripts/testBazelExitGuardFile.py new file mode 100755 index 00000000..b4d31b52 --- /dev/null +++ b/tests/TestScripts/testBazelExitGuardFile.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 + +# Copyright Catch2 Authors +# Distributed under the Boost Software License, Version 1.0. +# (See accompanying file LICENSE.txt or copy at +# https://www.boost.org/LICENSE_1_0.txt) + +# SPDX-License-Identifier: BSL-1.0 + +import os +import sys +import subprocess + +def generate_path_suffix() -> str: + return os.urandom(16).hex()[:16] + + +def run_common(cmd, env = None): + cmd_env = env if env is not None else os.environ.copy() + print('Running:', cmd) + + try: + ret = subprocess.run( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=True, + universal_newlines=True, + env=cmd_env + ) + except subprocess.SubprocessError as ex: + print('Could not run "{}"'.format(cmd)) + print("Return code: {}".format(ex.returncode)) + print("stdout: {}".format(ex.stdout)) + print("stderr: {}".format(ex.stderr)) + raise + + +def test_bazel_env_vars(bin_path, guard_path): + env = os.environ.copy() + env["TEST_PREMATURE_EXIT_FILE"] = guard_path + env["BAZEL_TEST"] = '1' + run_common([bin_path], env) + +def test_cli_parameter(bin_path, guard_path): + cmd = [ + bin_path, + '--premature-exit-guard-file', + guard_path + ] + run_common(cmd) + +def test_no_crash(bin_path, guard_path): + cmd = [ + bin_path, + '--premature-exit-guard-file', + guard_path, + # Disable the quick-exit test + '~[quick_exit]' + ] + run_common(cmd) + +checks = { + 'bazel': (test_bazel_env_vars, True), + 'cli': (test_cli_parameter, True), + 'no-crash': (test_no_crash, False), +} + + +bin_path = os.path.abspath(sys.argv[1]) +output_dir = os.path.abspath(sys.argv[2]) +test_kind = sys.argv[3] +guard_file_path = os.path.join(output_dir, f"guard_file.{generate_path_suffix()}") +print(f'Guard file path: "{guard_file_path}"') + +check_func, file_should_exist = checks[test_kind] +check_func(bin_path, guard_file_path) + +assert os.path.exists(guard_file_path) == file_should_exist +# With randomly generated file suffix, we should not run into name conflicts. +# However, we try to cleanup anyway, to avoid having infinity files in +# long living build directories. +if file_should_exist: + try: + os.remove(guard_file_path) + except Exception as ex: + print(f'Could not remove file {guard_file_path} because: {ex}') +