diff --git a/CMakeLists.txt b/CMakeLists.txt index f63d1d44..291af7c5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -35,7 +35,7 @@ if(CMAKE_BINARY_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) endif() project(Catch2 - VERSION 3.15.3 # CML version placeholder, don't delete + VERSION 3.16.0 # CML version placeholder, don't delete LANGUAGES CXX HOMEPAGE_URL "https://github.com/catchorg/Catch2" DESCRIPTION "A modern, C++-native, unit test framework." diff --git a/docs/command-line.md b/docs/command-line.md index cbb3449e..41c4ab75 100644 --- a/docs/command-line.md +++ b/docs/command-line.md @@ -208,7 +208,7 @@ hardcoded into Catch2. Currently there are 3 supported options: * ["colour-mode"](#colour-mode) * ["verbosity"](#output-verbosity) -> Support for per-reporter verbosity option was added in Catch2 vX.Y.Z +> Support for per-reporter verbosity option was added in Catch2 3.16.0 _Note that the reporter might still check the X-prefixed options for validity, and throw an error if they are wrong._ diff --git a/docs/release-notes.md b/docs/release-notes.md index 9fafa0cd..bab645de 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -2,6 +2,7 @@ # Release notes **Contents**
+[3.16.0](#3160)
[3.15.3](#3153)
[3.15.2](#3152)
[3.15.1](#3151)
@@ -79,6 +80,35 @@ +## 3.16.0 + +### Fixes +* Multiple fixes in `catch_discover_tests`: + * Fixed `_TESTS` variable accumulating JSON fragments alongside test names. + * This was introduced during the refactoring in last release. + * Fixed `_TESTS` variable from `catch_discover_tests` not escaping test names to be properly parsed by CMake. + * This means that e.g. test names with semicolons will not be split into multiple partial test names. + * This bug has existed since the first version of the script. + * Fixed `TEST_PREFIX`/`TEST_SUFFIX` args having leading/trailing whitespace stripped. +* Added workaround for Clang 20-21 compile error with `TEMPLATE_PRODUCT_TEST_CASE` (#3115, #3173) + +### Improvements +* Verbosity option is now handled per reporter. + * The standalone `--verbosity` flag is propagated to all reporters as default, just like `--colour-mode`. +* The JSON reporter considers verbosity when listing tests. +* Another set of performance improvements for `catch_discover_tests` performance + * The newest version can register about 3k tests in 1 second, up from 1k previously. +* The initial `TEST_CASE` registration is slightly faster. +* Reduced overhead from first instantiating `BENCHMARK` machinery in a TU. +* Improved compilation speed when multiple templated test case macros use the same types. +* Rewrote implementation of the string matchers + * Case-sensitive matching (the default) is significantly faster. + * Case-insensitive matching is faster in most cases. +* Added optimizer barrier to calls into benchmarks without return values + * This limits the optimizations compiler can perform **between calls** into the benchmarked function, improving the accuracy. +* Added warning that checks for using sharding without deterministic test order (#3186) + + ## 3.15.3 ### Fixes diff --git a/extras/catch_amalgamated.cpp b/extras/catch_amalgamated.cpp index abf540b4..73ef255d 100644 --- a/extras/catch_amalgamated.cpp +++ b/extras/catch_amalgamated.cpp @@ -6,8 +6,8 @@ // SPDX-License-Identifier: BSL-1.0 -// Catch v3.15.3 -// Generated: 2026-07-26 22:17:52.418168 +// Catch v3.16.0 +// Generated: 2026-08-25 09:29:23.172704 // ---------------------------------------------------------- // This file is an amalgamation of multiple different files. // You probably shouldn't edit it directly. @@ -40,6 +40,54 @@ +#include +#include +#include + +namespace Catch { + namespace Benchmark { + namespace Detail { + + Environment measure_environment_default() { + return Detail::measure_environment(); + } + + ExecutionPlan prepare_default( const IConfig& cfg, + Environment env, + BenchmarkFunction&& fun ) { + // This mirrors Benchmark::prepare(), but with the + // clock fixed so it is instantiated once here in the library. + auto min_time = env.clock_resolution.mean * Detail::minimum_ticks; + auto run_time = std::max( + min_time, + std::chrono::duration_cast( + cfg.benchmarkWarmupTime() ) ); + auto&& test = Detail::run_for_at_least( + std::chrono::duration_cast( run_time ), 1, fun ); + int new_iters = static_cast( + std::ceil( min_time * test.iterations / test.elapsed ) ); + return { new_iters, + test.elapsed / test.iterations * new_iters * + cfg.benchmarkSamples(), + CATCH_MOVE( fun ), + std::chrono::duration_cast( + cfg.benchmarkWarmupTime() ), + Detail::warmup_iterations }; + } + + std::vector run_plan_default( ExecutionPlan const& plan, + const IConfig& cfg, + Environment env ) { + return plan.run( cfg, env ); + } + + } // namespace Detail + } // namespace Benchmark +} // namespace Catch + + + + namespace Catch { namespace Benchmark { namespace Detail { @@ -795,6 +843,7 @@ namespace Catch { return lhs.name == rhs.name && lhs.outputFilename == rhs.outputFilename && lhs.colourMode == rhs.colourMode && + lhs.verbosity == rhs.verbosity && lhs.customOptions == rhs.customOptions; } @@ -863,6 +912,7 @@ namespace Catch { reporterSpec.outputFile() ? *reporterSpec.outputFile() : data.defaultOutputFilename, reporterSpec.colourMode().valueOr( data.defaultColourMode ), + reporterSpec.verbosity().valueOr( data.verbosity ), reporterSpec.customOptions() } ); } } @@ -913,6 +963,7 @@ namespace Catch { double Config::minDuration() const { return m_data.minDuration; } TestRunOrder Config::runOrder() const { return m_data.runOrder; } uint32_t Config::rngSeed() const { return m_data.rngSeed; } + bool Config::rngSeedWasFixed() const { return m_data.rngSeedWasFixed; } unsigned int Config::shardCount() const { return m_data.shardCount; } unsigned int Config::shardIndex() const { return m_data.shardIndex; } ColourMode Config::defaultColourMode() const { return m_data.defaultColourMode; } @@ -938,7 +989,7 @@ namespace Catch { if ( bazelOutputFile ) { m_data.reporterSpecifications.push_back( - { "junit", std::string( bazelOutputFile ), {}, {} } ); + { "junit", std::string( bazelOutputFile ), {}, {}, {} } ); } const auto bazelTestSpec = Detail::getEnv( "TESTBRIDGE_TEST_ONLY" ); @@ -977,6 +1028,7 @@ namespace Catch { << bazelRandomSeed << "') as proper seed.\n"; } else { m_data.rngSeed = *parsedSeed; + m_data.rngSeedWasFixed = true; } } } @@ -1162,6 +1214,10 @@ namespace Catch { #endif } + ITestCaseRegistry& getMutableTestCaseRegistry() override { + return m_testCaseRegistry; + } + private: TestRegistry m_testCaseRegistry; ReporterRegistry m_reporterRegistry; @@ -1217,6 +1273,7 @@ namespace Catch { ReporterConfig( config, makeStream( spec.outputFilename ), spec.colourMode, + spec.verbosity, spec.customOptions ) ); } @@ -1233,6 +1290,7 @@ namespace Catch { ReporterConfig( config, makeStream( reporterSpec.outputFilename ), reporterSpec.colourMode, + reporterSpec.verbosity, reporterSpec.customOptions ) ) ); } @@ -1303,9 +1361,7 @@ namespace Catch { }; void applyFilenamesAsTags() { - for (auto const& testInfo : getRegistryHub().getTestCaseRegistry().getAllInfos()) { - testInfo->addFilenameTag(); - } + getMutableRegistryHub().getMutableTestCaseRegistry().enableFilenameTags(); } // Creates empty file at path. The path must be writable, we do not @@ -1511,6 +1567,19 @@ namespace Catch { CATCH_TRY { config(); // Force config to be constructed + if ( m_config->shardCount() > 1 && + m_config->runOrder() == TestRunOrder::Randomized && + !m_config->rngSeedWasFixed() ) { + Catch::cerr() + << "Warning: using sharding (--shard-count) with random " + "order (--order rand, the default) and without a fixed " + "numeric --rng-seed does not guarantee disjoint coverage " + "between shard invocations. Pass the same numeric " + "--rng-seed to every shard, or use --order decl or " + "--order lex instead.\n" + << std::flush; + } + // We need to retrieve potential Bazel config with the full Config // constructor, so we have to create the guard file after it is created. setUpGuardFile( m_config->getExitGuardFilePath() ); @@ -2394,7 +2463,7 @@ namespace Catch { } Version const& libraryVersion() { - static Version version( 3, 15, 3, "", 0 ); + static Version version( 3, 16, 0, "", 0 ); return version; } @@ -2591,10 +2660,12 @@ namespace Catch { IConfig const* _fullConfig, Detail::unique_ptr _stream, ColourMode colourMode, + Verbosity verbosity, std::map customOptions ): m_stream( CATCH_MOVE(_stream) ), m_fullConfig( _fullConfig ), m_colourMode( colourMode ), + m_verbosity( verbosity ), m_customOptions( CATCH_MOVE( customOptions ) ) {} Detail::unique_ptr ReporterConfig::takeStream() && { @@ -2603,6 +2674,7 @@ namespace Catch { } IConfig const * ReporterConfig::fullConfig() const { return m_fullConfig; } ColourMode ReporterConfig::colourMode() const { return m_colourMode; } + Verbosity ReporterConfig::verbosity() const { return m_verbosity; } std::map const& ReporterConfig::customOptions() const { @@ -3297,9 +3369,11 @@ namespace Catch { auto const setRngSeed = [&]( std::string const& seed ) { if( seed == "time" ) { config.rngSeed = generateRandomSeed(GenerateFrom::Time); + config.rngSeedWasFixed = false; return ParserResult::ok(ParseResultType::Matched); } else if (seed == "random-device") { config.rngSeed = generateRandomSeed(GenerateFrom::RandomDevice); + config.rngSeedWasFixed = false; return ParserResult::ok(ParseResultType::Matched); } @@ -3310,6 +3384,7 @@ namespace Catch { return ParserResult::runtimeError( "Could not parse '" + seed + "' as seed" ); } config.rngSeed = *parsedSeed; + config.rngSeedWasFixed = true; return ParserResult::ok( ParseResultType::Matched ); }; auto const setDefaultColourMode = [&]( std::string const& colourMode ) { @@ -5753,6 +5828,18 @@ namespace Catch { return {}; } } + + Optional stringToVerbosity( StringRef verbosity ) { + if (verbosity == "quiet") { return Verbosity::Quiet; + } else if ( verbosity == "normal" ) { + return Verbosity::Normal; + } else if ( verbosity == "high" ) { + return Verbosity::High; + } else { + return {}; + } + } + } // namespace Detail @@ -5760,6 +5847,7 @@ namespace Catch { return lhs.m_name == rhs.m_name && lhs.m_outputFileName == rhs.m_outputFileName && lhs.m_colourMode == rhs.m_colourMode && + lhs.m_verbosity == rhs.m_verbosity && lhs.m_customOptions == rhs.m_customOptions; } @@ -5771,6 +5859,7 @@ namespace Catch { std::map kvPairs; Optional outputFileName; Optional colourMode; + Optional verbosity; // First part is always reporter name, so we skip it for ( size_t i = 1; i < parts.size(); ++i ) { @@ -5808,6 +5897,12 @@ namespace Catch { if ( !colourMode ) { return {}; } + } else if ( key == "verbosity" ) { + // Duplicated key + if ( verbosity ) { return {}; } + verbosity = Detail::stringToVerbosity( value ); + // Parsing failed + if ( !verbosity ) { return {}; } } else { // Unrecognized option return {}; @@ -5817,6 +5912,7 @@ namespace Catch { return ReporterSpec{ CATCH_MOVE( parts[0] ), CATCH_MOVE( outputFileName ), CATCH_MOVE( colourMode ), + CATCH_MOVE( verbosity), CATCH_MOVE( kvPairs ) }; } @@ -5824,10 +5920,12 @@ ReporterSpec::ReporterSpec( std::string name, Optional outputFileName, Optional colourMode, + Optional verbosity, std::map customOptions ): m_name( CATCH_MOVE( name ) ), m_outputFileName( CATCH_MOVE( outputFileName ) ), m_colourMode( CATCH_MOVE( colourMode ) ), + m_verbosity( CATCH_MOVE( verbosity ) ), m_customOptions( CATCH_MOVE( customOptions ) ) {} } // namespace Catch @@ -7234,6 +7332,9 @@ namespace Catch { namespace Catch { namespace { + // Picked small-ish number at random + static size_t kInitialTestCount = 120; + static void enforceNoDuplicateTestCases( std::vector const& tests ) { auto testInfoCmp = []( TestCaseInfo const* lhs, @@ -7335,17 +7436,26 @@ namespace Catch { return getRegistryHub().getTestCaseRegistry().getAllTestsSorted( config ); } + + TestRegistry::TestRegistry() { + // We pre-reserve some reasonable number of tests to avoid the + // initial geometric growth churning during test registration. + m_handles.reserve( kInitialTestCount ); + m_test_infos.reserve( kInitialTestCount ); + m_invokers.reserve( kInitialTestCount ); + } TestRegistry::~TestRegistry() = default; void TestRegistry::registerTest(Detail::unique_ptr testInfo, Detail::unique_ptr testInvoker) { m_handles.emplace_back(testInfo.get(), testInvoker.get()); - m_viewed_test_infos.push_back(testInfo.get()); - m_owned_test_infos.push_back(CATCH_MOVE(testInfo)); + m_test_infos.push_back(CATCH_MOVE(testInfo)); m_invokers.push_back(CATCH_MOVE(testInvoker)); } - std::vector const& TestRegistry::getAllInfos() const { - return m_viewed_test_infos; + void TestRegistry::enableFilenameTags() { + for (auto& info : m_test_infos) { + info->addFilenameTag(); + } } std::vector const& TestRegistry::getAllTests() const { @@ -9035,66 +9145,101 @@ namespace Catch { #include namespace Catch { + + namespace { + constexpr StringRef caseSensitivitySuffix( CaseSensitive caseSensitivity ) { + return caseSensitivity == CaseSensitive::Yes + ? StringRef{} + : " (case insensitive)"_sr; + } + } // namespace + namespace Matchers { - CasedString::CasedString( std::string const& str, CaseSensitive caseSensitivity ) - : m_caseSensitivity( caseSensitivity ), - m_str( adjustString( str ) ) - {} - std::string CasedString::adjustString( std::string const& str ) const { - return m_caseSensitivity == CaseSensitive::No - ? toLower( str ) - : str; - } - StringRef CasedString::caseSensitivitySuffix() const { - return m_caseSensitivity == CaseSensitive::Yes - ? StringRef() - : " (case insensitive)"_sr; - } + StringMatcherBase::StringMatcherBase( std::string target, + StringRef operation, + CaseSensitive caseSensitivity ): + m_target( CATCH_MOVE( target ) ), + m_operation( operation ), + m_caseSensitivity( caseSensitivity ) {} - StringMatcherBase::StringMatcherBase( StringRef operation, CasedString const& comparator ) - : m_comparator( comparator ), - m_operation( operation ) { - } - std::string StringMatcherBase::describe() const { std::string description; - description.reserve(5 + m_operation.size() + m_comparator.m_str.size() + - m_comparator.caseSensitivitySuffix().size()); + description.reserve(5 + m_operation.size() + m_target.size() + + caseSensitivitySuffix(m_caseSensitivity).size()); description += m_operation; description += ": \""; - description += m_comparator.m_str; + description += m_target; description += '"'; - description += m_comparator.caseSensitivitySuffix(); + description += caseSensitivitySuffix(m_caseSensitivity); return description; } - StringEqualsMatcher::StringEqualsMatcher( CasedString const& comparator ) : StringMatcherBase( "equals"_sr, comparator ) {} + + StringEqualsMatcher::StringEqualsMatcher( std::string comparator, CaseSensitive caseSensitivity ): + StringMatcherBase( CATCH_MOVE( comparator ), "equals"_sr, caseSensitivity ) {} bool StringEqualsMatcher::match( std::string const& source ) const { - return m_comparator.adjustString( source ) == m_comparator.m_str; + if (m_caseSensitivity == CaseSensitive::Yes) { + return m_target == source; + } + if (m_target.size() != source.size()) { return false; } + Catch::Detail::CaseInsensitiveEqualTo eq; + return eq( m_target, source ); } - StringContainsMatcher::StringContainsMatcher( CasedString const& comparator ) : StringMatcherBase( "contains"_sr, comparator ) {} + StringContainsMatcher::StringContainsMatcher( + std::string comparator, CaseSensitive caseSensitivity ): + StringMatcherBase( CATCH_MOVE( comparator ), "contains"_sr, caseSensitivity ) {} bool StringContainsMatcher::match( std::string const& source ) const { - return contains( m_comparator.adjustString( source ), m_comparator.m_str ); + if ( m_caseSensitivity == CaseSensitive::Yes ) { + return contains( source, m_target ); + } + if ( source.size() < m_target.size() ) { return false; } + StringRef as_ref( source ); + // The worst case of this is O(m*n), which is terrible, BUT: + // * The average case is much better, the worst case only happens rarely + // * We can implement BMH/other better searchers later if it matters + Catch::Detail::CaseInsensitiveEqualTo eq; + for (size_t i = 0; i < source.size(); ++i) { + const auto substr = as_ref.substr( i, m_target.size() ); + bool found = eq( substr, m_target ); + if ( found ) { return true; } + } + return false; } - StartsWithMatcher::StartsWithMatcher( CasedString const& comparator ) : StringMatcherBase( "starts with"_sr, comparator ) {} + StartsWithMatcher::StartsWithMatcher( std::string comparator, + CaseSensitive caseSensitivity ): + StringMatcherBase( CATCH_MOVE( comparator ), "starts with"_sr, caseSensitivity ) {} bool StartsWithMatcher::match( std::string const& source ) const { - return startsWith( m_comparator.adjustString( source ), m_comparator.m_str ); + if ( m_caseSensitivity == CaseSensitive::Yes ) { + return startsWith( source, m_target ); + } + if (source.size() < m_target.size()) { return false; } + Catch::Detail::CaseInsensitiveEqualTo eq; + return eq( + StringRef( source ).substr( 0, m_target.size() ), m_target ); } - EndsWithMatcher::EndsWithMatcher( CasedString const& comparator ) : StringMatcherBase( "ends with"_sr, comparator ) {} + EndsWithMatcher::EndsWithMatcher( std::string comparator, + CaseSensitive caseSensitivity ): + StringMatcherBase( CATCH_MOVE( comparator ), "ends with"_sr, caseSensitivity ) {} bool EndsWithMatcher::match( std::string const& source ) const { - return endsWith( m_comparator.adjustString( source ), m_comparator.m_str ); + if ( m_caseSensitivity == CaseSensitive::Yes ) { + return endsWith( source, m_target ); + } + if ( source.size() < m_target.size() ) { return false; } + Catch::Detail::CaseInsensitiveEqualTo eq; + const size_t start_point = source.size() - m_target.size(); + return eq( StringRef( source ).substr( start_point, m_target.size() ), m_target ); } @@ -9115,21 +9260,21 @@ namespace Matchers { } - StringEqualsMatcher Equals( std::string const& str, CaseSensitive caseSensitivity ) { - return StringEqualsMatcher( CasedString( str, caseSensitivity) ); + StringEqualsMatcher Equals( std::string str, CaseSensitive caseSensitivity ) { + return StringEqualsMatcher( CATCH_MOVE( str ), caseSensitivity ); } - StringContainsMatcher ContainsSubstring( std::string const& str, CaseSensitive caseSensitivity ) { - return StringContainsMatcher( CasedString( str, caseSensitivity) ); + StringContainsMatcher ContainsSubstring( std::string str, CaseSensitive caseSensitivity ) { + return StringContainsMatcher( CATCH_MOVE( str ), caseSensitivity ); } - EndsWithMatcher EndsWith( std::string const& str, CaseSensitive caseSensitivity ) { - return EndsWithMatcher( CasedString( str, caseSensitivity) ); + EndsWithMatcher EndsWith( std::string str, CaseSensitive caseSensitivity ) { + return EndsWithMatcher( CATCH_MOVE( str ), caseSensitivity ); } - StartsWithMatcher StartsWith( std::string const& str, CaseSensitive caseSensitivity ) { - return StartsWithMatcher( CasedString( str, caseSensitivity) ); + StartsWithMatcher StartsWith( std::string str, CaseSensitive caseSensitivity ) { + return StartsWithMatcher( CATCH_MOVE( str ), caseSensitivity ); } - RegexMatcher Matches(std::string const& regex, CaseSensitive caseSensitivity) { - return RegexMatcher(regex, caseSensitivity); + RegexMatcher Matches(std::string regex, CaseSensitive caseSensitivity) { + return RegexMatcher( CATCH_MOVE( regex ), caseSensitivity ); } } // namespace Matchers @@ -9231,6 +9376,7 @@ namespace Catch { m_wrapped_stream( CATCH_MOVE(config).takeStream() ), m_stream( m_wrapped_stream->stream() ), m_colour( makeColourImpl( config.colourMode(), m_wrapped_stream.get() ) ), + m_verbosity( config.verbosity() ), m_customOptions( config.customOptions() ) {} @@ -9238,12 +9384,12 @@ namespace Catch { void ReporterBase::listReporters( std::vector const& descriptions ) { - defaultListReporters( m_stream, descriptions, m_config->verbosity() ); + defaultListReporters( m_stream, descriptions, m_verbosity ); } void ReporterBase::listListeners( std::vector const& descriptions ) { - defaultListListeners( m_stream, descriptions, m_config->verbosity() ); + defaultListListeners( m_stream, descriptions, m_verbosity ); } void ReporterBase::listTests(std::vector const& tests) { @@ -9251,11 +9397,11 @@ namespace Catch { m_colour.get(), tests, m_config->hasTestFilters(), - m_config->verbosity()); + m_verbosity); } void ReporterBase::listTags(std::vector const& tags) { - defaultListTags( m_stream, tags, m_config->hasTestFilters(), m_config->verbosity() ); + defaultListTags( m_stream, tags, m_config->hasTestFilters(), m_verbosity ); } } // namespace Catch @@ -10705,6 +10851,8 @@ namespace Catch { namespace Catch { namespace { + static size_t kJsonOutputVersion = 2; + void writeSourceInfo( JsonObjectWriter& writer, SourceLineInfo const& sourceInfo ) { auto source_location_writer = @@ -10747,7 +10895,7 @@ namespace Catch { m_writers.emplace( Writer::Object ); auto& writer = m_objectWriters.top(); - writer.write( "version"_sr ).write( 1 ); + writer.write( "version"_sr ).write( kJsonOutputVersion ); { auto metadata_writer = writer.write( "metadata"_sr ).writeObject(); @@ -11033,14 +11181,18 @@ namespace Catch { auto const& info = test.getTestCaseInfo(); desc_writer.write( "name"_sr ).write( info.name ); - desc_writer.write( "class-name"_sr ).write( info.className ); - { + if (!info.className.empty()) { + desc_writer.write( "class-name"_sr ).write( info.className ); + } + if ( m_verbosity >= Verbosity::Normal ) { auto tag_writer = desc_writer.write( "tags"_sr ).writeArray(); for ( auto const& tag : info.tags ) { tag_writer.write( tag.original ); } } - writeSourceInfo( desc_writer, info.lineInfo ); + if ( m_verbosity >= Verbosity::High) { + writeSourceInfo( desc_writer, info.lineInfo ); + } } } void JsonReporter::listTags( std::vector const& tags ) { diff --git a/extras/catch_amalgamated.hpp b/extras/catch_amalgamated.hpp index a2d9a848..7a21dd51 100644 --- a/extras/catch_amalgamated.hpp +++ b/extras/catch_amalgamated.hpp @@ -6,8 +6,8 @@ // SPDX-License-Identifier: BSL-1.0 -// Catch v3.15.3 -// Generated: 2026-07-26 22:17:52.004020 +// Catch v3.16.0 +// Generated: 2026-08-25 09:29:22.652020 // ---------------------------------------------------------- // This file is an amalgamation of multiple different files. // You probably shouldn't edit it directly. @@ -1317,6 +1317,8 @@ namespace Catch { virtual void registerTranslator( Detail::unique_ptr&& translator ) = 0; virtual void registerTagAlias( std::string const& alias, std::string const& tag, SourceLineInfo const& lineInfo ) = 0; virtual void registerStartupException() noexcept = 0; + + virtual ITestCaseRegistry& getMutableTestCaseRegistry() = 0; }; IRegistryHub const& getRegistryHub(); @@ -1506,7 +1508,7 @@ namespace Catch { } // namespace Detail #elif defined(_MSC_VER) || defined(__IAR_SYSTEMS_ICC__) -#if defined(_MSVC_VER) +#if defined(_MSC_VER) #pragma optimize("", off) #elif defined(__IAR_SYSTEMS_ICC__) // For IAR the pragma only affects the following function @@ -1543,6 +1545,13 @@ namespace Catch { template inline auto invoke_deoptimized(Fn&& fn, Args&&... args) -> std::enable_if_t::value> { CATCH_FORWARD((fn)) (CATCH_FORWARD(args)...); + // In the non-void case, we pass the result through `deoptimize_value` + // to force the compiler to keep it. We have no return value here, + // but add an optimizer barrier (ideally a memory clobber) to force + // the _side effects_ of the loop be visible (e.g. writes to globals). + // Note that writes to benchmark-locals can be optimized away, as + // we would expect in normal code. + Detail::optimizer_barrier(); } } // namespace Benchmark } // namespace Catch @@ -2159,9 +2168,94 @@ namespace Catch { #include #include #include +#include namespace Catch { namespace Benchmark { + namespace Detail { + template + ExecutionPlan prepare( const IConfig& cfg, + Environment env, + BenchmarkFunction&& fun ) { + auto min_time = + env.clock_resolution.mean * Detail::minimum_ticks; + auto run_time = + std::max( min_time, + std::chrono::duration_cast( + cfg.benchmarkWarmupTime() ) ); + auto&& test = Detail::run_for_at_least( + std::chrono::duration_cast( run_time ), 1, fun ); + int new_iters = static_cast( + std::ceil( min_time * test.iterations / test.elapsed ) ); + return { new_iters, + test.elapsed / test.iterations * new_iters * + cfg.benchmarkSamples(), + CATCH_MOVE( fun ), + std::chrono::duration_cast( + cfg.benchmarkWarmupTime() ), + Detail::warmup_iterations }; + } + + // These are wrappers for their respective function templated + // over `default_clock`. This allows outlining the usual use + // of the template into single TU and save on compilation costs. + + Environment measure_environment_default(); + ExecutionPlan prepare_default( const IConfig& cfg, + Environment env, + BenchmarkFunction&& fun ); + std::vector run_plan_default( ExecutionPlan const& plan, + const IConfig& cfg, + Environment env ); + + template + std::enable_if_t::value, + Environment> + measureEnvironmentDispatch() { + return measure_environment_default(); + } + template + std::enable_if_t::value, + Environment> + measureEnvironmentDispatch() { + return measure_environment(); + } + + template + std::enable_if_t::value, + std::vector> + runPlanDispatch( ExecutionPlan const& plan, + const IConfig& cfg, + Environment env ) { + return run_plan_default( plan, cfg, env ); + } + template + std::enable_if_t::value, + std::vector> + runPlanDispatch( ExecutionPlan const& plan, + const IConfig& cfg, + Environment env ) { + return plan.template run( cfg, env ); + } + + template + std::enable_if_t::value, + ExecutionPlan> + prepareDispatch( const IConfig& cfg, + Environment env, + BenchmarkFunction&& fun ) { + return prepare_default( cfg, env, CATCH_MOVE( fun ) ); + } + template + std::enable_if_t::value, + ExecutionPlan> + prepareDispatch( const IConfig& cfg, + Environment env, + BenchmarkFunction&& fun ) { + return prepare( cfg, env, CATCH_MOVE( fun ) ); + } + } // namespace Detail + struct Benchmark { Benchmark(std::string&& benchmarkName) : name(CATCH_MOVE(benchmarkName)) {} @@ -2170,27 +2264,18 @@ namespace Catch { Benchmark(std::string&& benchmarkName , FUN &&func) : fun(CATCH_MOVE(func)), name(CATCH_MOVE(benchmarkName)) {} - template - ExecutionPlan prepare(const IConfig &cfg, Environment env) { - auto min_time = env.clock_resolution.mean * Detail::minimum_ticks; - auto run_time = std::max(min_time, std::chrono::duration_cast(cfg.benchmarkWarmupTime())); - auto&& test = Detail::run_for_at_least(std::chrono::duration_cast(run_time), 1, fun); - int new_iters = static_cast(std::ceil(min_time * test.iterations / test.elapsed)); - return { new_iters, test.elapsed / test.iterations * new_iters * cfg.benchmarkSamples(), CATCH_MOVE(fun), std::chrono::duration_cast(cfg.benchmarkWarmupTime()), Detail::warmup_iterations }; - } - template void run() { static_assert( Clock::is_steady, "Benchmarking clock should be steady" ); auto const* cfg = getCurrentContext().getConfig(); - auto env = Detail::measure_environment(); + auto env = Detail::measureEnvironmentDispatch(); getResultCapture().benchmarkPreparing(name); CATCH_TRY{ auto plan = user_code([&] { - return prepare(*cfg, env); + return Detail::prepareDispatch( *cfg, env, CATCH_MOVE(fun) ); }); BenchmarkInfo info { @@ -2206,7 +2291,7 @@ namespace Catch { getResultCapture().benchmarkStarting(info); auto samples = user_code([&] { - return plan.template run(*cfg, env); + return Detail::runPlanDispatch( plan, *cfg, env ); }); auto analysis = Detail::analyse(*cfg, samples.data(), samples.data() + samples.size()); @@ -3736,6 +3821,7 @@ namespace Catch { std::vector splitReporterSpec( StringRef reporterSpec ); Optional stringToColourMode( StringRef colourMode ); + Optional stringToVerbosity( StringRef verbosity ); } /** @@ -3750,6 +3836,7 @@ namespace Catch { std::string m_name; Optional m_outputFileName; Optional m_colourMode; + Optional m_verbosity; std::map m_customOptions; friend bool operator==( ReporterSpec const& lhs, @@ -3764,6 +3851,7 @@ namespace Catch { std::string name, Optional outputFileName, Optional colourMode, + Optional verbosity, std::map customOptions ); std::string const& name() const { return m_name; } @@ -3774,13 +3862,15 @@ namespace Catch { Optional const& colourMode() const { return m_colourMode; } + Optional const& verbosity() const { return m_verbosity; } + std::map const& customOptions() const { return m_customOptions; } }; /** - * Parses provided reporter spec string into + * Parses provided reporter spec string into actual `ReporterSpec` * * Returns empty optional on errors, e.g. * * field that is not first and not a key+value pair @@ -3813,6 +3903,7 @@ namespace Catch { std::string name; std::string outputFilename; ColourMode colourMode; + Verbosity verbosity; std::map customOptions; friend bool operator==( ProcessedReporterSpec const& lhs, ProcessedReporterSpec const& rhs ); @@ -3840,6 +3931,7 @@ namespace Catch { int abortAfter = -1; uint32_t rngSeed = generateRandomSeed(GenerateFrom::Default); + bool rngSeedWasFixed = false; unsigned int shardCount = 1; unsigned int shardIndex = 0; @@ -3911,6 +4003,7 @@ namespace Catch { double minDuration() const override; TestRunOrder runOrder() const override; uint32_t rngSeed() const override; + bool rngSeedWasFixed() const; unsigned int shardCount() const override; unsigned int shardIndex() const override; ColourMode defaultColourMode() const override; @@ -6624,6 +6717,51 @@ namespace Catch { struct priority_tag : priority_tag {}; template <> struct priority_tag<0> {}; + + // This is a bunch of helpers for the templated test case handling. + // They should live elsewhere in the long run, but as an in-between + // step we toss them all here. + template struct TypeList {}; + template + constexpr auto get_wrapper( priority_tag<1> ) noexcept -> TypeList { return {}; } + template