Support filtering on both sections and generators (#3069)

Not being able to filter generators to specific element has been regularly
causing problems. It was possible to use a dynamic section to run tests
for specific element in a generator, at least if the element had a nice
string representation, but the test case would still run once per element
in the generator.

With this change, it is possible to have the generator return only one
specific element, and do so based on the index, rather than the string
representation of the element. This enables simple debugging of tests
that fail for specific generator element.
This commit is contained in:
Martin Hořeňovský authored and GitHub committed 2026-02-15 20:27:15 +01:00
1 parent b6c7b217d4
commit 3a0cf7e75f
42 files changed
+1716 -159

No files matched your search

+3
View File
@@ -108,6 +108,7 @@ set(IMPL_HEADERS
${SOURCES_DIR}/internal/catch_optional.hpp
${SOURCES_DIR}/internal/catch_output_redirect.hpp
${SOURCES_DIR}/internal/catch_parse_numbers.hpp
${SOURCES_DIR}/internal/catch_path_filter.hpp
${SOURCES_DIR}/internal/catch_platform.hpp
${SOURCES_DIR}/internal/catch_polyfills.hpp
${SOURCES_DIR}/internal/catch_preprocessor.hpp
@@ -255,11 +256,13 @@ set(GENERATOR_HEADERS
${SOURCES_DIR}/generators/catch_generators_all.hpp
${SOURCES_DIR}/generators/catch_generators_random.hpp
${SOURCES_DIR}/generators/catch_generators_range.hpp
${SOURCES_DIR}/generators/catch_generators_throw.hpp
)
set(GENERATOR_SOURCES
${SOURCES_DIR}/generators/catch_generator_exception.cpp
${SOURCES_DIR}/generators/catch_generators.cpp
${SOURCES_DIR}/generators/catch_generators_random.cpp
${SOURCES_DIR}/generators/catch_generators_throw.cpp
)
set(GENERATOR_FILES ${GENERATOR_HEADERS} ${GENERATOR_SOURCES})
+1
View File
@@ -89,6 +89,7 @@
#include <catch2/internal/catch_optional.hpp>
#include <catch2/internal/catch_output_redirect.hpp>
#include <catch2/internal/catch_parse_numbers.hpp>
#include <catch2/internal/catch_path_filter.hpp>
#include <catch2/internal/catch_platform.hpp>
#include <catch2/internal/catch_polyfills.hpp>
#include <catch2/internal/catch_preprocessor.hpp>
+6 -4
View File
@@ -92,6 +92,10 @@ namespace Catch {
lhs.customOptions == rhs.customOptions;
}
bool operator==( PathFilter const& lhs, PathFilter const& rhs ) {
return lhs.type == rhs.type && lhs.filter == rhs.filter;
}
Config::Config( ConfigData const& data ):
m_data( data ) {
// We need to trim filter specs to avoid trouble with superfluous
@@ -101,9 +105,6 @@ namespace Catch {
for (auto& elem : m_data.testsOrTags) {
elem = trim(elem);
}
for (auto& elem : m_data.sectionsToRun) {
elem = trim(elem);
}
// Insert the default reporter if user hasn't asked for a specific one
if ( m_data.reporterSpecifications.empty() ) {
@@ -169,7 +170,8 @@ namespace Catch {
bool Config::listListeners() const { return m_data.listListeners; }
std::vector<std::string> const& Config::getTestsOrTags() const { return m_data.testsOrTags; }
std::vector<std::string> const& Config::getSectionsToRun() const { return m_data.sectionsToRun; }
std::vector<PathFilter> const& Config::getPathFilters() const { return m_data.pathFilters; }
bool Config::useNewFilterBehaviour() const { return m_data.useNewPathFilteringBehaviour; }
std::vector<ReporterSpec> const& Config::getReporterSpecs() const {
return m_data.reporterSpecifications;
+5 -2
View File
@@ -12,6 +12,7 @@
#include <catch2/interfaces/catch_interfaces_config.hpp>
#include <catch2/internal/catch_unique_ptr.hpp>
#include <catch2/internal/catch_optional.hpp>
#include <catch2/internal/catch_path_filter.hpp>
#include <catch2/internal/catch_stringref.hpp>
#include <catch2/internal/catch_random_seed_generation.hpp>
#include <catch2/internal/catch_reporter_spec_parser.hpp>
@@ -86,7 +87,8 @@ namespace Catch {
std::vector<ReporterSpec> reporterSpecifications;
std::vector<std::string> testsOrTags;
std::vector<std::string> sectionsToRun;
std::vector<PathFilter> pathFilters;
bool useNewPathFilteringBehaviour = false;
std::string prematureExitGuardFilePath;
};
@@ -109,7 +111,8 @@ namespace Catch {
getProcessedReporterSpecs() const;
std::vector<std::string> const& getTestsOrTags() const override;
std::vector<std::string> const& getSectionsToRun() const override;
std::vector<PathFilter> const& getPathFilters() const override;
bool useNewFilterBehaviour() const override;
TestSpec const& testSpec() const override;
bool hasTestFilters() const override;
@@ -7,8 +7,6 @@
// SPDX-License-Identifier: BSL-1.0
#include <catch2/generators/catch_generators.hpp>
#include <catch2/internal/catch_enforce.hpp>
#include <catch2/generators/catch_generator_exception.hpp>
#include <catch2/interfaces/catch_interfaces_capture.hpp>
namespace Catch {
@@ -17,14 +15,6 @@ namespace Catch {
namespace Generators {
namespace Detail {
[[noreturn]]
void throw_generator_exception(char const* msg) {
Catch::throw_exception(GeneratorException{ msg });
}
} // end namespace Detail
GeneratorUntypedBase::~GeneratorUntypedBase() = default;
IGeneratorTracker* acquireGeneratorTracker(StringRef generatorName, SourceLineInfo const& lineInfo ) {
+1 -8
View File
@@ -9,6 +9,7 @@
#define CATCH_GENERATORS_HPP_INCLUDED
#include <catch2/catch_tostring.hpp>
#include <catch2/generators/catch_generators_throw.hpp>
#include <catch2/interfaces/catch_interfaces_generatortracker.hpp>
#include <catch2/internal/catch_source_line_info.hpp>
#include <catch2/internal/catch_stringref.hpp>
@@ -22,14 +23,6 @@ namespace Catch {
namespace Generators {
namespace Detail {
//! Throws GeneratorException with the provided message
[[noreturn]]
void throw_generator_exception(char const * msg);
} // end namespace detail
template<typename T>
class IGenerator : public GeneratorUntypedBase {
std::string stringifyImpl() const override {
@@ -26,5 +26,6 @@
#include <catch2/generators/catch_generators_adapters.hpp>
#include <catch2/generators/catch_generators_random.hpp>
#include <catch2/generators/catch_generators_range.hpp>
#include <catch2/generators/catch_generators_throw.hpp>
#endif // CATCH_GENERATORS_ALL_HPP_INCLUDED
@@ -0,0 +1,24 @@
// Copyright Catch2 Authors
// Distributed under the Boost Software License, Version 1.0.
// (See accompanying file LICENSE.txt or copy at
// https://www.boost.org/LICENSE_1_0.txt)
// SPDX-License-Identifier: BSL-1.0
#include <catch2/generators/catch_generator_exception.hpp>
#include <catch2/generators/catch_generators_throw.hpp>
#include <catch2/internal/catch_enforce.hpp>
namespace Catch {
namespace Generators {
namespace Detail {
[[noreturn]]
void throw_generator_exception( char const* msg ) {
Catch::throw_exception( GeneratorException{ msg } );
}
} // namespace Detail
} // namespace Generators
} // namespace Catch
@@ -0,0 +1,23 @@
// Copyright Catch2 Authors
// Distributed under the Boost Software License, Version 1.0.
// (See accompanying file LICENSE.txt or copy at
// https://www.boost.org/LICENSE_1_0.txt)
// SPDX-License-Identifier: BSL-1.0
#ifndef CATCH_GENERATORS_THROW_HPP_INCLUDED
#define CATCH_GENERATORS_THROW_HPP_INCLUDED
namespace Catch {
namespace Generators {
namespace Detail {
//! Throws GeneratorException with the provided message
[[noreturn]]
void throw_generator_exception( char const* msg );
} // namespace Detail
} // namespace Generators
} // namespace Catch
#endif // CATCH_GENERATORS_THROW_HPP_INCLUDED
@@ -62,6 +62,7 @@ namespace Catch {
class TestSpec;
class IStream;
struct PathFilter;
class IConfig : public Detail::NonCopyable {
public:
@@ -87,7 +88,9 @@ namespace Catch {
virtual unsigned int shardCount() const = 0;
virtual unsigned int shardIndex() const = 0;
virtual ColourMode defaultColourMode() const = 0;
virtual std::vector<std::string> const& getSectionsToRun() const = 0;
virtual std::vector<PathFilter> const& getPathFilters() const = 0;
virtual bool useNewFilterBehaviour() const = 0;
virtual Verbosity verbosity() const = 0;
virtual bool skipBenchmarks() const = 0;
+44 -1
View File
@@ -216,6 +216,43 @@ namespace Catch {
return ParserResult::ok( ParseResultType::Matched );
};
auto const setSectionFilter = [&]( std::string const& sectionFilter ) {
config.pathFilters.emplace_back( PathFilter::For::Section, trim(sectionFilter) );
return ParserResult::ok( ParseResultType::Matched );
};
auto const setGeneratorFilter = [&]( std::string const& generatorFilter ) {
if (generatorFilter != "*") {
// TODO: avoid re-parsing the index?
auto parsedIndex = parseUInt( generatorFilter );
if ( !parsedIndex ) {
return ParserResult::runtimeError( "Could not parse '" +
generatorFilter +
"' as generator index" );
}
}
config.useNewPathFilteringBehaviour = true;
config.pathFilters.emplace_back( PathFilter::For::Generator, trim(generatorFilter) );
return ParserResult::ok( ParseResultType::Matched );
};
// Copy-capturing other `setFoo` functions enables calling them later,
// as the config ref remains valid, but the local lambda vars won't.
auto const setPathFilter = [=, &config]( std::string const& pathFilter ) {
config.useNewPathFilteringBehaviour = true;
if ( pathFilter.size() < 3 ) {
return ParserResult::runtimeError(
"Path filter '" + pathFilter + "' is too short" );
}
if ( startsWith( pathFilter, "g:" ) ) {
return setGeneratorFilter( pathFilter.substr( 2 ) );
}
if ( startsWith( pathFilter, "c:" ) ) {
return setSectionFilter( pathFilter.substr( 2 ) );
}
return ParserResult::runtimeError( "Path filter '" + pathFilter +
"' has unknown type prefix" );
};
auto cli
= ExeName( config.processName )
| Help( config.showHelp )
@@ -261,9 +298,15 @@ namespace Catch {
| Opt( config.filenamesAsTags )
["-#"]["--filenames-as-tags"]
( "adds a tag for the filename" )
| Opt( config.sectionsToRun, "section name" )
| Opt( accept_many, setSectionFilter, "section name" )
["-c"]["--section"]
( "specify section to run" )
| Opt( accept_many, setGeneratorFilter, "index spec" )
["-g"]["--generator-index"]
( "specify generator elements to try" )
| Opt( accept_many, setPathFilter, "path filter spec" )
["-p"]["--path-filter"]
( "qualified path filter" )
| Opt( setVerbosity, "quiet|normal|high" )
["-v"]["--verbosity"]
( "set output verbosity" )
+33
View File
@@ -0,0 +1,33 @@
// Copyright Catch2 Authors
// Distributed under the Boost Software License, Version 1.0.
// (See accompanying file LICENSE.txt or copy at
// https://www.boost.org/LICENSE_1_0.txt)
// SPDX-License-Identifier: BSL-1.0
#ifndef CATCH_PATH_FILTER_HPP_INCLUDED
#define CATCH_PATH_FILTER_HPP_INCLUDED
#include <catch2/internal/catch_move_and_forward.hpp>
#include <string>
namespace Catch {
struct PathFilter {
enum class For {
Section,
Generator,
};
PathFilter( For type_, std::string filter_ ):
type( type_ ), filter( CATCH_MOVE( filter_ ) ) {}
For type;
std::string filter;
friend bool operator==( PathFilter const& lhs, PathFilter const& rhs );
};
} // end namespace Catch
#endif // CATCH_PATH_FILTER_HPP_INCLUDED
+51 -18
View File
@@ -8,6 +8,7 @@
#include <catch2/internal/catch_run_context.hpp>
#include <catch2/catch_user_config.hpp>
#include <catch2/generators/catch_generators_throw.hpp>
#include <catch2/interfaces/catch_interfaces_config.hpp>
#include <catch2/interfaces/catch_interfaces_generatortracker.hpp>
#include <catch2/interfaces/catch_interfaces_reporter.hpp>
@@ -19,6 +20,7 @@
#include <catch2/catch_timer.hpp>
#include <catch2/internal/catch_output_redirect.hpp>
#include <catch2/internal/catch_assertion_handler.hpp>
#include <catch2/internal/catch_path_filter.hpp>
#include <catch2/internal/catch_test_failure_exception.hpp>
#include <catch2/internal/catch_thread_local.hpp>
#include <catch2/internal/catch_unreachable.hpp>
@@ -35,6 +37,9 @@ namespace Catch {
struct GeneratorTracker final : TestCaseTracking::TrackerBase,
IGeneratorTracker {
GeneratorBasePtr m_generator;
// Filtered generator has moved to specific index due to
// a filter, it needs special handling of `countedNext()`
bool m_isFiltered = false;
GeneratorTracker(
TestCaseTracking::NameAndLocation&& nameAndLocation,
@@ -45,6 +50,32 @@ namespace Catch {
m_generator( CATCH_MOVE( generator ) ) {
assert( m_generator &&
"Cannot create tracker without generator" );
// Handle potential filter and move forward here...
// Old style filters do not affect generators at all
if (m_newStyleFilters && m_allTrackerDepth < m_filterRef->size()) {
auto const& filter =
( *m_filterRef )[m_allTrackerDepth];
// Generator cannot be un-entered the way a section
// can be, so the tracker has to throw for a wrong
// filter to stop the execution flow.
if (filter.type == PathFilter::For::Section) {
// TBD: Explicit SKIP, or new exception that says
// "don't continue", but doesn't show in totals?
SKIP();
}
// '*' is the wildcard for "all elements in generator"
// used for filtering sections below the generator, but
// not the generator itself.
if ( filter.filter != "*" ) {
m_isFiltered = true;
// TBD: We assume that the filter was validated as
// number during parsing. We should pass it
// as number from the CLI parser.
size_t targetIndex = std::stoul( filter.filter );
m_generator->skipToNthElement( targetIndex );
}
}
}
static GeneratorTracker*
@@ -117,25 +148,24 @@ namespace Catch {
// _can_ start, and thus we should wait for them, or
// they cannot start (due to filters), and we shouldn't
// wait for them
ITracker* parent = m_parent;
// This is safe: there is always at least one section
// tracker in a test case tracking tree
while ( !parent->isSectionTracker() ) {
parent = parent->parent();
// No filters left -> no restrictions on running sections
size_t childDepth = 1 + (m_newStyleFilters ? m_allTrackerDepth : m_sectionOnlyDepth);
if ( childDepth >= m_filterRef->size() ) {
return true;
}
assert( parent &&
"Missing root (test case) level section" );
auto const& parentSection =
static_cast<SectionTracker const&>( *parent );
auto const& filters = parentSection.getFilters();
// No filters -> no restrictions on running sections
if ( filters.empty() ) { return true; }
// If we are using the new style filters, we need to check
// whether the successive filter is for section or a generator.
if ( m_newStyleFilters
&& (*m_filterRef)[childDepth].type != PathFilter::For::Section ) {
return false;
}
// Look for any child section that could match the remaining filters
for ( auto const& child : m_children ) {
if ( child->isSectionTracker() &&
static_cast<SectionTracker const&>( *child )
.trimmedName() == filters[0] ) {
.trimmedName() == StringRef((*m_filterRef)[childDepth].filter) ) {
return true;
}
}
@@ -147,9 +177,10 @@ namespace Catch {
// value, but we do not want to invoke the side-effect if
// this generator is still waiting for any child to start.
assert( m_generator && "Tracker without generator" );
if ( should_wait_for_child ||
( m_runState == CompletedSuccessfully &&
m_generator->countedNext() ) ) {
if ( should_wait_for_child
|| ( m_runState == CompletedSuccessfully
&& !m_isFiltered // filtered generators cannot meaningfully move forward, as they would get past the filter
&& m_generator->countedNext() ) ) {
m_children.clear();
m_runState = Executing;
}
@@ -312,7 +343,8 @@ namespace Catch {
ITracker& rootTracker = m_trackerContext.startRun();
assert(rootTracker.isSectionTracker());
static_cast<SectionTracker&>(rootTracker).addInitialFilters(m_config->getSectionsToRun());
rootTracker.setFilters( &m_config->getPathFilters(),
m_config->useNewFilterBehaviour() );
// We intentionally only seed the internal RNG once per test case,
// before it is first invoked. The reason for that is a complex
@@ -484,6 +516,7 @@ namespace Catch {
SourceLineInfo lineInfo,
Generators::GeneratorBasePtr&& generator ) {
// TBD: Do we want to avoid the warning if the generator is filtered?
if ( m_config->warnAboutInfiniteGenerators() &&
!generator->isFinite() ) {
// TBD: Would it be better to expand this macro inline?
+34 -28
View File
@@ -8,8 +8,9 @@
#include <catch2/internal/catch_test_case_tracker.hpp>
#include <catch2/internal/catch_enforce.hpp>
#include <catch2/internal/catch_string_manip.hpp>
#include <catch2/internal/catch_move_and_forward.hpp>
#include <catch2/internal/catch_path_filter.hpp>
#include <catch2/internal/catch_string_manip.hpp>
#include <algorithm>
#include <cassert>
@@ -27,6 +28,17 @@ namespace TestCaseTracking {
location( _location )
{}
ITracker::ITracker( NameAndLocation&& nameAndLoc, ITracker* parent ):
m_nameAndLocation( CATCH_MOVE( nameAndLoc ) ), m_parent( parent ) {
if ( m_parent ) {
m_allTrackerDepth = m_parent->m_allTrackerDepth + 1;
// We leave section trackers to bump themselves up, as
// we cannot use `isSectionTracker` in constructor
m_sectionOnlyDepth = m_parent->m_sectionOnlyDepth;
m_filterRef = m_parent->m_filterRef;
m_newStyleFilters = m_parent->m_newStyleFilters;
}
}
ITracker::~ITracker() = default;
@@ -159,25 +171,32 @@ namespace TestCaseTracking {
: TrackerBase( CATCH_MOVE(nameAndLocation), ctx, parent ),
m_trimmed_name(trim(StringRef(ITracker::nameAndLocation().name)))
{
if( parent ) {
while ( !parent->isSectionTracker() ) {
parent = parent->parent();
}
SectionTracker& parentSection = static_cast<SectionTracker&>( *parent );
addNextFilters( parentSection.m_filters );
if( m_parent ) {
++m_sectionOnlyDepth;
}
}
bool SectionTracker::isComplete() const {
bool complete = true;
if ( m_filters.empty()
|| m_filters[0].empty()
|| m_filters[0] == m_trimmed_name ) {
complete = TrackerBase::isComplete();
// If there are active filters AND we do not pass them,
// the section is always "completed"
const size_t filterIndex =
m_newStyleFilters ? m_allTrackerDepth : m_sectionOnlyDepth;
if ( filterIndex < m_filterRef->size() ) {
// There is active filter, check it
// 1) New style filter must explicitly target section
if ( m_newStyleFilters && ( *m_filterRef )[filterIndex].type !=
PathFilter::For::Section ) {
return true;
}
// 2) Both style filters must match the trimmed name exactly
if ( m_trimmed_name !=
StringRef( ( *m_filterRef )[filterIndex].filter ) ) {
return true;
}
}
return complete;
// Otherwise we delegate to the generic processing
return TrackerBase::isComplete();
}
bool SectionTracker::isSectionTracker() const { return true; }
@@ -213,19 +232,6 @@ namespace TestCaseTracking {
open();
}
void SectionTracker::addInitialFilters( std::vector<std::string> const& filters ) {
if( !filters.empty() ) {
m_filters.reserve( m_filters.size() + filters.size() + 2 );
m_filters.emplace_back(StringRef{}); // Root - should never be consulted
m_filters.emplace_back(StringRef{}); // Test Case - not a section filter
m_filters.insert( m_filters.end(), filters.begin(), filters.end() );
}
}
void SectionTracker::addNextFilters( std::vector<StringRef> const& filters ) {
if( filters.size() > 1 )
m_filters.insert( m_filters.end(), filters.begin()+1, filters.end() );
}
StringRef SectionTracker::trimmedName() const {
return m_trimmed_name;
}
+25 -11
View File
@@ -17,6 +17,9 @@
#include <vector>
namespace Catch {
struct PathFilter;
namespace TestCaseTracking {
struct NameAndLocation {
@@ -92,12 +95,23 @@ namespace TestCaseTracking {
Children m_children;
CycleState m_runState = NotStarted;
public:
ITracker( NameAndLocation&& nameAndLoc, ITracker* parent ):
m_nameAndLocation( CATCH_MOVE(nameAndLoc) ),
m_parent( parent )
{}
// Members for path filtering
std::vector<PathFilter> const* m_filterRef = nullptr;
// Note: There are 2 dummy section trackers (root, test-case) before
// the first "real" section tracker can be encountered. We start
// the default tracker at -2, so that the first "real" section
// tracker overflows to index 0.
// Nesting depth of this tracker, used to decide which new-style filter applies.
size_t m_allTrackerDepth = static_cast<size_t>( -2 );
// Nesting depth of sections (inc. this tracker), used for old-style filters.
// Must be updated by the section tracker on its own.
size_t m_sectionOnlyDepth = static_cast<size_t>( -2 );
// Transitory: Remove once we remove backwards compatibility with old-style (v3.x) filters
bool m_newStyleFilters = false;
public:
ITracker( NameAndLocation&& nameAndLoc, ITracker* parent );
// static queries
NameAndLocation const& nameAndLocation() const {
@@ -123,6 +137,11 @@ namespace TestCaseTracking {
//! Returns true iff tracker has started
bool hasStarted() const;
void setFilters( std::vector<PathFilter> const* filters, bool newStyleFilters ) {
m_filterRef = filters;
m_newStyleFilters = newStyleFilters;
}
// actions
virtual void close() = 0; // Successfully complete
virtual void fail() = 0;
@@ -208,8 +227,7 @@ namespace TestCaseTracking {
void moveToThis();
};
class SectionTracker : public TrackerBase {
std::vector<StringRef> m_filters;
class SectionTracker final : public TrackerBase {
// Note that lifetime-wise we piggy back off the name stored in the `ITracker` parent`.
// Currently it allocates owns the name, so this is safe. If it is later refactored
// to not own the name, the name still has to outlive the `ITracker` parent, so
@@ -226,10 +244,6 @@ namespace TestCaseTracking {
void tryOpen();
void addInitialFilters( std::vector<std::string> const& filters );
void addNextFilters( std::vector<StringRef> const& filters );
//! Returns filters active in this tracker
std::vector<StringRef> const& getFilters() const { return m_filters; }
//! Returns whitespace-trimmed name of the tracked section
StringRef trimmedName() const;
};
+3
View File
@@ -61,6 +61,7 @@ internal_headers = [
'generators/catch_generators_all.hpp',
'generators/catch_generators_random.hpp',
'generators/catch_generators_range.hpp',
'generators/catch_generators_throw.hpp',
'interfaces/catch_interfaces_all.hpp',
'interfaces/catch_interfaces_capture.hpp',
'interfaces/catch_interfaces_config.hpp',
@@ -115,6 +116,7 @@ internal_headers = [
'internal/catch_optional.hpp',
'internal/catch_output_redirect.hpp',
'internal/catch_parse_numbers.hpp',
'internal/catch_path_filter.hpp',
'internal/catch_platform.hpp',
'internal/catch_polyfills.hpp',
'internal/catch_preprocessor.hpp',
@@ -203,6 +205,7 @@ internal_sources = files(
'generators/catch_generator_exception.cpp',
'generators/catch_generators.cpp',
'generators/catch_generators_random.cpp',
'generators/catch_generators_throw.cpp',
'interfaces/catch_interfaces_capture.cpp',
'interfaces/catch_interfaces_config.cpp',
'interfaces/catch_interfaces_exception.cpp',