From 3a0cf7e75fdc571ff9ed25303fb268665f6260c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Ho=C5=99e=C5=88ovsk=C3=BD?= Date: Sun, 15 Feb 2026 20:27:15 +0100 Subject: [PATCH] Support filtering on both sections and generators (#3069) Not being able to filter generators to specific element has been regularly causing problems. It was possible to use a dynamic section to run tests for specific element in a generator, at least if the element had a nice string representation, but the test case would still run once per element in the generator. With this change, it is possible to have the generator return only one specific element, and do so based on the index, rather than the string representation of the element. This enables simple debugging of tests that fail for specific generator element. --- docs/Readme.md | 3 +- docs/command-line.md | 46 +--- docs/deprecations.md | 11 + docs/filtering-execution-path.md | 222 ++++++++++++++++++ src/CMakeLists.txt | 3 + src/catch2/catch_all.hpp | 1 + src/catch2/catch_config.cpp | 10 +- src/catch2/catch_config.hpp | 7 +- src/catch2/generators/catch_generators.cpp | 10 - src/catch2/generators/catch_generators.hpp | 9 +- .../generators/catch_generators_all.hpp | 1 + .../generators/catch_generators_throw.cpp | 24 ++ .../generators/catch_generators_throw.hpp | 23 ++ .../interfaces/catch_interfaces_config.hpp | 5 +- src/catch2/internal/catch_commandline.cpp | 45 +++- src/catch2/internal/catch_path_filter.hpp | 33 +++ src/catch2/internal/catch_run_context.cpp | 69 ++++-- .../internal/catch_test_case_tracker.cpp | 62 ++--- .../internal/catch_test_case_tracker.hpp | 36 ++- src/catch2/meson.build | 3 + tests/CMakeLists.txt | 27 ++- .../Baselines/automake.sw.approved.txt | 1 + .../Baselines/automake.sw.multi.approved.txt | 1 + .../Baselines/compact.sw.approved.txt | 27 ++- .../Baselines/compact.sw.multi.approved.txt | 27 ++- .../Baselines/console.std.approved.txt | 4 +- .../Baselines/console.sw.approved.txt | 189 ++++++++++++++- .../Baselines/console.sw.multi.approved.txt | 189 ++++++++++++++- .../SelfTest/Baselines/junit.sw.approved.txt | 13 +- .../Baselines/junit.sw.multi.approved.txt | 13 +- .../Baselines/sonarqube.sw.approved.txt | 11 + .../Baselines/sonarqube.sw.multi.approved.txt | 11 + tests/SelfTest/Baselines/tap.sw.approved.txt | 48 +++- .../Baselines/tap.sw.multi.approved.txt | 48 +++- .../Baselines/teamcity.sw.approved.txt | 2 + .../Baselines/teamcity.sw.multi.approved.txt | 2 + tests/SelfTest/Baselines/xml.sw.approved.txt | 221 ++++++++++++++++- .../Baselines/xml.sw.multi.approved.txt | 221 ++++++++++++++++- .../IntrospectiveTests/CmdLine.tests.cpp | 77 ++++++ .../IntrospectiveTests/Details.tests.cpp | 42 +++- .../IntrospectiveTests/PartTracker.tests.cpp | 5 +- tests/TestScripts/testSectionFiltering.py | 73 +++++- 42 files changed, 1716 insertions(+), 159 deletions(-) create mode 100644 docs/filtering-execution-path.md create mode 100644 src/catch2/generators/catch_generators_throw.cpp create mode 100644 src/catch2/generators/catch_generators_throw.hpp create mode 100644 src/catch2/internal/catch_path_filter.hpp diff --git a/docs/Readme.md b/docs/Readme.md index f4230506..fb54f917 100644 --- a/docs/Readme.md +++ b/docs/Readme.md @@ -24,7 +24,8 @@ Once you're up and running consider the following reference material. * [String Conversions](tostring.md#top) **Running:** -* [Command line](command-line.md#top) +* [Command line reference](command-line.md#top) +* [Running specific section/generator](filtering-execution-path.md#top) **Odds and ends:** * [Frequently Asked Questions (FAQ)](faq.md#top) diff --git a/docs/command-line.md b/docs/command-line.md index 640fb1d9..fbc4609d 100644 --- a/docs/command-line.md +++ b/docs/command-line.md @@ -52,6 +52,8 @@ Click one of the following links to take you straight to that option - or scroll ` -d, --durations`
` -f, --input-file`
` -c, --section`
+ ` -g, --generator-index`
+ ` -p, --path-filter`
` -#, --filenames-as-tags`
@@ -532,45 +534,19 @@ Prints the command line arguments to stdout -## Specify the section to run + +## Specify the section/generator element to run 
-c, --section <section name>
+
-g, --generator-index <index in generator>
+
-p, --path-filter <path filter spec>
-To limit execution to a specific section within a test case, use this option one or more times. -To narrow to sub-sections use multiple instances, where each subsequent instance specifies a deeper nesting level. +> The generator and generic path filtering was added in Catch2 vX.Y.Z -E.g. if you have: +These arguments allow you to run specific section(s) in a test case, or +only get specific element from a generator. All the variants form a shared +stack of filters. -
-TEST_CASE( "Test" ) {
-  SECTION( "sa" ) {
-    SECTION( "sb" ) {
-      /*...*/
-    }
-    SECTION( "sc" ) {
-      /*...*/
-    }
-  }
-  SECTION( "sd" ) {
-    /*...*/
-  }
-}
-
- -Then you can run `sb` with: -
./MyExe Test -c sa -c sb
- -Or run just `sd` with: -
./MyExe Test -c sd
- -To run all of `sa`, including `sb` and `sc` use: -
./MyExe Test -c sa
- -There are some limitations of this feature to be aware of: -- Code outside of sections being skipped will still be executed - e.g. any set-up code in the TEST_CASE before the -start of the first section.
-- At time of writing, wildcards are not supported in section names. -- If you specify a section without narrowing to a test case first then all test cases will be executed -(but only matching sections within them). +[See the full documentation of path filtering for more details](filtering-execution-path.md#top) diff --git a/docs/deprecations.md b/docs/deprecations.md index b7efa862..149f33b3 100644 --- a/docs/deprecations.md +++ b/docs/deprecations.md @@ -48,6 +48,17 @@ If you are mutating the fixture instance from within the test case, and want to keep doing so in the future, mark the mutated members as `mutable`. +### Section-only filtering with `-c/--section` + +> Deprecated in Catch2 vX.Y.Z + +Currently, if you use only `-c/--section` parameters to decide which +sections to enter, the filtering ignores generators completely. In the +future, using only `-c/--section` will behave the same way as if you +specified the filters through the new `-p/--path-filter` parameter, which +means that generators are taken into account. + + ### Generator interfaces #### Defaulted `UntypedGeneratorBase::isFinite()` diff --git a/docs/filtering-execution-path.md b/docs/filtering-execution-path.md new file mode 100644 index 00000000..a3dd675b --- /dev/null +++ b/docs/filtering-execution-path.md @@ -0,0 +1,222 @@ + +# How to run specific section/generator + +> The generator and generic path filtering was added in Catch2 vX.Y.Z + +Catch2 supports picking specific path through a test case by filtering +sections and generator indices to run through. This is done by using one +of the three commandline parameters, one or more times. + +``` +-c, --section
+-g, --generator-index +-p, --path-filter +``` + +All the variants form a shared stack of filters, but if you use only +`-c`/`--section` form to specify section filters, you will get the old +behaviour, which does not affect generators at all. If you also use either +`-g`/`--generator-index`, or `-p`/`--path-filter`, you will get the new +behaviour, which can also filter generator elements. + +Both the new and old filter behaviours include some potentially surprising +things: + * Code outside of sections being skipped will still be executed. E.g. + any setup code in the TEST_CASE that lives outside of sections. + * Path filters filter the prefix of the path. So if you specify single + filter, it affects only the top level sections/generator, with their + child sections/generators being unfiltered. + * Path filters are independent of test case selection, Catch2 will try + to follow the path filters in all selected test cases. This means + that if you specify path filters without a test case filter, Catch2 + will try to apply the path filters inside every registered test case. + +## Old behaviour + +> The old behaviour was deprecated in Catch2 vX.Y.Z + +``` +-c, --section
+``` +The argument to `-c`/`--section` can be any arbitrary string. When Catch2 +is deciding whether to enter a section, it will check its trimmed name +against the appropriate trimmed section filter. If they are the same, +the section can be opened. If not, Catch2 will skip over that section. + + +### Examples + +#### Simple section nesting +Given +```cpp +TEST_CASE( "foo" ) { + REQUIRE( true ); + SECTION( "A" ) { + SECTION( "A1" ) { REQUIRE( true ); } + SECTION( "A2" ) { REQUIRE( true ); } + } + SECTION( "B" ) { + SECTION( "B1" ) { REQUIRE( true ); } + SECTION( "B2" ) { REQUIRE( true ); } + } +} +``` + +* `./tests foo -c A` runs section "A" and both of its subsections, + resulting in 4 assertions. +* `./tests foo -c A -c B` runs section "A", but none of its subsections, + resulting in 1 assertion (the one before "A"). +* `./tests foo -c A -c A1` runs section "A" and only the "A1" subsection, + resulting in 2 assertions. + + +#### Sections with nested generators +Note that old behaviour completely _ignores_ generators. This means both +that they can't be filtered, but also that they aren't taken into account +for the filter depth. In other words, given +```cpp +TEST_CASE( "bar" ) { + REQUIRE( true ); + SECTION( "A" ) { REQUIRE( true ); } + SECTION( "B" ) { + auto i = GENERATE( 1, 2, 3 ); + DYNAMIC_SECTION( "i=" << i ) { + REQUIRE( true ); + } + } +} +``` +* `./tests bar -c A` results in 2 assertions. +* `./tests bar -c B -c i=2` results in 4 assertions, because the whole + generator in section "B" has to be used up, but the dynamic section is + only entered when the generator returns 2 as the value for `i`. +* `./tests bar -c B -c i=4` results in 3 assertions, because the assertion + outside of section is executed every time the test case is entered, and + the generator forces the test case to rerun 3 times before it is used up, + even though the dynamic section will never be entered. + + +#### Section with sibling generators +For cases where sections have sibling generators, the filtering can get +even more surprising. +```cpp +TEST_CASE( "qux" ) { + REQUIRE( true ); + SECTION( "A" ) { REQUIRE( true ); } + auto i = GENERATE( 1, 2, 3 ); + DYNAMIC_SECTION( "i=" << i ) { + REQUIRE( true ); + } +} +``` +* `./tests qux -c A` results in **4** assertions, because section "A" is + entered once, but the sibling generator has to be exhausted, and + the first assertion is executed once per generator element. +* `./tests qux -c i=2` also results in 4 assertions. Once again, + the generator has to be exhausted and the dynamic section is entered + once. + + +## New behaviour + +> The new behaviour was introduced in Catch2 vX.Y.Z + +``` +-g, --generator-index +-p, --path-filter +``` + +The argument to `-g`/`--generator-index` must be either a non-negative +number, which is interpreted as the index of the desired element from +the generator, or "\*", which allows all elements from the generator. + +Providing index outside of the generator is an error. + +The argument to `-p`/`--path-filter` must start with either "c:" for +a section filter, or with "g:" for a generator filter. Everything past +the colon is then parsed as either a section filter, or a generator filter. + +Note that using `p`/`--path-filter` enables new filtering behaviour, even +if it is only used to add section filters. + +There is another important difference between filtering out sections and +generators. A section can be left un-entered, but a generator always has +to be active. For this reason, if generator fails a filter +(e.g. there is a section filter at given depth instead), it has to stop +the execution of the test case. Currently, this is done via `SKIP()` +equivalent, causing the section to be considered skipped. + + +### Examples + +#### Nested generators +```cpp +TEST_CASE( "waldo" ) { + auto i = GENERATE( 1, 10, 100 ); + auto j = GENERATE( 2, 20, 200 ); + CAPTURE( i, j ); + REQUIRE( true ); +} +``` +* `./tests waldo -g 1` results in 3 assertions, with `i := 10`, because + the second nested generator is unfiltered. +* `./tests waldo -g 1 -g 2` results in 1 assertion, with `i := 10, j := 200`. +* `./tests waldo -g * -g 2` results in 3 assertions, all with `j := 200`. +* `./tests waldo -g 1 -g *` results in 3 assertions, all with `i := 10`. +* `./tests waldo -g 3` results in 1 **failed** assertion, because the first + generator does not have 3rd element. +* `./tests waldo -g * -g 3` results in 3 **failed** assertions, as the + second generator does not have 3rd element, but we have to exhaust the + first generator. + +#### Generator with a nested dynamic section +```cpp +TEST_CASE( "grault" ) { + REQUIRE( true ); + auto i = GENERATE( 1, 2, 3 ); + DYNAMIC_SECTION( "i=" << i ) { + REQUIRE( true ); + } +} +``` +* `./tests grault -p g:1` results in 2 assertions, as there is no filter + on the dynamic section. +* `./tests grault -p g:1 -p c:i=2` results in 2 assertions, as the filter + on the dynamic section matches the element given from the generator. +* `./tests grault -p g:1 -p c:i=3` results in 1 assertion, as the generator + is limited to only try `i := 2` and the dynamic section is filtered out. + +#### Section with a sibling generator +Because generators have to stop test execution when they don't pass filter, +it is impossible to run only a section with sibling generator without +triggering a test case skip. Consider this test case from an earlier example: +```cpp +TEST_CASE( "qux" ) { + REQUIRE( true ); + SECTION( "A" ) { REQUIRE( true ); } + auto i = GENERATE( 1, 2, 3 ); + DYNAMIC_SECTION( "i=" << i ) { + REQUIRE( true ); + } +} +``` +* `./tests qux -p g:1` results in 2 assertions, as the dynamic section is + entered only once. +* `./tests qux -p g:1 -p c:i=1` results in 1 assertion, as the dynamic + section filter is incompatible with the generator filter. +* `./tests qux -p c:A` results in 2 assertions **and a skipped test case**. + This is because the generator is sibling to section "A", and thus reads + the same section filter. However, it is not a section and as thus cannot + proceed. +* `./tests qux -p c:i=2` results in 1 assertion **and a skipped test case**. + Once again, the first filter in the filter stack is a section filter, + and thus the generator cannot proceed. + +Compare this with the old filter behaviour, where `./tests qux -c i=2` +would instead result in 4 assertions, because the generator would go +through all elements. + + +--- + +[Home](Readme.md#top) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index f8256f1e..11089ff5 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -108,6 +108,7 @@ set(IMPL_HEADERS ${SOURCES_DIR}/internal/catch_optional.hpp ${SOURCES_DIR}/internal/catch_output_redirect.hpp ${SOURCES_DIR}/internal/catch_parse_numbers.hpp + ${SOURCES_DIR}/internal/catch_path_filter.hpp ${SOURCES_DIR}/internal/catch_platform.hpp ${SOURCES_DIR}/internal/catch_polyfills.hpp ${SOURCES_DIR}/internal/catch_preprocessor.hpp @@ -255,11 +256,13 @@ set(GENERATOR_HEADERS ${SOURCES_DIR}/generators/catch_generators_all.hpp ${SOURCES_DIR}/generators/catch_generators_random.hpp ${SOURCES_DIR}/generators/catch_generators_range.hpp + ${SOURCES_DIR}/generators/catch_generators_throw.hpp ) set(GENERATOR_SOURCES ${SOURCES_DIR}/generators/catch_generator_exception.cpp ${SOURCES_DIR}/generators/catch_generators.cpp ${SOURCES_DIR}/generators/catch_generators_random.cpp + ${SOURCES_DIR}/generators/catch_generators_throw.cpp ) set(GENERATOR_FILES ${GENERATOR_HEADERS} ${GENERATOR_SOURCES}) diff --git a/src/catch2/catch_all.hpp b/src/catch2/catch_all.hpp index 693f6595..97226dd1 100644 --- a/src/catch2/catch_all.hpp +++ b/src/catch2/catch_all.hpp @@ -89,6 +89,7 @@ #include #include #include +#include #include #include #include diff --git a/src/catch2/catch_config.cpp b/src/catch2/catch_config.cpp index f83821a7..b4584f3c 100644 --- a/src/catch2/catch_config.cpp +++ b/src/catch2/catch_config.cpp @@ -92,6 +92,10 @@ namespace Catch { lhs.customOptions == rhs.customOptions; } + bool operator==( PathFilter const& lhs, PathFilter const& rhs ) { + return lhs.type == rhs.type && lhs.filter == rhs.filter; + } + Config::Config( ConfigData const& data ): m_data( data ) { // We need to trim filter specs to avoid trouble with superfluous @@ -101,9 +105,6 @@ namespace Catch { for (auto& elem : m_data.testsOrTags) { elem = trim(elem); } - for (auto& elem : m_data.sectionsToRun) { - elem = trim(elem); - } // Insert the default reporter if user hasn't asked for a specific one if ( m_data.reporterSpecifications.empty() ) { @@ -169,7 +170,8 @@ namespace Catch { bool Config::listListeners() const { return m_data.listListeners; } std::vector const& Config::getTestsOrTags() const { return m_data.testsOrTags; } - std::vector const& Config::getSectionsToRun() const { return m_data.sectionsToRun; } + std::vector const& Config::getPathFilters() const { return m_data.pathFilters; } + bool Config::useNewFilterBehaviour() const { return m_data.useNewPathFilteringBehaviour; } std::vector const& Config::getReporterSpecs() const { return m_data.reporterSpecifications; diff --git a/src/catch2/catch_config.hpp b/src/catch2/catch_config.hpp index 8014eb9e..c3fea561 100644 --- a/src/catch2/catch_config.hpp +++ b/src/catch2/catch_config.hpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -86,7 +87,8 @@ namespace Catch { std::vector reporterSpecifications; std::vector testsOrTags; - std::vector sectionsToRun; + std::vector pathFilters; + bool useNewPathFilteringBehaviour = false; std::string prematureExitGuardFilePath; }; @@ -109,7 +111,8 @@ namespace Catch { getProcessedReporterSpecs() const; std::vector const& getTestsOrTags() const override; - std::vector const& getSectionsToRun() const override; + std::vector const& getPathFilters() const override; + bool useNewFilterBehaviour() const override; TestSpec const& testSpec() const override; bool hasTestFilters() const override; diff --git a/src/catch2/generators/catch_generators.cpp b/src/catch2/generators/catch_generators.cpp index 3514e9f6..f7927eee 100644 --- a/src/catch2/generators/catch_generators.cpp +++ b/src/catch2/generators/catch_generators.cpp @@ -7,8 +7,6 @@ // SPDX-License-Identifier: BSL-1.0 #include -#include -#include #include namespace Catch { @@ -17,14 +15,6 @@ namespace Catch { namespace Generators { -namespace Detail { - - [[noreturn]] - void throw_generator_exception(char const* msg) { - Catch::throw_exception(GeneratorException{ msg }); - } -} // end namespace Detail - GeneratorUntypedBase::~GeneratorUntypedBase() = default; IGeneratorTracker* acquireGeneratorTracker(StringRef generatorName, SourceLineInfo const& lineInfo ) { diff --git a/src/catch2/generators/catch_generators.hpp b/src/catch2/generators/catch_generators.hpp index 719775fb..b833fd83 100644 --- a/src/catch2/generators/catch_generators.hpp +++ b/src/catch2/generators/catch_generators.hpp @@ -9,6 +9,7 @@ #define CATCH_GENERATORS_HPP_INCLUDED #include +#include #include #include #include @@ -22,14 +23,6 @@ namespace Catch { namespace Generators { -namespace Detail { - - //! Throws GeneratorException with the provided message - [[noreturn]] - void throw_generator_exception(char const * msg); - -} // end namespace detail - template class IGenerator : public GeneratorUntypedBase { std::string stringifyImpl() const override { diff --git a/src/catch2/generators/catch_generators_all.hpp b/src/catch2/generators/catch_generators_all.hpp index c12d314e..e6121858 100644 --- a/src/catch2/generators/catch_generators_all.hpp +++ b/src/catch2/generators/catch_generators_all.hpp @@ -26,5 +26,6 @@ #include #include #include +#include #endif // CATCH_GENERATORS_ALL_HPP_INCLUDED diff --git a/src/catch2/generators/catch_generators_throw.cpp b/src/catch2/generators/catch_generators_throw.cpp new file mode 100644 index 00000000..e7eda225 --- /dev/null +++ b/src/catch2/generators/catch_generators_throw.cpp @@ -0,0 +1,24 @@ + +// Copyright Catch2 Authors +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE.txt or copy at +// https://www.boost.org/LICENSE_1_0.txt) + +// SPDX-License-Identifier: BSL-1.0 + +#include +#include +#include + +namespace Catch { + namespace Generators { + namespace Detail { + + [[noreturn]] + void throw_generator_exception( char const* msg ) { + Catch::throw_exception( GeneratorException{ msg } ); + } + + } // namespace Detail + } // namespace Generators +} // namespace Catch diff --git a/src/catch2/generators/catch_generators_throw.hpp b/src/catch2/generators/catch_generators_throw.hpp new file mode 100644 index 00000000..2725eb6e --- /dev/null +++ b/src/catch2/generators/catch_generators_throw.hpp @@ -0,0 +1,23 @@ + +// Copyright Catch2 Authors +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE.txt or copy at +// https://www.boost.org/LICENSE_1_0.txt) + +// SPDX-License-Identifier: BSL-1.0 +#ifndef CATCH_GENERATORS_THROW_HPP_INCLUDED +#define CATCH_GENERATORS_THROW_HPP_INCLUDED + +namespace Catch { + namespace Generators { + namespace Detail { + + //! Throws GeneratorException with the provided message + [[noreturn]] + void throw_generator_exception( char const* msg ); + + } // namespace Detail + } // namespace Generators +} // namespace Catch + +#endif // CATCH_GENERATORS_THROW_HPP_INCLUDED diff --git a/src/catch2/interfaces/catch_interfaces_config.hpp b/src/catch2/interfaces/catch_interfaces_config.hpp index db4745c3..139d2c1e 100644 --- a/src/catch2/interfaces/catch_interfaces_config.hpp +++ b/src/catch2/interfaces/catch_interfaces_config.hpp @@ -62,6 +62,7 @@ namespace Catch { class TestSpec; class IStream; + struct PathFilter; class IConfig : public Detail::NonCopyable { public: @@ -87,7 +88,9 @@ namespace Catch { virtual unsigned int shardCount() const = 0; virtual unsigned int shardIndex() const = 0; virtual ColourMode defaultColourMode() const = 0; - virtual std::vector const& getSectionsToRun() const = 0; + virtual std::vector const& getPathFilters() const = 0; + virtual bool useNewFilterBehaviour() const = 0; + virtual Verbosity verbosity() const = 0; virtual bool skipBenchmarks() const = 0; diff --git a/src/catch2/internal/catch_commandline.cpp b/src/catch2/internal/catch_commandline.cpp index 34bda5e2..4ce42397 100644 --- a/src/catch2/internal/catch_commandline.cpp +++ b/src/catch2/internal/catch_commandline.cpp @@ -216,6 +216,43 @@ namespace Catch { return ParserResult::ok( ParseResultType::Matched ); }; + auto const setSectionFilter = [&]( std::string const& sectionFilter ) { + config.pathFilters.emplace_back( PathFilter::For::Section, trim(sectionFilter) ); + return ParserResult::ok( ParseResultType::Matched ); + }; + auto const setGeneratorFilter = [&]( std::string const& generatorFilter ) { + if (generatorFilter != "*") { + // TODO: avoid re-parsing the index? + auto parsedIndex = parseUInt( generatorFilter ); + if ( !parsedIndex ) { + return ParserResult::runtimeError( "Could not parse '" + + generatorFilter + + "' as generator index" ); + } + } + config.useNewPathFilteringBehaviour = true; + config.pathFilters.emplace_back( PathFilter::For::Generator, trim(generatorFilter) ); + return ParserResult::ok( ParseResultType::Matched ); + }; + // Copy-capturing other `setFoo` functions enables calling them later, + // as the config ref remains valid, but the local lambda vars won't. + auto const setPathFilter = [=, &config]( std::string const& pathFilter ) { + config.useNewPathFilteringBehaviour = true; + if ( pathFilter.size() < 3 ) { + return ParserResult::runtimeError( + "Path filter '" + pathFilter + "' is too short" ); + } + if ( startsWith( pathFilter, "g:" ) ) { + return setGeneratorFilter( pathFilter.substr( 2 ) ); + } + if ( startsWith( pathFilter, "c:" ) ) { + return setSectionFilter( pathFilter.substr( 2 ) ); + } + return ParserResult::runtimeError( "Path filter '" + pathFilter + + "' has unknown type prefix" ); + }; + + auto cli = ExeName( config.processName ) | Help( config.showHelp ) @@ -261,9 +298,15 @@ namespace Catch { | Opt( config.filenamesAsTags ) ["-#"]["--filenames-as-tags"] ( "adds a tag for the filename" ) - | Opt( config.sectionsToRun, "section name" ) + | Opt( accept_many, setSectionFilter, "section name" ) ["-c"]["--section"] ( "specify section to run" ) + | Opt( accept_many, setGeneratorFilter, "index spec" ) + ["-g"]["--generator-index"] + ( "specify generator elements to try" ) + | Opt( accept_many, setPathFilter, "path filter spec" ) + ["-p"]["--path-filter"] + ( "qualified path filter" ) | Opt( setVerbosity, "quiet|normal|high" ) ["-v"]["--verbosity"] ( "set output verbosity" ) diff --git a/src/catch2/internal/catch_path_filter.hpp b/src/catch2/internal/catch_path_filter.hpp new file mode 100644 index 00000000..9040efec --- /dev/null +++ b/src/catch2/internal/catch_path_filter.hpp @@ -0,0 +1,33 @@ + +// Copyright Catch2 Authors +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE.txt or copy at +// https://www.boost.org/LICENSE_1_0.txt) + +// SPDX-License-Identifier: BSL-1.0 +#ifndef CATCH_PATH_FILTER_HPP_INCLUDED +#define CATCH_PATH_FILTER_HPP_INCLUDED + +#include + +#include + +namespace Catch { + + struct PathFilter { + enum class For { + Section, + Generator, + }; + PathFilter( For type_, std::string filter_ ): + type( type_ ), filter( CATCH_MOVE( filter_ ) ) {} + + For type; + std::string filter; + + friend bool operator==( PathFilter const& lhs, PathFilter const& rhs ); + }; + +} // end namespace Catch + +#endif // CATCH_PATH_FILTER_HPP_INCLUDED diff --git a/src/catch2/internal/catch_run_context.cpp b/src/catch2/internal/catch_run_context.cpp index 86ae0446..93bcdf66 100644 --- a/src/catch2/internal/catch_run_context.cpp +++ b/src/catch2/internal/catch_run_context.cpp @@ -8,6 +8,7 @@ #include #include +#include #include #include #include @@ -19,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -35,6 +37,9 @@ namespace Catch { struct GeneratorTracker final : TestCaseTracking::TrackerBase, IGeneratorTracker { GeneratorBasePtr m_generator; + // Filtered generator has moved to specific index due to + // a filter, it needs special handling of `countedNext()` + bool m_isFiltered = false; GeneratorTracker( TestCaseTracking::NameAndLocation&& nameAndLocation, @@ -45,6 +50,32 @@ namespace Catch { m_generator( CATCH_MOVE( generator ) ) { assert( m_generator && "Cannot create tracker without generator" ); + + // Handle potential filter and move forward here... + // Old style filters do not affect generators at all + if (m_newStyleFilters && m_allTrackerDepth < m_filterRef->size()) { + auto const& filter = + ( *m_filterRef )[m_allTrackerDepth]; + // Generator cannot be un-entered the way a section + // can be, so the tracker has to throw for a wrong + // filter to stop the execution flow. + if (filter.type == PathFilter::For::Section) { + // TBD: Explicit SKIP, or new exception that says + // "don't continue", but doesn't show in totals? + SKIP(); + } + // '*' is the wildcard for "all elements in generator" + // used for filtering sections below the generator, but + // not the generator itself. + if ( filter.filter != "*" ) { + m_isFiltered = true; + // TBD: We assume that the filter was validated as + // number during parsing. We should pass it + // as number from the CLI parser. + size_t targetIndex = std::stoul( filter.filter ); + m_generator->skipToNthElement( targetIndex ); + } + } } static GeneratorTracker* @@ -117,25 +148,24 @@ namespace Catch { // _can_ start, and thus we should wait for them, or // they cannot start (due to filters), and we shouldn't // wait for them - ITracker* parent = m_parent; - // This is safe: there is always at least one section - // tracker in a test case tracking tree - while ( !parent->isSectionTracker() ) { - parent = parent->parent(); + + // No filters left -> no restrictions on running sections + size_t childDepth = 1 + (m_newStyleFilters ? m_allTrackerDepth : m_sectionOnlyDepth); + if ( childDepth >= m_filterRef->size() ) { + return true; } - assert( parent && - "Missing root (test case) level section" ); - - auto const& parentSection = - static_cast( *parent ); - auto const& filters = parentSection.getFilters(); - // No filters -> no restrictions on running sections - if ( filters.empty() ) { return true; } + // If we are using the new style filters, we need to check + // whether the successive filter is for section or a generator. + if ( m_newStyleFilters + && (*m_filterRef)[childDepth].type != PathFilter::For::Section ) { + return false; + } + // Look for any child section that could match the remaining filters for ( auto const& child : m_children ) { if ( child->isSectionTracker() && static_cast( *child ) - .trimmedName() == filters[0] ) { + .trimmedName() == StringRef((*m_filterRef)[childDepth].filter) ) { return true; } } @@ -147,9 +177,10 @@ namespace Catch { // value, but we do not want to invoke the side-effect if // this generator is still waiting for any child to start. assert( m_generator && "Tracker without generator" ); - if ( should_wait_for_child || - ( m_runState == CompletedSuccessfully && - m_generator->countedNext() ) ) { + if ( should_wait_for_child + || ( m_runState == CompletedSuccessfully + && !m_isFiltered // filtered generators cannot meaningfully move forward, as they would get past the filter + && m_generator->countedNext() ) ) { m_children.clear(); m_runState = Executing; } @@ -312,7 +343,8 @@ namespace Catch { ITracker& rootTracker = m_trackerContext.startRun(); assert(rootTracker.isSectionTracker()); - static_cast(rootTracker).addInitialFilters(m_config->getSectionsToRun()); + rootTracker.setFilters( &m_config->getPathFilters(), + m_config->useNewFilterBehaviour() ); // We intentionally only seed the internal RNG once per test case, // before it is first invoked. The reason for that is a complex @@ -484,6 +516,7 @@ namespace Catch { SourceLineInfo lineInfo, Generators::GeneratorBasePtr&& generator ) { + // TBD: Do we want to avoid the warning if the generator is filtered? if ( m_config->warnAboutInfiniteGenerators() && !generator->isFinite() ) { // TBD: Would it be better to expand this macro inline? diff --git a/src/catch2/internal/catch_test_case_tracker.cpp b/src/catch2/internal/catch_test_case_tracker.cpp index 541910d0..05bfff8e 100644 --- a/src/catch2/internal/catch_test_case_tracker.cpp +++ b/src/catch2/internal/catch_test_case_tracker.cpp @@ -8,8 +8,9 @@ #include #include -#include #include +#include +#include #include #include @@ -27,6 +28,17 @@ namespace TestCaseTracking { location( _location ) {} + ITracker::ITracker( NameAndLocation&& nameAndLoc, ITracker* parent ): + m_nameAndLocation( CATCH_MOVE( nameAndLoc ) ), m_parent( parent ) { + if ( m_parent ) { + m_allTrackerDepth = m_parent->m_allTrackerDepth + 1; + // We leave section trackers to bump themselves up, as + // we cannot use `isSectionTracker` in constructor + m_sectionOnlyDepth = m_parent->m_sectionOnlyDepth; + m_filterRef = m_parent->m_filterRef; + m_newStyleFilters = m_parent->m_newStyleFilters; + } + } ITracker::~ITracker() = default; @@ -159,25 +171,32 @@ namespace TestCaseTracking { : TrackerBase( CATCH_MOVE(nameAndLocation), ctx, parent ), m_trimmed_name(trim(StringRef(ITracker::nameAndLocation().name))) { - if( parent ) { - while ( !parent->isSectionTracker() ) { - parent = parent->parent(); - } - - SectionTracker& parentSection = static_cast( *parent ); - addNextFilters( parentSection.m_filters ); + if( m_parent ) { + ++m_sectionOnlyDepth; } } bool SectionTracker::isComplete() const { - bool complete = true; - - if ( m_filters.empty() - || m_filters[0].empty() - || m_filters[0] == m_trimmed_name ) { - complete = TrackerBase::isComplete(); + // If there are active filters AND we do not pass them, + // the section is always "completed" + const size_t filterIndex = + m_newStyleFilters ? m_allTrackerDepth : m_sectionOnlyDepth; + if ( filterIndex < m_filterRef->size() ) { + // There is active filter, check it + // 1) New style filter must explicitly target section + if ( m_newStyleFilters && ( *m_filterRef )[filterIndex].type != + PathFilter::For::Section ) { + return true; + } + // 2) Both style filters must match the trimmed name exactly + if ( m_trimmed_name != + StringRef( ( *m_filterRef )[filterIndex].filter ) ) { + return true; + } } - return complete; + + // Otherwise we delegate to the generic processing + return TrackerBase::isComplete(); } bool SectionTracker::isSectionTracker() const { return true; } @@ -213,19 +232,6 @@ namespace TestCaseTracking { open(); } - void SectionTracker::addInitialFilters( std::vector const& filters ) { - if( !filters.empty() ) { - m_filters.reserve( m_filters.size() + filters.size() + 2 ); - m_filters.emplace_back(StringRef{}); // Root - should never be consulted - m_filters.emplace_back(StringRef{}); // Test Case - not a section filter - m_filters.insert( m_filters.end(), filters.begin(), filters.end() ); - } - } - void SectionTracker::addNextFilters( std::vector const& filters ) { - if( filters.size() > 1 ) - m_filters.insert( m_filters.end(), filters.begin()+1, filters.end() ); - } - StringRef SectionTracker::trimmedName() const { return m_trimmed_name; } diff --git a/src/catch2/internal/catch_test_case_tracker.hpp b/src/catch2/internal/catch_test_case_tracker.hpp index 6bd749ff..48700e99 100644 --- a/src/catch2/internal/catch_test_case_tracker.hpp +++ b/src/catch2/internal/catch_test_case_tracker.hpp @@ -17,6 +17,9 @@ #include namespace Catch { + +struct PathFilter; + namespace TestCaseTracking { struct NameAndLocation { @@ -92,12 +95,23 @@ namespace TestCaseTracking { Children m_children; CycleState m_runState = NotStarted; - public: - ITracker( NameAndLocation&& nameAndLoc, ITracker* parent ): - m_nameAndLocation( CATCH_MOVE(nameAndLoc) ), - m_parent( parent ) - {} + // Members for path filtering + std::vector const* m_filterRef = nullptr; + // Note: There are 2 dummy section trackers (root, test-case) before + // the first "real" section tracker can be encountered. We start + // the default tracker at -2, so that the first "real" section + // tracker overflows to index 0. + // Nesting depth of this tracker, used to decide which new-style filter applies. + size_t m_allTrackerDepth = static_cast( -2 ); + // Nesting depth of sections (inc. this tracker), used for old-style filters. + // Must be updated by the section tracker on its own. + size_t m_sectionOnlyDepth = static_cast( -2 ); + // Transitory: Remove once we remove backwards compatibility with old-style (v3.x) filters + bool m_newStyleFilters = false; + + public: + ITracker( NameAndLocation&& nameAndLoc, ITracker* parent ); // static queries NameAndLocation const& nameAndLocation() const { @@ -123,6 +137,11 @@ namespace TestCaseTracking { //! Returns true iff tracker has started bool hasStarted() const; + void setFilters( std::vector const* filters, bool newStyleFilters ) { + m_filterRef = filters; + m_newStyleFilters = newStyleFilters; + } + // actions virtual void close() = 0; // Successfully complete virtual void fail() = 0; @@ -208,8 +227,7 @@ namespace TestCaseTracking { void moveToThis(); }; - class SectionTracker : public TrackerBase { - std::vector m_filters; + class SectionTracker final : public TrackerBase { // Note that lifetime-wise we piggy back off the name stored in the `ITracker` parent`. // Currently it allocates owns the name, so this is safe. If it is later refactored // to not own the name, the name still has to outlive the `ITracker` parent, so @@ -226,10 +244,6 @@ namespace TestCaseTracking { void tryOpen(); - void addInitialFilters( std::vector const& filters ); - void addNextFilters( std::vector const& filters ); - //! Returns filters active in this tracker - std::vector const& getFilters() const { return m_filters; } //! Returns whitespace-trimmed name of the tracked section StringRef trimmedName() const; }; diff --git a/src/catch2/meson.build b/src/catch2/meson.build index 45b9d2de..f491c602 100644 --- a/src/catch2/meson.build +++ b/src/catch2/meson.build @@ -61,6 +61,7 @@ internal_headers = [ 'generators/catch_generators_all.hpp', 'generators/catch_generators_random.hpp', 'generators/catch_generators_range.hpp', + 'generators/catch_generators_throw.hpp', 'interfaces/catch_interfaces_all.hpp', 'interfaces/catch_interfaces_capture.hpp', 'interfaces/catch_interfaces_config.hpp', @@ -115,6 +116,7 @@ internal_headers = [ 'internal/catch_optional.hpp', 'internal/catch_output_redirect.hpp', 'internal/catch_parse_numbers.hpp', + 'internal/catch_path_filter.hpp', 'internal/catch_platform.hpp', 'internal/catch_polyfills.hpp', 'internal/catch_preprocessor.hpp', @@ -203,6 +205,7 @@ internal_sources = files( 'generators/catch_generator_exception.cpp', 'generators/catch_generators.cpp', 'generators/catch_generators_random.cpp', + 'generators/catch_generators_throw.cpp', 'interfaces/catch_interfaces_capture.cpp', 'interfaces/catch_interfaces_config.cpp', 'interfaces/catch_interfaces_exception.cpp', diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 887c07f6..3329e90f 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -327,6 +327,24 @@ set_tests_properties(FilteredSections::GeneratorsDontCauseInfiniteLoop FAIL_REGULAR_EXPRESSION "inside with fov: 1" # This would mean there was no filtering ) +add_test(NAME "FilteredSections::DifferentSimpleFilters" + COMMAND + Python3::Interpreter "${CMAKE_CURRENT_LIST_DIR}/TestScripts/testSectionFiltering.py" $ +) +set_tests_properties("FilteredSections::DifferentSimpleFilters" + PROPERTIES + LABELS "uses-python" +) + +add_test(NAME "FilteredSections::HittingGeneratorsWithSectionFilterCausesSkip" + COMMAND + $ "baz" -p "c:A" -r "compact" +) +set_tests_properties("FilteredSections::HittingGeneratorsWithSectionFilterCausesSkip" + PROPERTIES + PASS_REGULAR_EXPRESSION "test cases: 1 \\| 1 skipped[\r\n\t ]*assertions: 1 \\| 1 passed" +) + add_test(NAME ApprovalTests COMMAND Python3::Interpreter @@ -679,14 +697,5 @@ set_tests_properties("Bazel::RngSeedEnvVar::MalformedValueIsIgnored" PASS_REGULAR_EXPRESSION "Randomness seeded to: 17171717" ) -add_test(NAME "FilteredSections::DifferentSimpleFilters" - COMMAND - Python3::Interpreter "${CMAKE_CURRENT_LIST_DIR}/TestScripts/testSectionFiltering.py" $ -) -set_tests_properties("FilteredSections::DifferentSimpleFilters" - PROPERTIES - LABELS "uses-python" -) - list(APPEND CATCH_TEST_TARGETS SelfTest) set(CATCH_TEST_TARGETS ${CATCH_TEST_TARGETS} PARENT_SCOPE) diff --git a/tests/SelfTest/Baselines/automake.sw.approved.txt b/tests/SelfTest/Baselines/automake.sw.approved.txt index bc019b38..557f58f9 100644 --- a/tests/SelfTest/Baselines/automake.sw.approved.txt +++ b/tests/SelfTest/Baselines/automake.sw.approved.txt @@ -215,6 +215,7 @@ Nor would this :test-result: PASS Overloaded comma or address-of operators are not used :test-result: PASS Parse uints :test-result: PASS Parsed tags are matched case insensitive +:test-result: PASS Parsing path filter specs :test-result: PASS Parsing sharding-related cli flags :test-result: PASS Parsing tags with non-alphabetical characters is pass-through :test-result: PASS Parsing warnings diff --git a/tests/SelfTest/Baselines/automake.sw.multi.approved.txt b/tests/SelfTest/Baselines/automake.sw.multi.approved.txt index 3079c349..1d3bc6a1 100644 --- a/tests/SelfTest/Baselines/automake.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/automake.sw.multi.approved.txt @@ -213,6 +213,7 @@ :test-result: PASS Overloaded comma or address-of operators are not used :test-result: PASS Parse uints :test-result: PASS Parsed tags are matched case insensitive +:test-result: PASS Parsing path filter specs :test-result: PASS Parsing sharding-related cli flags :test-result: PASS Parsing tags with non-alphabetical characters is pass-through :test-result: PASS Parsing warnings diff --git a/tests/SelfTest/Baselines/compact.sw.approved.txt b/tests/SelfTest/Baselines/compact.sw.approved.txt index e0a691f0..a7677dcd 100644 --- a/tests/SelfTest/Baselines/compact.sw.approved.txt +++ b/tests/SelfTest/Baselines/compact.sw.approved.txt @@ -1425,6 +1425,29 @@ Parse.tests.cpp:: passed: !(parseUInt( "0x", 10 )) for: TestSpecParser.tests.cpp:: passed: spec.hasFilters() for: true TestSpecParser.tests.cpp:: passed: spec.getInvalidSpecs().empty() for: true TestSpecParser.tests.cpp:: passed: spec.matches( testCase ) for: true +CmdLine.tests.cpp:: passed: result for: {?} +CmdLine.tests.cpp:: passed: !(config.useNewPathFilteringBehaviour) for: !false +CmdLine.tests.cpp:: passed: result for: {?} +CmdLine.tests.cpp:: passed: config.useNewPathFilteringBehaviour for: true +CmdLine.tests.cpp:: passed: !(result) for: !{?} +CmdLine.tests.cpp:: passed: result for: {?} +CmdLine.tests.cpp:: passed: config.pathFilters[0] == PathFilter( PathFilter::For::Generator, "*" ) for: {?} == {?} +CmdLine.tests.cpp:: passed: !(result) for: !{?} +CmdLine.tests.cpp:: passed: result for: {?} +CmdLine.tests.cpp:: passed: config.useNewPathFilteringBehaviour for: true +CmdLine.tests.cpp:: passed: config.pathFilters[0] == PathFilter( PathFilter::For::Generator, "1" ) for: {?} == {?} +CmdLine.tests.cpp:: passed: config.pathFilters[1] == PathFilter( PathFilter::For::Section, "foobar" ) for: {?} == {?} +CmdLine.tests.cpp:: passed: !(result) for: !{?} +CmdLine.tests.cpp:: passed: !(result1) for: !{?} +CmdLine.tests.cpp:: passed: !(result2) for: !{?} +CmdLine.tests.cpp:: passed: result for: {?} +CmdLine.tests.cpp:: passed: config.pathFilters[0] == PathFilter( PathFilter::For::Section, "foo-bar" ) for: {?} == {?} +CmdLine.tests.cpp:: passed: config.pathFilters[1] == PathFilter( PathFilter::For::Generator, "3" ) for: {?} == {?} +CmdLine.tests.cpp:: passed: config.pathFilters[2] == PathFilter( PathFilter::For::Generator, "123" ) for: {?} == {?} +CmdLine.tests.cpp:: passed: config.pathFilters[3] == PathFilter( PathFilter::For::Section, "baz" ) for: {?} == {?} +CmdLine.tests.cpp:: passed: result for: {?} +CmdLine.tests.cpp:: passed: config.pathFilters[0] == PathFilter(PathFilter::For::Section, "untrimmed" ) for: {?} == {?} +CmdLine.tests.cpp:: passed: config.pathFilters[1] == PathFilter(PathFilter::For::Generator, "42" ) for: {?} == {?} CmdLine.tests.cpp:: passed: cli.parse({ "test", "--shard-count=8" }) for: {?} CmdLine.tests.cpp:: passed: config.shardCount == 8 for: 8 == 8 CmdLine.tests.cpp:: passed: !(result) for: !{?} @@ -2960,7 +2983,7 @@ InternalBenchmark.tests.cpp:: passed: med == 18. for: 18.0 == 18.0 InternalBenchmark.tests.cpp:: passed: q3 == 23. for: 23.0 == 23.0 Misc.tests.cpp:: passed: Misc.tests.cpp:: passed: -test cases: 448 | 328 passed | 96 failed | 6 skipped | 18 failed as expected -assertions: 2376 | 2175 passed | 158 failed | 43 failed as expected +test cases: 449 | 329 passed | 96 failed | 6 skipped | 18 failed as expected +assertions: 2399 | 2198 passed | 158 failed | 43 failed as expected diff --git a/tests/SelfTest/Baselines/compact.sw.multi.approved.txt b/tests/SelfTest/Baselines/compact.sw.multi.approved.txt index b72f4a5e..65146fc5 100644 --- a/tests/SelfTest/Baselines/compact.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/compact.sw.multi.approved.txt @@ -1423,6 +1423,29 @@ Parse.tests.cpp:: passed: !(parseUInt( "0x", 10 )) for: TestSpecParser.tests.cpp:: passed: spec.hasFilters() for: true TestSpecParser.tests.cpp:: passed: spec.getInvalidSpecs().empty() for: true TestSpecParser.tests.cpp:: passed: spec.matches( testCase ) for: true +CmdLine.tests.cpp:: passed: result for: {?} +CmdLine.tests.cpp:: passed: !(config.useNewPathFilteringBehaviour) for: !false +CmdLine.tests.cpp:: passed: result for: {?} +CmdLine.tests.cpp:: passed: config.useNewPathFilteringBehaviour for: true +CmdLine.tests.cpp:: passed: !(result) for: !{?} +CmdLine.tests.cpp:: passed: result for: {?} +CmdLine.tests.cpp:: passed: config.pathFilters[0] == PathFilter( PathFilter::For::Generator, "*" ) for: {?} == {?} +CmdLine.tests.cpp:: passed: !(result) for: !{?} +CmdLine.tests.cpp:: passed: result for: {?} +CmdLine.tests.cpp:: passed: config.useNewPathFilteringBehaviour for: true +CmdLine.tests.cpp:: passed: config.pathFilters[0] == PathFilter( PathFilter::For::Generator, "1" ) for: {?} == {?} +CmdLine.tests.cpp:: passed: config.pathFilters[1] == PathFilter( PathFilter::For::Section, "foobar" ) for: {?} == {?} +CmdLine.tests.cpp:: passed: !(result) for: !{?} +CmdLine.tests.cpp:: passed: !(result1) for: !{?} +CmdLine.tests.cpp:: passed: !(result2) for: !{?} +CmdLine.tests.cpp:: passed: result for: {?} +CmdLine.tests.cpp:: passed: config.pathFilters[0] == PathFilter( PathFilter::For::Section, "foo-bar" ) for: {?} == {?} +CmdLine.tests.cpp:: passed: config.pathFilters[1] == PathFilter( PathFilter::For::Generator, "3" ) for: {?} == {?} +CmdLine.tests.cpp:: passed: config.pathFilters[2] == PathFilter( PathFilter::For::Generator, "123" ) for: {?} == {?} +CmdLine.tests.cpp:: passed: config.pathFilters[3] == PathFilter( PathFilter::For::Section, "baz" ) for: {?} == {?} +CmdLine.tests.cpp:: passed: result for: {?} +CmdLine.tests.cpp:: passed: config.pathFilters[0] == PathFilter(PathFilter::For::Section, "untrimmed" ) for: {?} == {?} +CmdLine.tests.cpp:: passed: config.pathFilters[1] == PathFilter(PathFilter::For::Generator, "42" ) for: {?} == {?} CmdLine.tests.cpp:: passed: cli.parse({ "test", "--shard-count=8" }) for: {?} CmdLine.tests.cpp:: passed: config.shardCount == 8 for: 8 == 8 CmdLine.tests.cpp:: passed: !(result) for: !{?} @@ -2949,7 +2972,7 @@ InternalBenchmark.tests.cpp:: passed: med == 18. for: 18.0 == 18.0 InternalBenchmark.tests.cpp:: passed: q3 == 23. for: 23.0 == 23.0 Misc.tests.cpp:: passed: Misc.tests.cpp:: passed: -test cases: 448 | 328 passed | 96 failed | 6 skipped | 18 failed as expected -assertions: 2376 | 2175 passed | 158 failed | 43 failed as expected +test cases: 449 | 329 passed | 96 failed | 6 skipped | 18 failed as expected +assertions: 2399 | 2198 passed | 158 failed | 43 failed as expected diff --git a/tests/SelfTest/Baselines/console.std.approved.txt b/tests/SelfTest/Baselines/console.std.approved.txt index cd178a0c..7e9a3cdc 100644 --- a/tests/SelfTest/Baselines/console.std.approved.txt +++ b/tests/SelfTest/Baselines/console.std.approved.txt @@ -1743,6 +1743,6 @@ due to unexpected exception with message: Why would you throw a std::string? =============================================================================== -test cases: 448 | 346 passed | 76 failed | 7 skipped | 19 failed as expected -assertions: 2354 | 2175 passed | 136 failed | 43 failed as expected +test cases: 449 | 347 passed | 76 failed | 7 skipped | 19 failed as expected +assertions: 2377 | 2198 passed | 136 failed | 43 failed as expected diff --git a/tests/SelfTest/Baselines/console.sw.approved.txt b/tests/SelfTest/Baselines/console.sw.approved.txt index 4af80f47..71a38625 100644 --- a/tests/SelfTest/Baselines/console.sw.approved.txt +++ b/tests/SelfTest/Baselines/console.sw.approved.txt @@ -9230,6 +9230,191 @@ TestSpecParser.tests.cpp:: PASSED: with expansion: true +------------------------------------------------------------------------------- +Parsing path filter specs + Only section specs leads to old filter behaviour +------------------------------------------------------------------------------- +CmdLine.tests.cpp: +............................................................................... + +CmdLine.tests.cpp:: PASSED: + REQUIRE( result ) +with expansion: + {?} + +CmdLine.tests.cpp:: PASSED: + REQUIRE_FALSE( config.useNewPathFilteringBehaviour ) +with expansion: + !false + +------------------------------------------------------------------------------- +Parsing path filter specs + Generator specs enable new filter behaviour +------------------------------------------------------------------------------- +CmdLine.tests.cpp: +............................................................................... + +CmdLine.tests.cpp:: PASSED: + REQUIRE( result ) +with expansion: + {?} + +CmdLine.tests.cpp:: PASSED: + REQUIRE( config.useNewPathFilteringBehaviour ) +with expansion: + true + +------------------------------------------------------------------------------- +Parsing path filter specs + Generator specs do not accept stringish arguments +------------------------------------------------------------------------------- +CmdLine.tests.cpp: +............................................................................... + +CmdLine.tests.cpp:: PASSED: + REQUIRE_FALSE( result ) +with expansion: + !{?} + +------------------------------------------------------------------------------- +Parsing path filter specs + Generator specs accept star as argument +------------------------------------------------------------------------------- +CmdLine.tests.cpp: +............................................................................... + +CmdLine.tests.cpp:: PASSED: + REQUIRE( result ) +with expansion: + {?} + +CmdLine.tests.cpp:: PASSED: + REQUIRE( config.pathFilters[0] == PathFilter( PathFilter::For::Generator, "*" ) ) +with expansion: + {?} == {?} + +------------------------------------------------------------------------------- +Parsing path filter specs + Generator specs do not accept negative numbers +------------------------------------------------------------------------------- +CmdLine.tests.cpp: +............................................................................... + +CmdLine.tests.cpp:: PASSED: + REQUIRE_FALSE( result ) +with expansion: + !{?} + +------------------------------------------------------------------------------- +Parsing path filter specs + Generic path spec enables new filter behaviour +------------------------------------------------------------------------------- +CmdLine.tests.cpp: +............................................................................... + +CmdLine.tests.cpp:: PASSED: + REQUIRE( result ) +with expansion: + {?} + +CmdLine.tests.cpp:: PASSED: + REQUIRE( config.useNewPathFilteringBehaviour ) +with expansion: + true + +CmdLine.tests.cpp:: PASSED: + REQUIRE( config.pathFilters[0] == PathFilter( PathFilter::For::Generator, "1" ) ) +with expansion: + {?} == {?} + +CmdLine.tests.cpp:: PASSED: + REQUIRE( config.pathFilters[1] == PathFilter( PathFilter::For::Section, "foobar" ) ) +with expansion: + {?} == {?} + +------------------------------------------------------------------------------- +Parsing path filter specs + Generic path spec for generator is validated +------------------------------------------------------------------------------- +CmdLine.tests.cpp: +............................................................................... + +CmdLine.tests.cpp:: PASSED: + REQUIRE_FALSE( result ) +with expansion: + !{?} + +------------------------------------------------------------------------------- +Parsing path filter specs + Generic path spec without colon is rejected +------------------------------------------------------------------------------- +CmdLine.tests.cpp: +............................................................................... + +CmdLine.tests.cpp:: PASSED: + REQUIRE_FALSE( result1 ) +with expansion: + !{?} + +CmdLine.tests.cpp:: PASSED: + REQUIRE_FALSE( result2 ) +with expansion: + !{?} + +------------------------------------------------------------------------------- +Parsing path filter specs + Using both section and generator filters creates filter stack +------------------------------------------------------------------------------- +CmdLine.tests.cpp: +............................................................................... + +CmdLine.tests.cpp:: PASSED: + REQUIRE( result ) +with expansion: + {?} + +CmdLine.tests.cpp:: PASSED: + REQUIRE( config.pathFilters[0] == PathFilter( PathFilter::For::Section, "foo-bar" ) ) +with expansion: + {?} == {?} + +CmdLine.tests.cpp:: PASSED: + REQUIRE( config.pathFilters[1] == PathFilter( PathFilter::For::Generator, "3" ) ) +with expansion: + {?} == {?} + +CmdLine.tests.cpp:: PASSED: + REQUIRE( config.pathFilters[2] == PathFilter( PathFilter::For::Generator, "123" ) ) +with expansion: + {?} == {?} + +CmdLine.tests.cpp:: PASSED: + REQUIRE( config.pathFilters[3] == PathFilter( PathFilter::For::Section, "baz" ) ) +with expansion: + {?} == {?} + +------------------------------------------------------------------------------- +Parsing path filter specs + Section/generator filters are whitespace trimmed +------------------------------------------------------------------------------- +CmdLine.tests.cpp: +............................................................................... + +CmdLine.tests.cpp:: PASSED: + REQUIRE( result ) +with expansion: + {?} + +CmdLine.tests.cpp:: PASSED: + REQUIRE( config.pathFilters[0] == PathFilter(PathFilter::For::Section, "untrimmed" ) ) +with expansion: + {?} == {?} + +CmdLine.tests.cpp:: PASSED: + REQUIRE( config.pathFilters[1] == PathFilter(PathFilter::For::Generator, "42" ) ) +with expansion: + {?} == {?} + ------------------------------------------------------------------------------- Parsing sharding-related cli flags shard-count @@ -19844,6 +20029,6 @@ Misc.tests.cpp: Misc.tests.cpp:: PASSED: =============================================================================== -test cases: 448 | 328 passed | 96 failed | 6 skipped | 18 failed as expected -assertions: 2376 | 2175 passed | 158 failed | 43 failed as expected +test cases: 449 | 329 passed | 96 failed | 6 skipped | 18 failed as expected +assertions: 2399 | 2198 passed | 158 failed | 43 failed as expected diff --git a/tests/SelfTest/Baselines/console.sw.multi.approved.txt b/tests/SelfTest/Baselines/console.sw.multi.approved.txt index 2eadff97..26723baa 100644 --- a/tests/SelfTest/Baselines/console.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/console.sw.multi.approved.txt @@ -9228,6 +9228,191 @@ TestSpecParser.tests.cpp:: PASSED: with expansion: true +------------------------------------------------------------------------------- +Parsing path filter specs + Only section specs leads to old filter behaviour +------------------------------------------------------------------------------- +CmdLine.tests.cpp: +............................................................................... + +CmdLine.tests.cpp:: PASSED: + REQUIRE( result ) +with expansion: + {?} + +CmdLine.tests.cpp:: PASSED: + REQUIRE_FALSE( config.useNewPathFilteringBehaviour ) +with expansion: + !false + +------------------------------------------------------------------------------- +Parsing path filter specs + Generator specs enable new filter behaviour +------------------------------------------------------------------------------- +CmdLine.tests.cpp: +............................................................................... + +CmdLine.tests.cpp:: PASSED: + REQUIRE( result ) +with expansion: + {?} + +CmdLine.tests.cpp:: PASSED: + REQUIRE( config.useNewPathFilteringBehaviour ) +with expansion: + true + +------------------------------------------------------------------------------- +Parsing path filter specs + Generator specs do not accept stringish arguments +------------------------------------------------------------------------------- +CmdLine.tests.cpp: +............................................................................... + +CmdLine.tests.cpp:: PASSED: + REQUIRE_FALSE( result ) +with expansion: + !{?} + +------------------------------------------------------------------------------- +Parsing path filter specs + Generator specs accept star as argument +------------------------------------------------------------------------------- +CmdLine.tests.cpp: +............................................................................... + +CmdLine.tests.cpp:: PASSED: + REQUIRE( result ) +with expansion: + {?} + +CmdLine.tests.cpp:: PASSED: + REQUIRE( config.pathFilters[0] == PathFilter( PathFilter::For::Generator, "*" ) ) +with expansion: + {?} == {?} + +------------------------------------------------------------------------------- +Parsing path filter specs + Generator specs do not accept negative numbers +------------------------------------------------------------------------------- +CmdLine.tests.cpp: +............................................................................... + +CmdLine.tests.cpp:: PASSED: + REQUIRE_FALSE( result ) +with expansion: + !{?} + +------------------------------------------------------------------------------- +Parsing path filter specs + Generic path spec enables new filter behaviour +------------------------------------------------------------------------------- +CmdLine.tests.cpp: +............................................................................... + +CmdLine.tests.cpp:: PASSED: + REQUIRE( result ) +with expansion: + {?} + +CmdLine.tests.cpp:: PASSED: + REQUIRE( config.useNewPathFilteringBehaviour ) +with expansion: + true + +CmdLine.tests.cpp:: PASSED: + REQUIRE( config.pathFilters[0] == PathFilter( PathFilter::For::Generator, "1" ) ) +with expansion: + {?} == {?} + +CmdLine.tests.cpp:: PASSED: + REQUIRE( config.pathFilters[1] == PathFilter( PathFilter::For::Section, "foobar" ) ) +with expansion: + {?} == {?} + +------------------------------------------------------------------------------- +Parsing path filter specs + Generic path spec for generator is validated +------------------------------------------------------------------------------- +CmdLine.tests.cpp: +............................................................................... + +CmdLine.tests.cpp:: PASSED: + REQUIRE_FALSE( result ) +with expansion: + !{?} + +------------------------------------------------------------------------------- +Parsing path filter specs + Generic path spec without colon is rejected +------------------------------------------------------------------------------- +CmdLine.tests.cpp: +............................................................................... + +CmdLine.tests.cpp:: PASSED: + REQUIRE_FALSE( result1 ) +with expansion: + !{?} + +CmdLine.tests.cpp:: PASSED: + REQUIRE_FALSE( result2 ) +with expansion: + !{?} + +------------------------------------------------------------------------------- +Parsing path filter specs + Using both section and generator filters creates filter stack +------------------------------------------------------------------------------- +CmdLine.tests.cpp: +............................................................................... + +CmdLine.tests.cpp:: PASSED: + REQUIRE( result ) +with expansion: + {?} + +CmdLine.tests.cpp:: PASSED: + REQUIRE( config.pathFilters[0] == PathFilter( PathFilter::For::Section, "foo-bar" ) ) +with expansion: + {?} == {?} + +CmdLine.tests.cpp:: PASSED: + REQUIRE( config.pathFilters[1] == PathFilter( PathFilter::For::Generator, "3" ) ) +with expansion: + {?} == {?} + +CmdLine.tests.cpp:: PASSED: + REQUIRE( config.pathFilters[2] == PathFilter( PathFilter::For::Generator, "123" ) ) +with expansion: + {?} == {?} + +CmdLine.tests.cpp:: PASSED: + REQUIRE( config.pathFilters[3] == PathFilter( PathFilter::For::Section, "baz" ) ) +with expansion: + {?} == {?} + +------------------------------------------------------------------------------- +Parsing path filter specs + Section/generator filters are whitespace trimmed +------------------------------------------------------------------------------- +CmdLine.tests.cpp: +............................................................................... + +CmdLine.tests.cpp:: PASSED: + REQUIRE( result ) +with expansion: + {?} + +CmdLine.tests.cpp:: PASSED: + REQUIRE( config.pathFilters[0] == PathFilter(PathFilter::For::Section, "untrimmed" ) ) +with expansion: + {?} == {?} + +CmdLine.tests.cpp:: PASSED: + REQUIRE( config.pathFilters[1] == PathFilter(PathFilter::For::Generator, "42" ) ) +with expansion: + {?} == {?} + ------------------------------------------------------------------------------- Parsing sharding-related cli flags shard-count @@ -19833,6 +20018,6 @@ Misc.tests.cpp: Misc.tests.cpp:: PASSED: =============================================================================== -test cases: 448 | 328 passed | 96 failed | 6 skipped | 18 failed as expected -assertions: 2376 | 2175 passed | 158 failed | 43 failed as expected +test cases: 449 | 329 passed | 96 failed | 6 skipped | 18 failed as expected +assertions: 2399 | 2198 passed | 158 failed | 43 failed as expected diff --git a/tests/SelfTest/Baselines/junit.sw.approved.txt b/tests/SelfTest/Baselines/junit.sw.approved.txt index e3010c82..efcd49cf 100644 --- a/tests/SelfTest/Baselines/junit.sw.approved.txt +++ b/tests/SelfTest/Baselines/junit.sw.approved.txt @@ -1,7 +1,7 @@ - + @@ -1276,6 +1276,17 @@ at Message.tests.cpp: + + + + + + + + + + + diff --git a/tests/SelfTest/Baselines/junit.sw.multi.approved.txt b/tests/SelfTest/Baselines/junit.sw.multi.approved.txt index cc2c9946..e59f9e92 100644 --- a/tests/SelfTest/Baselines/junit.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/junit.sw.multi.approved.txt @@ -1,6 +1,6 @@ - + @@ -1275,6 +1275,17 @@ at Message.tests.cpp: + + + + + + + + + + + diff --git a/tests/SelfTest/Baselines/sonarqube.sw.approved.txt b/tests/SelfTest/Baselines/sonarqube.sw.approved.txt index f790e0eb..ee6ad309 100644 --- a/tests/SelfTest/Baselines/sonarqube.sw.approved.txt +++ b/tests/SelfTest/Baselines/sonarqube.sw.approved.txt @@ -64,6 +64,17 @@ at AssertionHandler.tests.cpp: + + + + + + + + + + + diff --git a/tests/SelfTest/Baselines/sonarqube.sw.multi.approved.txt b/tests/SelfTest/Baselines/sonarqube.sw.multi.approved.txt index 405cd887..119782fe 100644 --- a/tests/SelfTest/Baselines/sonarqube.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/sonarqube.sw.multi.approved.txt @@ -63,6 +63,17 @@ at AssertionHandler.tests.cpp: + + + + + + + + + + + diff --git a/tests/SelfTest/Baselines/tap.sw.approved.txt b/tests/SelfTest/Baselines/tap.sw.approved.txt index c4e9dccb..1909b8f6 100644 --- a/tests/SelfTest/Baselines/tap.sw.approved.txt +++ b/tests/SelfTest/Baselines/tap.sw.approved.txt @@ -2328,6 +2328,52 @@ ok {test-number} - spec.hasFilters() for: true ok {test-number} - spec.getInvalidSpecs().empty() for: true # Parsed tags are matched case insensitive ok {test-number} - spec.matches( testCase ) for: true +# Parsing path filter specs +ok {test-number} - result for: {?} +# Parsing path filter specs +ok {test-number} - !(config.useNewPathFilteringBehaviour) for: !false +# Parsing path filter specs +ok {test-number} - result for: {?} +# Parsing path filter specs +ok {test-number} - config.useNewPathFilteringBehaviour for: true +# Parsing path filter specs +ok {test-number} - !(result) for: !{?} +# Parsing path filter specs +ok {test-number} - result for: {?} +# Parsing path filter specs +ok {test-number} - config.pathFilters[0] == PathFilter( PathFilter::For::Generator, "*" ) for: {?} == {?} +# Parsing path filter specs +ok {test-number} - !(result) for: !{?} +# Parsing path filter specs +ok {test-number} - result for: {?} +# Parsing path filter specs +ok {test-number} - config.useNewPathFilteringBehaviour for: true +# Parsing path filter specs +ok {test-number} - config.pathFilters[0] == PathFilter( PathFilter::For::Generator, "1" ) for: {?} == {?} +# Parsing path filter specs +ok {test-number} - config.pathFilters[1] == PathFilter( PathFilter::For::Section, "foobar" ) for: {?} == {?} +# Parsing path filter specs +ok {test-number} - !(result) for: !{?} +# Parsing path filter specs +ok {test-number} - !(result1) for: !{?} +# Parsing path filter specs +ok {test-number} - !(result2) for: !{?} +# Parsing path filter specs +ok {test-number} - result for: {?} +# Parsing path filter specs +ok {test-number} - config.pathFilters[0] == PathFilter( PathFilter::For::Section, "foo-bar" ) for: {?} == {?} +# Parsing path filter specs +ok {test-number} - config.pathFilters[1] == PathFilter( PathFilter::For::Generator, "3" ) for: {?} == {?} +# Parsing path filter specs +ok {test-number} - config.pathFilters[2] == PathFilter( PathFilter::For::Generator, "123" ) for: {?} == {?} +# Parsing path filter specs +ok {test-number} - config.pathFilters[3] == PathFilter( PathFilter::For::Section, "baz" ) for: {?} == {?} +# Parsing path filter specs +ok {test-number} - result for: {?} +# Parsing path filter specs +ok {test-number} - config.pathFilters[0] == PathFilter(PathFilter::For::Section, "untrimmed" ) for: {?} == {?} +# Parsing path filter specs +ok {test-number} - config.pathFilters[1] == PathFilter(PathFilter::For::Generator, "42" ) for: {?} == {?} # Parsing sharding-related cli flags ok {test-number} - cli.parse({ "test", "--shard-count=8" }) for: {?} # Parsing sharding-related cli flags @@ -4771,5 +4817,5 @@ ok {test-number} - q3 == 23. for: 23.0 == 23.0 ok {test-number} - # xmlentitycheck ok {test-number} - -1..2388 +1..2411 diff --git a/tests/SelfTest/Baselines/tap.sw.multi.approved.txt b/tests/SelfTest/Baselines/tap.sw.multi.approved.txt index 54f36be0..8c765346 100644 --- a/tests/SelfTest/Baselines/tap.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/tap.sw.multi.approved.txt @@ -2326,6 +2326,52 @@ ok {test-number} - spec.hasFilters() for: true ok {test-number} - spec.getInvalidSpecs().empty() for: true # Parsed tags are matched case insensitive ok {test-number} - spec.matches( testCase ) for: true +# Parsing path filter specs +ok {test-number} - result for: {?} +# Parsing path filter specs +ok {test-number} - !(config.useNewPathFilteringBehaviour) for: !false +# Parsing path filter specs +ok {test-number} - result for: {?} +# Parsing path filter specs +ok {test-number} - config.useNewPathFilteringBehaviour for: true +# Parsing path filter specs +ok {test-number} - !(result) for: !{?} +# Parsing path filter specs +ok {test-number} - result for: {?} +# Parsing path filter specs +ok {test-number} - config.pathFilters[0] == PathFilter( PathFilter::For::Generator, "*" ) for: {?} == {?} +# Parsing path filter specs +ok {test-number} - !(result) for: !{?} +# Parsing path filter specs +ok {test-number} - result for: {?} +# Parsing path filter specs +ok {test-number} - config.useNewPathFilteringBehaviour for: true +# Parsing path filter specs +ok {test-number} - config.pathFilters[0] == PathFilter( PathFilter::For::Generator, "1" ) for: {?} == {?} +# Parsing path filter specs +ok {test-number} - config.pathFilters[1] == PathFilter( PathFilter::For::Section, "foobar" ) for: {?} == {?} +# Parsing path filter specs +ok {test-number} - !(result) for: !{?} +# Parsing path filter specs +ok {test-number} - !(result1) for: !{?} +# Parsing path filter specs +ok {test-number} - !(result2) for: !{?} +# Parsing path filter specs +ok {test-number} - result for: {?} +# Parsing path filter specs +ok {test-number} - config.pathFilters[0] == PathFilter( PathFilter::For::Section, "foo-bar" ) for: {?} == {?} +# Parsing path filter specs +ok {test-number} - config.pathFilters[1] == PathFilter( PathFilter::For::Generator, "3" ) for: {?} == {?} +# Parsing path filter specs +ok {test-number} - config.pathFilters[2] == PathFilter( PathFilter::For::Generator, "123" ) for: {?} == {?} +# Parsing path filter specs +ok {test-number} - config.pathFilters[3] == PathFilter( PathFilter::For::Section, "baz" ) for: {?} == {?} +# Parsing path filter specs +ok {test-number} - result for: {?} +# Parsing path filter specs +ok {test-number} - config.pathFilters[0] == PathFilter(PathFilter::For::Section, "untrimmed" ) for: {?} == {?} +# Parsing path filter specs +ok {test-number} - config.pathFilters[1] == PathFilter(PathFilter::For::Generator, "42" ) for: {?} == {?} # Parsing sharding-related cli flags ok {test-number} - cli.parse({ "test", "--shard-count=8" }) for: {?} # Parsing sharding-related cli flags @@ -4760,5 +4806,5 @@ ok {test-number} - q3 == 23. for: 23.0 == 23.0 ok {test-number} - # xmlentitycheck ok {test-number} - -1..2388 +1..2411 diff --git a/tests/SelfTest/Baselines/teamcity.sw.approved.txt b/tests/SelfTest/Baselines/teamcity.sw.approved.txt index e7859ebd..ccd012ba 100644 --- a/tests/SelfTest/Baselines/teamcity.sw.approved.txt +++ b/tests/SelfTest/Baselines/teamcity.sw.approved.txt @@ -549,6 +549,8 @@ ##teamcity[testFinished name='Parse uints' duration="{duration}"] ##teamcity[testStarted name='Parsed tags are matched case insensitive'] ##teamcity[testFinished name='Parsed tags are matched case insensitive' duration="{duration}"] +##teamcity[testStarted name='Parsing path filter specs'] +##teamcity[testFinished name='Parsing path filter specs' duration="{duration}"] ##teamcity[testStarted name='Parsing sharding-related cli flags'] ##teamcity[testFinished name='Parsing sharding-related cli flags' duration="{duration}"] ##teamcity[testStarted name='Parsing tags with non-alphabetical characters is pass-through'] diff --git a/tests/SelfTest/Baselines/teamcity.sw.multi.approved.txt b/tests/SelfTest/Baselines/teamcity.sw.multi.approved.txt index 4f96fab8..f89bcde6 100644 --- a/tests/SelfTest/Baselines/teamcity.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/teamcity.sw.multi.approved.txt @@ -549,6 +549,8 @@ ##teamcity[testFinished name='Parse uints' duration="{duration}"] ##teamcity[testStarted name='Parsed tags are matched case insensitive'] ##teamcity[testFinished name='Parsed tags are matched case insensitive' duration="{duration}"] +##teamcity[testStarted name='Parsing path filter specs'] +##teamcity[testFinished name='Parsing path filter specs' duration="{duration}"] ##teamcity[testStarted name='Parsing sharding-related cli flags'] ##teamcity[testFinished name='Parsing sharding-related cli flags' duration="{duration}"] ##teamcity[testStarted name='Parsing tags with non-alphabetical characters is pass-through'] diff --git a/tests/SelfTest/Baselines/xml.sw.approved.txt b/tests/SelfTest/Baselines/xml.sw.approved.txt index f8f1ff29..92852769 100644 --- a/tests/SelfTest/Baselines/xml.sw.approved.txt +++ b/tests/SelfTest/Baselines/xml.sw.approved.txt @@ -11071,6 +11071,223 @@ Approx( 1.21999999999999997 ) + +
+ + + result + + + {?} + + + + + !(config.useNewPathFilteringBehaviour) + + + !false + + + +
+
+ + + result + + + {?} + + + + + config.useNewPathFilteringBehaviour + + + true + + + +
+
+ + + !(result) + + + !{?} + + + +
+
+ + + result + + + {?} + + + + + config.pathFilters[0] == PathFilter( PathFilter::For::Generator, "*" ) + + + {?} == {?} + + + +
+
+ + + !(result) + + + !{?} + + + +
+
+ + + result + + + {?} + + + + + config.useNewPathFilteringBehaviour + + + true + + + + + config.pathFilters[0] == PathFilter( PathFilter::For::Generator, "1" ) + + + {?} == {?} + + + + + config.pathFilters[1] == PathFilter( PathFilter::For::Section, "foobar" ) + + + {?} == {?} + + + +
+
+ + + !(result) + + + !{?} + + + +
+
+ + + !(result1) + + + !{?} + + + + + !(result2) + + + !{?} + + + +
+
+ + + result + + + {?} + + + + + config.pathFilters[0] == PathFilter( PathFilter::For::Section, "foo-bar" ) + + + {?} == {?} + + + + + config.pathFilters[1] == PathFilter( PathFilter::For::Generator, "3" ) + + + {?} == {?} + + + + + config.pathFilters[2] == PathFilter( PathFilter::For::Generator, "123" ) + + + {?} == {?} + + + + + config.pathFilters[3] == PathFilter( PathFilter::For::Section, "baz" ) + + + {?} == {?} + + + +
+
+ + + result + + + {?} + + + + + config.pathFilters[0] == PathFilter(PathFilter::For::Section, "untrimmed" ) + + + {?} == {?} + + + + + config.pathFilters[1] == PathFilter(PathFilter::For::Generator, "42" ) + + + {?} == {?} + + + +
+ +
@@ -23020,6 +23237,6 @@ Approx( -1.95996398454005449 )
- - + + diff --git a/tests/SelfTest/Baselines/xml.sw.multi.approved.txt b/tests/SelfTest/Baselines/xml.sw.multi.approved.txt index f37883d1..91ac5a60 100644 --- a/tests/SelfTest/Baselines/xml.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/xml.sw.multi.approved.txt @@ -11071,6 +11071,223 @@ Approx( 1.21999999999999997 ) + +
+ + + result + + + {?} + + + + + !(config.useNewPathFilteringBehaviour) + + + !false + + + +
+
+ + + result + + + {?} + + + + + config.useNewPathFilteringBehaviour + + + true + + + +
+
+ + + !(result) + + + !{?} + + + +
+
+ + + result + + + {?} + + + + + config.pathFilters[0] == PathFilter( PathFilter::For::Generator, "*" ) + + + {?} == {?} + + + +
+
+ + + !(result) + + + !{?} + + + +
+
+ + + result + + + {?} + + + + + config.useNewPathFilteringBehaviour + + + true + + + + + config.pathFilters[0] == PathFilter( PathFilter::For::Generator, "1" ) + + + {?} == {?} + + + + + config.pathFilters[1] == PathFilter( PathFilter::For::Section, "foobar" ) + + + {?} == {?} + + + +
+
+ + + !(result) + + + !{?} + + + +
+
+ + + !(result1) + + + !{?} + + + + + !(result2) + + + !{?} + + + +
+
+ + + result + + + {?} + + + + + config.pathFilters[0] == PathFilter( PathFilter::For::Section, "foo-bar" ) + + + {?} == {?} + + + + + config.pathFilters[1] == PathFilter( PathFilter::For::Generator, "3" ) + + + {?} == {?} + + + + + config.pathFilters[2] == PathFilter( PathFilter::For::Generator, "123" ) + + + {?} == {?} + + + + + config.pathFilters[3] == PathFilter( PathFilter::For::Section, "baz" ) + + + {?} == {?} + + + +
+
+ + + result + + + {?} + + + + + config.pathFilters[0] == PathFilter(PathFilter::For::Section, "untrimmed" ) + + + {?} == {?} + + + + + config.pathFilters[1] == PathFilter(PathFilter::For::Generator, "42" ) + + + {?} == {?} + + + +
+ +
@@ -23019,6 +23236,6 @@ Approx( -1.95996398454005449 )
- - + + diff --git a/tests/SelfTest/IntrospectiveTests/CmdLine.tests.cpp b/tests/SelfTest/IntrospectiveTests/CmdLine.tests.cpp index c797ce5c..154d8ed9 100644 --- a/tests/SelfTest/IntrospectiveTests/CmdLine.tests.cpp +++ b/tests/SelfTest/IntrospectiveTests/CmdLine.tests.cpp @@ -483,3 +483,80 @@ TEST_CASE( "Parse rng seed in different formats", "[approvals][cli][rng-seed]" ) REQUIRE_FALSE( cli.parse( { "tests", "--rng-seed", seed_string } ) ); } } + +TEST_CASE( "Parsing path filter specs", + "[cli][section-spec]" ) { + using Catch::PathFilter; + + Catch::ConfigData config; + auto cli = Catch::makeCommandLineParser( config ); + SECTION( "Only section specs leads to old filter behaviour" ) { + auto result = cli.parse( { "tests", "-c", "1", "--section", "a section" } ); + REQUIRE( result ); + REQUIRE_FALSE( config.useNewPathFilteringBehaviour ); + } + SECTION( "Generator specs enable new filter behaviour" ) { + auto result = + cli.parse( { "tests", "-g", "1", "--generator-index", "2" } ); + REQUIRE( result ); + REQUIRE( config.useNewPathFilteringBehaviour ); + } + SECTION("Generator specs do not accept stringish arguments") { + auto result = cli.parse( { "tests", "--generator-index", "foo-baz" } ); + REQUIRE_FALSE( result ); + } + SECTION( "Generator specs accept star as argument" ) { + auto result = cli.parse( { "tests", "--generator-index", "*" } ); + REQUIRE( result ); + + REQUIRE( config.pathFilters[0] == + PathFilter( PathFilter::For::Generator, "*" ) ); + } + SECTION( "Generator specs do not accept negative numbers" ) { + auto result = cli.parse( { "tests", "--generator-index", "-2" } ); + REQUIRE_FALSE( result ); + } + SECTION( "Generic path spec enables new filter behaviour" ) { + auto result = + cli.parse( { "tests", "-p", "g:1", "--path-filter", "c:foobar" } ); + REQUIRE( result ); + REQUIRE( config.useNewPathFilteringBehaviour ); + + REQUIRE( config.pathFilters[0] == + PathFilter( PathFilter::For::Generator, "1" ) ); + REQUIRE( config.pathFilters[1] == + PathFilter( PathFilter::For::Section, "foobar" ) ); + } + SECTION( "Generic path spec for generator is validated" ) { + auto result = cli.parse( { "tests", "-p", "g:foo-bar" } ); + REQUIRE_FALSE( result ); + } + SECTION( "Generic path spec without colon is rejected" ) { + auto result1 = cli.parse( { "tests", "-p", "g123" } ); + REQUIRE_FALSE( result1 ); + auto result2 = cli.parse( { "tests", "-p", "carp" } ); + REQUIRE_FALSE( result2 ); + } + SECTION( "Using both section and generator filters creates filter stack" ) { + auto result = cli.parse( { "tests", + "--section", "foo-bar", + "--generator-index", "3", + "-g", "123", + "-c", "baz" + }); + REQUIRE( result ); + REQUIRE( config.pathFilters[0] == PathFilter( PathFilter::For::Section, "foo-bar" ) ); + REQUIRE( config.pathFilters[1] == PathFilter( PathFilter::For::Generator, "3" ) ); + REQUIRE( config.pathFilters[2] == PathFilter( PathFilter::For::Generator, "123" ) ); + REQUIRE( config.pathFilters[3] == PathFilter( PathFilter::For::Section, "baz" ) ); + } + SECTION( "Section/generator filters are whitespace trimmed" ) { + auto result = cli.parse( { "tests", + "--section", " untrimmed ", + "--generator-index", " 42 " + }); + REQUIRE( result ); + REQUIRE( config.pathFilters[0] == PathFilter(PathFilter::For::Section, "untrimmed" ) ); + REQUIRE( config.pathFilters[1] == PathFilter(PathFilter::For::Generator, "42" ) ); + } +} diff --git a/tests/SelfTest/IntrospectiveTests/Details.tests.cpp b/tests/SelfTest/IntrospectiveTests/Details.tests.cpp index d968d942..522b4c22 100644 --- a/tests/SelfTest/IntrospectiveTests/Details.tests.cpp +++ b/tests/SelfTest/IntrospectiveTests/Details.tests.cpp @@ -173,7 +173,7 @@ TEST_CASE( "Decomposer checks that the argument is 0 when handling " } -TEST_CASE( "foo", "[approvals]" ) { +TEST_CASE( "foo", "[approvals][path-filters]" ) { SECTION( "A" ) { SECTION( "B1" ) { REQUIRE( true ); } SECTION( "B2" ) { REQUIRE( true ); } @@ -181,7 +181,7 @@ TEST_CASE( "foo", "[approvals]" ) { } } -TEST_CASE( "bar", "[approvals]" ) { +TEST_CASE( "bar", "[approvals][path-filters]" ) { REQUIRE( true ); SECTION( "A" ) { SECTION( "B1" ) { REQUIRE( true ); } @@ -191,14 +191,14 @@ TEST_CASE( "bar", "[approvals]" ) { REQUIRE( true ); } -TEST_CASE( "baz", "[approvals]" ) { +TEST_CASE( "baz", "[approvals][path-filters]" ) { SECTION( "A" ) { REQUIRE( true ); } auto _ = GENERATE( 1, 2, 3 ); (void)_; SECTION( "B" ) { REQUIRE( true ); } } -TEST_CASE( "qux", "[approvals]" ) { +TEST_CASE( "qux", "[approvals][path-filters]" ) { REQUIRE( true ); SECTION( "A" ) { REQUIRE( true ); } auto _ = GENERATE( 1, 2, 3 ); @@ -207,7 +207,7 @@ TEST_CASE( "qux", "[approvals]" ) { REQUIRE( true ); } -TEST_CASE( "corge", "[approvals]" ) { +TEST_CASE( "corge", "[approvals][path-filters]" ) { REQUIRE( true ); SECTION( "A" ) { REQUIRE( true ); @@ -219,7 +219,7 @@ TEST_CASE( "corge", "[approvals]" ) { REQUIRE( true ); } -TEST_CASE("grault", "[approvals]") { +TEST_CASE( "grault", "[approvals][path-filters]" ) { REQUIRE( true ); SECTION( "A" ) { REQUIRE( true ); @@ -232,3 +232,33 @@ TEST_CASE("grault", "[approvals]") { } REQUIRE( true ); } + +TEST_CASE( "garply", "[approvals][path-filters]" ) { + auto i = GENERATE( 1, 10, 100 ); + CAPTURE( i ); + REQUIRE( true ); +} + +TEST_CASE( "waldo", "[approvals][path-filters]" ) { + auto i = GENERATE( 1, 10, 100 ); + auto j = GENERATE( 2, 20, 200 ); + CAPTURE( i, j ); + REQUIRE( true ); +} + +TEST_CASE( "fred", "[approvals][path-filters]" ) { + REQUIRE( true ); + SECTION( "A" ) { + auto _ = GENERATE( 0, 1, 2 ); + (void)_; + SECTION( "A1" ) { REQUIRE( true ); } + SECTION( "A2" ) { REQUIRE( true ); } + } + SECTION( "B" ) { + auto _ = GENERATE( 0, 1, 2 ); + (void)_; + SECTION( "B1" ) { REQUIRE( true ); } + SECTION( "B2" ) { REQUIRE( true ); } + } +} +// More metasyntactic names if needed: plugh, xyzzy, thud diff --git a/tests/SelfTest/IntrospectiveTests/PartTracker.tests.cpp b/tests/SelfTest/IntrospectiveTests/PartTracker.tests.cpp index c13ec573..09c765aa 100644 --- a/tests/SelfTest/IntrospectiveTests/PartTracker.tests.cpp +++ b/tests/SelfTest/IntrospectiveTests/PartTracker.tests.cpp @@ -9,6 +9,7 @@ #include #include #include +#include using namespace Catch; @@ -22,7 +23,9 @@ Catch::TestCaseTracking::NameAndLocationRef makeNAL( StringRef name ) { TEST_CASE( "Tracker" ) { TrackerContext ctx; - ctx.startRun(); + ITracker& root = ctx.startRun(); + std::vector dummyFilters; + root.setFilters( &dummyFilters, false ); ctx.startCycle(); diff --git a/tests/TestScripts/testSectionFiltering.py b/tests/TestScripts/testSectionFiltering.py index e4f33ad0..4f56df53 100755 --- a/tests/TestScripts/testSectionFiltering.py +++ b/tests/TestScripts/testSectionFiltering.py @@ -22,20 +22,34 @@ import sys from typing import Tuple, List import xml.etree.ElementTree as ET -def make_cli_filter(section_names: Tuple[str, ...]) -> List[str]: +def make_old_filters(section_names: Tuple[str, ...]) -> List[str]: final = [] for name in section_names: final.append('--section') final.append(name) return final -def run_one_test(binary_path: str, test_name: str, section_names: Tuple[str, ...], expected_assertions: int): +def make_new_filters(path_filters: Tuple[str, ...]) -> List[str]: + final = [] + for component in path_filters: + final.append('--path-filter') + final.append(component) + return final + +def run_one_test(binary_path: str, + test_name: str, + section_names: Tuple[str, ...], + expected_assertions: int, + new_filters: bool): cmd = [ binary_path, '--reporter', 'xml', test_name ] - cmd.extend(make_cli_filter(section_names)) + if new_filters: + cmd.extend(make_new_filters(section_names)) + else: + cmd.extend(make_old_filters(section_names)) try: ret = subprocess.run( cmd, @@ -70,7 +84,7 @@ def run_one_test(binary_path: str, test_name: str, section_names: Tuple[str, ... # Inputs taken from issue #3038 -tests = { +old_section_tests = { 'foo': ( ((), 3), (('A',), 3), @@ -117,12 +131,59 @@ tests = { ), } +new_filter_tests = { + 'grault': ( + ((), 12), + (('c:B', 'g:1'), 3), + (('c:B', 'g:1', 'c:i=2'), 3), + # We filter the generator to different element than the dynamic section + (('c:B', 'g:1', 'c:i=1'), 2), + ), + 'garply': ( + ((), 3), + (('g:1',), 1), + (('g:*',), 3), + # Followup section filters do not change results (there are no sections underneath) + (('g:1', 'c:abcdef'), 1), + (('g:*', 'c:abcdef'), 3), + ), + 'waldo': ( + ((), 9), + (('g:1',), 3), + (('g:*',), 9), + (('g:1', 'g:2'), 1), + (('g:*', 'g:1'), 3), + (('g:0', 'g:*'), 3), + (('g:*', 'g:*'), 9), + (('g:*', 'g:1', 'c:abcdef'), 3), + ), + 'fred': ( + ((), 24), + # No sections entered, but we still hit assertion at the start of test + (('c:abcdef',), 1), + (('c:A', ), 12), + (('c:A', 'g:*', 'c:A1'), 6), + (('c:A', 'g:1', 'c:A1'), 2), + (('c:A', 'g:1'), 4), + # Section 'A3' does not exist + (('c:A', 'g:*', 'c:A3'), 3), + (('c:A', 'g:1', 'c:A3'), 1), + # We would like something like 'c:*', 'g:2', but we do not support globs + # for sections (yet?) + ), +} + + if len(sys.argv) != 2: print("Wrong number of arguments, expected just the path to Catch2 SelfTest binary") exit(1) bin_path = os.path.abspath(sys.argv[1]) -for test_filter, specs in tests.items(): +for test_filter, specs in old_section_tests.items(): for section_path, expected_assertions in specs: - run_one_test(bin_path, test_filter, section_path, expected_assertions) + run_one_test(bin_path, test_filter, section_path, expected_assertions, new_filters=False) + +for test_filter, specs in new_filter_tests.items(): + for section_path, expected_assertions in specs: + run_one_test(bin_path, test_filter, section_path, expected_assertions, new_filters=True)