From a0eba50e9d56c743aff1de7a50d02a6e01e73cdc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Ho=C5=99e=C5=88ovsk=C3=BD?= Date: Sat, 1 Aug 2026 16:52:08 +0200 Subject: [PATCH] Make verbosity per-reporter --- docs/command-line.md | 9 ++++-- docs/reporters.md | 8 ++--- src/catch2/catch_config.cpp | 4 ++- src/catch2/catch_config.hpp | 1 + src/catch2/catch_session.cpp | 2 ++ .../interfaces/catch_interfaces_reporter.cpp | 3 ++ .../interfaces/catch_interfaces_reporter.hpp | 4 +++ .../internal/catch_reporter_spec_parser.cpp | 23 ++++++++++++++ .../internal/catch_reporter_spec_parser.hpp | 7 ++++- .../reporters/catch_reporter_common_base.cpp | 9 +++--- .../reporters/catch_reporter_common_base.hpp | 2 ++ tests/CMakeLists.txt | 8 +++++ .../Baselines/compact.sw.approved.txt | 18 +++++------ .../Baselines/compact.sw.multi.approved.txt | 18 +++++------ .../Baselines/console.sw.approved.txt | 18 +++++------ .../Baselines/console.sw.multi.approved.txt | 18 +++++------ tests/SelfTest/Baselines/tap.sw.approved.txt | 18 +++++------ .../Baselines/tap.sw.multi.approved.txt | 18 +++++------ tests/SelfTest/Baselines/xml.sw.approved.txt | 18 +++++------ .../Baselines/xml.sw.multi.approved.txt | 18 +++++------ .../IntrospectiveTests/CmdLine.tests.cpp | 21 +++++++------ .../CmdLineHelpers.tests.cpp | 31 +++++++++++++++++-- .../IntrospectiveTests/Reporters.tests.cpp | 3 +- 23 files changed, 182 insertions(+), 97 deletions(-) diff --git a/docs/command-line.md b/docs/command-line.md index 7df8ecf6..cbb3449e 100644 --- a/docs/command-line.md +++ b/docs/command-line.md @@ -202,8 +202,13 @@ as many times as you want, e.g. `--reporter xml::out=someFile.xml` or The keys must either be prefixed by "X", in which case they are not parsed by Catch2 and are only passed down to the reporter, or one of options -hardcoded into Catch2. Currently there are only 2, -["out"](#sending-output-to-a-file), and ["colour-mode"](#colour-mode). +hardcoded into Catch2. Currently there are 3 supported options: + +* ["out"](#sending-output-to-a-file) +* ["colour-mode"](#colour-mode) +* ["verbosity"](#output-verbosity) + +> Support for per-reporter verbosity option was added in Catch2 vX.Y.Z _Note that the reporter might still check the X-prefixed options for validity, and throw an error if they are wrong._ diff --git a/docs/reporters.md b/docs/reporters.md index d42648cf..63297089 100644 --- a/docs/reporters.md +++ b/docs/reporters.md @@ -43,13 +43,13 @@ them write into different destinations. The two main uses of this are Specifying multiple reporter looks like this: ``` ---reporter JUnit::out=result-junit.xml --reporter console::out=-::colour-mode=ansi +--reporter JUnit::out=result-junit.xml --reporter console::out=-::colour-mode=ansi::verbosity=quiet ``` This tells Catch2 to use two reporters, `JUnit` reporter that writes its machine-readable XML output to file `result-junit.xml`, and the -`console` reporter that writes its user-friendly output to stdout and -uses ANSI colour codes for colouring the output. +`console` reporter that writes its user-friendly output to stdout, uses +ANSI colour codes for colouring the output and is set to "quiet" verbosity. Using multiple reporters (or one reporter and one-or-more [event listeners](event-listeners.md#top)) can have surprisingly complex semantics @@ -110,7 +110,7 @@ passing and failing assertions. _Generally we recommend that if you override a member function from either of the bases, you call into the base's implementation first. This is not -necessarily in all cases, but it is safer and easier._ +necessary in all cases, but it is safer and easier._ Writing your own reporter then looks like this: diff --git a/src/catch2/catch_config.cpp b/src/catch2/catch_config.cpp index b4584f3c..86b9b8dd 100644 --- a/src/catch2/catch_config.cpp +++ b/src/catch2/catch_config.cpp @@ -89,6 +89,7 @@ namespace Catch { return lhs.name == rhs.name && lhs.outputFilename == rhs.outputFilename && lhs.colourMode == rhs.colourMode && + lhs.verbosity == rhs.verbosity && lhs.customOptions == rhs.customOptions; } @@ -157,6 +158,7 @@ namespace Catch { reporterSpec.outputFile() ? *reporterSpec.outputFile() : data.defaultOutputFilename, reporterSpec.colourMode().valueOr( data.defaultColourMode ), + reporterSpec.verbosity().valueOr( data.verbosity ), reporterSpec.customOptions() } ); } } @@ -232,7 +234,7 @@ namespace Catch { if ( bazelOutputFile ) { m_data.reporterSpecifications.push_back( - { "junit", std::string( bazelOutputFile ), {}, {} } ); + { "junit", std::string( bazelOutputFile ), {}, {}, {} } ); } const auto bazelTestSpec = Detail::getEnv( "TESTBRIDGE_TEST_ONLY" ); diff --git a/src/catch2/catch_config.hpp b/src/catch2/catch_config.hpp index c3fea561..429c7156 100644 --- a/src/catch2/catch_config.hpp +++ b/src/catch2/catch_config.hpp @@ -35,6 +35,7 @@ namespace Catch { std::string name; std::string outputFilename; ColourMode colourMode; + Verbosity verbosity; std::map customOptions; friend bool operator==( ProcessedReporterSpec const& lhs, ProcessedReporterSpec const& rhs ); diff --git a/src/catch2/catch_session.cpp b/src/catch2/catch_session.cpp index 0487fe75..97def8dc 100644 --- a/src/catch2/catch_session.cpp +++ b/src/catch2/catch_session.cpp @@ -52,6 +52,7 @@ namespace Catch { ReporterConfig( config, makeStream( spec.outputFilename ), spec.colourMode, + spec.verbosity, spec.customOptions ) ); } @@ -68,6 +69,7 @@ namespace Catch { ReporterConfig( config, makeStream( reporterSpec.outputFilename ), reporterSpec.colourMode, + reporterSpec.verbosity, reporterSpec.customOptions ) ) ); } diff --git a/src/catch2/interfaces/catch_interfaces_reporter.cpp b/src/catch2/interfaces/catch_interfaces_reporter.cpp index 90536bb3..929c6424 100644 --- a/src/catch2/interfaces/catch_interfaces_reporter.cpp +++ b/src/catch2/interfaces/catch_interfaces_reporter.cpp @@ -19,10 +19,12 @@ namespace Catch { IConfig const* _fullConfig, Detail::unique_ptr _stream, ColourMode colourMode, + Verbosity verbosity, std::map customOptions ): m_stream( CATCH_MOVE(_stream) ), m_fullConfig( _fullConfig ), m_colourMode( colourMode ), + m_verbosity( verbosity ), m_customOptions( CATCH_MOVE( customOptions ) ) {} Detail::unique_ptr ReporterConfig::takeStream() && { @@ -31,6 +33,7 @@ namespace Catch { } IConfig const * ReporterConfig::fullConfig() const { return m_fullConfig; } ColourMode ReporterConfig::colourMode() const { return m_colourMode; } + Verbosity ReporterConfig::verbosity() const { return m_verbosity; } std::map const& ReporterConfig::customOptions() const { diff --git a/src/catch2/interfaces/catch_interfaces_reporter.hpp b/src/catch2/interfaces/catch_interfaces_reporter.hpp index c7217889..edf2955f 100644 --- a/src/catch2/interfaces/catch_interfaces_reporter.hpp +++ b/src/catch2/interfaces/catch_interfaces_reporter.hpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -36,6 +37,7 @@ namespace Catch { ReporterConfig( IConfig const* _fullConfig, Detail::unique_ptr _stream, ColourMode colourMode, + Verbosity verbosity, std::map customOptions ); ReporterConfig( ReporterConfig&& ) = default; @@ -45,12 +47,14 @@ namespace Catch { Detail::unique_ptr takeStream() &&; IConfig const* fullConfig() const; ColourMode colourMode() const; + Verbosity verbosity() const; std::map const& customOptions() const; private: Detail::unique_ptr m_stream; IConfig const* m_fullConfig; ColourMode m_colourMode; + Verbosity m_verbosity; std::map m_customOptions; }; diff --git a/src/catch2/internal/catch_reporter_spec_parser.cpp b/src/catch2/internal/catch_reporter_spec_parser.cpp index 2b08758a..3347c082 100644 --- a/src/catch2/internal/catch_reporter_spec_parser.cpp +++ b/src/catch2/internal/catch_reporter_spec_parser.cpp @@ -93,6 +93,18 @@ namespace Catch { return {}; } } + + Optional stringToVerbosity( StringRef verbosity ) { + if (verbosity == "quiet") { return Verbosity::Quiet; + } else if ( verbosity == "normal" ) { + return Verbosity::Normal; + } else if ( verbosity == "high" ) { + return Verbosity::High; + } else { + return {}; + } + } + } // namespace Detail @@ -100,6 +112,7 @@ namespace Catch { return lhs.m_name == rhs.m_name && lhs.m_outputFileName == rhs.m_outputFileName && lhs.m_colourMode == rhs.m_colourMode && + lhs.m_verbosity == rhs.m_verbosity && lhs.m_customOptions == rhs.m_customOptions; } @@ -111,6 +124,7 @@ namespace Catch { std::map kvPairs; Optional outputFileName; Optional colourMode; + Optional verbosity; // First part is always reporter name, so we skip it for ( size_t i = 1; i < parts.size(); ++i ) { @@ -148,6 +162,12 @@ namespace Catch { if ( !colourMode ) { return {}; } + } else if ( key == "verbosity" ) { + // Duplicated key + if ( verbosity ) { return {}; } + verbosity = Detail::stringToVerbosity( value ); + // Parsing failed + if ( !verbosity ) { return {}; } } else { // Unrecognized option return {}; @@ -157,6 +177,7 @@ namespace Catch { return ReporterSpec{ CATCH_MOVE( parts[0] ), CATCH_MOVE( outputFileName ), CATCH_MOVE( colourMode ), + CATCH_MOVE( verbosity), CATCH_MOVE( kvPairs ) }; } @@ -164,10 +185,12 @@ ReporterSpec::ReporterSpec( std::string name, Optional outputFileName, Optional colourMode, + Optional verbosity, std::map customOptions ): m_name( CATCH_MOVE( name ) ), m_outputFileName( CATCH_MOVE( outputFileName ) ), m_colourMode( CATCH_MOVE( colourMode ) ), + m_verbosity( CATCH_MOVE( verbosity ) ), m_customOptions( CATCH_MOVE( customOptions ) ) {} } // namespace Catch diff --git a/src/catch2/internal/catch_reporter_spec_parser.hpp b/src/catch2/internal/catch_reporter_spec_parser.hpp index 9f447ee2..41102c2c 100644 --- a/src/catch2/internal/catch_reporter_spec_parser.hpp +++ b/src/catch2/internal/catch_reporter_spec_parser.hpp @@ -25,6 +25,7 @@ namespace Catch { std::vector splitReporterSpec( StringRef reporterSpec ); Optional stringToColourMode( StringRef colourMode ); + Optional stringToVerbosity( StringRef verbosity ); } /** @@ -39,6 +40,7 @@ namespace Catch { std::string m_name; Optional m_outputFileName; Optional m_colourMode; + Optional m_verbosity; std::map m_customOptions; friend bool operator==( ReporterSpec const& lhs, @@ -53,6 +55,7 @@ namespace Catch { std::string name, Optional outputFileName, Optional colourMode, + Optional verbosity, std::map customOptions ); std::string const& name() const { return m_name; } @@ -63,13 +66,15 @@ namespace Catch { Optional const& colourMode() const { return m_colourMode; } + Optional const& verbosity() const { return m_verbosity; } + std::map const& customOptions() const { return m_customOptions; } }; /** - * Parses provided reporter spec string into + * Parses provided reporter spec string into actual `ReporterSpec` * * Returns empty optional on errors, e.g. * * field that is not first and not a key+value pair diff --git a/src/catch2/reporters/catch_reporter_common_base.cpp b/src/catch2/reporters/catch_reporter_common_base.cpp index 0a3671e1..638824d2 100644 --- a/src/catch2/reporters/catch_reporter_common_base.cpp +++ b/src/catch2/reporters/catch_reporter_common_base.cpp @@ -19,6 +19,7 @@ namespace Catch { m_wrapped_stream( CATCH_MOVE(config).takeStream() ), m_stream( m_wrapped_stream->stream() ), m_colour( makeColourImpl( config.colourMode(), m_wrapped_stream.get() ) ), + m_verbosity( config.verbosity() ), m_customOptions( config.customOptions() ) {} @@ -26,12 +27,12 @@ namespace Catch { void ReporterBase::listReporters( std::vector const& descriptions ) { - defaultListReporters( m_stream, descriptions, m_config->verbosity() ); + defaultListReporters( m_stream, descriptions, m_verbosity ); } void ReporterBase::listListeners( std::vector const& descriptions ) { - defaultListListeners( m_stream, descriptions, m_config->verbosity() ); + defaultListListeners( m_stream, descriptions, m_verbosity ); } void ReporterBase::listTests(std::vector const& tests) { @@ -39,11 +40,11 @@ namespace Catch { m_colour.get(), tests, m_config->hasTestFilters(), - m_config->verbosity()); + m_verbosity); } void ReporterBase::listTags(std::vector const& tags) { - defaultListTags( m_stream, tags, m_config->hasTestFilters(), m_config->verbosity() ); + defaultListTags( m_stream, tags, m_config->hasTestFilters(), m_verbosity ); } } // namespace Catch diff --git a/src/catch2/reporters/catch_reporter_common_base.hpp b/src/catch2/reporters/catch_reporter_common_base.hpp index b4f0a9ff..a7497a35 100644 --- a/src/catch2/reporters/catch_reporter_common_base.hpp +++ b/src/catch2/reporters/catch_reporter_common_base.hpp @@ -35,6 +35,8 @@ namespace Catch { std::ostream& m_stream; //! Colour implementation this reporter was configured for Detail::unique_ptr m_colour; + //! Verbosity configured for this reporter + Verbosity m_verbosity; //! The custom reporter options user passed down to the reporter std::map m_customOptions; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 6a2028b6..eba8c22b 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -184,6 +184,13 @@ set_tests_properties(List::Tests::Output PROPERTIES PASS_REGULAR_EXPRESSION "[0-9]+ test cases" FAIL_REGULAR_EXPRESSION "Hidden Test" ) + +add_test(NAME VerbosityIsPerReporter COMMAND $ --list-tests --verbosity high --reporter console::verbosity=quiet) +set_tests_properties(VerbosityIsPerReporter + PROPERTIES + FAIL_REGULAR_EXPRESSION "\.cpp" +) + # This should be equivalent to the old --list-test-names-only and be usable # with --input-file. add_test(NAME List::Tests::Quiet COMMAND $ --list-tests --verbosity quiet) @@ -193,6 +200,7 @@ set_tests_properties(List::Tests::Quiet PROPERTIES PASS_REGULAR_EXPRESSION "\"#1905 -- test spec parser properly clears internal state between compound tests\"[\r\n]" FAIL_REGULAR_EXPRESSION "[ \t]\"#1905 -- test spec parser properly clears internal state between compound tests\"" ) + add_test(NAME List::Tests::ExitCode COMMAND $ --list-tests --verbosity high) add_test(NAME List::Tests::XmlOutput COMMAND $ --list-tests --verbosity high -r xml) set_tests_properties(List::Tests::XmlOutput PROPERTIES diff --git a/tests/SelfTest/Baselines/compact.sw.approved.txt b/tests/SelfTest/Baselines/compact.sw.approved.txt index 7989d276..95809e24 100644 --- a/tests/SelfTest/Baselines/compact.sw.approved.txt +++ b/tests/SelfTest/Baselines/compact.sw.approved.txt @@ -1531,9 +1531,9 @@ CmdLine.tests.cpp:: passed: config.noThrow == false for: false == f CmdLine.tests.cpp:: passed: config.reporterSpecifications.empty() for: true CmdLine.tests.cpp:: passed: !(cfg.hasTestFilters()) for: !false CmdLine.tests.cpp:: passed: cfg.getReporterSpecs().size() == 1 for: 1 == 1 -CmdLine.tests.cpp:: passed: cfg.getReporterSpecs()[0] == Catch::ReporterSpec{ expectedReporter, {}, {}, {} } for: {?} == {?} +CmdLine.tests.cpp:: passed: cfg.getReporterSpecs()[0] == Catch::ReporterSpec{ expectedReporter, {}, {}, {}, {} } for: {?} == {?} CmdLine.tests.cpp:: passed: cfg.getProcessedReporterSpecs().size() == 1 for: 1 == 1 -CmdLine.tests.cpp:: passed: cfg.getProcessedReporterSpecs()[0] == Catch::ProcessedReporterSpec{ expectedReporter, std::string{}, Catch::ColourMode::PlatformDefault, {} } for: {?} == {?} +CmdLine.tests.cpp:: passed: cfg.getProcessedReporterSpecs()[0] == Catch::ProcessedReporterSpec{ expectedReporter, std::string{}, Catch::ColourMode::PlatformDefault, Catch::Verbosity::Normal, {} } for: {?} == {?} CmdLine.tests.cpp:: passed: result for: {?} CmdLine.tests.cpp:: passed: cfg.hasTestFilters() for: true CmdLine.tests.cpp:: passed: cfg.testSpec().matches(*fakeTestCase("notIncluded")) == false for: false == false @@ -1547,21 +1547,21 @@ CmdLine.tests.cpp:: passed: cfg.hasTestFilters() for: true CmdLine.tests.cpp:: passed: cfg.testSpec().matches(*fakeTestCase("test1")) == false for: false == false CmdLine.tests.cpp:: passed: cfg.testSpec().matches(*fakeTestCase("alwaysIncluded")) for: true CmdLine.tests.cpp:: passed: result for: {?} with 1 message: 'result.errorMessage() := ""' -CmdLine.tests.cpp:: passed: config.reporterSpecifications == vec_Specs{ { "console", {}, {}, {} } } for: { {?} } == { {?} } with 1 message: 'result.errorMessage() := ""' +CmdLine.tests.cpp:: passed: config.reporterSpecifications == vec_Specs{ { "console", {}, {}, {}, {} } } for: { {?} } == { {?} } with 1 message: 'result.errorMessage() := ""' CmdLine.tests.cpp:: passed: result for: {?} with 1 message: 'result.errorMessage() := ""' -CmdLine.tests.cpp:: passed: config.reporterSpecifications == vec_Specs{ { "xml", {}, {}, {} } } for: { {?} } == { {?} } with 1 message: 'result.errorMessage() := ""' +CmdLine.tests.cpp:: passed: config.reporterSpecifications == vec_Specs{ { "xml", {}, {}, {}, {} } } for: { {?} } == { {?} } with 1 message: 'result.errorMessage() := ""' CmdLine.tests.cpp:: passed: result for: {?} with 1 message: 'result.errorMessage() := ""' -CmdLine.tests.cpp:: passed: config.reporterSpecifications == vec_Specs{ { "junit", {}, {}, {} } } for: { {?} } == { {?} } with 1 message: 'result.errorMessage() := ""' +CmdLine.tests.cpp:: passed: config.reporterSpecifications == vec_Specs{ { "junit", {}, {}, {}, {} } } for: { {?} } == { {?} } with 1 message: 'result.errorMessage() := ""' CmdLine.tests.cpp:: passed: !result for: true CmdLine.tests.cpp:: passed: result.errorMessage(), ContainsSubstring("Unrecognized reporter") for: "Unrecognized reporter, 'unsupported'. Check available with --list-reporters" contains: "Unrecognized reporter" CmdLine.tests.cpp:: passed: result for: {?} with 1 message: 'result.errorMessage() := ""' -CmdLine.tests.cpp:: passed: config.reporterSpecifications == vec_Specs{ { "console", "out.txt"s, {}, {} } } for: { {?} } == { {?} } with 1 message: 'result.errorMessage() := ""' +CmdLine.tests.cpp:: passed: config.reporterSpecifications == vec_Specs{ { "console", "out.txt"s, {}, {}, {} } } for: { {?} } == { {?} } with 1 message: 'result.errorMessage() := ""' CmdLine.tests.cpp:: passed: result for: {?} with 1 message: 'result.errorMessage() := ""' -CmdLine.tests.cpp:: passed: config.reporterSpecifications == vec_Specs{ { "console", "C:\\Temp\\out.txt"s, {}, {} } } for: { {?} } == { {?} } with 1 message: 'result.errorMessage() := ""' +CmdLine.tests.cpp:: passed: config.reporterSpecifications == vec_Specs{ { "console", "C:\\Temp\\out.txt"s, {}, {}, {} } } for: { {?} } == { {?} } with 1 message: 'result.errorMessage() := ""' CmdLine.tests.cpp:: passed: cli.parse({ "test", "-r", "xml::out=output.xml", "-r", "junit::out=output-junit.xml" }) for: {?} -CmdLine.tests.cpp:: passed: config.reporterSpecifications == vec_Specs{ { "xml", "output.xml"s, {}, {} }, { "junit", "output-junit.xml"s, {}, {} } } for: { {?}, {?} } == { {?}, {?} } +CmdLine.tests.cpp:: passed: config.reporterSpecifications == vec_Specs{ { "xml", "output.xml"s, {}, {}, {} }, { "junit", "output-junit.xml"s, {}, {}, {} } } for: { {?}, {?} } == { {?}, {?} } CmdLine.tests.cpp:: passed: cli.parse({ "test", "-r", "xml::out=output.xml", "-r", "console" }) for: {?} -CmdLine.tests.cpp:: passed: config.reporterSpecifications == vec_Specs{ { "xml", "output.xml"s, {}, {} }, { "console", {}, {}, {} } } for: { {?}, {?} } == { {?}, {?} } +CmdLine.tests.cpp:: passed: config.reporterSpecifications == vec_Specs{ { "xml", "output.xml"s, {}, {}, {} }, { "console", {}, {}, {}, {} } } for: { {?}, {?} } == { {?}, {?} } CmdLine.tests.cpp:: passed: !result for: true CmdLine.tests.cpp:: passed: result.errorMessage(), ContainsSubstring("Only one reporter may have unspecified output file.") for: "Only one reporter may have unspecified output file." contains: "Only one reporter may have unspecified output file." CmdLine.tests.cpp:: passed: cli.parse({"test", "-b"}) for: {?} diff --git a/tests/SelfTest/Baselines/compact.sw.multi.approved.txt b/tests/SelfTest/Baselines/compact.sw.multi.approved.txt index ffe7a2f6..889ad483 100644 --- a/tests/SelfTest/Baselines/compact.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/compact.sw.multi.approved.txt @@ -1529,9 +1529,9 @@ CmdLine.tests.cpp:: passed: config.noThrow == false for: false == f CmdLine.tests.cpp:: passed: config.reporterSpecifications.empty() for: true CmdLine.tests.cpp:: passed: !(cfg.hasTestFilters()) for: !false CmdLine.tests.cpp:: passed: cfg.getReporterSpecs().size() == 1 for: 1 == 1 -CmdLine.tests.cpp:: passed: cfg.getReporterSpecs()[0] == Catch::ReporterSpec{ expectedReporter, {}, {}, {} } for: {?} == {?} +CmdLine.tests.cpp:: passed: cfg.getReporterSpecs()[0] == Catch::ReporterSpec{ expectedReporter, {}, {}, {}, {} } for: {?} == {?} CmdLine.tests.cpp:: passed: cfg.getProcessedReporterSpecs().size() == 1 for: 1 == 1 -CmdLine.tests.cpp:: passed: cfg.getProcessedReporterSpecs()[0] == Catch::ProcessedReporterSpec{ expectedReporter, std::string{}, Catch::ColourMode::PlatformDefault, {} } for: {?} == {?} +CmdLine.tests.cpp:: passed: cfg.getProcessedReporterSpecs()[0] == Catch::ProcessedReporterSpec{ expectedReporter, std::string{}, Catch::ColourMode::PlatformDefault, Catch::Verbosity::Normal, {} } for: {?} == {?} CmdLine.tests.cpp:: passed: result for: {?} CmdLine.tests.cpp:: passed: cfg.hasTestFilters() for: true CmdLine.tests.cpp:: passed: cfg.testSpec().matches(*fakeTestCase("notIncluded")) == false for: false == false @@ -1545,21 +1545,21 @@ CmdLine.tests.cpp:: passed: cfg.hasTestFilters() for: true CmdLine.tests.cpp:: passed: cfg.testSpec().matches(*fakeTestCase("test1")) == false for: false == false CmdLine.tests.cpp:: passed: cfg.testSpec().matches(*fakeTestCase("alwaysIncluded")) for: true CmdLine.tests.cpp:: passed: result for: {?} with 1 message: 'result.errorMessage() := ""' -CmdLine.tests.cpp:: passed: config.reporterSpecifications == vec_Specs{ { "console", {}, {}, {} } } for: { {?} } == { {?} } with 1 message: 'result.errorMessage() := ""' +CmdLine.tests.cpp:: passed: config.reporterSpecifications == vec_Specs{ { "console", {}, {}, {}, {} } } for: { {?} } == { {?} } with 1 message: 'result.errorMessage() := ""' CmdLine.tests.cpp:: passed: result for: {?} with 1 message: 'result.errorMessage() := ""' -CmdLine.tests.cpp:: passed: config.reporterSpecifications == vec_Specs{ { "xml", {}, {}, {} } } for: { {?} } == { {?} } with 1 message: 'result.errorMessage() := ""' +CmdLine.tests.cpp:: passed: config.reporterSpecifications == vec_Specs{ { "xml", {}, {}, {}, {} } } for: { {?} } == { {?} } with 1 message: 'result.errorMessage() := ""' CmdLine.tests.cpp:: passed: result for: {?} with 1 message: 'result.errorMessage() := ""' -CmdLine.tests.cpp:: passed: config.reporterSpecifications == vec_Specs{ { "junit", {}, {}, {} } } for: { {?} } == { {?} } with 1 message: 'result.errorMessage() := ""' +CmdLine.tests.cpp:: passed: config.reporterSpecifications == vec_Specs{ { "junit", {}, {}, {}, {} } } for: { {?} } == { {?} } with 1 message: 'result.errorMessage() := ""' CmdLine.tests.cpp:: passed: !result for: true CmdLine.tests.cpp:: passed: result.errorMessage(), ContainsSubstring("Unrecognized reporter") for: "Unrecognized reporter, 'unsupported'. Check available with --list-reporters" contains: "Unrecognized reporter" CmdLine.tests.cpp:: passed: result for: {?} with 1 message: 'result.errorMessage() := ""' -CmdLine.tests.cpp:: passed: config.reporterSpecifications == vec_Specs{ { "console", "out.txt"s, {}, {} } } for: { {?} } == { {?} } with 1 message: 'result.errorMessage() := ""' +CmdLine.tests.cpp:: passed: config.reporterSpecifications == vec_Specs{ { "console", "out.txt"s, {}, {}, {} } } for: { {?} } == { {?} } with 1 message: 'result.errorMessage() := ""' CmdLine.tests.cpp:: passed: result for: {?} with 1 message: 'result.errorMessage() := ""' -CmdLine.tests.cpp:: passed: config.reporterSpecifications == vec_Specs{ { "console", "C:\\Temp\\out.txt"s, {}, {} } } for: { {?} } == { {?} } with 1 message: 'result.errorMessage() := ""' +CmdLine.tests.cpp:: passed: config.reporterSpecifications == vec_Specs{ { "console", "C:\\Temp\\out.txt"s, {}, {}, {} } } for: { {?} } == { {?} } with 1 message: 'result.errorMessage() := ""' CmdLine.tests.cpp:: passed: cli.parse({ "test", "-r", "xml::out=output.xml", "-r", "junit::out=output-junit.xml" }) for: {?} -CmdLine.tests.cpp:: passed: config.reporterSpecifications == vec_Specs{ { "xml", "output.xml"s, {}, {} }, { "junit", "output-junit.xml"s, {}, {} } } for: { {?}, {?} } == { {?}, {?} } +CmdLine.tests.cpp:: passed: config.reporterSpecifications == vec_Specs{ { "xml", "output.xml"s, {}, {}, {} }, { "junit", "output-junit.xml"s, {}, {}, {} } } for: { {?}, {?} } == { {?}, {?} } CmdLine.tests.cpp:: passed: cli.parse({ "test", "-r", "xml::out=output.xml", "-r", "console" }) for: {?} -CmdLine.tests.cpp:: passed: config.reporterSpecifications == vec_Specs{ { "xml", "output.xml"s, {}, {} }, { "console", {}, {}, {} } } for: { {?}, {?} } == { {?}, {?} } +CmdLine.tests.cpp:: passed: config.reporterSpecifications == vec_Specs{ { "xml", "output.xml"s, {}, {}, {} }, { "console", {}, {}, {}, {} } } for: { {?}, {?} } == { {?}, {?} } CmdLine.tests.cpp:: passed: !result for: true CmdLine.tests.cpp:: passed: result.errorMessage(), ContainsSubstring("Only one reporter may have unspecified output file.") for: "Only one reporter may have unspecified output file." contains: "Only one reporter may have unspecified output file." CmdLine.tests.cpp:: passed: cli.parse({"test", "-b"}) for: {?} diff --git a/tests/SelfTest/Baselines/console.sw.approved.txt b/tests/SelfTest/Baselines/console.sw.approved.txt index 0729823f..1ae34284 100644 --- a/tests/SelfTest/Baselines/console.sw.approved.txt +++ b/tests/SelfTest/Baselines/console.sw.approved.txt @@ -9977,7 +9977,7 @@ with expansion: 1 == 1 CmdLine.tests.cpp:: PASSED: - CHECK( cfg.getReporterSpecs()[0] == Catch::ReporterSpec{ expectedReporter, {}, {}, {} } ) + CHECK( cfg.getReporterSpecs()[0] == Catch::ReporterSpec{ expectedReporter, {}, {}, {}, {} } ) with expansion: {?} == {?} @@ -9987,7 +9987,7 @@ with expansion: 1 == 1 CmdLine.tests.cpp:: PASSED: - CHECK( cfg.getProcessedReporterSpecs()[0] == Catch::ProcessedReporterSpec{ expectedReporter, std::string{}, Catch::ColourMode::PlatformDefault, {} } ) + CHECK( cfg.getProcessedReporterSpecs()[0] == Catch::ProcessedReporterSpec{ expectedReporter, std::string{}, Catch::ColourMode::PlatformDefault, Catch::Verbosity::Normal, {} } ) with expansion: {?} == {?} @@ -10091,7 +10091,7 @@ with message: result.errorMessage() := "" CmdLine.tests.cpp:: PASSED: - REQUIRE( config.reporterSpecifications == vec_Specs{ { "console", {}, {}, {} } } ) + REQUIRE( config.reporterSpecifications == vec_Specs{ { "console", {}, {}, {}, {} } } ) with expansion: { {?} } == { {?} } with message: @@ -10113,7 +10113,7 @@ with message: result.errorMessage() := "" CmdLine.tests.cpp:: PASSED: - REQUIRE( config.reporterSpecifications == vec_Specs{ { "xml", {}, {}, {} } } ) + REQUIRE( config.reporterSpecifications == vec_Specs{ { "xml", {}, {}, {}, {} } } ) with expansion: { {?} } == { {?} } with message: @@ -10135,7 +10135,7 @@ with message: result.errorMessage() := "" CmdLine.tests.cpp:: PASSED: - REQUIRE( config.reporterSpecifications == vec_Specs{ { "junit", {}, {}, {} } } ) + REQUIRE( config.reporterSpecifications == vec_Specs{ { "junit", {}, {}, {}, {} } } ) with expansion: { {?} } == { {?} } with message: @@ -10176,7 +10176,7 @@ with message: result.errorMessage() := "" CmdLine.tests.cpp:: PASSED: - REQUIRE( config.reporterSpecifications == vec_Specs{ { "console", "out.txt"s, {}, {} } } ) + REQUIRE( config.reporterSpecifications == vec_Specs{ { "console", "out.txt"s, {}, {}, {} } } ) with expansion: { {?} } == { {?} } with message: @@ -10198,7 +10198,7 @@ with message: result.errorMessage() := "" CmdLine.tests.cpp:: PASSED: - REQUIRE( config.reporterSpecifications == vec_Specs{ { "console", "C:\\Temp\\out.txt"s, {}, {} } } ) + REQUIRE( config.reporterSpecifications == vec_Specs{ { "console", "C:\\Temp\\out.txt"s, {}, {}, {} } } ) with expansion: { {?} } == { {?} } with message: @@ -10219,7 +10219,7 @@ with expansion: {?} CmdLine.tests.cpp:: PASSED: - REQUIRE( config.reporterSpecifications == vec_Specs{ { "xml", "output.xml"s, {}, {} }, { "junit", "output-junit.xml"s, {}, {} } } ) + REQUIRE( config.reporterSpecifications == vec_Specs{ { "xml", "output.xml"s, {}, {}, {} }, { "junit", "output-junit.xml"s, {}, {}, {} } } ) with expansion: { {?}, {?} } == { {?}, {?} } @@ -10238,7 +10238,7 @@ with expansion: {?} CmdLine.tests.cpp:: PASSED: - REQUIRE( config.reporterSpecifications == vec_Specs{ { "xml", "output.xml"s, {}, {} }, { "console", {}, {}, {} } } ) + REQUIRE( config.reporterSpecifications == vec_Specs{ { "xml", "output.xml"s, {}, {}, {} }, { "console", {}, {}, {}, {} } } ) with expansion: { {?}, {?} } == { {?}, {?} } diff --git a/tests/SelfTest/Baselines/console.sw.multi.approved.txt b/tests/SelfTest/Baselines/console.sw.multi.approved.txt index 3db4bfc8..897f79ef 100644 --- a/tests/SelfTest/Baselines/console.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/console.sw.multi.approved.txt @@ -9975,7 +9975,7 @@ with expansion: 1 == 1 CmdLine.tests.cpp:: PASSED: - CHECK( cfg.getReporterSpecs()[0] == Catch::ReporterSpec{ expectedReporter, {}, {}, {} } ) + CHECK( cfg.getReporterSpecs()[0] == Catch::ReporterSpec{ expectedReporter, {}, {}, {}, {} } ) with expansion: {?} == {?} @@ -9985,7 +9985,7 @@ with expansion: 1 == 1 CmdLine.tests.cpp:: PASSED: - CHECK( cfg.getProcessedReporterSpecs()[0] == Catch::ProcessedReporterSpec{ expectedReporter, std::string{}, Catch::ColourMode::PlatformDefault, {} } ) + CHECK( cfg.getProcessedReporterSpecs()[0] == Catch::ProcessedReporterSpec{ expectedReporter, std::string{}, Catch::ColourMode::PlatformDefault, Catch::Verbosity::Normal, {} } ) with expansion: {?} == {?} @@ -10089,7 +10089,7 @@ with message: result.errorMessage() := "" CmdLine.tests.cpp:: PASSED: - REQUIRE( config.reporterSpecifications == vec_Specs{ { "console", {}, {}, {} } } ) + REQUIRE( config.reporterSpecifications == vec_Specs{ { "console", {}, {}, {}, {} } } ) with expansion: { {?} } == { {?} } with message: @@ -10111,7 +10111,7 @@ with message: result.errorMessage() := "" CmdLine.tests.cpp:: PASSED: - REQUIRE( config.reporterSpecifications == vec_Specs{ { "xml", {}, {}, {} } } ) + REQUIRE( config.reporterSpecifications == vec_Specs{ { "xml", {}, {}, {}, {} } } ) with expansion: { {?} } == { {?} } with message: @@ -10133,7 +10133,7 @@ with message: result.errorMessage() := "" CmdLine.tests.cpp:: PASSED: - REQUIRE( config.reporterSpecifications == vec_Specs{ { "junit", {}, {}, {} } } ) + REQUIRE( config.reporterSpecifications == vec_Specs{ { "junit", {}, {}, {}, {} } } ) with expansion: { {?} } == { {?} } with message: @@ -10174,7 +10174,7 @@ with message: result.errorMessage() := "" CmdLine.tests.cpp:: PASSED: - REQUIRE( config.reporterSpecifications == vec_Specs{ { "console", "out.txt"s, {}, {} } } ) + REQUIRE( config.reporterSpecifications == vec_Specs{ { "console", "out.txt"s, {}, {}, {} } } ) with expansion: { {?} } == { {?} } with message: @@ -10196,7 +10196,7 @@ with message: result.errorMessage() := "" CmdLine.tests.cpp:: PASSED: - REQUIRE( config.reporterSpecifications == vec_Specs{ { "console", "C:\\Temp\\out.txt"s, {}, {} } } ) + REQUIRE( config.reporterSpecifications == vec_Specs{ { "console", "C:\\Temp\\out.txt"s, {}, {}, {} } } ) with expansion: { {?} } == { {?} } with message: @@ -10217,7 +10217,7 @@ with expansion: {?} CmdLine.tests.cpp:: PASSED: - REQUIRE( config.reporterSpecifications == vec_Specs{ { "xml", "output.xml"s, {}, {} }, { "junit", "output-junit.xml"s, {}, {} } } ) + REQUIRE( config.reporterSpecifications == vec_Specs{ { "xml", "output.xml"s, {}, {}, {} }, { "junit", "output-junit.xml"s, {}, {}, {} } } ) with expansion: { {?}, {?} } == { {?}, {?} } @@ -10236,7 +10236,7 @@ with expansion: {?} CmdLine.tests.cpp:: PASSED: - REQUIRE( config.reporterSpecifications == vec_Specs{ { "xml", "output.xml"s, {}, {} }, { "console", {}, {}, {} } } ) + REQUIRE( config.reporterSpecifications == vec_Specs{ { "xml", "output.xml"s, {}, {}, {} }, { "console", {}, {}, {}, {} } } ) with expansion: { {?}, {?} } == { {?}, {?} } diff --git a/tests/SelfTest/Baselines/tap.sw.approved.txt b/tests/SelfTest/Baselines/tap.sw.approved.txt index 4ac7f09f..9a4826ed 100644 --- a/tests/SelfTest/Baselines/tap.sw.approved.txt +++ b/tests/SelfTest/Baselines/tap.sw.approved.txt @@ -2515,11 +2515,11 @@ ok {test-number} - !(cfg.hasTestFilters()) for: !false # Process can be configured on command line ok {test-number} - cfg.getReporterSpecs().size() == 1 for: 1 == 1 # Process can be configured on command line -ok {test-number} - cfg.getReporterSpecs()[0] == Catch::ReporterSpec{ expectedReporter, {}, {}, {} } for: {?} == {?} +ok {test-number} - cfg.getReporterSpecs()[0] == Catch::ReporterSpec{ expectedReporter, {}, {}, {}, {} } for: {?} == {?} # Process can be configured on command line ok {test-number} - cfg.getProcessedReporterSpecs().size() == 1 for: 1 == 1 # Process can be configured on command line -ok {test-number} - cfg.getProcessedReporterSpecs()[0] == Catch::ProcessedReporterSpec{ expectedReporter, std::string{}, Catch::ColourMode::PlatformDefault, {} } for: {?} == {?} +ok {test-number} - cfg.getProcessedReporterSpecs()[0] == Catch::ProcessedReporterSpec{ expectedReporter, std::string{}, Catch::ColourMode::PlatformDefault, Catch::Verbosity::Normal, {} } for: {?} == {?} # Process can be configured on command line ok {test-number} - result for: {?} # Process can be configured on command line @@ -2547,15 +2547,15 @@ ok {test-number} - cfg.testSpec().matches(*fakeTestCase("alwaysIncluded")) for: # Process can be configured on command line ok {test-number} - result for: {?} with 1 message: 'result.errorMessage() := ""' # Process can be configured on command line -ok {test-number} - config.reporterSpecifications == vec_Specs{ { "console", {}, {}, {} } } for: { {?} } == { {?} } with 1 message: 'result.errorMessage() := ""' +ok {test-number} - config.reporterSpecifications == vec_Specs{ { "console", {}, {}, {}, {} } } for: { {?} } == { {?} } with 1 message: 'result.errorMessage() := ""' # Process can be configured on command line ok {test-number} - result for: {?} with 1 message: 'result.errorMessage() := ""' # Process can be configured on command line -ok {test-number} - config.reporterSpecifications == vec_Specs{ { "xml", {}, {}, {} } } for: { {?} } == { {?} } with 1 message: 'result.errorMessage() := ""' +ok {test-number} - config.reporterSpecifications == vec_Specs{ { "xml", {}, {}, {}, {} } } for: { {?} } == { {?} } with 1 message: 'result.errorMessage() := ""' # Process can be configured on command line ok {test-number} - result for: {?} with 1 message: 'result.errorMessage() := ""' # Process can be configured on command line -ok {test-number} - config.reporterSpecifications == vec_Specs{ { "junit", {}, {}, {} } } for: { {?} } == { {?} } with 1 message: 'result.errorMessage() := ""' +ok {test-number} - config.reporterSpecifications == vec_Specs{ { "junit", {}, {}, {}, {} } } for: { {?} } == { {?} } with 1 message: 'result.errorMessage() := ""' # Process can be configured on command line ok {test-number} - !result for: true # Process can be configured on command line @@ -2563,19 +2563,19 @@ ok {test-number} - result.errorMessage(), ContainsSubstring("Unrecognized report # Process can be configured on command line ok {test-number} - result for: {?} with 1 message: 'result.errorMessage() := ""' # Process can be configured on command line -ok {test-number} - config.reporterSpecifications == vec_Specs{ { "console", "out.txt"s, {}, {} } } for: { {?} } == { {?} } with 1 message: 'result.errorMessage() := ""' +ok {test-number} - config.reporterSpecifications == vec_Specs{ { "console", "out.txt"s, {}, {}, {} } } for: { {?} } == { {?} } with 1 message: 'result.errorMessage() := ""' # Process can be configured on command line ok {test-number} - result for: {?} with 1 message: 'result.errorMessage() := ""' # Process can be configured on command line -ok {test-number} - config.reporterSpecifications == vec_Specs{ { "console", "C:\\Temp\\out.txt"s, {}, {} } } for: { {?} } == { {?} } with 1 message: 'result.errorMessage() := ""' +ok {test-number} - config.reporterSpecifications == vec_Specs{ { "console", "C:\\Temp\\out.txt"s, {}, {}, {} } } for: { {?} } == { {?} } with 1 message: 'result.errorMessage() := ""' # Process can be configured on command line ok {test-number} - cli.parse({ "test", "-r", "xml::out=output.xml", "-r", "junit::out=output-junit.xml" }) for: {?} # Process can be configured on command line -ok {test-number} - config.reporterSpecifications == vec_Specs{ { "xml", "output.xml"s, {}, {} }, { "junit", "output-junit.xml"s, {}, {} } } for: { {?}, {?} } == { {?}, {?} } +ok {test-number} - config.reporterSpecifications == vec_Specs{ { "xml", "output.xml"s, {}, {}, {} }, { "junit", "output-junit.xml"s, {}, {}, {} } } for: { {?}, {?} } == { {?}, {?} } # Process can be configured on command line ok {test-number} - cli.parse({ "test", "-r", "xml::out=output.xml", "-r", "console" }) for: {?} # Process can be configured on command line -ok {test-number} - config.reporterSpecifications == vec_Specs{ { "xml", "output.xml"s, {}, {} }, { "console", {}, {}, {} } } for: { {?}, {?} } == { {?}, {?} } +ok {test-number} - config.reporterSpecifications == vec_Specs{ { "xml", "output.xml"s, {}, {}, {} }, { "console", {}, {}, {}, {} } } for: { {?}, {?} } == { {?}, {?} } # Process can be configured on command line ok {test-number} - !result for: true # Process can be configured on command line diff --git a/tests/SelfTest/Baselines/tap.sw.multi.approved.txt b/tests/SelfTest/Baselines/tap.sw.multi.approved.txt index 3f1d56aa..cad770a2 100644 --- a/tests/SelfTest/Baselines/tap.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/tap.sw.multi.approved.txt @@ -2513,11 +2513,11 @@ ok {test-number} - !(cfg.hasTestFilters()) for: !false # Process can be configured on command line ok {test-number} - cfg.getReporterSpecs().size() == 1 for: 1 == 1 # Process can be configured on command line -ok {test-number} - cfg.getReporterSpecs()[0] == Catch::ReporterSpec{ expectedReporter, {}, {}, {} } for: {?} == {?} +ok {test-number} - cfg.getReporterSpecs()[0] == Catch::ReporterSpec{ expectedReporter, {}, {}, {}, {} } for: {?} == {?} # Process can be configured on command line ok {test-number} - cfg.getProcessedReporterSpecs().size() == 1 for: 1 == 1 # Process can be configured on command line -ok {test-number} - cfg.getProcessedReporterSpecs()[0] == Catch::ProcessedReporterSpec{ expectedReporter, std::string{}, Catch::ColourMode::PlatformDefault, {} } for: {?} == {?} +ok {test-number} - cfg.getProcessedReporterSpecs()[0] == Catch::ProcessedReporterSpec{ expectedReporter, std::string{}, Catch::ColourMode::PlatformDefault, Catch::Verbosity::Normal, {} } for: {?} == {?} # Process can be configured on command line ok {test-number} - result for: {?} # Process can be configured on command line @@ -2545,15 +2545,15 @@ ok {test-number} - cfg.testSpec().matches(*fakeTestCase("alwaysIncluded")) for: # Process can be configured on command line ok {test-number} - result for: {?} with 1 message: 'result.errorMessage() := ""' # Process can be configured on command line -ok {test-number} - config.reporterSpecifications == vec_Specs{ { "console", {}, {}, {} } } for: { {?} } == { {?} } with 1 message: 'result.errorMessage() := ""' +ok {test-number} - config.reporterSpecifications == vec_Specs{ { "console", {}, {}, {}, {} } } for: { {?} } == { {?} } with 1 message: 'result.errorMessage() := ""' # Process can be configured on command line ok {test-number} - result for: {?} with 1 message: 'result.errorMessage() := ""' # Process can be configured on command line -ok {test-number} - config.reporterSpecifications == vec_Specs{ { "xml", {}, {}, {} } } for: { {?} } == { {?} } with 1 message: 'result.errorMessage() := ""' +ok {test-number} - config.reporterSpecifications == vec_Specs{ { "xml", {}, {}, {}, {} } } for: { {?} } == { {?} } with 1 message: 'result.errorMessage() := ""' # Process can be configured on command line ok {test-number} - result for: {?} with 1 message: 'result.errorMessage() := ""' # Process can be configured on command line -ok {test-number} - config.reporterSpecifications == vec_Specs{ { "junit", {}, {}, {} } } for: { {?} } == { {?} } with 1 message: 'result.errorMessage() := ""' +ok {test-number} - config.reporterSpecifications == vec_Specs{ { "junit", {}, {}, {}, {} } } for: { {?} } == { {?} } with 1 message: 'result.errorMessage() := ""' # Process can be configured on command line ok {test-number} - !result for: true # Process can be configured on command line @@ -2561,19 +2561,19 @@ ok {test-number} - result.errorMessage(), ContainsSubstring("Unrecognized report # Process can be configured on command line ok {test-number} - result for: {?} with 1 message: 'result.errorMessage() := ""' # Process can be configured on command line -ok {test-number} - config.reporterSpecifications == vec_Specs{ { "console", "out.txt"s, {}, {} } } for: { {?} } == { {?} } with 1 message: 'result.errorMessage() := ""' +ok {test-number} - config.reporterSpecifications == vec_Specs{ { "console", "out.txt"s, {}, {}, {} } } for: { {?} } == { {?} } with 1 message: 'result.errorMessage() := ""' # Process can be configured on command line ok {test-number} - result for: {?} with 1 message: 'result.errorMessage() := ""' # Process can be configured on command line -ok {test-number} - config.reporterSpecifications == vec_Specs{ { "console", "C:\\Temp\\out.txt"s, {}, {} } } for: { {?} } == { {?} } with 1 message: 'result.errorMessage() := ""' +ok {test-number} - config.reporterSpecifications == vec_Specs{ { "console", "C:\\Temp\\out.txt"s, {}, {}, {} } } for: { {?} } == { {?} } with 1 message: 'result.errorMessage() := ""' # Process can be configured on command line ok {test-number} - cli.parse({ "test", "-r", "xml::out=output.xml", "-r", "junit::out=output-junit.xml" }) for: {?} # Process can be configured on command line -ok {test-number} - config.reporterSpecifications == vec_Specs{ { "xml", "output.xml"s, {}, {} }, { "junit", "output-junit.xml"s, {}, {} } } for: { {?}, {?} } == { {?}, {?} } +ok {test-number} - config.reporterSpecifications == vec_Specs{ { "xml", "output.xml"s, {}, {}, {} }, { "junit", "output-junit.xml"s, {}, {}, {} } } for: { {?}, {?} } == { {?}, {?} } # Process can be configured on command line ok {test-number} - cli.parse({ "test", "-r", "xml::out=output.xml", "-r", "console" }) for: {?} # Process can be configured on command line -ok {test-number} - config.reporterSpecifications == vec_Specs{ { "xml", "output.xml"s, {}, {} }, { "console", {}, {}, {} } } for: { {?}, {?} } == { {?}, {?} } +ok {test-number} - config.reporterSpecifications == vec_Specs{ { "xml", "output.xml"s, {}, {}, {} }, { "console", {}, {}, {}, {} } } for: { {?}, {?} } == { {?}, {?} } # Process can be configured on command line ok {test-number} - !result for: true # Process can be configured on command line diff --git a/tests/SelfTest/Baselines/xml.sw.approved.txt b/tests/SelfTest/Baselines/xml.sw.approved.txt index a7f8d3ae..cf9dfdb2 100644 --- a/tests/SelfTest/Baselines/xml.sw.approved.txt +++ b/tests/SelfTest/Baselines/xml.sw.approved.txt @@ -11976,7 +11976,7 @@ Approx( 1.21999999999999997 ) - cfg.getReporterSpecs()[0] == Catch::ReporterSpec{ expectedReporter, {}, {}, {} } + cfg.getReporterSpecs()[0] == Catch::ReporterSpec{ expectedReporter, {}, {}, {}, {} } {?} == {?} @@ -11992,7 +11992,7 @@ Approx( 1.21999999999999997 ) - cfg.getProcessedReporterSpecs()[0] == Catch::ProcessedReporterSpec{ expectedReporter, std::string{}, Catch::ColourMode::PlatformDefault, {} } + cfg.getProcessedReporterSpecs()[0] == Catch::ProcessedReporterSpec{ expectedReporter, std::string{}, Catch::ColourMode::PlatformDefault, Catch::Verbosity::Normal, {} } {?} == {?} @@ -12132,7 +12132,7 @@ Approx( 1.21999999999999997 ) - config.reporterSpecifications == vec_Specs{ { "console", {}, {}, {} } } + config.reporterSpecifications == vec_Specs{ { "console", {}, {}, {}, {} } } { {?} } == { {?} } @@ -12160,7 +12160,7 @@ Approx( 1.21999999999999997 ) - config.reporterSpecifications == vec_Specs{ { "xml", {}, {}, {} } } + config.reporterSpecifications == vec_Specs{ { "xml", {}, {}, {}, {} } } { {?} } == { {?} } @@ -12188,7 +12188,7 @@ Approx( 1.21999999999999997 ) - config.reporterSpecifications == vec_Specs{ { "junit", {}, {}, {} } } + config.reporterSpecifications == vec_Specs{ { "junit", {}, {}, {}, {} } } { {?} } == { {?} } @@ -12238,7 +12238,7 @@ Approx( 1.21999999999999997 ) - config.reporterSpecifications == vec_Specs{ { "console", "out.txt"s, {}, {} } } + config.reporterSpecifications == vec_Specs{ { "console", "out.txt"s, {}, {}, {} } } { {?} } == { {?} } @@ -12266,7 +12266,7 @@ Approx( 1.21999999999999997 ) - config.reporterSpecifications == vec_Specs{ { "console", "C:\\Temp\\out.txt"s, {}, {} } } + config.reporterSpecifications == vec_Specs{ { "console", "C:\\Temp\\out.txt"s, {}, {}, {} } } { {?} } == { {?} } @@ -12289,7 +12289,7 @@ Approx( 1.21999999999999997 ) - config.reporterSpecifications == vec_Specs{ { "xml", "output.xml"s, {}, {} }, { "junit", "output-junit.xml"s, {}, {} } } + config.reporterSpecifications == vec_Specs{ { "xml", "output.xml"s, {}, {}, {} }, { "junit", "output-junit.xml"s, {}, {}, {} } } { {?}, {?} } == { {?}, {?} } @@ -12314,7 +12314,7 @@ Approx( 1.21999999999999997 ) - config.reporterSpecifications == vec_Specs{ { "xml", "output.xml"s, {}, {} }, { "console", {}, {}, {} } } + config.reporterSpecifications == vec_Specs{ { "xml", "output.xml"s, {}, {}, {} }, { "console", {}, {}, {}, {} } } { {?}, {?} } == { {?}, {?} } diff --git a/tests/SelfTest/Baselines/xml.sw.multi.approved.txt b/tests/SelfTest/Baselines/xml.sw.multi.approved.txt index cbfe936b..0ff255a2 100644 --- a/tests/SelfTest/Baselines/xml.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/xml.sw.multi.approved.txt @@ -11976,7 +11976,7 @@ Approx( 1.21999999999999997 ) - cfg.getReporterSpecs()[0] == Catch::ReporterSpec{ expectedReporter, {}, {}, {} } + cfg.getReporterSpecs()[0] == Catch::ReporterSpec{ expectedReporter, {}, {}, {}, {} } {?} == {?} @@ -11992,7 +11992,7 @@ Approx( 1.21999999999999997 ) - cfg.getProcessedReporterSpecs()[0] == Catch::ProcessedReporterSpec{ expectedReporter, std::string{}, Catch::ColourMode::PlatformDefault, {} } + cfg.getProcessedReporterSpecs()[0] == Catch::ProcessedReporterSpec{ expectedReporter, std::string{}, Catch::ColourMode::PlatformDefault, Catch::Verbosity::Normal, {} } {?} == {?} @@ -12132,7 +12132,7 @@ Approx( 1.21999999999999997 ) - config.reporterSpecifications == vec_Specs{ { "console", {}, {}, {} } } + config.reporterSpecifications == vec_Specs{ { "console", {}, {}, {}, {} } } { {?} } == { {?} } @@ -12160,7 +12160,7 @@ Approx( 1.21999999999999997 ) - config.reporterSpecifications == vec_Specs{ { "xml", {}, {}, {} } } + config.reporterSpecifications == vec_Specs{ { "xml", {}, {}, {}, {} } } { {?} } == { {?} } @@ -12188,7 +12188,7 @@ Approx( 1.21999999999999997 ) - config.reporterSpecifications == vec_Specs{ { "junit", {}, {}, {} } } + config.reporterSpecifications == vec_Specs{ { "junit", {}, {}, {}, {} } } { {?} } == { {?} } @@ -12238,7 +12238,7 @@ Approx( 1.21999999999999997 ) - config.reporterSpecifications == vec_Specs{ { "console", "out.txt"s, {}, {} } } + config.reporterSpecifications == vec_Specs{ { "console", "out.txt"s, {}, {}, {} } } { {?} } == { {?} } @@ -12266,7 +12266,7 @@ Approx( 1.21999999999999997 ) - config.reporterSpecifications == vec_Specs{ { "console", "C:\\Temp\\out.txt"s, {}, {} } } + config.reporterSpecifications == vec_Specs{ { "console", "C:\\Temp\\out.txt"s, {}, {}, {} } } { {?} } == { {?} } @@ -12289,7 +12289,7 @@ Approx( 1.21999999999999997 ) - config.reporterSpecifications == vec_Specs{ { "xml", "output.xml"s, {}, {} }, { "junit", "output-junit.xml"s, {}, {} } } + config.reporterSpecifications == vec_Specs{ { "xml", "output.xml"s, {}, {}, {} }, { "junit", "output-junit.xml"s, {}, {}, {} } } { {?}, {?} } == { {?}, {?} } @@ -12314,7 +12314,7 @@ Approx( 1.21999999999999997 ) - config.reporterSpecifications == vec_Specs{ { "xml", "output.xml"s, {}, {} }, { "console", {}, {}, {} } } + config.reporterSpecifications == vec_Specs{ { "xml", "output.xml"s, {}, {}, {} }, { "console", {}, {}, {}, {} } } { {?}, {?} } == { {?}, {?} } diff --git a/tests/SelfTest/IntrospectiveTests/CmdLine.tests.cpp b/tests/SelfTest/IntrospectiveTests/CmdLine.tests.cpp index 154d8ed9..0717ae2e 100644 --- a/tests/SelfTest/IntrospectiveTests/CmdLine.tests.cpp +++ b/tests/SelfTest/IntrospectiveTests/CmdLine.tests.cpp @@ -58,12 +58,13 @@ TEST_CASE( "Process can be configured on command line", "[config][command-line]" CHECK( cfg.getReporterSpecs().size() == 1 ); CHECK( cfg.getReporterSpecs()[0] == - Catch::ReporterSpec{ expectedReporter, {}, {}, {} } ); + Catch::ReporterSpec{ expectedReporter, {}, {}, {}, {} } ); CHECK( cfg.getProcessedReporterSpecs().size() == 1 ); CHECK( cfg.getProcessedReporterSpecs()[0] == Catch::ProcessedReporterSpec{ expectedReporter, std::string{}, Catch::ColourMode::PlatformDefault, + Catch::Verbosity::Normal, {} } ); } @@ -108,7 +109,7 @@ TEST_CASE( "Process can be configured on command line", "[config][command-line]" CHECK(result); REQUIRE( config.reporterSpecifications == - vec_Specs{ { "console", {}, {}, {} } } ); + vec_Specs{ { "console", {}, {}, {}, {} } } ); } SECTION("-r/xml") { auto result = cli.parse({"test", "-r", "xml"}); @@ -116,7 +117,7 @@ TEST_CASE( "Process can be configured on command line", "[config][command-line]" CHECK(result); REQUIRE( config.reporterSpecifications == - vec_Specs{ { "xml", {}, {}, {} } } ); + vec_Specs{ { "xml", {}, {}, {}, {} } } ); } SECTION("--reporter/junit") { auto result = cli.parse({"test", "--reporter", "junit"}); @@ -124,7 +125,7 @@ TEST_CASE( "Process can be configured on command line", "[config][command-line]" CHECK(result); REQUIRE( config.reporterSpecifications == - vec_Specs{ { "junit", {}, {}, {} } } ); + vec_Specs{ { "junit", {}, {}, {}, {} } } ); } SECTION("must match one of the available ones") { auto result = cli.parse({"test", "--reporter", "unsupported"}); @@ -137,27 +138,27 @@ TEST_CASE( "Process can be configured on command line", "[config][command-line]" CAPTURE(result.errorMessage()); CHECK(result); REQUIRE( config.reporterSpecifications == - vec_Specs{ { "console", "out.txt"s, {}, {} } } ); + vec_Specs{ { "console", "out.txt"s, {}, {}, {} } } ); } SECTION("With Windows-like absolute path as output file") { auto result = cli.parse({ "test", "-r", "console::out=C:\\Temp\\out.txt" }); CAPTURE(result.errorMessage()); CHECK(result); REQUIRE( config.reporterSpecifications == - vec_Specs{ { "console", "C:\\Temp\\out.txt"s, {}, {} } } ); + vec_Specs{ { "console", "C:\\Temp\\out.txt"s, {}, {}, {} } } ); } SECTION("Multiple reporters") { SECTION("All with output files") { CHECK(cli.parse({ "test", "-r", "xml::out=output.xml", "-r", "junit::out=output-junit.xml" })); REQUIRE( config.reporterSpecifications == - vec_Specs{ { "xml", "output.xml"s, {}, {} }, - { "junit", "output-junit.xml"s, {}, {} } } ); + vec_Specs{ { "xml", "output.xml"s, {}, {}, {} }, + { "junit", "output-junit.xml"s, {}, {}, {} } } ); } SECTION("Mixed output files and default output") { CHECK(cli.parse({ "test", "-r", "xml::out=output.xml", "-r", "console" })); REQUIRE( config.reporterSpecifications == - vec_Specs{ { "xml", "output.xml"s, {}, {} }, - { "console", {}, {}, {} } } ); + vec_Specs{ { "xml", "output.xml"s, {}, {}, {} }, + { "console", {}, {}, {}, {} } } ); } SECTION("cannot have multiple reporters with default output") { auto result = cli.parse({ "test", "-r", "console", "-r", "xml::out=output.xml", "-r", "junit" }); diff --git a/tests/SelfTest/IntrospectiveTests/CmdLineHelpers.tests.cpp b/tests/SelfTest/IntrospectiveTests/CmdLineHelpers.tests.cpp index 4df8ecf9..d38679d1 100644 --- a/tests/SelfTest/IntrospectiveTests/CmdLineHelpers.tests.cpp +++ b/tests/SelfTest/IntrospectiveTests/CmdLineHelpers.tests.cpp @@ -63,6 +63,20 @@ TEST_CASE( "Parsing colour mode", "[cli][colour][approvals]" ) { } } +TEST_CASE( "Parsing verbosity", "[cli][colour][approvals]" ) { + using Catch::Detail::stringToVerbosity; + using Catch::Verbosity; + SECTION( "Valid strings" ) { + REQUIRE( stringToVerbosity( "quiet" ) == Verbosity::Quiet ); + REQUIRE( stringToVerbosity( "normal" ) == Verbosity::Normal ); + REQUIRE( stringToVerbosity( "high" ) == Verbosity::High ); + } + SECTION( "Wrong strings" ) { + REQUIRE_FALSE( stringToVerbosity( "QUIET" ) ); + REQUIRE_FALSE( stringToVerbosity( "medium" ) ); + REQUIRE_FALSE( stringToVerbosity( "vvv" ) ); + } +} TEST_CASE("Parsing reporter specs", "[cli][reporter-spec][approvals]") { using Catch::parseReporterSpec; @@ -71,12 +85,13 @@ TEST_CASE("Parsing reporter specs", "[cli][reporter-spec][approvals]") { SECTION( "Correct specs" ) { REQUIRE( parseReporterSpec( "someReporter" ) == - ReporterSpec( "someReporter"s, {}, {}, {} ) ); + ReporterSpec( "someReporter"s, {}, {}, {}, {} ) ); REQUIRE( parseReporterSpec( "otherReporter::Xk=v::out=c:\\blah" ) == ReporterSpec( - "otherReporter"s, "c:\\blah"s, {}, { { "Xk"s, "v"s } } ) ); + "otherReporter"s, "c:\\blah"s, {}, {}, { { "Xk"s, "v"s } } ) ); REQUIRE( parseReporterSpec( "diffReporter::Xk1=v1::Xk2==v2" ) == ReporterSpec( "diffReporter", + {}, {}, {}, { { "Xk1"s, "v1"s }, { "Xk2"s, "=v2"s } } ) ); @@ -85,7 +100,15 @@ TEST_CASE("Parsing reporter specs", "[cli][reporter-spec][approvals]") { ReporterSpec( "Foo:bar:reporter", {}, Catch::ColourMode::ANSI, + {}, { { "Xk 1"s, "v 1"s }, { "Xk2"s, "v:3"s } } ) ); + REQUIRE( + parseReporterSpec( "my:reporter::X1=V1::verbosity=high::X2=V2" ) == + ReporterSpec( "my:reporter", + {}, + {}, + Catch::Verbosity::High, + { { "X1"s, "V1"s }, { "X2"s, "V2"s } } ) ); } SECTION( "Bad specs" ) { @@ -107,5 +130,9 @@ TEST_CASE("Parsing reporter specs", "[cli][reporter-spec][approvals]") { REQUIRE_FALSE( parseReporterSpec( "reporter::Xa=" ) ); // non-key value later field REQUIRE_FALSE( parseReporterSpec( "reporter::Xab" ) ); + // Invalid verbosity value + REQUIRE_FALSE( parseReporterSpec( "reporter::verbosity=medium" ) ); + // Duplicated verbosity + REQUIRE_FALSE( parseReporterSpec( "reporter::verbosity=high::X1=X2::verbosity=high" ) ); } } diff --git a/tests/SelfTest/IntrospectiveTests/Reporters.tests.cpp b/tests/SelfTest/IntrospectiveTests/Reporters.tests.cpp index afb5143a..4ff793f0 100644 --- a/tests/SelfTest/IntrospectiveTests/Reporters.tests.cpp +++ b/tests/SelfTest/IntrospectiveTests/Reporters.tests.cpp @@ -42,6 +42,7 @@ namespace { &config, Catch::Detail::make_unique(), Catch::ColourMode::None, + Catch::Verbosity::Normal, {} }; } } @@ -114,7 +115,7 @@ TEST_CASE( "Reporter's write listings to provided stream", "[reporters]" ) { cfg_data.rngSeed = 1234; Catch::Config config( cfg_data ); auto reporter = factory.second->create( Catch::ReporterConfig{ - &config, CATCH_MOVE( sstream ), Catch::ColourMode::None, {} } ); + &config, CATCH_MOVE( sstream ), Catch::ColourMode::None, Catch::Verbosity::Normal, {} } ); DYNAMIC_SECTION( factory.first << " reporter lists tags" ) { std::vector tags(1);