diff --git a/benchmarks/discover_tests/benchmark_discovery.py b/benchmarks/discover_tests/benchmark_discovery.py index b3358607..55e6d494 100644 --- a/benchmarks/discover_tests/benchmark_discovery.py +++ b/benchmarks/discover_tests/benchmark_discovery.py @@ -8,6 +8,7 @@ # SPDX-License-Identifier: BSL-1.0 import argparse +import copy import json import os import re @@ -33,14 +34,19 @@ def load_template(path: str) -> dict: return catch_out -def synthesize_listing(template_doc: dict, count: int, file) -> str: +def synthesize_listing(template_doc: dict, count: int, file, with_tags: bool) -> str: """Writes JSON listing with exactly `count` test cases into `file`. The template entries are cycled through, and each name is made unique by appending an index, so that every registered CTest test has a distinct name. """ - doc_copy = template_doc.copy() - template_tests = template_doc['listings']['tests'] + # To avoid messing up the template tests by deletion, we have + # to take a deepcopy before the changes. + doc_copy = copy.deepcopy(template_doc) + template_tests = doc_copy['listings']['tests'] + if not with_tags: + for test in template_tests: + del test['tags'] num_original = len(template_tests) out_tests = [] @@ -176,10 +182,10 @@ def main(): os.makedirs(workdir, exist_ok=True) for count in COUNTS: listing_path = os.path.join(workdir, f"listing-{count}.json") - with open(listing_path, "w", encoding="utf-8") as f: - synthesize_listing(template_doc, count, f) for add_tags in tag_modes: + with open(listing_path, "w", encoding="utf-8") as f: + synthesize_listing(template_doc, count, f, add_tags) mode = "on" if add_tags else "off" ctest_file = os.path.join(workdir, f"ctest-{count}-{mode}.cmake") cmd = build_command(args, listing_path, ctest_file, add_tags) diff --git a/benchmarks/discover_tests/listing_template.json b/benchmarks/discover_tests/listing_template.json index 7e976162..d434dd5c 100644 --- a/benchmarks/discover_tests/listing_template.json +++ b/benchmarks/discover_tests/listing_template.json @@ -11,22 +11,14 @@ "tags": [ "function pointer", "Tricky" - ], - "source-location": { - "filename": "/mnt/c/ubuntu/Catch2/tests/SelfTest/UsageTests/Tricky.tests.cpp", - "line": 266 - } + ] }, { "name": "Testing checked-if 4", "tags": [ "!shouldfail", "checked-if" - ], - "source-location": { - "filename": "/mnt/c/ubuntu/Catch2/tests/SelfTest/UsageTests/Misc.tests.cpp", - "line": 216 - } + ] }, { "name": "count_equidistant_floats - double", @@ -34,11 +26,7 @@ "approvals", "distance", "floating-point" - ], - "source-location": { - "filename": "/mnt/c/ubuntu/Catch2/tests/SelfTest/IntrospectiveTests/FloatingPoint.tests.cpp", - "line": 103 - } + ] }, { "name": "Usage of AllTrue range matcher", @@ -46,11 +34,7 @@ "matchers", "quantifiers", "templated" - ], - "source-location": { - "filename": "/mnt/c/ubuntu/Catch2/tests/SelfTest/UsageTests/MatchersRanges.tests.cpp", - "line": 372 - } + ] }, { "name": "Exception matchers that succeed", @@ -58,11 +42,7 @@ "!throws", "exceptions", "matchers" - ], - "source-location": { - "filename": "/mnt/c/ubuntu/Catch2/tests/SelfTest/UsageTests/Matchers.tests.cpp", - "line": 428 - } + ] }, { "name": "A TEMPLATE_TEST_CASE_METHOD based test run that succeeds - float", @@ -70,22 +50,14 @@ "tags": [ "class", "template" - ], - "source-location": { - "filename": "/mnt/c/ubuntu/Catch2/tests/SelfTest/UsageTests/Class.tests.cpp", - "line": 82 - } + ] }, { "name": "Approximate PI", "tags": [ "Approx", "PI" - ], - "source-location": { - "filename": "/mnt/c/ubuntu/Catch2/tests/SelfTest/UsageTests/Approx.tests.cpp", - "line": 134 - } + ] }, { "name": "TextFlow::Column respects width setting", @@ -93,32 +65,20 @@ "approvals", "column", "TextFlow" - ], - "source-location": { - "filename": "/mnt/c/ubuntu/Catch2/tests/SelfTest/IntrospectiveTests/TextFlow.tests.cpp", - "line": 38 - } + ] }, { "name": "Generators internals", "tags": [ "generators", "internals" - ], - "source-location": { - "filename": "/mnt/c/ubuntu/Catch2/tests/SelfTest/IntrospectiveTests/GeneratorsImpl.tests.cpp", - "line": 24 - } + ] }, { "name": "Mayfail test case with nested sections", "tags": [ "!mayfail" - ], - "source-location": { - "filename": "/mnt/c/ubuntu/Catch2/tests/SelfTest/UsageTests/Condition.tests.cpp", - "line": 83 - } + ] } ] } diff --git a/extras/CatchAddTests.cmake b/extras/CatchAddTests.cmake index b0d8c02b..6116e4b8 100644 --- a/extras/CatchAddTests.cmake +++ b/extras/CatchAddTests.cmake @@ -253,10 +253,15 @@ function(catch_discover_tests_impl) make_temp_file_path(listing_output_path "${_TEST_WORKING_DIR}") + set(_Verbosity "quiet") + if (add_tags) + set(_Verbosity "normal") + endif() + execute_process( COMMAND ${_TEST_EXECUTOR} "${_TEST_EXECUTABLE}" ${spec} --list-tests - --reporter json + --reporter "json::verbosity=${_Verbosity}" --out "${listing_output_path}" --order lex # Make sure the output order, and thus test registration order, is consistent across runs. OUTPUT_VARIABLE listing_output diff --git a/src/catch2/reporters/catch_reporter_json.cpp b/src/catch2/reporters/catch_reporter_json.cpp index 6c093b40..39a2105d 100644 --- a/src/catch2/reporters/catch_reporter_json.cpp +++ b/src/catch2/reporters/catch_reporter_json.cpp @@ -349,13 +349,15 @@ namespace Catch { if (!info.className.empty()) { desc_writer.write( "class-name"_sr ).write( info.className ); } - { + if ( m_verbosity >= Verbosity::Normal ) { auto tag_writer = desc_writer.write( "tags"_sr ).writeArray(); for ( auto const& tag : info.tags ) { tag_writer.write( tag.original ); } } - writeSourceInfo( desc_writer, info.lineInfo ); + if ( m_verbosity >= Verbosity::High) { + writeSourceInfo( desc_writer, info.lineInfo ); + } } } void JsonReporter::listTags( std::vector const& tags ) { diff --git a/src/catch2/reporters/catch_reporter_json.hpp b/src/catch2/reporters/catch_reporter_json.hpp index b28aaf8d..6de1bd77 100644 --- a/src/catch2/reporters/catch_reporter_json.hpp +++ b/src/catch2/reporters/catch_reporter_json.hpp @@ -85,9 +85,6 @@ namespace Catch { std::stack m_writers{}; bool m_startedListing = false; - - // std::size_t m_sectionDepth = 0; - // std::size_t m_sectionStarted = 0; }; } // namespace Catch diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index eba8c22b..573c38e7 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -208,6 +208,21 @@ set_tests_properties(List::Tests::XmlOutput PROPERTIES FAIL_REGULAR_EXPRESSION "[0-9]+ test cases" ) +add_test(NAME List::Tests::Json::Normal COMMAND $ --list-tests -r json) +set_tests_properties(List::Tests::Json::Normal PROPERTIES + PASS_REGULAR_EXPRESSION "\"tags\": \\[" + FAIL_REGULAR_EXPRESSION "\"source-location\": {" +) +add_test(NAME List::Tests::Json::Quiet COMMAND $ --list-tests -r json::verbosity=quiet) +set_tests_properties(List::Tests::Json::Quiet PROPERTIES + PASS_REGULAR_EXPRESSION "\"name\": \"" + FAIL_REGULAR_EXPRESSION "\"tags\": \\[" +) +add_test(NAME List::Tests::Json::High COMMAND $ --list-tests -r json::verbosity=high) +set_tests_properties(List::Tests::Json::High PROPERTIES + PASS_REGULAR_EXPRESSION "\"source-location\": {" +) + add_test(NAME List::Tags::Output COMMAND $ --list-tags) set_tests_properties(List::Tags::Output PROPERTIES PASS_REGULAR_EXPRESSION "[0-9]+ tags" @@ -218,6 +233,11 @@ set_tests_properties(List::Tags::XmlOutput PROPERTIES PASS_REGULAR_EXPRESSION "18" FAIL_REGULAR_EXPRESSION "[0-9]+ tags" ) +add_test(NAME List::Tags::JsonOutput COMMAND $ --list-tags -r json) +set_tests_properties(List::Tags::JsonOutput PROPERTIES + PASS_REGULAR_EXPRESSION "\"count\": 18" + FAIL_REGULAR_EXPRESSION "[0-9]+ tags" +) add_test(NAME List::Reporters::Output COMMAND $ --list-reporters) set_tests_properties(List::Reporters::Output PROPERTIES PASS_REGULAR_EXPRESSION "Available reporters:") @@ -227,6 +247,12 @@ set_tests_properties(List::Reporters::XmlOutput PROPERTIES PASS_REGULAR_EXPRESSION "compact" FAIL_REGULAR_EXPRESSION "Available reporters:" ) +add_test(NAME List::Reporters::JsonOutput COMMAND $ --list-reporters -r json) +set_tests_properties(List::Reporters::JsonOutput PROPERTIES + PASS_REGULAR_EXPRESSION "\"name\": \"compact\"" + FAIL_REGULAR_EXPRESSION "Available reporters:" +) + add_test(NAME List::Listeners::Output COMMAND @@ -251,6 +277,18 @@ set_tests_properties(List::Listeners::XmlOutput PASS_REGULAR_EXPRESSION "" FAIL_REGULAR_EXPRESSION "Registered listeners:" ) +add_test(NAME List::Listeners::JsonOutput + COMMAND + $ + --list-listeners + --reporter json +) +set_tests_properties(List::Listeners::JsonOutput + PROPERTIES + PASS_REGULAR_EXPRESSION "\"listeners\": \\[" + FAIL_REGULAR_EXPRESSION "Registered listeners:" +) + add_test(NAME NoAssertions COMMAND $ -w NoAssertions "An empty test with no assertions") set_tests_properties(NoAssertions PROPERTIES PASS_REGULAR_EXPRESSION "No assertions in test case") diff --git a/tests/SelfTest/Baselines/compact.sw.approved.txt b/tests/SelfTest/Baselines/compact.sw.approved.txt index 94898c37..673c9dd7 100644 --- a/tests/SelfTest/Baselines/compact.sw.approved.txt +++ b/tests/SelfTest/Baselines/compact.sw.approved.txt @@ -1740,11 +1740,7 @@ Reporters.tests.cpp:: passed: listingString, ContainsSubstring( "fa "name": "fake test name", "tags": [ "fakeTestTag" - ], - "source-location": { - "filename": "fake-file.cpp", - "line": 123456789 - } + ] } ]" ( contains: "fake test name" and contains: "fakeTestTag" ) with 1 message: 'Tested reporter: JSON' Reporters.tests.cpp:: passed: !(factories.empty()) for: !false diff --git a/tests/SelfTest/Baselines/compact.sw.multi.approved.txt b/tests/SelfTest/Baselines/compact.sw.multi.approved.txt index 2d268b2f..73195967 100644 --- a/tests/SelfTest/Baselines/compact.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/compact.sw.multi.approved.txt @@ -1738,11 +1738,7 @@ Reporters.tests.cpp:: passed: listingString, ContainsSubstring( "fa "name": "fake test name", "tags": [ "fakeTestTag" - ], - "source-location": { - "filename": "fake-file.cpp", - "line": 123456789 - } + ] } ]" ( contains: "fake test name" and contains: "fakeTestTag" ) with 1 message: 'Tested reporter: JSON' Reporters.tests.cpp:: passed: !(factories.empty()) for: !false diff --git a/tests/SelfTest/Baselines/console.sw.approved.txt b/tests/SelfTest/Baselines/console.sw.approved.txt index 9d8fa88f..6ec67e3f 100644 --- a/tests/SelfTest/Baselines/console.sw.approved.txt +++ b/tests/SelfTest/Baselines/console.sw.approved.txt @@ -11323,11 +11323,7 @@ with expansion: "name": "fake test name", "tags": [ "fakeTestTag" - ], - "source-location": { - "filename": "fake-file.cpp", - "line": 123456789 - } + ] } ]" ( contains: "fake test name" and contains: "fakeTestTag" ) with message: diff --git a/tests/SelfTest/Baselines/console.sw.multi.approved.txt b/tests/SelfTest/Baselines/console.sw.multi.approved.txt index fcdf5f9f..9af23aa5 100644 --- a/tests/SelfTest/Baselines/console.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/console.sw.multi.approved.txt @@ -11321,11 +11321,7 @@ with expansion: "name": "fake test name", "tags": [ "fakeTestTag" - ], - "source-location": { - "filename": "fake-file.cpp", - "line": 123456789 - } + ] } ]" ( contains: "fake test name" and contains: "fakeTestTag" ) with message: diff --git a/tests/SelfTest/Baselines/tap.sw.approved.txt b/tests/SelfTest/Baselines/tap.sw.approved.txt index 657b86ff..69476eef 100644 --- a/tests/SelfTest/Baselines/tap.sw.approved.txt +++ b/tests/SelfTest/Baselines/tap.sw.approved.txt @@ -2775,7 +2775,7 @@ ok {test-number} - listingString, ContainsSubstring("fake reporter"s) for: "{ # Reporter's write listings to provided stream ok {test-number} - !(factories.empty()) for: !false # Reporter's write listings to provided stream -ok {test-number} - listingString, ContainsSubstring( "fake test name"s ) && ContainsSubstring( "fakeTestTag"s ) for: "{ "version": 2, "metadata": { "name": "", "rng-seed": 1234, "catch2-version": "" }, "listings": { "tests": [ { "name": "fake test name", "tags": [ "fakeTestTag" ], "source-location": { "filename": "fake-file.cpp", "line": 123456789 } } ]" ( contains: "fake test name" and contains: "fakeTestTag" ) with 1 message: 'Tested reporter: JSON' +ok {test-number} - listingString, ContainsSubstring( "fake test name"s ) && ContainsSubstring( "fakeTestTag"s ) for: "{ "version": 2, "metadata": { "name": "", "rng-seed": 1234, "catch2-version": "" }, "listings": { "tests": [ { "name": "fake test name", "tags": [ "fakeTestTag" ] } ]" ( contains: "fake test name" and contains: "fakeTestTag" ) with 1 message: 'Tested reporter: JSON' # Reporter's write listings to provided stream ok {test-number} - !(factories.empty()) for: !false # Reporter's write listings to provided stream diff --git a/tests/SelfTest/Baselines/tap.sw.multi.approved.txt b/tests/SelfTest/Baselines/tap.sw.multi.approved.txt index 5626093a..01b8a422 100644 --- a/tests/SelfTest/Baselines/tap.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/tap.sw.multi.approved.txt @@ -2773,7 +2773,7 @@ ok {test-number} - listingString, ContainsSubstring("fake reporter"s) for: "{ # Reporter's write listings to provided stream ok {test-number} - !(factories.empty()) for: !false # Reporter's write listings to provided stream -ok {test-number} - listingString, ContainsSubstring( "fake test name"s ) && ContainsSubstring( "fakeTestTag"s ) for: "{ "version": 2, "metadata": { "name": "", "rng-seed": 1234, "catch2-version": "" }, "listings": { "tests": [ { "name": "fake test name", "tags": [ "fakeTestTag" ], "source-location": { "filename": "fake-file.cpp", "line": 123456789 } } ]" ( contains: "fake test name" and contains: "fakeTestTag" ) with 1 message: 'Tested reporter: JSON' +ok {test-number} - listingString, ContainsSubstring( "fake test name"s ) && ContainsSubstring( "fakeTestTag"s ) for: "{ "version": 2, "metadata": { "name": "", "rng-seed": 1234, "catch2-version": "" }, "listings": { "tests": [ { "name": "fake test name", "tags": [ "fakeTestTag" ] } ]" ( contains: "fake test name" and contains: "fakeTestTag" ) with 1 message: 'Tested reporter: JSON' # Reporter's write listings to provided stream ok {test-number} - !(factories.empty()) for: !false # Reporter's write listings to provided stream diff --git a/tests/SelfTest/Baselines/xml.sw.approved.txt b/tests/SelfTest/Baselines/xml.sw.approved.txt index 42b41eca..d48496b9 100644 --- a/tests/SelfTest/Baselines/xml.sw.approved.txt +++ b/tests/SelfTest/Baselines/xml.sw.approved.txt @@ -13503,11 +13503,7 @@ Approx( 0.98999999999999999 ) "name": "fake test name", "tags": [ "fakeTestTag" - ], - "source-location": { - "filename": "fake-file.cpp", - "line": 123456789 - } + ] } ]" ( contains: "fake test name" and contains: "fakeTestTag" ) diff --git a/tests/SelfTest/Baselines/xml.sw.multi.approved.txt b/tests/SelfTest/Baselines/xml.sw.multi.approved.txt index 3d265d40..db82d34d 100644 --- a/tests/SelfTest/Baselines/xml.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/xml.sw.multi.approved.txt @@ -13503,11 +13503,7 @@ Approx( 0.98999999999999999 ) "name": "fake test name", "tags": [ "fakeTestTag" - ], - "source-location": { - "filename": "fake-file.cpp", - "line": 123456789 - } + ] } ]" ( contains: "fake test name" and contains: "fakeTestTag" )