JSONReporter considers verbosity when listing tests

* Quiet verbosity provides just the test names * Normal verbosity adds tags * High verbosity add the source location of the tests Listing tests with the quiet verbosity results in about 1/4 of the previous output, which leads to measurably faster execution of `catch_discover_tests` when it does not need tags for labels. (If it does need tags, the output is about 1/2 of previous).
2026-08-08 00:05:18 +02:00
parent 0b4d7a5a16
commit 4cde128517
14 changed files with 77 additions and 93 deletions
@@ -8,6 +8,7 @@
 # SPDX-License-Identifier: BSL-1.0

 import argparse
+import copy
 import json
 import os
 import re
@@ -33,14 +34,19 @@ def load_template(path: str) -> dict:
    return catch_out


-def synthesize_listing(template_doc: dict, count: int, file) -> str:
+def synthesize_listing(template_doc: dict, count: int, file, with_tags: bool) -> str:
    """Writes JSON listing with exactly `count` test cases into `file`.

    The template entries are cycled through, and each name is made unique by
    appending an index, so that every registered CTest test has a distinct name.
    """
-    doc_copy = template_doc.copy()
-    template_tests = template_doc['listings']['tests']
+    # To avoid messing up the template tests by deletion, we have
+    # to take a deepcopy before the changes.
+    doc_copy = copy.deepcopy(template_doc)
+    template_tests = doc_copy['listings']['tests']
+    if not with_tags:
+        for test in template_tests:
+            del test['tags']
    num_original = len(template_tests)

    out_tests = []
@@ -176,10 +182,10 @@ def main():
        os.makedirs(workdir, exist_ok=True)
        for count in COUNTS:
            listing_path = os.path.join(workdir, f"listing-{count}.json")
-            with open(listing_path, "w", encoding="utf-8") as f:
-                synthesize_listing(template_doc, count, f)

            for add_tags in tag_modes:
+                with open(listing_path, "w", encoding="utf-8") as f:
+                    synthesize_listing(template_doc, count, f, add_tags)
                mode = "on" if add_tags else "off"
                ctest_file = os.path.join(workdir, f"ctest-{count}-{mode}.cmake")
                cmd = build_command(args, listing_path, ctest_file, add_tags)
@@ -11,22 +11,14 @@
        "tags": [
          "function pointer",
          "Tricky"
-        ],
-        "source-location": {
-          "filename": "/mnt/c/ubuntu/Catch2/tests/SelfTest/UsageTests/Tricky.tests.cpp",
-          "line": 266
-        }
+        ]
      },
      {
        "name": "Testing checked-if 4",
        "tags": [
          "!shouldfail",
          "checked-if"
-        ],
-        "source-location": {
-          "filename": "/mnt/c/ubuntu/Catch2/tests/SelfTest/UsageTests/Misc.tests.cpp",
-          "line": 216
-        }
+        ]
      },
      {
        "name": "count_equidistant_floats - double",
@@ -34,11 +26,7 @@
          "approvals",
          "distance",
          "floating-point"
-        ],
-        "source-location": {
-          "filename": "/mnt/c/ubuntu/Catch2/tests/SelfTest/IntrospectiveTests/FloatingPoint.tests.cpp",
-          "line": 103
-        }
+        ]
      },
      {
        "name": "Usage of AllTrue range matcher",
@@ -46,11 +34,7 @@
          "matchers",
          "quantifiers",
          "templated"
-        ],
-        "source-location": {
-          "filename": "/mnt/c/ubuntu/Catch2/tests/SelfTest/UsageTests/MatchersRanges.tests.cpp",
-          "line": 372
-        }
+        ]
      },
      {
        "name": "Exception matchers that succeed",
@@ -58,11 +42,7 @@
          "!throws",
          "exceptions",
          "matchers"
-        ],
-        "source-location": {
-          "filename": "/mnt/c/ubuntu/Catch2/tests/SelfTest/UsageTests/Matchers.tests.cpp",
-          "line": 428
-        }
+        ]
      },
      {
        "name": "A TEMPLATE_TEST_CASE_METHOD based test run that succeeds - float",
@@ -70,22 +50,14 @@
        "tags": [
          "class",
          "template"
-        ],
-        "source-location": {
-          "filename": "/mnt/c/ubuntu/Catch2/tests/SelfTest/UsageTests/Class.tests.cpp",
-          "line": 82
-        }
+        ]
      },
      {
        "name": "Approximate PI",
        "tags": [
          "Approx",
          "PI"
-        ],
-        "source-location": {
-          "filename": "/mnt/c/ubuntu/Catch2/tests/SelfTest/UsageTests/Approx.tests.cpp",
-          "line": 134
-        }
+        ]
      },
      {
        "name": "TextFlow::Column respects width setting",
@@ -93,32 +65,20 @@
          "approvals",
          "column",
          "TextFlow"
-        ],
-        "source-location": {
-          "filename": "/mnt/c/ubuntu/Catch2/tests/SelfTest/IntrospectiveTests/TextFlow.tests.cpp",
-          "line": 38
-        }
+        ]
      },
      {
        "name": "Generators internals",
        "tags": [
          "generators",
          "internals"
-        ],
-        "source-location": {
-          "filename": "/mnt/c/ubuntu/Catch2/tests/SelfTest/IntrospectiveTests/GeneratorsImpl.tests.cpp",
-          "line": 24
-        }
+        ]
      },
      {
        "name": "Mayfail test case with nested sections",
        "tags": [
          "!mayfail"
-        ],
-        "source-location": {
-          "filename": "/mnt/c/ubuntu/Catch2/tests/SelfTest/UsageTests/Condition.tests.cpp",
-          "line": 83
-        }
+        ]
      }
    ]
  }
@@ -253,10 +253,15 @@ function(catch_discover_tests_impl)

  make_temp_file_path(listing_output_path "${_TEST_WORKING_DIR}")

+  set(_Verbosity "quiet")
+  if (add_tags)
+    set(_Verbosity "normal")
+  endif()
+
  execute_process(
    COMMAND ${_TEST_EXECUTOR} "${_TEST_EXECUTABLE}" ${spec}
      --list-tests
-      --reporter json
+      --reporter "json::verbosity=${_Verbosity}"
      --out "${listing_output_path}"
      --order lex # Make sure the output order, and thus test registration order, is consistent across runs.
    OUTPUT_VARIABLE listing_output
@@ -349,13 +349,15 @@ namespace Catch {
            if (!info.className.empty()) {
                desc_writer.write( "class-name"_sr ).write( info.className );
            }
-            {
+            if ( m_verbosity >= Verbosity::Normal ) {
                auto tag_writer = desc_writer.write( "tags"_sr ).writeArray();
                for ( auto const& tag : info.tags ) {
                    tag_writer.write( tag.original );
                }
            }
-            writeSourceInfo( desc_writer, info.lineInfo );
+            if ( m_verbosity >= Verbosity::High) {
+                writeSourceInfo( desc_writer, info.lineInfo );
+            }
        }
    }
    void JsonReporter::listTags( std::vector<TagInfo> const& tags ) {
@@ -85,9 +85,6 @@ namespace Catch {
        std::stack<Writer> m_writers{};

        bool m_startedListing = false;
-
-        // std::size_t m_sectionDepth = 0;
-        // std::size_t m_sectionStarted = 0;
    };
 } // namespace Catch

@@ -208,6 +208,21 @@ set_tests_properties(List::Tests::XmlOutput PROPERTIES
  FAIL_REGULAR_EXPRESSION "[0-9]+ test cases"
 )

+add_test(NAME List::Tests::Json::Normal COMMAND $<TARGET_FILE:SelfTest> --list-tests -r json)
+set_tests_properties(List::Tests::Json::Normal PROPERTIES
+  PASS_REGULAR_EXPRESSION "\"tags\": \\["
+  FAIL_REGULAR_EXPRESSION "\"source-location\": {"
+)
+add_test(NAME List::Tests::Json::Quiet COMMAND $<TARGET_FILE:SelfTest> --list-tests -r json::verbosity=quiet)
+set_tests_properties(List::Tests::Json::Quiet PROPERTIES
+  PASS_REGULAR_EXPRESSION "\"name\": \""
+  FAIL_REGULAR_EXPRESSION "\"tags\": \\["
+)
+add_test(NAME List::Tests::Json::High COMMAND $<TARGET_FILE:SelfTest> --list-tests -r json::verbosity=high)
+set_tests_properties(List::Tests::Json::High PROPERTIES
+  PASS_REGULAR_EXPRESSION "\"source-location\": {"
+)
+
 add_test(NAME List::Tags::Output COMMAND $<TARGET_FILE:SelfTest> --list-tags)
 set_tests_properties(List::Tags::Output PROPERTIES
  PASS_REGULAR_EXPRESSION "[0-9]+ tags"
@@ -218,6 +233,11 @@ set_tests_properties(List::Tags::XmlOutput PROPERTIES
  PASS_REGULAR_EXPRESSION "<Count>18</Count>"
  FAIL_REGULAR_EXPRESSION "[0-9]+ tags"
 )
+add_test(NAME List::Tags::JsonOutput COMMAND $<TARGET_FILE:SelfTest> --list-tags -r json)
+set_tests_properties(List::Tags::JsonOutput PROPERTIES
+  PASS_REGULAR_EXPRESSION "\"count\": 18"
+  FAIL_REGULAR_EXPRESSION "[0-9]+ tags"
+)

 add_test(NAME List::Reporters::Output COMMAND $<TARGET_FILE:SelfTest> --list-reporters)
 set_tests_properties(List::Reporters::Output PROPERTIES PASS_REGULAR_EXPRESSION "Available reporters:")
@@ -227,6 +247,12 @@ set_tests_properties(List::Reporters::XmlOutput PROPERTIES
  PASS_REGULAR_EXPRESSION "<Name>compact</Name>"
  FAIL_REGULAR_EXPRESSION "Available reporters:"
 )
+add_test(NAME List::Reporters::JsonOutput COMMAND $<TARGET_FILE:SelfTest> --list-reporters -r json)
+set_tests_properties(List::Reporters::JsonOutput PROPERTIES
+  PASS_REGULAR_EXPRESSION "\"name\": \"compact\""
+  FAIL_REGULAR_EXPRESSION "Available reporters:"
+)
+

 add_test(NAME List::Listeners::Output
  COMMAND
@@ -251,6 +277,18 @@ set_tests_properties(List::Listeners::XmlOutput
    PASS_REGULAR_EXPRESSION "<RegisteredListeners>"
    FAIL_REGULAR_EXPRESSION "Registered listeners:"
 )
+add_test(NAME List::Listeners::JsonOutput
+  COMMAND
+    $<TARGET_FILE:SelfTest>
+      --list-listeners
+      --reporter json
+)
+set_tests_properties(List::Listeners::JsonOutput
+  PROPERTIES
+    PASS_REGULAR_EXPRESSION "\"listeners\": \\["
+    FAIL_REGULAR_EXPRESSION "Registered listeners:"
+)
+

 add_test(NAME NoAssertions COMMAND $<TARGET_FILE:SelfTest> -w NoAssertions "An empty test with no assertions")
 set_tests_properties(NoAssertions PROPERTIES PASS_REGULAR_EXPRESSION "No assertions in test case")
@@ -1740,11 +1740,7 @@ Reporters.tests.cpp:<line number>: passed: listingString, ContainsSubstring( "fa
        "name": "fake test name",
        "tags": [
          "fakeTestTag"
-        ],
-        "source-location": {
-          "filename": "fake-file.cpp",
-          "line": 123456789
-        }
+        ]
      }
    ]" ( contains: "fake test name" and contains: "fakeTestTag" ) with 1 message: 'Tested reporter: JSON'
 Reporters.tests.cpp:<line number>: passed: !(factories.empty()) for: !false
@@ -1738,11 +1738,7 @@ Reporters.tests.cpp:<line number>: passed: listingString, ContainsSubstring( "fa
        "name": "fake test name",
        "tags": [
          "fakeTestTag"
-        ],
-        "source-location": {
-          "filename": "fake-file.cpp",
-          "line": 123456789
-        }
+        ]
      }
    ]" ( contains: "fake test name" and contains: "fakeTestTag" ) with 1 message: 'Tested reporter: JSON'
 Reporters.tests.cpp:<line number>: passed: !(factories.empty()) for: !false
@@ -11323,11 +11323,7 @@ with expansion:
          "name": "fake test name",
          "tags": [
            "fakeTestTag"
-          ],
-          "source-location": {
-            "filename": "fake-file.cpp",
-            "line": 123456789
-          }
+          ]
        }
      ]" ( contains: "fake test name" and contains: "fakeTestTag" )
 with message:
@@ -11321,11 +11321,7 @@ with expansion:
          "name": "fake test name",
          "tags": [
            "fakeTestTag"
-          ],
-          "source-location": {
-            "filename": "fake-file.cpp",
-            "line": 123456789
-          }
+          ]
        }
      ]" ( contains: "fake test name" and contains: "fakeTestTag" )
 with message:
@@ -2775,7 +2775,7 @@ ok {test-number} - listingString, ContainsSubstring("fake reporter"s) for: "{
 # Reporter's write listings to provided stream
 ok {test-number} - !(factories.empty()) for: !false
 # Reporter's write listings to provided stream
-ok {test-number} - listingString, ContainsSubstring( "fake test name"s ) && ContainsSubstring( "fakeTestTag"s ) for: "{   "version": 2,   "metadata": {     "name": "",     "rng-seed": 1234,     "catch2-version": "<version>"   },   "listings": {     "tests": [       {         "name": "fake test name",         "tags": [           "fakeTestTag"         ],         "source-location": {           "filename": "fake-file.cpp",           "line": 123456789         }       }     ]" ( contains: "fake test name" and contains: "fakeTestTag" ) with 1 message: 'Tested reporter: JSON'
+ok {test-number} - listingString, ContainsSubstring( "fake test name"s ) && ContainsSubstring( "fakeTestTag"s ) for: "{   "version": 2,   "metadata": {     "name": "",     "rng-seed": 1234,     "catch2-version": "<version>"   },   "listings": {     "tests": [       {         "name": "fake test name",         "tags": [           "fakeTestTag"         ]       }     ]" ( contains: "fake test name" and contains: "fakeTestTag" ) with 1 message: 'Tested reporter: JSON'
 # Reporter's write listings to provided stream
 ok {test-number} - !(factories.empty()) for: !false
 # Reporter's write listings to provided stream
@@ -2773,7 +2773,7 @@ ok {test-number} - listingString, ContainsSubstring("fake reporter"s) for: "{
 # Reporter's write listings to provided stream
 ok {test-number} - !(factories.empty()) for: !false
 # Reporter's write listings to provided stream
-ok {test-number} - listingString, ContainsSubstring( "fake test name"s ) && ContainsSubstring( "fakeTestTag"s ) for: "{   "version": 2,   "metadata": {     "name": "",     "rng-seed": 1234,     "catch2-version": "<version>"   },   "listings": {     "tests": [       {         "name": "fake test name",         "tags": [           "fakeTestTag"         ],         "source-location": {           "filename": "fake-file.cpp",           "line": 123456789         }       }     ]" ( contains: "fake test name" and contains: "fakeTestTag" ) with 1 message: 'Tested reporter: JSON'
+ok {test-number} - listingString, ContainsSubstring( "fake test name"s ) && ContainsSubstring( "fakeTestTag"s ) for: "{   "version": 2,   "metadata": {     "name": "",     "rng-seed": 1234,     "catch2-version": "<version>"   },   "listings": {     "tests": [       {         "name": "fake test name",         "tags": [           "fakeTestTag"         ]       }     ]" ( contains: "fake test name" and contains: "fakeTestTag" ) with 1 message: 'Tested reporter: JSON'
 # Reporter's write listings to provided stream
 ok {test-number} - !(factories.empty()) for: !false
 # Reporter's write listings to provided stream
@@ -13503,11 +13503,7 @@ Approx( 0.98999999999999999 )
        "name": "fake test name",
        "tags": [
          "fakeTestTag"
-        ],
-        "source-location": {
-          "filename": "fake-file.cpp",
-          "line": 123456789
-        }
+        ]
      }
    ]" ( contains: "fake test name" and contains: "fakeTestTag" )
        </Expanded>
@@ -13503,11 +13503,7 @@ Approx( 0.98999999999999999 )
        "name": "fake test name",
        "tags": [
          "fakeTestTag"
-        ],
-        "source-location": {
-          "filename": "fake-file.cpp",
-          "line": 123456789
-        }
+        ]
      }
    ]" ( contains: "fake test name" and contains: "fakeTestTag" )
        </Expanded>