forked from catchorg/Catch2
Martin Hořeňovský
630840c500
There were 2 separate bugs, one recently added and one ancient (since roughly the first version of the registration script). 1) The recent refactoring of how the JSON output from Catch2 is parsed caused shadowing between the variable containing the per-test JSON fragments and the accumulator of test names for <target>_TESTS variable. This led to the variable containing both the names and the JSON fragments, and thus being completely wrong. 2) The test name accumulation has never accounted for characters that need escaping to be present in a CMake list. This means that e.g. test names with semicolon in them would end up with two elements in the test list. Both of these are now fixed, at the cost of extra complexity and my sanity as I had to learn more about CMake escaping rules. (CMake escaping rules are dumb)
299 lines
10 KiB
Python
299 lines
10 KiB
Python
#!/usr/bin/env python3
|
|
|
|
# Copyright Catch2 Authors
|
|
# Distributed under the Boost Software License, Version 1.0.
|
|
# (See accompanying file LICENSE.txt or copy at
|
|
# https://www.boost.org/LICENSE_1_0.txt)
|
|
|
|
# SPDX-License-Identifier: BSL-1.0
|
|
|
|
import glob
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import re
|
|
import json
|
|
import tempfile
|
|
from collections import namedtuple
|
|
from typing import List
|
|
|
|
TestInfo = namedtuple('TestInfo', ['name', 'tags'])
|
|
|
|
cmake_version_regex = re.compile(r'cmake version (\d+)\.(\d+)\.(\d+)')
|
|
|
|
def get_cmake_version():
|
|
result = subprocess.run(['cmake', '--version'],
|
|
capture_output = True,
|
|
check = True,
|
|
text = True)
|
|
version_match = cmake_version_regex.match(result.stdout)
|
|
if not version_match:
|
|
print('Could not find cmake version in output')
|
|
print(f"output: '{result.stdout}'")
|
|
exit(4)
|
|
return (int(version_match.group(1)),
|
|
int(version_match.group(2)),
|
|
int(version_match.group(3)))
|
|
|
|
def build_project(sources_dir, output_base_path, catch2_path):
|
|
build_dir = os.path.join(output_base_path, 'ctest-registration-test')
|
|
config_cmd = ['cmake',
|
|
'-B', build_dir,
|
|
'-S', sources_dir,
|
|
f'-DCATCH2_PATH={catch2_path}',
|
|
'-DCMAKE_BUILD_TYPE=Debug']
|
|
|
|
build_cmd = ['cmake',
|
|
'--build', build_dir,
|
|
'--config', 'Debug']
|
|
|
|
try:
|
|
subprocess.run(config_cmd,
|
|
capture_output = True,
|
|
check = True,
|
|
text = True)
|
|
subprocess.run(build_cmd,
|
|
capture_output = True,
|
|
check = True,
|
|
text = True)
|
|
except subprocess.CalledProcessError as err:
|
|
print('Error when building the test project')
|
|
print(f'cmd: {err.cmd}')
|
|
print(f'stderr: {err.stderr}')
|
|
print(f'stdout: {err.stdout}')
|
|
exit(3)
|
|
|
|
return build_dir
|
|
|
|
|
|
def get_test_names(build_path: str) -> List[TestInfo]:
|
|
# For now we assume that Windows builds are done using MSBuild under
|
|
# Debug configuration. This means that we need to add "Debug" folder
|
|
# to the path when constructing it. On Linux, we don't add anything.
|
|
config_path = "Debug" if os.name == 'nt' else ""
|
|
full_path = os.path.join(build_path, config_path, 'tests')
|
|
|
|
with tempfile.TemporaryDirectory() as tmpdir:
|
|
fname = f'{tmpdir}/listing-output.json'
|
|
cmd = [full_path,
|
|
'--list-tests',
|
|
'--reporter', 'json',
|
|
'--out', fname
|
|
]
|
|
result = subprocess.run(cmd,
|
|
capture_output = False,
|
|
check = True,
|
|
text = True)
|
|
with open(fname, mode='r', encoding='utf-8') as file:
|
|
test_listing = json.load(file)
|
|
|
|
assert test_listing['version'] == 2
|
|
|
|
tests = []
|
|
for test in test_listing['listings']['tests']:
|
|
test_name = test['name']
|
|
tags = test['tags']
|
|
tests.append(TestInfo(test_name, tags))
|
|
|
|
return tests
|
|
|
|
|
|
def get_ctest_listing(build_path):
|
|
old_path = os.getcwd()
|
|
os.chdir(build_path)
|
|
|
|
cmd = ['ctest', '-C', 'debug', '--show-only=json-v1']
|
|
try:
|
|
result = subprocess.run(cmd,
|
|
capture_output = True,
|
|
check = True,
|
|
text = True)
|
|
except subprocess.CalledProcessError as err:
|
|
print('Error when getting output from CTest')
|
|
print(f'cmd: {err.cmd}')
|
|
print(f'stderr: {err.stderr}')
|
|
print(f'stdout: {err.stdout}')
|
|
exit(4)
|
|
|
|
os.chdir(old_path)
|
|
return result.stdout
|
|
|
|
def extract_tests_from_ctest(ctest_output) -> List[TestInfo]:
|
|
ctest_response = json.loads(ctest_output)
|
|
tests = ctest_response['tests']
|
|
test_infos = []
|
|
for test in tests:
|
|
test_command = test['command']
|
|
# First part of the command is the binary, second is the filter.
|
|
# If there are less, registration has failed. If there are more,
|
|
# registration has changed and the script needs updating.
|
|
assert len(test_command) == 2
|
|
test_name = test_command[1]
|
|
labels = []
|
|
for prop in test['properties']:
|
|
if prop['name'] == 'LABELS':
|
|
labels = prop['value']
|
|
|
|
test_infos.append(TestInfo(test_name, labels))
|
|
|
|
return test_infos
|
|
|
|
|
|
def check_DL_PATHS(ctest_output):
|
|
ctest_response = json.loads(ctest_output)
|
|
tests = ctest_response['tests']
|
|
for test in tests:
|
|
properties = test['properties']
|
|
for property in properties:
|
|
if property['name'] == 'ENVIRONMENT_MODIFICATION':
|
|
assert len(property['value']) == 2, f"The test provides 2 arguments to DL_PATHS, but instead found {len(property['value'])}"
|
|
|
|
|
|
def add_test_list_extractor(build_path: str) -> str:
|
|
# The actual CTest script file has one of two names:
|
|
# * `<target>-<short-hash>_tests.cmake` on single-config generators
|
|
# * `<target>-<short-hash>_tests-<Config>.cmake` on multi-config generators
|
|
#
|
|
# We know the target name (`tests`), so we glob for the hash part
|
|
patterns = [
|
|
os.path.join(build_path, 'tests-*_tests.cmake'),
|
|
os.path.join(build_path, 'tests-*_tests-Debug.cmake'),
|
|
]
|
|
matches = []
|
|
for pattern in patterns:
|
|
matches.extend(glob.glob(pattern))
|
|
if len(matches) != 1:
|
|
print(f"Found {len(matches)} CTest files in '{build_path}'. Expected only 1.")
|
|
exit(5)
|
|
test_script_file = matches[0]
|
|
basename = os.path.basename(test_script_file)
|
|
extractor_fname = os.path.join(build_path, 'extractor.cmake')
|
|
with open(extractor_fname, 'w') as f:
|
|
f.write(fr"""
|
|
cmake_minimum_required(VERSION 3.19)
|
|
|
|
cmake_policy(VERSION 3.19...4.4)
|
|
|
|
# This dummies out the `add_test` and `set_tests_properties` commands
|
|
# inside the CTest script, so we can include it during CMake script call.
|
|
macro(add_test)
|
|
endmacro()
|
|
macro(set_tests_properties)
|
|
endmacro()
|
|
|
|
include(${{CMAKE_CURRENT_LIST_DIR}}/{basename})
|
|
|
|
list(LENGTH tests_TESTS num_tests)
|
|
message(STATUS "NUM TESTS: ${{num_tests}}")
|
|
|
|
# '[' and ']' were escaped into ASCII 2 and 3 respectively, we have to
|
|
# unescape them back here. Note that this has to be done per-element,
|
|
# or CMake's list parsing breaks (which is why they were escaped).
|
|
string(ASCII 2 _LeftBracketListingEscape)
|
|
string(ASCII 3 _RightBracketListingEscape)
|
|
|
|
foreach(test IN LISTS tests_TESTS)
|
|
string(REPLACE "${{_LeftBracketListingEscape}}" "[" test "${{test}}")
|
|
string(REPLACE "${{_RightBracketListingEscape}}" "]" test "${{test}}")
|
|
string(REPLACE "\\" "\\\\" test "${{test}}")
|
|
string(REPLACE "\r" "\\r" test "${{test}}")
|
|
string(REPLACE "\n" "\\n" test "${{test}}")
|
|
message(STATUS "TEST_NAME: ${{test}}")
|
|
endforeach()
|
|
""")
|
|
return extractor_fname
|
|
|
|
|
|
def extract_tests_list_from_ctest_script(build_path: str) -> List[str]:
|
|
extractor = add_test_list_extractor(build_path)
|
|
|
|
cmd = ['cmake', '-P', extractor]
|
|
|
|
try:
|
|
result = subprocess.run(cmd,
|
|
capture_output = True,
|
|
check = True,
|
|
text = True)
|
|
except subprocess.CalledProcessError as err:
|
|
print('Error when calling CTest test extractor')
|
|
print(f'cmd: {err.cmd}')
|
|
print(f'stderr: {err.stderr}')
|
|
print(f'stdout: {err.stdout}')
|
|
exit(4)
|
|
|
|
lines = result.stdout.strip().split('\n')
|
|
test_num_line = lines[0]
|
|
test_lines = lines[1:]
|
|
|
|
test_num_prefix = '-- NUM TESTS: '
|
|
assert test_num_prefix in test_num_line, test_num_line
|
|
test_num_line = test_num_line[len(test_num_prefix):]
|
|
num_tests = int(test_num_line)
|
|
|
|
assert num_tests == len(test_lines), len(test_lines)
|
|
test_name_prefix = '-- TEST_NAME: '
|
|
assert all(test_name_prefix in x for x in test_lines)
|
|
test_names = [x[len(test_name_prefix):] for x in test_lines]
|
|
|
|
# Unescape the names, so that names with literal newlines have newlines in them again
|
|
test_names = [x.encode('utf-8').decode('unicode-escape') for x in test_names]
|
|
return test_names
|
|
|
|
|
|
def escape_catch2_test_names(infos: List[TestInfo]):
|
|
escaped = []
|
|
for info in infos:
|
|
name = info.name
|
|
for char in ('\\', ',', '[', ']'):
|
|
name = name.replace(char, f"\\{char}")
|
|
escaped.append(TestInfo(name, info.tags))
|
|
return escaped
|
|
|
|
|
|
if __name__ == '__main__':
|
|
if len(sys.argv) != 3:
|
|
print(f'Usage: {sys.argv[0]} path-to-catch2-cml output-path')
|
|
exit(2)
|
|
catch2_path = sys.argv[1]
|
|
output_base_path = sys.argv[2]
|
|
sources_dir = os.path.dirname(os.path.abspath(sys.argv[0]))
|
|
|
|
build_path = build_project(sources_dir, output_base_path, catch2_path)
|
|
|
|
raw_catch_test_names = get_test_names(build_path)
|
|
catch_test_names = escape_catch2_test_names(raw_catch_test_names)
|
|
ctest_output = get_ctest_listing(build_path)
|
|
ctest_test_names = extract_tests_from_ctest(ctest_output)
|
|
|
|
mismatched = 0
|
|
for catch_test in catch_test_names:
|
|
if catch_test not in ctest_test_names:
|
|
print(f"Catch2 test '{catch_test}' not found in CTest")
|
|
mismatched += 1
|
|
for ctest_test in ctest_test_names:
|
|
if ctest_test not in catch_test_names:
|
|
print(f"CTest test '{ctest_test}' not found in Catch2")
|
|
mismatched += 1
|
|
|
|
if mismatched:
|
|
print(f"Found {mismatched} mismatched tests catch test names and ctest test commands!")
|
|
exit(1)
|
|
print(f"{len(catch_test_names)} tests matched in CTest listing")
|
|
|
|
test_list_names = sorted(extract_tests_list_from_ctest_script(build_path))
|
|
expected_names = sorted(info.name for info in raw_catch_test_names)
|
|
if test_list_names != expected_names:
|
|
print("TEST_LIST variable (tests_TESTS) does not match Catch2 test listing!")
|
|
for name in test_list_names:
|
|
if name not in expected_names:
|
|
print(f" TEST_LIST name '{name}' not in Catch2 listing")
|
|
for name in expected_names:
|
|
if name not in test_list_names:
|
|
print(f" Catch2 name '{name}' not in TEST_LIST")
|
|
exit(1)
|
|
print(f"{len(test_list_names)} tests matched in TEST_LIST variable")
|
|
|
|
cmake_version = get_cmake_version()
|
|
if cmake_version >= (3, 27):
|
|
check_DL_PATHS(ctest_output)
|