mirror of
https://github.com/catchorg/Catch2.git
synced 2026-08-23 22:03:41 +02:00
Martin Hořeňovský
1079da4c5f
Using CMake's `string(JSON` to parse JSON array leads to quadratic running time in number of tests, see https://gitlab.kitware.com/cmake/cmake/-/work_items/27985 This leads to _terrible_ runtime for `catch_discover_tests` when called on binaries with lot of tests (1k+). To get reasonable runtimes, we have to avoid using `string(JSON` to parse out the individual test objects from the array with all tests. This commit replaces the sane approach of using real JSON parser with a set of terrible hacks, where we use CMake's string APIs to split the JSON array on what looks like object boundary (`}<ws>*,<ws>*{`), and then checking whether the resulting thing can be parsed as JSON object. If not, we append the next piece and check again. And again, and again, until we get a proper JSON object. This is all around a hilariously terrible idea, however: 1) It works in practice for all tested inputs. 2) It improves the time it takes to run `catch_discover_tests` on binary with 1k tests from 4.2s to 1.1s and 2k tests from 16s to 3.9s.
464 lines
17 KiB
CMake
464 lines
17 KiB
CMake
# Distributed under the OSI-approved BSD 3-Clause License. See accompanying
|
|
# file Copyright.txt or https://cmake.org/licensing for details.
|
|
|
|
# Because natively using CMake's JSON processing for arrays leads to quadratic
|
|
# running times, we do terrible hack and split JSON array by object boundary
|
|
# + commas and try to reconstruct valid JSON objects. During this, we need
|
|
# to replace CMake characters that could be in the test name/tags with
|
|
# placeholder, so it doesn't affect CMake's processing of the strings/lists
|
|
# we create during the parsing.
|
|
#
|
|
# We use 0x01, 0x02, 0x03, and 0x04 as placeholder bytes, as those cannot
|
|
# exist in JSON unescaped.
|
|
#
|
|
# 0x01 <=> ';' (CMake list separator)
|
|
# 0x02 == element boundary marker used while splitting the tests array
|
|
# 0x03 <-> '[' (opens a CMake bracket-argument context)
|
|
# 0x04 <-> ']' (closes a CMake bracket-argument context)
|
|
#
|
|
string(ASCII 1 _SemicolonEscape)
|
|
string(ASCII 2 _BoundaryEscape)
|
|
string(ASCII 3 _OpenBracketEscape)
|
|
string(ASCII 4 _CloseBracketEscape)
|
|
|
|
|
|
# Placeholder bytes in the listing would break our parsing hack, so
|
|
# we check they don't exist. They shouldn't exist in valid JSON, but
|
|
# the reporter might not be escaping them properly.
|
|
function(validate_input_noescapes listing_var)
|
|
foreach(byte "${_SemicolonEscape}" "${_BoundaryEscape}" "${_OpenBracketEscape}" "${_CloseBracketEscape}")
|
|
string(FIND "${${listing_var}}" "${byte}" found)
|
|
if(NOT found EQUAL -1)
|
|
message(FATAL_ERROR
|
|
"The test listing contains raw control byte (0x01-0x04) which should not "
|
|
"be there. This means either bad escaping in JSON reporter, or corrupted file. "
|
|
)
|
|
endif()
|
|
endforeach()
|
|
endfunction()
|
|
|
|
|
|
# Replaces relevant characters with their placeholders, see the top of this file.
|
|
# Modifies argument `var` in place.
|
|
function(magic_escape_chars var)
|
|
set(value "${${var}}")
|
|
string(REPLACE ";" "${_SemicolonEscape}" value "${value}")
|
|
string(REPLACE "[" "${_OpenBracketEscape}" value "${value}")
|
|
string(REPLACE "]" "${_CloseBracketEscape}" value "${value}")
|
|
set(${var} "${value}" PARENT_SCOPE)
|
|
endfunction()
|
|
|
|
|
|
# Turns placeholders back into original characters, see the top of this file.
|
|
# Modifies argument `var` in place.
|
|
function(magic_unescape_chars var)
|
|
set(value "${${var}}")
|
|
string(REPLACE "${_SemicolonEscape}" ";" value "${value}")
|
|
string(REPLACE "${_OpenBracketEscape}" "[" value "${value}")
|
|
string(REPLACE "${_CloseBracketEscape}" "]" value "${value}")
|
|
set(${var} "${value}" PARENT_SCOPE)
|
|
endfunction()
|
|
|
|
|
|
# Abuses knowledge of Catch2's JSON reporter output for listing tests to
|
|
# split JSON array of the test listings into a CMake list of strings,
|
|
# with each element being the JSON string of one array entry.
|
|
#
|
|
# This avoids the terrible quadratic running time of using CMake's JSON
|
|
# support to parse the JSON reporter output "properly", where the whole
|
|
# JSON array of tests is parsed again for every element. Instead, we can
|
|
# use the CMake's API to only parse the individual test's objects, which
|
|
# are usually small and only have to be reparsed fixed number of times
|
|
# (once for test names, once for labels).
|
|
#
|
|
# We process the string representing the JSON array by splitting it on
|
|
# `}<ws>*,<ws>*{` and then checking for each chunk whether it is a valid
|
|
# JSON object representing Catch2's test. If not (e.g. because we split
|
|
# on the presence of `}<ws>*,<ws>*{` inside a test name), then we append
|
|
# the next chunk to the current one and check again. And again, until
|
|
# we get back to a valid JSON.
|
|
#
|
|
# Note that to support passing the object strings back from the function,
|
|
# they will still contain the placeholders and need to be unescaped before
|
|
# further processing (e.g. sending them into CMake's JSON parsing API).
|
|
function(split_json_array json_array_var out_var)
|
|
# We have to pass the input by var name to avoid CMake processing
|
|
# the input as an arg.
|
|
set(json_in "${${json_array_var}}")
|
|
|
|
# Strip the array brackets at the start and end of the JSON array.
|
|
# Must happen before we escape the other [] instances below from the
|
|
# actual array data.
|
|
string(REGEX REPLACE "^[ \t\r\n]*\\[" "" json_in "${json_in}")
|
|
string(REGEX REPLACE "\\][ \t\r\n]*$" "" json_in "${json_in}")
|
|
|
|
magic_escape_chars(json_in)
|
|
|
|
# We need to keep the whitespace around comma around, so that if we
|
|
# split inside the test object, we can reconstruct it losslessly.
|
|
string(REGEX REPLACE "(}[ \t\r\n]*)[,]([ \t\r\n]*{)" "\\1${_BoundaryEscape}\\2" json_in "${json_in}")
|
|
|
|
# We escaped all list separators above, so now we can turn the JSON
|
|
# string into a CMake list of fragments in single pass.
|
|
string(REPLACE "${_BoundaryEscape}" ";" fragments "${json_in}")
|
|
|
|
# And now we have to reconstruct the actual JSON structure from fragments.
|
|
set(array_elements "")
|
|
set(accumulator "")
|
|
foreach(next_fragment IN LISTS fragments)
|
|
if(accumulator)
|
|
set(accumulator "${accumulator},${next_fragment}")
|
|
else()
|
|
set(accumulator "${next_fragment}")
|
|
endif()
|
|
|
|
# Because the fragments (might) contain invalid JSON characters due
|
|
# to escaping, we have to unescape it before checking if we can parse it.
|
|
set(maybe_json "${accumulator}")
|
|
magic_unescape_chars(maybe_json)
|
|
string(JSON unused ERROR_VARIABLE err GET "${maybe_json}" "name")
|
|
if(err STREQUAL "NOTFOUND")
|
|
list(APPEND array_elements "${accumulator}")
|
|
set(accumulator "")
|
|
endif()
|
|
endforeach()
|
|
|
|
set(${out_var} "${array_elements}" PARENT_SCOPE)
|
|
endfunction()
|
|
|
|
# TBD: Further possible optimization is that most arguments for per-test
|
|
# `prepare_command` call are constant across one invocation of
|
|
# `catch_discover_tests`, and thus need checking and escaping only
|
|
# once, instead of for each test.
|
|
# This would provide nice speed-up of the actual command preparation,
|
|
# but it will make the script much harder to read, and it is utterly
|
|
# dwarfed by the quadratic scaling of parsing JSON arrays in CMake.
|
|
|
|
|
|
# Prepare command with escaped (bracketed) arguments and return it via `_Command` out variable.
|
|
#
|
|
# To avoid quadratic performance when concatenating all commands together,
|
|
# the actual concatenation must be done by the caller, by appending it
|
|
# into a string of all other commands.
|
|
function(prepare_command NAME)
|
|
set(_args "")
|
|
# use ARGV* instead of ARGN, because ARGN splits arrays into multiple arguments
|
|
math(EXPR _last_arg ${ARGC}-1)
|
|
foreach(_n RANGE 1 ${_last_arg})
|
|
set(_arg "${ARGV${_n}}")
|
|
if(_arg MATCHES "[^-./:a-zA-Z0-9_]")
|
|
set(_args "${_args} [==[${_arg}]==]") # form a bracket_argument
|
|
else()
|
|
set(_args "${_args} ${_arg}")
|
|
endif()
|
|
endforeach()
|
|
set(_Command "${NAME}(${_args})\n" PARENT_SCOPE)
|
|
endfunction()
|
|
|
|
# Generates random filename in the temp folder.
|
|
# Temp folder is retrieved by checking env vars from various platforms.
|
|
function(make_temp_file_path OUT_VARIABLE FALLBACK_PATH)
|
|
set(TEMP_DIR "")
|
|
set(ENV_VARS
|
|
# From XDG base dir specification
|
|
XDG_RUNTIME_DIR
|
|
# From POSIX standard
|
|
TMPDIR
|
|
# From Windows
|
|
TMP
|
|
TEMP
|
|
)
|
|
|
|
foreach(var ${ENV_VARS})
|
|
if(DEFINED ENV{${var}} AND NOT "$ENV{${var}}" STREQUAL "")
|
|
set(TEMP_DIR "$ENV{${var}}")
|
|
break()
|
|
endif()
|
|
endforeach()
|
|
|
|
# If all checks fail, we use the fallback path
|
|
if(TEMP_DIR STREQUAL "")
|
|
set(TEMP_DIR "${FALLBACK_PATH}")
|
|
endif()
|
|
|
|
file(TO_CMAKE_PATH "${TEMP_DIR}" TEMP_DIR)
|
|
|
|
# Generate the random file name
|
|
string(RANDOM LENGTH 8 RAND_ID)
|
|
set(FINAL_TEMP_PATH "${TEMP_DIR}/Catch2-test-listing.${RAND_ID}.json")
|
|
|
|
set(${OUT_VARIABLE} "${FINAL_TEMP_PATH}" PARENT_SCOPE)
|
|
endfunction()
|
|
|
|
function(catch_discover_tests_impl)
|
|
cmake_parse_arguments(
|
|
""
|
|
""
|
|
"TEST_EXECUTABLE;TEST_WORKING_DIR;TEST_OUTPUT_DIR;TEST_OUTPUT_PREFIX;TEST_OUTPUT_SUFFIX;TEST_PREFIX;TEST_REPORTER;TEST_SPEC;TEST_SUFFIX;TEST_LIST;CTEST_FILE"
|
|
"TEST_EXTRA_ARGS;TEST_PROPERTIES;TEST_EXECUTOR;TEST_DL_PATHS;TEST_DL_FRAMEWORK_PATHS;ADD_TAGS_AS_LABELS"
|
|
${ARGN}
|
|
)
|
|
|
|
# We periodically append to the output file below, so we have to ensure
|
|
# that it is empty at the start, or we get duplicated test scripts.
|
|
file(REMOVE "${_CTEST_FILE}")
|
|
# Size (in Bytes) at which the intermediate `script` var is dumped to file.
|
|
set(_WriteToFileThreshold 50000)
|
|
|
|
set(add_tags "${_ADD_TAGS_AS_LABELS}")
|
|
set(prefix "${_TEST_PREFIX}")
|
|
set(suffix "${_TEST_SUFFIX}")
|
|
set(spec ${_TEST_SPEC})
|
|
set(extra_args ${_TEST_EXTRA_ARGS})
|
|
set(properties ${_TEST_PROPERTIES})
|
|
set(reporter ${_TEST_REPORTER})
|
|
set(output_dir ${_TEST_OUTPUT_DIR})
|
|
set(output_prefix ${_TEST_OUTPUT_PREFIX})
|
|
set(output_suffix ${_TEST_OUTPUT_SUFFIX})
|
|
set(dl_paths ${_TEST_DL_PATHS})
|
|
set(dl_framework_paths ${_TEST_DL_FRAMEWORK_PATHS})
|
|
set(environment_modifications "")
|
|
set(script)
|
|
set(suite)
|
|
set(tests)
|
|
|
|
if(WIN32)
|
|
set(dl_paths_variable_name PATH)
|
|
elseif(APPLE)
|
|
set(dl_paths_variable_name DYLD_LIBRARY_PATH)
|
|
else()
|
|
set(dl_paths_variable_name LD_LIBRARY_PATH)
|
|
endif()
|
|
|
|
# Run test executable to get list of available tests
|
|
if(NOT EXISTS "${_TEST_EXECUTABLE}")
|
|
message(FATAL_ERROR
|
|
"Specified test executable '${_TEST_EXECUTABLE}' does not exist"
|
|
)
|
|
endif()
|
|
|
|
if(dl_paths)
|
|
cmake_path(CONVERT "$ENV{${dl_paths_variable_name}}" TO_NATIVE_PATH_LIST env_dl_paths)
|
|
list(PREPEND env_dl_paths "${dl_paths}")
|
|
cmake_path(CONVERT "${env_dl_paths}" TO_NATIVE_PATH_LIST paths)
|
|
set(ENV{${dl_paths_variable_name}} "${paths}")
|
|
endif()
|
|
|
|
if(APPLE AND dl_framework_paths)
|
|
cmake_path(CONVERT "$ENV{DYLD_FRAMEWORK_PATH}" TO_NATIVE_PATH_LIST env_dl_framework_paths)
|
|
list(PREPEND env_dl_framework_paths "${dl_framework_paths}")
|
|
cmake_path(CONVERT "${env_dl_framework_paths}" TO_NATIVE_PATH_LIST paths)
|
|
set(ENV{DYLD_FRAMEWORK_PATH} "${paths}")
|
|
endif()
|
|
|
|
make_temp_file_path(listing_output_path "${_TEST_WORKING_DIR}")
|
|
|
|
execute_process(
|
|
COMMAND ${_TEST_EXECUTOR} "${_TEST_EXECUTABLE}" ${spec}
|
|
--list-tests
|
|
--reporter json
|
|
--out "${listing_output_path}"
|
|
--order lex # Make sure the output order, and thus test registration order, is consistent across runs.
|
|
OUTPUT_VARIABLE listing_output
|
|
RESULT_VARIABLE result
|
|
WORKING_DIRECTORY "${_TEST_WORKING_DIR}"
|
|
)
|
|
if(NOT ${result} EQUAL 0)
|
|
message(FATAL_ERROR
|
|
"Error listing tests from executable '${_TEST_EXECUTABLE}':\n"
|
|
" Result: ${result}\n"
|
|
" Output: ${listing_output}\n"
|
|
)
|
|
endif()
|
|
|
|
# Read the JSON output back from the output file and validate it.
|
|
file(READ ${listing_output_path} listing_output)
|
|
file(REMOVE ${listing_output_path})
|
|
validate_input_noescapes(listing_output)
|
|
|
|
# Prepare reporter
|
|
if(reporter)
|
|
set(reporter_arg "--reporter ${reporter}")
|
|
|
|
# Run test executable to check whether reporter is available
|
|
# note that the use of --list-reporters is not the important part,
|
|
# we only want to check whether the execution succeeds with ${reporter_arg}
|
|
execute_process(
|
|
COMMAND ${_TEST_EXECUTOR} "${_TEST_EXECUTABLE}" ${spec} ${reporter_arg} --list-reporters
|
|
OUTPUT_VARIABLE reporter_check_output
|
|
RESULT_VARIABLE reporter_check_result
|
|
WORKING_DIRECTORY "${_TEST_WORKING_DIR}"
|
|
)
|
|
if(${reporter_check_result} EQUAL 255)
|
|
message(FATAL_ERROR
|
|
"\"${reporter}\" is not a valid reporter!\n"
|
|
)
|
|
elseif(NOT ${reporter_check_result} EQUAL 0)
|
|
message(FATAL_ERROR
|
|
"Error checking for reporter in test executable '${_TEST_EXECUTABLE}':\n"
|
|
" Result: ${reporter_check_result}\n"
|
|
" Output: ${reporter_check_output}\n"
|
|
)
|
|
endif()
|
|
endif()
|
|
|
|
# Prepare output dir
|
|
if(output_dir AND NOT IS_ABSOLUTE ${output_dir})
|
|
set(output_dir "${_TEST_WORKING_DIR}/${output_dir}")
|
|
if(NOT EXISTS ${output_dir})
|
|
file(MAKE_DIRECTORY ${output_dir})
|
|
endif()
|
|
endif()
|
|
|
|
if(dl_paths)
|
|
foreach(path ${dl_paths})
|
|
cmake_path(NATIVE_PATH path native_path)
|
|
list(PREPEND environment_modifications "${dl_paths_variable_name}=path_list_prepend:${native_path}")
|
|
endforeach()
|
|
endif()
|
|
|
|
if(APPLE AND dl_framework_paths)
|
|
foreach(path ${dl_framework_paths})
|
|
cmake_path(NATIVE_PATH path native_path)
|
|
list(PREPEND environment_modifications "DYLD_FRAMEWORK_PATH=path_list_prepend:${native_path}")
|
|
endforeach()
|
|
endif()
|
|
|
|
# Parse JSON output for list of tests/class names/tags
|
|
string(JSON version GET "${listing_output}" "version")
|
|
if(NOT version STREQUAL "1")
|
|
message(FATAL_ERROR "Unsupported catch output version: '${version}'")
|
|
endif()
|
|
|
|
# Extract just the JSON array with tests and then split them into
|
|
# individual objects.
|
|
string(JSON test_listing GET "${listing_output}" "listings" "tests")
|
|
split_json_array(test_listing tests)
|
|
|
|
# Exit early if no tests are detected
|
|
if(NOT tests)
|
|
file(WRITE "${_CTEST_FILE}" "")
|
|
return()
|
|
endif()
|
|
|
|
# Each element in the tests is JSON-string representing one test object.
|
|
# We have to parse it and then turn it into CTest script commands.
|
|
foreach(single_test IN LISTS tests)
|
|
# Because appending to the same string in CMake has quadratic runtime,
|
|
# we flush the script into the file periodically to avoid the worst case.
|
|
string(LENGTH "${script}" script_len)
|
|
if (script_len GREATER _WriteToFileThreshold)
|
|
file(APPEND "${_CTEST_FILE}" "${script}")
|
|
set(script "")
|
|
endif()
|
|
|
|
# The elements are still escaped and contain JSON-invalid characters,
|
|
# they have to be unescaped before parsing them as JSON.
|
|
magic_unescape_chars(single_test)
|
|
if(add_tags)
|
|
string(JSON test_tags GET "${single_test}" "tags")
|
|
endif()
|
|
string(JSON plain_name GET "${single_test}" "name")
|
|
|
|
# Escape characters in test case names that would be parsed by Catch2
|
|
# Note that the \ escaping must happen FIRST! Do not change the order.
|
|
set(escaped_name "${plain_name}")
|
|
foreach(char \\ , [ ] ;)
|
|
string(REPLACE ${char} "\\${char}" escaped_name "${escaped_name}")
|
|
endforeach(char)
|
|
# ...add output dir
|
|
if(output_dir)
|
|
string(REGEX REPLACE "[^A-Za-z0-9_]" "_" escaped_name_clean "${escaped_name}")
|
|
set(output_dir_arg "--out ${output_dir}/${output_prefix}${escaped_name_clean}${output_suffix}")
|
|
endif()
|
|
|
|
# ...and add to script
|
|
prepare_command(add_test
|
|
"${prefix}${plain_name}${suffix}"
|
|
${_TEST_EXECUTOR}
|
|
"${_TEST_EXECUTABLE}"
|
|
"${escaped_name}"
|
|
${extra_args}
|
|
"${reporter_arg}"
|
|
"${output_dir_arg}"
|
|
)
|
|
string(APPEND script "${_Command}")
|
|
prepare_command(set_tests_properties
|
|
"${prefix}${plain_name}${suffix}"
|
|
PROPERTIES
|
|
WORKING_DIRECTORY "${_TEST_WORKING_DIR}"
|
|
${properties}
|
|
)
|
|
string(APPEND script "${_Command}")
|
|
|
|
if(add_tags)
|
|
string(JSON num_tags LENGTH "${test_tags}")
|
|
math(EXPR num_tags "${num_tags} - 1")
|
|
set(tag_list "")
|
|
if(num_tags GREATER_EQUAL "0")
|
|
foreach(tag_idx RANGE ${num_tags})
|
|
string(JSON a_tag GET "${test_tags}" "${tag_idx}")
|
|
# Catch2's tags can contain semicolons, which are list element separators
|
|
# in CMake, so we have to escape them. Ideally we could use the [=[...]=]
|
|
# syntax for this, but CTest currently keeps the square quotes in the label
|
|
# name. So we add 2 backslashes to escape it instead.
|
|
# **IMPORTANT**: The number of backslashes depends on how many layers
|
|
# of CMake the tag goes. If this script is changed, the
|
|
# number of backslashes to escape may change as well.
|
|
string(REPLACE ";" "\\;" a_tag "${a_tag}")
|
|
list(APPEND tag_list "${a_tag}")
|
|
endforeach()
|
|
|
|
prepare_command(set_tests_properties
|
|
"${prefix}${plain_name}${suffix}"
|
|
PROPERTIES
|
|
LABELS "${tag_list}"
|
|
)
|
|
string(APPEND script "${_Command}")
|
|
endif()
|
|
endif(add_tags)
|
|
|
|
if(environment_modifications)
|
|
prepare_command(set_tests_properties
|
|
"${prefix}${plain_name}${suffix}"
|
|
PROPERTIES
|
|
ENVIRONMENT_MODIFICATION "${environment_modifications}")
|
|
string(APPEND script "${_Command}")
|
|
endif()
|
|
|
|
list(APPEND tests "${prefix}${plain_name}${suffix}")
|
|
endforeach()
|
|
|
|
# Create a list of all discovered tests, which users may use to e.g. set
|
|
# properties on the tests
|
|
prepare_command(set ${_TEST_LIST} ${tests})
|
|
string(APPEND script "${_Command}")
|
|
|
|
# Write any script leftovers we have
|
|
file(APPEND "${_CTEST_FILE}" "${script}")
|
|
endfunction()
|
|
|
|
# To enable `include`ing this file in the unit test scripts, we only run
|
|
# the impl if an actual `TEST_EXECUTABLE` is provided.
|
|
if(CMAKE_SCRIPT_MODE_FILE AND DEFINED TEST_EXECUTABLE)
|
|
catch_discover_tests_impl(
|
|
TEST_EXECUTABLE ${TEST_EXECUTABLE}
|
|
TEST_EXECUTOR ${TEST_EXECUTOR}
|
|
TEST_WORKING_DIR ${TEST_WORKING_DIR}
|
|
TEST_SPEC ${TEST_SPEC}
|
|
TEST_EXTRA_ARGS ${TEST_EXTRA_ARGS}
|
|
TEST_PROPERTIES ${TEST_PROPERTIES}
|
|
TEST_PREFIX ${TEST_PREFIX}
|
|
TEST_SUFFIX ${TEST_SUFFIX}
|
|
TEST_LIST ${TEST_LIST}
|
|
TEST_REPORTER ${TEST_REPORTER}
|
|
TEST_OUTPUT_DIR ${TEST_OUTPUT_DIR}
|
|
TEST_OUTPUT_PREFIX ${TEST_OUTPUT_PREFIX}
|
|
TEST_OUTPUT_SUFFIX ${TEST_OUTPUT_SUFFIX}
|
|
TEST_DL_PATHS ${TEST_DL_PATHS}
|
|
TEST_DL_FRAMEWORK_PATHS ${TEST_DL_FRAMEWORK_PATHS}
|
|
CTEST_FILE ${CTEST_FILE}
|
|
ADD_TAGS_AS_LABELS ${ADD_TAGS_AS_LABELS}
|
|
)
|
|
endif()
|