Martin Hořeňovský
60c8b87829
Repeatedly calling `string(APPEND` on the same destination string leads to quadratic running time. We avoid this by flushing the CTest script into a file periodically, currently every 50kB of text. This can cause _slight_ slowdown just around the flush boundary (each file write is quite expensive, so if we flush just before the last test is written, it hurts), but it avoids terrible performance for large test suites. The performance is roughly equal at 500 tests to discover, and clearly wins at more; for 1/2/4/8 k tests, the improvements are 0.3/1.6/6.4/27 s.
340 lines
12 KiB
CMake
340 lines
12 KiB
CMake
# Distributed under the OSI-approved BSD 3-Clause License. See accompanying
|
|
# file Copyright.txt or https://cmake.org/licensing for details.
|
|
|
|
# TBD: Further possible optimization is that most arguments for per-test
|
|
# `prepare_command` call are constant across one invocation of
|
|
# `catch_discover_tests`, and thus need checking and escaping only
|
|
# once, instead of for each test.
|
|
# This would provide nice speed-up of the actual command preparation,
|
|
# but it will make the script much harder to read, and it is utterly
|
|
# dwarfed by the quadratic scaling of parsing JSON arrays in CMake.
|
|
|
|
|
|
# Prepare command with escaped (bracketed) arguments and return it via `_Command` out variable.
|
|
#
|
|
# To avoid quadratic performance when concatenating all commands together,
|
|
# the actual concatenation must be done by the caller, by appending it
|
|
# into a string of all other commands.
|
|
function(prepare_command NAME)
|
|
set(_args "")
|
|
# use ARGV* instead of ARGN, because ARGN splits arrays into multiple arguments
|
|
math(EXPR _last_arg ${ARGC}-1)
|
|
foreach(_n RANGE 1 ${_last_arg})
|
|
set(_arg "${ARGV${_n}}")
|
|
if(_arg MATCHES "[^-./:a-zA-Z0-9_]")
|
|
set(_args "${_args} [==[${_arg}]==]") # form a bracket_argument
|
|
else()
|
|
set(_args "${_args} ${_arg}")
|
|
endif()
|
|
endforeach()
|
|
set(_Command "${NAME}(${_args})\n" PARENT_SCOPE)
|
|
endfunction()
|
|
|
|
# Generates random filename in the temp folder.
|
|
# Temp folder is retrieved by checking env vars from various platforms.
|
|
function(make_temp_file_path OUT_VARIABLE FALLBACK_PATH)
|
|
set(TEMP_DIR "")
|
|
set(ENV_VARS
|
|
# From XDG base dir specification
|
|
XDG_RUNTIME_DIR
|
|
# From POSIX standard
|
|
TMPDIR
|
|
# From Windows
|
|
TMP
|
|
TEMP
|
|
)
|
|
|
|
foreach(var ${ENV_VARS})
|
|
if(DEFINED ENV{${var}} AND NOT "$ENV{${var}}" STREQUAL "")
|
|
set(TEMP_DIR "$ENV{${var}}")
|
|
break()
|
|
endif()
|
|
endforeach()
|
|
|
|
# If all checks fail, we use the fallback path
|
|
if(TEMP_DIR STREQUAL "")
|
|
set(TEMP_DIR "${FALLBACK_PATH}")
|
|
endif()
|
|
|
|
file(TO_CMAKE_PATH "${TEMP_DIR}" TEMP_DIR)
|
|
|
|
# Generate the random file name
|
|
string(RANDOM LENGTH 8 RAND_ID)
|
|
set(FINAL_TEMP_PATH "${TEMP_DIR}/Catch2-test-listing.${RAND_ID}.json")
|
|
|
|
set(${OUT_VARIABLE} "${FINAL_TEMP_PATH}" PARENT_SCOPE)
|
|
endfunction()
|
|
|
|
function(catch_discover_tests_impl)
|
|
|
|
cmake_parse_arguments(
|
|
""
|
|
""
|
|
"TEST_EXECUTABLE;TEST_WORKING_DIR;TEST_OUTPUT_DIR;TEST_OUTPUT_PREFIX;TEST_OUTPUT_SUFFIX;TEST_PREFIX;TEST_REPORTER;TEST_SPEC;TEST_SUFFIX;TEST_LIST;CTEST_FILE"
|
|
"TEST_EXTRA_ARGS;TEST_PROPERTIES;TEST_EXECUTOR;TEST_DL_PATHS;TEST_DL_FRAMEWORK_PATHS;ADD_TAGS_AS_LABELS"
|
|
${ARGN}
|
|
)
|
|
|
|
# We periodically append to the output file below, so we have to ensure
|
|
# that it is empty at the start, or we get duplicated test scripts.
|
|
file(REMOVE "${_CTEST_FILE}")
|
|
# Size (in Bytes) at which the intermediate `script` var is dumped to file.
|
|
set(_WriteToFileThreshold 50000)
|
|
|
|
set(add_tags "${_ADD_TAGS_AS_LABELS}")
|
|
set(prefix "${_TEST_PREFIX}")
|
|
set(suffix "${_TEST_SUFFIX}")
|
|
set(spec ${_TEST_SPEC})
|
|
set(extra_args ${_TEST_EXTRA_ARGS})
|
|
set(properties ${_TEST_PROPERTIES})
|
|
set(reporter ${_TEST_REPORTER})
|
|
set(output_dir ${_TEST_OUTPUT_DIR})
|
|
set(output_prefix ${_TEST_OUTPUT_PREFIX})
|
|
set(output_suffix ${_TEST_OUTPUT_SUFFIX})
|
|
set(dl_paths ${_TEST_DL_PATHS})
|
|
set(dl_framework_paths ${_TEST_DL_FRAMEWORK_PATHS})
|
|
set(environment_modifications "")
|
|
set(script)
|
|
set(suite)
|
|
set(tests)
|
|
|
|
if(WIN32)
|
|
set(dl_paths_variable_name PATH)
|
|
elseif(APPLE)
|
|
set(dl_paths_variable_name DYLD_LIBRARY_PATH)
|
|
else()
|
|
set(dl_paths_variable_name LD_LIBRARY_PATH)
|
|
endif()
|
|
|
|
# Run test executable to get list of available tests
|
|
if(NOT EXISTS "${_TEST_EXECUTABLE}")
|
|
message(FATAL_ERROR
|
|
"Specified test executable '${_TEST_EXECUTABLE}' does not exist"
|
|
)
|
|
endif()
|
|
|
|
if(dl_paths)
|
|
cmake_path(CONVERT "$ENV{${dl_paths_variable_name}}" TO_NATIVE_PATH_LIST env_dl_paths)
|
|
list(PREPEND env_dl_paths "${dl_paths}")
|
|
cmake_path(CONVERT "${env_dl_paths}" TO_NATIVE_PATH_LIST paths)
|
|
set(ENV{${dl_paths_variable_name}} "${paths}")
|
|
endif()
|
|
|
|
if(APPLE AND dl_framework_paths)
|
|
cmake_path(CONVERT "$ENV{DYLD_FRAMEWORK_PATH}" TO_NATIVE_PATH_LIST env_dl_framework_paths)
|
|
list(PREPEND env_dl_framework_paths "${dl_framework_paths}")
|
|
cmake_path(CONVERT "${env_dl_framework_paths}" TO_NATIVE_PATH_LIST paths)
|
|
set(ENV{DYLD_FRAMEWORK_PATH} "${paths}")
|
|
endif()
|
|
|
|
make_temp_file_path(listing_output_path "${_TEST_WORKING_DIR}")
|
|
|
|
execute_process(
|
|
COMMAND ${_TEST_EXECUTOR} "${_TEST_EXECUTABLE}" ${spec}
|
|
--list-tests
|
|
--reporter json
|
|
--out "${listing_output_path}"
|
|
--order lex # Make sure the output order, and thus test registration order, is consistent across runs.
|
|
OUTPUT_VARIABLE listing_output
|
|
RESULT_VARIABLE result
|
|
WORKING_DIRECTORY "${_TEST_WORKING_DIR}"
|
|
)
|
|
if(NOT ${result} EQUAL 0)
|
|
message(FATAL_ERROR
|
|
"Error listing tests from executable '${_TEST_EXECUTABLE}':\n"
|
|
" Result: ${result}\n"
|
|
" Output: ${listing_output}\n"
|
|
)
|
|
endif()
|
|
|
|
# Read the JSON output back from the output file (and then get rid of the file)
|
|
file(READ ${listing_output_path} listing_output)
|
|
file(REMOVE ${listing_output_path})
|
|
|
|
# Prepare reporter
|
|
if(reporter)
|
|
set(reporter_arg "--reporter ${reporter}")
|
|
|
|
# Run test executable to check whether reporter is available
|
|
# note that the use of --list-reporters is not the important part,
|
|
# we only want to check whether the execution succeeds with ${reporter_arg}
|
|
execute_process(
|
|
COMMAND ${_TEST_EXECUTOR} "${_TEST_EXECUTABLE}" ${spec} ${reporter_arg} --list-reporters
|
|
OUTPUT_VARIABLE reporter_check_output
|
|
RESULT_VARIABLE reporter_check_result
|
|
WORKING_DIRECTORY "${_TEST_WORKING_DIR}"
|
|
)
|
|
if(${reporter_check_result} EQUAL 255)
|
|
message(FATAL_ERROR
|
|
"\"${reporter}\" is not a valid reporter!\n"
|
|
)
|
|
elseif(NOT ${reporter_check_result} EQUAL 0)
|
|
message(FATAL_ERROR
|
|
"Error checking for reporter in test executable '${_TEST_EXECUTABLE}':\n"
|
|
" Result: ${reporter_check_result}\n"
|
|
" Output: ${reporter_check_output}\n"
|
|
)
|
|
endif()
|
|
endif()
|
|
|
|
# Prepare output dir
|
|
if(output_dir AND NOT IS_ABSOLUTE ${output_dir})
|
|
set(output_dir "${_TEST_WORKING_DIR}/${output_dir}")
|
|
if(NOT EXISTS ${output_dir})
|
|
file(MAKE_DIRECTORY ${output_dir})
|
|
endif()
|
|
endif()
|
|
|
|
if(dl_paths)
|
|
foreach(path ${dl_paths})
|
|
cmake_path(NATIVE_PATH path native_path)
|
|
list(PREPEND environment_modifications "${dl_paths_variable_name}=path_list_prepend:${native_path}")
|
|
endforeach()
|
|
endif()
|
|
|
|
if(APPLE AND dl_framework_paths)
|
|
foreach(path ${dl_framework_paths})
|
|
cmake_path(NATIVE_PATH path native_path)
|
|
list(PREPEND environment_modifications "DYLD_FRAMEWORK_PATH=path_list_prepend:${native_path}")
|
|
endforeach()
|
|
endif()
|
|
|
|
# Parse JSON output for list of tests/class names/tags
|
|
string(JSON version GET "${listing_output}" "version")
|
|
if(NOT version STREQUAL "1")
|
|
message(FATAL_ERROR "Unsupported catch output version: '${version}'")
|
|
endif()
|
|
|
|
# Speed-up reparsing by cutting away unneeded parts of JSON.
|
|
string(JSON test_listing GET "${listing_output}" "listings" "tests")
|
|
string(JSON num_tests LENGTH "${test_listing}")
|
|
|
|
# Exit early if no tests are detected
|
|
if(num_tests STREQUAL "0")
|
|
file(WRITE "${_CTEST_FILE}" "")
|
|
return()
|
|
endif()
|
|
|
|
# CMake's foreach-RANGE is inclusive, so we have to subtract 1
|
|
math(EXPR num_tests "${num_tests} - 1")
|
|
|
|
foreach(idx RANGE ${num_tests})
|
|
string(LENGTH "${script}" script_len)
|
|
# Because appending to the same string in CMake has quadratic runtime,
|
|
# we flush the script into the file periodically to avoid the worst case.
|
|
if (script_len GREATER _WriteToFileThreshold)
|
|
file(APPEND "${_CTEST_FILE}" "${script}")
|
|
set(script "")
|
|
endif()
|
|
|
|
|
|
if(add_tags)
|
|
string(JSON single_test GET "${test_listing}" ${idx})
|
|
string(JSON test_tags GET "${single_test}" "tags")
|
|
string(JSON plain_name GET "${single_test}" "name")
|
|
else()
|
|
string(JSON plain_name GET "${test_listing}" ${idx} "name")
|
|
endif()
|
|
|
|
# Escape characters in test case names that would be parsed by Catch2
|
|
# Note that the \ escaping must happen FIRST! Do not change the order.
|
|
set(escaped_name "${plain_name}")
|
|
foreach(char \\ , [ ] ;)
|
|
string(REPLACE ${char} "\\${char}" escaped_name "${escaped_name}")
|
|
endforeach(char)
|
|
# ...add output dir
|
|
if(output_dir)
|
|
string(REGEX REPLACE "[^A-Za-z0-9_]" "_" escaped_name_clean "${escaped_name}")
|
|
set(output_dir_arg "--out ${output_dir}/${output_prefix}${escaped_name_clean}${output_suffix}")
|
|
endif()
|
|
|
|
# ...and add to script
|
|
prepare_command(add_test
|
|
"${prefix}${plain_name}${suffix}"
|
|
${_TEST_EXECUTOR}
|
|
"${_TEST_EXECUTABLE}"
|
|
"${escaped_name}"
|
|
${extra_args}
|
|
"${reporter_arg}"
|
|
"${output_dir_arg}"
|
|
)
|
|
string(APPEND script "${_Command}")
|
|
prepare_command(set_tests_properties
|
|
"${prefix}${plain_name}${suffix}"
|
|
PROPERTIES
|
|
WORKING_DIRECTORY "${_TEST_WORKING_DIR}"
|
|
${properties}
|
|
)
|
|
string(APPEND script "${_Command}")
|
|
|
|
if(add_tags)
|
|
string(JSON num_tags LENGTH "${test_tags}")
|
|
math(EXPR num_tags "${num_tags} - 1")
|
|
set(tag_list "")
|
|
if(num_tags GREATER_EQUAL "0")
|
|
foreach(tag_idx RANGE ${num_tags})
|
|
string(JSON a_tag GET "${test_tags}" "${tag_idx}")
|
|
# Catch2's tags can contain semicolons, which are list element separators
|
|
# in CMake, so we have to escape them. Ideally we could use the [=[...]=]
|
|
# syntax for this, but CTest currently keeps the square quotes in the label
|
|
# name. So we add 2 backslashes to escape it instead.
|
|
# **IMPORTANT**: The number of backslashes depends on how many layers
|
|
# of CMake the tag goes. If this script is changed, the
|
|
# number of backslashes to escape may change as well.
|
|
string(REPLACE ";" "\\;" a_tag "${a_tag}")
|
|
list(APPEND tag_list "${a_tag}")
|
|
endforeach()
|
|
|
|
prepare_command(set_tests_properties
|
|
"${prefix}${plain_name}${suffix}"
|
|
PROPERTIES
|
|
LABELS "${tag_list}"
|
|
)
|
|
string(APPEND script "${_Command}")
|
|
endif()
|
|
endif(add_tags)
|
|
|
|
if(environment_modifications)
|
|
prepare_command(set_tests_properties
|
|
"${prefix}${plain_name}${suffix}"
|
|
PROPERTIES
|
|
ENVIRONMENT_MODIFICATION "${environment_modifications}")
|
|
string(APPEND script "${_Command}")
|
|
endif()
|
|
|
|
list(APPEND tests "${prefix}${plain_name}${suffix}")
|
|
endforeach()
|
|
|
|
# Create a list of all discovered tests, which users may use to e.g. set
|
|
# properties on the tests
|
|
prepare_command(set ${_TEST_LIST} ${tests})
|
|
string(APPEND script "${_Command}")
|
|
|
|
# Write any script leftovers we have
|
|
file(APPEND "${_CTEST_FILE}" "${script}")
|
|
endfunction()
|
|
|
|
# To enable `include`ing this file in the unit test scripts, we only run
|
|
# the impl if an actual `TEST_EXECUTABLE` is provided.
|
|
if(CMAKE_SCRIPT_MODE_FILE AND DEFINED TEST_EXECUTABLE)
|
|
catch_discover_tests_impl(
|
|
TEST_EXECUTABLE ${TEST_EXECUTABLE}
|
|
TEST_EXECUTOR ${TEST_EXECUTOR}
|
|
TEST_WORKING_DIR ${TEST_WORKING_DIR}
|
|
TEST_SPEC ${TEST_SPEC}
|
|
TEST_EXTRA_ARGS ${TEST_EXTRA_ARGS}
|
|
TEST_PROPERTIES ${TEST_PROPERTIES}
|
|
TEST_PREFIX ${TEST_PREFIX}
|
|
TEST_SUFFIX ${TEST_SUFFIX}
|
|
TEST_LIST ${TEST_LIST}
|
|
TEST_REPORTER ${TEST_REPORTER}
|
|
TEST_OUTPUT_DIR ${TEST_OUTPUT_DIR}
|
|
TEST_OUTPUT_PREFIX ${TEST_OUTPUT_PREFIX}
|
|
TEST_OUTPUT_SUFFIX ${TEST_OUTPUT_SUFFIX}
|
|
TEST_DL_PATHS ${TEST_DL_PATHS}
|
|
TEST_DL_FRAMEWORK_PATHS ${TEST_DL_FRAMEWORK_PATHS}
|
|
CTEST_FILE ${CTEST_FILE}
|
|
ADD_TAGS_AS_LABELS ${ADD_TAGS_AS_LABELS}
|
|
)
|
|
endif()
|