Catch2/extras/CatchAddTests.cmake

# Distributed under the OSI-approved BSD 3-Clause License.  See accompanying
# file Copyright.txt or https://cmake.org/licensing for details.

# TBD: Further possible optimization is that most arguments for per-test
#      `prepare_command` call are constant across one invocation of
#      `catch_discover_tests`, and thus need checking and escaping only
#      once, instead of for each test.
#      This would provide nice speed-up of the actual command preparation,
#      but it will make the script much harder to read, and it is utterly
#      dwarfed by the quadratic scaling of parsing JSON arrays in CMake.


# Prepare command with escaped (bracketed) arguments and return it via `_Command` out variable.
#
# To avoid quadratic performance when concatenating all commands together,
# the actual concatenation must be done by the caller, by appending it
# into a string of all other commands.
function(prepare_command NAME)
  set(_args "")
  # use ARGV* instead of ARGN, because ARGN splits arrays into multiple arguments
  math(EXPR _last_arg ${ARGC}-1)
  foreach(_n RANGE 1 ${_last_arg})
    set(_arg "${ARGV${_n}}")
    if(_arg MATCHES "[^-./:a-zA-Z0-9_]")
      set(_args "${_args} [==[${_arg}]==]") # form a bracket_argument
    else()
      set(_args "${_args} ${_arg}")
    endif()
  endforeach()
  set(_Command "${NAME}(${_args})\n" PARENT_SCOPE)
endfunction()

# Generates random filename in the temp folder.
# Temp folder is retrieved by checking env vars from various platforms.
function(make_temp_file_path OUT_VARIABLE FALLBACK_PATH)
  set(TEMP_DIR "")
  set(ENV_VARS
    # From XDG base dir specification
    XDG_RUNTIME_DIR
    # From POSIX standard
    TMPDIR
    # From Windows
    TMP
    TEMP
  )

  foreach(var ${ENV_VARS})
    if(DEFINED ENV{${var}} AND NOT "$ENV{${var}}" STREQUAL "")
        set(TEMP_DIR "$ENV{${var}}")
        break()
    endif()
  endforeach()

  # If all checks fail, we use the fallback path
  if(TEMP_DIR STREQUAL "")
    set(TEMP_DIR "${FALLBACK_PATH}")
  endif()

  file(TO_CMAKE_PATH "${TEMP_DIR}" TEMP_DIR)

  # Generate the random file name
  string(RANDOM LENGTH 8 RAND_ID)
  set(FINAL_TEMP_PATH "${TEMP_DIR}/Catch2-test-listing.${RAND_ID}.json")

  set(${OUT_VARIABLE} "${FINAL_TEMP_PATH}" PARENT_SCOPE)
endfunction()

function(catch_discover_tests_impl)

  cmake_parse_arguments(
    ""
    ""
    "TEST_EXECUTABLE;TEST_WORKING_DIR;TEST_OUTPUT_DIR;TEST_OUTPUT_PREFIX;TEST_OUTPUT_SUFFIX;TEST_PREFIX;TEST_REPORTER;TEST_SPEC;TEST_SUFFIX;TEST_LIST;CTEST_FILE"
    "TEST_EXTRA_ARGS;TEST_PROPERTIES;TEST_EXECUTOR;TEST_DL_PATHS;TEST_DL_FRAMEWORK_PATHS;ADD_TAGS_AS_LABELS"
    ${ARGN}
  )

  set(add_tags "${_ADD_TAGS_AS_LABELS}")
  set(prefix "${_TEST_PREFIX}")
  set(suffix "${_TEST_SUFFIX}")
  set(spec ${_TEST_SPEC})
  set(extra_args ${_TEST_EXTRA_ARGS})
  set(properties ${_TEST_PROPERTIES})
  set(reporter ${_TEST_REPORTER})
  set(output_dir ${_TEST_OUTPUT_DIR})
  set(output_prefix ${_TEST_OUTPUT_PREFIX})
  set(output_suffix ${_TEST_OUTPUT_SUFFIX})
  set(dl_paths ${_TEST_DL_PATHS})
  set(dl_framework_paths ${_TEST_DL_FRAMEWORK_PATHS})
  set(environment_modifications "")
  set(script)
  set(suite)
  set(tests)

  if(WIN32)
    set(dl_paths_variable_name PATH)
  elseif(APPLE)
    set(dl_paths_variable_name DYLD_LIBRARY_PATH)
  else()
    set(dl_paths_variable_name LD_LIBRARY_PATH)
  endif()

  # Run test executable to get list of available tests
  if(NOT EXISTS "${_TEST_EXECUTABLE}")
    message(FATAL_ERROR
      "Specified test executable '${_TEST_EXECUTABLE}' does not exist"
    )
  endif()

  if(dl_paths)
    cmake_path(CONVERT "$ENV{${dl_paths_variable_name}}" TO_NATIVE_PATH_LIST env_dl_paths)
    list(PREPEND env_dl_paths "${dl_paths}")
    cmake_path(CONVERT "${env_dl_paths}" TO_NATIVE_PATH_LIST paths)
    set(ENV{${dl_paths_variable_name}} "${paths}")
  endif()

  if(APPLE AND dl_framework_paths)
    cmake_path(CONVERT "$ENV{DYLD_FRAMEWORK_PATH}" TO_NATIVE_PATH_LIST env_dl_framework_paths)
    list(PREPEND env_dl_framework_paths "${dl_framework_paths}")
    cmake_path(CONVERT "${env_dl_framework_paths}" TO_NATIVE_PATH_LIST paths)
    set(ENV{DYLD_FRAMEWORK_PATH} "${paths}")
  endif()

  make_temp_file_path(listing_output_path "${_TEST_WORKING_DIR}")

  execute_process(
    COMMAND ${_TEST_EXECUTOR} "${_TEST_EXECUTABLE}" ${spec}
      --list-tests
      --reporter json
      --out "${listing_output_path}"
      --order lex # Make sure the output order, and thus test registration order, is consistent across runs.
    OUTPUT_VARIABLE listing_output
    RESULT_VARIABLE result
    WORKING_DIRECTORY "${_TEST_WORKING_DIR}"
  )
  if(NOT ${result} EQUAL 0)
    message(FATAL_ERROR
      "Error listing tests from executable '${_TEST_EXECUTABLE}':\n"
      "  Result: ${result}\n"
      "  Output: ${listing_output}\n"
    )
  endif()

  # Read the JSON output back from the output file (and then get rid of the file)
  file(READ ${listing_output_path} listing_output)
  file(REMOVE ${listing_output_path})

  # Prepare reporter
  if(reporter)
    set(reporter_arg "--reporter ${reporter}")

    # Run test executable to check whether reporter is available
    # note that the use of --list-reporters is not the important part,
    # we only want to check whether the execution succeeds with ${reporter_arg}
    execute_process(
      COMMAND ${_TEST_EXECUTOR} "${_TEST_EXECUTABLE}" ${spec} ${reporter_arg} --list-reporters
      OUTPUT_VARIABLE reporter_check_output
      RESULT_VARIABLE reporter_check_result
      WORKING_DIRECTORY "${_TEST_WORKING_DIR}"
    )
    if(${reporter_check_result} EQUAL 255)
      message(FATAL_ERROR
        "\"${reporter}\" is not a valid reporter!\n"
      )
    elseif(NOT ${reporter_check_result} EQUAL 0)
      message(FATAL_ERROR
        "Error checking for reporter in test executable '${_TEST_EXECUTABLE}':\n"
        "  Result: ${reporter_check_result}\n"
        "  Output: ${reporter_check_output}\n"
      )
    endif()
  endif()

  # Prepare output dir
  if(output_dir AND NOT IS_ABSOLUTE ${output_dir})
    set(output_dir "${_TEST_WORKING_DIR}/${output_dir}")
    if(NOT EXISTS ${output_dir})
      file(MAKE_DIRECTORY ${output_dir})
    endif()
  endif()

  if(dl_paths)
    foreach(path ${dl_paths})
      cmake_path(NATIVE_PATH path native_path)
      list(PREPEND environment_modifications "${dl_paths_variable_name}=path_list_prepend:${native_path}")
    endforeach()
  endif()

  if(APPLE AND dl_framework_paths)
    foreach(path ${dl_framework_paths})
      cmake_path(NATIVE_PATH path native_path)
      list(PREPEND environment_modifications "DYLD_FRAMEWORK_PATH=path_list_prepend:${native_path}")
    endforeach()
  endif()

  # Parse JSON output for list of tests/class names/tags
  string(JSON version GET "${listing_output}" "version")
  if(NOT version STREQUAL "1")
    message(FATAL_ERROR "Unsupported catch output version: '${version}'")
  endif()

  # Speed-up reparsing by cutting away unneeded parts of JSON.
  string(JSON test_listing GET "${listing_output}" "listings" "tests")
  string(JSON num_tests LENGTH "${test_listing}")

  # Exit early if no tests are detected
  if(num_tests STREQUAL "0")
    file(WRITE "${_CTEST_FILE}" "")
    return()
  endif()

  # CMake's foreach-RANGE is inclusive, so we have to subtract 1
  math(EXPR num_tests "${num_tests} - 1")

  foreach(idx RANGE ${num_tests})
    if(add_tags)
      string(JSON single_test GET "${test_listing}" ${idx})
      string(JSON test_tags GET "${single_test}" "tags")
      string(JSON plain_name GET "${single_test}" "name")
    else()
      string(JSON plain_name GET "${test_listing}" ${idx} "name")
    endif()

    # Escape characters in test case names that would be parsed by Catch2
    # Note that the \ escaping must happen FIRST! Do not change the order.
    set(escaped_name "${plain_name}")
    foreach(char \\ , [ ] ;)
      string(REPLACE ${char} "\\${char}" escaped_name "${escaped_name}")
    endforeach(char)
    # ...add output dir
    if(output_dir)
      string(REGEX REPLACE "[^A-Za-z0-9_]" "_" escaped_name_clean "${escaped_name}")
      set(output_dir_arg "--out ${output_dir}/${output_prefix}${escaped_name_clean}${output_suffix}")
    endif()

    # ...and add to script
    prepare_command(add_test
      "${prefix}${plain_name}${suffix}"
      ${_TEST_EXECUTOR}
      "${_TEST_EXECUTABLE}"
      "${escaped_name}"
      ${extra_args}
      "${reporter_arg}"
      "${output_dir_arg}"
    )
    string(APPEND script "${_Command}")
    prepare_command(set_tests_properties
      "${prefix}${plain_name}${suffix}"
      PROPERTIES
      WORKING_DIRECTORY "${_TEST_WORKING_DIR}"
      ${properties}
    )
    string(APPEND script "${_Command}")

    if(add_tags)
      string(JSON num_tags LENGTH "${test_tags}")
      math(EXPR num_tags "${num_tags} - 1")
      set(tag_list "")
      if(num_tags GREATER_EQUAL "0")
        foreach(tag_idx RANGE ${num_tags})
          string(JSON a_tag GET "${test_tags}" "${tag_idx}")
          # Catch2's tags can contain semicolons, which are list element separators
          # in CMake, so we have to escape them. Ideally we could use the [=[...]=]
          # syntax for this, but CTest currently keeps the square quotes in the label
          # name. So we add 2 backslashes to escape it instead.
          # **IMPORTANT**: The number of backslashes depends on how many layers
          #                of CMake the tag goes. If this script is changed, the
          #                number of backslashes to escape may change as well.
          string(REPLACE ";" "\\;" a_tag "${a_tag}")
          list(APPEND tag_list "${a_tag}")
        endforeach()

        prepare_command(set_tests_properties
          "${prefix}${plain_name}${suffix}"
          PROPERTIES
          LABELS "${tag_list}"
        )
        string(APPEND script "${_Command}")
      endif()
    endif(add_tags)

    if(environment_modifications)
      prepare_command(set_tests_properties
        "${prefix}${plain_name}${suffix}"
        PROPERTIES
        ENVIRONMENT_MODIFICATION "${environment_modifications}")
      string(APPEND script "${_Command}")
    endif()

    list(APPEND tests "${prefix}${plain_name}${suffix}")
  endforeach()

  # Create a list of all discovered tests, which users may use to e.g. set
  # properties on the tests
  prepare_command(set ${_TEST_LIST} ${tests})
  string(APPEND script "${_Command}")

  # Write CTest script
  file(WRITE "${_CTEST_FILE}" "${script}")
endfunction()

# To enable `include`ing this file in the unit test scripts, we only run
# the impl if an actual `TEST_EXECUTABLE` is provided.
if(CMAKE_SCRIPT_MODE_FILE AND DEFINED TEST_EXECUTABLE)
  catch_discover_tests_impl(
    TEST_EXECUTABLE ${TEST_EXECUTABLE}
    TEST_EXECUTOR ${TEST_EXECUTOR}
    TEST_WORKING_DIR ${TEST_WORKING_DIR}
    TEST_SPEC ${TEST_SPEC}
    TEST_EXTRA_ARGS ${TEST_EXTRA_ARGS}
    TEST_PROPERTIES ${TEST_PROPERTIES}
    TEST_PREFIX ${TEST_PREFIX}
    TEST_SUFFIX ${TEST_SUFFIX}
    TEST_LIST ${TEST_LIST}
    TEST_REPORTER ${TEST_REPORTER}
    TEST_OUTPUT_DIR ${TEST_OUTPUT_DIR}
    TEST_OUTPUT_PREFIX ${TEST_OUTPUT_PREFIX}
    TEST_OUTPUT_SUFFIX ${TEST_OUTPUT_SUFFIX}
    TEST_DL_PATHS ${TEST_DL_PATHS}
    TEST_DL_FRAMEWORK_PATHS ${TEST_DL_FRAMEWORK_PATHS}
    CTEST_FILE ${CTEST_FILE}
    ADD_TAGS_AS_LABELS ${ADD_TAGS_AS_LABELS}
  )
endif()