COIN: Arm auto tests

Some of our auto tests fail, so add an EXCLUDE_FROM_PRECHECK option for
add_qtc_test, which sets a corresponding label on the ctest, and exclude
these when running ctest on COIN.

Disable the Googletest based unittests for now, they are inconsistently
failing on CI, e.g. sometimes timing out after 60 seconds for tests that
usually take only a fraction of a second.

tst_perfdata from the perfparser submodule also fails. Temporarily
disable it explicitly, until the EXCLUDE_FROM_PRECHECK change can be
merged there.

Change-Id: I7e9d7aded75bbe8800f82f7aa125c181271a2a1f
Reviewed-by: Qt CI Bot <qt_ci_bot@qt-project.org>
Reviewed-by: Jarek Kobus <jaroslaw.kobus@qt.io>
Reviewed-by: <github-actions-qt-creator@cristianadam.eu>

cmake/QtCreatorAPI.cmake

@@ -807,10 +807,10 @@ function(extend_qtc_executable name)
 endfunction()
 
 function(add_qtc_test name)
-  cmake_parse_arguments(_arg "GTEST;MANUALTEST" "TIMEOUT"
+  cmake_parse_arguments(_arg "GTEST;MANUALTEST;EXCLUDE_FROM_PRECHECK" "TIMEOUT"
       "DEFINES;DEPENDS;INCLUDES;SOURCES;EXPLICIT_MOC;SKIP_AUTOMOC;SKIP_PCH;CONDITION" ${ARGN})
 
-  if ($_arg_UNPARSED_ARGUMENTS)
+  if (${_arg_UNPARSED_ARGUMENTS})
     message(FATAL_ERROR "add_qtc_test had unparsed arguments!")
   endif()
 
@@ -871,6 +871,9 @@ function(add_qtc_test name)
 
   if (NOT _arg_GTEST AND NOT _arg_MANUALTEST)
     add_test(NAME ${name} COMMAND ${name})
+    if (_arg_EXCLUDE_FROM_PRECHECK)
+      set_tests_properties(${name} PROPERTIES LABELS exclude_from_precheck)
+    endif()
     if (DEFINED _arg_TIMEOUT)
       set(timeout_option TIMEOUT ${_arg_TIMEOUT})
     else()
@@ -880,17 +883,32 @@ function(add_qtc_test name)
   endif()
 endfunction()
 
-function(finalize_qtc_gtest test_name exclude_sources_regex)
+function(finalize_qtc_gtest test_name)
   if (NOT TARGET ${test_name})
     return()
   endif()
+
+  cmake_parse_arguments(_arg "EXCLUDE_ALL_FROM_PRECHECK" "EXCLUDE_SOURCES_REGEX"
+      "EXCLUDE_FROM_PRECHECK" ${ARGN})
+
+  if (${_arg_UNPARSED_ARGUMENTS})
+    message(FATAL_ERROR "finalize_qtc_gtest had unparsed arguments!")
+  endif()
+
   get_target_property(test_sources ${test_name} SOURCES)
-  if (exclude_sources_regex)
+  if (_arg_EXCLUDE_SOURCES_REGEX)
-    list(FILTER test_sources EXCLUDE REGEX "${exclude_sources_regex}")
+    list(FILTER test_sources EXCLUDE REGEX "${_arg_EXCLUDE_SOURCES_REGEX}")
   endif()
   include(GoogleTest)
   gtest_add_tests(TARGET ${test_name} SOURCES ${test_sources} TEST_LIST test_list SKIP_DEPENDENCY)
 
+  if(_arg_EXCLUDE_ALL_FROM_PRECHECK)
+    set_tests_properties(${test_list}
+      PROPERTIES LABELS exclude_from_precheck)
+  elseif(_arg_EXCLUDE_FROM_PRECHECK)
+    set_tests_properties(${_arg_EXCLUDE_FROM_PRECHECK}
+      PROPERTIES LABELS exclude_from_precheck)
+  endif()
   foreach(test IN LISTS test_list)
     finalize_test_setup(${test})
   endforeach()

coin/instructions/test.yaml

@@ -3,8 +3,7 @@ instructions:
   - type: ChangeDirectory
     directory: "{{.AgentWorkingDir}}/qt-creator/qt-creator_build/build"
   - type: ExecuteCommand
-    command: "ctest -j 4 --timeout 60 --output-on-failure"
+    command: "ctest -j 4 --timeout 60 --output-on-failure --label-exclude exclude_from_precheck --exclude-regex tst_perfdata"
     maxTimeInSeconds: 600
     maxTimeBetweenOutput: 600
     userMessageOnFailure: "Failed to run tests, check logs"
-    ignoreExitCode: true

src/plugins/mesonprojectmanager/CMakeLists.txt

@@ -91,6 +91,7 @@ file(RELATIVE_PATH TEST_RELATIVE_LIBEXEC_PATH "/${RELATIVE_TEST_PATH}" "/${IDE_L
 
 if(WITH_TESTS)
 add_qtc_test(tst_mesonwrapper
+  EXCLUDE_FROM_PRECHECK
   INCLUDES
     BEFORE "."
   DEPENDS
@@ -111,6 +112,7 @@ add_qtc_test(tst_mesonwrapper
 )
 
 add_qtc_test(tst_mesoninfoparser
+  EXCLUDE_FROM_PRECHECK
   INCLUDES
     BEFORE "."
   DEPENDS

tests/auto/debugger/CMakeLists.txt

@@ -51,6 +51,7 @@ if (WITH_DEBUGGER_DUMPERS)
   get_target_property(qmake_binary Qt5::qmake IMPORTED_LOCATION)
 
   add_qtc_test(tst_debugger_dumpers
+    EXCLUDE_FROM_PRECHECK
     TIMEOUT 0
     DEPENDS Qt5::Network Utils
     DEFINES

tests/auto/qml/codemodel/check/CMakeLists.txt

@@ -1,4 +1,5 @@
 add_qtc_test(tst_qml_check
+  EXCLUDE_FROM_PRECHECK
   DEPENDS QmlJS QmlJSTools ExtensionSystem Utils
   DEFINES
     QT_CREATOR

tests/auto/qml/codemodel/dependencies/CMakeLists.txt

@@ -1,4 +1,5 @@
 add_qtc_test(tst_qml_dependencies
+  EXCLUDE_FROM_PRECHECK
   DEPENDS QmlJS QmlJSTools ExtensionSystem Utils
   INCLUDES "${PROJECT_SOURCE_DIR}/src/plugins"
   DEFINES

tests/auto/qml/codemodel/importscheck/CMakeLists.txt

@@ -1,4 +1,5 @@
 add_qtc_test(tst_qml_importscheck
+  EXCLUDE_FROM_PRECHECK
   DEPENDS QmlJS QmlJSTools Utils CPlusPlus
   INCLUDES "${PROJECT_SOURCE_DIR}/src/plugins"
   DEFINES

tests/auto/qml/qmldesigner/coretests/CMakeLists.txt

@@ -1,4 +1,5 @@
 add_qtc_test(tst_qml_testcore
+  EXCLUDE_FROM_PRECHECK
   CONDITION TARGET QmlProjectManager
   DEFINES
     QT_CREATOR

tests/auto/tracing/flamegraphview/CMakeLists.txt

@@ -5,6 +5,7 @@ set(TSTFLAMEGRAPHVIEW_CPP_SOURCES
 
 if(${Qt5_VERSION} VERSION_LESS "6.2.0")
   add_qtc_test(tst_tracing_flamegraphview
+    EXCLUDE_FROM_PRECHECK
     DEPENDS Tracing Qt5::QuickWidgets Qt5::Quick Utils
     SOURCES
       ${TSTFLAMEGRAPHVIEW_CPP_SOURCES}
@@ -12,6 +13,7 @@ if(${Qt5_VERSION} VERSION_LESS "6.2.0")
   )
 else() # < Qt 6.2
   add_qtc_test(tst_tracing_flamegraphview
+    EXCLUDE_FROM_PRECHECK
     DEPENDS Tracing Qt5::QuickWidgets Qt5::Quick Utils
   )
 

tests/auto/tracing/timelinerenderer/CMakeLists.txt

@@ -1,4 +1,5 @@
 add_qtc_test(tst_tracing_timelinerenderer
+  EXCLUDE_FROM_PRECHECK
   DEPENDS Tracing Qt5::Gui Qt5::Quick
   SOURCES tst_timelinerenderer.cpp
 )

tests/auto/utils/qtcprocess/CMakeLists.txt

@@ -4,6 +4,7 @@ file(RELATIVE_PATH RELATIVE_TEST_PATH "${PROJECT_BINARY_DIR}" "${CMAKE_CURRENT_B
 file(RELATIVE_PATH TEST_RELATIVE_LIBEXEC_PATH "/${RELATIVE_TEST_PATH}" "/${IDE_LIBEXEC_PATH}")
 
 add_qtc_test(tst_qtcprocess
+  CONDITION Qt5_VERSION VERSION_GREATER_EQUAL 6.2.0
   TIMEOUT 30
   DEFINES "TEST_RELATIVE_LIBEXEC_PATH=\"${TEST_RELATIVE_LIBEXEC_PATH}\""
           "PROCESS_TESTAPP=\"${CMAKE_CURRENT_BINARY_DIR}/processtestapp\""

tests/unit/unittest/CMakeLists.txt

@@ -147,7 +147,9 @@ extend_qtc_test(unittest
     smallstring-benchmark.cpp
 )
 
-finalize_qtc_gtest(unittest ".c$")
+finalize_qtc_gtest(unittest
+  EXCLUDE_SOURCES_REGEX ".c$"
+  EXCLUDE_ALL_FROM_PRECHECK)
 
 # Path needs to be before CppEditor
 target_include_directories(unittest