From d059a610b1243f99984a15fdda38a8e3db048708 Mon Sep 17 00:00:00 2001 From: Eike Ziller Date: Fri, 24 Jun 2022 09:57:23 +0200 Subject: [PATCH] COIN: Arm auto tests Some of our auto tests fail, so add an EXCLUDE_FROM_PRECHECK option for add_qtc_test, which sets a corresponding label on the ctest, and exclude these when running ctest on COIN. Disable the Googletest based unittests for now, they are inconsistently failing on CI, e.g. sometimes timing out after 60 seconds for tests that usually take only a fraction of a second. tst_perfdata from the perfparser submodule also fails. Temporarily disable it explicitly, until the EXCLUDE_FROM_PRECHECK change can be merged there. Change-Id: I7e9d7aded75bbe8800f82f7aa125c181271a2a1f Reviewed-by: Qt CI Bot Reviewed-by: Jarek Kobus Reviewed-by: --- cmake/QtCreatorAPI.cmake | 28 +++++++++++++++---- coin/instructions/test.yaml | 3 +- .../mesonprojectmanager/CMakeLists.txt | 2 ++ tests/auto/debugger/CMakeLists.txt | 1 + tests/auto/qml/codemodel/check/CMakeLists.txt | 1 + .../qml/codemodel/dependencies/CMakeLists.txt | 1 + .../qml/codemodel/importscheck/CMakeLists.txt | 1 + .../qml/qmldesigner/coretests/CMakeLists.txt | 1 + .../tracing/flamegraphview/CMakeLists.txt | 2 ++ .../tracing/timelinerenderer/CMakeLists.txt | 1 + tests/auto/utils/qtcprocess/CMakeLists.txt | 1 + tests/unit/unittest/CMakeLists.txt | 4 ++- 12 files changed, 38 insertions(+), 8 deletions(-) diff --git a/cmake/QtCreatorAPI.cmake b/cmake/QtCreatorAPI.cmake index d61d4d4c43e..257d351426b 100644 --- a/cmake/QtCreatorAPI.cmake +++ b/cmake/QtCreatorAPI.cmake @@ -807,10 +807,10 @@ function(extend_qtc_executable name) endfunction() function(add_qtc_test name) - cmake_parse_arguments(_arg "GTEST;MANUALTEST" "TIMEOUT" + cmake_parse_arguments(_arg "GTEST;MANUALTEST;EXCLUDE_FROM_PRECHECK" "TIMEOUT" "DEFINES;DEPENDS;INCLUDES;SOURCES;EXPLICIT_MOC;SKIP_AUTOMOC;SKIP_PCH;CONDITION" ${ARGN}) - if ($_arg_UNPARSED_ARGUMENTS) + if (${_arg_UNPARSED_ARGUMENTS}) message(FATAL_ERROR "add_qtc_test had unparsed arguments!") endif() @@ -871,6 +871,9 @@ function(add_qtc_test name) if (NOT _arg_GTEST AND NOT _arg_MANUALTEST) add_test(NAME ${name} COMMAND ${name}) + if (_arg_EXCLUDE_FROM_PRECHECK) + set_tests_properties(${name} PROPERTIES LABELS exclude_from_precheck) + endif() if (DEFINED _arg_TIMEOUT) set(timeout_option TIMEOUT ${_arg_TIMEOUT}) else() @@ -880,17 +883,32 @@ function(add_qtc_test name) endif() endfunction() -function(finalize_qtc_gtest test_name exclude_sources_regex) +function(finalize_qtc_gtest test_name) if (NOT TARGET ${test_name}) return() endif() + + cmake_parse_arguments(_arg "EXCLUDE_ALL_FROM_PRECHECK" "EXCLUDE_SOURCES_REGEX" + "EXCLUDE_FROM_PRECHECK" ${ARGN}) + + if (${_arg_UNPARSED_ARGUMENTS}) + message(FATAL_ERROR "finalize_qtc_gtest had unparsed arguments!") + endif() + get_target_property(test_sources ${test_name} SOURCES) - if (exclude_sources_regex) - list(FILTER test_sources EXCLUDE REGEX "${exclude_sources_regex}") + if (_arg_EXCLUDE_SOURCES_REGEX) + list(FILTER test_sources EXCLUDE REGEX "${_arg_EXCLUDE_SOURCES_REGEX}") endif() include(GoogleTest) gtest_add_tests(TARGET ${test_name} SOURCES ${test_sources} TEST_LIST test_list SKIP_DEPENDENCY) + if(_arg_EXCLUDE_ALL_FROM_PRECHECK) + set_tests_properties(${test_list} + PROPERTIES LABELS exclude_from_precheck) + elseif(_arg_EXCLUDE_FROM_PRECHECK) + set_tests_properties(${_arg_EXCLUDE_FROM_PRECHECK} + PROPERTIES LABELS exclude_from_precheck) + endif() foreach(test IN LISTS test_list) finalize_test_setup(${test}) endforeach() diff --git a/coin/instructions/test.yaml b/coin/instructions/test.yaml index f40036cf611..7786d75f5ed 100644 --- a/coin/instructions/test.yaml +++ b/coin/instructions/test.yaml @@ -3,8 +3,7 @@ instructions: - type: ChangeDirectory directory: "{{.AgentWorkingDir}}/qt-creator/qt-creator_build/build" - type: ExecuteCommand - command: "ctest -j 4 --timeout 60 --output-on-failure" + command: "ctest -j 4 --timeout 60 --output-on-failure --label-exclude exclude_from_precheck --exclude-regex tst_perfdata" maxTimeInSeconds: 600 maxTimeBetweenOutput: 600 userMessageOnFailure: "Failed to run tests, check logs" - ignoreExitCode: true diff --git a/src/plugins/mesonprojectmanager/CMakeLists.txt b/src/plugins/mesonprojectmanager/CMakeLists.txt index ea66df93f88..cea74e07510 100644 --- a/src/plugins/mesonprojectmanager/CMakeLists.txt +++ b/src/plugins/mesonprojectmanager/CMakeLists.txt @@ -91,6 +91,7 @@ file(RELATIVE_PATH TEST_RELATIVE_LIBEXEC_PATH "/${RELATIVE_TEST_PATH}" "/${IDE_L if(WITH_TESTS) add_qtc_test(tst_mesonwrapper + EXCLUDE_FROM_PRECHECK INCLUDES BEFORE "." DEPENDS @@ -111,6 +112,7 @@ add_qtc_test(tst_mesonwrapper ) add_qtc_test(tst_mesoninfoparser + EXCLUDE_FROM_PRECHECK INCLUDES BEFORE "." DEPENDS diff --git a/tests/auto/debugger/CMakeLists.txt b/tests/auto/debugger/CMakeLists.txt index 5ab4c63f244..8dbf475ca6c 100644 --- a/tests/auto/debugger/CMakeLists.txt +++ b/tests/auto/debugger/CMakeLists.txt @@ -51,6 +51,7 @@ if (WITH_DEBUGGER_DUMPERS) get_target_property(qmake_binary Qt5::qmake IMPORTED_LOCATION) add_qtc_test(tst_debugger_dumpers + EXCLUDE_FROM_PRECHECK TIMEOUT 0 DEPENDS Qt5::Network Utils DEFINES diff --git a/tests/auto/qml/codemodel/check/CMakeLists.txt b/tests/auto/qml/codemodel/check/CMakeLists.txt index 462c2765171..ad78f01f5b7 100644 --- a/tests/auto/qml/codemodel/check/CMakeLists.txt +++ b/tests/auto/qml/codemodel/check/CMakeLists.txt @@ -1,4 +1,5 @@ add_qtc_test(tst_qml_check + EXCLUDE_FROM_PRECHECK DEPENDS QmlJS QmlJSTools ExtensionSystem Utils DEFINES QT_CREATOR diff --git a/tests/auto/qml/codemodel/dependencies/CMakeLists.txt b/tests/auto/qml/codemodel/dependencies/CMakeLists.txt index 43e915176a0..606663fb2c7 100644 --- a/tests/auto/qml/codemodel/dependencies/CMakeLists.txt +++ b/tests/auto/qml/codemodel/dependencies/CMakeLists.txt @@ -1,4 +1,5 @@ add_qtc_test(tst_qml_dependencies + EXCLUDE_FROM_PRECHECK DEPENDS QmlJS QmlJSTools ExtensionSystem Utils INCLUDES "${PROJECT_SOURCE_DIR}/src/plugins" DEFINES diff --git a/tests/auto/qml/codemodel/importscheck/CMakeLists.txt b/tests/auto/qml/codemodel/importscheck/CMakeLists.txt index a7c10f9d5e2..60a22d77696 100644 --- a/tests/auto/qml/codemodel/importscheck/CMakeLists.txt +++ b/tests/auto/qml/codemodel/importscheck/CMakeLists.txt @@ -1,4 +1,5 @@ add_qtc_test(tst_qml_importscheck + EXCLUDE_FROM_PRECHECK DEPENDS QmlJS QmlJSTools Utils CPlusPlus INCLUDES "${PROJECT_SOURCE_DIR}/src/plugins" DEFINES diff --git a/tests/auto/qml/qmldesigner/coretests/CMakeLists.txt b/tests/auto/qml/qmldesigner/coretests/CMakeLists.txt index 253c91e4b72..349a721d9e1 100644 --- a/tests/auto/qml/qmldesigner/coretests/CMakeLists.txt +++ b/tests/auto/qml/qmldesigner/coretests/CMakeLists.txt @@ -1,4 +1,5 @@ add_qtc_test(tst_qml_testcore + EXCLUDE_FROM_PRECHECK CONDITION TARGET QmlProjectManager DEFINES QT_CREATOR diff --git a/tests/auto/tracing/flamegraphview/CMakeLists.txt b/tests/auto/tracing/flamegraphview/CMakeLists.txt index 23f73fde899..e34941bd636 100644 --- a/tests/auto/tracing/flamegraphview/CMakeLists.txt +++ b/tests/auto/tracing/flamegraphview/CMakeLists.txt @@ -5,6 +5,7 @@ set(TSTFLAMEGRAPHVIEW_CPP_SOURCES if(${Qt5_VERSION} VERSION_LESS "6.2.0") add_qtc_test(tst_tracing_flamegraphview + EXCLUDE_FROM_PRECHECK DEPENDS Tracing Qt5::QuickWidgets Qt5::Quick Utils SOURCES ${TSTFLAMEGRAPHVIEW_CPP_SOURCES} @@ -12,6 +13,7 @@ if(${Qt5_VERSION} VERSION_LESS "6.2.0") ) else() # < Qt 6.2 add_qtc_test(tst_tracing_flamegraphview + EXCLUDE_FROM_PRECHECK DEPENDS Tracing Qt5::QuickWidgets Qt5::Quick Utils ) diff --git a/tests/auto/tracing/timelinerenderer/CMakeLists.txt b/tests/auto/tracing/timelinerenderer/CMakeLists.txt index 30e44a5cd20..db71a10c4a1 100644 --- a/tests/auto/tracing/timelinerenderer/CMakeLists.txt +++ b/tests/auto/tracing/timelinerenderer/CMakeLists.txt @@ -1,4 +1,5 @@ add_qtc_test(tst_tracing_timelinerenderer + EXCLUDE_FROM_PRECHECK DEPENDS Tracing Qt5::Gui Qt5::Quick SOURCES tst_timelinerenderer.cpp ) diff --git a/tests/auto/utils/qtcprocess/CMakeLists.txt b/tests/auto/utils/qtcprocess/CMakeLists.txt index c03eee4945b..4040548b334 100644 --- a/tests/auto/utils/qtcprocess/CMakeLists.txt +++ b/tests/auto/utils/qtcprocess/CMakeLists.txt @@ -4,6 +4,7 @@ file(RELATIVE_PATH RELATIVE_TEST_PATH "${PROJECT_BINARY_DIR}" "${CMAKE_CURRENT_B file(RELATIVE_PATH TEST_RELATIVE_LIBEXEC_PATH "/${RELATIVE_TEST_PATH}" "/${IDE_LIBEXEC_PATH}") add_qtc_test(tst_qtcprocess + CONDITION Qt5_VERSION VERSION_GREATER_EQUAL 6.2.0 TIMEOUT 30 DEFINES "TEST_RELATIVE_LIBEXEC_PATH=\"${TEST_RELATIVE_LIBEXEC_PATH}\"" "PROCESS_TESTAPP=\"${CMAKE_CURRENT_BINARY_DIR}/processtestapp\"" diff --git a/tests/unit/unittest/CMakeLists.txt b/tests/unit/unittest/CMakeLists.txt index a0b39262ef2..70cff54788a 100644 --- a/tests/unit/unittest/CMakeLists.txt +++ b/tests/unit/unittest/CMakeLists.txt @@ -147,7 +147,9 @@ extend_qtc_test(unittest smallstring-benchmark.cpp ) -finalize_qtc_gtest(unittest ".c$") +finalize_qtc_gtest(unittest + EXCLUDE_SOURCES_REGEX ".c$" + EXCLUDE_ALL_FROM_PRECHECK) # Path needs to be before CppEditor target_include_directories(unittest