AutoTest: Support some ctest settings

Add some useful settings for ctest based testing. Adapt output
parser accordingly to get at least relevant output and display
significant information inside the visual display.
Current implementation loses or mixes some information when
running jobs in parallel.
Complete output is still available inside the text display.

Fixes: QTCREATORBUG-26029
Change-Id: Id040f83900a6cfd89f5c2aff987278a297aa14c4
Reviewed-by: David Schulz <david.schulz@qt.io>
Reviewed-by: Christian Stenger <christian.stenger@qt.io>

doc/qtcreator/src/howto/creator-only/creator-autotest.qdoc

@@ -538,6 +538,34 @@
             statistical analysis and bootstrapping.
     \endlist
 
+    \section2 Specifying Settings for Running CTest-Based Tests
+    \list 1
+        \li To specify settings for running CTest-based tests, select
+        \uicontrol Tools > \uicontrol Options > \uicontrol {Testing} >
+        \uicontrol {CTest}.
+        //! insert image here
+        \li Select the \uicontrol {Output on failure} check box to show test
+            specific output if a test fails. Contrary to the CTest default
+            this is enabled by default.
+        \li Select \uicontrol {Schedule random} to execute the tests in
+            random order.
+        \li Select \uicontrol {Stop on failure} to automatically stop the
+            test execution on the first failing test.
+        \li In the \uicontrol {Output mode} field, select the verbosity level
+            of the CTest output.
+            \note This only influences the output on the text display.
+        \li Select \uicontrol {Repeat tests} if you want to re-run tests
+            under certain circumstances.
+        \li In the \uicontrol {Repetition mode} field, select the mode for
+            re-running tests. The maximum count for repeating a test can be
+            specified in the \uicontrol {Count} field.
+        \li Select \uicontrol {Run in parallel} to run the tests in parallel
+            using the specified number of \uicontrol {Jobs}.
+        \li Select \uicontrol {Test load} to be able to limit the parallel
+            execution. CTest will not start a new test if it would cause the
+            CPU load to pass the threshold given in \uicontrol {Threshold}.
+    \endlist
+
     \section1 Viewing Test Output
 
     The test results are displayed in the \uicontrol {Test Results} output pane

src/plugins/autotest/CMakeLists.txt

@@ -25,6 +25,7 @@ add_qtc_plugin(AutoTest
     catch/catchtestsettings.cpp catch/catchtestsettings.h
     ctest/ctestconfiguration.cpp ctest/ctestconfiguration.h
     ctest/ctestoutputreader.cpp ctest/ctestoutputreader.h
+    ctest/ctestsettings.cpp ctest/ctestsettings.h
     ctest/ctesttool.cpp ctest/ctesttool.h
     ctest/ctesttreeitem.cpp ctest/ctesttreeitem.h
     gtest/gtest_utils.cpp gtest/gtest_utils.h

src/plugins/autotest/autotest.pro

@@ -9,6 +9,7 @@ SOURCES += \
     autotestplugin.cpp \
     ctest/ctestconfiguration.cpp \
     ctest/ctestoutputreader.cpp \
+    ctest/ctestsettings.cpp \
     ctest/ctesttool.cpp \
     ctest/ctesttreeitem.cpp \
     itestframework.cpp \
@@ -80,6 +81,7 @@ HEADERS += \
     autotestplugin.h \
     ctest/ctestconfiguration.h \
     ctest/ctestoutputreader.h \
+    ctest/ctestsettings.h \
     ctest/ctesttool.h \
     ctest/ctesttreeitem.h \
     itemdatacache.h \

src/plugins/autotest/ctest/ctestoutputreader.cpp

@@ -79,9 +79,12 @@ CTestOutputReader::CTestOutputReader(const QFutureInterface<TestResultPtr> &futu
 
 void CTestOutputReader::processOutputLine(const QByteArray &outputLine)
 {
+    static const QRegularExpression verbose("^\\d+: .*$");
     static const QRegularExpression testProject("^Test project (.*)$");
-    static const QRegularExpression testCase("^(test \\d+)|(    Start\\s+\\d+: .*)$");
-    static const QRegularExpression testResult("^\\s*\\d+/\\d+ Test\\s+#\\d+: (.*) (\\.+)\\s*"
+    static const QRegularExpression testCase1("^test (?<current>\\d+)$");
+    static const QRegularExpression testCase2("^    Start\\s+(?<current>\\d+): (?<name>.*)$");
+    static const QRegularExpression testResult("^\\s*(?<first>\\d+/\\d+)? "
+                                               "Test\\s+#(?<current>\\d+): (.*) (\\.+)\\s*"
                                                "(Passed|\\*\\*\\*Failed|\\*\\*\\*Not Run|"
                                                ".*\\*\\*\\*Exception:.*)\\s+(.*) sec$");
     static const QRegularExpression testCrash("^\\s*\\d+/\\d+ Test\\s+#\\d+: (.*) (\\.+)\\s*"
@@ -100,7 +103,9 @@ void CTestOutputReader::processOutputLine(const QByteArray &outputLine)
         operator bool() const { return hasMatch(); }
     };
 
-    if (ExactMatch match = testProject.match(line)) {
+    if (ExactMatch match = verbose.match(line)) { // ignore verbose output for visual display
+        return;
+    } else if (ExactMatch match = testProject.match(line)) {
         if (!m_testName.isEmpty()) // possible?
             sendCompleteInformation();
         m_project = match.captured(1);
@@ -108,13 +113,27 @@ void CTestOutputReader::processOutputLine(const QByteArray &outputLine)
         testResult->setResult(ResultType::TestStart);
         testResult->setDescription(tr("Running tests for %1").arg(m_project));
         reportResult(testResult);
-    } else if (ExactMatch match = testCase.match(line)) {
-        if (!m_testName.isEmpty())
+    } else if (ExactMatch match = testCase1.match(line)) {
+        int current = match.captured("current").toInt();
+        if (m_result != ResultType::Invalid && m_currentTestNo != -1 && current != m_currentTestNo)
             sendCompleteInformation();
+        m_currentTestNo = -1;
+    } else if (ExactMatch match = testCase2.match(line)) {
+        int current = match.captured("current").toInt();
+        if (m_result != ResultType::Invalid && m_currentTestNo != -1 && current != m_currentTestNo)
+            sendCompleteInformation();
+        m_currentTestNo = -1;
     } else if (ExactMatch match = testResult.match(line)) {
-        m_description = match.captured();
-        m_testName = match.captured(1);
-        const QString resultType = match.captured(3);
+        int current = match.captured("current").toInt();
+        if (m_result != ResultType::Invalid && m_currentTestNo != -1 && current != m_currentTestNo)
+            sendCompleteInformation();
+        if (!m_description.isEmpty() && match.captured("first").isEmpty())
+            m_description.append('\n').append(match.captured());
+        else
+            m_description = match.captured();
+        m_currentTestNo = current;
+        m_testName = match.captured(3);
+        const QString resultType = match.captured(5);
         if (resultType == "Passed")
             m_result = ResultType::Pass;
         else if (resultType == "***Failed" || resultType == "***Not Run")
@@ -162,12 +181,19 @@ TestResultPtr CTestOutputReader::createDefaultResult() const
 
 void CTestOutputReader::sendCompleteInformation()
 {
+    // some verbose output we did not ignore
+    if (m_result == ResultType::Invalid) {
+        QTC_CHECK(m_currentTestNo == -1 && m_testName.isEmpty());
+        return;
+    }
+
     TestResultPtr testResult = createDefaultResult();
     testResult->setResult(m_result);
     testResult->setDescription(m_description);
     reportResult(testResult);
     m_testName.clear();
     m_description.clear();
+    m_currentTestNo = -1;
     m_result = ResultType::Invalid;
 }
 

src/plugins/autotest/ctest/ctestoutputreader.h

@@ -41,7 +41,9 @@ public:
 protected:
     void processOutputLine(const QByteArray &outputLineWithNewLine) final;
     TestResultPtr createDefaultResult() const final;
+private:
     void sendCompleteInformation();
+    int m_currentTestNo = -1;
     QString m_project;
     QString m_testName;
     QString m_description;

src/plugins/autotest/ctest/ctestsettings.cpp

@@ -0,0 +1,173 @@
+/****************************************************************************
+**
+** Copyright (C) 2021 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of Qt Creator.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+****************************************************************************/
+
+#include "ctestsettings.h"
+
+#include "../autotestconstants.h"
+
+#include <utils/layoutbuilder.h>
+
+namespace Autotest {
+namespace Internal {
+
+CTestSettings::CTestSettings()
+{
+    setSettingsGroups("Autotest", "CTest");
+    setAutoApply(false);
+
+    registerAspect(&outputOnFail);
+    outputOnFail.setSettingsKey("OutputOnFail");
+    outputOnFail.setLabelText(tr("Output on failure"));
+    outputOnFail.setDefaultValue(true);
+
+    registerAspect(&outputMode);
+    outputMode.setSettingsKey("OutputMode");
+    outputMode.setLabelText(tr("Output mode"));
+    outputMode.setDisplayStyle(Utils::SelectionAspect::DisplayStyle::ComboBox);
+    outputMode.addOption({tr("Default"), {}, 0});
+    outputMode.addOption({tr("Verbose"), {}, 1});
+    outputMode.addOption({tr("Very Verbose"), {}, 2});
+
+    registerAspect(&repetitionMode);
+    repetitionMode.setSettingsKey("RepetitionMode");
+    repetitionMode.setLabelText(tr("Repetition mode"));
+    repetitionMode.setDisplayStyle(Utils::SelectionAspect::DisplayStyle::ComboBox);
+    repetitionMode.addOption({tr("Until Fail"), {}, 0});
+    repetitionMode.addOption({tr("Until Pass"), {}, 1});
+    repetitionMode.addOption({tr("After Timeout"), {}, 2});
+
+    registerAspect(&repetitionCount);
+    repetitionCount.setSettingsKey("RepetitionCount");
+    repetitionCount.setDefaultValue(1);
+    repetitionCount.setLabelText(tr("Count"));
+    repetitionCount.setToolTip(tr("Number of re-runs for the test."));
+    repetitionCount.setRange(1, 10000);
+
+    registerAspect(&repeat);
+    repeat.setSettingsKey("Repeat");
+
+    registerAspect(&scheduleRandom);
+    scheduleRandom.setSettingsKey("ScheduleRandom");
+    scheduleRandom.setLabelText(tr("Schedule random"));
+
+    registerAspect(&stopOnFailure);
+    stopOnFailure.setSettingsKey("StopOnFail");
+    stopOnFailure.setLabelText(tr("Stop on failure"));
+
+    registerAspect(&parallel);
+    parallel.setSettingsKey("Parallel");
+    parallel.setToolTip(tr("Run tests in parallel mode using given number of jobs."));
+
+    registerAspect(&jobs);
+    jobs.setSettingsKey("Jobs");
+    jobs.setLabelText(tr("Jobs"));
+    jobs.setDefaultValue(1);
+    jobs.setRange(1, 128);
+
+    registerAspect(&testLoad);
+    testLoad.setSettingsKey("TestLoad");
+    testLoad.setLabelText(tr("Test load"));
+    testLoad.setToolTip(tr("Try not to start tests when they may cause CPU load to pass a "
+                           "threshold."));
+
+    registerAspect(&threshold);
+    threshold.setSettingsKey("Threshold");
+    threshold.setLabelText(tr("Threshold"));
+    threshold.setDefaultValue(1);
+    threshold.setRange(1, 128);
+    threshold.setEnabler(&testLoad);
+}
+
+QStringList CTestSettings::activeSettingsAsOptions() const
+{
+    QStringList options;
+    if (outputOnFail.value())
+        options << "--output-on-failure";
+    switch (outputMode.value()) {
+    case 1: options << "-V"; break;
+    case 2: options << "-VV"; break;
+    default: break;
+    }
+    if (repeat.value()) {
+        QString repeatOption;
+        switch (repetitionMode.value()) {
+        case 0: repeatOption = "until-fail"; break;
+        case 1: repeatOption = "until-pass"; break;
+        case 2: repeatOption = "after-timeout"; break;
+        default: break;
+        }
+        if (!repeatOption.isEmpty()) {
+            repeatOption.append(':');
+            repeatOption.append(QString::number(repetitionCount.value()));
+            options << "--repeat" << repeatOption;
+        }
+    }
+    if (scheduleRandom.value())
+        options << "--schedule-random";
+    if (stopOnFailure.value())
+        options << "--stop-on-failure";
+    if (parallel.value()) {
+        options << "-j" << QString::number(jobs.value());
+        if (testLoad.value())
+            options << "--test-load" << QString::number(threshold.value());
+    }
+    return options;
+}
+
+CTestSettingsPage::CTestSettingsPage(CTestSettings *settings, Utils::Id settingsId)
+{
+    setId(settingsId);
+    setCategory(Constants::AUTOTEST_SETTINGS_CATEGORY);
+    setDisplayName(tr("CTest"));
+
+    setSettings(settings);
+
+    setLayouter([settings](QWidget *widget) {
+        CTestSettings &s = *settings;
+        using namespace Utils::Layouting;
+        const Break nl;
+
+        Form form {
+            Row {s.outputOnFail}, nl,
+            Row {s.scheduleRandom}, nl,
+            Row {s.stopOnFailure}, nl,
+            Row {s.outputMode}, nl,
+            Group {
+                Title(tr("Repeat tests"), &s.repeat), nl,
+                Row {s.repetitionMode, s.repetitionCount},
+            }, nl,
+            Group {
+                Title(tr("Run in parallel"), &s.parallel), nl,
+                Row {s.jobs}, nl,
+                Row {s.testLoad, s.threshold}
+            }
+        };
+
+        Column {Row {Column { form , Stretch() }, Stretch() } }.attachTo(widget);
+    });
+}
+
+} // namespace Internal
+} // namespace Autotest

src/plugins/autotest/ctest/ctestsettings.h

@@ -0,0 +1,66 @@
+/****************************************************************************
+**
+** Copyright (C) 2021 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of Qt Creator.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+****************************************************************************/
+
+#pragma once
+
+#include <coreplugin/dialogs/ioptionspage.h>
+
+#include <utils/aspects.h>
+
+namespace Autotest {
+namespace Internal {
+
+class CTestSettings : public Utils::AspectContainer
+{
+    Q_DECLARE_TR_FUNCTIONS(Autotest::Internal::CTestSettings)
+public:
+    CTestSettings();
+
+    QStringList activeSettingsAsOptions() const;
+
+    Utils::IntegerAspect repetitionCount;
+    Utils::SelectionAspect repetitionMode;
+    Utils::SelectionAspect outputMode;
+    Utils::BoolAspect outputOnFail;
+    Utils::BoolAspect stopOnFailure;
+    Utils::BoolAspect scheduleRandom;
+    Utils::BoolAspect repeat;
+    // FIXME.. this makes the outputreader fail to get all results correctly for visual display
+    Utils::BoolAspect parallel;
+    Utils::IntegerAspect jobs;
+    Utils::BoolAspect testLoad;
+    Utils::IntegerAspect threshold;
+};
+
+class CTestSettingsPage final : public Core::IOptionsPage
+{
+    Q_DECLARE_TR_FUNCTIONS(Autotest::Internal::CTestSettingsPage)
+public:
+    CTestSettingsPage(CTestSettings *settings, Utils::Id settingsId);
+};
+
+} // namespace Internal
+} // namespace Autotest
+

src/plugins/autotest/ctest/ctesttool.h

@@ -26,6 +26,7 @@
 #pragma once
 
 #include "../itestframework.h"
+#include "ctestsettings.h"
 
 namespace Autotest {
 namespace Internal {
@@ -42,6 +43,12 @@ public:
 protected:
     const char *name() const final;
     ITestTreeItem *createRootNode() final;
+
+private:
+    ITestSettings *testSettings() override { return &m_settings; }
+
+    CTestSettings m_settings;
+    CTestSettingsPage m_settingsPage{&m_settings, settingsId()};
 };
 
 } // namespace Internal

src/plugins/autotest/ctest/ctesttreeitem.cpp

@@ -26,6 +26,7 @@
 #include "ctesttreeitem.h"
 
 #include "ctestconfiguration.h"
+#include "ctestsettings.h"
 
 #include "../autotestplugin.h"
 #include "../itestframework.h"
@@ -107,8 +108,8 @@ QList<ITestConfiguration *> CTestTreeItem::testConfigurationsFor(const QStringLi
 
     const ProjectExplorer::BuildSystem *buildSystem = target->buildSystem();
     QStringList options{"--timeout", QString::number(AutotestPlugin::settings()->timeout / 1000)};
-    // TODO add ctest options from settings?
-    options << "--output-on-failure";
+    auto ctestSettings = static_cast<CTestSettings *>(testBase()->testSettings());
+    options << ctestSettings->activeSettingsAsOptions();
     Utils::CommandLine command = buildSystem->commandLineForTests(selected, options);
     if (command.executable().isEmpty())
         return {};

src/plugins/autotest/testframeworkmanager.cpp

@@ -125,6 +125,10 @@ void TestFrameworkManager::synchronizeSettings(QSettings *s)
         if (ITestSettings *fSettings = framework->testSettings())
             fSettings->readSettings(s);
     }
+    for (ITestTool *testTool : qAsConst(m_registeredTestTools)) {
+        if (ITestSettings *tSettings = testTool->testSettings())
+            tSettings->readSettings(s);
+    }
 }
 
 } // namespace Autotest

src/plugins/autotest/testresultmodel.cpp

@@ -129,7 +129,7 @@ static bool isSignificant(ResultType type)
     case ResultType::MessageCurrentTest:
     case ResultType::Application:
     case ResultType::Invalid:
-        QTC_ASSERT_STRING("Got unexpedted type in isSignificant check");
+        QTC_ASSERT_STRING("Got unexpected type in isSignificant check");
         return false;
     default:
         return true;