diff --git a/doc/qtcreator/src/howto/creator-only/creator-autotest.qdoc b/doc/qtcreator/src/howto/creator-only/creator-autotest.qdoc index cae258a28bd..22d6082d95e 100644 --- a/doc/qtcreator/src/howto/creator-only/creator-autotest.qdoc +++ b/doc/qtcreator/src/howto/creator-only/creator-autotest.qdoc @@ -538,6 +538,34 @@ statistical analysis and bootstrapping. \endlist + \section2 Specifying Settings for Running CTest-Based Tests + \list 1 + \li To specify settings for running CTest-based tests, select + \uicontrol Tools > \uicontrol Options > \uicontrol {Testing} > + \uicontrol {CTest}. + //! insert image here + \li Select the \uicontrol {Output on failure} check box to show test + specific output if a test fails. Contrary to the CTest default + this is enabled by default. + \li Select \uicontrol {Schedule random} to execute the tests in + random order. + \li Select \uicontrol {Stop on failure} to automatically stop the + test execution on the first failing test. + \li In the \uicontrol {Output mode} field, select the verbosity level + of the CTest output. + \note This only influences the output on the text display. + \li Select \uicontrol {Repeat tests} if you want to re-run tests + under certain circumstances. + \li In the \uicontrol {Repetition mode} field, select the mode for + re-running tests. The maximum count for repeating a test can be + specified in the \uicontrol {Count} field. + \li Select \uicontrol {Run in parallel} to run the tests in parallel + using the specified number of \uicontrol {Jobs}. + \li Select \uicontrol {Test load} to be able to limit the parallel + execution. CTest will not start a new test if it would cause the + CPU load to pass the threshold given in \uicontrol {Threshold}. + \endlist + \section1 Viewing Test Output The test results are displayed in the \uicontrol {Test Results} output pane diff --git a/src/plugins/autotest/CMakeLists.txt b/src/plugins/autotest/CMakeLists.txt index 64c2392d382..881b5d2bc85 100644 --- a/src/plugins/autotest/CMakeLists.txt +++ b/src/plugins/autotest/CMakeLists.txt @@ -25,6 +25,7 @@ add_qtc_plugin(AutoTest catch/catchtestsettings.cpp catch/catchtestsettings.h ctest/ctestconfiguration.cpp ctest/ctestconfiguration.h ctest/ctestoutputreader.cpp ctest/ctestoutputreader.h + ctest/ctestsettings.cpp ctest/ctestsettings.h ctest/ctesttool.cpp ctest/ctesttool.h ctest/ctesttreeitem.cpp ctest/ctesttreeitem.h gtest/gtest_utils.cpp gtest/gtest_utils.h diff --git a/src/plugins/autotest/autotest.pro b/src/plugins/autotest/autotest.pro index d7f708d63c8..2e0d2ff9055 100644 --- a/src/plugins/autotest/autotest.pro +++ b/src/plugins/autotest/autotest.pro @@ -9,6 +9,7 @@ SOURCES += \ autotestplugin.cpp \ ctest/ctestconfiguration.cpp \ ctest/ctestoutputreader.cpp \ + ctest/ctestsettings.cpp \ ctest/ctesttool.cpp \ ctest/ctesttreeitem.cpp \ itestframework.cpp \ @@ -80,6 +81,7 @@ HEADERS += \ autotestplugin.h \ ctest/ctestconfiguration.h \ ctest/ctestoutputreader.h \ + ctest/ctestsettings.h \ ctest/ctesttool.h \ ctest/ctesttreeitem.h \ itemdatacache.h \ diff --git a/src/plugins/autotest/ctest/ctestoutputreader.cpp b/src/plugins/autotest/ctest/ctestoutputreader.cpp index 31b41b41349..93315c0bea6 100644 --- a/src/plugins/autotest/ctest/ctestoutputreader.cpp +++ b/src/plugins/autotest/ctest/ctestoutputreader.cpp @@ -79,9 +79,12 @@ CTestOutputReader::CTestOutputReader(const QFutureInterface &futu void CTestOutputReader::processOutputLine(const QByteArray &outputLine) { + static const QRegularExpression verbose("^\\d+: .*$"); static const QRegularExpression testProject("^Test project (.*)$"); - static const QRegularExpression testCase("^(test \\d+)|( Start\\s+\\d+: .*)$"); - static const QRegularExpression testResult("^\\s*\\d+/\\d+ Test\\s+#\\d+: (.*) (\\.+)\\s*" + static const QRegularExpression testCase1("^test (?\\d+)$"); + static const QRegularExpression testCase2("^ Start\\s+(?\\d+): (?.*)$"); + static const QRegularExpression testResult("^\\s*(?\\d+/\\d+)? " + "Test\\s+#(?\\d+): (.*) (\\.+)\\s*" "(Passed|\\*\\*\\*Failed|\\*\\*\\*Not Run|" ".*\\*\\*\\*Exception:.*)\\s+(.*) sec$"); static const QRegularExpression testCrash("^\\s*\\d+/\\d+ Test\\s+#\\d+: (.*) (\\.+)\\s*" @@ -100,7 +103,9 @@ void CTestOutputReader::processOutputLine(const QByteArray &outputLine) operator bool() const { return hasMatch(); } }; - if (ExactMatch match = testProject.match(line)) { + if (ExactMatch match = verbose.match(line)) { // ignore verbose output for visual display + return; + } else if (ExactMatch match = testProject.match(line)) { if (!m_testName.isEmpty()) // possible? sendCompleteInformation(); m_project = match.captured(1); @@ -108,13 +113,27 @@ void CTestOutputReader::processOutputLine(const QByteArray &outputLine) testResult->setResult(ResultType::TestStart); testResult->setDescription(tr("Running tests for %1").arg(m_project)); reportResult(testResult); - } else if (ExactMatch match = testCase.match(line)) { - if (!m_testName.isEmpty()) + } else if (ExactMatch match = testCase1.match(line)) { + int current = match.captured("current").toInt(); + if (m_result != ResultType::Invalid && m_currentTestNo != -1 && current != m_currentTestNo) sendCompleteInformation(); + m_currentTestNo = -1; + } else if (ExactMatch match = testCase2.match(line)) { + int current = match.captured("current").toInt(); + if (m_result != ResultType::Invalid && m_currentTestNo != -1 && current != m_currentTestNo) + sendCompleteInformation(); + m_currentTestNo = -1; } else if (ExactMatch match = testResult.match(line)) { - m_description = match.captured(); - m_testName = match.captured(1); - const QString resultType = match.captured(3); + int current = match.captured("current").toInt(); + if (m_result != ResultType::Invalid && m_currentTestNo != -1 && current != m_currentTestNo) + sendCompleteInformation(); + if (!m_description.isEmpty() && match.captured("first").isEmpty()) + m_description.append('\n').append(match.captured()); + else + m_description = match.captured(); + m_currentTestNo = current; + m_testName = match.captured(3); + const QString resultType = match.captured(5); if (resultType == "Passed") m_result = ResultType::Pass; else if (resultType == "***Failed" || resultType == "***Not Run") @@ -162,12 +181,19 @@ TestResultPtr CTestOutputReader::createDefaultResult() const void CTestOutputReader::sendCompleteInformation() { + // some verbose output we did not ignore + if (m_result == ResultType::Invalid) { + QTC_CHECK(m_currentTestNo == -1 && m_testName.isEmpty()); + return; + } + TestResultPtr testResult = createDefaultResult(); testResult->setResult(m_result); testResult->setDescription(m_description); reportResult(testResult); m_testName.clear(); m_description.clear(); + m_currentTestNo = -1; m_result = ResultType::Invalid; } diff --git a/src/plugins/autotest/ctest/ctestoutputreader.h b/src/plugins/autotest/ctest/ctestoutputreader.h index baff9a9ca17..bbec3f70c77 100644 --- a/src/plugins/autotest/ctest/ctestoutputreader.h +++ b/src/plugins/autotest/ctest/ctestoutputreader.h @@ -41,7 +41,9 @@ public: protected: void processOutputLine(const QByteArray &outputLineWithNewLine) final; TestResultPtr createDefaultResult() const final; +private: void sendCompleteInformation(); + int m_currentTestNo = -1; QString m_project; QString m_testName; QString m_description; diff --git a/src/plugins/autotest/ctest/ctestsettings.cpp b/src/plugins/autotest/ctest/ctestsettings.cpp new file mode 100644 index 00000000000..4c8bfb643d8 --- /dev/null +++ b/src/plugins/autotest/ctest/ctestsettings.cpp @@ -0,0 +1,173 @@ +/**************************************************************************** +** +** Copyright (C) 2021 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of Qt Creator. +** +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 3 as published by the Free Software +** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-3.0.html. +** +****************************************************************************/ + +#include "ctestsettings.h" + +#include "../autotestconstants.h" + +#include + +namespace Autotest { +namespace Internal { + +CTestSettings::CTestSettings() +{ + setSettingsGroups("Autotest", "CTest"); + setAutoApply(false); + + registerAspect(&outputOnFail); + outputOnFail.setSettingsKey("OutputOnFail"); + outputOnFail.setLabelText(tr("Output on failure")); + outputOnFail.setDefaultValue(true); + + registerAspect(&outputMode); + outputMode.setSettingsKey("OutputMode"); + outputMode.setLabelText(tr("Output mode")); + outputMode.setDisplayStyle(Utils::SelectionAspect::DisplayStyle::ComboBox); + outputMode.addOption({tr("Default"), {}, 0}); + outputMode.addOption({tr("Verbose"), {}, 1}); + outputMode.addOption({tr("Very Verbose"), {}, 2}); + + registerAspect(&repetitionMode); + repetitionMode.setSettingsKey("RepetitionMode"); + repetitionMode.setLabelText(tr("Repetition mode")); + repetitionMode.setDisplayStyle(Utils::SelectionAspect::DisplayStyle::ComboBox); + repetitionMode.addOption({tr("Until Fail"), {}, 0}); + repetitionMode.addOption({tr("Until Pass"), {}, 1}); + repetitionMode.addOption({tr("After Timeout"), {}, 2}); + + registerAspect(&repetitionCount); + repetitionCount.setSettingsKey("RepetitionCount"); + repetitionCount.setDefaultValue(1); + repetitionCount.setLabelText(tr("Count")); + repetitionCount.setToolTip(tr("Number of re-runs for the test.")); + repetitionCount.setRange(1, 10000); + + registerAspect(&repeat); + repeat.setSettingsKey("Repeat"); + + registerAspect(&scheduleRandom); + scheduleRandom.setSettingsKey("ScheduleRandom"); + scheduleRandom.setLabelText(tr("Schedule random")); + + registerAspect(&stopOnFailure); + stopOnFailure.setSettingsKey("StopOnFail"); + stopOnFailure.setLabelText(tr("Stop on failure")); + + registerAspect(¶llel); + parallel.setSettingsKey("Parallel"); + parallel.setToolTip(tr("Run tests in parallel mode using given number of jobs.")); + + registerAspect(&jobs); + jobs.setSettingsKey("Jobs"); + jobs.setLabelText(tr("Jobs")); + jobs.setDefaultValue(1); + jobs.setRange(1, 128); + + registerAspect(&testLoad); + testLoad.setSettingsKey("TestLoad"); + testLoad.setLabelText(tr("Test load")); + testLoad.setToolTip(tr("Try not to start tests when they may cause CPU load to pass a " + "threshold.")); + + registerAspect(&threshold); + threshold.setSettingsKey("Threshold"); + threshold.setLabelText(tr("Threshold")); + threshold.setDefaultValue(1); + threshold.setRange(1, 128); + threshold.setEnabler(&testLoad); +} + +QStringList CTestSettings::activeSettingsAsOptions() const +{ + QStringList options; + if (outputOnFail.value()) + options << "--output-on-failure"; + switch (outputMode.value()) { + case 1: options << "-V"; break; + case 2: options << "-VV"; break; + default: break; + } + if (repeat.value()) { + QString repeatOption; + switch (repetitionMode.value()) { + case 0: repeatOption = "until-fail"; break; + case 1: repeatOption = "until-pass"; break; + case 2: repeatOption = "after-timeout"; break; + default: break; + } + if (!repeatOption.isEmpty()) { + repeatOption.append(':'); + repeatOption.append(QString::number(repetitionCount.value())); + options << "--repeat" << repeatOption; + } + } + if (scheduleRandom.value()) + options << "--schedule-random"; + if (stopOnFailure.value()) + options << "--stop-on-failure"; + if (parallel.value()) { + options << "-j" << QString::number(jobs.value()); + if (testLoad.value()) + options << "--test-load" << QString::number(threshold.value()); + } + return options; +} + +CTestSettingsPage::CTestSettingsPage(CTestSettings *settings, Utils::Id settingsId) +{ + setId(settingsId); + setCategory(Constants::AUTOTEST_SETTINGS_CATEGORY); + setDisplayName(tr("CTest")); + + setSettings(settings); + + setLayouter([settings](QWidget *widget) { + CTestSettings &s = *settings; + using namespace Utils::Layouting; + const Break nl; + + Form form { + Row {s.outputOnFail}, nl, + Row {s.scheduleRandom}, nl, + Row {s.stopOnFailure}, nl, + Row {s.outputMode}, nl, + Group { + Title(tr("Repeat tests"), &s.repeat), nl, + Row {s.repetitionMode, s.repetitionCount}, + }, nl, + Group { + Title(tr("Run in parallel"), &s.parallel), nl, + Row {s.jobs}, nl, + Row {s.testLoad, s.threshold} + } + }; + + Column {Row {Column { form , Stretch() }, Stretch() } }.attachTo(widget); + }); +} + +} // namespace Internal +} // namespace Autotest diff --git a/src/plugins/autotest/ctest/ctestsettings.h b/src/plugins/autotest/ctest/ctestsettings.h new file mode 100644 index 00000000000..5683987e9d4 --- /dev/null +++ b/src/plugins/autotest/ctest/ctestsettings.h @@ -0,0 +1,66 @@ +/**************************************************************************** +** +** Copyright (C) 2021 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of Qt Creator. +** +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 3 as published by the Free Software +** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-3.0.html. +** +****************************************************************************/ + +#pragma once + +#include + +#include + +namespace Autotest { +namespace Internal { + +class CTestSettings : public Utils::AspectContainer +{ + Q_DECLARE_TR_FUNCTIONS(Autotest::Internal::CTestSettings) +public: + CTestSettings(); + + QStringList activeSettingsAsOptions() const; + + Utils::IntegerAspect repetitionCount; + Utils::SelectionAspect repetitionMode; + Utils::SelectionAspect outputMode; + Utils::BoolAspect outputOnFail; + Utils::BoolAspect stopOnFailure; + Utils::BoolAspect scheduleRandom; + Utils::BoolAspect repeat; + // FIXME.. this makes the outputreader fail to get all results correctly for visual display + Utils::BoolAspect parallel; + Utils::IntegerAspect jobs; + Utils::BoolAspect testLoad; + Utils::IntegerAspect threshold; +}; + +class CTestSettingsPage final : public Core::IOptionsPage +{ + Q_DECLARE_TR_FUNCTIONS(Autotest::Internal::CTestSettingsPage) +public: + CTestSettingsPage(CTestSettings *settings, Utils::Id settingsId); +}; + +} // namespace Internal +} // namespace Autotest + diff --git a/src/plugins/autotest/ctest/ctesttool.h b/src/plugins/autotest/ctest/ctesttool.h index 30cce7df53b..ff0d3b75221 100644 --- a/src/plugins/autotest/ctest/ctesttool.h +++ b/src/plugins/autotest/ctest/ctesttool.h @@ -26,6 +26,7 @@ #pragma once #include "../itestframework.h" +#include "ctestsettings.h" namespace Autotest { namespace Internal { @@ -42,6 +43,12 @@ public: protected: const char *name() const final; ITestTreeItem *createRootNode() final; + +private: + ITestSettings *testSettings() override { return &m_settings; } + + CTestSettings m_settings; + CTestSettingsPage m_settingsPage{&m_settings, settingsId()}; }; } // namespace Internal diff --git a/src/plugins/autotest/ctest/ctesttreeitem.cpp b/src/plugins/autotest/ctest/ctesttreeitem.cpp index d9230461fbb..5e08d62c488 100644 --- a/src/plugins/autotest/ctest/ctesttreeitem.cpp +++ b/src/plugins/autotest/ctest/ctesttreeitem.cpp @@ -26,6 +26,7 @@ #include "ctesttreeitem.h" #include "ctestconfiguration.h" +#include "ctestsettings.h" #include "../autotestplugin.h" #include "../itestframework.h" @@ -107,8 +108,8 @@ QList CTestTreeItem::testConfigurationsFor(const QStringLi const ProjectExplorer::BuildSystem *buildSystem = target->buildSystem(); QStringList options{"--timeout", QString::number(AutotestPlugin::settings()->timeout / 1000)}; - // TODO add ctest options from settings? - options << "--output-on-failure"; + auto ctestSettings = static_cast(testBase()->testSettings()); + options << ctestSettings->activeSettingsAsOptions(); Utils::CommandLine command = buildSystem->commandLineForTests(selected, options); if (command.executable().isEmpty()) return {}; diff --git a/src/plugins/autotest/testframeworkmanager.cpp b/src/plugins/autotest/testframeworkmanager.cpp index b7ec3dc0221..0352a3dd61b 100644 --- a/src/plugins/autotest/testframeworkmanager.cpp +++ b/src/plugins/autotest/testframeworkmanager.cpp @@ -125,6 +125,10 @@ void TestFrameworkManager::synchronizeSettings(QSettings *s) if (ITestSettings *fSettings = framework->testSettings()) fSettings->readSettings(s); } + for (ITestTool *testTool : qAsConst(m_registeredTestTools)) { + if (ITestSettings *tSettings = testTool->testSettings()) + tSettings->readSettings(s); + } } } // namespace Autotest diff --git a/src/plugins/autotest/testresultmodel.cpp b/src/plugins/autotest/testresultmodel.cpp index c4060f4d42d..ad8b4bd177b 100644 --- a/src/plugins/autotest/testresultmodel.cpp +++ b/src/plugins/autotest/testresultmodel.cpp @@ -129,7 +129,7 @@ static bool isSignificant(ResultType type) case ResultType::MessageCurrentTest: case ResultType::Application: case ResultType::Invalid: - QTC_ASSERT_STRING("Got unexpedted type in isSignificant check"); + QTC_ASSERT_STRING("Got unexpected type in isSignificant check"); return false; default: return true;