From ba61f2946d85d6674ca56b4845d598c9d30692b8 Mon Sep 17 00:00:00 2001 From: Christian Stenger Date: Tue, 23 Feb 2016 17:40:10 +0100 Subject: [PATCH] AutoTest: Add some gtest related settings This patch enables execution of disabled tests, repetition of test runs and shuffling of test sets. Change-Id: I3668afde5766d18acdf1afb7edd913655b13a76c Reviewed-by: David Schulz Reviewed-by: Leena Miettinen --- src/plugins/autotest/testcodeparser.cpp | 1 + src/plugins/autotest/testoutputreader.cpp | 16 ++- src/plugins/autotest/testoutputreader.h | 1 + src/plugins/autotest/testresult.h | 1 + src/plugins/autotest/testresultmodel.cpp | 3 +- src/plugins/autotest/testrunner.cpp | 23 ++-- src/plugins/autotest/testsettings.cpp | 20 +++- src/plugins/autotest/testsettings.h | 5 + src/plugins/autotest/testsettingspage.cpp | 13 +++ src/plugins/autotest/testsettingspage.ui | 132 +++++++++++++++++++++- src/plugins/autotest/testtreeitem.cpp | 14 ++- src/plugins/autotest/testtreeitem.h | 1 + src/plugins/autotest/testtreemodel.h | 1 + src/plugins/autotest/testvisitor.cpp | 9 +- src/plugins/autotest/testvisitor.h | 10 +- 15 files changed, 224 insertions(+), 26 deletions(-) diff --git a/src/plugins/autotest/testcodeparser.cpp b/src/plugins/autotest/testcodeparser.cpp index ad982494c9e..76e5037ef63 100644 --- a/src/plugins/autotest/testcodeparser.cpp +++ b/src/plugins/autotest/testcodeparser.cpp @@ -498,6 +498,7 @@ static void handleGTest(QFutureInterface futureInterface, const parseResult.testCaseName = testSpec.testCaseName; parseResult.parameterized = testSpec.parameterized; parseResult.typed = testSpec.typed; + parseResult.disabled = testSpec.disabled; parseResult.proFile = proFile; parseResult.dataTagsOrTestSets.insert(QString(), result.value(testSpec)); futureInterface.reportResult(parseResult); diff --git a/src/plugins/autotest/testoutputreader.cpp b/src/plugins/autotest/testoutputreader.cpp index 862a061effb..c30a5a88a7f 100644 --- a/src/plugins/autotest/testoutputreader.cpp +++ b/src/plugins/autotest/testoutputreader.cpp @@ -314,6 +314,7 @@ void GTestOutputReader::processOutput() static QRegExp testSetFail(QStringLiteral("^\\[ FAILED \\] (.*) \\((.*)\\)$")); static QRegExp disabledTests(QStringLiteral("^ YOU HAVE (\\d+) DISABLED TESTS?$")); static QRegExp failureLocation(QStringLiteral("^(.*):(\\d+): Failure$")); + static QRegExp iterations(QStringLiteral("^Repeating all tests \\(iteration (\\d+)\\) . . .$")); while (m_testApplication->canReadLine()) { if (m_futureInterface.isCanceled()) @@ -337,7 +338,10 @@ void GTestOutputReader::processOutput() if (!line.startsWith(QLatin1Char('['))) { m_description.append(line).append(QLatin1Char('\n')); - if (line.startsWith(QStringLiteral("Note:"))) { + if (iterations.exactMatch(line)) { + m_iteration = iterations.cap(1).toInt(); + m_description.clear(); + } else if (line.startsWith(QStringLiteral("Note:"))) { auto testResult = new GTestResult(); testResult->setResult(Result::MessageInternal); testResult->setDescription(line); @@ -366,8 +370,14 @@ void GTestOutputReader::processOutput() } else if (newTestStarts.exactMatch(line)) { m_currentTestName = newTestStarts.cap(1); auto testResult = new GTestResult(m_currentTestName); - testResult->setResult(Result::MessageTestCaseStart); - testResult->setDescription(tr("Executing test case %1").arg(m_currentTestName)); + if (m_iteration > 1) { + testResult->setResult(Result::MessageTestCaseRepetition); + testResult->setDescription(tr("Repeating test case %1 (iteration %2)") + .arg(m_currentTestName).arg(m_iteration)); + } else { + testResult->setResult(Result::MessageTestCaseStart); + testResult->setDescription(tr("Executing test case %1").arg(m_currentTestName)); + } m_futureInterface.reportResult(testResult); } else if (newTestSetStarts.exactMatch(line)) { m_currentTestSet = newTestSetStarts.cap(1); diff --git a/src/plugins/autotest/testoutputreader.h b/src/plugins/autotest/testoutputreader.h index 44a973145c9..012b28b8bec 100644 --- a/src/plugins/autotest/testoutputreader.h +++ b/src/plugins/autotest/testoutputreader.h @@ -100,6 +100,7 @@ private: QString m_currentTestSet; QString m_description; QByteArray m_unprocessed; + int m_iteration = 0; }; diff --git a/src/plugins/autotest/testresult.h b/src/plugins/autotest/testresult.h index 91e380c9c35..17ae5dc82ba 100644 --- a/src/plugins/autotest/testresult.h +++ b/src/plugins/autotest/testresult.h @@ -56,6 +56,7 @@ enum Type { MessageTestCaseWarn, MessageTestCaseFail, MessageTestCaseEnd, + MessageTestCaseRepetition, MessageCurrentTest, INTERNAL_MESSAGES_END = MessageCurrentTest, Invalid, diff --git a/src/plugins/autotest/testresultmodel.cpp b/src/plugins/autotest/testresultmodel.cpp index 835850fe733..05a361267cc 100644 --- a/src/plugins/autotest/testresultmodel.cpp +++ b/src/plugins/autotest/testresultmodel.cpp @@ -292,7 +292,8 @@ void TestResultFilterModel::enableAllResultTypes() << Result::BlacklistedFail << Result::Benchmark << Result::MessageCurrentTest << Result::MessageTestCaseStart << Result::MessageTestCaseSuccess << Result::MessageTestCaseWarn - << Result::MessageTestCaseFail << Result::MessageTestCaseEnd; + << Result::MessageTestCaseFail << Result::MessageTestCaseEnd + << Result::MessageTestCaseRepetition; invalidateFilter(); } diff --git a/src/plugins/autotest/testrunner.cpp b/src/plugins/autotest/testrunner.cpp index b61413336bb..ca4eddb90da 100644 --- a/src/plugins/autotest/testrunner.cpp +++ b/src/plugins/autotest/testrunner.cpp @@ -118,9 +118,11 @@ void TestRunner::setSelectedTests(const QList &selected) } static void performTestRun(QFutureInterface &futureInterface, - const QList selectedTests, const int timeout, - const QString metricsOption) + const QList selectedTests, + const TestSettings &settings) { + const int timeout = settings.timeout; + const QString &metricsOption = TestSettings::metricsTypeToOption(settings.metrics); QEventLoop eventLoop; int testCaseCount = 0; foreach (TestConfiguration *config, selectedTests) { @@ -177,13 +179,21 @@ static void performTestRun(QFutureInterface &futureInterface, argumentList << testConfiguration->testCases(); testProcess.setArguments(argumentList); } else { // TestTypeGTest + QStringList argumentList; const QStringList &testSets = testConfiguration->testCases(); if (testSets.size()) { - QStringList argumentList; argumentList << QLatin1String("--gtest_filter=") + testSets.join(QLatin1Char(':')); - testProcess.setArguments(argumentList); } + if (settings.gtestRunDisabled) + argumentList << QLatin1String("--gtest_also_run_disabled_tests"); + if (settings.gtestRepeat) + argumentList << QString::fromLatin1("--gtest_repeat=%1").arg(settings.gtestIterations); + if (settings.gtestShuffle) { + argumentList << QLatin1String("--gtest_shuffle"); + argumentList << QString::fromLatin1("--gtest_random_seed=%1").arg(settings.gtestSeed); + } + testProcess.setArguments(argumentList); } testProcess.setWorkingDirectory(testConfiguration->workingDirectory()); @@ -282,11 +292,8 @@ void TestRunner::prepareToRunTests() void TestRunner::runTests() { - const QSharedPointer settings = AutotestPlugin::instance()->settings(); - const QString &metricsOption = TestSettings::metricsTypeToOption(settings->metrics); - QFuture future = Utils::runAsync(&performTestRun, m_selectedTests, - settings->timeout, metricsOption); + *AutotestPlugin::instance()->settings()); m_futureWatcher.setFuture(future); Core::ProgressManager::addTask(future, tr("Running Tests"), Autotest::Constants::TASK_INDEX); } diff --git a/src/plugins/autotest/testsettings.cpp b/src/plugins/autotest/testsettings.cpp index a6d13930a82..870b35f124a 100644 --- a/src/plugins/autotest/testsettings.cpp +++ b/src/plugins/autotest/testsettings.cpp @@ -38,6 +38,11 @@ static const char omitRunConfigWarnKey[] = "OmitRCWarnings"; static const char limitResultOutputKey[] = "LimitResultOutput"; static const char autoScrollKey[] = "AutoScrollResults"; static const char alwaysParseKey[] = "AlwaysParse"; +static const char gtestRunDisabledKey[] = "RunDisabledGTests"; +static const char gtestRepeatKey[] = "RepeatGTests"; +static const char gtestShuffleKey[] = "ShuffleGTests"; +static const char gtestIterationsKey[] = "IterationsGTests"; +static const char gtestSeedKey[] = "SeedGTests"; static const int defaultTimeout = 60000; TestSettings::TestSettings() @@ -56,6 +61,11 @@ void TestSettings::toSettings(QSettings *s) const s->setValue(QLatin1String(limitResultOutputKey), limitResultOutput); s->setValue(QLatin1String(autoScrollKey), autoScroll); s->setValue(QLatin1String(alwaysParseKey), alwaysParse); + s->setValue(QLatin1String(gtestRunDisabledKey), gtestRunDisabled); + s->setValue(QLatin1String(gtestRepeatKey), gtestRepeat); + s->setValue(QLatin1String(gtestShuffleKey), gtestShuffle); + s->setValue(QLatin1String(gtestIterationsKey), gtestIterations); + s->setValue(QLatin1String(gtestSeedKey), gtestSeed); s->endGroup(); } @@ -87,6 +97,11 @@ void TestSettings::fromSettings(const QSettings *s) limitResultOutput = s->value(root + QLatin1String(limitResultOutputKey), true).toBool(); autoScroll = s->value(root + QLatin1String(autoScrollKey), true).toBool(); alwaysParse = s->value(root + QLatin1String(alwaysParseKey), false).toBool(); + gtestRunDisabled = s->value(root + QLatin1String(gtestRunDisabledKey), false).toBool(); + gtestRepeat = s->value(root + QLatin1String(gtestRepeatKey), false).toBool(); + gtestShuffle = s->value(root + QLatin1String(gtestShuffleKey), false).toBool(); + gtestIterations = s->value(root + QLatin1String(gtestIterationsKey), 1).toInt(); + gtestSeed = s->value(root + QLatin1String(gtestSeedKey), 0).toInt(); } bool TestSettings::equals(const TestSettings &rhs) const @@ -96,7 +111,10 @@ bool TestSettings::equals(const TestSettings &rhs) const && omitRunConfigWarn == rhs.omitRunConfigWarn && limitResultOutput == rhs.limitResultOutput && autoScroll == rhs.autoScroll - && alwaysParse == rhs.alwaysParse; + && alwaysParse == rhs.alwaysParse + && gtestRunDisabled == rhs.gtestRunDisabled + && gtestRepeat == rhs.gtestRepeat && gtestIterations == rhs.gtestIterations + && gtestShuffle == rhs.gtestShuffle && gtestSeed == rhs.gtestSeed; } QString TestSettings::metricsTypeToOption(const MetricsType type) diff --git a/src/plugins/autotest/testsettings.h b/src/plugins/autotest/testsettings.h index 4e10fc59c9c..7a9aa1d97d3 100644 --- a/src/plugins/autotest/testsettings.h +++ b/src/plugins/autotest/testsettings.h @@ -52,12 +52,17 @@ struct TestSettings static QString metricsTypeToOption(const MetricsType type); int timeout; + int gtestIterations; + int gtestSeed; MetricsType metrics; bool omitInternalMssg; bool omitRunConfigWarn; bool limitResultOutput; bool autoScroll; bool alwaysParse; + bool gtestRunDisabled; + bool gtestShuffle; + bool gtestRepeat; }; inline bool operator==(const TestSettings &s1, const TestSettings &s2) { return s1.equals(s2); } diff --git a/src/plugins/autotest/testsettingspage.cpp b/src/plugins/autotest/testsettingspage.cpp index f003cce54ed..1cf2535f522 100644 --- a/src/plugins/autotest/testsettingspage.cpp +++ b/src/plugins/autotest/testsettingspage.cpp @@ -41,6 +41,9 @@ TestSettingsWidget::TestSettingsWidget(QWidget *parent) m_ui.setupUi(this); m_ui.callgrindRB->setEnabled(Utils::HostOsInfo::isAnyUnixHost()); // valgrind available on UNIX m_ui.perfRB->setEnabled(Utils::HostOsInfo::isLinuxHost()); // according to docs perf Linux only + + connect(m_ui.repeatGTestsCB, &QCheckBox::toggled, m_ui.repetitionSpin, &QSpinBox::setEnabled); + connect(m_ui.shuffleGTestsCB, &QCheckBox::toggled, m_ui.seedSpin, &QSpinBox::setEnabled); } void TestSettingsWidget::setSettings(const TestSettings &settings) @@ -51,6 +54,11 @@ void TestSettingsWidget::setSettings(const TestSettings &settings) m_ui.limitResultOutputCB->setChecked(settings.limitResultOutput); m_ui.autoScrollCB->setChecked(settings.autoScroll); m_ui.alwaysParseCB->setChecked(settings.alwaysParse); + m_ui.runDisabledGTestsCB->setChecked(settings.gtestRunDisabled); + m_ui.repeatGTestsCB->setChecked(settings.gtestRepeat); + m_ui.shuffleGTestsCB->setChecked(settings.gtestShuffle); + m_ui.repetitionSpin->setValue(settings.gtestIterations); + m_ui.seedSpin->setValue(settings.gtestSeed); switch (settings.metrics) { case MetricsType::Walltime: @@ -82,6 +90,11 @@ TestSettings TestSettingsWidget::settings() const result.limitResultOutput = m_ui.limitResultOutputCB->isChecked(); result.autoScroll = m_ui.autoScrollCB->isChecked(); result.alwaysParse = m_ui.alwaysParseCB->isChecked(); + result.gtestRunDisabled = m_ui.runDisabledGTestsCB->isChecked(); + result.gtestRepeat = m_ui.repeatGTestsCB->isChecked(); + result.gtestShuffle = m_ui.shuffleGTestsCB->isChecked(); + result.gtestIterations = m_ui.repetitionSpin->value(); + result.gtestSeed = m_ui.seedSpin->value(); if (m_ui.walltimeRB->isChecked()) result.metrics = MetricsType::Walltime; diff --git a/src/plugins/autotest/testsettingspage.ui b/src/plugins/autotest/testsettingspage.ui index 37fe0c048a2..ad70fd1617a 100644 --- a/src/plugins/autotest/testsettingspage.ui +++ b/src/plugins/autotest/testsettingspage.ui @@ -6,8 +6,8 @@ 0 0 - 451 - 271 + 485 + 404 @@ -150,6 +150,134 @@ + + + + Google Test + + + + + + Executes disabled tests when performing a test run. + + + Run disabled tests + + + + + + + + + Repeats a test run (you might be required to increase the timeout to avoid canceling the tests). + + + Repeat tests + + + + + + + + 0 + 0 + + + + Iterations: + + + + + + + false + + + 1 + + + 9999 + + + + + + + Qt::Horizontal + + + + 40 + 20 + + + + + + + + + + + + Shuffle tests automatically on every iteration by the given seed. + + + Shuffle tests + + + + + + + + 0 + 0 + + + + Seed: + + + + + + + false + + + A seed of 0 generates a seed based on the current timestamp. + + + + + + 99999 + + + + + + + Qt::Horizontal + + + + 40 + 20 + + + + + + + + + diff --git a/src/plugins/autotest/testtreeitem.cpp b/src/plugins/autotest/testtreeitem.cpp index 9754bf2487a..d3a1207dbed 100644 --- a/src/plugins/autotest/testtreeitem.cpp +++ b/src/plugins/autotest/testtreeitem.cpp @@ -388,6 +388,8 @@ GoogleTestTreeItem *GoogleTestTreeItem::createTestItem(const TestParseResult &re item->setState(Parameterized); if (result.typed) item->setState(Typed); + if (result.disabled) + item->setState(Disabled); foreach (const TestCodeLocationAndType &location, result.dataTagsOrTestSets.first()) item->appendChild(createTestSetItem(result, location)); return item; @@ -408,10 +410,14 @@ GoogleTestTreeItem *GoogleTestTreeItem::createTestSetItem(const TestParseResult QVariant GoogleTestTreeItem::data(int column, int role) const { switch (role) { - case Qt::DisplayRole: - if (type() == TestCase) - return QVariant(name() + nameSuffix()); - break; + case Qt::DisplayRole: { + if (type() == TestTreeItem::Root) + return TestTreeItem::data(column, role); + + const QString &displayName = (m_state & GoogleTestTreeItem::Disabled) + ? name().mid(9) : name(); + return QVariant(displayName + nameSuffix()); + } case StateRole: return (int)m_state; default: diff --git a/src/plugins/autotest/testtreeitem.h b/src/plugins/autotest/testtreeitem.h index 8123fd4cfd9..4d44923c2f8 100644 --- a/src/plugins/autotest/testtreeitem.h +++ b/src/plugins/autotest/testtreeitem.h @@ -212,6 +212,7 @@ struct GTestCaseSpec QString testCaseName; bool parameterized; bool typed; + bool disabled; }; } // namespace Internal diff --git a/src/plugins/autotest/testtreemodel.h b/src/plugins/autotest/testtreemodel.h index 9c7f08d7b06..dd2437431f0 100644 --- a/src/plugins/autotest/testtreemodel.h +++ b/src/plugins/autotest/testtreemodel.h @@ -160,6 +160,7 @@ struct TestParseResult unsigned column = 0; bool parameterized = false; bool typed = false; + bool disabled = false; QMap functions; QMap dataTagsOrTestSets; }; diff --git a/src/plugins/autotest/testvisitor.cpp b/src/plugins/autotest/testvisitor.cpp index a203ed46083..c4c9373ca35 100644 --- a/src/plugins/autotest/testvisitor.cpp +++ b/src/plugins/autotest/testvisitor.cpp @@ -383,16 +383,17 @@ bool GTestVisitor::visit(CPlusPlus::FunctionDefinitionAST *ast) m_document->translationUnit()->getTokenStartPosition(token, &line, &column); TestCodeLocationAndType locationAndType; - locationAndType.m_name = disabled ? testName.mid(9) : testName; + locationAndType.m_name = testName; locationAndType.m_line = line; locationAndType.m_column = column - 1; locationAndType.m_type = TestTreeItem::TestFunctionOrSet; - locationAndType.m_state = (disabled || disabledCase) ? GoogleTestTreeItem::Disabled - : GoogleTestTreeItem::Enabled; + locationAndType.m_state = disabled ? GoogleTestTreeItem::Disabled + : GoogleTestTreeItem::Enabled; GTestCaseSpec spec; - spec.testCaseName = disabledCase ? testCaseName.mid(9) : testCaseName; + spec.testCaseName = testCaseName; spec.parameterized = TestUtils::isGTestParameterized(prettyName); spec.typed = TestUtils::isGTestTyped(prettyName); + spec.disabled = disabledCase; m_gtestFunctions[spec].append(locationAndType); } diff --git a/src/plugins/autotest/testvisitor.h b/src/plugins/autotest/testvisitor.h index cbc1c7d4be4..5d171216f58 100644 --- a/src/plugins/autotest/testvisitor.h +++ b/src/plugins/autotest/testvisitor.h @@ -134,10 +134,14 @@ inline bool operator<(const GTestCaseSpec &spec1, const GTestCaseSpec &spec2) if (spec1.testCaseName != spec2.testCaseName) return spec1.testCaseName < spec2.testCaseName; if (spec1.parameterized == spec2.parameterized) { - if (spec1.typed == spec2.typed) - return false; - else + if (spec1.typed == spec2.typed) { + if (spec1.disabled == spec2.disabled) + return false; + else + return !spec1.disabled; + } else { return !spec1.typed; + } } else { return !spec1.parameterized; }