From cc8bff67b312fe8f18304859eeb421744f1fac6d Mon Sep 17 00:00:00 2001 From: Christian Stenger Date: Mon, 19 Jun 2017 15:58:21 +0200 Subject: [PATCH] AutoTest: Support plaintext output for Qt tests Change-Id: I88ec477777d79c69e699dd906ec4ef1550bcaf44 Reviewed-by: David Schulz --- .../autotest/qtest/qttestconfiguration.cpp | 21 +- .../autotest/qtest/qttestoutputreader.cpp | 237 ++++++++++++++++-- .../autotest/qtest/qttestoutputreader.h | 22 +- src/plugins/autotest/qtest/qttestsettings.cpp | 3 + src/plugins/autotest/qtest/qttestsettings.h | 1 + .../autotest/qtest/qttestsettingspage.cpp | 2 + .../autotest/qtest/qttestsettingspage.ui | 15 ++ .../autotest/quick/quicktestconfiguration.cpp | 20 +- src/plugins/autotest/testresult.cpp | 8 +- 9 files changed, 289 insertions(+), 40 deletions(-) diff --git a/src/plugins/autotest/qtest/qttestconfiguration.cpp b/src/plugins/autotest/qtest/qttestconfiguration.cpp index 57ab6fc86a6..2329bfe6bc5 100644 --- a/src/plugins/autotest/qtest/qttestconfiguration.cpp +++ b/src/plugins/autotest/qtest/qttestconfiguration.cpp @@ -35,7 +35,17 @@ namespace Internal { TestOutputReader *QtTestConfiguration::outputReader(const QFutureInterface &fi, QProcess *app) const { - return new QtTestOutputReader(fi, app, buildDirectory()); + static const Core::Id id + = Core::Id(Constants::FRAMEWORK_PREFIX).withSuffix(QtTest::Constants::FRAMEWORK_NAME); + TestFrameworkManager *manager = TestFrameworkManager::instance(); + auto qtSettings = qSharedPointerCast(manager->settingsForTestFramework(id)); + if (qtSettings.isNull()) + return nullptr; + + if (qtSettings->useXMLOutput) + return new QtTestOutputReader(fi, app, buildDirectory(), QtTestOutputReader::XML); + else + return new QtTestOutputReader(fi, app, buildDirectory(), QtTestOutputReader::PlainText); } QStringList QtTestConfiguration::argumentsForTestRunner() const @@ -43,14 +53,15 @@ QStringList QtTestConfiguration::argumentsForTestRunner() const static const Core::Id id = Core::Id(Constants::FRAMEWORK_PREFIX).withSuffix(QtTest::Constants::FRAMEWORK_NAME); - QStringList arguments("-xml"); - if (testCases().count()) - arguments << testCases(); - + QStringList arguments; TestFrameworkManager *manager = TestFrameworkManager::instance(); auto qtSettings = qSharedPointerCast(manager->settingsForTestFramework(id)); if (qtSettings.isNull()) return arguments; + if (qtSettings->useXMLOutput) + arguments << "-xml"; + if (testCases().count()) + arguments << testCases(); const QString &metricsOption = QtTestSettings::metricsTypeToOption(qtSettings->metrics); if (!metricsOption.isEmpty()) diff --git a/src/plugins/autotest/qtest/qttestoutputreader.cpp b/src/plugins/autotest/qtest/qttestoutputreader.cpp index 015bcc179c2..aa86681a486 100644 --- a/src/plugins/autotest/qtest/qttestoutputreader.cpp +++ b/src/plugins/autotest/qtest/qttestoutputreader.cpp @@ -31,6 +31,7 @@ #include #include #include +#include namespace Autotest { namespace Internal { @@ -128,12 +129,26 @@ static QString constructSourceFilePath(const QString &path, const QString &fileP } QtTestOutputReader::QtTestOutputReader(const QFutureInterface &futureInterface, - QProcess *testApplication, const QString &buildDirectory) + QProcess *testApplication, const QString &buildDirectory, + OutputMode mode) : TestOutputReader(futureInterface, testApplication, buildDirectory) + , m_mode(mode) { } void QtTestOutputReader::processOutput(const QByteArray &outputLine) +{ + switch (m_mode) { + case PlainText: + processPlainTextOutput(outputLine); + break; + case XML: + processXMLOutput(outputLine); + break; + } +} + +void QtTestOutputReader::processXMLOutput(const QByteArray &outputLine) { static QStringList validEndTags = {QStringLiteral("Incident"), QStringLiteral("Message"), @@ -162,24 +177,14 @@ void QtTestOutputReader::processOutput(const QByteArray &outputLine) if (currentTag == QStringLiteral("TestCase")) { m_className = m_xmlReader.attributes().value(QStringLiteral("name")).toString(); QTC_ASSERT(!m_className.isEmpty(), continue); - TestResultPtr testResult = TestResultPtr(createDefaultResult()); - testResult->setResult(Result::MessageTestCaseStart); - testResult->setDescription(tr("Executing test case %1").arg(m_className)); - m_futureInterface.reportResult(testResult); + sendStartMessage(false); } else if (currentTag == QStringLiteral("TestFunction")) { m_testCase = m_xmlReader.attributes().value(QStringLiteral("name")).toString(); QTC_ASSERT(!m_testCase.isEmpty(), continue); if (m_testCase == m_formerTestCase) // don't report "Executing..." more than once continue; - TestResultPtr testResult = TestResultPtr(createDefaultResult()); - testResult->setResult(Result::MessageTestCaseStart); - testResult->setDescription(tr("Executing test function %1").arg(m_testCase)); - m_futureInterface.reportResult(testResult); - testResult = TestResultPtr(new QtTestResult); - testResult->setResult(Result::MessageCurrentTest); - testResult->setDescription(tr("Entering test function %1::%2").arg(m_className, - m_testCase)); - m_futureInterface.reportResult(testResult); + sendStartMessage(true); + sendMessageCurrentTest(); } else if (currentTag == QStringLiteral("Duration")) { m_duration = m_xmlReader.attributes().value(QStringLiteral("msecs")).toString(); QTC_ASSERT(!m_duration.isEmpty(), continue); @@ -255,23 +260,13 @@ void QtTestOutputReader::processOutput(const QByteArray &outputLine) m_cdataMode = None; const QStringRef currentTag = m_xmlReader.name(); if (currentTag == QStringLiteral("TestFunction")) { - QtTestResult *testResult = createDefaultResult(); - testResult->setResult(Result::MessageTestCaseEnd); - testResult->setDescription( - m_duration.isEmpty() ? tr("Test function finished.") - : tr("Execution took %1 ms.").arg(m_duration)); - m_futureInterface.reportResult(TestResultPtr(testResult)); + sendFinishMessage(true); m_futureInterface.setProgressValue(m_futureInterface.progressValue() + 1); m_dataTag.clear(); m_formerTestCase = m_testCase; m_testCase.clear(); } else if (currentTag == QStringLiteral("TestCase")) { - QtTestResult *testResult = createDefaultResult(); - testResult->setResult(Result::MessageTestCaseEnd); - testResult->setDescription( - m_duration.isEmpty() ? tr("Test finished.") - : tr("Test execution took %1 ms.").arg(m_duration)); - m_futureInterface.reportResult(TestResultPtr(testResult)); + sendFinishMessage(false); } else if (validEndTags.contains(currentTag.toString())) { QtTestResult *testResult = createDefaultResult(); testResult->setResult(m_result); @@ -290,6 +285,138 @@ void QtTestOutputReader::processOutput(const QByteArray &outputLine) } } +static QStringList extractFunctionInformation(const QString &testClassName, + const QString &lineWithoutResultType, + Result::Type resultType) +{ + static QRegularExpression classInformation("^(.+?)\\((.*?)\\)(.*)$"); + QStringList result; + const QRegularExpressionMatch match = classInformation.match(lineWithoutResultType); + if (match.hasMatch()) { + QString fullQualifiedFunc = match.captured(1); + QTC_ASSERT(fullQualifiedFunc.startsWith(testClassName + "::"), return result); + fullQualifiedFunc = fullQualifiedFunc.mid(testClassName.length() + 2); + result.append(fullQualifiedFunc); + if (resultType == Result::Benchmark) { // tag is displayed differently + QString possiblyTag = match.captured(3); + if (!possiblyTag.isEmpty()) + possiblyTag = possiblyTag.mid(2, possiblyTag.length() - 4); + result.append(possiblyTag); + result.append(QString()); + } else { + result.append(match.captured(2)); + result.append(match.captured(3)); + } + } + return result; +} + +void QtTestOutputReader::processPlainTextOutput(const QByteArray &outputLine) +{ + static QRegExp start("^[*]{9} Start testing of (.*) [*]{9}$"); + static QRegExp config("^Config: Using QtTest library (.*), (Qt (\\d+(\\.\\d+){2}) \\(.*\\))$"); + static QRegExp summary("^Totals: \\d+ passed, \\d+ failed, \\d+ skipped(, \\d+ blacklisted)?$"); + static QRegExp finish("^[*]{9} Finished testing of (.*) [*]{9}$"); + + static QRegExp result("^(PASS |FAIL! |XFAIL |XPASS |SKIP |BPASS |BFAIL |RESULT " + "|INFO |QWARN |WARNING|QDEBUG ): (.*)$"); + + static QRegExp benchDetails("^\\s+([\\d,.]+ .* per iteration \\(total: [\\d,.]+, iterations: \\d+\\))$"); + static QRegExp locationUnix("^ Loc: \\[(.*)\\]$"); + static QRegExp locationWin("^(.*\\(\\d+\\)) : failure location$"); + + if (m_futureInterface.isCanceled()) + return; + + const QString &line = QString::fromLatin1(outputLine); + + if (result.exactMatch(line)) { + processResultOutput(result.cap(1).toLower().trimmed(), result.cap(2)); + } else if (locationUnix.exactMatch(line)) { + processLocationOutput(locationUnix.cap(1)); + } else if (locationWin.exactMatch(line)) { + processLocationOutput(locationWin.cap(1)); + } else if (benchDetails.exactMatch(line)) { + m_description = benchDetails.cap(1); + } else if (config.exactMatch(line)) { + handleAndSendConfigMessage(config); + } else if (start.exactMatch(line)) { + m_className = start.cap(1); + QTC_CHECK(!m_className.isEmpty()); + sendStartMessage(false); + } else if (summary.exactMatch(line) || finish.exactMatch(line)) { + processSummaryFinishOutput(); + } else { // we have some plain output, but we cannot say where for sure it belongs to.. + if (!m_description.isEmpty()) + m_description.append('\n'); + m_description.append(line); + } +} + +void QtTestOutputReader::processResultOutput(const QString &result, const QString &message) +{ + if (!m_testCase.isEmpty()) { // report the former result if there is any + sendCompleteInformation(); + m_dataTag.clear(); + m_description.clear(); + m_file.clear(); + m_lineNumber = 0; + } + m_result = TestResult::resultFromString(result); + const QStringList funcWithTag = extractFunctionInformation(m_className, message, m_result); + QTC_ASSERT(funcWithTag.size() == 3, return); + m_testCase = funcWithTag.at(0); + if (m_testCase != m_formerTestCase) { // new test function executed + if (!m_formerTestCase.isEmpty()) { + using namespace std; + swap(m_testCase, m_formerTestCase); // we want formerTestCase to be reported + sendFinishMessage(true); + swap(m_testCase, m_formerTestCase); + } + sendStartMessage(true); + sendMessageCurrentTest(); + } + m_dataTag = funcWithTag.at(1); + const QString description = funcWithTag.at(2); + if (!description.isEmpty()) { + if (!m_description.isEmpty()) + m_description.append('\n'); + m_description.append(description.mid(1)); // cut the first whitespace + } + m_formerTestCase = m_testCase; +} + +void QtTestOutputReader::processLocationOutput(const QString &fileWithLine) +{ + QTC_ASSERT(fileWithLine.endsWith(')'), return); + int openBrace = fileWithLine.lastIndexOf('('); + QTC_ASSERT(openBrace != -1, return); + m_file = constructSourceFilePath(m_buildDir, fileWithLine.left(openBrace)); + QString numberStr = fileWithLine.mid(openBrace + 1); + numberStr.chop(1); + m_lineNumber = numberStr.toInt(); +} + +void QtTestOutputReader::processSummaryFinishOutput() +{ + if (m_className.isEmpty()) // we have reported already + return; + // we still have something to report + sendCompleteInformation(); + m_dataTag.clear(); + // report finished function + sendFinishMessage(true); + m_testCase.clear(); + m_formerTestCase.clear(); + // create and report the finish message for this test class + sendFinishMessage(false); + m_className.clear(); + m_description.clear(); + m_result = Result::Invalid; + m_file.clear(); + m_lineNumber = 0; +} + QtTestResult *QtTestOutputReader::createDefaultResult() const { QtTestResult *result = new QtTestResult(m_className); @@ -298,5 +425,63 @@ QtTestResult *QtTestOutputReader::createDefaultResult() const return result; } +void QtTestOutputReader::sendCompleteInformation() +{ + TestResultPtr testResult = TestResultPtr(createDefaultResult()); + testResult->setResult(m_result); + testResult->setFileName(m_file); + testResult->setLine(m_lineNumber); + testResult->setDescription(m_description); + m_futureInterface.reportResult(testResult); +} + +void QtTestOutputReader::sendMessageCurrentTest() +{ + TestResultPtr testResult = TestResultPtr(new QtTestResult); + testResult->setResult(Result::MessageCurrentTest); + testResult->setDescription(tr("Entering test function %1::%2").arg(m_className, m_testCase)); + m_futureInterface.reportResult(testResult); +} + +void QtTestOutputReader::sendStartMessage(bool isFunction) +{ + TestResultPtr testResult = TestResultPtr(createDefaultResult()); + testResult->setResult(Result::MessageTestCaseStart); + testResult->setDescription(isFunction ? tr("Executing test function %1").arg(m_testCase) + : tr("Executing test case %1").arg(m_className)); + m_futureInterface.reportResult(testResult); +} + +void QtTestOutputReader::sendFinishMessage(bool isFunction) +{ + TestResultPtr testResult = TestResultPtr(createDefaultResult()); + testResult->setResult(Result::MessageTestCaseEnd); + if (m_duration.isEmpty()) { + testResult->setDescription(isFunction ? tr("Execution took %1 ms.").arg(m_duration) + : tr("Test execution took %1 ms.").arg(m_duration)); + } else { + testResult->setDescription(isFunction ? tr("Test function finished.") + : tr("Test finished.")); + } + m_futureInterface.reportResult(testResult); +} + +// TODO factor out tr() strings to avoid duplication (see XML processing of Characters) +void QtTestOutputReader::handleAndSendConfigMessage(const QRegExp &config) +{ + QtTestResult *testResult = createDefaultResult(); + testResult->setResult(Result::MessageInternal); + testResult->setDescription(tr("Qt version: %1").arg(config.cap(3))); + m_futureInterface.reportResult(TestResultPtr(testResult)); + testResult = createDefaultResult(); + testResult->setResult(Result::MessageInternal); + testResult->setDescription(tr("Qt build: %1").arg(config.cap(2))); + m_futureInterface.reportResult(TestResultPtr(testResult)); + testResult = createDefaultResult(); + testResult->setResult(Result::MessageInternal); + testResult->setDescription(tr("QTest version: %1").arg(config.cap(1))); + m_futureInterface.reportResult(TestResultPtr(testResult)); +} + } // namespace Internal } // namespace Autotest diff --git a/src/plugins/autotest/qtest/qttestoutputreader.h b/src/plugins/autotest/qtest/qttestoutputreader.h index 027300c6e9c..28305160642 100644 --- a/src/plugins/autotest/qtest/qttestoutputreader.h +++ b/src/plugins/autotest/qtest/qttestoutputreader.h @@ -40,14 +40,32 @@ class QtTestOutputReader : public TestOutputReader Q_DECLARE_TR_FUNCTIONS(Autotest::Internal::QtTestOutputReader) public: + enum OutputMode + { + XML, + PlainText + }; + QtTestOutputReader(const QFutureInterface &futureInterface, - QProcess *testApplication, const QString &buildDirectory); + QProcess *testApplication, const QString &buildDirectory, + OutputMode mode); protected: void processOutput(const QByteArray &outputLine) override; private: + void processXMLOutput(const QByteArray &outputLine); + void processPlainTextOutput(const QByteArray &outputLine); + void processResultOutput(const QString &result, const QString &message); + void processLocationOutput(const QString &fileWithLine); + void processSummaryFinishOutput(); + // helper functions QtTestResult *createDefaultResult() const; + void sendCompleteInformation(); + void sendMessageCurrentTest(); + void sendStartMessage(bool isFunction); + void sendFinishMessage(bool isFunction); + void handleAndSendConfigMessage(const QRegExp &config); enum CDATAMode { @@ -70,6 +88,8 @@ private: int m_lineNumber = 0; QString m_duration; QXmlStreamReader m_xmlReader; + OutputMode m_mode = XML; + }; } // namespace Internal diff --git a/src/plugins/autotest/qtest/qttestsettings.cpp b/src/plugins/autotest/qtest/qttestsettings.cpp index 103d416989a..1b672cd29dd 100644 --- a/src/plugins/autotest/qtest/qttestsettings.cpp +++ b/src/plugins/autotest/qtest/qttestsettings.cpp @@ -30,6 +30,7 @@ namespace Internal { static const char metricsKey[] = "Metrics"; static const char noCrashhandlerKey[] = "NoCrashhandlerOnDebug"; +static const char useXMLOutputKey[] = "UseXMLOutput"; static MetricsType intToMetrics(int value) { @@ -58,12 +59,14 @@ void QtTestSettings::fromFrameworkSettings(const QSettings *s) { metrics = intToMetrics(s->value(metricsKey, Walltime).toInt()); noCrashHandler = s->value(noCrashhandlerKey, true).toBool(); + useXMLOutput = s->value(useXMLOutputKey, true).toBool(); } void QtTestSettings::toFrameworkSettings(QSettings *s) const { s->setValue(metricsKey, metrics); s->setValue(noCrashhandlerKey, noCrashHandler); + s->setValue(useXMLOutputKey, useXMLOutput); } QString QtTestSettings::metricsTypeToOption(const MetricsType type) diff --git a/src/plugins/autotest/qtest/qttestsettings.h b/src/plugins/autotest/qtest/qttestsettings.h index c80a93165e3..9418059243b 100644 --- a/src/plugins/autotest/qtest/qttestsettings.h +++ b/src/plugins/autotest/qtest/qttestsettings.h @@ -48,6 +48,7 @@ public: MetricsType metrics = Walltime; bool noCrashHandler = true; + bool useXMLOutput = true; protected: void fromFrameworkSettings(const QSettings *s) override; diff --git a/src/plugins/autotest/qtest/qttestsettingspage.cpp b/src/plugins/autotest/qtest/qttestsettingspage.cpp index 54e081a02d3..8258440ec4e 100644 --- a/src/plugins/autotest/qtest/qttestsettingspage.cpp +++ b/src/plugins/autotest/qtest/qttestsettingspage.cpp @@ -46,6 +46,7 @@ QtTestSettingsWidget::QtTestSettingsWidget(QWidget *parent) void QtTestSettingsWidget::setSettings(const QtTestSettings &settings) { m_ui.disableCrashhandlerCB->setChecked(settings.noCrashHandler); + m_ui.useXMLOutputCB->setChecked(settings.useXMLOutput); switch (settings.metrics) { case MetricsType::Walltime: m_ui.walltimeRB->setChecked(true); @@ -72,6 +73,7 @@ QtTestSettings QtTestSettingsWidget::settings() const QtTestSettings result; result.noCrashHandler = m_ui.disableCrashhandlerCB->isChecked(); + result.useXMLOutput = m_ui.useXMLOutputCB->isChecked(); if (m_ui.walltimeRB->isChecked()) result.metrics = MetricsType::Walltime; else if (m_ui.tickcounterRB->isChecked()) diff --git a/src/plugins/autotest/qtest/qttestsettingspage.ui b/src/plugins/autotest/qtest/qttestsettingspage.ui index c45233e4154..38837afee87 100644 --- a/src/plugins/autotest/qtest/qttestsettingspage.ui +++ b/src/plugins/autotest/qtest/qttestsettingspage.ui @@ -31,6 +31,21 @@ + + + + XML output recommended as it avoids parsing issues, while plain text is more human readable. + +Warning: Plain text output is missing some information (e.g. duration) + + + Use XML output + + + true + + + diff --git a/src/plugins/autotest/quick/quicktestconfiguration.cpp b/src/plugins/autotest/quick/quicktestconfiguration.cpp index b92c1252c02..4d3241f0f27 100644 --- a/src/plugins/autotest/quick/quicktestconfiguration.cpp +++ b/src/plugins/autotest/quick/quicktestconfiguration.cpp @@ -35,7 +35,16 @@ namespace Internal { TestOutputReader *QuickTestConfiguration::outputReader(const QFutureInterface &fi, QProcess *app) const { - return new QtTestOutputReader(fi, app, buildDirectory()); + static const Core::Id id + = Core::Id(Constants::FRAMEWORK_PREFIX).withSuffix(QtTest::Constants::FRAMEWORK_NAME); + TestFrameworkManager *manager = TestFrameworkManager::instance(); + auto qtSettings = qSharedPointerCast(manager->settingsForTestFramework(id)); + if (qtSettings.isNull()) + return nullptr; + if (qtSettings->useXMLOutput) + return new QtTestOutputReader(fi, app, buildDirectory(), QtTestOutputReader::XML); + else + return new QtTestOutputReader(fi, app, buildDirectory(), QtTestOutputReader::PlainText); } QStringList QuickTestConfiguration::argumentsForTestRunner() const @@ -43,14 +52,15 @@ QStringList QuickTestConfiguration::argumentsForTestRunner() const static const Core::Id id = Core::Id(Constants::FRAMEWORK_PREFIX).withSuffix(QtTest::Constants::FRAMEWORK_NAME); - QStringList arguments("-xml"); - if (testCases().count()) - arguments << testCases(); - + QStringList arguments; TestFrameworkManager *manager = TestFrameworkManager::instance(); auto qtSettings = qSharedPointerCast(manager->settingsForTestFramework(id)); if (qtSettings.isNull()) return arguments; + if (qtSettings->useXMLOutput) + arguments << "-xml"; + if (testCases().count()) + arguments << testCases(); const QString &metricsOption = QtTestSettings::metricsTypeToOption(qtSettings->metrics); if (!metricsOption.isEmpty()) diff --git a/src/plugins/autotest/testresult.cpp b/src/plugins/autotest/testresult.cpp index 794f7ad18e0..bfef7164034 100644 --- a/src/plugins/autotest/testresult.cpp +++ b/src/plugins/autotest/testresult.cpp @@ -56,7 +56,7 @@ Result::Type TestResult::resultFromString(const QString &resultString) { if (resultString == "pass") return Result::Pass; - if (resultString == "fail") + if (resultString == "fail" || resultString == "fail!") return Result::Fail; if (resultString == "xfail") return Result::ExpectedFail; @@ -64,11 +64,13 @@ Result::Type TestResult::resultFromString(const QString &resultString) return Result::UnexpectedPass; if (resultString == "skip") return Result::Skip; + if (resultString == "result") + return Result::Benchmark; if (resultString == "qdebug") return Result::MessageDebug; - if (resultString == "qinfo") + if (resultString == "qinfo" || resultString == "info") return Result::MessageInfo; - if (resultString == "warn" || resultString == "qwarn") + if (resultString == "warn" || resultString == "qwarn" || resultString == "warning") return Result::MessageWarn; if (resultString == "qfatal") return Result::MessageFatal;