AutoTest: Support plaintext output for Qt tests

Change-Id: I88ec477777d79c69e699dd906ec4ef1550bcaf44
Reviewed-by: David Schulz <david.schulz@qt.io>

src/plugins/autotest/qtest/qttestconfiguration.cpp

@@ -35,7 +35,17 @@ namespace Internal {
 TestOutputReader *QtTestConfiguration::outputReader(const QFutureInterface<TestResultPtr> &fi,
                                                     QProcess *app) const
 {
-    return new QtTestOutputReader(fi, app, buildDirectory());
+    static const Core::Id id
+            = Core::Id(Constants::FRAMEWORK_PREFIX).withSuffix(QtTest::Constants::FRAMEWORK_NAME);
+    TestFrameworkManager *manager = TestFrameworkManager::instance();
+    auto qtSettings = qSharedPointerCast<QtTestSettings>(manager->settingsForTestFramework(id));
+    if (qtSettings.isNull())
+        return nullptr;
+
+    if (qtSettings->useXMLOutput)
+        return new QtTestOutputReader(fi, app, buildDirectory(), QtTestOutputReader::XML);
+    else
+        return new QtTestOutputReader(fi, app, buildDirectory(), QtTestOutputReader::PlainText);
 }
 
 QStringList QtTestConfiguration::argumentsForTestRunner() const
@@ -43,14 +53,15 @@ QStringList QtTestConfiguration::argumentsForTestRunner() const
     static const Core::Id id
             = Core::Id(Constants::FRAMEWORK_PREFIX).withSuffix(QtTest::Constants::FRAMEWORK_NAME);
 
-    QStringList arguments("-xml");
+    QStringList arguments;
-    if (testCases().count())
-        arguments << testCases();
-
     TestFrameworkManager *manager = TestFrameworkManager::instance();
     auto qtSettings = qSharedPointerCast<QtTestSettings>(manager->settingsForTestFramework(id));
     if (qtSettings.isNull())
         return arguments;
+    if (qtSettings->useXMLOutput)
+        arguments << "-xml";
+    if (testCases().count())
+        arguments << testCases();
 
     const QString &metricsOption = QtTestSettings::metricsTypeToOption(qtSettings->metrics);
     if (!metricsOption.isEmpty())

src/plugins/autotest/qtest/qttestoutputreader.cpp

@@ -31,6 +31,7 @@
 #include <QDir>
 #include <QFileInfo>
 #include <QRegExp>
+#include <QRegularExpression>
 
 namespace Autotest {
 namespace Internal {
@@ -128,12 +129,26 @@ static QString constructSourceFilePath(const QString &path, const QString &fileP
 }
 
 QtTestOutputReader::QtTestOutputReader(const QFutureInterface<TestResultPtr> &futureInterface,
-                                       QProcess *testApplication, const QString &buildDirectory)
+                                       QProcess *testApplication, const QString &buildDirectory,
+                                       OutputMode mode)
     : TestOutputReader(futureInterface, testApplication, buildDirectory)
+    , m_mode(mode)
 {
 }
 
 void QtTestOutputReader::processOutput(const QByteArray &outputLine)
+{
+    switch (m_mode) {
+    case PlainText:
+        processPlainTextOutput(outputLine);
+        break;
+    case XML:
+        processXMLOutput(outputLine);
+        break;
+    }
+}
+
+void QtTestOutputReader::processXMLOutput(const QByteArray &outputLine)
 {
     static QStringList validEndTags = {QStringLiteral("Incident"),
                                        QStringLiteral("Message"),
@@ -162,24 +177,14 @@ void QtTestOutputReader::processOutput(const QByteArray &outputLine)
             if (currentTag == QStringLiteral("TestCase")) {
                 m_className = m_xmlReader.attributes().value(QStringLiteral("name")).toString();
                 QTC_ASSERT(!m_className.isEmpty(), continue);
-                TestResultPtr testResult = TestResultPtr(createDefaultResult());
+                sendStartMessage(false);
-                testResult->setResult(Result::MessageTestCaseStart);
-                testResult->setDescription(tr("Executing test case %1").arg(m_className));
-                m_futureInterface.reportResult(testResult);
             } else if (currentTag == QStringLiteral("TestFunction")) {
                 m_testCase = m_xmlReader.attributes().value(QStringLiteral("name")).toString();
                 QTC_ASSERT(!m_testCase.isEmpty(), continue);
                 if (m_testCase == m_formerTestCase)  // don't report "Executing..." more than once
                     continue;
-                TestResultPtr testResult = TestResultPtr(createDefaultResult());
+                sendStartMessage(true);
-                testResult->setResult(Result::MessageTestCaseStart);
+                sendMessageCurrentTest();
-                testResult->setDescription(tr("Executing test function %1").arg(m_testCase));
-                m_futureInterface.reportResult(testResult);
-                testResult = TestResultPtr(new QtTestResult);
-                testResult->setResult(Result::MessageCurrentTest);
-                testResult->setDescription(tr("Entering test function %1::%2").arg(m_className,
-                                                                                   m_testCase));
-                m_futureInterface.reportResult(testResult);
             } else if (currentTag == QStringLiteral("Duration")) {
                 m_duration = m_xmlReader.attributes().value(QStringLiteral("msecs")).toString();
                 QTC_ASSERT(!m_duration.isEmpty(), continue);
@@ -255,23 +260,13 @@ void QtTestOutputReader::processOutput(const QByteArray &outputLine)
             m_cdataMode = None;
             const QStringRef currentTag = m_xmlReader.name();
             if (currentTag == QStringLiteral("TestFunction")) {
-                QtTestResult *testResult = createDefaultResult();
+                sendFinishMessage(true);
-                testResult->setResult(Result::MessageTestCaseEnd);
-                testResult->setDescription(
-                            m_duration.isEmpty() ? tr("Test function finished.")
-                                                 : tr("Execution took %1 ms.").arg(m_duration));
-                m_futureInterface.reportResult(TestResultPtr(testResult));
                 m_futureInterface.setProgressValue(m_futureInterface.progressValue() + 1);
                 m_dataTag.clear();
                 m_formerTestCase = m_testCase;
                 m_testCase.clear();
             } else if (currentTag == QStringLiteral("TestCase")) {
-                QtTestResult *testResult = createDefaultResult();
+                sendFinishMessage(false);
-                testResult->setResult(Result::MessageTestCaseEnd);
-                testResult->setDescription(
-                            m_duration.isEmpty() ? tr("Test finished.")
-                                                 : tr("Test execution took %1 ms.").arg(m_duration));
-                m_futureInterface.reportResult(TestResultPtr(testResult));
             } else if (validEndTags.contains(currentTag.toString())) {
                 QtTestResult *testResult = createDefaultResult();
                 testResult->setResult(m_result);
@@ -290,6 +285,138 @@ void QtTestOutputReader::processOutput(const QByteArray &outputLine)
     }
 }
 
+static QStringList extractFunctionInformation(const QString &testClassName,
+                                              const QString &lineWithoutResultType,
+                                              Result::Type resultType)
+{
+    static QRegularExpression classInformation("^(.+?)\\((.*?)\\)(.*)$");
+    QStringList result;
+    const QRegularExpressionMatch match = classInformation.match(lineWithoutResultType);
+    if (match.hasMatch()) {
+        QString fullQualifiedFunc = match.captured(1);
+        QTC_ASSERT(fullQualifiedFunc.startsWith(testClassName + "::"), return result);
+        fullQualifiedFunc = fullQualifiedFunc.mid(testClassName.length() + 2);
+        result.append(fullQualifiedFunc);
+        if (resultType == Result::Benchmark) { // tag is displayed differently
+            QString possiblyTag = match.captured(3);
+            if (!possiblyTag.isEmpty())
+                possiblyTag = possiblyTag.mid(2, possiblyTag.length() - 4);
+            result.append(possiblyTag);
+            result.append(QString());
+        } else {
+            result.append(match.captured(2));
+            result.append(match.captured(3));
+        }
+    }
+    return result;
+}
+
+void QtTestOutputReader::processPlainTextOutput(const QByteArray &outputLine)
+{
+    static QRegExp start("^[*]{9} Start testing of (.*) [*]{9}$");
+    static QRegExp config("^Config: Using QtTest library (.*), (Qt (\\d+(\\.\\d+){2}) \\(.*\\))$");
+    static QRegExp summary("^Totals: \\d+ passed, \\d+ failed, \\d+ skipped(, \\d+ blacklisted)?$");
+    static QRegExp finish("^[*]{9} Finished testing of (.*) [*]{9}$");
+
+    static QRegExp result("^(PASS   |FAIL!  |XFAIL  |XPASS  |SKIP   |BPASS   |BFAIL   |RESULT "
+                          "|INFO    |QWARN  |WARNING|QDEBUG ): (.*)$");
+
+    static QRegExp benchDetails("^\\s+([\\d,.]+ .* per iteration \\(total: [\\d,.]+, iterations: \\d+\\))$");
+    static QRegExp locationUnix("^   Loc: \\[(.*)\\]$");
+    static QRegExp locationWin("^(.*\\(\\d+\\)) : failure location$");
+
+    if (m_futureInterface.isCanceled())
+        return;
+
+    const QString &line = QString::fromLatin1(outputLine);
+
+    if (result.exactMatch(line)) {
+        processResultOutput(result.cap(1).toLower().trimmed(), result.cap(2));
+    } else if (locationUnix.exactMatch(line)) {
+        processLocationOutput(locationUnix.cap(1));
+    } else if (locationWin.exactMatch(line)) {
+        processLocationOutput(locationWin.cap(1));
+    } else if (benchDetails.exactMatch(line)) {
+        m_description = benchDetails.cap(1);
+    } else if (config.exactMatch(line)) {
+        handleAndSendConfigMessage(config);
+    } else if (start.exactMatch(line)) {
+        m_className = start.cap(1);
+        QTC_CHECK(!m_className.isEmpty());
+        sendStartMessage(false);
+    } else if (summary.exactMatch(line) || finish.exactMatch(line)) {
+        processSummaryFinishOutput();
+    } else { // we have some plain output, but we cannot say where for sure it belongs to..
+        if (!m_description.isEmpty())
+            m_description.append('\n');
+        m_description.append(line);
+    }
+}
+
+void QtTestOutputReader::processResultOutput(const QString &result, const QString &message)
+{
+    if (!m_testCase.isEmpty()) { // report the former result if there is any
+        sendCompleteInformation();
+        m_dataTag.clear();
+        m_description.clear();
+        m_file.clear();
+        m_lineNumber = 0;
+    }
+    m_result = TestResult::resultFromString(result);
+    const QStringList funcWithTag = extractFunctionInformation(m_className, message, m_result);
+    QTC_ASSERT(funcWithTag.size() == 3, return);
+    m_testCase = funcWithTag.at(0);
+    if (m_testCase != m_formerTestCase) { // new test function executed
+        if (!m_formerTestCase.isEmpty()) {
+            using namespace std;
+            swap(m_testCase, m_formerTestCase); // we want formerTestCase to be reported
+            sendFinishMessage(true);
+            swap(m_testCase, m_formerTestCase);
+        }
+        sendStartMessage(true);
+        sendMessageCurrentTest();
+    }
+    m_dataTag = funcWithTag.at(1);
+    const QString description = funcWithTag.at(2);
+    if (!description.isEmpty()) {
+        if (!m_description.isEmpty())
+            m_description.append('\n');
+        m_description.append(description.mid(1)); // cut the first whitespace
+    }
+    m_formerTestCase = m_testCase;
+}
+
+void QtTestOutputReader::processLocationOutput(const QString &fileWithLine)
+{
+    QTC_ASSERT(fileWithLine.endsWith(')'), return);
+    int openBrace = fileWithLine.lastIndexOf('(');
+    QTC_ASSERT(openBrace != -1, return);
+    m_file = constructSourceFilePath(m_buildDir, fileWithLine.left(openBrace));
+    QString numberStr = fileWithLine.mid(openBrace + 1);
+    numberStr.chop(1);
+    m_lineNumber = numberStr.toInt();
+}
+
+void QtTestOutputReader::processSummaryFinishOutput()
+{
+    if (m_className.isEmpty()) // we have reported already
+        return;
+    // we still have something to report
+    sendCompleteInformation();
+    m_dataTag.clear();
+    // report finished function
+    sendFinishMessage(true);
+    m_testCase.clear();
+    m_formerTestCase.clear();
+    // create and report the finish message for this test class
+    sendFinishMessage(false);
+    m_className.clear();
+    m_description.clear();
+    m_result = Result::Invalid;
+    m_file.clear();
+    m_lineNumber = 0;
+}
+
 QtTestResult *QtTestOutputReader::createDefaultResult() const
 {
     QtTestResult *result = new QtTestResult(m_className);
@@ -298,5 +425,63 @@ QtTestResult *QtTestOutputReader::createDefaultResult() const
     return result;
 }
 
+void QtTestOutputReader::sendCompleteInformation()
+{
+    TestResultPtr testResult = TestResultPtr(createDefaultResult());
+    testResult->setResult(m_result);
+    testResult->setFileName(m_file);
+    testResult->setLine(m_lineNumber);
+    testResult->setDescription(m_description);
+    m_futureInterface.reportResult(testResult);
+}
+
+void QtTestOutputReader::sendMessageCurrentTest()
+{
+    TestResultPtr testResult = TestResultPtr(new QtTestResult);
+    testResult->setResult(Result::MessageCurrentTest);
+    testResult->setDescription(tr("Entering test function %1::%2").arg(m_className, m_testCase));
+    m_futureInterface.reportResult(testResult);
+}
+
+void QtTestOutputReader::sendStartMessage(bool isFunction)
+{
+    TestResultPtr testResult = TestResultPtr(createDefaultResult());
+    testResult->setResult(Result::MessageTestCaseStart);
+    testResult->setDescription(isFunction ? tr("Executing test function %1").arg(m_testCase)
+                                          : tr("Executing test case %1").arg(m_className));
+    m_futureInterface.reportResult(testResult);
+}
+
+void QtTestOutputReader::sendFinishMessage(bool isFunction)
+{
+    TestResultPtr testResult = TestResultPtr(createDefaultResult());
+    testResult->setResult(Result::MessageTestCaseEnd);
+    if (m_duration.isEmpty()) {
+        testResult->setDescription(isFunction ? tr("Execution took %1 ms.").arg(m_duration)
+                                              : tr("Test execution took %1 ms.").arg(m_duration));
+    } else {
+        testResult->setDescription(isFunction ? tr("Test function finished.")
+                                              : tr("Test finished."));
+    }
+    m_futureInterface.reportResult(testResult);
+}
+
+// TODO factor out tr() strings to avoid duplication (see XML processing of Characters)
+void QtTestOutputReader::handleAndSendConfigMessage(const QRegExp &config)
+{
+    QtTestResult *testResult = createDefaultResult();
+    testResult->setResult(Result::MessageInternal);
+    testResult->setDescription(tr("Qt version: %1").arg(config.cap(3)));
+    m_futureInterface.reportResult(TestResultPtr(testResult));
+    testResult = createDefaultResult();
+    testResult->setResult(Result::MessageInternal);
+    testResult->setDescription(tr("Qt build: %1").arg(config.cap(2)));
+    m_futureInterface.reportResult(TestResultPtr(testResult));
+    testResult = createDefaultResult();
+    testResult->setResult(Result::MessageInternal);
+    testResult->setDescription(tr("QTest version: %1").arg(config.cap(1)));
+    m_futureInterface.reportResult(TestResultPtr(testResult));
+}
+
 } // namespace Internal
 } // namespace Autotest

src/plugins/autotest/qtest/qttestoutputreader.h

@@ -40,14 +40,32 @@ class QtTestOutputReader : public TestOutputReader
     Q_DECLARE_TR_FUNCTIONS(Autotest::Internal::QtTestOutputReader)
 
 public:
+    enum OutputMode
+    {
+        XML,
+        PlainText
+    };
+
     QtTestOutputReader(const QFutureInterface<TestResultPtr> &futureInterface,
-                       QProcess *testApplication, const QString &buildDirectory);
+                       QProcess *testApplication, const QString &buildDirectory,
+                       OutputMode mode);
 
 protected:
     void processOutput(const QByteArray &outputLine) override;
 
 private:
+    void processXMLOutput(const QByteArray &outputLine);
+    void processPlainTextOutput(const QByteArray &outputLine);
+    void processResultOutput(const QString &result, const QString &message);
+    void processLocationOutput(const QString &fileWithLine);
+    void processSummaryFinishOutput();
+    // helper functions
     QtTestResult *createDefaultResult() const;
+    void sendCompleteInformation();
+    void sendMessageCurrentTest();
+    void sendStartMessage(bool isFunction);
+    void sendFinishMessage(bool isFunction);
+    void handleAndSendConfigMessage(const QRegExp &config);
 
     enum CDATAMode
     {
@@ -70,6 +88,8 @@ private:
     int m_lineNumber = 0;
     QString m_duration;
     QXmlStreamReader m_xmlReader;
+    OutputMode m_mode = XML;
+
 };
 
 } // namespace Internal

src/plugins/autotest/qtest/qttestsettings.cpp

@@ -30,6 +30,7 @@ namespace Internal {
 
 static const char metricsKey[]           = "Metrics";
 static const char noCrashhandlerKey[]    = "NoCrashhandlerOnDebug";
+static const char useXMLOutputKey[]      = "UseXMLOutput";
 
 static MetricsType intToMetrics(int value)
 {
@@ -58,12 +59,14 @@ void QtTestSettings::fromFrameworkSettings(const QSettings *s)
 {
     metrics = intToMetrics(s->value(metricsKey, Walltime).toInt());
     noCrashHandler = s->value(noCrashhandlerKey, true).toBool();
+    useXMLOutput = s->value(useXMLOutputKey, true).toBool();
 }
 
 void QtTestSettings::toFrameworkSettings(QSettings *s) const
 {
     s->setValue(metricsKey, metrics);
     s->setValue(noCrashhandlerKey, noCrashHandler);
+    s->setValue(useXMLOutputKey, useXMLOutput);
 }
 
 QString QtTestSettings::metricsTypeToOption(const MetricsType type)

src/plugins/autotest/qtest/qttestsettings.h

@@ -48,6 +48,7 @@ public:
 
     MetricsType metrics = Walltime;
     bool noCrashHandler = true;
+    bool useXMLOutput = true;
 
 protected:
     void fromFrameworkSettings(const QSettings *s) override;

src/plugins/autotest/qtest/qttestsettingspage.cpp

@@ -46,6 +46,7 @@ QtTestSettingsWidget::QtTestSettingsWidget(QWidget *parent)
 void QtTestSettingsWidget::setSettings(const QtTestSettings &settings)
 {
     m_ui.disableCrashhandlerCB->setChecked(settings.noCrashHandler);
+    m_ui.useXMLOutputCB->setChecked(settings.useXMLOutput);
     switch (settings.metrics) {
     case MetricsType::Walltime:
         m_ui.walltimeRB->setChecked(true);
@@ -72,6 +73,7 @@ QtTestSettings QtTestSettingsWidget::settings() const
     QtTestSettings result;
 
     result.noCrashHandler = m_ui.disableCrashhandlerCB->isChecked();
+    result.useXMLOutput = m_ui.useXMLOutputCB->isChecked();
     if (m_ui.walltimeRB->isChecked())
         result.metrics = MetricsType::Walltime;
     else if (m_ui.tickcounterRB->isChecked())

src/plugins/autotest/qtest/qttestsettingspage.ui

@@ -31,6 +31,21 @@
          </property>
         </widget>
        </item>
+       <item>
+        <widget class="QCheckBox" name="useXMLOutputCB">
+         <property name="toolTip">
+          <string>XML output recommended as it avoids parsing issues, while plain text is more human readable.
+
+Warning: Plain text output is missing some information (e.g. duration)</string>
+         </property>
+         <property name="text">
+          <string>Use XML output</string>
+         </property>
+         <property name="checked">
+          <bool>true</bool>
+         </property>
+        </widget>
+       </item>
        <item>
         <widget class="QGroupBox" name="groupBox">
          <property name="title">

src/plugins/autotest/quick/quicktestconfiguration.cpp

@@ -35,7 +35,16 @@ namespace Internal {
 TestOutputReader *QuickTestConfiguration::outputReader(const QFutureInterface<TestResultPtr> &fi,
                                                        QProcess *app) const
 {
-    return new QtTestOutputReader(fi, app, buildDirectory());
+    static const Core::Id id
+            = Core::Id(Constants::FRAMEWORK_PREFIX).withSuffix(QtTest::Constants::FRAMEWORK_NAME);
+    TestFrameworkManager *manager = TestFrameworkManager::instance();
+    auto qtSettings = qSharedPointerCast<QtTestSettings>(manager->settingsForTestFramework(id));
+    if (qtSettings.isNull())
+        return nullptr;
+    if (qtSettings->useXMLOutput)
+        return new QtTestOutputReader(fi, app, buildDirectory(), QtTestOutputReader::XML);
+    else
+        return new QtTestOutputReader(fi, app, buildDirectory(), QtTestOutputReader::PlainText);
 }
 
 QStringList QuickTestConfiguration::argumentsForTestRunner() const
@@ -43,14 +52,15 @@ QStringList QuickTestConfiguration::argumentsForTestRunner() const
     static const Core::Id id
             = Core::Id(Constants::FRAMEWORK_PREFIX).withSuffix(QtTest::Constants::FRAMEWORK_NAME);
 
-    QStringList arguments("-xml");
+    QStringList arguments;
-    if (testCases().count())
-        arguments << testCases();
-
     TestFrameworkManager *manager = TestFrameworkManager::instance();
     auto qtSettings = qSharedPointerCast<QtTestSettings>(manager->settingsForTestFramework(id));
     if (qtSettings.isNull())
         return arguments;
+    if (qtSettings->useXMLOutput)
+        arguments << "-xml";
+    if (testCases().count())
+        arguments << testCases();
 
     const QString &metricsOption = QtTestSettings::metricsTypeToOption(qtSettings->metrics);
     if (!metricsOption.isEmpty())

src/plugins/autotest/testresult.cpp

@@ -56,7 +56,7 @@ Result::Type TestResult::resultFromString(const QString &resultString)
 {
     if (resultString == "pass")
         return Result::Pass;
-    if (resultString == "fail")
+    if (resultString == "fail" || resultString == "fail!")
         return Result::Fail;
     if (resultString == "xfail")
         return Result::ExpectedFail;
@@ -64,11 +64,13 @@ Result::Type TestResult::resultFromString(const QString &resultString)
         return Result::UnexpectedPass;
     if (resultString == "skip")
         return Result::Skip;
+    if (resultString == "result")
+        return Result::Benchmark;
     if (resultString == "qdebug")
         return Result::MessageDebug;
-    if (resultString == "qinfo")
+    if (resultString == "qinfo" || resultString == "info")
         return Result::MessageInfo;
-    if (resultString == "warn" || resultString == "qwarn")
+    if (resultString == "warn" || resultString == "qwarn" || resultString == "warning")
         return Result::MessageWarn;
     if (resultString == "qfatal")
         return Result::MessageFatal;