AutoTest: Improve process error handling

If the test executable cannot be started or it seems
not to be the correct one or wrong arguments have been
passed to it the user now gets a respective message
inside the results pane.

Task-number: QTCREATORBUG-18955
Change-Id: Ica68cdbb9e401c8d48a9ce8b23b65d5410d2075a
Reviewed-by: Leena Miettinen <riitta-leena.miettinen@qt.io>
Reviewed-by: Oliver Wolff <oliver.wolff@qt.io>

src/plugins/autotest/gtest/gtestoutputreader.cpp

@@ -77,7 +77,7 @@ void GTestOutputReader::processOutput(const QByteArray &outputLine)
             TestResultPtr testResult = TestResultPtr(new GTestResult(m_projectFile));
             testResult->setResult(Result::MessageInternal);
             testResult->setDescription(m_description);
-            m_futureInterface.reportResult(testResult);
+            reportResult(testResult);
             m_description.clear();
         } else if (disabledTests.exactMatch(line)) {
             TestResultPtr testResult = TestResultPtr(new GTestResult(m_projectFile));
@@ -85,7 +85,7 @@ void GTestOutputReader::processOutput(const QByteArray &outputLine)
             int disabled = disabledTests.cap(1).toInt();
             testResult->setDescription(tr("You have %n disabled test(s).", 0, disabled));
             testResult->setLine(disabled); // misuse line property to hold number of disabled
-            m_futureInterface.reportResult(testResult);
+            reportResult(testResult);
             m_description.clear();
         }
         return;
@@ -95,7 +95,7 @@ void GTestOutputReader::processOutput(const QByteArray &outputLine)
         GTestResult *testResult = createDefaultResult();
         testResult->setResult(Result::MessageTestCaseEnd);
         testResult->setDescription(tr("Test execution took %1").arg(testEnds.cap(2)));
-        m_futureInterface.reportResult(TestResultPtr(testResult));
+        reportResult(TestResultPtr(testResult));
         m_currentTestName.clear();
         m_currentTestSet.clear();
     } else if (newTestStarts.exactMatch(line)) {
@@ -108,24 +108,24 @@ void GTestOutputReader::processOutput(const QByteArray &outputLine)
         } else {
             testResult->setDescription(tr("Executing test case %1").arg(m_currentTestName));
         }
-        m_futureInterface.reportResult(testResult);
+        reportResult(testResult);
     } else if (newTestSetStarts.exactMatch(line)) {
         setCurrentTestSet(newTestSetStarts.cap(1));
         TestResultPtr testResult = TestResultPtr(new GTestResult(m_projectFile));
         testResult->setResult(Result::MessageCurrentTest);
         testResult->setDescription(tr("Entering test set %1").arg(m_currentTestSet));
-        m_futureInterface.reportResult(testResult);
+        reportResult(testResult);
         m_description.clear();
     } else if (testSetSuccess.exactMatch(line)) {
         GTestResult *testResult = createDefaultResult();
         testResult->setResult(Result::Pass);
         testResult->setDescription(m_description);
-        m_futureInterface.reportResult(TestResultPtr(testResult));
+        reportResult(TestResultPtr(testResult));
         m_description.clear();
         testResult = createDefaultResult();
         testResult->setResult(Result::MessageInternal);
         testResult->setDescription(tr("Execution took %1.").arg(testSetSuccess.cap(2)));
-        m_futureInterface.reportResult(TestResultPtr(testResult));
+        reportResult(TestResultPtr(testResult));
         m_futureInterface.setProgressValue(m_futureInterface.progressValue() + 1);
     } else if (testSetFail.exactMatch(line)) {
         GTestResult *testResult = createDefaultResult();
@@ -149,12 +149,12 @@ void GTestOutputReader::processOutput(const QByteArray &outputLine)
                 }
             }
         }
-        m_futureInterface.reportResult(TestResultPtr(testResult));
+        reportResult(TestResultPtr(testResult));
         m_description.clear();
         testResult = createDefaultResult();
         testResult->setResult(Result::MessageInternal);
         testResult->setDescription(tr("Execution took %1.").arg(testSetFail.cap(2)));
-        m_futureInterface.reportResult(TestResultPtr(testResult));
+        reportResult(TestResultPtr(testResult));
         m_futureInterface.setProgressValue(m_futureInterface.progressValue() + 1);
     }
 }

src/plugins/autotest/qtest/qttestoutputreader.cpp

@@ -274,7 +274,7 @@ void QtTestOutputReader::processXMLOutput(const QByteArray &outputLine)
                 testResult->setFileName(m_file);
                 testResult->setLine(m_lineNumber);
                 testResult->setDescription(m_description);
-                m_futureInterface.reportResult(TestResultPtr(testResult));
+                reportResult(TestResultPtr(testResult));
                 if (currentTag == QStringLiteral("Incident"))
                     m_dataTag.clear();
             }
@@ -433,7 +433,7 @@ void QtTestOutputReader::sendCompleteInformation()
     testResult->setFileName(m_file);
     testResult->setLine(m_lineNumber);
     testResult->setDescription(m_description);
-    m_futureInterface.reportResult(testResult);
+    reportResult(testResult);
 }
 
 void QtTestOutputReader::sendMessageCurrentTest()
@@ -441,7 +441,7 @@ void QtTestOutputReader::sendMessageCurrentTest()
     TestResultPtr testResult = TestResultPtr(new QtTestResult);
     testResult->setResult(Result::MessageCurrentTest);
     testResult->setDescription(tr("Entering test function %1::%2").arg(m_className, m_testCase));
-    m_futureInterface.reportResult(testResult);
+    reportResult(testResult);
 }
 
 void QtTestOutputReader::sendStartMessage(bool isFunction)
@@ -450,7 +450,7 @@ void QtTestOutputReader::sendStartMessage(bool isFunction)
     testResult->setResult(Result::MessageTestCaseStart);
     testResult->setDescription(isFunction ? tr("Executing test function %1").arg(m_testCase)
                                           : tr("Executing test case %1").arg(m_className));
-    m_futureInterface.reportResult(testResult);
+    reportResult(testResult);
 }
 
 void QtTestOutputReader::sendFinishMessage(bool isFunction)
@@ -464,7 +464,7 @@ void QtTestOutputReader::sendFinishMessage(bool isFunction)
         testResult->setDescription(isFunction ? tr("Test function finished.")
                                               : tr("Test finished."));
     }
-    m_futureInterface.reportResult(testResult);
+    reportResult(testResult);
 }
 
 // TODO factor out tr() strings to avoid duplication (see XML processing of Characters)
@@ -473,15 +473,15 @@ void QtTestOutputReader::handleAndSendConfigMessage(const QRegExp &config)
     QtTestResult *testResult = createDefaultResult();
     testResult->setResult(Result::MessageInternal);
     testResult->setDescription(tr("Qt version: %1").arg(config.cap(3)));
-    m_futureInterface.reportResult(TestResultPtr(testResult));
+    reportResult(TestResultPtr(testResult));
     testResult = createDefaultResult();
     testResult->setResult(Result::MessageInternal);
     testResult->setDescription(tr("Qt build: %1").arg(config.cap(2)));
-    m_futureInterface.reportResult(TestResultPtr(testResult));
+    reportResult(TestResultPtr(testResult));
     testResult = createDefaultResult();
     testResult->setResult(Result::MessageInternal);
     testResult->setDescription(tr("QTest version: %1").arg(config.cap(1)));
-    m_futureInterface.reportResult(TestResultPtr(testResult));
+    reportResult(TestResultPtr(testResult));
 }
 
 } // namespace Internal

src/plugins/autotest/testoutputreader.cpp

@@ -66,5 +66,11 @@ void TestOutputReader::processStdError(const QByteArray &output)
     qWarning() << "AutoTest.Run: Ignored plain output:" << output;
 }
 
+void TestOutputReader::reportResult(const TestResultPtr &result)
+{
+    m_futureInterface.reportResult(result);
+    m_hadValidOutput = true;
+}
+
 } // namespace Internal
 } // namespace Autotest

src/plugins/autotest/testoutputreader.h

@@ -44,13 +44,17 @@ public:
 
     virtual void processOutput(const QByteArray &outputLine) = 0;
     virtual void processStdError(const QByteArray &output);
+    bool hadValidOutput() const { return m_hadValidOutput; }
 
 signals:
     void newOutputAvailable(const QByteArray &output);
 protected:
+    void reportResult(const TestResultPtr &result);
     QFutureInterface<TestResultPtr> m_futureInterface;
     QProcess *m_testApplication;  // not owned
     QString m_buildDir;
+private:
+    bool m_hadValidOutput = false;
 };
 
 } // namespace Internal

src/plugins/autotest/testrunner.cpp

@@ -257,6 +257,11 @@ static void performTestRun(QFutureInterface<TestResultPtr> &futureInterface,
                 TestRunner::tr("Test for project \"%1\" crashed.")
                     .arg(testConfiguration->displayName()) + processInformation(testProcess)
                                                            + rcInfo(testConfiguration))));
+        } else if (!outputReader->hadValidOutput()) {
+            futureInterface.reportResult(TestResultPtr(new FaultyTestResult(Result::MessageFatal,
+                TestRunner::tr("Test for project \"%1\" did not produce any expected output.")
+                    .arg(testConfiguration->displayName()) + processInformation(testProcess)
+                                                           + rcInfo(testConfiguration))));
         }
 
         if (canceledByTimeout) {