AutoTest: Refactor output handling

Handle getting output from application's process inside base class
and just process output inside the sub classes.
Additionally this is a preparation for being able to process output
for debugging tests as well.

Change-Id: I8a2289dc7faab25afe08530b5021a0318f3ba6a6
Reviewed-by: David Schulz <david.schulz@qt.io>
This commit is contained in:
Christian Stenger
2016-07-21 10:08:11 +02:00
parent ce8bff4b31
commit bb643a3cdc
6 changed files with 240 additions and 245 deletions

View File

@@ -43,11 +43,8 @@ GTestOutputReader::GTestOutputReader(const QFutureInterface<TestResultPtr> &futu
{
}
void GTestOutputReader::processOutput()
void GTestOutputReader::processOutput(const QByteArray &outputLine)
{
if (!m_testApplication || m_testApplication->state() != QProcess::Running)
return;
static QRegExp newTestStarts(QStringLiteral("^\\[-{10}\\] \\d+ tests? from (.*)$"));
static QRegExp testEnds(QStringLiteral("^\\[-{10}\\] \\d+ tests? from (.*) \\((.*)\\)$"));
static QRegExp newTestSetStarts(QStringLiteral("^\\[ RUN \\] (.*)$"));
@@ -58,121 +55,117 @@ void GTestOutputReader::processOutput()
static QRegExp errorLocation(QStringLiteral("^(.*)\\((\\d+)\\): error:.*$"));
static QRegExp iterations(QStringLiteral("^Repeating all tests \\(iteration (\\d+)\\) . . .$"));
while (m_testApplication->canReadLine()) {
if (m_futureInterface.isCanceled())
return;
QByteArray read = m_testApplication->readLine();
if (!m_unprocessed.isEmpty()) {
read = m_unprocessed + read;
m_unprocessed.clear();
}
if (!read.endsWith('\n')) {
m_unprocessed = read;
continue;
}
read.chop(1); // remove the newline from the output
if (read.endsWith('\r'))
read.chop(1);
QByteArray read = outputLine;
if (!m_unprocessed.isEmpty()) {
read = m_unprocessed + read;
m_unprocessed.clear();
}
if (!read.endsWith('\n')) {
m_unprocessed = read;
return;
}
read.chop(1); // remove the newline from the output
if (read.endsWith('\r'))
read.chop(1);
const QString line = QString::fromLatin1(read);
if (line.trimmed().isEmpty())
continue;
const QString line = QString::fromLatin1(read);
if (line.trimmed().isEmpty())
return;
if (!line.startsWith(QLatin1Char('['))) {
m_description.append(line).append(QLatin1Char('\n'));
if (iterations.exactMatch(line)) {
m_iteration = iterations.cap(1).toInt();
m_description.clear();
} else if (line.startsWith(QStringLiteral("Note:"))) {
TestResultPtr testResult = TestResultPtr(new GTestResult());
testResult->setResult(Result::MessageInternal);
testResult->setDescription(line);
m_futureInterface.reportResult(testResult);
m_description.clear();
} else if (disabledTests.exactMatch(line)) {
TestResultPtr testResult = TestResultPtr(new GTestResult());
testResult->setResult(Result::MessageDisabledTests);
int disabled = disabledTests.cap(1).toInt();
testResult->setDescription(tr("You have %n disabled test(s).", 0, disabled));
testResult->setLine(disabled); // misuse line property to hold number of disabled
m_futureInterface.reportResult(testResult);
m_description.clear();
}
continue;
}
if (testEnds.exactMatch(line)) {
GTestResult *testResult = new GTestResult(m_currentTestName);
testResult->setTestSetName(m_currentTestSet);
testResult->setResult(Result::MessageTestCaseEnd);
testResult->setDescription(tr("Test execution took %1").arg(testEnds.cap(2)));
m_futureInterface.reportResult(TestResultPtr(testResult));
m_currentTestName.clear();
m_currentTestSet.clear();
} else if (newTestStarts.exactMatch(line)) {
m_currentTestName = newTestStarts.cap(1);
TestResultPtr testResult = TestResultPtr(new GTestResult(m_currentTestName));
if (m_iteration > 1) {
testResult->setResult(Result::MessageTestCaseRepetition);
testResult->setDescription(tr("Repeating test case %1 (iteration %2)")
.arg(m_currentTestName).arg(m_iteration));
} else {
testResult->setResult(Result::MessageTestCaseStart);
testResult->setDescription(tr("Executing test case %1").arg(m_currentTestName));
}
m_futureInterface.reportResult(testResult);
} else if (newTestSetStarts.exactMatch(line)) {
m_currentTestSet = newTestSetStarts.cap(1);
if (!line.startsWith(QLatin1Char('['))) {
m_description.append(line).append(QLatin1Char('\n'));
if (iterations.exactMatch(line)) {
m_iteration = iterations.cap(1).toInt();
m_description.clear();
} else if (line.startsWith(QStringLiteral("Note:"))) {
TestResultPtr testResult = TestResultPtr(new GTestResult());
testResult->setResult(Result::MessageCurrentTest);
testResult->setDescription(tr("Entering test set %1").arg(m_currentTestSet));
testResult->setResult(Result::MessageInternal);
testResult->setDescription(line);
m_futureInterface.reportResult(testResult);
m_description.clear();
} else if (testSetSuccess.exactMatch(line)) {
GTestResult *testResult = new GTestResult(m_currentTestName);
testResult->setTestSetName(m_currentTestSet);
testResult->setResult(Result::Pass);
testResult->setDescription(m_description);
m_futureInterface.reportResult(TestResultPtr(testResult));
} else if (disabledTests.exactMatch(line)) {
TestResultPtr testResult = TestResultPtr(new GTestResult());
testResult->setResult(Result::MessageDisabledTests);
int disabled = disabledTests.cap(1).toInt();
testResult->setDescription(tr("You have %n disabled test(s).", 0, disabled));
testResult->setLine(disabled); // misuse line property to hold number of disabled
m_futureInterface.reportResult(testResult);
m_description.clear();
testResult = new GTestResult(m_currentTestName);
testResult->setTestSetName(m_currentTestSet);
testResult->setResult(Result::MessageInternal);
testResult->setDescription(tr("Execution took %1.").arg(testSetSuccess.cap(2)));
m_futureInterface.reportResult(TestResultPtr(testResult));
m_futureInterface.setProgressValue(m_futureInterface.progressValue() + 1);
} else if (testSetFail.exactMatch(line)) {
GTestResult *testResult = new GTestResult(m_currentTestName);
testResult->setTestSetName(m_currentTestSet);
testResult->setResult(Result::Fail);
m_description.chop(1);
testResult->setDescription(m_description);
}
return; //continue;
}
foreach (const QString &output, m_description.split(QLatin1Char('\n'))) {
QRegExp *match = 0;
if (failureLocation.exactMatch(output))
match = &failureLocation;
else if (errorLocation.exactMatch(output))
match = &errorLocation;
if (testEnds.exactMatch(line)) {
GTestResult *testResult = new GTestResult(m_currentTestName);
testResult->setTestSetName(m_currentTestSet);
testResult->setResult(Result::MessageTestCaseEnd);
testResult->setDescription(tr("Test execution took %1").arg(testEnds.cap(2)));
m_futureInterface.reportResult(TestResultPtr(testResult));
m_currentTestName.clear();
m_currentTestSet.clear();
} else if (newTestStarts.exactMatch(line)) {
m_currentTestName = newTestStarts.cap(1);
TestResultPtr testResult = TestResultPtr(new GTestResult(m_currentTestName));
if (m_iteration > 1) {
testResult->setResult(Result::MessageTestCaseRepetition);
testResult->setDescription(tr("Repeating test case %1 (iteration %2)")
.arg(m_currentTestName).arg(m_iteration));
} else {
testResult->setResult(Result::MessageTestCaseStart);
testResult->setDescription(tr("Executing test case %1").arg(m_currentTestName));
}
m_futureInterface.reportResult(testResult);
} else if (newTestSetStarts.exactMatch(line)) {
m_currentTestSet = newTestSetStarts.cap(1);
TestResultPtr testResult = TestResultPtr(new GTestResult());
testResult->setResult(Result::MessageCurrentTest);
testResult->setDescription(tr("Entering test set %1").arg(m_currentTestSet));
m_futureInterface.reportResult(testResult);
m_description.clear();
} else if (testSetSuccess.exactMatch(line)) {
GTestResult *testResult = new GTestResult(m_currentTestName);
testResult->setTestSetName(m_currentTestSet);
testResult->setResult(Result::Pass);
testResult->setDescription(m_description);
m_futureInterface.reportResult(TestResultPtr(testResult));
m_description.clear();
testResult = new GTestResult(m_currentTestName);
testResult->setTestSetName(m_currentTestSet);
testResult->setResult(Result::MessageInternal);
testResult->setDescription(tr("Execution took %1.").arg(testSetSuccess.cap(2)));
m_futureInterface.reportResult(TestResultPtr(testResult));
m_futureInterface.setProgressValue(m_futureInterface.progressValue() + 1);
} else if (testSetFail.exactMatch(line)) {
GTestResult *testResult = new GTestResult(m_currentTestName);
testResult->setTestSetName(m_currentTestSet);
testResult->setResult(Result::Fail);
m_description.chop(1);
testResult->setDescription(m_description);
if (match) {
QString file = constructSourceFilePath(m_buildDir, match->cap(1));
if (!file.isEmpty()) {
testResult->setFileName(file);
testResult->setLine(match->cap(2).toInt());
break;
}
foreach (const QString &output, m_description.split(QLatin1Char('\n'))) {
QRegExp *match = 0;
if (failureLocation.exactMatch(output))
match = &failureLocation;
else if (errorLocation.exactMatch(output))
match = &errorLocation;
if (match) {
QString file = constructSourceFilePath(m_buildDir, match->cap(1));
if (!file.isEmpty()) {
testResult->setFileName(file);
testResult->setLine(match->cap(2).toInt());
break;
}
}
m_futureInterface.reportResult(TestResultPtr(testResult));
m_description.clear();
testResult = new GTestResult(m_currentTestName);
testResult->setTestSetName(m_currentTestSet);
testResult->setResult(Result::MessageInternal);
testResult->setDescription(tr("Execution took %1.").arg(testSetFail.cap(2)));
m_futureInterface.reportResult(TestResultPtr(testResult));
m_futureInterface.setProgressValue(m_futureInterface.progressValue() + 1);
}
m_futureInterface.reportResult(TestResultPtr(testResult));
m_description.clear();
testResult = new GTestResult(m_currentTestName);
testResult->setTestSetName(m_currentTestSet);
testResult->setResult(Result::MessageInternal);
testResult->setDescription(tr("Execution took %1.").arg(testSetFail.cap(2)));
m_futureInterface.reportResult(TestResultPtr(testResult));
m_futureInterface.setProgressValue(m_futureInterface.progressValue() + 1);
}
}

View File

@@ -37,7 +37,7 @@ public:
QProcess *testApplication, const QString &buildDirectory);
protected:
void processOutput() override;
void processOutput(const QByteArray &outputLine) override;
private:
QString m_currentTestName;

View File

@@ -133,10 +133,8 @@ QtTestOutputReader::QtTestOutputReader(const QFutureInterface<TestResultPtr> &fu
{
}
void QtTestOutputReader::processOutput()
void QtTestOutputReader::processOutput(const QByteArray &outputLine)
{
if (!m_testApplication || m_testApplication->state() != QProcess::Running)
return;
static QStringList validEndTags = { QStringLiteral("Incident"),
QStringLiteral("Message"),
QStringLiteral("BenchmarkResult"),
@@ -144,142 +142,140 @@ void QtTestOutputReader::processOutput()
QStringLiteral("QtBuild"),
QStringLiteral("QTestVersion") };
while (m_testApplication->canReadLine()) {
m_xmlReader.addData(m_testApplication->readLine());
while (!m_xmlReader.atEnd()) {
if (m_futureInterface.isCanceled())
return;
QXmlStreamReader::TokenType token = m_xmlReader.readNext();
switch (token) {
case QXmlStreamReader::StartDocument:
m_className.clear();
break;
case QXmlStreamReader::EndDocument:
m_xmlReader.clear();
return;
case QXmlStreamReader::StartElement: {
const QString currentTag = m_xmlReader.name().toString();
if (currentTag == QStringLiteral("TestCase")) {
m_className = m_xmlReader.attributes().value(QStringLiteral("name")).toString();
QTC_ASSERT(!m_className.isEmpty(), continue);
TestResultPtr testResult = TestResultPtr(new QtTestResult(m_className));
testResult->setResult(Result::MessageTestCaseStart);
testResult->setDescription(tr("Executing test case %1").arg(m_className));
m_futureInterface.reportResult(testResult);
} else if (currentTag == QStringLiteral("TestFunction")) {
m_testCase = m_xmlReader.attributes().value(QStringLiteral("name")).toString();
QTC_ASSERT(!m_testCase.isEmpty(), continue);
TestResultPtr testResult = TestResultPtr(new QtTestResult());
testResult->setResult(Result::MessageCurrentTest);
testResult->setDescription(tr("Entering test function %1::%2").arg(m_className,
m_testCase));
m_futureInterface.reportResult(testResult);
} else if (currentTag == QStringLiteral("Duration")) {
m_duration = m_xmlReader.attributes().value(QStringLiteral("msecs")).toString();
QTC_ASSERT(!m_duration.isEmpty(), continue);
} else if (currentTag == QStringLiteral("Message")
|| currentTag == QStringLiteral("Incident")) {
m_dataTag.clear();
m_description.clear();
m_duration.clear();
m_file.clear();
m_result = Result::Invalid;
m_lineNumber = 0;
const QXmlStreamAttributes &attributes = m_xmlReader.attributes();
m_result = TestResult::resultFromString(
attributes.value(QStringLiteral("type")).toString());
m_file = decode(attributes.value(QStringLiteral("file")).toString());
if (!m_file.isEmpty()) {
m_file = constructSourceFilePath(m_buildDir, m_file);
}
m_lineNumber = attributes.value(QStringLiteral("line")).toInt();
} else if (currentTag == QStringLiteral("BenchmarkResult")) {
const QXmlStreamAttributes &attributes = m_xmlReader.attributes();
const QString metric = attributes.value(QStringLiteral("metrics")).toString();
const double value = attributes.value(QStringLiteral("value")).toDouble();
const int iterations = attributes.value(QStringLiteral("iterations")).toInt();
m_description = constructBenchmarkInformation(metric, value, iterations);
m_result = Result::Benchmark;
} else if (currentTag == QStringLiteral("DataTag")) {
m_cdataMode = DataTag;
} else if (currentTag == QStringLiteral("Description")) {
m_cdataMode = Description;
} else if (currentTag == QStringLiteral("QtVersion")) {
m_result = Result::MessageInternal;
m_cdataMode = QtVersion;
} else if (currentTag == QStringLiteral("QtBuild")) {
m_result = Result::MessageInternal;
m_cdataMode = QtBuild;
} else if (currentTag == QStringLiteral("QTestVersion")) {
m_result = Result::MessageInternal;
m_cdataMode = QTestVersion;
m_xmlReader.addData(outputLine);
while (!m_xmlReader.atEnd()) {
if (m_futureInterface.isCanceled())
return;
QXmlStreamReader::TokenType token = m_xmlReader.readNext();
switch (token) {
case QXmlStreamReader::StartDocument:
m_className.clear();
break;
case QXmlStreamReader::EndDocument:
m_xmlReader.clear();
return;
case QXmlStreamReader::StartElement: {
const QString currentTag = m_xmlReader.name().toString();
if (currentTag == QStringLiteral("TestCase")) {
m_className = m_xmlReader.attributes().value(QStringLiteral("name")).toString();
QTC_ASSERT(!m_className.isEmpty(), continue);
TestResultPtr testResult = TestResultPtr(new QtTestResult(m_className));
testResult->setResult(Result::MessageTestCaseStart);
testResult->setDescription(tr("Executing test case %1").arg(m_className));
m_futureInterface.reportResult(testResult);
} else if (currentTag == QStringLiteral("TestFunction")) {
m_testCase = m_xmlReader.attributes().value(QStringLiteral("name")).toString();
QTC_ASSERT(!m_testCase.isEmpty(), continue);
TestResultPtr testResult = TestResultPtr(new QtTestResult());
testResult->setResult(Result::MessageCurrentTest);
testResult->setDescription(tr("Entering test function %1::%2").arg(m_className,
m_testCase));
m_futureInterface.reportResult(testResult);
} else if (currentTag == QStringLiteral("Duration")) {
m_duration = m_xmlReader.attributes().value(QStringLiteral("msecs")).toString();
QTC_ASSERT(!m_duration.isEmpty(), continue);
} else if (currentTag == QStringLiteral("Message")
|| currentTag == QStringLiteral("Incident")) {
m_dataTag.clear();
m_description.clear();
m_duration.clear();
m_file.clear();
m_result = Result::Invalid;
m_lineNumber = 0;
const QXmlStreamAttributes &attributes = m_xmlReader.attributes();
m_result = TestResult::resultFromString(
attributes.value(QStringLiteral("type")).toString());
m_file = decode(attributes.value(QStringLiteral("file")).toString());
if (!m_file.isEmpty()) {
m_file = constructSourceFilePath(m_buildDir, m_file);
}
break;
m_lineNumber = attributes.value(QStringLiteral("line")).toInt();
} else if (currentTag == QStringLiteral("BenchmarkResult")) {
const QXmlStreamAttributes &attributes = m_xmlReader.attributes();
const QString metric = attributes.value(QStringLiteral("metrics")).toString();
const double value = attributes.value(QStringLiteral("value")).toDouble();
const int iterations = attributes.value(QStringLiteral("iterations")).toInt();
m_description = constructBenchmarkInformation(metric, value, iterations);
m_result = Result::Benchmark;
} else if (currentTag == QStringLiteral("DataTag")) {
m_cdataMode = DataTag;
} else if (currentTag == QStringLiteral("Description")) {
m_cdataMode = Description;
} else if (currentTag == QStringLiteral("QtVersion")) {
m_result = Result::MessageInternal;
m_cdataMode = QtVersion;
} else if (currentTag == QStringLiteral("QtBuild")) {
m_result = Result::MessageInternal;
m_cdataMode = QtBuild;
} else if (currentTag == QStringLiteral("QTestVersion")) {
m_result = Result::MessageInternal;
m_cdataMode = QTestVersion;
}
case QXmlStreamReader::Characters: {
QStringRef text = m_xmlReader.text().trimmed();
if (text.isEmpty())
break;
break;
}
case QXmlStreamReader::Characters: {
QStringRef text = m_xmlReader.text().trimmed();
if (text.isEmpty())
break;
switch (m_cdataMode) {
case DataTag:
m_dataTag = text.toString();
break;
case Description:
if (!m_description.isEmpty())
m_description.append(QLatin1Char('\n'));
m_description.append(text);
break;
case QtVersion:
m_description = tr("Qt version: %1").arg(text.toString());
break;
case QtBuild:
m_description = tr("Qt build: %1").arg(text.toString());
break;
case QTestVersion:
m_description = tr("QTest version: %1").arg(text.toString());
break;
default:
// this must come from plain printf() calls - but this will be ignored anyhow
qWarning() << "AutoTest.Run: Ignored plain output:" << text.toString();
break;
}
switch (m_cdataMode) {
case DataTag:
m_dataTag = text.toString();
break;
case Description:
if (!m_description.isEmpty())
m_description.append(QLatin1Char('\n'));
m_description.append(text);
break;
case QtVersion:
m_description = tr("Qt version: %1").arg(text.toString());
break;
case QtBuild:
m_description = tr("Qt build: %1").arg(text.toString());
break;
case QTestVersion:
m_description = tr("QTest version: %1").arg(text.toString());
break;
default:
// this must come from plain printf() calls - but this will be ignored anyhow
qWarning() << "AutoTest.Run: Ignored plain output:" << text.toString();
break;
}
case QXmlStreamReader::EndElement: {
m_cdataMode = None;
const QStringRef currentTag = m_xmlReader.name();
if (currentTag == QStringLiteral("TestFunction")) {
if (!m_duration.isEmpty()) {
QtTestResult *testResult = new QtTestResult(m_className);
testResult->setFunctionName(m_testCase);
testResult->setResult(Result::MessageInternal);
testResult->setDescription(tr("Execution took %1 ms.").arg(m_duration));
m_futureInterface.reportResult(TestResultPtr(testResult));
}
m_futureInterface.setProgressValue(m_futureInterface.progressValue() + 1);
} else if (currentTag == QStringLiteral("TestCase")) {
QtTestResult *testResult = new QtTestResult(m_className);
testResult->setResult(Result::MessageTestCaseEnd);
testResult->setDescription(
m_duration.isEmpty() ? tr("Test finished.")
: tr("Test execution took %1 ms.").arg(m_duration));
m_futureInterface.reportResult(TestResultPtr(testResult));
} else if (validEndTags.contains(currentTag.toString())) {
break;
}
case QXmlStreamReader::EndElement: {
m_cdataMode = None;
const QStringRef currentTag = m_xmlReader.name();
if (currentTag == QStringLiteral("TestFunction")) {
if (!m_duration.isEmpty()) {
QtTestResult *testResult = new QtTestResult(m_className);
testResult->setFunctionName(m_testCase);
testResult->setDataTag(m_dataTag);
testResult->setResult(m_result);
testResult->setFileName(m_file);
testResult->setLine(m_lineNumber);
testResult->setDescription(m_description);
testResult->setResult(Result::MessageInternal);
testResult->setDescription(tr("Execution took %1 ms.").arg(m_duration));
m_futureInterface.reportResult(TestResultPtr(testResult));
}
break;
}
default:
break;
m_futureInterface.setProgressValue(m_futureInterface.progressValue() + 1);
} else if (currentTag == QStringLiteral("TestCase")) {
QtTestResult *testResult = new QtTestResult(m_className);
testResult->setResult(Result::MessageTestCaseEnd);
testResult->setDescription(
m_duration.isEmpty() ? tr("Test finished.")
: tr("Test execution took %1 ms.").arg(m_duration));
m_futureInterface.reportResult(TestResultPtr(testResult));
} else if (validEndTags.contains(currentTag.toString())) {
QtTestResult *testResult = new QtTestResult(m_className);
testResult->setFunctionName(m_testCase);
testResult->setDataTag(m_dataTag);
testResult->setResult(m_result);
testResult->setFileName(m_file);
testResult->setLine(m_lineNumber);
testResult->setDescription(m_description);
m_futureInterface.reportResult(TestResultPtr(testResult));
}
break;
}
default:
break;
}
}
}

View File

@@ -39,7 +39,7 @@ public:
QProcess *testApplication, const QString &buildDirectory);
protected:
void processOutput() override;
void processOutput(const QByteArray &outputLine) override;
private:
enum CDATAMode

View File

@@ -38,14 +38,20 @@ TestOutputReader::TestOutputReader(const QFutureInterface<TestResultPtr> &future
, m_testApplication(testApplication)
, m_buildDir(buildDirectory)
{
connect(m_testApplication, &QProcess::readyRead, this, &TestOutputReader::processOutput);
connect(m_testApplication, &QProcess::readyRead,
this, [this] () {
while (m_testApplication->canReadLine())
processOutput(m_testApplication->readLine());
});
connect(m_testApplication, &QProcess::readyReadStandardError,
this, &TestOutputReader::processStdError);
this, [this] () {
processStdError(m_testApplication->readAllStandardError());
});
}
void TestOutputReader::processStdError()
void TestOutputReader::processStdError(const QByteArray &output)
{
qWarning() << "AutoTest.Run: Ignored plain output:" << m_testApplication->readAllStandardError();
qWarning() << "AutoTest.Run: Ignored plain output:" << output;
}
} // namespace Internal

View File

@@ -43,8 +43,8 @@ public:
QProcess *testApplication, const QString &buildDirectory);
protected:
virtual void processOutput() = 0;
virtual void processStdError();
virtual void processOutput(const QByteArray &outputLine) = 0;
virtual void processStdError(const QByteArray &output);
QFutureInterface<TestResultPtr> m_futureInterface;
QProcess *m_testApplication; // not owned
QString m_buildDir;