feedc0de/qt-creator
1
0
Fork 0
forked from qt-creator/qt-creator
Code Pull Requests Activity

AutoTest: Use reporting summary from Boost UTF

Browse Source 
Handle all combinations of LogFormat and ReportFormat
correctly and be able to summarize the test result
correctly by reporting the correct number of Passes,
Fails and Skips.

Change-Id: I47e6fefa2bb2cbb7a25f2ed474a3bb2da1e1e769
Reviewed-by: David Schulz <david.schulz@qt.io>
This commit is contained in:
Christian Stenger
2019-05-27 13:31:32 +02:00
parent 12038dd012
commit 77902649a8
1 changed files with 42 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

48
src/plugins/autotest/boost/boosttestoutputreader.cpp
View File

@@ -133,6 +133,8 @@ void BoostTestOutputReader::handleMessageMatch(const QRegularExpressionMatch &ma
if (m_currentTest.isEmpty() || m_logLevel > LogLevel::UnitScope)
m_currentTest = caseFromContent(content);
m_result = ResultType::Fail;
if (m_reportLevel == ReportLevel::No)
++m_summary[ResultType::Fail];
m_description = content;
} else if (content.startsWith("fatal error:")) {
if (m_currentTest.isEmpty() || m_logLevel > LogLevel::UnitScope)
@@ -171,6 +173,8 @@ void BoostTestOutputReader::handleMessageMatch(const QRegularExpressionMatch &ma
} else if (content.startsWith("Test case ")) {
m_currentTest = match.captured(4);
m_result = ResultType::Skip;
if (m_reportLevel == ReportLevel::Confirm || m_reportLevel == ReportLevel::No)
++m_summary[ResultType::Skip];
m_description = content;
}
@@ -193,10 +197,10 @@ void BoostTestOutputReader::processOutputLine(const QByteArray &outputLineWithNe
static QRegularExpression noAssertion("^Test case (.*) did not check any assertions$");
static QRegularExpression summaryPreamble("^\\s*Test (module|suite|case) \"(.*)\" has "
"(failed|passed) with:$");
"(failed|passed)( with:)?$");
static QRegularExpression summarySkip("^\\s+Test case \"(.*)\" was skipped$");
static QRegularExpression summaryDetail("^\\s+(\\d+) test cases? out of (\\d+) "
"(failed|passed)$");
"(failed|passed|skipped)$");
static QRegularExpression summaryAssertion("^\\s+(\\d+) assertions? out of (\\d+) "
"(failed|passed)$");
@@ -284,16 +288,34 @@ void BoostTestOutputReader::processOutputLine(const QByteArray &outputLineWithNe
return;
}
// should summary get reported unconditionally?
match = summaryPreamble.match(line);
if (match.hasMatch()) {
createAndReportResult(match.captured(0), ResultType::MessageInfo);
if (m_reportLevel == ReportLevel::Detailed || match.captured(4).isEmpty()) {
if (match.captured(1) == "case") {
if (match.captured(3) == "passed")
++m_summary[ResultType::Pass];
else
++m_summary[ResultType::Fail];
}
}
return;
}
match = summaryDetail.match(line);
if (match.hasMatch()) {
createAndReportResult(match.captured(0), ResultType::MessageInfo);
int report = match.captured(1).toInt();
QString type = match.captured(3);
if (m_reportLevel != ReportLevel::Detailed) {
if (type == "passed")
m_summary[ResultType::Pass] += report;
else if (type == "failed")
m_summary[ResultType::Fail] += report;
else if (type == "skipped")
m_summary[ResultType::Skip] += report;
}
return;
}
@@ -306,6 +328,8 @@ void BoostTestOutputReader::processOutputLine(const QByteArray &outputLineWithNe
match = summarySkip.match(line);
if (match.hasMatch()) {
createAndReportResult(match.captured(0), ResultType::MessageInfo);
if (m_reportLevel == ReportLevel::Detailed)
++m_summary[ResultType::Skip];
return;
}
@@ -316,11 +340,17 @@ void BoostTestOutputReader::processOutputLine(const QByteArray &outputLineWithNe
BoostTestResult *result = new BoostTestResult(id(), m_projectFile, QString());
int failed = match.captured(1).toInt();
QString txt = tr("%1 failures detected in %2.").arg(failed).arg(match.captured(3));
int passed = (m_testCaseCount != -1)
? m_testCaseCount - failed - m_summary[ResultType::Skip] : -1;
if (m_testCaseCount != -1)
txt.append(' ').append(tr("%1 tests passed.").arg(m_testCaseCount - failed));
txt.append(' ').append(tr("%1 tests passed.").arg(passed));
result->setDescription(txt);
result->setResult(ResultType::MessageInfo); // TODO report similar to disabled tests
result->setResult(ResultType::MessageInfo);
reportResult(TestResultPtr(result));
if (m_reportLevel == ReportLevel::Confirm) { // for the final summary
m_summary[ResultType::Pass] += passed;
m_summary[ResultType::Fail] += failed;
}
m_testCaseCount = -1;
return;
}
@@ -333,8 +363,10 @@ void BoostTestOutputReader::processOutputLine(const QByteArray &outputLineWithNe
if (m_testCaseCount != -1)
txt.append(' ').append(tr("%1 tests passed.").arg(m_testCaseCount));
result->setDescription(txt);
result->setResult(ResultType::MessageInfo); // TODO report similar to disabled tests
result->setResult(ResultType::MessageInfo);
reportResult(TestResultPtr(result));
if (m_reportLevel == ReportLevel::Confirm) // for the final summary
m_summary.insert(ResultType::Pass, m_testCaseCount);
return;
}
@@ -379,6 +411,10 @@ TestResultPtr BoostTestOutputReader::createDefaultResult() const
}
void BoostTestOutputReader::onFinished(int exitCode, QProcess::ExitStatus /*exitState*/) {
if (m_reportLevel == ReportLevel::No && m_testCaseCount != -1) {
int reportedFailsAndSkips = m_summary[ResultType::Fail] + m_summary[ResultType::Skip];
m_summary.insert(ResultType::Pass, m_testCaseCount - reportedFailsAndSkips);
}
// boost::exit_success (0), boost::exit_test_failure (201)
// or boost::exit_exception_failure (200)
// be graceful and do not add a fatal for exit_test_failure