forked from qt-creator/qt-creator
2c7e769e31
Instead of transforming forth and back the output try to handle the output once correctly and pass it line-wise around. This also ensures that we always get a single line when appending the output which will be necessary later on. Change-Id: I3e9c6db5f81172997dfe566eee9a86bfe2f17a1f Reviewed-by: David Schulz <david.schulz@qt.io>
441 lines
18 KiB
C++
441 lines
18 KiB
C++
/****************************************************************************
|
|
**
|
|
** Copyright (C) 2019 The Qt Company Ltd.
|
|
** Contact: https://www.qt.io/licensing/
|
|
**
|
|
** This file is part of Qt Creator.
|
|
**
|
|
** Commercial License Usage
|
|
** Licensees holding valid commercial Qt licenses may use this file in
|
|
** accordance with the commercial license agreement provided with the
|
|
** Software or, alternatively, in accordance with the terms contained in
|
|
** a written agreement between you and The Qt Company. For licensing terms
|
|
** and conditions see https://www.qt.io/terms-conditions. For further
|
|
** information use the contact form at https://www.qt.io/contact-us.
|
|
**
|
|
** GNU General Public License Usage
|
|
** Alternatively, this file may be used under the terms of the GNU
|
|
** General Public License version 3 as published by the Free Software
|
|
** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
|
|
** included in the packaging of this file. Please review the following
|
|
** information to ensure the GNU General Public License requirements will
|
|
** be met: https://www.gnu.org/licenses/gpl-3.0.html.
|
|
**
|
|
****************************************************************************/
|
|
|
|
#include "boosttestoutputreader.h"
|
|
#include "boosttestsettings.h"
|
|
#include "boosttestresult.h"
|
|
|
|
#include <utils/qtcassert.h>
|
|
|
|
#include <QDir>
|
|
#include <QFileInfo>
|
|
#include <QLoggingCategory>
|
|
#include <QRegularExpression>
|
|
|
|
namespace Autotest {
|
|
namespace Internal {
|
|
|
|
static Q_LOGGING_CATEGORY(orLog, "qtc.autotest.boost.outputreader", QtWarningMsg)
|
|
|
|
static QString constructSourceFilePath(const QString &path, const QString &filePath)
|
|
{
|
|
return QFileInfo(path, filePath).canonicalFilePath();
|
|
}
|
|
|
|
BoostTestOutputReader::BoostTestOutputReader(const QFutureInterface<TestResultPtr> &futureInterface,
|
|
QProcess *testApplication,
|
|
const QString &buildDirectory,
|
|
const QString &projectFile,
|
|
LogLevel log, ReportLevel report)
|
|
: TestOutputReader(futureInterface, testApplication, buildDirectory)
|
|
, m_projectFile(projectFile)
|
|
, m_logLevel(log)
|
|
, m_reportLevel(report)
|
|
{
|
|
if (m_testApplication) {
|
|
connect(m_testApplication, QOverload<int, QProcess::ExitStatus>::of(&QProcess::finished),
|
|
this, &BoostTestOutputReader::onFinished);
|
|
}
|
|
}
|
|
|
|
// content of "error:..." / "info:..." / ... messages
|
|
static QString caseFromContent(const QString &content)
|
|
{
|
|
const int length = content.length();
|
|
if (content.startsWith("last checkpoint:")) {
|
|
int index = content.indexOf('"');
|
|
if (index != 17 || length <= 18) {
|
|
qCDebug(orLog) << "double quote position" << index << " or content length" << length
|
|
<< "wrong on content" << content;
|
|
return QString();
|
|
}
|
|
index = content.indexOf('"', 18);
|
|
if (index == -1) {
|
|
qCDebug(orLog) << "no closing double quote" << content;
|
|
return QString();
|
|
}
|
|
return content.mid(18, index - 1);
|
|
}
|
|
|
|
int index = content.indexOf(": in ");
|
|
if (index == -1) // "info: check true has passed"
|
|
return QString();
|
|
|
|
if (index <= 4 || length < index + 4) {
|
|
qCDebug(orLog) << "unexpected position" << index << "for info" << content;
|
|
return QString();
|
|
}
|
|
|
|
QString result = content.mid(index + 5);
|
|
static QRegularExpression functionName("\"(.+)\":.*");
|
|
const QRegularExpressionMatch matcher = functionName.match(result);
|
|
if (!matcher.hasMatch()) {
|
|
qCDebug(orLog) << "got no match";
|
|
return QString();
|
|
}
|
|
return matcher.captured(1);
|
|
}
|
|
|
|
void BoostTestOutputReader::sendCompleteInformation()
|
|
{
|
|
QTC_ASSERT(m_result != ResultType::Invalid, return);
|
|
BoostTestResult *result = new BoostTestResult(id(), m_projectFile, m_currentModule);
|
|
result->setTestSuite(m_currentSuite);
|
|
result->setTestCase(m_currentTest);
|
|
if (m_lineNumber) {
|
|
result->setLine(m_lineNumber);
|
|
result->setFileName(m_fileName);
|
|
}
|
|
|
|
result->setDescription(m_description);
|
|
result->setResult(m_result);
|
|
reportResult(TestResultPtr(result));
|
|
m_result = ResultType::Invalid;
|
|
}
|
|
|
|
void BoostTestOutputReader::handleMessageMatch(const QRegularExpressionMatch &match)
|
|
{
|
|
m_fileName = constructSourceFilePath(m_buildDir, match.captured(1));
|
|
m_lineNumber = match.captured(2).toInt();
|
|
|
|
const QString &content = match.captured(3);
|
|
if (content.startsWith("info:")) {
|
|
if (m_currentTest.isEmpty() || m_logLevel > LogLevel::UnitScope) {
|
|
QString tmp = caseFromContent(content);
|
|
if (!tmp.isEmpty())
|
|
m_currentTest = tmp;
|
|
}
|
|
m_result = ResultType::Pass;
|
|
m_description = content;
|
|
} else if (content.startsWith("error:")) {
|
|
if (m_currentTest.isEmpty() || m_logLevel > LogLevel::UnitScope)
|
|
m_currentTest = caseFromContent(content);
|
|
m_result = ResultType::Fail;
|
|
if (m_reportLevel == ReportLevel::No)
|
|
++m_summary[ResultType::Fail];
|
|
m_description = content;
|
|
} else if (content.startsWith("fatal error:")) {
|
|
if (m_currentTest.isEmpty() || m_logLevel > LogLevel::UnitScope)
|
|
m_currentTest = caseFromContent(content);
|
|
m_result = ResultType::MessageFatal;
|
|
m_description = content;
|
|
} else if (content.startsWith("last checkpoint:")) {
|
|
if (m_currentTest.isEmpty() || m_logLevel > LogLevel::UnitScope)
|
|
m_currentTest = caseFromContent(content);
|
|
m_result = ResultType::MessageInfo;
|
|
m_description = content;
|
|
} else if (content.startsWith("Entering")) {
|
|
m_result = ResultType::TestStart;
|
|
const QString type = match.captured(8);
|
|
if (type == "case") {
|
|
m_currentTest = match.captured(9);
|
|
m_description = tr("Executing test case %1").arg(m_currentTest);
|
|
} else if (type == "suite") {
|
|
m_currentSuite = match.captured(9);
|
|
m_description = tr("Executing test suite %1").arg(m_currentSuite);
|
|
}
|
|
} else if (content.startsWith("Leaving")) {
|
|
const QString type = match.captured(10);
|
|
if (type == "case") {
|
|
if (m_currentTest != match.captured(11) && m_currentTest.isEmpty())
|
|
m_currentTest = match.captured(11);
|
|
m_result = ResultType::TestEnd;
|
|
m_description = tr("Test execution took %1").arg(match.captured(12));
|
|
} else if (type == "suite") {
|
|
if (m_currentSuite != match.captured(11) && m_currentSuite.isEmpty())
|
|
m_currentSuite = match.captured(11);
|
|
m_currentTest.clear();
|
|
m_result = ResultType::TestEnd;
|
|
m_description = tr("Test suite execution took %1").arg(match.captured(12));
|
|
}
|
|
} else if (content.startsWith("Test case ")) {
|
|
m_currentTest = match.captured(4);
|
|
m_result = ResultType::Skip;
|
|
if (m_reportLevel == ReportLevel::Confirm || m_reportLevel == ReportLevel::No)
|
|
++m_summary[ResultType::Skip];
|
|
m_description = content;
|
|
}
|
|
|
|
if (m_result != ResultType::Invalid) // we got a new result
|
|
sendCompleteInformation();
|
|
}
|
|
|
|
void BoostTestOutputReader::processOutputLine(const QByteArray &outputLine)
|
|
{
|
|
static QRegularExpression newTestStart("^Running (\\d+) test cases?\\.\\.\\.$");
|
|
static QRegularExpression dependency("^Including test case (.+) as a dependency of "
|
|
"test case (.+)$");
|
|
static QRegularExpression messages("^(.+)\\((\\d+)\\): (info: (.+)|error: (.+)|"
|
|
"fatal error: (.+)|last checkpoint: (.+)"
|
|
"|Entering test (case|suite) \"(.+)\""
|
|
"|Leaving test (case|suite) \"(.+)\"; testing time: (\\d+.+)"
|
|
"|Test case \"(.+)\" is skipped because .+$)$");
|
|
static QRegularExpression moduleMssg("^(Entering test module \"(.+)\"|"
|
|
"Leaving test module \"(.+)\"; testing time: (\\d+.+))$");
|
|
static QRegularExpression noAssertion("^Test case (.*) did not check any assertions$");
|
|
|
|
static QRegularExpression summaryPreamble("^\\s*Test (module|suite|case) \"(.*)\" has "
|
|
"(failed|passed)( with:)?$");
|
|
static QRegularExpression summarySkip("^\\s+Test case \"(.*)\" was skipped$");
|
|
static QRegularExpression summaryDetail("^\\s+(\\d+) test cases? out of (\\d+) "
|
|
"(failed|passed|skipped)$");
|
|
static QRegularExpression summaryAssertion("^\\s+(\\d+) assertions? out of (\\d+) "
|
|
"(failed|passed)$");
|
|
|
|
static QRegularExpression finish("^\\*{3} (\\d+) failure(s are| is) detected in the "
|
|
"test module \"(.*)\"$");
|
|
static QRegularExpression errDetect("^\\*{3} Errors where detected in the "
|
|
"test module \"(.*}\"; see standard output for details");
|
|
QString noErrors("*** No errors detected");
|
|
|
|
const QString line = QString::fromUtf8(outputLine);
|
|
if (line.trimmed().isEmpty())
|
|
return;
|
|
|
|
QRegularExpressionMatch match = messages.match(line);
|
|
if (match.hasMatch()) {
|
|
handleMessageMatch(match);
|
|
return;
|
|
}
|
|
|
|
match = dependency.match(line);
|
|
if (match.hasMatch()) {
|
|
if (m_result != ResultType::Invalid)
|
|
sendCompleteInformation();
|
|
BoostTestResult *result = new BoostTestResult(id(), m_projectFile, m_currentModule);
|
|
result->setDescription(match.captured(0));
|
|
result->setResult(ResultType::MessageInfo);
|
|
reportResult(TestResultPtr(result));
|
|
return;
|
|
}
|
|
|
|
match = newTestStart.match(line);
|
|
if (match.hasMatch()) {
|
|
if (m_result != ResultType::Invalid)
|
|
sendCompleteInformation();
|
|
m_testCaseCount = match.captured(1).toInt();
|
|
m_description.clear();
|
|
return;
|
|
}
|
|
|
|
match = moduleMssg.match(line);
|
|
if (match.hasMatch()) {
|
|
if (m_result != ResultType::Invalid)
|
|
sendCompleteInformation();
|
|
if (match.captured(1).startsWith("Entering")) {
|
|
m_currentModule = match.captured(2);
|
|
BoostTestResult *result = new BoostTestResult(id(), m_projectFile, m_currentModule);
|
|
result->setDescription(tr("Executing test module %1").arg(m_currentModule));
|
|
result->setResult(ResultType::TestStart);
|
|
reportResult(TestResultPtr(result));
|
|
m_description.clear();
|
|
} else {
|
|
QTC_CHECK(m_currentModule == match.captured(3));
|
|
BoostTestResult *result = new BoostTestResult(id(), m_projectFile, m_currentModule);
|
|
result->setDescription(tr("Test module execution took %1").arg(match.captured(4)));
|
|
result->setResult(ResultType::TestEnd);
|
|
reportResult(TestResultPtr(result));
|
|
|
|
m_currentTest.clear();
|
|
m_currentSuite.clear();
|
|
m_currentModule.clear();
|
|
m_description.clear();
|
|
}
|
|
return;
|
|
}
|
|
|
|
match = noAssertion.match(line);
|
|
if (match.hasMatch()) {
|
|
if (m_result != ResultType::Invalid)
|
|
sendCompleteInformation();
|
|
const QString caseWithOptionalSuite = match.captured(1);
|
|
int index = caseWithOptionalSuite.lastIndexOf('/');
|
|
if (index == -1) {
|
|
QTC_CHECK(caseWithOptionalSuite == m_currentTest);
|
|
} else {
|
|
QTC_CHECK(caseWithOptionalSuite.mid(index + 1) == m_currentTest);
|
|
int sIndex = caseWithOptionalSuite.lastIndexOf('/', index - 1);
|
|
if (sIndex == -1) {
|
|
QTC_CHECK(caseWithOptionalSuite.left(index) == m_currentSuite);
|
|
m_currentSuite = caseWithOptionalSuite.left(index); // FIXME should not be necessary - but we currently do not care for the whole suite path
|
|
} else {
|
|
QTC_CHECK(caseWithOptionalSuite.mid(sIndex + 1, index - sIndex - 1) == m_currentSuite);
|
|
}
|
|
}
|
|
createAndReportResult(match.captured(0), ResultType::MessageWarn);
|
|
return;
|
|
}
|
|
|
|
match = summaryPreamble.match(line);
|
|
if (match.hasMatch()) {
|
|
createAndReportResult(match.captured(0), ResultType::MessageInfo);
|
|
if (m_reportLevel == ReportLevel::Detailed || match.captured(4).isEmpty()) {
|
|
if (match.captured(1) == "case") {
|
|
if (match.captured(3) == "passed")
|
|
++m_summary[ResultType::Pass];
|
|
else
|
|
++m_summary[ResultType::Fail];
|
|
}
|
|
}
|
|
return;
|
|
}
|
|
|
|
match = summaryDetail.match(line);
|
|
if (match.hasMatch()) {
|
|
createAndReportResult(match.captured(0), ResultType::MessageInfo);
|
|
int report = match.captured(1).toInt();
|
|
QString type = match.captured(3);
|
|
if (m_reportLevel != ReportLevel::Detailed) {
|
|
if (type == "passed")
|
|
m_summary[ResultType::Pass] += report;
|
|
else if (type == "failed")
|
|
m_summary[ResultType::Fail] += report;
|
|
else if (type == "skipped")
|
|
m_summary[ResultType::Skip] += report;
|
|
}
|
|
|
|
return;
|
|
}
|
|
|
|
match = summaryAssertion.match(line);
|
|
if (match.hasMatch()) {
|
|
createAndReportResult(match.captured(0), ResultType::MessageInfo);
|
|
return;
|
|
}
|
|
|
|
match = summarySkip.match(line);
|
|
if (match.hasMatch()) {
|
|
createAndReportResult(match.captured(0), ResultType::MessageInfo);
|
|
if (m_reportLevel == ReportLevel::Detailed)
|
|
++m_summary[ResultType::Skip];
|
|
return;
|
|
}
|
|
|
|
match = finish.match(line);
|
|
if (match.hasMatch()) {
|
|
if (m_result != ResultType::Invalid)
|
|
sendCompleteInformation();
|
|
BoostTestResult *result = new BoostTestResult(id(), m_projectFile, QString());
|
|
int failed = match.captured(1).toInt();
|
|
QString txt = tr("%1 failures detected in %2.").arg(failed).arg(match.captured(3));
|
|
int passed = (m_testCaseCount != -1)
|
|
? m_testCaseCount - failed - m_summary[ResultType::Skip] : -1;
|
|
if (m_testCaseCount != -1)
|
|
txt.append(' ').append(tr("%1 tests passed.").arg(passed));
|
|
result->setDescription(txt);
|
|
result->setResult(ResultType::MessageInfo);
|
|
reportResult(TestResultPtr(result));
|
|
if (m_reportLevel == ReportLevel::Confirm) { // for the final summary
|
|
m_summary[ResultType::Pass] += passed;
|
|
m_summary[ResultType::Fail] += failed;
|
|
}
|
|
m_testCaseCount = -1;
|
|
return;
|
|
}
|
|
|
|
if (line == noErrors) {
|
|
if (m_result != ResultType::Invalid)
|
|
sendCompleteInformation();
|
|
BoostTestResult *result = new BoostTestResult(id(), m_projectFile, QString());
|
|
QString txt = tr("No errors detected.");
|
|
if (m_testCaseCount != -1)
|
|
txt.append(' ').append(tr("%1 tests passed.").arg(m_testCaseCount));
|
|
result->setDescription(txt);
|
|
result->setResult(ResultType::MessageInfo);
|
|
reportResult(TestResultPtr(result));
|
|
if (m_reportLevel == ReportLevel::Confirm) // for the final summary
|
|
m_summary.insert(ResultType::Pass, m_testCaseCount);
|
|
return;
|
|
}
|
|
|
|
// some plain output...
|
|
if (!m_description.isEmpty())
|
|
m_description.append('\n');
|
|
m_description.append(line);
|
|
}
|
|
|
|
void BoostTestOutputReader::processStdError(const QByteArray &outputLine)
|
|
{
|
|
// we need to process the output, Boost UTF uses both out streams
|
|
processOutputLine(outputLine);
|
|
emit newOutputLineAvailable(outputLine);
|
|
}
|
|
|
|
TestResultPtr BoostTestOutputReader::createDefaultResult() const
|
|
{
|
|
BoostTestResult *result = new BoostTestResult(id(), m_projectFile, m_currentModule);
|
|
result->setTestSuite(m_currentSuite);
|
|
result->setTestCase(m_currentTest);
|
|
|
|
return TestResultPtr(result);
|
|
}
|
|
|
|
void BoostTestOutputReader::onFinished(int exitCode, QProcess::ExitStatus /*exitState*/) {
|
|
if (m_reportLevel == ReportLevel::No && m_testCaseCount != -1) {
|
|
int reportedFailsAndSkips = m_summary[ResultType::Fail] + m_summary[ResultType::Skip];
|
|
m_summary.insert(ResultType::Pass, m_testCaseCount - reportedFailsAndSkips);
|
|
}
|
|
// boost::exit_success (0), boost::exit_test_failure (201)
|
|
// or boost::exit_exception_failure (200)
|
|
// be graceful and do not add a fatal for exit_test_failure
|
|
if (m_logLevel == LogLevel::Nothing && m_reportLevel == ReportLevel::No) {
|
|
switch (exitCode) {
|
|
case 0:
|
|
reportNoOutputFinish(tr("Running tests exited with ") + "boost::exit_success.",
|
|
ResultType::Pass);
|
|
break;
|
|
case 200:
|
|
reportNoOutputFinish(
|
|
tr("Running tests exited with ") + "boost::exit_test_exception.",
|
|
ResultType::MessageFatal);
|
|
break;
|
|
case 201:
|
|
reportNoOutputFinish(tr("Running tests exited with ")
|
|
+ "boost::exit_test_failure.", ResultType::Fail);
|
|
break;
|
|
}
|
|
} else if (exitCode != 0 && exitCode != 201 && !m_description.isEmpty()) {
|
|
if (m_description.startsWith("Test setup error:")) {
|
|
createAndReportResult(m_description + '\n' + tr("Executable: %1")
|
|
.arg(id()), ResultType::MessageWarn);
|
|
} else {
|
|
createAndReportResult(tr("Running tests failed.\n%1\nExecutable: %2")
|
|
.arg(m_description).arg(id()), ResultType::MessageFatal);
|
|
}
|
|
}
|
|
}
|
|
|
|
void BoostTestOutputReader::reportNoOutputFinish(const QString &description, ResultType type)
|
|
{
|
|
BoostTestResult *result = new BoostTestResult(id(), m_projectFile, m_currentModule);
|
|
result->setTestCase(tr("Running tests without output."));
|
|
result->setDescription(description);
|
|
result->setResult(type);
|
|
reportResult(TestResultPtr(result));
|
|
}
|
|
|
|
} // namespace Internal
|
|
} // namespace Autotest
|