forked from qt-creator/qt-creator
AutoTest: Add basic boost test support
Provide experimental support for Boost UTF. This patch adds the basic implementation for * parsing the code for Boost tests * executing the found tests * displaying respective results This is just a basic and limited support which needs to be enhanced and improved later on. Task-number: QTCREATORBUG-21169 Change-Id: Ie0da5f51f90fb1fa7217eac461ebfc5214395ef6 Reviewed-by: David Schulz <david.schulz@qt.io>
This commit is contained in:
Christian Stenger
423
src/plugins/autotest/boost/boosttestoutputreader.cpp
Normal file
423
src/plugins/autotest/boost/boosttestoutputreader.cpp
Normal file
@@ -0,0 +1,423 @@
|
||||
/****************************************************************************
|
||||
**
|
||||
** Copyright (C) 2019 The Qt Company Ltd.
|
||||
** Contact: https://www.qt.io/licensing/
|
||||
**
|
||||
** This file is part of Qt Creator.
|
||||
**
|
||||
** Commercial License Usage
|
||||
** Licensees holding valid commercial Qt licenses may use this file in
|
||||
** accordance with the commercial license agreement provided with the
|
||||
** Software or, alternatively, in accordance with the terms contained in
|
||||
** a written agreement between you and The Qt Company. For licensing terms
|
||||
** and conditions see https://www.qt.io/terms-conditions. For further
|
||||
** information use the contact form at https://www.qt.io/contact-us.
|
||||
**
|
||||
** GNU General Public License Usage
|
||||
** Alternatively, this file may be used under the terms of the GNU
|
||||
** General Public License version 3 as published by the Free Software
|
||||
** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
|
||||
** included in the packaging of this file. Please review the following
|
||||
** information to ensure the GNU General Public License requirements will
|
||||
** be met: https://www.gnu.org/licenses/gpl-3.0.html.
|
||||
**
|
||||
****************************************************************************/
|
||||
|
||||
#include "boosttestoutputreader.h"
|
||||
#include "boosttestsettings.h"
|
||||
#include "boosttestresult.h"
|
||||
|
||||
#include <utils/qtcassert.h>
|
||||
|
||||
#include <QDir>
|
||||
#include <QFileInfo>
|
||||
#include <QLoggingCategory>
|
||||
#include <QRegularExpression>
|
||||
|
||||
namespace Autotest {
|
||||
namespace Internal {
|
||||
|
||||
static Q_LOGGING_CATEGORY(orLog, "qtc.autotest.boost.outputreader", QtWarningMsg)
|
||||
|
||||
static QString constructSourceFilePath(const QString &path, const QString &filePath)
|
||||
{
|
||||
return QFileInfo(path, filePath).canonicalFilePath();
|
||||
}
|
||||
|
||||
BoostTestOutputReader::BoostTestOutputReader(const QFutureInterface<TestResultPtr> &futureInterface,
|
||||
QProcess *testApplication,
|
||||
const QString &buildDirectory,
|
||||
const QString &projectFile,
|
||||
LogLevel log, ReportLevel report)
|
||||
: TestOutputReader(futureInterface, testApplication, buildDirectory)
|
||||
, m_projectFile(projectFile)
|
||||
, m_logLevel(log)
|
||||
, m_reportLevel(report)
|
||||
{
|
||||
if (m_testApplication) {
|
||||
connect(m_testApplication, QOverload<int, QProcess::ExitStatus>::of(&QProcess::finished),
|
||||
this, &BoostTestOutputReader::onFinished);
|
||||
}
|
||||
}
|
||||
|
||||
// content of "error:..." / "info:..." / ... messages
|
||||
static QString caseFromContent(const QString &content)
|
||||
{
|
||||
const int length = content.length();
|
||||
if (content.startsWith("last checkpoint:")) {
|
||||
int index = content.indexOf('"');
|
||||
if (index != 17 || length <= 18) {
|
||||
qCDebug(orLog) << "double quote position" << index << " or content length" << length
|
||||
<< "wrong on content" << content;
|
||||
return QString();
|
||||
}
|
||||
index = content.indexOf('"', 18);
|
||||
if (index == -1) {
|
||||
qCDebug(orLog) << "no closing double quote" << content;
|
||||
return QString();
|
||||
}
|
||||
return content.mid(18, index - 1);
|
||||
}
|
||||
|
||||
int index = content.indexOf(": in ");
|
||||
if (index == -1) // "info: check true has passed"
|
||||
return QString();
|
||||
|
||||
if (index <= 4 || length < index + 4) {
|
||||
qCDebug(orLog) << "unexpected position" << index << "for info" << content;
|
||||
return QString();
|
||||
}
|
||||
|
||||
QString result = content.mid(index + 5);
|
||||
static QRegularExpression functionName("([^\\s]+):.*");
|
||||
const QRegularExpressionMatch matcher = functionName.match(result);
|
||||
if (!matcher.hasMatch()) {
|
||||
qCDebug(orLog) << "got no match";
|
||||
return QString();
|
||||
}
|
||||
return matcher.captured(1);
|
||||
}
|
||||
|
||||
void BoostTestOutputReader::sendCompleteInformation()
|
||||
{
|
||||
QTC_ASSERT(m_result != ResultType::Invalid, return);
|
||||
BoostTestResult *result = new BoostTestResult(id(), m_projectFile, m_currentModule);
|
||||
result->setTestSuite(m_currentSuite);
|
||||
result->setTestCase(m_currentTest);
|
||||
if (m_lineNumber) {
|
||||
result->setLine(m_lineNumber);
|
||||
result->setFileName(m_fileName);
|
||||
} // else TODO
|
||||
|
||||
result->setDescription(m_description);
|
||||
result->setResult(m_result);
|
||||
reportResult(TestResultPtr(result));
|
||||
m_result = ResultType::Invalid;
|
||||
}
|
||||
|
||||
void BoostTestOutputReader::handleMessageMatch(const QRegularExpressionMatch &match)
|
||||
{
|
||||
if (m_result != ResultType::Invalid)
|
||||
sendCompleteInformation();
|
||||
|
||||
m_fileName = constructSourceFilePath(m_buildDir, match.captured(1));
|
||||
m_lineNumber = match.captured(2).toInt();
|
||||
|
||||
const QString &content = match.captured(3);
|
||||
if (content.startsWith("info:")) {
|
||||
if (m_currentTest.isEmpty() || m_logLevel > LogLevel::UnitScope) {
|
||||
QString tmp = caseFromContent(content);
|
||||
if (!tmp.isEmpty())
|
||||
m_currentTest = tmp;
|
||||
}
|
||||
m_result = ResultType::Pass;
|
||||
m_description = content;
|
||||
} else if (content.startsWith("error:")) {
|
||||
if (m_currentTest.isEmpty() || m_logLevel > LogLevel::UnitScope)
|
||||
m_currentTest = caseFromContent(content);
|
||||
m_result = ResultType::Fail;
|
||||
m_description = content;
|
||||
} else if (content.startsWith("fatal error:")) {
|
||||
if (m_currentTest.isEmpty() || m_logLevel > LogLevel::UnitScope)
|
||||
m_currentTest = caseFromContent(content);
|
||||
m_result = ResultType::MessageFatal;
|
||||
m_description = content;
|
||||
} else if (content.startsWith("last checkpoint:")) {
|
||||
if (m_currentTest.isEmpty() || m_logLevel > LogLevel::UnitScope)
|
||||
m_currentTest = caseFromContent(content);
|
||||
m_result = ResultType::MessageInfo;
|
||||
m_description = content;
|
||||
} else if (content.startsWith("Entering")) {
|
||||
m_result = ResultType::TestStart;
|
||||
const QString type = match.captured(8);
|
||||
if (type == "case") {
|
||||
m_currentTest = match.captured(9);
|
||||
m_description = tr("Executing test case %1").arg(m_currentTest);
|
||||
} else if (type == "suite") {
|
||||
m_currentSuite = match.captured(9);
|
||||
m_description = tr("Executing test suite %1").arg(m_currentSuite);
|
||||
}
|
||||
} else if (content.startsWith("Leaving")) {
|
||||
const QString type = match.captured(10);
|
||||
if (type == "case") {
|
||||
if (m_currentTest != match.captured(11) && m_currentTest.isEmpty())
|
||||
m_currentTest = match.captured(11);
|
||||
m_result = ResultType::TestEnd;
|
||||
m_description = tr("Test execution took %1").arg(match.captured(12));
|
||||
} else if (type == "suite") {
|
||||
if (m_currentSuite != match.captured(11) && m_currentSuite.isEmpty())
|
||||
m_currentSuite = match.captured(11);
|
||||
m_currentTest.clear();
|
||||
m_result = ResultType::TestEnd;
|
||||
m_description = tr("Test suite execution took %1").arg(match.captured(12));
|
||||
}
|
||||
} else if (content.startsWith("Test case ")) {
|
||||
m_currentTest = match.captured(4);
|
||||
m_result = ResultType::Skip;
|
||||
m_description = content;
|
||||
}
|
||||
}
|
||||
|
||||
void BoostTestOutputReader::processOutputLine(const QByteArray &outputLineWithNewLine)
|
||||
{
|
||||
static QRegularExpression newTestStart("^Running (\\d+) test cases?\\.\\.\\.$");
|
||||
static QRegularExpression dependency("^Including test case (.+) as a dependency of "
|
||||
"test case (.+)$");
|
||||
static QRegularExpression messages("^(.+)\\((\\d+)\\): (info: (.+)|error: (.+)|"
|
||||
"fatal error: (.+)|last checkpoint: (.+)"
|
||||
"|Entering test (case|suite) \"(.+)\""
|
||||
"|Leaving test (case|suite) \"(.+)\"; testing time: (\\d+.+)"
|
||||
"|Test case \"(.+)\" is skipped because .+$)$");
|
||||
static QRegularExpression moduleMssg("^(Entering test module \"(.+)\"|"
|
||||
"Leaving test module \"(.+)\"; testing time: (\\d+.+))$");
|
||||
static QRegularExpression noAssertion("^Test case (.*) did not check any assertions$");
|
||||
|
||||
static QRegularExpression summaryPreamble("^\\s*Test (module|suite|case) \"(.*)\" has "
|
||||
"(failed|passed) with:$");
|
||||
static QRegularExpression summarySkip("^\\s+Test case \"(.*)\" was skipped$");
|
||||
static QRegularExpression summaryDetail("^\\s+(\\d+) test cases? out of (\\d+) "
|
||||
"(failed|passed)$");
|
||||
static QRegularExpression summaryAssertion("^\\s+(\\d+) assertions? out of (\\d+) "
|
||||
"(failed|passed)$");
|
||||
|
||||
static QRegularExpression finish("^\\*{3} (\\d+) failure(s are| is) detected in the "
|
||||
"test module \"(.*)\"$");
|
||||
static QRegularExpression errDetect("^\\*{3} Errors where detected in the "
|
||||
"test module \"(.*}\"; see standard output for details");
|
||||
QString noErrors("*** No errors detected");
|
||||
|
||||
const QString line = QString::fromUtf8(chopLineBreak(outputLineWithNewLine));
|
||||
if (line.trimmed().isEmpty())
|
||||
return;
|
||||
|
||||
QRegularExpressionMatch match = messages.match(line);
|
||||
if (match.hasMatch()) {
|
||||
handleMessageMatch(match);
|
||||
return;
|
||||
}
|
||||
|
||||
match = dependency.match(line);
|
||||
if (match.hasMatch()) {
|
||||
if (m_result != ResultType::Invalid)
|
||||
sendCompleteInformation();
|
||||
BoostTestResult *result = new BoostTestResult(id(), m_projectFile, m_currentModule);
|
||||
result->setDescription(match.captured(0));
|
||||
result->setResult(ResultType::MessageInfo);
|
||||
reportResult(TestResultPtr(result));
|
||||
return;
|
||||
}
|
||||
|
||||
match = newTestStart.match(line);
|
||||
if (match.hasMatch()) {
|
||||
if (m_result != ResultType::Invalid)
|
||||
sendCompleteInformation();
|
||||
m_testCaseCount = match.captured(1).toInt();
|
||||
m_description.clear();
|
||||
return;
|
||||
}
|
||||
|
||||
match = moduleMssg.match(line);
|
||||
if (match.hasMatch()) {
|
||||
if (m_result != ResultType::Invalid)
|
||||
sendCompleteInformation();
|
||||
if (match.captured(1).startsWith("Entering")) {
|
||||
m_currentModule = match.captured(2);
|
||||
BoostTestResult *result = new BoostTestResult(id(), m_projectFile, m_currentModule);
|
||||
result->setDescription(tr("Executing test module %1").arg(m_currentModule));
|
||||
result->setResult(ResultType::TestStart);
|
||||
reportResult(TestResultPtr(result));
|
||||
m_description.clear();
|
||||
} else {
|
||||
QTC_CHECK(m_currentModule == match.captured(3));
|
||||
BoostTestResult *result = new BoostTestResult(id(), m_projectFile, m_currentModule);
|
||||
result->setDescription(tr("Test module execution took %1").arg(match.captured(4)));
|
||||
result->setResult(ResultType::TestEnd);
|
||||
reportResult(TestResultPtr(result));
|
||||
|
||||
m_currentTest.clear();
|
||||
m_currentSuite.clear();
|
||||
m_currentModule.clear();
|
||||
m_description.clear();
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
match = noAssertion.match(line);
|
||||
if (match.hasMatch()) {
|
||||
if (m_result != ResultType::Invalid)
|
||||
sendCompleteInformation();
|
||||
const QString caseWithOptionalSuite = match.captured(1);
|
||||
int index = caseWithOptionalSuite.lastIndexOf('/');
|
||||
if (index == -1) {
|
||||
QTC_CHECK(caseWithOptionalSuite == m_currentTest);
|
||||
} else {
|
||||
QTC_CHECK(caseWithOptionalSuite.mid(index + 1) == m_currentTest);
|
||||
int sIndex = caseWithOptionalSuite.lastIndexOf('/', index - 1);
|
||||
if (sIndex == -1) {
|
||||
QTC_CHECK(caseWithOptionalSuite.left(index) == m_currentSuite);
|
||||
m_currentSuite = caseWithOptionalSuite.left(index); // FIXME should not be necessary - but we currently do not care for the whole suite path
|
||||
} else {
|
||||
QTC_CHECK(caseWithOptionalSuite.mid(sIndex + 1, index - sIndex - 1) == m_currentSuite);
|
||||
}
|
||||
}
|
||||
createAndReportResult(match.captured(0), ResultType::MessageWarn);
|
||||
return;
|
||||
}
|
||||
|
||||
// should summary get reported unconditionally?
|
||||
match = summaryPreamble.match(line);
|
||||
if (match.hasMatch()) {
|
||||
createAndReportResult(match.captured(0), ResultType::MessageInfo);
|
||||
return;
|
||||
}
|
||||
|
||||
match = summaryDetail.match(line);
|
||||
if (match.hasMatch()) {
|
||||
createAndReportResult(match.captured(0), ResultType::MessageInfo);
|
||||
return;
|
||||
}
|
||||
|
||||
match = summaryAssertion.match(line);
|
||||
if (match.hasMatch()) {
|
||||
createAndReportResult(match.captured(0), ResultType::MessageInfo);
|
||||
return;
|
||||
}
|
||||
|
||||
match = summarySkip.match(line);
|
||||
if (match.hasMatch()) {
|
||||
createAndReportResult(match.captured(0), ResultType::MessageInfo);
|
||||
return;
|
||||
}
|
||||
|
||||
match = finish.match(line);
|
||||
if (match.hasMatch()) {
|
||||
if (m_result != ResultType::Invalid)
|
||||
sendCompleteInformation();
|
||||
BoostTestResult *result = new BoostTestResult(id(), m_projectFile, QString());
|
||||
int failed = match.captured(1).toInt();
|
||||
QString txt = tr("%1 failures detected in %2.").arg(failed).arg(match.captured(3));
|
||||
if (m_testCaseCount != -1)
|
||||
txt.append(' ').append(tr("%1 tests passed.").arg(m_testCaseCount - failed));
|
||||
result->setDescription(txt);
|
||||
result->setResult(ResultType::MessageInfo); // TODO report similar to disabled tests
|
||||
reportResult(TestResultPtr(result));
|
||||
m_testCaseCount = -1;
|
||||
return;
|
||||
}
|
||||
|
||||
if (line == noErrors) {
|
||||
if (m_result != ResultType::Invalid)
|
||||
sendCompleteInformation();
|
||||
BoostTestResult *result = new BoostTestResult(id(), m_projectFile, QString());
|
||||
QString txt = tr("No errors detected.");
|
||||
if (m_testCaseCount != -1)
|
||||
txt.append(' ').append(tr("%1 tests passed.").arg(m_testCaseCount));
|
||||
result->setDescription(txt);
|
||||
result->setResult(ResultType::MessageInfo); // TODO report similar to disabled tests
|
||||
reportResult(TestResultPtr(result));
|
||||
return;
|
||||
}
|
||||
|
||||
// some plain output...
|
||||
if (!m_description.isEmpty())
|
||||
m_description.append('\n');
|
||||
m_description.append(line);
|
||||
}
|
||||
|
||||
void BoostTestOutputReader::processStdError(const QByteArray &output)
|
||||
{
|
||||
// we need to process the output, Boost UTF uses both out streams
|
||||
int start = 0;
|
||||
int index = -1;
|
||||
while ((index = output.indexOf('\n', start)) != -1) {
|
||||
const QByteArray &line = output.mid(start, index - start + 1);
|
||||
if (!line.isEmpty()) {
|
||||
if (line != QByteArray(1, '\n'))
|
||||
processOutputLine(line);
|
||||
emit newOutputAvailable(line);
|
||||
}
|
||||
start = index + 1;
|
||||
}
|
||||
if (start > 0) { // remove? this never happens
|
||||
const QByteArray lastLine = output.mid(start) + '\n';
|
||||
if (!lastLine.isEmpty()) {
|
||||
if (lastLine != QByteArray(1, '\n'))
|
||||
processOutputLine(lastLine);
|
||||
emit newOutputAvailable(lastLine);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TestResultPtr BoostTestOutputReader::createDefaultResult() const
|
||||
{
|
||||
BoostTestResult *result = new BoostTestResult(id(), m_projectFile, m_currentModule);
|
||||
result->setTestSuite(m_currentSuite);
|
||||
result->setTestCase(m_currentTest);
|
||||
|
||||
// TODO find corresponding TestTreeItem and set filename/line
|
||||
return TestResultPtr(result);
|
||||
}
|
||||
|
||||
void BoostTestOutputReader::onFinished(int exitCode, QProcess::ExitStatus /*exitState*/) {
|
||||
// boost::exit_success (0), boost::exit_test_failure (201)
|
||||
// or boost::exit_exception_failure (200)
|
||||
// be graceful and do not add a fatal for exit_test_failure
|
||||
// but exit code 0 can be forced with an option - what todo in that case?
|
||||
if (m_logLevel == LogLevel::Nothing && m_reportLevel == ReportLevel::No) {
|
||||
switch (exitCode) {
|
||||
case 0:
|
||||
reportNoOutputFinish(tr("Running tests exited with ") + "boost::exit_success.",
|
||||
ResultType::Pass);
|
||||
break;
|
||||
case 200:
|
||||
reportNoOutputFinish(
|
||||
tr("Running tests exited with ") + "boost::exit_test_exception.",
|
||||
ResultType::MessageFatal);
|
||||
break;
|
||||
case 201:
|
||||
reportNoOutputFinish(tr("Running tests exited with ")
|
||||
+ "boost::exit_test_failure.", ResultType::Fail);
|
||||
break;
|
||||
}
|
||||
} else if (exitCode != 0 && exitCode != 201 && !m_description.isEmpty()) {
|
||||
if (m_description.startsWith("Test setup error:")) {
|
||||
createAndReportResult(m_description + '\n' + tr("Executable: %1")
|
||||
.arg(id()), ResultType::MessageWarn);
|
||||
} else {
|
||||
createAndReportResult(tr("Running tests failed.\n%1\nExecutable: %2")
|
||||
.arg(m_description).arg(id()), ResultType::MessageFatal);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void BoostTestOutputReader::reportNoOutputFinish(const QString &description, ResultType type)
|
||||
{
|
||||
BoostTestResult *result = new BoostTestResult(id(), m_projectFile, m_currentModule);
|
||||
result->setTestCase(tr("Running tests without output."));
|
||||
result->setDescription(description);
|
||||
result->setResult(type);
|
||||
reportResult(TestResultPtr(result));
|
||||
}
|
||||
|
||||
} // namespace Internal
|
||||
} // namespace Autotest
|
||||
Reference in New Issue
Block a user