feedc0de/qt-creator
1
0
Fork 0
forked from qt-creator/qt-creator
Code Pull Requests Activity

AutoTest: Support plaintext output for Qt tests

Browse Source 
Change-Id: I88ec477777d79c69e699dd906ec4ef1550bcaf44
Reviewed-by: David Schulz <david.schulz@qt.io>
This commit is contained in:
Christian Stenger
2017-06-19 15:58:21 +02:00
parent 4edc715b1c
commit cc8bff67b3
9 changed files with 289 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
src/plugins/autotest/qtest/qttestconfiguration.cpp
View File

@@ -35,7 +35,17 @@ namespace Internal {
TestOutputReader *QtTestConfiguration::outputReader(const QFutureInterface<TestResultPtr> &fi,
QProcess *app) const
{
return new QtTestOutputReader(fi, app, buildDirectory());
static const Core::Id id
= Core::Id(Constants::FRAMEWORK_PREFIX).withSuffix(QtTest::Constants::FRAMEWORK_NAME);
TestFrameworkManager *manager = TestFrameworkManager::instance();
auto qtSettings = qSharedPointerCast<QtTestSettings>(manager->settingsForTestFramework(id));
if (qtSettings.isNull())
return nullptr;
if (qtSettings->useXMLOutput)
return new QtTestOutputReader(fi, app, buildDirectory(), QtTestOutputReader::XML);
else
return new QtTestOutputReader(fi, app, buildDirectory(), QtTestOutputReader::PlainText);
}
QStringList QtTestConfiguration::argumentsForTestRunner() const
@@ -43,14 +53,15 @@ QStringList QtTestConfiguration::argumentsForTestRunner() const
static const Core::Id id
= Core::Id(Constants::FRAMEWORK_PREFIX).withSuffix(QtTest::Constants::FRAMEWORK_NAME);
QStringList arguments("-xml");
if (testCases().count())
arguments << testCases();
QStringList arguments;
TestFrameworkManager *manager = TestFrameworkManager::instance();
auto qtSettings = qSharedPointerCast<QtTestSettings>(manager->settingsForTestFramework(id));
if (qtSettings.isNull())
return arguments;
if (qtSettings->useXMLOutput)
arguments << "-xml";
if (testCases().count())
arguments << testCases();
const QString &metricsOption = QtTestSettings::metricsTypeToOption(qtSettings->metrics);
if (!metricsOption.isEmpty())

237
src/plugins/autotest/qtest/qttestoutputreader.cpp
View File

@@ -31,6 +31,7 @@
#include <QDir>
#include <QFileInfo>
#include <QRegExp>
#include <QRegularExpression>
namespace Autotest {
namespace Internal {
@@ -128,12 +129,26 @@ static QString constructSourceFilePath(const QString &path, const QString &fileP
}
QtTestOutputReader::QtTestOutputReader(const QFutureInterface<TestResultPtr> &futureInterface,
QProcess *testApplication, const QString &buildDirectory)
QProcess *testApplication, const QString &buildDirectory,
OutputMode mode)
: TestOutputReader(futureInterface, testApplication, buildDirectory)
, m_mode(mode)
{
}
void QtTestOutputReader::processOutput(const QByteArray &outputLine)
{
switch (m_mode) {
case PlainText:
processPlainTextOutput(outputLine);
break;
case XML:
processXMLOutput(outputLine);
break;
}
}
void QtTestOutputReader::processXMLOutput(const QByteArray &outputLine)
{
static QStringList validEndTags = {QStringLiteral("Incident"),
QStringLiteral("Message"),
@@ -162,24 +177,14 @@ void QtTestOutputReader::processOutput(const QByteArray &outputLine)
if (currentTag == QStringLiteral("TestCase")) {
m_className = m_xmlReader.attributes().value(QStringLiteral("name")).toString();
QTC_ASSERT(!m_className.isEmpty(), continue);
TestResultPtr testResult = TestResultPtr(createDefaultResult());
testResult->setResult(Result::MessageTestCaseStart);
testResult->setDescription(tr("Executing test case %1").arg(m_className));
m_futureInterface.reportResult(testResult);
sendStartMessage(false);
} else if (currentTag == QStringLiteral("TestFunction")) {
m_testCase = m_xmlReader.attributes().value(QStringLiteral("name")).toString();
QTC_ASSERT(!m_testCase.isEmpty(), continue);
if (m_testCase == m_formerTestCase) // don't report "Executing..." more than once
continue;
TestResultPtr testResult = TestResultPtr(createDefaultResult());
testResult->setResult(Result::MessageTestCaseStart);
testResult->setDescription(tr("Executing test function %1").arg(m_testCase));
m_futureInterface.reportResult(testResult);
testResult = TestResultPtr(new QtTestResult);
testResult->setResult(Result::MessageCurrentTest);
testResult->setDescription(tr("Entering test function %1::%2").arg(m_className,
m_testCase));
m_futureInterface.reportResult(testResult);
sendStartMessage(true);
sendMessageCurrentTest();
} else if (currentTag == QStringLiteral("Duration")) {
m_duration = m_xmlReader.attributes().value(QStringLiteral("msecs")).toString();
QTC_ASSERT(!m_duration.isEmpty(), continue);
@@ -255,23 +260,13 @@ void QtTestOutputReader::processOutput(const QByteArray &outputLine)
m_cdataMode = None;
const QStringRef currentTag = m_xmlReader.name();
if (currentTag == QStringLiteral("TestFunction")) {
QtTestResult *testResult = createDefaultResult();
testResult->setResult(Result::MessageTestCaseEnd);
testResult->setDescription(
m_duration.isEmpty() ? tr("Test function finished.")
: tr("Execution took %1 ms.").arg(m_duration));
m_futureInterface.reportResult(TestResultPtr(testResult));
sendFinishMessage(true);
m_futureInterface.setProgressValue(m_futureInterface.progressValue() + 1);
m_dataTag.clear();
m_formerTestCase = m_testCase;
m_testCase.clear();
} else if (currentTag == QStringLiteral("TestCase")) {
QtTestResult *testResult = createDefaultResult();
testResult->setResult(Result::MessageTestCaseEnd);
testResult->setDescription(
m_duration.isEmpty() ? tr("Test finished.")
: tr("Test execution took %1 ms.").arg(m_duration));
m_futureInterface.reportResult(TestResultPtr(testResult));
sendFinishMessage(false);
} else if (validEndTags.contains(currentTag.toString())) {
QtTestResult *testResult = createDefaultResult();
testResult->setResult(m_result);
@@ -290,6 +285,138 @@ void QtTestOutputReader::processOutput(const QByteArray &outputLine)
}
}
static QStringList extractFunctionInformation(const QString &testClassName,
const QString &lineWithoutResultType,
Result::Type resultType)
{
static QRegularExpression classInformation("^(.+?)\\((.*?)\\)(.*)$");
QStringList result;
const QRegularExpressionMatch match = classInformation.match(lineWithoutResultType);
if (match.hasMatch()) {
QString fullQualifiedFunc = match.captured(1);
QTC_ASSERT(fullQualifiedFunc.startsWith(testClassName + "::"), return result);
fullQualifiedFunc = fullQualifiedFunc.mid(testClassName.length() + 2);
result.append(fullQualifiedFunc);
if (resultType == Result::Benchmark) { // tag is displayed differently
QString possiblyTag = match.captured(3);
if (!possiblyTag.isEmpty())
possiblyTag = possiblyTag.mid(2, possiblyTag.length() - 4);
result.append(possiblyTag);
result.append(QString());
} else {
result.append(match.captured(2));
result.append(match.captured(3));
}
}
return result;
}
void QtTestOutputReader::processPlainTextOutput(const QByteArray &outputLine)
{
static QRegExp start("^[*]{9} Start testing of (.*) [*]{9}$");
static QRegExp config("^Config: Using QtTest library (.*), (Qt (\\d+(\\.\\d+){2}) \\(.*\\))$");
static QRegExp summary("^Totals: \\d+ passed, \\d+ failed, \\d+ skipped(, \\d+ blacklisted)?$");
static QRegExp finish("^[*]{9} Finished testing of (.*) [*]{9}$");
static QRegExp result("^(PASS |FAIL! |XFAIL |XPASS |SKIP |BPASS |BFAIL |RESULT "
"|INFO |QWARN |WARNING|QDEBUG ): (.*)$");
static QRegExp benchDetails("^\\s+([\\d,.]+ .* per iteration \\(total: [\\d,.]+, iterations: \\d+\\))$");
static QRegExp locationUnix("^ Loc: \\[(.*)\\]$");
static QRegExp locationWin("^(.*\\(\\d+\\)) : failure location$");
if (m_futureInterface.isCanceled())
return;
const QString &line = QString::fromLatin1(outputLine);
if (result.exactMatch(line)) {
processResultOutput(result.cap(1).toLower().trimmed(), result.cap(2));
} else if (locationUnix.exactMatch(line)) {
processLocationOutput(locationUnix.cap(1));
} else if (locationWin.exactMatch(line)) {
processLocationOutput(locationWin.cap(1));
} else if (benchDetails.exactMatch(line)) {
m_description = benchDetails.cap(1);
} else if (config.exactMatch(line)) {
handleAndSendConfigMessage(config);
} else if (start.exactMatch(line)) {
m_className = start.cap(1);
QTC_CHECK(!m_className.isEmpty());
sendStartMessage(false);
} else if (summary.exactMatch(line) || finish.exactMatch(line)) {
processSummaryFinishOutput();
} else { // we have some plain output, but we cannot say where for sure it belongs to..
if (!m_description.isEmpty())
m_description.append('\n');
m_description.append(line);
}
}
void QtTestOutputReader::processResultOutput(const QString &result, const QString &message)
{
if (!m_testCase.isEmpty()) { // report the former result if there is any
sendCompleteInformation();
m_dataTag.clear();
m_description.clear();
m_file.clear();
m_lineNumber = 0;
}
m_result = TestResult::resultFromString(result);
const QStringList funcWithTag = extractFunctionInformation(m_className, message, m_result);
QTC_ASSERT(funcWithTag.size() == 3, return);
m_testCase = funcWithTag.at(0);
if (m_testCase != m_formerTestCase) { // new test function executed
if (!m_formerTestCase.isEmpty()) {
using namespace std;
swap(m_testCase, m_formerTestCase); // we want formerTestCase to be reported
sendFinishMessage(true);
swap(m_testCase, m_formerTestCase);
}
sendStartMessage(true);
sendMessageCurrentTest();
}
m_dataTag = funcWithTag.at(1);
const QString description = funcWithTag.at(2);
if (!description.isEmpty()) {
if (!m_description.isEmpty())
m_description.append('\n');
m_description.append(description.mid(1)); // cut the first whitespace
}
m_formerTestCase = m_testCase;
}
void QtTestOutputReader::processLocationOutput(const QString &fileWithLine)
{
QTC_ASSERT(fileWithLine.endsWith(')'), return);
int openBrace = fileWithLine.lastIndexOf('(');
QTC_ASSERT(openBrace != -1, return);
m_file = constructSourceFilePath(m_buildDir, fileWithLine.left(openBrace));
QString numberStr = fileWithLine.mid(openBrace + 1);
numberStr.chop(1);
m_lineNumber = numberStr.toInt();
}
void QtTestOutputReader::processSummaryFinishOutput()
{
if (m_className.isEmpty()) // we have reported already
return;
// we still have something to report
sendCompleteInformation();
m_dataTag.clear();
// report finished function
sendFinishMessage(true);
m_testCase.clear();
m_formerTestCase.clear();
// create and report the finish message for this test class
sendFinishMessage(false);
m_className.clear();
m_description.clear();
m_result = Result::Invalid;
m_file.clear();
m_lineNumber = 0;
}
QtTestResult *QtTestOutputReader::createDefaultResult() const
{
QtTestResult *result = new QtTestResult(m_className);
@@ -298,5 +425,63 @@ QtTestResult *QtTestOutputReader::createDefaultResult() const
return result;
}
void QtTestOutputReader::sendCompleteInformation()
{
TestResultPtr testResult = TestResultPtr(createDefaultResult());
testResult->setResult(m_result);
testResult->setFileName(m_file);
testResult->setLine(m_lineNumber);
testResult->setDescription(m_description);
m_futureInterface.reportResult(testResult);
}
void QtTestOutputReader::sendMessageCurrentTest()
{
TestResultPtr testResult = TestResultPtr(new QtTestResult);
testResult->setResult(Result::MessageCurrentTest);
testResult->setDescription(tr("Entering test function %1::%2").arg(m_className, m_testCase));
m_futureInterface.reportResult(testResult);
}
void QtTestOutputReader::sendStartMessage(bool isFunction)
{
TestResultPtr testResult = TestResultPtr(createDefaultResult());
testResult->setResult(Result::MessageTestCaseStart);
testResult->setDescription(isFunction ? tr("Executing test function %1").arg(m_testCase)
: tr("Executing test case %1").arg(m_className));
m_futureInterface.reportResult(testResult);
}
void QtTestOutputReader::sendFinishMessage(bool isFunction)
{
TestResultPtr testResult = TestResultPtr(createDefaultResult());
testResult->setResult(Result::MessageTestCaseEnd);
if (m_duration.isEmpty()) {
testResult->setDescription(isFunction ? tr("Execution took %1 ms.").arg(m_duration)
: tr("Test execution took %1 ms.").arg(m_duration));
} else {
testResult->setDescription(isFunction ? tr("Test function finished.")
: tr("Test finished."));
}
m_futureInterface.reportResult(testResult);
}
// TODO factor out tr() strings to avoid duplication (see XML processing of Characters)
void QtTestOutputReader::handleAndSendConfigMessage(const QRegExp &config)
{
QtTestResult *testResult = createDefaultResult();
testResult->setResult(Result::MessageInternal);
testResult->setDescription(tr("Qt version: %1").arg(config.cap(3)));
m_futureInterface.reportResult(TestResultPtr(testResult));
testResult = createDefaultResult();
testResult->setResult(Result::MessageInternal);
testResult->setDescription(tr("Qt build: %1").arg(config.cap(2)));
m_futureInterface.reportResult(TestResultPtr(testResult));
testResult = createDefaultResult();
testResult->setResult(Result::MessageInternal);
testResult->setDescription(tr("QTest version: %1").arg(config.cap(1)));
m_futureInterface.reportResult(TestResultPtr(testResult));
}
} // namespace Internal
} // namespace Autotest

22
src/plugins/autotest/qtest/qttestoutputreader.h
View File

@@ -40,14 +40,32 @@ class QtTestOutputReader : public TestOutputReader
Q_DECLARE_TR_FUNCTIONS(Autotest::Internal::QtTestOutputReader)
public:
enum OutputMode
{
XML,
PlainText
};
QtTestOutputReader(const QFutureInterface<TestResultPtr> &futureInterface,
QProcess *testApplication, const QString &buildDirectory);
QProcess *testApplication, const QString &buildDirectory,
OutputMode mode);
protected:
void processOutput(const QByteArray &outputLine) override;
private:
void processXMLOutput(const QByteArray &outputLine);
void processPlainTextOutput(const QByteArray &outputLine);
void processResultOutput(const QString &result, const QString &message);
void processLocationOutput(const QString &fileWithLine);
void processSummaryFinishOutput();
// helper functions
QtTestResult *createDefaultResult() const;
void sendCompleteInformation();
void sendMessageCurrentTest();
void sendStartMessage(bool isFunction);
void sendFinishMessage(bool isFunction);
void handleAndSendConfigMessage(const QRegExp &config);
enum CDATAMode
{
@@ -70,6 +88,8 @@ private:
int m_lineNumber = 0;
QString m_duration;
QXmlStreamReader m_xmlReader;
OutputMode m_mode = XML;
};
} // namespace Internal

3
src/plugins/autotest/qtest/qttestsettings.cpp
View File

@@ -30,6 +30,7 @@ namespace Internal {
static const char metricsKey[] = "Metrics";
static const char noCrashhandlerKey[] = "NoCrashhandlerOnDebug";
static const char useXMLOutputKey[] = "UseXMLOutput";
static MetricsType intToMetrics(int value)
{
@@ -58,12 +59,14 @@ void QtTestSettings::fromFrameworkSettings(const QSettings *s)
{
metrics = intToMetrics(s->value(metricsKey, Walltime).toInt());
noCrashHandler = s->value(noCrashhandlerKey, true).toBool();
useXMLOutput = s->value(useXMLOutputKey, true).toBool();
}
void QtTestSettings::toFrameworkSettings(QSettings *s) const
{
s->setValue(metricsKey, metrics);
s->setValue(noCrashhandlerKey, noCrashHandler);
s->setValue(useXMLOutputKey, useXMLOutput);
}
QString QtTestSettings::metricsTypeToOption(const MetricsType type)

1
src/plugins/autotest/qtest/qttestsettings.h
View File

@@ -48,6 +48,7 @@ public:
MetricsType metrics = Walltime;
bool noCrashHandler = true;
bool useXMLOutput = true;
protected:
void fromFrameworkSettings(const QSettings *s) override;

2
src/plugins/autotest/qtest/qttestsettingspage.cpp
View File

@@ -46,6 +46,7 @@ QtTestSettingsWidget::QtTestSettingsWidget(QWidget *parent)
void QtTestSettingsWidget::setSettings(const QtTestSettings &settings)
{
m_ui.disableCrashhandlerCB->setChecked(settings.noCrashHandler);
m_ui.useXMLOutputCB->setChecked(settings.useXMLOutput);
switch (settings.metrics) {
case MetricsType::Walltime:
m_ui.walltimeRB->setChecked(true);
@@ -72,6 +73,7 @@ QtTestSettings QtTestSettingsWidget::settings() const
QtTestSettings result;
result.noCrashHandler = m_ui.disableCrashhandlerCB->isChecked();
result.useXMLOutput = m_ui.useXMLOutputCB->isChecked();
if (m_ui.walltimeRB->isChecked())
result.metrics = MetricsType::Walltime;
else if (m_ui.tickcounterRB->isChecked())

15
src/plugins/autotest/qtest/qttestsettingspage.ui
View File

@@ -31,6 +31,21 @@
</property>
</widget>
</item>
<item>
<widget class="QCheckBox" name="useXMLOutputCB">
<property name="toolTip">
<string>XML output recommended as it avoids parsing issues, while plain text is more human readable.
Warning: Plain text output is missing some information (e.g. duration)</string>
</property>
<property name="text">
<string>Use XML output</string>
</property>
<property name="checked">
<bool>true</bool>
</property>
</widget>
</item>
<item>
<widget class="QGroupBox" name="groupBox">
<property name="title">

20
src/plugins/autotest/quick/quicktestconfiguration.cpp
View File

@@ -35,7 +35,16 @@ namespace Internal {
TestOutputReader *QuickTestConfiguration::outputReader(const QFutureInterface<TestResultPtr> &fi,
QProcess *app) const
{
return new QtTestOutputReader(fi, app, buildDirectory());
static const Core::Id id
= Core::Id(Constants::FRAMEWORK_PREFIX).withSuffix(QtTest::Constants::FRAMEWORK_NAME);
TestFrameworkManager *manager = TestFrameworkManager::instance();
auto qtSettings = qSharedPointerCast<QtTestSettings>(manager->settingsForTestFramework(id));
if (qtSettings.isNull())
return nullptr;
if (qtSettings->useXMLOutput)
return new QtTestOutputReader(fi, app, buildDirectory(), QtTestOutputReader::XML);
else
return new QtTestOutputReader(fi, app, buildDirectory(), QtTestOutputReader::PlainText);
}
QStringList QuickTestConfiguration::argumentsForTestRunner() const
@@ -43,14 +52,15 @@ QStringList QuickTestConfiguration::argumentsForTestRunner() const
static const Core::Id id
= Core::Id(Constants::FRAMEWORK_PREFIX).withSuffix(QtTest::Constants::FRAMEWORK_NAME);
QStringList arguments("-xml");
if (testCases().count())
arguments << testCases();
QStringList arguments;
TestFrameworkManager *manager = TestFrameworkManager::instance();
auto qtSettings = qSharedPointerCast<QtTestSettings>(manager->settingsForTestFramework(id));
if (qtSettings.isNull())
return arguments;
if (qtSettings->useXMLOutput)
arguments << "-xml";
if (testCases().count())
arguments << testCases();
const QString &metricsOption = QtTestSettings::metricsTypeToOption(qtSettings->metrics);
if (!metricsOption.isEmpty())

8
src/plugins/autotest/testresult.cpp
View File

@@ -56,7 +56,7 @@ Result::Type TestResult::resultFromString(const QString &resultString)
{
if (resultString == "pass")
return Result::Pass;
if (resultString == "fail")
if (resultString == "fail" || resultString == "fail!")
return Result::Fail;
if (resultString == "xfail")
return Result::ExpectedFail;
@@ -64,11 +64,13 @@ Result::Type TestResult::resultFromString(const QString &resultString)
return Result::UnexpectedPass;
if (resultString == "skip")
return Result::Skip;
if (resultString == "result")
return Result::Benchmark;
if (resultString == "qdebug")
return Result::MessageDebug;
if (resultString == "qinfo")
if (resultString == "qinfo" || resultString == "info")
return Result::MessageInfo;
if (resultString == "warn" || resultString == "qwarn")
if (resultString == "warn" || resultString == "qwarn" || resultString == "warning")
return Result::MessageWarn;
if (resultString == "qfatal")
return Result::MessageFatal;