feedc0de/qt-creator
1
0
Fork 0
forked from qt-creator/qt-creator
Code Pull Requests Activity

TestRunner: Reuse TaskTree

Browse Source 
Get rid of QFutureInterface argument from
ITestConfiguration::createOutputReader() and from
TestOutputReader c'tor.

The fine-grained progress reporting was broken anyway:

1. The assumption was that testCaseCount was meant to be
   the total number of test functions executed. It didn't
   include the initTestCase() and cleanupTestCase(),
   while those were reported on runtime apparently
   (and exceeding the max progress by 2).
2. In case of tst_qtcprocess, when the whole test was run,
   the testCaseCount reported 41, while the real
   number of functions was 26 (+2 = 28 for init/cleanup).
3. While the max progress was set to testCaseCount initially,
   the corresponding FutureProgress rendered the progress
   always in 0-100 range, what didn't match the reality.

Instead, rely on TaskTree progress, which resolution
is per test as a whole. So, when executing a series
of tests this should scale fine. In addition, the
progress advances fluently according to the expected
run time - with 10 seconds hardcoded.

The original code locations, where progress was bumped,
are left with a TODO comment for any possible future tweaks.
Like in case of result reporting, fine-grained progress
reporting may be implemented by providing additional signal,
so there is no need for QFutureInterface inside
TestOutputReader.

Change-Id: Idc11d55e3a49dac8d1788948b9a82f68199203c6
Reviewed-by: <github-actions-qt-creator@cristianadam.eu>
Reviewed-by: Christian Stenger <christian.stenger@qt.io>
This commit is contained in:
Jarek Kobus
2023-01-17 00:45:50 +01:00
parent 6908130c83
commit 4f70aa7052
28 changed files with 219 additions and 351 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
src/plugins/autotest/boost/boosttestconfiguration.cpp
View File

@@ -11,18 +11,16 @@
#include "../testsettings.h"
#include <utils/algorithm.h>
#include <utils/stringutils.h>
using namespace Utils;
namespace Autotest {
namespace Internal {
TestOutputReader *BoostTestConfiguration::createOutputReader(
const QFutureInterface<TestResult> &fi, QtcProcess *app) const
TestOutputReader *BoostTestConfiguration::createOutputReader(QtcProcess *app) const
{
auto settings = static_cast<BoostTestSettings *>(framework()->testSettings());
return new BoostTestOutputReader(fi, app, buildDirectory(), projectFile(),
return new BoostTestOutputReader(app, buildDirectory(), projectFile(),
LogLevel(settings->logLevel.value()),
ReportLevel(settings->reportLevel.value()));
}

3
src/plugins/autotest/boost/boosttestconfiguration.h
View File

@@ -13,8 +13,7 @@ class BoostTestConfiguration : public DebuggableTestConfiguration
public:
explicit BoostTestConfiguration(ITestFramework *framework)
: DebuggableTestConfiguration(framework) {}
TestOutputReader *createOutputReader(const QFutureInterface<TestResult> &fi,
Utils::QtcProcess *app) const override;
TestOutputReader *createOutputReader(Utils::QtcProcess *app) const override;
QStringList argumentsForTestRunner(QStringList *omitted = nullptr) const override;
Utils::Environment filteredEnvironment(const Utils::Environment &original) const override;
};

6
src/plugins/autotest/boost/boosttestoutputreader.cpp
View File

@@ -5,6 +5,7 @@
#include "boosttestsettings.h"
#include "boosttestresult.h"
#include "../autotesttr.h"
#include "../testtreeitem.h"
@@ -21,12 +22,11 @@ namespace Internal {
static Q_LOGGING_CATEGORY(orLog, "qtc.autotest.boost.outputreader", QtWarningMsg)
BoostTestOutputReader::BoostTestOutputReader(const QFutureInterface<TestResult> &futureInterface,
QtcProcess *testApplication,
BoostTestOutputReader::BoostTestOutputReader(QtcProcess *testApplication,
const FilePath &buildDirectory,
const FilePath &projectFile,
LogLevel log, ReportLevel report)
: TestOutputReader(futureInterface, testApplication, buildDirectory)
: TestOutputReader(testApplication, buildDirectory)
, m_projectFile(projectFile)
, m_logLevel(log)
, m_reportLevel(report)

3
src/plugins/autotest/boost/boosttestoutputreader.h
View File

@@ -15,8 +15,7 @@ class BoostTestOutputReader : public TestOutputReader
{
Q_OBJECT
public:
BoostTestOutputReader(const QFutureInterface<TestResult> &futureInterface,
Utils::QtcProcess *testApplication, const Utils::FilePath &buildDirectory,
BoostTestOutputReader(Utils::QtcProcess *testApplication, const Utils::FilePath &buildDirectory,
const Utils::FilePath &projectFile, LogLevel log, ReportLevel report);
protected:
void processOutputLine(const QByteArray &outputLine) override;

8
src/plugins/autotest/catch/catchconfiguration.cpp
View File

@@ -2,6 +2,7 @@
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0
#include "catchconfiguration.h"
#include "catchoutputreader.h"
#include "catchtestsettings.h"
@@ -9,17 +10,14 @@
#include "../itestframework.h"
#include "../testsettings.h"
#include <utils/stringutils.h>
using namespace Utils;
namespace Autotest {
namespace Internal {
TestOutputReader *CatchConfiguration::createOutputReader(const QFutureInterface<TestResult> &fi,
QtcProcess *app) const
TestOutputReader *CatchConfiguration::createOutputReader(QtcProcess *app) const
{
return new CatchOutputReader(fi, app, buildDirectory(), projectFile());
return new CatchOutputReader(app, buildDirectory(), projectFile());
}
static QStringList filterInterfering(const QStringList &provided, QStringList *omitted)

3
src/plugins/autotest/catch/catchconfiguration.h
View File

@@ -12,8 +12,7 @@ class CatchConfiguration : public DebuggableTestConfiguration
{
public:
CatchConfiguration(ITestFramework *framework) : DebuggableTestConfiguration(framework) {}
TestOutputReader *createOutputReader(const QFutureInterface<TestResult> &fi,
Utils::QtcProcess *app) const override;
TestOutputReader *createOutputReader(Utils::QtcProcess *app) const override;
QStringList argumentsForTestRunner(QStringList *omitted = nullptr) const override;
Utils::Environment filteredEnvironment(const Utils::Environment &original) const override;
};

9
src/plugins/autotest/catch/catchoutputreader.cpp
View File

@@ -2,13 +2,11 @@
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0
#include "catchoutputreader.h"
#include "catchresult.h"
#include "../autotesttr.h"
#include <utils/fileutils.h>
#include <utils/qtcassert.h>
using namespace Utils;
namespace Autotest {
@@ -31,11 +29,10 @@ namespace CatchXml {
const char TestCaseResultElement[] = "OverallResult";
}
CatchOutputReader::CatchOutputReader(const QFutureInterface<TestResult> &futureInterface,
QtcProcess *testApplication,
CatchOutputReader::CatchOutputReader(QtcProcess *testApplication,
const FilePath &buildDirectory,
const FilePath &projectFile)
: TestOutputReader (futureInterface, testApplication, buildDirectory)
: TestOutputReader(testApplication, buildDirectory)
, m_projectFile(projectFile)
{
}

3
src/plugins/autotest/catch/catchoutputreader.h
View File

@@ -14,8 +14,7 @@ namespace Internal {
class CatchOutputReader : public TestOutputReader
{
public:
CatchOutputReader(const QFutureInterface<TestResult> &futureInterface,
Utils::QtcProcess *testApplication, const Utils::FilePath &buildDirectory,
CatchOutputReader(Utils::QtcProcess *testApplication, const Utils::FilePath &buildDirectory,
const Utils::FilePath &projectFile);
protected:

5
src/plugins/autotest/ctest/ctestconfiguration.cpp
View File

@@ -13,10 +13,9 @@ CTestConfiguration::CTestConfiguration(ITestBase *testBase)
setDisplayName("CTest");
}
TestOutputReader *CTestConfiguration::createOutputReader(const QFutureInterface<TestResult> &fi,
Utils::QtcProcess *app) const
TestOutputReader *CTestConfiguration::createOutputReader(Utils::QtcProcess *app) const
{
return new CTestOutputReader(fi, app, workingDirectory());
return new CTestOutputReader(app, workingDirectory());
}
} // namespace Internal

3
src/plugins/autotest/ctest/ctestconfiguration.h
View File

@@ -13,8 +13,7 @@ class CTestConfiguration final : public Autotest::TestToolConfiguration
public:
explicit CTestConfiguration(ITestBase *testBase);
TestOutputReader *createOutputReader(const QFutureInterface<TestResult> &fi,
Utils::QtcProcess *app) const final;
TestOutputReader *createOutputReader(Utils::QtcProcess *app) const final;
};
} // namespace Internal

7
src/plugins/autotest/ctest/ctestoutputreader.cpp
View File

@@ -5,13 +5,11 @@
#include "../autotesttr.h"
#include "../testframeworkmanager.h"
#include "../testresult.h"
#include "../testtreeitem.h"
#include <cmakeprojectmanager/cmakeprojectconstants.h>
#include <utils/qtcassert.h>
#include <utils/treemodel.h>
#include <QRegularExpression>
@@ -52,10 +50,9 @@ public:
{}
};
CTestOutputReader::CTestOutputReader(const QFutureInterface<TestResult> &futureInterface,
QtcProcess *testApplication,
CTestOutputReader::CTestOutputReader(QtcProcess *testApplication,
const FilePath &buildDirectory)
: TestOutputReader(futureInterface, testApplication, buildDirectory)
: TestOutputReader(testApplication, buildDirectory)
{
}

3
src/plugins/autotest/ctest/ctestoutputreader.h
View File

@@ -13,8 +13,7 @@ namespace Internal {
class CTestOutputReader final : public Autotest::TestOutputReader
{
public:
CTestOutputReader(const QFutureInterface<TestResult> &futureInterface,
Utils::QtcProcess *testApplication, const Utils::FilePath &buildDirectory);
CTestOutputReader(Utils::QtcProcess *testApplication, const Utils::FilePath &buildDirectory);
protected:
void processOutputLine(const QByteArray &outputLineWithNewLine) final;

7
src/plugins/autotest/gtest/gtestconfiguration.cpp
View File

@@ -5,22 +5,21 @@
#include "gtestoutputreader.h"
#include "gtestsettings.h"
#include "../autotestplugin.h"
#include "../itestframework.h"
#include "../testsettings.h"
#include <utils/algorithm.h>
#include <utils/stringutils.h>
using namespace Utils;
namespace Autotest {
namespace Internal {
TestOutputReader *GTestConfiguration::createOutputReader(const QFutureInterface<TestResult> &fi,
QtcProcess *app) const
TestOutputReader *GTestConfiguration::createOutputReader(QtcProcess *app) const
{
return new GTestOutputReader(fi, app, buildDirectory(), projectFile());
return new GTestOutputReader(app, buildDirectory(), projectFile());
}
QStringList filterInterfering(const QStringList &provided, QStringList *omitted)

3
src/plugins/autotest/gtest/gtestconfiguration.h
View File

@@ -14,8 +14,7 @@ public:
explicit GTestConfiguration(ITestFramework *framework)
: DebuggableTestConfiguration(framework) {}
TestOutputReader *createOutputReader(const QFutureInterface<TestResult> &fi,
Utils::QtcProcess *app) const override;
TestOutputReader *createOutputReader(Utils::QtcProcess *app) const override;
QStringList argumentsForTestRunner(QStringList *omitted = nullptr) const override;
Utils::Environment filteredEnvironment(const Utils::Environment &original) const override;
};

9
src/plugins/autotest/gtest/gtestoutputreader.cpp
View File

@@ -17,11 +17,10 @@ using namespace Utils;
namespace Autotest {
namespace Internal {
GTestOutputReader::GTestOutputReader(const QFutureInterface<TestResult> &futureInterface,
QtcProcess *testApplication,
GTestOutputReader::GTestOutputReader(QtcProcess *testApplication,
const FilePath &buildDirectory,
const FilePath &projectFile)
: TestOutputReader(futureInterface, testApplication, buildDirectory)
: TestOutputReader(testApplication, buildDirectory)
, m_projectFile(projectFile)
{
if (testApplication) {
@@ -121,7 +120,7 @@ void GTestOutputReader::processOutputLine(const QByteArray &outputLine)
testResult.setResult(ResultType::MessageInternal);
testResult.setDescription(Tr::tr("Execution took %1.").arg(match.captured(2)));
reportResult(testResult);
m_futureInterface.setProgressValue(m_futureInterface.progressValue() + 1);
// TODO: bump progress?
} else if (ExactMatch match = testSetFail.match(line)) {
m_testSetStarted = false;
TestResult testResult = createDefaultResult();
@@ -132,7 +131,7 @@ void GTestOutputReader::processOutputLine(const QByteArray &outputLine)
testResult.setResult(ResultType::MessageInternal);
testResult.setDescription(Tr::tr("Execution took %1.").arg(match.captured(2)));
reportResult(testResult);
m_futureInterface.setProgressValue(m_futureInterface.progressValue() + 1);
// TODO: bump progress?
} else if (ExactMatch match = testSetSkipped.match(line)) {
if (!m_testSetStarted) // ignore SKIPPED at summary
return;

3
src/plugins/autotest/gtest/gtestoutputreader.h
View File

@@ -11,8 +11,7 @@ namespace Internal {
class GTestOutputReader : public TestOutputReader
{
public:
GTestOutputReader(const QFutureInterface<TestResult> &futureInterface,
Utils::QtcProcess *testApplication, const Utils::FilePath &buildDirectory,
GTestOutputReader(Utils::QtcProcess *testApplication, const Utils::FilePath &buildDirectory,
const Utils::FilePath &projectFile);
protected:
void processOutputLine(const QByteArray &outputLine) override;

9
src/plugins/autotest/qtest/qttestconfiguration.cpp
View File

@@ -2,16 +2,16 @@
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0
#include "qttestconfiguration.h"
#include "qttestconstants.h"
#include "qttestoutputreader.h"
#include "qttestsettings.h"
#include "qttest_utils.h"
#include "../autotestplugin.h"
#include "../itestframework.h"
#include "../testsettings.h"
#include <utils/algorithm.h>
#include <utils/stringutils.h>
using namespace Utils;
@@ -28,14 +28,13 @@ static QStringList quoteIfNeeded(const QStringList &testCases, bool debugMode)
});
}
TestOutputReader *QtTestConfiguration::createOutputReader(const QFutureInterface<TestResult> &fi,
QtcProcess *app) const
TestOutputReader *QtTestConfiguration::createOutputReader(QtcProcess *app) const
{
auto qtSettings = static_cast<QtTestSettings *>(framework()->testSettings());
const QtTestOutputReader::OutputMode mode = qtSettings && qtSettings->useXMLOutput.value()
? QtTestOutputReader::XML
: QtTestOutputReader::PlainText;
return new QtTestOutputReader(fi, app, buildDirectory(), projectFile(), mode, TestType::QtTest);
return new QtTestOutputReader(app, buildDirectory(), projectFile(), mode, TestType::QtTest);
}
QStringList QtTestConfiguration::argumentsForTestRunner(QStringList *omitted) const

3
src/plugins/autotest/qtest/qttestconfiguration.h
View File

@@ -13,8 +13,7 @@ class QtTestConfiguration : public DebuggableTestConfiguration
public:
explicit QtTestConfiguration(ITestFramework *framework)
: DebuggableTestConfiguration(framework) {}
TestOutputReader *createOutputReader(const QFutureInterface<TestResult> &fi,
Utils::QtcProcess *app) const override;
TestOutputReader *createOutputReader(Utils::QtcProcess *app) const override;
QStringList argumentsForTestRunner(QStringList *omitted = nullptr) const override;
Utils::Environment filteredEnvironment(const Utils::Environment &original) const override;
};

18
src/plugins/autotest/qtest/qttestoutputreader.cpp
View File

@@ -12,8 +12,6 @@
#include <QRegularExpression>
#include <cctype>
using namespace Utils;
namespace Autotest {
@@ -99,18 +97,15 @@ static QString constructBenchmarkInformation(const QString &metric, double value
else if (metric == "CPUCycles") // -perf
metricsText = "CPU cycles";
return Tr::tr("%1 %2 per iteration (total: %3, iterations: %4)")
.arg(formatResult(value))
.arg(metricsText)
.arg(formatResult(value * double(iterations)))
.arg(formatResult(value), metricsText, formatResult(value * double(iterations)))
.arg(iterations);
}
QtTestOutputReader::QtTestOutputReader(const QFutureInterface<TestResult> &futureInterface,
QtcProcess *testApplication,
QtTestOutputReader::QtTestOutputReader(QtcProcess *testApplication,
const FilePath &buildDirectory,
const FilePath &projectFile,
OutputMode mode, TestType type)
: TestOutputReader(futureInterface, testApplication, buildDirectory)
: TestOutputReader(testApplication, buildDirectory)
, m_projectFile(projectFile)
, m_mode(mode)
, m_testType(type)
@@ -177,8 +172,6 @@ void QtTestOutputReader::processXMLOutput(const QByteArray &outputLine)
m_xmlReader.addData("\n");
m_xmlReader.addData(QString::fromUtf8(outputLine));
while (!m_xmlReader.atEnd()) {
if (m_futureInterface.isCanceled())
return;
QXmlStreamReader::TokenType token = m_xmlReader.readNext();
switch (token) {
case QXmlStreamReader::StartDocument:
@@ -277,7 +270,7 @@ void QtTestOutputReader::processXMLOutput(const QByteArray &outputLine)
const QStringView currentTag = m_xmlReader.name();
if (currentTag == QStringLiteral("TestFunction")) {
sendFinishMessage(true);
m_futureInterface.setProgressValue(m_futureInterface.progressValue() + 1);
// TODO: bump progress?
m_dataTag.clear();
m_formerTestCase = m_testCase;
m_testCase.clear();
@@ -347,9 +340,6 @@ void QtTestOutputReader::processPlainTextOutput(const QByteArray &outputLine)
static const QRegularExpression locationUnix(QT_TEST_FAIL_UNIX_REGEXP);
static const QRegularExpression locationWin(QT_TEST_FAIL_WIN_REGEXP);
if (m_futureInterface.isCanceled())
return;
const QString line = QString::fromUtf8(outputLine);
QRegularExpressionMatch match;

6
src/plugins/autotest/qtest/qttestoutputreader.h
View File

@@ -3,9 +3,10 @@
#pragma once
#include "qttestconstants.h"
#include "../testoutputreader.h"
#include "qttestconstants.h"
#include <QXmlStreamReader>
namespace Autotest {
@@ -22,8 +23,7 @@ public:
PlainText
};
QtTestOutputReader(const QFutureInterface<TestResult> &futureInterface,
Utils::QtcProcess *testApplication, const Utils::FilePath &buildDirectory,
QtTestOutputReader(Utils::QtcProcess *testApplication, const Utils::FilePath &buildDirectory,
const Utils::FilePath &projectFile, OutputMode mode, TestType type);
protected:
void processOutputLine(const QByteArray &outputLine) override;

14
src/plugins/autotest/quick/quicktestconfiguration.cpp
View File

@@ -2,16 +2,14 @@
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0
#include "quicktestconfiguration.h"
#include "../qtest/qttestconstants.h"
#include "../autotestplugin.h"
#include "../itestframework.h"
#include "../qtest/qttestoutputreader.h"
#include "../qtest/qttestsettings.h"
#include "../qtest/qttest_utils.h"
#include "../autotestplugin.h"
#include "../itestframework.h"
#include "../testsettings.h"
#include <utils/stringutils.h>
using namespace Utils;
namespace Autotest {
@@ -23,15 +21,13 @@ QuickTestConfiguration::QuickTestConfiguration(ITestFramework *framework)
setMixedDebugging(true);
}
TestOutputReader *QuickTestConfiguration::createOutputReader(
const QFutureInterface<TestResult> &fi, QtcProcess *app) const
TestOutputReader *QuickTestConfiguration::createOutputReader(QtcProcess *app) const
{
auto qtSettings = static_cast<QtTestSettings *>(framework()->testSettings());
const QtTestOutputReader::OutputMode mode = qtSettings && qtSettings->useXMLOutput.value()
? QtTestOutputReader::XML
: QtTestOutputReader::PlainText;
return new QtTestOutputReader(fi, app, buildDirectory(), projectFile(),
mode, TestType::QuickTest);
return new QtTestOutputReader(app, buildDirectory(), projectFile(), mode, TestType::QuickTest);
}
QStringList QuickTestConfiguration::argumentsForTestRunner(QStringList *omitted) const

3
src/plugins/autotest/quick/quicktestconfiguration.h
View File

@@ -12,8 +12,7 @@ class QuickTestConfiguration : public DebuggableTestConfiguration
{
public:
explicit QuickTestConfiguration(ITestFramework *framework);
TestOutputReader *createOutputReader(const QFutureInterface<TestResult> &fi,
Utils::QtcProcess *app) const override;
TestOutputReader *createOutputReader(Utils::QtcProcess *app) const override;
QStringList argumentsForTestRunner(QStringList *omitted = nullptr) const override;
Utils::Environment filteredEnvironment(const Utils::Environment &original) const override;
};

10
src/plugins/autotest/testconfiguration.cpp
View File

@@ -4,22 +4,19 @@
#include "testconfiguration.h"
#include "itestframework.h"
#include "testoutputreader.h"
#include "testrunconfiguration.h"
#include <cppeditor/cppmodelmanager.h>
#include <cppeditor/projectinfo.h>
#include <projectexplorer/buildconfiguration.h>
#include <projectexplorer/buildsystem.h>
#include <projectexplorer/buildtargetinfo.h>
#include <projectexplorer/deploymentdata.h>
#include <projectexplorer/environmentaspect.h>
#include <projectexplorer/kitinformation.h>
#include <projectexplorer/runconfiguration.h>
#include <projectexplorer/session.h>
#include <projectexplorer/target.h>
#include <utils/algorithm.h>
#include <QLoggingCategory>
static Q_LOGGING_CATEGORY(LOG, "qtc.autotest.testconfiguration", QtWarningMsg)
@@ -29,7 +26,6 @@ using namespace Utils;
namespace Autotest {
ITestConfiguration::ITestConfiguration(ITestBase *testBase)
: m_testBase(testBase)
{
@@ -94,7 +90,7 @@ static FilePath ensureExeEnding(const FilePath &file)
return file.withExecutableSuffix();
}
void TestConfiguration::completeTestInformation(ProjectExplorer::RunConfiguration *rc,
void TestConfiguration::completeTestInformation(RunConfiguration *rc,
TestRunMode runMode)
{
QTC_ASSERT(rc, return);

4
src/plugins/autotest/testconfiguration.h
View File

@@ -9,7 +9,6 @@
#include <projectexplorer/runcontrol.h>
#include <utils/environment.h>
#include <QFutureInterface>
#include <QPointer>
#include <QStringList>
@@ -40,8 +39,7 @@ public:
Utils::FilePath executableFilePath() const;
virtual Utils::FilePath testExecutable() const { return executableFilePath(); };
virtual TestOutputReader *createOutputReader(const QFutureInterface<TestResult> &fi,
Utils::QtcProcess *app) const = 0;
virtual TestOutputReader *createOutputReader(Utils::QtcProcess *app) const = 0;
virtual Utils::Environment filteredEnvironment(const Utils::Environment &original) const;
ITestBase *testBase() const { return m_testBase; }

17
src/plugins/autotest/testoutputreader.cpp
View File

@@ -4,17 +4,12 @@
#include "testoutputreader.h"
#include "autotesttr.h"
#include "testresult.h"
#include "testresultspane.h"
#include "testtreeitem.h"
#include <utils/qtcassert.h>
#include <utils/qtcprocess.h>
#include <QDebug>
#include <QDir>
#include <QFileInfo>
#include <QProcess>
#include <QRegularExpression>
using namespace Utils;
@@ -26,11 +21,8 @@ FilePath TestOutputReader::constructSourceFilePath(const FilePath &path, const Q
return filePath.isReadableFile() ? filePath : FilePath();
}
TestOutputReader::TestOutputReader(const QFutureInterface<TestResult> &futureInterface,
QtcProcess *testApplication, const FilePath &buildDirectory)
: m_futureInterface(futureInterface)
, m_buildDir(buildDirectory)
, m_id(testApplication ? testApplication->commandLine().executable().toUserOutput() : QString())
TestOutputReader::TestOutputReader(QtcProcess *testApplication, const FilePath &buildDirectory)
: m_buildDir(buildDirectory)
{
auto chopLineBreak = [](QByteArray line) {
if (line.endsWith('\n'))
@@ -41,6 +33,9 @@ TestOutputReader::TestOutputReader(const QFutureInterface<TestResult> &futureInt
};
if (testApplication) {
connect(testApplication, &QtcProcess::started, this, [this, testApplication] {
m_id = testApplication->commandLine().executable().toUserOutput();
});
testApplication->setStdOutLineCallback([this, &chopLineBreak](const QString &line) {
processStdOutput(chopLineBreak(line.toUtf8()));
});

6
src/plugins/autotest/testoutputreader.h
View File

@@ -5,9 +5,7 @@
#include "testresult.h"
#include <QFutureInterface>
#include <QObject>
#include <QString>
namespace Utils { class QtcProcess; }
@@ -17,8 +15,7 @@ class TestOutputReader : public QObject
{
Q_OBJECT
public:
TestOutputReader(const QFutureInterface<TestResult> &futureInterface,
Utils::QtcProcess *testApplication, const Utils::FilePath &buildDirectory);
TestOutputReader(Utils::QtcProcess *testApplication, const Utils::FilePath &buildDirectory);
virtual ~TestOutputReader();
void processStdOutput(const QByteArray &outputLine);
virtual void processStdError(const QByteArray &outputLine);
@@ -46,7 +43,6 @@ protected:
void sendAndResetSanitizerResult();
void reportResult(const TestResult &result);
QFutureInterface<TestResult> m_futureInterface;
Utils::FilePath m_buildDir;
QString m_id;
QHash<ResultType, int> m_summary;

365
src/plugins/autotest/testrunner.cpp
View File

@@ -6,18 +6,15 @@
#include "autotestconstants.h"
#include "autotestplugin.h"
#include "autotesttr.h"
#include "itestframework.h"
#include "testoutputreader.h"
#include "testprojectsettings.h"
#include "testresultspane.h"
#include "testrunconfiguration.h"
#include "testsettings.h"
#include "testtreeitem.h"
#include "testtreemodel.h"
#include <coreplugin/icore.h>
#include <coreplugin/progressmanager/futureprogress.h>
#include <coreplugin/progressmanager/progressmanager.h>
#include <coreplugin/progressmanager/taskprogress.h>
#include <debugger/debuggerkitinformation.h>
#include <debugger/debuggerruncontrol.h>
@@ -47,10 +44,9 @@
#include <QLabel>
#include <QLoggingCategory>
#include <QPointer>
#include <QProcess>
#include <QPushButton>
#include <QTimer>
using namespace Core;
using namespace ProjectExplorer;
using namespace Utils;
@@ -72,14 +68,7 @@ TestRunner::TestRunner()
m_cancelTimer.setSingleShot(true);
connect(&m_cancelTimer, &QTimer::timeout, this, [this] { cancelCurrent(Timeout); });
connect(&m_futureWatcher, &QFutureWatcher<TestResult>::finished,
this, &TestRunner::onFinished);
connect(this, &TestRunner::requestStopTestRun,
&m_futureWatcher, &QFutureWatcher<TestResult>::cancel);
connect(&m_futureWatcher, &QFutureWatcher<TestResult>::canceled, this, [this] {
cancelCurrent(UserCanceled);
reportResult(ResultType::MessageFatal, Tr::tr("Test run canceled by user."));
});
connect(this, &TestRunner::requestStopTestRun, this, [this] { cancelCurrent(UserCanceled); });
connect(BuildManager::instance(), &BuildManager::buildQueueFinished,
this, &TestRunner::onBuildQueueFinished);
}
@@ -93,7 +82,7 @@ TestRunner::~TestRunner()
void TestRunner::runTest(TestRunMode mode, const ITestTreeItem *item)
{
QTC_ASSERT(!m_executingTests, return);
QTC_ASSERT(!isTestRunning(), return);
ITestConfiguration *configuration = item->asConfiguration(mode);
if (configuration)
@@ -144,180 +133,21 @@ static QString constructOmittedVariablesDetailsString(const EnvironmentItems &di
+ '\n' + removedVars.join('\n');
}
bool TestRunner::currentConfigValid()
{
const FilePath commandFilePath = m_currentConfig->testExecutable();
if (!commandFilePath.isEmpty())
return true;
reportResult(ResultType::MessageFatal,
Tr::tr("Executable path is empty. (%1)").arg(m_currentConfig->displayName()));
delete m_currentConfig;
m_currentConfig = nullptr;
if (m_selectedTests.isEmpty()) {
if (m_fakeFutureInterface)
m_fakeFutureInterface->reportFinished();
onFinished();
} else {
onProcessDone();
}
return false;
}
void TestRunner::setUpProcessEnv()
{
CommandLine command = m_currentProcess->commandLine();
if (m_currentConfig->testBase()->type() == ITestBase::Framework) {
TestConfiguration *current = static_cast<TestConfiguration *>(m_currentConfig);
QStringList omitted;
command.addArgs(current->argumentsForTestRunner(&omitted).join(' '), CommandLine::Raw);
if (!omitted.isEmpty()) {
const QString &details = constructOmittedDetailsString(omitted);
reportResult(ResultType::MessageWarn, details.arg(current->displayName()));
}
} else {
TestToolConfiguration *current = static_cast<TestToolConfiguration *>(m_currentConfig);
command.setArguments(current->commandLine().arguments());
}
m_currentProcess->setCommand(command);
m_currentProcess->setWorkingDirectory(m_currentConfig->workingDirectory());
const Environment &original = m_currentConfig->environment();
Environment environment = m_currentConfig->filteredEnvironment(original);
const EnvironmentItems removedVariables = Utils::filtered(
original.diff(environment), [](const EnvironmentItem &it) {
return it.operation == EnvironmentItem::Unset;
});
if (!removedVariables.isEmpty()) {
const QString &details = constructOmittedVariablesDetailsString(removedVariables)
.arg(m_currentConfig->displayName());
reportResult(ResultType::MessageWarn, details);
}
m_currentProcess->setEnvironment(environment);
}
void TestRunner::scheduleNext()
{
QTC_ASSERT(!m_selectedTests.isEmpty(), onFinished(); return);
QTC_ASSERT(!m_currentConfig && !m_currentProcess, resetInternalPointers());
QTC_ASSERT(m_fakeFutureInterface, onFinished(); return);
QTC_ASSERT(!m_canceled, onFinished(); return);
m_currentConfig = m_selectedTests.takeFirst();
if (!currentConfigValid())
return;
if (!m_currentConfig->project())
onProcessDone();
m_currentProcess = new QtcProcess;
m_currentProcess->setCommand({m_currentConfig->testExecutable(), {}});
QTC_ASSERT(!m_currentOutputReader, delete m_currentOutputReader);
m_currentOutputReader = m_currentConfig->createOutputReader(*m_fakeFutureInterface, m_currentProcess);
QTC_ASSERT(m_currentOutputReader, onProcessDone(); return);
connect(m_currentOutputReader, &TestOutputReader::newResult, this, &TestRunner::testResultReady);
connect(m_currentOutputReader, &TestOutputReader::newOutputLineAvailable,
TestResultsPane::instance(), &TestResultsPane::addOutputLine);
setUpProcessEnv();
connect(m_currentProcess, &QtcProcess::done, this, &TestRunner::onProcessDone);
const int timeout = AutotestPlugin::settings()->timeout;
m_cancelTimer.setInterval(timeout);
m_cancelTimer.start();
qCInfo(runnerLog) << "Command:" << m_currentProcess->commandLine().executable();
qCInfo(runnerLog) << "Arguments:" << m_currentProcess->commandLine().arguments();
qCInfo(runnerLog) << "Working directory:" << m_currentProcess->workingDirectory();
qCDebug(runnerLog) << "Environment:" << m_currentProcess->environment().toStringList();
m_currentProcess->start();
}
void TestRunner::cancelCurrent(TestRunner::CancelReason reason)
{
m_canceled = true;
if (m_fakeFutureInterface)
m_fakeFutureInterface->reportCanceled();
if (reason == KitChanged)
reportResult(ResultType::MessageWarn, Tr::tr("Current kit has changed. Canceling test run."));
else if (reason == Timeout)
reportResult(ResultType::MessageFatal, Tr::tr("Test case canceled due to timeout.\nMaybe raise the timeout?"));
// if user or timeout cancels the current run ensure to kill the running process
if (m_currentProcess && m_currentProcess->state() != QProcess::NotRunning) {
m_currentProcess->kill();
m_currentProcess->waitForFinished();
}
}
void TestRunner::onProcessDone()
{
if (m_currentProcess->result() == ProcessResult::StartFailed) {
reportResult(ResultType::MessageFatal,
Tr::tr("Failed to start test for project \"%1\".").arg(m_currentConfig->displayName())
+ processInformation(m_currentProcess) + rcInfo(m_currentConfig));
}
if (m_executingTests && m_currentConfig) {
QTC_CHECK(m_fakeFutureInterface);
m_fakeFutureInterface->setProgressValue(m_fakeFutureInterface->progressValue()
+ m_currentConfig->testCaseCount());
if (m_currentProcess && !m_fakeFutureInterface->isCanceled()) {
if (m_currentProcess->exitStatus() == QProcess::CrashExit) {
if (m_currentOutputReader)
m_currentOutputReader->reportCrash();
reportResult(ResultType::MessageFatal,
Tr::tr("Test for project \"%1\" crashed.").arg(m_currentConfig->displayName())
+ processInformation(m_currentProcess) + rcInfo(m_currentConfig));
} else if (m_currentOutputReader && !m_currentOutputReader->hadValidOutput()) {
reportResult(ResultType::MessageFatal,
Tr::tr("Test for project \"%1\" did not produce any expected output.")
.arg(m_currentConfig->displayName()) + processInformation(m_currentProcess)
+ rcInfo(m_currentConfig));
}
}
}
if (m_currentOutputReader) {
const int disabled = m_currentOutputReader->disabledTests();
if (disabled > 0)
emit hadDisabledTests(disabled);
if (m_currentOutputReader->hasSummary())
emit reportSummary(m_currentOutputReader->id(), m_currentOutputReader->summary());
m_currentOutputReader->resetCommandlineColor();
}
resetInternalPointers();
if (!m_fakeFutureInterface) {
QTC_ASSERT(!m_executingTests, m_executingTests = false);
return;
}
if (!m_selectedTests.isEmpty() && !m_fakeFutureInterface->isCanceled())
scheduleNext();
else
m_fakeFutureInterface->reportFinished();
}
void TestRunner::resetInternalPointers()
{
delete m_currentOutputReader;
if (m_currentProcess)
m_currentProcess->deleteLater();
delete m_currentConfig;
m_currentOutputReader = nullptr;
m_currentProcess = nullptr;
m_currentConfig = nullptr;
else if (reason == UserCanceled)
reportResult(ResultType::MessageFatal, Tr::tr("Test run canceled by user."));
m_taskTree.reset();
onFinished();
}
void TestRunner::runTests(TestRunMode mode, const QList<ITestConfiguration *> &selectedTests)
{
QTC_ASSERT(!m_executingTests, return);
QTC_ASSERT(!isTestRunning(), return);
qDeleteAll(m_selectedTests);
m_selectedTests = selectedTests;
@@ -332,8 +162,6 @@ void TestRunner::runTests(TestRunMode mode, const QList<ITestConfiguration *> &s
return;
}
m_executingTests = true;
m_canceled = false;
emit testRunStarted();
// clear old log and output pane
@@ -411,7 +239,7 @@ static RunConfiguration *getRunConfiguration(const QString &buildTargetKey)
if (runConfigurations.size() == 1)
return runConfigurations.first();
RunConfigurationSelectionDialog dialog(buildTargetKey, Core::ICore::dialogParent());
RunConfigurationSelectionDialog dialog(buildTargetKey, ICore::dialogParent());
if (dialog.exec() == QDialog::Accepted) {
const QString dName = dialog.displayName();
if (dName.isEmpty())
@@ -513,19 +341,136 @@ void TestRunner::runTestsHelper()
return;
}
int testCaseCount = precheckTestConfigurations();
const int testCaseCount = precheckTestConfigurations();
Q_UNUSED(testCaseCount) // TODO: may be useful for fine-grained progress reporting, when fixed
// Fake future interface - destruction will be handled by QFuture/QFutureWatcher
m_fakeFutureInterface = new QFutureInterface<TestResult>(QFutureInterfaceBase::Running);
QFuture<TestResult> future = m_fakeFutureInterface->future();
m_fakeFutureInterface->setProgressRange(0, testCaseCount);
m_fakeFutureInterface->setProgressValue(0);
m_futureWatcher.setFuture(future);
struct TestStorage {
std::unique_ptr<TestOutputReader> m_outputReader;
};
using namespace Tasking;
QList<TaskItem> tasks{optional};
for (ITestConfiguration *config : m_selectedTests) {
QTC_ASSERT(config, continue);
const TreeStorage<TestStorage> storage;
const auto onGroupSetup = [this, config] {
if (!config->project())
return TaskAction::StopWithDone;
if (config->testExecutable().isEmpty()) {
reportResult(ResultType::MessageFatal,
Tr::tr("Executable path is empty. (%1)").arg(config->displayName()));
return TaskAction::StopWithDone;
}
return TaskAction::Continue;
};
const auto onSetup = [this, config, storage](QtcProcess &process) {
TestStorage *testStorage = storage.activeStorage();
QTC_ASSERT(testStorage, return);
testStorage->m_outputReader.reset(config->createOutputReader(&process));
QTC_ASSERT(testStorage->m_outputReader, return);
connect(testStorage->m_outputReader.get(), &TestOutputReader::newResult,
this, &TestRunner::testResultReady);
connect(testStorage->m_outputReader.get(), &TestOutputReader::newOutputLineAvailable,
TestResultsPane::instance(), &TestResultsPane::addOutputLine);
CommandLine command{config->testExecutable(), {}};
if (config->testBase()->type() == ITestBase::Framework) {
TestConfiguration *current = static_cast<TestConfiguration *>(config);
QStringList omitted;
command.addArgs(current->argumentsForTestRunner(&omitted).join(' '), CommandLine::Raw);
if (!omitted.isEmpty()) {
const QString &details = constructOmittedDetailsString(omitted);
reportResult(ResultType::MessageWarn, details.arg(current->displayName()));
}
} else {
TestToolConfiguration *current = static_cast<TestToolConfiguration *>(config);
command.setArguments(current->commandLine().arguments());
}
process.setCommand(command);
process.setWorkingDirectory(config->workingDirectory());
const Environment &original = config->environment();
Environment environment = config->filteredEnvironment(original);
const EnvironmentItems removedVariables = Utils::filtered(
original.diff(environment), [](const EnvironmentItem &it) {
return it.operation == EnvironmentItem::Unset;
});
if (!removedVariables.isEmpty()) {
const QString &details = constructOmittedVariablesDetailsString(removedVariables)
.arg(config->displayName());
reportResult(ResultType::MessageWarn, details);
}
process.setEnvironment(environment);
m_cancelTimer.setInterval(AutotestPlugin::settings()->timeout);
m_cancelTimer.start();
qCInfo(runnerLog) << "Command:" << process.commandLine().executable();
qCInfo(runnerLog) << "Arguments:" << process.commandLine().arguments();
qCInfo(runnerLog) << "Working directory:" << process.workingDirectory();
qCDebug(runnerLog) << "Environment:" << process.environment().toStringList();
};
const auto onDone = [this, config, storage](const QtcProcess &process) {
TestStorage *testStorage = storage.activeStorage();
QTC_ASSERT(testStorage, return);
if (process.result() == ProcessResult::StartFailed) {
reportResult(ResultType::MessageFatal,
Tr::tr("Failed to start test for project \"%1\".").arg(config->displayName())
+ processInformation(&process) + rcInfo(config));
}
if (process.exitStatus() == QProcess::CrashExit) {
if (testStorage->m_outputReader)
testStorage->m_outputReader->reportCrash();
reportResult(ResultType::MessageFatal,
Tr::tr("Test for project \"%1\" crashed.").arg(config->displayName())
+ processInformation(&process) + rcInfo(config));
} else if (testStorage->m_outputReader && !testStorage->m_outputReader->hadValidOutput()) {
reportResult(ResultType::MessageFatal,
Tr::tr("Test for project \"%1\" did not produce any expected output.")
.arg(config->displayName()) + processInformation(&process)
+ rcInfo(config));
}
if (testStorage->m_outputReader) {
const int disabled = testStorage->m_outputReader->disabledTests();
if (disabled > 0)
emit hadDisabledTests(disabled);
if (testStorage->m_outputReader->hasSummary())
emit reportSummary(testStorage->m_outputReader->id(), testStorage->m_outputReader->summary());
testStorage->m_outputReader->resetCommandlineColor();
}
};
const Group group {
optional,
Storage(storage),
OnGroupSetup(onGroupSetup),
Process(onSetup, onDone, onDone)
};
tasks.append(group);
}
m_taskTree.reset(new TaskTree(tasks));
connect(m_taskTree.get(), &TaskTree::done, this, &TestRunner::onFinished);
connect(m_taskTree.get(), &TaskTree::errorOccurred, this, &TestRunner::onFinished);
auto progress = new TaskProgress(m_taskTree.get());
progress->setDisplayName(tr("Running Tests"));
progress->setAutoStopOnCancel(false);
progress->setHalfLifeTimePerTask(10000); // 10 seconds
connect(progress, &TaskProgress::canceled, this, [this, progress] {
// progress was a child of task tree which is going to be deleted directly. Unwind properly.
progress->setParent(nullptr);
progress->deleteLater();
cancelCurrent(UserCanceled);
});
Core::ProgressManager::addTask(future, Tr::tr("Running Tests"), Autotest::Constants::TASK_INDEX);
if (AutotestPlugin::settings()->popupOnStart)
AutotestPlugin::popupResultsPane();
scheduleNext();
m_taskTree->start();
}
static void processOutput(TestOutputReader *outputreader, const QString &msg, OutputFormat format)
@@ -626,13 +571,8 @@ void TestRunner::debugTests()
}
}
// We need a fake QFuture for the results. TODO: replace with QtConcurrent::run
QFutureInterface<TestResult> *futureInterface
= new QFutureInterface<TestResult>(QFutureInterfaceBase::Running);
m_futureWatcher.setFuture(futureInterface->future());
if (useOutputProcessor) {
TestOutputReader *outputreader = config->createOutputReader(*futureInterface, nullptr);
TestOutputReader *outputreader = config->createOutputReader(nullptr);
connect(outputreader, &TestOutputReader::newResult, this, &TestRunner::testResultReady);
outputreader->setId(inferior.command.executable().toString());
connect(outputreader, &TestOutputReader::newOutputLineAvailable,
@@ -641,9 +581,7 @@ void TestRunner::debugTests()
this, [outputreader](const QString &msg, OutputFormat format) {
processOutput(outputreader, msg, format);
});
connect(runControl, &RunControl::stopped,
outputreader, &QObject::deleteLater);
connect(runControl, &RunControl::stopped, outputreader, &QObject::deleteLater);
}
m_stopDebugConnect = connect(this, &TestRunner::requestStopTestRun,
@@ -706,8 +644,7 @@ void TestRunner::buildProject(Project *project)
BuildManager *buildManager = BuildManager::instance();
m_buildConnect = connect(this, &TestRunner::requestStopTestRun,
buildManager, &BuildManager::cancel);
connect(buildManager, &BuildManager::buildQueueFinished,
this, &TestRunner::buildFinished);
connect(buildManager, &BuildManager::buildQueueFinished, this, &TestRunner::buildFinished);
BuildManager::buildProjectWithDependencies(project);
if (!BuildManager::isBuilding())
buildFinished(false);
@@ -717,19 +654,15 @@ void TestRunner::buildFinished(bool success)
{
disconnect(m_buildConnect);
BuildManager *buildManager = BuildManager::instance();
disconnect(buildManager, &BuildManager::buildQueueFinished,
this, &TestRunner::buildFinished);
disconnect(buildManager, &BuildManager::buildQueueFinished, this, &TestRunner::buildFinished);
if (success) {
if (!m_canceled)
runOrDebugTests();
else if (m_executingTests)
onFinished();
} else {
return;
}
reportResult(ResultType::MessageFatal, Tr::tr("Build failed. Canceling test run."));
onFinished();
}
}
static RunAfterBuildMode runAfterBuild()
{
@@ -747,7 +680,7 @@ static RunAfterBuildMode runAfterBuild()
void TestRunner::onBuildQueueFinished(bool success)
{
if (m_executingTests || !m_selectedTests.isEmpty()) // paranoia!
if (isTestRunning() || !m_selectedTests.isEmpty()) // paranoia!
return;
if (!success || m_runMode != TestRunMode::None)
@@ -768,17 +701,15 @@ void TestRunner::onBuildQueueFinished(bool success)
void TestRunner::onFinished()
{
m_cancelTimer.stop();
// if we've been canceled and we still have test configurations queued just throw them away
qDeleteAll(m_selectedTests);
m_selectedTests.clear();
if (m_taskTree)
m_taskTree.release()->deleteLater();
disconnect(m_stopDebugConnect);
disconnect(m_finishDebugConnect);
disconnect(m_targetConnect);
m_fakeFutureInterface = nullptr;
qDeleteAll(m_selectedTests);
m_selectedTests.clear();
m_cancelTimer.stop();
m_runMode = TestRunMode::None;
m_executingTests = false;
emit testRunFinished();
}

28
src/plugins/autotest/testrunner.h
View File

@@ -4,12 +4,11 @@
#pragma once
#include "autotest_global.h"
#include "testresult.h"
#include "autotestconstants.h"
#include <QDialog>
#include <QFutureWatcher>
#include <QList>
#include <QObject>
#include <QTimer>
QT_BEGIN_NAMESPACE
@@ -20,13 +19,14 @@ class QLabel;
QT_END_NAMESPACE
namespace ProjectExplorer { class Project; }
namespace Utils { class QtcProcess; }
namespace Utils { class TaskTree; }
namespace Autotest {
enum class TestRunMode;
class ITestConfiguration;
class TestOutputReader;
class ITestTreeItem;
class TestResult;
enum class ResultType;
namespace Internal {
@@ -44,7 +44,7 @@ public:
void runTests(TestRunMode mode, const QList<ITestConfiguration *> &selectedTests);
void runTest(TestRunMode mode, const ITestTreeItem *item);
bool isTestRunning() const { return m_executingTests; }
bool isTestRunning() const { return m_buildConnect || m_stopDebugConnect || m_taskTree.get(); }
signals:
void testRunStarted();
@@ -61,12 +61,7 @@ private:
void onFinished();
int precheckTestConfigurations();
bool currentConfigValid();
void setUpProcessEnv();
void scheduleNext();
void cancelCurrent(CancelReason reason);
void onProcessDone();
void resetInternalPointers();
void runTestsHelper();
void debugTests();
@@ -75,14 +70,9 @@ private:
bool postponeTestRunWithEmptyExecutable(ProjectExplorer::Project *project);
void onBuildSystemUpdated();
QFutureWatcher<TestResult> m_futureWatcher;
QFutureInterface<TestResult> *m_fakeFutureInterface = nullptr;
std::unique_ptr<Utils::TaskTree> m_taskTree;
QList<ITestConfiguration *> m_selectedTests;
bool m_executingTests = false;
bool m_canceled = false;
ITestConfiguration *m_currentConfig = nullptr;
Utils::QtcProcess *m_currentProcess = nullptr;
TestOutputReader *m_currentOutputReader = nullptr;
TestRunMode m_runMode = TestRunMode::None;
// temporarily used if building before running is necessary