AutoTest: Provide basic ctest integration

This allows to have ctest based tests listed inside the test
integration and to run them.

Task-number: QTCREATORBUG-23332
Change-Id: I1cb855697d7bb4afcd7c887506abdea16c252cb0
Reviewed-by: David Schulz <david.schulz@qt.io>
This commit is contained in:
Christian Stenger
2020-10-19 14:47:45 +02:00
parent f70e7e6c99
commit a59e04b848
12 changed files with 610 additions and 0 deletions

View File

@@ -25,6 +25,10 @@ add_qtc_plugin(AutoTest
catch/catchtestparser.cpp catch/catchtreeitem.h catch/catchtreeitem.cpp
catch/catchtestsettings.cpp catch/catchtestsettings.h
catch/catchtestsettingspage.cpp catch/catchtestsettingspage.h catch/catchtestsettingspage.ui
ctest/ctestconfiguration.cpp ctest/ctestconfiguration.h
ctest/ctestoutputreader.cpp ctest/ctestoutputreader.h
ctest/ctesttool.cpp ctest/ctesttool.h
ctest/ctesttreeitem.cpp ctest/ctesttreeitem.h
gtest/gtest_utils.cpp gtest/gtest_utils.h
gtest/gtestconfiguration.cpp gtest/gtestconfiguration.h
gtest/gtestconstants.h

View File

@@ -7,6 +7,10 @@ DEFINES += AUTOTEST_LIBRARY
SOURCES += \
autotestplugin.cpp \
ctest/ctestconfiguration.cpp \
ctest/ctestoutputreader.cpp \
ctest/ctesttool.cpp \
ctest/ctesttreeitem.cpp \
itestframework.cpp \
itestparser.cpp \
projectsettingswidget.cpp \
@@ -78,6 +82,10 @@ HEADERS += \
autotestconstants.h \
autotesticons.h \
autotestplugin.h \
ctest/ctestconfiguration.h \
ctest/ctestoutputreader.h \
ctest/ctesttool.h \
ctest/ctesttreeitem.h \
itemdatacache.h \
itestframework.h \
itestparser.h \

View File

@@ -120,6 +120,13 @@ QtcPlugin {
]
}
Group {
name: "CTest support files"
files: [
"ctest/*"
]
}
Group {
name: "Test sources"
condition: qtc.testsEnabled

View File

@@ -42,6 +42,7 @@
#include "boost/boosttestframework.h"
#include "catch/catchframework.h"
#include "ctest/ctesttool.h"
#include "gtest/gtestframework.h"
#include "qtest/qttestframework.h"
#include "quick/quicktestframework.h"
@@ -141,6 +142,8 @@ AutotestPluginPrivate::AutotestPluginPrivate()
m_frameworkManager.registerTestFramework(new BoostTestFramework);
m_frameworkManager.registerTestFramework(new CatchFramework);
m_frameworkManager.registerTestTool(new CTestTool);
m_frameworkManager.synchronizeSettings(ICore::settings());
m_navigationWidgetFactory = new TestNavigationWidgetFactory;
m_resultsPane = TestResultsPane::instance();

View File

@@ -0,0 +1,45 @@
/****************************************************************************
**
** Copyright (C) 2020 The Qt Company Ltd.
** Contact: https://www.qt.io/licensing/
**
** This file is part of Qt Creator.
**
** Commercial License Usage
** Licensees holding valid commercial Qt licenses may use this file in
** accordance with the commercial license agreement provided with the
** Software or, alternatively, in accordance with the terms contained in
** a written agreement between you and The Qt Company. For licensing terms
** and conditions see https://www.qt.io/terms-conditions. For further
** information use the contact form at https://www.qt.io/contact-us.
**
** GNU General Public License Usage
** Alternatively, this file may be used under the terms of the GNU
** General Public License version 3 as published by the Free Software
** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
** included in the packaging of this file. Please review the following
** information to ensure the GNU General Public License requirements will
** be met: https://www.gnu.org/licenses/gpl-3.0.html.
**
****************************************************************************/
#include "ctestconfiguration.h"
#include "ctestoutputreader.h"
namespace Autotest {
namespace Internal {
CTestConfiguration::CTestConfiguration(ITestBase *testBase)
: TestToolConfiguration(testBase)
{
setDisplayName("CTest");
}
TestOutputReader *CTestConfiguration::outputReader(const QFutureInterface<TestResultPtr> &fi,
QProcess *app) const
{
return new CTestOutputReader(fi, app, workingDirectory());
}
} // namespace Internal
} // namespace Autotest

View File

@@ -0,0 +1,43 @@
/****************************************************************************
**
** Copyright (C) 2020 The Qt Company Ltd.
** Contact: https://www.qt.io/licensing/
**
** This file is part of Qt Creator.
**
** Commercial License Usage
** Licensees holding valid commercial Qt licenses may use this file in
** accordance with the commercial license agreement provided with the
** Software or, alternatively, in accordance with the terms contained in
** a written agreement between you and The Qt Company. For licensing terms
** and conditions see https://www.qt.io/terms-conditions. For further
** information use the contact form at https://www.qt.io/contact-us.
**
** GNU General Public License Usage
** Alternatively, this file may be used under the terms of the GNU
** General Public License version 3 as published by the Free Software
** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
** included in the packaging of this file. Please review the following
** information to ensure the GNU General Public License requirements will
** be met: https://www.gnu.org/licenses/gpl-3.0.html.
**
****************************************************************************/
#pragma once
#include "../testconfiguration.h"
namespace Autotest {
namespace Internal {
class CTestConfiguration final : public Autotest::TestToolConfiguration
{
public:
explicit CTestConfiguration(ITestBase *testBase);
TestOutputReader *outputReader(const QFutureInterface<TestResultPtr> &fi,
QProcess *app) const final;
};
} // namespace Internal
} // namespace Autotest

View File

@@ -0,0 +1,156 @@
/****************************************************************************
**
** Copyright (C) 2020 The Qt Company Ltd.
**
** This file is part of Qt Creator.
**
** Commercial License Usage
** Licensees holding valid commercial Qt licenses may use this file in
** accordance with the commercial license agreement provided with the
** Software or, alternatively, in accordance with the terms contained in
** a written agreement between you and The Qt Company. For licensing terms
** and conditions see https://www.qt.io/terms-conditions. For further
** information use the contact form at https://www.qt.io/contact-us.
**
** GNU General Public License Usage
** Alternatively, this file may be used under the terms of the GNU
** General Public License version 3 as published by the Free Software
** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
** included in the packaging of this file. Please review the following
** information to ensure the GNU General Public License requirements will
** be met: https://www.gnu.org/licenses/gpl-3.0.html.
**
****************************************************************************/
#include "ctestoutputreader.h"
#include "ctesttreeitem.h"
#include "../testframeworkmanager.h"
#include "../testresult.h"
#include <cmakeprojectmanager/cmakeprojectconstants.h>
#include <utils/qtcassert.h>
#include <QRegularExpression>
namespace Autotest {
namespace Internal {
class CTestResult : public TestResult
{
public:
CTestResult(const QString &id, const QString &project, const QString &testCase)
: TestResult(id, project)
, m_testCase(testCase)
{}
bool isDirectParentOf(const TestResult *other, bool *needsIntermediate) const override
{
if (!TestResult::isDirectParentOf(other, needsIntermediate))
return false;
return result() == ResultType::TestStart;
}
const ITestTreeItem *findTestTreeItem() const override
{
ITestTool *testTool = TestFrameworkManager::testToolForBuildSystemId(
CMakeProjectManager::Constants::CMAKE_PROJECT_ID);
QTC_ASSERT(testTool, return nullptr);
const ITestTreeItem *rootNode = testTool->rootNode();
if (!rootNode)
return nullptr;
return rootNode->findFirstLevelChild([this](const ITestTreeItem *item) {
return item && item->name() == m_testCase;
});
}
private:
QString m_testCase;
};
CTestOutputReader::CTestOutputReader(const QFutureInterface<TestResultPtr> &futureInterface,
QProcess *testApplication, const QString &buildDirectory)
: TestOutputReader(futureInterface, testApplication, buildDirectory)
{
}
void CTestOutputReader::processOutputLine(const QByteArray &outputLine)
{
static const QRegularExpression testProject("^Test project (.*)$");
static const QRegularExpression testCase("^(test \\d+)|( Start\\s+\\d+: .*)$");
static const QRegularExpression testResult("^\\s*\\d+/\\d+ Test\\s+#\\d+: (.*) (\\.+)\\s*"
"(Passed|\\*\\*\\*Failed)\\s+(.*) sec$");
static const QRegularExpression summary("^\\d+% tests passed, (\\d+) tests failed "
"out of (\\d+)");
static const QRegularExpression summaryTime("^Total Test time .* =\\s+(.*) sec$");
const QString line = removeCommandlineColors(QString::fromLatin1(outputLine));
if (line.trimmed().isEmpty())
return;
struct ExactMatch : public QRegularExpressionMatch
{
ExactMatch(const QRegularExpressionMatch &other) : QRegularExpressionMatch(other) {}
operator bool() const { return hasMatch(); }
};
if (ExactMatch match = testProject.match(line)) {
if (!m_testName.isEmpty()) // possible?
sendCompleteInformation();
m_project = match.captured(1);
TestResultPtr testResult = createDefaultResult();
testResult->setResult(ResultType::TestStart);
testResult->setDescription(tr("Running tests for %1").arg(m_project));
reportResult(testResult);
} else if (ExactMatch match = testCase.match(line)) {
if (!m_testName.isEmpty())
sendCompleteInformation();
} else if (ExactMatch match = testResult.match(line)) {
m_description = match.captured();
m_testName = match.captured(1);
m_result = (match.captured(3) == "Passed") ? ResultType::Pass : ResultType::Fail;
} else if (ExactMatch match = summary.match(line)) {
if (!m_testName.isEmpty())
sendCompleteInformation();
TestResultPtr testResult = createDefaultResult();
testResult->setResult(ResultType::MessageInfo);
testResult->setDescription(match.captured());
reportResult(testResult);
int failed = match.captured(1).toInt();
int testCount = match.captured(2).toInt();
m_summary.insert(ResultType::Fail, failed);
m_summary.insert(ResultType::Pass, testCount - failed);
} else if (ExactMatch match = summaryTime.match(line)) {
if (!m_testName.isEmpty()) // possible?
sendCompleteInformation();
TestResultPtr testResult = createDefaultResult();
testResult->setResult(ResultType::TestEnd);
testResult->setDescription(match.captured());
reportResult(testResult);
} else {
if (!m_description.isEmpty())
m_description.append('\n');
m_description.append(line);
}
}
TestResultPtr CTestOutputReader::createDefaultResult() const
{
return TestResultPtr(new CTestResult(id(), m_project, m_testName));
}
void CTestOutputReader::sendCompleteInformation()
{
TestResultPtr testResult = createDefaultResult();
testResult->setResult(m_result);
testResult->setDescription(m_description);
reportResult(testResult);
m_testName.clear();
m_description.clear();
m_result = ResultType::Invalid;
}
} // namespace Internal
} // namespace Autotest

View File

@@ -0,0 +1,52 @@
/****************************************************************************
**
** Copyright (C) 2020 The Qt Company Ltd.
**
** This file is part of Qt Creator.
**
** Commercial License Usage
** Licensees holding valid commercial Qt licenses may use this file in
** accordance with the commercial license agreement provided with the
** Software or, alternatively, in accordance with the terms contained in
** a written agreement between you and The Qt Company. For licensing terms
** and conditions see https://www.qt.io/terms-conditions. For further
** information use the contact form at https://www.qt.io/contact-us.
**
** GNU General Public License Usage
** Alternatively, this file may be used under the terms of the GNU
** General Public License version 3 as published by the Free Software
** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
** included in the packaging of this file. Please review the following
** information to ensure the GNU General Public License requirements will
** be met: https://www.gnu.org/licenses/gpl-3.0.html.
**
****************************************************************************/
#pragma once
#include "../testoutputreader.h"
#include <QCoreApplication>
namespace Autotest {
namespace Internal {
class CTestOutputReader final : public Autotest::TestOutputReader
{
Q_DECLARE_TR_FUNCTIONS(Autotest::Internal::CTestOutputReader)
public:
CTestOutputReader(const QFutureInterface<TestResultPtr> &futureInterface,
QProcess *testApplication, const QString &buildDirectory);
protected:
void processOutputLine(const QByteArray &outputLineWithNewLine) final;
TestResultPtr createDefaultResult() const final;
void sendCompleteInformation();
QString m_project;
QString m_testName;
QString m_description;
ResultType m_result = ResultType::Invalid;
};
} // namespace Internal
} // namespace Autotest

View File

@@ -0,0 +1,62 @@
/****************************************************************************
**
** Copyright (C) 2020 The Qt Company Ltd.
** Contact: https://www.qt.io/licensing/
**
** This file is part of Qt Creator.
**
** Commercial License Usage
** Licensees holding valid commercial Qt licenses may use this file in
** accordance with the commercial license agreement provided with the
** Software or, alternatively, in accordance with the terms contained in
** a written agreement between you and The Qt Company. For licensing terms
** and conditions see https://www.qt.io/terms-conditions. For further
** information use the contact form at https://www.qt.io/contact-us.
**
** GNU General Public License Usage
** Alternatively, this file may be used under the terms of the GNU
** General Public License version 3 as published by the Free Software
** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
** included in the packaging of this file. Please review the following
** information to ensure the GNU General Public License requirements will
** be met: https://www.gnu.org/licenses/gpl-3.0.html.
**
****************************************************************************/
#include "ctesttool.h"
#include "ctesttreeitem.h"
#include <cmakeprojectmanager/cmakeprojectconstants.h>
#include <projectexplorer/buildsystem.h>
namespace Autotest {
namespace Internal {
Utils::Id CTestTool::buildSystemId() const
{
return Utils::Id(CMakeProjectManager::Constants::CMAKE_PROJECT_ID);
}
ITestTreeItem *CTestTool::createItemFromTestCaseInfo(const ProjectExplorer::TestCaseInfo &tci)
{
CTestTreeItem *item = new CTestTreeItem(this, tci.name, tci.path.toString(),
TestTreeItem::TestCase);
item->setLine(tci.line);
return item;
}
const char *CTestTool::name() const
{
return "CTest";
}
ITestTreeItem *CTestTool::createRootNode()
{
return new CTestTreeItem(this,
QCoreApplication::translate("CTestTool", "CTest"),
QString(), ITestTreeItem::Root);
}
} // namespace Internal
} // namespace Autotest

View File

@@ -0,0 +1,48 @@
/****************************************************************************
**
** Copyright (C) 2020 The Qt Company Ltd.
** Contact: https://www.qt.io/licensing/
**
** This file is part of Qt Creator.
**
** Commercial License Usage
** Licensees holding valid commercial Qt licenses may use this file in
** accordance with the commercial license agreement provided with the
** Software or, alternatively, in accordance with the terms contained in
** a written agreement between you and The Qt Company. For licensing terms
** and conditions see https://www.qt.io/terms-conditions. For further
** information use the contact form at https://www.qt.io/contact-us.
**
** GNU General Public License Usage
** Alternatively, this file may be used under the terms of the GNU
** General Public License version 3 as published by the Free Software
** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
** included in the packaging of this file. Please review the following
** information to ensure the GNU General Public License requirements will
** be met: https://www.gnu.org/licenses/gpl-3.0.html.
**
****************************************************************************/
#pragma once
#include "../itestframework.h"
namespace Autotest {
namespace Internal {
class CTestTool final : public Autotest::ITestTool
{
public:
CTestTool() : Autotest::ITestTool(false) {}
Utils::Id buildSystemId() const final;
ITestTreeItem *createItemFromTestCaseInfo(const ProjectExplorer::TestCaseInfo &tci) final;
protected:
const char *name() const final;
ITestTreeItem *createRootNode() final;
};
} // namespace Internal
} // namespace Autotest

View File

@@ -0,0 +1,132 @@
/****************************************************************************
**
** Copyright (C) 2020 The Qt Company Ltd.
** Contact: https://www.qt.io/licensing/
**
** This file is part of Qt Creator.
**
** Commercial License Usage
** Licensees holding valid commercial Qt licenses may use this file in
** accordance with the commercial license agreement provided with the
** Software or, alternatively, in accordance with the terms contained in
** a written agreement between you and The Qt Company. For licensing terms
** and conditions see https://www.qt.io/terms-conditions. For further
** information use the contact form at https://www.qt.io/contact-us.
**
** GNU General Public License Usage
** Alternatively, this file may be used under the terms of the GNU
** General Public License version 3 as published by the Free Software
** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
** included in the packaging of this file. Please review the following
** information to ensure the GNU General Public License requirements will
** be met: https://www.gnu.org/licenses/gpl-3.0.html.
**
****************************************************************************/
#include "ctesttreeitem.h"
#include "ctestconfiguration.h"
#include "../autotestplugin.h"
#include "../itestframework.h"
#include "../testsettings.h"
#include <projectexplorer/buildsystem.h>
#include <projectexplorer/project.h>
#include <projectexplorer/session.h>
#include <projectexplorer/target.h>
#include <utils/link.h>
#include <utils/qtcassert.h>
namespace Autotest {
namespace Internal {
CTestTreeItem::CTestTreeItem(ITestBase *testBase, const QString &name,
const QString &filepath, Type type)
: ITestTreeItem(testBase, name, filepath, type)
{
}
QList<ITestConfiguration *> CTestTreeItem::getAllTestConfigurations() const
{
return testConfigurationsFor({});
}
QList<ITestConfiguration *> CTestTreeItem::getSelectedTestConfigurations() const
{
QStringList selectedTests;
forFirstLevelChildren([&selectedTests](ITestTreeItem *it) {
if (it->checked())
selectedTests.append(it->name());
});
return selectedTests.isEmpty() ? QList<ITestConfiguration *>()
: testConfigurationsFor(selectedTests);
}
QList<ITestConfiguration *> CTestTreeItem::getFailedTestConfigurations() const
{
QStringList selectedTests;
forFirstLevelChildren([&selectedTests](ITestTreeItem *it) {
if (it->data(0, FailedRole).toBool())
selectedTests.append(it->name());
});
return selectedTests.isEmpty() ? QList<ITestConfiguration *>()
: testConfigurationsFor(selectedTests);
}
ITestConfiguration *CTestTreeItem::testConfiguration() const
{
return testConfigurationsFor({name()}).value(0);
}
QVariant CTestTreeItem::data(int column, int role) const
{
if (role == Qt::CheckStateRole)
return checked();
if (role == LinkRole) {
QVariant itemLink;
itemLink.setValue(Utils::Link(filePath(), line()));
return itemLink;
}
return ITestTreeItem::data(column, role);
}
QList<ITestConfiguration *> CTestTreeItem::testConfigurationsFor(const QStringList &selected) const
{
ProjectExplorer::Project *project = ProjectExplorer::SessionManager::startupProject();
if (!project)
return {};
const ProjectExplorer::Target *target = project->targets().value(0);
if (!target)
return {};
const ProjectExplorer::BuildSystem *buildSystem = target->buildSystem();
QStringList options{"--timeout", QString::number(AutotestPlugin::settings()->timeout / 1000)};
// TODO add ctest options from settings?
Utils::CommandLine command = buildSystem->commandLineForTests(selected, options);
if (command.executable().isEmpty())
return {};
CTestConfiguration *config = new CTestConfiguration(testBase());
config->setProject(project);
config->setCommandLine(command);
const QList<ProjectExplorer::BuildConfiguration *> buildConfigs = target->buildConfigurations();
if (QTC_GUARD(!buildConfigs.isEmpty())) {
config->setEnvironment(buildConfigs.first()->environment());
config->setWorkingDirectory(buildConfigs.first()->buildDirectory().toString());
} else {
config->setEnvironment(Utils::Environment::systemEnvironment());
}
if (selected.isEmpty())
config->setTestCaseCount(testBase()->asTestTool()->rootNode()->childCount());
else
config->setTestCaseCount(selected.size());
return {config};
}
} // namespace Internal
} // namespace Autotest

View File

@@ -0,0 +1,50 @@
/****************************************************************************
**
** Copyright (C) 2020 The Qt Company Ltd.
** Contact: https://www.qt.io/licensing/
**
** This file is part of Qt Creator.
**
** Commercial License Usage
** Licensees holding valid commercial Qt licenses may use this file in
** accordance with the commercial license agreement provided with the
** Software or, alternatively, in accordance with the terms contained in
** a written agreement between you and The Qt Company. For licensing terms
** and conditions see https://www.qt.io/terms-conditions. For further
** information use the contact form at https://www.qt.io/contact-us.
**
** GNU General Public License Usage
** Alternatively, this file may be used under the terms of the GNU
** General Public License version 3 as published by the Free Software
** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
** included in the packaging of this file. Please review the following
** information to ensure the GNU General Public License requirements will
** be met: https://www.gnu.org/licenses/gpl-3.0.html.
**
****************************************************************************/
#pragma once
#include "../testtreeitem.h"
namespace Autotest {
namespace Internal {
class CTestTreeItem final : public Autotest::ITestTreeItem
{
public:
CTestTreeItem(ITestBase *testBase, const QString &name, const QString &filepath, Type type);
QList<ITestConfiguration *> getAllTestConfigurations() const final;
QList<ITestConfiguration *> getSelectedTestConfigurations() const final;
QList<ITestConfiguration *> getFailedTestConfigurations() const final;
ITestConfiguration *testConfiguration() const final;
bool canProvideTestConfiguration() const final { return true; }
QVariant data(int column, int role) const final;
private:
QList<ITestConfiguration *> testConfigurationsFor(const QStringList &selected) const;
};
} // namespace Internal
} // namespace Autotest