AutoTest: Add gtest filter mode

This adds another grouping mode to the gtest framework based on
gtest filtering. You can now specify a filter that will be used
to group the gtest tree items into matching and non-matching
tests.

Change-Id: Iaf0e55c9e57e2720f4fa84ab4b51ecaeb614df88
Reviewed-by: David Schulz <david.schulz@qt.io>
This commit is contained in:
Christian Stenger
2018-02-28 09:42:13 +01:00
parent 070b5fdbbb
commit aee959ea1d
15 changed files with 343 additions and 23 deletions

View File

@@ -25,6 +25,7 @@
#include "gtest_utils.h"
#include <QRegularExpression>
#include <QStringList>
namespace Autotest {
@@ -51,6 +52,17 @@ bool isGTestTyped(const QString &macro)
return macro == QStringLiteral("TYPED_TEST") || macro == QStringLiteral("TYPED_TEST_P");
}
bool isValidGTestFilter(const QString &filterExpression)
{
// this still is not a 100% validation - but a compromise
// - numbers after '.' should get prohibited
// - more than one '.' inside a single filter should be prohibited
static const QRegularExpression regex("^(:*([_a-zA-Z*.?][_a-zA-Z0-9*.?]*:*)*)?"
"(-(:*([_a-zA-Z*.?][_a-zA-Z0-9*.?]*:*)*)?)?$");
return regex.match(filterExpression).hasMatch();
}
} // namespace GTestUtils
} // namespace Internal
} // namespace Autotest

View File

@@ -34,6 +34,7 @@ namespace GTestUtils {
bool isGTestMacro(const QString &macro);
bool isGTestParameterized(const QString &macro);
bool isGTestTyped(const QString &macro);
bool isValidGTestFilter(const QString &filterExpression);
} // namespace GTestUtils
} // namespace Internal

View File

@@ -35,6 +35,13 @@ const char FRAMEWORK_NAME[] = "GTest";
const char FRAMEWORK_SETTINGS_CATEGORY[] = QT_TRANSLATE_NOOP("GTestFramework", "Google Test");
const unsigned FRAMEWORK_PRIORITY = 10;
enum GroupMode
{
None,
Directory,
GTestFilter
};
} // namespace Constants
} // namespace GTest
} // namespace AutoTest

View File

@@ -24,11 +24,11 @@
****************************************************************************/
#include "gtestframework.h"
#include "gtestconstants.h"
#include "gtestsettings.h"
#include "gtestsettingspage.h"
#include "gtesttreeitem.h"
#include "gtestparser.h"
#include "../testframeworkmanager.h"
namespace Autotest {
namespace Internal {
@@ -71,5 +71,27 @@ bool GTestFramework::hasFrameworkSettings() const
return true;
}
QString GTestFramework::currentGTestFilter()
{
static const Core::Id id
= Core::Id(Constants::FRAMEWORK_PREFIX).withSuffix(GTest::Constants::FRAMEWORK_NAME);
const auto manager = TestFrameworkManager::instance();
auto gSettings = qSharedPointerCast<GTestSettings>(manager->settingsForTestFramework(id));
return gSettings.isNull() ? QString("*.*") : gSettings->gtestFilter;
}
GTest::Constants::GroupMode GTestFramework::groupMode()
{
static const Core::Id id
= Core::Id(Constants::FRAMEWORK_PREFIX).withSuffix(GTest::Constants::FRAMEWORK_NAME);
const auto manager = TestFrameworkManager::instance();
if (!manager->groupingEnabled(id))
return GTest::Constants::None;
auto gSettings = qSharedPointerCast<GTestSettings>(manager->settingsForTestFramework(id));
return gSettings.isNull() ? GTest::Constants::Directory : gSettings->groupMode;
}
} // namespace Internal
} // namespace Autotest

View File

@@ -26,6 +26,7 @@
#pragma once
#include "../itestframework.h"
#include "gtestconstants.h"
namespace Autotest {
namespace Internal {
@@ -39,6 +40,8 @@ public:
IFrameworkSettings *createFrameworkSettings() const override;
ITestSettingsPage *createSettingsPage(QSharedPointer<IFrameworkSettings> settings) const override;
bool hasFrameworkSettings() const override;
static GTest::Constants::GroupMode groupMode();
static QString currentGTestFilter();
protected:
ITestParser *createTestParser() const override;
TestTreeItem *createRootNode() const override;

View File

@@ -24,6 +24,7 @@
****************************************************************************/
#include "gtestsettings.h"
#include "gtest_utils.h"
namespace Autotest {
namespace Internal {
@@ -35,6 +36,8 @@ static const char runDisabledKey[] = "RunDisabled";
static const char seedKey[] = "Seed";
static const char shuffleKey[] = "Shuffle";
static const char throwOnFailureKey[] = "ThrowOnFailure";
static const char groupModeKey[] = "GroupMode";
static const char gtestFilterKey[] = "GTestFilter";
QString GTestSettings::name() const
{
@@ -50,6 +53,13 @@ void GTestSettings::fromFrameworkSettings(const QSettings *s)
seed = s->value(seedKey, 0).toInt();
breakOnFailure = s->value(breakOnFailureKey, true).toBool();
throwOnFailure = s->value(throwOnFailureKey, false).toBool();
// avoid problems if user messes around with the settings file
bool ok = false;
const int tmp = s->value(groupModeKey, GTest::Constants::Directory).toInt(&ok);
groupMode = ok ? static_cast<GTest::Constants::GroupMode>(tmp) : GTest::Constants::Directory;
gtestFilter = s->value(gtestFilterKey, "*.*").toString();
if (!GTestUtils::isValidGTestFilter(gtestFilter))
gtestFilter = "*.*";
}
void GTestSettings::toFrameworkSettings(QSettings *s) const
@@ -61,6 +71,8 @@ void GTestSettings::toFrameworkSettings(QSettings *s) const
s->setValue(seedKey, seed);
s->setValue(breakOnFailureKey, breakOnFailure);
s->setValue(throwOnFailureKey, throwOnFailure);
s->setValue(groupModeKey, groupMode);
s->setValue(gtestFilterKey, gtestFilter);
}
} // namespace Internal

View File

@@ -26,6 +26,7 @@
#pragma once
#include "../iframeworksettings.h"
#include "gtestconstants.h"
namespace Autotest {
namespace Internal {
@@ -43,6 +44,8 @@ public:
bool repeat = false;
bool throwOnFailure = false;
bool breakOnFailure = true;
GTest::Constants::GroupMode groupMode = GTest::Constants::Directory;
QString gtestFilter{"*.*"};
protected:
void fromFrameworkSettings(const QSettings *s) override;

View File

@@ -23,20 +23,34 @@
**
****************************************************************************/
#include "../autotestconstants.h"
#include "gtestconstants.h"
#include "gtestsettingspage.h"
#include "gtestsettings.h"
#include "gtest_utils.h"
#include "../autotestconstants.h"
#include "../testframeworkmanager.h"
#include <coreplugin/icore.h>
namespace Autotest {
namespace Internal {
static bool validateFilter(Utils::FancyLineEdit *edit, QString */*error*/)
{
return edit && GTestUtils::isValidGTestFilter(edit->text());
}
GTestSettingsWidget::GTestSettingsWidget(QWidget *parent)
: QWidget(parent)
{
m_ui.setupUi(this);
m_ui.filterLineEdit->setValidationFunction(&validateFilter);
m_ui.filterLineEdit->setEnabled(m_ui.groupModeCombo->currentIndex() == 1);
connect(m_ui.groupModeCombo, &QComboBox::currentTextChanged,
this, [this] () {
m_ui.filterLineEdit->setEnabled(m_ui.groupModeCombo->currentIndex() == 1);
});
connect(m_ui.repeatGTestsCB, &QCheckBox::toggled, m_ui.repetitionSpin, &QSpinBox::setEnabled);
connect(m_ui.shuffleGTestsCB, &QCheckBox::toggled, m_ui.seedSpin, &QSpinBox::setEnabled);
}
@@ -50,6 +64,9 @@ void GTestSettingsWidget::setSettings(const GTestSettings &settings)
m_ui.seedSpin->setValue(settings.seed);
m_ui.breakOnFailureCB->setChecked(settings.breakOnFailure);
m_ui.throwOnFailureCB->setChecked(settings.throwOnFailure);
m_ui.groupModeCombo->setCurrentIndex(settings.groupMode - 1); // there's None for internal use
m_ui.filterLineEdit->setText(settings.gtestFilter);
m_currentGTestFilter = settings.gtestFilter; // store it temporarily (if edit is invalid)
}
GTestSettings GTestSettingsWidget::settings() const
@@ -62,6 +79,12 @@ GTestSettings GTestSettingsWidget::settings() const
result.seed = m_ui.seedSpin->value();
result.breakOnFailure = m_ui.breakOnFailureCB->isChecked();
result.throwOnFailure = m_ui.throwOnFailureCB->isChecked();
result.groupMode = static_cast<GTest::Constants::GroupMode>(
m_ui.groupModeCombo->currentIndex() + 1);
if (m_ui.filterLineEdit->isValid())
result.gtestFilter = m_ui.filterLineEdit->text();
else
result.gtestFilter = m_currentGTestFilter;
return result;
}
@@ -88,8 +111,16 @@ void GTestSettingsPage::apply()
{
if (!m_widget) // page was not shown at all
return;
GTest::Constants::GroupMode oldGroupMode = m_settings->groupMode;
const QString oldFilter = m_settings->gtestFilter;
*m_settings = m_widget->settings();
m_settings->toSettings(Core::ICore::settings());
if (m_settings->groupMode == oldGroupMode && oldFilter == m_settings->gtestFilter)
return;
auto id = Core::Id(Constants::FRAMEWORK_PREFIX).withSuffix(GTest::Constants::FRAMEWORK_NAME);
TestTreeModel::instance()->rebuild({id});
}
} // namespace Internal

View File

@@ -48,6 +48,7 @@ public:
private:
Ui::GTestSettingsPage m_ui;
QString m_currentGTestFilter;
};
class GTestSettingsPage : public ITestSettingsPage

View File

@@ -7,7 +7,7 @@
<x>0</x>
<y>0</y>
<width>449</width>
<height>210</height>
<height>232</height>
</rect>
</property>
<property name="windowTitle">
@@ -128,6 +128,65 @@
</item>
</layout>
</item>
<item>
<layout class="QGridLayout" name="gridLayout_2">
<item row="0" column="0">
<widget class="QLabel" name="label_3">
<property name="text">
<string>Group mode:</string>
</property>
</widget>
</item>
<item row="1" column="0">
<widget class="QLabel" name="label_4">
<property name="text">
<string>Active filter:</string>
</property>
</widget>
</item>
<item row="0" column="2">
<widget class="QComboBox" name="groupModeCombo">
<property name="toolTip">
<string>Select on what grouping the tests should be based.</string>
</property>
<item>
<property name="text">
<string>Directory</string>
</property>
</item>
<item>
<property name="text">
<string>GTest Filter</string>
</property>
</item>
</widget>
</item>
<item row="1" column="2">
<widget class="Utils::FancyLineEdit" name="filterLineEdit">
<property name="toolTip">
<string>Set the GTest filter to be used for grouping.
See Google Test documentation for further information on GTest filters.</string>
</property>
</widget>
</item>
<item row="1" column="1">
<spacer name="horizontalSpacer_2">
<property name="orientation">
<enum>Qt::Horizontal</enum>
</property>
<property name="sizeType">
<enum>QSizePolicy::Fixed</enum>
</property>
<property name="sizeHint" stdset="0">
<size>
<width>60</width>
<height>20</height>
</size>
</property>
</spacer>
</item>
</layout>
</item>
<item>
<spacer name="verticalSpacer">
<property name="orientation">
@@ -158,6 +217,13 @@
</item>
</layout>
</widget>
<customwidgets>
<customwidget>
<class>Utils::FancyLineEdit</class>
<extends>QLineEdit</extends>
<header location="global">utils/fancylineedit.h</header>
</customwidget>
</customwidgets>
<resources/>
<connections/>
</ui>

View File

@@ -25,6 +25,8 @@
#include "gtesttreeitem.h"
#include "gtestconfiguration.h"
#include "gtestconstants.h"
#include "gtestframework.h"
#include "gtestparser.h"
#include "../testframeworkmanager.h"
@@ -32,10 +34,23 @@
#include <projectexplorer/session.h>
#include <utils/algorithm.h>
#include <utils/qtcassert.h>
#include <utils/utilsicons.h>
#include <QRegExp>
namespace Autotest {
namespace Internal {
static QString matchingString()
{
return QCoreApplication::translate("GTestTreeItem", "<matching>");
}
static QString notMatchingString()
{
return QCoreApplication::translate("GTestTreeItem", "<not matching>");
}
static QString gtestFilter(GTestTreeItem::TestStates states)
{
if ((states & GTestTreeItem::Parameterized) && (states & GTestTreeItem::Typed))
@@ -55,6 +70,35 @@ TestTreeItem *GTestTreeItem::copyWithoutChildren()
return copied;
}
static bool matchesFilter(const QString &filter, const QString &fullTestName)
{
QStringList positive;
QStringList negative;
int startOfNegative = filter.indexOf('-');
if (startOfNegative == -1) {
positive.append(filter.split(':', QString::SkipEmptyParts));
} else {
positive.append(filter.left(startOfNegative).split(':', QString::SkipEmptyParts));
negative.append(filter.mid(startOfNegative + 1).split(':', QString::SkipEmptyParts));
}
QString testName = fullTestName;
if (!testName.contains('.'))
testName.append('.');
for (const QString &curr : negative) {
QRegExp regex(curr, Qt::CaseSensitive, QRegExp::Wildcard);
if (regex.exactMatch(testName))
return false;
}
for (const QString &curr : positive) {
QRegExp regex(curr, Qt::CaseSensitive, QRegExp::Wildcard);
if (regex.exactMatch(testName))
return true;
}
return positive.isEmpty();
}
QVariant GTestTreeItem::data(int column, int role) const
{
switch (role) {
@@ -65,6 +109,20 @@ QVariant GTestTreeItem::data(int column, int role) const
const QString &displayName = (m_state & Disabled) ? name().mid(9) : name();
return QVariant(displayName + nameSuffix());
}
case Qt::DecorationRole:
if (type() == GroupNode
&& GTestFramework::groupMode() == GTest::Constants::GTestFilter) {
return Utils::Icons::FILTER.icon(); // TODO replace by an 'inked' filter w/o arrow
}
break;
case Qt::ToolTipRole:
if (type() == GroupNode
&& GTestFramework::groupMode() == GTest::Constants::GTestFilter) {
const auto tpl = QString("<p>%1</p><p>%2</p>").arg(filePath());
return tpl.arg(QCoreApplication::translate(
"GTestTreeItem", "Change GTest filter in use inside the settings."));
}
break;
case Qt::CheckStateRole:
switch (type()) {
case Root:
@@ -225,18 +283,35 @@ TestTreeItem *GTestTreeItem::find(const TestParseResult *result)
switch (type()) {
case Root:
if (TestFrameworkManager::instance()->groupingEnabled(result->frameworkId)) {
const QFileInfo fileInfo(parseResult->fileName);
const QFileInfo base(fileInfo.absolutePath());
for (int row = 0; row < childCount(); ++row) {
GTestTreeItem *group = static_cast<GTestTreeItem *>(childAt(row));
if (group->filePath() != base.absoluteFilePath())
continue;
if (auto groupChild = group->findChildByNameStateAndFile(parseResult->name, states,
parseResult->proFile)) {
return groupChild;
if (GTestFramework::groupMode() == GTest::Constants::Directory) {
const QFileInfo fileInfo(parseResult->fileName);
const QFileInfo base(fileInfo.absolutePath());
for (int row = 0; row < childCount(); ++row) {
GTestTreeItem *group = static_cast<GTestTreeItem *>(childAt(row));
if (group->filePath() != base.absoluteFilePath())
continue;
if (auto groupChild = group->findChildByNameStateAndFile(
parseResult->name, states,parseResult->proFile)) {
return groupChild;
}
}
return nullptr;
} else { // GTestFilter
QTC_ASSERT(parseResult->children.size(), return nullptr);
auto fstChild = static_cast<const GTestParseResult *>(parseResult->children.at(0));
bool matching = matchesFilter(GTestFramework::currentGTestFilter(),
parseResult->name + '.' + fstChild->name);
for (int row = 0; row < childCount(); ++row) {
GTestTreeItem *group = static_cast<GTestTreeItem *>(childAt(row));
if ((matching && group->name() == matchingString())
|| (!matching && group->name() == notMatchingString())) {
if (auto groupChild = group->findChildByNameStateAndFile(
parseResult->name, states, parseResult->proFile))
return groupChild;
}
}
return nullptr;
}
return nullptr;
}
return findChildByNameStateAndFile(parseResult->name, states, parseResult->proFile);
case GroupNode:
@@ -264,9 +339,22 @@ TestTreeItem *GTestTreeItem::createParentGroupNode() const
{
if (type() != TestCase)
return nullptr;
const QFileInfo fileInfo(filePath());
const QFileInfo base(fileInfo.absolutePath());
return new GTestTreeItem(base.baseName(), fileInfo.absolutePath(), TestTreeItem::GroupNode);
if (GTestFramework::groupMode() == GTest::Constants::Directory) {
const QFileInfo fileInfo(filePath());
const QFileInfo base(fileInfo.absolutePath());
return new GTestTreeItem(base.baseName(), fileInfo.absolutePath(), TestTreeItem::GroupNode);
} else { // GTestFilter
QTC_ASSERT(childCount(), return nullptr); // paranoia
const TestTreeItem *firstChild = childItem(0);
const QString activeFilter = GTestFramework::currentGTestFilter();
const QString fullTestName = name() + '.' + firstChild->name();
const QString groupNodeName =
matchesFilter(activeFilter, fullTestName) ? matchingString() : notMatchingString();
auto groupNode = new GTestTreeItem(groupNodeName, activeFilter, TestTreeItem::GroupNode);
if (groupNodeName == notMatchingString())
groupNode->setData(0, Qt::Unchecked, Qt::CheckStateRole);
return groupNode;
}
}
bool GTestTreeItem::modifyTestSetContent(const GTestParseResult *result)
@@ -329,5 +417,61 @@ QSet<QString> GTestTreeItem::internalTargets() const
return result;
}
bool GTestTreeItem::isGroupNodeFor(const TestTreeItem *other) const
{
QTC_ASSERT(other, return false);
if (type() != TestTreeItem::GroupNode)
return false;
if (GTestFramework::groupMode() == GTest::Constants::Directory) {
return QFileInfo(other->filePath()).absolutePath() == filePath();
} else { // GTestFilter
QString fullName;
if (other->type() == TestCase) {
fullName = other->name();
if (other->childCount())
fullName += '.' + other->childItem(0)->name();
} else if (other->type() == TestFunctionOrSet) {
QTC_ASSERT(other->parentItem(), return false);
fullName = other->parentItem()->name() + '.' + other->name();
} else if (other->type() == GroupNode) { // can happen on a rebuild if only filter changes
return false;
} else {
QTC_ASSERT(false, return false);
}
if (GTestFramework::currentGTestFilter() != filePath()) // filter has changed in settings
return false;
bool matches = matchesFilter(filePath(), fullName);
return (matches && name() == matchingString())
|| (!matches && name() == notMatchingString());
}
}
TestTreeItem *GTestTreeItem::applyFilters()
{
if (type() != TestCase)
return nullptr;
if (GTestFramework::groupMode() != GTest::Constants::GTestFilter)
return nullptr;
const QString gtestFilter = GTestFramework::currentGTestFilter();
TestTreeItem *filtered = nullptr;
for (int row = childCount() - 1; row >= 0; --row) {
GTestTreeItem *child = static_cast<GTestTreeItem *>(childItem(row));
if (!matchesFilter(gtestFilter, name() + '.' + child->name())) {
if (!filtered) {
filtered = copyWithoutChildren();
filtered->setData(0, Qt::Unchecked, Qt::CheckStateRole);
}
auto childCopy = child->copyWithoutChildren();
childCopy->setData(0, Qt::Unchecked, Qt::CheckStateRole);
filtered->appendChild(childCopy);
removeChildAt(row);
}
}
return filtered;
}
} // namespace Internal
} // namespace Autotest

View File

@@ -69,7 +69,8 @@ public:
const QString &proFile) const;
QString nameSuffix() const;
QSet<QString> internalTargets() const override;
bool isGroupNodeFor(const TestTreeItem *other) const override;
TestTreeItem *applyFilters() override;
private:
bool modifyTestSetContent(const GTestParseResult *result);
QList<TestConfiguration *> getTestConfigurations(bool ignoreCheckState) const;

View File

@@ -118,6 +118,9 @@ public:
virtual bool modify(const TestParseResult *result) = 0;
virtual bool isGroupNodeFor(const TestTreeItem *other) const;
virtual TestTreeItem *createParentGroupNode() const = 0;
// based on (internal) filters this will be used to filter out sub items (and remove them)
// returns a copy of the item that contains the filtered out children or nullptr
virtual TestTreeItem *applyFilters() { return nullptr; }
virtual QSet<QString> internalTargets() const;
protected:
void copyBasicDataFrom(const TestTreeItem *other);

View File

@@ -211,6 +211,17 @@ void TestTreeModel::syncTestFrameworks()
emit updatedActiveFrameworks(sortedIds.size());
}
void TestTreeModel::filterAndInsert(TestTreeItem *item, TestTreeItem *root, bool groupingEnabled)
{
TestTreeItem *filtered = item->applyFilters();
if (item->type() != TestTreeItem::TestCase || item->childCount())
insertItemInParent(item, root, groupingEnabled);
else // might be that all children have been filtered out
delete item;
if (filtered)
insertItemInParent(filtered, root, groupingEnabled);
}
void TestTreeModel::rebuild(const QList<Core::Id> &frameworkIds)
{
TestFrameworkManager *frameworkManager = TestFrameworkManager::instance();
@@ -219,18 +230,19 @@ void TestTreeModel::rebuild(const QList<Core::Id> &frameworkIds)
const bool groupingEnabled = TestFrameworkManager::instance()->groupingEnabled(id);
for (int row = frameworkRoot->childCount() - 1; row >= 0; --row) {
auto testItem = frameworkRoot->childItem(row);
if (!groupingEnabled && testItem->type() == TestTreeItem::GroupNode) {
// do not re-insert the GroupNode, but process its children and delete it afterwards
if (testItem->type() == TestTreeItem::GroupNode) {
// process children of group node and delete it afterwards if necessary
for (int childRow = testItem->childCount() - 1; childRow >= 0; --childRow) {
// FIXME should this be done recursively until we have a non-GroupNode?
TestTreeItem *childTestItem = testItem->childItem(childRow);
takeItem(childTestItem);
insertItemInParent(childTestItem, frameworkRoot, groupingEnabled);
filterAndInsert(childTestItem, frameworkRoot, groupingEnabled);
}
delete takeItem(testItem);
if (!groupingEnabled || testItem->childCount() == 0)
delete takeItem(testItem);
} else {
takeItem(testItem);
insertItemInParent(testItem, frameworkRoot, groupingEnabled);
filterAndInsert(testItem, frameworkRoot, groupingEnabled);
}
}
}
@@ -404,7 +416,8 @@ void TestTreeModel::handleParseResult(const TestParseResult *result, TestTreeIte
TestTreeItem *newItem = result->createTestTreeItem();
QTC_ASSERT(newItem, return);
insertItemInParent(newItem, parentNode, groupingEnabled);
// it might be necessary to "split" created item
filterAndInsert(newItem, parentNode, groupingEnabled);
}
void TestTreeModel::removeAllTestItems()

View File

@@ -91,6 +91,7 @@ private:
void revalidateCheckState(TestTreeItem *item);
explicit TestTreeModel(QObject *parent = 0);
void setupParsingConnections();
void filterAndInsert(TestTreeItem *item, TestTreeItem *root, bool groupingEnabled);
QList<TestTreeItem *> testItemsByName(TestTreeItem *root, const QString &testName);
TestCodeParser *m_parser;