From 5f22126a79016f765d752d1aa84d8c193b03791a Mon Sep 17 00:00:00 2001 From: Christian Stenger Date: Fri, 11 Sep 2020 14:30:15 +0200 Subject: [PATCH] AutoTest: Support gathering failed tests Mark test tree items as failed for the last run to be able to re-run them in an easier way. Change-Id: I7ea3dcd16e5a02797d41f13e02b2fa95b857cf5e Reviewed-by: David Schulz --- .../autotest/boost/boosttesttreeitem.cpp | 21 ++++++- .../autotest/boost/boosttesttreeitem.h | 3 + src/plugins/autotest/catch/catchtreeitem.cpp | 43 +++++++++++++ src/plugins/autotest/catch/catchtreeitem.h | 1 + src/plugins/autotest/gtest/gtesttreeitem.cpp | 44 +++++++++++++ src/plugins/autotest/gtest/gtesttreeitem.h | 1 + src/plugins/autotest/qtest/qttesttreeitem.cpp | 41 +++++++++++++ src/plugins/autotest/qtest/qttesttreeitem.h | 1 + .../autotest/quick/quicktesttreeitem.cpp | 61 +++++++++++++++++++ .../autotest/quick/quicktesttreeitem.h | 1 + src/plugins/autotest/testresultmodel.cpp | 19 ++++++ src/plugins/autotest/testrunner.cpp | 1 + src/plugins/autotest/testtreeitem.cpp | 10 +++ src/plugins/autotest/testtreeitem.h | 5 +- src/plugins/autotest/testtreemodel.cpp | 25 ++++++++ src/plugins/autotest/testtreemodel.h | 3 + 16 files changed, 276 insertions(+), 4 deletions(-) diff --git a/src/plugins/autotest/boost/boosttesttreeitem.cpp b/src/plugins/autotest/boost/boosttesttreeitem.cpp index faf50624941..67c798369ee 100644 --- a/src/plugins/autotest/boost/boosttesttreeitem.cpp +++ b/src/plugins/autotest/boost/boosttesttreeitem.cpp @@ -215,7 +215,8 @@ QList BoostTestTreeItem::getAllTestConfigurations() const return result; } -QList BoostTestTreeItem::getSelectedTestConfigurations() const +QList BoostTestTreeItem::getTestConfigurations( + std::function predicate) const { QList result; ProjectExplorer::Project *project = ProjectExplorer::SessionManager::startupProject(); @@ -228,13 +229,13 @@ QList BoostTestTreeItem::getSelectedTestConfigurations() co }; QHash testCasesForProjectFile; - forAllChildren([&testCasesForProjectFile](TreeItem *it){ + forAllChildren([&testCasesForProjectFile, &predicate](TreeItem *it){ auto item = static_cast(it); if (item->type() != TestCase) return; if (!item->enabled()) // ignore child tests known to be disabled when using run selected return; - if (item->checked() == Qt::Checked) { + if (predicate(item)) { QString tcName = item->name(); if (item->state().testFlag(BoostTestTreeItem::Templated)) tcName.append("<*"); @@ -261,6 +262,20 @@ QList BoostTestTreeItem::getSelectedTestConfigurations() co return result; } +QList BoostTestTreeItem::getSelectedTestConfigurations() const +{ + return getTestConfigurations([](BoostTestTreeItem *it) { + return it->checked() == Qt::Checked; + }); +} + +QList BoostTestTreeItem::getFailedTestConfigurations() const +{ + return getTestConfigurations([](BoostTestTreeItem *it) { + return it->data(0, FailedRole).toBool(); + }); +} + TestConfiguration *BoostTestTreeItem::testConfiguration() const { ProjectExplorer::Project *project = ProjectExplorer::SessionManager::startupProject(); diff --git a/src/plugins/autotest/boost/boosttesttreeitem.h b/src/plugins/autotest/boost/boosttesttreeitem.h index f9dc154a562..436525f2119 100644 --- a/src/plugins/autotest/boost/boosttesttreeitem.h +++ b/src/plugins/autotest/boost/boosttesttreeitem.h @@ -71,6 +71,7 @@ public: QList getAllTestConfigurations() const override; QList getSelectedTestConfigurations() const override; + QList getFailedTestConfigurations() const override; bool canProvideTestConfiguration() const override { return type() != Root; } bool canProvideDebugConfiguration() const override { return canProvideTestConfiguration(); } TestConfiguration *testConfiguration() const override; @@ -83,6 +84,8 @@ private: BoostTestTreeItem::TestStates state, const QString &proFile) const; QString prependWithParentsSuitePaths(const QString &testName) const; + QList getTestConfigurations( + std::function predicate) const; bool modifyTestContent(const BoostTestParseResult *result); TestStates m_state = Enabled; QString m_fullName; diff --git a/src/plugins/autotest/catch/catchtreeitem.cpp b/src/plugins/autotest/catch/catchtreeitem.cpp index e62cbe0da78..926c86c4ee3 100644 --- a/src/plugins/autotest/catch/catchtreeitem.cpp +++ b/src/plugins/autotest/catch/catchtreeitem.cpp @@ -221,6 +221,25 @@ static void collectTestInfo(const TestTreeItem *item, } } +static void collectFailedTestInfo(const CatchTreeItem *item, + QHash &testCasesForProfile) +{ + QTC_ASSERT(item, return); + QTC_ASSERT(item->type() == TestTreeItem::Root, return); + + item->forAllChildren([&testCasesForProfile](TestTreeItem *it) { + QTC_ASSERT(it, return); + CatchTreeItem *parent = static_cast(it->parentItem()); + QTC_ASSERT(parent, return); + if (it->type() == TestTreeItem::TestCase && it->data(0, FailedRole).toBool()) { + CatchTreeItem *current = static_cast(it); + testCasesForProfile[it->proFile()].names.append(current->testCasesString()); + testCasesForProfile[it->proFile()].internalTargets.unite( + it->internalTargets()); + } + }); +} + QList CatchTreeItem::getAllTestConfigurations() const { return getTestConfigurations(true); @@ -231,6 +250,30 @@ QList CatchTreeItem::getSelectedTestConfigurations() const return getTestConfigurations(false); } +QList CatchTreeItem::getFailedTestConfigurations() const +{ + QList result; + ProjectExplorer::Project *project = ProjectExplorer::SessionManager::startupProject(); + if (!project || type() != Root) + return result; + + QHash testCasesForProFile; + collectFailedTestInfo(this, testCasesForProFile); + + for (auto it = testCasesForProFile.begin(), end = testCasesForProFile.end(); it != end; ++it) { + for (const QString &target : qAsConst(it.value().internalTargets)) { + CatchConfiguration *tc = new CatchConfiguration(framework()); + tc->setTestCases(it.value().names); + tc->setProjectFile(it.key()); + tc->setProject(project); + tc->setInternalTarget(target); + result << tc; + } + } + + return result; +} + QList CatchTreeItem::getTestConfigurationsForFile(const Utils::FilePath &fileName) const { QList result; diff --git a/src/plugins/autotest/catch/catchtreeitem.h b/src/plugins/autotest/catch/catchtreeitem.h index 1e69d916181..05c5e834595 100644 --- a/src/plugins/autotest/catch/catchtreeitem.h +++ b/src/plugins/autotest/catch/catchtreeitem.h @@ -63,6 +63,7 @@ public: TestConfiguration *debugConfiguration() const override; QList getAllTestConfigurations() const override; QList getSelectedTestConfigurations() const override; + QList getFailedTestConfigurations() const override; QList getTestConfigurationsForFile(const Utils::FilePath &fileName) const override; private: diff --git a/src/plugins/autotest/gtest/gtesttreeitem.cpp b/src/plugins/autotest/gtest/gtesttreeitem.cpp index ec219793807..efa09c60f2f 100644 --- a/src/plugins/autotest/gtest/gtesttreeitem.cpp +++ b/src/plugins/autotest/gtest/gtesttreeitem.cpp @@ -248,6 +248,25 @@ static void collectTestInfo(const GTestTreeItem *item, } } +static void collectFailedTestInfo(const GTestTreeItem *item, + QHash &testCasesForProfile) +{ + QTC_ASSERT(item, return); + QTC_ASSERT(item->type() == TestTreeItem::Root, return); + + item->forAllChildren([&testCasesForProfile](TestTreeItem *it) { + QTC_ASSERT(it, return); + GTestTreeItem *parent = static_cast(it->parentItem()); + QTC_ASSERT(parent, return); + if (it->type() == TestTreeItem::TestCase && it->data(0, FailedRole).toBool()) { + testCasesForProfile[it->proFile()].filters.append( + gtestFilter(parent->state()).arg(parent->name()).arg(it->name())); + testCasesForProfile[it->proFile()].internalTargets.unite( + it->internalTargets()); + } + }); +} + QList GTestTreeItem::getTestConfigurations(bool ignoreCheckState) const { QList result; @@ -287,6 +306,31 @@ QList GTestTreeItem::getSelectedTestConfigurations() const return getTestConfigurations(false); } +QList GTestTreeItem::getFailedTestConfigurations() const +{ + QList result; + ProjectExplorer::Project *project = ProjectExplorer::SessionManager::startupProject(); + if (!project || type() != Root) + return result; + + QHash testCasesForProFile; + collectFailedTestInfo(this, testCasesForProFile); + + for (auto it = testCasesForProFile.begin(), end = testCasesForProFile.end(); it != end; ++it) { + for (const QString &target : qAsConst(it.value().internalTargets)) { + GTestConfiguration *tc = new GTestConfiguration(framework()); + tc->setTestCases(it.value().filters); + tc->setTestCaseCount(tc->testCaseCount() + it.value().testSetCount); + tc->setProjectFile(it.key()); + tc->setProject(project); + tc->setInternalTarget(target); + result << tc; + } + } + + return result; +} + QList GTestTreeItem::getTestConfigurationsForFile(const Utils::FilePath &fileName) const { QList result; diff --git a/src/plugins/autotest/gtest/gtesttreeitem.h b/src/plugins/autotest/gtest/gtesttreeitem.h index 3e5c1620a10..0464153ca7d 100644 --- a/src/plugins/autotest/gtest/gtesttreeitem.h +++ b/src/plugins/autotest/gtest/gtesttreeitem.h @@ -61,6 +61,7 @@ public: TestConfiguration *debugConfiguration() const override; QList getAllTestConfigurations() const override; QList getSelectedTestConfigurations() const override; + QList getFailedTestConfigurations() const override; QList getTestConfigurationsForFile(const Utils::FilePath &fileName) const override; TestTreeItem *find(const TestParseResult *result) override; TestTreeItem *findChild(const TestTreeItem *other) override; diff --git a/src/plugins/autotest/qtest/qttesttreeitem.cpp b/src/plugins/autotest/qtest/qttesttreeitem.cpp index a128a809416..e8964ffb878 100644 --- a/src/plugins/autotest/qtest/qttesttreeitem.cpp +++ b/src/plugins/autotest/qtest/qttesttreeitem.cpp @@ -190,6 +190,37 @@ static void fillTestConfigurationsFromCheckState(const TestTreeItem *item, } } +static void collectFailedTestInfo(TestTreeItem *item, QList &testConfigs) +{ + QTC_ASSERT(item, return); + if (item->type() == TestTreeItem::GroupNode) { + for (int row = 0, count = item->childCount(); row < count; ++row) + collectFailedTestInfo(item->childAt(row), testConfigs); + return; + } + QTC_ASSERT(item->type() == TestTreeItem::TestCase, return); + QStringList testCases; + item->forFirstLevelChildren([&testCases](TestTreeItem *func) { + if (func->data(0, FailedRole).toBool()) { + testCases << func->name(); + } else { + func->forFirstLevelChildren([&testCases, func](TestTreeItem *dataTag) { + if (dataTag->data(0, FailedRole).toBool()) + testCases << func->name() + ':' + dataTag->name(); + }); + } + }); + if (testCases.isEmpty()) + return; + + QtTestConfiguration *testConfig = new QtTestConfiguration(item->framework()); + testConfig->setTestCases(testCases); + testConfig->setProjectFile(item->proFile()); + testConfig->setProject(ProjectExplorer::SessionManager::startupProject()); + testConfig->setInternalTargets(item->internalTargets()); + testConfigs << testConfig; +} + TestConfiguration *QtTestTreeItem::debugConfiguration() const { QtTestConfiguration *config = static_cast(testConfiguration()); @@ -235,6 +266,16 @@ QList QtTestTreeItem::getSelectedTestConfigurations() const return result; } +QList QtTestTreeItem::getFailedTestConfigurations() const +{ + QList result; + QTC_ASSERT(type() == TestTreeItem::Root, return result); + forFirstLevelChildren([&result](TestTreeItem *child) { + collectFailedTestInfo(child, result); + }); + return result; +} + QList QtTestTreeItem::getTestConfigurationsForFile(const Utils::FilePath &fileName) const { QList result; diff --git a/src/plugins/autotest/qtest/qttesttreeitem.h b/src/plugins/autotest/qtest/qttesttreeitem.h index 9b803ee92eb..62c1235f712 100644 --- a/src/plugins/autotest/qtest/qttesttreeitem.h +++ b/src/plugins/autotest/qtest/qttesttreeitem.h @@ -45,6 +45,7 @@ public: TestConfiguration *debugConfiguration() const override; QList getAllTestConfigurations() const override; QList getSelectedTestConfigurations() const override; + QList getFailedTestConfigurations() const override; QList getTestConfigurationsForFile(const Utils::FilePath &fileName) const override; TestTreeItem *find(const TestParseResult *result) override; TestTreeItem *findChild(const TestTreeItem *other) override; diff --git a/src/plugins/autotest/quick/quicktesttreeitem.cpp b/src/plugins/autotest/quick/quicktesttreeitem.cpp index 5c6cf57aa93..650b8a8576e 100644 --- a/src/plugins/autotest/quick/quicktesttreeitem.cpp +++ b/src/plugins/autotest/quick/quicktesttreeitem.cpp @@ -195,6 +195,45 @@ static void testConfigurationFromCheckState(const TestTreeItem *item, } } +static void testConfigurationsForFailed(const TestTreeItem *item, + QHash &foundProfiles) +{ + QTC_ASSERT(item, return); + if (item->type() == TestTreeItem::GroupNode) { + for (int row = 0, count = item->childCount(); row < count; ++row) + testConfigurationsForFailed(item->childAt(row), foundProfiles); + return; + } + QTC_ASSERT(item->type() == TestTreeItem::TestCase, return); + + const QString testName = item->name(); + if (testName.isEmpty()) // skip unnamed quick tests as we cannot address them + return; + + QStringList testFunctions; + item->forFirstLevelChildren([&testFunctions, &testName](TestTreeItem *child) { + if (child->data(0, FailedRole).toBool()) + testFunctions << testName + "::" + child->name(); + }); + if (testFunctions.isEmpty()) + return; + + QuickTestConfiguration *tc = nullptr; + if (foundProfiles.contains(item->proFile())) { + tc = foundProfiles[item->proFile()]; + QStringList oldFunctions(tc->testCases()); + oldFunctions << testFunctions; + tc->setTestCases(oldFunctions); + } else { + tc = new QuickTestConfiguration(item->framework()); + tc->setTestCases(testFunctions); + tc->setProjectFile(item->proFile()); + tc->setProject(ProjectExplorer::SessionManager::startupProject()); + tc->setInternalTargets(item->internalTargets()); + foundProfiles.insert(item->proFile(), tc); + } +} + TestConfiguration *QuickTestTreeItem::debugConfiguration() const { QuickTestConfiguration *config = static_cast(testConfiguration()); @@ -280,6 +319,28 @@ QList QuickTestTreeItem::getSelectedTestConfigurations() co return result; } +QList QuickTestTreeItem::getFailedTestConfigurations() const +{ + QList result; + ProjectExplorer::Project *project = ProjectExplorer::SessionManager::startupProject(); + if (!project || type() != Root) + return result; + + QHash foundProFiles; + forFirstLevelChildren([&foundProFiles](TestTreeItem *child) { + testConfigurationsForFailed(child, foundProFiles); + }); + + for (auto it = foundProFiles.begin(), end = foundProFiles.end(); it != end; ++it) { + QuickTestConfiguration *config = it.value(); + if (!config->unnamedOnly()) + result << config; + else + delete config; + } + return result; +} + QList QuickTestTreeItem::getTestConfigurationsForFile(const Utils::FilePath &fileName) const { QList result; diff --git a/src/plugins/autotest/quick/quicktesttreeitem.h b/src/plugins/autotest/quick/quicktesttreeitem.h index 050c7c4dd14..d21787805ad 100644 --- a/src/plugins/autotest/quick/quicktesttreeitem.h +++ b/src/plugins/autotest/quick/quicktesttreeitem.h @@ -49,6 +49,7 @@ public: TestConfiguration *debugConfiguration() const override; QList getAllTestConfigurations() const override; QList getSelectedTestConfigurations() const override; + QList getFailedTestConfigurations() const override; QList getTestConfigurationsForFile(const Utils::FilePath &fileName) const override; TestTreeItem *find(const TestParseResult *result) override; TestTreeItem *findChild(const TestTreeItem *other) override; diff --git a/src/plugins/autotest/testresultmodel.cpp b/src/plugins/autotest/testresultmodel.cpp index 7ea5f7d9d8f..0649ffec3dd 100644 --- a/src/plugins/autotest/testresultmodel.cpp +++ b/src/plugins/autotest/testresultmodel.cpp @@ -29,6 +29,8 @@ #include "testresultdelegate.h" #include "testrunner.h" #include "testsettings.h" +#include "testtreeitem.h" +#include "testtreemodel.h" #include #include @@ -242,6 +244,16 @@ void TestResultModel::updateParent(const TestResultItem *item) updateParent(parentItem); } +static bool isFailed(ResultType type) +{ + switch (type) { + case ResultType::Fail: case ResultType::UnexpectedPass: case ResultType::MessageFatal: + return true; + default: + return false; + } +} + void TestResultModel::addTestResult(const TestResultPtr &testResult, bool autoExpand) { const int lastRow = rootItem()->childCount() - 1; @@ -307,6 +319,13 @@ void TestResultModel::addTestResult(const TestResultPtr &testResult, bool autoEx // there is no MessageCurrentTest at the last row, but we have a toplevel item - just add it rootItem()->appendChild(newItem); } + + if (isFailed(testResult->result())) { + if (const TestTreeItem *it = testResult->findTestTreeItem()) { + TestTreeModel *model = TestTreeModel::instance(); + model->setData(model->indexForItem(it), true, FailedRole); + } + } } void TestResultModel::removeCurrentTestMessage() diff --git a/src/plugins/autotest/testrunner.cpp b/src/plugins/autotest/testrunner.cpp index 56973b01df5..4581b91bd53 100644 --- a/src/plugins/autotest/testrunner.cpp +++ b/src/plugins/autotest/testrunner.cpp @@ -335,6 +335,7 @@ void TestRunner::prepareToRunTests(TestRunMode mode) // clear old log and output pane TestResultsPane::instance()->clearContents(); + TestTreeModel::instance()->clearFailedMarks(); if (m_selectedTests.empty()) { reportResult(ResultType::MessageWarn, tr("No tests selected. Canceling test run.")); diff --git a/src/plugins/autotest/testtreeitem.cpp b/src/plugins/autotest/testtreeitem.cpp index 5debb983d64..b0d6634511e 100644 --- a/src/plugins/autotest/testtreeitem.cpp +++ b/src/plugins/autotest/testtreeitem.cpp @@ -102,6 +102,8 @@ QVariant TestTreeItem::data(int /*column*/, int role) const return m_type; case EnabledRole: return true; + case FailedRole: + return m_failed; } return QVariant(); } @@ -112,6 +114,8 @@ bool TestTreeItem::setData(int /*column*/, const QVariant &data, int role) Qt::CheckState old = m_checked; m_checked = Qt::CheckState(data.toInt()); return m_checked != old; + } else if (role == FailedRole) { + m_failed = data.toBool(); } return false; } @@ -262,6 +266,11 @@ QList TestTreeItem::getSelectedTestConfigurations() const return QList(); } +QList TestTreeItem::getFailedTestConfigurations() const +{ + return QList(); +} + QList TestTreeItem::getTestConfigurationsForFile(const Utils::FilePath &) const { return QList(); @@ -333,6 +342,7 @@ void TestTreeItem::copyBasicDataFrom(const TestTreeItem *other) m_filePath = other->m_filePath; m_type = other->m_type; m_checked = other->m_checked; + m_failed = other->m_failed; m_line = other->m_line; m_column = other->m_column; m_proFile = other->m_proFile; diff --git a/src/plugins/autotest/testtreeitem.h b/src/plugins/autotest/testtreeitem.h index ea46cb5dc18..154a4ea147d 100644 --- a/src/plugins/autotest/testtreeitem.h +++ b/src/plugins/autotest/testtreeitem.h @@ -37,7 +37,8 @@ namespace { LinkRole = Qt::UserRole + 2, // can be removed if AnnotationRole comes back ItalicRole, // used only inside the delegate TypeRole, - EnabledRole + EnabledRole, + FailedRole // marker for having failed in last run }; } @@ -119,6 +120,7 @@ public: TestConfiguration *asConfiguration(TestRunMode mode) const; virtual QList getAllTestConfigurations() const; virtual QList getSelectedTestConfigurations() const; + virtual QList getFailedTestConfigurations() const; virtual QList getTestConfigurationsForFile(const Utils::FilePath &fileName) const; virtual bool lessThan(const TestTreeItem *other, SortMode mode) const; virtual TestTreeItem *find(const TestParseResult *result) = 0; @@ -156,6 +158,7 @@ private: QString m_name; QString m_filePath; Qt::CheckState m_checked; + bool m_failed = false; Type m_type; int m_line = 0; int m_column = 0; diff --git a/src/plugins/autotest/testtreemodel.cpp b/src/plugins/autotest/testtreemodel.cpp index b42e0311489..0c32a716a36 100644 --- a/src/plugins/autotest/testtreemodel.cpp +++ b/src/plugins/autotest/testtreemodel.cpp @@ -170,6 +170,14 @@ QList TestTreeModel::getSelectedTests() const return result; } +QList TestTreeModel::getFailedTests() const +{ + QList result; + for (Utils::TreeItem *frameworkRoot : *rootItem()) + result.append(static_cast(frameworkRoot)->getFailedTestConfigurations()); + return result; +} + QList TestTreeModel::getTestsForFile(const Utils::FilePath &fileName) const { QList result; @@ -305,6 +313,23 @@ void TestTreeModel::updateCheckStateCache() } } +bool TestTreeModel::hasFailedTests() const +{ + auto failedItem = rootItem()->findAnyChild([](Utils::TreeItem *it) { + return it->data(0, FailedRole).toBool(); + }); + return failedItem != nullptr; +} + +void TestTreeModel::clearFailedMarks() +{ + for (Utils::TreeItem *rootNode : *rootItem()) { + rootNode->forAllChildren([](Utils::TreeItem *child) { + child->setData(0, false, FailedRole); + }); + } +} + void TestTreeModel::removeFiles(const QStringList &files) { for (const QString &file : files) diff --git a/src/plugins/autotest/testtreemodel.h b/src/plugins/autotest/testtreemodel.h index 34f51cf7bb4..9d07705e80b 100644 --- a/src/plugins/autotest/testtreemodel.h +++ b/src/plugins/autotest/testtreemodel.h @@ -63,12 +63,15 @@ public: bool hasTests() const; QList getAllTestCases() const; QList getSelectedTests() const; + QList getFailedTests() const; QList getTestsForFile(const Utils::FilePath &fileName) const; QList testItemsByName(const QString &testName); void synchronizeTestFrameworks(); void rebuild(const QList &frameworkIds); void updateCheckStateCache(); + bool hasFailedTests() const; + void clearFailedMarks(); #ifdef WITH_TESTS int autoTestsCount() const; int namedQuickTestsCount() const;