AutoTest: Support gathering failed tests

Mark test tree items as failed for the last run
to be able to re-run them in an easier way.

Change-Id: I7ea3dcd16e5a02797d41f13e02b2fa95b857cf5e
Reviewed-by: David Schulz <david.schulz@qt.io>

src/plugins/autotest/boost/boosttesttreeitem.cpp

@@ -215,7 +215,8 @@ QList<TestConfiguration *> BoostTestTreeItem::getAllTestConfigurations() const
     return result;
 }
 
-QList<TestConfiguration *> BoostTestTreeItem::getSelectedTestConfigurations() const
+QList<TestConfiguration *> BoostTestTreeItem::getTestConfigurations(
+        std::function<bool(BoostTestTreeItem *)> predicate) const
 {
     QList<TestConfiguration *> result;
     ProjectExplorer::Project *project = ProjectExplorer::SessionManager::startupProject();
@@ -228,13 +229,13 @@ QList<TestConfiguration *> BoostTestTreeItem::getSelectedTestConfigurations() co
     };
 
     QHash<QString, BoostTestCases> testCasesForProjectFile;
-    forAllChildren([&testCasesForProjectFile](TreeItem *it){
+    forAllChildren([&testCasesForProjectFile, &predicate](TreeItem *it){
         auto item = static_cast<BoostTestTreeItem *>(it);
         if (item->type() != TestCase)
             return;
         if (!item->enabled()) // ignore child tests known to be disabled when using run selected
             return;
-        if (item->checked() == Qt::Checked) {
+        if (predicate(item)) {
             QString tcName = item->name();
             if (item->state().testFlag(BoostTestTreeItem::Templated))
                 tcName.append("<*");
@@ -261,6 +262,20 @@ QList<TestConfiguration *> BoostTestTreeItem::getSelectedTestConfigurations() co
     return result;
 }
 
+QList<TestConfiguration *> BoostTestTreeItem::getSelectedTestConfigurations() const
+{
+    return getTestConfigurations([](BoostTestTreeItem *it) {
+        return it->checked() == Qt::Checked;
+    });
+}
+
+QList<TestConfiguration *> BoostTestTreeItem::getFailedTestConfigurations() const
+{
+    return getTestConfigurations([](BoostTestTreeItem *it) {
+        return it->data(0, FailedRole).toBool();
+    });
+}
+
 TestConfiguration *BoostTestTreeItem::testConfiguration() const
 {
     ProjectExplorer::Project *project = ProjectExplorer::SessionManager::startupProject();

src/plugins/autotest/boost/boosttesttreeitem.h

@@ -71,6 +71,7 @@ public:
 
     QList<TestConfiguration *> getAllTestConfigurations() const override;
     QList<TestConfiguration *> getSelectedTestConfigurations() const override;
+    QList<TestConfiguration *> getFailedTestConfigurations() const override;
     bool canProvideTestConfiguration() const override { return type() != Root; }
     bool canProvideDebugConfiguration() const override { return canProvideTestConfiguration(); }
     TestConfiguration *testConfiguration() const override;
@@ -83,6 +84,8 @@ private:
                                               BoostTestTreeItem::TestStates state,
                                               const QString &proFile) const;
     QString prependWithParentsSuitePaths(const QString &testName) const;
+    QList<TestConfiguration *> getTestConfigurations(
+            std::function<bool(BoostTestTreeItem *)> predicate) const;
     bool modifyTestContent(const BoostTestParseResult *result);
     TestStates m_state = Enabled;
     QString m_fullName;

src/plugins/autotest/catch/catchtreeitem.cpp

@@ -221,6 +221,25 @@ static void collectTestInfo(const TestTreeItem *item,
     }
 }
 
+static void collectFailedTestInfo(const CatchTreeItem *item,
+                                  QHash<QString, CatchTestCases> &testCasesForProfile)
+{
+    QTC_ASSERT(item, return);
+    QTC_ASSERT(item->type() == TestTreeItem::Root, return);
+
+    item->forAllChildren([&testCasesForProfile](TestTreeItem *it) {
+        QTC_ASSERT(it, return);
+        CatchTreeItem *parent = static_cast<CatchTreeItem *>(it->parentItem());
+        QTC_ASSERT(parent, return);
+        if (it->type() == TestTreeItem::TestCase && it->data(0, FailedRole).toBool()) {
+            CatchTreeItem *current = static_cast<CatchTreeItem *>(it);
+            testCasesForProfile[it->proFile()].names.append(current->testCasesString());
+            testCasesForProfile[it->proFile()].internalTargets.unite(
+                        it->internalTargets());
+        }
+    });
+}
+
 QList<TestConfiguration *> CatchTreeItem::getAllTestConfigurations() const
 {
     return getTestConfigurations(true);
@@ -231,6 +250,30 @@ QList<TestConfiguration *> CatchTreeItem::getSelectedTestConfigurations() const
     return getTestConfigurations(false);
 }
 
+QList<TestConfiguration *> CatchTreeItem::getFailedTestConfigurations() const
+{
+    QList<TestConfiguration *> result;
+    ProjectExplorer::Project *project = ProjectExplorer::SessionManager::startupProject();
+    if (!project || type() != Root)
+        return result;
+
+    QHash<QString, CatchTestCases> testCasesForProFile;
+    collectFailedTestInfo(this, testCasesForProFile);
+
+    for (auto it = testCasesForProFile.begin(), end = testCasesForProFile.end(); it != end; ++it) {
+        for (const QString &target : qAsConst(it.value().internalTargets)) {
+            CatchConfiguration *tc = new CatchConfiguration(framework());
+            tc->setTestCases(it.value().names);
+            tc->setProjectFile(it.key());
+            tc->setProject(project);
+            tc->setInternalTarget(target);
+            result << tc;
+        }
+    }
+
+    return result;
+}
+
 QList<TestConfiguration *> CatchTreeItem::getTestConfigurationsForFile(const Utils::FilePath &fileName) const
 {
     QList<TestConfiguration *> result;

src/plugins/autotest/catch/catchtreeitem.h

@@ -63,6 +63,7 @@ public:
     TestConfiguration *debugConfiguration() const override;
     QList<TestConfiguration *> getAllTestConfigurations() const override;
     QList<TestConfiguration *> getSelectedTestConfigurations() const override;
+    QList<TestConfiguration *> getFailedTestConfigurations() const override;
     QList<TestConfiguration *> getTestConfigurationsForFile(const Utils::FilePath &fileName) const override;
 
 private:

src/plugins/autotest/gtest/gtesttreeitem.cpp

@@ -248,6 +248,25 @@ static void collectTestInfo(const GTestTreeItem *item,
     }
 }
 
+static void collectFailedTestInfo(const GTestTreeItem *item,
+                                  QHash<QString, GTestCases> &testCasesForProfile)
+{
+    QTC_ASSERT(item, return);
+    QTC_ASSERT(item->type() == TestTreeItem::Root, return);
+
+    item->forAllChildren([&testCasesForProfile](TestTreeItem *it) {
+        QTC_ASSERT(it, return);
+        GTestTreeItem *parent = static_cast<GTestTreeItem *>(it->parentItem());
+        QTC_ASSERT(parent, return);
+        if (it->type() == TestTreeItem::TestCase && it->data(0, FailedRole).toBool()) {
+            testCasesForProfile[it->proFile()].filters.append(
+                        gtestFilter(parent->state()).arg(parent->name()).arg(it->name()));
+            testCasesForProfile[it->proFile()].internalTargets.unite(
+                        it->internalTargets());
+        }
+    });
+}
+
 QList<TestConfiguration *> GTestTreeItem::getTestConfigurations(bool ignoreCheckState) const
 {
     QList<TestConfiguration *> result;
@@ -287,6 +306,31 @@ QList<TestConfiguration *> GTestTreeItem::getSelectedTestConfigurations() const
     return getTestConfigurations(false);
 }
 
+QList<TestConfiguration *> GTestTreeItem::getFailedTestConfigurations() const
+{
+    QList<TestConfiguration *> result;
+    ProjectExplorer::Project *project = ProjectExplorer::SessionManager::startupProject();
+    if (!project || type() != Root)
+        return result;
+
+    QHash<QString, GTestCases> testCasesForProFile;
+    collectFailedTestInfo(this, testCasesForProFile);
+
+    for (auto it = testCasesForProFile.begin(), end = testCasesForProFile.end(); it != end; ++it) {
+        for (const QString &target : qAsConst(it.value().internalTargets)) {
+            GTestConfiguration *tc = new GTestConfiguration(framework());
+            tc->setTestCases(it.value().filters);
+            tc->setTestCaseCount(tc->testCaseCount() + it.value().testSetCount);
+            tc->setProjectFile(it.key());
+            tc->setProject(project);
+            tc->setInternalTarget(target);
+            result << tc;
+        }
+    }
+
+    return result;
+}
+
 QList<TestConfiguration *> GTestTreeItem::getTestConfigurationsForFile(const Utils::FilePath &fileName) const
 {
     QList<TestConfiguration *> result;

src/plugins/autotest/gtest/gtesttreeitem.h

@@ -61,6 +61,7 @@ public:
     TestConfiguration *debugConfiguration() const override;
     QList<TestConfiguration *> getAllTestConfigurations() const override;
     QList<TestConfiguration *> getSelectedTestConfigurations() const override;
+    QList<TestConfiguration *> getFailedTestConfigurations() const override;
     QList<TestConfiguration *> getTestConfigurationsForFile(const Utils::FilePath &fileName) const override;
     TestTreeItem *find(const TestParseResult *result) override;
     TestTreeItem *findChild(const TestTreeItem *other) override;

src/plugins/autotest/qtest/qttesttreeitem.cpp

@@ -190,6 +190,37 @@ static void fillTestConfigurationsFromCheckState(const TestTreeItem *item,
     }
 }
 
+static void collectFailedTestInfo(TestTreeItem *item, QList<TestConfiguration *> &testConfigs)
+{
+    QTC_ASSERT(item, return);
+    if (item->type() == TestTreeItem::GroupNode) {
+        for (int row = 0, count = item->childCount(); row < count; ++row)
+            collectFailedTestInfo(item->childAt(row), testConfigs);
+        return;
+    }
+    QTC_ASSERT(item->type() == TestTreeItem::TestCase, return);
+    QStringList testCases;
+    item->forFirstLevelChildren([&testCases](TestTreeItem *func) {
+        if (func->data(0, FailedRole).toBool()) {
+            testCases << func->name();
+        } else {
+            func->forFirstLevelChildren([&testCases, func](TestTreeItem *dataTag) {
+                if (dataTag->data(0, FailedRole).toBool())
+                    testCases << func->name() + ':' + dataTag->name();
+            });
+        }
+    });
+    if (testCases.isEmpty())
+        return;
+
+    QtTestConfiguration *testConfig = new QtTestConfiguration(item->framework());
+    testConfig->setTestCases(testCases);
+    testConfig->setProjectFile(item->proFile());
+    testConfig->setProject(ProjectExplorer::SessionManager::startupProject());
+    testConfig->setInternalTargets(item->internalTargets());
+    testConfigs << testConfig;
+}
+
 TestConfiguration *QtTestTreeItem::debugConfiguration() const
 {
     QtTestConfiguration *config = static_cast<QtTestConfiguration *>(testConfiguration());
@@ -235,6 +266,16 @@ QList<TestConfiguration *> QtTestTreeItem::getSelectedTestConfigurations() const
     return result;
 }
 
+QList<TestConfiguration *> QtTestTreeItem::getFailedTestConfigurations() const
+{
+    QList<TestConfiguration *> result;
+    QTC_ASSERT(type() == TestTreeItem::Root, return result);
+    forFirstLevelChildren([&result](TestTreeItem *child) {
+        collectFailedTestInfo(child, result);
+    });
+    return result;
+}
+
 QList<TestConfiguration *> QtTestTreeItem::getTestConfigurationsForFile(const Utils::FilePath &fileName) const
 {
     QList<TestConfiguration *> result;

src/plugins/autotest/qtest/qttesttreeitem.h

@@ -45,6 +45,7 @@ public:
     TestConfiguration *debugConfiguration() const override;
     QList<TestConfiguration *> getAllTestConfigurations() const override;
     QList<TestConfiguration *> getSelectedTestConfigurations() const override;
+    QList<TestConfiguration *> getFailedTestConfigurations() const override;
     QList<TestConfiguration *> getTestConfigurationsForFile(const Utils::FilePath &fileName) const override;
     TestTreeItem *find(const TestParseResult *result) override;
     TestTreeItem *findChild(const TestTreeItem *other) override;

src/plugins/autotest/quick/quicktesttreeitem.cpp

@@ -195,6 +195,45 @@ static void testConfigurationFromCheckState(const TestTreeItem *item,
     }
 }
 
+static void testConfigurationsForFailed(const TestTreeItem *item,
+                                        QHash<QString, QuickTestConfiguration *> &foundProfiles)
+{
+    QTC_ASSERT(item, return);
+    if (item->type() == TestTreeItem::GroupNode) {
+        for (int row = 0, count = item->childCount(); row < count; ++row)
+            testConfigurationsForFailed(item->childAt(row), foundProfiles);
+        return;
+    }
+    QTC_ASSERT(item->type() == TestTreeItem::TestCase, return);
+
+    const QString testName = item->name();
+    if (testName.isEmpty()) // skip unnamed quick tests as we cannot address them
+        return;
+
+    QStringList testFunctions;
+    item->forFirstLevelChildren([&testFunctions, &testName](TestTreeItem *child) {
+        if (child->data(0, FailedRole).toBool())
+            testFunctions << testName + "::" + child->name();
+    });
+    if (testFunctions.isEmpty())
+        return;
+
+    QuickTestConfiguration *tc = nullptr;
+    if (foundProfiles.contains(item->proFile())) {
+        tc = foundProfiles[item->proFile()];
+        QStringList oldFunctions(tc->testCases());
+        oldFunctions << testFunctions;
+        tc->setTestCases(oldFunctions);
+    } else {
+        tc = new QuickTestConfiguration(item->framework());
+        tc->setTestCases(testFunctions);
+        tc->setProjectFile(item->proFile());
+        tc->setProject(ProjectExplorer::SessionManager::startupProject());
+        tc->setInternalTargets(item->internalTargets());
+        foundProfiles.insert(item->proFile(), tc);
+    }
+}
+
 TestConfiguration *QuickTestTreeItem::debugConfiguration() const
 {
     QuickTestConfiguration *config = static_cast<QuickTestConfiguration *>(testConfiguration());
@@ -280,6 +319,28 @@ QList<TestConfiguration *> QuickTestTreeItem::getSelectedTestConfigurations() co
     return result;
 }
 
+QList<TestConfiguration *> QuickTestTreeItem::getFailedTestConfigurations() const
+{
+    QList<TestConfiguration *> result;
+    ProjectExplorer::Project *project = ProjectExplorer::SessionManager::startupProject();
+    if (!project || type() != Root)
+        return result;
+
+    QHash<QString, QuickTestConfiguration *> foundProFiles;
+    forFirstLevelChildren([&foundProFiles](TestTreeItem *child) {
+        testConfigurationsForFailed(child, foundProFiles);
+    });
+
+    for (auto it = foundProFiles.begin(), end = foundProFiles.end(); it != end; ++it) {
+        QuickTestConfiguration *config = it.value();
+        if (!config->unnamedOnly())
+            result << config;
+        else
+            delete config;
+    }
+    return result;
+}
+
 QList<TestConfiguration *> QuickTestTreeItem::getTestConfigurationsForFile(const Utils::FilePath &fileName) const
 {
     QList<TestConfiguration *> result;

src/plugins/autotest/quick/quicktesttreeitem.h

@@ -49,6 +49,7 @@ public:
     TestConfiguration *debugConfiguration() const override;
     QList<TestConfiguration *> getAllTestConfigurations() const override;
     QList<TestConfiguration *> getSelectedTestConfigurations() const override;
+    QList<TestConfiguration *> getFailedTestConfigurations() const override;
     QList<TestConfiguration *> getTestConfigurationsForFile(const Utils::FilePath &fileName) const override;
     TestTreeItem *find(const TestParseResult *result) override;
     TestTreeItem *findChild(const TestTreeItem *other) override;

src/plugins/autotest/testresultmodel.cpp

@@ -29,6 +29,8 @@
 #include "testresultdelegate.h"
 #include "testrunner.h"
 #include "testsettings.h"
+#include "testtreeitem.h"
+#include "testtreemodel.h"
 
 #include <projectexplorer/projectexplorericons.h>
 #include <utils/qtcassert.h>
@@ -242,6 +244,16 @@ void TestResultModel::updateParent(const TestResultItem *item)
     updateParent(parentItem);
 }
 
+static bool isFailed(ResultType type)
+{
+    switch (type) {
+    case ResultType::Fail: case ResultType::UnexpectedPass: case ResultType::MessageFatal:
+        return true;
+    default:
+        return false;
+    }
+}
+
 void TestResultModel::addTestResult(const TestResultPtr &testResult, bool autoExpand)
 {
     const int lastRow = rootItem()->childCount() - 1;
@@ -307,6 +319,13 @@ void TestResultModel::addTestResult(const TestResultPtr &testResult, bool autoEx
         // there is no MessageCurrentTest at the last row, but we have a toplevel item - just add it
         rootItem()->appendChild(newItem);
     }
+
+    if (isFailed(testResult->result())) {
+        if (const TestTreeItem *it = testResult->findTestTreeItem()) {
+            TestTreeModel *model = TestTreeModel::instance();
+            model->setData(model->indexForItem(it), true, FailedRole);
+        }
+    }
 }
 
 void TestResultModel::removeCurrentTestMessage()

src/plugins/autotest/testrunner.cpp

@@ -335,6 +335,7 @@ void TestRunner::prepareToRunTests(TestRunMode mode)
 
     // clear old log and output pane
     TestResultsPane::instance()->clearContents();
+    TestTreeModel::instance()->clearFailedMarks();
 
     if (m_selectedTests.empty()) {
         reportResult(ResultType::MessageWarn, tr("No tests selected. Canceling test run."));

src/plugins/autotest/testtreeitem.cpp

@@ -102,6 +102,8 @@ QVariant TestTreeItem::data(int /*column*/, int role) const
         return m_type;
     case EnabledRole:
         return true;
+    case FailedRole:
+        return m_failed;
     }
     return QVariant();
 }
@@ -112,6 +114,8 @@ bool TestTreeItem::setData(int /*column*/, const QVariant &data, int role)
         Qt::CheckState old = m_checked;
         m_checked = Qt::CheckState(data.toInt());
         return m_checked != old;
+    } else if (role == FailedRole) {
+        m_failed = data.toBool();
     }
     return false;
 }
@@ -262,6 +266,11 @@ QList<TestConfiguration *> TestTreeItem::getSelectedTestConfigurations() const
     return QList<TestConfiguration *>();
 }
 
+QList<TestConfiguration *> TestTreeItem::getFailedTestConfigurations() const
+{
+    return QList<TestConfiguration *>();
+}
+
 QList<TestConfiguration *> TestTreeItem::getTestConfigurationsForFile(const Utils::FilePath &) const
 {
     return QList<TestConfiguration *>();
@@ -333,6 +342,7 @@ void TestTreeItem::copyBasicDataFrom(const TestTreeItem *other)
     m_filePath = other->m_filePath;
     m_type = other->m_type;
     m_checked = other->m_checked;
+    m_failed = other->m_failed;
     m_line = other->m_line;
     m_column = other->m_column;
     m_proFile = other->m_proFile;

src/plugins/autotest/testtreeitem.h

@@ -37,7 +37,8 @@ namespace {
         LinkRole = Qt::UserRole + 2, // can be removed if AnnotationRole comes back
         ItalicRole, // used only inside the delegate
         TypeRole,
-        EnabledRole
+        EnabledRole,
+        FailedRole  // marker for having failed in last run
     };
 }
 
@@ -119,6 +120,7 @@ public:
     TestConfiguration *asConfiguration(TestRunMode mode) const;
     virtual QList<TestConfiguration *> getAllTestConfigurations() const;
     virtual QList<TestConfiguration *> getSelectedTestConfigurations() const;
+    virtual QList<TestConfiguration *> getFailedTestConfigurations() const;
     virtual QList<TestConfiguration *> getTestConfigurationsForFile(const Utils::FilePath &fileName) const;
     virtual bool lessThan(const TestTreeItem *other, SortMode mode) const;
     virtual TestTreeItem *find(const TestParseResult *result) = 0;
@@ -156,6 +158,7 @@ private:
     QString m_name;
     QString m_filePath;
     Qt::CheckState m_checked;
+    bool m_failed = false;
     Type m_type;
     int m_line = 0;
     int m_column = 0;

src/plugins/autotest/testtreemodel.cpp

@@ -170,6 +170,14 @@ QList<TestConfiguration *> TestTreeModel::getSelectedTests() const
     return result;
 }
 
+QList<TestConfiguration *> TestTreeModel::getFailedTests() const
+{
+    QList<TestConfiguration *> result;
+    for (Utils::TreeItem *frameworkRoot : *rootItem())
+        result.append(static_cast<TestTreeItem *>(frameworkRoot)->getFailedTestConfigurations());
+    return result;
+}
+
 QList<TestConfiguration *> TestTreeModel::getTestsForFile(const Utils::FilePath &fileName) const
 {
     QList<TestConfiguration *> result;
@@ -305,6 +313,23 @@ void TestTreeModel::updateCheckStateCache()
     }
 }
 
+bool TestTreeModel::hasFailedTests() const
+{
+    auto failedItem = rootItem()->findAnyChild([](Utils::TreeItem *it) {
+        return it->data(0, FailedRole).toBool();
+    });
+    return failedItem != nullptr;
+}
+
+void TestTreeModel::clearFailedMarks()
+{
+    for (Utils::TreeItem *rootNode : *rootItem()) {
+        rootNode->forAllChildren([](Utils::TreeItem *child) {
+            child->setData(0, false, FailedRole);
+        });
+    }
+}
+
 void TestTreeModel::removeFiles(const QStringList &files)
 {
     for (const QString &file : files)

src/plugins/autotest/testtreemodel.h

@@ -63,12 +63,15 @@ public:
     bool hasTests() const;
     QList<TestConfiguration *> getAllTestCases() const;
     QList<TestConfiguration *> getSelectedTests() const;
+    QList<TestConfiguration *> getFailedTests() const;
     QList<TestConfiguration *> getTestsForFile(const Utils::FilePath &fileName) const;
     QList<TestTreeItem *> testItemsByName(const QString &testName);
     void synchronizeTestFrameworks();
     void rebuild(const QList<Utils::Id> &frameworkIds);
 
     void updateCheckStateCache();
+    bool hasFailedTests() const;
+    void clearFailedMarks();
 #ifdef WITH_TESTS
     int autoTestsCount() const;
     int namedQuickTestsCount() const;