AutoTest: Allow registering ITestTools

Task-number: QTCREATORBUG-23332
Change-Id: I529b1cc1f110739c264c7a021aada063f697b1db
Reviewed-by: David Schulz <david.schulz@qt.io>

src/plugins/autotest/autotestplugin.cpp

@@ -154,8 +154,9 @@ AutotestPluginPrivate::AutotestPluginPrivate()
     });
     ProjectExplorer::ProjectPanelFactory::registerFactory(panelFactory);
 
-    TestFrameworkManager::activateFrameworksFromSettings(&m_settings);
+    TestFrameworkManager::activateFrameworksAndToolsFromSettings(&m_settings);
     m_testTreeModel.synchronizeTestFrameworks();
+    m_testTreeModel.synchronizeTestTools();
 
     auto sessionManager = ProjectExplorer::SessionManager::instance();
     connect(sessionManager, &ProjectExplorer::SessionManager::startupProjectChanged,

src/plugins/autotest/itemdatacache.h

@@ -25,6 +25,7 @@
 
 #pragma once
 
+#include "itestframework.h"
 #include "testtreeitem.h"
 
 #include <utils/optional.h>
@@ -38,12 +39,22 @@ template<class T>
 class ItemDataCache
 {
 public:
-    void insert(ITestTreeItem *item, const T &value) { m_cache[item->cacheName()] = {0, value}; }
+    void insert(ITestTreeItem *item, const T &value)
-    void evolve()
+    {
+        m_cache[item->cacheName()] = {
+                0, value, item->testBase()->asTestTool() ? ITestBase::Tool : ITestBase::Framework };
+    }
+
+    /* \a type represents an OR'ed value of ITestBase::TestBaseType */
+    void evolve(int type)
     {
         auto it = m_cache.begin(), end = m_cache.end();
-        while (it != end)
+        while (it != end) {
-            it = it->generation++ >= maxGen ? m_cache.erase(it) : ++it;
+            if ((it->type & type) && it->generation++ >= maxGen)
+                it = m_cache.erase(it);
+            else
+                ++it;
+        }
     }
 
     Utils::optional<T> get(ITestTreeItem *item)
@@ -62,15 +73,22 @@ public:
     {
         QVariantMap result;
         for (auto it = m_cache.cbegin(), end = m_cache.cend(); it != end; ++it)
-            result.insert(it.key(), QVariant::fromValue(it.value().value));
+            result.insert(QString::number(it.value().type) + '@'
+                          + it.key(), QVariant::fromValue(it.value().value));
         return result;
     }
 
     void fromSettings(const QVariantMap &stored)
     {
+        const QRegularExpression regex("^((\\d+)@)?(.*)$");
         m_cache.clear();
-        for (auto it = stored.cbegin(), end = stored.cend(); it != end; ++it)
+        for (auto it = stored.cbegin(), end = stored.cend(); it != end; ++it) {
-            m_cache[it.key()] = {0, qvariant_cast<T>(it.value())};
+            const QRegularExpressionMatch match = regex.match(it.key());
+            ITestBase::TestBaseType type = match.hasMatch()
+                    ? static_cast<ITestBase::TestBaseType>(match.captured(2).toInt())
+                    : ITestBase::Framework;
+            m_cache[match.captured(3)] = {0, qvariant_cast<T>(it.value()), type};
+        }
     }
 
 private:
@@ -79,6 +97,7 @@ private:
     {
         int generation = 0;
         T value;
+        ITestBase::TestBaseType type;
     };
     QHash<QString, Entry> m_cache;
 };

src/plugins/autotest/itestframework.h

@@ -41,6 +41,13 @@ class TestTreeItem;
 class ITestBase
 {
 public:
+    enum TestBaseType
+    {
+        None      = 0x0,
+        Framework = 0x1,
+        Tool      = 0x2
+    };
+
     explicit ITestBase(bool activeByDefault);
     virtual ~ITestBase() = default;
 

src/plugins/autotest/testframeworkmanager.cpp

@@ -62,25 +62,52 @@ bool TestFrameworkManager::registerTestFramework(ITestFramework *framework)
     return true;
 }
 
-void TestFrameworkManager::activateFrameworksFromSettings(const Internal::TestSettings *settings)
+bool TestFrameworkManager::registerTestTool(ITestTool *testTool)
+{
+    QTC_ASSERT(testTool, return false);
+    QTC_ASSERT(!m_registeredTestTools.contains(testTool), return false);
+    m_registeredTestTools.append(testTool);
+    return true;
+}
+
+void TestFrameworkManager::activateFrameworksAndToolsFromSettings(
+        const Internal::TestSettings *settings)
 {
     for (ITestFramework *framework : qAsConst(s_instance->m_registeredFrameworks)) {
         framework->setActive(settings->frameworks.value(framework->id(), false));
         framework->setGrouping(settings->frameworksGrouping.value(framework->id(), false));
     }
+    for (ITestTool *testTool : qAsConst(s_instance->m_registeredTestTools))
+        testTool->setActive(settings->tools.value(testTool->id(), false));
 }
 
-TestFrameworks TestFrameworkManager::registeredFrameworks()
+const TestFrameworks TestFrameworkManager::registeredFrameworks()
 {
     return s_instance->m_registeredFrameworks;
 }
 
+const TestTools TestFrameworkManager::registeredTestTools()
+{
+    return s_instance->m_registeredTestTools;
+}
+
 ITestFramework *TestFrameworkManager::frameworkForId(Id frameworkId)
 {
     return Utils::findOrDefault(s_instance->m_registeredFrameworks,
             [frameworkId](ITestFramework *framework) {
                 return framework->id() == frameworkId;
-            });
+    });
+}
+
+ITestTool *TestFrameworkManager::testToolForBuildSystemId(Id buildSystemId)
+{
+    if (!buildSystemId.isValid())
+        return nullptr;
+
+    return Utils::findOrDefault(s_instance->m_registeredTestTools,
+                                [&buildSystemId](ITestTool *testTool) {
+        return testTool->buildSystemId() == buildSystemId;
+    });
 }
 
 void TestFrameworkManager::synchronizeSettings(QSettings *s)

src/plugins/autotest/testframeworkmanager.h

@@ -44,14 +44,18 @@ public:
     ~TestFrameworkManager();
 
     bool registerTestFramework(ITestFramework *framework);
+    bool registerTestTool(ITestTool *testTool);
     void synchronizeSettings(QSettings *s);
 
     static ITestFramework *frameworkForId(Utils::Id frameworkId);
-    static void activateFrameworksFromSettings(const Internal::TestSettings *settings);
+    static ITestTool *testToolForBuildSystemId(Utils::Id buildSystemId);
-    static TestFrameworks registeredFrameworks();
+    static void activateFrameworksAndToolsFromSettings(const Internal::TestSettings *settings);
+    static const TestFrameworks registeredFrameworks();
+    static const TestTools registeredTestTools();
 
 private:
     TestFrameworks m_registeredFrameworks;
+    TestTools m_registeredTestTools;
 };
 
 } // namespace Autotest

src/plugins/autotest/testnavigationwidget.cpp

@@ -252,7 +252,7 @@ QList<QToolButton *> TestNavigationWidget::createToolButtons()
 
 void TestNavigationWidget::updateExpandedStateCache()
 {
-    m_expandedStateCache.evolve();
+    m_expandedStateCache.evolve(ITestBase::Framework);
 
     for (Utils::TreeItem *rootNode : *m_model->rootItem()) {
         rootNode->forAllChildren([this](Utils::TreeItem *child) {

src/plugins/autotest/testsettings.cpp

@@ -74,6 +74,9 @@ void TestSettings::toSettings(QSettings *s) const
         s->setValue(id.toString(), frameworks.value(id));
         s->setValue(id.toString() + groupSuffix, frameworksGrouping.value(id));
     }
+    // ..and the testtools as well
+    for (const Utils::Id &id : tools.keys())
+        s->setValue(id.toString(), tools.value(id));
     s->endGroup();
 }
 
@@ -104,6 +107,13 @@ void TestSettings::fromSettings(QSettings *s)
         // and whether grouping is enabled
         frameworksGrouping.insert(id, s->value(key + groupSuffix, framework->grouping()).toBool());
     }
+    // ..and for test tools as well
+    const TestTools &registeredTools = TestFrameworkManager::registeredTestTools();
+    tools.clear();
+    for (const ITestTool *testTool : registeredTools) {
+        const Utils::Id id = testTool->id();
+        tools.insert(id, s->value(id.toString(), testTool->active()).toBool());
+    }
     s->endGroup();
 }
 

src/plugins/autotest/testsettings.h

@@ -64,6 +64,7 @@ struct TestSettings
     RunAfterBuildMode runAfterBuild = RunAfterBuildMode::None;
     QHash<Utils::Id, bool> frameworks;
     QHash<Utils::Id, bool> frameworksGrouping;
+    QHash<Utils::Id, bool> tools;
 };
 
 } // namespace Internal

src/plugins/autotest/testsettingspage.cpp

@@ -48,9 +48,6 @@ TestSettingsWidget::TestSettingsWidget(QWidget *parent)
     m_ui.frameworksWarn->setVisible(false);
     m_ui.frameworksWarn->setElideMode(Qt::ElideNone);
     m_ui.frameworksWarn->setType(Utils::InfoLabel::Warning);
-    m_ui.frameworksWarn->setText(tr("No active test frameworks."));
-    m_ui.frameworksWarn->setToolTip(tr("You will not be able to use the AutoTest plugin without "
-                                       "having at least one active test framework."));
     connect(m_ui.frameworkTreeWidget, &QTreeWidget::itemChanged,
             this, &TestSettingsWidget::onFrameworkItemChanged);
     connect(m_ui.resetChoicesButton, &QPushButton::clicked,
@@ -72,7 +69,7 @@ void TestSettingsWidget::setSettings(const TestSettings &settings)
     m_ui.openResultsOnFinishCB->setChecked(settings.popupOnFinish);
     m_ui.openResultsOnFailCB->setChecked(settings.popupOnFail);
     m_ui.runAfterBuildCB->setCurrentIndex(int(settings.runAfterBuild));
-    populateFrameworksListWidget(settings.frameworks);
+    populateFrameworksListWidget(settings.frameworks, settings.tools);
 }
 
 TestSettings TestSettingsWidget::settings() const
@@ -90,10 +87,22 @@ TestSettings TestSettingsWidget::settings() const
     result.popupOnFail = m_ui.openResultsOnFailCB->isChecked();
     result.runAfterBuild = RunAfterBuildMode(m_ui.runAfterBuildCB->currentIndex());
     testSettings(result);
+    testToolsSettings(result);
     return result;
 }
 
-void TestSettingsWidget::populateFrameworksListWidget(const QHash<Utils::Id, bool> &frameworks)
+namespace {
+
+enum TestBaseInfo
+{
+    BaseId = Qt::UserRole,
+    BaseType
+};
+
+}
+
+void TestSettingsWidget::populateFrameworksListWidget(const QHash<Utils::Id, bool> &frameworks,
+                                                      const QHash<Utils::Id, bool> &testTools)
 {
     const TestFrameworks &registered = TestFrameworkManager::registeredFrameworks();
     m_ui.frameworkTreeWidget->clear();
@@ -102,7 +111,8 @@ void TestSettingsWidget::populateFrameworksListWidget(const QHash<Utils::Id, boo
         auto item = new QTreeWidgetItem(m_ui.frameworkTreeWidget, QStringList(QLatin1String(framework->name())));
         item->setFlags(Qt::ItemIsEnabled | Qt::ItemIsSelectable | Qt::ItemIsUserCheckable);
         item->setCheckState(0, frameworks.value(id) ? Qt::Checked : Qt::Unchecked);
-        item->setData(0, Qt::UserRole, id.toSetting());
+        item->setData(0, BaseId, id.toSetting());
+        item->setData(0, BaseType, ITestBase::Framework);
         item->setData(1, Qt::CheckStateRole, framework->grouping() ? Qt::Checked : Qt::Unchecked);
         item->setToolTip(0, tr("Enable or disable test frameworks to be handled by the AutoTest "
                                "plugin."));
@@ -111,33 +121,76 @@ void TestSettingsWidget::populateFrameworksListWidget(const QHash<Utils::Id, boo
             toolTip = tr("Enable or disable grouping of test cases by folder.");
         item->setToolTip(1, toolTip);
     }
+    // ...and now the test tools
+    const TestTools &registeredTools = TestFrameworkManager::registeredTestTools();
+    for (const ITestTool *testTool : registeredTools) {
+        const Utils::Id id = testTool->id();
+        auto item = new QTreeWidgetItem(m_ui.frameworkTreeWidget, {QLatin1String(testTool->name())});
+        item->setFlags(Qt::ItemIsEnabled | Qt::ItemIsSelectable | Qt::ItemIsUserCheckable);
+        item->setCheckState(0, testTools.value(id) ? Qt::Checked : Qt::Unchecked);
+        item->setData(0, BaseId, id.toSetting());
+        item->setData(0, BaseType, ITestBase::Tool);
+    }
 }
 
 void TestSettingsWidget::testSettings(TestSettings &settings) const
 {
     const QAbstractItemModel *model = m_ui.frameworkTreeWidget->model();
     QTC_ASSERT(model, return);
-    const int itemCount = model->rowCount();
+    const int itemCount = TestFrameworkManager::registeredFrameworks().size();
+    QTC_ASSERT(itemCount <= model->rowCount(), return);
     for (int row = 0; row < itemCount; ++row) {
         QModelIndex idx = model->index(row, 0);
-        const Utils::Id id = Utils::Id::fromSetting(idx.data(Qt::UserRole));
+        const Utils::Id id = Utils::Id::fromSetting(idx.data(BaseId));
         settings.frameworks.insert(id, idx.data(Qt::CheckStateRole) == Qt::Checked);
         idx = model->index(row, 1);
         settings.frameworksGrouping.insert(id, idx.data(Qt::CheckStateRole) == Qt::Checked);
     }
 }
 
+void TestSettingsWidget::testToolsSettings(TestSettings &settings) const
+{
+    const QAbstractItemModel *model = m_ui.frameworkTreeWidget->model();
+    QTC_ASSERT(model, return);
+    // frameworks are listed before tools
+    int row = TestFrameworkManager::registeredFrameworks().size();
+    const int end = model->rowCount();
+    QTC_ASSERT(row <= end, return);
+    for ( ; row < end; ++row) {
+        const QModelIndex idx = model->index(row, 0);
+        const Utils::Id id = Utils::Id::fromSetting(idx.data(BaseId));
+        settings.tools.insert(id, idx.data(Qt::CheckStateRole) == Qt::Checked);
+    }
+}
+
 void TestSettingsWidget::onFrameworkItemChanged()
 {
+    bool atLeastOneEnabled = false;
+    int mixed = ITestBase::None;
     if (QAbstractItemModel *model = m_ui.frameworkTreeWidget->model()) {
         for (int row = 0, count = model->rowCount(); row < count; ++row) {
-            if (model->index(row, 0).data(Qt::CheckStateRole) == Qt::Checked) {
+            const QModelIndex idx = model->index(row, 0);
-                m_ui.frameworksWarn->setVisible(false);
+            if (idx.data(Qt::CheckStateRole) == Qt::Checked) {
-                return;
+                atLeastOneEnabled = true;
+                mixed |= idx.data(BaseType).toInt();
             }
         }
     }
-    m_ui.frameworksWarn->setVisible(true);
+
+    if (!atLeastOneEnabled || (mixed == (ITestBase::Framework | ITestBase::Tool))) {
+        if (!atLeastOneEnabled) {
+            m_ui.frameworksWarn->setText(tr("No active test frameworks or tools."));
+            m_ui.frameworksWarn->setToolTip(tr("You will not be able to use the AutoTest plugin "
+                                               "without having at least one active test framework."));
+        } else {
+            m_ui.frameworksWarn->setText(tr("Mixing test frameworks and test tools."));
+            m_ui.frameworksWarn->setToolTip(tr("Mixing test frameworks and test tools can lead "
+                                               "to duplicating run information when using e.g. "
+                                                "'Run All Tests'."));
+        }
+    }
+    m_ui.frameworksWarn->setVisible(!atLeastOneEnabled
+                                    || (mixed == (ITestBase::Framework | ITestBase::Tool)));
 }
 
 TestSettingsPage::TestSettingsPage(TestSettings *settings)
@@ -176,7 +229,11 @@ void TestSettingsPage::apply()
         framework->setGrouping(m_settings->frameworksGrouping.value(framework->id(), false));
     }
 
+    for (ITestTool *testTool : TestFrameworkManager::registeredTestTools())
+        testTool->setActive(m_settings->tools.value(testTool->id(), false));
+
     TestTreeModel::instance()->synchronizeTestFrameworks();
+    TestTreeModel::instance()->synchronizeTestTools();
     if (!changedIds.isEmpty())
         TestTreeModel::instance()->rebuild(changedIds);
 }

src/plugins/autotest/testsettingspage.h

@@ -46,8 +46,10 @@ public:
     TestSettings settings() const;
 
 private:
-    void populateFrameworksListWidget(const QHash<Utils::Id, bool> &frameworks);
+    void populateFrameworksListWidget(const QHash<Utils::Id, bool> &frameworks,
+                                      const QHash<Utils::Id, bool> &testTools);
     void testSettings(TestSettings &settings) const;
+    void testToolsSettings(TestSettings &settings) const;
     void onFrameworkItemChanged();
     Ui::TestSettingsPage m_ui;
 

src/plugins/autotest/testtreemodel.cpp

@@ -33,8 +33,10 @@
 #include "testsettings.h"
 
 #include <cpptools/cppmodelmanager.h>
+#include <projectexplorer/buildsystem.h>
 #include <projectexplorer/project.h>
 #include <projectexplorer/session.h>
+#include <projectexplorer/target.h>
 #include <qmljs/qmljsmodelmanagerinterface.h>
 #include <texteditor/texteditor.h>
 #include <utils/algorithm.h>
@@ -90,12 +92,23 @@ void TestTreeModel::setupParsingConnections()
     m_parser->setState(TestCodeParser::Idle);
 
     SessionManager *sm = SessionManager::instance();
-    connect(sm, &SessionManager::startupProjectChanged, [this](Project *project) {
+    connect(sm, &SessionManager::startupProjectChanged, [this, sm](Project *project) {
         synchronizeTestFrameworks(); // we might have project settings
         m_parser->onStartupProjectChanged(project);
+        removeAllTestToolItems();
         m_checkStateCache = project ? AutotestPlugin::projectSettings(project)->checkStateCache()
                                     : nullptr;
+        onBuildSystemTestsUpdated(); // we may have old results if project was open before switching
         m_failedStateCache.clear();
+        if (project) {
+            if (sm->startupBuildSystem()) {
+                connect(sm->startupBuildSystem(), &BuildSystem::testInformationUpdated,
+                        this, &TestTreeModel::onBuildSystemTestsUpdated, Qt::UniqueConnection);
+            } else {
+                connect(project, &Project::activeTargetChanged,
+                        this, &TestTreeModel::onTargetChanged);
+            }
+        }
     });
 
     CppTools::CppModelManager *cppMM = CppTools::CppModelManager::instance();
@@ -221,6 +234,43 @@ QList<TestTreeItem *> TestTreeModel::testItemsByName(TestTreeItem *root, const Q
     return result;
 }
 
+void TestTreeModel::onTargetChanged(Target *target)
+{
+    if (target && target->buildSystem()) {
+        const Target *topLevelTarget = SessionManager::startupProject()->targets().first();
+        connect(topLevelTarget->buildSystem(), &BuildSystem::testInformationUpdated,
+                this, &TestTreeModel::onBuildSystemTestsUpdated, Qt::UniqueConnection);
+        disconnect(target->project(), &Project::activeTargetChanged,
+                   this, &TestTreeModel::onTargetChanged);
+    }
+}
+
+void TestTreeModel::onBuildSystemTestsUpdated()
+{
+    const BuildSystem *bs = SessionManager::startupBuildSystem();
+    if (!bs || !bs->project())
+        return;
+
+    m_checkStateCache->evolve(ITestBase::Tool);
+
+    ITestTool *testTool = TestFrameworkManager::testToolForBuildSystemId(bs->project()->id());
+    if (!testTool || !testTool->active())
+        return;
+
+    ITestTreeItem *rootNode = testTool->rootNode();
+    QTC_ASSERT(rootNode, return);
+    rootNode->removeChildren();
+    for (const auto &tci : bs->testcasesInfo()) {
+        ITestTreeItem *item = testTool->createItemFromTestCaseInfo(tci);
+        QTC_ASSERT(item, continue);
+        if (Utils::optional<Qt::CheckState> cached = m_checkStateCache->get(item))
+            item->setData(0, cached.value(), Qt::CheckStateRole);
+        m_checkStateCache->insert(item, item->checked());
+        rootNode->appendChild(item);
+    }
+    revalidateCheckState(rootNode);
+}
+
 QList<TestTreeItem *> TestTreeModel::testItemsByName(const QString &testName)
 {
     QList<TestTreeItem *> result;
@@ -255,26 +305,80 @@ void TestTreeModel::synchronizeTestFrameworks()
     // pre-check to avoid further processing when frameworks are unchanged
     Utils::TreeItem *invisibleRoot = rootItem();
     QSet<ITestFramework *> newlyAdded;
-    QList<Utils::TreeItem *> oldFrameworkRoots;
+    QList<ITestTreeItem *> oldFrameworkRoots;
     for (Utils::TreeItem *oldFrameworkRoot : *invisibleRoot)
-        oldFrameworkRoots.append(oldFrameworkRoot);
+        oldFrameworkRoots.append(static_cast<ITestTreeItem *>(oldFrameworkRoot));
 
-    for (Utils::TreeItem *oldFrameworkRoot : oldFrameworkRoots)
+    for (ITestTreeItem *oldFrameworkRoot : oldFrameworkRoots)
         takeItem(oldFrameworkRoot);  // do NOT delete the ptr is still held by TestFrameworkManager
 
-    for (ITestFramework *framework : sorted) {
+    for (ITestFramework *framework : qAsConst(sorted)) {
         TestTreeItem *frameworkRootNode = framework->rootNode();
         invisibleRoot->appendChild(frameworkRootNode);
         if (!oldFrameworkRoots.removeOne(frameworkRootNode))
             newlyAdded.insert(framework);
     }
-    for (Utils::TreeItem *oldFrameworkRoot : oldFrameworkRoots)
+    for (ITestTreeItem *oldFrameworkRoot : oldFrameworkRoots) {
-        oldFrameworkRoot->removeChildren();
+        if (oldFrameworkRoot->testBase()->asFramework())
+            oldFrameworkRoot->removeChildren();
+        else // re-add the test tools - they are handled separately
+            invisibleRoot->appendChild(oldFrameworkRoot);
+    }
 
     m_parser->syncTestFrameworks(sorted);
     if (!newlyAdded.isEmpty())
         m_parser->updateTestTree(newlyAdded);
-    emit updatedActiveFrameworks(sorted.size());
+    emit updatedActiveFrameworks(invisibleRoot->childCount());
+}
+
+void TestTreeModel::synchronizeTestTools()
+{
+    // currently no project settings...
+    // pre-check to avoid further processing when test tools are unchanged
+    Utils::TreeItem *invisibleRoot = rootItem();
+    QSet<ITestTool *> newlyAdded;
+    QList<ITestTreeItem *> oldFrameworkRoots;
+    for (Utils::TreeItem *oldFrameworkRoot : *invisibleRoot) {
+        auto item = static_cast<TestTreeItem *>(oldFrameworkRoot);
+        if (item->testBase()->asTestTool())
+            oldFrameworkRoots.append(item);
+    }
+
+    for (ITestTreeItem *oldFrameworkRoot : oldFrameworkRoots)
+        takeItem(oldFrameworkRoot);  // do NOT delete the ptr is still held by TestFrameworkManager
+
+    for (ITestTool *testTool : TestFrameworkManager::registeredTestTools()) {
+        ITestTreeItem *testToolRootNode = testTool->rootNode();
+        if (testTool->active()) {
+            invisibleRoot->appendChild(testToolRootNode);
+            if (!oldFrameworkRoots.removeOne(testToolRootNode))
+                newlyAdded.insert(testTool);
+        }
+    }
+
+    const Project *project = SessionManager::startupProject();
+    if (project) {
+        const QList<Target *> &allTargets = project->targets();
+        auto target = allTargets.empty() ? nullptr : allTargets.first();
+        if (QTC_GUARD(target)) {
+            auto bs = target->buildSystem();
+            for (ITestTool *testTool : newlyAdded) {
+                ITestTreeItem *rootNode = testTool->rootNode();
+                QTC_ASSERT(rootNode, return);
+                rootNode->removeChildren();
+                for (const auto &tci : bs->testcasesInfo()) {
+                    ITestTreeItem *item = testTool->createItemFromTestCaseInfo(tci);
+                    QTC_ASSERT(item, continue);
+                    if (Utils::optional<Qt::CheckState> cached = m_checkStateCache->get(item))
+                        item->setData(0, cached.value(), Qt::CheckStateRole);
+                    m_checkStateCache->insert(item, item->checked());
+                    rootNode->appendChild(item);
+                }
+                revalidateCheckState(rootNode);
+            }
+        }
+    }
+    emit updatedActiveFrameworks(invisibleRoot->childCount());
 }
 
 void TestTreeModel::filterAndInsert(TestTreeItem *item, TestTreeItem *root, bool groupingEnabled)
@@ -317,7 +421,7 @@ void TestTreeModel::rebuild(const QList<Utils::Id> &frameworkIds)
 
 void TestTreeModel::updateCheckStateCache()
 {
-    m_checkStateCache->evolve();
+    m_checkStateCache->evolve(ITestBase::Framework);
 
     for (Utils::TreeItem *rootNode : *rootItem()) {
         // FIXME limit to framework items
@@ -608,11 +712,26 @@ void TestTreeModel::handleParseResult(const TestParseResult *result, TestTreeIte
 
 void TestTreeModel::removeAllTestItems()
 {
-    for (Utils::TreeItem *item : *rootItem()) {
+    for (Utils::TreeItem *it : *rootItem()) {
+        ITestTreeItem *item = static_cast<ITestTreeItem *>(it);
+        if (item->testBase()->asTestTool())
+            continue;
         item->removeChildren();
-        TestTreeItem *testTreeItem = static_cast<TestTreeItem *>(item);
+        if (item->checked() == Qt::PartiallyChecked)
-        if (testTreeItem->checked() == Qt::PartiallyChecked)
+            item->setData(0, Qt::Checked, Qt::CheckStateRole);
-            testTreeItem->setData(0, Qt::Checked, Qt::CheckStateRole);
+    }
+    emit testTreeModelChanged();
+}
+
+void TestTreeModel::removeAllTestToolItems()
+{
+    for (Utils::TreeItem *it : *rootItem()) {
+        ITestTreeItem * item = static_cast<ITestTreeItem *>(it);
+        if (item->testBase()->asFramework())
+            continue;
+        item->removeChildren();
+        if (item->checked() == Qt::PartiallyChecked)
+            item->setData(0, Qt::Checked, Qt::CheckStateRole);
     }
     emit testTreeModelChanged();
 }

src/plugins/autotest/testtreemodel.h

@@ -35,6 +35,8 @@
 
 #include <QSortFilterProxyModel>
 
+namespace ProjectExplorer { class Target; }
+
 namespace Autotest {
 namespace Internal {
 class AutotestPluginPrivate;
@@ -66,6 +68,7 @@ public:
     QList<ITestConfiguration *> getTestsForFile(const Utils::FilePath &fileName) const;
     QList<TestTreeItem *> testItemsByName(const QString &testName);
     void synchronizeTestFrameworks();
+    void synchronizeTestTools();
     void rebuild(const QList<Utils::Id> &frameworkIds);
 
     void updateCheckStateCache();
@@ -101,6 +104,7 @@ private:
                        const QVector<int> &roles);
     void handleParseResult(const TestParseResult *result, TestTreeItem *rootNode);
     void removeAllTestItems();
+    void removeAllTestToolItems();
     void removeFiles(const QStringList &files);
     bool sweepChildren(TestTreeItem *item);
     void insertItemInParent(TestTreeItem *item, TestTreeItem *root, bool groupingEnabled);
@@ -108,6 +112,8 @@ private:
     void setupParsingConnections();
     void filterAndInsert(TestTreeItem *item, TestTreeItem *root, bool groupingEnabled);
     QList<TestTreeItem *> testItemsByName(TestTreeItem *root, const QString &testName);
+    void onTargetChanged(ProjectExplorer::Target *target);
+    void onBuildSystemTestsUpdated();
 
     Internal::TestCodeParser *m_parser = nullptr;
     Internal::ItemDataCache<Qt::CheckState> *m_checkStateCache = nullptr; // not owned