diff --git a/src/plugins/projectexplorer/taskmodel.h b/src/plugins/projectexplorer/taskmodel.h
index 5fa37c02d1a..64df152cdf9 100644
--- a/src/plugins/projectexplorer/taskmodel.h
+++ b/src/plugins/projectexplorer/taskmodel.h
@@ -117,8 +117,12 @@ public:
     bool filterIncludesErrors() const { return m_includeErrors; }
     void setFilterIncludesErrors(bool b) { m_includeErrors = b; invalidateFilter(); }
 
-    QList<Utils::Id> filteredCategories() const { return m_categoryIds; }
-    void setFilteredCategories(const QList<Utils::Id> &categoryIds) { m_categoryIds = categoryIds; invalidateFilter(); }
+    QSet<Utils::Id> filteredCategories() const { return m_categoryIds; }
+    void setFilteredCategories(const QSet<Utils::Id> &categoryIds)
+    {
+        m_categoryIds = categoryIds;
+        invalidateFilter();
+    }
 
     Task task(const QModelIndex &index) const { return taskModel()->task(mapToSource(index)); }
     Tasks tasks(const QModelIndexList &indexes) const;
@@ -144,7 +148,7 @@ private:
     bool m_filterStringIsRegexp = false;
     bool m_filterIsInverted = false;
     Qt::CaseSensitivity m_filterCaseSensitivity = Qt::CaseInsensitive;
-    QList<Utils::Id> m_categoryIds;
+    QSet<Utils::Id> m_categoryIds;
     QString m_filterText;
     QRegularExpression m_filterRegexp;
 };
diff --git a/src/plugins/projectexplorer/taskwindow.cpp b/src/plugins/projectexplorer/taskwindow.cpp
index 703a277f159..7bad834230d 100644
--- a/src/plugins/projectexplorer/taskwindow.cpp
+++ b/src/plugins/projectexplorer/taskwindow.cpp
@@ -327,19 +327,20 @@ void TaskWindow::setCategoryVisibility(Id categoryId, bool visible)
     if (!categoryId.isValid())
         return;
 
-    QList<Id> categories = d->m_filter->filteredCategories();
+    QSet<Id> categories = d->m_filter->filteredCategories();
 
     if (visible)
-        categories.removeOne(categoryId);
+        categories.remove(categoryId);
     else
-        categories.append(categoryId);
+        categories.insert(categoryId);
 
     d->m_filter->setFilteredCategories(categories);
 }
 
 void TaskWindow::saveSettings()
 {
-    QStringList categories = Utils::transform(d->m_filter->filteredCategories(), &Id::toString);
+    const QStringList categories = Utils::toList(
+        Utils::transform(d->m_filter->filteredCategories(), &Id::toString));
     SessionManager::setValue(QLatin1String(SESSION_FILTER_CATEGORIES), categories);
     SessionManager::setValue(QLatin1String(SESSION_FILTER_WARNINGS), d->m_filter->filterIncludesWarnings());
 }
@@ -348,7 +349,8 @@ void TaskWindow::loadSettings()
 {
     QVariant value = SessionManager::value(QLatin1String(SESSION_FILTER_CATEGORIES));
     if (value.isValid()) {
-        QList<Id> categories = Utils::transform(value.toStringList(), &Id::fromString);
+        const QSet<Id> categories = Utils::toSet(
+            Utils::transform(value.toStringList(), &Id::fromString));
         d->m_filter->setFilteredCategories(categories);
     }
     value = SessionManager::value(QLatin1String(SESSION_FILTER_WARNINGS));
@@ -369,8 +371,8 @@ void TaskWindow::addCategory(Id categoryId, const QString &displayName, bool vis
 {
     d->m_model->addCategory(categoryId, displayName, priority);
     if (!visible) {
-        QList<Id> filters = d->m_filter->filteredCategories();
-        filters += categoryId;
+        QSet<Id> filters = d->m_filter->filteredCategories();
+        filters.insert(categoryId);
         d->m_filter->setFilteredCategories(filters);
     }
 }
@@ -469,7 +471,7 @@ void TaskWindow::updateCategoriesMenu()
 
     d->m_categoriesMenu->clear();
 
-    const QList<Id> filteredCategories = d->m_filter->filteredCategories();
+    const QSet<Id> filteredCategories = d->m_filter->filteredCategories();
 
     QMap<QString, Id> nameToIds;
     const QList<Id> ids = d->m_model->categoryIds();