diff --git a/src/plugins/coreplugin/locator/directoryfilter.cpp b/src/plugins/coreplugin/locator/directoryfilter.cpp
index a18c92b5736..6100181ff43 100644
--- a/src/plugins/coreplugin/locator/directoryfilter.cpp
+++ b/src/plugins/coreplugin/locator/directoryfilter.cpp
@@ -83,8 +83,7 @@ DirectoryFilter::DirectoryFilter(Id id)
     const auto groupSetup = [this] {
         if (!m_directories.isEmpty())
             return TaskAction::Continue; // Async task will run
-        m_files.clear();
-        updateFileIterator();
+        m_cache.setFilePaths({});
         return TaskAction::StopWithDone; // Group stops, skips async task
     };
     const auto asyncSetup = [this](Async<FilePaths> &async) {
@@ -92,8 +91,8 @@ DirectoryFilter::DirectoryFilter(Id id)
                                     displayName());
     };
     const auto asyncDone = [this](const Async<FilePaths> &async) {
-        m_files = async.isResultAvailable() ? async.result() : FilePaths();
-        updateFileIterator();
+        if (async.isResultAvailable())
+            m_cache.setFilePaths(async.result());
     };
     const Group root {
         OnGroupSetup(groupSetup),
@@ -113,10 +112,11 @@ void DirectoryFilter::saveState(QJsonObject &object) const
     }
     if (m_filters != kFiltersDefault)
         object.insert(kFiltersKey, QJsonArray::fromStringList(m_filters));
-    if (!m_files.isEmpty())
-        object.insert(kFilesKey,
-                      QJsonArray::fromStringList(
-                          Utils::transform(m_files, &Utils::FilePath::toString)));
+    const std::optional<FilePaths> files = m_cache.filePaths();
+    if (files) {
+        object.insert(kFilesKey, QJsonArray::fromStringList(
+                                     Utils::transform(*files, &FilePath::toString)));
+    }
     if (m_exclusionFilters != kExclusionFiltersDefault)
         object.insert(kExclusionFiltersKey, QJsonArray::fromStringList(m_exclusionFilters));
 }
@@ -138,7 +138,10 @@ void DirectoryFilter::restoreState(const QJsonObject &object)
     m_directories = toFilePaths(object.value(kDirectoriesKey).toArray());
     m_filters = toStringList(
         object.value(kFiltersKey).toArray(QJsonArray::fromStringList(kFiltersDefault)));
-    m_files = FileUtils::toFilePathList(toStringList(object.value(kFilesKey).toArray()));
+    if (object.contains(kFilesKey)) {
+        m_cache.setFilePaths(FileUtils::toFilePathList(
+            toStringList(object.value(kFilesKey).toArray())));
+    }
     m_exclusionFilters = toStringList(
         object.value(kExclusionFiltersKey)
             .toArray(QJsonArray::fromStringList(kExclusionFiltersDefault)));
@@ -161,7 +164,7 @@ void DirectoryFilter::restoreState(const QByteArray &state)
         in >> shortcut;
         in >> defaultFilter;
         in >> files;
-        m_files = FileUtils::toFilePathList(files);
+        m_cache.setFilePaths(FileUtils::toFilePathList(files));
         if (!in.atEnd()) // Qt Creator 4.3 and later
             in >> m_exclusionFilters;
         else
@@ -178,7 +181,6 @@ void DirectoryFilter::restoreState(const QByteArray &state)
     } else {
         ILocatorFilter::restoreState(state);
     }
-    updateFileIterator();
 }
 
 class DirectoryFilterOptions : public QDialog
@@ -381,12 +383,6 @@ void DirectoryFilter::updateOptionButtons()
     m_dialog->removeButton->setEnabled(haveSelectedItem);
 }
 
-void DirectoryFilter::updateFileIterator()
-{
-    m_cache.setFilePaths(m_files);
-    setFileIterator(new BaseFileFilter::ListIterator(m_files));
-}
-
 void DirectoryFilter::setIsCustomFilter(bool value)
 {
     m_isCustomFilter = value;
diff --git a/src/plugins/coreplugin/locator/directoryfilter.h b/src/plugins/coreplugin/locator/directoryfilter.h
index 556c9937a30..485377d2a15 100644
--- a/src/plugins/coreplugin/locator/directoryfilter.h
+++ b/src/plugins/coreplugin/locator/directoryfilter.h
@@ -3,19 +3,14 @@
 
 #pragma once
 
-#include "basefilefilter.h"
+#include "ilocatorfilter.h"
 
 #include <coreplugin/core_global.h>
 
-#include <QByteArray>
-
 namespace Core {
 
-// TODO: Don't derive from BaseFileFilter, flatten the hierarchy
-class CORE_EXPORT DirectoryFilter : public BaseFileFilter
+class CORE_EXPORT DirectoryFilter : public ILocatorFilter
 {
-    Q_OBJECT
-
 public:
     DirectoryFilter(Utils::Id id);
     void restoreState(const QByteArray &state) override;
@@ -38,8 +33,6 @@ private:
     void handleEditDirectory();
     void handleRemoveDirectory();
     void updateOptionButtons();
-    // TODO: Remove me, replace with direct "m_cache.setFilePaths()" call
-    void updateFileIterator();
 
     Utils::FilePaths m_directories;
     QStringList m_filters;
@@ -47,8 +40,6 @@ private:
     // Our config dialog, uses in addDirectory and editDirectory
     // to give their dialogs the right parent
     class DirectoryFilterOptions *m_dialog = nullptr;
-    // TODO: Remove me, use the cache instead.
-    Utils::FilePaths m_files;
     bool m_isCustomFilter = true;
     LocatorFileCache m_cache;
 };
diff --git a/src/plugins/coreplugin/locator/ilocatorfilter.cpp b/src/plugins/coreplugin/locator/ilocatorfilter.cpp
index 4fadef96d8e..45ae41c8fdd 100644
--- a/src/plugins/coreplugin/locator/ilocatorfilter.cpp
+++ b/src/plugins/coreplugin/locator/ilocatorfilter.cpp
@@ -1361,6 +1361,11 @@ void LocatorFileCache::setGeneratorProvider(const GeneratorProvider &provider)
     d->setGeneratorProvider(provider);
 }
 
+std::optional<FilePaths> LocatorFileCache::filePaths() const
+{
+    return d->m_filePaths;
+}
+
 /*!
     Sets the file path generator.
 
@@ -1413,6 +1418,7 @@ void LocatorFileCache::setFilePathsGenerator(const FilePathsGenerator &generator
 void LocatorFileCache::setFilePaths(const FilePaths &filePaths)
 {
     setFilePathsGenerator(filePathsGenerator(filePaths));
+    d->m_filePaths = filePaths;
 }
 
 /*!
diff --git a/src/plugins/coreplugin/locator/ilocatorfilter.h b/src/plugins/coreplugin/locator/ilocatorfilter.h
index b7b8618ea96..fb70648967e 100644
--- a/src/plugins/coreplugin/locator/ilocatorfilter.h
+++ b/src/plugins/coreplugin/locator/ilocatorfilter.h
@@ -321,6 +321,8 @@ public:
     void setFilePaths(const Utils::FilePaths &filePaths);
     void setGeneratorProvider(const GeneratorProvider &provider);
 
+    std::optional<Utils::FilePaths> filePaths() const;
+
     static FilePathsGenerator filePathsGenerator(const Utils::FilePaths &filePaths);
     LocatorMatcherTask matcher() const;