Utils: Use callbacks when iterating remote file systems

This will make recursion easier and has the potential to avoid creating
big intermediate lists.

Change-Id: I44d42925dae9c0048338c7d0a6aa26606f314c28
Reviewed-by: Eike Ziller <eike.ziller@qt.io>
Reviewed-by: <github-actions-qt-creator@cristianadam.eu>

src/libs/utils/filepath.cpp

@@ -721,8 +721,11 @@ FilePaths FilePath::dirEntries(const QStringList &nameFilters,
                                QDir::SortFlags sort) const
 {
     if (needsDevice()) {
-        QTC_ASSERT(s_deviceHooks.dirEntries, return {});
+        QTC_ASSERT(s_deviceHooks.iterateDirectory, return {});
-        return s_deviceHooks.dirEntries(*this, nameFilters, filters, sort);
+        FilePaths result;
+        const auto callBack = [&result](const FilePath &path) { result.append(path); return true; };
+        s_deviceHooks.iterateDirectory(*this, callBack, nameFilters, filters);
+        return result;
     }
 
     const QFileInfoList entryInfoList = QDir(m_data).entryInfoList(nameFilters, filters, sort);
@@ -741,9 +744,8 @@ void FilePath::iterateDirectory(const std::function<bool(const FilePath &item)>
                                 QDirIterator::IteratorFlags flags) const
 {
     if (needsDevice()) {
-        for (const FilePath &filePath :
+        QTC_ASSERT(s_deviceHooks.iterateDirectory, return);
-             s_deviceHooks.dirEntries(*this, nameFilters, filters, QDir::NoSort))
+        s_deviceHooks.iterateDirectory(*this, callBack, nameFilters, filters);
-            if (!callBack(filePath))
         return;
     }
 

src/libs/utils/fileutils.h

@@ -80,8 +80,10 @@ public:
     std::function<FilePath(const FilePath &)> symLinkTarget;
     std::function<FilePath(const FilePath &)> mapToGlobalPath;
     std::function<QString(const FilePath &)> mapToDevicePath;
-    std::function<QList<FilePath>(const FilePath &, const QStringList &,
+    std::function<void(const FilePath &,
-                                  QDir::Filters, QDir::SortFlags)> dirEntries;
+                       const std::function<bool(const FilePath &)> &, // Abort on 'false' return.
+                       const QStringList &,
+                       QDir::Filters)> iterateDirectory;
     std::function<QByteArray(const FilePath &, qint64, qint64)> fileContents;
     std::function<bool(const FilePath &, const QByteArray &)> writeFileContents;
     std::function<QDateTime(const FilePath &)> lastModified;

src/plugins/debugger/debuggeritemmanager.cpp

@@ -32,7 +32,6 @@
 
 #include <extensionsystem/pluginmanager.h>
 
-#include <projectexplorer/devicesupport/devicemanager.h>
 #include <projectexplorer/projectexplorerconstants.h>
 #include <projectexplorer/projectexplorericons.h>
 
@@ -755,12 +754,9 @@ void DebuggerItemManagerPrivate::autoDetectGdbOrLldbDebuggers(const FilePaths &s
     if (searchPaths.isEmpty())
         return;
 
-    IDevice::ConstPtr device = DeviceManager::deviceForPath(searchPaths.front());
-    QTC_ASSERT(device, return);
-
     FilePaths suspects;
 
-    if (device->osType() == OsTypeMac) {
+    if (searchPaths.front().osType() == OsTypeMac) {
         QtcProcess proc;
         proc.setTimeoutS(2);
         proc.setCommand({"xcrun", {"--find", "lldb"}});
@@ -782,9 +778,9 @@ void DebuggerItemManagerPrivate::autoDetectGdbOrLldbDebuggers(const FilePaths &s
 
     paths = Utils::filteredUnique(paths);
 
-    for (const FilePath &path : paths) {
+    const auto addSuspect = [&suspects](const FilePath &entry) { suspects.append(entry); return true; };
-        suspects.append(device->directoryEntries(path, filters, QDir::Files | QDir::Executable));
+    for (const FilePath &path : paths)
-    }
+        path.iterateDirectory(addSuspect, filters, QDir::Files | QDir::Executable);
 
     QStringList logMessages{tr("Searching debuggers...")};
     for (const FilePath &command : qAsConst(suspects)) {

src/plugins/docker/dockerdevice.cpp

@@ -1429,11 +1429,12 @@ bool DockerDevice::setPermissions(const FilePath &filePath, QFileDevice::Permiss
     return false;
 }
 
-FilePaths DockerDevice::findFilesWithFind(const FilePath &filePath,
+void DockerDevice::iterateWithFind(const FilePath &filePath,
+                                   const std::function<bool(const Utils::FilePath &)> &callBack,
                                    const QStringList &nameFilters,
-                                          QDir::Filters filters,
+                                   QDir::Filters filters) const
-                                          QDir::SortFlags sort) const
 {
+    QTC_ASSERT(callBack, return);
     QTC_CHECK(filePath.isAbsolutePath());
     QStringList arguments{filePath.path(), "-maxdepth", "1"};
     if (filters & QDir::NoSymLinks)
@@ -1481,28 +1482,23 @@ FilePaths DockerDevice::findFilesWithFind(const FilePath &filePath,
     if (!output.isEmpty() && !output.startsWith(filePath.path())) { // missing find, unknown option
         LOG("Setting 'do not use find'" << output.left(output.indexOf('\n')));
         d->m_useFind = false;
-        return {};
+        return;
     }
 
-    QStringList entries = output.split("\n", Qt::SkipEmptyParts);
+    const QStringList entries = output.split("\n", Qt::SkipEmptyParts);
-    if (sort & QDir::Name)
+    for (const QString &entry : entries) {
-        entries.sort(filters & QDir::CaseSensitive ? Qt::CaseSensitive : Qt::CaseInsensitive);
+        if (entry.startsWith("find: "))
-    QTC_CHECK(sort == QDir::Name || sort == QDir::NoSort || sort == QDir::Unsorted);
+            continue;
-
+        if (!callBack(FilePath::fromString(entry).onDevice(filePath)))
-    // strip out find messages
+            break;
-    entries = Utils::filtered(entries,
+    }
-                              [](const QString &entry) { return !entry.startsWith("find: "); });
-    FilePaths result;
-    for (const QString &entry : qAsConst(entries))
-        result.append(FilePath::fromString(entry).onDevice(filePath));
-    return result;
 }
 
-static FilePaths filterEntriesHelper(const FilePath &base,
+static void filterEntriesHelper(const FilePath &base,
+                                const std::function<bool(const FilePath &)> &callBack,
                                 const QStringList &entries,
                                 const QStringList &nameFilters,
-                                     QDir::Filters filters,
+                                QDir::Filters filters)
-                                     QDir::SortFlags sort)
 {
     const QList<QRegularExpression> nameRegexps = transform(nameFilters, [](const QString &filter) {
         QRegularExpression re;
@@ -1520,43 +1516,46 @@ static FilePaths filterEntriesHelper(const FilePath &base,
         return nameRegexps.isEmpty();
     };
 
-    // FIXME: Handle sort and filters. For now bark on unsupported options.
+    // FIXME: Handle filters. For now bark on unsupported options.
     QTC_CHECK(filters == QDir::NoFilter);
-    QTC_CHECK(sort == QDir::NoSort);
 
     FilePaths result;
     for (const QString &entry : entries) {
         if (!nameMatches(entry))
             continue;
-        result.append(base.pathAppended(entry));
+        if (!callBack(base.pathAppended(entry)))
+            break;
     }
-    return result;
 }
 
-FilePaths DockerDevice::directoryEntries(const FilePath &filePath,
+void DockerDevice::iterateDirectory(const FilePath &filePath,
+                                    const std::function<bool(const FilePath &)> &callBack,
                                     const QStringList &nameFilters,
-                                         QDir::Filters filters,
+                                    QDir::Filters filters) const
-                                         QDir::SortFlags sort) const
 {
-    QTC_ASSERT(handlesFile(filePath), return {});
+    QTC_ASSERT(handlesFile(filePath), return);
     tryCreateLocalFileAccess();
     if (hasLocalFileAccess()) {
-        const FilePaths entries = mapToLocalAccess(filePath).dirEntries(nameFilters, filters, sort);
+        const FilePath local = mapToLocalAccess(filePath);
-        return Utils::transform(entries, [this](const FilePath &entry) {
+        local.iterateDirectory([&callBack, this](const FilePath &entry) {
-            return mapFromLocalAccess(entry);
+                                    return callBack(mapFromLocalAccess(entry));
-        });
+                               },
+                               nameFilters, filters);
+        return;
     }
 
     if (d->m_useFind) {
-        const FilePaths result = findFilesWithFind(filePath, nameFilters, filters, sort);
+        iterateWithFind(filePath, callBack, nameFilters, filters);
+        // d->m_useFind will be set to false if 'find' is not found. In this
+        // case fall back to 'ls' below.
         if (d->m_useFind)
-            return result;
+            return;
     }
 
     // if we do not have find - use ls as fallback
     const QString output = d->outputForRunInShell({"ls", {"-1", "-b", "--", filePath.path()}});
-    QStringList entries = output.split('\n', Qt::SkipEmptyParts);
+    const QStringList entries = output.split('\n', Qt::SkipEmptyParts);
-    return filterEntriesHelper(filePath, entries, nameFilters, filters, sort);
+    filterEntriesHelper(filePath, callBack, entries, nameFilters, filters);
 }
 
 QByteArray DockerDevice::fileContents(const FilePath &filePath, qint64 limit, qint64 offset) const

src/plugins/docker/dockerdevice.h

@@ -94,10 +94,10 @@ public:
     bool copyFile(const Utils::FilePath &filePath, const Utils::FilePath &target) const override;
     bool renameFile(const Utils::FilePath &filePath, const Utils::FilePath &target) const override;
     Utils::FilePath symLinkTarget(const Utils::FilePath &filePath) const override;
-    QList<Utils::FilePath> directoryEntries(const Utils::FilePath &filePath,
+    void iterateDirectory(const Utils::FilePath &filePath,
+                          const std::function<bool(const Utils::FilePath &)> &callBack,
                           const QStringList &nameFilters,
-                                            QDir::Filters filters,
+                          QDir::Filters filters) const override;
-                                            QDir::SortFlags sort) const override;
     QByteArray fileContents(const Utils::FilePath &filePath, qint64 limit, qint64 offset) const override;
     bool writeFileContents(const Utils::FilePath &filePath, const QByteArray &data) const override;
     QDateTime lastModified(const Utils::FilePath &filePath) const override;
@@ -124,10 +124,10 @@ protected:
     QVariantMap toMap() const final;
 
 private:
-    Utils::FilePaths findFilesWithFind(const Utils::FilePath &filePath,
+    void iterateWithFind(const Utils::FilePath &filePath,
+                         const std::function<bool(const Utils::FilePath &)> &callBack,
                          const QStringList &nameFilters,
-                                       QDir::Filters filters,
+                         QDir::Filters filters) const;
-                                       QDir::SortFlags sort) const;
 
     void aboutToBeRemoved() const final;
 

src/plugins/projectexplorer/devicesupport/desktopdevice.cpp

@@ -176,15 +176,13 @@ bool DesktopDevice::handlesFile(const FilePath &filePath) const
     return !filePath.needsDevice();
 }
 
-QList<FilePath> DesktopDevice::directoryEntries(const FilePath &filePath,
+void DesktopDevice::iterateDirectory(const FilePath &filePath,
+                                     const std::function<bool(const Utils::FilePath &)> &callBack,
                                      const QStringList &nameFilters,
-                                                QDir::Filters filters,
+                                     QDir::Filters filters) const
-                                                QDir::SortFlags sort) const
 {
     QTC_CHECK(!filePath.needsDevice());
-    const QDir dir(filePath.path());
+    filePath.iterateDirectory(callBack, nameFilters, filters);
-    const QFileInfoList entryInfoList = dir.entryInfoList(nameFilters, filters, sort);
-    return Utils::transform(entryInfoList, &FilePath::fromFileInfo);
 }
 
 qint64 DesktopDevice::fileSize(const FilePath &filePath) const

src/plugins/projectexplorer/devicesupport/desktopdevice.h

@@ -56,10 +56,6 @@ public:
     QUrl toolControlChannel(const ControlChannelHint &) const override;
 
     bool handlesFile(const Utils::FilePath &filePath) const override;
-    QList<Utils::FilePath> directoryEntries(const Utils::FilePath &filePath,
-                                            const QStringList &nameFilters,
-                                            QDir::Filters filters,
-                                            QDir::SortFlags sort) const override;
     Utils::Environment systemEnvironment() const override;
     bool isExecutableFile(const Utils::FilePath &filePath) const override;
     bool isReadableFile(const Utils::FilePath &filePath) const override;
@@ -77,6 +73,10 @@ public:
     bool renameFile(const Utils::FilePath &filePath, const Utils::FilePath &target) const override;
     QDateTime lastModified(const Utils::FilePath &filePath) const override;
     Utils::FilePath symLinkTarget(const Utils::FilePath &filePath) const override;
+    void iterateDirectory(const Utils::FilePath &filePath,
+                          const std::function<bool(const Utils::FilePath &)> &callBack,
+                          const QStringList &nameFilters,
+                          QDir::Filters filters) const override;
     QByteArray fileContents(const Utils::FilePath &filePath, qint64 limit, qint64 offset) const override;
     bool writeFileContents(const Utils::FilePath &filePath, const QByteArray &data) const override;
     qint64 fileSize(const Utils::FilePath &filePath) const override;

src/plugins/projectexplorer/devicesupport/devicemanager.cpp

@@ -496,11 +496,13 @@ DeviceManager::DeviceManager(bool isInstance) : d(std::make_unique<DeviceManager
         return device->mapToDevicePath(filePath);
     };
 
-    deviceHooks.dirEntries = [](const FilePath &filePath, const QStringList &nameFilters,
+    deviceHooks.iterateDirectory = [](const FilePath &filePath,
-                                QDir::Filters filters, QDir::SortFlags sort) {
+                                      const std::function<bool(const FilePath &)> &callBack,
+                                      const QStringList &nameFilters,
+                                      QDir::Filters filters) {
         auto device = DeviceManager::deviceForPath(filePath);
-        QTC_ASSERT(device, return FilePaths());
+        QTC_ASSERT(device, return);
-        return device->directoryEntries(filePath, nameFilters, filters, sort);
+        device->iterateDirectory(filePath, callBack, nameFilters, filters);
     };
 
     deviceHooks.fileContents = [](const FilePath &filePath, qint64 maxSize, qint64 offset) {

src/plugins/projectexplorer/devicesupport/idevice.cpp

@@ -361,17 +361,16 @@ FilePath IDevice::symLinkTarget(const FilePath &filePath) const
     return {};
 }
 
-QList<FilePath> IDevice::directoryEntries(const FilePath &filePath,
+void IDevice::iterateDirectory(const FilePath &filePath,
+                               const std::function<bool(const FilePath &)> &callBack,
                                const QStringList &nameFilters,
-                                          QDir::Filters filters,
+                               QDir::Filters filters) const
-                                          QDir::SortFlags sort) const
 {
     Q_UNUSED(filePath);
+    Q_UNUSED(callBack);
     Q_UNUSED(nameFilters);
     Q_UNUSED(filters);
-    Q_UNUSED(sort);
     QTC_CHECK(false);
-    return {};
 }
 
 QByteArray IDevice::fileContents(const FilePath &filePath, qint64 limit, qint64 offset) const

src/plugins/projectexplorer/devicesupport/idevice.h

@@ -259,10 +259,10 @@ public:
     virtual Utils::FilePath searchExecutable(const QString &fileName,
                                              const QList<Utils::FilePath> &dirs) const;
     virtual Utils::FilePath symLinkTarget(const Utils::FilePath &filePath) const;
-    virtual QList<Utils::FilePath> directoryEntries(const Utils::FilePath &filePath,
+    virtual void iterateDirectory(const Utils::FilePath &filePath,
+                                  const std::function<bool(const Utils::FilePath &)> &callBack,
                                   const QStringList &nameFilters,
-                                                    QDir::Filters filters,
+                                  QDir::Filters filters) const;
-                                                    QDir::SortFlags sort = QDir::NoSort) const;
     virtual QByteArray fileContents(const Utils::FilePath &filePath,
                                     qint64 limit,
                                     qint64 offset) const;

src/plugins/projectexplorer/gcctoolchain.cpp

@@ -1062,15 +1062,15 @@ static FilePaths findCompilerCandidates(const IDevice::Ptr &device,
         FilePaths searchPaths = device->systemEnvironment().path();
         for (const FilePath &deviceDir : qAsConst(searchPaths)) {
             static const QRegularExpression regexp(binaryRegexp);
+            const auto callBack = [&compilerPaths, compilerName](const FilePath &candidate) {
+                if (candidate.fileName() == compilerName)
+                    compilerPaths << candidate;
+                else if (regexp.match(candidate.path()).hasMatch())
+                    compilerPaths << candidate;
+                return true;
+            };
             const FilePath globalDir = device->mapToGlobalPath(deviceDir);
-            const FilePaths fileNames = device->directoryEntries(globalDir, nameFilters,
+            device->iterateDirectory(globalDir, callBack, nameFilters, QDir::Files | QDir::Executable);
-                                                                 QDir::Files | QDir::Executable);
-            for (const FilePath &fileName : fileNames) {
-                if (fileName.fileName() == compilerName)
-                    compilerPaths << fileName;
-                else if (regexp.match(fileName.path()).hasMatch())
-                    compilerPaths << fileName;
-            }
         }
     } else {
         // The normal, local host case.