From 7677dc4ba0bd52d65596486a8becda74908203d5 Mon Sep 17 00:00:00 2001 From: hjk Date: Tue, 14 Dec 2021 12:15:40 +0100 Subject: [PATCH] Utils: Use callbacks when iterating remote file systems This will make recursion easier and has the potential to avoid creating big intermediate lists. Change-Id: I44d42925dae9c0048338c7d0a6aa26606f314c28 Reviewed-by: Eike Ziller Reviewed-by: --- src/libs/utils/filepath.cpp | 14 ++-- src/libs/utils/fileutils.h | 6 +- src/plugins/debugger/debuggeritemmanager.cpp | 12 +-- src/plugins/docker/dockerdevice.cpp | 77 +++++++++---------- src/plugins/docker/dockerdevice.h | 16 ++-- .../devicesupport/desktopdevice.cpp | 12 ++- .../devicesupport/desktopdevice.h | 8 +- .../devicesupport/devicemanager.cpp | 10 ++- .../projectexplorer/devicesupport/idevice.cpp | 11 ++- .../projectexplorer/devicesupport/idevice.h | 8 +- src/plugins/projectexplorer/gcctoolchain.cpp | 16 ++-- 11 files changed, 94 insertions(+), 96 deletions(-) diff --git a/src/libs/utils/filepath.cpp b/src/libs/utils/filepath.cpp index 57faa92c825..1e8d0eed8dc 100644 --- a/src/libs/utils/filepath.cpp +++ b/src/libs/utils/filepath.cpp @@ -721,8 +721,11 @@ FilePaths FilePath::dirEntries(const QStringList &nameFilters, QDir::SortFlags sort) const { if (needsDevice()) { - QTC_ASSERT(s_deviceHooks.dirEntries, return {}); - return s_deviceHooks.dirEntries(*this, nameFilters, filters, sort); + QTC_ASSERT(s_deviceHooks.iterateDirectory, return {}); + FilePaths result; + const auto callBack = [&result](const FilePath &path) { result.append(path); return true; }; + s_deviceHooks.iterateDirectory(*this, callBack, nameFilters, filters); + return result; } const QFileInfoList entryInfoList = QDir(m_data).entryInfoList(nameFilters, filters, sort); @@ -741,10 +744,9 @@ void FilePath::iterateDirectory(const std::function QDirIterator::IteratorFlags flags) const { if (needsDevice()) { - for (const FilePath &filePath : - s_deviceHooks.dirEntries(*this, nameFilters, filters, QDir::NoSort)) - if (!callBack(filePath)) - return; + QTC_ASSERT(s_deviceHooks.iterateDirectory, return); + s_deviceHooks.iterateDirectory(*this, callBack, nameFilters, filters); + return; } QDirIterator it(m_data, nameFilters, filters, flags); diff --git a/src/libs/utils/fileutils.h b/src/libs/utils/fileutils.h index c87ca5d5dd0..bcdba3e7761 100644 --- a/src/libs/utils/fileutils.h +++ b/src/libs/utils/fileutils.h @@ -80,8 +80,10 @@ public: std::function symLinkTarget; std::function mapToGlobalPath; std::function mapToDevicePath; - std::function(const FilePath &, const QStringList &, - QDir::Filters, QDir::SortFlags)> dirEntries; + std::function &, // Abort on 'false' return. + const QStringList &, + QDir::Filters)> iterateDirectory; std::function fileContents; std::function writeFileContents; std::function lastModified; diff --git a/src/plugins/debugger/debuggeritemmanager.cpp b/src/plugins/debugger/debuggeritemmanager.cpp index 4c6a482e2c2..1b1e198d562 100644 --- a/src/plugins/debugger/debuggeritemmanager.cpp +++ b/src/plugins/debugger/debuggeritemmanager.cpp @@ -32,7 +32,6 @@ #include -#include #include #include @@ -755,12 +754,9 @@ void DebuggerItemManagerPrivate::autoDetectGdbOrLldbDebuggers(const FilePaths &s if (searchPaths.isEmpty()) return; - IDevice::ConstPtr device = DeviceManager::deviceForPath(searchPaths.front()); - QTC_ASSERT(device, return); - FilePaths suspects; - if (device->osType() == OsTypeMac) { + if (searchPaths.front().osType() == OsTypeMac) { QtcProcess proc; proc.setTimeoutS(2); proc.setCommand({"xcrun", {"--find", "lldb"}}); @@ -782,9 +778,9 @@ void DebuggerItemManagerPrivate::autoDetectGdbOrLldbDebuggers(const FilePaths &s paths = Utils::filteredUnique(paths); - for (const FilePath &path : paths) { - suspects.append(device->directoryEntries(path, filters, QDir::Files | QDir::Executable)); - } + const auto addSuspect = [&suspects](const FilePath &entry) { suspects.append(entry); return true; }; + for (const FilePath &path : paths) + path.iterateDirectory(addSuspect, filters, QDir::Files | QDir::Executable); QStringList logMessages{tr("Searching debuggers...")}; for (const FilePath &command : qAsConst(suspects)) { diff --git a/src/plugins/docker/dockerdevice.cpp b/src/plugins/docker/dockerdevice.cpp index d5fda74e024..3bab1ef594e 100644 --- a/src/plugins/docker/dockerdevice.cpp +++ b/src/plugins/docker/dockerdevice.cpp @@ -1429,11 +1429,12 @@ bool DockerDevice::setPermissions(const FilePath &filePath, QFileDevice::Permiss return false; } -FilePaths DockerDevice::findFilesWithFind(const FilePath &filePath, - const QStringList &nameFilters, - QDir::Filters filters, - QDir::SortFlags sort) const +void DockerDevice::iterateWithFind(const FilePath &filePath, + const std::function &callBack, + const QStringList &nameFilters, + QDir::Filters filters) const { + QTC_ASSERT(callBack, return); QTC_CHECK(filePath.isAbsolutePath()); QStringList arguments{filePath.path(), "-maxdepth", "1"}; if (filters & QDir::NoSymLinks) @@ -1481,28 +1482,23 @@ FilePaths DockerDevice::findFilesWithFind(const FilePath &filePath, if (!output.isEmpty() && !output.startsWith(filePath.path())) { // missing find, unknown option LOG("Setting 'do not use find'" << output.left(output.indexOf('\n'))); d->m_useFind = false; - return {}; + return; } - QStringList entries = output.split("\n", Qt::SkipEmptyParts); - if (sort & QDir::Name) - entries.sort(filters & QDir::CaseSensitive ? Qt::CaseSensitive : Qt::CaseInsensitive); - QTC_CHECK(sort == QDir::Name || sort == QDir::NoSort || sort == QDir::Unsorted); - - // strip out find messages - entries = Utils::filtered(entries, - [](const QString &entry) { return !entry.startsWith("find: "); }); - FilePaths result; - for (const QString &entry : qAsConst(entries)) - result.append(FilePath::fromString(entry).onDevice(filePath)); - return result; + const QStringList entries = output.split("\n", Qt::SkipEmptyParts); + for (const QString &entry : entries) { + if (entry.startsWith("find: ")) + continue; + if (!callBack(FilePath::fromString(entry).onDevice(filePath))) + break; + } } -static FilePaths filterEntriesHelper(const FilePath &base, - const QStringList &entries, - const QStringList &nameFilters, - QDir::Filters filters, - QDir::SortFlags sort) +static void filterEntriesHelper(const FilePath &base, + const std::function &callBack, + const QStringList &entries, + const QStringList &nameFilters, + QDir::Filters filters) { const QList nameRegexps = transform(nameFilters, [](const QString &filter) { QRegularExpression re; @@ -1520,43 +1516,46 @@ static FilePaths filterEntriesHelper(const FilePath &base, return nameRegexps.isEmpty(); }; - // FIXME: Handle sort and filters. For now bark on unsupported options. + // FIXME: Handle filters. For now bark on unsupported options. QTC_CHECK(filters == QDir::NoFilter); - QTC_CHECK(sort == QDir::NoSort); FilePaths result; for (const QString &entry : entries) { if (!nameMatches(entry)) continue; - result.append(base.pathAppended(entry)); + if (!callBack(base.pathAppended(entry))) + break; } - return result; } -FilePaths DockerDevice::directoryEntries(const FilePath &filePath, - const QStringList &nameFilters, - QDir::Filters filters, - QDir::SortFlags sort) const +void DockerDevice::iterateDirectory(const FilePath &filePath, + const std::function &callBack, + const QStringList &nameFilters, + QDir::Filters filters) const { - QTC_ASSERT(handlesFile(filePath), return {}); + QTC_ASSERT(handlesFile(filePath), return); tryCreateLocalFileAccess(); if (hasLocalFileAccess()) { - const FilePaths entries = mapToLocalAccess(filePath).dirEntries(nameFilters, filters, sort); - return Utils::transform(entries, [this](const FilePath &entry) { - return mapFromLocalAccess(entry); - }); + const FilePath local = mapToLocalAccess(filePath); + local.iterateDirectory([&callBack, this](const FilePath &entry) { + return callBack(mapFromLocalAccess(entry)); + }, + nameFilters, filters); + return; } if (d->m_useFind) { - const FilePaths result = findFilesWithFind(filePath, nameFilters, filters, sort); + iterateWithFind(filePath, callBack, nameFilters, filters); + // d->m_useFind will be set to false if 'find' is not found. In this + // case fall back to 'ls' below. if (d->m_useFind) - return result; + return; } // if we do not have find - use ls as fallback const QString output = d->outputForRunInShell({"ls", {"-1", "-b", "--", filePath.path()}}); - QStringList entries = output.split('\n', Qt::SkipEmptyParts); - return filterEntriesHelper(filePath, entries, nameFilters, filters, sort); + const QStringList entries = output.split('\n', Qt::SkipEmptyParts); + filterEntriesHelper(filePath, callBack, entries, nameFilters, filters); } QByteArray DockerDevice::fileContents(const FilePath &filePath, qint64 limit, qint64 offset) const diff --git a/src/plugins/docker/dockerdevice.h b/src/plugins/docker/dockerdevice.h index 3a6bff1a59b..dea644ae24e 100644 --- a/src/plugins/docker/dockerdevice.h +++ b/src/plugins/docker/dockerdevice.h @@ -94,10 +94,10 @@ public: bool copyFile(const Utils::FilePath &filePath, const Utils::FilePath &target) const override; bool renameFile(const Utils::FilePath &filePath, const Utils::FilePath &target) const override; Utils::FilePath symLinkTarget(const Utils::FilePath &filePath) const override; - QList directoryEntries(const Utils::FilePath &filePath, - const QStringList &nameFilters, - QDir::Filters filters, - QDir::SortFlags sort) const override; + void iterateDirectory(const Utils::FilePath &filePath, + const std::function &callBack, + const QStringList &nameFilters, + QDir::Filters filters) const override; QByteArray fileContents(const Utils::FilePath &filePath, qint64 limit, qint64 offset) const override; bool writeFileContents(const Utils::FilePath &filePath, const QByteArray &data) const override; QDateTime lastModified(const Utils::FilePath &filePath) const override; @@ -124,10 +124,10 @@ protected: QVariantMap toMap() const final; private: - Utils::FilePaths findFilesWithFind(const Utils::FilePath &filePath, - const QStringList &nameFilters, - QDir::Filters filters, - QDir::SortFlags sort) const; + void iterateWithFind(const Utils::FilePath &filePath, + const std::function &callBack, + const QStringList &nameFilters, + QDir::Filters filters) const; void aboutToBeRemoved() const final; diff --git a/src/plugins/projectexplorer/devicesupport/desktopdevice.cpp b/src/plugins/projectexplorer/devicesupport/desktopdevice.cpp index 8553f932f46..c97e00f6e0e 100644 --- a/src/plugins/projectexplorer/devicesupport/desktopdevice.cpp +++ b/src/plugins/projectexplorer/devicesupport/desktopdevice.cpp @@ -176,15 +176,13 @@ bool DesktopDevice::handlesFile(const FilePath &filePath) const return !filePath.needsDevice(); } -QList DesktopDevice::directoryEntries(const FilePath &filePath, - const QStringList &nameFilters, - QDir::Filters filters, - QDir::SortFlags sort) const +void DesktopDevice::iterateDirectory(const FilePath &filePath, + const std::function &callBack, + const QStringList &nameFilters, + QDir::Filters filters) const { QTC_CHECK(!filePath.needsDevice()); - const QDir dir(filePath.path()); - const QFileInfoList entryInfoList = dir.entryInfoList(nameFilters, filters, sort); - return Utils::transform(entryInfoList, &FilePath::fromFileInfo); + filePath.iterateDirectory(callBack, nameFilters, filters); } qint64 DesktopDevice::fileSize(const FilePath &filePath) const diff --git a/src/plugins/projectexplorer/devicesupport/desktopdevice.h b/src/plugins/projectexplorer/devicesupport/desktopdevice.h index 1e0b59d7daf..9e5fed0211f 100644 --- a/src/plugins/projectexplorer/devicesupport/desktopdevice.h +++ b/src/plugins/projectexplorer/devicesupport/desktopdevice.h @@ -56,10 +56,6 @@ public: QUrl toolControlChannel(const ControlChannelHint &) const override; bool handlesFile(const Utils::FilePath &filePath) const override; - QList directoryEntries(const Utils::FilePath &filePath, - const QStringList &nameFilters, - QDir::Filters filters, - QDir::SortFlags sort) const override; Utils::Environment systemEnvironment() const override; bool isExecutableFile(const Utils::FilePath &filePath) const override; bool isReadableFile(const Utils::FilePath &filePath) const override; @@ -77,6 +73,10 @@ public: bool renameFile(const Utils::FilePath &filePath, const Utils::FilePath &target) const override; QDateTime lastModified(const Utils::FilePath &filePath) const override; Utils::FilePath symLinkTarget(const Utils::FilePath &filePath) const override; + void iterateDirectory(const Utils::FilePath &filePath, + const std::function &callBack, + const QStringList &nameFilters, + QDir::Filters filters) const override; QByteArray fileContents(const Utils::FilePath &filePath, qint64 limit, qint64 offset) const override; bool writeFileContents(const Utils::FilePath &filePath, const QByteArray &data) const override; qint64 fileSize(const Utils::FilePath &filePath) const override; diff --git a/src/plugins/projectexplorer/devicesupport/devicemanager.cpp b/src/plugins/projectexplorer/devicesupport/devicemanager.cpp index 1e671e5fe31..5b1b5322a8c 100644 --- a/src/plugins/projectexplorer/devicesupport/devicemanager.cpp +++ b/src/plugins/projectexplorer/devicesupport/devicemanager.cpp @@ -496,11 +496,13 @@ DeviceManager::DeviceManager(bool isInstance) : d(std::make_uniquemapToDevicePath(filePath); }; - deviceHooks.dirEntries = [](const FilePath &filePath, const QStringList &nameFilters, - QDir::Filters filters, QDir::SortFlags sort) { + deviceHooks.iterateDirectory = [](const FilePath &filePath, + const std::function &callBack, + const QStringList &nameFilters, + QDir::Filters filters) { auto device = DeviceManager::deviceForPath(filePath); - QTC_ASSERT(device, return FilePaths()); - return device->directoryEntries(filePath, nameFilters, filters, sort); + QTC_ASSERT(device, return); + device->iterateDirectory(filePath, callBack, nameFilters, filters); }; deviceHooks.fileContents = [](const FilePath &filePath, qint64 maxSize, qint64 offset) { diff --git a/src/plugins/projectexplorer/devicesupport/idevice.cpp b/src/plugins/projectexplorer/devicesupport/idevice.cpp index 746890a6cae..c0d877c8ed7 100644 --- a/src/plugins/projectexplorer/devicesupport/idevice.cpp +++ b/src/plugins/projectexplorer/devicesupport/idevice.cpp @@ -361,17 +361,16 @@ FilePath IDevice::symLinkTarget(const FilePath &filePath) const return {}; } -QList IDevice::directoryEntries(const FilePath &filePath, - const QStringList &nameFilters, - QDir::Filters filters, - QDir::SortFlags sort) const +void IDevice::iterateDirectory(const FilePath &filePath, + const std::function &callBack, + const QStringList &nameFilters, + QDir::Filters filters) const { Q_UNUSED(filePath); + Q_UNUSED(callBack); Q_UNUSED(nameFilters); Q_UNUSED(filters); - Q_UNUSED(sort); QTC_CHECK(false); - return {}; } QByteArray IDevice::fileContents(const FilePath &filePath, qint64 limit, qint64 offset) const diff --git a/src/plugins/projectexplorer/devicesupport/idevice.h b/src/plugins/projectexplorer/devicesupport/idevice.h index c36091b5543..b8e5765b493 100644 --- a/src/plugins/projectexplorer/devicesupport/idevice.h +++ b/src/plugins/projectexplorer/devicesupport/idevice.h @@ -259,10 +259,10 @@ public: virtual Utils::FilePath searchExecutable(const QString &fileName, const QList &dirs) const; virtual Utils::FilePath symLinkTarget(const Utils::FilePath &filePath) const; - virtual QList directoryEntries(const Utils::FilePath &filePath, - const QStringList &nameFilters, - QDir::Filters filters, - QDir::SortFlags sort = QDir::NoSort) const; + virtual void iterateDirectory(const Utils::FilePath &filePath, + const std::function &callBack, + const QStringList &nameFilters, + QDir::Filters filters) const; virtual QByteArray fileContents(const Utils::FilePath &filePath, qint64 limit, qint64 offset) const; diff --git a/src/plugins/projectexplorer/gcctoolchain.cpp b/src/plugins/projectexplorer/gcctoolchain.cpp index ea45d93ec59..886dfbed9cb 100644 --- a/src/plugins/projectexplorer/gcctoolchain.cpp +++ b/src/plugins/projectexplorer/gcctoolchain.cpp @@ -1062,15 +1062,15 @@ static FilePaths findCompilerCandidates(const IDevice::Ptr &device, FilePaths searchPaths = device->systemEnvironment().path(); for (const FilePath &deviceDir : qAsConst(searchPaths)) { static const QRegularExpression regexp(binaryRegexp); + const auto callBack = [&compilerPaths, compilerName](const FilePath &candidate) { + if (candidate.fileName() == compilerName) + compilerPaths << candidate; + else if (regexp.match(candidate.path()).hasMatch()) + compilerPaths << candidate; + return true; + }; const FilePath globalDir = device->mapToGlobalPath(deviceDir); - const FilePaths fileNames = device->directoryEntries(globalDir, nameFilters, - QDir::Files | QDir::Executable); - for (const FilePath &fileName : fileNames) { - if (fileName.fileName() == compilerName) - compilerPaths << fileName; - else if (regexp.match(fileName.path()).hasMatch()) - compilerPaths << fileName; - } + device->iterateDirectory(globalDir, callBack, nameFilters, QDir::Files | QDir::Executable); } } else { // The normal, local host case.