feedc0de/qt-creator
1
0
Fork 0
forked from qt-creator/qt-creator
Code Pull Requests Activity

ProjectExplorer: use multiple threads to scan for files recursively

Browse Source 
Change-Id: I845d2e2eaffd8d6a4e85d3186435d57846789506
Reviewed-by: Eike Ziller <eike.ziller@qt.io>
Reviewed-by: Jarek Kobus <jaroslaw.kobus@qt.io>
This commit is contained in:
David Schulz
2024-05-30 10:19:10 +02:00
parent caa5c3947f
commit c5564559cc
1 changed files with 88 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

125
src/plugins/projectexplorer/projectnodeshelper.h
View File

@@ -8,67 +8,121 @@
#include <coreplugin/iversioncontrol.h>
#include <coreplugin/vcsmanager.h>
#include <solutions/tasking/tasktreerunner.h>
#include <utils/algorithm.h>
#include <utils/async.h>
#include <utils/filepath.h>
#include <utils/hostosinfo.h>
#include <QPromise>
namespace ProjectExplorer {
namespace Internal {
struct DirectoryScanResult
{
QList<FileNode *> nodes;
Utils::FilePaths subDirectories;
};
template<typename Result>
QList<FileNode *> scanForFilesRecursively(
DirectoryScanResult scanForFiles(
QPromise<Result> &promise,
double progressStart,
double progressRange,
const Utils::FilePath &directory,
const QDir::Filters &filter,
const std::function<FileNode *(const Utils::FilePath &)> factory,
QSet<Utils::FilePath> &visited,
const QList<Core::IVersionControl *> &versionControls)
{
QList<FileNode *> result;
// Do not follow directory loops:
if (!Utils::insert(visited, directory.canonicalPath()))
return result;
DirectoryScanResult result;
const Utils::FilePaths entries = directory.dirEntries(filter);
double progress = 0;
const double progressIncrement = progressRange / static_cast<double>(entries.count());
int lastIntProgress = 0;
for (const Utils::FilePath &entry : entries) {
if (promise.isCanceled())
return result;
if (!Utils::contains(versionControls, [entry](const Core::IVersionControl *vc) {
if (Utils::anyOf(versionControls, [entry](const Core::IVersionControl *vc) {
return vc->isVcsFileOrDirectory(entry);
})) {
if (entry.isDir()) {
result.append(scanForFilesRecursively(promise,
progress,
progressIncrement,
entry,
filter,
factory,
visited,
versionControls));
} else if (FileNode *node = factory(entry)) {
result.append(node);
continue;
}
if (entry.isDir())
result.subDirectories.append(entry);
else if (FileNode *node = factory(entry))
result.nodes.append(node);
}
progress += progressIncrement;
const int intProgress = std::min(static_cast<int>(progressStart + progress),
promise.future().progressMaximum());
if (lastIntProgress < intProgress) {
promise.setProgressValue(intProgress);
lastIntProgress = intProgress;
}
}
promise.setProgressValue(std::min(static_cast<int>(progressStart + progressRange),
promise.future().progressMaximum()));
return result;
}
template<typename Result>
QList<FileNode *> scanForFilesRecursively(
QPromise<Result> &promise,
double progressRange,
const Utils::FilePath &directory,
const QDir::Filters &filter,
const std::function<FileNode *(const Utils::FilePath &)> factory,
const QList<Core::IVersionControl *> &versionControls)
{
QSet<Utils::FilePath> visited;
const DirectoryScanResult result
= scanForFiles(promise, directory, filter, factory, versionControls);
QList<FileNode *> fileNodes = result.nodes;
const double progressIncrement = progressRange
/ static_cast<double>(
fileNodes.count() + result.subDirectories.count());
promise.setProgressValue(fileNodes.count() * progressIncrement);
QList<QPair<Utils::FilePath, int>> subDirectories;
auto addSubDirectories = [&](const Utils::FilePaths &subdirs, int progressIncrement) {
for (const Utils::FilePath &subdir : subdirs) {
if (Utils::insert(visited, subdir.canonicalPath()))
subDirectories.append(qMakePair(subdir, progressIncrement));
else
promise.setProgressValue(promise.future().progressValue() + progressIncrement);
}
};
addSubDirectories(result.subDirectories, progressIncrement);
while (!subDirectories.isEmpty()) {
using namespace Tasking;
LoopList iterator(subDirectories);
subDirectories.clear();
auto onSetup = [&, iterator](Utils::Async<DirectoryScanResult> &task) {
task.setConcurrentCallData(
[&filter, &factory, &promise, &versionControls, subdir = iterator->first](
QPromise<DirectoryScanResult> &p) {
p.addResult(scanForFiles(promise, subdir, filter, factory, versionControls));
});
};
auto onDone = [&, iterator](const Utils::Async<DirectoryScanResult> &task) {
const int progressRange = iterator->second;
const DirectoryScanResult result = task.result();
fileNodes.append(result.nodes);
const int subDirCount = result.subDirectories.count();
if (subDirCount == 0) {
promise.setProgressValue(promise.future().progressValue() + progressRange);
} else {
const int fileCount = result.nodes.count();
const int increment = progressRange / static_cast<double>(fileCount + subDirCount);
promise.setProgressValue(
promise.future().progressValue() + increment * fileCount);
addSubDirectories(result.subDirectories, increment);
}
};
const Group group{
Utils::HostOsInfo::isLinuxHost() ? parallelLimit(2) : parallelIdealThreadCountLimit,
iterator,
Utils::AsyncTask<DirectoryScanResult>(onSetup, onDone)
};
TaskTree::runBlocking(group);
}
return fileNodes;
}
} // namespace Internal
template<typename Result>
@@ -78,15 +132,12 @@ QList<FileNode *> scanForFiles(
const QDir::Filters &filter,
const std::function<FileNode *(const Utils::FilePath &)> factory)
{
QSet<Utils::FilePath> visited;
promise.setProgressRange(0, 1000000);
return Internal::scanForFilesRecursively(promise,
0.0,
1000000.0,
directory,
filter,
factory,
visited,
Core::VcsManager::versionControls());
}