ProjectExplorer: use multiple threads to scan for files recursively

Change-Id: I845d2e2eaffd8d6a4e85d3186435d57846789506
Reviewed-by: Eike Ziller <eike.ziller@qt.io>
Reviewed-by: Jarek Kobus <jaroslaw.kobus@qt.io>

src/plugins/projectexplorer/projectnodeshelper.h

@@ -8,67 +8,121 @@
 #include <coreplugin/iversioncontrol.h>
 #include <coreplugin/vcsmanager.h>
 
+#include <solutions/tasking/tasktreerunner.h>
+
 #include <utils/algorithm.h>
+#include <utils/async.h>
 #include <utils/filepath.h>
+#include <utils/hostosinfo.h>
 
 #include <QPromise>
 
 namespace ProjectExplorer {
 namespace Internal {
 
+struct DirectoryScanResult
+{
+    QList<FileNode *> nodes;
+    Utils::FilePaths subDirectories;
+};
+
 template<typename Result>
-QList<FileNode *> scanForFilesRecursively(
+DirectoryScanResult scanForFiles(
     QPromise<Result> &promise,
-    double progressStart,
-    double progressRange,
     const Utils::FilePath &directory,
     const QDir::Filters &filter,
     const std::function<FileNode *(const Utils::FilePath &)> factory,
-    QSet<Utils::FilePath> &visited,
     const QList<Core::IVersionControl *> &versionControls)
 {
-    QList<FileNode *> result;
+    DirectoryScanResult result;
-
-    // Do not follow directory loops:
-    if (!Utils::insert(visited, directory.canonicalPath()))
-        return result;
 
     const Utils::FilePaths entries = directory.dirEntries(filter);
-    double progress = 0;
-    const double progressIncrement = progressRange / static_cast<double>(entries.count());
-    int lastIntProgress = 0;
     for (const Utils::FilePath &entry : entries) {
         if (promise.isCanceled())
             return result;
 
-        if (!Utils::contains(versionControls, [entry](const Core::IVersionControl *vc) {
+        if (Utils::anyOf(versionControls, [entry](const Core::IVersionControl *vc) {
                 return vc->isVcsFileOrDirectory(entry);
             })) {
-            if (entry.isDir()) {
+            continue;
-                result.append(scanForFilesRecursively(promise,
-                                                      progress,
-                                                      progressIncrement,
-                                                      entry,
-                                                      filter,
-                                                      factory,
-                                                      visited,
-                                                      versionControls));
-            } else if (FileNode *node = factory(entry)) {
-                result.append(node);
         }
+
+        if (entry.isDir())
+            result.subDirectories.append(entry);
+        else if (FileNode *node = factory(entry))
+            result.nodes.append(node);
     }
-        progress += progressIncrement;
+
-        const int intProgress = std::min(static_cast<int>(progressStart + progress),
-                                         promise.future().progressMaximum());
-        if (lastIntProgress < intProgress) {
-            promise.setProgressValue(intProgress);
-            lastIntProgress = intProgress;
-        }
-    }
-    promise.setProgressValue(std::min(static_cast<int>(progressStart + progressRange),
-                                      promise.future().progressMaximum()));
     return result;
 }
+
+template<typename Result>
+QList<FileNode *> scanForFilesRecursively(
+    QPromise<Result> &promise,
+    double progressRange,
+    const Utils::FilePath &directory,
+    const QDir::Filters &filter,
+    const std::function<FileNode *(const Utils::FilePath &)> factory,
+    const QList<Core::IVersionControl *> &versionControls)
+{
+    QSet<Utils::FilePath> visited;
+    const DirectoryScanResult result
+        = scanForFiles(promise, directory, filter, factory, versionControls);
+    QList<FileNode *> fileNodes = result.nodes;
+    const double progressIncrement = progressRange
+                                     / static_cast<double>(
+                                         fileNodes.count() + result.subDirectories.count());
+    promise.setProgressValue(fileNodes.count() * progressIncrement);
+    QList<QPair<Utils::FilePath, int>> subDirectories;
+    auto addSubDirectories = [&](const Utils::FilePaths &subdirs, int progressIncrement) {
+        for (const Utils::FilePath &subdir : subdirs) {
+            if (Utils::insert(visited, subdir.canonicalPath()))
+                subDirectories.append(qMakePair(subdir, progressIncrement));
+            else
+                promise.setProgressValue(promise.future().progressValue() + progressIncrement);
+        }
+    };
+    addSubDirectories(result.subDirectories, progressIncrement);
+
+    while (!subDirectories.isEmpty()) {
+        using namespace Tasking;
+        LoopList iterator(subDirectories);
+        subDirectories.clear();
+
+        auto onSetup = [&, iterator](Utils::Async<DirectoryScanResult> &task) {
+            task.setConcurrentCallData(
+                [&filter, &factory, &promise, &versionControls, subdir = iterator->first](
+                    QPromise<DirectoryScanResult> &p) {
+                    p.addResult(scanForFiles(promise, subdir, filter, factory, versionControls));
+                });
+        };
+
+        auto onDone = [&, iterator](const Utils::Async<DirectoryScanResult> &task) {
+            const int progressRange = iterator->second;
+            const DirectoryScanResult result = task.result();
+            fileNodes.append(result.nodes);
+            const int subDirCount = result.subDirectories.count();
+            if (subDirCount == 0) {
+                promise.setProgressValue(promise.future().progressValue() + progressRange);
+            } else {
+                const int fileCount = result.nodes.count();
+                const int increment = progressRange / static_cast<double>(fileCount + subDirCount);
+                promise.setProgressValue(
+                    promise.future().progressValue() + increment * fileCount);
+                addSubDirectories(result.subDirectories, increment);
+            }
+        };
+
+        const Group group{
+            Utils::HostOsInfo::isLinuxHost() ? parallelLimit(2) : parallelIdealThreadCountLimit,
+            iterator,
+            Utils::AsyncTask<DirectoryScanResult>(onSetup, onDone)
+        };
+        TaskTree::runBlocking(group);
+    }
+    return fileNodes;
+}
+
 } // namespace Internal
 
 template<typename Result>
@@ -78,15 +132,12 @@ QList<FileNode *> scanForFiles(
     const QDir::Filters &filter,
     const std::function<FileNode *(const Utils::FilePath &)> factory)
 {
-    QSet<Utils::FilePath> visited;
     promise.setProgressRange(0, 1000000);
     return Internal::scanForFilesRecursively(promise,
-                                             0.0,
                                              1000000.0,
                                              directory,
                                              filter,
                                              factory,
-                                             visited,
                                              Core::VcsManager::versionControls());
 }