CMake: Do not create file system node in main thread

The file system scanning was already in a thread, but creating a tree
from the flat list of file nodes was still done in the main thread.
Creating the tree looks for and creates folder nodes as needed for each
file node, which is not that big of a deal but still takes 1/3 of a
second for the Qt Creator source tree.

Task-number: QTCREATORBUG-25783
Change-Id: I28948ed3ff5233f6fc4b86e93da94d882b81e231
Reviewed-by: Qt CI Bot <qt_ci_bot@qt-project.org>
Reviewed-by: Cristian Adam <cristian.adam@qt.io>

src/plugins/cmakeprojectmanager/cmakebuildsystem.cpp

@@ -213,7 +213,7 @@ CMakeBuildSystem::~CMakeBuildSystem()
 
     delete m_cppCodeModelUpdater;
     qDeleteAll(m_extraCompilers);
-    qDeleteAll(m_allFiles);
+    qDeleteAll(m_allFiles.allFiles);
 }
 
 void CMakeBuildSystem::triggerParsing()
@@ -244,7 +244,7 @@ void CMakeBuildSystem::triggerParsing()
 
     qCDebug(cmakeBuildSystemLog) << "ParseGuard acquired.";
 
-    if (m_allFiles.isEmpty()) {
+    if (m_allFiles.allFiles.isEmpty()) {
         qCDebug(cmakeBuildSystemLog)
             << "No treescanner information available, forcing treescanner run.";
         updateReparseParameters(REPARSE_SCAN);
@@ -466,8 +466,8 @@ void CMakeBuildSystem::handleTreeScanningFinished()
 {
     QTC_CHECK(m_waitingForScan);
 
-    qDeleteAll(m_allFiles);
+    qDeleteAll(m_allFiles.allFiles);
-    m_allFiles = Utils::transform(m_treeScanner.release(), [](const FileNode *fn) { return fn; });
+    m_allFiles = m_treeScanner.release();
 
     m_waitingForScan = false;
 
@@ -528,7 +528,7 @@ void CMakeBuildSystem::clearCMakeCache()
 }
 
 std::unique_ptr<CMakeProjectNode> CMakeBuildSystem::generateProjectTree(
-    const QList<const FileNode *> &allFiles, bool includeHeaderNodes)
+    const TreeScanner::Result &allFiles, bool includeHeaderNodes)
 {
     QString errorMessage;
     auto root = m_reader.generateProjectTree(allFiles, errorMessage, includeHeaderNodes);

src/plugins/cmakeprojectmanager/cmakebuildsystem.h

@@ -133,7 +133,7 @@ private:
     void combineScanAndParse();
 
     std::unique_ptr<CMakeProjectNode> generateProjectTree(
-        const QList<const ProjectExplorer::FileNode *> &allFiles, bool includeHeadersNode);
+        const ProjectExplorer::TreeScanner::Result &allFiles, bool includeHeadersNode);
     void checkAndReportError(QString &errorMessage);
 
     void updateCMakeConfiguration(QString &errorMessage);
@@ -159,8 +159,8 @@ private:
     void runCTest();
 
     ProjectExplorer::TreeScanner m_treeScanner;
+    ProjectExplorer::TreeScanner::Result m_allFiles;
     QHash<QString, bool> m_mimeBinaryCache;
-    QList<const ProjectExplorer::FileNode *> m_allFiles;
 
     bool m_waitingForScan = false;
     bool m_waitingForParse = false;

src/plugins/cmakeprojectmanager/fileapireader.cpp

@@ -209,14 +209,16 @@ bool FileApiReader::usesAllCapsTargets() const
 }
 
 std::unique_ptr<CMakeProjectNode> FileApiReader::generateProjectTree(
-    const QList<const FileNode *> &allFiles, QString &errorMessage, bool includeHeaderNodes)
+    const ProjectExplorer::TreeScanner::Result &allFiles,
+    QString &errorMessage,
+    bool includeHeaderNodes)
 {
     Q_UNUSED(errorMessage)
 
     if (includeHeaderNodes) {
-        addHeaderNodes(m_rootProjectNode.get(), m_knownHeaders, allFiles);
+        addHeaderNodes(m_rootProjectNode.get(), m_knownHeaders, allFiles.allFiles);
     }
-    addFileSystemNodes(m_rootProjectNode.get(), allFiles);
+    addFileSystemNodes(m_rootProjectNode.get(), allFiles.folderNode);
     return std::exchange(m_rootProjectNode, {});
 }
 

src/plugins/cmakeprojectmanager/fileapireader.h

@@ -30,6 +30,7 @@
 #include "cmakeprojectnodes.h"
 
 #include <projectexplorer/rawprojectpart.h>
+#include <projectexplorer/treescanner.h>
 
 #include <utils/filesystemwatcher.h>
 #include <utils/optional.h>
@@ -70,7 +71,7 @@ public:
     CMakeConfig takeParsedConfiguration(QString &errorMessage);
     QString ctestPath() const;
     std::unique_ptr<CMakeProjectNode> generateProjectTree(
-        const QList<const ProjectExplorer::FileNode *> &allFiles,
+        const ProjectExplorer::TreeScanner::Result &allFiles,
         QString &errorMessage,
         bool includeHeaderNodes);
     ProjectExplorer::RawProjectParts createRawProjectParts(QString &errorMessage);

src/plugins/cmakeprojectmanager/projecttreehelper.cpp

@@ -175,7 +175,7 @@ CMakeTargetNode *createTargetNode(const QHash<Utils::FilePath, ProjectNode *> &c
 
 void addHeaderNodes(ProjectNode *root,
                     QSet<Utils::FilePath> &seenHeaders,
-                    const QList<const FileNode *> &allFiles)
+                    const QList<FileNode *> &allFiles)
 {
     QTC_ASSERT(root, return );
 
@@ -206,11 +206,28 @@ void addHeaderNodes(ProjectNode *root,
         root->addNode(std::move(headerNode));
 }
 
-void addFileSystemNodes(ProjectNode *root, const QList<const FileNode *> &allFiles)
+template<typename Result>
+static std::unique_ptr<Result> cloneFolderNode(FolderNode *node)
+{
+    auto folderNode = std::make_unique<Result>(node->filePath());
+    folderNode->setDisplayName(node->displayName());
+    for (Node *node : node->nodes()) {
+        if (FileNode *fn = node->asFileNode()) {
+            folderNode->addNode(std::unique_ptr<FileNode>(fn->clone()));
+        } else if (FolderNode *fn = node->asFolderNode()) {
+            folderNode->addNode(cloneFolderNode<FolderNode>(fn));
+        } else {
+            QTC_CHECK(false);
+        }
+    }
+    return folderNode;
+}
+
+void addFileSystemNodes(ProjectNode *root, const std::shared_ptr<FolderNode> &folderNode)
 {
     QTC_ASSERT(root, return );
 
-    auto fileSystemNode = std::make_unique<VirtualFolderNode>(root->filePath());
+    auto fileSystemNode = cloneFolderNode<VirtualFolderNode>(folderNode.get());
     // just before special nodes like "CMake Modules"
     fileSystemNode->setPriority(Node::DefaultPriority - 6);
     fileSystemNode->setDisplayName(
@@ -218,19 +235,12 @@ void addFileSystemNodes(ProjectNode *root, const QList<const FileNode *> &allFil
                                     "<File System>"));
     fileSystemNode->setIcon(DirectoryIcon(ProjectExplorer::Constants::FILEOVERLAY_UNKNOWN));
 
-    for (const FileNode *fn : allFiles) {
-        if (!fn->filePath().isChildOf(root->filePath()))
-            continue;
-
-        std::unique_ptr<FileNode> node(fn->clone());
-        node->setEnabled(false);
-        fileSystemNode->addNestedNode(std::move(node));
-    }
-
     if (!fileSystemNode->isEmpty()) {
         // make file system nodes less probable to be selected when syncing with the current document
-        fileSystemNode->forEachGenericNode(
+        fileSystemNode->forEachGenericNode([](Node *n) {
-            [](Node *n) { n->setPriority(n->priority() + Node::DefaultProjectFilePriority + 1); });
+            n->setPriority(n->priority() + Node::DefaultProjectFilePriority + 1);
+            n->setEnabled(false);
+        });
         root->addNode(std::move(fileSystemNode));
     }
 }

src/plugins/cmakeprojectmanager/projecttreehelper.h

@@ -66,9 +66,9 @@ CMakeTargetNode *createTargetNode(
 
 void addHeaderNodes(ProjectExplorer::ProjectNode *root,
                     QSet<Utils::FilePath> &seenHeaders,
-                    const QList<const ProjectExplorer::FileNode *> &allFiles);
+                    const QList<ProjectExplorer::FileNode *> &allFiles);
 
 void addFileSystemNodes(ProjectExplorer::ProjectNode *root,
-                        const QList<const ProjectExplorer::FileNode *> &allFiles);
+                        const std::shared_ptr<ProjectExplorer::FolderNode> &folderNode);
 } // namespace Internal
 } // namespace CMakeProjectManager

src/plugins/compilationdatabaseprojectmanager/compilationdbparser.cpp

@@ -138,8 +138,9 @@ void CompilationDbParser::stop()
 
 QList<FileNode *> CompilationDbParser::scannedFiles() const
 {
-    return m_treeScanner && !m_treeScanner->future().isCanceled()
+    const TreeScanner::Result result = m_treeScanner->release();
-            ? m_treeScanner->release() : QList<FileNode *>();
+    return m_treeScanner && !m_treeScanner->future().isCanceled() ? result.allFiles
+                                                                  : QList<FileNode *>();
 }
 
 void CompilationDbParser::finish(ParseResult result)

src/plugins/nim/project/nimbuildsystem.cpp

@@ -69,7 +69,8 @@ NimProjectScanner::NimProjectScanner(Project *project)
     connect(&m_scanner, &TreeScanner::finished, this, [this] {
         // Collect scanned nodes
         std::vector<std::unique_ptr<FileNode>> nodes;
-        for (FileNode *node : m_scanner.release()) {
+        const TreeScanner::Result scanResult = m_scanner.release();
+        for (FileNode *node : scanResult.allFiles) {
             if (!node->path().endsWith(".nim") && !node->path().endsWith(".nimble"))
                 node->setEnabled(false); // Disable files that do not end in .nim
             nodes.emplace_back(node);

src/plugins/projectexplorer/projectnodes.cpp

@@ -37,7 +37,6 @@
 #include <coreplugin/iversioncontrol.h>
 #include <coreplugin/vcsmanager.h>
 
-#include <utils/algorithm.h>
 #include <utils/fileutils.h>
 #include <utils/hostosinfo.h>
 #include <utils/mimetypes/mimedatabase.h>
@@ -384,62 +383,6 @@ FileType FileNode::fileType() const
     return m_fileType;
 }
 
-static QList<FileNode *> scanForFilesRecursively(QFutureInterface<QList<FileNode*>> &future,
-                                                 double progressStart, double progressRange,
-                                                 const Utils::FilePath &directory,
-                                                 const std::function<FileNode *(const Utils::FilePath &)> factory,
-                                                 QSet<QString> &visited,
-                                                 const QList<Core::IVersionControl*> &versionControls)
-{
-    QList<FileNode *> result;
-
-    const QDir baseDir = QDir(directory.toString());
-
-    // Do not follow directory loops:
-    const int visitedCount = visited.count();
-    visited.insert(baseDir.canonicalPath());
-    if (visitedCount == visited.count())
-        return result;
-
-    const QFileInfoList entries = baseDir.entryInfoList(QStringList(), QDir::AllEntries|QDir::NoDotAndDotDot);
-    double progress = 0;
-    const double progressIncrement = progressRange / static_cast<double>(entries.count());
-    int lastIntProgress = 0;
-    for (const QFileInfo &entry : entries) {
-        if (future.isCanceled())
-            return result;
-
-        const Utils::FilePath entryName = Utils::FilePath::fromString(entry.absoluteFilePath());
-        if (!Utils::contains(versionControls, [&entryName](const Core::IVersionControl *vc) {
-                             return vc->isVcsFileOrDirectory(entryName);
-            })) {
-            if (entry.isDir())
-                result.append(scanForFilesRecursively(future, progress, progressIncrement, entryName, factory, visited, versionControls));
-            else if (FileNode *node = factory(entryName))
-                result.append(node);
-        }
-        progress += progressIncrement;
-        const int intProgress = std::min(static_cast<int>(progressStart + progress), future.progressMaximum());
-        if (lastIntProgress < intProgress) {
-            future.setProgressValue(intProgress);
-            lastIntProgress = intProgress;
-        }
-    }
-    future.setProgressValue(std::min(static_cast<int>(progressStart + progressRange), future.progressMaximum()));
-    return result;
-}
-
-QList<FileNode *>
-FileNode::scanForFiles(QFutureInterface<QList<FileNode *>> &future,
-                       const Utils::FilePath &directory,
-                       const std::function<FileNode *(const Utils::FilePath &)> factory)
-{
-    QSet<QString> visited;
-    future.setProgressRange(0, 1000000);
-    return scanForFilesRecursively(future, 0.0, 1000000.0, directory, factory, visited,
-                                   Core::VcsManager::versionControls());
-}
-
 bool FileNode::supportsAction(ProjectAction action, const Node *node) const
 {
     if (action == InheritedFromParent)

src/plugins/projectexplorer/projectnodes.h

@@ -215,10 +215,6 @@ public:
     FileNode *asFileNode() final { return this; }
     const FileNode *asFileNode() const final { return this; }
 
-    static QList<FileNode *>
-    scanForFiles(QFutureInterface<QList<FileNode *>> &future,
-                 const Utils::FilePath &directory,
-                 const std::function<FileNode *(const Utils::FilePath &fileName)> factory);
     bool supportsAction(ProjectAction action, const Node *node) const override;
     QString displayName() const override;
 

src/plugins/projectexplorer/projectnodeshelper.h

@@ -0,0 +1,120 @@
+/****************************************************************************
+**
+** Copyright (C) 2021 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of Qt Creator.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+****************************************************************************/
+
+#pragma once
+
+#include "projectnodes.h"
+
+#include <coreplugin/iversioncontrol.h>
+#include <coreplugin/vcsmanager.h>
+
+#include <utils/algorithm.h>
+#include <utils/fileutils.h>
+
+namespace ProjectExplorer {
+
+template<typename Result>
+QList<FileNode *> scanForFiles(QFutureInterface<Result> &future,
+                               const Utils::FilePath &directory,
+                               const std::function<FileNode *(const Utils::FilePath &)> factory);
+
+// IMPLEMENTATION:
+
+namespace Internal {
+template<typename Result>
+QList<FileNode *> scanForFilesRecursively(
+    QFutureInterface<Result> &future,
+    double progressStart,
+    double progressRange,
+    const Utils::FilePath &directory,
+    const std::function<FileNode *(const Utils::FilePath &)> factory,
+    QSet<QString> &visited,
+    const QList<Core::IVersionControl *> &versionControls)
+{
+    QList<FileNode *> result;
+
+    const QDir baseDir = QDir(directory.toString());
+
+    // Do not follow directory loops:
+    const int visitedCount = visited.count();
+    visited.insert(baseDir.canonicalPath());
+    if (visitedCount == visited.count())
+        return result;
+
+    const QFileInfoList entries = baseDir.entryInfoList(QStringList(),
+                                                        QDir::AllEntries | QDir::NoDotAndDotDot);
+    double progress = 0;
+    const double progressIncrement = progressRange / static_cast<double>(entries.count());
+    int lastIntProgress = 0;
+    for (const QFileInfo &entry : entries) {
+        if (future.isCanceled())
+            return result;
+
+        const Utils::FilePath entryName = Utils::FilePath::fromString(entry.absoluteFilePath());
+        if (!Utils::contains(versionControls, [&entryName](const Core::IVersionControl *vc) {
+                return vc->isVcsFileOrDirectory(entryName);
+            })) {
+            if (entry.isDir())
+                result.append(scanForFilesRecursively(future,
+                                                      progress,
+                                                      progressIncrement,
+                                                      entryName,
+                                                      factory,
+                                                      visited,
+                                                      versionControls));
+            else if (FileNode *node = factory(entryName))
+                result.append(node);
+        }
+        progress += progressIncrement;
+        const int intProgress = std::min(static_cast<int>(progressStart + progress),
+                                         future.progressMaximum());
+        if (lastIntProgress < intProgress) {
+            future.setProgressValue(intProgress);
+            lastIntProgress = intProgress;
+        }
+    }
+    future.setProgressValue(
+        std::min(static_cast<int>(progressStart + progressRange), future.progressMaximum()));
+    return result;
+}
+} // namespace Internal
+
+template<typename Result>
+QList<FileNode *> scanForFiles(QFutureInterface<Result> &future,
+                               const Utils::FilePath &directory,
+                               const std::function<FileNode *(const Utils::FilePath &)> factory)
+{
+    QSet<QString> visited;
+    future.setProgressRange(0, 1000000);
+    return Internal::scanForFilesRecursively(future,
+                                             0.0,
+                                             1000000.0,
+                                             directory,
+                                             factory,
+                                             visited,
+                                             Core::VcsManager::versionControls());
+}
+
+} // namespace ProjectExplorer

src/plugins/projectexplorer/treescanner.cpp

@@ -24,7 +24,9 @@
 ****************************************************************************/
 
 #include "treescanner.h"
+
 #include "projectexplorerconstants.h"
+#include "projectnodeshelper.h"
 
 #include <coreplugin/iversioncontrol.h>
 #include <coreplugin/vcsmanager.h>
@@ -145,11 +147,25 @@ FileType TreeScanner::genericFileType(const Utils::MimeType &mimeType, const Uti
     return Node::fileTypeForMimeType(mimeType);
 }
 
+static std::unique_ptr<FolderNode> createFolderNode(const Utils::FilePath &directory,
+                                                    const QList<FileNode *> &allFiles)
+{
+    auto fileSystemNode = std::make_unique<FolderNode>(directory);
+    for (const FileNode *fn : allFiles) {
+        if (!fn->filePath().isChildOf(directory))
+            continue;
+
+        std::unique_ptr<FileNode> node(fn->clone());
+        fileSystemNode->addNestedNode(std::move(node));
+    }
+    return fileSystemNode;
+}
+
 void TreeScanner::scanForFiles(FutureInterface &fi, const Utils::FilePath& directory,
                                const FileFilter &filter, const FileTypeFactory &factory)
 {
-    Result nodes = FileNode::scanForFiles(fi, directory,
+    QList<FileNode *> nodes = ProjectExplorer::scanForFiles(fi, directory,
-                [&filter, &factory](const Utils::FilePath &fn) -> FileNode * {
+                           [&filter, &factory](const Utils::FilePath &fn) -> FileNode * {
         const Utils::MimeType mimeType = Utils::mimeTypeForFile(fn);
 
         // Skip some files during scan.
@@ -167,7 +183,9 @@ void TreeScanner::scanForFiles(FutureInterface &fi, const Utils::FilePath& direc
     Utils::sort(nodes, ProjectExplorer::Node::sortByPath);
 
     fi.setProgressValue(fi.progressMaximum());
-    fi.reportResult(nodes);
+    Result result{createFolderNode(directory, nodes), nodes};
+
+    fi.reportResult(result);
 }
 
 } // namespace ProjectExplorer

src/plugins/projectexplorer/treescanner.h

@@ -46,7 +46,11 @@ class PROJECTEXPLORER_EXPORT TreeScanner : public QObject
     Q_OBJECT
 
 public:
-    using Result = QList<ProjectExplorer::FileNode *>;
+    struct Result
+    {
+        std::shared_ptr<FolderNode> folderNode;
+        QList<FileNode *> allFiles;
+    };
     using Future = QFuture<Result>;
     using FutureWatcher = QFutureWatcher<Result>;
     using FutureInterface = QFutureInterface<Result>;