From 9109a6895ee02f02fe675643433529b605819f5d Mon Sep 17 00:00:00 2001 From: Eike Ziller Date: Wed, 16 Jun 2021 11:33:35 +0200 Subject: [PATCH] CMake: Do not create file system node in main thread The file system scanning was already in a thread, but creating a tree from the flat list of file nodes was still done in the main thread. Creating the tree looks for and creates folder nodes as needed for each file node, which is not that big of a deal but still takes 1/3 of a second for the Qt Creator source tree. Task-number: QTCREATORBUG-25783 Change-Id: I28948ed3ff5233f6fc4b86e93da94d882b81e231 Reviewed-by: Qt CI Bot Reviewed-by: Cristian Adam --- .../cmakeprojectmanager/cmakebuildsystem.cpp | 10 +- .../cmakeprojectmanager/cmakebuildsystem.h | 4 +- .../cmakeprojectmanager/fileapireader.cpp | 8 +- .../cmakeprojectmanager/fileapireader.h | 3 +- .../cmakeprojectmanager/projecttreehelper.cpp | 38 ++++-- .../cmakeprojectmanager/projecttreehelper.h | 4 +- .../compilationdbparser.cpp | 5 +- src/plugins/nim/project/nimbuildsystem.cpp | 3 +- src/plugins/projectexplorer/projectnodes.cpp | 57 --------- src/plugins/projectexplorer/projectnodes.h | 4 - .../projectexplorer/projectnodeshelper.h | 120 ++++++++++++++++++ src/plugins/projectexplorer/treescanner.cpp | 24 +++- src/plugins/projectexplorer/treescanner.h | 6 +- 13 files changed, 191 insertions(+), 95 deletions(-) create mode 100644 src/plugins/projectexplorer/projectnodeshelper.h diff --git a/src/plugins/cmakeprojectmanager/cmakebuildsystem.cpp b/src/plugins/cmakeprojectmanager/cmakebuildsystem.cpp index aa57a88c1f0..c3a20a928c1 100644 --- a/src/plugins/cmakeprojectmanager/cmakebuildsystem.cpp +++ b/src/plugins/cmakeprojectmanager/cmakebuildsystem.cpp @@ -213,7 +213,7 @@ CMakeBuildSystem::~CMakeBuildSystem() delete m_cppCodeModelUpdater; qDeleteAll(m_extraCompilers); - qDeleteAll(m_allFiles); + qDeleteAll(m_allFiles.allFiles); } void CMakeBuildSystem::triggerParsing() @@ -244,7 +244,7 @@ void CMakeBuildSystem::triggerParsing() qCDebug(cmakeBuildSystemLog) << "ParseGuard acquired."; - if (m_allFiles.isEmpty()) { + if (m_allFiles.allFiles.isEmpty()) { qCDebug(cmakeBuildSystemLog) << "No treescanner information available, forcing treescanner run."; updateReparseParameters(REPARSE_SCAN); @@ -466,8 +466,8 @@ void CMakeBuildSystem::handleTreeScanningFinished() { QTC_CHECK(m_waitingForScan); - qDeleteAll(m_allFiles); - m_allFiles = Utils::transform(m_treeScanner.release(), [](const FileNode *fn) { return fn; }); + qDeleteAll(m_allFiles.allFiles); + m_allFiles = m_treeScanner.release(); m_waitingForScan = false; @@ -528,7 +528,7 @@ void CMakeBuildSystem::clearCMakeCache() } std::unique_ptr CMakeBuildSystem::generateProjectTree( - const QList &allFiles, bool includeHeaderNodes) + const TreeScanner::Result &allFiles, bool includeHeaderNodes) { QString errorMessage; auto root = m_reader.generateProjectTree(allFiles, errorMessage, includeHeaderNodes); diff --git a/src/plugins/cmakeprojectmanager/cmakebuildsystem.h b/src/plugins/cmakeprojectmanager/cmakebuildsystem.h index 27446052b54..6f0bcea68cb 100644 --- a/src/plugins/cmakeprojectmanager/cmakebuildsystem.h +++ b/src/plugins/cmakeprojectmanager/cmakebuildsystem.h @@ -133,7 +133,7 @@ private: void combineScanAndParse(); std::unique_ptr generateProjectTree( - const QList &allFiles, bool includeHeadersNode); + const ProjectExplorer::TreeScanner::Result &allFiles, bool includeHeadersNode); void checkAndReportError(QString &errorMessage); void updateCMakeConfiguration(QString &errorMessage); @@ -159,8 +159,8 @@ private: void runCTest(); ProjectExplorer::TreeScanner m_treeScanner; + ProjectExplorer::TreeScanner::Result m_allFiles; QHash m_mimeBinaryCache; - QList m_allFiles; bool m_waitingForScan = false; bool m_waitingForParse = false; diff --git a/src/plugins/cmakeprojectmanager/fileapireader.cpp b/src/plugins/cmakeprojectmanager/fileapireader.cpp index 22b4e0ce1bf..fd2705049d1 100644 --- a/src/plugins/cmakeprojectmanager/fileapireader.cpp +++ b/src/plugins/cmakeprojectmanager/fileapireader.cpp @@ -209,14 +209,16 @@ bool FileApiReader::usesAllCapsTargets() const } std::unique_ptr FileApiReader::generateProjectTree( - const QList &allFiles, QString &errorMessage, bool includeHeaderNodes) + const ProjectExplorer::TreeScanner::Result &allFiles, + QString &errorMessage, + bool includeHeaderNodes) { Q_UNUSED(errorMessage) if (includeHeaderNodes) { - addHeaderNodes(m_rootProjectNode.get(), m_knownHeaders, allFiles); + addHeaderNodes(m_rootProjectNode.get(), m_knownHeaders, allFiles.allFiles); } - addFileSystemNodes(m_rootProjectNode.get(), allFiles); + addFileSystemNodes(m_rootProjectNode.get(), allFiles.folderNode); return std::exchange(m_rootProjectNode, {}); } diff --git a/src/plugins/cmakeprojectmanager/fileapireader.h b/src/plugins/cmakeprojectmanager/fileapireader.h index 2f1ab749b7d..3961dfc3b97 100644 --- a/src/plugins/cmakeprojectmanager/fileapireader.h +++ b/src/plugins/cmakeprojectmanager/fileapireader.h @@ -30,6 +30,7 @@ #include "cmakeprojectnodes.h" #include +#include #include #include @@ -70,7 +71,7 @@ public: CMakeConfig takeParsedConfiguration(QString &errorMessage); QString ctestPath() const; std::unique_ptr generateProjectTree( - const QList &allFiles, + const ProjectExplorer::TreeScanner::Result &allFiles, QString &errorMessage, bool includeHeaderNodes); ProjectExplorer::RawProjectParts createRawProjectParts(QString &errorMessage); diff --git a/src/plugins/cmakeprojectmanager/projecttreehelper.cpp b/src/plugins/cmakeprojectmanager/projecttreehelper.cpp index edd3dd3c77f..50789ec0a20 100644 --- a/src/plugins/cmakeprojectmanager/projecttreehelper.cpp +++ b/src/plugins/cmakeprojectmanager/projecttreehelper.cpp @@ -175,7 +175,7 @@ CMakeTargetNode *createTargetNode(const QHash &c void addHeaderNodes(ProjectNode *root, QSet &seenHeaders, - const QList &allFiles) + const QList &allFiles) { QTC_ASSERT(root, return ); @@ -206,11 +206,28 @@ void addHeaderNodes(ProjectNode *root, root->addNode(std::move(headerNode)); } -void addFileSystemNodes(ProjectNode *root, const QList &allFiles) +template +static std::unique_ptr cloneFolderNode(FolderNode *node) +{ + auto folderNode = std::make_unique(node->filePath()); + folderNode->setDisplayName(node->displayName()); + for (Node *node : node->nodes()) { + if (FileNode *fn = node->asFileNode()) { + folderNode->addNode(std::unique_ptr(fn->clone())); + } else if (FolderNode *fn = node->asFolderNode()) { + folderNode->addNode(cloneFolderNode(fn)); + } else { + QTC_CHECK(false); + } + } + return folderNode; +} + +void addFileSystemNodes(ProjectNode *root, const std::shared_ptr &folderNode) { QTC_ASSERT(root, return ); - auto fileSystemNode = std::make_unique(root->filePath()); + auto fileSystemNode = cloneFolderNode(folderNode.get()); // just before special nodes like "CMake Modules" fileSystemNode->setPriority(Node::DefaultPriority - 6); fileSystemNode->setDisplayName( @@ -218,19 +235,12 @@ void addFileSystemNodes(ProjectNode *root, const QList &allFil "")); fileSystemNode->setIcon(DirectoryIcon(ProjectExplorer::Constants::FILEOVERLAY_UNKNOWN)); - for (const FileNode *fn : allFiles) { - if (!fn->filePath().isChildOf(root->filePath())) - continue; - - std::unique_ptr node(fn->clone()); - node->setEnabled(false); - fileSystemNode->addNestedNode(std::move(node)); - } - if (!fileSystemNode->isEmpty()) { // make file system nodes less probable to be selected when syncing with the current document - fileSystemNode->forEachGenericNode( - [](Node *n) { n->setPriority(n->priority() + Node::DefaultProjectFilePriority + 1); }); + fileSystemNode->forEachGenericNode([](Node *n) { + n->setPriority(n->priority() + Node::DefaultProjectFilePriority + 1); + n->setEnabled(false); + }); root->addNode(std::move(fileSystemNode)); } } diff --git a/src/plugins/cmakeprojectmanager/projecttreehelper.h b/src/plugins/cmakeprojectmanager/projecttreehelper.h index eabb4a0096b..e25d933141b 100644 --- a/src/plugins/cmakeprojectmanager/projecttreehelper.h +++ b/src/plugins/cmakeprojectmanager/projecttreehelper.h @@ -66,9 +66,9 @@ CMakeTargetNode *createTargetNode( void addHeaderNodes(ProjectExplorer::ProjectNode *root, QSet &seenHeaders, - const QList &allFiles); + const QList &allFiles); void addFileSystemNodes(ProjectExplorer::ProjectNode *root, - const QList &allFiles); + const std::shared_ptr &folderNode); } // namespace Internal } // namespace CMakeProjectManager diff --git a/src/plugins/compilationdatabaseprojectmanager/compilationdbparser.cpp b/src/plugins/compilationdatabaseprojectmanager/compilationdbparser.cpp index 23c65a014af..0a0485e1896 100644 --- a/src/plugins/compilationdatabaseprojectmanager/compilationdbparser.cpp +++ b/src/plugins/compilationdatabaseprojectmanager/compilationdbparser.cpp @@ -138,8 +138,9 @@ void CompilationDbParser::stop() QList CompilationDbParser::scannedFiles() const { - return m_treeScanner && !m_treeScanner->future().isCanceled() - ? m_treeScanner->release() : QList(); + const TreeScanner::Result result = m_treeScanner->release(); + return m_treeScanner && !m_treeScanner->future().isCanceled() ? result.allFiles + : QList(); } void CompilationDbParser::finish(ParseResult result) diff --git a/src/plugins/nim/project/nimbuildsystem.cpp b/src/plugins/nim/project/nimbuildsystem.cpp index a9af84cf9e7..b02d2131e4a 100644 --- a/src/plugins/nim/project/nimbuildsystem.cpp +++ b/src/plugins/nim/project/nimbuildsystem.cpp @@ -69,7 +69,8 @@ NimProjectScanner::NimProjectScanner(Project *project) connect(&m_scanner, &TreeScanner::finished, this, [this] { // Collect scanned nodes std::vector> nodes; - for (FileNode *node : m_scanner.release()) { + const TreeScanner::Result scanResult = m_scanner.release(); + for (FileNode *node : scanResult.allFiles) { if (!node->path().endsWith(".nim") && !node->path().endsWith(".nimble")) node->setEnabled(false); // Disable files that do not end in .nim nodes.emplace_back(node); diff --git a/src/plugins/projectexplorer/projectnodes.cpp b/src/plugins/projectexplorer/projectnodes.cpp index f2705678973..b160a479791 100644 --- a/src/plugins/projectexplorer/projectnodes.cpp +++ b/src/plugins/projectexplorer/projectnodes.cpp @@ -37,7 +37,6 @@ #include #include -#include #include #include #include @@ -384,62 +383,6 @@ FileType FileNode::fileType() const return m_fileType; } -static QList scanForFilesRecursively(QFutureInterface> &future, - double progressStart, double progressRange, - const Utils::FilePath &directory, - const std::function factory, - QSet &visited, - const QList &versionControls) -{ - QList result; - - const QDir baseDir = QDir(directory.toString()); - - // Do not follow directory loops: - const int visitedCount = visited.count(); - visited.insert(baseDir.canonicalPath()); - if (visitedCount == visited.count()) - return result; - - const QFileInfoList entries = baseDir.entryInfoList(QStringList(), QDir::AllEntries|QDir::NoDotAndDotDot); - double progress = 0; - const double progressIncrement = progressRange / static_cast(entries.count()); - int lastIntProgress = 0; - for (const QFileInfo &entry : entries) { - if (future.isCanceled()) - return result; - - const Utils::FilePath entryName = Utils::FilePath::fromString(entry.absoluteFilePath()); - if (!Utils::contains(versionControls, [&entryName](const Core::IVersionControl *vc) { - return vc->isVcsFileOrDirectory(entryName); - })) { - if (entry.isDir()) - result.append(scanForFilesRecursively(future, progress, progressIncrement, entryName, factory, visited, versionControls)); - else if (FileNode *node = factory(entryName)) - result.append(node); - } - progress += progressIncrement; - const int intProgress = std::min(static_cast(progressStart + progress), future.progressMaximum()); - if (lastIntProgress < intProgress) { - future.setProgressValue(intProgress); - lastIntProgress = intProgress; - } - } - future.setProgressValue(std::min(static_cast(progressStart + progressRange), future.progressMaximum())); - return result; -} - -QList -FileNode::scanForFiles(QFutureInterface> &future, - const Utils::FilePath &directory, - const std::function factory) -{ - QSet visited; - future.setProgressRange(0, 1000000); - return scanForFilesRecursively(future, 0.0, 1000000.0, directory, factory, visited, - Core::VcsManager::versionControls()); -} - bool FileNode::supportsAction(ProjectAction action, const Node *node) const { if (action == InheritedFromParent) diff --git a/src/plugins/projectexplorer/projectnodes.h b/src/plugins/projectexplorer/projectnodes.h index f6b8e0d9d96..8fd2092ecc1 100644 --- a/src/plugins/projectexplorer/projectnodes.h +++ b/src/plugins/projectexplorer/projectnodes.h @@ -215,10 +215,6 @@ public: FileNode *asFileNode() final { return this; } const FileNode *asFileNode() const final { return this; } - static QList - scanForFiles(QFutureInterface> &future, - const Utils::FilePath &directory, - const std::function factory); bool supportsAction(ProjectAction action, const Node *node) const override; QString displayName() const override; diff --git a/src/plugins/projectexplorer/projectnodeshelper.h b/src/plugins/projectexplorer/projectnodeshelper.h new file mode 100644 index 00000000000..6e159571517 --- /dev/null +++ b/src/plugins/projectexplorer/projectnodeshelper.h @@ -0,0 +1,120 @@ +/**************************************************************************** +** +** Copyright (C) 2021 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of Qt Creator. +** +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 3 as published by the Free Software +** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-3.0.html. +** +****************************************************************************/ + +#pragma once + +#include "projectnodes.h" + +#include +#include + +#include +#include + +namespace ProjectExplorer { + +template +QList scanForFiles(QFutureInterface &future, + const Utils::FilePath &directory, + const std::function factory); + +// IMPLEMENTATION: + +namespace Internal { +template +QList scanForFilesRecursively( + QFutureInterface &future, + double progressStart, + double progressRange, + const Utils::FilePath &directory, + const std::function factory, + QSet &visited, + const QList &versionControls) +{ + QList result; + + const QDir baseDir = QDir(directory.toString()); + + // Do not follow directory loops: + const int visitedCount = visited.count(); + visited.insert(baseDir.canonicalPath()); + if (visitedCount == visited.count()) + return result; + + const QFileInfoList entries = baseDir.entryInfoList(QStringList(), + QDir::AllEntries | QDir::NoDotAndDotDot); + double progress = 0; + const double progressIncrement = progressRange / static_cast(entries.count()); + int lastIntProgress = 0; + for (const QFileInfo &entry : entries) { + if (future.isCanceled()) + return result; + + const Utils::FilePath entryName = Utils::FilePath::fromString(entry.absoluteFilePath()); + if (!Utils::contains(versionControls, [&entryName](const Core::IVersionControl *vc) { + return vc->isVcsFileOrDirectory(entryName); + })) { + if (entry.isDir()) + result.append(scanForFilesRecursively(future, + progress, + progressIncrement, + entryName, + factory, + visited, + versionControls)); + else if (FileNode *node = factory(entryName)) + result.append(node); + } + progress += progressIncrement; + const int intProgress = std::min(static_cast(progressStart + progress), + future.progressMaximum()); + if (lastIntProgress < intProgress) { + future.setProgressValue(intProgress); + lastIntProgress = intProgress; + } + } + future.setProgressValue( + std::min(static_cast(progressStart + progressRange), future.progressMaximum())); + return result; +} +} // namespace Internal + +template +QList scanForFiles(QFutureInterface &future, + const Utils::FilePath &directory, + const std::function factory) +{ + QSet visited; + future.setProgressRange(0, 1000000); + return Internal::scanForFilesRecursively(future, + 0.0, + 1000000.0, + directory, + factory, + visited, + Core::VcsManager::versionControls()); +} + +} // namespace ProjectExplorer diff --git a/src/plugins/projectexplorer/treescanner.cpp b/src/plugins/projectexplorer/treescanner.cpp index f677c87e48b..4f2eb958681 100644 --- a/src/plugins/projectexplorer/treescanner.cpp +++ b/src/plugins/projectexplorer/treescanner.cpp @@ -24,7 +24,9 @@ ****************************************************************************/ #include "treescanner.h" + #include "projectexplorerconstants.h" +#include "projectnodeshelper.h" #include #include @@ -145,11 +147,25 @@ FileType TreeScanner::genericFileType(const Utils::MimeType &mimeType, const Uti return Node::fileTypeForMimeType(mimeType); } +static std::unique_ptr createFolderNode(const Utils::FilePath &directory, + const QList &allFiles) +{ + auto fileSystemNode = std::make_unique(directory); + for (const FileNode *fn : allFiles) { + if (!fn->filePath().isChildOf(directory)) + continue; + + std::unique_ptr node(fn->clone()); + fileSystemNode->addNestedNode(std::move(node)); + } + return fileSystemNode; +} + void TreeScanner::scanForFiles(FutureInterface &fi, const Utils::FilePath& directory, const FileFilter &filter, const FileTypeFactory &factory) { - Result nodes = FileNode::scanForFiles(fi, directory, - [&filter, &factory](const Utils::FilePath &fn) -> FileNode * { + QList nodes = ProjectExplorer::scanForFiles(fi, directory, + [&filter, &factory](const Utils::FilePath &fn) -> FileNode * { const Utils::MimeType mimeType = Utils::mimeTypeForFile(fn); // Skip some files during scan. @@ -167,7 +183,9 @@ void TreeScanner::scanForFiles(FutureInterface &fi, const Utils::FilePath& direc Utils::sort(nodes, ProjectExplorer::Node::sortByPath); fi.setProgressValue(fi.progressMaximum()); - fi.reportResult(nodes); + Result result{createFolderNode(directory, nodes), nodes}; + + fi.reportResult(result); } } // namespace ProjectExplorer diff --git a/src/plugins/projectexplorer/treescanner.h b/src/plugins/projectexplorer/treescanner.h index 28fe47ba5b2..26992448928 100644 --- a/src/plugins/projectexplorer/treescanner.h +++ b/src/plugins/projectexplorer/treescanner.h @@ -46,7 +46,11 @@ class PROJECTEXPLORER_EXPORT TreeScanner : public QObject Q_OBJECT public: - using Result = QList; + struct Result + { + std::shared_ptr folderNode; + QList allFiles; + }; using Future = QFuture; using FutureWatcher = QFutureWatcher; using FutureInterface = QFutureInterface;