ProjectExplorer: improve treescanner scan result

Allow returning all folders scanned by the treescanner. This will be
used in the WorkspaceProject to also show empty folders in the project
tree. Additionally we are now able to iterate over the complete tree of
folder nodes and add the paths to the watcher instead of walking the
directory tree again to collect all directories to watch.

Change-Id: Ibe7bed9ccee8317918e06fc78ca85f74102d46fc
Reviewed-by: Marcus Tillmanns <marcus.tillmanns@qt.io>

src/plugins/cmakeprojectmanager/cmakebuildsystem.cpp

@@ -1505,8 +1505,9 @@ void CMakeBuildSystem::updateProjectData()
 void CMakeBuildSystem::handleTreeScanningFinished()
 {
     TreeScanner::Result result = m_treeScanner.release();
-    m_allFiles = result.folderNode;
+    m_allFiles = std::make_shared<ProjectExplorer::FolderNode>(projectDirectory());
-    qDeleteAll(result.allFiles);
+    for (auto node : result.takeFirstLevelNodes())
+        m_allFiles->addNode(std::unique_ptr<Node>(node));
 
     updateFileSystemNodes();
 }

src/plugins/compilationdatabaseprojectmanager/compilationdbparser.cpp

@@ -221,8 +221,8 @@ void CompilationDbParser::stop()
 QList<FileNode *> CompilationDbParser::scannedFiles() const
 {
     const bool canceled = m_treeScanner->future().isCanceled();
-    const TreeScanner::Result result = m_treeScanner->release();
+    TreeScanner::Result result = m_treeScanner->release();
-    return !canceled ? result.allFiles : QList<FileNode *>();
+    return !canceled ? result.takeAllFiles() : QList<FileNode *>();
 }
 
 void CompilationDbParser::parserJobFinished()

src/plugins/haskell/haskellproject.cpp

@@ -67,7 +67,7 @@ HaskellBuildSystem::HaskellBuildSystem(Target *t)
         auto root = std::make_unique<ProjectNode>(projectDirectory());
         root->setDisplayName(target()->project()->displayName());
         std::vector<std::unique_ptr<FileNode>> nodePtrs
-            = Utils::transform<std::vector>(m_scanner.release().allFiles, [](FileNode *fn) {
+            = Utils::transform<std::vector>(m_scanner.release().takeAllFiles(), [](FileNode *fn) {
                   return std::unique_ptr<FileNode>(fn);
               });
         root->addNestedNodes(std::move(nodePtrs));

src/plugins/nim/project/nimbuildsystem.cpp

@@ -36,8 +36,8 @@ NimProjectScanner::NimProjectScanner(Project *project)
     connect(&m_scanner, &TreeScanner::finished, this, [this] {
         // Collect scanned nodes
         std::vector<std::unique_ptr<FileNode>> nodes;
-        const TreeScanner::Result scanResult = m_scanner.release();
+        TreeScanner::Result scanResult = m_scanner.release();
-        for (FileNode *node : scanResult.allFiles) {
+        for (FileNode *node : scanResult.takeAllFiles()) {
             if (!node->path().endsWith(".nim") && !node->path().endsWith(".nimble"))
                 node->setEnabled(false); // Disable files that do not end in .nim
             nodes.emplace_back(node);

src/plugins/projectexplorer/treescanner.cpp

@@ -133,35 +133,23 @@ FileType TreeScanner::genericFileType(const Utils::MimeType &mimeType, const Uti
     return Node::fileTypeForMimeType(mimeType);
 }
 
-static std::unique_ptr<FolderNode> createFolderNode(const Utils::FilePath &directory,
-                                                    const QList<FileNode *> &allFiles)
-{
-    auto fileSystemNode = std::make_unique<FolderNode>(directory);
-    for (const FileNode *fn : allFiles) {
-        if (!fn->filePath().isChildOf(directory))
-            continue;
-
-        std::unique_ptr<FileNode> node(fn->clone());
-        fileSystemNode->addNestedNode(std::move(node));
-    }
-    ProjectTree::applyTreeManager(fileSystemNode.get(), ProjectTree::AsyncPhase); // QRC nodes
-    return fileSystemNode;
-}
-
 struct DirectoryScanResult
 {
     QList<FileNode *> nodes;
-    Utils::FilePaths subDirectories;
+    QList<FolderNode *> subDirectories;
+    FolderNode *parentNode;
 };
 
 static DirectoryScanResult scanForFilesImpl(
     const QFuture<void> &future,
     const Utils::FilePath &directory,
+    FolderNode *parent,
     QDir::Filters filter,
     const std::function<FileNode *(const Utils::FilePath &)> &factory,
     const QList<Core::IVersionControl *> &versionControls)
 {
     DirectoryScanResult result;
+    result.parentNode = parent;
 
     const Utils::FilePaths entries = directory.dirEntries(filter);
     for (const Utils::FilePath &entry : entries) {
@@ -175,7 +163,7 @@ static DirectoryScanResult scanForFilesImpl(
         }
 
         if (entry.isDir())
-            result.subDirectories.append(entry);
+            result.subDirectories.append(new FolderNode(entry));
         else if (FileNode *node = factory(entry))
             result.nodes.append(node);
     }
@@ -188,7 +176,7 @@ static const Utils::MimeType &directoryMimeType()
     return mimeType;
 }
 
-static QList<FileNode *> scanForFilesHelper(
+static TreeScanner::Result scanForFilesHelper(
     TreeScanner::Promise &promise,
     const Utils::FilePath &directory,
     QDir::Filters dirfilter,
@@ -202,23 +190,32 @@ static QList<FileNode *> scanForFilesHelper(
     promise.setProgressRange(0, progressRange);
 
     QSet<Utils::FilePath> visited;
-    const DirectoryScanResult result = scanForFilesImpl(future, directory, dirfilter, factory, versionControls);
+    const DirectoryScanResult result
+        = scanForFilesImpl(future, directory, nullptr, dirfilter, factory, versionControls);
     QList<FileNode *> fileNodes = result.nodes;
+    QList<Node *> firstLevelNodes;
+    for (auto fileNode : fileNodes)
+        firstLevelNodes.append(fileNode->clone());
     const int progressIncrement = int(
         progressRange / static_cast<double>(fileNodes.count() + result.subDirectories.count()));
     promise.setProgressValue(int(fileNodes.count() * progressIncrement));
-    QList<QPair<Utils::FilePath, int>> subDirectories;
+    QList<QPair<FolderNode *, int>> subDirectories;
-    auto addSubDirectories = [&](const Utils::FilePaths &subdirs, int progressIncrement) {
+    auto addSubDirectories = [&](const QList<FolderNode *> &subdirs, FolderNode * parent, int progressIncrement) {
-        for (const Utils::FilePath &subdir : subdirs) {
+        for (FolderNode *subdir : subdirs) {
-            if (Utils::insert(visited, subdir.canonicalPath())
+            if (Utils::insert(visited, subdir->filePath().canonicalPath())
-                && !(filter && filter(directoryMimeType(), subdir))) {
+                && !(filter && filter(directoryMimeType(), subdir->filePath()))) {
                 subDirectories.append(qMakePair(subdir, progressIncrement));
+                subdir->setDisplayName(subdir->filePath().fileName());
+                if (parent)
+                    parent->addNode(std::unique_ptr<FolderNode>(subdir));
+                else
+                    firstLevelNodes << subdir;
             } else {
                 promise.setProgressValue(future.progressValue() + progressIncrement);
             }
         }
     };
-    addSubDirectories(result.subDirectories, progressIncrement);
+    addSubDirectories(result.subDirectories, nullptr, progressIncrement);
 
     while (!subDirectories.isEmpty()) {
         using namespace Tasking;
@@ -227,7 +224,13 @@ static QList<FileNode *> scanForFilesHelper(
 
         auto onSetup = [&, iterator](Utils::Async<DirectoryScanResult> &task) {
             task.setConcurrentCallData(
-                scanForFilesImpl, future, iterator->first, dirfilter, factory, versionControls);
+                scanForFilesImpl,
+                future,
+                iterator->first->filePath(),
+                iterator->first,
+                dirfilter,
+                factory,
+                versionControls);
         };
 
         auto onDone = [&, iterator](const Utils::Async<DirectoryScanResult> &task) {
@@ -235,6 +238,10 @@ static QList<FileNode *> scanForFilesHelper(
             const DirectoryScanResult result = task.result();
             fileNodes.append(result.nodes);
             const qsizetype subDirCount = result.subDirectories.count();
+            if (iterator->first) {
+                for (auto fn : result.nodes)
+                    iterator->first->addNode(std::unique_ptr<FileNode>(fn->clone()));
+            }
             if (subDirCount == 0) {
                 promise.setProgressValue(future.progressValue() + progressRange);
             } else {
@@ -242,7 +249,7 @@ static QList<FileNode *> scanForFilesHelper(
                 const int increment = int(
                     progressRange / static_cast<double>(fileCount + subDirCount));
                 promise.setProgressValue(future.progressValue() + increment * fileCount);
-                addSubDirectories(result.subDirectories, increment);
+                addSubDirectories(result.subDirectories, result.parentNode, increment);
             }
         };
 
@@ -252,7 +259,10 @@ static QList<FileNode *> scanForFilesHelper(
         };
         TaskTree::runBlocking(recipe);
     }
-    return fileNodes;
+
+    Utils::sort(fileNodes, ProjectExplorer::Node::sortByPath);
+
+    return {fileNodes, firstLevelNodes};
 }
 
 void TreeScanner::scanForFiles(
@@ -262,8 +272,12 @@ void TreeScanner::scanForFiles(
     QDir::Filters dirFilter,
     const FileTypeFactory &factory)
 {
-    QList<FileNode *> nodes = scanForFilesHelper(
+    Result result = scanForFilesHelper(
-        promise, directory, dirFilter, filter, [&filter, &factory](const Utils::FilePath &fn) -> FileNode * {
+        promise,
+        directory,
+        dirFilter,
+        filter,
+        [&filter, &factory](const Utils::FilePath &fn) -> FileNode * {
             const Utils::MimeType mimeType = Utils::mimeTypesForFileName(fn.path()).value(0);
 
             // Skip some files during scan.
@@ -278,12 +292,31 @@ void TreeScanner::scanForFiles(
             return new FileNode(fn, type);
         });
 
-    Utils::sort(nodes, ProjectExplorer::Node::sortByPath);
-
     promise.setProgressValue(promise.future().progressMaximum());
-    Result result{createFolderNode(directory, nodes), nodes};
-
     promise.addResult(result);
 }
 
+TreeScanner::Result::Result(QList<FileNode *> files, QList<Node *> nodes)
+    : allFiles(files)
+    , firstLevelNodes(nodes)
+{}
+
+QList<FileNode *> TreeScanner::Result::takeAllFiles()
+{
+    qDeleteAll(firstLevelNodes);
+    firstLevelNodes.clear();
+    QList<FileNode *> result = allFiles;
+    allFiles.clear();
+    return result;
+}
+
+QList<Node *> TreeScanner::Result::takeFirstLevelNodes()
+{
+    qDeleteAll(allFiles);
+    allFiles.clear();
+    QList<Node *> result = firstLevelNodes;
+    firstLevelNodes.clear();
+    return result;
+}
+
 } // namespace ProjectExplorer

src/plugins/projectexplorer/treescanner.h

@@ -24,10 +24,16 @@ class PROJECTEXPLORER_EXPORT TreeScanner : public QObject
     Q_OBJECT
 
 public:
-    struct Result
+    class PROJECTEXPLORER_EXPORT Result
     {
-        std::shared_ptr<FolderNode> folderNode;
+    public:
+        Result() = default;
+        Result(QList<FileNode *> files, QList<Node *> nodes);
+        QList<FileNode *> takeAllFiles();
+        QList<Node *> takeFirstLevelNodes();
+    private:
         QList<FileNode *> allFiles;
+        QList<Node *> firstLevelNodes;
     };
     using Future = QFuture<Result>;
     using FutureWatcher = QFutureWatcher<Result>;

src/plugins/projectexplorer/workspaceproject.cpp

@@ -70,8 +70,6 @@ public:
     bool renameFiles(Node *context, const FilePairs &filesToRename, FilePaths *notRenamed) final;
     bool supportsAction(Node *context, ProjectAction action, const Node *node) const final;
 
-    void watchFolder(const FilePath &path, const QList<IVersionControl *> &versionControls);
-
     void handleDirectoryChanged(const FilePath &directory);
 
     void scan(const FilePath &path);
@@ -108,13 +106,28 @@ WorkspaceBuildSystem::WorkspaceBuildSystem(Target *t)
         QTC_ASSERT(!m_scanQueue.isEmpty(), return);
         const FilePath scannedDir = m_scanQueue.takeFirst();
 
-        std::vector<std::unique_ptr<FileNode>> nodePtrs
+        TreeScanner::Result result = m_scanner.release();
-            = Utils::transform<std::vector>(m_scanner.release().allFiles, [](FileNode *fn) {
+        auto addNodes = [this, &result](FolderNode *parent) {
-                  return std::unique_ptr<FileNode>(fn);
+            QElapsedTimer timer;
-              });
+            timer.start();
+            const QList<IVersionControl *> versionControls = VcsManager::versionControls();
+            for (auto node : result.takeFirstLevelNodes())
+                parent->addNode(std::unique_ptr<Node>(node));
+            qCDebug(wsbs) << "Added nodes in" << timer.elapsed() << "ms";
+            FilePaths toWatch;
+            auto collectWatchFolders = [this, &toWatch, &versionControls](FolderNode *fn) {
+                if (!isFiltered(fn->path(), versionControls))
+                    toWatch << fn->path();
+            };
+            collectWatchFolders(parent);
+            parent->forEachNode({}, collectWatchFolders);
+            qCDebug(wsbs) << "Added and collected nodes in" << timer.elapsed() << "ms" << toWatch.size() << "dirs";
+            m_watcher->addDirectories(toWatch, FileSystemWatcher::WatchAllChanges);
+            qCDebug(wsbs) << "Added and and collected and watched nodes in" << timer.elapsed() << "ms";
+        };
+
         if (scannedDir == projectDirectory()) {
-            qCDebug(wsbs) << "Finished scanning new root" << scannedDir << "found"
+            qCDebug(wsbs) << "Finished scanning new root" << scannedDir;
-                          << nodePtrs.size() << "file entries";
             auto root = std::make_unique<ProjectNode>(scannedDir);
             root->setDisplayName(target()->project()->displayName());
             m_watcher.reset(new FileSystemWatcher);
@@ -123,24 +136,22 @@ WorkspaceBuildSystem::WorkspaceBuildSystem(Target *t)
                 &FileSystemWatcher::directoryChanged,
                 this,
                 [this](const QString &path) {
-                    handleDirectoryChanged(Utils::FilePath::fromPathPart(path));
+                    handleDirectoryChanged(FilePath::fromPathPart(path));
                 });
 
-            root->addNestedNodes(std::move(nodePtrs));
+            addNodes(root.get());
             setRootProjectNode(std::move(root));
         } else {
-            qCDebug(wsbs) << "Finished scanning subdir" << scannedDir << "found"
+            qCDebug(wsbs) << "Finished scanning subdir" << scannedDir;
-                          << nodePtrs.size() << "file entries";
             FolderNode *parent = findAvailableParent(project()->rootProjectNode(), scannedDir);
             const FilePath relativePath = scannedDir.relativeChildPath(parent->filePath());
             const QString firstRelativeFolder = relativePath.path().left(relativePath.path().indexOf('/'));
             const FilePath nodePath = parent->filePath() / firstRelativeFolder;
             auto newNode = std::make_unique<FolderNode>(nodePath);
             newNode->setDisplayName(firstRelativeFolder);
-            newNode->addNestedNodes(std::move(nodePtrs));
+            addNodes(newNode.get());
             parent->replaceSubtree(nullptr, std::move(newNode));
         }
-        watchFolder(scannedDir, VcsManager::versionControls());
 
         scanNext();
     });
@@ -258,19 +269,6 @@ bool WorkspaceBuildSystem::supportsAction(Node *, ProjectAction action, const No
            || action == ProjectAction::EraseFile;
 }
 
-void WorkspaceBuildSystem::watchFolder(
-    const FilePath &path, const QList<IVersionControl *> &versionControls)
-{
-    if (!m_watcher->watchesDirectory(path)) {
-        qCDebug(wsbs) << "Adding watch for " << path;
-        m_watcher->addDirectory(path, FileSystemWatcher::WatchAllChanges);
-    }
-    for (auto entry : path.dirEntries(QDir::NoDotAndDotDot | QDir::Hidden | QDir::Dirs)) {
-        if (!isFiltered(entry, versionControls))
-            watchFolder(entry, versionControls);
-    }
-}
-
 void WorkspaceBuildSystem::handleDirectoryChanged(const FilePath &directory)
 {
     ProjectNode *root = project()->rootProjectNode();