Don't update extra compilers individually after project load

Each call of CppModelManager::updateSourceFiles detaches the current
snapshot.
The extra compilers where set up and triggered individually, and
resulted in individual updateSourceFiles calls with the single result
file of the extra compiler. For Qt Creator this would lead to 200
calls in quick succession after project load, potentially leading to a
freeze of multiple seconds.

Instead of updating the result files of the extra compilers individually
after project load, integrate the update into the regular project source
file update. So we end up with only a single call of updateSourceFiles.
For this the project updater needs to trigger the extra compilers, and
wait for all to finish as well as the regular project part update,
before
triggering the parser.

Task-number: QTCREATORBUG-25783
Change-Id: I34f6df0fc0f96bcb42ee65019bee39cf49176c1f
Reviewed-by: Jarek Kobus <jaroslaw.kobus@qt.io>
Reviewed-by: Christian Kandeler <christian.kandeler@qt.io>

src/plugins/cmakeprojectmanager/cmakebuildsystem.cpp

@@ -630,8 +630,7 @@ void CMakeBuildSystem::updateProjectData()
     {
         qDeleteAll(m_extraCompilers);
         m_extraCompilers = findExtraCompilers();
-        CppTools::GeneratedCodeModelSupport::update(m_extraCompilers);
+        qCDebug(cmakeBuildSystemLog) << "Extra compilers created.";
-        qCDebug(cmakeBuildSystemLog) << "Extra compilers updated.";
     }
 
     QtSupport::CppKitInfo kitInfo(kit());
@@ -658,7 +657,8 @@ void CMakeBuildSystem::updateProjectData()
             }
         }
 
-        m_cppCodeModelUpdater->update({p, kitInfo, cmakeBuildConfiguration()->environment(), rpps});
+        m_cppCodeModelUpdater->update({p, kitInfo, cmakeBuildConfiguration()->environment(), rpps},
+                                      m_extraCompilers);
     }
     {
         const bool mergedHeaderPathsAndQmlImportPaths = kit()->value(

src/plugins/cpptools/cppmodelmanager.cpp

@@ -1145,7 +1145,8 @@ void CppModelManager::updateCppEditorDocuments(bool projectsUpdated) const
     }
 }
 
-QFuture<void> CppModelManager::updateProjectInfo(const ProjectInfo &newProjectInfo)
+QFuture<void> CppModelManager::updateProjectInfo(const ProjectInfo &newProjectInfo,
+                                                 const QSet<QString> &additionalFiles)
 {
     if (!newProjectInfo.isValid())
         return QFuture<void>();
@@ -1236,6 +1237,7 @@ QFuture<void> CppModelManager::updateProjectInfo(const ProjectInfo &newProjectIn
     // resolved includes that we could rely on.
     updateCppEditorDocuments(/*projectsUpdated = */ true);
 
+    filesToReindex.unite(additionalFiles);
     // Trigger reindexing
     const QFuture<void> indexingFuture = updateSourceFiles(filesToReindex,
                                                            ForcedProgressNotification);

src/plugins/cpptools/cppmodelmanager.h

@@ -116,7 +116,8 @@ public:
 
     QList<ProjectInfo> projectInfos() const;
     ProjectInfo projectInfo(ProjectExplorer::Project *project) const;
-    QFuture<void> updateProjectInfo(const ProjectInfo &newProjectInfo);
+    QFuture<void> updateProjectInfo(const ProjectInfo &newProjectInfo,
+                                    const QSet<QString> &additionalFiles = {});
 
     /// \return The project part with the given project file
     ProjectPart::Ptr projectPartForId(const QString &projectPartId) const override;

src/plugins/cpptools/cppprojectupdater.cpp

@@ -27,14 +27,21 @@
 
 #include "cppmodelmanager.h"
 #include "cppprojectinfogenerator.h"
+#include "generatedcodemodelsupport.h"
+
+#include <coreplugin/progressmanager/progressmanager.h>
 
 #include <projectexplorer/toolchainmanager.h>
 
+#include <utils/algorithm.h>
+#include <utils/fileutils.h>
 #include <utils/qtcassert.h>
 #include <utils/runextensions.h>
 
 #include <QFutureInterface>
 
+using namespace ProjectExplorer;
+
 namespace CppTools {
 
 CppProjectUpdater::CppProjectUpdater()
@@ -46,11 +53,25 @@ CppProjectUpdater::CppProjectUpdater()
     m_futureSynchronizer.setCancelOnWait(true);
 }
 
-void CppProjectUpdater::update(const ProjectExplorer::ProjectUpdateInfo &projectUpdateInfo)
+CppProjectUpdater::~CppProjectUpdater()
+{
+    cancel();
+}
+
+void CppProjectUpdater::update(const ProjectUpdateInfo &projectUpdateInfo)
+{
+    update(projectUpdateInfo, {});
+}
+
+void CppProjectUpdater::update(const ProjectUpdateInfo &projectUpdateInfo,
+                               const QList<ProjectExplorer::ExtraCompiler *> &extraCompilers)
 {
     // Stop previous update.
     cancel();
 
+    m_extraCompilers = Utils::transform(extraCompilers, [](ExtraCompiler *compiler) {
+        return QPointer<ExtraCompiler>(compiler);
+    });
     m_projectUpdateInfo = projectUpdateInfo;
 
     // Ensure that we do not operate on a deleted toolchain.
@@ -68,15 +89,52 @@ void CppProjectUpdater::update(const ProjectExplorer::ProjectUpdateInfo &project
     });
     m_generateFutureWatcher.setFuture(generateFuture);
     m_futureSynchronizer.addFuture(generateFuture);
+
+    // extra compilers
+    for (QPointer<ExtraCompiler> compiler : qAsConst(m_extraCompilers)) {
+        if (compiler->isDirty()) {
+            auto watcher = new QFutureWatcher<void>;
+            // queued connection to delay after the extra compiler updated its result contents,
+            // which is also done in the main thread when compiler->run() finished
+            connect(watcher, &QFutureWatcherBase::finished,
+                    this, [this, watcher] {
+                        m_projectUpdateFutureInterface->setProgressValue(
+                            m_projectUpdateFutureInterface->progressValue() + 1);
+                        m_extraCompilersFutureWatchers.remove(watcher);
+                        watcher->deleteLater();
+                        if (!watcher->isCanceled())
+                            checkForExtraCompilersFinished();
+                    },
+                    Qt::QueuedConnection);
+            m_extraCompilersFutureWatchers += watcher;
+            watcher->setFuture(QFuture<void>(compiler->run()));
+            m_futureSynchronizer.addFuture(watcher->future());
+        }
+    }
+
+    m_projectUpdateFutureInterface.reset(new QFutureInterface<void>);
+    m_projectUpdateFutureInterface->setProgressRange(0, m_extraCompilersFutureWatchers.size()
+                                                        + 1 /*generateFuture*/);
+    m_projectUpdateFutureInterface->setProgressValue(0);
+    m_projectUpdateFutureInterface->reportStarted();
+    Core::ProgressManager::addTask(m_projectUpdateFutureInterface->future(),
+                                   tr("Preparing C++ Code Model"),
+                                   "CppProjectUpdater");
 }
 
 void CppProjectUpdater::cancel()
 {
+    if (m_projectUpdateFutureInterface && m_projectUpdateFutureInterface->isRunning())
+        m_projectUpdateFutureInterface->reportFinished();
     m_generateFutureWatcher.setFuture({});
+    m_isProjectInfoGenerated = false;
+    qDeleteAll(m_extraCompilersFutureWatchers);
+    m_extraCompilersFutureWatchers.clear();
+    m_extraCompilers.clear();
     m_futureSynchronizer.cancelAllFutures();
 }
 
-void CppProjectUpdater::onToolChainRemoved(ProjectExplorer::ToolChain *t)
+void CppProjectUpdater::onToolChainRemoved(ToolChain *t)
 {
     QTC_ASSERT(t, return);
     if (t == m_projectUpdateInfo.cToolChain || t == m_projectUpdateInfo.cxxToolChain)
@@ -93,8 +151,33 @@ void CppProjectUpdater::onProjectInfoGenerated()
     if (m_generateFutureWatcher.isCanceled() || m_generateFutureWatcher.future().resultCount() < 1)
         return;
 
-    auto updateFuture = CppModelManager::instance()->updateProjectInfo(
+    m_projectUpdateFutureInterface->setProgressValue(m_projectUpdateFutureInterface->progressValue()
-        m_generateFutureWatcher.result());
+                                                     + 1);
+    m_isProjectInfoGenerated = true;
+    checkForExtraCompilersFinished();
+}
+
+void CppProjectUpdater::checkForExtraCompilersFinished()
+{
+    if (!m_extraCompilersFutureWatchers.isEmpty() || !m_isProjectInfoGenerated)
+        return; // still need to wait
+
+    m_projectUpdateFutureInterface->reportFinished();
+    m_projectUpdateFutureInterface.reset();
+
+    QList<ExtraCompiler *> extraCompilers;
+    QSet<QString> compilerFiles;
+    for (const QPointer<ExtraCompiler> &compiler : qAsConst(m_extraCompilers)) {
+        if (compiler) {
+            extraCompilers += compiler.data();
+            compilerFiles += Utils::transform<QSet>(compiler->targets(), &Utils::FilePath::toString);
+        }
+    }
+    GeneratedCodeModelSupport::update(extraCompilers);
+    m_extraCompilers.clear();
+
+    auto updateFuture = CppModelManager::instance()
+                            ->updateProjectInfo(m_generateFutureWatcher.result(), compilerFiles);
     m_futureSynchronizer.addFuture(updateFuture);
 }
 

src/plugins/cpptools/cppprojectupdater.h

@@ -29,6 +29,7 @@
 #include "cpptools_global.h"
 #include "projectinfo.h"
 
+#include <projectexplorer/extracompiler.h>
 #include <utils/futuresynchronizer.h>
 
 #include <QFutureWatcher>
@@ -54,18 +55,26 @@ class CPPTOOLS_EXPORT CppProjectUpdater final : public QObject, public CppProjec
 
 public:
     CppProjectUpdater();
+    ~CppProjectUpdater() override;
 
     void update(const ProjectExplorer::ProjectUpdateInfo &projectUpdateInfo) override;
+    void update(const ProjectExplorer::ProjectUpdateInfo &projectUpdateInfo,
+                const QList<ProjectExplorer::ExtraCompiler *> &extraCompilers);
     void cancel() override;
 
 private:
     void onToolChainRemoved(ProjectExplorer::ToolChain *);
     void onProjectInfoGenerated();
+    void checkForExtraCompilersFinished();
 
 private:
     ProjectExplorer::ProjectUpdateInfo m_projectUpdateInfo;
+    QList<QPointer<ProjectExplorer::ExtraCompiler>> m_extraCompilers;
 
     QFutureWatcher<ProjectInfo> m_generateFutureWatcher;
+    bool m_isProjectInfoGenerated = false;
+    QSet<QFutureWatcher<void> *> m_extraCompilersFutureWatchers;
+    std::unique_ptr<QFutureInterface<void>> m_projectUpdateFutureInterface;
     Utils::FutureSynchronizer m_futureSynchronizer;
 };
 

src/plugins/projectexplorer/extracompiler.cpp

@@ -123,11 +123,6 @@ ExtraCompiler::ExtraCompiler(const Project *project, const Utils::FilePath &sour
         if (file.open(QFile::ReadOnly | QFile::Text))
             setContent(target, file.readAll());
     }
-
-    if (d->dirty) {
-        d->dirty = false;
-        QTimer::singleShot(0, this, [this]() { run(d->source); }); // delay till available.
-    }
 }
 
 ExtraCompiler::~ExtraCompiler() = default;
@@ -173,6 +168,11 @@ QThreadPool *ExtraCompiler::extraCompilerThreadPool()
     return s_extraCompilerThreadPool();
 }
 
+bool ExtraCompiler::isDirty() const
+{
+    return d->dirty;
+}
+
 void ExtraCompiler::onTargetsBuilt(Project *project)
 {
     if (project != d->project || BuildManager::isBuilding(project))
@@ -346,15 +346,16 @@ void ProcessExtraCompiler::run(const QByteArray &sourceContents)
     runImpl(contents);
 }
 
-void ProcessExtraCompiler::run(const Utils::FilePath &fileName)
+QFuture<FileNameToContentsHash> ProcessExtraCompiler::run()
 {
+    const Utils::FilePath fileName = source();
     ContentProvider contents = [fileName]() {
         QFile file(fileName.toString());
         if (!file.open(QFile::ReadOnly | QFile::Text))
             return QByteArray();
         return file.readAll();
     };
-    runImpl(contents);
+    return runImpl(contents);
 }
 
 Utils::FilePath ProcessExtraCompiler::workingDirectory() const
@@ -379,7 +380,7 @@ Tasks ProcessExtraCompiler::parseIssues(const QByteArray &stdErr)
     return {};
 }
 
-void ProcessExtraCompiler::runImpl(const ContentProvider &provider)
+QFuture<FileNameToContentsHash> ProcessExtraCompiler::runImpl(const ContentProvider &provider)
 {
     if (m_watcher)
         delete m_watcher;
@@ -392,6 +393,7 @@ void ProcessExtraCompiler::runImpl(const ContentProvider &provider)
                                          &ProcessExtraCompiler::runInThread, this,
                                          command(), workingDirectory(), arguments(), provider,
                                          buildEnvironment()));
+    return m_watcher->future();
 }
 
 void ProcessExtraCompiler::runInThread(

src/plugins/projectexplorer/extracompiler.h

@@ -80,6 +80,9 @@ public:
 
     static QThreadPool *extraCompilerThreadPool();
 
+    virtual QFuture<FileNameToContentsHash> run() = 0;
+    bool isDirty() const;
+
 signals:
     void contentsChanged(const Utils::FilePath &file);
 
@@ -94,7 +97,6 @@ private:
     void setDirty();
     // This method may not block!
     virtual void run(const QByteArray &sourceContent) = 0;
-    virtual void run(const Utils::FilePath &file) = 0;
 
     const std::unique_ptr<ExtraCompilerPrivate> d;
 };
@@ -115,7 +117,7 @@ protected:
     //  * prepareToRun returns true
     //  * The process is not yet running
     void run(const QByteArray &sourceContents) override;
-    void run(const Utils::FilePath &fileName) override;
+    QFuture<FileNameToContentsHash> run() override;
 
     // Information about the process to run:
     virtual Utils::FilePath workingDirectory() const;
@@ -133,7 +135,7 @@ protected:
 
 private:
     using ContentProvider = std::function<QByteArray()>;
-    void runImpl(const ContentProvider &sourceContents);
+    QFuture<FileNameToContentsHash> runImpl(const ContentProvider &sourceContents);
     void runInThread(QFutureInterface<FileNameToContentsHash> &futureInterface,
                      const Utils::FilePath &cmd, const Utils::FilePath &workDir,
                      const QStringList &args, const ContentProvider &provider,

src/plugins/qbsprojectmanager/qbsproject.cpp

@@ -193,6 +193,10 @@ QbsBuildSystem::QbsBuildSystem(QbsBuildConfiguration *bc)
             }
         }
         CppTools::GeneratedCodeModelSupport::update(m_extraCompilers);
+        for (ExtraCompiler *compiler : m_extraCompilers) {
+            if (compiler->isDirty())
+                compiler->run();
+        }
         m_sourcesForGeneratedFiles.clear();
     });
     connect(m_session, &QbsSession::errorOccurred, this, [](QbsSession::Error e) {

src/plugins/qmakeprojectmanager/qmakeproject.cpp

@@ -399,8 +399,7 @@ void QmakeBuildSystem::updateCppCodeModel()
         rpps.append(rpp);
     }
 
-    CppTools::GeneratedCodeModelSupport::update(generators);
+    m_cppCodeModelUpdater->update({project(), kitInfo, activeParseEnvironment(), rpps}, generators);
-    m_cppCodeModelUpdater->update({project(), kitInfo, activeParseEnvironment(), rpps});
 }
 
 void QmakeBuildSystem::updateQmlJSCodeModel()