QMakeProjectManager: Move some parsing code out of the UI thread

When parsing larger qmake project, the callbacks from the parser threads
are currently overloading the UI thread, often rendering the application
non-responsive until the project is completely loaded.
This patch moves some expensive operations from the UI thread into the
parser threads, at the cost of a somewhat ugly two-stage setup for some
types of objects.
On my Linux machine, I measured that the time spent in parser callback code
went down by almost 50% when loading the Qt Creator super project.

Task-number: QTCREATORBUG-18533
Change-Id: If9624da5b07e81a50c180693580b20a70e1aaea7
Reviewed-by: hjk <hjk@qt.io>

src/plugins/qmakeprojectmanager/qmakeparsernodes.cpp

@@ -118,6 +118,8 @@ public:
     QtSupport::ProFileReader *readerCumulative;
     QMakeGlobals *qmakeGlobals;
     QMakeVfs *qmakeVfs;
+    QSet<FilePath> parentFilePaths;
+    bool includedInExcactParse;
 };
 
 class QmakePriFileEvalResult
@@ -146,6 +148,8 @@ public:
 class QmakeEvalResult
 {
 public:
+    ~QmakeEvalResult() { qDeleteAll(directChildren); }
+
     enum EvalResultState { EvalAbort, EvalFail, EvalPartial, EvalOk };
     EvalResultState state;
     ProjectType projectType;
@@ -158,23 +162,33 @@ public:
     QHash<Variable, QStringList> newVarValues;
     QStringList errors;
     QSet<QString> directoriesWithWildcards;
+    QList<QmakePriFile *> directChildren;
+    QList<QPair<QmakePriFile *, QmakePriFileEvalResult>> priFiles;
+    QList<QmakeProFile *> proFiles;
 };
 
 } // namespace Internal
 
 QmakePriFile::QmakePriFile(QmakeBuildSystem *buildSystem, QmakeProFile *qmakeProFile,
-                           const FilePath &filePath) :
+                           const FilePath &filePath) : m_filePath(filePath)
-    m_buildSystem(buildSystem),
-    m_qmakeProFile(qmakeProFile)
 {
-    Q_ASSERT(buildSystem);
+    finishInitialization(buildSystem, qmakeProFile);
-    m_priFileDocument = std::make_unique<QmakePriFileDocument>(this, filePath);
+}
+
+QmakePriFile::QmakePriFile(const FilePath &filePath) : m_filePath(filePath) { }
+
+void QmakePriFile::finishInitialization(QmakeBuildSystem *buildSystem, QmakeProFile *qmakeProFile)
+{
+    QTC_ASSERT(buildSystem, return);
+    m_buildSystem = buildSystem;
+    m_qmakeProFile = qmakeProFile;
+    m_priFileDocument = std::make_unique<QmakePriFileDocument>(this, filePath());
     Core::DocumentManager::addDocument(m_priFileDocument.get());
 }
 
 FilePath QmakePriFile::filePath() const
 {
-    return m_priFileDocument->filePath();
+    return m_filePath;
 }
 
 FilePath QmakePriFile::directoryPath() const
@@ -1180,23 +1194,30 @@ QByteArray QmakeProFile::cxxDefines() const
 QmakeProFile::QmakeProFile(QmakeBuildSystem *buildSystem, const FilePath &filePath) :
     QmakePriFile(buildSystem, this, filePath)
 {
-    // The lifetime of the m_parserFutureWatcher is shorter
+    setupFutureWatcher();
-    // than of this, so this is all safe
-    QObject::connect(&m_parseFutureWatcher, &QFutureWatcherBase::finished,
-                     [this](){ applyAsyncEvaluate(); });
 }
 
+QmakeProFile::QmakeProFile(const FilePath &filePath) : QmakePriFile(filePath) { }
+
 QmakeProFile::~QmakeProFile()
 {
     qDeleteAll(m_extraCompilers);
-    m_parseFutureWatcher.cancel();
+    m_parseFutureWatcher->cancel();
-    m_parseFutureWatcher.waitForFinished();
+    m_parseFutureWatcher->waitForFinished();
+    delete m_parseFutureWatcher;
     if (m_readerExact)
         applyAsyncEvaluate();
 
     cleanupProFileReaders();
 }
 
+void QmakeProFile::setupFutureWatcher()
+{
+    m_parseFutureWatcher = new QFutureWatcher<Internal::QmakeEvalResult *>;
+    QObject::connect(m_parseFutureWatcher, &QFutureWatcherBase::finished,
+                     [this](){ applyAsyncEvaluate(); });
+}
+
 bool QmakeProFile::isParent(QmakeProFile *node)
 {
     while ((node = dynamic_cast<QmakeProFile *>(node->parent()))) {
@@ -1287,13 +1308,13 @@ void QmakeProFile::asyncUpdate()
     setupReader();
     if (!includedInExactParse())
         m_readerExact->setExact(false);
-    m_parseFutureWatcher.waitForFinished();
+    m_parseFutureWatcher->waitForFinished();
     QmakeEvalInput input = evalInput();
     QFuture<QmakeEvalResult *> future = Utils::runAsync(ProjectExplorerPlugin::sharedThreadPool(),
                                                         QThread::LowestPriority,
                                                         &QmakeProFile::asyncEvaluate,
                                                         this, input);
-    m_parseFutureWatcher.setFuture(future);
+    m_parseFutureWatcher->setFuture(future);
 }
 
 bool QmakeProFile::isFileFromWildcard(const QString &filePath) const
@@ -1315,6 +1336,9 @@ QmakeEvalInput QmakeProFile::evalInput() const
     input.readerCumulative = m_readerCumulative;
     input.qmakeGlobals = m_buildSystem->qmakeGlobals();
     input.qmakeVfs = m_buildSystem->qmakeVfs();
+    input.includedInExcactParse = includedInExactParse();
+    for (const QmakePriFile *pri = this; pri; pri = pri->parent())
+        input.parentFilePaths.insert(pri->filePath());
     return input;
 }
 
@@ -1580,6 +1604,47 @@ QmakeEvalResult *QmakeProFile::evaluate(const QmakeEvalInput &input)
     if (cumulativeBuildPassReader && cumulativeBuildPassReader != input.readerCumulative)
         delete cumulativeBuildPassReader;
 
+    QList<QPair<QmakePriFile *, QmakeIncludedPriFile *>> toCompare;
+    toCompare.append(qMakePair(nullptr, &result->includedFiles));
+    while (!toCompare.isEmpty()) {
+        QmakePriFile *pn = toCompare.first().first;
+        QmakeIncludedPriFile *tree = toCompare.first().second;
+        toCompare.pop_front();
+
+        // Loop prevention: Make sure that exact same node is not in our parent chain
+        for (QmakeIncludedPriFile *priFile : tree->children) {
+            bool loop = input.parentFilePaths.contains(priFile->name);
+            for (const QmakePriFile *n = pn; n && !loop; n = n->parent()) {
+                if (n->filePath() == priFile->name)
+                    loop = true;
+            }
+            if (loop)
+                continue; // Do nothing
+
+            if (priFile->proFile) {
+                auto *qmakePriFileNode = new QmakePriFile(priFile->name);
+                if (pn)
+                    pn->addChild(qmakePriFileNode);
+                else
+                    result->directChildren << qmakePriFileNode;
+                qmakePriFileNode->setIncludedInExactParse(input.includedInExcactParse
+                        && result->state == QmakeEvalResult::EvalOk);
+                result->priFiles.append(qMakePair(qmakePriFileNode, priFile->result));
+                toCompare.append(qMakePair(qmakePriFileNode, priFile));
+            } else {
+                auto *qmakeProFileNode = new QmakeProFile(priFile->name);
+                if (pn)
+                    pn->addChild(qmakeProFileNode);
+                else
+                    result->directChildren << qmakeProFileNode;
+                qmakeProFileNode->setIncludedInExactParse(input.includedInExcactParse
+                            && result->exactSubdirs.contains(qmakeProFileNode->filePath()));
+                qmakeProFileNode->setParseInProgress(true);
+                result->proFiles << qmakeProFileNode;
+            }
+        }
+    }
+
     return result;
 }
 
@@ -1591,8 +1656,8 @@ void QmakeProFile::asyncEvaluate(QFutureInterface<QmakeEvalResult *> &fi, QmakeE
 
 void QmakeProFile::applyAsyncEvaluate()
 {
-    if (m_parseFutureWatcher.isFinished())
+    if (m_parseFutureWatcher->isFinished())
-        applyEvaluate(m_parseFutureWatcher.result());
+        applyEvaluate(m_parseFutureWatcher->result());
     m_buildSystem->decrementPendingEvaluateFutures();
 }
 
@@ -1655,53 +1720,22 @@ void QmakeProFile::applyEvaluate(QmakeEvalResult *evalResult)
     //
     // Add/Remove pri files, sub projects
     //
-
     FilePath buildDirectory = buildDir();
-
-    QList<QPair<QmakePriFile *, QmakeIncludedPriFile *>> toCompare;
-
-    toCompare.append(qMakePair(this, &result->includedFiles));
-
     makeEmpty();
+    for (QmakePriFile * const toAdd : qAsConst(result->directChildren))
+        addChild(toAdd);
+    result->directChildren.clear();
 
-    while (!toCompare.isEmpty()) {
+    for (const auto priFiles : qAsConst(result->priFiles)) {
-        QmakePriFile *pn = toCompare.first().first;
+        priFiles.first->finishInitialization(m_buildSystem, this);
-        QmakeIncludedPriFile *tree = toCompare.first().second;
+        priFiles.first->update(priFiles.second);
-        toCompare.pop_front();
-
-        for (QmakeIncludedPriFile *priFile : tree->children) {
-            // Loop preventation, make sure that exact same node is not in our parent chain
-            bool loop = false;
-            QmakePriFile *n = pn;
-            while ((n = n->parent())) {
-                if (n->filePath() == priFile->name) {
-                    loop = true;
-                    break;
-                }
-            }
-
-            if (loop)
-                continue; // Do nothing
-
-            if (priFile->proFile) {
-                auto *qmakePriFileNode = new QmakePriFile(m_buildSystem, this, priFile->name);
-                pn->addChild(qmakePriFileNode);
-                qmakePriFileNode->setIncludedInExactParse(
-                            (result->state == QmakeEvalResult::EvalOk) && pn->includedInExactParse());
-                qmakePriFileNode->update(priFile->result);
-                toCompare.append(qMakePair(qmakePriFileNode, priFile));
-            } else {
-                auto *qmakeProFileNode = new QmakeProFile(m_buildSystem, priFile->name);
-                pn->addChild(qmakeProFileNode);
-                qmakeProFileNode->setIncludedInExactParse(
-                            result->exactSubdirs.contains(qmakeProFileNode->filePath())
-                            && pn->includedInExactParse());
-                qmakeProFileNode->setParseInProgress(true);
-                qmakeProFileNode->asyncUpdate();
-            }
-        }
     }
 
+    for (QmakeProFile * const proFile : qAsConst(result->proFiles)) {
+        proFile->finishInitialization(m_buildSystem, proFile);
+        proFile->setupFutureWatcher();
+        proFile->asyncUpdate();
+    }
     QmakePriFile::update(result->includedFiles.result);
 
     m_validParse = (result->state == QmakeEvalResult::EvalOk);

src/plugins/qmakeprojectmanager/qmakeparsernodes.h

@@ -125,8 +125,10 @@ class QMAKEPROJECTMANAGER_EXPORT QmakePriFile
 {
 public:
     QmakePriFile(QmakeBuildSystem *buildSystem, QmakeProFile *qmakeProFile, const Utils::FilePath &filePath);
+    explicit QmakePriFile(const Utils::FilePath &filePath);
     virtual ~QmakePriFile();
 
+    void finishInitialization(QmakeBuildSystem *buildSystem, QmakeProFile *qmakeProFile);
     Utils::FilePath filePath() const;
     Utils::FilePath directoryPath() const;
     virtual QString displayName() const;
@@ -241,6 +243,7 @@ private:
     QMap<ProjectExplorer::FileType, SourceFiles> m_files;
     QSet<Utils::FilePath> m_recursiveEnumerateFiles; // FIXME: Remove this?!
     QSet<QString> m_watchedFolders;
+    const Utils::FilePath m_filePath;
     bool m_includedInExactParse = true;
 
     friend class QmakeProFile;
@@ -294,8 +297,11 @@ class QMAKEPROJECTMANAGER_EXPORT QmakeProFile : public QmakePriFile
 {
 public:
     QmakeProFile(QmakeBuildSystem *buildSystem, const Utils::FilePath &filePath);
+    explicit QmakeProFile(const Utils::FilePath &filePath);
     ~QmakeProFile() override;
 
+    void setupFutureWatcher();
+
     bool isParent(QmakeProFile *node);
     QString displayName() const final;
 
@@ -390,7 +396,7 @@ private:
     QMap<QString, QStringList> m_wildcardDirectoryContents;
 
     // Async stuff
-    QFutureWatcher<Internal::QmakeEvalResult *> m_parseFutureWatcher;
+    QFutureWatcher<Internal::QmakeEvalResult *> *m_parseFutureWatcher = nullptr;
     QtSupport::ProFileReader *m_readerExact = nullptr;
     QtSupport::ProFileReader *m_readerCumulative = nullptr;
 };

src/plugins/qmakeprojectmanager/qmakeproject.cpp

@@ -67,6 +67,7 @@
 #include <qtsupport/qtversionmanager.h>
 
 #include <utils/algorithm.h>
+#include <utils/runextensions.h>
 #include <qmljs/qmljsmodelmanagerinterface.h>
 
 #include <QDebug>
@@ -710,16 +711,19 @@ QString QmakeBuildSystem::qmakeSysroot()
 
 void QmakeBuildSystem::destroyProFileReader(QtSupport::ProFileReader *reader)
 {
-    delete reader;
+    // The ProFileReader destructor is super expensive (but thread-safe).
-    if (!--m_qmakeGlobalsRefCnt) {
+    const auto deleteFuture = runAsync(ProjectExplorerPlugin::sharedThreadPool(), QThread::LowestPriority,
-        QString dir = projectFilePath().toString();
+                    [reader] { delete reader; });
-        if (!dir.endsWith(QLatin1Char('/')))
+    onFinished(deleteFuture, this, [this](const QFuture<void> &) {
-            dir += QLatin1Char('/');
+        if (!--m_qmakeGlobalsRefCnt) {
-        QtSupport::ProFileCacheManager::instance()->discardFiles(dir, qmakeVfs());
+            QString dir = projectFilePath().toString();
-        QtSupport::ProFileCacheManager::instance()->decRefCount();
+            if (!dir.endsWith(QLatin1Char('/')))
-
+                dir += QLatin1Char('/');
-        m_qmakeGlobals.reset();
+            QtSupport::ProFileCacheManager::instance()->discardFiles(dir, qmakeVfs());
-    }
+            QtSupport::ProFileCacheManager::instance()->decRefCount();
+            m_qmakeGlobals.reset();
+        }
+    });
 }
 
 void QmakeBuildSystem::activeTargetWasChanged(Target *t)