ExtraCompiler: Run extra compiler in a thread

and make sure there are not too many of these threads
running at any time. This stops the massive process
startup when loading a project with many UI files, etc.

Task-number: QTCREATORBUG-15795
Change-Id: Icfcddd80d04e36b61ecafbbefe5a1a8b7ea02ec6
Reviewed-by: Tobias Hunger <tobias.hunger@theqtcompany.com>
Reviewed-by: Ulf Hermann <ulf.hermann@theqtcompany.com>

src/plugins/cpptools/generatedcodemodelsupport.cpp

@@ -98,7 +98,7 @@ void GeneratedCodeModelSupport::onContentsChanged(const Utils::FileName &file)
 void GeneratedCodeModelSupport::init() const
 {
     connect(m_generator, &ProjectExplorer::ExtraCompiler::contentsChanged,
-            this, &GeneratedCodeModelSupport::onContentsChanged);
+            this, &GeneratedCodeModelSupport::onContentsChanged, Qt::QueuedConnection);
 }
 
 QByteArray GeneratedCodeModelSupport::contents() const

src/plugins/projectexplorer/extracompiler.cpp

@@ -24,26 +24,33 @@
 ****************************************************************************/
 
 #include "extracompiler.h"
+
+#include "buildconfiguration.h"
 #include "buildmanager.h"
+#include "kitinformation.h"
 #include "session.h"
 #include "target.h"
-#include "buildconfiguration.h"
-#include "kitinformation.h"
 
+#include <coreplugin/editormanager/editormanager.h>
+#include <coreplugin/idocument.h>
 #include <texteditor/texteditor.h>
 #include <texteditor/texteditorsettings.h>
 #include <texteditor/texteditorconstants.h>
 #include <texteditor/fontsettings.h>
+
 #include <utils/qtcassert.h>
-#include <coreplugin/idocument.h>
+#include <utils/runextensions.h>
-#include <coreplugin/editormanager/editormanager.h>
 
 #include <QDateTime>
+#include <QFutureWatcher>
+#include <QProcess>
+#include <QThreadPool>
 #include <QTimer>
 #include <QTextBlock>
 
 namespace ProjectExplorer {
 
+Q_GLOBAL_STATIC(QThreadPool, s_extraCompilerThreadPool);
 Q_GLOBAL_STATIC(QList<ExtraCompilerFactory *>, factories);
 
 class ExtraCompilerPrivate
@@ -55,7 +62,7 @@ public:
     Utils::FileNameList targets;
     QList<Task> issues;
     QDateTime compileTime;
-    Core::IEditor *lastEditor = 0;
+    Core::IEditor *lastEditor = nullptr;
     QMetaObject::Connection activeBuildConfigConnection;
     QMetaObject::Connection activeEnvironmentConnection;
     bool dirty = false;
@@ -80,8 +87,8 @@ ExtraCompiler::ExtraCompiler(const Project *project, const Utils::FileName &sour
 
     connect(&d->timer, &QTimer::timeout, this, [this](){
         if (d->dirty && d->lastEditor) {
-            run(d->lastEditor->document()->contents());
             d->dirty = false;
+            run(d->lastEditor->document()->contents());
         }
     });
 
@@ -122,13 +129,8 @@ ExtraCompiler::ExtraCompiler(const Project *project, const Utils::FileName &sour
     }
 
     if (d->dirty) {
-        // Run in the next event loop, as run() is not available yet in the ctor.
+        d->dirty = false;
-        QTimer::singleShot(0, this, [this](){
+        QTimer::singleShot(0, this, [this]() { run(d->source); }); // delay till available.
-            QFile file(d->source.toString());
-            if (file.open(QFile::ReadOnly | QFile::Text))
-                run(file.readAll());
-            d->dirty = false;
-        });
     }
 }
 
@@ -167,6 +169,11 @@ QDateTime ExtraCompiler::compileTime() const
     return d->compileTime;
 }
 
+QThreadPool *ExtraCompiler::extraCompilerThreadPool()
+{
+    return s_extraCompilerThreadPool();
+}
+
 void ExtraCompiler::onTargetsBuilt(Project *project)
 {
     if (project != d->project || BuildManager::isBuilding(project))
@@ -203,8 +210,8 @@ void ExtraCompiler::onEditorChanged(Core::IEditor *editor)
                    this, &ExtraCompiler::setDirty);
 
         if (d->dirty) {
-            run(doc->contents());
             d->dirty = false;
+            run(doc->contents());
         }
     }
 
@@ -216,7 +223,7 @@ void ExtraCompiler::onEditorChanged(Core::IEditor *editor)
         connect(d->lastEditor->document(), &Core::IDocument::contentsChanged,
                 this, &ExtraCompiler::setDirty);
     } else {
-        d->lastEditor = 0;
+        d->lastEditor = nullptr;
     }
 }
 
@@ -237,10 +244,10 @@ void ExtraCompiler::onEditorAboutToClose(Core::IEditor *editor)
     disconnect(doc, &Core::IDocument::contentsChanged,
                this, &ExtraCompiler::setDirty);
     if (d->dirty) {
-        run(doc->contents());
         d->dirty = false;
+        run(doc->contents());
     }
-    d->lastEditor = 0;
+    d->lastEditor = nullptr;
 }
 
 void ExtraCompiler::onActiveTargetChanged()
@@ -355,4 +362,114 @@ QList<ExtraCompilerFactory *> ExtraCompilerFactory::extraCompilerFactories()
     return *factories();
 }
 
+ProcessExtraCompiler::ProcessExtraCompiler(const Project *project, const Utils::FileName &source,
+                                           const Utils::FileNameList &targets, QObject *parent) :
+    ExtraCompiler(project, source, targets, parent)
+{ }
+
+void ProcessExtraCompiler::run(const QByteArray &sourceContents)
+{
+    ContentProvider contents = [this, sourceContents]() { return sourceContents; };
+    runImpl(contents);
+}
+
+void ProcessExtraCompiler::run(const Utils::FileName &fileName)
+{
+    ContentProvider contents = [this, fileName]() {
+        QFile file(fileName.toString());
+        if (!file.open(QFile::ReadOnly | QFile::Text))
+            return QByteArray();
+        return file.readAll();
+    };
+    runImpl(contents);
+}
+
+Utils::FileName ProcessExtraCompiler::workingDirectory() const
+{
+    return Utils::FileName();
+}
+
+QStringList ProcessExtraCompiler::arguments() const
+{
+    return QStringList();
+}
+
+bool ProcessExtraCompiler::prepareToRun(const QByteArray &sourceContents)
+{
+    Q_UNUSED(sourceContents);
+    return true;
+}
+
+QList<Task> ProcessExtraCompiler::parseIssues(const QByteArray &stdErr)
+{
+    Q_UNUSED(stdErr);
+    return QList<Task>();
+}
+
+void ProcessExtraCompiler::runImpl(const ContentProvider &provider)
+{
+    if (m_watcher)
+        delete m_watcher;
+
+    m_watcher = new QFutureWatcher<QList<QByteArray>>();
+    connect(m_watcher, &QFutureWatcher<QList<QByteArray>>::finished,
+            this, &ProcessExtraCompiler::cleanUp);
+
+    m_watcher->setFuture(Utils::runAsync(extraCompilerThreadPool(),
+                                         &ProcessExtraCompiler::runInThread, this,
+                                         command(), workingDirectory(), arguments(), provider,
+                                         buildEnvironment()));
+}
+
+QList<QByteArray> ProcessExtraCompiler::runInThread(const Utils::FileName &cmd, const Utils::FileName &workDir,
+                                                    const QStringList &args, const ContentProvider &provider,
+                                                    const Utils::Environment &env)
+{
+    if (cmd.isEmpty() || !cmd.toFileInfo().isExecutable())
+        return QList<QByteArray>();
+
+    const QByteArray sourceContents = provider();
+    if (sourceContents.isNull() || !prepareToRun(sourceContents))
+        return QList<QByteArray>();
+
+    QProcess process;
+
+    process.setProcessEnvironment(env.toProcessEnvironment());
+    if (!workDir.isEmpty())
+        process.setWorkingDirectory(workDir.toString());
+    process.start(cmd.toString(), args, QIODevice::ReadWrite);
+    if (!process.waitForStarted()) {
+        handleProcessError(&process);
+        return QList<QByteArray>();
+    }
+    handleProcessStarted(&process, sourceContents);
+    process.waitForFinished();
+
+    if (process.state() == QProcess::Running) {
+        process.kill();
+        process.waitForFinished(3000);
+    }
+
+    return handleProcessFinished(&process);
+}
+
+void ProcessExtraCompiler::cleanUp()
+{
+    QTC_ASSERT(m_watcher, return);
+    const QList<QByteArray> data = m_watcher->future().result();
+    delete m_watcher;
+    m_watcher = nullptr;
+
+    if (data.isEmpty())
+        return; // There was some kind of error...
+
+    const Utils::FileNameList targetList = targets();
+    QTC_ASSERT(data.count() == targetList.count(), return);
+
+    for (int i = 0; i < targetList.count(); ++i)
+        setContent(targetList.at(i), data.at(i));
+
+    setCompileTime(QDateTime::currentDateTime());
+}
+
 } // namespace ProjectExplorer

src/plugins/projectexplorer/extracompiler.h

@@ -33,6 +33,13 @@
 #include <utils/fileutils.h>
 #include <utils/environment.h>
 
+#include <QByteArray>
+#include <QFuture>
+#include <QList>
+
+QT_FORWARD_DECLARE_CLASS(QProcess);
+QT_FORWARD_DECLARE_CLASS(QThreadPool);
+
 namespace ProjectExplorer {
 
 class ExtraCompilerPrivate;
@@ -42,8 +49,8 @@ class PROJECTEXPLORER_EXPORT ExtraCompiler : public QObject
 public:
 
     ExtraCompiler(const Project *project, const Utils::FileName &source,
-                  const Utils::FileNameList &targets, QObject *parent = 0);
+                  const Utils::FileNameList &targets, QObject *parent = nullptr);
-    virtual ~ExtraCompiler() override;
+    ~ExtraCompiler() override;
 
     const Project *project() const;
     Utils::FileName source() const;
@@ -58,6 +65,8 @@ public:
     void setCompileTime(const QDateTime &time);
     QDateTime compileTime() const;
 
+    static QThreadPool *extraCompilerThreadPool();
+
 signals:
     void contentsChanged(const Utils::FileName &file);
 
@@ -72,16 +81,60 @@ private:
     void onActiveTargetChanged();
     void onActiveBuildConfigurationChanged();
     void setDirty();
+    // This method may not block!
     virtual void run(const QByteArray &sourceContent) = 0;
+    virtual void run(const Utils::FileName &file) = 0;
 
     ExtraCompilerPrivate *const d;
 };
 
+class PROJECTEXPLORER_EXPORT ProcessExtraCompiler : public ExtraCompiler
+{
+    Q_OBJECT
+public:
+
+    ProcessExtraCompiler(const Project *project, const Utils::FileName &source,
+                         const Utils::FileNameList &targets, QObject *parent = nullptr);
+
+protected:
+    // This will run a process in a thread, if
+    //  * command() does not return an empty file name
+    //  * command() is exectuable
+    //  * prepareToRun returns true
+    //  * The process is not yet running
+    void run(const QByteArray &sourceContents) override;
+    void run(const Utils::FileName &fileName) override;
+
+    // Information about the process to run:
+    virtual Utils::FileName workingDirectory() const;
+    virtual Utils::FileName command() const = 0;
+    virtual QStringList arguments() const;
+
+    virtual bool prepareToRun(const QByteArray &sourceContents);
+
+    virtual void handleProcessError(QProcess *process) { Q_UNUSED(process); }
+    virtual void handleProcessStarted(QProcess *process, const QByteArray &sourceContents)
+    { Q_UNUSED(process); Q_UNUSED(sourceContents); }
+    virtual QList<QByteArray> handleProcessFinished(QProcess *process) = 0;
+
+    virtual QList<Task> parseIssues(const QByteArray &stdErr);
+
+private:
+    using ContentProvider = std::function<QByteArray()>;
+    void runImpl(const ContentProvider &sourceContents);
+    QList<QByteArray> runInThread(const Utils::FileName &cmd, const Utils::FileName &workDir,
+                                  const QStringList &args, const ContentProvider &provider,
+                                  const Utils::Environment &env);
+    void cleanUp();
+
+    QFutureWatcher<QList<QByteArray>> *m_watcher = nullptr;
+};
+
 class PROJECTEXPLORER_EXPORT ExtraCompilerFactory : public QObject
 {
     Q_OBJECT
 public:
-    explicit ExtraCompilerFactory(QObject *parent = 0);
+    explicit ExtraCompilerFactory(QObject *parent = nullptr);
 
     virtual FileType sourceType() const = 0;
     virtual QString sourceTag() const = 0;

src/plugins/qtsupport/qscxmlcgenerator.cpp

@@ -42,13 +42,10 @@ static const char TaskCategory[] = "Task.Category.ExtraCompiler.QScxmlc";
 QScxmlcGenerator::QScxmlcGenerator(const ProjectExplorer::Project *project,
                                    const Utils::FileName &source,
                                    const Utils::FileNameList &targets, QObject *parent) :
-    ProjectExplorer::ExtraCompiler(project, source, targets, parent)
+    ProjectExplorer::ProcessExtraCompiler(project, source, targets, parent)
-{
+{ }
-    connect(&m_process, static_cast<void(QProcess::*)(int)>(&QProcess::finished),
-            this, &QScxmlcGenerator::finishProcess);
-}
 
-void QScxmlcGenerator::parseIssues(const QByteArray &processStderr)
+QList<ProjectExplorer::Task> QScxmlcGenerator::parseIssues(const QByteArray &processStderr)
 {
     QList<ProjectExplorer::Task> issues;
     foreach (const QByteArray &line, processStderr.split('\n')) {
@@ -64,30 +61,13 @@ void QScxmlcGenerator::parseIssues(const QByteArray &processStderr)
             issues.append(ProjectExplorer::Task(type, message, file, line, TaskCategory));
         }
     }
-    setCompileIssues(issues);
+    return issues;
 }
 
-void QScxmlcGenerator::finishProcess()
+
+Utils::FileName QScxmlcGenerator::command() const
 {
-    parseIssues(m_process.readAllStandardError());
+    QtSupport::BaseQtVersion *version = nullptr;
-
-    setCompileTime(QDateTime::currentDateTime());
-    foreach (const Utils::FileName &target, targets()) {
-        QFile generated(m_tmpdir.path() + QLatin1Char('/') + target.fileName());
-        if (!generated.open(QIODevice::ReadOnly))
-            continue;
-        setContent(target, generated.readAll());
-    }
-}
-
-void QScxmlcGenerator::run(const QByteArray &sourceContent)
-{
-    if (m_process.state() != QProcess::NotRunning) {
-        m_process.kill();
-        m_process.waitForFinished(3000);
-    }
-
-    QtSupport::BaseQtVersion *version = 0;
     ProjectExplorer::Target *target;
     if ((target = project()->activeTarget()))
         version = QtSupport::QtKitInformation::qtVersion(target->kit());
@@ -95,28 +75,61 @@ void QScxmlcGenerator::run(const QByteArray &sourceContent)
         version = QtSupport::QtKitInformation::qtVersion(ProjectExplorer::KitManager::defaultKit());
 
     if (!version)
-        return;
+        return Utils::FileName();
 
-    const QString generator = version->qscxmlcCommand();
+    return Utils::FileName::fromString(version->qscxmlcCommand());
-    if (!QFileInfo(generator).isExecutable())
+}
-        return;
 
-    m_process.setProcessEnvironment(buildEnvironment().toProcessEnvironment());
+QStringList QScxmlcGenerator::arguments() const
-    m_process.setWorkingDirectory(m_tmpdir.path());
+{
+    QTC_ASSERT(targets().count() == 2, return QStringList());
 
-    QFile input(m_tmpdir.path() + QLatin1Char('/') + source().fileName());
+    const Utils::FileName fn = tmpFile();
+    const QString header = m_tmpdir.path() + QLatin1Char('/') + targets()[0].fileName();
+    const QString impl = m_tmpdir.path() + QLatin1Char('/') + targets()[1].fileName();
+
+    return QStringList({ QLatin1String("--header"), header, QLatin1String("--impl"), impl,
+                         fn.fileName() });
+}
+
+Utils::FileName QScxmlcGenerator::workingDirectory() const
+{
+    return Utils::FileName::fromString(m_tmpdir.path());
+}
+
+bool QScxmlcGenerator::prepareToRun(const QByteArray &sourceContents)
+{
+    const Utils::FileName fn = tmpFile();
+    QFile input(fn.toString());
     if (!input.open(QIODevice::WriteOnly))
-        return;
+        return false;
-    input.write(sourceContent);
+    input.write(sourceContents);
     input.close();
 
-    qCDebug(log) << "  QScxmlcGenerator::run " << generator << " on "
+    return true;
-                 << sourceContent.size() << " bytes";
+}
 
-    m_process.start(generator, QStringList({
+QList<QByteArray> QScxmlcGenerator::handleProcessFinished(QProcess *process)
-            QLatin1String("--header"), m_tmpdir.path() + QLatin1Char('/') + targets()[0].fileName(),
+{
-            QLatin1String("--impl"), m_tmpdir.path() + QLatin1Char('/') + targets()[1].fileName(),
+    Q_UNUSED(process);
-            input.fileName()}));
+    const Utils::FileName wd = workingDirectory();
+    QList<QByteArray> result;
+    foreach (const Utils::FileName &target, targets()) {
+        Utils::FileName file = wd;
+        file.appendPath(target.fileName());
+        QFile generated(file.toString());
+        if (!generated.open(QIODevice::ReadOnly))
+            continue;
+        result << generated.readAll();
+    }
+    return result;
+}
+
+Utils::FileName QScxmlcGenerator::tmpFile() const
+{
+    Utils::FileName wd = workingDirectory();
+    wd.appendPath(source().fileName());
+    return wd;
 }
 
 ProjectExplorer::FileType QScxmlcGeneratorFactory::sourceType() const

src/plugins/qtsupport/qscxmlcgenerator.h

@@ -33,20 +33,25 @@
 
 namespace QtSupport {
 
-class QScxmlcGenerator : public ProjectExplorer::ExtraCompiler
+class QScxmlcGenerator : public ProjectExplorer::ProcessExtraCompiler
 {
     Q_OBJECT
 public:
     QScxmlcGenerator(const ProjectExplorer::Project *project, const Utils::FileName &source,
                      const Utils::FileNameList &targets, QObject *parent = 0);
 
-private:
+protected:
-    void finishProcess();
+    Utils::FileName command() const override;
-    void run(const QByteArray &sourceContent) override;
+    QStringList arguments() const override;
+    Utils::FileName workingDirectory() const override;
+
+private:
+    Utils::FileName tmpFile() const;
+    QList<QByteArray> handleProcessFinished(QProcess *process) override;
+    bool prepareToRun(const QByteArray &sourceContents) override;
+    QList<ProjectExplorer::Task> parseIssues(const QByteArray &processStderr) override;
 
-    QProcess m_process;
     QTemporaryDir m_tmpdir;
-    void parseIssues(const QByteArray &processStderr);
 };
 
 class QScxmlcGeneratorFactory : public ProjectExplorer::ExtraCompilerFactory

src/plugins/qtsupport/uicgenerator.cpp

@@ -39,43 +39,14 @@
 
 namespace QtSupport {
 
-QLoggingCategory UicGenerator::m_log("qtc.uicgenerator");
-
 UicGenerator::UicGenerator(const ProjectExplorer::Project *project, const Utils::FileName &source,
                            const Utils::FileNameList &targets, QObject *parent) :
-    ProjectExplorer::ExtraCompiler(project, source, targets, parent)
+    ProjectExplorer::ProcessExtraCompiler(project, source, targets, parent)
+{ }
+
+Utils::FileName UicGenerator::command() const
 {
-    connect(&m_process, static_cast<void(QProcess::*)(int)>(&QProcess::finished),
+    QtSupport::BaseQtVersion *version = nullptr;
-            this, &UicGenerator::finishProcess);
-}
-
-void UicGenerator::finishProcess()
-{
-    if (!m_process.waitForFinished(3000)
-            && m_process.exitStatus() != QProcess::NormalExit
-            && m_process.exitCode() != 0) {
-
-        qCDebug(m_log) << "finish process: failed" << m_process.readAllStandardError();
-        m_process.kill();
-        return;
-    }
-
-    // As far as I can discover in the UIC sources, it writes out local 8-bit encoding. The
-    // conversion below is to normalize both the encoding, and the line terminators.
-    QByteArray normalized = QString::fromLocal8Bit(m_process.readAllStandardOutput()).toUtf8();
-    qCDebug(m_log) << "finish process: ok" << normalized.size() << "bytes.";
-    setCompileTime(QDateTime::currentDateTime());
-    setContent(targets()[0], normalized);
-}
-
-void UicGenerator::run(const QByteArray &sourceContent)
-{
-    if (m_process.state() != QProcess::NotRunning) {
-        m_process.kill();
-        m_process.waitForFinished(3000);
-    }
-
-    QtSupport::BaseQtVersion *version = 0;
     ProjectExplorer::Target *target;
     if ((target = project()->activeTarget()))
         version = QtSupport::QtKitInformation::qtVersion(target->kit());
@@ -83,28 +54,25 @@ void UicGenerator::run(const QByteArray &sourceContent)
         version = QtSupport::QtKitInformation::qtVersion(ProjectExplorer::KitManager::defaultKit());
 
     if (!version)
-        return;
+        return Utils::FileName();
 
-    const QString generator = version->uicCommand();
+    return Utils::FileName::fromString(version->uicCommand());
-    if (generator.isEmpty())
+}
-        return;
 
-    m_process.setProcessEnvironment(buildEnvironment().toProcessEnvironment());
+void UicGenerator::handleProcessStarted(QProcess *process, const QByteArray &sourceContents)
+{
+    process->write(sourceContents);
+    process->closeWriteChannel();
+}
 
-    qCDebug(m_log) << "  UicGenerator::run " << generator << " on "
+QList<QByteArray> UicGenerator::handleProcessFinished(QProcess *process)
-                 << sourceContent.size() << " bytes";
+{
-    m_process.start(generator, QStringList(), QIODevice::ReadWrite);
+    if (process->exitStatus() != QProcess::NormalExit && process->exitCode() != 0)
-    if (!m_process.waitForStarted())
+        return QList<QByteArray>();
-        return;
 
-    m_process.write(sourceContent);
+    // As far as I can discover in the UIC sources, it writes out local 8-bit encoding. The
-    if (!m_process.waitForBytesWritten(3000)) {
+    // conversion below is to normalize both the encoding, and the line terminators.
-        qCDebug(m_log) << "failed" << m_process.readAllStandardError();
+    return { QString::fromLocal8Bit(process->readAllStandardOutput()).toUtf8() };
-        m_process.kill();
-        return;
-    }
-
-    m_process.closeWriteChannel();
 }
 
 ProjectExplorer::FileType UicGeneratorFactory::sourceType() const

src/plugins/qtsupport/uicgenerator.h

@@ -33,20 +33,17 @@
 
 namespace QtSupport {
 
-class UicGenerator : public ProjectExplorer::ExtraCompiler
+class UicGenerator : public ProjectExplorer::ProcessExtraCompiler
 {
     Q_OBJECT
 public:
     UicGenerator(const ProjectExplorer::Project *project, const Utils::FileName &source,
                  const Utils::FileNameList &targets, QObject *parent = 0);
 
-private slots:
+protected:
-    void finishProcess();
+    Utils::FileName command() const override;
-    void run(const QByteArray &sourceContent) override;
+    void handleProcessStarted(QProcess *process, const QByteArray &sourceContents) override;
-
+    QList<QByteArray> handleProcessFinished(QProcess *process) override;
-private:
-    QProcess m_process;
-    static QLoggingCategory m_log;
 };
 
 class UicGeneratorFactory : public ProjectExplorer::ExtraCompilerFactory