QmlDesigner: Queue puppet processes during import

On some systems launcing multiple simultaneous import processes causes
imports to fail. Fixed by only launching single process at a time
and queuing the rest.

Fixes: QDS-7107
Change-Id: I330c5920dcbd74d3b4f2e7f40899795a4fbaf3ac
Reviewed-by: Thomas Hartmann <thomas.hartmann@qt.io>

share/qtcreator/qml/qmlpuppet/qml2puppet/iconrenderer/iconrenderer.cpp

@@ -198,8 +198,7 @@ void IconRenderer::finishCreateIcon()
 
     render(saveFile);
 
-    // Allow little time for file operations to finish
+    QTimer::singleShot(0, qGuiApp, &QGuiApplication::quit);
-    QTimer::singleShot(1000, qGuiApp, &QGuiApplication::quit);
 }
 
 void IconRenderer::render(const QString &fileName)

src/plugins/qmldesigner/components/itemlibrary/itemlibraryassetimporter.cpp

@@ -37,6 +37,7 @@
 
 #include <utils/algorithm.h>
 #include <utils/runextensions.h>
+#include <utils/qtcassert.h>
 
 #include <QApplication>
 #include <QDir>
@@ -90,21 +91,16 @@ void ItemLibraryAssetImporter::importQuick3D(const QStringList &inputFiles,
 
     if (!isCancelled()) {
         const auto parseData = m_parseData;
-        for (const auto &pd : parseData) {
+        for (const auto &pd : parseData)
-            if (!startImportProcess(pd)) {
+            m_puppetQueue.append(pd.importId);
-                addError(tr("Failed to start import 3D asset process."),
+        startNextImportProcess();
-                         pd.sourceInfo.absoluteFilePath());
-                m_parseData.remove(pd.importId);
-            }
-        }
     }
 
     if (!isCancelled()) {
         // Wait for puppet processes to finish
-        if (m_qmlPuppetProcesses.empty()) {
+        if (m_puppetQueue.isEmpty() && !m_puppetProcess) {
             postImport();
         } else {
-            m_qmlPuppetCount = static_cast<int>(m_qmlPuppetProcesses.size());
             const QString progressTitle = tr("Importing 3D assets.");
             addInfo(progressTitle);
             notifyProgress(0, progressTitle);
@@ -142,21 +138,14 @@ void ItemLibraryAssetImporter::addInfo(const QString &infoMsg, const QString &sr
     emit infoReported(infoMsg, srcPath);
 }
 
-void ItemLibraryAssetImporter::importProcessFinished(int exitCode, QProcess::ExitStatus exitStatus,
+void ItemLibraryAssetImporter::importProcessFinished(int exitCode, QProcess::ExitStatus exitStatus)
-                                                     int importId)
 {
     Q_UNUSED(exitCode)
 
-    ++m_qmlImportFinishedCount;
+    m_puppetProcess.reset();
 
-    m_qmlPuppetProcesses.erase(
+    if (m_parseData.contains(m_currentImportId)) {
-        std::remove_if(m_qmlPuppetProcesses.begin(), m_qmlPuppetProcesses.end(),
+        const ParseData &pd = m_parseData[m_currentImportId];
-                       [&](const auto &entry) {
-        return !entry || entry->state() == QProcess::NotRunning;
-    }));
-
-    if (m_parseData.contains(importId)) {
-        const ParseData &pd = m_parseData[importId];
         QString errStr;
         if (exitStatus == QProcess::ExitStatus::CrashExit) {
             errStr = tr("Import process crashed.");
@@ -179,15 +168,19 @@ void ItemLibraryAssetImporter::importProcessFinished(int exitCode, QProcess::Exi
             addError(tr("Asset import process failed: \"%1\".")
                      .arg(pd.sourceInfo.absoluteFilePath()));
             addError(errStr);
-            m_parseData.remove(importId);
+            m_parseData.remove(m_currentImportId);
         }
     }
 
-    if (m_qmlImportFinishedCount == m_qmlPuppetCount) {
+    int finishedCount = m_parseData.size() - m_puppetQueue.size();
+    if (!m_puppetQueue.isEmpty())
+        startNextImportProcess();
+
+    if (m_puppetQueue.isEmpty() && !m_puppetProcess) {
         notifyProgress(100);
         QTimer::singleShot(0, this, &ItemLibraryAssetImporter::postImport);
     } else {
-        notifyProgress(int(100. * (double(m_qmlImportFinishedCount) / double(m_qmlPuppetCount))));
+        notifyProgress(int(100. * (double(finishedCount) / double(m_parseData.size()))));
     }
 }
 
@@ -196,17 +189,17 @@ void ItemLibraryAssetImporter::iconProcessFinished(int exitCode, QProcess::ExitS
     Q_UNUSED(exitCode)
     Q_UNUSED(exitStatus)
 
-    m_qmlPuppetProcesses.erase(
+    m_puppetProcess.reset();
-        std::remove_if(m_qmlPuppetProcesses.begin(), m_qmlPuppetProcesses.end(),
-                       [&](const auto &entry) {
-        return !entry || entry->state() == QProcess::NotRunning;
-    }));
 
-    if (m_qmlPuppetProcesses.empty()) {
+    int finishedCount = m_parseData.size() - m_puppetQueue.size();
+    if (!m_puppetQueue.isEmpty())
+        startNextIconProcess();
+
+    if (m_puppetQueue.isEmpty() && !m_puppetProcess) {
         notifyProgress(100);
         QTimer::singleShot(0, this, &ItemLibraryAssetImporter::finalizeQuick3DImport);
     } else {
-        notifyProgress(int(100. * (1. - (double(m_qmlPuppetProcesses.size()) / double(m_qmlPuppetCount)))));
+        notifyProgress(int(100. * (double(finishedCount) / double(m_parseData.size()))));
     }
 }
 
@@ -225,11 +218,11 @@ void ItemLibraryAssetImporter::reset()
     m_tempDir = new QTemporaryDir;
     m_importFiles.clear();
     m_overwrittenImports.clear();
-    m_qmlPuppetProcesses.clear();
+    m_puppetProcess.reset();
-    m_qmlPuppetCount = 0;
-    m_qmlImportFinishedCount = 0;
     m_parseData.clear();
     m_requiredImports.clear();
+    m_currentImportId = 0;
+    m_puppetQueue.clear();
 }
 
 void ItemLibraryAssetImporter::parseFiles(const QStringList &filePaths,
@@ -351,7 +344,7 @@ bool ItemLibraryAssetImporter::preParseQuick3DAsset(const QString &file, ParseDa
     return true;
 }
 
-void ItemLibraryAssetImporter::postParseQuick3DAsset(const ParseData &pd)
+void ItemLibraryAssetImporter::postParseQuick3DAsset(ParseData &pd)
 {
     QDir outDir = pd.outDir;
     if (pd.originalAssetName != pd.assetName) {
@@ -452,8 +445,10 @@ void ItemLibraryAssetImporter::postParseQuick3DAsset(const ParseData &pd)
                                                                  "QtQuick3D", impVersionStr));
                                 }
                             }
-                            if (impVersionMajor > 0 && impVersionMajor < 6
+                            if (impVersionMajor > 0 && impVersionMajor < 6) {
-                                && startIconProcess(24, iconFileName, qmlIt.filePath())) {
+                                pd.iconFile = iconFileName;
+                                pd.iconSource = qmlIt.filePath();
+                                m_puppetQueue.append(pd.importId);
                                 // Since icon is generated by external process, the file won't be
                                 // ready for asset gathering below, so assume its generation succeeds
                                 // and add it now.
@@ -589,84 +584,101 @@ ItemLibraryAssetImporter::OverwriteResult ItemLibraryAssetImporter::confirmAsset
     return OverwriteResult::Skip;
 }
 
-bool ItemLibraryAssetImporter::startImportProcess(const ParseData &pd)
+void ItemLibraryAssetImporter::startNextImportProcess()
 {
+    if (m_puppetQueue.isEmpty())
+        return;
+
     auto doc = QmlDesignerPlugin::instance()->currentDesignDocument();
     Model *model = doc ? doc->currentModel() : nullptr;
 
     if (model) {
         PuppetCreator puppetCreator(doc->currentTarget(), model);
         puppetCreator.createQml2PuppetExecutableIfMissing();
-        QStringList puppetArgs;
-        QJsonDocument optDoc(pd.options);
 
-        puppetArgs << "--import3dAsset" << pd.sourceInfo.absoluteFilePath()
+        bool done = false;
-                   << pd.outDir.absolutePath() << QString::fromUtf8(optDoc.toJson());
+        while (!m_puppetQueue.isEmpty() && !done) {
+            const ParseData pd = m_parseData.value(m_puppetQueue.takeLast());
+            QStringList puppetArgs;
+            QJsonDocument optDoc(pd.options);
 
-        QProcessUniquePointer process = puppetCreator.createPuppetProcess(
+            puppetArgs << "--import3dAsset" << pd.sourceInfo.absoluteFilePath()
-            "custom",
+                       << pd.outDir.absolutePath() << QString::fromUtf8(optDoc.toJson());
-            {},
-            [&] {},
-            [&](int exitCode, QProcess::ExitStatus exitStatus) {
-                importProcessFinished(exitCode, exitStatus, pd.importId);
-            },
-            puppetArgs);
 
-        if (process->waitForStarted(5000)) {
+            m_currentImportId = pd.importId;
-            m_qmlPuppetProcesses.push_back(std::move(process));
+            m_puppetProcess = puppetCreator.createPuppetProcess(
-            return true;
+                "custom",
-        } else {
+                {},
-            process.reset();
+                [&] {},
+                [&](int exitCode, QProcess::ExitStatus exitStatus) {
+                    importProcessFinished(exitCode, exitStatus);
+                },
+                puppetArgs);
+
+            if (m_puppetProcess->waitForStarted(10000)) {
+                done = true;
+            } else {
+                addError(tr("Failed to start import 3D asset process."),
+                         pd.sourceInfo.absoluteFilePath());
+                m_parseData.remove(pd.importId);
+                m_puppetProcess.reset();
+            }
         }
     }
-    return false;
 }
 
-bool ItemLibraryAssetImporter::startIconProcess(int size, const QString &iconFile,
+void ItemLibraryAssetImporter::startNextIconProcess()
-                                                const QString &iconSource)
 {
+    if (m_puppetQueue.isEmpty())
+        return;
+
     auto doc = QmlDesignerPlugin::instance()->currentDesignDocument();
     Model *model = doc ? doc->currentModel() : nullptr;
 
     if (model) {
         PuppetCreator puppetCreator(doc->currentTarget(), model);
         puppetCreator.createQml2PuppetExecutableIfMissing();
-        QStringList puppetArgs;
-        puppetArgs << "--rendericon" << QString::number(size) << iconFile << iconSource;
-        QProcessUniquePointer process = puppetCreator.createPuppetProcess(
-            "custom",
-            {},
-            [&] {},
-            [&](int exitCode, QProcess::ExitStatus exitStatus) {
-                iconProcessFinished(exitCode, exitStatus);
-            },
-            puppetArgs);
 
-        if (process->waitForStarted(5000)) {
+        bool done = false;
-            m_qmlPuppetProcesses.push_back(std::move(process));
+        while (!m_puppetQueue.isEmpty() && !done) {
-            return true;
+            const ParseData pd = m_parseData.value(m_puppetQueue.takeLast());
-        } else {
+            QStringList puppetArgs;
-            process.reset();
+            puppetArgs << "--rendericon" << QString::number(24) << pd.iconFile << pd.iconSource;
+            m_puppetProcess = puppetCreator.createPuppetProcess(
+                "custom",
+                {},
+                [&] {},
+                [&](int exitCode, QProcess::ExitStatus exitStatus) {
+                    iconProcessFinished(exitCode, exitStatus);
+                },
+                puppetArgs);
+
+            if (m_puppetProcess->waitForStarted(10000)) {
+                done = true;
+            } else {
+                addError(tr("Failed to start icon generation process."),
+                         pd.sourceInfo.absoluteFilePath());
+                m_puppetProcess.reset();
+            }
         }
     }
-    return false;
 }
 
 void ItemLibraryAssetImporter::postImport()
 {
-    Q_ASSERT(m_qmlPuppetProcesses.empty());
+    QTC_ASSERT(m_puppetQueue.isEmpty() && !m_puppetProcess, return);
 
     if (!isCancelled()) {
-        for (const auto &pd : qAsConst(m_parseData))
+        for (auto &pd : m_parseData)
             postParseQuick3DAsset(pd);
+        startNextIconProcess();
     }
 
     if (!isCancelled()) {
         // Wait for icon generation processes to finish
-        if (m_qmlPuppetProcesses.empty()) {
+        if (m_puppetQueue.isEmpty() && !m_puppetProcess) {
             finalizeQuick3DImport();
         } else {
-            m_qmlPuppetCount = static_cast<int>(m_qmlPuppetProcesses.size());
             const QString progressTitle = tr("Generating icons.");
             addInfo(progressTitle);
             notifyProgress(0, progressTitle);

src/plugins/qmldesigner/components/itemlibrary/itemlibraryassetimporter.h

@@ -74,7 +74,7 @@ signals:
     void importFinished();
 
 private slots:
-    void importProcessFinished(int exitCode, QProcess::ExitStatus exitStatus, int importId);
+    void importProcessFinished(int exitCode, QProcess::ExitStatus exitStatus);
     void iconProcessFinished(int exitCode, QProcess::ExitStatus exitStatus);
 
 private:
@@ -87,6 +87,8 @@ private:
         QString assetName;
         QString originalAssetName;
         int importId;
+        QString iconFile;
+        QString iconSource;
     };
 
     void notifyFinished();
@@ -96,7 +98,7 @@ private:
                     const QSet<QString> &preselectedFilesForOverwrite);
     bool preParseQuick3DAsset(const QString &file, ParseData &pd,
                               const QSet<QString> &preselectedFilesForOverwrite);
-    void postParseQuick3DAsset(const ParseData &pd);
+    void postParseQuick3DAsset(ParseData &pd);
     void copyImportedFiles();
 
     void notifyProgress(int value, const QString &text);
@@ -110,8 +112,8 @@ private:
     };
 
     OverwriteResult confirmAssetOverwrite(const QString &assetName);
-    bool startImportProcess(const ParseData &pd);
+    void startNextImportProcess();
-    bool startIconProcess(int size, const QString &iconFile, const QString &iconSource);
+    void startNextIconProcess();
     void postImport();
     void finalizeQuick3DImport();
     QString sourceSceneTargetFilePath(const ParseData &pd);
@@ -122,12 +124,12 @@ private:
     bool m_cancelled = false;
     QString m_importPath;
     QTemporaryDir *m_tempDir = nullptr;
-    std::vector<QProcessUniquePointer> m_qmlPuppetProcesses;
+    QProcessUniquePointer m_puppetProcess;
-    int m_qmlPuppetCount = 0;
-    int m_qmlImportFinishedCount = 0;
     int m_importIdCounter = 0;
+    int m_currentImportId = 0;
     QHash<int, ParseData> m_parseData;
     QString m_progressTitle;
     QList<Import> m_requiredImports;
+    QList<int> m_puppetQueue;
 };
 } // QmlDesigner