Resolve qmlproject dependencies during conversion

Qt for MCUs modules have their own qmlproject files. To make it
easier to know which qmlproject files belong to a project
the qmlproject dependencies are resolved and listed in the
internal project JSON object when converting a qmlproject file
in QDS.

In Qt for MCUs, qmlproject files are found either in ModuleFiles
nodes or in the importPaths (with some extra restrictions).
This implementation mirrors the dependency resolution in
Qt for MCUs also in QDS.

Task-number: QDS-12636
Change-Id: I7ae874d26beeea0deb440fba031b7a4b11eef1e0
Reviewed-by: Marco Bubke <marco.bubke@qt.io>
Reviewed-by: Burak Hancerli <burak.hancerli@qt.io>

src/plugins/qmlprojectmanager/buildsystem/projectitem/converters.cpp

@@ -2,6 +2,7 @@
 // SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0+ OR GPL-3.0 WITH Qt-GPL-exception-1.0
 
 #include "converters.h"
+#include "utils/algorithm.h"
 
 #include <QJsonDocument>
 
@@ -224,6 +225,103 @@ QString jsonToQmlProject(const QJsonObject &rootObject)
     return qmlProjectString;
 }
 
+QStringList qmlprojectsFromFilesNodes(const QJsonArray &fileGroups,
+                                      const Utils::FilePath &projectRootPath)
+{
+    QStringList qmlProjectFiles;
+    for (const QJsonValue &fileGroup : fileGroups) {
+        if (fileGroup["type"].toString() != "Module") {
+            continue;
+        }
+        // In Qul, paths are relative to the project root directory, not the "directory" entry
+        qmlProjectFiles.append(fileGroup["files"].toVariant().toStringList());
+
+        // If the "directory" property is set, all qmlproject files in the directory are also added
+        // as relative paths from the project root directory, in addition to explicitly added files
+        const QString directoryProp = fileGroup["directory"].toString("");
+        if (directoryProp.isEmpty()) {
+            continue;
+        }
+        const Utils::FilePath dir = projectRootPath / directoryProp;
+        qmlProjectFiles.append(Utils::transform<QStringList>(
+            dir.dirEntries(Utils::FileFilter({"*.qmlproject"}, QDir::Files)),
+            [&projectRootPath](Utils::FilePath file) {
+                return file.absoluteFilePath().relativePathFrom(projectRootPath).toFSPathString();
+            }));
+    }
+
+    return qmlProjectFiles;
+}
+
+QString moduleUriFromQmlProject(const QString &qmlProjectFilePath)
+{
+    QmlJS::SimpleReader simpleReader;
+    const auto rootNode = simpleReader.readFile(qmlProjectFilePath);
+    // Since the file wasn't explicitly added, skip qmlproject files with errors
+    if (!rootNode || !simpleReader.errors().isEmpty()) {
+        return QString();
+    }
+
+    for (const auto &child : rootNode->children()) {
+        if (child->name() == "MCU.Module") {
+            const auto prop = child->property("uri").isValid() ? child->property("uri")
+                                                               : child->property("MCU.uri");
+            if (prop.isValid()) {
+                return prop.value.toString();
+            }
+            break;
+        }
+    }
+
+    return QString();
+}
+
+// Returns a list of qmlproject files in currentSearchPath which are valid modules,
+// with URIs matching the relative path from importPathBase.
+QStringList getModuleQmlProjectFiles(const Utils::FilePath &importPath,
+                                     const Utils::FilePath &projectRootPath)
+{
+    QStringList qmlProjectFiles;
+
+    QDirIterator it(importPath.toFSPathString(),
+                    QDir::NoDotAndDotDot | QDir::Files,
+                    QDirIterator::Subdirectories);
+    while (it.hasNext()) {
+        const QString file = it.next();
+        if (!file.endsWith(".qmlproject")) {
+            continue;
+        }
+
+        // Add if matching
+        QString uri = moduleUriFromQmlProject(file);
+        if (uri.isEmpty()) {
+            // If the qmlproject file is not a valid module, skip it
+            continue;
+        }
+
+        const auto filePath = Utils::FilePath::fromUserInput(file);
+        const bool isBaseImportPath = filePath.parentDir() == importPath;
+
+        // Check the URI against the original import path before adding
+        // If we look directly in the search path, the URI doesn't matter
+        const QString relativePath = filePath.parentDir().relativePathFrom(importPath).path();
+        if (isBaseImportPath || uri.replace(".", "/") == relativePath) {
+            // If the URI matches the path or the file is directly in the import path, add it
+            qmlProjectFiles.emplace_back(filePath.relativePathFrom(projectRootPath).toFSPathString());
+        }
+    }
+    return qmlProjectFiles;
+}
+
+QStringList qmlprojectsFromImportPaths(const QStringList &importPaths,
+                                       const Utils::FilePath &projectRootPath)
+{
+    return Utils::transform<QStringList>(importPaths, [&projectRootPath](const QString &importPath) {
+        const auto importDir = projectRootPath / importPath;
+        return getModuleQmlProjectFiles(importDir, projectRootPath);
+    });
+}
+
 QJsonObject qmlProjectTojson(const Utils::FilePath &projectFile)
 {
     QmlJS::SimpleReader simpleQmlJSReader;
@@ -265,6 +363,9 @@ QJsonObject qmlProjectTojson(const Utils::FilePath &projectFile)
 
     bool qtForMCUs = false;
 
+    QStringList importPaths;
+    Utils::FilePath projectRootPath = projectFile.parentDir();
+
     // convert the non-object props
     for (const QString &propName : rootNode->propertyNames()) {
         QJsonObject *currentObj = &rootObject;
@@ -300,6 +401,7 @@ QJsonObject qmlProjectTojson(const Utils::FilePath &projectFile)
             value = rootNode->property(propName).value.toBool() ? "6" : "5";
         } else if (propName.contains("importpaths", Qt::CaseInsensitive)) {
             objKey = "importPaths";
+            importPaths = value.toVariant().toStringList();
         } else {
             currentObj = &otherProperties;
             objKey = propName; // With prefix
@@ -413,6 +515,12 @@ QJsonObject qmlProjectTojson(const Utils::FilePath &projectFile)
         }
     }
 
+    QStringList qmlProjectDependencies;
+    qmlProjectDependencies.append(qmlprojectsFromImportPaths(importPaths, projectRootPath));
+    qmlProjectDependencies.append(qmlprojectsFromFilesNodes(fileGroupsObject, projectRootPath));
+    qmlProjectDependencies.sort();
+    rootObject.insert("qmlprojectDependencies", QJsonArray::fromStringList(qmlProjectDependencies));
+
     mcuObject.insert("config", mcuConfigObject);
     mcuObject.insert("module", mcuModuleObject);
     qtForMCUs |= !(mcuModuleObject.isEmpty() && mcuConfigObject.isEmpty());

src/plugins/qmlprojectmanager/buildsystem/projectitem/qmlprojectitem.cpp

@@ -209,6 +209,11 @@ void QmlProjectItem::addImportPath(const QString &importPath)
     insertAndUpdateProjectFile("importPaths", importPaths);
 }
 
+QStringList QmlProjectItem::qmlProjectModules() const
+{
+    return m_project["qmlprojectDependencies"].toVariant().toStringList();
+}
+
 QStringList QmlProjectItem::fileSelectors() const
 {
     return m_project["runConfig"].toObject()["fileSelectors"].toVariant().toStringList();

src/plugins/qmlprojectmanager/buildsystem/projectitem/qmlprojectitem.h

@@ -46,6 +46,8 @@ public:
     void setImportPaths(const QStringList &paths);
     void addImportPath(const QString &importPath);
 
+    QStringList qmlProjectModules() const;
+
     QStringList fileSelectors() const;
     void setFileSelectors(const QStringList &selectors);
     void addFileSelector(const QString &selector);

tests/unit/tests/unittests/qmlprojectmanager/data/README.md

@@ -38,6 +38,9 @@ that QDS and QUL are aligned on the qmlproject format and that new features in Q
 qmlproject files for MCU projects. The test set will be tested in the Qt for MCUs repositories
 to make sure both the original and the converted qmlprojects build correctly.
 
+The test set also includes some dummy qmlproject files to test that the converter correctly picks up
+qmlproject modules in the same way Qt for MCUs does.
+
 The qmlproject files in the test set aim to cover all the possible contents of a Qt for MCUs qmlproject,
 but since new features are added with every release, it is not guaranteed to be exhaustive.
 

tests/unit/tests/unittests/qmlprojectmanager/data/converter/test-set-1/testfile.qmltojson

@@ -260,6 +260,8 @@
     },
     "otherProperties": {
     },
+    "qmlprojectDependencies": [
+    ],
     "runConfig": {
         "fileSelectors": [
         ],

tests/unit/tests/unittests/qmlprojectmanager/data/converter/test-set-2/testfile.qmltojson

@@ -113,6 +113,8 @@
     },
     "otherProperties": {
     },
+    "qmlprojectDependencies": [
+    ],
     "runConfig": {
         "fileSelectors": [
             "WXGA",

tests/unit/tests/unittests/qmlprojectmanager/data/converter/test-set-3/testfile.qmltojson

@@ -260,6 +260,8 @@
     },
     "otherProperties": {
     },
+    "qmlprojectDependencies": [
+    ],
     "runConfig": {
         "fileSelectors": [
         ],

tests/unit/tests/unittests/qmlprojectmanager/data/converter/test-set-mcu-1/mcu-modules/from_importpath/imported_module.qmlproject

@@ -0,0 +1,7 @@
+import QmlProject 1.3
+
+Project {
+    MCU.Module {
+        uri: "from_importpath"
+    }
+}

tests/unit/tests/unittests/qmlprojectmanager/data/converter/test-set-mcu-1/mcu-modules/from_importpath/mismatched_uri.qmlproject

@@ -0,0 +1,9 @@
+import QmlProject 1.3
+
+// This qmlproject file will not be picked up from the import path "mcu-modules"
+// since the URI doesn't match a relative path to the importPath
+Project {
+    MCU.Module {
+        uri: "SomeModule"
+    }
+}

tests/unit/tests/unittests/qmlprojectmanager/data/converter/test-set-mcu-1/mcu-modules/module.qmlproject

@@ -0,0 +1,8 @@
+import QmlProject 1.3
+
+// This file will be picked up as an MCU module from importPaths
+Project {
+    MCU.Module {
+        uri: "MyModule"
+    }
+}

tests/unit/tests/unittests/qmlprojectmanager/data/converter/test-set-mcu-1/mcu-modules/no_module.qmlproject

@@ -0,0 +1,5 @@
+import QmlProject 1.3
+
+// This file will NOT be picked up as an MCU module from importPaths
+Project {
+}

tests/unit/tests/unittests/qmlprojectmanager/data/converter/test-set-mcu-1/testfile.jsontoqml

@@ -8,7 +8,7 @@ Project {
     targetDirectory: "/opt/UntitledProject13"
     enableCMakeGeneration: false
     widgetApp: true
-    importPaths: [ "imports","asset_imports" ]
+    importPaths: [ "imports","asset_imports","mcu-modules" ]
 
     qdsVersion: "4.0"
     quickVersion: "6.2"
@@ -81,6 +81,10 @@ Project {
         MCU.qulModules: ["Controls","Timeline"]
     }
 
+    ModuleFiles {
+        directory: "../test-set-mcu-2"
+    }
+
     FontFiles {
         files: [
             "fonts/RobotoFonts.fmp"

tests/unit/tests/unittests/qmlprojectmanager/data/converter/test-set-mcu-1/testfile.qmlproject

@@ -11,7 +11,7 @@ Project {
     mainFile: "Main.qml"
     supportedLanguages: ["no"]
     primaryLanguage: "en"
-    importPaths: [ "imports", "asset_imports" ]
+    importPaths: [ "imports", "asset_imports", "mcu-modules"]
     idBasedTranslations: true
     projectRootPath: ".."
 // END of common properties
@@ -84,6 +84,10 @@ Project {
         ]
     }
 
+    ModuleFiles {
+        directory: "../test-set-mcu-2"
+    }
+
     FontFiles {
         files: [
             "fonts/RobotoFonts.fmp"

tests/unit/tests/unittests/qmlprojectmanager/data/converter/test-set-mcu-1/testfile.qmltojson

@@ -108,6 +108,18 @@
             },
             "type": "Module"
         },
+        {
+            "directory": "../test-set-mcu-2",
+            "files": [
+            ],
+            "filters": [
+            ],
+            "mcuProperties": {
+            },
+            "otherProperties": {
+            },
+            "type": "Module"
+        },
         {
             "directory": "",
             "files": [
@@ -180,7 +192,8 @@
     "fileVersion": 1,
     "importPaths": [
         "imports",
-        "asset_imports"
+        "asset_imports",
+        "mcu-modules"
     ],
     "language": {
         "multiLanguageSupport": true,
@@ -213,6 +226,12 @@
         "idBasedTranslations": true,
         "projectRootPath": ".."
     },
+    "qmlprojectDependencies": [
+        "../test-set-mcu-2/testfile.qmlproject",
+        "mcu-modules/from_importpath/imported_module.qmlproject",
+        "mcu-modules/module.qmlproject",
+        "qmlproject/module/module.qmlproject"
+    ],
     "runConfig": {
         "fileSelectors": [
         ],

tests/unit/tests/unittests/qmlprojectmanager/data/converter/test-set-mcu-2/testfile.qmltojson

@@ -46,6 +46,8 @@
     "otherProperties": {
         "projectRootPath": "../.."
     },
+    "qmlprojectDependencies": [
+    ],
     "runConfig": {
         "fileSelectors": [
         ]

tests/unit/tests/unittests/qmlprojectmanager/data/getter-setter/mcu_project_with_modules.qmlproject

@@ -0,0 +1,15 @@
+import QmlProject 1.3
+
+Project {
+    importPaths: [ "../converter/test-set-mcu-1/mcu-modules" ]
+
+    mainFile: "Main.qml"
+
+    ModuleFiles {
+        files: [
+            "file1.qmlproject",
+            "file2.qmlproject"
+        ]
+        directory: "../converter/test-set-mcu-2"
+    }
+}

tests/unit/tests/unittests/qmlprojectmanager/projectitem-test.cpp

@@ -35,6 +35,11 @@ protected:
             Utils::FilePath::fromString(localTestDataDir
                                         + "/file-filters/MaterialBundle.qmlproject"),
             true);
+
+        projectItemMcuWithModules = std::make_unique<const QmlProjectManager::QmlProjectItem>(
+            Utils::FilePath::fromString(localTestDataDir
+                                        + "/getter-setter/mcu_project_with_modules.qmlproject"),
+            true);
     }
 
     static void TearDownTestSuite()
@@ -43,6 +48,7 @@ protected:
         projectItemWithQdsPrefix.reset();
         projectItemWithoutQdsPrefix.reset();
         projectItemFileFilters.reset();
+        projectItemMcuWithModules.reset();
     }
 
 protected:
@@ -54,6 +60,7 @@ protected:
                                                localTestDataDir + "/getter-setter/empty.qmlproject"),
                                            true);
     inline static std::unique_ptr<const QmlProjectManager::QmlProjectItem> projectItemFileFilters;
+    inline static std::unique_ptr<const QmlProjectManager::QmlProjectItem> projectItemMcuWithModules;
 };
 
 auto createAbsoluteFilePaths(const QStringList &fileList)
@@ -742,4 +749,24 @@ TEST_F(QmlProjectItem, not_matches_file)
     ASSERT_FALSE(fileFound);
 }
 
+TEST_F(QmlProjectItem, qmlproject_modules)
+{
+    auto qmlProjectModules = projectItemMcuWithModules->qmlProjectModules();
+
+    ASSERT_THAT(
+        qmlProjectModules,
+        UnorderedElementsAre(
+            "file1.qmlproject",
+            "file2.qmlproject",
+            "../converter/test-set-mcu-1/mcu-modules/from_importpath/imported_module.qmlproject",
+            "../converter/test-set-mcu-2/testfile.qmlproject"));
+}
+
+TEST_F(QmlProjectItem, no_qmlproject_modules)
+{
+    auto qmlProjectModules = projectItemEmpty->qmlProjectModules();
+
+    ASSERT_THAT(qmlProjectModules, IsEmpty());
+}
+
 } // namespace