McuSupport: Evaluate environment variables in paths

Packages are created as defined in JSON. Some paths
contain variables. This creates dependencies between
packages. After all packages are created and collected
environment variables are evaluated in one pass. McuTarget
packages get updated.

This is needed because we shouldn't show the user paths
with variables in them. Also file picker wouldn't work.

There will be cmake variables support in separate commit.

Change-Id: Id210ea394f3f5bb5a14d87f3cf6a0a9a99e690bf
Reviewed-by: Alessandro Portale <alessandro.portale@qt.io>

src/plugins/mcusupport/mcuabstractpackage.h

@@ -57,6 +57,7 @@ public:
 
     virtual Utils::FilePath basePath() const = 0;
     virtual Utils::FilePath path() const = 0;
+    virtual void setPath(const Utils::FilePath &) = 0;
     virtual Utils::FilePath defaultPath() const = 0;
     virtual Utils::FilePath detectionPath() const = 0;
     virtual QString settingsKey() const = 0;

src/plugins/mcusupport/mcupackage.cpp

@@ -124,7 +124,7 @@ FilePath McuPackage::basePath() const
 
 FilePath McuPackage::path() const
 {
-    return (basePath() / m_relativePathModifier.path()).absoluteFilePath().cleanPath();
+    return (basePath() / m_relativePathModifier.path()).cleanPath();
 }
 
 FilePath McuPackage::defaultPath() const
@@ -137,6 +137,13 @@ FilePath McuPackage::detectionPath() const
     return m_detectionPath;
 }
 
+void McuPackage::setPath(const FilePath &newPath)
+{
+    m_path = newPath;
+    m_defaultPath = newPath;
+    updateStatus();
+}
+
 void McuPackage::updatePath()
 {
     m_path = m_fileChooser->rawFilePath();

src/plugins/mcusupport/mcupackage.h

@@ -90,6 +90,8 @@ public:
     QWidget *widget() override;
     const McuPackageVersionDetector *getVersionDetector() const override;
 
+    void setPath(const Utils::FilePath &);
+
 private:
     void updatePath();
     void updateStatusUi();
@@ -100,7 +102,7 @@ private:
     Utils::InfoLabel *m_infoLabel = nullptr;
 
     const QString m_label;
-    const Utils::FilePath m_defaultPath;
+    Utils::FilePath m_defaultPath;
     const Utils::FilePath m_detectionPath;
     const QString m_settingsKey;
     QScopedPointer<const McuPackageVersionDetector> m_versionDetector;

src/plugins/mcusupport/mcutargetfactory.cpp

@@ -78,6 +78,34 @@ McuPackageVersionDetector *createVersionDetection(const VersionDetection &versio
                                                       versionDetection.isFile);
 }
 
+static void evaluateVariables(McuTarget &target)
+{
+    const static QRegularExpression variableRegex{R"(\${*\w+}*)",
+                                                  QRegularExpression::CaseInsensitiveOption};
+
+    for (const auto &package : target.packages()) {
+        const QRegularExpressionMatch match{variableRegex.match(package->path().toString())};
+        if (!match.hasMatch())
+            continue;
+        const QString variable{match.captured(0).remove(0, 1)};
+
+        McuPackagePtr packageDefiningVariable{
+            Utils::findOrDefault(target.packages(), [variable](const McuPackagePtr &pkg) {
+                return pkg->cmakeVariableName() == variable;
+                // return pkg->cmakeVariableName() == variable || pkg->environmentVariableName() == variable;
+            })};
+
+        if (packageDefiningVariable == nullptr) // nothing provides the variable
+            continue;
+
+        const auto evaluatedPath{Utils::FilePath::fromUserInput(
+            package->path().toString().replace(match.capturedStart(),
+                                               match.capturedLength(),
+                                               packageDefiningVariable->path().toString()))};
+        package->setPath(evaluatedPath);
+    }
+}
+
 McuTargetFactory::McuTargetFactory(const SettingsHandler::Ptr &settingsHandler)
     : settingsHandler{settingsHandler}
 {}
@@ -102,13 +130,17 @@ QPair<Targets, Packages> McuTargetFactory::createTargets(const McuTargetDescript
         targetPackages.insert({toolchainPtr});
         targetPackages.unite({toolchainFile});
         packages.unite(targetPackages);
-        mcuTargets.append(McuTargetPtr{new McuTarget{QVersionNumber::fromString(desc.qulVersion),
+
-                                                     platform,
+        McuTargetPtr target{new McuTarget{QVersionNumber::fromString(desc.qulVersion),
-                                                     deduceOperatingSystem(desc),
+                                          platform,
-                                                     targetPackages,
+                                          deduceOperatingSystem(desc),
-                                                     toolchainPtr,
+                                          targetPackages,
-                                                     toolchainFile,
+                                          toolchainPtr,
-                                                     colorDepth}});
+                                          toolchainFile,
+                                          colorDepth}};
+
+        evaluateVariables(*target);
+        mcuTargets.append(target);
     }
     return {mcuTargets, packages};
 }

src/plugins/mcusupport/mcutargetfactorylegacy.cpp

@@ -76,7 +76,7 @@ QPair<Targets, Packages> McuTargetFactory::createTargets(const McuTargetDescript
                 boardSdkPkgs.insert(desc.boardSdk.envVar, boardSdkPkg);
             }
             McuPackagePtr boardSdkPkg{boardSdkPkgs.value(desc.boardSdk.envVar)};
-            boardSdkDefaultPath = boardSdkPkg->defaultPath();
+            boardSdkDefaultPath = boardSdkPkg->path();
             required3rdPartyPkgs.insert(boardSdkPkg);
         }
 

src/plugins/mcusupport/test/armgcc_stm32f769i_freertos_json.h

@@ -53,6 +53,7 @@ constexpr auto armgcc_stm32f769i_freertos_json = R"({
         "id": "Qul_DIR",
         "label": "Qt for MCUs SDK",
         "type": "path",
+        "setting": "QtForMCUsSdk",
         "cmakeVar": "Qul_ROOT",
         "optional": false
       }
@@ -83,7 +84,7 @@ constexpr auto armgcc_stm32f769i_freertos_json = R"({
         "label": "CMake Toolchain File",
         "cmakeVar": "CMAKE_TOOLCHAIN_FILE",
         "type": "file",
-        "defaultValue": "/opt/qtformcu/2.2//lib/cmake/Qul/toolchain/armgcc.cmake",
+        "defaultValue": "$Qul_ROOT//lib/cmake/Qul/toolchain/armgcc.cmake",
         "visible": false,
         "optional": false
       }

src/plugins/mcusupport/test/packagemock.h

@@ -37,6 +37,7 @@ class PackageMock : public McuAbstractPackage
 public:
     MOCK_METHOD(Utils::FilePath, basePath, (), (const));
     MOCK_METHOD(Utils::FilePath, path, (), (const));
+    MOCK_METHOD(void, setPath, (const Utils::FilePath &) );
     MOCK_METHOD(QString, label, (), (const));
     MOCK_METHOD(Utils::FilePath, defaultPath, (), (const));
     MOCK_METHOD(Utils::FilePath, detectionPath, (), (const));

src/plugins/mcusupport/test/unittest.cpp

@@ -79,7 +79,10 @@ const char armGccDirectorySetting[]{"GNUArmEmbeddedToolchain"};
 const char armGccEnvVar[]{"ARMGCC_DIR"};
 const char armGccLabel[]{"GNU Arm Embedded Toolchain"};
 const char armGccSuffix[]{"bin/arm-none-eabi-g++"};
-const char armGccToolchainFilePath[]{"/opt/qtformcu/2.2/lib/cmake/Qul/toolchain/armgcc.cmake"};
+const char armGccToolchainFilePath[]{"/opt/toolchain/armgcc.cmake"};
+const char armGccToolchainFileDefaultPath[]{
+    "/opt/qtformcu/2.2/lib/cmake/Qul/toolchain/armgcc.cmake"};
+const char armGccToolchainFilePathWithVariable[]{"$Qul_ROOT/lib/cmake/Qul/toolchain/armgcc.cmake"};
 const char armGccVersion[]{"9.3.1"};
 const char armGccNewVersion[]{"10.3.1"};
 const char msvcVersion[]{"14.29"};
@@ -92,7 +95,9 @@ const char freeRtosEnvVar[]{"EVK_MIMXRT1170_FREERTOS_PATH"};
 const char freeRtosLabel[]{"FreeRTOS directory"};
 const char freeRtosPath[]{"/opt/freertos/default"};
 const char freeRtosSetting[]{"Freertos"};
-const char greenhillToolchainFilePath[]{"/opt/qtformcu/2.2/lib/cmake/Qul/toolchain/ghs.cmake"};
+const char greenhillToolchainFilePath[]{"/opt/toolchain/ghs.cmake"};
+const char greenhillToolchainFileDefaultPath[]{
+    "/opt/qtformcu/2.2/lib/cmake/Qul/toolchain/ghs.cmake"};
 const char greenhillCompilerDir[]{"/abs/ghs"};
 const char greenhillSetting[]{"GHSToolchain"};
 const QStringList greenhillVersions{{"2018.1.5"}};
@@ -100,7 +105,8 @@ const char iarDir[]{"/opt/iar/compiler"};
 const char iarEnvVar[]{"IAR_ARM_COMPILER_DIR"};
 const char iarLabel[]{"IAR ARM Compiler"};
 const char iarSetting[]{"IARToolchain"};
-const char iarToolchainFilePath[]{"/opt/qtformcu/2.2/lib/cmake/Qul/toolchain/iar.cmake"};
+const char iarToolchainFilePath[]{"/opt/toolchain/iar.cmake"};
+const char iarToolchainFileDefaultPath[]{"/opt/qtformcu/2.2/lib/cmake/Qul/toolchain/iar.cmake"};
 const char iarVersionDetectionRegex[]{R"(\bV(\d+\.\d+\.\d+)\.\d+\b)"};
 const QStringList iarVersions{{"8.50.9"}};
 const char iar[]{"iar"};
@@ -247,6 +253,7 @@ void verifyTargetToolchains(const Targets &targets,
     QCOMPARE(toolchainFile->cmakeVariableName(), Legacy::Constants::TOOLCHAIN_FILE_CMAKE_VARIABLE);
     QCOMPARE(toolchainFile->settingsKey(), empty);
     QCOMPARE(toolchainFile->path().toString(), toolchainFilePath);
+    QCOMPARE(toolchainFile->defaultPath().toString(), toolchainFilePath);
 
     const auto toolchainCompiler{target->toolChainPackage()};
     QVERIFY(toolchainCompiler);
@@ -393,26 +400,26 @@ void McuSupportTest::test_parseToolchainFromJSON_data()
     QTest::addColumn<QString>("json");
     QTest::addColumn<QString>("environmentVariable");
     QTest::addColumn<QString>("label");
-    QTest::addColumn<QString>("toolchainFile");
+    QTest::addColumn<QString>("toolchainFileDefaultPath");
     QTest::addColumn<QString>("id");
 
     //TODO(me): Add ghs nxp 1064 nxp 1070.
-    QTest::newRow("armgcc_nxp_1050_json")
+    QTest::newRow("armgcc_nxp_1050_json") << armgcc_nxp_1050_json << armGccEnvVar << armGccLabel
-        << armgcc_nxp_1050_json << armGccEnvVar << armGccLabel << armGccToolchainFilePath << armGcc;
+                                          << armGccToolchainFileDefaultPath << armGcc;
     QTest::newRow("armgcc_stm32f769i_freertos_json")
-        << armgcc_stm32f769i_freertos_json << armGccEnvVar << armGccLabel << armGccToolchainFilePath
+        << armgcc_stm32f769i_freertos_json << armGccEnvVar << armGccLabel
-        << armGcc;
+        << armGccToolchainFilePathWithVariable << armGcc;
 
     QTest::newRow("armgcc_stm32h750b_metal_json")
-        << armgcc_stm32h750b_metal_json << armGccEnvVar << armGccLabel << armGccToolchainFilePath
+        << armgcc_stm32h750b_metal_json << armGccEnvVar << armGccLabel
-        << armGcc;
+        << armGccToolchainFileDefaultPath << armGcc;
 
     QTest::newRow("armgcc_nxp_mimxrt1170_evk_freertos_json")
         << armgcc_nxp_mimxrt1170_evk_freertos_json << armGccEnvVar << armGccLabel
-        << armGccToolchainFilePath << armGcc;
+        << armGccToolchainFileDefaultPath << armGcc;
 
     QTest::newRow("iar_stm32f469i_metal_json")
-        << iar_stm32f469i_metal_json << iarEnvVar << iarLabel << iarToolchainFilePath << iar;
+        << iar_stm32f469i_metal_json << iarEnvVar << iarLabel << iarToolchainFileDefaultPath << iar;
 }
 
 void McuSupportTest::test_parseToolchainFromJSON()
@@ -420,8 +427,9 @@ void McuSupportTest::test_parseToolchainFromJSON()
     QFETCH(QString, json);
     QFETCH(QString, environmentVariable);
     QFETCH(QString, label);
-    QFETCH(QString, toolchainFile);
+    QFETCH(QString, toolchainFileDefaultPath);
     QFETCH(QString, id);
+
     McuTargetDescription description{parseDescriptionJson(json.toLocal8Bit())};
     QCOMPARE(description.toolchain.id, id);
 
@@ -433,7 +441,7 @@ void McuSupportTest::test_parseToolchainFromJSON()
     QCOMPARE(toolchainFilePackage.label, cmakeToolchainLabel);
     QCOMPARE(toolchainFilePackage.envVar, QString{});
     QCOMPARE(toolchainFilePackage.cmakeVar, Legacy::Constants::TOOLCHAIN_FILE_CMAKE_VARIABLE);
-    QCOMPARE(toolchainFilePackage.defaultPath.cleanPath().toString(), toolchainFile);
+    QCOMPARE(toolchainFilePackage.defaultPath.cleanPath().toString(), toolchainFileDefaultPath);
 }
 
 void McuSupportTest::test_legacy_createIarToolchain()
@@ -836,22 +844,22 @@ void McuSupportTest::test_legacy_createTargetWithToolchainPackages_data()
     QTest::addColumn<QString>("compilerSetting");
     QTest::addColumn<QStringList>("versions");
 
-    QTest::newRow("nxp1050") << armgcc_nxp_1050_json << armGccToolchainFilePath << armGccDir
+    QTest::newRow("nxp1050") << armgcc_nxp_1050_json << armGccToolchainFileDefaultPath << armGccDir
                              << armGccDirectorySetting
                              << QStringList{armGccVersion, armGccNewVersion};
-    QTest::newRow("stm32h750b") << armgcc_stm32h750b_metal_json << armGccToolchainFilePath
+    QTest::newRow("stm32h750b") << armgcc_stm32h750b_metal_json << armGccToolchainFileDefaultPath
                                 << armGccDir << armGccDirectorySetting
                                 << QStringList{armGccVersion};
-    QTest::newRow("stm32f769i") << armgcc_stm32f769i_freertos_json << armGccToolchainFilePath
+    QTest::newRow("stm32f769i") << armgcc_stm32f769i_freertos_json << armGccToolchainFileDefaultPath
                                 << armGccDir << armGccDirectorySetting
                                 << QStringList{armGccVersion, armGccNewVersion};
-    QTest::newRow("stm32f469i") << iar_stm32f469i_metal_json << iarToolchainFilePath << iarDir
+    QTest::newRow("stm32f469i") << iar_stm32f469i_metal_json << iarToolchainFileDefaultPath
-                                << iarSetting << iarVersions;
+                                << iarDir << iarSetting << iarVersions;
     QTest::newRow("iar_nxp_1064_json")
-        << iar_nxp_1064_json << iarToolchainFilePath << iarDir << iarSetting << iarVersions;
+        << iar_nxp_1064_json << iarToolchainFileDefaultPath << iarDir << iarSetting << iarVersions;
     QTest::newRow("ghs_rh850_d1m1a_baremetal_json")
-        << ghs_rh850_d1m1a_baremetal_json << greenhillToolchainFilePath << greenhillCompilerDir
+        << ghs_rh850_d1m1a_baremetal_json << greenhillToolchainFileDefaultPath
-        << greenhillSetting << greenhillVersions;
+        << greenhillCompilerDir << greenhillSetting << greenhillVersions;
 }
 
 void McuSupportTest::test_legacy_createTargetWithToolchainPackages()
@@ -893,8 +901,9 @@ void McuSupportTest::test_createTargetWithToolchainPackages()
     EXPECT_CALL(*settingsMockPtr, getPath(compilerSetting, _, _))
         .WillRepeatedly(Return(FilePath::fromUserInput(compilerPath)));
 
-    EXPECT_CALL(*settingsMockPtr, getPath(compilerSetting, _, _))
+    EXPECT_CALL(*settingsMockPtr,
-        .WillRepeatedly(Return(FilePath::fromUserInput(compilerPath)));
+                getPath(QString{Constants::SETTINGS_KEY_PACKAGE_QT_FOR_MCUS_SDK}, _, _))
+        .WillRepeatedly(Return(FilePath::fromUserInput(qtForMcuSdkPath)));
 
     const McuTargetDescription description = parseDescriptionJson(json.toLocal8Bit());
     const auto [targets, packages]{
@@ -1134,4 +1143,35 @@ void McuSupportTest::test_passDirectoryVersionDetectorToRenesasBoardSdkPackage()
     QCOMPARE(typeid(*versionDetector).name(), typeid(McuPackageDirectoryVersionDetector).name());
 }
 
+void McuSupportTest::test_resolveEnvironmentVariablesInDefaultPaths()
+{
+    QVERIFY(qputenv(qulEnvVar, qtForMcuSdkPath));
+    QCOMPARE(qEnvironmentVariable(qulEnvVar), qtForMcuSdkPath);
+
+    toochainFileDescription.defaultPath = "$Qul_ROOT/lib/cmake/Qul/toolchain/iar.cmake";
+    targetDescription.toolchain.file = toochainFileDescription;
+
+    auto [targets, packages] = targetFactory.createTargets(targetDescription, qtForMcuSdkPath);
+    auto qtForMCUPkg = findOrDefault(packages, [](const McuPackagePtr &pkg) {
+        return (pkg->cmakeVariableName() == Legacy::Constants::QUL_ENV_VAR);
+    });
+
+    QVERIFY(qtForMCUPkg);
+    QCOMPARE(qtForMCUPkg->path().toString(), qtForMcuSdkPath);
+
+    auto toolchainFilePkg = findOrDefault(packages, [](const McuPackagePtr &pkg) {
+        return (pkg->cmakeVariableName() == Legacy::Constants::TOOLCHAIN_FILE_CMAKE_VARIABLE);
+    });
+
+    QVERIFY(toolchainFilePkg);
+    QVERIFY(targets.size() == 1);
+
+    QString expectedPkgPath = QString{qtForMcuSdkPath} + "/lib/cmake/Qul/toolchain/iar.cmake";
+    QCOMPARE(toolchainFilePkg->path().toString(), expectedPkgPath);
+    QVERIFY(toolchainFilePkg->path().toString().startsWith(qtForMcuSdkPath));
+    QCOMPARE(toolchainFilePkg->defaultPath().toString(), expectedPkgPath);
+
+    QVERIFY(qunsetenv(qulEnvVar));
+}
+
 } // namespace McuSupport::Internal::Test

src/plugins/mcusupport/test/unittest.h

@@ -106,6 +106,8 @@ private slots:
     void test_createBoardSdk_data();
     void test_createBoardSdk();
 
+    void test_resolveEnvironmentVariablesInDefaultPaths();
+
 private:
     QVersionNumber currentQulVersion{2, 0};
     PackageMock *freeRtosPackage{new PackageMock};