Enhanced the handling of built-in variables during interpolation // Resolve #4695

HISTORY.rst

@@ -23,6 +23,7 @@ test-driven methodologies, and modern toolchains for unrivaled success.
 * Introduced a warning during the verification of MCU maximum RAM usage, signaling when the allocated RAM surpasses 100% (`issue #4791 <https://github.com/platformio/platformio-core/issues/4791>`_)
 * Drastically enhanced the speed of project building when operating in verbose mode (`issue #4783 <https://github.com/platformio/platformio-core/issues/4783>`_)
 * Upgraded the build engine to the latest version of SCons (4.6.0) to improve build performance, reliability, and compatibility with other tools and systems (`release notes <https://github.com/SCons/scons/releases/tag/4.6.0>`__)
+* Enhanced the handling of built-in variables in |PIOCONF| during |INTERPOLATION| (`issue #4695 <https://github.com/platformio/platformio-core/issues/4695>`_)
 * Resolved an issue where the ``COMPILATIONDB_INCLUDE_TOOLCHAIN`` setting was not correctly applying to private libraries (`issue #4762 <https://github.com/platformio/platformio-core/issues/4762>`_)
 * Resolved an issue where ``get_systype()`` inaccurately returned the architecture when executed within a Docker container on a 64-bit kernel with a 32-bit userspace (`issue #4777 <https://github.com/platformio/platformio-core/issues/4777>`_)
 * Resolved an issue with incorrect handling of the ``check_src_filters`` option when used in multiple environments (`issue #4788 <https://github.com/platformio/platformio-core/issues/4788>`_)

platformio/__init__.py

@@ -67,7 +67,7 @@ __install_requires__ = [
 ] + [
     # PIO Home requirements
     "ajsonrpc == 1.2.*",
-    "starlette >=0.19, <0.33",
+    "starlette >=0.19, <=0.33",
     "uvicorn >=0.16, <0.24",
     "wsproto == 1.*",
 ]

platformio/project/config.py

@@ -14,14 +14,16 @@
 
 import configparser
 import glob
+import hashlib
 import json
 import os
 import re
+import time
 
 import click
 
 from platformio import fs
-from platformio.compat import MISSING, string_types
+from platformio.compat import MISSING, hashlib_encode_data, string_types
 from platformio.project import exception
 from platformio.project.options import ProjectOptions
 
@@ -41,7 +43,17 @@ CONFIG_HEADER = """
 class ProjectConfigBase:
     ENVNAME_RE = re.compile(r"^[a-z\d\_\-]+$", flags=re.I)
     INLINE_COMMENT_RE = re.compile(r"\s+;.*$")
-    VARTPL_RE = re.compile(r"\$\{([^\.\}\()]+)\.([^\}]+)\}")
+    VARTPL_RE = re.compile(r"\$\{(?:([^\.\}\()]+)\.)?([^\}]+)\}")
+
+    BUILTIN_VARS = {
+        "PROJECT_DIR": lambda: os.getcwd(),  # pylint: disable=unnecessary-lambda
+        "PROJECT_HASH": lambda: "%s-%s"
+        % (
+            os.path.basename(os.getcwd()),
+            hashlib.sha1(hashlib_encode_data(os.getcwd())).hexdigest()[:10],
+        ),
+        "UNIX_TIME": lambda: str(int(time.time())),
+    }
 
     CUSTOM_OPTION_PREFIXES = ("custom_", "board_")
 
@@ -274,7 +286,7 @@ class ProjectConfigBase:
                 value = (
                     default if default != MISSING else self._parser.get(section, option)
                 )
-            return self._expand_interpolations(section, value)
+            return self._expand_interpolations(section, option, value)
 
         if option_meta.sysenvvar:
             envvar_value = os.getenv(option_meta.sysenvvar)
@@ -297,24 +309,46 @@ class ProjectConfigBase:
         if value == MISSING:
             return None
 
-        return self._expand_interpolations(section, value)
+        return self._expand_interpolations(section, option, value)
 
-    def _expand_interpolations(self, parent_section, value):
+    def _expand_interpolations(self, section, option, value):
-        if (
+        if not value or not isinstance(value, string_types) or not "$" in value:
-            not value
+            return value
-            or not isinstance(value, string_types)
+
-            or not all(["${" in value, "}" in value])
+        # legacy support for variables delclared without "${}"
-        ):
+        stop = False
+        while not stop:
+            stop = True
+            for name in self.BUILTIN_VARS:
+                x = value.find(f"${name}")
+                if x < 0 or value[x - 1] == "$":
+                    continue
+                value = "%s${%s}%s" % (value[:x], name, value[x + len(name) + 1 :])
+                stop = False
+                warn_msg = (
+                    "Invalid variable declaration. Please use "
+                    f"`${{{name}}}` instead of `${name}`"
+                )
+                if warn_msg not in self.warnings:
+                    self.warnings.append(warn_msg)
+
+        if not all(["${" in value, "}" in value]):
             return value
         return self.VARTPL_RE.sub(
-            lambda match: self._re_interpolation_handler(parent_section, match), value
+            lambda match: self._re_interpolation_handler(section, option, match), value
         )
 
-    def _re_interpolation_handler(self, parent_section, match):
+    def _re_interpolation_handler(self, parent_section, parent_option, match):
         section, option = match.group(1), match.group(2)
+
+        # handle built-in variables
+        if section is None and option in self.BUILTIN_VARS:
+            return self.BUILTIN_VARS[option]()
+
         # handle system environment variables
         if section == "sysenv":
             return os.getenv(option)
+
         # handle ${this.*}
         if section == "this":
             section = parent_section
@@ -322,21 +356,18 @@ class ProjectConfigBase:
                 if not parent_section.startswith("env:"):
                     raise exception.ProjectOptionValueError(
                         f"`${{this.__env__}}` is called from the `{parent_section}` "
-                        "section that is not valid PlatformIO environment, see",
+                        "section that is not valid PlatformIO environment. Please "
-                        option,
+                        f"check `{parent_option}` option in the `{section}` section"
-                        " ",
-                        section,
                     )
                 return parent_section[4:]
+
         # handle nested calls
         try:
             value = self.get(section, option)
         except RecursionError as exc:
             raise exception.ProjectOptionValueError(
-                "Infinite recursion has been detected",
+                f"Infinite recursion has been detected for `{option}` "
-                option,
+                f"option in the `{section}` section"
-                " ",
-                section,
             ) from exc
         if isinstance(value, list):
             return "\n".join(value)
@@ -363,10 +394,8 @@ class ProjectConfigBase:
             if not self.expand_interpolations:
                 return value
             raise exception.ProjectOptionValueError(
-                exc.format_message(),
+                "%s for `%s` option in the `%s` section (%s)"
-                option,
+                % (exc.format_message(), option, section, option_meta.description)
-                " (%s) " % option_meta.description,
-                section,
             )
 
     @staticmethod
@@ -439,8 +468,9 @@ class ProjectConfigLintMixin:
         try:
             config = cls.get_instance(path)
             config.validate(silent=True)
-            warnings = config.warnings
+            warnings = config.warnings  # in case "as_tuple" fails
             config.as_tuple()
+            warnings = config.warnings
         except Exception as exc:  # pylint: disable=broad-exception-caught
             if exc.__cause__ is not None:
                 exc = exc.__cause__

platformio/project/exception.py

@@ -51,4 +51,4 @@ class InvalidEnvNameError(ProjectError, UserSideException):
 
 
 class ProjectOptionValueError(ProjectError, UserSideException):
-    MESSAGE = "{0} for option `{1}`{2}in section [{3}]"
+    pass

platformio/project/options.py

@@ -14,14 +14,13 @@
 
 # pylint: disable=redefined-builtin, too-many-arguments
 
-import hashlib
 import os
 from collections import OrderedDict
 
 import click
 
 from platformio import fs
-from platformio.compat import IS_WINDOWS, hashlib_encode_data
+from platformio.compat import IS_WINDOWS
 
 
 class ConfigOption:  # pylint: disable=too-many-instance-attributes
@@ -80,30 +79,6 @@ def ConfigEnvOption(*args, **kwargs):
     return ConfigOption("env", *args, **kwargs)
 
 
-def calculate_path_hash(path):
-    return "%s-%s" % (
-        os.path.basename(path),
-        hashlib.sha1(hashlib_encode_data(path)).hexdigest()[:10],
-    )
-
-
-def expand_dir_templates(path):
-    project_dir = os.getcwd()
-    tpls = {
-        "$PROJECT_DIR": lambda: project_dir,
-        "$PROJECT_HASH": lambda: calculate_path_hash(project_dir),
-    }
-    done = False
-    while not done:
-        done = True
-        for tpl, cb in tpls.items():
-            if tpl not in path:
-                continue
-            path = path.replace(tpl, cb())
-            done = False
-    return path
-
-
 def validate_dir(path):
     if not path:
         return path
@@ -112,8 +87,6 @@ def validate_dir(path):
         return path
     if path.startswith("~"):
         path = fs.expanduser(path)
-    if "$" in path:
-        path = expand_dir_templates(path)
     return os.path.abspath(path)
 
 
@@ -240,7 +213,7 @@ ProjectOptions = OrderedDict(
                     "external library dependencies"
                 ),
                 sysenvvar="PLATFORMIO_WORKSPACE_DIR",
-                default=os.path.join("$PROJECT_DIR", ".pio"),
+                default=os.path.join("${PROJECT_DIR}", ".pio"),
                 validate=validate_dir,
             ),
             ConfigPlatformioOption(
@@ -274,7 +247,7 @@ ProjectOptions = OrderedDict(
                     "System automatically adds this path to CPPPATH scope"
                 ),
                 sysenvvar="PLATFORMIO_INCLUDE_DIR",
-                default=os.path.join("$PROJECT_DIR", "include"),
+                default=os.path.join("${PROJECT_DIR}", "include"),
                 validate=validate_dir,
             ),
             ConfigPlatformioOption(
@@ -285,7 +258,7 @@ ProjectOptions = OrderedDict(
                     "project C/C++ source files"
                 ),
                 sysenvvar="PLATFORMIO_SRC_DIR",
-                default=os.path.join("$PROJECT_DIR", "src"),
+                default=os.path.join("${PROJECT_DIR}", "src"),
                 validate=validate_dir,
             ),
             ConfigPlatformioOption(
@@ -293,7 +266,7 @@ ProjectOptions = OrderedDict(
                 name="lib_dir",
                 description="A storage for the custom/private project libraries",
                 sysenvvar="PLATFORMIO_LIB_DIR",
-                default=os.path.join("$PROJECT_DIR", "lib"),
+                default=os.path.join("${PROJECT_DIR}", "lib"),
                 validate=validate_dir,
             ),
             ConfigPlatformioOption(
@@ -304,7 +277,7 @@ ProjectOptions = OrderedDict(
                     "file system (SPIFFS, etc.)"
                 ),
                 sysenvvar="PLATFORMIO_DATA_DIR",
-                default=os.path.join("$PROJECT_DIR", "data"),
+                default=os.path.join("${PROJECT_DIR}", "data"),
                 validate=validate_dir,
             ),
             ConfigPlatformioOption(
@@ -315,7 +288,7 @@ ProjectOptions = OrderedDict(
                     "test source files"
                 ),
                 sysenvvar="PLATFORMIO_TEST_DIR",
-                default=os.path.join("$PROJECT_DIR", "test"),
+                default=os.path.join("${PROJECT_DIR}", "test"),
                 validate=validate_dir,
             ),
             ConfigPlatformioOption(
@@ -323,7 +296,7 @@ ProjectOptions = OrderedDict(
                 name="boards_dir",
                 description="A storage for custom board manifests",
                 sysenvvar="PLATFORMIO_BOARDS_DIR",
-                default=os.path.join("$PROJECT_DIR", "boards"),
+                default=os.path.join("${PROJECT_DIR}", "boards"),
                 validate=validate_dir,
             ),
             ConfigPlatformioOption(
@@ -331,7 +304,7 @@ ProjectOptions = OrderedDict(
                 name="monitor_dir",
                 description="A storage for custom monitor filters",
                 sysenvvar="PLATFORMIO_MONITOR_DIR",
-                default=os.path.join("$PROJECT_DIR", "monitor"),
+                default=os.path.join("${PROJECT_DIR}", "monitor"),
                 validate=validate_dir,
             ),
             ConfigPlatformioOption(
@@ -342,7 +315,7 @@ ProjectOptions = OrderedDict(
                     "synchronize extra files between remote machines"
                 ),
                 sysenvvar="PLATFORMIO_SHARED_DIR",
-                default=os.path.join("$PROJECT_DIR", "shared"),
+                default=os.path.join("${PROJECT_DIR}", "shared"),
                 validate=validate_dir,
             ),
             #

tests/project/test_config.py

@@ -33,7 +33,6 @@ BASE_CONFIG = """
 [platformio]
 env_default = base, extra_2
 src_dir = ${custom.src_dir}
-build_dir = ${custom.build_dir}
 extra_configs =
   extra_envs.ini
   extra_debug.ini
@@ -61,7 +60,6 @@ build_flags = -D RELEASE
 
 [custom]
 src_dir = source
-build_dir = ~/tmp/pio-$PROJECT_HASH
 debug_flags = -D RELEASE
 lib_flags = -lc -lm
 extra_flags = ${sysenv.__PIO_TEST_CNF_EXTRA_FLAGS}
@@ -319,7 +317,6 @@ def test_getraw_value(config):
         config.getraw("custom", "debug_server")
         == f"\n{packages_dir}/tool-openocd/openocd\n--help"
     )
-    assert config.getraw("platformio", "build_dir") == "~/tmp/pio-$PROJECT_HASH"
 
     # renamed option
     assert config.getraw("env:extra_1", "lib_install") == "574"
@@ -360,7 +357,6 @@ def test_get_value(config):
     assert config.get("platformio", "src_dir") == os.path.abspath(
         os.path.join(os.getcwd(), "source")
     )
-    assert "$PROJECT_HASH" not in config.get("platformio", "build_dir")
 
     # renamed option
     assert config.get("env:extra_1", "lib_install") == ["574"]
@@ -371,7 +367,6 @@ def test_get_value(config):
 def test_items(config):
     assert config.items("custom") == [
         ("src_dir", "source"),
-        ("build_dir", "~/tmp/pio-$PROJECT_HASH"),
         ("debug_flags", "-D DEBUG=1"),
         ("lib_flags", "-lc -lm"),
         ("extra_flags", ""),
@@ -525,7 +520,6 @@ def test_dump(tmpdir_factory):
             [
                 ("env_default", ["base", "extra_2"]),
                 ("src_dir", "${custom.src_dir}"),
-                ("build_dir", "${custom.build_dir}"),
                 ("extra_configs", ["extra_envs.ini", "extra_debug.ini"]),
             ],
         ),
@@ -549,7 +543,6 @@ def test_dump(tmpdir_factory):
             "custom",
             [
                 ("src_dir", "source"),
-                ("build_dir", "~/tmp/pio-$PROJECT_HASH"),
                 ("debug_flags", "-D RELEASE"),
                 ("lib_flags", "-lc -lm"),
                 ("extra_flags", "${sysenv.__PIO_TEST_CNF_EXTRA_FLAGS}"),
@@ -636,9 +629,10 @@ def test_nested_interpolation(tmp_path: Path):
     project_conf.write_text(
         """
 [platformio]
-build_dir = ~/tmp/pio-$PROJECT_HASH
+build_dir = /tmp/pio-$PROJECT_HASH
 
 [env:myenv]
+build_flags = -D UTIME=${UNIX_TIME}
 test_testing_command =
     ${platformio.packages_dir}/tool-simavr/bin/simavr
      -m
@@ -651,6 +645,7 @@ test_testing_command =
     config = ProjectConfig(str(project_conf))
     testing_command = config.get("env:myenv", "test_testing_command")
     assert "$" not in " ".join(testing_command)
+    assert config.get("env:myenv", "build_flags")[0][-10:].isdigit()
 
 
 def test_extends_order(tmp_path: Path):
@@ -707,11 +702,16 @@ def test_linting_warnings(tmp_path: Path):
     project_conf = tmp_path / "platformio.ini"
     project_conf.write_text(
         """
+[platformio]
+build_dir = /tmp/pio-$PROJECT_HASH
+
 [env:app1]
 lib_use = 1
+test_testing_command = /usr/bin/flash-tool -p $UPLOAD_PORT -b $UPLOAD_SPEED
     """
     )
     result = ProjectConfig.lint(str(project_conf))
     assert not result["errors"]
-    assert result["warnings"] and len(result["warnings"]) == 1
+    assert result["warnings"] and len(result["warnings"]) == 2
     assert "deprecated" in result["warnings"][0]
+    assert "Invalid variable declaration" in result["warnings"][1]

tox.ini

@@ -55,7 +55,7 @@ commands =
 [testenv:docs]
 deps =
     sphinx
-    sphinx-rtd-theme==1.2.2
+    sphinx-rtd-theme==2.0.0
     sphinx-notfound-page
     sphinx-copybutton
     restructuredtext-lint