diff --git a/HISTORY.rst b/HISTORY.rst index b88d8b32..2adfdf86 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -23,6 +23,7 @@ test-driven methodologies, and modern toolchains for unrivaled success. * Introduced a warning during the verification of MCU maximum RAM usage, signaling when the allocated RAM surpasses 100% (`issue #4791 `_) * Drastically enhanced the speed of project building when operating in verbose mode (`issue #4783 `_) * Upgraded the build engine to the latest version of SCons (4.6.0) to improve build performance, reliability, and compatibility with other tools and systems (`release notes `__) +* Enhanced the handling of built-in variables in |PIOCONF| during |INTERPOLATION| (`issue #4695 `_) * Resolved an issue where the ``COMPILATIONDB_INCLUDE_TOOLCHAIN`` setting was not correctly applying to private libraries (`issue #4762 `_) * Resolved an issue where ``get_systype()`` inaccurately returned the architecture when executed within a Docker container on a 64-bit kernel with a 32-bit userspace (`issue #4777 `_) * Resolved an issue with incorrect handling of the ``check_src_filters`` option when used in multiple environments (`issue #4788 `_) diff --git a/docs b/docs index 07f966a6..5f5efa22 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 07f966a65c2392cae4bcfc1b0e4f9f8a85b831d0 +Subproject commit 5f5efa22b88562fe755d206eea0faa2f097ea70c diff --git a/platformio/__init__.py b/platformio/__init__.py index 5c82336a..a61cd4f4 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -67,7 +67,7 @@ __install_requires__ = [ ] + [ # PIO Home requirements "ajsonrpc == 1.2.*", - "starlette >=0.19, <0.33", + "starlette >=0.19, <=0.33", "uvicorn >=0.16, <0.24", "wsproto == 1.*", ] diff --git a/platformio/project/config.py b/platformio/project/config.py index 8020fdd3..f19473f8 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -14,14 +14,16 @@ import configparser import glob +import hashlib import json import os import re +import time import click from platformio import fs -from platformio.compat import MISSING, string_types +from platformio.compat import MISSING, hashlib_encode_data, string_types from platformio.project import exception from platformio.project.options import ProjectOptions @@ -41,7 +43,17 @@ CONFIG_HEADER = """ class ProjectConfigBase: ENVNAME_RE = re.compile(r"^[a-z\d\_\-]+$", flags=re.I) INLINE_COMMENT_RE = re.compile(r"\s+;.*$") - VARTPL_RE = re.compile(r"\$\{([^\.\}\()]+)\.([^\}]+)\}") + VARTPL_RE = re.compile(r"\$\{(?:([^\.\}\()]+)\.)?([^\}]+)\}") + + BUILTIN_VARS = { + "PROJECT_DIR": lambda: os.getcwd(), # pylint: disable=unnecessary-lambda + "PROJECT_HASH": lambda: "%s-%s" + % ( + os.path.basename(os.getcwd()), + hashlib.sha1(hashlib_encode_data(os.getcwd())).hexdigest()[:10], + ), + "UNIX_TIME": lambda: str(int(time.time())), + } CUSTOM_OPTION_PREFIXES = ("custom_", "board_") @@ -274,7 +286,7 @@ class ProjectConfigBase: value = ( default if default != MISSING else self._parser.get(section, option) ) - return self._expand_interpolations(section, value) + return self._expand_interpolations(section, option, value) if option_meta.sysenvvar: envvar_value = os.getenv(option_meta.sysenvvar) @@ -297,24 +309,46 @@ class ProjectConfigBase: if value == MISSING: return None - return self._expand_interpolations(section, value) + return self._expand_interpolations(section, option, value) - def _expand_interpolations(self, parent_section, value): - if ( - not value - or not isinstance(value, string_types) - or not all(["${" in value, "}" in value]) - ): + def _expand_interpolations(self, section, option, value): + if not value or not isinstance(value, string_types) or not "$" in value: + return value + + # legacy support for variables delclared without "${}" + stop = False + while not stop: + stop = True + for name in self.BUILTIN_VARS: + x = value.find(f"${name}") + if x < 0 or value[x - 1] == "$": + continue + value = "%s${%s}%s" % (value[:x], name, value[x + len(name) + 1 :]) + stop = False + warn_msg = ( + "Invalid variable declaration. Please use " + f"`${{{name}}}` instead of `${name}`" + ) + if warn_msg not in self.warnings: + self.warnings.append(warn_msg) + + if not all(["${" in value, "}" in value]): return value return self.VARTPL_RE.sub( - lambda match: self._re_interpolation_handler(parent_section, match), value + lambda match: self._re_interpolation_handler(section, option, match), value ) - def _re_interpolation_handler(self, parent_section, match): + def _re_interpolation_handler(self, parent_section, parent_option, match): section, option = match.group(1), match.group(2) + + # handle built-in variables + if section is None and option in self.BUILTIN_VARS: + return self.BUILTIN_VARS[option]() + # handle system environment variables if section == "sysenv": return os.getenv(option) + # handle ${this.*} if section == "this": section = parent_section @@ -322,21 +356,18 @@ class ProjectConfigBase: if not parent_section.startswith("env:"): raise exception.ProjectOptionValueError( f"`${{this.__env__}}` is called from the `{parent_section}` " - "section that is not valid PlatformIO environment, see", - option, - " ", - section, + "section that is not valid PlatformIO environment. Please " + f"check `{parent_option}` option in the `{section}` section" ) return parent_section[4:] + # handle nested calls try: value = self.get(section, option) except RecursionError as exc: raise exception.ProjectOptionValueError( - "Infinite recursion has been detected", - option, - " ", - section, + f"Infinite recursion has been detected for `{option}` " + f"option in the `{section}` section" ) from exc if isinstance(value, list): return "\n".join(value) @@ -363,10 +394,8 @@ class ProjectConfigBase: if not self.expand_interpolations: return value raise exception.ProjectOptionValueError( - exc.format_message(), - option, - " (%s) " % option_meta.description, - section, + "%s for `%s` option in the `%s` section (%s)" + % (exc.format_message(), option, section, option_meta.description) ) @staticmethod @@ -439,8 +468,9 @@ class ProjectConfigLintMixin: try: config = cls.get_instance(path) config.validate(silent=True) - warnings = config.warnings + warnings = config.warnings # in case "as_tuple" fails config.as_tuple() + warnings = config.warnings except Exception as exc: # pylint: disable=broad-exception-caught if exc.__cause__ is not None: exc = exc.__cause__ diff --git a/platformio/project/exception.py b/platformio/project/exception.py index 3821c865..cf97c69e 100644 --- a/platformio/project/exception.py +++ b/platformio/project/exception.py @@ -51,4 +51,4 @@ class InvalidEnvNameError(ProjectError, UserSideException): class ProjectOptionValueError(ProjectError, UserSideException): - MESSAGE = "{0} for option `{1}`{2}in section [{3}]" + pass diff --git a/platformio/project/options.py b/platformio/project/options.py index 2e13815a..70f3219e 100644 --- a/platformio/project/options.py +++ b/platformio/project/options.py @@ -14,14 +14,13 @@ # pylint: disable=redefined-builtin, too-many-arguments -import hashlib import os from collections import OrderedDict import click from platformio import fs -from platformio.compat import IS_WINDOWS, hashlib_encode_data +from platformio.compat import IS_WINDOWS class ConfigOption: # pylint: disable=too-many-instance-attributes @@ -80,30 +79,6 @@ def ConfigEnvOption(*args, **kwargs): return ConfigOption("env", *args, **kwargs) -def calculate_path_hash(path): - return "%s-%s" % ( - os.path.basename(path), - hashlib.sha1(hashlib_encode_data(path)).hexdigest()[:10], - ) - - -def expand_dir_templates(path): - project_dir = os.getcwd() - tpls = { - "$PROJECT_DIR": lambda: project_dir, - "$PROJECT_HASH": lambda: calculate_path_hash(project_dir), - } - done = False - while not done: - done = True - for tpl, cb in tpls.items(): - if tpl not in path: - continue - path = path.replace(tpl, cb()) - done = False - return path - - def validate_dir(path): if not path: return path @@ -112,8 +87,6 @@ def validate_dir(path): return path if path.startswith("~"): path = fs.expanduser(path) - if "$" in path: - path = expand_dir_templates(path) return os.path.abspath(path) @@ -240,7 +213,7 @@ ProjectOptions = OrderedDict( "external library dependencies" ), sysenvvar="PLATFORMIO_WORKSPACE_DIR", - default=os.path.join("$PROJECT_DIR", ".pio"), + default=os.path.join("${PROJECT_DIR}", ".pio"), validate=validate_dir, ), ConfigPlatformioOption( @@ -274,7 +247,7 @@ ProjectOptions = OrderedDict( "System automatically adds this path to CPPPATH scope" ), sysenvvar="PLATFORMIO_INCLUDE_DIR", - default=os.path.join("$PROJECT_DIR", "include"), + default=os.path.join("${PROJECT_DIR}", "include"), validate=validate_dir, ), ConfigPlatformioOption( @@ -285,7 +258,7 @@ ProjectOptions = OrderedDict( "project C/C++ source files" ), sysenvvar="PLATFORMIO_SRC_DIR", - default=os.path.join("$PROJECT_DIR", "src"), + default=os.path.join("${PROJECT_DIR}", "src"), validate=validate_dir, ), ConfigPlatformioOption( @@ -293,7 +266,7 @@ ProjectOptions = OrderedDict( name="lib_dir", description="A storage for the custom/private project libraries", sysenvvar="PLATFORMIO_LIB_DIR", - default=os.path.join("$PROJECT_DIR", "lib"), + default=os.path.join("${PROJECT_DIR}", "lib"), validate=validate_dir, ), ConfigPlatformioOption( @@ -304,7 +277,7 @@ ProjectOptions = OrderedDict( "file system (SPIFFS, etc.)" ), sysenvvar="PLATFORMIO_DATA_DIR", - default=os.path.join("$PROJECT_DIR", "data"), + default=os.path.join("${PROJECT_DIR}", "data"), validate=validate_dir, ), ConfigPlatformioOption( @@ -315,7 +288,7 @@ ProjectOptions = OrderedDict( "test source files" ), sysenvvar="PLATFORMIO_TEST_DIR", - default=os.path.join("$PROJECT_DIR", "test"), + default=os.path.join("${PROJECT_DIR}", "test"), validate=validate_dir, ), ConfigPlatformioOption( @@ -323,7 +296,7 @@ ProjectOptions = OrderedDict( name="boards_dir", description="A storage for custom board manifests", sysenvvar="PLATFORMIO_BOARDS_DIR", - default=os.path.join("$PROJECT_DIR", "boards"), + default=os.path.join("${PROJECT_DIR}", "boards"), validate=validate_dir, ), ConfigPlatformioOption( @@ -331,7 +304,7 @@ ProjectOptions = OrderedDict( name="monitor_dir", description="A storage for custom monitor filters", sysenvvar="PLATFORMIO_MONITOR_DIR", - default=os.path.join("$PROJECT_DIR", "monitor"), + default=os.path.join("${PROJECT_DIR}", "monitor"), validate=validate_dir, ), ConfigPlatformioOption( @@ -342,7 +315,7 @@ ProjectOptions = OrderedDict( "synchronize extra files between remote machines" ), sysenvvar="PLATFORMIO_SHARED_DIR", - default=os.path.join("$PROJECT_DIR", "shared"), + default=os.path.join("${PROJECT_DIR}", "shared"), validate=validate_dir, ), # diff --git a/tests/project/test_config.py b/tests/project/test_config.py index 6b742855..47c8f283 100644 --- a/tests/project/test_config.py +++ b/tests/project/test_config.py @@ -33,7 +33,6 @@ BASE_CONFIG = """ [platformio] env_default = base, extra_2 src_dir = ${custom.src_dir} -build_dir = ${custom.build_dir} extra_configs = extra_envs.ini extra_debug.ini @@ -61,7 +60,6 @@ build_flags = -D RELEASE [custom] src_dir = source -build_dir = ~/tmp/pio-$PROJECT_HASH debug_flags = -D RELEASE lib_flags = -lc -lm extra_flags = ${sysenv.__PIO_TEST_CNF_EXTRA_FLAGS} @@ -319,7 +317,6 @@ def test_getraw_value(config): config.getraw("custom", "debug_server") == f"\n{packages_dir}/tool-openocd/openocd\n--help" ) - assert config.getraw("platformio", "build_dir") == "~/tmp/pio-$PROJECT_HASH" # renamed option assert config.getraw("env:extra_1", "lib_install") == "574" @@ -360,7 +357,6 @@ def test_get_value(config): assert config.get("platformio", "src_dir") == os.path.abspath( os.path.join(os.getcwd(), "source") ) - assert "$PROJECT_HASH" not in config.get("platformio", "build_dir") # renamed option assert config.get("env:extra_1", "lib_install") == ["574"] @@ -371,7 +367,6 @@ def test_get_value(config): def test_items(config): assert config.items("custom") == [ ("src_dir", "source"), - ("build_dir", "~/tmp/pio-$PROJECT_HASH"), ("debug_flags", "-D DEBUG=1"), ("lib_flags", "-lc -lm"), ("extra_flags", ""), @@ -525,7 +520,6 @@ def test_dump(tmpdir_factory): [ ("env_default", ["base", "extra_2"]), ("src_dir", "${custom.src_dir}"), - ("build_dir", "${custom.build_dir}"), ("extra_configs", ["extra_envs.ini", "extra_debug.ini"]), ], ), @@ -549,7 +543,6 @@ def test_dump(tmpdir_factory): "custom", [ ("src_dir", "source"), - ("build_dir", "~/tmp/pio-$PROJECT_HASH"), ("debug_flags", "-D RELEASE"), ("lib_flags", "-lc -lm"), ("extra_flags", "${sysenv.__PIO_TEST_CNF_EXTRA_FLAGS}"), @@ -636,9 +629,10 @@ def test_nested_interpolation(tmp_path: Path): project_conf.write_text( """ [platformio] -build_dir = ~/tmp/pio-$PROJECT_HASH +build_dir = /tmp/pio-$PROJECT_HASH [env:myenv] +build_flags = -D UTIME=${UNIX_TIME} test_testing_command = ${platformio.packages_dir}/tool-simavr/bin/simavr -m @@ -651,6 +645,7 @@ test_testing_command = config = ProjectConfig(str(project_conf)) testing_command = config.get("env:myenv", "test_testing_command") assert "$" not in " ".join(testing_command) + assert config.get("env:myenv", "build_flags")[0][-10:].isdigit() def test_extends_order(tmp_path: Path): @@ -707,11 +702,16 @@ def test_linting_warnings(tmp_path: Path): project_conf = tmp_path / "platformio.ini" project_conf.write_text( """ +[platformio] +build_dir = /tmp/pio-$PROJECT_HASH + [env:app1] lib_use = 1 +test_testing_command = /usr/bin/flash-tool -p $UPLOAD_PORT -b $UPLOAD_SPEED """ ) result = ProjectConfig.lint(str(project_conf)) assert not result["errors"] - assert result["warnings"] and len(result["warnings"]) == 1 + assert result["warnings"] and len(result["warnings"]) == 2 assert "deprecated" in result["warnings"][0] + assert "Invalid variable declaration" in result["warnings"][1] diff --git a/tox.ini b/tox.ini index 2eda9b90..02b96152 100644 --- a/tox.ini +++ b/tox.ini @@ -55,7 +55,7 @@ commands = [testenv:docs] deps = sphinx - sphinx-rtd-theme==1.2.2 + sphinx-rtd-theme==2.0.0 sphinx-notfound-page sphinx-copybutton restructuredtext-lint