platformio/platformio-core
1
0
Fork 1
mirror of https://github.com/platformio/platformio-core.git synced 2025-07-29 17:47:14 +02:00
Code Issues Packages Projects Releases Wiki Activity

Enhanced the handling of built-in variables during interpolation // Resolve #4695

Browse Source
This commit is contained in:
Ivan Kravets
2023-12-08 19:12:00 +02:00
parent fb93c1937c
commit 8c61f0f6b6
8 changed files with 79 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
HISTORY.rst
View File

@ -23,6 +23,7 @@ test-driven methodologies, and modern toolchains for unrivaled success.
* Introduced a warning during the verification of MCU maximum RAM usage, signaling when the allocated RAM surpasses 100% (`issue #4791 <https://github.com/platformio/platformio-core/issues/4791>`_)
* Drastically enhanced the speed of project building when operating in verbose mode (`issue #4783 <https://github.com/platformio/platformio-core/issues/4783>`_)
* Upgraded the build engine to the latest version of SCons (4.6.0) to improve build performance, reliability, and compatibility with other tools and systems (`release notes <https://github.com/SCons/scons/releases/tag/4.6.0>`__)
* Enhanced the handling of built-in variables in |PIOCONF| during |INTERPOLATION| (`issue #4695 <https://github.com/platformio/platformio-core/issues/4695>`_)
* Resolved an issue where the ``COMPILATIONDB_INCLUDE_TOOLCHAIN`` setting was not correctly applying to private libraries (`issue #4762 <https://github.com/platformio/platformio-core/issues/4762>`_)
* Resolved an issue where ``get_systype()`` inaccurately returned the architecture when executed within a Docker container on a 64-bit kernel with a 32-bit userspace (`issue #4777 <https://github.com/platformio/platformio-core/issues/4777>`_)
* Resolved an issue with incorrect handling of the ``check_src_filters`` option when used in multiple environments (`issue #4788 <https://github.com/platformio/platformio-core/issues/4788>`_)

2
docs

Submodule docs updated: 07f966a65c...5f5efa22b8

2
platformio/__init__.py
View File

@ -67,7 +67,7 @@ __install_requires__ = [
] + [
# PIO Home requirements
"ajsonrpc == 1.2.*",
"starlette >=0.19, <0.33",
"starlette >=0.19, <=0.33",
"uvicorn >=0.16, <0.24",
"wsproto == 1.*",
]

80
platformio/project/config.py
View File

@ -14,14 +14,16 @@
import configparser
import glob
import hashlib
import json
import os
import re
import time
import click
from platformio import fs
from platformio.compat import MISSING, string_types
from platformio.compat import MISSING, hashlib_encode_data, string_types
from platformio.project import exception
from platformio.project.options import ProjectOptions
@ -41,7 +43,17 @@ CONFIG_HEADER = """
class ProjectConfigBase:
ENVNAME_RE = re.compile(r"^[a-z\d\_\-]+$", flags=re.I)
INLINE_COMMENT_RE = re.compile(r"\s+;.*$")
VARTPL_RE = re.compile(r"\$\{([^\.\}\()]+)\.([^\}]+)\}")
VARTPL_RE = re.compile(r"\$\{(?:([^\.\}\()]+)\.)?([^\}]+)\}")
BUILTIN_VARS = {
"PROJECT_DIR": lambda: os.getcwd(), # pylint: disable=unnecessary-lambda
"PROJECT_HASH": lambda: "%s-%s"
% (
os.path.basename(os.getcwd()),
hashlib.sha1(hashlib_encode_data(os.getcwd())).hexdigest()[:10],
),
"UNIX_TIME": lambda: str(int(time.time())),
}
CUSTOM_OPTION_PREFIXES = ("custom_", "board_")
@ -274,7 +286,7 @@ class ProjectConfigBase:
value = (
default if default != MISSING else self._parser.get(section, option)
)
return self._expand_interpolations(section, value)
return self._expand_interpolations(section, option, value)
if option_meta.sysenvvar:
envvar_value = os.getenv(option_meta.sysenvvar)
@ -297,24 +309,46 @@ class ProjectConfigBase:
if value == MISSING:
return None
return self._expand_interpolations(section, value)
return self._expand_interpolations(section, option, value)
def _expand_interpolations(self, parent_section, value):
if (
not value
or not isinstance(value, string_types)
or not all(["${" in value, "}" in value])
):
def _expand_interpolations(self, section, option, value):
if not value or not isinstance(value, string_types) or not "$" in value:
return value
# legacy support for variables delclared without "${}"
stop = False
while not stop:
stop = True
for name in self.BUILTIN_VARS:
x = value.find(f"${name}")
if x < 0 or value[x - 1] == "$":
continue
value = "%s${%s}%s" % (value[:x], name, value[x + len(name) + 1 :])
stop = False
warn_msg = (
"Invalid variable declaration. Please use "
f"`${{{name}}}` instead of `${name}`"
)
if warn_msg not in self.warnings:
self.warnings.append(warn_msg)
if not all(["${" in value, "}" in value]):
return value
return self.VARTPL_RE.sub(
lambda match: self._re_interpolation_handler(parent_section, match), value
lambda match: self._re_interpolation_handler(section, option, match), value
)
def _re_interpolation_handler(self, parent_section, match):
def _re_interpolation_handler(self, parent_section, parent_option, match):
section, option = match.group(1), match.group(2)
# handle built-in variables
if section is None and option in self.BUILTIN_VARS:
return self.BUILTIN_VARS[option]()
# handle system environment variables
if section == "sysenv":
return os.getenv(option)
# handle ${this.*}
if section == "this":
section = parent_section
@ -322,21 +356,18 @@ class ProjectConfigBase:
if not parent_section.startswith("env:"):
raise exception.ProjectOptionValueError(
f"`${{this.__env__}}` is called from the `{parent_section}` "
"section that is not valid PlatformIO environment, see",
option,
" ",
section,
"section that is not valid PlatformIO environment. Please "
f"check `{parent_option}` option in the `{section}` section"
)
return parent_section[4:]
# handle nested calls
try:
value = self.get(section, option)
except RecursionError as exc:
raise exception.ProjectOptionValueError(
"Infinite recursion has been detected",
option,
" ",
section,
f"Infinite recursion has been detected for `{option}` "
f"option in the `{section}` section"
) from exc
if isinstance(value, list):
return "\n".join(value)
@ -363,10 +394,8 @@ class ProjectConfigBase:
if not self.expand_interpolations:
return value
raise exception.ProjectOptionValueError(
exc.format_message(),
option,
" (%s) " % option_meta.description,
section,
"%s for `%s` option in the `%s` section (%s)"
% (exc.format_message(), option, section, option_meta.description)
)
@staticmethod
@ -439,8 +468,9 @@ class ProjectConfigLintMixin:
try:
config = cls.get_instance(path)
config.validate(silent=True)
warnings = config.warnings
warnings = config.warnings # in case "as_tuple" fails
config.as_tuple()
warnings = config.warnings
except Exception as exc: # pylint: disable=broad-exception-caught
if exc.__cause__ is not None:
exc = exc.__cause__

2
platformio/project/exception.py
View File

@ -51,4 +51,4 @@ class InvalidEnvNameError(ProjectError, UserSideException):
class ProjectOptionValueError(ProjectError, UserSideException):
MESSAGE = "{0} for option `{1}`{2}in section [{3}]"
pass

47
platformio/project/options.py
View File

@ -14,14 +14,13 @@
# pylint: disable=redefined-builtin, too-many-arguments
import hashlib
import os
from collections import OrderedDict
import click
from platformio import fs
from platformio.compat import IS_WINDOWS, hashlib_encode_data
from platformio.compat import IS_WINDOWS
class ConfigOption: # pylint: disable=too-many-instance-attributes
@ -80,30 +79,6 @@ def ConfigEnvOption(*args, **kwargs):
return ConfigOption("env", *args, **kwargs)
def calculate_path_hash(path):
return "%s-%s" % (
os.path.basename(path),
hashlib.sha1(hashlib_encode_data(path)).hexdigest()[:10],
)
def expand_dir_templates(path):
project_dir = os.getcwd()
tpls = {
"$PROJECT_DIR": lambda: project_dir,
"$PROJECT_HASH": lambda: calculate_path_hash(project_dir),
}
done = False
while not done:
done = True
for tpl, cb in tpls.items():
if tpl not in path:
continue
path = path.replace(tpl, cb())
done = False
return path
def validate_dir(path):
if not path:
return path
@ -112,8 +87,6 @@ def validate_dir(path):
return path
if path.startswith("~"):
path = fs.expanduser(path)
if "$" in path:
path = expand_dir_templates(path)
return os.path.abspath(path)
@ -240,7 +213,7 @@ ProjectOptions = OrderedDict(
"external library dependencies"
),
sysenvvar="PLATFORMIO_WORKSPACE_DIR",
default=os.path.join("$PROJECT_DIR", ".pio"),
default=os.path.join("${PROJECT_DIR}", ".pio"),
validate=validate_dir,
),
ConfigPlatformioOption(
@ -274,7 +247,7 @@ ProjectOptions = OrderedDict(
"System automatically adds this path to CPPPATH scope"
),
sysenvvar="PLATFORMIO_INCLUDE_DIR",
default=os.path.join("$PROJECT_DIR", "include"),
default=os.path.join("${PROJECT_DIR}", "include"),
validate=validate_dir,
),
ConfigPlatformioOption(
@ -285,7 +258,7 @@ ProjectOptions = OrderedDict(
"project C/C++ source files"
),
sysenvvar="PLATFORMIO_SRC_DIR",
default=os.path.join("$PROJECT_DIR", "src"),
default=os.path.join("${PROJECT_DIR}", "src"),
validate=validate_dir,
),
ConfigPlatformioOption(
@ -293,7 +266,7 @@ ProjectOptions = OrderedDict(
name="lib_dir",
description="A storage for the custom/private project libraries",
sysenvvar="PLATFORMIO_LIB_DIR",
default=os.path.join("$PROJECT_DIR", "lib"),
default=os.path.join("${PROJECT_DIR}", "lib"),
validate=validate_dir,
),
ConfigPlatformioOption(
@ -304,7 +277,7 @@ ProjectOptions = OrderedDict(
"file system (SPIFFS, etc.)"
),
sysenvvar="PLATFORMIO_DATA_DIR",
default=os.path.join("$PROJECT_DIR", "data"),
default=os.path.join("${PROJECT_DIR}", "data"),
validate=validate_dir,
),
ConfigPlatformioOption(
@ -315,7 +288,7 @@ ProjectOptions = OrderedDict(
"test source files"
),
sysenvvar="PLATFORMIO_TEST_DIR",
default=os.path.join("$PROJECT_DIR", "test"),
default=os.path.join("${PROJECT_DIR}", "test"),
validate=validate_dir,
),
ConfigPlatformioOption(
@ -323,7 +296,7 @@ ProjectOptions = OrderedDict(
name="boards_dir",
description="A storage for custom board manifests",
sysenvvar="PLATFORMIO_BOARDS_DIR",
default=os.path.join("$PROJECT_DIR", "boards"),
default=os.path.join("${PROJECT_DIR}", "boards"),
validate=validate_dir,
),
ConfigPlatformioOption(
@ -331,7 +304,7 @@ ProjectOptions = OrderedDict(
name="monitor_dir",
description="A storage for custom monitor filters",
sysenvvar="PLATFORMIO_MONITOR_DIR",
default=os.path.join("$PROJECT_DIR", "monitor"),
default=os.path.join("${PROJECT_DIR}", "monitor"),
validate=validate_dir,
),
ConfigPlatformioOption(
@ -342,7 +315,7 @@ ProjectOptions = OrderedDict(
"synchronize extra files between remote machines"
),
sysenvvar="PLATFORMIO_SHARED_DIR",
default=os.path.join("$PROJECT_DIR", "shared"),
default=os.path.join("${PROJECT_DIR}", "shared"),
validate=validate_dir,
),
#

18
tests/project/test_config.py
View File

@ -33,7 +33,6 @@ BASE_CONFIG = """
[platformio]
env_default = base, extra_2
src_dir = ${custom.src_dir}
build_dir = ${custom.build_dir}
extra_configs =
extra_envs.ini
extra_debug.ini
@ -61,7 +60,6 @@ build_flags = -D RELEASE
[custom]
src_dir = source
build_dir = ~/tmp/pio-$PROJECT_HASH
debug_flags = -D RELEASE
lib_flags = -lc -lm
extra_flags = ${sysenv.__PIO_TEST_CNF_EXTRA_FLAGS}
@ -319,7 +317,6 @@ def test_getraw_value(config):
config.getraw("custom", "debug_server")
== f"\n{packages_dir}/tool-openocd/openocd\n--help"
)
assert config.getraw("platformio", "build_dir") == "~/tmp/pio-$PROJECT_HASH"
# renamed option
assert config.getraw("env:extra_1", "lib_install") == "574"
@ -360,7 +357,6 @@ def test_get_value(config):
assert config.get("platformio", "src_dir") == os.path.abspath(
os.path.join(os.getcwd(), "source")
)
assert "$PROJECT_HASH" not in config.get("platformio", "build_dir")
# renamed option
assert config.get("env:extra_1", "lib_install") == ["574"]
@ -371,7 +367,6 @@ def test_get_value(config):
def test_items(config):
assert config.items("custom") == [
("src_dir", "source"),
("build_dir", "~/tmp/pio-$PROJECT_HASH"),
("debug_flags", "-D DEBUG=1"),
("lib_flags", "-lc -lm"),
("extra_flags", ""),
@ -525,7 +520,6 @@ def test_dump(tmpdir_factory):
[
("env_default", ["base", "extra_2"]),
("src_dir", "${custom.src_dir}"),
("build_dir", "${custom.build_dir}"),
("extra_configs", ["extra_envs.ini", "extra_debug.ini"]),
],
),
@ -549,7 +543,6 @@ def test_dump(tmpdir_factory):
"custom",
[
("src_dir", "source"),
("build_dir", "~/tmp/pio-$PROJECT_HASH"),
("debug_flags", "-D RELEASE"),
("lib_flags", "-lc -lm"),
("extra_flags", "${sysenv.__PIO_TEST_CNF_EXTRA_FLAGS}"),
@ -636,9 +629,10 @@ def test_nested_interpolation(tmp_path: Path):
project_conf.write_text(
"""
[platformio]
build_dir = ~/tmp/pio-$PROJECT_HASH
build_dir = /tmp/pio-$PROJECT_HASH
[env:myenv]
build_flags = -D UTIME=${UNIX_TIME}
test_testing_command =
${platformio.packages_dir}/tool-simavr/bin/simavr
-m
@ -651,6 +645,7 @@ test_testing_command =
config = ProjectConfig(str(project_conf))
testing_command = config.get("env:myenv", "test_testing_command")
assert "$" not in " ".join(testing_command)
assert config.get("env:myenv", "build_flags")[0][-10:].isdigit()
def test_extends_order(tmp_path: Path):
@ -707,11 +702,16 @@ def test_linting_warnings(tmp_path: Path):
project_conf = tmp_path / "platformio.ini"
project_conf.write_text(
"""
[platformio]
build_dir = /tmp/pio-$PROJECT_HASH
[env:app1]
lib_use = 1
test_testing_command = /usr/bin/flash-tool -p $UPLOAD_PORT -b $UPLOAD_SPEED
"""
)
result = ProjectConfig.lint(str(project_conf))
assert not result["errors"]
assert result["warnings"] and len(result["warnings"]) == 1
assert result["warnings"] and len(result["warnings"]) == 2
assert "deprecated" in result["warnings"][0]
assert "Invalid variable declaration" in result["warnings"][1]

2
tox.ini
View File

@ -55,7 +55,7 @@ commands =
[testenv:docs]
deps =
sphinx
sphinx-rtd-theme==1.2.2
sphinx-rtd-theme==2.0.0
sphinx-notfound-page
sphinx-copybutton
restructuredtext-lint