Merge branch 'develop' into feature/v7

# Conflicts:
#	docs
#	platformio/builder/tools/piobuild.py
#	platformio/debug/config/base.py
#	platformio/project/helpers.py

HISTORY.rst

@@ -20,8 +20,12 @@ test-driven methodologies, and modern toolchains for unrivaled success.
 6.1.14 (2024-??-??)
 ~~~~~~~~~~~~~~~~~~~
 
+* Introduced the ``--json-output`` option to the `pio test <https://docs.platformio.org/en/latest/core/userguide/cmd_test.html>`__ command, enabling users to generate test results in the JSON format
 * Broadened version support for the ``pyelftools`` dependency, enabling compatibility with lower versions and facilitating integration with a wider range of third-party tools (`issue #4834 <https://github.com/platformio/platformio-core/issues/4834>`_)
+* Addressed an issue where passing a relative path (``--project-dir``) to the `pio project init <https://docs.platformio.org/en/latest/core/userguide/project/cmd_init.html>`__ command resulted in an error (`issue #4847 <https://github.com/platformio/platformio-core/issues/4847>`_)
 * Resolved an issue related to the relative package path in the `pio pkg publish <https://docs.platformio.org/en/latest/core/userguide/pkg/cmd_publish.html>`__ command
+* Resolved an issue where the |LDF| selected an incorrect library version (`issue #4860 <https://github.com/platformio/platformio-core/issues/4860>`_)
+* Resolved an issue with the ``hexlify`` filter in the `device monitor <https://docs.platformio.org/en/latest/core/userguide/device/cmd_monitor.html>`__ command, ensuring proper representation of characters with Unicode code points higher than 127 (`issue #4732 <https://github.com/platformio/platformio-core/issues/4732>`_)
 
 6.1.13 (2024-01-12)
 ~~~~~~~~~~~~~~~~~~~

platformio/__init__.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-VERSION = (6, 1, "14a1")
+VERSION = (6, 1, "14b1")
 __version__ = ".".join([str(s) for s in VERSION])
 
 __title__ = "platformio"

platformio/assets/system/99-platformio-udev.rules

@@ -36,6 +36,8 @@ ATTRS{idVendor}=="067b", ATTRS{idProduct}=="2303", MODE:="0666", ENV{ID_MM_DEVIC
 
 # QinHeng Electronics HL-340 USB-Serial adapter
 ATTRS{idVendor}=="1a86", ATTRS{idProduct}=="7523", MODE:="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
+# QinHeng Electronics CH343 USB-Serial adapter
+ATTRS{idVendor}=="1a86", ATTRS{idProduct}=="55d3", MODE:="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
 # QinHeng Electronics CH9102 USB-Serial adapter
 ATTRS{idVendor}=="1a86", ATTRS{idProduct}=="55d4", MODE:="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
 
@@ -85,6 +87,8 @@ ATTRS{idVendor}=="2e8a", ATTRS{idProduct}=="[01]*", MODE:="0666", ENV{ID_MM_DEVI
 # AIR32F103
 ATTRS{idVendor}=="0d28", ATTRS{idProduct}=="0204", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
 
+# STM32 virtual COM port
+ATTRS{idVendor}=="0483", ATTRS{idProduct}=="5740", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
 
 #
 # Debuggers

platformio/builder/tools/piobuild.py

@@ -45,7 +45,16 @@ def scons_patched_match_splitext(path, suffixes=None):
 
 
 def GetBuildType(env):
-    return env["BUILD_TYPE"]
+    modes = []
+    if (
+        set(["__debug", "sizedata"])  # sizedata = for memory inspection
+        & set(COMMAND_LINE_TARGETS)
+        or env.GetProjectOption("build_type") == "debug"
+    ):
+        modes.append("debug")
+    if "__test" in COMMAND_LINE_TARGETS or env.GetProjectOption("build_type") == "test":
+        modes.append("test")
+    return ", ".join(modes or ["release"])
 
 
 def BuildProgram(env):
@@ -65,7 +74,7 @@ def BuildProgram(env):
         env.Prepend(_LIBFLAGS="-Wl,--start-group ")
         env.Append(_LIBFLAGS=" -Wl,--end-group")
 
-    program = env.Program(env.subst("$PROGPATH"), env.get("PIOBUILDFILES", []))
+    program = env.Program(env.subst("$PROGPATH"), env["PIOBUILDFILES"])
     env.Replace(PIOMAINPROG=program)
 
     AlwaysBuild(
@@ -153,9 +162,6 @@ def ProcessProjectDeps(env):
         }
     )
 
-    if env.IsIntegrationDump():
-        return
-
     if "test" in env["BUILD_TYPE"]:
         build_files_before_nums = len(env.get("PIOBUILDFILES", []))
         plb.env.BuildSources(

platformio/builder/tools/piolib.py

@@ -309,10 +309,10 @@ class LibBuilderBase:
         if not self.dependencies or self._deps_are_processed:
             return
         self._deps_are_processed = True
-        for item in self.dependencies:
+        for dependency in self.dependencies:
             found = False
             for lb in self.env.GetLibBuilders():
-                if item["name"] != lb.name:
+                if not lb.is_dependency_compatible(dependency):
                     continue
                 found = True
                 if lb not in self.depbuilders:
@@ -322,9 +322,20 @@ class LibBuilderBase:
             if not found and self.verbose:
                 sys.stderr.write(
                     "Warning: Ignored `%s` dependency for `%s` "
-                    "library\n" % (item["name"], self.name)
+                    "library\n" % (dependency["name"], self.name)
                 )
 
+    def is_dependency_compatible(self, dependency):
+        pkg = PackageItem(self.path)
+        qualifiers = {"name": self.name, "version": self.version}
+        if pkg.metadata:
+            qualifiers = {"name": pkg.metadata.name, "version": pkg.metadata.version}
+            if pkg.metadata.spec and pkg.metadata.spec.owner:
+                qualifiers["owner"] = pkg.metadata.spec.owner
+        return PackageCompatibility.from_dependency(
+            {k: v for k, v in dependency.items() if k in ("owner", "name", "version")}
+        ).is_compatible(PackageCompatibility(**qualifiers))
+
     def get_search_files(self):
         return [
             os.path.join(self.src_dir, item)

platformio/builder/tools/piomaxlen.py

@@ -23,10 +23,10 @@ from SCons.Subst import quote_spaces  # pylint: disable=import-error
 from platformio.compat import IS_WINDOWS, hashlib_encode_data
 
 # There are the next limits depending on a platform:
-# - Windows = 8192
+# - Windows = 8191
 # - Unix    = 131072
 # We need ~512 characters for compiler and temporary file paths
-MAX_LINE_LENGTH = (8192 if IS_WINDOWS else 131072) - 512
+MAX_LINE_LENGTH = (8191 if IS_WINDOWS else 131072) - 512
 
 WINPATHSEP_RE = re.compile(r"\\([^\"'\\]|$)")
 

platformio/debug/config/base.py

@@ -31,7 +31,7 @@ class DebugConfigBase:  # pylint: disable=too-many-instance-attributes
         self.project_config = project_config
         self.env_name = env_name
         self.env_options = project_config.items(env=env_name, as_dict=True)
-        self.build_metadata = self._load_debug_build_metadata()
+        self.build_data = self._load_build_data()
 
         self.tool_name = None
         self.board_config = {}
@@ -64,11 +64,11 @@ class DebugConfigBase:  # pylint: disable=too-many-instance-attributes
 
     @property
     def program_path(self):
-        return self.build_metadata["prog_path"]
+        return self.build_data["prog_path"]
 
     @property
     def client_executable_path(self):
-        return self.build_metadata["gdb_path"]
+        return self.build_data["gdb_path"]
 
     @property
     def load_cmds(self):
@@ -147,13 +147,9 @@ class DebugConfigBase:  # pylint: disable=too-many-instance-attributes
             "debug_server_ready_pattern", (self.server or {}).get("ready_pattern")
         )
 
-    def _load_debug_build_metadata(self):
+    def _load_build_data(self):
         data = load_build_metadata(
-            None,
+            os.getcwd(), self.env_name, cache=True, build_type="debug"
-            self.env_name,
-            cache=True,
-            debug=True,
-            test=self.env_options.get("debug_test", False),
         )
         if not data:
             raise DebugInvalidOptionsError("Could not load a build configuration")

platformio/device/monitor/command.py

@@ -58,7 +58,7 @@ from platformio.project.options import ProjectOptions
     "--encoding",
     help=(
         "Set the encoding for the serial port "
-        "(e.g. hexlify, Latin1, UTF-8) [default=%s]"
+        "(e.g. hexlify, Latin-1, UTF-8) [default=%s]"
         % ProjectOptions["env.monitor_encoding"].default
     ),
 )

platformio/device/monitor/filters/base.py

@@ -25,11 +25,12 @@ from platformio.project.config import ProjectConfig
 class DeviceMonitorFilterBase(miniterm.Transform):
     def __init__(self, options=None):
         """Called by PlatformIO to pass context"""
-        miniterm.Transform.__init__(self)
+        super().__init__()
 
         self.options = options or {}
         self.project_dir = self.options.get("project_dir")
         self.environment = self.options.get("environment")
+        self._running_terminal = None
 
         self.config = ProjectConfig.get_instance()
         if not self.environment:
@@ -47,6 +48,12 @@ class DeviceMonitorFilterBase(miniterm.Transform):
     def NAME(self):
         raise NotImplementedError("Please declare NAME attribute for the filter class")
 
+    def set_running_terminal(self, terminal):
+        self._running_terminal = terminal
+
+    def get_running_terminal(self):
+        return self._running_terminal
+
 
 def register_filters(platform=None, options=None):
     # project filters

platformio/device/monitor/filters/hexlify.py

@@ -24,12 +24,18 @@ class Hexlify(DeviceMonitorFilterBase):
         super().__init__(*args, **kwargs)
         self._counter = 0
 
+    def set_running_terminal(self, terminal):
+        # force to Latin-1, issue #4732
+        if terminal.input_encoding == "UTF-8":
+            terminal.set_rx_encoding("Latin-1")
+        super().set_running_terminal(terminal)
+
     def rx(self, text):
         result = ""
-        for b in serial.iterbytes(text):
+        for c in serial.iterbytes(text):
             if (self._counter % 16) == 0:
                 result += "\n{:04X} | ".format(self._counter)
-            asciicode = ord(b)
+            asciicode = ord(c)
             if asciicode <= 255:
                 result += "{:02X} ".format(asciicode)
             else:

platformio/device/monitor/terminal.py

@@ -110,6 +110,12 @@ def new_terminal(options):
     term.raw = options["raw"]
     term.set_rx_encoding(options["encoding"])
     term.set_tx_encoding(options["encoding"])
+    for ts in (term.tx_transformations, term.rx_transformations):
+        for t in ts:
+            try:
+                t.set_running_terminal(term)
+            except AttributeError:
+                pass
     return term
 
 

platformio/package/manager/_install.py

@@ -99,7 +99,11 @@ class PackageManagerInstallMixin:
             pkg = self.install_from_registry(
                 spec,
                 search_qualifiers=(
-                    compatibility.to_search_qualifiers() if compatibility else None
+                    compatibility.to_search_qualifiers(
+                        ["platforms", "frameworks", "authors"]
+                    )
+                    if compatibility
+                    else None
                 ),
             )
 

platformio/package/manifest/schema.py

@@ -276,7 +276,7 @@ class ManifestSchema(BaseSchema):
     @staticmethod
     @memoized(expire="1h")
     def load_spdx_licenses():
-        version = "3.22"
+        version = "3.23"
         spdx_data_url = (
             "https://raw.githubusercontent.com/spdx/license-list-data/"
             f"v{version}/json/licenses.json"

platformio/package/meta.py

@@ -65,7 +65,14 @@ class PackageType:
 
 
 class PackageCompatibility:
-    KNOWN_QUALIFIERS = ("platforms", "frameworks", "authors")
+    KNOWN_QUALIFIERS = (
+        "owner",
+        "name",
+        "version",
+        "platforms",
+        "frameworks",
+        "authors",
+    )
 
     @classmethod
     def from_dependency(cls, dependency):
@@ -89,19 +96,45 @@ class PackageCompatibility:
     def __repr__(self):
         return "PackageCompatibility <%s>" % self.qualifiers
 
-    def to_search_qualifiers(self):
+    def to_search_qualifiers(self, fields=None):
-        return self.qualifiers
+        result = {}
+        for name, value in self.qualifiers.items():
+            if not fields or name in fields:
+                result[name] = value
+        return result
 
     def is_compatible(self, other):
         assert isinstance(other, PackageCompatibility)
-        for key, value in self.qualifiers.items():
+        for key, current_value in self.qualifiers.items():
             other_value = other.qualifiers.get(key)
-            if not value or not other_value:
+            if not current_value or not other_value:
                 continue
-            if not items_in_list(value, other_value):
+            if any(isinstance(v, list) for v in (current_value, other_value)):
+                if not items_in_list(current_value, other_value):
+                    return False
+                continue
+            if key == "version":
+                if not self._compare_versions(current_value, other_value):
+                    return False
+                continue
+            if current_value != other_value:
                 return False
         return True
 
+    def _compare_versions(self, current, other):
+        if current == other:
+            return True
+        try:
+            version = (
+                other
+                if isinstance(other, semantic_version.Version)
+                else cast_version_to_semver(other)
+            )
+            return version in semantic_version.SimpleSpec(current)
+        except ValueError:
+            pass
+        return False
+
 
 class PackageOutdatedResult:
     UPDATE_INCREMENT_MAJOR = "major"

platformio/pipdeps.py

@@ -35,7 +35,7 @@ def get_pip_dependencies():
     home = [
         # PIO Home requirements
         "ajsonrpc == 1.2.*",
-        "starlette >=0.19, <0.36",
+        "starlette >=0.19, <0.38",
         "uvicorn %s" % ("== 0.16.0" if PY36 else ">=0.16, <0.28"),
         "wsproto == 1.*",
     ]

platformio/project/commands/init.py

@@ -79,6 +79,7 @@ def project_init_cmd(
     env_prefix,
     silent,
 ):
+    project_dir = os.path.abspath(project_dir)
     is_new_project = not is_platformio_project(project_dir)
     if is_new_project:
         if not silent:
@@ -223,7 +224,7 @@ def init_lib_readme(lib_dir):
 This directory is intended for project specific (private) libraries.
 PlatformIO will compile them to static libraries and link into executable file.
 
-The source code of each library should be placed in a an own separate directory
+The source code of each library should be placed in an own separate directory
 ("lib/your_library_name/[here are source files]").
 
 For example, see a structure of the following two libraries `Foo` and `Bar`:

platformio/project/helpers.py

@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import hashlib
 import os
 import re
 import subprocess
@@ -29,12 +28,6 @@ def get_project_dir():
     return os.getcwd()
 
 
-def get_project_id(project_dir=None):
-    return hashlib.sha1(
-        hashlib_encode_data(project_dir or get_project_dir())
-    ).hexdigest()
-
-
 def is_platformio_project(project_dir=None):
     if not project_dir:
         project_dir = get_project_dir()
@@ -138,36 +131,27 @@ def compute_project_checksum(config):
     return checksum.hexdigest()
 
 
-def get_build_type(config, env, run_targets=None):
+def load_build_metadata(project_dir, env_or_envs, cache=False, build_type=None):
-    types = []
-    run_targets = run_targets or []
-    declared_build_type = config.get(f"env:{env}", "build_type")
-    if (
-        set(["__debug", "__memusage"]) & set(run_targets)
-        or declared_build_type == "debug"
-    ):
-        types.append("debug")
-    if "__test" in run_targets or declared_build_type == "test":
-        types.append("test")
-    return "+".join(types or ["release"])
-
-
-def load_build_metadata(project_dir, env_or_envs, cache=False, debug=False, test=False):
     assert env_or_envs
-    envs = env_or_envs
+    env_names = env_or_envs
-    if not isinstance(envs, list):
+    if not isinstance(env_names, list):
-        envs = [envs]
+        env_names = [env_names]
-    run_targets = []
-    if debug:
-        run_targets.append("__debug")
-    if test:
-        run_targets.append("__test")
 
-    with fs.cd(project_dir or os.getcwd()):
+    with fs.cd(project_dir):
-        result = _get_cached_build_metadata(envs, run_targets) if cache else {}
+        result = _get_cached_build_metadata(env_names) if cache else {}
-        missed_envs = set(envs) - set(result.keys())
+        # incompatible build-type data
-        if missed_envs:
+        for env_name in list(result.keys()):
-            result.update(_load_build_metadata(missed_envs, run_targets))
+            if build_type is None:
+                build_type = ProjectConfig.get_instance().get(
+                    f"env:{env_name}", "build_type"
+                )
+            if result[env_name].get("build_type", "") != build_type:
+                del result[env_name]
+        missed_env_names = set(env_names) - set(result.keys())
+        if missed_env_names:
+            result.update(
+                _load_build_metadata(project_dir, missed_env_names, build_type)
+            )
 
     if not isinstance(env_or_envs, list) and env_or_envs in result:
         return result[env_or_envs]
@@ -178,28 +162,18 @@ def load_build_metadata(project_dir, env_or_envs, cache=False, debug=False, test
 load_project_ide_data = load_build_metadata
 
 
-def _get_cached_build_metadata(envs, run_targets=None):
+def _load_build_metadata(project_dir, env_names, build_type=None):
-    config = ProjectConfig.get_instance(os.path.join(os.getcwd(), "platformio.ini"))
-    build_dir = config.get("platformio", "build_dir")
-    result = {}
-    for env in envs:
-        build_type = get_build_type(config, env, run_targets)
-        json_path = os.path.join(build_dir, env, build_type, "metadata.json")
-        if os.path.isfile(json_path):
-            result[env] = fs.load_json(json_path)
-    return result
-
-
-def _load_build_metadata(envs, run_targets=None):
     # pylint: disable=import-outside-toplevel
     from platformio import app
     from platformio.run.cli import cli as cmd_run
 
-    args = ["--target", "__metadata"]
+    args = ["--project-dir", project_dir, "--target", "__idedata"]
-    for target in run_targets or []:
+    if build_type == "debug":
-        args.extend(["--target", target])
+        args.extend(["--target", "__debug"])
-    for env in envs:
+    # if build_type == "test":
-        args.extend(["-e", env])
+    #     args.extend(["--target", "__test"])
+    for name in env_names:
+        args.extend(["-e", name])
     app.set_session_var("pause_telemetry", True)
     result = CliRunner().invoke(cmd_run, args)
     app.set_session_var("pause_telemetry", False)
@@ -207,6 +181,18 @@ def _load_build_metadata(envs, run_targets=None):
         result.exception, exception.ReturnErrorCode
     ):
         raise result.exception
-    if "Metadata has been saved to the following location" not in result.output:
+    if '"includes":' not in result.output:
         raise exception.UserSideException(result.output)
-    return _get_cached_build_metadata(envs, run_targets)
+    return _get_cached_build_metadata(env_names)
+
+
+def _get_cached_build_metadata(env_names):
+    build_dir = ProjectConfig.get_instance().get("platformio", "build_dir")
+    result = {}
+    for env_name in env_names:
+        if not os.path.isfile(os.path.join(build_dir, env_name, "idedata.json")):
+            continue
+        result[env_name] = fs.load_json(
+            os.path.join(build_dir, env_name, "idedata.json")
+        )
+    return result

platformio/project/options.py

@@ -560,7 +560,7 @@ ProjectOptions = OrderedDict(
             ConfigEnvOption(
                 group="monitor",
                 name="monitor_encoding",
-                description="Custom encoding (e.g. hexlify, Latin1, UTF-8)",
+                description="Custom encoding (e.g. hexlify, Latin-1, UTF-8)",
                 default="UTF-8",
             ),
             # Library

platformio/test/cli.py

@@ -14,6 +14,7 @@
 
 import os
 import shutil
+import subprocess
 
 import click
 
@@ -79,6 +80,7 @@ from platformio.test.runners.factory import TestRunnerFactory
     help="A program argument (multiple are allowed)",
 )
 @click.option("--list-tests", is_flag=True)
+@click.option("--json-output", is_flag=True)
 @click.option("--json-output-path", type=click.Path())
 @click.option("--junit-output-path", type=click.Path())
 @click.option(
@@ -105,6 +107,7 @@ def cli(  # pylint: disable=too-many-arguments,too-many-locals,redefined-builtin
     monitor_dtr,
     program_args,
     list_tests,
+    json_output,
     json_output_path,
     junit_output_path,
     verbose,
@@ -156,6 +159,7 @@ def cli(  # pylint: disable=too-many-arguments,too-many-locals,redefined-builtin
     stdout_report.generate(verbose=verbose or list_tests)
 
     for output_format, output_path in [
+        ("json", subprocess.STDOUT if json_output else None),
         ("json", json_output_path),
         ("junit", junit_output_path),
     ]:

platformio/test/reports/json.py

@@ -15,6 +15,7 @@
 import datetime
 import json
 import os
+import subprocess
 
 import click
 
@@ -24,6 +25,9 @@ from platformio.test.result import TestStatus
 
 class JsonTestReport(TestReportBase):
     def generate(self, output_path, verbose=False):
+        if output_path == subprocess.STDOUT:
+            return click.echo("\n\n" + json.dumps(self.to_json()))
+
         if os.path.isdir(output_path):
             output_path = os.path.join(
                 output_path,
@@ -40,6 +44,8 @@ class JsonTestReport(TestReportBase):
         if verbose:
             click.secho(f"Saved JSON report to the {output_path}", fg="green")
 
+        return True
+
     def to_json(self):
         result = dict(
             version="1.0",

platformio/test/runners/unity.py

@@ -184,10 +184,6 @@ void unityOutputComplete(void) { unittest_uart_end(); }
         ),
     )
 
-    def __init__(self, *args, **kwargs):
-        """Delete when Unity > 2.5.2 is released"""
-        super().__init__(*args, **kwargs)
-
     def get_unity_framework_config(self):
         if not self.platform.is_embedded():
             return self.UNITY_FRAMEWORK_CONFIG["native"]

tests/commands/test_init.py

@@ -15,6 +15,7 @@
 import json
 import os
 
+from platformio import fs
 from platformio.commands.boards import cli as cmd_boards
 from platformio.project.commands.init import project_init_cmd
 from platformio.project.config import ProjectConfig
@@ -36,27 +37,28 @@ def test_init_default(clirunner, validate_cliresult):
         validate_pioproject(os.getcwd())
 
 
-def test_init_ext_folder(clirunner, validate_cliresult):
-    with clirunner.isolated_filesystem():
-        ext_folder_name = "ext_folder"
-        os.makedirs(ext_folder_name)
-        result = clirunner.invoke(project_init_cmd, ["-d", ext_folder_name])
-        validate_cliresult(result)
-        validate_pioproject(os.path.join(os.getcwd(), ext_folder_name))
-
-
 def test_init_duplicated_boards(clirunner, validate_cliresult, tmpdir):
-    with tmpdir.as_cwd():
+    project_dir = str(tmpdir.join("ext_folder"))
-        for _ in range(2):
+    os.makedirs(project_dir)
-            result = clirunner.invoke(
+
-                project_init_cmd,
+    with fs.cd(os.path.dirname(project_dir)):
-                ["-b", "uno", "-b", "uno", "--no-install-dependencies"],
+        result = clirunner.invoke(
-            )
+            project_init_cmd,
-            validate_cliresult(result)
+            [
-            validate_pioproject(str(tmpdir))
+                "-d",
-        config = ProjectConfig(os.path.join(os.getcwd(), "platformio.ini"))
+                os.path.basename(project_dir),
-        config.validate()
+                "-b",
-        assert set(config.sections()) == set(["env:uno"])
+                "uno",
+                "-b",
+                "uno",
+                "--no-install-dependencies",
+            ],
+        )
+    validate_cliresult(result)
+    validate_pioproject(project_dir)
+    config = ProjectConfig(os.path.join(project_dir, "platformio.ini"))
+    config.validate()
+    assert set(config.sections()) == set(["env:uno"])
 
 
 def test_init_ide_without_board(clirunner, tmpdir):