Merge branch 'release/v6.1.1'

.github/workflows/deployment.yml

@@ -4,8 +4,6 @@ on:
   push:
     branches:
       - "release/**"
-    tags:
-      - v*
 
 jobs:
   deployment:
@@ -35,8 +33,11 @@ jobs:
         run: |
           tox -e testcore
 
+      - name: Build Python source tarball
+        run:  python setup.py sdist
+
       - name: Publish package to PyPI
-        if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
+        if: ${{ github.ref == 'refs/heads/master' }}
         uses: pypa/gh-action-pypi-publish@release/v1
         with:
           user: __token__

HISTORY.rst

@@ -13,6 +13,16 @@ PlatformIO Core 6
 
 **A professional collaborative platform for declarative, safety-critical, and test-driven embedded development.**
 
+6.1.1 (2022-07-11)
+~~~~~~~~~~~~~~~~~~
+
+* Added new ``monitor_encoding`` project configuration option to configure `Device Monitor <https://docs.platformio.org/en/latest/core/userguide/device/cmd_monitor.html>`__ (`issue #4350 <https://github.com/platformio/platformio-core/issues/4350>`_)
+* Allowed specifying project environments for `pio ci <https://docs.platformio.org/en/latest/core/userguide/cmd_ci.html>`__ command (`issue #4347 <https://github.com/platformio/platformio-core/issues/4347>`_)
+* Show "TimeoutError" only in the verbose mode when can not find a serial port
+* Fixed an issue when a serial port was not automatically detected if the board has predefined HWIDs
+* Fixed an issue with endless scanning of project dependencies (`issue #4349 <https://github.com/platformio/platformio-core/issues/4349>`_)
+* Fixed an issue with |LDF| when incompatible libraries were used for the working project environment with the missed framework (`pull #4346 <https://github.com/platformio/platformio-core/pull/4346>`_)
+
 6.1.0 (2022-07-06)
 ~~~~~~~~~~~~~~~~~~
 

platformio/__init__.py

@@ -14,7 +14,7 @@
 
 import sys
 
-VERSION = (6, 1, 0)
+VERSION = (6, 1, 1)
 __version__ = ".".join([str(s) for s in VERSION])
 
 __title__ = "platformio"

platformio/builder/tools/piolib.py

@@ -30,7 +30,7 @@ from SCons.Script import DefaultEnvironment  # pylint: disable=import-error
 
 from platformio import exception, fs
 from platformio.builder.tools import platformio as piotool
-from platformio.compat import IS_WINDOWS, hashlib_encode_data, string_types
+from platformio.compat import IS_WINDOWS, MISSING, hashlib_encode_data, string_types
 from platformio.http import HTTPClientError, InternetIsOffline
 from platformio.package.exception import (
     MissingPackageManifestError,
@@ -143,7 +143,7 @@ class LibBuilderBase:
 
         self._deps_are_processed = False
         self._circular_deps = []
-        self._processed_files = []
+        self._processed_search_files = []
 
         # reset source filter, could be overridden with extra script
         self.env["SRC_FILTER"] = ""
@@ -154,20 +154,27 @@ class LibBuilderBase:
     def __repr__(self):
         return "%s(%r)" % (self.__class__, self.path)
 
-    def __contains__(self, path):
-        p1 = self.path
-        p2 = path
+    def __contains__(self, child_path):
+        return self.is_common_builder(self.path, child_path)
+
+    def is_common_builder(self, root_path, child_path):
         if IS_WINDOWS:
-            p1 = p1.lower()
-            p2 = p2.lower()
-        if p1 == p2:
+            root_path = root_path.lower()
+            child_path = child_path.lower()
+        if root_path == child_path:
             return True
-        if os.path.commonprefix([p1 + os.path.sep, p2]) == p1 + os.path.sep:
+        if (
+            os.path.commonprefix([root_path + os.path.sep, child_path])
+            == root_path + os.path.sep
+        ):
             return True
         # try to resolve paths
-        p1 = os.path.os.path.realpath(p1)
-        p2 = os.path.os.path.realpath(p2)
-        return os.path.commonprefix([p1 + os.path.sep, p2]) == p1 + os.path.sep
+        root_path = os.path.realpath(root_path)
+        child_path = os.path.realpath(child_path)
+        return (
+            os.path.commonprefix([root_path + os.path.sep, child_path])
+            == root_path + os.path.sep
+        )
 
     @property
     def name(self):
@@ -324,7 +331,7 @@ class LibBuilderBase:
             )
         ]
 
-    def _get_found_includes(  # pylint: disable=too-many-branches
+    def get_implicit_includes(  # pylint: disable=too-many-branches
         self, search_files=None
     ):
         # all include directories
@@ -345,15 +352,17 @@ class LibBuilderBase:
         include_dirs.extend(LibBuilderBase._INCLUDE_DIRS_CACHE)
 
         result = []
-        for path in search_files or []:
-            if path in self._processed_files:
+        search_files = search_files or []
+        while search_files:
+            node = self.env.File(search_files.pop(0))
+            if node.get_abspath() in self._processed_search_files:
                 continue
-            self._processed_files.append(path)
+            self._processed_search_files.append(node.get_abspath())
 
             try:
                 assert "+" in self.lib_ldf_mode
                 candidates = LibBuilderBase.CCONDITIONAL_SCANNER(
-                    self.env.File(path),
+                    node,
                     self.env,
                     tuple(include_dirs),
                     depth=self.CCONDITIONAL_SCANNER_DEPTH,
@@ -363,39 +372,35 @@ class LibBuilderBase:
                 if self.verbose and "+" in self.lib_ldf_mode:
                     sys.stderr.write(
                         "Warning! Classic Pre Processor is used for `%s`, "
-                        "advanced has failed with `%s`\n" % (path, exc)
+                        "advanced has failed with `%s`\n" % (node.get_abspath(), exc)
                     )
-                candidates = self.env.File(path).get_implicit_deps(
-                    self.env,
-                    LibBuilderBase.CLASSIC_SCANNER,
-                    lambda _: tuple(include_dirs),
+                candidates = LibBuilderBase.CLASSIC_SCANNER(
+                    node, self.env, tuple(include_dirs)
                 )
 
-            # mark candidates already processed
-            self._processed_files.extend(
-                [
-                    c.get_abspath()
-                    for c in candidates
-                    if c.get_abspath() not in self._processed_files
-                ]
-            )
-
-            # print(path, [c.get_abspath() for c in candidates])
+            # print(node.get_abspath(), [c.get_abspath() for c in candidates])
             for item in candidates:
+                item_path = item.get_abspath()
+                # process internal files recursively
+                if (
+                    item_path not in self._processed_search_files
+                    and item_path not in search_files
+                    and item_path in self
+                ):
+                    search_files.append(item_path)
                 if item not in result:
                     result.append(item)
                 if not self.PARSE_SRC_BY_H_NAME:
                     continue
-                _h_path = item.get_abspath()
-                if not fs.path_endswith_ext(_h_path, piotool.SRC_HEADER_EXT):
+                if not fs.path_endswith_ext(item_path, piotool.SRC_HEADER_EXT):
                     continue
-                _f_part = _h_path[: _h_path.rindex(".")]
+                item_fname = item_path[: item_path.rindex(".")]
                 for ext in piotool.SRC_C_EXT + piotool.SRC_CXX_EXT:
-                    if not os.path.isfile("%s.%s" % (_f_part, ext)):
+                    if not os.path.isfile("%s.%s" % (item_fname, ext)):
                         continue
-                    _c_path = self.env.File("%s.%s" % (_f_part, ext))
-                    if _c_path not in result:
-                        result.append(_c_path)
+                    item_c_node = self.env.File("%s.%s" % (item_fname, ext))
+                    if item_c_node not in result:
+                        result.append(item_c_node)
 
         return result
 
@@ -410,7 +415,7 @@ class LibBuilderBase:
             search_files = self.get_search_files()
 
         lib_inc_map = {}
-        for inc in self._get_found_includes(search_files):
+        for inc in self.get_implicit_includes(search_files):
             inc_path = inc.get_abspath()
             for lb in self.env.GetLibBuilders():
                 if inc_path in lb:
@@ -571,11 +576,10 @@ class ArduinoLibBuilder(LibBuilderBase):
         # pylint: disable=no-member
         if not self._manifest.get("dependencies"):
             return LibBuilderBase.lib_ldf_mode.fget(self)
-        missing = object()
         global_value = self.env.GetProjectConfig().getraw(
-            "env:" + self.env["PIOENV"], "lib_ldf_mode", missing
+            "env:" + self.env["PIOENV"], "lib_ldf_mode", MISSING
         )
-        if global_value != missing:
+        if global_value != MISSING:
             return LibBuilderBase.lib_ldf_mode.fget(self)
         # automatically enable C++ Preprocessing in runtime
         # (Arduino IDE has this behavior)
@@ -827,11 +831,10 @@ class PlatformIOLibBuilder(LibBuilderBase):
 
     @property
     def lib_archive(self):
-        missing = object()
         global_value = self.env.GetProjectConfig().getraw(
-            "env:" + self.env["PIOENV"], "lib_archive", missing
+            "env:" + self.env["PIOENV"], "lib_archive", MISSING
         )
-        if global_value != missing:
+        if global_value != MISSING:
             return self.env.GetProjectConfig().get(
                 "env:" + self.env["PIOENV"], "lib_archive"
             )
@@ -881,6 +884,12 @@ class ProjectAsLibBuilder(LibBuilderBase):
         if export_projenv:
             env.Export(dict(projenv=self.env))
 
+    def __contains__(self, child_path):
+        for root_path in (self.include_dir, self.src_dir, self.test_dir):
+            if root_path and self.is_common_builder(root_path, child_path):
+                return True
+        return False
+
     @property
     def include_dir(self):
         include_dir = self.env.subst("$PROJECT_INCLUDE_DIR")
@@ -890,6 +899,10 @@ class ProjectAsLibBuilder(LibBuilderBase):
     def src_dir(self):
         return self.env.subst("$PROJECT_SRC_DIR")
 
+    @property
+    def test_dir(self):
+        return self.env.subst("$PROJECT_TEST_DIR")
+
     def get_search_files(self):
         items = []
         build_type = self.env.GetBuildType()
@@ -1035,7 +1048,7 @@ def IsCompatibleLibBuilder(env, lb, verbose=int(ARGUMENTS.get("PIOVERBOSE", 0)))
             sys.stderr.write("Platform incompatible library %s\n" % lb.path)
         return False
     if compat_mode in ("soft", "strict") and not lb.is_frameworks_compatible(
-        env.get("PIOFRAMEWORK", [])
+        env.get("PIOFRAMEWORK", "__noframework__")
     ):
         if verbose:
             sys.stderr.write("Framework incompatible library %s\n" % lb.path)

platformio/builder/tools/pioupload.py

@@ -117,6 +117,7 @@ def AutodetectUploadPort(*args, **kwargs):
                 board_config=env.BoardConfig() if "BOARD" in env else None,
                 upload_protocol=upload_protocol,
                 prefer_gdb_port="blackmagic" in upload_protocol,
+                verbose=int(ARGUMENTS.get("PIOVERBOSE", 0)),
             )
         )
 

platformio/commands/ci.py

@@ -62,6 +62,7 @@ def validate_path(ctx, param, value):  # pylint: disable=unused-argument
     ),
 )
 @click.option("-O", "--project-option", multiple=True)
+@click.option("-e", "--environment", "environments", multiple=True)
 @click.option("-v", "--verbose", is_flag=True)
 @click.pass_context
 def cli(  # pylint: disable=too-many-arguments, too-many-branches
@@ -74,9 +75,9 @@ def cli(  # pylint: disable=too-many-arguments, too-many-branches
     keep_build_dir,
     project_conf,
     project_option,
+    environments,
     verbose,
 ):
-
     if not src and os.getenv("PLATFORMIO_CI_SRC"):
         src = validate_path(ctx, None, os.getenv("PLATFORMIO_CI_SRC").split(":"))
     if not src:
@@ -115,7 +116,9 @@ def cli(  # pylint: disable=too-many-arguments, too-many-branches
         )
 
         # process project
-        ctx.invoke(cmd_run, project_dir=build_dir, verbose=verbose)
+        ctx.invoke(
+            cmd_run, project_dir=build_dir, environment=environments, verbose=verbose
+        )
     finally:
         if not keep_build_dir:
             fs.rmtree(build_dir)

platformio/debug/config/base.py

@@ -16,7 +16,7 @@ import json
 import os
 
 from platformio import fs, proc, util
-from platformio.compat import string_types
+from platformio.compat import MISSING, string_types
 from platformio.debug.exception import DebugInvalidOptionsError
 from platformio.project.config import ProjectConfig
 from platformio.project.helpers import load_build_metadata
@@ -96,9 +96,8 @@ class DebugConfigBase:  # pylint: disable=too-many-instance-attributes
 
     @property
     def init_break(self):
-        missed = object()
-        result = self.env_options.get("debug_init_break", missed)
-        if result != missed:
+        result = self.env_options.get("debug_init_break", MISSING)
+        if result != MISSING:
             return result
         result = None
         if not result:

platformio/device/finder.py

@@ -88,6 +88,7 @@ def find_serial_port(  # pylint: disable=too-many-arguments
     ensure_ready=False,
     prefer_gdb_port=False,
     timeout=2,
+    verbose=False,
 ):
     if initial_port:
         if not is_pattern_port(initial_port):
@@ -96,9 +97,11 @@ def find_serial_port(  # pylint: disable=too-many-arguments
 
     if upload_protocol and upload_protocol.startswith("blackmagic"):
         return find_blackmagic_serial_port(prefer_gdb_port, timeout)
+    port = None
     if board_config and board_config.get("build.hwids", []):
-        return find_board_serial_port(board_config, timeout)
-    port = find_known_uart_port(ensure_ready, timeout)
+        port = find_board_serial_port(board_config, timeout, verbose)
+    if not port:
+        port = find_known_uart_port(ensure_ready, timeout, verbose)
     if port:
         return port
 
@@ -158,7 +161,7 @@ def find_blackmagic_serial_port(prefer_gdb_port=False, timeout=0):
     return None
 
 
-def find_board_serial_port(board_config, timeout=0):
+def find_board_serial_port(board_config, timeout=0, verbose=False):
     hwids = board_config.get("build.hwids", [])
     try:
 
@@ -175,18 +178,19 @@ def find_board_serial_port(board_config, timeout=0):
     except retry.RetryStopException:
         pass
 
-    click.secho(
-        "TimeoutError: Could not automatically find serial port "
-        "for the `%s` board based on the declared HWIDs=%s"
-        % (board_config.get("name", "unknown"), hwids),
-        fg="yellow",
-        err=True,
-    )
+    if verbose:
+        click.secho(
+            "TimeoutError: Could not automatically find serial port "
+            "for the `%s` board based on the declared HWIDs=%s"
+            % (board_config.get("name", "unknown"), hwids),
+            fg="yellow",
+            err=True,
+        )
 
     return None
 
 
-def find_known_uart_port(ensure_ready=False, timeout=0):
+def find_known_uart_port(ensure_ready=False, timeout=0, verbose=False):
     known_hwids = list(BLACK_MAGIC_HWIDS)
 
     # load from UDEV rules
@@ -222,12 +226,13 @@ def find_known_uart_port(ensure_ready=False, timeout=0):
     except retry.RetryStopException:
         pass
 
-    click.secho(
-        "TimeoutError: Could not automatically find serial port "
-        "based on the known UART bridges",
-        fg="yellow",
-        err=True,
-    )
+    if verbose:
+        click.secho(
+            "TimeoutError: Could not automatically find serial port "
+            "based on the known UART bridges",
+            fg="yellow",
+            err=True,
+        )
 
     return None
 

platformio/device/monitor/command.py

@@ -56,9 +56,11 @@ from platformio.project.options import ProjectOptions
 @click.option("--echo", is_flag=True, help="Enable local echo")
 @click.option(
     "--encoding",
-    default="UTF-8",
-    show_default=True,
-    help="Set the encoding for the serial port (e.g. hexlify, Latin1, UTF-8)",
+    help=(
+        "Set the encoding for the serial port "
+        "(e.g. hexlify, Latin1, UTF-8) [default=%s]"
+        % ProjectOptions["env.monitor_encoding"].default
+    ),
 )
 @click.option(
     "-f",

platformio/project/options.py

@@ -567,6 +567,12 @@ ProjectOptions = OrderedDict(
                 type=click.BOOL,
                 default=False,
             ),
+            ConfigEnvOption(
+                group="monitor",
+                name="monitor_encoding",
+                description="Custom encoding (e.g. hexlify, Latin1, UTF-8)",
+                default="UTF-8",
+            ),
             # Library
             ConfigEnvOption(
                 group="library",

platformio/test/runners/readers/serial.py

@@ -73,6 +73,7 @@ class SerialTestOutputReader:
             ),
             upload_protocol=project_options.get("upload_protocol"),
             ensure_ready=True,
+            verbose=self.test_runner.options.verbose,
         )
         if port:
             return port

scripts/99-platformio-udev.rules

@@ -36,6 +36,8 @@ ATTRS{idVendor}=="067b", ATTRS{idProduct}=="2303", MODE:="0666", ENV{ID_MM_DEVIC
 
 # QinHeng Electronics HL-340 USB-Serial adapter
 ATTRS{idVendor}=="1a86", ATTRS{idProduct}=="7523", MODE:="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
+# QinHeng Electronics CH9102 USB-Serial adapter
+ATTRS{idVendor}=="1a86", ATTRS{idProduct}=="55d4", MODE:="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
 
 # Arduino boards
 ATTRS{idVendor}=="2341", ATTRS{idProduct}=="[08][023]*", MODE:="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"

tests/commands/test_run.py

@@ -335,3 +335,73 @@ projenv.Append(CPPDEFINES=[
     assert 'MACRO_2=<Text is "Quoted">' in result.output
     assert 'MACRO_3=<Hello "World"! Isn\'t true?>' in result.output
     assert "MACRO_4=<Special chars: ',(,),[,],:>" in result.output
+
+
+def test_ldf(clirunner, validate_cliresult, tmp_path: Path):
+    project_dir = tmp_path / "project"
+
+    # libs
+    lib_dir = project_dir / "lib"
+    a_lib_dir = lib_dir / "a"
+    a_lib_dir.mkdir(parents=True)
+    (a_lib_dir / "a.h").write_text(
+        """
+#include <some_from_b.h>
+"""
+    )
+    # b
+    b_lib_dir = lib_dir / "b"
+    b_lib_dir.mkdir(parents=True)
+    (b_lib_dir / "some_from_b.h").write_text("")
+    # c
+    c_lib_dir = lib_dir / "c"
+    c_lib_dir.mkdir(parents=True)
+    (c_lib_dir / "parse_c_by_name.h").write_text(
+        """
+void some_func();
+    """
+    )
+    (c_lib_dir / "parse_c_by_name.c").write_text(
+        """
+#include <d.h>
+#include <parse_c_by_name.h>
+
+void some_func() {
+}
+    """
+    )
+    (c_lib_dir / "some.c").write_text(
+        """
+#include <d.h>
+    """
+    )
+    # d
+    d_lib_dir = lib_dir / "d"
+    d_lib_dir.mkdir(parents=True)
+    (d_lib_dir / "d.h").write_text("")
+
+    # project
+    src_dir = project_dir / "src"
+    src_dir.mkdir(parents=True)
+    (src_dir / "main.h").write_text(
+        """
+#include <a.h>
+#include <parse_c_by_name.h>
+"""
+    )
+    (src_dir / "main.c").write_text(
+        """
+#include <main.h>
+
+int main() {
+}
+"""
+    )
+    (project_dir / "platformio.ini").write_text(
+        """
+[env:native]
+platform = native
+    """
+    )
+    result = clirunner.invoke(cmd_run, ["--project-dir", str(project_dir)])
+    validate_cliresult(result)