Merge branch 'release/v5.2.1'

.github/workflows/examples.yml

@@ -7,7 +7,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-16.04, windows-latest, macos-latest]
+        os: [ubuntu-18.04, windows-latest, macos-latest]
         python-version: [3.7]
     runs-on: ${{ matrix.os }}
     steps:

.pylintrc

@@ -16,6 +16,7 @@ disable=
     useless-import-alias,
     bad-option-value,
     consider-using-dict-items,
+    consider-using-f-string,
 
     ; PY2 Compat
     super-with-arguments,

HISTORY.rst

@@ -8,6 +8,18 @@ PlatformIO Core 5
 
 **A professional collaborative platform for embedded development**
 
+5.2.1 (2021-10-11)
+~~~~~~~~~~~~~~~~~~
+
+- Clean a build environment and installed library dependencies using a new ``cleanall`` target (`issue #4062 <https://github.com/platformio/platformio-core/issues/4062>`_)
+- Override a default library builder via a new ``builder`` field in a ``build`` group of `library.json <https://docs.platformio.org/page/librarymanager/config.html#build>`__ manifest (`issue #3957 <https://github.com/platformio/platformio-core/issues/3957>`_)
+- Updated `Cppcheck <https://docs.platformio.org/page/plus/check-tools/cppcheck.html>`__ v2.6 with new checks, increased reliability of advanced addons (MISRA/CERT) and various improvements
+- Handle the "test" folder as a part of CLion project (`issue #4005 <https://github.com/platformio/platformio-core/issues/4005>`_)
+- Improved handling of a library root based on "Conan" or "CMake" build systems (`issue #3887 <https://github.com/platformio/platformio-core/issues/3887>`_)
+- Fixed a "KeyError: Invalid board option 'build.cpu'" when using a precompiled library with a board that does not have a CPU field in the manifest (`issue #4056 <https://github.com/platformio/platformio-core/issues/4056>`_)
+- Fixed a "FileExist" error when the `platformio ci <https://docs.platformio.org/en/latest/userguide/cmd_ci.html>`__ command is used in pair with the ``--keep-build-dir`` option (`issue #4011 <https://github.com/platformio/platformio-core/issues/4011>`_)
+- Fixed an issue with draft values of C++ language standards that broke static analysis via Cppcheck (`issue #3944 <https://github.com/platformio/platformio-core/issues/3944>`_)
+
 5.2.0 (2021-09-13)
 ~~~~~~~~~~~~~~~~~~
 

platformio/__init__.py

@@ -14,7 +14,7 @@
 
 import sys
 
-VERSION = (5, 2, 0)
+VERSION = (5, 2, 1)
 __version__ = ".".join([str(s) for s in VERSION])
 
 __title__ = "platformio"
@@ -51,7 +51,7 @@ __core_packages__ = {
     "contrib-pysite": "~2.%d%d.0" % (sys.version_info.major, sys.version_info.minor),
     "tool-unity": "~1.20500.0",
     "tool-scons": "~4.40200.0",
-    "tool-cppcheck": "~1.250.0",
+    "tool-cppcheck": "~1.260.0",
     "tool-clangtidy": "~1.120001.0",
     "tool-pvs-studio": "~7.14.0",
 }

platformio/__main__.py

@@ -67,9 +67,24 @@ def cli(ctx, force, caller, no_ansi):
     maintenance.on_platformio_start(ctx, force, caller)
 
 
+try:
+
+    @cli.result_callback()
+    @click.pass_context
+    def process_result(ctx, result, *_, **__):
+        _process_result(ctx, result)
+
+
+except (AttributeError, TypeError):  # legacy support for CLick > 8.0.1
+    print("legacy Click")
+
     @cli.resultcallback()
     @click.pass_context
     def process_result(ctx, result, *_, **__):
+        _process_result(ctx, result)
+
+
+def _process_result(ctx, result):
     from platformio import maintenance
 
     maintenance.on_platformio_end(ctx, result)

platformio/builder/main.py

@@ -149,10 +149,12 @@ if int(ARGUMENTS.get("ISATTY", 0)):
     # pylint: disable=protected-access
     click._compat.isatty = lambda stream: True
 
-if env.GetOption("clean"):
+is_clean_all = "cleanall" in COMMAND_LINE_TARGETS
-    env.PioClean(env.subst("$BUILD_DIR"))
+if env.GetOption("clean") or is_clean_all:
+    env.PioClean(is_clean_all)
     env.Exit(0)
-elif not int(ARGUMENTS.get("PIOVERBOSE", 0)):
+
+if not int(ARGUMENTS.get("PIOVERBOSE", 0)):
     click.echo("Verbose mode can be enabled via `-v, --verbose` option")
 
 # Dynamically load dependent tools

platformio/builder/tools/piolib.py

@@ -59,6 +59,16 @@ class LibBuilderFactory(object):
                 clsname = "%sLibBuilder" % used_frameworks[0].title()
 
         obj = getattr(sys.modules[__name__], clsname)(env, path, verbose=verbose)
+
+        # Handle PlatformIOLibBuilder.manifest.build.builder
+        # pylint: disable=protected-access
+        if isinstance(obj, PlatformIOLibBuilder) and obj._manifest.get("build", {}).get(
+            "builder"
+        ):
+            obj = getattr(
+                sys.modules[__name__], obj._manifest.get("build", {}).get("builder")
+            )(env, path, verbose=verbose)
+
         assert isinstance(obj, LibBuilderBase)
         return obj
 
@@ -174,19 +184,19 @@ class LibBuilderBase(object):
 
     @property
     def include_dir(self):
-        if not all(
+        for name in ("include", "Include"):
-            os.path.isdir(os.path.join(self.path, d)) for d in ("include", "src")
+            d = os.path.join(self.path, name)
-        ):
+            if os.path.isdir(d):
+                return d
         return None
-        return os.path.join(self.path, "include")
 
     @property
     def src_dir(self):
-        return (
+        for name in ("src", "Src"):
-            os.path.join(self.path, "src")
+            d = os.path.join(self.path, name)
-            if os.path.isdir(os.path.join(self.path, "src"))
+            if os.path.isdir(d):
-            else self.path
+                return d
-        )
+        return self.path
 
     def get_include_dirs(self):
         items = []
@@ -491,6 +501,14 @@ class ArduinoLibBuilder(LibBuilderBase):
             return {}
         return ManifestParserFactory.new_from_file(manifest_path).as_dict()
 
+    @property
+    def include_dir(self):
+        if not all(
+            os.path.isdir(os.path.join(self.path, d)) for d in ("include", "src")
+        ):
+            return None
+        return os.path.join(self.path, "include")
+
     def get_include_dirs(self):
         include_dirs = LibBuilderBase.get_include_dirs(self)
         if os.path.isdir(os.path.join(self.path, "src")):
@@ -566,9 +584,12 @@ class ArduinoLibBuilder(LibBuilderBase):
         if self._manifest.get("precompiled") in ("true", "full"):
             # add to LDPATH {build.mcu} folder
             board_config = self.env.BoardConfig()
-            self.env.PrependUnique(
+            for key in ("build.mcu", "build.cpu"):
-                LIBPATH=os.path.join(self.src_dir, board_config.get("build.cpu"))
+                libpath = os.path.join(self.src_dir, board_config.get(key, ""))
-            )
+                if not os.path.isdir(libpath):
+                    continue
+                self.env.PrependUnique(LIBPATH=libpath)
+                break
         ldflags = [flag for flag in ldflags if flag]  # remove empty
         return " ".join(ldflags) if ldflags else None
 
@@ -580,12 +601,6 @@ class MbedLibBuilder(LibBuilderBase):
             return {}
         return ManifestParserFactory.new_from_file(manifest_path).as_dict()
 
-    @property
-    def include_dir(self):
-        if os.path.isdir(os.path.join(self.path, "include")):
-            return os.path.join(self.path, "include")
-        return None
-
     @property
     def src_dir(self):
         if os.path.isdir(os.path.join(self.path, "source")):

platformio/builder/tools/piotarget.py

@@ -29,7 +29,7 @@ def VerboseAction(_, act, actstr):
     return Action(act, actstr)
 
 
-def PioClean(env, clean_dir):
+def PioClean(env, clean_all=False):
     def _relpath(path):
         if compat.IS_WINDOWS:
             prefix = os.getcwd()[:2].lower()
@@ -41,11 +41,9 @@ def PioClean(env, clean_dir):
                 return path
         return os.path.relpath(path)
 
-    if not os.path.isdir(clean_dir):
+    def _clean_dir(path):
-        print("Build environment is clean")
+        clean_rel_path = _relpath(path)
-        env.Exit(0)
+        for root, _, files in os.walk(path):
-    clean_rel_path = _relpath(clean_dir)
-    for root, _, files in os.walk(clean_dir):
             for f in files:
                 dst = os.path.join(root, f)
                 os.remove(dst)
@@ -53,9 +51,20 @@ def PioClean(env, clean_dir):
                     "Removed %s"
                     % (dst if not clean_rel_path.startswith(".") else _relpath(dst))
                 )
+
+    build_dir = env.subst("$BUILD_DIR")
+    libdeps_dir = env.subst("$PROJECT_LIBDEPS_DIR")
+    if os.path.isdir(build_dir):
+        _clean_dir(build_dir)
+        fs.rmtree(build_dir)
+    else:
+        print("Build environment is clean")
+
+    if clean_all and os.path.isdir(libdeps_dir):
+        _clean_dir(libdeps_dir)
+        fs.rmtree(libdeps_dir)
+
     print("Done cleaning")
-    fs.rmtree(clean_dir)
-    env.Exit(0)
 
 
 def AddTarget(  # pylint: disable=too-many-arguments
@@ -65,7 +74,7 @@ def AddTarget(  # pylint: disable=too-many-arguments
     actions,
     title=None,
     description=None,
-    group="Generic",
+    group="General",
     always_build=True,
 ):
     if "__PIO_TARGETS" not in env:
@@ -101,7 +110,13 @@ def DumpTargets(env):
         description="Generate compilation database `compile_commands.json`",
         group="Advanced",
     )
-    targets["clean"] = dict(name="clean", title="Clean", group="Generic")
+    targets["clean"] = dict(name="clean", title="Clean", group="General")
+    targets["cleanall"] = dict(
+        name="cleanall",
+        title="Clean All",
+        group="General",
+        description="Clean a build environment and installed library dependencies",
+    )
     return list(targets.values())
 
 

platformio/commands/check/tools/cppcheck.py

@@ -141,10 +141,11 @@ class CppcheckCheckTool(CheckToolBase):
 
         build_flags = self.cxx_flags if language == "c++" else self.cc_flags
 
+        if not self.is_flag_set("--std", flags):
+            # Try to guess the standard version from the build flags
             for flag in build_flags:
                 if "-std" in flag:
-                # Standards with GNU extensions are not allowed
+                    cmd.append("-" + self.convert_language_standard(flag))
-                cmd.append("-" + flag.replace("gnu", "c"))
 
         cmd.extend(
             ["-D%s" % d for d in self.cpp_defines + self.toolchain_defines[language]]
@@ -224,6 +225,21 @@ class CppcheckCheckTool(CheckToolBase):
         # Cppcheck is configured to return '3' if a defect is found
         return cmd_result["returncode"] in (0, 3)
 
+    @staticmethod
+    def convert_language_standard(flag):
+        cpp_standards_map = {
+            "0x": "11",
+            "1y": "14",
+            "1z": "17",
+            "2a": "20",
+        }
+
+        standard = flag[-2:]
+        # Note: GNU extensions are not supported and converted to regular standards
+        return flag.replace("gnu", "c").replace(
+            standard, cpp_standards_map.get(standard, standard)
+        )
+
     def check(self, on_defect_callback=None):
         self._on_defect_callback = on_defect_callback
 

platformio/commands/ci.py

@@ -122,7 +122,7 @@ def cli(  # pylint: disable=too-many-arguments, too-many-branches
             fs.rmtree(build_dir)
 
 
-def _copy_contents(dst_dir, contents):
+def _copy_contents(dst_dir, contents):  # pylint: disable=too-many-branches
     items = {"dirs": set(), "files": set()}
 
     for path in contents:
@@ -134,14 +134,15 @@ def _copy_contents(dst_dir, contents):
     dst_dir_name = os.path.basename(dst_dir)
 
     if dst_dir_name == "src" and len(items["dirs"]) == 1:
+        if not os.path.isdir(dst_dir):
             shutil.copytree(list(items["dirs"]).pop(), dst_dir, symlinks=True)
     else:
         if not os.path.isdir(dst_dir):
             os.makedirs(dst_dir)
         for d in items["dirs"]:
-            shutil.copytree(
+            src_dst_dir = os.path.join(dst_dir, os.path.basename(d))
-                d, os.path.join(dst_dir, os.path.basename(d)), symlinks=True
+            if not os.path.isdir(src_dst_dir):
-            )
+                shutil.copytree(d, src_dst_dir, symlinks=True)
 
     if not items["files"]:
         return

platformio/commands/home/rpc/handlers/project.py

@@ -265,7 +265,8 @@ class ProjectRPC:
                 fp.write(main_content.strip())
         return project_dir
 
-    async def import_arduino(self, board, use_arduino_libs, arduino_project_dir):
+    @staticmethod
+    async def import_arduino(board, use_arduino_libs, arduino_project_dir):
         board = str(board)
         # don't import PIO Project
         if is_platformio_project(arduino_project_dir):

platformio/commands/remote/command.py

@@ -336,7 +336,10 @@ def device_monitor(ctx, agents, **kwargs):
     kwargs["baud"] = kwargs["baud"] or 9600
 
     def _tx_target(sock_dir):
-        subcmd_argv = ["remote", "device", "monitor"]
+        subcmd_argv = ["remote"]
+        for agent in agents:
+            subcmd_argv.extend(["--agent", agent])
+        subcmd_argv.extend(["device", "monitor"])
         subcmd_argv.extend(device_helpers.options_to_argv(kwargs, project_options))
         subcmd_argv.extend(["--sock", sock_dir])
         subprocess.call([proc.where_is_program("platformio")] + subcmd_argv)

platformio/ide/projectgenerator.py

@@ -81,6 +81,7 @@ class ProjectGenerator(object):
                     "src_files": self.get_src_files(),
                     "project_src_dir": self.config.get_optional_dir("src"),
                     "project_lib_dir": self.config.get_optional_dir("lib"),
+                    "project_test_dir": self.config.get_optional_dir("test"),
                     "project_libdeps_dir": os.path.join(
                         self.config.get_optional_dir("libdeps"), self.env_name
                     ),

platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl

@@ -115,7 +115,7 @@ endif()
 % end
 
 FILE(GLOB_RECURSE SRC_LIST
-%   for path in (project_src_dir, project_lib_dir):
+%   for path in (project_src_dir, project_lib_dir, project_test_dir):
     {{  _normalize_path(path) + "/*.*" }}
 %   end
 )

platformio/package/manager/library.py

@@ -60,14 +60,22 @@ class LibraryPackageManager(BasePackageManager):  # pylint: disable=too-many-anc
 
     @staticmethod
     def find_library_root(path):
+        root_dir_signs = set(["include", "Include", "src", "Src"])
+        root_file_signs = set(
+            [
+                "conanfile.py",  # Conan-based library
+                "CMakeLists.txt",  # CMake-based library
+            ]
+        )
         for root, dirs, files in os.walk(path):
             if not files and len(dirs) == 1:
                 continue
+            if set(root_dir_signs) & set(dirs):
+                return root
+            if set(root_file_signs) & set(files):
+                return root
             for fname in files:
-                if not fname.endswith((".c", ".cpp", ".h", ".S")):
+                if fname.endswith((".c", ".cpp", ".h", ".hpp", ".S")):
-                    continue
-                if os.path.isdir(os.path.join(os.path.dirname(root), "src")):
-                    return os.path.dirname(root)
                     return root
         return path
 

platformio/telemetry.py

@@ -187,7 +187,7 @@ class MeasurementProtocol(TelemetryBase):
     def _ignore_hit(self):
         if not app.get_setting("enable_telemetry"):
             return True
-        if all(c in sys.argv for c in ("run", "idedata")) or self["ea"] == "Idedata":
+        if self["ea"] in ("Idedata", "_Idedata"):
             return True
         return False
 

setup.py

@@ -23,12 +23,12 @@ from platformio import (
     __url__,
     __version__,
 )
-from platformio.compat import PY2, WINDOWS
+from platformio.compat import PY2
 
 
 minimal_requirements = [
     "bottle==0.12.*",
-    "click>=5,<9%s" % (",!=7.1,!=7.1.1" if WINDOWS else ""),
+    "click>=7.1.2,<9,!=8.0.2",
     "colorama",
     "marshmallow%s" % (">=2,<3" if PY2 else ">=2,<4"),
     "pyelftools>=0.27,<1",

tests/commands/test_check.py

@@ -15,7 +15,6 @@
 # pylint: disable=redefined-outer-name
 
 import json
-import sys
 from os.path import isfile, join
 
 import pytest
@@ -132,6 +131,47 @@ def test_check_language_standard_definition_passed(clirunner, tmpdir):
     assert "--std=c++17" in result.output
 
 
+def test_check_language_standard_option_is_converted(clirunner, tmpdir):
+    config = (
+        DEFAULT_CONFIG
+        + """
+build_flags = -std=gnu++1y
+    """
+    )
+    tmpdir.join("platformio.ini").write(config)
+    tmpdir.mkdir("src").join("main.cpp").write(TEST_CODE)
+    result = clirunner.invoke(cmd_check, ["--project-dir", str(tmpdir), "-v"])
+
+    assert "--std=c++14" in result.output
+
+
+def test_check_language_standard_is_prioritized_over_build_flags(clirunner, tmpdir):
+    config = (
+        DEFAULT_CONFIG
+        + """
+check_flags = --std=c++03
+build_flags = -std=c++17
+    """
+    )
+    tmpdir.join("platformio.ini").write(config)
+    tmpdir.mkdir("src").join("main.cpp").write(TEST_CODE)
+    result = clirunner.invoke(cmd_check, ["--project-dir", str(tmpdir), "-v"])
+
+    assert "--std=c++03" in result.output
+    assert "--std=c++17" not in result.output
+
+
+def test_check_language_standard_for_c_language(clirunner, tmpdir):
+    config = DEFAULT_CONFIG + "\nbuild_flags = -std=c11"
+    tmpdir.join("platformio.ini").write(config)
+    tmpdir.mkdir("src").join("main.c").write(TEST_CODE)
+    result = clirunner.invoke(cmd_check, ["--project-dir", str(tmpdir), "-v"])
+
+    assert "--std=c11" in result.output
+    assert "__STDC_VERSION__=201112L" in result.output
+    assert "__cplusplus" not in result.output
+
+
 def test_check_severity_threshold(clirunner, validate_cliresult, check_dir):
     result = clirunner.invoke(
         cmd_check, ["--project-dir", str(check_dir), "--severity=high"]
@@ -451,12 +491,11 @@ int main() {
 """
     )
 
-    frameworks = ["arduino", "stm32cube"]
+    for framework in (
-    if sys.version_info[0] == 3:
+        "arduino",
-        # Zephyr only supports Python 3
+        "stm32cube",
-        frameworks.append("zephyr")
+        "zephyr",
-
+    ):
-    for framework in frameworks:
         for tool in ("cppcheck", "clangtidy", "pvs-studio"):
             tmpdir.join("platformio.ini").write(config % (framework, tool))
             result = clirunner.invoke(cmd_check, ["--project-dir", str(tmpdir)])

tests/commands/test_ci.py

@@ -88,6 +88,69 @@ def test_ci_keep_build_dir(clirunner, tmpdir_factory, validate_cliresult):
     assert "board: metro" in result.output
 
 
+def test_ci_keep_build_dir_single_src_dir(
+    clirunner, tmpdir_factory, validate_cliresult
+):
+    build_dir = str(tmpdir_factory.mktemp("ci_build_dir"))
+
+    # Run two times to detect possible "AlreadyExists" errors
+    for _ in range(2):
+        result = clirunner.invoke(
+            cmd_ci,
+            [
+                join("examples", "wiring-blink", "src"),
+                "-b",
+                "uno",
+                "--build-dir",
+                build_dir,
+                "--keep-build-dir",
+            ],
+        )
+        validate_cliresult(result)
+
+
+def test_ci_keep_build_dir_nested_src_dirs(
+    clirunner, tmpdir_factory, validate_cliresult
+):
+
+    build_dir = str(tmpdir_factory.mktemp("ci_build_dir"))
+
+    # Split default Arduino project in two parts
+    src_dir1 = tmpdir_factory.mktemp("src_1")
+    src_dir1.join("src1.cpp").write(
+        """
+void setup() {}
+"""
+    )
+
+    src_dir2 = tmpdir_factory.mktemp("src_2")
+    src_dir2.join("src2.cpp").write(
+        """
+void loop() {}
+"""
+    )
+
+    src_dir1 = str(src_dir1)
+    src_dir2 = str(src_dir2)
+
+    # Run two times to detect possible "AlreadyExists" errors
+    for _ in range(2):
+        result = clirunner.invoke(
+            cmd_ci,
+            [
+                src_dir1,
+                src_dir2,
+                "-b",
+                "teensy40",
+                "--build-dir",
+                build_dir,
+                "--keep-build-dir",
+            ],
+        )
+
+        validate_cliresult(result)
+
+
 def test_ci_project_conf(clirunner, validate_cliresult):
     project_dir = join("examples", "wiring-blink")
     result = clirunner.invoke(

tests/commands/test_test.py

@@ -162,6 +162,8 @@ void unittest_uart_end(){}
     validate_cliresult(native_result)
     validate_cliresult(embedded_result)
 
+    print("native_result.output", native_result.output)
+    print("embedded_result.output", embedded_result.output)
     assert all(f in native_result.output for f in ("setUp called", "tearDown called"))
     assert all(
         "[FAILED]" not in out for out in (native_result.output, embedded_result.output)