diff --git a/.github/workflows/examples.yml b/.github/workflows/examples.yml index 29c76103..6e9fc7df 100644 --- a/.github/workflows/examples.yml +++ b/.github/workflows/examples.yml @@ -7,7 +7,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-16.04, windows-latest, macos-latest] + os: [ubuntu-18.04, windows-latest, macos-latest] python-version: [3.7] runs-on: ${{ matrix.os }} steps: diff --git a/.pylintrc b/.pylintrc index 4a7f6601..3943bac6 100644 --- a/.pylintrc +++ b/.pylintrc @@ -16,6 +16,7 @@ disable= useless-import-alias, bad-option-value, consider-using-dict-items, + consider-using-f-string, ; PY2 Compat super-with-arguments, diff --git a/HISTORY.rst b/HISTORY.rst index fa81cec0..d84c84da 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -8,6 +8,18 @@ PlatformIO Core 5 **A professional collaborative platform for embedded development** +5.2.1 (2021-10-11) +~~~~~~~~~~~~~~~~~~ + +- Clean a build environment and installed library dependencies using a new ``cleanall`` target (`issue #4062 `_) +- Override a default library builder via a new ``builder`` field in a ``build`` group of `library.json `__ manifest (`issue #3957 `_) +- Updated `Cppcheck `__ v2.6 with new checks, increased reliability of advanced addons (MISRA/CERT) and various improvements +- Handle the "test" folder as a part of CLion project (`issue #4005 `_) +- Improved handling of a library root based on "Conan" or "CMake" build systems (`issue #3887 `_) +- Fixed a "KeyError: Invalid board option 'build.cpu'" when using a precompiled library with a board that does not have a CPU field in the manifest (`issue #4056 `_) +- Fixed a "FileExist" error when the `platformio ci `__ command is used in pair with the ``--keep-build-dir`` option (`issue #4011 `_) +- Fixed an issue with draft values of C++ language standards that broke static analysis via Cppcheck (`issue #3944 `_) + 5.2.0 (2021-09-13) ~~~~~~~~~~~~~~~~~~ diff --git a/docs b/docs index c9d2ef9a..8a613430 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit c9d2ef9abe4d349465609616fc5c2b69b4a0823d +Subproject commit 8a613430957704dfc2e4fc30337020a0a80cfba8 diff --git a/platformio/__init__.py b/platformio/__init__.py index 0314286b..31da28ca 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (5, 2, 0) +VERSION = (5, 2, 1) __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" @@ -51,7 +51,7 @@ __core_packages__ = { "contrib-pysite": "~2.%d%d.0" % (sys.version_info.major, sys.version_info.minor), "tool-unity": "~1.20500.0", "tool-scons": "~4.40200.0", - "tool-cppcheck": "~1.250.0", + "tool-cppcheck": "~1.260.0", "tool-clangtidy": "~1.120001.0", "tool-pvs-studio": "~7.14.0", } diff --git a/platformio/__main__.py b/platformio/__main__.py index 0cc7dca3..f2c38808 100644 --- a/platformio/__main__.py +++ b/platformio/__main__.py @@ -67,9 +67,24 @@ def cli(ctx, force, caller, no_ansi): maintenance.on_platformio_start(ctx, force, caller) -@cli.resultcallback() -@click.pass_context -def process_result(ctx, result, *_, **__): +try: + + @cli.result_callback() + @click.pass_context + def process_result(ctx, result, *_, **__): + _process_result(ctx, result) + + +except (AttributeError, TypeError): # legacy support for CLick > 8.0.1 + print("legacy Click") + + @cli.resultcallback() + @click.pass_context + def process_result(ctx, result, *_, **__): + _process_result(ctx, result) + + +def _process_result(ctx, result): from platformio import maintenance maintenance.on_platformio_end(ctx, result) diff --git a/platformio/builder/main.py b/platformio/builder/main.py index 04d6e342..7f719cca 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -149,10 +149,12 @@ if int(ARGUMENTS.get("ISATTY", 0)): # pylint: disable=protected-access click._compat.isatty = lambda stream: True -if env.GetOption("clean"): - env.PioClean(env.subst("$BUILD_DIR")) +is_clean_all = "cleanall" in COMMAND_LINE_TARGETS +if env.GetOption("clean") or is_clean_all: + env.PioClean(is_clean_all) env.Exit(0) -elif not int(ARGUMENTS.get("PIOVERBOSE", 0)): + +if not int(ARGUMENTS.get("PIOVERBOSE", 0)): click.echo("Verbose mode can be enabled via `-v, --verbose` option") # Dynamically load dependent tools diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index 186485d2..bdebf6ab 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -59,6 +59,16 @@ class LibBuilderFactory(object): clsname = "%sLibBuilder" % used_frameworks[0].title() obj = getattr(sys.modules[__name__], clsname)(env, path, verbose=verbose) + + # Handle PlatformIOLibBuilder.manifest.build.builder + # pylint: disable=protected-access + if isinstance(obj, PlatformIOLibBuilder) and obj._manifest.get("build", {}).get( + "builder" + ): + obj = getattr( + sys.modules[__name__], obj._manifest.get("build", {}).get("builder") + )(env, path, verbose=verbose) + assert isinstance(obj, LibBuilderBase) return obj @@ -174,19 +184,19 @@ class LibBuilderBase(object): @property def include_dir(self): - if not all( - os.path.isdir(os.path.join(self.path, d)) for d in ("include", "src") - ): - return None - return os.path.join(self.path, "include") + for name in ("include", "Include"): + d = os.path.join(self.path, name) + if os.path.isdir(d): + return d + return None @property def src_dir(self): - return ( - os.path.join(self.path, "src") - if os.path.isdir(os.path.join(self.path, "src")) - else self.path - ) + for name in ("src", "Src"): + d = os.path.join(self.path, name) + if os.path.isdir(d): + return d + return self.path def get_include_dirs(self): items = [] @@ -491,6 +501,14 @@ class ArduinoLibBuilder(LibBuilderBase): return {} return ManifestParserFactory.new_from_file(manifest_path).as_dict() + @property + def include_dir(self): + if not all( + os.path.isdir(os.path.join(self.path, d)) for d in ("include", "src") + ): + return None + return os.path.join(self.path, "include") + def get_include_dirs(self): include_dirs = LibBuilderBase.get_include_dirs(self) if os.path.isdir(os.path.join(self.path, "src")): @@ -566,9 +584,12 @@ class ArduinoLibBuilder(LibBuilderBase): if self._manifest.get("precompiled") in ("true", "full"): # add to LDPATH {build.mcu} folder board_config = self.env.BoardConfig() - self.env.PrependUnique( - LIBPATH=os.path.join(self.src_dir, board_config.get("build.cpu")) - ) + for key in ("build.mcu", "build.cpu"): + libpath = os.path.join(self.src_dir, board_config.get(key, "")) + if not os.path.isdir(libpath): + continue + self.env.PrependUnique(LIBPATH=libpath) + break ldflags = [flag for flag in ldflags if flag] # remove empty return " ".join(ldflags) if ldflags else None @@ -580,12 +601,6 @@ class MbedLibBuilder(LibBuilderBase): return {} return ManifestParserFactory.new_from_file(manifest_path).as_dict() - @property - def include_dir(self): - if os.path.isdir(os.path.join(self.path, "include")): - return os.path.join(self.path, "include") - return None - @property def src_dir(self): if os.path.isdir(os.path.join(self.path, "source")): diff --git a/platformio/builder/tools/piotarget.py b/platformio/builder/tools/piotarget.py index 70b7c41b..6ebe1f99 100644 --- a/platformio/builder/tools/piotarget.py +++ b/platformio/builder/tools/piotarget.py @@ -29,7 +29,7 @@ def VerboseAction(_, act, actstr): return Action(act, actstr) -def PioClean(env, clean_dir): +def PioClean(env, clean_all=False): def _relpath(path): if compat.IS_WINDOWS: prefix = os.getcwd()[:2].lower() @@ -41,21 +41,30 @@ def PioClean(env, clean_dir): return path return os.path.relpath(path) - if not os.path.isdir(clean_dir): + def _clean_dir(path): + clean_rel_path = _relpath(path) + for root, _, files in os.walk(path): + for f in files: + dst = os.path.join(root, f) + os.remove(dst) + print( + "Removed %s" + % (dst if not clean_rel_path.startswith(".") else _relpath(dst)) + ) + + build_dir = env.subst("$BUILD_DIR") + libdeps_dir = env.subst("$PROJECT_LIBDEPS_DIR") + if os.path.isdir(build_dir): + _clean_dir(build_dir) + fs.rmtree(build_dir) + else: print("Build environment is clean") - env.Exit(0) - clean_rel_path = _relpath(clean_dir) - for root, _, files in os.walk(clean_dir): - for f in files: - dst = os.path.join(root, f) - os.remove(dst) - print( - "Removed %s" - % (dst if not clean_rel_path.startswith(".") else _relpath(dst)) - ) + + if clean_all and os.path.isdir(libdeps_dir): + _clean_dir(libdeps_dir) + fs.rmtree(libdeps_dir) + print("Done cleaning") - fs.rmtree(clean_dir) - env.Exit(0) def AddTarget( # pylint: disable=too-many-arguments @@ -65,7 +74,7 @@ def AddTarget( # pylint: disable=too-many-arguments actions, title=None, description=None, - group="Generic", + group="General", always_build=True, ): if "__PIO_TARGETS" not in env: @@ -101,7 +110,13 @@ def DumpTargets(env): description="Generate compilation database `compile_commands.json`", group="Advanced", ) - targets["clean"] = dict(name="clean", title="Clean", group="Generic") + targets["clean"] = dict(name="clean", title="Clean", group="General") + targets["cleanall"] = dict( + name="cleanall", + title="Clean All", + group="General", + description="Clean a build environment and installed library dependencies", + ) return list(targets.values()) diff --git a/platformio/commands/check/tools/cppcheck.py b/platformio/commands/check/tools/cppcheck.py index 8dd6041a..bc06799a 100644 --- a/platformio/commands/check/tools/cppcheck.py +++ b/platformio/commands/check/tools/cppcheck.py @@ -141,10 +141,11 @@ class CppcheckCheckTool(CheckToolBase): build_flags = self.cxx_flags if language == "c++" else self.cc_flags - for flag in build_flags: - if "-std" in flag: - # Standards with GNU extensions are not allowed - cmd.append("-" + flag.replace("gnu", "c")) + if not self.is_flag_set("--std", flags): + # Try to guess the standard version from the build flags + for flag in build_flags: + if "-std" in flag: + cmd.append("-" + self.convert_language_standard(flag)) cmd.extend( ["-D%s" % d for d in self.cpp_defines + self.toolchain_defines[language]] @@ -224,6 +225,21 @@ class CppcheckCheckTool(CheckToolBase): # Cppcheck is configured to return '3' if a defect is found return cmd_result["returncode"] in (0, 3) + @staticmethod + def convert_language_standard(flag): + cpp_standards_map = { + "0x": "11", + "1y": "14", + "1z": "17", + "2a": "20", + } + + standard = flag[-2:] + # Note: GNU extensions are not supported and converted to regular standards + return flag.replace("gnu", "c").replace( + standard, cpp_standards_map.get(standard, standard) + ) + def check(self, on_defect_callback=None): self._on_defect_callback = on_defect_callback diff --git a/platformio/commands/ci.py b/platformio/commands/ci.py index 5bc5b38b..58d8fef7 100644 --- a/platformio/commands/ci.py +++ b/platformio/commands/ci.py @@ -122,7 +122,7 @@ def cli( # pylint: disable=too-many-arguments, too-many-branches fs.rmtree(build_dir) -def _copy_contents(dst_dir, contents): +def _copy_contents(dst_dir, contents): # pylint: disable=too-many-branches items = {"dirs": set(), "files": set()} for path in contents: @@ -134,14 +134,15 @@ def _copy_contents(dst_dir, contents): dst_dir_name = os.path.basename(dst_dir) if dst_dir_name == "src" and len(items["dirs"]) == 1: - shutil.copytree(list(items["dirs"]).pop(), dst_dir, symlinks=True) + if not os.path.isdir(dst_dir): + shutil.copytree(list(items["dirs"]).pop(), dst_dir, symlinks=True) else: if not os.path.isdir(dst_dir): os.makedirs(dst_dir) for d in items["dirs"]: - shutil.copytree( - d, os.path.join(dst_dir, os.path.basename(d)), symlinks=True - ) + src_dst_dir = os.path.join(dst_dir, os.path.basename(d)) + if not os.path.isdir(src_dst_dir): + shutil.copytree(d, src_dst_dir, symlinks=True) if not items["files"]: return diff --git a/platformio/commands/home/rpc/handlers/project.py b/platformio/commands/home/rpc/handlers/project.py index 2263beb1..4c48a5d9 100644 --- a/platformio/commands/home/rpc/handlers/project.py +++ b/platformio/commands/home/rpc/handlers/project.py @@ -265,7 +265,8 @@ class ProjectRPC: fp.write(main_content.strip()) return project_dir - async def import_arduino(self, board, use_arduino_libs, arduino_project_dir): + @staticmethod + async def import_arduino(board, use_arduino_libs, arduino_project_dir): board = str(board) # don't import PIO Project if is_platformio_project(arduino_project_dir): diff --git a/platformio/commands/remote/command.py b/platformio/commands/remote/command.py index f9486d51..03da8389 100644 --- a/platformio/commands/remote/command.py +++ b/platformio/commands/remote/command.py @@ -336,7 +336,10 @@ def device_monitor(ctx, agents, **kwargs): kwargs["baud"] = kwargs["baud"] or 9600 def _tx_target(sock_dir): - subcmd_argv = ["remote", "device", "monitor"] + subcmd_argv = ["remote"] + for agent in agents: + subcmd_argv.extend(["--agent", agent]) + subcmd_argv.extend(["device", "monitor"]) subcmd_argv.extend(device_helpers.options_to_argv(kwargs, project_options)) subcmd_argv.extend(["--sock", sock_dir]) subprocess.call([proc.where_is_program("platformio")] + subcmd_argv) diff --git a/platformio/ide/projectgenerator.py b/platformio/ide/projectgenerator.py index 30eaa97d..56930b9a 100644 --- a/platformio/ide/projectgenerator.py +++ b/platformio/ide/projectgenerator.py @@ -81,6 +81,7 @@ class ProjectGenerator(object): "src_files": self.get_src_files(), "project_src_dir": self.config.get_optional_dir("src"), "project_lib_dir": self.config.get_optional_dir("lib"), + "project_test_dir": self.config.get_optional_dir("test"), "project_libdeps_dir": os.path.join( self.config.get_optional_dir("libdeps"), self.env_name ), diff --git a/platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl b/platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl index dc03b2e5..a5c0010b 100644 --- a/platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl +++ b/platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl @@ -115,7 +115,7 @@ endif() % end FILE(GLOB_RECURSE SRC_LIST -% for path in (project_src_dir, project_lib_dir): +% for path in (project_src_dir, project_lib_dir, project_test_dir): {{ _normalize_path(path) + "/*.*" }} % end ) diff --git a/platformio/package/manager/library.py b/platformio/package/manager/library.py index 03960306..3ce2a1fa 100644 --- a/platformio/package/manager/library.py +++ b/platformio/package/manager/library.py @@ -60,15 +60,23 @@ class LibraryPackageManager(BasePackageManager): # pylint: disable=too-many-anc @staticmethod def find_library_root(path): + root_dir_signs = set(["include", "Include", "src", "Src"]) + root_file_signs = set( + [ + "conanfile.py", # Conan-based library + "CMakeLists.txt", # CMake-based library + ] + ) for root, dirs, files in os.walk(path): if not files and len(dirs) == 1: continue - for fname in files: - if not fname.endswith((".c", ".cpp", ".h", ".S")): - continue - if os.path.isdir(os.path.join(os.path.dirname(root), "src")): - return os.path.dirname(root) + if set(root_dir_signs) & set(dirs): return root + if set(root_file_signs) & set(files): + return root + for fname in files: + if fname.endswith((".c", ".cpp", ".h", ".hpp", ".S")): + return root return path def _install( # pylint: disable=too-many-arguments diff --git a/platformio/telemetry.py b/platformio/telemetry.py index 94c4222a..ed2bf9b0 100644 --- a/platformio/telemetry.py +++ b/platformio/telemetry.py @@ -187,7 +187,7 @@ class MeasurementProtocol(TelemetryBase): def _ignore_hit(self): if not app.get_setting("enable_telemetry"): return True - if all(c in sys.argv for c in ("run", "idedata")) or self["ea"] == "Idedata": + if self["ea"] in ("Idedata", "_Idedata"): return True return False diff --git a/setup.py b/setup.py index fb592ed2..0c057437 100644 --- a/setup.py +++ b/setup.py @@ -23,12 +23,12 @@ from platformio import ( __url__, __version__, ) -from platformio.compat import PY2, WINDOWS +from platformio.compat import PY2 minimal_requirements = [ "bottle==0.12.*", - "click>=5,<9%s" % (",!=7.1,!=7.1.1" if WINDOWS else ""), + "click>=7.1.2,<9,!=8.0.2", "colorama", "marshmallow%s" % (">=2,<3" if PY2 else ">=2,<4"), "pyelftools>=0.27,<1", diff --git a/tests/commands/test_check.py b/tests/commands/test_check.py index 98400c17..d4b3b79b 100644 --- a/tests/commands/test_check.py +++ b/tests/commands/test_check.py @@ -15,7 +15,6 @@ # pylint: disable=redefined-outer-name import json -import sys from os.path import isfile, join import pytest @@ -132,6 +131,47 @@ def test_check_language_standard_definition_passed(clirunner, tmpdir): assert "--std=c++17" in result.output +def test_check_language_standard_option_is_converted(clirunner, tmpdir): + config = ( + DEFAULT_CONFIG + + """ +build_flags = -std=gnu++1y + """ + ) + tmpdir.join("platformio.ini").write(config) + tmpdir.mkdir("src").join("main.cpp").write(TEST_CODE) + result = clirunner.invoke(cmd_check, ["--project-dir", str(tmpdir), "-v"]) + + assert "--std=c++14" in result.output + + +def test_check_language_standard_is_prioritized_over_build_flags(clirunner, tmpdir): + config = ( + DEFAULT_CONFIG + + """ +check_flags = --std=c++03 +build_flags = -std=c++17 + """ + ) + tmpdir.join("platformio.ini").write(config) + tmpdir.mkdir("src").join("main.cpp").write(TEST_CODE) + result = clirunner.invoke(cmd_check, ["--project-dir", str(tmpdir), "-v"]) + + assert "--std=c++03" in result.output + assert "--std=c++17" not in result.output + + +def test_check_language_standard_for_c_language(clirunner, tmpdir): + config = DEFAULT_CONFIG + "\nbuild_flags = -std=c11" + tmpdir.join("platformio.ini").write(config) + tmpdir.mkdir("src").join("main.c").write(TEST_CODE) + result = clirunner.invoke(cmd_check, ["--project-dir", str(tmpdir), "-v"]) + + assert "--std=c11" in result.output + assert "__STDC_VERSION__=201112L" in result.output + assert "__cplusplus" not in result.output + + def test_check_severity_threshold(clirunner, validate_cliresult, check_dir): result = clirunner.invoke( cmd_check, ["--project-dir", str(check_dir), "--severity=high"] @@ -451,12 +491,11 @@ int main() { """ ) - frameworks = ["arduino", "stm32cube"] - if sys.version_info[0] == 3: - # Zephyr only supports Python 3 - frameworks.append("zephyr") - - for framework in frameworks: + for framework in ( + "arduino", + "stm32cube", + "zephyr", + ): for tool in ("cppcheck", "clangtidy", "pvs-studio"): tmpdir.join("platformio.ini").write(config % (framework, tool)) result = clirunner.invoke(cmd_check, ["--project-dir", str(tmpdir)]) diff --git a/tests/commands/test_ci.py b/tests/commands/test_ci.py index 0ea22dd6..95b31a8f 100644 --- a/tests/commands/test_ci.py +++ b/tests/commands/test_ci.py @@ -88,6 +88,69 @@ def test_ci_keep_build_dir(clirunner, tmpdir_factory, validate_cliresult): assert "board: metro" in result.output +def test_ci_keep_build_dir_single_src_dir( + clirunner, tmpdir_factory, validate_cliresult +): + build_dir = str(tmpdir_factory.mktemp("ci_build_dir")) + + # Run two times to detect possible "AlreadyExists" errors + for _ in range(2): + result = clirunner.invoke( + cmd_ci, + [ + join("examples", "wiring-blink", "src"), + "-b", + "uno", + "--build-dir", + build_dir, + "--keep-build-dir", + ], + ) + validate_cliresult(result) + + +def test_ci_keep_build_dir_nested_src_dirs( + clirunner, tmpdir_factory, validate_cliresult +): + + build_dir = str(tmpdir_factory.mktemp("ci_build_dir")) + + # Split default Arduino project in two parts + src_dir1 = tmpdir_factory.mktemp("src_1") + src_dir1.join("src1.cpp").write( + """ +void setup() {} +""" + ) + + src_dir2 = tmpdir_factory.mktemp("src_2") + src_dir2.join("src2.cpp").write( + """ +void loop() {} +""" + ) + + src_dir1 = str(src_dir1) + src_dir2 = str(src_dir2) + + # Run two times to detect possible "AlreadyExists" errors + for _ in range(2): + result = clirunner.invoke( + cmd_ci, + [ + src_dir1, + src_dir2, + "-b", + "teensy40", + "--build-dir", + build_dir, + "--keep-build-dir", + ], + ) + + validate_cliresult(result) + + def test_ci_project_conf(clirunner, validate_cliresult): project_dir = join("examples", "wiring-blink") result = clirunner.invoke( diff --git a/tests/commands/test_test.py b/tests/commands/test_test.py index 08642eaf..f7552ed8 100644 --- a/tests/commands/test_test.py +++ b/tests/commands/test_test.py @@ -162,6 +162,8 @@ void unittest_uart_end(){} validate_cliresult(native_result) validate_cliresult(embedded_result) + print("native_result.output", native_result.output) + print("embedded_result.output", embedded_result.output) assert all(f in native_result.output for f in ("setUp called", "tearDown called")) assert all( "[FAILED]" not in out for out in (native_result.output, embedded_result.output)