diff --git a/.github/workflows/examples.yml b/.github/workflows/examples.yml index a2239d5b..5825fd81 100644 --- a/.github/workflows/examples.yml +++ b/.github/workflows/examples.yml @@ -15,7 +15,7 @@ jobs: with: submodules: "recursive" - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v1 + uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - name: Install dependencies @@ -26,7 +26,8 @@ jobs: - name: Run on Linux if: startsWith(matrix.os, 'ubuntu') env: - PIO_INSTALL_DEVPLATFORMS_IGNORE: "ststm8,infineonxmc,siwigsm,intel_mcs51,aceinna_imu" + PIO_INSTALL_DEVPLATFORMS_OWNERNAMES: "platformio" + PIO_INSTALL_DEVPLATFORMS_IGNORE: "ststm8,infineonxmc,intel_mcs51" run: | # ChipKIT issue: install 32-bit support for GCC PIC32 sudo apt-get install libc6-i386 @@ -40,7 +41,8 @@ jobs: - name: Run on macOS if: startsWith(matrix.os, 'macos') env: - PIO_INSTALL_DEVPLATFORMS_IGNORE: "ststm8,infineonxmc,siwigsm,microchippic32,gd32v,nuclei,lattice_ice40" + PIO_INSTALL_DEVPLATFORMS_OWNERNAMES: "platformio" + PIO_INSTALL_DEVPLATFORMS_IGNORE: "ststm8,infineonxmc,microchippic32,lattice_ice40,gd32v" run: | df -h tox -e testexamples @@ -50,7 +52,8 @@ jobs: env: PLATFORMIO_CORE_DIR: C:/pio PLATFORMIO_WORKSPACE_DIR: C:/pio-workspace/$PROJECT_HASH - PIO_INSTALL_DEVPLATFORMS_IGNORE: "ststm8,infineonxmc,siwigsm,riscv_gap" + PIO_INSTALL_DEVPLATFORMS_OWNERNAMES: "platformio" + PIO_INSTALL_DEVPLATFORMS_IGNORE: "ststm8,infineonxmc,riscv_gap" run: | tox -e testexamples diff --git a/HISTORY.rst b/HISTORY.rst index 19b9b921..9e5bff4e 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -8,6 +8,21 @@ PlatformIO Core 5 **A professional collaborative platform for embedded development** +5.0.4 (2020-12-30) +~~~~~~~~~~~~~~~~~~ + +- Added "Core" suffix when showing PlatformIO Core version using ``pio --version`` command +- Improved ``.ccls`` configuration file for Emacs, Vim, and Sublime Text integrations +- Updated analysis tools: + + * `Cppcheck `__ v2.3 with improved C++ parser and several new MISRA rules + * `PVS-Studio `__ v7.11 with new diagnostics and updated mass suppression mechanism + +- Show a warning message about deprecated support for Python 2 and Python 3.5 +- Do not provide "intelliSenseMode" option when generating configuration for VSCode C/C++ extension +- Fixed a "git-sh-setup: file not found" error when installing project dependencies from Git VCS (`issue #3740 `_) +- Fixed an issue with package publishing on Windows when Unix permissions are not preserved (`issue #3776 `_) + 5.0.3 (2020-11-12) ~~~~~~~~~~~~~~~~~~ @@ -128,7 +143,8 @@ Please check `Migration guide from 4.x to 5.0 `__ command (`issue #3521 `_) - Remove unused data using a new `pio system prune `__ command (`issue #3522 `_) - Show ignored project environments only in the verbose mode (`issue #3641 `_) - - Do not escape compiler arguments in VSCode template on Windows. + - Do not escape compiler arguments in VSCode template on Windows + - Drop support for Python 2 and 3.5. .. _release_notes_4: diff --git a/docs b/docs index 8d9e8ef0..9db46dcc 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 8d9e8ef02b730c01c39a6e9355ede0111b3eee81 +Subproject commit 9db46dccef0a765770e71b64bf29b6e1d91df403 diff --git a/examples b/examples index 7255d5b8..161ae730 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit 7255d5b897b402d981b843f88ee001b69c9f1407 +Subproject commit 161ae7302b3508cadb10f5552de9f996731976ac diff --git a/platformio/__init__.py b/platformio/__init__.py index 5fa70b0a..a540b3ff 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (5, 0, 3) +VERSION = (5, 0, 4) __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" @@ -31,11 +31,11 @@ __description__ = ( ) __url__ = "https://platformio.org" -__author__ = "PlatformIO" -__email__ = "contact@platformio.org" +__author__ = "PlatformIO Labs" +__email__ = "contact@piolabs.com" __license__ = "Apache Software License" -__copyright__ = "Copyright 2014-present PlatformIO" +__copyright__ = "Copyright 2014-present PlatformIO Labs" __accounts_api__ = "https://api.accounts.platformio.org" __registry_api__ = [ @@ -51,9 +51,9 @@ __core_packages__ = { "contrib-pysite": "~2.%d%d.0" % (sys.version_info.major, sys.version_info.minor), "tool-unity": "~1.20500.0", "tool-scons": "~2.20501.7" if sys.version_info.major == 2 else "~4.40001.0", - "tool-cppcheck": "~1.210.0", + "tool-cppcheck": "~1.230.0", "tool-clangtidy": "~1.100000.0", - "tool-pvs-studio": "~7.9.0", + "tool-pvs-studio": "~7.11.0", } __check_internet_hosts__ = [ diff --git a/platformio/__main__.py b/platformio/__main__.py index 8420544b..537b1d0a 100644 --- a/platformio/__main__.py +++ b/platformio/__main__.py @@ -33,7 +33,7 @@ except: # pylint: disable=bare-except @click.command( cls=PlatformioCLI, context_settings=dict(help_option_names=["-h", "--help"]) ) -@click.version_option(__version__, prog_name="PlatformIO") +@click.version_option(__version__, prog_name="PlatformIO Core") @click.option("--force", "-f", is_flag=True, help="DEPRECATE") @click.option("--caller", "-c", help="Caller ID (service)") @click.option("--no-ansi", is_flag=True, help="Do not print ANSI control characters") diff --git a/platformio/commands/check/tools/base.py b/platformio/commands/check/tools/base.py index dc9f476f..7eda6936 100644 --- a/platformio/commands/check/tools/base.py +++ b/platformio/commands/check/tools/base.py @@ -167,6 +167,29 @@ class CheckToolBase(object): # pylint: disable=too-many-instance-attributes if os.path.isfile(f): os.remove(f) + @staticmethod + def is_check_successful(cmd_result): + return cmd_result["returncode"] == 0 + + def execute_check_cmd(self, cmd): + result = proc.exec_command( + cmd, + stdout=proc.LineBufferedAsyncPipe(self.on_tool_output), + stderr=proc.LineBufferedAsyncPipe(self.on_tool_output), + ) + + if not self.is_check_successful(result): + click.echo( + "\nError: Failed to execute check command! Exited with code %d." + % result["returncode"] + ) + if self.options.get("verbose"): + click.echo(result["out"]) + click.echo(result["err"]) + self._bad_input = True + + return result + @staticmethod def get_project_target_files(patterns): c_extension = (".c",) @@ -200,11 +223,7 @@ class CheckToolBase(object): # pylint: disable=too-many-instance-attributes if self.options.get("verbose"): click.echo(" ".join(cmd)) - proc.exec_command( - cmd, - stdout=proc.LineBufferedAsyncPipe(self.on_tool_output), - stderr=proc.LineBufferedAsyncPipe(self.on_tool_output), - ) + self.execute_check_cmd(cmd) else: if self.options.get("verbose"): diff --git a/platformio/commands/check/tools/clangtidy.py b/platformio/commands/check/tools/clangtidy.py index 06f3ff76..3206cdba 100644 --- a/platformio/commands/check/tools/clangtidy.py +++ b/platformio/commands/check/tools/clangtidy.py @@ -49,6 +49,12 @@ class ClangtidyCheckTool(CheckToolBase): return DefectItem(severity, category, message, file_, line, column, defect_id) + @staticmethod + def is_check_successful(cmd_result): + # Note: Clang-Tidy returns 1 for not critical compilation errors, + # so 0 and 1 are only acceptable values + return cmd_result["returncode"] < 2 + def configure_command(self): tool_path = join(get_core_package_dir("tool-clangtidy"), "clang-tidy") diff --git a/platformio/commands/check/tools/cppcheck.py b/platformio/commands/check/tools/cppcheck.py index b38bb8d6..46d15d07 100644 --- a/platformio/commands/check/tools/cppcheck.py +++ b/platformio/commands/check/tools/cppcheck.py @@ -109,7 +109,7 @@ class CppcheckCheckTool(CheckToolBase): cmd = [ tool_path, "--addon-python=%s" % proc.get_pythonexe_path(), - "--error-exitcode=1", + "--error-exitcode=3", "--verbose" if self.options.get("verbose") else "--quiet", ] @@ -220,6 +220,11 @@ class CppcheckCheckTool(CheckToolBase): if os.path.isfile(dump_file): os.remove(dump_file) + @staticmethod + def is_check_successful(cmd_result): + # Cppcheck is configured to return '3' if a defect is found + return cmd_result["returncode"] in (0, 3) + def check(self, on_defect_callback=None): self._on_defect_callback = on_defect_callback project_files = self.get_project_target_files(self.options["patterns"]) @@ -238,11 +243,7 @@ class CppcheckCheckTool(CheckToolBase): if self.options.get("verbose"): click.echo(" ".join(cmd)) - proc.exec_command( - cmd, - stdout=proc.LineBufferedAsyncPipe(self.on_tool_output), - stderr=proc.LineBufferedAsyncPipe(self.on_tool_output), - ) + self.execute_check_cmd(cmd) self.clean_up() diff --git a/platformio/commands/check/tools/pvsstudio.py b/platformio/commands/check/tools/pvsstudio.py index ce5d93ec..88b1a513 100644 --- a/platformio/commands/check/tools/pvsstudio.py +++ b/platformio/commands/check/tools/pvsstudio.py @@ -52,6 +52,11 @@ class PvsStudioCheckTool(CheckToolBase): # pylint: disable=too-many-instance-at ) ) + def tool_output_filter(self, line): + if "license was not entered" in line.lower(): + self._bad_input = True + return line + def _process_defects(self, defects): for defect in defects: if not isinstance(defect, DefectItem): @@ -203,6 +208,12 @@ class PvsStudioCheckTool(CheckToolBase): # pylint: disable=too-many-instance-at if os.path.isdir(self._tmp_dir): shutil.rmtree(self._tmp_dir) + @staticmethod + def is_check_successful(cmd_result): + return ( + "license" not in cmd_result["err"].lower() and cmd_result["returncode"] == 0 + ) + def check(self, on_defect_callback=None): self._on_defect_callback = on_defect_callback for scope, files in self.get_project_target_files( @@ -219,11 +230,8 @@ class PvsStudioCheckTool(CheckToolBase): # pylint: disable=too-many-instance-at self._bad_input = True continue - result = proc.exec_command(cmd) - # pylint: disable=unsupported-membership-test - if result["returncode"] != 0 or "license" in result["err"].lower(): - self._bad_input = True - click.echo(result["err"]) + result = self.execute_check_cmd(cmd) + if result["returncode"] != 0: continue self._process_defects(self.parse_defects(self._tmp_output_file)) diff --git a/platformio/commands/package.py b/platformio/commands/package.py index bda36570..0013d158 100644 --- a/platformio/commands/package.py +++ b/platformio/commands/package.py @@ -23,6 +23,7 @@ from platformio.clients.registry import RegistryClient from platformio.compat import ensure_python3 from platformio.package.meta import PackageSpec, PackageType from platformio.package.pack import PackagePacker +from platformio.package.unpack import FileUnpacker, TARArchiver def validate_datetime(ctx, param, value): # pylint: disable=unused-argument @@ -81,6 +82,17 @@ def package_pack(package, output): ) def package_publish(package, owner, released_at, private, notify): assert ensure_python3() + + # publish .tar.gz instantly without repacking + if not os.path.isdir(package) and isinstance( + FileUnpacker.new_archiver(package), TARArchiver + ): + response = RegistryClient().publish_package( + package, owner, released_at, private, notify + ) + click.secho(response.get("message"), fg="green") + return + with tempfile.TemporaryDirectory() as tmp_dir: # pylint: disable=no-member with fs.cd(tmp_dir): p = PackagePacker(package) diff --git a/platformio/compat.py b/platformio/compat.py index 1ef2883b..974bdf2f 100644 --- a/platformio/compat.py +++ b/platformio/compat.py @@ -62,10 +62,11 @@ def ci_strings_are_equal(a, b): def ensure_python3(raise_exception=True): - if not raise_exception or not PY2: - return not PY2 + compatible = sys.version_info >= (3, 6) + if not raise_exception or compatible: + return compatible raise UserSideException( - "Python 3.5 or later is required for this operation. \n" + "Python 3.6 or later is required for this operation. \n" "Please install the latest Python 3 and reinstall PlatformIO Core using " "installation script:\n" "https://docs.platformio.org/page/core/installation.html" diff --git a/platformio/ide/tpls/emacs/.ccls.tpl b/platformio/ide/tpls/emacs/.ccls.tpl index 232d2f8d..53c4afeb 100644 --- a/platformio/ide/tpls/emacs/.ccls.tpl +++ b/platformio/ide/tpls/emacs/.ccls.tpl @@ -1,22 +1,12 @@ -% import re -% STD_RE = re.compile(r"\-std=[a-z\+]+(\w+)") -% cc_stds = STD_RE.findall(cc_flags) -% cxx_stds = STD_RE.findall(cxx_flags) -% -% {{ cxx_path }} -% if cc_stds: -{{"%c"}} -std=c{{ cc_stds[-1] }} -% end -% if cxx_stds: -{{"%cpp"}} -std=c++{{ cxx_stds[-1] }} -% end +{{"%c"}} {{ !cc_flags }} +{{"%cpp"}} {{ !cxx_flags }} % for include in filter_includes(includes): --I{{ include }} +-I{{ !include }} % end % for define in defines: --D{{ define }} +-D{{ !define }} % end diff --git a/platformio/ide/tpls/emacs/.clang_complete.tpl b/platformio/ide/tpls/emacs/.clang_complete.tpl deleted file mode 100644 index 6d8e70ed..00000000 --- a/platformio/ide/tpls/emacs/.clang_complete.tpl +++ /dev/null @@ -1,6 +0,0 @@ -% for include in filter_includes(includes): --I{{include}} -% end -% for define in defines: --D{{!define}} -% end diff --git a/platformio/ide/tpls/sublimetext/.ccls.tpl b/platformio/ide/tpls/sublimetext/.ccls.tpl index 232d2f8d..53c4afeb 100644 --- a/platformio/ide/tpls/sublimetext/.ccls.tpl +++ b/platformio/ide/tpls/sublimetext/.ccls.tpl @@ -1,22 +1,12 @@ -% import re -% STD_RE = re.compile(r"\-std=[a-z\+]+(\w+)") -% cc_stds = STD_RE.findall(cc_flags) -% cxx_stds = STD_RE.findall(cxx_flags) -% -% {{ cxx_path }} -% if cc_stds: -{{"%c"}} -std=c{{ cc_stds[-1] }} -% end -% if cxx_stds: -{{"%cpp"}} -std=c++{{ cxx_stds[-1] }} -% end +{{"%c"}} {{ !cc_flags }} +{{"%cpp"}} {{ !cxx_flags }} % for include in filter_includes(includes): --I{{ include }} +-I{{ !include }} % end % for define in defines: --D{{ define }} +-D{{ !define }} % end diff --git a/platformio/ide/tpls/vim/.ccls.tpl b/platformio/ide/tpls/vim/.ccls.tpl index 39e29f87..53c4afeb 100644 --- a/platformio/ide/tpls/vim/.ccls.tpl +++ b/platformio/ide/tpls/vim/.ccls.tpl @@ -1,20 +1,10 @@ -% import re -% STD_RE = re.compile(r"\-std=[a-z\+]+(\w+)") -% cc_stds = STD_RE.findall(cc_flags) -% cxx_stds = STD_RE.findall(cxx_flags) -% -% -clang +{{ cxx_path }} -% if cc_stds: -{{"%c"}} -std=c{{ cc_stds[-1] }} -% end -% if cxx_stds: -{{"%cpp"}} -std=c++{{ cxx_stds[-1] }} -% end +{{"%c"}} {{ !cc_flags }} +{{"%cpp"}} {{ !cxx_flags }} % for include in filter_includes(includes): --I{{ include }} +-I{{ !include }} % end % for define in defines: diff --git a/platformio/ide/tpls/vim/.clang_complete.tpl b/platformio/ide/tpls/vim/.clang_complete.tpl deleted file mode 100644 index 5db01b75..00000000 --- a/platformio/ide/tpls/vim/.clang_complete.tpl +++ /dev/null @@ -1,6 +0,0 @@ -% for include in filter_includes(includes): --I"{{include}}" -% end -% for define in defines: --D{{!define}} -% end diff --git a/platformio/ide/tpls/vim/.gcc-flags.json.tpl b/platformio/ide/tpls/vim/.gcc-flags.json.tpl deleted file mode 100644 index 153396fe..00000000 --- a/platformio/ide/tpls/vim/.gcc-flags.json.tpl +++ /dev/null @@ -1,9 +0,0 @@ -% _defines = " ".join(["-D%s" % d.replace(" ", "\\\\ ") for d in defines]) -{ - "execPath": "{{ cxx_path }}", - "gccDefaultCFlags": "-fsyntax-only {{! cc_flags.replace(' -MMD ', ' ').replace('"', '\\"') }} {{ !_defines.replace('"', '\\"') }}", - "gccDefaultCppFlags": "-fsyntax-only {{! cxx_flags.replace(' -MMD ', ' ').replace('"', '\\"') }} {{ !_defines.replace('"', '\\"') }}", - "gccErrorLimit": 15, - "gccIncludePaths": "{{ ','.join(filter_includes(includes)) }}", - "gccSuppressWarnings": false -} diff --git a/platformio/ide/tpls/vscode/.vscode/c_cpp_properties.json.tpl b/platformio/ide/tpls/vscode/.vscode/c_cpp_properties.json.tpl index dee165d4..ac525e56 100644 --- a/platformio/ide/tpls/vscode/.vscode/c_cpp_properties.json.tpl +++ b/platformio/ide/tpls/vscode/.vscode/c_cpp_properties.json.tpl @@ -83,20 +83,15 @@ % forced_includes = _find_forced_includes( % filter_args(cc_m_flags, ["-include", "-imacros"]), cleaned_includes) % +// +// !!! WARNING !!! AUTO-GENERATED FILE! +// PLEASE DO NOT MODIFY IT AND USE "platformio.ini": +// https://docs.platformio.org/page/projectconf/section_env_build.html#build-flags +// { "configurations": [ { - "name": "!!! WARNING !!! AUTO-GENERATED FILE, PLEASE DO NOT MODIFY IT AND USE https://docs.platformio.org/page/projectconf/section_env_build.html#build-flags" - }, - { -% if systype == "windows": - "name": "Win32", -% elif systype == "darwin": - "name": "Mac", - "macFrameworkPath": [], -% else: - "name": "Linux", -% end + "name": "PlatformIO", "includePath": [ % for include in cleaned_includes: "{{ include }}", @@ -118,9 +113,6 @@ % end "" ], -% if compiler_type == "gcc": - "intelliSenseMode": "gcc-x64", -% end % if cc_stds: "cStandard": "c{{ cc_stds[-1] }}", % end diff --git a/platformio/maintenance.py b/platformio/maintenance.py index 63674ad7..9d2f8245 100644 --- a/platformio/maintenance.py +++ b/platformio/maintenance.py @@ -27,6 +27,7 @@ from platformio.commands.lib.command import CTX_META_STORAGE_DIRS_KEY from platformio.commands.lib.command import lib_update as cmd_lib_update from platformio.commands.platform import platform_update as cmd_platform_update from platformio.commands.upgrade import get_latest_version +from platformio.compat import ensure_python3 from platformio.package.manager.core import update_core_packages from platformio.package.manager.library import LibraryPackageManager from platformio.package.manager.platform import PlatformPackageManager @@ -43,8 +44,25 @@ def on_platformio_start(ctx, force, caller): set_caller(caller) telemetry.on_command() - if not PlatformioCLI.in_silence(): - after_upgrade(ctx) + if PlatformioCLI.in_silence(): + return + + after_upgrade(ctx) + + if not ensure_python3(raise_exception=False): + click.secho( + """ +Python 2 and Python 3.5 are not compatible with PlatformIO Core 5.0. +Please check the migration guide on how to fix this warning message: +""", + fg="yellow", + ) + click.secho( + "https://docs.platformio.org/en/latest/core/migration.html" + "#drop-support-for-python-2-and-3-5", + fg="blue", + ) + click.echo("") def on_platformio_end(ctx, result): # pylint: disable=unused-argument diff --git a/platformio/package/manager/_install.py b/platformio/package/manager/_install.py index 6cdccf09..11684563 100644 --- a/platformio/package/manager/_install.py +++ b/platformio/package/manager/_install.py @@ -153,7 +153,7 @@ class PackageManagerInstallMixin(object): finally: if os.path.isdir(tmp_dir): try: - shutil.rmtree(tmp_dir) + fs.rmtree(tmp_dir) except: # pylint: disable=bare-except pass diff --git a/platformio/package/manager/_update.py b/platformio/package/manager/_update.py index 3b6dd2d4..1487d0bf 100644 --- a/platformio/package/manager/_update.py +++ b/platformio/package/manager/_update.py @@ -104,7 +104,7 @@ class PackageManagerUpdateMixin(object): outdated = self.outdated(pkg, to_spec) if not silent: - self.print_outdated_state(outdated, show_incompatible) + self.print_outdated_state(outdated, only_check, show_incompatible) if only_check or not outdated.is_outdated(allow_incompatible=False): return pkg @@ -116,24 +116,39 @@ class PackageManagerUpdateMixin(object): self.unlock() @staticmethod - def print_outdated_state(outdated, show_incompatible=True): + def print_outdated_state(outdated, only_check, show_incompatible): if outdated.detached: return click.echo("[%s]" % (click.style("Detached", fg="yellow"))) + if ( not outdated.latest or outdated.current == outdated.latest or (not show_incompatible and outdated.current == outdated.wanted) ): return click.echo("[%s]" % (click.style("Up-to-date", fg="green"))) + if outdated.wanted and outdated.current == outdated.wanted: return click.echo( "[%s]" % (click.style("Incompatible %s" % outdated.latest, fg="yellow")) ) + + if only_check: + return click.echo( + "[%s]" + % ( + click.style( + "Outdated %s" % str(outdated.wanted or outdated.latest), + fg="red", + ) + ) + ) + return click.echo( "[%s]" % ( click.style( - "Outdated %s" % str(outdated.wanted or outdated.latest), fg="red" + "Updating to %s" % str(outdated.wanted or outdated.latest), + fg="green", ) ) ) diff --git a/platformio/package/manifest/schema.py b/platformio/package/manifest/schema.py index fdf2f4bb..a3d940ee 100644 --- a/platformio/package/manifest/schema.py +++ b/platformio/package/manifest/schema.py @@ -143,7 +143,8 @@ class ExampleSchema(StrictSchema): validate=[ validate.Length(min=1, max=255), validate.Regexp( - r"^[a-zA-Z\d\-\_/]+$", error="Only [a-zA-Z0-9-_/] chars are allowed" + r"^[a-zA-Z\d\-\_/\. ]+$", + error="Only [a-zA-Z0-9-_/. ] chars are allowed", ), ], ) diff --git a/platformio/package/pack.py b/platformio/package/pack.py index 164ce96c..66dff1d5 100644 --- a/platformio/package/pack.py +++ b/platformio/package/pack.py @@ -20,8 +20,8 @@ import tarfile import tempfile from platformio import fs -from platformio.compat import ensure_python3 -from platformio.package.exception import PackageException +from platformio.compat import WINDOWS, ensure_python3 +from platformio.package.exception import PackageException, UserSideException from platformio.package.manifest.parser import ManifestFileType, ManifestParserFactory from platformio.package.manifest.schema import ManifestSchema from platformio.package.meta import PackageItem @@ -117,6 +117,12 @@ class PackagePacker(object): # if zip/tar.gz -> unpack to tmp dir if not os.path.isdir(src): + if WINDOWS: + raise UserSideException( + "Packaging from an archive does not work on Windows OS. Please " + "extract data from `%s` manually and pack a folder instead" + % src + ) with FileUnpacker(src) as fu: assert fu.unpack(tmp_dir, silent=True) src = tmp_dir diff --git a/platformio/package/unpack.py b/platformio/package/unpack.py index 9956b46a..2913660d 100644 --- a/platformio/package/unpack.py +++ b/platformio/package/unpack.py @@ -134,27 +134,28 @@ class FileUnpacker(object): self.path = path self._archiver = None - def _init_archiver(self): + def __enter__(self): + self._archiver = self.new_archiver(self.path) + return self + + def __exit__(self, *args): + if self._archiver: + self._archiver.close() + + @staticmethod + def new_archiver(path): magic_map = { b"\x1f\x8b\x08": TARArchiver, b"\x42\x5a\x68": TARArchiver, b"\x50\x4b\x03\x04": ZIPArchiver, } magic_len = max(len(k) for k in magic_map) - with open(self.path, "rb") as fp: + with open(path, "rb") as fp: data = fp.read(magic_len) for magic, archiver in magic_map.items(): if data.startswith(magic): - return archiver(self.path) - raise PackageException("Unknown archive type '%s'" % self.path) - - def __enter__(self): - self._archiver = self._init_archiver() - return self - - def __exit__(self, *args): - if self._archiver: - self._archiver.close() + return archiver(path) + raise PackageException("Unknown archive type '%s'" % path) def unpack( self, dest_dir=None, with_progress=True, check_unpacked=True, silent=False diff --git a/platformio/package/vcsclient.py b/platformio/package/vcsclient.py index 2e9bb238..eb85ae37 100644 --- a/platformio/package/vcsclient.py +++ b/platformio/package/vcsclient.py @@ -12,17 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os import re -from os.path import join -from subprocess import CalledProcessError, check_call -from sys import modules +import subprocess +import sys +from platformio import proc from platformio.package.exception import ( PackageException, PlatformioException, UserSideException, ) -from platformio.proc import exec_command try: from urllib.parse import urlparse @@ -51,7 +51,7 @@ class VCSClientFactory(object): if not type_: raise VCSBaseException("VCS: Unknown repository type %s" % remote_url) try: - obj = getattr(modules[__name__], "%sClient" % type_.title())( + obj = getattr(sys.modules[__name__], "%sClient" % type_.title())( src_dir, remote_url, tag, silent ) assert isinstance(obj, VCSClientBase) @@ -86,7 +86,7 @@ class VCSClientBase(object): @property def storage_dir(self): - return join(self.src_dir, "." + self.command) + return os.path.join(self.src_dir, "." + self.command) def export(self): raise NotImplementedError @@ -108,17 +108,19 @@ class VCSClientBase(object): args = [self.command] + args if "cwd" not in kwargs: kwargs["cwd"] = self.src_dir + if "env" not in kwargs: + kwargs["env"] = os.environ try: - check_call(args, **kwargs) + subprocess.check_call(args, **kwargs) return True - except CalledProcessError as e: + except subprocess.CalledProcessError as e: raise VCSBaseException("VCS: Could not process command %s" % e.cmd) def get_cmd_output(self, args, **kwargs): args = [self.command] + args if "cwd" not in kwargs: kwargs["cwd"] = self.src_dir - result = exec_command(args, **kwargs) + result = proc.exec_command(args, **kwargs) if result["returncode"] == 0: return result["out"].strip() raise VCSBaseException( @@ -129,6 +131,28 @@ class VCSClientBase(object): class GitClient(VCSClientBase): command = "git" + _configured = False + + def __init__(self, *args, **kwargs): + self.configure() + super(GitClient, self).__init__(*args, **kwargs) + + @classmethod + def configure(cls): + if cls._configured: + return True + cls._configured = True + try: + result = proc.exec_command([cls.command, "--exec-path"]) + if result["returncode"] != 0: + return False + path = result["out"].strip() + if path: + proc.append_env_path("PATH", path) + return True + except subprocess.CalledProcessError: + pass + return False def check_client(self): try: @@ -173,7 +197,7 @@ class GitClient(VCSClientBase): if self.tag: args += ["--branch", self.tag] args += [self.remote_url, self.src_dir] - assert self.run_cmd(args) + assert self.run_cmd(args, cwd=os.getcwd()) if is_commit: assert self.run_cmd(["reset", "--hard", self.tag]) return self.run_cmd( diff --git a/platformio/proc.py b/platformio/proc.py index 82f5a9cf..8db7153e 100644 --- a/platformio/proc.py +++ b/platformio/proc.py @@ -20,6 +20,7 @@ from threading import Thread from platformio import exception from platformio.compat import ( + PY2, WINDOWS, get_filesystem_encoding, get_locale_encoding, @@ -125,7 +126,9 @@ def exec_command(*args, **kwargs): result[s[3:]] = kwargs[s].get_buffer() for k, v in result.items(): - if isinstance(result[k], bytes): + if PY2 and isinstance(v, unicode): # pylint: disable=undefined-variable + result[k] = v.encode() + elif not PY2 and isinstance(result[k], bytes): try: result[k] = result[k].decode( get_locale_encoding() or get_filesystem_encoding() @@ -203,3 +206,11 @@ def where_is_program(program, envpath=None): return os.path.join(bin_dir, "%s.exe" % program) return program + + +def append_env_path(name, value): + cur_value = os.environ.get(name) or "" + if cur_value and value in cur_value.split(os.pathsep): + return cur_value + os.environ[name] = os.pathsep.join([cur_value, value]) + return os.environ[name] diff --git a/scripts/install_devplatforms.py b/scripts/install_devplatforms.py index 7252be69..af579e17 100644 --- a/scripts/install_devplatforms.py +++ b/scripts/install_devplatforms.py @@ -26,15 +26,25 @@ import click envvar="PIO_INSTALL_DEVPLATFORMS_IGNORE", help="Ignore names split by comma", ) -def main(desktop, ignore): +@click.option( + "--ownernames", + envvar="PIO_INSTALL_DEVPLATFORMS_OWNERNAMES", + help="Filter dev-platforms by ownernames (split by comma)", +) +def main(desktop, ignore, ownernames): platforms = json.loads( subprocess.check_output( ["platformio", "platform", "search", "--json-output"] ).decode() ) ignore = [n.strip() for n in (ignore or "").split(",") if n.strip()] + ownernames = [n.strip() for n in (ownernames or "").split(",") if n.strip()] for platform in platforms: - skip = [not desktop and platform["forDesktop"], platform["name"] in ignore] + skip = [ + not desktop and platform["forDesktop"], + platform["name"] in ignore, + ownernames and platform["ownername"] not in ownernames, + ] if any(skip): continue subprocess.check_call(["platformio", "platform", "install", platform["name"]]) diff --git a/tests/commands/test_check.py b/tests/commands/test_check.py index 06c133fc..b8c8e65a 100644 --- a/tests/commands/test_check.py +++ b/tests/commands/test_check.py @@ -410,6 +410,22 @@ check_tool = pvs-studio assert style == 0 +def test_check_pvs_studio_fails_without_license(clirunner, tmpdir): + config = DEFAULT_CONFIG + "\ncheck_tool = pvs-studio" + + tmpdir.join("platformio.ini").write(config) + tmpdir.mkdir("src").join("main.c").write(TEST_CODE) + + default_result = clirunner.invoke(cmd_check, ["--project-dir", str(tmpdir)]) + verbose_result = clirunner.invoke(cmd_check, ["--project-dir", str(tmpdir), "-v"]) + + assert default_result.exit_code != 0 + assert "failed to perform check" in default_result.output.lower() + + assert verbose_result.exit_code != 0 + assert "license was not entered" in verbose_result.output.lower() + + def test_check_embedded_platform_all_tools(clirunner, validate_cliresult, tmpdir): config = """ [env:test] diff --git a/tests/test_maintenance.py b/tests/test_maintenance.py index 46bc82c8..0d863b0d 100644 --- a/tests/test_maintenance.py +++ b/tests/test_maintenance.py @@ -91,7 +91,7 @@ def test_check_and_update_libraries(clirunner, isolated_pio_core, validate_clire assert "There are the new updates for libraries (ArduinoJson)" in result.output assert "Please wait while updating libraries" in result.output assert re.search( - r"Updating bblanchon/ArduinoJson\s+6\.12\.0\s+\[Outdated [\d\.]+\]", + r"Updating bblanchon/ArduinoJson\s+6\.12\.0\s+\[Updating to [\d\.]+\]", result.output, ) @@ -143,7 +143,9 @@ def test_check_and_update_platforms(clirunner, isolated_pio_core, validate_clire validate_cliresult(result) assert "There are the new updates for platforms (native)" in result.output assert "Please wait while updating platforms" in result.output - assert re.search(r"Updating native\s+0.0.0\s+\[Outdated [\d\.]+\]", result.output) + assert re.search( + r"Updating native\s+0.0.0\s+\[Updating to [\d\.]+\]", result.output + ) # check updated version result = clirunner.invoke(cli_pio, ["platform", "list", "--json-output"])