diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml index df920e4e..e566e02c 100644 --- a/.github/workflows/core.yml +++ b/.github/workflows/core.yml @@ -33,6 +33,7 @@ jobs: pip install tox - name: Python Lint + if: ${{ matrix.python-version != '3.6' }} run: | tox -e lint diff --git a/.github/workflows/projects.yml b/.github/workflows/projects.yml index 1e21bed1..ad62bc64 100644 --- a/.github/workflows/projects.yml +++ b/.github/workflows/projects.yml @@ -13,11 +13,11 @@ jobs: folder: "Marlin" config_dir: "Marlin" env_name: "mega2560" - - esphome: - repository: "esphome/esphome" - folder: "esphome" - config_dir: "esphome" - env_name: "esp32-arduino" + # - esphome: + # repository: "esphome/esphome" + # folder: "esphome" + # config_dir: "esphome" + # env_name: "esp32-arduino" - smartknob: repository: "scottbez1/smartknob" folder: "smartknob" diff --git a/.pylintrc b/.pylintrc index 3943bac6..0b9ff1f8 100644 --- a/.pylintrc +++ b/.pylintrc @@ -3,21 +3,9 @@ output-format=colorized [MESSAGES CONTROL] disable= - bad-continuation, - bad-whitespace, missing-docstring, - ungrouped-imports, - invalid-name, - cyclic-import, duplicate-code, - superfluous-parens, + invalid-name, too-few-public-methods, - useless-object-inheritance, - useless-import-alias, - bad-option-value, - consider-using-dict-items, consider-using-f-string, - - ; PY2 Compat - super-with-arguments, - raise-missing-from + cyclic-import diff --git a/HISTORY.rst b/HISTORY.rst index e37671e5..7ad9d795 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -13,6 +13,54 @@ PlatformIO Core 6 **A professional collaborative platform for declarative, safety-critical, and test-driven embedded development.** +6.1.0 (2022-07-06) +~~~~~~~~~~~~~~~~~~ + +* **Device Manager** + + - Automatically reconnect device monitor if a connection fails + - Added new `pio device monitor --no-reconnect `__ option to disable automatic reconnection + - Handle device monitor disconnects more gracefully (`issue #3939 `_) + - Improved a serial port finder for `Black Magic Probe `__ (`issue #4023 `_) + - Improved a serial port finder for a board with predefined HWIDs + - Replaced ``monitor_flags`` with independent project configuration options: `monitor_parity `__, `monitor_eol `__, `monitor_raw `__, `monitor_echo `__ + - Fixed an issue when the monitor filters were not applied in their order (`issue #4320 `_) + +* **Unit Testing** + + - Updated "Getting Started" documentation for `GoogleTest `__ testing and mocking framework + - Export |UNITTESTING| flags only to the project build environment (``projenv``, files in "src" folder) + - Merged the "building" stage with "uploading" for the embedded target (`issue #4307 `_) + - Do not resolve dependencies from the project "src" folder when the `test_build_src `__ option is not enabled + - Do not immediately terminate a testing program when results are received + - Fixed an issue when a custom `pio test --project-config `__ was not handled properly (`issue #4299 `_) + - Fixed an issue when testing results were wrong in the verbose mode (`issue #4336 `_) + +* **Build System** + + - Significantly improved support for `Pre & Post Actions `__ + + * Allowed to declare actions in the `PRE-type scripts `__ even if the target is not ready yet + * Allowed library maintainers to use Pre & Post Actions in the library `extraScript `__ + + - Documented `Stringification `__ – converting a macro argument into a string constant (`issue #4310 `_) + - Added new `pio run --monitor-port `__ option to specify custom device monitor port to the ``monitor`` target (`issue #4337 `_) + - Added ``env.StringifyMacro(value)`` helper function for the `Advanced Scripting `__ + - Allowed to ``Import("projenv")`` in a library extra script (`issue #4305 `_) + - Fixed an issue when the `build_unflags `__ operation ignores a flag value (`issue #4309 `_) + - Fixed an issue when the `build_unflags `__ option was not applied to the ``ASPPFLAGS`` scope + - Fixed an issue on Windows OS when flags were wrapped to the temporary file while generating the `Compilation database "compile_commands.json" `__ + - Fixed an issue with the |LDF| when recursively scanning dependencies in the ``chain`` mode + - Fixed a "PermissionError" on Windows when running "clean" or "cleanall" targets (`issue #4331 `_) + +* **Package Management** + + - Fixed an issue when library dependencies were installed for the incompatible project environment (`issue #4338 `_) + +* **Miscellaneous** + + - Warn about incompatible Bash version for the `Shell Completion `__ (`issue #4326 `_) + 6.0.2 (2022-06-01) ~~~~~~~~~~~~~~~~~~ diff --git a/Makefile b/Makefile index 8b280334..09c0b838 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ lint: - pylint -j 6 --rcfile=./.pylintrc ./tests - pylint -j 6 --rcfile=./.pylintrc ./platformio + pylint --rcfile=./.pylintrc ./tests + pylint --rcfile=./.pylintrc ./platformio isort: isort ./platformio diff --git a/docs b/docs index 300060ea..f5958b87 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 300060ea08be494465b03b427186bee66eda1766 +Subproject commit f5958b875629eac7b9b95932d524952731e79480 diff --git a/examples b/examples index 6c52fd32..7fbb0ec1 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit 6c52fd327753f2ca14b575bd8719674b479e1181 +Subproject commit 7fbb0ec1532e98af213ffd242d725a8cde1061f8 diff --git a/platformio/__init__.py b/platformio/__init__.py index 7fe4bec7..1ceb17a7 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (6, 0, 2) +VERSION = (6, 1, 0) __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" @@ -47,7 +47,7 @@ __pioremote_endpoint__ = "ssl:host=remote.platformio.org:port=4413" __default_requests_timeout__ = (10, None) # (connect, read) __core_packages__ = { - "contrib-piohome": "~3.4.1", + "contrib-piohome": "~3.4.2", "contrib-pysite": "~2.%d%d.0" % (sys.version_info.major, sys.version_info.minor), "tool-scons": "~4.40300.0", "tool-cppcheck": "~1.270.0", diff --git a/platformio/__main__.py b/platformio/__main__.py index c7f8e245..6a1cda94 100644 --- a/platformio/__main__.py +++ b/platformio/__main__.py @@ -100,15 +100,15 @@ def main(argv=None): ensure_python3(raise_exception=True) configure() cli() # pylint: disable=no-value-for-parameter - except SystemExit as e: - if e.code and str(e.code).isdigit(): - exit_code = int(e.code) - except Exception as e: # pylint: disable=broad-except - if not isinstance(e, exception.ReturnErrorCode): - maintenance.on_platformio_exception(e) + except SystemExit as exc: + if exc.code and str(exc.code).isdigit(): + exit_code = int(exc.code) + except Exception as exc: # pylint: disable=broad-except + if not isinstance(exc, exception.ReturnErrorCode): + maintenance.on_platformio_exception(exc) error_str = "Error: " - if isinstance(e, exception.PlatformioException): - error_str += str(e) + if isinstance(exc, exception.PlatformioException): + error_str += str(exc) else: error_str += format_exc() error_str += """ @@ -128,7 +128,7 @@ An unexpected error occurred. Further steps: ============================================================ """ click.secho(error_str, fg="red", err=True) - exit_code = int(str(e)) if str(e).isdigit() else 1 + exit_code = int(str(exc)) if str(exc).isdigit() else 1 sys.argv = prev_sys_argv return exit_code diff --git a/platformio/account/client.py b/platformio/account/client.py index a2eb0c28..7aabb24d 100644 --- a/platformio/account/client.py +++ b/platformio/account/client.py @@ -46,8 +46,8 @@ class AccountClient(HTTPClient): # pylint:disable=too-many-public-methods def get_refresh_token(): try: return app.get_state_item("account").get("auth").get("refresh_token") - except: # pylint:disable=bare-except - raise AccountNotAuthorized() + except Exception as exc: + raise AccountNotAuthorized() from exc @staticmethod def delete_local_session(): diff --git a/platformio/account/team/commands/list.py b/platformio/account/team/commands/list.py index e41e872d..d0395d7c 100644 --- a/platformio/account/team/commands/list.py +++ b/platformio/account/team/commands/list.py @@ -37,8 +37,8 @@ def team_list_cmd(orgname, json_output): return click.echo(json.dumps(data[orgname] if orgname else data)) if not any(data.values()): return click.secho("You do not have any teams.", fg="yellow") - for org_name in data: - for team in data[org_name]: + for org_name, teams in data.items(): + for team in teams: click.echo() click.secho("%s:%s" % (org_name, team.get("name")), fg="cyan") click.echo("-" * len("%s:%s" % (org_name, team.get("name")))) diff --git a/platformio/app.py b/platformio/app.py index 26b4c80b..b2c78bc1 100644 --- a/platformio/app.py +++ b/platformio/app.py @@ -70,7 +70,7 @@ SESSION_VARS = { } -class State(object): +class State: def __init__(self, path=None, lock=False): self.path = path self.lock = lock @@ -103,8 +103,10 @@ class State(object): try: with open(self.path, mode="w", encoding="utf8") as fp: fp.write(json.dumps(self._storage)) - except IOError: - raise exception.HomeDirPermissionsError(os.path.dirname(self.path)) + except IOError as exc: + raise exception.HomeDirPermissionsError( + os.path.dirname(self.path) + ) from exc self._unlock_state_file() def _lock_state_file(self): @@ -113,8 +115,8 @@ class State(object): self._lockfile = LockFile(self.path) try: self._lockfile.acquire() - except IOError: - raise exception.HomeDirPermissionsError(os.path.dirname(self.path)) + except IOError as exc: + raise exception.HomeDirPermissionsError(os.path.dirname(self.path)) from exc def _unlock_state_file(self): if hasattr(self, "_lockfile") and self._lockfile: @@ -169,8 +171,8 @@ def sanitize_setting(name, value): value = str(value).lower() in ("true", "yes", "y", "1") elif isinstance(defdata["value"], int): value = int(value) - except Exception: - raise exception.InvalidSettingValue(value, name) + except Exception as exc: + raise exception.InvalidSettingValue(value, name) from exc return value diff --git a/platformio/builder/main.py b/platformio/builder/main.py index 6aeee4f4..74f6acd3 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -28,7 +28,7 @@ from SCons.Script import DefaultEnvironment # pylint: disable=import-error from SCons.Script import Import # pylint: disable=import-error from SCons.Script import Variables # pylint: disable=import-error -from platformio import compat, fs +from platformio import app, compat, fs from platformio.platform.base import PlatformBase from platformio.proc import get_pythonexe_path from platformio.project.helpers import get_project_dir @@ -53,19 +53,20 @@ DEFAULT_ENV_OPTIONS = dict( "cc", "c++", "link", + "piohooks", "pioasm", "platformio", "pioproject", "pioplatform", "piotest", "piotarget", - "piomaxlen", "piolib", "pioupload", "piosize", "pioino", "piomisc", "piointegration", + "piomaxlen", ], toolpath=[os.path.join(fs.get_source_dir(), "builder", "tools")], variables=clivars, @@ -78,7 +79,8 @@ DEFAULT_ENV_OPTIONS = dict( COMPILATIONDB_PATH=os.path.join("$PROJECT_DIR", "compile_commands.json"), LIBPATH=["$BUILD_DIR"], PROGNAME="program", - PROG_PATH=os.path.join("$BUILD_DIR", "$PROGNAME$PROGSUFFIX"), + PROGPATH=os.path.join("$BUILD_DIR", "$PROGNAME$PROGSUFFIX"), + PROG_PATH="$PROGPATH", # deprecated PYTHONEXE=get_pythonexe_path(), IDE_EXTRA_DATA={}, ) @@ -110,6 +112,8 @@ env.Replace( # Setup project optional directories config = env.GetProjectConfig() +app.set_session_var("custom_project_conf", config.path) + env.Replace( PROJECT_DIR=get_project_dir(), PROJECT_CORE_DIR=config.get("platformio", "core_dir"), @@ -197,7 +201,7 @@ for item in env.GetExtraScripts("post"): if env.get("SIZETOOL") and not ( set(["nobuild", "sizedata"]) & set(COMMAND_LINE_TARGETS) ): - env.Depends(["upload", "program"], "checkprogsize") + env.Depends("upload", "checkprogsize") # Replace platform's "size" target with our _new_targets = [t for t in DEFAULT_TARGETS if str(t) != "size"] Default(None) @@ -209,7 +213,7 @@ if "compiledb" in COMMAND_LINE_TARGETS: # Print configured protocols env.AddPreAction( - ["upload", "program"], + "upload", env.VerboseAction( lambda source, target, env: env.PrintUploadInfo(), "Configuring upload protocol...", @@ -219,6 +223,8 @@ env.AddPreAction( AlwaysBuild(env.Alias("__debug", DEFAULT_TARGETS)) AlwaysBuild(env.Alias("__test", DEFAULT_TARGETS)) +env.ProcessDelayedActions() + ############################################################################## if "envdump" in COMMAND_LINE_TARGETS: diff --git a/platformio/builder/tools/piohooks.py b/platformio/builder/tools/piohooks.py new file mode 100644 index 00000000..acd43fbf --- /dev/null +++ b/platformio/builder/tools/piohooks.py @@ -0,0 +1,52 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import + + +def AddActionWrapper(handler): + def wraps(env, files, action): + if not isinstance(files, (list, tuple, set)): + files = [files] + known_nodes = [] + unknown_files = [] + for item in files: + nodes = env.arg2nodes(item, env.fs.Entry) + if nodes and nodes[0].exists(): + known_nodes.extend(nodes) + else: + unknown_files.append(item) + if unknown_files: + env.Append(**{"_PIO_DELAYED_ACTIONS": [(handler, unknown_files, action)]}) + if known_nodes: + return handler(known_nodes, action) + return [] + + return wraps + + +def ProcessDelayedActions(env): + for func, nodes, action in env.get("_PIO_DELAYED_ACTIONS", []): + func(nodes, action) + + +def generate(env): + env.Replace(**{"_PIO_DELAYED_ACTIONS": []}) + env.AddMethod(AddActionWrapper(env.AddPreAction), "AddPreAction") + env.AddMethod(AddActionWrapper(env.AddPostAction), "AddPostAction") + env.AddMethod(ProcessDelayedActions) + + +def exists(_): + return True diff --git a/platformio/builder/tools/pioino.py b/platformio/builder/tools/pioino.py index 0c1f59a2..453d1f51 100644 --- a/platformio/builder/tools/pioino.py +++ b/platformio/builder/tools/pioino.py @@ -26,7 +26,7 @@ import click from platformio.compat import get_filesystem_encoding, get_locale_encoding -class InoToCPPConverter(object): +class InoToCPPConverter: PROTOTYPE_RE = re.compile( r"""^( diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index 104e8496..c830c881 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -12,9 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -# pylint: disable=no-self-use, unused-argument, too-many-lines # pylint: disable=too-many-instance-attributes, too-many-public-methods -# pylint: disable=assignment-from-no-return +# pylint: disable=assignment-from-no-return, unused-argument, too-many-lines from __future__ import absolute_import @@ -29,7 +28,7 @@ import SCons.Scanner # pylint: disable=import-error from SCons.Script import ARGUMENTS # pylint: disable=import-error from SCons.Script import DefaultEnvironment # pylint: disable=import-error -from platformio import exception, fs, util +from platformio import exception, fs from platformio.builder.tools import platformio as piotool from platformio.compat import IS_WINDOWS, hashlib_encode_data, string_types from platformio.http import HTTPClientError, InternetIsOffline @@ -42,11 +41,11 @@ from platformio.package.manifest.parser import ( ManifestParserError, ManifestParserFactory, ) -from platformio.package.meta import PackageItem +from platformio.package.meta import PackageCompatibility, PackageItem from platformio.project.options import ProjectOptions -class LibBuilderFactory(object): +class LibBuilderFactory: @staticmethod def new(env, path, verbose=int(ARGUMENTS.get("PIOVERBOSE", 0))): clsname = "UnknownLibBuilder" @@ -318,19 +317,12 @@ class LibBuilderBase: ) def get_search_files(self): - items = [ + return [ os.path.join(self.src_dir, item) - for item in self.env.MatchSourceFiles(self.src_dir, self.src_filter) - ] - include_dir = self.include_dir - if include_dir: - items.extend( - [ - os.path.join(include_dir, item) - for item in self.env.MatchSourceFiles(include_dir) - ] + for item in self.env.MatchSourceFiles( + self.src_dir, self.src_filter, piotool.SRC_BUILD_EXT ) - return items + ] def _get_found_includes( # pylint: disable=too-many-branches self, search_files=None @@ -340,7 +332,7 @@ class LibBuilderBase: LibBuilderBase._INCLUDE_DIRS_CACHE = [ self.env.Dir(d) for d in ProjectAsLibBuilder( - self.envorigin, "$PROJECT_DIR" + self.envorigin, "$PROJECT_DIR", export_projenv=False ).get_include_dirs() ] for lb in self.env.GetLibBuilders(): @@ -366,24 +358,28 @@ class LibBuilderBase: tuple(include_dirs), depth=self.CCONDITIONAL_SCANNER_DEPTH, ) - # mark candidates already processed via Conditional Scanner - self._processed_files.extend( - [ - c.get_abspath() - for c in candidates - if c.get_abspath() not in self._processed_files - ] - ) - except Exception as e: # pylint: disable=broad-except + + except Exception as exc: # pylint: disable=broad-except if self.verbose and "+" in self.lib_ldf_mode: sys.stderr.write( "Warning! Classic Pre Processor is used for `%s`, " - "advanced has failed with `%s`\n" % (path, e) + "advanced has failed with `%s`\n" % (path, exc) ) - candidates = LibBuilderBase.CLASSIC_SCANNER( - self.env.File(path), self.env, tuple(include_dirs) + candidates = self.env.File(path).get_implicit_deps( + self.env, + LibBuilderBase.CLASSIC_SCANNER, + lambda _: tuple(include_dirs), ) + # mark candidates already processed + self._processed_files.extend( + [ + c.get_abspath() + for c in candidates + if c.get_abspath() not in self._processed_files + ] + ) + # print(path, [c.get_abspath() for c in candidates]) for item in candidates: if item not in result: @@ -415,11 +411,12 @@ class LibBuilderBase: lib_inc_map = {} for inc in self._get_found_includes(search_files): + inc_path = inc.get_abspath() for lb in self.env.GetLibBuilders(): - if inc.get_abspath() in lb: + if inc_path in lb: if lb not in lib_inc_map: lib_inc_map[lb] = [] - lib_inc_map[lb].append(inc.get_abspath()) + lib_inc_map[lb].append(inc_path) break for lb, lb_search_files in lib_inc_map.items(): @@ -585,10 +582,14 @@ class ArduinoLibBuilder(LibBuilderBase): return "chain+" def is_frameworks_compatible(self, frameworks): - return util.items_in_list(frameworks, ["arduino", "energia"]) + return PackageCompatibility(frameworks=frameworks).is_compatible( + PackageCompatibility(frameworks=["arduino", "energia"]) + ) def is_platforms_compatible(self, platforms): - return util.items_in_list(platforms, self._manifest.get("platforms") or ["*"]) + return PackageCompatibility(platforms=platforms).is_compatible( + PackageCompatibility(platforms=self._manifest.get("platforms")) + ) @property def build_flags(self): @@ -643,7 +644,9 @@ class MbedLibBuilder(LibBuilderBase): return include_dirs def is_frameworks_compatible(self, frameworks): - return util.items_in_list(frameworks, ["mbed"]) + return PackageCompatibility(frameworks=frameworks).is_compatible( + PackageCompatibility(frameworks=["mbed"]) + ) def process_extra_options(self): self._process_mbed_lib_confs() @@ -768,6 +771,24 @@ class PlatformIOLibBuilder(LibBuilderBase): return os.path.abspath(self._manifest.get("build").get("includeDir")) return LibBuilderBase.include_dir.fget(self) # pylint: disable=no-member + def get_include_dirs(self): + include_dirs = super().get_include_dirs() + + # backwards compatibility with PlatformIO 2.0 + if ( + "build" not in self._manifest + and self._has_arduino_manifest() + and not os.path.isdir(os.path.join(self.path, "src")) + and os.path.isdir(os.path.join(self.path, "utility")) + ): + include_dirs.append(os.path.join(self.path, "utility")) + + for path in self.env.get("CPPPATH", []): + if path not in self.envorigin.get("CPPPATH", []): + include_dirs.append(self.env.subst(path)) + + return include_dirs + @property def src_dir(self): if "srcDir" in self._manifest.get("build", {}): @@ -838,36 +859,27 @@ class PlatformIOLibBuilder(LibBuilderBase): ) def is_platforms_compatible(self, platforms): - return util.items_in_list(platforms, self._manifest.get("platforms") or ["*"]) + return PackageCompatibility(platforms=platforms).is_compatible( + PackageCompatibility(platforms=self._manifest.get("platforms")) + ) def is_frameworks_compatible(self, frameworks): - return util.items_in_list(frameworks, self._manifest.get("frameworks") or ["*"]) - - def get_include_dirs(self): - include_dirs = super().get_include_dirs() - - # backwards compatibility with PlatformIO 2.0 - if ( - "build" not in self._manifest - and self._has_arduino_manifest() - and not os.path.isdir(os.path.join(self.path, "src")) - and os.path.isdir(os.path.join(self.path, "utility")) - ): - include_dirs.append(os.path.join(self.path, "utility")) - - for path in self.env.get("CPPPATH", []): - if path not in self.envorigin.get("CPPPATH", []): - include_dirs.append(self.env.subst(path)) - - return include_dirs + return PackageCompatibility(frameworks=frameworks).is_compatible( + PackageCompatibility(frameworks=self._manifest.get("frameworks")) + ) class ProjectAsLibBuilder(LibBuilderBase): def __init__(self, env, *args, **kwargs): + export_projenv = kwargs.get("export_projenv", True) + if "export_projenv" in kwargs: + del kwargs["export_projenv"] # backup original value, will be reset in base.__init__ project_src_filter = env.get("SRC_FILTER") super().__init__(env, *args, **kwargs) self.env["SRC_FILTER"] = project_src_filter + if export_projenv: + env.Export(dict(projenv=self.env)) @property def include_dir(self): @@ -878,21 +890,14 @@ class ProjectAsLibBuilder(LibBuilderBase): def src_dir(self): return self.env.subst("$PROJECT_SRC_DIR") - def get_include_dirs(self): - include_dirs = [] - project_include_dir = self.env.subst("$PROJECT_INCLUDE_DIR") - if os.path.isdir(project_include_dir): - include_dirs.append(project_include_dir) - for include_dir in super().get_include_dirs(): - if include_dir not in include_dirs: - include_dirs.append(include_dir) - return include_dirs - def get_search_files(self): + items = [] + build_type = self.env.GetBuildType() # project files - items = super().get_search_files() + if "test" not in build_type or self.env.GetProjectOption("test_build_src"): + items.extend(super().get_search_files()) # test files - if "test" in self.env.GetBuildType(): + if "test" in build_type: items.extend( [ os.path.join("$PROJECT_TEST_DIR", item) @@ -960,8 +965,8 @@ class ProjectAsLibBuilder(LibBuilderBase): try: lm.install(spec) did_install = True - except (HTTPClientError, UnknownPackageError, InternetIsOffline) as e: - click.secho("Warning! %s" % e, fg="yellow") + except (HTTPClientError, UnknownPackageError, InternetIsOffline) as exc: + click.secho("Warning! %s" % exc, fg="yellow") # reset cache if did_install: @@ -1139,6 +1144,10 @@ def ConfigureProjectLibBuilder(env): _print_deps_tree(lb, level + 1) project = ProjectAsLibBuilder(env, "$PROJECT_DIR") + + if "test" in env.GetBuildType(): + project.env.ConfigureTestTarget() + ldf_mode = LibBuilderBase.lib_ldf_mode.fget(project) # pylint: disable=no-member click.echo("LDF: Library Dependency Finder -> https://bit.ly/configure-pio-ldf") diff --git a/platformio/builder/tools/piomaxlen.py b/platformio/builder/tools/piomaxlen.py index c7360418..b6a0ebb0 100644 --- a/platformio/builder/tools/piomaxlen.py +++ b/platformio/builder/tools/piomaxlen.py @@ -19,6 +19,7 @@ import os import re from SCons.Platform import TempFileMunge # pylint: disable=import-error +from SCons.Script import COMMAND_LINE_TARGETS # pylint: disable=import-error from SCons.Subst import quote_spaces # pylint: disable=import-error from platformio.compat import IS_WINDOWS, hashlib_encode_data @@ -70,11 +71,13 @@ def _file_long_data(env, data): return tmp_file -def exists(_): - return True +def exists(env): + return "compiledb" not in COMMAND_LINE_TARGETS and not env.IsIntegrationDump() def generate(env): + if not exists(env): + return env kwargs = dict( _long_sources_hook=long_sources_hook, TEMPFILE=TempFileMunge, diff --git a/platformio/builder/tools/pioplatform.py b/platformio/builder/tools/pioplatform.py index c8c2785f..d6e71e58 100644 --- a/platformio/builder/tools/pioplatform.py +++ b/platformio/builder/tools/pioplatform.py @@ -51,8 +51,8 @@ def BoardConfig(env, board=None): board = board or env.get("BOARD") assert board, "BoardConfig: Board is not defined" return p.board_config(board) - except (AssertionError, UnknownBoard) as e: - sys.stderr.write("Error: %s\n" % str(e)) + except (AssertionError, UnknownBoard) as exc: + sys.stderr.write("Error: %s\n" % str(exc)) env.Exit(1) return None diff --git a/platformio/builder/tools/piotarget.py b/platformio/builder/tools/piotarget.py index 6ebe1f99..4d7c3a74 100644 --- a/platformio/builder/tools/piotarget.py +++ b/platformio/builder/tools/piotarget.py @@ -43,26 +43,18 @@ def PioClean(env, clean_all=False): def _clean_dir(path): clean_rel_path = _relpath(path) - for root, _, files in os.walk(path): - for f in files: - dst = os.path.join(root, f) - os.remove(dst) - print( - "Removed %s" - % (dst if not clean_rel_path.startswith(".") else _relpath(dst)) - ) + print(f"Removing {clean_rel_path}") + fs.rmtree(path) build_dir = env.subst("$BUILD_DIR") libdeps_dir = env.subst("$PROJECT_LIBDEPS_DIR") if os.path.isdir(build_dir): _clean_dir(build_dir) - fs.rmtree(build_dir) else: print("Build environment is clean") if clean_all and os.path.isdir(libdeps_dir): _clean_dir(libdeps_dir) - fs.rmtree(libdeps_dir) print("Done cleaning") @@ -104,19 +96,6 @@ def DumpTargets(env): t["group"] == "Platform" for t in targets.values() ): targets["upload"] = dict(name="upload", group="Platform", title="Upload") - targets["compiledb"] = dict( - name="compiledb", - title="Compilation Database", - description="Generate compilation database `compile_commands.json`", - group="Advanced", - ) - targets["clean"] = dict(name="clean", title="Clean", group="General") - targets["cleanall"] = dict( - name="cleanall", - title="Clean All", - group="General", - description="Clean a build environment and installed library dependencies", - ) return list(targets.values()) diff --git a/platformio/builder/tools/pioupload.py b/platformio/builder/tools/pioupload.py index 14ee59e1..2229a9fb 100644 --- a/platformio/builder/tools/pioupload.py +++ b/platformio/builder/tools/pioupload.py @@ -27,7 +27,7 @@ from serial import Serial, SerialException from platformio import exception, fs from platformio.device.finder import find_mbed_disk, find_serial_port, is_pattern_port -from platformio.device.list import list_serial_ports +from platformio.device.list.util import list_serial_ports from platformio.proc import exec_command @@ -109,13 +109,14 @@ def AutodetectUploadPort(*args, **kwargs): else: try: fs.ensure_udev_rules() - except exception.InvalidUdevRules as e: - sys.stderr.write("\n%s\n\n" % e) + except exception.InvalidUdevRules as exc: + sys.stderr.write("\n%s\n\n" % exc) env.Replace( UPLOAD_PORT=find_serial_port( initial_port=initial_port, board_config=env.BoardConfig() if "BOARD" in env else None, upload_protocol=upload_protocol, + prefer_gdb_port="blackmagic" in upload_protocol, ) ) diff --git a/platformio/builder/tools/platformio.py b/platformio/builder/tools/platformio.py index 69875a70..04b1b1d7 100644 --- a/platformio/builder/tools/platformio.py +++ b/platformio/builder/tools/platformio.py @@ -23,7 +23,6 @@ from SCons.Node import FS # pylint: disable=import-error from SCons.Script import COMMAND_LINE_TARGETS # pylint: disable=import-error from SCons.Script import AlwaysBuild # pylint: disable=import-error from SCons.Script import DefaultEnvironment # pylint: disable=import-error -from SCons.Script import Export # pylint: disable=import-error from SCons.Script import SConscript # pylint: disable=import-error from platformio import __version__, fs @@ -76,10 +75,7 @@ def BuildProgram(env): env.Prepend(_LIBFLAGS="-Wl,--start-group ") env.Append(_LIBFLAGS=" -Wl,--end-group") - program = env.Program( - os.path.join("$BUILD_DIR", env.subst("$PROGNAME$PROGSUFFIX")), - env["PIOBUILDFILES"], - ) + program = env.Program(env.subst("$PROGPATH"), env["PIOBUILDFILES"]) env.Replace(PIOMAINPROG=program) AlwaysBuild( @@ -127,8 +123,6 @@ def ProcessProgramDeps(env): if "debug" in env.GetBuildType(): env.ConfigureDebugTarget() - if "test" in env.GetBuildType(): - env.ConfigureTestTarget() # remove specified flags env.ProcessUnFlags(env.get("BUILD_UNFLAGS")) @@ -142,23 +136,22 @@ def ProcessProgramDeps(env): def ProcessProjectDeps(env): - project_lib_builder = env.ConfigureProjectLibBuilder() - projenv = project_lib_builder.env + plb = env.ConfigureProjectLibBuilder() # prepend project libs to the beginning of list - env.Prepend(LIBS=project_lib_builder.build()) + env.Prepend(LIBS=plb.build()) # prepend extra linker related options from libs env.PrependUnique( **{ - key: project_lib_builder.env.get(key) + key: plb.env.get(key) for key in ("LIBS", "LIBPATH", "LINKFLAGS") - if project_lib_builder.env.get(key) + if plb.env.get(key) } ) if "test" in env.GetBuildType(): build_files_before_nums = len(env.get("PIOBUILDFILES", [])) - projenv.BuildSources( + plb.env.BuildSources( "$BUILD_TEST_DIR", "$PROJECT_TEST_DIR", "$PIOTEST_SRC_FILTER" ) if len(env.get("PIOBUILDFILES", [])) - build_files_before_nums < 1: @@ -169,7 +162,7 @@ def ProcessProjectDeps(env): env.Exit(1) if "test" not in env.GetBuildType() or env.GetProjectOption("test_build_src"): - projenv.BuildSources( + plb.env.BuildSources( "$BUILD_SRC_DIR", "$PROJECT_SRC_DIR", env.get("SRC_FILTER") ) @@ -180,8 +173,6 @@ def ProcessProjectDeps(env): ) env.Exit(1) - Export("projenv") - def ParseFlagsExtended(env, flags): # pylint: disable=too-many-branches if not isinstance(flags, list): @@ -246,33 +237,30 @@ def ProcessUnFlags(env, flags): if not flags: return parsed = env.ParseFlagsExtended(flags) - - # get all flags and copy them to each "*FLAGS" variable - all_flags = [] - for key, unflags in parsed.items(): - if key.endswith("FLAGS"): - all_flags.extend(unflags) - for key, unflags in parsed.items(): - if key.endswith("FLAGS"): - parsed[key].extend(all_flags) - - for key, unflags in parsed.items(): - for unflag in unflags: - for current in env.get(key, []): - conditions = [ - unflag == current, - isinstance(current, (tuple, list)) and unflag[0] == current[0], - ] - if any(conditions): - env[key].remove(current) + unflag_scopes = tuple(set(["ASPPFLAGS"] + list(parsed.keys()))) + for scope in unflag_scopes: + for unflags in parsed.values(): + for unflag in unflags: + for current in env.get(scope, []): + conditions = [ + unflag == current, + not isinstance(unflag, (tuple, list)) + and isinstance(current, (tuple, list)) + and unflag == current[0], + ] + if any(conditions): + env[scope].remove(current) -def MatchSourceFiles(env, src_dir, src_filter=None): +def StringifyMacro(env, value): # pylint: disable=unused-argument + return '\\"%s\\"' % value.replace('"', '\\\\\\"') + + +def MatchSourceFiles(env, src_dir, src_filter=None, src_exts=None): src_filter = env.subst(src_filter) if src_filter else None src_filter = src_filter or SRC_FILTER_DEFAULT - return fs.match_src_files( - env.subst(src_dir), src_filter, SRC_BUILD_EXT + SRC_HEADER_EXT - ) + src_exts = src_exts or (SRC_BUILD_EXT + SRC_HEADER_EXT) + return fs.match_src_files(env.subst(src_dir), src_filter, src_exts) def CollectBuildFiles( @@ -285,7 +273,7 @@ def CollectBuildFiles( if src_dir.endswith(os.sep): src_dir = src_dir[:-1] - for item in env.MatchSourceFiles(src_dir, src_filter): + for item in env.MatchSourceFiles(src_dir, src_filter, SRC_BUILD_EXT): _reldir = os.path.dirname(item) _src_dir = os.path.join(src_dir, _reldir) if _reldir else src_dir _var_dir = os.path.join(variant_dir, _reldir) if _reldir else variant_dir @@ -294,8 +282,7 @@ def CollectBuildFiles( variants.append(_var_dir) env.VariantDir(_var_dir, _src_dir, duplicate) - if fs.path_endswith_ext(item, SRC_BUILD_EXT): - sources.append(env.File(os.path.join(_var_dir, os.path.basename(item)))) + sources.append(env.File(os.path.join(_var_dir, os.path.basename(item)))) middlewares = env.get("__PIO_BUILD_MIDDLEWARES") if not middlewares: @@ -371,6 +358,7 @@ def generate(env): env.AddMethod(ParseFlagsExtended) env.AddMethod(ProcessFlags) env.AddMethod(ProcessUnFlags) + env.AddMethod(StringifyMacro) env.AddMethod(MatchSourceFiles) env.AddMethod(CollectBuildFiles) env.AddMethod(AddBuildMiddleware) diff --git a/platformio/cache.py b/platformio/cache.py index e8b7982d..20901545 100644 --- a/platformio/cache.py +++ b/platformio/cache.py @@ -23,7 +23,7 @@ from platformio.package.lockfile import LockFile from platformio.project.helpers import get_project_cache_dir -class ContentCache(object): +class ContentCache: def __init__(self, namespace=None): self.cache_dir = os.path.join(get_project_cache_dir(), namespace or "content") self._db_path = os.path.join(self.cache_dir, "db.data") diff --git a/platformio/check/defect.py b/platformio/check/defect.py index d271ab48..15f9df70 100644 --- a/platformio/check/defect.py +++ b/platformio/check/defect.py @@ -22,7 +22,7 @@ from platformio.project.helpers import get_project_dir # pylint: disable=too-many-arguments -class DefectItem(object): +class DefectItem: SEVERITY_HIGH = 1 SEVERITY_MEDIUM = 2 diff --git a/platformio/check/tools/__init__.py b/platformio/check/tools/__init__.py index 824df5bb..58a9263d 100644 --- a/platformio/check/tools/__init__.py +++ b/platformio/check/tools/__init__.py @@ -18,7 +18,7 @@ from platformio.check.tools.cppcheck import CppcheckCheckTool from platformio.check.tools.pvsstudio import PvsStudioCheckTool -class CheckToolFactory(object): +class CheckToolFactory: @staticmethod def new(tool, project_dir, config, envname, options): cls = None diff --git a/platformio/check/tools/base.py b/platformio/check/tools/base.py index 90c58bc1..d51b3158 100644 --- a/platformio/check/tools/base.py +++ b/platformio/check/tools/base.py @@ -25,7 +25,7 @@ from platformio.package.meta import PackageSpec from platformio.project.helpers import load_build_metadata -class CheckToolBase(object): # pylint: disable=too-many-instance-attributes +class CheckToolBase: # pylint: disable=too-many-instance-attributes def __init__(self, project_dir, config, envname, options): self.config = config self.envname = envname diff --git a/platformio/check/tools/clangtidy.py b/platformio/check/tools/clangtidy.py index 682a2cb7..1dd4165e 100644 --- a/platformio/check/tools/clangtidy.py +++ b/platformio/check/tools/clangtidy.py @@ -67,8 +67,8 @@ class ClangtidyCheckTool(CheckToolBase): project_files = self.get_project_target_files(self.options["patterns"]) src_files = [] - for scope in project_files: - src_files.extend(project_files[scope]) + for items in project_files.values(): + src_files.extend(items) cmd.extend(flags + src_files + ["--"]) cmd.extend( diff --git a/platformio/commands/ci.py b/platformio/commands/ci.py index f1875b23..9364d1e9 100644 --- a/platformio/commands/ci.py +++ b/platformio/commands/ci.py @@ -39,8 +39,8 @@ def validate_path(ctx, param, value): # pylint: disable=unused-argument try: assert invalid_path is None return value - except AssertionError: - raise click.BadParameter("Found invalid path: %s" % invalid_path) + except AssertionError as exc: + raise click.BadParameter("Found invalid path: %s" % invalid_path) from exc @click.command("ci", short_help="Continuous Integration") diff --git a/platformio/commands/device/__init__.py b/platformio/commands/device/__init__.py index 1af0f8d4..d78b0035 100644 --- a/platformio/commands/device/__init__.py +++ b/platformio/commands/device/__init__.py @@ -13,6 +13,6 @@ # limitations under the License. # pylint: disable=unused-import -from platformio.device.filters.base import ( +from platformio.device.monitor.filters.base import ( DeviceMonitorFilterBase as DeviceMonitorFilter, ) diff --git a/platformio/commands/lib/command.py b/platformio/commands/lib.py similarity index 54% rename from platformio/commands/lib/command.py rename to platformio/commands/lib.py index 3b9af87e..ed3c9e8d 100644 --- a/platformio/commands/lib/command.py +++ b/platformio/commands/lib.py @@ -17,16 +17,18 @@ import json import logging import os -import time -from urllib.parse import quote import click -from tabulate import tabulate -from platformio import exception, fs, util +from platformio import exception, fs from platformio.cli import PlatformioCLI -from platformio.commands.lib.helpers import get_builtin_libs, save_project_libdeps -from platformio.package.exception import NotGlobalLibDir, UnknownPackageError +from platformio.package.commands.install import package_install_cmd +from platformio.package.commands.list import package_list_cmd +from platformio.package.commands.search import package_search_cmd +from platformio.package.commands.show import package_show_cmd +from platformio.package.commands.uninstall import package_uninstall_cmd +from platformio.package.commands.update import package_update_cmd +from platformio.package.exception import NotGlobalLibDir from platformio.package.manager.library import LibraryPackageManager from platformio.package.meta import PackageItem, PackageSpec from platformio.proc import is_ci @@ -43,6 +45,20 @@ def get_project_global_lib_dir(): return ProjectConfig.get_instance().get("platformio", "globallib_dir") +def invoke_command(ctx, cmd, **kwargs): + input_dirs = ctx.meta.get(CTX_META_INPUT_DIRS_KEY, []) + project_environments = ctx.meta[CTX_META_PROJECT_ENVIRONMENTS_KEY] + for input_dir in input_dirs: + cmd_kwargs = kwargs.copy() + if is_platformio_project(input_dir): + cmd_kwargs["project_dir"] = input_dir + cmd_kwargs["environments"] = project_environments + else: + cmd_kwargs["global"] = True + cmd_kwargs["storage_dir"] = input_dir + ctx.invoke(cmd, **cmd_kwargs) + + @click.group(short_help="Library manager", hidden=True) @click.option( "-d", @@ -146,55 +162,14 @@ def lib_install( # pylint: disable=too-many-arguments,unused-argument "the next releases. \nPlease use `pio pkg install` instead.\n", fg="yellow", ) - storage_dirs = ctx.meta[CTX_META_STORAGE_DIRS_KEY] - storage_libdeps = ctx.meta.get(CTX_META_STORAGE_LIBDEPS_KEY, []) - - installed_pkgs = {} - for storage_dir in storage_dirs: - if not silent and (libraries or storage_dir in storage_libdeps): - print_storage_header(storage_dirs, storage_dir) - lm = LibraryPackageManager(storage_dir) - lm.set_log_level(logging.WARN if silent else logging.DEBUG) - - if libraries: - installed_pkgs = { - library: lm.install(library, force=force) for library in libraries - } - - elif storage_dir in storage_libdeps: - for library in storage_libdeps[storage_dir]: - lm.install(library, force=force) - - if save and installed_pkgs: - _save_deps(ctx, installed_pkgs) - - -def _save_deps(ctx, pkgs, action="add"): - specs = [] - for library, pkg in pkgs.items(): - spec = PackageSpec(library) - if spec.external: - specs.append(spec) - else: - specs.append( - PackageSpec( - owner=pkg.metadata.spec.owner, - name=pkg.metadata.spec.name, - requirements=spec.requirements - or ( - ("^%s" % pkg.metadata.version) - if not pkg.metadata.version.build - else pkg.metadata.version - ), - ) - ) - - input_dirs = ctx.meta.get(CTX_META_INPUT_DIRS_KEY, []) - project_environments = ctx.meta[CTX_META_PROJECT_ENVIRONMENTS_KEY] - for input_dir in input_dirs: - if not is_platformio_project(input_dir): - continue - save_project_libdeps(input_dir, specs, project_environments, action=action) + return invoke_command( + ctx, + package_install_cmd, + libraries=libraries, + no_save=not save, + force=force, + silent=silent, + ) @cli.command("uninstall", short_help="Remove libraries") @@ -214,16 +189,13 @@ def lib_uninstall(ctx, libraries, save, silent): "the next releases. \nPlease use `pio pkg uninstall` instead.\n", fg="yellow", ) - storage_dirs = ctx.meta[CTX_META_STORAGE_DIRS_KEY] - uninstalled_pkgs = {} - for storage_dir in storage_dirs: - print_storage_header(storage_dirs, storage_dir) - lm = LibraryPackageManager(storage_dir) - lm.set_log_level(logging.WARN if silent else logging.DEBUG) - uninstalled_pkgs = {library: lm.uninstall(library) for library in libraries} - - if save and uninstalled_pkgs: - _save_deps(ctx, uninstalled_pkgs, action="remove") + invoke_command( + ctx, + package_uninstall_cmd, + libraries=libraries, + no_save=not save, + silent=silent, + ) @cli.command("update", short_help="Update installed libraries") @@ -255,60 +227,51 @@ def lib_update( # pylint: disable=too-many-arguments "the next releases. \nPlease use `pio pkg update` instead.\n", fg="yellow", ) + return invoke_command( + ctx, + package_update_cmd, + libraries=libraries, + silent=silent, + ) storage_dirs = ctx.meta[CTX_META_STORAGE_DIRS_KEY] json_result = {} for storage_dir in storage_dirs: - if not json_output: - print_storage_header(storage_dirs, storage_dir) lib_deps = ctx.meta.get(CTX_META_STORAGE_LIBDEPS_KEY, {}).get(storage_dir, []) lm = LibraryPackageManager(storage_dir) lm.set_log_level(logging.WARN if silent else logging.DEBUG) _libraries = libraries or lib_deps or lm.get_installed() - if only_check and json_output: - result = [] - for library in _libraries: - spec = None - pkg = None - if isinstance(library, PackageItem): - pkg = library - else: - spec = PackageSpec(library) - pkg = lm.get_package(spec) - if not pkg: - continue - outdated = lm.outdated(pkg, spec) - if not outdated.is_outdated(allow_incompatible=True): - continue - manifest = lm.legacy_load_manifest(pkg) - manifest["versionWanted"] = ( - str(outdated.wanted) if outdated.wanted else None - ) - manifest["versionLatest"] = ( - str(outdated.latest) if outdated.latest else None - ) - result.append(manifest) - json_result[storage_dir] = result - else: - for library in _libraries: - to_spec = ( - None if isinstance(library, PackageItem) else PackageSpec(library) - ) - try: - lm.update(library, to_spec=to_spec) - except UnknownPackageError as e: - if library not in lib_deps: - raise e - - if json_output: - return click.echo( - json.dumps( - json_result[storage_dirs[0]] if len(storage_dirs) == 1 else json_result + result = [] + for library in _libraries: + spec = None + pkg = None + if isinstance(library, PackageItem): + pkg = library + else: + spec = PackageSpec(library) + pkg = lm.get_package(spec) + if not pkg: + continue + outdated = lm.outdated(pkg, spec) + if not outdated.is_outdated(allow_incompatible=True): + continue + manifest = lm.legacy_load_manifest(pkg) + manifest["versionWanted"] = ( + str(outdated.wanted) if outdated.wanted else None ) - ) + manifest["versionLatest"] = ( + str(outdated.latest) if outdated.latest else None + ) + result.append(manifest) - return True + json_result[storage_dir] = result + + return click.echo( + json.dumps( + json_result[storage_dirs[0]] if len(storage_dirs) == 1 else json_result + ) + ) @cli.command("list", short_help="List installed libraries") @@ -321,29 +284,18 @@ def lib_list(ctx, json_output): "the next releases. \nPlease use `pio pkg list` instead.\n", fg="yellow", ) + return invoke_command(ctx, package_list_cmd, only_libraries=True) + storage_dirs = ctx.meta[CTX_META_STORAGE_DIRS_KEY] json_result = {} for storage_dir in storage_dirs: - if not json_output: - print_storage_header(storage_dirs, storage_dir) lm = LibraryPackageManager(storage_dir) - items = lm.legacy_get_installed() - if json_output: - json_result[storage_dir] = items - elif items: - for item in sorted(items, key=lambda i: i["name"]): - print_lib_item(item) - else: - click.echo("No items found") - - if json_output: - return click.echo( - json.dumps( - json_result[storage_dirs[0]] if len(storage_dirs) == 1 else json_result - ) + json_result[storage_dir] = lm.legacy_get_installed() + return click.echo( + json.dumps( + json_result[storage_dirs[0]] if len(storage_dirs) == 1 else json_result ) - - return True + ) @cli.command("search", short_help="Search for a library") @@ -363,14 +315,10 @@ def lib_list(ctx, json_output): is_flag=True, help="Do not prompt, automatically paginate with delay", ) -def lib_search(query, json_output, page, noninteractive, **filters): - if not json_output: - click.secho( - "\nWARNING: This command is deprecated and will be removed in " - "the next releases. \nPlease use `pio pkg search` instead.\n", - fg="yellow", - ) - regclient = LibraryPackageManager().get_registry_client_instance() +@click.pass_context +def lib_search( # pylint: disable=unused-argument + ctx, query, json_output, page, noninteractive, **filters +): if not query: query = [] if not isinstance(query, list): @@ -380,72 +328,30 @@ def lib_search(query, json_output, page, noninteractive, **filters): for value in values: query.append('%s:"%s"' % (key, value)) + if not json_output: + click.secho( + "\nWARNING: This command is deprecated and will be removed in " + "the next releases. \nPlease use `pio pkg search` instead.\n", + fg="yellow", + ) + query.append("type:library") + return ctx.invoke(package_search_cmd, query=" ".join(query), page=page) + + regclient = LibraryPackageManager().get_registry_client_instance() result = regclient.fetch_json_data( "get", "/v2/lib/search", params=dict(query=" ".join(query), page=page), x_cache_valid="1d", ) - - if json_output: - click.echo(json.dumps(result)) - return - - if result["total"] == 0: - click.secho( - "Nothing has been found by your request\n" - "Try a less-specific search or use truncation (or wildcard) " - "operator", - fg="yellow", - nl=False, - ) - click.secho(" *", fg="green") - click.secho("For example: DS*, PCA*, DHT* and etc.\n", fg="yellow") - click.echo( - "For more examples and advanced search syntax, please use documentation:" - ) - click.secho( - "https://docs.platformio.org/page/userguide/lib/cmd_search.html\n", - fg="cyan", - ) - return - - click.secho( - "Found %d libraries:\n" % result["total"], - fg="green" if result["total"] else "yellow", - ) - - while True: - for item in result["items"]: - print_lib_item(item) - - if int(result["page"]) * int(result["perpage"]) >= int(result["total"]): - break - - if noninteractive: - click.echo() - click.secho( - "Loading next %d libraries... Press Ctrl+C to stop!" - % result["perpage"], - fg="yellow", - ) - click.echo() - time.sleep(5) - elif not click.confirm("Show next libraries?"): - break - result = regclient.fetch_json_data( - "get", - "/v2/lib/search", - params=dict(query=" ".join(query), page=int(result["page"]) + 1), - x_cache_valid="1d", - ) + return click.echo(json.dumps(result)) @cli.command("builtin", short_help="List built-in libraries") @click.option("--storage", multiple=True) @click.option("--json-output", is_flag=True) def lib_builtin(storage, json_output): - items = get_builtin_libs(storage) + items = LibraryPackageManager.get_builtin_libs(storage) if json_output: return click.echo(json.dumps(items)) @@ -465,13 +371,16 @@ def lib_builtin(storage, json_output): @cli.command("show", short_help="Show detailed info about a library") @click.argument("library", metavar="[LIBRARY]") @click.option("--json-output", is_flag=True) -def lib_show(library, json_output): +@click.pass_context +def lib_show(ctx, library, json_output): if not json_output: click.secho( "\nWARNING: This command is deprecated and will be removed in " "the next releases. \nPlease use `pio pkg show` instead.\n", fg="yellow", ) + return ctx.invoke(package_show_cmd, pkg_type="library", spec=library) + lm = LibraryPackageManager() lm.set_log_level(logging.ERROR if json_output else logging.DEBUG) lib_id = lm.reveal_registry_package_id(library) @@ -479,86 +388,7 @@ def lib_show(library, json_output): lib = regclient.fetch_json_data( "get", "/v2/lib/info/%d" % lib_id, x_cache_valid="1h" ) - if json_output: - return click.echo(json.dumps(lib)) - - title = "{ownername}/{name}".format(**lib) - click.secho(title, fg="cyan") - click.echo("=" * len(title)) - click.echo(lib["description"]) - click.echo() - - click.secho("ID: %d" % lib["id"]) - click.echo( - "Version: %s, released %s" - % ( - lib["version"]["name"], - util.parse_datetime(lib["version"]["released"]).strftime("%c"), - ) - ) - click.echo("Manifest: %s" % lib["confurl"]) - for key in ("homepage", "repository", "license"): - if key not in lib or not lib[key]: - continue - if isinstance(lib[key], list): - click.echo("%s: %s" % (key.capitalize(), ", ".join(lib[key]))) - else: - click.echo("%s: %s" % (key.capitalize(), lib[key])) - - blocks = [] - - _authors = [] - for author in lib.get("authors", []): - _data = [] - for key in ("name", "email", "url", "maintainer"): - if not author.get(key): - continue - if key == "email": - _data.append("<%s>" % author[key]) - elif key == "maintainer": - _data.append("(maintainer)") - else: - _data.append(author[key]) - _authors.append(" ".join(_data)) - if _authors: - blocks.append(("Authors", _authors)) - - blocks.append(("Keywords", lib["keywords"])) - for key in ("frameworks", "platforms"): - if key not in lib or not lib[key]: - continue - blocks.append(("Compatible %s" % key, [i["title"] for i in lib[key]])) - blocks.append(("Headers", lib["headers"])) - blocks.append(("Examples", lib["examples"])) - blocks.append( - ( - "Versions", - [ - "%s, released %s" - % (v["name"], util.parse_datetime(v["released"]).strftime("%c")) - for v in lib["versions"] - ], - ) - ) - blocks.append( - ( - "Unique Downloads", - [ - "Today: %s" % lib["dlstats"]["day"], - "Week: %s" % lib["dlstats"]["week"], - "Month: %s" % lib["dlstats"]["month"], - ], - ) - ) - - for (title, rows) in blocks: - click.echo() - click.secho(title, bold=True) - click.echo("-" * len(title)) - for row in rows: - click.echo(row) - - return True + return click.echo(json.dumps(lib)) @cli.command("register", short_help="Deprecated") @@ -572,76 +402,18 @@ def lib_register(config_url): # pylint: disable=unused-argument @cli.command("stats", short_help="Library Registry Statistics") @click.option("--json-output", is_flag=True) def lib_stats(json_output): + if not json_output: + click.secho( + "\nWARNING: This command is deprecated and will be removed in " + "the next releases. \nPlease visit " + "https://registry.platformio.org\n", + fg="yellow", + ) + return None + regclient = LibraryPackageManager().get_registry_client_instance() result = regclient.fetch_json_data("get", "/v2/lib/stats", x_cache_valid="1h") - - if json_output: - return click.echo(json.dumps(result)) - - for key in ("updated", "added"): - tabular_data = [ - ( - click.style(item["name"], fg="cyan"), - util.parse_datetime(item["date"]).strftime("%c"), - "https://platformio.org/lib/show/%s/%s" - % (item["id"], quote(item["name"])), - ) - for item in result.get(key, []) - ] - table = tabulate( - tabular_data, - headers=[click.style("RECENTLY " + key.upper(), bold=True), "Date", "URL"], - ) - click.echo(table) - click.echo() - - for key in ("lastkeywords", "topkeywords"): - tabular_data = [ - ( - click.style(name, fg="cyan"), - "https://platformio.org/lib/search?query=" + quote("keyword:%s" % name), - ) - for name in result.get(key, []) - ] - table = tabulate( - tabular_data, - headers=[ - click.style( - ("RECENT" if key == "lastkeywords" else "POPULAR") + " KEYWORDS", - bold=True, - ), - "URL", - ], - ) - click.echo(table) - click.echo() - - for key, title in (("dlday", "Today"), ("dlweek", "Week"), ("dlmonth", "Month")): - tabular_data = [ - ( - click.style(item["name"], fg="cyan"), - "https://platformio.org/lib/show/%s/%s" - % (item["id"], quote(item["name"])), - ) - for item in result.get(key, []) - ] - table = tabulate( - tabular_data, - headers=[click.style("FEATURED: " + title.upper(), bold=True), "URL"], - ) - click.echo(table) - click.echo() - - return True - - -def print_storage_header(storage_dirs, storage_dir): - if storage_dirs and storage_dirs[0] != storage_dir: - click.echo("") - click.echo( - click.style("Library Storage: ", bold=True) - + click.style(storage_dir, fg="blue") - ) + return click.echo(json.dumps(result)) def print_lib_item(item): diff --git a/platformio/commands/lib/helpers.py b/platformio/commands/lib/helpers.py deleted file mode 100644 index 9b99cd6c..00000000 --- a/platformio/commands/lib/helpers.py +++ /dev/null @@ -1,104 +0,0 @@ -# Copyright (c) 2014-present PlatformIO -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os - -from platformio import util -from platformio.compat import ci_strings_are_equal -from platformio.package.manager.platform import PlatformPackageManager -from platformio.package.meta import PackageSpec -from platformio.platform.factory import PlatformFactory -from platformio.project.config import ProjectConfig -from platformio.project.exception import InvalidProjectConfError - - -@util.memoized(expire="60s") -def get_builtin_libs(storage_names=None): - # pylint: disable=import-outside-toplevel - from platformio.package.manager.library import LibraryPackageManager - - items = [] - storage_names = storage_names or [] - pm = PlatformPackageManager() - for pkg in pm.get_installed(): - p = PlatformFactory.new(pkg) - for storage in p.get_lib_storages(): - if storage_names and storage["name"] not in storage_names: - continue - lm = LibraryPackageManager(storage["path"]) - items.append( - { - "name": storage["name"], - "path": storage["path"], - "items": lm.legacy_get_installed(), - } - ) - return items - - -def is_builtin_lib(name): - for storage in get_builtin_libs(): - for lib in storage["items"]: - if lib.get("name") == name: - return True - return False - - -def ignore_deps_by_specs(deps, specs): - result = [] - for dep in deps: - depspec = PackageSpec(dep) - if depspec.external: - result.append(dep) - continue - ignore_conditions = [] - for spec in specs: - if depspec.owner: - ignore_conditions.append( - ci_strings_are_equal(depspec.owner, spec.owner) - and ci_strings_are_equal(depspec.name, spec.name) - ) - else: - ignore_conditions.append(ci_strings_are_equal(depspec.name, spec.name)) - if not any(ignore_conditions): - result.append(dep) - return result - - -def save_project_libdeps(project_dir, specs, environments=None, action="add"): - config = ProjectConfig.get_instance(os.path.join(project_dir, "platformio.ini")) - config.validate(environments) - for env in config.envs(): - if environments and env not in environments: - continue - config.expand_interpolations = False - candidates = [] - try: - candidates = ignore_deps_by_specs( - config.get("env:" + env, "lib_deps"), specs - ) - except InvalidProjectConfError: - pass - if action == "add": - candidates.extend(spec.as_dependency() for spec in specs) - if candidates: - result = [] - for item in candidates: - item = item.strip() - if item and item not in result: - result.append(item) - config.set("env:" + env, "lib_deps", result) - elif config.has_option("env:" + env, "lib_deps"): - config.remove_option("env:" + env, "lib_deps") - config.save() diff --git a/platformio/commands/platform.py b/platformio/commands/platform.py index d9e8af2d..2f2ae47c 100644 --- a/platformio/commands/platform.py +++ b/platformio/commands/platform.py @@ -18,9 +18,13 @@ import os import click -from platformio.commands.boards import print_boards from platformio.exception import UserSideException -from platformio.package.exception import UnknownPackageError +from platformio.package.commands.install import package_install_cmd +from platformio.package.commands.list import package_list_cmd +from platformio.package.commands.search import package_search_cmd +from platformio.package.commands.show import package_show_cmd +from platformio.package.commands.uninstall import package_uninstall_cmd +from platformio.package.commands.update import package_update_cmd from platformio.package.manager.platform import PlatformPackageManager from platformio.package.meta import PackageItem, PackageSpec from platformio.package.version import get_original_version @@ -36,13 +40,17 @@ def cli(): @cli.command("search", short_help="Search for development platform") @click.argument("query", required=False) @click.option("--json-output", is_flag=True) -def platform_search(query, json_output): +@click.pass_context +def platform_search(ctx, query, json_output): if not json_output: click.secho( "\nWARNING: This command is deprecated and will be removed in " "the next releases. \nPlease use `pio pkg search` instead.\n", fg="yellow", ) + query = query or "" + return ctx.invoke(package_search_cmd, query=f"type:platform {query}".strip()) + platforms = [] for platform in _get_registry_platforms(): if query == "all": @@ -55,17 +63,23 @@ def platform_search(query, json_output): platform["name"], with_boards=False, expose_packages=False ) ) - - if json_output: - click.echo(json.dumps(platforms)) - else: - _print_platforms(platforms) + click.echo(json.dumps(platforms)) + return None @cli.command("frameworks", short_help="List supported frameworks, SDKs") @click.argument("query", required=False) @click.option("--json-output", is_flag=True) def platform_frameworks(query, json_output): + if not json_output: + click.secho( + "\nWARNING: This command is deprecated and will be removed in " + "the next releases. \nPlease visit https://docs.platformio.org" + "/en/latest/frameworks/index.html\n", + fg="yellow", + ) + return + regclient = PlatformPackageManager().get_registry_client_instance() frameworks = [] for framework in regclient.fetch_json_data( @@ -85,21 +99,21 @@ def platform_frameworks(query, json_output): frameworks.append(framework) frameworks = sorted(frameworks, key=lambda manifest: manifest["name"]) - if json_output: - click.echo(json.dumps(frameworks)) - else: - _print_platforms(frameworks) + click.echo(json.dumps(frameworks)) @cli.command("list", short_help="List installed development platforms") @click.option("--json-output", is_flag=True) -def platform_list(json_output): +@click.pass_context +def platform_list(ctx, json_output): if not json_output: click.secho( "\nWARNING: This command is deprecated and will be removed in " "the next releases. \nPlease use `pio pkg list` instead.\n", fg="yellow", ) + return ctx.invoke(package_list_cmd, **{"global": True, "only_platforms": True}) + platforms = [] pm = PlatformPackageManager() for pkg in pm.get_installed(): @@ -108,81 +122,27 @@ def platform_list(json_output): ) platforms = sorted(platforms, key=lambda manifest: manifest["name"]) - if json_output: - click.echo(json.dumps(platforms)) - else: - _print_platforms(platforms) + click.echo(json.dumps(platforms)) + return None @cli.command("show", short_help="Show details about development platform") @click.argument("platform") @click.option("--json-output", is_flag=True) -def platform_show(platform, json_output): # pylint: disable=too-many-branches +@click.pass_context +def platform_show(ctx, platform, json_output): # pylint: disable=too-many-branches if not json_output: click.secho( "\nWARNING: This command is deprecated and will be removed in " "the next releases. \nPlease use `pio pkg show` instead.\n", fg="yellow", ) + return ctx.invoke(package_show_cmd, pkg_type="platform", spec=platform) + data = _get_platform_data(platform) if not data: raise UnknownPlatform(platform) - if json_output: - return click.echo(json.dumps(data)) - - dep = "{ownername}/{name}".format(**data) if "ownername" in data else data["name"] - click.echo( - "{dep} ~ {title}".format(dep=click.style(dep, fg="cyan"), title=data["title"]) - ) - click.echo("=" * (3 + len(dep + data["title"]))) - click.echo(data["description"]) - click.echo() - if "version" in data: - click.echo("Version: %s" % data["version"]) - if data["homepage"]: - click.echo("Home: %s" % data["homepage"]) - if data["repository"]: - click.echo("Repository: %s" % data["repository"]) - if data["url"]: - click.echo("Vendor: %s" % data["url"]) - if data["license"]: - click.echo("License: %s" % data["license"]) - if data["frameworks"]: - click.echo("Frameworks: %s" % ", ".join(data["frameworks"])) - - if not data["packages"]: - return None - - if not isinstance(data["packages"][0], dict): - click.echo("Packages: %s" % ", ".join(data["packages"])) - else: - click.echo() - click.secho("Packages", bold=True) - click.echo("--------") - for item in data["packages"]: - click.echo() - click.echo("Package %s" % click.style(item["name"], fg="yellow")) - click.echo("-" * (8 + len(item["name"]))) - if item["type"]: - click.echo("Type: %s" % item["type"]) - click.echo("Requirements: %s" % item["requirements"]) - click.echo( - "Installed: %s" % ("Yes" if item.get("version") else "No (optional)") - ) - if "version" in item: - click.echo("Version: %s" % item["version"]) - if "originalVersion" in item: - click.echo("Original version: %s" % item["originalVersion"]) - if "description" in item: - click.echo("Description: %s" % item["description"]) - - if data["boards"]: - click.echo() - click.secho("Boards", bold=True) - click.echo("------") - print_boards(data["boards"]) - - return True + return click.echo(json.dumps(data)) @cli.command("install", short_help="Install new development platform") @@ -198,7 +158,9 @@ def platform_show(platform, json_output): # pylint: disable=too-many-branches is_flag=True, help="Reinstall/redownload dev/platform and its packages if exist", ) -def platform_install( # pylint: disable=too-many-arguments,too-many-locals +@click.pass_context +def platform_install( # pylint: disable=too-many-arguments + ctx, platforms, with_package, without_package, @@ -212,76 +174,37 @@ def platform_install( # pylint: disable=too-many-arguments,too-many-locals "the next releases. \nPlease use `pio pkg install` instead.\n", fg="yellow", ) - - def _find_pkg_names(p, candidates): - result = [] - for candidate in candidates: - found = False - # lookup by package types - for _name, _opts in p.packages.items(): - if _opts.get("type") == candidate: - result.append(_name) - found = True - if ( - p.frameworks - and candidate.startswith("framework-") - and candidate[10:] in p.frameworks - ): - result.append(p.frameworks[candidate[10:]]["package"]) - found = True - if not found: - result.append(candidate) - return result - - pm = PlatformPackageManager() - pm.set_log_level(logging.WARN if silent else logging.DEBUG) - for platform in platforms: - if with_package or without_package or with_all_packages: - pkg = pm.install(platform, skip_dependencies=True) - p = PlatformFactory.new(pkg) - if with_all_packages: - with_package = list(p.packages) - with_package = set(_find_pkg_names(p, with_package or [])) - without_package = set(_find_pkg_names(p, without_package or [])) - upkgs = with_package | without_package - ppkgs = set(p.packages) - if not upkgs.issubset(ppkgs): - raise UnknownPackageError(", ".join(upkgs - ppkgs)) - for name, options in p.packages.items(): - if name in without_package: - continue - if name in with_package or not ( - skip_default_package or options.get("optional", False) - ): - p.pm.install(p.get_package_spec(name), force=force) - else: - pkg = pm.install(platform, skip_dependencies=skip_default_package) - - if pkg and not silent: - click.secho( - "The platform '%s' has been successfully installed!\n" - "The rest of the packages will be installed later " - "depending on your build environment." % platform, - fg="green", - ) + ctx.invoke( + package_install_cmd, + **{ + "global": True, + "platforms": platforms, + "skip_dependencies": ( + not with_all_packages + and (with_package or without_package or skip_default_package) + ), + "silent": silent, + "force": force, + }, + ) @cli.command("uninstall", short_help="Uninstall development platform") @click.argument("platforms", nargs=-1, required=True, metavar="[PLATFORM...]") -def platform_uninstall(platforms): +@click.pass_context +def platform_uninstall(ctx, platforms): click.secho( "\nWARNING: This command is deprecated and will be removed in " "the next releases. \nPlease use `pio pkg uninstall` instead.\n", fg="yellow", ) - pm = PlatformPackageManager() - pm.set_log_level(logging.DEBUG) - for platform in platforms: - if pm.uninstall(platform): - click.secho( - "The platform '%s' has been successfully removed!" % platform, - fg="green", - ) + ctx.invoke( + package_uninstall_cmd, + **{ + "global": True, + "platforms": platforms, + }, + ) @cli.command("update", short_help="Update installed development platforms") @@ -300,9 +223,12 @@ def platform_uninstall(platforms): ) @click.option("-s", "--silent", is_flag=True, help="Suppress progress reporting") @click.option("--json-output", is_flag=True) +@click.pass_context def platform_update( # pylint: disable=too-many-locals, too-many-arguments - platforms, only_check, dry_run, silent, json_output, **_ + ctx, platforms, only_check, dry_run, silent, json_output, **_ ): + only_check = dry_run or only_check + if only_check and not json_output: raise UserSideException( "This command is deprecated, please use `pio pkg outdated` instead" @@ -314,54 +240,42 @@ def platform_update( # pylint: disable=too-many-locals, too-many-arguments "the next releases. \nPlease use `pio pkg update` instead.\n", fg="yellow", ) + return ctx.invoke( + package_update_cmd, + **{ + "global": True, + "platforms": platforms, + "silent": silent, + }, + ) pm = PlatformPackageManager() pm.set_log_level(logging.WARN if silent else logging.DEBUG) platforms = platforms or pm.get_installed() - only_check = dry_run or only_check - - if only_check and json_output: - result = [] - for platform in platforms: - spec = None - pkg = None - if isinstance(platform, PackageItem): - pkg = platform - else: - spec = PackageSpec(platform) - pkg = pm.get_package(spec) - if not pkg: - continue - outdated = pm.outdated(pkg, spec) - if ( - not outdated.is_outdated(allow_incompatible=True) - and not PlatformFactory.new(pkg).are_outdated_packages() - ): - continue - data = _get_installed_platform_data( - pkg, with_boards=False, expose_packages=False - ) - if outdated.is_outdated(allow_incompatible=True): - data["versionLatest"] = ( - str(outdated.latest) if outdated.latest else None - ) - result.append(data) - return click.echo(json.dumps(result)) - + result = [] for platform in platforms: - click.echo( - "Platform %s" - % click.style( - platform.metadata.name - if isinstance(platform, PackageItem) - else platform, - fg="cyan", - ) + spec = None + pkg = None + if isinstance(platform, PackageItem): + pkg = platform + else: + spec = PackageSpec(platform) + pkg = pm.get_package(spec) + if not pkg: + continue + outdated = pm.outdated(pkg, spec) + if ( + not outdated.is_outdated(allow_incompatible=True) + and not PlatformFactory.new(pkg).are_outdated_packages() + ): + continue + data = _get_installed_platform_data( + pkg, with_boards=False, expose_packages=False ) - click.echo("--------") - pm.update(platform) - click.echo() - + if outdated.is_outdated(allow_incompatible=True): + data["versionLatest"] = str(outdated.latest) if outdated.latest else None + result.append(data) + click.echo(json.dumps(result)) return True @@ -370,32 +284,6 @@ def platform_update( # pylint: disable=too-many-locals, too-many-arguments # -def _print_platforms(platforms): - for platform in platforms: - click.echo( - "{name} ~ {title}".format( - name=click.style(platform["name"], fg="cyan"), title=platform["title"] - ) - ) - click.echo("=" * (3 + len(platform["name"] + platform["title"]))) - click.echo(platform["description"]) - click.echo() - if "homepage" in platform: - click.echo("Home: %s" % platform["homepage"]) - if "frameworks" in platform and platform["frameworks"]: - click.echo("Frameworks: %s" % ", ".join(platform["frameworks"])) - if "packages" in platform: - click.echo("Packages: %s" % ", ".join(platform["packages"])) - if "version" in platform: - if "__src_url" in platform: - click.echo( - "Version: %s (%s)" % (platform["version"], platform["__src_url"]) - ) - else: - click.echo("Version: " + platform["version"]) - click.echo() - - def _get_registry_platforms(): regclient = PlatformPackageManager().get_registry_client_instance() return regclient.fetch_json_data("get", "/v2/platforms", x_cache_valid="1d") diff --git a/platformio/commands/upgrade.py b/platformio/commands/upgrade.py index 7664732a..2766dbd5 100644 --- a/platformio/commands/upgrade.py +++ b/platformio/commands/upgrade.py @@ -71,9 +71,9 @@ def cli(dev): click.secho( "Warning! Please restart IDE to affect PIO Home changes", fg="yellow" ) - except Exception as e: # pylint: disable=broad-except + except Exception as exc: if not r: - raise exception.UpgradeError("\n".join([str(cmd), str(e)])) + raise exception.UpgradeError("\n".join([str(cmd), str(exc)])) from exc permission_errors = ("permission denied", "not permitted") if any(m in r["err"].lower() for m in permission_errors) and not IS_WINDOWS: click.secho( @@ -127,8 +127,8 @@ def get_latest_version(): except: # pylint: disable=bare-except pass return get_pypi_latest_version() - except: - raise exception.GetLatestVersionError() + except Exception as exc: + raise exception.GetLatestVersionError() from exc def get_develop_latest_version(): diff --git a/platformio/debug/cli.py b/platformio/debug/cli.py index 01e81f36..64bd1b43 100644 --- a/platformio/debug/cli.py +++ b/platformio/debug/cli.py @@ -129,11 +129,11 @@ def _debug_in_project_dir( try: fs.ensure_udev_rules() - except exception.InvalidUdevRules as e: + except exception.InvalidUdevRules as exc: click.echo( - helpers.escape_gdbmi_stream("~", str(e) + "\n") + helpers.escape_gdbmi_stream("~", str(exc) + "\n") if helpers.is_gdbmi_mode() - else str(e) + "\n", + else str(exc) + "\n", nl=False, ) diff --git a/platformio/debug/config/base.py b/platformio/debug/config/base.py index c53c7d0f..a5867340 100644 --- a/platformio/debug/config/base.py +++ b/platformio/debug/config/base.py @@ -18,14 +18,13 @@ import os from platformio import fs, proc, util from platformio.compat import string_types from platformio.debug.exception import DebugInvalidOptionsError -from platformio.debug.helpers import reveal_debug_port from platformio.project.config import ProjectConfig from platformio.project.helpers import load_build_metadata from platformio.project.options import ProjectOptions class DebugConfigBase: # pylint: disable=too-many-instance-attributes - def __init__(self, platform, project_config, env_name): + def __init__(self, platform, project_config, env_name, port=None): self.platform = platform self.project_config = project_config self.env_name = env_name @@ -49,6 +48,7 @@ class DebugConfigBase: # pylint: disable=too-many-instance-attributes self._load_cmds = None self._port = None + self.port = port self.server = self._configure_server() try: @@ -119,11 +119,9 @@ class DebugConfigBase: # pylint: disable=too-many-instance-attributes @property def port(self): - return reveal_debug_port( + return ( self.env_options.get("debug_port", self.tool_settings.get("port")) - or self._port, - self.tool_name, - self.tool_settings, + or self._port ) @port.setter @@ -205,8 +203,8 @@ class DebugConfigBase: # pylint: disable=too-many-instance-attributes def get_init_script(self, debugger): try: return getattr(self, "%s_INIT_SCRIPT" % debugger.upper()) - except AttributeError: - raise NotImplementedError + except AttributeError as exc: + raise NotImplementedError from exc def reveal_patterns(self, source, recursive=True): program_path = self.program_path or "" diff --git a/platformio/debug/config/blackmagic.py b/platformio/debug/config/blackmagic.py index bfc16246..5a89c2f9 100644 --- a/platformio/debug/config/blackmagic.py +++ b/platformio/debug/config/blackmagic.py @@ -13,6 +13,8 @@ # limitations under the License. from platformio.debug.config.base import DebugConfigBase +from platformio.debug.exception import DebugInvalidOptionsError +from platformio.device.finder import find_serial_port, is_pattern_port class BlackmagicDebugConfig(DebugConfigBase): @@ -47,3 +49,25 @@ while ($busy) end set language auto """ + + @property + def port(self): + # pylint: disable=assignment-from-no-return + initial_port = DebugConfigBase.port.fget(self) + if initial_port and not is_pattern_port(initial_port): + return initial_port + port = find_serial_port( + initial_port, + board_config=self.board_config, + upload_protocol=self.tool_name, + prefer_gdb_port=True, + ) + if port: + return port + raise DebugInvalidOptionsError( + "Please specify `debug_port` for the working environment" + ) + + @port.setter + def port(self, value): + self._port = value diff --git a/platformio/debug/config/factory.py b/platformio/debug/config/factory.py index 4741b800..d7358b92 100644 --- a/platformio/debug/config/factory.py +++ b/platformio/debug/config/factory.py @@ -19,7 +19,7 @@ from platformio.debug.config.generic import GenericDebugConfig from platformio.debug.config.native import NativeDebugConfig -class DebugConfigFactory(object): +class DebugConfigFactory: @staticmethod def get_clsname(name): name = re.sub(r"[^\da-z\_\-]+", "", name, flags=re.I) diff --git a/platformio/debug/config/generic.py b/platformio/debug/config/generic.py index 870aad7b..1f155ecb 100644 --- a/platformio/debug/config/generic.py +++ b/platformio/debug/config/generic.py @@ -34,5 +34,6 @@ $INIT_BREAK """ def __init__(self, *args, **kwargs): + if "port" not in kwargs: + kwargs["port"] = ":3333" super().__init__(*args, **kwargs) - self.port = ":3333" diff --git a/platformio/debug/config/jlink.py b/platformio/debug/config/jlink.py index ed5f9966..03a1bd3a 100644 --- a/platformio/debug/config/jlink.py +++ b/platformio/debug/config/jlink.py @@ -38,8 +38,9 @@ $INIT_BREAK """ def __init__(self, *args, **kwargs): + if "port" not in kwargs: + kwargs["port"] = ":2331" super().__init__(*args, **kwargs) - self.port = ":2331" @property def server_ready_pattern(self): diff --git a/platformio/debug/config/mspdebug.py b/platformio/debug/config/mspdebug.py index 86ee8d6a..09266b3b 100644 --- a/platformio/debug/config/mspdebug.py +++ b/platformio/debug/config/mspdebug.py @@ -32,5 +32,6 @@ $INIT_BREAK """ def __init__(self, *args, **kwargs): + if "port" not in kwargs: + kwargs["port"] = ":2000" super().__init__(*args, **kwargs) - self.port = ":2000" diff --git a/platformio/debug/config/qemu.py b/platformio/debug/config/qemu.py index e272a373..e9a57409 100644 --- a/platformio/debug/config/qemu.py +++ b/platformio/debug/config/qemu.py @@ -33,5 +33,6 @@ $INIT_BREAK """ def __init__(self, *args, **kwargs): + if "port" not in kwargs: + kwargs["port"] = ":1234" super().__init__(*args, **kwargs) - self.port = ":1234" diff --git a/platformio/debug/config/renode.py b/platformio/debug/config/renode.py index 0a4164de..724be407 100644 --- a/platformio/debug/config/renode.py +++ b/platformio/debug/config/renode.py @@ -35,8 +35,9 @@ monitor start """ def __init__(self, *args, **kwargs): + if "port" not in kwargs: + kwargs["port"] = ":3333" super().__init__(*args, **kwargs) - self.port = ":3333" @property def server_ready_pattern(self): diff --git a/platformio/debug/helpers.py b/platformio/debug/helpers.py index f156cf45..a2d89de3 100644 --- a/platformio/debug/helpers.py +++ b/platformio/debug/helpers.py @@ -16,14 +16,12 @@ import os import re import sys import time -from fnmatch import fnmatch from hashlib import sha1 from io import BytesIO from platformio.cli import PlatformioCLI -from platformio.compat import IS_WINDOWS, is_bytes +from platformio.compat import is_bytes from platformio.debug.exception import DebugInvalidOptionsError -from platformio.device.list import list_serial_ports from platformio.run.cli import cli as cmd_run from platformio.run.cli import print_processing_header from platformio.test.helpers import list_test_names @@ -161,44 +159,3 @@ def is_prog_obsolete(prog_path): with open(prog_hash_path, mode="w", encoding="utf8") as fp: fp.write(new_digest) return True - - -def reveal_debug_port(env_debug_port, tool_name, tool_settings): - def _get_pattern(): - if not env_debug_port: - return None - if set(["*", "?", "[", "]"]) & set(env_debug_port): - return env_debug_port - return None - - def _is_match_pattern(port): - pattern = _get_pattern() - if not pattern: - return True - return fnmatch(port, pattern) - - def _look_for_serial_port(hwids): - for item in list_serial_ports(filter_hwid=True): - if not _is_match_pattern(item["port"]): - continue - port = item["port"] - if tool_name.startswith("blackmagic"): - if IS_WINDOWS and port.startswith("COM") and len(port) > 4: - port = "\\\\.\\%s" % port - if "GDB" in item["description"]: - return port - for hwid in hwids: - hwid_str = ("%s:%s" % (hwid[0], hwid[1])).replace("0x", "") - if hwid_str in item["hwid"]: - return port - return None - - if env_debug_port and not _get_pattern(): - return env_debug_port - if not tool_settings.get("require_debug_port"): - return None - - debug_port = _look_for_serial_port(tool_settings.get("hwids", [])) - if not debug_port: - raise DebugInvalidOptionsError("Please specify `debug_port` for environment") - return debug_port diff --git a/platformio/device/cli.py b/platformio/device/cli.py index 5865c43e..13dec793 100644 --- a/platformio/device/cli.py +++ b/platformio/device/cli.py @@ -14,8 +14,8 @@ import click -from platformio.device.commands.list import device_list_cmd -from platformio.device.commands.monitor import device_monitor_cmd +from platformio.device.list.command import device_list_cmd +from platformio.device.monitor.command import device_monitor_cmd @click.group( diff --git a/platformio/device/commands/monitor.py b/platformio/device/commands/monitor.py deleted file mode 100644 index f3d4ace5..00000000 --- a/platformio/device/commands/monitor.py +++ /dev/null @@ -1,184 +0,0 @@ -# Copyright (c) 2014-present PlatformIO -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import sys - -import click -from serial.tools import miniterm - -from platformio import exception, fs -from platformio.device.filters.base import register_filters -from platformio.device.finder import find_serial_port -from platformio.platform.factory import PlatformFactory -from platformio.project.config import ProjectConfig -from platformio.project.exception import NotPlatformIOProjectError -from platformio.project.options import ProjectOptions - - -@click.command("monitor", short_help="Monitor device (Serial/Socket)") -@click.option("--port", "-p", help="Port, a number or a device name") -@click.option( - "--baud", - "-b", - type=int, - help="Set baud rate, default=%d" % ProjectOptions["env.monitor_speed"].default, -) -@click.option( - "--parity", - default="N", - type=click.Choice(["N", "E", "O", "S", "M"]), - help="Set parity, default=N", -) -@click.option("--rtscts", is_flag=True, help="Enable RTS/CTS flow control, default=Off") -@click.option( - "--xonxoff", is_flag=True, help="Enable software flow control, default=Off" -) -@click.option( - "--rts", default=None, type=click.IntRange(0, 1), help="Set initial RTS line state" -) -@click.option( - "--dtr", default=None, type=click.IntRange(0, 1), help="Set initial DTR line state" -) -@click.option("--echo", is_flag=True, help="Enable local echo, default=Off") -@click.option( - "--encoding", - default="UTF-8", - help="Set the encoding for the serial port (e.g. hexlify, " - "Latin1, UTF-8), default: UTF-8", -) -@click.option("--filter", "-f", multiple=True, help="Add filters/text transformations") -@click.option( - "--eol", - default="CRLF", - type=click.Choice(["CR", "LF", "CRLF"]), - help="End of line mode, default=CRLF", -) -@click.option("--raw", is_flag=True, help="Do not apply any encodings/transformations") -@click.option( - "--exit-char", - type=int, - default=3, - help="ASCII code of special character that is used to exit " - "the application, default=3 (Ctrl+C)", -) -@click.option( - "--menu-char", - type=int, - default=20, - help="ASCII code of special character that is used to " - "control miniterm (menu), default=20 (DEC)", -) -@click.option( - "--quiet", - is_flag=True, - help="Diagnostics: suppress non-error messages, default=Off", -) -@click.option( - "-d", - "--project-dir", - default=os.getcwd, - type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True), -) -@click.option( - "-e", - "--environment", - help="Load configuration from `platformio.ini` and specified environment", -) -def device_monitor_cmd(**kwargs): # pylint: disable=too-many-branches - project_options = {} - platform = None - with fs.cd(kwargs["project_dir"]): - try: - project_options = get_project_options(kwargs["environment"]) - kwargs = apply_project_monitor_options(kwargs, project_options) - if "platform" in project_options: - platform = PlatformFactory.new(project_options["platform"]) - except NotPlatformIOProjectError: - pass - register_filters(platform=platform, options=kwargs) - kwargs["port"] = find_serial_port( - initial_port=kwargs["port"], - board_config=platform.board_config(project_options.get("board")) - if platform and project_options.get("board") - else None, - upload_protocol=project_options.get("upload_port"), - ) - - # override system argv with patched options - sys.argv = ["monitor"] + project_options_to_monitor_argv( - kwargs, - project_options, - ignore=("port", "baud", "rts", "dtr", "environment", "project_dir"), - ) - - if not kwargs["quiet"]: - click.echo( - "--- Available filters and text transformations: %s" - % ", ".join(sorted(miniterm.TRANSFORMATIONS.keys())) - ) - click.echo("--- More details at https://bit.ly/pio-monitor-filters") - try: - miniterm.main( - default_port=kwargs["port"], - default_baudrate=kwargs["baud"] - or ProjectOptions["env.monitor_speed"].default, - default_rts=kwargs["rts"], - default_dtr=kwargs["dtr"], - ) - except Exception as e: - raise exception.MinitermException(e) - - -def get_project_options(environment=None): - config = ProjectConfig.get_instance() - config.validate(envs=[environment] if environment else None) - environment = environment or config.get_default_env() - return config.items(env=environment, as_dict=True) - - -def apply_project_monitor_options(cli_options, project_options): - for k in ("port", "speed", "rts", "dtr"): - k2 = "monitor_%s" % k - if k == "speed": - k = "baud" - if cli_options[k] is None and k2 in project_options: - cli_options[k] = project_options[k2] - if k != "port": - cli_options[k] = int(cli_options[k]) - return cli_options - - -def project_options_to_monitor_argv(cli_options, project_options, ignore=None): - confmon_flags = project_options.get("monitor_flags", []) - result = confmon_flags[::] - - for f in project_options.get("monitor_filters", []): - result.extend(["--filter", f]) - - for k, v in cli_options.items(): - if v is None or (ignore and k in ignore): - continue - k = "--" + k.replace("_", "-") - if k in confmon_flags: - continue - if isinstance(v, bool): - if v: - result.append(k) - elif isinstance(v, tuple): - for i in v: - result.extend([k, i]) - else: - result.extend([k, str(v)]) - return result diff --git a/platformio/device/finder.py b/platformio/device/finder.py index 0fe98baa..98aef4f6 100644 --- a/platformio/device/finder.py +++ b/platformio/device/finder.py @@ -15,10 +15,48 @@ import os from fnmatch import fnmatch +import click import serial -from platformio.compat import IS_WINDOWS -from platformio.device.list import list_logical_devices, list_serial_ports +from platformio.compat import IS_MACOS, IS_WINDOWS +from platformio.device.list.util import list_logical_devices, list_serial_ports +from platformio.fs import get_platformio_udev_rules_path +from platformio.package.manager.platform import PlatformPackageManager +from platformio.platform.factory import PlatformFactory +from platformio.util import retry + +BLACK_MAGIC_HWIDS = [ + "1D50:6018", +] + + +def parse_udev_rules_hwids(path): + result = [] + with open(path, mode="r", encoding="utf8") as fp: + for line in fp.readlines(): + line = line.strip() + if not line or line.startswith("#"): + continue + attrs = {} + for attr in line.split(","): + attr = attr.replace("==", "=").replace('"', "").strip() + if "=" not in attr: + continue + name, value = attr.split("=", 1) + attrs[name] = value + hwid = "%s:%s" % ( + attrs.get("ATTRS{idVendor}", "*"), + attrs.get("ATTRS{idProduct}", "*"), + ) + if hwid != "*:*": + result.append(hwid.upper()) + return result + + +def normalize_board_hwid(value): + if isinstance(value, (list, tuple)): + value = ("%s:%s" % (value[0], value[1])).replace("0x", "") + return value.upper() def is_pattern_port(port): @@ -43,52 +81,154 @@ def is_serial_port_ready(port, timeout=1): return False -def find_serial_port( - initial_port, board_config=None, upload_protocol=None, ensure_ready=False +def find_serial_port( # pylint: disable=too-many-arguments + initial_port, + board_config=None, + upload_protocol=None, + ensure_ready=False, + prefer_gdb_port=False, + timeout=2, ): if initial_port: if not is_pattern_port(initial_port): return initial_port return match_serial_port(initial_port) - port = None + if upload_protocol and upload_protocol.startswith("blackmagic"): - port = find_blackmagic_serial_port() - if not port and board_config: - port = find_board_serial_port(board_config) + return find_blackmagic_serial_port(prefer_gdb_port, timeout) + if board_config and board_config.get("build.hwids", []): + return find_board_serial_port(board_config, timeout) + port = find_known_uart_port(ensure_ready, timeout) if port: return port - # pick the last PID:VID USB device - usb_port = None + # pick the best PID:VID USB device + best_port = None for item in list_serial_ports(): if ensure_ready and not is_serial_port_ready(item["port"]): continue port = item["port"] if "VID:PID" in item["hwid"]: - usb_port = port - return usb_port or port + best_port = port + return best_port or port -def find_blackmagic_serial_port(): - for item in list_serial_ports(): - port = item["port"] - if IS_WINDOWS and port.startswith("COM") and len(port) > 4: - port = "\\\\.\\%s" % port - if "GDB" in item["description"]: - return port +def find_blackmagic_serial_port(prefer_gdb_port=False, timeout=0): + try: + + @retry(timeout=timeout) + def wrapper(): + candidates = [] + for item in list_serial_ports(filter_hwid=True): + if ( + not any(hwid in item["hwid"].upper() for hwid in BLACK_MAGIC_HWIDS) + and not "Black Magic" in item["description"] + ): + continue + if ( + IS_WINDOWS + and item["port"].startswith("COM") + and len(item["port"]) > 4 + ): + item["port"] = "\\\\.\\%s" % item["port"] + candidates.append(item) + + if not candidates: + raise retry.RetryNextException() + + for item in candidates: + if ("GDB" if prefer_gdb_port else "UART") in item["description"]: + return item["port"] + if IS_MACOS: + # 1 - GDB, 3 - UART + for item in candidates: + if item["port"].endswith("1" if prefer_gdb_port else "3"): + return item["port"] + + candidates = sorted(candidates, key=lambda item: item["port"]) + return ( + candidates[0] # first port is GDB? + if len(candidates) == 1 or prefer_gdb_port + else candidates[1] + )["port"] + + return wrapper() + except retry.RetryStopException: + pass return None -def find_board_serial_port(board_config): - board_hwids = board_config.get("build.hwids", []) - if not board_hwids: - return None - for item in list_serial_ports(filter_hwid=True): - port = item["port"] - for hwid in board_hwids: - hwid_str = ("%s:%s" % (hwid[0], hwid[1])).replace("0x", "") - if hwid_str in item["hwid"]: - return port +def find_board_serial_port(board_config, timeout=0): + hwids = board_config.get("build.hwids", []) + try: + + @retry(timeout=timeout) + def wrapper(): + for item in list_serial_ports(filter_hwid=True): + hwid = item["hwid"].upper() + for board_hwid in hwids: + if normalize_board_hwid(board_hwid) in hwid: + return item["port"] + raise retry.RetryNextException() + + return wrapper() + except retry.RetryStopException: + pass + + click.secho( + "TimeoutError: Could not automatically find serial port " + "for the `%s` board based on the declared HWIDs=%s" + % (board_config.get("name", "unknown"), hwids), + fg="yellow", + err=True, + ) + + return None + + +def find_known_uart_port(ensure_ready=False, timeout=0): + known_hwids = list(BLACK_MAGIC_HWIDS) + + # load from UDEV rules + udev_rules_path = get_platformio_udev_rules_path() + if os.path.isfile(udev_rules_path): + known_hwids.extend(parse_udev_rules_hwids(udev_rules_path)) + + # load from installed dev-platforms + for platform in PlatformPackageManager().get_installed(): + p = PlatformFactory.new(platform) + for board_config in p.get_boards().values(): + for board_hwid in board_config.get("build.hwids", []): + board_hwid = normalize_board_hwid(board_hwid) + if board_hwid not in known_hwids: + known_hwids.append(board_hwid) + + try: + + @retry(timeout=timeout) + def wrapper(): + for item in list_serial_ports(as_objects=True): + if not item.vid or not item.pid: + continue + hwid = "{:04X}:{:04X}".format(item.vid, item.pid) + for pattern in known_hwids: + if fnmatch(hwid, pattern) and ( + not ensure_ready or is_serial_port_ready(item.device) + ): + return item.device + raise retry.RetryNextException() + + return wrapper() + except retry.RetryStopException: + pass + + click.secho( + "TimeoutError: Could not automatically find serial port " + "based on the known UART bridges", + fg="yellow", + err=True, + ) + return None diff --git a/platformio/commands/lib/__init__.py b/platformio/device/list/__init__.py similarity index 100% rename from platformio/commands/lib/__init__.py rename to platformio/device/list/__init__.py diff --git a/platformio/device/commands/list.py b/platformio/device/list/command.py similarity index 98% rename from platformio/device/commands/list.py rename to platformio/device/list/command.py index 9cd3364f..078371cf 100644 --- a/platformio/device/commands/list.py +++ b/platformio/device/list/command.py @@ -16,7 +16,7 @@ import json import click -from platformio.device.list import ( +from platformio.device.list.util import ( list_logical_devices, list_mdns_services, list_serial_ports, diff --git a/platformio/device/list.py b/platformio/device/list/util.py similarity index 95% rename from platformio/device/list.py rename to platformio/device/list/util.py index 3695f760..2c1d0385 100644 --- a/platformio/device/list.py +++ b/platformio/device/list/util.py @@ -24,12 +24,15 @@ from platformio import __version__, exception, proc from platformio.compat import IS_MACOS, IS_WINDOWS -def list_serial_ports(filter_hwid=False): +def list_serial_ports(filter_hwid=False, as_objects=False): try: # pylint: disable=import-outside-toplevel from serial.tools.list_ports import comports - except ImportError: - raise exception.GetSerialPortsError(os.name) + except ImportError as exc: + raise exception.GetSerialPortsError(os.name) from exc + + if as_objects: + return comports() result = [] for p, d, h in comports(): @@ -81,7 +84,7 @@ def list_logical_devices(): def list_mdns_services(): - class mDNSListener(object): + class mDNSListener: def __init__(self): self._zc = zeroconf.Zeroconf(interfaces=zeroconf.InterfaceChoice.All) self._found_types = [] diff --git a/platformio/device/commands/__init__.py b/platformio/device/monitor/__init__.py similarity index 100% rename from platformio/device/commands/__init__.py rename to platformio/device/monitor/__init__.py diff --git a/platformio/device/monitor/command.py b/platformio/device/monitor/command.py new file mode 100644 index 00000000..44df4043 --- /dev/null +++ b/platformio/device/monitor/command.py @@ -0,0 +1,164 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys + +import click + +from platformio import exception, fs +from platformio.device.finder import find_serial_port +from platformio.device.monitor.filters.base import register_filters +from platformio.device.monitor.terminal import start_terminal +from platformio.platform.factory import PlatformFactory +from platformio.project.config import ProjectConfig +from platformio.project.exception import NotPlatformIOProjectError +from platformio.project.options import ProjectOptions + + +@click.command("monitor", short_help="Monitor device (Serial/Socket)") +@click.option("--port", "-p", help="Port, a number or a device name") +@click.option( + "-b", + "--baud", + type=ProjectOptions["env.monitor_speed"].type, + help="Set baud/speed [default=%d]" % ProjectOptions["env.monitor_speed"].default, +) +@click.option( + "--parity", + type=ProjectOptions["env.monitor_parity"].type, + help="Enable parity checking [default=%s]" + % ProjectOptions["env.monitor_parity"].default, +) +@click.option("--rtscts", is_flag=True, help="Enable RTS/CTS flow control") +@click.option("--xonxoff", is_flag=True, help="Enable software flow control") +@click.option( + "--rts", + type=ProjectOptions["env.monitor_rts"].type, + help="Set initial RTS line state", +) +@click.option( + "--dtr", + type=ProjectOptions["env.monitor_dtr"].type, + help="Set initial DTR line state", +) +@click.option("--echo", is_flag=True, help="Enable local echo") +@click.option( + "--encoding", + default="UTF-8", + show_default=True, + help="Set the encoding for the serial port (e.g. hexlify, Latin1, UTF-8)", +) +@click.option( + "-f", + "--filter", + "filters", + multiple=True, + help="Apply filters/text transformations", +) +@click.option( + "--eol", + type=ProjectOptions["env.monitor_eol"].type, + help="End of line mode [default=%s]" % ProjectOptions["env.monitor_eol"].default, +) +@click.option("--raw", is_flag=True, help=ProjectOptions["env.monitor_raw"].description) +@click.option( + "--exit-char", + type=int, + default=3, + show_default=True, + help="ASCII code of special character that is used to exit " + "the application [default=3 (Ctrl+C)]", +) +@click.option( + "--menu-char", + type=int, + default=20, + help="ASCII code of special character that is used to " + "control terminal (menu) [default=20 (DEC)]", +) +@click.option( + "--quiet", + is_flag=True, + help="Diagnostics: suppress non-error messages", +) +@click.option( + "--no-reconnect", + is_flag=True, + help="Disable automatic reconnection if the established connection fails", +) +@click.option( + "-d", + "--project-dir", + default=os.getcwd, + type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True), +) +@click.option( + "-e", + "--environment", + help="Load configuration from `platformio.ini` and the specified environment", +) +def device_monitor_cmd(**options): + with fs.cd(options["project_dir"]): + platform = None + project_options = {} + try: + project_options = get_project_options(options["environment"]) + if "platform" in project_options: + platform = PlatformFactory.new(project_options["platform"]) + except NotPlatformIOProjectError: + pass + + options = apply_project_monitor_options(options, project_options) + register_filters(platform=platform, options=options) + options["port"] = find_serial_port( + initial_port=options["port"], + board_config=platform.board_config(project_options.get("board")) + if platform and project_options.get("board") + else None, + upload_protocol=project_options.get("upload_protocol"), + ensure_ready=True, + ) + + if options["menu_char"] == options["exit_char"]: + raise exception.UserSideException( + "--exit-char can not be the same as --menu-char" + ) + + start_terminal(options) + + +def get_project_options(environment=None): + config = ProjectConfig.get_instance() + config.validate(envs=[environment] if environment else None) + environment = environment or config.get_default_env() + return config.items(env=environment, as_dict=True) + + +def apply_project_monitor_options(initial_options, project_options): + for option_meta in ProjectOptions.values(): + if option_meta.group != "monitor": + continue + cli_key = option_meta.name.split("_", 1)[1] + if cli_key == "speed": + cli_key = "baud" + # value set from CLI, skip overriding + if initial_options[cli_key] not in (None, (), []) and ( + option_meta.type != click.BOOL or f"--{cli_key}" in sys.argv[1:] + ): + continue + initial_options[cli_key] = project_options.get( + option_meta.name, option_meta.default + ) + return initial_options diff --git a/platformio/device/filters/__init__.py b/platformio/device/monitor/filters/__init__.py similarity index 100% rename from platformio/device/filters/__init__.py rename to platformio/device/monitor/filters/__init__.py diff --git a/platformio/device/filters/base.py b/platformio/device/monitor/filters/base.py similarity index 93% rename from platformio/device/filters/base.py rename to platformio/device/monitor/filters/base.py index 203cdb77..e773ed65 100644 --- a/platformio/device/filters/base.py +++ b/platformio/device/monitor/filters/base.py @@ -17,7 +17,6 @@ import os from serial.tools import miniterm -from platformio import fs from platformio.compat import get_object_members, load_python_module from platformio.package.manager.tool import ToolPackageManager from platformio.project.config import ProjectConfig @@ -70,10 +69,7 @@ def register_filters(platform=None, options=None): os.path.join(pkg.path, "monitor"), prefix="filter_", options=options ) # default filters - load_monitor_filters( - os.path.join(fs.get_source_dir(), "device", "filters"), - options=options, - ) + load_monitor_filters(os.path.dirname(__file__), options=options) def load_monitor_filters(monitor_dir, prefix=None, options=None): @@ -91,7 +87,7 @@ def load_monitor_filters(monitor_dir, prefix=None, options=None): def load_monitor_filter(path, options=None): name = os.path.basename(path) name = name[: name.find(".")] - module = load_python_module("platformio.device.filters.%s" % name, path) + module = load_python_module("platformio.device.monitor.filters.%s" % name, path) for cls in get_object_members(module).values(): if ( not inspect.isclass(cls) diff --git a/platformio/device/filters/hexlify.py b/platformio/device/monitor/filters/hexlify.py similarity index 94% rename from platformio/device/filters/hexlify.py rename to platformio/device/monitor/filters/hexlify.py index 045f637e..28e83bfb 100644 --- a/platformio/device/filters/hexlify.py +++ b/platformio/device/monitor/filters/hexlify.py @@ -14,7 +14,7 @@ import serial -from platformio.device.filters.base import DeviceMonitorFilterBase +from platformio.device.monitor.filters.base import DeviceMonitorFilterBase class Hexlify(DeviceMonitorFilterBase): diff --git a/platformio/device/filters/log2file.py b/platformio/device/monitor/filters/log2file.py similarity index 94% rename from platformio/device/filters/log2file.py rename to platformio/device/monitor/filters/log2file.py index e4c622d1..bf97b551 100644 --- a/platformio/device/filters/log2file.py +++ b/platformio/device/monitor/filters/log2file.py @@ -16,7 +16,7 @@ import io import os.path from datetime import datetime -from platformio.device.filters.base import DeviceMonitorFilterBase +from platformio.device.monitor.filters.base import DeviceMonitorFilterBase class LogToFile(DeviceMonitorFilterBase): diff --git a/platformio/device/filters/send_on_enter.py b/platformio/device/monitor/filters/send_on_enter.py similarity index 94% rename from platformio/device/filters/send_on_enter.py rename to platformio/device/monitor/filters/send_on_enter.py index ec002295..c28888cd 100644 --- a/platformio/device/filters/send_on_enter.py +++ b/platformio/device/monitor/filters/send_on_enter.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from platformio.device.filters.base import DeviceMonitorFilterBase +from platformio.device.monitor.filters.base import DeviceMonitorFilterBase class SendOnEnter(DeviceMonitorFilterBase): diff --git a/platformio/device/filters/time.py b/platformio/device/monitor/filters/time.py similarity index 94% rename from platformio/device/filters/time.py rename to platformio/device/monitor/filters/time.py index d7ba1c7f..cde4e772 100644 --- a/platformio/device/filters/time.py +++ b/platformio/device/monitor/filters/time.py @@ -14,7 +14,7 @@ from datetime import datetime -from platformio.device.filters.base import DeviceMonitorFilterBase +from platformio.device.monitor.filters.base import DeviceMonitorFilterBase class Timestamp(DeviceMonitorFilterBase): diff --git a/platformio/device/monitor/terminal.py b/platformio/device/monitor/terminal.py new file mode 100644 index 00000000..38c18390 --- /dev/null +++ b/platformio/device/monitor/terminal.py @@ -0,0 +1,185 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import signal +import sys +import threading + +import click +import serial +from serial.tools import miniterm + +from platformio.exception import UserSideException + + +class Terminal(miniterm.Miniterm): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.pio_unexpected_exception = None + + def reader(self): + try: + super().reader() + except Exception as exc: # pylint: disable=broad-except + self.pio_unexpected_exception = exc + + def writer(self): + try: + super().writer() + except Exception as exc: # pylint: disable=broad-except + self.pio_unexpected_exception = exc + + +def start_terminal(options): + retries = 0 + is_port_valid = False + while True: + term = None + try: + term = new_terminal(options) + is_port_valid = True + options["port"] = term.serial.name + if retries: + click.echo("\t Connected!", err=True) + elif not options["quiet"]: + print_terminal_settings(term) + retries = 0 # reset + term.start() + try: + term.join(True) + except KeyboardInterrupt: + pass + term.join() + + # cleanup + term.console.cleanup() + + # restore original standard streams + sys.stdin = sys.__stdin__ + sys.stdout = sys.__stdout__ + sys.stderr = sys.__stderr__ + + term.close() + + if term.pio_unexpected_exception: + click.secho( + "Disconnected (%s)" % term.pio_unexpected_exception, + fg="red", + err=True, + ) + if not options["no_reconnect"]: + raise UserSideException(term.pio_unexpected_exception) + + return + except UserSideException as exc: + if not is_port_valid: + raise exc + if not retries: + click.echo("Reconnecting to %s " % options["port"], err=True, nl=False) + signal.signal(signal.SIGINT, signal.SIG_DFL) + else: + click.echo(".", err=True, nl=False) + retries += 1 + threading.Event().wait(retries / 2) + + +def new_terminal(options): + term = Terminal( + new_serial_instance(options), + echo=options["echo"], + eol=options["eol"].lower(), + filters=list(reversed(options["filters"] or ["default"])), + ) + term.exit_character = chr(options["exit_char"]) + term.menu_character = chr(options["menu_char"]) + term.raw = options["raw"] + term.set_rx_encoding(options["encoding"]) + term.set_tx_encoding(options["encoding"]) + return term + + +def print_terminal_settings(terminal): + click.echo( + "--- Terminal on {p.name} | " + "{p.baudrate} {p.bytesize}-{p.parity}-{p.stopbits}".format(p=terminal.serial) + ) + click.echo( + "--- Available filters and text transformations: %s" + % ", ".join(sorted(miniterm.TRANSFORMATIONS.keys())) + ) + click.echo("--- More details at https://bit.ly/pio-monitor-filters") + click.echo( + "--- Quit: {} | Menu: {} | Help: {} followed by {}".format( + miniterm.key_description(terminal.exit_character), + miniterm.key_description(terminal.menu_character), + miniterm.key_description(terminal.menu_character), + miniterm.key_description("\x08"), + ) + ) + + +def new_serial_instance(options): # pylint: disable=too-many-branches + serial_instance = None + port = options["port"] + while serial_instance is None: + # no port given on command line -> ask user now + if port is None or port == "-": + try: + port = miniterm.ask_for_port() + except KeyboardInterrupt as exc: + click.echo("", err=True) + raise UserSideException("User aborted and port is not given") from exc + else: + if not port: + raise UserSideException("Port is not given") + try: + serial_instance = serial.serial_for_url( + port, + options["baud"], + parity=options["parity"], + rtscts=options["rtscts"], + xonxoff=options["xonxoff"], + do_not_open=True, + ) + + if not hasattr(serial_instance, "cancel_read"): + # enable timeout for alive flag polling if cancel_read is not available + serial_instance.timeout = 1 + + if options["dtr"] is not None: + if not options["quiet"]: + click.echo( + "--- forcing DTR {}".format( + "active" if options["dtr"] else "inactive" + ) + ) + serial_instance.dtr = options["dtr"] + + if options["rts"] is not None: + if not options["quiet"]: + click.echo( + "--- forcing RTS {}".format( + "active" if options["rts"] else "inactive" + ) + ) + serial_instance.rts = options["rts"] + + if isinstance(serial_instance, serial.Serial): + serial_instance.exclusive = True + + serial_instance.open() + except serial.SerialException as exc: + raise UserSideException(exc) from exc + + return serial_instance diff --git a/platformio/exception.py b/platformio/exception.py index 5c0b44ea..a8287c04 100644 --- a/platformio/exception.py +++ b/platformio/exception.py @@ -30,10 +30,6 @@ class ReturnErrorCode(PlatformioException): MESSAGE = "{0}" -class MinitermException(PlatformioException): - pass - - class UserSideException(PlatformioException): pass diff --git a/platformio/fs.py b/platformio/fs.py index 2ede27b7..60111694 100644 --- a/platformio/fs.py +++ b/platformio/fs.py @@ -28,7 +28,7 @@ from platformio import exception, proc from platformio.compat import IS_WINDOWS -class cd(object): +class cd: def __init__(self, new_path): self.new_path = new_path self.prev_path = os.getcwd() @@ -54,8 +54,8 @@ def load_json(file_path): try: with open(file_path, mode="r", encoding="utf8") as f: return json.load(f) - except ValueError: - raise exception.InvalidJSONFile(file_path) + except ValueError as exc: + raise exception.InvalidJSONFile(file_path) from exc def humanize_file_size(filesize): @@ -97,6 +97,12 @@ def calculate_folder_size(path): return result +def get_platformio_udev_rules_path(): + return os.path.abspath( + os.path.join(get_source_dir(), "..", "scripts", "99-platformio-udev.rules") + ) + + def ensure_udev_rules(): from platformio.util import get_systype # pylint: disable=import-outside-toplevel @@ -119,9 +125,7 @@ def ensure_udev_rules(): if not any(os.path.isfile(p) for p in installed_rules): raise exception.MissedUdevRules - origin_path = os.path.abspath( - os.path.join(get_source_dir(), "..", "scripts", "99-platformio-udev.rules") - ) + origin_path = get_platformio_udev_rules_path() if not os.path.isfile(origin_path): return None @@ -227,9 +231,9 @@ def rmtree(path): if st_mode & stat.S_IREAD: os.chmod(path, st_mode | stat.S_IWRITE) func(path) - except Exception as e: # pylint: disable=broad-except + except Exception as exc: # pylint: disable=broad-except click.secho( - "%s \nPlease manually remove the file `%s`" % (str(e), path), + "%s \nPlease manually remove the file `%s`" % (str(exc), path), fg="red", err=True, ) diff --git a/platformio/home/rpc/handlers/account.py b/platformio/home/rpc/handlers/account.py index 23777437..d857d587 100644 --- a/platformio/home/rpc/handlers/account.py +++ b/platformio/home/rpc/handlers/account.py @@ -23,7 +23,7 @@ class AccountRPC: try: client = AccountClient() return getattr(client, method)(*args, **kwargs) - except Exception as e: # pylint: disable=bare-except + except Exception as exc: # pylint: disable=bare-except raise JSONRPC20DispatchException( - code=4003, message="PIO Account Call Error", data=str(e) - ) + code=4003, message="PIO Account Call Error", data=str(exc) + ) from exc diff --git a/platformio/home/rpc/handlers/os.py b/platformio/home/rpc/handlers/os.py index 1aaf6d63..fad11318 100644 --- a/platformio/home/rpc/handlers/os.py +++ b/platformio/home/rpc/handlers/os.py @@ -24,7 +24,7 @@ import click from platformio import __default_requests_timeout__, fs from platformio.cache import ContentCache -from platformio.device.list import list_logical_devices +from platformio.device.list.util import list_logical_devices from platformio.home import helpers from platformio.http import ensure_internet_on diff --git a/platformio/home/rpc/handlers/piocore.py b/platformio/home/rpc/handlers/piocore.py index add05a31..7a0052d4 100644 --- a/platformio/home/rpc/handlers/piocore.py +++ b/platformio/home/rpc/handlers/piocore.py @@ -29,7 +29,7 @@ from platformio.compat import get_locale_encoding, is_bytes from platformio.home import helpers -class MultiThreadingStdStream(object): +class MultiThreadingStdStream: def __init__(self, parent_stream): self._buffers = {threading.get_ident(): parent_stream} @@ -94,10 +94,10 @@ class PIOCoreRPC: # fall-back to subprocess method result = await PIOCoreRPC._call_subprocess(args, options) return PIOCoreRPC._process_result(result, to_json) - except Exception as e: # pylint: disable=bare-except + except Exception as exc: # pylint: disable=bare-except raise JSONRPC20DispatchException( - code=4003, message="PIO Core Call Error", data=str(e) - ) + code=4003, message="PIO Core Call Error", data=str(exc) + ) from exc @staticmethod async def _call_subprocess(args, options): @@ -139,8 +139,8 @@ class PIOCoreRPC: return text try: return json.loads(out) - except ValueError as e: - click.secho("%s => `%s`" % (e, out), fg="red", err=True) + except ValueError as exc: + click.secho("%s => `%s`" % (exc, out), fg="red", err=True) # if PIO Core prints unhandled warnings for line in out.split("\n"): line = line.strip() @@ -150,4 +150,4 @@ class PIOCoreRPC: return json.loads(line) except ValueError: pass - raise e + raise exc diff --git a/platformio/home/rpc/handlers/project.py b/platformio/home/rpc/handlers/project.py index 13a9ebbb..146629ca 100644 --- a/platformio/home/rpc/handlers/project.py +++ b/platformio/home/rpc/handlers/project.py @@ -26,8 +26,8 @@ from platformio.home.rpc.handlers.piocore import PIOCoreRPC from platformio.package.manager.platform import PlatformPackageManager from platformio.project.config import ProjectConfig from platformio.project.exception import ProjectError -from platformio.project.generator import ProjectGenerator from platformio.project.helpers import get_project_dir, is_platformio_project +from platformio.project.integration.generator import ProjectGenerator from platformio.project.options import get_config_options_schema @@ -247,7 +247,7 @@ class ProjectRPC: if not isinstance(platforms, list): platforms = [platforms] c_based_platforms = ["intel_mcs51", "ststm8"] - is_cpp_project = not (set(platforms) & set(c_based_platforms)) + is_cpp_project = not set(platforms) & set(c_based_platforms) except exception.PlatformioException: pass diff --git a/platformio/http.py b/platformio/http.py index 5f5fbdd2..ff88a85b 100644 --- a/platformio/http.py +++ b/platformio/http.py @@ -57,7 +57,7 @@ class EndpointSession(requests.Session): return super().request(method, urljoin(self.base_url, url), *args, **kwargs) -class EndpointSessionIterator(object): +class EndpointSessionIterator: def __init__(self, endpoints): if not isinstance(endpoints, list): endpoints = [endpoints] @@ -82,7 +82,7 @@ class EndpointSessionIterator(object): return session -class HTTPClient(object): +class HTTPClient: def __init__(self, endpoints): self._session_iter = EndpointSessionIterator(endpoints) self._session = None @@ -132,11 +132,11 @@ class HTTPClient(object): except ( requests.exceptions.ConnectionError, requests.exceptions.Timeout, - ) as e: + ) as exc: try: self._next_session() - except: # pylint: disable=bare-except - raise HTTPClientError(str(e)) + except Exception as exc2: + raise HTTPClientError(str(exc2)) from exc def fetch_json_data(self, method, path, **kwargs): if method not in ("get", "head", "options"): diff --git a/platformio/maintenance.py b/platformio/maintenance.py index 925d2f3a..bd9ccd9f 100644 --- a/platformio/maintenance.py +++ b/platformio/maintenance.py @@ -79,7 +79,7 @@ def set_caller(caller=None): return app.set_session_var("caller_id", caller) -class Upgrader(object): +class Upgrader: def __init__(self, from_version, to_version): self.from_version = pepver_to_semver(from_version) self.to_version = pepver_to_semver(to_version) diff --git a/platformio/package/commands/exec.py b/platformio/package/commands/exec.py index b5484f73..fc100f19 100644 --- a/platformio/package/commands/exec.py +++ b/platformio/package/commands/exec.py @@ -67,7 +67,7 @@ def package_exec_cmd(obj, package, call, args): if force_click_stream: click.echo(result.stdout.decode().strip(), err=result.returncode != 0) except Exception as exc: - raise UserSideException(exc) + raise UserSideException(exc) from exc if result and result.returncode != 0: raise ReturnErrorCode(result.returncode) diff --git a/platformio/package/commands/install.py b/platformio/package/commands/install.py index 8a6e3017..ecfbe7bb 100644 --- a/platformio/package/commands/install.py +++ b/platformio/package/commands/install.py @@ -23,7 +23,9 @@ from platformio.package.exception import UnknownPackageError from platformio.package.manager.library import LibraryPackageManager from platformio.package.manager.platform import PlatformPackageManager from platformio.package.manager.tool import ToolPackageManager -from platformio.package.meta import PackageSpec +from platformio.package.meta import PackageCompatibility, PackageSpec +from platformio.platform.exception import UnknownPlatform +from platformio.platform.factory import PlatformFactory from platformio.project.config import ProjectConfig from platformio.project.savedeps import pkg_to_save_spec, save_project_dependencies from platformio.test.result import TestSuite @@ -100,9 +102,7 @@ def install_project_dependencies(options): if environments and env not in environments: continue if not options.get("silent"): - click.echo( - "Resolving %s environment packages..." % click.style(env, fg="cyan") - ) + click.echo("Resolving %s dependencies..." % click.style(env, fg="cyan")) already_up_to_date = not install_project_env_dependencies(env, options) if not options.get("silent") and already_up_to_date: click.secho("Already up-to-date.", fg="green") @@ -204,8 +204,24 @@ def _install_project_env_libraries(project_env, options): _uninstall_project_unused_libdeps(project_env, options) already_up_to_date = not options.get("force") config = ProjectConfig.get_instance() + + compatibility_qualifiers = {} + if config.get(f"env:{project_env}", "platform"): + try: + p = PlatformFactory.new(config.get(f"env:{project_env}", "platform")) + compatibility_qualifiers["platforms"] = [p.name] + except UnknownPlatform: + pass + if config.get(f"env:{project_env}", "framework"): + compatibility_qualifiers["frameworks"] = config.get( + f"env:{project_env}", "framework" + ) + env_lm = LibraryPackageManager( - os.path.join(config.get("platformio", "libdeps_dir"), project_env) + os.path.join(config.get("platformio", "libdeps_dir"), project_env), + compatibility=PackageCompatibility(**compatibility_qualifiers) + if compatibility_qualifiers + else None, ) private_lm = LibraryPackageManager( os.path.join(config.get("platformio", "lib_dir")) diff --git a/platformio/package/commands/list.py b/platformio/package/commands/list.py index 578ecd20..c92cc1b4 100644 --- a/platformio/package/commands/list.py +++ b/platformio/package/commands/list.py @@ -163,9 +163,7 @@ def list_project_packages(options): for env in config.envs(): if environments and env not in environments: continue - click.echo( - "Resolving %s environment packages..." % click.style(env, fg="cyan") - ) + click.echo("Resolving %s dependencies..." % click.style(env, fg="cyan")) found = False if not only_packages or only_platform_packages: _found = print_project_env_platform_packages(env, options) diff --git a/platformio/package/commands/pack.py b/platformio/package/commands/pack.py index c80995e4..038ffc15 100644 --- a/platformio/package/commands/pack.py +++ b/platformio/package/commands/pack.py @@ -39,7 +39,7 @@ def package_pack_cmd(package, output): ManifestSchema().load_manifest( ManifestParserFactory.new_from_archive(archive_path).as_dict() ) - except ManifestValidationError as e: + except ManifestValidationError as exc: os.remove(archive_path) - raise e + raise exc click.secho('Wrote a tarball to "%s"' % archive_path, fg="green") diff --git a/platformio/package/commands/publish.py b/platformio/package/commands/publish.py index c1c12a5a..8d282fa0 100644 --- a/platformio/package/commands/publish.py +++ b/platformio/package/commands/publish.py @@ -36,8 +36,8 @@ def validate_datetime(ctx, param, value): # pylint: disable=unused-argument return value try: datetime.strptime(value, "%Y-%m-%d %H:%M:%S") - except ValueError as e: - raise click.BadParameter(e) + except ValueError as exc: + raise click.BadParameter(exc) return value @@ -71,14 +71,21 @@ def validate_datetime(ctx, param, value): # pylint: disable=unused-argument help="Notify by email when package is processed", ) @click.option( - "--non-interactive", + "--no-interactive", is_flag=True, help="Do not show interactive prompt", ) +@click.option( + "--non-interactive", + is_flag=True, + help="Do not show interactive prompt", + hidden=True, +) def package_publish_cmd( # pylint: disable=too-many-arguments, too-many-locals - package, owner, type_, released_at, private, notify, non_interactive + package, owner, type_, released_at, private, notify, no_interactive, non_interactive ): click.secho("Preparing a package...", fg="cyan") + no_interactive = no_interactive or non_interactive owner = owner or AccountClient().get_logged_username() do_not_pack = ( not os.path.isdir(package) @@ -118,7 +125,7 @@ def package_publish_cmd( # pylint: disable=too-many-arguments, too-many-locals # look for duplicates check_package_duplicates(owner, type_, name, version, manifest.get("system")) - if not non_interactive: + if not no_interactive: click.confirm( "Are you sure you want to publish the %s %s to the registry?\n" % ( diff --git a/platformio/package/commands/show.py b/platformio/package/commands/show.py index 82b6e47c..5be6790c 100644 --- a/platformio/package/commands/show.py +++ b/platformio/package/commands/show.py @@ -59,14 +59,6 @@ def package_show_cmd(spec, pkg_type): ) ) - click.echo() - type_plural = "libraries" if data["type"] == "library" else (data["type"] + "s") - click.secho( - "https://registry.platformio.org/%s/%s/%s" - % (type_plural, data["owner"]["username"], quote(data["name"])), - fg="blue", - ) - # Description click.echo() click.echo(data["description"]) @@ -87,7 +79,17 @@ def package_show_cmd(spec, pkg_type): ("frameworks", "Compatible Frameworks"), ("keywords", "Keywords"), ] - extra = [] + type_plural = "libraries" if data["type"] == "library" else (data["type"] + "s") + extra = [ + ( + "Registry", + click.style( + "https://registry.platformio.org/%s/%s/%s" + % (type_plural, data["owner"]["username"], quote(data["name"])), + fg="blue", + ), + ) + ] for key, title in fields: if "." in key: k1, k2 = key.split(".") @@ -127,7 +129,11 @@ def fetch_package_data(spec, pkg_type=None): return client.get_package( pkg_type, spec.owner, spec.name, version=spec.requirements ) - qualifiers = dict(names=spec.name.lower()) + qualifiers = {} + if spec.id: + qualifiers["ids"] = str(spec.id) + if spec.name: + qualifiers["names"] = spec.name.lower() if pkg_type: qualifiers["types"] = pkg_type if spec.owner: diff --git a/platformio/package/commands/uninstall.py b/platformio/package/commands/uninstall.py index 5fff8160..e393d2a8 100644 --- a/platformio/package/commands/uninstall.py +++ b/platformio/package/commands/uninstall.py @@ -92,9 +92,7 @@ def uninstall_project_dependencies(options): if environments and env not in environments: continue if not options["silent"]: - click.echo( - "Resolving %s environment packages..." % click.style(env, fg="cyan") - ) + click.echo("Resolving %s dependencies..." % click.style(env, fg="cyan")) already_up_to_date = not uninstall_project_env_dependencies(env, options) if not options["silent"] and already_up_to_date: click.secho("Already up-to-date.", fg="green") diff --git a/platformio/package/commands/update.py b/platformio/package/commands/update.py index a520b7b3..67fc5dd1 100644 --- a/platformio/package/commands/update.py +++ b/platformio/package/commands/update.py @@ -95,9 +95,7 @@ def update_project_dependencies(options): if environments and env not in environments: continue if not options["silent"]: - click.echo( - "Resolving %s environment packages..." % click.style(env, fg="cyan") - ) + click.echo("Resolving %s dependencies..." % click.style(env, fg="cyan")) already_up_to_date = not update_project_env_dependencies(env, options) if not options["silent"] and already_up_to_date: click.secho("Already up-to-date.", fg="green") diff --git a/platformio/package/download.py b/platformio/package/download.py index ffc57d50..f8b67ba8 100644 --- a/platformio/package/download.py +++ b/platformio/package/download.py @@ -25,7 +25,7 @@ from platformio import __default_requests_timeout__, app, fs from platformio.package.exception import PackageException -class FileDownloader(object): +class FileDownloader: def __init__(self, url, dest_dir=None): self._request = None # make connection diff --git a/platformio/package/lockfile.py b/platformio/package/lockfile.py index 296036aa..3c6b2047 100644 --- a/platformio/package/lockfile.py +++ b/platformio/package/lockfile.py @@ -44,7 +44,7 @@ class LockFileTimeoutError(PlatformioException): pass -class LockFile(object): +class LockFile: def __init__(self, path, timeout=LOCKFILE_TIMEOUT, delay=LOCKFILE_DELAY): self.timeout = timeout self.delay = delay @@ -72,10 +72,10 @@ class LockFile(object): msvcrt.locking( # pylint: disable=used-before-assignment self._fp.fileno(), msvcrt.LK_NBLCK, 1 ) - except (BlockingIOError, IOError): + except (BlockingIOError, IOError) as exc: self._fp.close() self._fp = None - raise LockFileExists + raise LockFileExists from exc return True def _unlock(self): diff --git a/platformio/package/manager/_download.py b/platformio/package/manager/_download.py index e408908a..9d9c6118 100644 --- a/platformio/package/manager/_download.py +++ b/platformio/package/manager/_download.py @@ -25,7 +25,7 @@ from platformio.package.download import FileDownloader from platformio.package.lockfile import LockFile -class PackageManagerDownloadMixin(object): +class PackageManagerDownloadMixin: DOWNLOAD_CACHE_EXPIRE = 86400 * 30 # keep package in a local cache for 1 month @@ -70,7 +70,7 @@ class PackageManagerDownloadMixin(object): fd = FileDownloader(url) fd.set_destination(tmp_path) fd.start(with_progress=with_progress, silent=silent) - except IOError as e: + except IOError as exc: raise_error = not with_progress if with_progress: try: @@ -86,7 +86,7 @@ class PackageManagerDownloadMixin(object): fg="red", ) ) - raise e + raise exc if checksum: fd.verify(checksum) os.close(tmp_fd) diff --git a/platformio/package/manager/_install.py b/platformio/package/manager/_install.py index 8129d9b0..c89d7b86 100644 --- a/platformio/package/manager/_install.py +++ b/platformio/package/manager/_install.py @@ -21,12 +21,12 @@ import click from platformio import app, compat, fs, util from platformio.package.exception import PackageException, UnknownPackageError -from platformio.package.meta import PackageItem +from platformio.package.meta import PackageCompatibility, PackageItem from platformio.package.unpack import FileUnpacker from platformio.package.vcsclient import VCSClientFactory -class PackageManagerInstallMixin(object): +class PackageManagerInstallMixin: _INSTALL_HISTORY = None # avoid circle dependencies @@ -36,9 +36,9 @@ class PackageManagerInstallMixin(object): try: with FileUnpacker(src) as fu: return fu.unpack(dst, with_progress=with_progress) - except IOError as e: + except IOError as exc: if not with_progress: - raise e + raise exc with FileUnpacker(src) as fu: return fu.unpack(dst, with_progress=False) @@ -55,9 +55,9 @@ class PackageManagerInstallMixin(object): def _install( self, spec, - search_qualifiers=None, skip_dependencies=False, force=False, + compatibility: PackageCompatibility = None, ): spec = self.ensure_spec(spec) @@ -97,7 +97,12 @@ class PackageManagerInstallMixin(object): if spec.external: pkg = self.install_from_uri(spec.uri, spec) else: - pkg = self.install_from_registry(spec, search_qualifiers) + pkg = self.install_from_registry( + spec, + search_qualifiers=compatibility.to_search_qualifiers() + if compatibility + else None, + ) if not pkg or not pkg.metadata: raise PackageException( @@ -137,20 +142,29 @@ class PackageManagerInstallMixin(object): if dependency.get("owner"): self.log.warning( click.style( - "Warning! Could not install dependency %s for package '%s'" - % (dependency, pkg.metadata.name), + "Warning! Could not install `%s` dependency " + "for the`%s` package" % (dependency, pkg.metadata.name), fg="yellow", ) ) def install_dependency(self, dependency): - spec = self.dependency_to_spec(dependency) - search_qualifiers = { - key: value - for key, value in dependency.items() - if key in ("authors", "platforms", "frameworks") - } - return self._install(spec, search_qualifiers=search_qualifiers or None) + dependency_compatibility = PackageCompatibility.from_dependency(dependency) + if self.compatibility and not dependency_compatibility.is_compatible( + self.compatibility + ): + self.log.debug( + click.style( + "Skip incompatible `%s` dependency with `%s`" + % (dependency, self.compatibility), + fg="yellow", + ) + ) + return None + return self._install( + spec=self.dependency_to_spec(dependency), + compatibility=dependency_compatibility, + ) def install_from_uri(self, uri, spec, checksum=None): spec = self.ensure_spec(spec) diff --git a/platformio/package/manager/_legacy.py b/platformio/package/manager/_legacy.py index 978efc9c..4b02a1df 100644 --- a/platformio/package/manager/_legacy.py +++ b/platformio/package/manager/_legacy.py @@ -18,7 +18,7 @@ from platformio import fs from platformio.package.meta import PackageItem, PackageSpec -class PackageManagerLegacyMixin(object): +class PackageManagerLegacyMixin: def build_legacy_spec(self, pkg_dir): # find src manifest src_manifest_name = ".piopkgmanager.json" diff --git a/platformio/package/manager/_registry.py b/platformio/package/manager/_registry.py index 0a9f39c5..4015cffb 100644 --- a/platformio/package/manager/_registry.py +++ b/platformio/package/manager/_registry.py @@ -23,7 +23,7 @@ from platformio.registry.client import RegistryClient from platformio.registry.mirror import RegistryFileMirrorIterator -class PackageManagerRegistryMixin(object): +class PackageManagerRegistryMixin: def install_from_registry(self, spec, search_qualifiers=None): if spec.owner and spec.name and not search_qualifiers: package = self.fetch_registry_package(spec) @@ -56,9 +56,9 @@ class PackageManagerRegistryMixin(object): ), checksum or pkgfile["checksum"]["sha256"], ) - except Exception as e: # pylint: disable=broad-except + except Exception as exc: # pylint: disable=broad-except self.log.warning( - click.style("Warning! Package Mirror: %s" % e, fg="yellow") + click.style("Warning! Package Mirror: %s" % exc, fg="yellow") ) self.log.warning( click.style("Looking for another mirror...", fg="yellow") diff --git a/platformio/package/manager/_symlink.py b/platformio/package/manager/_symlink.py index 8c5eae38..33ff03f1 100644 --- a/platformio/package/manager/_symlink.py +++ b/platformio/package/manager/_symlink.py @@ -20,7 +20,7 @@ from platformio.package.exception import PackageException from platformio.package.meta import PackageItem, PackageSpec -class PackageManagerSymlinkMixin(object): +class PackageManagerSymlinkMixin: @staticmethod def is_symlink(path): return path and path.endswith(".pio-link") and os.path.isfile(path) diff --git a/platformio/package/manager/_uninstall.py b/platformio/package/manager/_uninstall.py index 9c6b5772..c845145a 100644 --- a/platformio/package/manager/_uninstall.py +++ b/platformio/package/manager/_uninstall.py @@ -22,7 +22,7 @@ from platformio.package.exception import UnknownPackageError from platformio.package.meta import PackageItem, PackageSpec -class PackageManagerUninstallMixin(object): +class PackageManagerUninstallMixin: def uninstall(self, spec, skip_dependencies=False): try: self.lock() diff --git a/platformio/package/manager/_update.py b/platformio/package/manager/_update.py index 5d689ba6..353e9e45 100644 --- a/platformio/package/manager/_update.py +++ b/platformio/package/manager/_update.py @@ -21,7 +21,7 @@ from platformio.package.meta import PackageItem, PackageOutdatedResult, PackageS from platformio.package.vcsclient import VCSBaseException, VCSClientFactory -class PackageManagerUpdateMixin(object): +class PackageManagerUpdateMixin: def outdated(self, pkg, spec=None): assert isinstance(pkg, PackageItem) assert pkg.metadata diff --git a/platformio/package/manager/base.py b/platformio/package/manager/base.py index b692140f..8369f736 100644 --- a/platformio/package/manager/base.py +++ b/platformio/package/manager/base.py @@ -59,9 +59,10 @@ class BasePackageManager( # pylint: disable=too-many-public-methods,too-many-in ): _MEMORY_CACHE = {} - def __init__(self, pkg_type, package_dir): + def __init__(self, pkg_type, package_dir, compatibility=None): self.pkg_type = pkg_type self.package_dir = package_dir + self.compatibility = compatibility self.log = self._setup_logger() self._MEMORY_CACHE = {} @@ -187,9 +188,9 @@ class BasePackageManager( # pylint: disable=too-many-public-methods,too-many-in result = ManifestParserFactory.new_from_file(item).as_dict() self.memcache_set(cache_key, result) return result - except ManifestException as e: + except ManifestException as exc: if not PlatformioCLI.in_silence(): - self.log.warning(click.style(str(e), fg="yellow")) + self.log.warning(click.style(str(exc), fg="yellow")) raise MissingPackageManifestError(", ".join(self.manifest_names)) @staticmethod diff --git a/platformio/package/manager/core.py b/platformio/package/manager/core.py index d9a05cb5..28fab07b 100644 --- a/platformio/package/manager/core.py +++ b/platformio/package/manager/core.py @@ -156,7 +156,7 @@ def build_contrib_pysite_package(target_dir, with_metadata=True): raise UserSideException( "\n\nPlease ensure that the next packages are installed:\n\n" "sudo apt install python3-dev libffi-dev libssl-dev\n" - ) + ) from exc raise exc # build manifests diff --git a/platformio/package/manager/library.py b/platformio/package/manager/library.py index 802e0cfd..6babfc9c 100644 --- a/platformio/package/manager/library.py +++ b/platformio/package/manager/library.py @@ -15,19 +15,21 @@ import json import os -from platformio.commands.lib.helpers import is_builtin_lib +from platformio import util from platformio.package.exception import MissingPackageManifestError from platformio.package.manager.base import BasePackageManager from platformio.package.meta import PackageSpec, PackageType +from platformio.platform.factory import PlatformFactory from platformio.project.config import ProjectConfig class LibraryPackageManager(BasePackageManager): # pylint: disable=too-many-ancestors - def __init__(self, package_dir=None): + def __init__(self, package_dir=None, **kwargs): super().__init__( PackageType.LIBRARY, package_dir or ProjectConfig.get_instance().get("platformio", "globallib_dir"), + **kwargs ) @property @@ -84,7 +86,39 @@ class LibraryPackageManager(BasePackageManager): # pylint: disable=too-many-anc # skip built-in dependencies not_builtin_conds = [spec.external, spec.owner] if not any(not_builtin_conds): - not_builtin_conds.append(not is_builtin_lib(spec.name)) + not_builtin_conds.append(not self.is_builtin_lib(spec.name)) if any(not_builtin_conds): return super().install_dependency(dependency) return None + + @staticmethod + @util.memoized(expire="60s") + def get_builtin_libs(storage_names=None): + # pylint: disable=import-outside-toplevel + from platformio.package.manager.platform import PlatformPackageManager + + items = [] + storage_names = storage_names or [] + pm = PlatformPackageManager() + for pkg in pm.get_installed(): + p = PlatformFactory.new(pkg) + for storage in p.get_lib_storages(): + if storage_names and storage["name"] not in storage_names: + continue + lm = LibraryPackageManager(storage["path"]) + items.append( + { + "name": storage["name"], + "path": storage["path"], + "items": lm.legacy_get_installed(), + } + ) + return items + + @classmethod + def is_builtin_lib(cls, name): + for storage in cls.get_builtin_libs(): + for lib in storage["items"]: + if lib.get("name") == name: + return True + return False diff --git a/platformio/package/manager/platform.py b/platformio/package/manager/platform.py index 41f7b41e..e5e948ba 100644 --- a/platformio/package/manager/platform.py +++ b/platformio/package/manager/platform.py @@ -53,9 +53,9 @@ class PlatformPackageManager(BasePackageManager): # pylint: disable=too-many-an # set logging level for underlying tool manager p.pm.set_log_level(self.log.getEffectiveLevel()) p.ensure_engine_compatible() - except IncompatiblePlatform as e: + except IncompatiblePlatform as exc: super().uninstall(pkg, skip_dependencies=True) - raise e + raise exc if project_env: p.configure_project_packages(project_env, project_targets) if not skip_dependencies: diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py index 8735bcff..a70af7ea 100644 --- a/platformio/package/manifest/parser.py +++ b/platformio/package/manifest/parser.py @@ -27,7 +27,7 @@ from platformio.package.exception import ManifestParserError, UnknownManifestErr from platformio.project.helpers import is_platformio_project -class ManifestFileType(object): +class ManifestFileType: PLATFORM_JSON = "platform.json" LIBRARY_JSON = "library.json" LIBRARY_PROPERTIES = "library.properties" @@ -53,7 +53,7 @@ class ManifestFileType(object): return None -class ManifestParserFactory(object): +class ManifestParserFactory: @staticmethod def read_manifest_contents(path): last_err = None @@ -61,9 +61,9 @@ class ManifestParserFactory(object): try: with io.open(path, encoding=encoding) as fp: return fp.read() - except UnicodeDecodeError as e: - last_err = e - raise last_err # pylint: disable=raising-bad-type + except UnicodeDecodeError as exc: + last_err = exc + raise last_err @classmethod def new_from_file(cls, path, remote_url=False): @@ -139,14 +139,14 @@ class ManifestParserFactory(object): raise UnknownManifestError("Unknown manifest file type %s" % type) -class BaseManifestParser(object): +class BaseManifestParser: def __init__(self, contents, remote_url=None, package_dir=None): self.remote_url = remote_url self.package_dir = package_dir try: self._data = self.parse(contents) - except Exception as e: - raise ManifestParserError("Could not parse manifest -> %s" % e) + except Exception as exc: + raise ManifestParserError("Could not parse manifest -> %s" % exc) from exc self._data = self.normalize_repository(self._data) self._data = self.parse_examples(self._data) diff --git a/platformio/package/manifest/schema.py b/platformio/package/manifest/schema.py index 8258dfe8..5b74033c 100644 --- a/platformio/package/manifest/schema.py +++ b/platformio/package/manifest/schema.py @@ -28,7 +28,7 @@ from platformio.util import memoized class BaseSchema(Schema): - class Meta(object): # pylint: disable=no-init + class Meta: unknown = marshmallow.EXCLUDE # pylint: disable=no-member def load_manifest(self, data): @@ -232,7 +232,7 @@ class ManifestSchema(BaseSchema): ) @validates("version") - def validate_version(self, value): # pylint: disable=no-self-use + def validate_version(self, value): try: value = str(value) assert "." in value @@ -243,17 +243,19 @@ class ManifestSchema(BaseSchema): if "Invalid leading zero" in str(exc): raise exc semantic_version.Version.coerce(value) - except (AssertionError, ValueError): + except (AssertionError, ValueError) as exc: raise ValidationError( "Invalid semantic versioning format, see https://semver.org/" - ) + ) from exc @validates("license") def validate_license(self, value): try: spdx = self.load_spdx_licenses() - except requests.exceptions.RequestException: - raise ValidationError("Could not load SPDX licenses for validation") + except requests.exceptions.RequestException as exc: + raise ValidationError( + "Could not load SPDX licenses for validation" + ) from exc known_ids = set(item.get("licenseId") for item in spdx.get("licenses", [])) if value in known_ids: return True diff --git a/platformio/package/meta.py b/platformio/package/meta.py index 20199527..fbd2b734 100644 --- a/platformio/package/meta.py +++ b/platformio/package/meta.py @@ -25,9 +25,10 @@ from platformio import fs from platformio.compat import get_object_members, hashlib_encode_data, string_types from platformio.package.manifest.parser import ManifestFileType from platformio.package.version import cast_version_to_semver +from platformio.util import items_in_list -class PackageType(object): +class PackageType: LIBRARY = "library" PLATFORM = "platform" TOOL = "tool" @@ -63,7 +64,47 @@ class PackageType(object): return None -class PackageOutdatedResult(object): +class PackageCompatibility: + + KNOWN_QUALIFIERS = ("platforms", "frameworks", "authors") + + @classmethod + def from_dependency(cls, dependency): + assert isinstance(dependency, dict) + qualifiers = { + key: value + for key, value in dependency.items() + if key in cls.KNOWN_QUALIFIERS + } + return PackageCompatibility(**qualifiers) + + def __init__(self, **kwargs): + self.qualifiers = {} + for key, value in kwargs.items(): + if key not in self.KNOWN_QUALIFIERS: + raise ValueError( + "Unknown package compatibility qualifier -> `%s`" % key + ) + self.qualifiers[key] = value + + def __repr__(self): + return "PackageCompatibility <%s>" % self.qualifiers + + def to_search_qualifiers(self): + return self.qualifiers + + def is_compatible(self, other): + assert isinstance(other, PackageCompatibility) + for key, value in self.qualifiers.items(): + other_value = other.qualifiers.get(key) + if not value or not other_value: + continue + if not items_in_list(value, other_value): + return False + return True + + +class PackageOutdatedResult: UPDATE_INCREMENT_MAJOR = "major" UPDATE_INCREMENT_MINOR = "minor" UPDATE_INCREMENT_PATCH = "patch" @@ -122,7 +163,7 @@ class PackageOutdatedResult(object): return True -class PackageSpec(object): # pylint: disable=too-many-instance-attributes +class PackageSpec: # pylint: disable=too-many-instance-attributes def __init__( # pylint: disable=redefined-builtin,too-many-arguments self, raw=None, owner=None, id=None, name=None, requirements=None, uri=None ): @@ -358,7 +399,7 @@ class PackageSpec(object): # pylint: disable=too-many-instance-attributes return name -class PackageMetaData(object): +class PackageMetaData: def __init__( # pylint: disable=redefined-builtin self, type, name, version, spec=None ): @@ -426,7 +467,7 @@ class PackageMetaData(object): return PackageMetaData(**data) -class PackageItem(object): +class PackageItem: METAFILE_NAME = ".piopm" diff --git a/platformio/package/pack.py b/platformio/package/pack.py index ed9a9a21..0170ee3d 100644 --- a/platformio/package/pack.py +++ b/platformio/package/pack.py @@ -32,7 +32,7 @@ from platformio.package.meta import PackageItem from platformio.package.unpack import FileUnpacker -class PackagePacker(object): +class PackagePacker: INCLUDE_DEFAULT = list(ManifestFileType.items().values()) + [ "README", "README.md", diff --git a/platformio/package/unpack.py b/platformio/package/unpack.py index d8544a25..e39222c2 100644 --- a/platformio/package/unpack.py +++ b/platformio/package/unpack.py @@ -31,7 +31,7 @@ class ExtractArchiveItemError(PackageException): ) -class BaseArchiver(object): +class BaseArchiver: def __init__(self, arhfileobj): self._afo = arhfileobj @@ -129,7 +129,7 @@ class ZIPArchiver(BaseArchiver): self.preserve_mtime(item, dest_dir) -class FileUnpacker(object): +class FileUnpacker: def __init__(self, path): self.path = path self._archiver = None diff --git a/platformio/package/vcsclient.py b/platformio/package/vcsclient.py index dc4c090d..b4dafc03 100644 --- a/platformio/package/vcsclient.py +++ b/platformio/package/vcsclient.py @@ -29,7 +29,7 @@ class VCSBaseException(PackageException): pass -class VCSClientFactory(object): +class VCSClientFactory: @staticmethod def new(src_dir, remote_url, silent=False): result = urlparse(remote_url) @@ -51,11 +51,13 @@ class VCSClientFactory(object): ) assert isinstance(obj, VCSClientBase) return obj - except (KeyError, AssertionError): - raise VCSBaseException("VCS: Unknown repository type %s" % remote_url) + except (KeyError, AssertionError) as exc: + raise VCSBaseException( + "VCS: Unknown repository type %s" % remote_url + ) from exc -class VCSClientBase(object): +class VCSClientBase: command = None @@ -73,10 +75,10 @@ class VCSClientBase(object): self.get_cmd_output(["--version"]) else: assert self.run_cmd(["--version"]) - except (AssertionError, OSError, PlatformioException): + except (AssertionError, OSError, PlatformioException) as exc: raise UserSideException( "VCS: `%s` client is not installed in your system" % self.command - ) + ) from exc return True @property @@ -108,8 +110,10 @@ class VCSClientBase(object): try: subprocess.check_call(args, **kwargs) return True - except subprocess.CalledProcessError as e: - raise VCSBaseException("VCS: Could not process command %s" % e.cmd) + except subprocess.CalledProcessError as exc: + raise VCSBaseException( + "VCS: Could not process command %s" % exc.cmd + ) from exc def get_cmd_output(self, args, **kwargs): args = [self.command] + args @@ -152,10 +156,10 @@ class GitClient(VCSClientBase): def check_client(self): try: return VCSClientBase.check_client(self) - except UserSideException: + except UserSideException as exc: raise UserSideException( "Please install Git client from https://git-scm.com/downloads" - ) + ) from exc def get_branches(self): output = self.get_cmd_output(["branch"]) diff --git a/platformio/platform/_packages.py b/platformio/platform/_packages.py index c741c790..2a3aa490 100644 --- a/platformio/platform/_packages.py +++ b/platformio/platform/_packages.py @@ -15,7 +15,7 @@ from platformio.package.meta import PackageSpec -class PlatformPackagesMixin(object): +class PlatformPackagesMixin: def get_package_spec(self, name, version=None): return PackageSpec( owner=self.packages[name].get("owner"), diff --git a/platformio/platform/_run.py b/platformio/platform/_run.py index 5bac13ae..79afff94 100644 --- a/platformio/platform/_run.py +++ b/platformio/platform/_run.py @@ -27,7 +27,7 @@ from platformio.package.manager.core import get_core_package_dir from platformio.platform.exception import BuildScriptNotFound -class PlatformRunMixin(object): +class PlatformRunMixin: LINE_ERROR_RE = re.compile(r"(^|\s+)error:?\s+", re.I) diff --git a/platformio/platform/board.py b/platformio/platform/board.py index 2e998ac0..73a3ebd7 100644 --- a/platformio/platform/board.py +++ b/platformio/platform/board.py @@ -21,15 +21,15 @@ from platformio.exception import UserSideException from platformio.platform.exception import InvalidBoardManifest -class PlatformBoardConfig(object): +class PlatformBoardConfig: def __init__(self, manifest_path): self._id = os.path.basename(manifest_path)[:-5] assert os.path.isfile(manifest_path) self.manifest_path = manifest_path try: self._manifest = fs.load_json(manifest_path) - except ValueError: - raise InvalidBoardManifest(manifest_path) + except ValueError as exc: + raise InvalidBoardManifest(manifest_path) from exc if not set(["name", "url", "vendor"]) <= set(self._manifest): raise UserSideException( "Please specify name, url and vendor fields for " + manifest_path diff --git a/platformio/platform/factory.py b/platformio/platform/factory.py index 0345cb2a..b628590c 100644 --- a/platformio/platform/factory.py +++ b/platformio/platform/factory.py @@ -23,7 +23,7 @@ from platformio.platform import base from platformio.platform.exception import UnknownPlatform -class PlatformFactory(object): +class PlatformFactory: @staticmethod def get_clsname(name): name = re.sub(r"[^\da-z\_]+", "", name, flags=re.I) @@ -35,8 +35,8 @@ class PlatformFactory(object): sys.modules["platformio.managers.platform"] = base try: return load_python_module("platformio.platform.%s" % name, path) - except ImportError: - raise UnknownPlatform(name) + except ImportError as exc: + raise UnknownPlatform(name) from exc @classmethod def new(cls, pkg_or_spec, autoinstall=False) -> base.PlatformBase: diff --git a/platformio/proc.py b/platformio/proc.py index 83943273..fb66bc7f 100644 --- a/platformio/proc.py +++ b/platformio/proc.py @@ -27,7 +27,7 @@ from platformio.compat import ( ) -class AsyncPipeBase(object): +class AsyncPipeBase: def __init__(self): self._fd_read, self._fd_write = os.pipe() self._pipe_reader = os.fdopen( @@ -115,8 +115,8 @@ def exec_command(*args, **kwargs): try: result["out"], result["err"] = p.communicate() result["returncode"] = p.returncode - except KeyboardInterrupt: - raise exception.AbortedByUser() + except KeyboardInterrupt as exc: + raise exception.AbortedByUser() from exc finally: for s in ("stdout", "stderr"): if isinstance(kwargs[s], AsyncPipeBase): diff --git a/platformio/project/commands/init.py b/platformio/project/commands/init.py index 7c3b77cb..baae29ec 100644 --- a/platformio/project/commands/init.py +++ b/platformio/project/commands/init.py @@ -25,8 +25,8 @@ from platformio.package.commands.install import install_project_dependencies from platformio.package.manager.platform import PlatformPackageManager from platformio.platform.exception import UnknownBoard from platformio.project.config import ProjectConfig -from platformio.project.generator import ProjectGenerator from platformio.project.helpers import is_platformio_project +from platformio.project.integration.generator import ProjectGenerator def validate_boards(ctx, param, value): # pylint: disable=W0613 @@ -34,11 +34,11 @@ def validate_boards(ctx, param, value): # pylint: disable=W0613 for id_ in value: try: pm.board_config(id_) - except UnknownBoard: + except UnknownBoard as exc: raise click.BadParameter( "`%s`. Please search for board ID using `platformio boards` " "command" % id_ - ) + ) from exc return value diff --git a/platformio/project/config.py b/platformio/project/config.py index 999827d5..3443863a 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -38,7 +38,7 @@ CONFIG_HEADER = """ """ -class ProjectConfigBase(object): +class ProjectConfigBase: INLINE_COMMENT_RE = re.compile(r"\s+;.*$") VARTPL_RE = re.compile(r"\$\{([^\.\}\()]+)\.([^\}]+)\}") @@ -97,8 +97,8 @@ class ProjectConfigBase(object): self._parsed.append(path) try: self._parser.read(path, "utf-8") - except configparser.Error as e: - raise exception.InvalidProjectConfError(path, str(e)) + except configparser.Error as exc: + raise exception.InvalidProjectConfError(path, str(exc)) if not parse_extra: return @@ -324,10 +324,10 @@ class ProjectConfigBase(object): # handle nested calls try: value = self.get(section, option) - except RecursionError: + except RecursionError as exc: raise exception.ProjectOptionValueError( "Infinite recursion has been detected", option, section - ) + ) from exc if isinstance(value, list): return "\n".join(value) return str(value) @@ -336,8 +336,8 @@ class ProjectConfigBase(object): value = None try: value = self.getraw(section, option, default) - except configparser.Error as e: - raise exception.InvalidProjectConfError(self.path, str(e)) + except configparser.Error as exc: + raise exception.InvalidProjectConfError(self.path, str(exc)) option_meta = self.find_option_meta(section, option) if not option_meta: @@ -349,10 +349,12 @@ class ProjectConfigBase(object): value = self.parse_multi_values(value or []) try: return self.cast_to(value, option_meta.type) - except click.BadParameter as e: + except click.BadParameter as exc: if not self.expand_interpolations: return value - raise exception.ProjectOptionValueError(e.format_message(), option, section) + raise exception.ProjectOptionValueError( + exc.format_message(), option, section + ) @staticmethod def cast_to(value, to_type): @@ -394,7 +396,7 @@ class ProjectConfigBase(object): return True -class ProjectConfigDirsMixin(object): +class ProjectConfigDirsMixin: def get_optional_dir(self, name): """ Deprecated, used by platformio-node-helpers.project.observer.fetchLibDirs diff --git a/platformio/project/integration/__init__.py b/platformio/project/integration/__init__.py new file mode 100644 index 00000000..b0514903 --- /dev/null +++ b/platformio/project/integration/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/project/generator.py b/platformio/project/integration/generator.py similarity index 95% rename from platformio/project/generator.py rename to platformio/project/integration/generator.py index 10efba7d..7ffd05e3 100644 --- a/platformio/project/generator.py +++ b/platformio/project/integration/generator.py @@ -15,6 +15,7 @@ import codecs import os import sys +from pathlib import Path import bottle @@ -23,7 +24,7 @@ from platformio.proc import where_is_program from platformio.project.helpers import load_build_metadata -class ProjectGenerator(object): +class ProjectGenerator: def __init__(self, config, env_name, ide, board_ids=None): self.config = config self.project_dir = os.path.dirname(config.path) @@ -51,12 +52,11 @@ class ProjectGenerator(object): @staticmethod def get_supported_ides(): - tpls_dir = os.path.join(fs.get_source_dir(), "project", "tpls") return sorted( [ - d - for d in os.listdir(tpls_dir) - if os.path.isdir(os.path.join(tpls_dir, d)) + item.name + for item in (Path(__file__).parent / "tpls").iterdir() + if item.is_dir() ] ) @@ -132,7 +132,7 @@ class ProjectGenerator(object): def get_tpls(self): tpls = [] - tpls_dir = os.path.join(fs.get_source_dir(), "project", "tpls", self.ide) + tpls_dir = str(Path(__file__).parent / "tpls" / self.ide) for root, _, files in os.walk(tpls_dir): for f in files: if not f.endswith(".tpl"): diff --git a/platformio/project/tpls/atom/.clang_complete.tpl b/platformio/project/integration/tpls/atom/.clang_complete.tpl similarity index 100% rename from platformio/project/tpls/atom/.clang_complete.tpl rename to platformio/project/integration/tpls/atom/.clang_complete.tpl diff --git a/platformio/project/tpls/atom/.gcc-flags.json.tpl b/platformio/project/integration/tpls/atom/.gcc-flags.json.tpl similarity index 100% rename from platformio/project/tpls/atom/.gcc-flags.json.tpl rename to platformio/project/integration/tpls/atom/.gcc-flags.json.tpl diff --git a/platformio/project/tpls/atom/.gitignore.tpl b/platformio/project/integration/tpls/atom/.gitignore.tpl similarity index 100% rename from platformio/project/tpls/atom/.gitignore.tpl rename to platformio/project/integration/tpls/atom/.gitignore.tpl diff --git a/platformio/project/tpls/clion/.gitignore.tpl b/platformio/project/integration/tpls/clion/.gitignore.tpl similarity index 100% rename from platformio/project/tpls/clion/.gitignore.tpl rename to platformio/project/integration/tpls/clion/.gitignore.tpl diff --git a/platformio/project/tpls/clion/CMakeLists.txt.tpl b/platformio/project/integration/tpls/clion/CMakeLists.txt.tpl similarity index 100% rename from platformio/project/tpls/clion/CMakeLists.txt.tpl rename to platformio/project/integration/tpls/clion/CMakeLists.txt.tpl diff --git a/platformio/project/tpls/clion/CMakeListsPrivate.txt.tpl b/platformio/project/integration/tpls/clion/CMakeListsPrivate.txt.tpl similarity index 100% rename from platformio/project/tpls/clion/CMakeListsPrivate.txt.tpl rename to platformio/project/integration/tpls/clion/CMakeListsPrivate.txt.tpl diff --git a/platformio/project/tpls/codeblocks/platformio.cbp.tpl b/platformio/project/integration/tpls/codeblocks/platformio.cbp.tpl similarity index 100% rename from platformio/project/tpls/codeblocks/platformio.cbp.tpl rename to platformio/project/integration/tpls/codeblocks/platformio.cbp.tpl diff --git a/platformio/project/tpls/eclipse/.cproject.tpl b/platformio/project/integration/tpls/eclipse/.cproject.tpl similarity index 100% rename from platformio/project/tpls/eclipse/.cproject.tpl rename to platformio/project/integration/tpls/eclipse/.cproject.tpl diff --git a/platformio/project/tpls/eclipse/.project.tpl b/platformio/project/integration/tpls/eclipse/.project.tpl similarity index 100% rename from platformio/project/tpls/eclipse/.project.tpl rename to platformio/project/integration/tpls/eclipse/.project.tpl diff --git a/platformio/project/tpls/eclipse/.settings/PlatformIO Debugger.launch.tpl b/platformio/project/integration/tpls/eclipse/.settings/PlatformIO Debugger.launch.tpl similarity index 100% rename from platformio/project/tpls/eclipse/.settings/PlatformIO Debugger.launch.tpl rename to platformio/project/integration/tpls/eclipse/.settings/PlatformIO Debugger.launch.tpl diff --git a/platformio/project/tpls/eclipse/.settings/language.settings.xml.tpl b/platformio/project/integration/tpls/eclipse/.settings/language.settings.xml.tpl similarity index 100% rename from platformio/project/tpls/eclipse/.settings/language.settings.xml.tpl rename to platformio/project/integration/tpls/eclipse/.settings/language.settings.xml.tpl diff --git a/platformio/project/tpls/eclipse/.settings/org.eclipse.cdt.core.prefs.tpl b/platformio/project/integration/tpls/eclipse/.settings/org.eclipse.cdt.core.prefs.tpl similarity index 100% rename from platformio/project/tpls/eclipse/.settings/org.eclipse.cdt.core.prefs.tpl rename to platformio/project/integration/tpls/eclipse/.settings/org.eclipse.cdt.core.prefs.tpl diff --git a/platformio/project/tpls/emacs/.ccls.tpl b/platformio/project/integration/tpls/emacs/.ccls.tpl similarity index 100% rename from platformio/project/tpls/emacs/.ccls.tpl rename to platformio/project/integration/tpls/emacs/.ccls.tpl diff --git a/platformio/project/tpls/emacs/.gitignore.tpl b/platformio/project/integration/tpls/emacs/.gitignore.tpl similarity index 100% rename from platformio/project/tpls/emacs/.gitignore.tpl rename to platformio/project/integration/tpls/emacs/.gitignore.tpl diff --git a/platformio/project/tpls/netbeans/nbproject/configurations.xml.tpl b/platformio/project/integration/tpls/netbeans/nbproject/configurations.xml.tpl similarity index 100% rename from platformio/project/tpls/netbeans/nbproject/configurations.xml.tpl rename to platformio/project/integration/tpls/netbeans/nbproject/configurations.xml.tpl diff --git a/platformio/project/tpls/netbeans/nbproject/private/configurations.xml.tpl b/platformio/project/integration/tpls/netbeans/nbproject/private/configurations.xml.tpl similarity index 100% rename from platformio/project/tpls/netbeans/nbproject/private/configurations.xml.tpl rename to platformio/project/integration/tpls/netbeans/nbproject/private/configurations.xml.tpl diff --git a/platformio/project/tpls/netbeans/nbproject/private/launcher.properties.tpl b/platformio/project/integration/tpls/netbeans/nbproject/private/launcher.properties.tpl similarity index 100% rename from platformio/project/tpls/netbeans/nbproject/private/launcher.properties.tpl rename to platformio/project/integration/tpls/netbeans/nbproject/private/launcher.properties.tpl diff --git a/platformio/project/tpls/netbeans/nbproject/private/private.xml.tpl b/platformio/project/integration/tpls/netbeans/nbproject/private/private.xml.tpl similarity index 100% rename from platformio/project/tpls/netbeans/nbproject/private/private.xml.tpl rename to platformio/project/integration/tpls/netbeans/nbproject/private/private.xml.tpl diff --git a/platformio/project/tpls/netbeans/nbproject/project.xml.tpl b/platformio/project/integration/tpls/netbeans/nbproject/project.xml.tpl similarity index 100% rename from platformio/project/tpls/netbeans/nbproject/project.xml.tpl rename to platformio/project/integration/tpls/netbeans/nbproject/project.xml.tpl diff --git a/platformio/project/tpls/qtcreator/.gitignore.tpl b/platformio/project/integration/tpls/qtcreator/.gitignore.tpl similarity index 100% rename from platformio/project/tpls/qtcreator/.gitignore.tpl rename to platformio/project/integration/tpls/qtcreator/.gitignore.tpl diff --git a/platformio/project/tpls/qtcreator/Makefile.tpl b/platformio/project/integration/tpls/qtcreator/Makefile.tpl similarity index 100% rename from platformio/project/tpls/qtcreator/Makefile.tpl rename to platformio/project/integration/tpls/qtcreator/Makefile.tpl diff --git a/platformio/project/tpls/qtcreator/platformio.cflags.tpl b/platformio/project/integration/tpls/qtcreator/platformio.cflags.tpl similarity index 100% rename from platformio/project/tpls/qtcreator/platformio.cflags.tpl rename to platformio/project/integration/tpls/qtcreator/platformio.cflags.tpl diff --git a/platformio/project/tpls/qtcreator/platformio.config.tpl b/platformio/project/integration/tpls/qtcreator/platformio.config.tpl similarity index 100% rename from platformio/project/tpls/qtcreator/platformio.config.tpl rename to platformio/project/integration/tpls/qtcreator/platformio.config.tpl diff --git a/platformio/project/tpls/qtcreator/platformio.creator.tpl b/platformio/project/integration/tpls/qtcreator/platformio.creator.tpl similarity index 100% rename from platformio/project/tpls/qtcreator/platformio.creator.tpl rename to platformio/project/integration/tpls/qtcreator/platformio.creator.tpl diff --git a/platformio/project/tpls/qtcreator/platformio.cxxflags.tpl b/platformio/project/integration/tpls/qtcreator/platformio.cxxflags.tpl similarity index 100% rename from platformio/project/tpls/qtcreator/platformio.cxxflags.tpl rename to platformio/project/integration/tpls/qtcreator/platformio.cxxflags.tpl diff --git a/platformio/project/tpls/qtcreator/platformio.files.tpl b/platformio/project/integration/tpls/qtcreator/platformio.files.tpl similarity index 100% rename from platformio/project/tpls/qtcreator/platformio.files.tpl rename to platformio/project/integration/tpls/qtcreator/platformio.files.tpl diff --git a/platformio/project/tpls/qtcreator/platformio.includes.tpl b/platformio/project/integration/tpls/qtcreator/platformio.includes.tpl similarity index 100% rename from platformio/project/tpls/qtcreator/platformio.includes.tpl rename to platformio/project/integration/tpls/qtcreator/platformio.includes.tpl diff --git a/platformio/project/tpls/sublimetext/.ccls.tpl b/platformio/project/integration/tpls/sublimetext/.ccls.tpl similarity index 100% rename from platformio/project/tpls/sublimetext/.ccls.tpl rename to platformio/project/integration/tpls/sublimetext/.ccls.tpl diff --git a/platformio/project/tpls/sublimetext/platformio.sublime-project.tpl b/platformio/project/integration/tpls/sublimetext/platformio.sublime-project.tpl similarity index 100% rename from platformio/project/tpls/sublimetext/platformio.sublime-project.tpl rename to platformio/project/integration/tpls/sublimetext/platformio.sublime-project.tpl diff --git a/platformio/project/tpls/vim/.ccls.tpl b/platformio/project/integration/tpls/vim/.ccls.tpl similarity index 100% rename from platformio/project/tpls/vim/.ccls.tpl rename to platformio/project/integration/tpls/vim/.ccls.tpl diff --git a/platformio/project/tpls/vim/.gitignore.tpl b/platformio/project/integration/tpls/vim/.gitignore.tpl similarity index 100% rename from platformio/project/tpls/vim/.gitignore.tpl rename to platformio/project/integration/tpls/vim/.gitignore.tpl diff --git a/platformio/project/tpls/visualstudio/platformio.vcxproj.filters.tpl b/platformio/project/integration/tpls/visualstudio/platformio.vcxproj.filters.tpl similarity index 100% rename from platformio/project/tpls/visualstudio/platformio.vcxproj.filters.tpl rename to platformio/project/integration/tpls/visualstudio/platformio.vcxproj.filters.tpl diff --git a/platformio/project/tpls/visualstudio/platformio.vcxproj.tpl b/platformio/project/integration/tpls/visualstudio/platformio.vcxproj.tpl similarity index 100% rename from platformio/project/tpls/visualstudio/platformio.vcxproj.tpl rename to platformio/project/integration/tpls/visualstudio/platformio.vcxproj.tpl diff --git a/platformio/project/tpls/vscode/.gitignore.tpl b/platformio/project/integration/tpls/vscode/.gitignore.tpl similarity index 100% rename from platformio/project/tpls/vscode/.gitignore.tpl rename to platformio/project/integration/tpls/vscode/.gitignore.tpl diff --git a/platformio/project/tpls/vscode/.vscode/c_cpp_properties.json.tpl b/platformio/project/integration/tpls/vscode/.vscode/c_cpp_properties.json.tpl similarity index 100% rename from platformio/project/tpls/vscode/.vscode/c_cpp_properties.json.tpl rename to platformio/project/integration/tpls/vscode/.vscode/c_cpp_properties.json.tpl diff --git a/platformio/project/tpls/vscode/.vscode/extensions.json.tpl b/platformio/project/integration/tpls/vscode/.vscode/extensions.json.tpl similarity index 100% rename from platformio/project/tpls/vscode/.vscode/extensions.json.tpl rename to platformio/project/integration/tpls/vscode/.vscode/extensions.json.tpl diff --git a/platformio/project/tpls/vscode/.vscode/launch.json.tpl b/platformio/project/integration/tpls/vscode/.vscode/launch.json.tpl similarity index 100% rename from platformio/project/tpls/vscode/.vscode/launch.json.tpl rename to platformio/project/integration/tpls/vscode/.vscode/launch.json.tpl diff --git a/platformio/project/options.py b/platformio/project/options.py index ad96b24c..4c29b925 100644 --- a/platformio/project/options.py +++ b/platformio/project/options.py @@ -24,7 +24,7 @@ from platformio import fs from platformio.compat import IS_WINDOWS, hashlib_encode_data -class ConfigOption(object): # pylint: disable=too-many-instance-attributes +class ConfigOption: # pylint: disable=too-many-instance-attributes def __init__( self, scope, @@ -519,6 +519,13 @@ ProjectOptions = OrderedDict( oldnames=["monitor_baud"], default=9600, ), + ConfigEnvOption( + group="monitor", + name="monitor_parity", + description="A monitor parity checking", + type=click.Choice(["N", "E", "O", "S", "M"]), + default="N", + ), ConfigEnvOption( group="monitor", name="monitor_filters", @@ -541,12 +548,24 @@ ProjectOptions = OrderedDict( ), ConfigEnvOption( group="monitor", - name="monitor_flags", - description=( - "The extra flags and options for `platformio device monitor` " - "command" - ), - multiple=True, + name="monitor_eol", + description="A monitor end of line mode", + type=click.Choice(["CR", "LF", "CRLF"]), + default="CRLF", + ), + ConfigEnvOption( + group="monitor", + name="monitor_raw", + description="Disable encodings/transformations of device output", + type=click.BOOL, + default=False, + ), + ConfigEnvOption( + group="monitor", + name="monitor_echo", + description="Enable a monitor local echo", + type=click.BOOL, + default=False, ), # Library ConfigEnvOption( diff --git a/platformio/public.py b/platformio/public.py index 51eecc52..681ab126 100644 --- a/platformio/public.py +++ b/platformio/public.py @@ -14,8 +14,8 @@ # pylint: disable=unused-import -from platformio.device.filters.base import DeviceMonitorFilterBase -from platformio.device.list import list_serial_ports +from platformio.device.list.util import list_serial_ports +from platformio.device.monitor.filters.base import DeviceMonitorFilterBase from platformio.fs import to_unix_path from platformio.platform.base import PlatformBase from platformio.project.config import ProjectConfig diff --git a/platformio/registry/client.py b/platformio/registry/client.py index 2c465e0c..a682db57 100644 --- a/platformio/registry/client.py +++ b/platformio/registry/client.py @@ -153,7 +153,7 @@ class RegistryClient(HTTPClient): x_cache_valid="1h", x_with_authorization=self.allowed_private_packages(), ) - except HTTPClientError as e: - if e.response is not None and e.response.status_code == 404: + except HTTPClientError as exc: + if exc.response is not None and exc.response.status_code == 404: return None - raise e + raise exc diff --git a/platformio/registry/mirror.py b/platformio/registry/mirror.py index d967838e..4235e88d 100644 --- a/platformio/registry/mirror.py +++ b/platformio/registry/mirror.py @@ -21,7 +21,7 @@ from platformio.http import HTTPClient from platformio.registry.client import RegistryClient -class RegistryFileMirrorIterator(object): +class RegistryFileMirrorIterator: HTTP_CLIENT_INSTANCES = {} diff --git a/platformio/remote/ac/base.py b/platformio/remote/ac/base.py index 7b76a327..8105062c 100644 --- a/platformio/remote/ac/base.py +++ b/platformio/remote/ac/base.py @@ -16,7 +16,7 @@ from twisted.internet import defer # pylint: disable=import-error from twisted.spread import pb # pylint: disable=import-error -class AsyncCommandBase(object): +class AsyncCommandBase: MAX_BUFFER_SIZE = 1024 * 1024 # 1Mb @@ -30,8 +30,8 @@ class AsyncCommandBase(object): try: self.start() - except Exception as e: - raise pb.Error(str(e)) + except Exception as exc: + raise pb.Error(str(exc)) from exc @property def id(self): diff --git a/platformio/remote/cli.py b/platformio/remote/cli.py index 1a32c549..bede41d1 100644 --- a/platformio/remote/cli.py +++ b/platformio/remote/cli.py @@ -24,11 +24,10 @@ from time import sleep import click from platformio import fs, proc -from platformio.device.commands.monitor import ( +from platformio.device.monitor.command import ( apply_project_monitor_options, device_monitor_cmd, get_project_options, - project_options_to_monitor_argv, ) from platformio.package.manager.core import inject_contrib_pysite from platformio.project.exception import NotPlatformIOProjectError @@ -271,60 +270,67 @@ def device_list(agents, json_output): @remote_device.command("monitor", short_help="Monitor remote device") @click.option("--port", "-p", help="Port, a number or a device name") @click.option( - "--baud", "-b", - type=int, - help="Set baud rate, default=%d" % ProjectOptions["env.monitor_speed"].default, + "--baud", + type=ProjectOptions["env.monitor_speed"].type, + help="Set baud/speed [default=%d]" % ProjectOptions["env.monitor_speed"].default, ) @click.option( "--parity", - default="N", - type=click.Choice(["N", "E", "O", "S", "M"]), - help="Set parity, default=N", + type=ProjectOptions["env.monitor_parity"].type, + help="Set parity [default=%s]" % ProjectOptions["env.monitor_parity"].default, ) -@click.option("--rtscts", is_flag=True, help="Enable RTS/CTS flow control, default=Off") +@click.option("--rtscts", is_flag=True, help="Enable RTS/CTS flow control") +@click.option("--xonxoff", is_flag=True, help="Enable software flow control") @click.option( - "--xonxoff", is_flag=True, help="Enable software flow control, default=Off" + "--rts", + type=ProjectOptions["env.monitor_rts"].type, + help="Set initial RTS line state", ) @click.option( - "--rts", default=None, type=click.IntRange(0, 1), help="Set initial RTS line state" + "--dtr", + type=ProjectOptions["env.monitor_dtr"].type, + help="Set initial DTR line state", ) -@click.option( - "--dtr", default=None, type=click.IntRange(0, 1), help="Set initial DTR line state" -) -@click.option("--echo", is_flag=True, help="Enable local echo, default=Off") +@click.option("--echo", is_flag=True, help="Enable local echo") @click.option( "--encoding", default="UTF-8", - help="Set the encoding for the serial port (e.g. hexlify, " - "Latin1, UTF-8), default: UTF-8", + show_default=True, + help="Set the encoding for the serial port (e.g. hexlify, Latin1, UTF-8)", +) +@click.option( + "-f", + "--filter", + "filters", + multiple=True, + help="Apply filters/text transformations", ) -@click.option("--filter", "-f", multiple=True, help="Add text transformation") @click.option( "--eol", - default="CRLF", - type=click.Choice(["CR", "LF", "CRLF"]), - help="End of line mode, default=CRLF", + type=ProjectOptions["env.monitor_eol"].type, + help="End of line mode [default=%s]" % ProjectOptions["env.monitor_eol"].default, ) -@click.option("--raw", is_flag=True, help="Do not apply any encodings/transformations") +@click.option("--raw", is_flag=True, help=ProjectOptions["env.monitor_raw"].description) @click.option( "--exit-char", type=int, default=3, + show_default=True, help="ASCII code of special character that is used to exit " - "the application, default=3 (Ctrl+C)", + "the application [default=3 (Ctrl+C)]", ) @click.option( "--menu-char", type=int, default=20, help="ASCII code of special character that is used to " - "control miniterm (menu), default=20 (DEC)", + "control terminal (menu) [default=20 (DEC)]", ) @click.option( "--quiet", is_flag=True, - help="Diagnostics: suppress non-error messages, default=Off", + help="Diagnostics: suppress non-error messages", ) @click.option( "-d", @@ -355,18 +361,17 @@ def device_monitor(ctx, agents, **kwargs): try: with fs.cd(kwargs["project_dir"]): project_options = get_project_options(kwargs["environment"]) - kwargs = apply_project_monitor_options(kwargs, project_options) except NotPlatformIOProjectError: pass - kwargs["baud"] = kwargs["baud"] or ProjectOptions["env.monitor_speed"].default + kwargs = apply_project_monitor_options(kwargs, project_options) def _tx_target(sock_dir): subcmd_argv = ["remote"] for agent in agents: subcmd_argv.extend(["--agent", agent]) subcmd_argv.extend(["device", "monitor"]) - subcmd_argv.extend(project_options_to_monitor_argv(kwargs, project_options)) + subcmd_argv.extend(project_options_to_monitor_argv(kwargs)) subcmd_argv.extend(["--sock", sock_dir]) subprocess.call([proc.where_is_program("platformio")] + subcmd_argv) @@ -381,9 +386,29 @@ def device_monitor(ctx, agents, **kwargs): return with open(sock_file, encoding="utf8") as fp: kwargs["port"] = fp.read() + kwargs["no_reconnect"] = True ctx.invoke(device_monitor_cmd, **kwargs) t.join(2) finally: fs.rmtree(sock_dir) return True + + +def project_options_to_monitor_argv(cli_options): + result = [] + for item in cli_options["filters"] or []: + result.extend(["--filter", item]) + for k, v in cli_options.items(): + if v is None or k == "filters": + continue + k = "--" + k.replace("_", "-") + if isinstance(v, bool): + if v: + result.append(k) + elif isinstance(v, tuple): + for i in v: + result.extend([k, i]) + else: + result.extend([k, str(v)]) + return result diff --git a/platformio/remote/client/agent_service.py b/platformio/remote/client/agent_service.py index a2f0a05a..3a884e6e 100644 --- a/platformio/remote/client/agent_service.py +++ b/platformio/remote/client/agent_service.py @@ -18,7 +18,7 @@ from twisted.logger import LogLevel # pylint: disable=import-error from twisted.spread import pb # pylint: disable=import-error from platformio import proc -from platformio.device.list import list_serial_ports +from platformio.device.list.util import list_serial_ports from platformio.project.config import ProjectConfig from platformio.project.exception import NotPlatformIOProjectError from platformio.remote.ac.process import ProcessAsyncCmd @@ -164,8 +164,8 @@ class RemoteAgentService(RemoteClientBase): origin_pio_ini, (os.path.getatime(back_pio_ini), os.path.getmtime(back_pio_ini)), ) - except NotPlatformIOProjectError as e: - raise pb.Error(str(e)) + except NotPlatformIOProjectError as exc: + raise pb.Error(str(exc)) from exc cmd_args = ["platformio", "--force", command, "-d", project_dir] for env in options.get("environment", []): diff --git a/platformio/remote/client/device_monitor.py b/platformio/remote/client/device_monitor.py index 7ddc0048..46ca19e4 100644 --- a/platformio/remote/client/device_monitor.py +++ b/platformio/remote/client/device_monitor.py @@ -22,7 +22,7 @@ from twisted.spread import pb # pylint: disable=import-error from platformio.remote.client.base import RemoteClientBase -class SMBridgeProtocol(protocol.Protocol): # pylint: disable=no-init +class SMBridgeProtocol(protocol.Protocol): def connectionMade(self): self.factory.add_client(self) diff --git a/platformio/remote/factory/client.py b/platformio/remote/factory/client.py index 2565e6ad..b9e3cd5a 100644 --- a/platformio/remote/factory/client.py +++ b/platformio/remote/factory/client.py @@ -38,10 +38,10 @@ class RemoteClientFactory(pb.PBClientFactory, protocol.ReconnectingClientFactory auth_token = None try: auth_token = AccountClient().fetch_authentication_token() - except Exception as e: # pylint:disable=broad-except + except Exception as exc: # pylint:disable=broad-except d = defer.Deferred() d.addErrback(self.clientAuthorizationFailed) - d.errback(pb.Error(e)) + d.errback(pb.Error(exc)) return d d = self.login( diff --git a/platformio/remote/projectsync.py b/platformio/remote/projectsync.py index 867922bd..820034ae 100644 --- a/platformio/remote/projectsync.py +++ b/platformio/remote/projectsync.py @@ -31,7 +31,7 @@ class PROJECT_SYNC_STAGE(constants.Flags): COMPLETED = constants.FlagConstant() -class ProjectSync(object): +class ProjectSync: def __init__(self, path): self.path = path if not isdir(self.path): diff --git a/platformio/run/cli.py b/platformio/run/cli.py index aa1c0d6c..eb85b10c 100644 --- a/platformio/run/cli.py +++ b/platformio/run/cli.py @@ -22,7 +22,7 @@ import click from tabulate import tabulate from platformio import app, exception, fs, util -from platformio.device.commands.monitor import device_monitor_cmd +from platformio.device.monitor.command import device_monitor_cmd from platformio.project.config import ProjectConfig from platformio.project.helpers import find_project_dir_above, load_build_metadata from platformio.run.helpers import clean_build_dir, handle_legacy_libdeps @@ -41,6 +41,7 @@ except NotImplementedError: @click.option("-e", "--environment", multiple=True) @click.option("-t", "--target", multiple=True) @click.option("--upload-port") +@click.option("--monitor-port") @click.option( "-d", "--project-dir", @@ -83,6 +84,7 @@ def cli( environment, target, upload_port, + monitor_port, project_dir, project_conf, jobs, @@ -146,6 +148,7 @@ def cli( environment, target, upload_port, + monitor_port, jobs, program_args, is_test_running, @@ -174,6 +177,7 @@ def process_env( environments, targets, upload_port, + monitor_port, jobs, program_args, is_test_running, @@ -207,7 +211,9 @@ def process_env( and "nobuild" not in ep.get_build_targets() ): ctx.invoke( - device_monitor_cmd, environment=environments[0] if environments else None + device_monitor_cmd, + port=monitor_port, + environment=environments[0] if environments else None, ) return result diff --git a/platformio/run/processor.py b/platformio/run/processor.py index 4751d352..00a96f02 100644 --- a/platformio/run/processor.py +++ b/platformio/run/processor.py @@ -20,7 +20,7 @@ from platformio.test.runners.base import CTX_META_TEST_RUNNING_NAME # pylint: disable=too-many-instance-attributes -class EnvironmentProcessor(object): +class EnvironmentProcessor: def __init__( # pylint: disable=too-many-arguments self, cmd_ctx, diff --git a/platformio/system/completion.py b/platformio/system/completion.py index 3ea80b99..3192021c 100644 --- a/platformio/system/completion.py +++ b/platformio/system/completion.py @@ -13,6 +13,8 @@ # limitations under the License. import os +import re +import subprocess from enum import Enum import click @@ -26,6 +28,14 @@ class ShellType(Enum): BASH = "bash" +def get_bash_version(): + result = subprocess.run(["bash", "--version"], capture_output=True, check=True) + match = re.search(r"version\s+(\d+)\.(\d+)", result.stdout.decode()) + if match: + return (int(match.group(1)), int(match.group(2))) + return (0, 0) + + def get_completion_install_path(shell): home_dir = os.path.expanduser("~") prog_name = click.get_current_context().find_root().info_name @@ -59,6 +69,8 @@ def is_completion_code_installed(shell, path): def install_completion_code(shell, path): + if shell == ShellType.BASH and get_bash_version() < (4, 4): + raise click.ClickException("The minimal supported Bash version is 4.4") if is_completion_code_installed(shell, path): return None append = shell != ShellType.FISH diff --git a/platformio/system/prune.py b/platformio/system/prune.py index e0ef2dd8..e00da69c 100644 --- a/platformio/system/prune.py +++ b/platformio/system/prune.py @@ -66,7 +66,7 @@ def _prune_packages(force, dry_run, silent, handler): for pkg in handler(dry_run=True) ] items = sorted(items, key=itemgetter(1), reverse=True) - reclaimed_space = sum([item[1] for item in items]) + reclaimed_space = sum(item[1] for item in items) if items and not silent: click.echo( tabulate( diff --git a/platformio/telemetry.py b/platformio/telemetry.py index c0fbf97d..a3aded5a 100644 --- a/platformio/telemetry.py +++ b/platformio/telemetry.py @@ -34,7 +34,7 @@ from platformio.proc import is_ci, is_container from platformio.project.helpers import is_platformio_project -class TelemetryBase(object): +class TelemetryBase: def __init__(self): self._params = {} @@ -198,7 +198,7 @@ class MeasurementProtocol(TelemetryBase): @util.singleton -class MPDataPusher(object): +class MPDataPusher: MAX_WORKERS = 5 @@ -274,11 +274,11 @@ class MPDataPusher(object): ) r.raise_for_status() return True - except requests.exceptions.HTTPError as e: + except requests.exceptions.HTTPError as exc: # skip Bad Request - if 400 >= e.response.status_code < 500: + if 400 >= exc.response.status_code < 500: return True - except: # pylint: disable=W0702 + except: # pylint: disable=bare-except pass self._http_offline = True return False diff --git a/platformio/test/exception.py b/platformio/test/exception.py index 048e7399..12b76ea4 100644 --- a/platformio/test/exception.py +++ b/platformio/test/exception.py @@ -23,7 +23,7 @@ class TestDirNotExistsError(UnitTestError, UserSideException): MESSAGE = ( "A test folder '{0}' does not exist.\nPlease create 'test' " - "directory in the project root and put a test set.\n" + "directory in the project root and put a test suite.\n" "More details about Unit " "Testing: https://docs.platformio.org/en/latest/advanced/" "unit-testing/index.html" diff --git a/platformio/test/helpers.py b/platformio/test/helpers.py index a789a6ea..908173d0 100644 --- a/platformio/test/helpers.py +++ b/platformio/test/helpers.py @@ -42,10 +42,10 @@ def list_test_suites(project_config, environments, filters, ignores): # filter and ignore patterns patterns = dict(filter=list(filters), ignore=list(ignores)) - for key in patterns: - if patterns[key]: # overriden from CLI + for key, value in patterns.items(): + if value: # overridden from CLI continue - patterns[key].extend( + patterns[key].extend( # pylint: disable=unnecessary-dict-index-lookup project_config.get(f"env:{env_name}", f"test_{key}", []) ) diff --git a/platformio/test/runners/base.py b/platformio/test/runners/base.py index 2a0e6cec..8f9136f1 100644 --- a/platformio/test/runners/base.py +++ b/platformio/test/runners/base.py @@ -117,6 +117,9 @@ class TestRunnerBase: def stage_building(self): if self.options.without_building: return None + # run "building" once at the "uploading" stage for the embedded target + if not self.options.without_uploading and self.platform.is_embedded(): + return None click.secho("Building...", bold=True) targets = ["__test"] if not self.options.without_debugging: @@ -125,16 +128,19 @@ class TestRunnerBase: targets.append("checkprogsize") try: return self.run_project_targets(targets) - except ReturnErrorCode: + except ReturnErrorCode as exc: raise UnitTestSuiteError( "Building stage has failed, see errors above. " "Use `pio test -vvv` option to enable verbose output." - ) + ) from exc def stage_uploading(self): - if self.options.without_uploading or not self.platform.is_embedded(): + is_embedded = self.platform.is_embedded() + if self.options.without_uploading or not is_embedded: return None - click.secho("Uploading...", bold=True) + click.secho( + "Building & Uploading..." if is_embedded else "Uploading...", bold=True + ) targets = ["upload"] if self.options.without_building: targets.append("nobuild") @@ -144,11 +150,11 @@ class TestRunnerBase: targets.append("__debug") try: return self.run_project_targets(targets) - except ReturnErrorCode: + except ReturnErrorCode as exc: raise UnitTestSuiteError( "Uploading stage has failed, see errors above. " "Use `pio test -vvv` option to enable verbose output." - ) + ) from exc def stage_testing(self): if self.options.without_testing: @@ -188,7 +194,7 @@ class TestRunnerBase: target=targets, ) - def configure_build_env(self, env): # pylint: disable=no-self-use + def configure_build_env(self, env): """ Configure SCons build environment Called in "builder/tools/piotest" tool @@ -206,5 +212,5 @@ class TestRunnerBase: self._testing_output_buffer = self._testing_output_buffer[nl_pos + 1 :] self.on_testing_line_output(line) - def on_testing_line_output(self, line): # pylint: disable=no-self-use + def on_testing_line_output(self, line): click.echo(line, nl=False) diff --git a/platformio/test/runners/doctest.py b/platformio/test/runners/doctest.py index d0fc931f..52a0916f 100644 --- a/platformio/test/runners/doctest.py +++ b/platformio/test/runners/doctest.py @@ -102,26 +102,21 @@ class DoctestTestCaseParser: class DoctestTestRunner(TestRunnerBase): - EXTRA_LIB_DEPS = ["doctest/doctest@^2.4.8"] + EXTRA_LIB_DEPS = ["doctest/doctest@^2.4.9"] def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self._tc_parser = DoctestTestCaseParser() - def configure_build_env(self, env): - env.Append(CPPDEFINES=["DOCTEST_CONFIG_COLORS_NONE"]) - if self.platform.is_embedded(): - return - env.Append(CXXFLAGS=["-std=c++11"]) - def on_testing_line_output(self, line): if self.options.verbose: click.echo(line, nl=False) test_case = self._tc_parser.parse(line) - if test_case and not self.options.verbose: - click.echo(test_case.humanize()) + if test_case: self.test_suite.add_case(test_case) + if not self.options.verbose: + click.echo(test_case.humanize()) if "[doctest] Status:" in line: self.test_suite.on_finish() diff --git a/platformio/test/runners/factory.py b/platformio/test/runners/factory.py index 6c428316..ecb6f884 100644 --- a/platformio/test/runners/factory.py +++ b/platformio/test/runners/factory.py @@ -23,7 +23,7 @@ from platformio.test.result import TestSuite from platformio.test.runners.base import TestRunnerBase, TestRunnerOptions -class TestRunnerFactory(object): +class TestRunnerFactory: @staticmethod def get_clsname(name): name = re.sub(r"[^\da-z\_\-]+", "", name, flags=re.I) @@ -56,11 +56,11 @@ class TestRunnerFactory(object): try: mod = load_python_module(module_name, custom_runner_path) - except (FileNotFoundError, ImportError): + except (FileNotFoundError, ImportError) as exc: raise UserSideException( "Could not find custom test runner " f"by this path -> {custom_runner_path}" - ) + ) from exc else: mod = importlib.import_module(module_name) runner_cls = getattr(mod, cls.get_clsname(test_framework)) diff --git a/platformio/test/runners/googletest.py b/platformio/test/runners/googletest.py index 11e51ff6..b299002e 100644 --- a/platformio/test/runners/googletest.py +++ b/platformio/test/runners/googletest.py @@ -17,7 +17,6 @@ import re import click -from platformio.compat import IS_WINDOWS from platformio.test.result import TestCase, TestCaseSource, TestStatus from platformio.test.runners.base import TestRunnerBase @@ -91,28 +90,22 @@ class GoogletestTestCaseParser: class GoogletestTestRunner(TestRunnerBase): - EXTRA_LIB_DEPS = ["google/googletest@^1.11.0"] + EXTRA_LIB_DEPS = ["google/googletest@^1.12.1"] def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self._tc_parser = GoogletestTestCaseParser() os.environ["GTEST_COLOR"] = "no" # disable ANSI symbols - def configure_build_env(self, env): - if self.platform.is_embedded(): - return - env.Append(CXXFLAGS=["-std=c++11"]) - if not IS_WINDOWS: - env.Append(CCFLAGS=["-pthread"], LINKFLAGS=["-pthread"]) - def on_testing_line_output(self, line): if self.options.verbose: click.echo(line, nl=False) test_case = self._tc_parser.parse(line) - if test_case and not self.options.verbose: - click.echo(test_case.humanize()) + if test_case: self.test_suite.add_case(test_case) + if not self.options.verbose: + click.echo(test_case.humanize()) if "Global test environment tear-down" in line: self.test_suite.on_finish() diff --git a/platformio/test/runners/readers/program.py b/platformio/test/runners/readers/program.py index b0b33130..d80d170d 100644 --- a/platformio/test/runners/readers/program.py +++ b/platformio/test/runners/readers/program.py @@ -18,14 +18,22 @@ import signal import subprocess import time -from platformio.compat import IS_WINDOWS, get_filesystem_encoding, get_locale_encoding +from platformio.compat import ( + IS_WINDOWS, + aio_get_running_loop, + get_filesystem_encoding, + get_locale_encoding, +) from platformio.test.exception import UnitTestError +EXITING_TIMEOUT = 5 # seconds + class ProgramProcessProtocol(asyncio.SubprocessProtocol): def __init__(self, test_runner, exit_future): self.test_runner = test_runner self.exit_future = exit_future + self._exit_timer = None def pipe_data_received(self, _, data): try: @@ -34,7 +42,9 @@ class ProgramProcessProtocol(asyncio.SubprocessProtocol): data = data.decode("latin-1") self.test_runner.on_testing_data_output(data) if self.test_runner.test_suite.is_finished(): - self._stop_testing() + self._exit_timer = aio_get_running_loop().call_later( + EXITING_TIMEOUT, self._stop_testing + ) def process_exited(self): self._stop_testing() @@ -42,12 +52,11 @@ class ProgramProcessProtocol(asyncio.SubprocessProtocol): def _stop_testing(self): if not self.exit_future.done(): self.exit_future.set_result(True) + if self._exit_timer: + self._exit_timer.cancel() class ProgramTestOutputReader: - - KILLING_TIMEOUT = 5 # seconds - def __init__(self, test_runner): self.test_runner = test_runner self.aio_loop = ( @@ -89,7 +98,7 @@ class ProgramTestOutputReader: # wait until subprocess will be killed start = time.time() while ( - start > (time.time() - self.KILLING_TIMEOUT) + start > (time.time() - EXITING_TIMEOUT) and transport.get_returncode() is None ): await asyncio.sleep(0.5) @@ -108,8 +117,8 @@ class ProgramTestOutputReader: raise UnitTestError( f"Program received signal {sig.name} ({signal_description})" ) - except ValueError: - raise UnitTestError("Program errored with %d code" % return_code) + except ValueError as exc: + raise UnitTestError("Program errored with %d code" % return_code) from exc def begin(self): try: diff --git a/platformio/test/runners/readers/serial.py b/platformio/test/runners/readers/serial.py index 195298a0..da4b1a05 100644 --- a/platformio/test/runners/readers/serial.py +++ b/platformio/test/runners/readers/serial.py @@ -45,8 +45,8 @@ class SerialTestOutputReader: ser.rts = self.test_runner.options.monitor_rts ser.dtr = self.test_runner.options.monitor_dtr ser.open() - except serial.SerialException as e: - click.secho(str(e), fg="red", err=True) + except serial.SerialException as exc: + click.secho(str(exc), fg="red", err=True) return None if not self.test_runner.options.no_reset: @@ -66,23 +66,16 @@ class SerialTestOutputReader: project_options = self.test_runner.project_config.items( env=self.test_runner.test_suite.env_name, as_dict=True ) - scan_options = dict( + port = find_serial_port( initial_port=self.test_runner.get_test_port(), board_config=self.test_runner.platform.board_config( project_options["board"] ), - upload_protocol=project_options.get("upload_port"), + upload_protocol=project_options.get("upload_protocol"), ensure_ready=True, ) - - elapsed = 0 - while elapsed < 5: - port = find_serial_port(**scan_options) - if port: - return port - sleep(0.25) - elapsed += 0.25 - + if port: + return port raise UserSideException( "Please specify `test_port` for environment or use " "global `--test-port` option." diff --git a/platformio/test/runners/unity.py b/platformio/test/runners/unity.py index e048c1ed..4b28a8ad 100644 --- a/platformio/test/runners/unity.py +++ b/platformio/test/runners/unity.py @@ -19,6 +19,7 @@ from pathlib import Path import click +from platformio.package.manager.library import LibraryPackageManager from platformio.test.exception import UnitTestSuiteError from platformio.test.result import TestCase, TestCaseSource, TestStatus from platformio.test.runners.base import TestRunnerBase @@ -184,6 +185,24 @@ void unityOutputComplete(void) { unittest_uart_end(); } ), ) + def __init__(self, *args, **kwargs): + """Delete when Unity > 2.5.2 is released""" + super().__init__(*args, **kwargs) + self._tmp_pre_upgrade() + + def _tmp_pre_upgrade(self): + """Delete when Unity > 2.5.2 is released""" + lm = LibraryPackageManager( + os.path.join( + self.project_config.get("platformio", "libdeps_dir"), + self.test_suite.env_name, + ), + ) + pkg = lm.get_package(self.EXTRA_LIB_DEPS[0]) + if not pkg or os.path.isfile(os.path.join(pkg.path, "platformio-build.py")): + return + lm.uninstall(pkg) + def get_unity_framework_config(self): if not self.platform.is_embedded(): return self.UNITY_FRAMEWORK_CONFIG["native"] @@ -265,8 +284,10 @@ void unityOutputComplete(void) { unittest_uart_end(); } return test_case = self.parse_test_case(line) - if test_case and not self.options.verbose: - click.echo(test_case.humanize()) + if test_case: + self.test_suite.add_case(test_case) + if not self.options.verbose: + click.echo(test_case.humanize()) if all(s in line for s in ("Tests", "Failures", "Ignored")): self.test_suite.on_finish() @@ -286,12 +307,10 @@ void unityOutputComplete(void) { unittest_uart_end(); } source = TestCaseSource( filename=data["source_file"], line=int(data.get("source_line")) ) - test_case = TestCase( + return TestCase( name=data.get("name").strip(), status=TestStatus.from_string(data.get("status")), message=(data.get("message") or "").strip() or None, stdout=line, source=source, ) - self.test_suite.add_case(test_case) - return test_case diff --git a/platformio/util.py b/platformio/util.py index 00158d45..f235c2ee 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -27,14 +27,18 @@ import click from platformio import __version__ # pylint: disable=unused-import -from platformio.device.list import list_serial_ports as get_serial_ports +from platformio.device.list.util import list_serial_ports as get_serial_ports from platformio.fs import cd, load_json from platformio.proc import exec_command # pylint: enable=unused-import +# also export list_serial_ports as get_serialports to be +# backward compatibility with arduinosam versions 3.9.0 to 3.5.0 (and possibly others) +get_serialports = get_serial_ports -class memoized(object): + +class memoized: def __init__(self, expire=0): expire = str(expire) if expire.isdigit(): @@ -61,7 +65,7 @@ class memoized(object): self.cache.clear() -class throttle(object): +class throttle: def __init__(self, threshhold): self.threshhold = threshhold # milliseconds self.last = 0 @@ -78,6 +82,50 @@ class throttle(object): return wrapper +# Retry: Begin + + +class RetryException(Exception): + pass + + +class RetryNextException(RetryException): + pass + + +class RetryStopException(RetryException): + pass + + +class retry: + + RetryNextException = RetryNextException + RetryStopException = RetryStopException + + def __init__(self, timeout=0, step=0.25): + self.timeout = timeout + self.step = step + + def __call__(self, func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + elapsed = 0 + while True: + try: + return func(*args, **kwargs) + except self.RetryNextException: + pass + if elapsed >= self.timeout: + raise self.RetryStopException() + elapsed += self.step + time.sleep(self.step) + + return wrapper + + +# Retry: End + + def singleton(cls): """From PEP-318 http://www.python.org/dev/peps/pep-0318/#examples""" _instances = {} diff --git a/scripts/99-platformio-udev.rules b/scripts/99-platformio-udev.rules index 574b6fe6..05c6ab45 100644 --- a/scripts/99-platformio-udev.rules +++ b/scripts/99-platformio-udev.rules @@ -25,7 +25,8 @@ # # CP210X USB UART -ATTRS{idVendor}=="10c4", ATTRS{idProduct}=="ea60", MODE:="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" +ATTRS{idVendor}=="10c4", ATTRS{idProduct}=="ea[67][013]", MODE:="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" +ATTRS{idVendor}=="10c4", ATTRS{idProduct}=="80a9", MODE:="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # FT231XS USB UART ATTRS{idVendor}=="0403", ATTRS{idProduct}=="6015", MODE:="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" @@ -61,13 +62,13 @@ ATTRS{idVendor}=="16c0", ATTRS{idProduct}=="04[789A]?", ENV{MTP_NO_PROBE}="1" SUBSYSTEMS=="usb", ATTRS{idVendor}=="16c0", ATTRS{idProduct}=="04[789ABCD]?", MODE:="0666" KERNEL=="ttyACM*", ATTRS{idVendor}=="16c0", ATTRS{idProduct}=="04[789B]?", MODE:="0666" -#TI Stellaris Launchpad +# TI Stellaris Launchpad ATTRS{idVendor}=="1cbe", ATTRS{idProduct}=="00fd", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" -#TI MSP430 Launchpad +# TI MSP430 Launchpad ATTRS{idVendor}=="0451", ATTRS{idProduct}=="f432", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" -#GD32V DFU Bootloader +# GD32V DFU Bootloader ATTRS{idVendor}=="28e9", ATTRS{idProduct}=="0189", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # FireBeetle-ESP32 diff --git a/setup.py b/setup.py index 51853205..2a41db90 100644 --- a/setup.py +++ b/setup.py @@ -32,8 +32,9 @@ minimal_requirements = [ "colorama", "marshmallow==%s" % ("3.*" if sys.version_info >= (3, 7) else "3.14.1"), "pyelftools>=0.27,<1", - "pyserial==3.*", + "pyserial==3.5.*", # keep in sync "device/monitor/terminal.py" "requests==2.*", + "requests==%s" % ("2.*" if sys.version_info >= (3, 7) else "2.27.1"), "semantic_version==2.10.*", "tabulate==0.8.*", "zeroconf<1", @@ -43,7 +44,7 @@ home_requirements = [ "aiofiles==0.8.*", "ajsonrpc==1.*", "starlette==%s" % ("0.20.*" if sys.version_info >= (3, 7) else "0.19.1"), - "uvicorn==%s" % ("0.17.*" if sys.version_info >= (3, 7) else "0.16.0"), + "uvicorn==%s" % ("0.18.*" if sys.version_info >= (3, 7) else "0.16.0"), "wsproto==%s" % ("1.1.*" if sys.version_info >= (3, 7) else "1.0.0"), ] @@ -61,10 +62,10 @@ setup( packages=find_packages(exclude=["tests.*", "tests"]) + ["scripts"], package_data={ "platformio": [ - "project/tpls/*/.*.tpl", - "project/tpls/*/*.tpl", - "project/tpls/*/*/*.tpl", - "project/tpls/*/.*/*.tpl", + "project/integration/tpls/*/.*.tpl", + "project/integration/tpls/*/*.tpl", + "project/integration/tpls/*/*/*.tpl", + "project/integration/tpls/*/.*/*.tpl", ], "scripts": ["99-platformio-udev.rules"], }, diff --git a/tests/commands/pkg/test_install.py b/tests/commands/pkg/test_install.py index 3f3fcc96..dcbd2d98 100644 --- a/tests/commands/pkg/test_install.py +++ b/tests/commands/pkg/test_install.py @@ -29,7 +29,9 @@ from platformio.project.config import ProjectConfig PROJECT_CONFIG_TPL = """ [env] platform = platformio/atmelavr@^3.4.0 -lib_deps = milesburton/DallasTemperature@^3.9.1 +lib_deps = + milesburton/DallasTemperature@^3.9.1 + https://github.com/esphome/ESPAsyncWebServer/archive/refs/tags/v2.1.0.zip [env:baremetal] board = uno @@ -48,7 +50,11 @@ def pkgs_to_specs(pkgs): def test_global_packages( - clirunner, validate_cliresult, func_isolated_pio_core, tmp_path + clirunner, + validate_cliresult, + func_isolated_pio_core, + get_pkg_latest_version, + tmp_path, ): # libraries result = clirunner.invoke( @@ -79,7 +85,7 @@ def test_global_packages( assert pkgs_to_specs(LibraryPackageManager().get_installed()) == [ PackageSpec("ArduinoJson@5.13.4"), PackageSpec("DallasTemperature@3.9.0+sha.964939d"), - PackageSpec("OneWire@2.3.6"), + PackageSpec("OneWire@%s" % get_pkg_latest_version("paulstoffregen/OneWire")), ] # custom storage storage_dir = tmp_path / "custom_lib_storage" @@ -120,7 +126,9 @@ def test_global_packages( ] -def test_skip_dependencies(clirunner, validate_cliresult, isolated_pio_core, tmp_path): +def test_skip_dependencies( + clirunner, validate_cliresult, isolated_pio_core, get_pkg_latest_version, tmp_path +): project_dir = tmp_path / "project" project_dir.mkdir() (project_dir / "platformio.ini").write_text(PROJECT_CONFIG_TPL) @@ -134,12 +142,18 @@ def test_skip_dependencies(clirunner, validate_cliresult, isolated_pio_core, tmp os.path.join(ProjectConfig().get("platformio", "libdeps_dir"), "devkit") ).get_installed() assert pkgs_to_specs(installed_lib_pkgs) == [ - PackageSpec("DallasTemperature@3.9.1") + PackageSpec( + "DallasTemperature@%s" + % get_pkg_latest_version("milesburton/DallasTemperature") + ), + PackageSpec("ESPAsyncWebServer-esphome@2.1.0"), ] assert len(ToolPackageManager().get_installed()) == 0 -def test_baremetal_project(clirunner, validate_cliresult, isolated_pio_core, tmp_path): +def test_baremetal_project( + clirunner, validate_cliresult, isolated_pio_core, get_pkg_latest_version, tmp_path +): project_dir = tmp_path / "project" project_dir.mkdir() (project_dir / "platformio.ini").write_text(PROJECT_CONFIG_TPL) @@ -153,15 +167,23 @@ def test_baremetal_project(clirunner, validate_cliresult, isolated_pio_core, tmp os.path.join(ProjectConfig().get("platformio", "libdeps_dir"), "baremetal") ).get_installed() assert pkgs_to_specs(installed_lib_pkgs) == [ - PackageSpec("DallasTemperature@3.9.1"), - PackageSpec("OneWire@2.3.6"), + PackageSpec( + "DallasTemperature@%s" + % get_pkg_latest_version("milesburton/DallasTemperature") + ), + PackageSpec("ESPAsyncWebServer-esphome@2.1.0"), + PackageSpec( + "OneWire@%s" % get_pkg_latest_version("paulstoffregen/OneWire") + ), ] assert pkgs_to_specs(ToolPackageManager().get_installed()) == [ PackageSpec("toolchain-atmelavr@1.70300.191015"), ] -def test_project(clirunner, validate_cliresult, isolated_pio_core, tmp_path): +def test_project( + clirunner, validate_cliresult, isolated_pio_core, get_pkg_latest_version, tmp_path +): project_dir = tmp_path / "project" project_dir.mkdir() (project_dir / "platformio.ini").write_text(PROJECT_CONFIG_TPL) @@ -176,15 +198,22 @@ def test_project(clirunner, validate_cliresult, isolated_pio_core, tmp_path): os.path.join(config.get("platformio", "libdeps_dir"), "devkit") ) assert pkgs_to_specs(lm.get_installed()) == [ - PackageSpec("DallasTemperature@3.9.1"), - PackageSpec("OneWire@2.3.6"), + PackageSpec( + "DallasTemperature@%s" + % get_pkg_latest_version("milesburton/DallasTemperature") + ), + PackageSpec("ESPAsyncWebServer-esphome@2.1.0"), + PackageSpec( + "OneWire@%s" % get_pkg_latest_version("paulstoffregen/OneWire") + ), ] assert pkgs_to_specs(ToolPackageManager().get_installed()) == [ PackageSpec("framework-arduino-avr-attiny@1.5.2"), PackageSpec("toolchain-atmelavr@1.70300.191015"), ] assert config.get("env:devkit", "lib_deps") == [ - "milesburton/DallasTemperature@^3.9.1" + "milesburton/DallasTemperature@^3.9.1", + "https://github.com/esphome/ESPAsyncWebServer/archive/refs/tags/v2.1.0.zip", ] # test "Already up-to-date" @@ -196,7 +225,9 @@ def test_project(clirunner, validate_cliresult, isolated_pio_core, tmp_path): assert "Already up-to-date" in result.output -def test_private_lib_deps(clirunner, validate_cliresult, isolated_pio_core, tmp_path): +def test_private_lib_deps( + clirunner, validate_cliresult, isolated_pio_core, get_pkg_latest_version, tmp_path +): project_dir = tmp_path / "project" private_lib_dir = project_dir / "lib" / "private" private_lib_dir.mkdir(parents=True) @@ -241,7 +272,9 @@ platform = native config.get("platformio", "lib_dir") ).get_installed() assert pkgs_to_specs(installed_private_pkgs) == [ - PackageSpec("OneWire@2.3.6"), + PackageSpec( + "OneWire@%s" % get_pkg_latest_version("paulstoffregen/OneWire") + ), PackageSpec("My Private Lib@1.0.0"), ] installed_env_pkgs = LibraryPackageManager( @@ -249,12 +282,15 @@ platform = native ).get_installed() assert pkgs_to_specs(installed_env_pkgs) == [ PackageSpec("ArduinoJson@5.13.4"), - PackageSpec("DallasTemperature@3.9.1"), + PackageSpec( + "DallasTemperature@%s" + % get_pkg_latest_version("milesburton/DallasTemperature") + ), ] def test_remove_project_unused_libdeps( - clirunner, validate_cliresult, isolated_pio_core, tmp_path + clirunner, validate_cliresult, isolated_pio_core, get_pkg_latest_version, tmp_path ): project_dir = tmp_path / "project" project_dir.mkdir() @@ -269,8 +305,14 @@ def test_remove_project_unused_libdeps( storage_dir = os.path.join(config.get("platformio", "libdeps_dir"), "baremetal") lm = LibraryPackageManager(storage_dir) assert pkgs_to_specs(lm.get_installed()) == [ - PackageSpec("DallasTemperature@3.9.1"), - PackageSpec("OneWire@2.3.6"), + PackageSpec( + "DallasTemperature@%s" + % get_pkg_latest_version("milesburton/DallasTemperature") + ), + PackageSpec("ESPAsyncWebServer-esphome@2.1.0"), + PackageSpec( + "OneWire@%s" % get_pkg_latest_version("paulstoffregen/OneWire") + ), ] # add new deps @@ -285,8 +327,14 @@ def test_remove_project_unused_libdeps( lm = LibraryPackageManager(storage_dir) assert pkgs_to_specs(lm.get_installed()) == [ PackageSpec("ArduinoJson@5.13.4"), - PackageSpec("DallasTemperature@3.9.1"), - PackageSpec("OneWire@2.3.6"), + PackageSpec( + "DallasTemperature@%s" + % get_pkg_latest_version("milesburton/DallasTemperature") + ), + PackageSpec("ESPAsyncWebServer-esphome@2.1.0"), + PackageSpec( + "OneWire@%s" % get_pkg_latest_version("paulstoffregen/OneWire") + ), ] # manually remove from cofiguration file diff --git a/tests/commands/pkg/test_update.py b/tests/commands/pkg/test_update.py index c70e2b25..06ab92a7 100644 --- a/tests/commands/pkg/test_update.py +++ b/tests/commands/pkg/test_update.py @@ -145,7 +145,9 @@ def test_global_packages( assert isinstance(result.exception, UnknownPackageError) -def test_project(clirunner, validate_cliresult, isolated_pio_core, tmp_path): +def test_project( + clirunner, validate_cliresult, isolated_pio_core, get_pkg_latest_version, tmp_path +): project_dir = tmp_path / "project" project_dir.mkdir() (project_dir / "platformio.ini").write_text(PROJECT_OUTDATED_CONFIG_TPL) @@ -161,7 +163,9 @@ def test_project(clirunner, validate_cliresult, isolated_pio_core, tmp_path): ) assert pkgs_to_specs(lm.get_installed()) == [ PackageSpec("DallasTemperature@3.8.1"), - PackageSpec("OneWire@2.3.6"), + PackageSpec( + "OneWire@%s" % get_pkg_latest_version("paulstoffregen/OneWire") + ), ] assert pkgs_to_specs(PlatformPackageManager().get_installed()) == [ PackageSpec("atmelavr@2.2.0") @@ -187,8 +191,13 @@ def test_project(clirunner, validate_cliresult, isolated_pio_core, tmp_path): assert pkgs[0].metadata.name == "atmelavr" assert pkgs[0].metadata.version.major == 3 assert pkgs_to_specs(lm.get_installed()) == [ - PackageSpec("DallasTemperature@3.9.1"), - PackageSpec("OneWire@2.3.6"), + PackageSpec( + "DallasTemperature@%s" + % get_pkg_latest_version("milesburton/DallasTemperature") + ), + PackageSpec( + "OneWire@%s" % get_pkg_latest_version("paulstoffregen/OneWire") + ), ] assert pkgs_to_specs(ToolPackageManager().get_installed()) == [ PackageSpec("framework-arduino-avr-attiny@1.3.2"), @@ -211,7 +220,7 @@ def test_project(clirunner, validate_cliresult, isolated_pio_core, tmp_path): def test_custom_project_libraries( - clirunner, validate_cliresult, isolated_pio_core, tmp_path + clirunner, validate_cliresult, isolated_pio_core, get_pkg_latest_version, tmp_path ): project_dir = tmp_path / "project" project_dir.mkdir() @@ -230,7 +239,9 @@ def test_custom_project_libraries( ) assert pkgs_to_specs(lm.get_installed()) == [ PackageSpec("DallasTemperature@3.8.1"), - PackageSpec("OneWire@2.3.6"), + PackageSpec( + "OneWire@%s" % get_pkg_latest_version("paulstoffregen/OneWire") + ), ] # update package result = clirunner.invoke( @@ -260,8 +271,13 @@ def test_custom_project_libraries( os.path.join(config.get("platformio", "libdeps_dir"), "devkit") ) assert pkgs_to_specs(lm.get_installed()) == [ - PackageSpec("DallasTemperature@3.9.1"), - PackageSpec("OneWire@2.3.6"), + PackageSpec( + "DallasTemperature@%s" + % get_pkg_latest_version("milesburton/DallasTemperature") + ), + PackageSpec( + "OneWire@%s" % get_pkg_latest_version("paulstoffregen/OneWire") + ), ] assert config.get("env:devkit", "lib_deps") == [ "milesburton/DallasTemperature@^3.8.0" diff --git a/tests/commands/test_ci.py b/tests/commands/test_ci.py index 01ac9c37..ae632280 100644 --- a/tests/commands/test_ci.py +++ b/tests/commands/test_ci.py @@ -15,7 +15,7 @@ from os.path import isfile, join from platformio.commands.ci import cli as cmd_ci -from platformio.commands.lib.command import cli as cmd_lib +from platformio.package.commands.install import package_install_cmd def test_ci_empty(clirunner): @@ -170,7 +170,8 @@ def test_ci_project_conf(clirunner, validate_cliresult): def test_ci_lib_and_board(clirunner, tmpdir_factory, validate_cliresult): storage_dir = str(tmpdir_factory.mktemp("lib")) result = clirunner.invoke( - cmd_lib, ["--storage-dir", storage_dir, "install", "1@2.3.2"] + package_install_cmd, + ["--global", "--storage-dir", storage_dir, "--library", "1"], ) validate_cliresult(result) @@ -182,7 +183,7 @@ def test_ci_lib_and_board(clirunner, tmpdir_factory, validate_cliresult): "OneWire", "examples", "DS2408_Switch", - "DS2408_Switch.pde", + "DS2408_Switch.ino", ), "-l", join(storage_dir, "OneWire"), diff --git a/tests/commands/test_lib.py b/tests/commands/test_lib.py index cf1e53b6..8f47ad61 100644 --- a/tests/commands/test_lib.py +++ b/tests/commands/test_lib.py @@ -20,7 +20,7 @@ import os import pytest import semantic_version -from platformio.commands.lib.command import cli as cmd_lib +from platformio.commands.lib import cli as cmd_lib from platformio.package.meta import PackageType from platformio.package.vcsclient import VCSClientFactory from platformio.project.config import ProjectConfig @@ -137,11 +137,7 @@ lib_deps = # test list result = clirunner.invoke(cmd_lib, ["-d", str(project_dir), "list"]) validate_cliresult(result) - assert "Version: 0.8.3+sha." in result.stdout - assert ( - "Source: git+https://github.com/OttoWinter/async-mqtt-client.git#v0.8.3" - in result.stdout - ) + assert "AsyncMqttClient-esphome @ 0.8.3+sha.f5aa899" in result.stdout result = clirunner.invoke( cmd_lib, ["-d", str(project_dir), "list", "--json-output"] ) diff --git a/tests/commands/test_lib_complex.py b/tests/commands/test_lib_complex.py index 31f61fc6..eb56c5b7 100644 --- a/tests/commands/test_lib_complex.py +++ b/tests/commands/test_lib_complex.py @@ -18,7 +18,7 @@ import json import re from platformio.cli import PlatformioCLI -from platformio.commands.lib.command import cli as cmd_lib +from platformio.commands.lib import cli as cmd_lib from platformio.package.exception import UnknownPackageError from platformio.util import strip_ansi_codes @@ -28,12 +28,12 @@ PlatformioCLI.leftover_args = ["--json-output"] # hook for click def test_search(clirunner, validate_cliresult): result = clirunner.invoke(cmd_lib, ["search", "DHT22"]) validate_cliresult(result) - match = re.search(r"Found\s+(\d+)\slibraries:", result.output) + match = re.search(r"Found\s+(\d+)\spackages", result.output) assert int(match.group(1)) > 2 result = clirunner.invoke(cmd_lib, ["search", "DHT22", "--platform=timsp430"]) validate_cliresult(result) - match = re.search(r"Found\s+(\d+)\slibraries:", result.output) + match = re.search(r"Found\s+(\d+)\spackages", result.output) assert int(match.group(1)) > 1 @@ -175,10 +175,10 @@ def test_global_lib_list(clirunner, validate_cliresult): assert all( n in result.output for n in ( - "Source: https://github.com/Pedroalbuquerque/ESP32WebServer/archive/master.zip", - "Version: 5.10.1", - "Source: git+https://github.com/gioblu/PJON.git#3.0", - "Version: 3.0.0+sha.1fb26fd", + "required: https://github.com/Pedroalbuquerque/ESP32WebServer/archive/master.zip", + "ArduinoJson @ 5.10.1", + "required: git+https://github.com/gioblu/PJON.git#3.0", + "PJON @ 3.0.0+sha.1fb26f", ) ) @@ -251,11 +251,12 @@ def test_global_lib_update(clirunner, validate_cliresult): validate_cliresult(result) assert "Removing NeoPixelBus @ 2.2.4" in strip_ansi_codes(result.output) - # update rest libraries - result = clirunner.invoke(cmd_lib, ["-g", "update"]) + # update all libraries + result = clirunner.invoke( + cmd_lib, + ["-g", "update", "adafruit/Adafruit PN532", "marvinroger/AsyncMqttClient"], + ) validate_cliresult(result) - assert result.output.count("+sha.") == 4 - assert result.output.count("already up-to-date") == 14 # update unknown library result = clirunner.invoke(cmd_lib, ["-g", "update", "Unknown"]) @@ -271,7 +272,7 @@ def test_global_lib_uninstall(clirunner, validate_cliresult, isolated_pio_core): items = sorted(items, key=lambda item: item["__pkg_dir"]) result = clirunner.invoke(cmd_lib, ["-g", "uninstall", items[0]["__pkg_dir"]]) validate_cliresult(result) - assert ("Removing %s" % items[0]["name"]) in strip_ansi_codes(result.output) + assert "Removing %s" % items[0]["name"] in strip_ansi_codes(result.output) # uninstall the rest libraries result = clirunner.invoke( @@ -314,7 +315,7 @@ def test_global_lib_uninstall(clirunner, validate_cliresult, isolated_pio_core): def test_lib_show(clirunner, validate_cliresult): result = clirunner.invoke(cmd_lib, ["show", "64"]) validate_cliresult(result) - assert all(s in result.output for s in ("ArduinoJson", "Arduino", "Atmel AVR")) + assert all(s in result.output for s in ("ArduinoJson", "Arduino")) result = clirunner.invoke(cmd_lib, ["show", "OneWire", "--json-output"]) validate_cliresult(result) assert "OneWire" in result.output @@ -328,13 +329,6 @@ def test_lib_builtin(clirunner, validate_cliresult): def test_lib_stats(clirunner, validate_cliresult): - result = clirunner.invoke(cmd_lib, ["stats"]) - validate_cliresult(result) - assert all( - s in result.output - for s in ("UPDATED", "POPULAR", "https://platformio.org/lib/show") - ) - result = clirunner.invoke(cmd_lib, ["stats", "--json-output"]) validate_cliresult(result) assert set( diff --git a/tests/commands/test_platform.py b/tests/commands/test_platform.py index 604e392f..967bf51e 100644 --- a/tests/commands/test_platform.py +++ b/tests/commands/test_platform.py @@ -15,6 +15,7 @@ # pylint: disable=unused-argument import json +import os from platformio.commands import platform as cli_platform from platformio.package.exception import UnknownPackageError @@ -36,7 +37,7 @@ def test_search_json_output(clirunner, validate_cliresult, isolated_pio_core): def test_search_raw_output(clirunner, validate_cliresult): result = clirunner.invoke(cli_platform.platform_search, ["arduino"]) validate_cliresult(result) - assert "teensy" in result.output + assert "atmelavr" in result.output def test_install_unknown_version(clirunner): @@ -75,8 +76,7 @@ def test_install_known_version(clirunner, validate_cliresult, isolated_pio_core) validate_cliresult(result) output = strip_ansi_codes(result.output) assert "atmelavr @ 2.0.0" in output - assert "Installing tool-avrdude @" in output - assert len(isolated_pio_core.join("packages").listdir()) == 1 + assert not os.path.isdir(str(isolated_pio_core.join("packages"))) def test_install_from_vcs(clirunner, validate_cliresult, isolated_pio_core): @@ -89,7 +89,7 @@ def test_install_from_vcs(clirunner, validate_cliresult, isolated_pio_core): ) validate_cliresult(result) assert "espressif8266" in result.output - assert len(isolated_pio_core.join("packages").listdir()) == 1 + assert not os.path.isdir(str(isolated_pio_core.join("packages"))) def test_list_json_output(clirunner, validate_cliresult): @@ -109,6 +109,11 @@ def test_list_raw_output(clirunner, validate_cliresult): def test_update_check(clirunner, validate_cliresult, isolated_pio_core): + result = clirunner.invoke( + cli_platform.package_install_cmd, + ["--global", "--tool", "platformio/tool-avrdude@~1.60300.0"], + ) + validate_cliresult(result) result = clirunner.invoke( cli_platform.platform_update, ["--dry-run", "--json-output"] ) @@ -120,7 +125,7 @@ def test_update_check(clirunner, validate_cliresult, isolated_pio_core): def test_update_raw(clirunner, validate_cliresult, isolated_pio_core): - result = clirunner.invoke(cli_platform.platform_update) + result = clirunner.invoke(cli_platform.platform_update, ["atmelavr"]) validate_cliresult(result) output = strip_ansi_codes(result.output) assert "Removing atmelavr @ 2.0.0" in output diff --git a/tests/commands/test_run.py b/tests/commands/test_run.py index 5a2617fe..23d1e096 100644 --- a/tests/commands/test_run.py +++ b/tests/commands/test_run.py @@ -17,7 +17,7 @@ from pathlib import Path from platformio.run.cli import cli as cmd_run -def test_build_flags(clirunner, validate_cliresult, tmpdir): +def test_generic_build(clirunner, validate_cliresult, tmpdir): build_flags = [ ("-D TEST_INT=13", "-DTEST_INT=13"), ("-DTEST_SINGLE_MACRO", "-DTEST_SINGLE_MACRO"), @@ -28,7 +28,9 @@ def test_build_flags(clirunner, validate_cliresult, tmpdir): """ [env:native] platform = native -extra_scripts = extra.py +extra_scripts = + pre:pre_script.py + post_script.py lib_ldf_mode = deep+ build_src_flags = -DI_AM_ONLY_SRC_FLAG build_flags = @@ -38,7 +40,17 @@ build_flags = % " ".join([f[0] for f in build_flags]) ) - tmpdir.join("extra.py").write( + tmpdir.join("pre_script.py").write( + """ +Import("env") + +def post_prog_action(source, target, env): + print("post_prog_action is called") + +env.AddPostAction("$PROGPATH", post_prog_action) + """ + ) + tmpdir.join("post_script.py").write( """ Import("projenv") @@ -102,6 +114,7 @@ void dummy(void ) {}; result = clirunner.invoke(cmd_run, ["--project-dir", str(tmpdir), "--verbose"]) validate_cliresult(result) + assert "post_prog_action is called" in result.output build_output = result.output[result.output.find("Scanning dependencies...") :] for flag in build_flags: assert flag[1] in build_output, flag @@ -112,7 +125,16 @@ def test_build_unflags(clirunner, validate_cliresult, tmpdir): """ [env:native] platform = native -build_unflags = -DTMP_MACRO1=45 -I. -DNON_EXISTING_MACRO -lunknownLib -Os +build_unflags = + -DTMP_MACRO_1=45 + -DTMP_MACRO_3=13 + -DTMP_MACRO_4 + -DNON_EXISTING_MACRO + -I. + -lunknownLib + -Os +build_flags = + -DTMP_MACRO_3=10 extra_scripts = pre:extra.py """ ) @@ -121,9 +143,10 @@ extra_scripts = pre:extra.py """ Import("env") env.Append(CPPPATH="%s") -env.Append(CPPDEFINES="TMP_MACRO1") -env.Append(CPPDEFINES=["TMP_MACRO2"]) -env.Append(CPPDEFINES=("TMP_MACRO3", 13)) +env.Append(CPPDEFINES="TMP_MACRO_1") +env.Append(CPPDEFINES=["TMP_MACRO_2"]) +env.Append(CPPDEFINES=[("TMP_MACRO_3", 13)]) +env.Append(CPPDEFINES=[("TMP_MACRO_4", 4)]) env.Append(CCFLAGS=["-Os"]) env.Append(LIBS=["unknownLib"]) """ @@ -132,8 +155,20 @@ env.Append(LIBS=["unknownLib"]) tmpdir.mkdir("src").join("main.c").write( """ -#ifdef TMP_MACRO1 -#error "TMP_MACRO1 should be removed" +#ifndef TMP_MACRO_1 +#error "TMP_MACRO_1 should be defined" +#endif + +#ifndef TMP_MACRO_2 +#error "TMP_MACRO_2 should be defined" +#endif + +#if TMP_MACRO_3 != 10 +#error "TMP_MACRO_3 should be 10" +#endif + +#ifdef TMP_MACRO_4 +#error "TMP_MACRO_4 should not be defined" #endif int main() { @@ -252,5 +287,51 @@ platform = native lib_deps = symlink://../External """ ) - result = clirunner.invoke(cmd_run, ["--project-dir", str(project_dir), "--verbose"]) + result = clirunner.invoke(cmd_run, ["--project-dir", str(project_dir)]) validate_cliresult(result) + + +def test_stringification(clirunner, validate_cliresult, tmp_path: Path): + project_dir = tmp_path / "project" + src_dir = project_dir / "src" + src_dir.mkdir(parents=True) + (src_dir / "main.c").write_text( + """ +#include +int main(void) { + printf("MACRO_1=<%s>\\n", MACRO_1); + printf("MACRO_2=<%s>\\n", MACRO_2); + printf("MACRO_3=<%s>\\n", MACRO_3); + printf("MACRO_4=<%s>\\n", MACRO_4); + return(0); +} +""" + ) + (project_dir / "platformio.ini").write_text( + """ +[env:native] +platform = native +extra_scripts = script.py +build_flags = + '-DMACRO_1="Hello World!"' + '-DMACRO_2="Text is \\\\"Quoted\\\\""' + """ + ) + (project_dir / "script.py").write_text( + """ +Import("projenv") + +projenv.Append(CPPDEFINES=[ + ("MACRO_3", projenv.StringifyMacro('Hello "World"! Isn\\'t true?')), + ("MACRO_4", projenv.StringifyMacro("Special chars: ',(,),[,],:")) +]) + """ + ) + result = clirunner.invoke( + cmd_run, ["--project-dir", str(project_dir), "-t", "exec"] + ) + validate_cliresult(result) + assert "MACRO_1=" in result.output + assert 'MACRO_2=' in result.output + assert 'MACRO_3=' in result.output + assert "MACRO_4=" in result.output diff --git a/tests/commands/test_test.py b/tests/commands/test_test.py index 2a6c7c56..26089500 100644 --- a/tests/commands/test_test.py +++ b/tests/commands/test_test.py @@ -13,6 +13,7 @@ # limitations under the License. import os +import shutil import sys import xml.etree.ElementTree as ET from pathlib import Path @@ -26,12 +27,16 @@ from platformio.test.cli import cli as pio_test_cmd def test_calculator_example(tmp_path: Path): junit_output_path = tmp_path / "junit.xml" + project_dir = tmp_path / "project" + shutil.copytree( + os.path.join("examples", "unit-testing", "calculator"), str(project_dir) + ) result = proc.exec_command( [ "platformio", "test", "-d", - os.path.join("examples", "unit-testing", "calculator"), + str(project_dir), "-e", "uno", "-e", @@ -67,11 +72,15 @@ def test_calculator_example(tmp_path: Path): def test_list_tests(clirunner, validate_cliresult, tmp_path: Path): json_output_path = tmp_path / "report.json" + project_dir = tmp_path / "project" + shutil.copytree( + os.path.join("examples", "unit-testing", "calculator"), str(project_dir) + ) result = clirunner.invoke( pio_test_cmd, [ "-d", - os.path.join("examples", "unit-testing", "calculator"), + str(project_dir), "--list-tests", "--json-output-path", str(json_output_path), @@ -309,10 +318,15 @@ platform = native """ ) test_dir = project_dir.mkdir("test") - test_dir.join("test_main.c").write( + test_dir.join("test_main.h").write( """ #include #include + """ + ) + test_dir.join("test_main.c").write( + """ +#include "test_main.h" void setUp(){ printf("setUp called"); @@ -587,14 +601,21 @@ int main(int argc, char **argv) assert json_report["failure_nums"] == 1 +@pytest.mark.skipif( + sys.platform == "win32" and os.environ.get("GITHUB_ACTIONS") == "true", + reason="skip Github Actions on Windows (MinGW issue)", +) def test_googletest_framework(clirunner, tmp_path: Path): - project_dir = os.path.join("examples", "unit-testing", "googletest") + project_dir = tmp_path / "project" + shutil.copytree( + os.path.join("examples", "unit-testing", "googletest"), str(project_dir) + ) junit_output_path = tmp_path / "junit.xml" result = clirunner.invoke( pio_test_cmd, [ "-d", - project_dir, + str(project_dir), "-e", "native", "--junit-output-path", diff --git a/tests/conftest.py b/tests/conftest.py index b26aaf49..eedf54c5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -13,6 +13,7 @@ # limitations under the License. import email +import functools import imaplib import os import time @@ -21,6 +22,8 @@ import pytest from click.testing import CliRunner from platformio import http +from platformio.package.meta import PackageSpec, PackageType +from platformio.registry.client import RegistryClient def pytest_configure(config): @@ -131,3 +134,17 @@ def receive_email(): # pylint:disable=redefined-outer-name, too-many-locals return result return _receive_email + + +@pytest.fixture(scope="session") +def get_pkg_latest_version(): + @functools.lru_cache() + def wrap(spec, pkg_type=None): + if not isinstance(spec, PackageSpec): + spec = PackageSpec(spec) + pkg_type = pkg_type or PackageType.LIBRARY + client = RegistryClient() + pkg = client.get_package(pkg_type, spec.owner, spec.name) + return pkg["version"]["name"] + + return wrap diff --git a/tests/package/test_manifest.py b/tests/package/test_manifest.py index c16ce04b..828412f4 100644 --- a/tests/package/test_manifest.py +++ b/tests/package/test_manifest.py @@ -531,9 +531,9 @@ includes=MozziGuts.h errors = None try: ManifestSchema().load_manifest(raw_data) - except ManifestValidationError as e: - data = e.valid_data - errors = e.messages + except ManifestValidationError as exc: + data = exc.valid_data + errors = exc.messages assert errors["authors"] diff --git a/tests/package/test_meta.py b/tests/package/test_meta.py index 4faabeba..f7eae748 100644 --- a/tests/package/test_meta.py +++ b/tests/package/test_meta.py @@ -18,6 +18,7 @@ import jsondiff import semantic_version from platformio.package.meta import ( + PackageCompatibility, PackageMetaData, PackageOutdatedResult, PackageSpec, @@ -312,3 +313,25 @@ def test_metadata_load(tmpdir_factory): metadata.dump(str(piopm_path)) restored_metadata = PackageMetaData.load(str(piopm_path)) assert metadata == restored_metadata + + +def test_compatibility(): + assert PackageCompatibility().is_compatible(PackageCompatibility()) + assert PackageCompatibility().is_compatible( + PackageCompatibility(platforms=["espressif32"]) + ) + assert PackageCompatibility(frameworks=["arduino"]).is_compatible( + PackageCompatibility(platforms=["espressif32"]) + ) + assert PackageCompatibility(platforms="espressif32").is_compatible( + PackageCompatibility(platforms=["espressif32"]) + ) + assert PackageCompatibility( + platforms="espressif32", frameworks=["arduino"] + ).is_compatible(PackageCompatibility(platforms=None)) + assert PackageCompatibility( + platforms="espressif32", frameworks=["arduino"] + ).is_compatible(PackageCompatibility(platforms=["*"])) + assert not PackageCompatibility( + platforms="espressif32", frameworks=["arduino"] + ).is_compatible(PackageCompatibility(platforms=["atmelavr"])) diff --git a/tests/project/test_savedeps.py b/tests/project/test_savedeps.py index 9740ce71..2a16511d 100644 --- a/tests/project/test_savedeps.py +++ b/tests/project/test_savedeps.py @@ -54,7 +54,7 @@ def test_save_libraries(tmp_path): PackageSpec("https://github.com/nanopb/nanopb.git"), ] - # add to the sepcified environment + # add to the specified environment save_project_dependencies( str(project_dir), specs, scope="lib_deps", action="add", environments=["debug"] ) @@ -140,7 +140,7 @@ def test_save_tools(tmp_path): PackageSpec("platformio/tool-esptoolpy"), ] - # add to the sepcified environment + # add to the specified environment save_project_dependencies( str(project_dir), specs, diff --git a/tox.ini b/tox.ini index 92555cf5..8332f8d3 100644 --- a/tox.ini +++ b/tox.ini @@ -41,7 +41,8 @@ commands = [testenv:lint] commands = {envpython} --version - pylint --rcfile=./.pylintrc ./platformio ./tests + pylint --rcfile=./.pylintrc ./platformio + pylint --rcfile=./.pylintrc ./tests [testenv:testcore] commands = @@ -54,7 +55,6 @@ commands = py.test -v --basetemp="{envtmpdir}" tests/test_examples.py [testenv:docs] -; basepython = ~/.pyenv/versions/3.6.12/bin/python deps = sphinx sphinx_rtd_theme