From 99d049a6dd557c270d05cc6da4e71a8b474ac05b Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 1 Jun 2022 17:24:02 +0300 Subject: [PATCH 01/93] Run deployment on tags --- .github/workflows/deployment.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/deployment.yml b/.github/workflows/deployment.yml index d961da8c..d7035d5a 100644 --- a/.github/workflows/deployment.yml +++ b/.github/workflows/deployment.yml @@ -3,8 +3,9 @@ name: Deployment on: push: branches: - - master - "release/**" + tags: + - v* jobs: deployment: From 26f897cb55ad883db2d4793465d22db21d0c4a15 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 1 Jun 2022 18:29:49 +0300 Subject: [PATCH 02/93] Fix PyLint --- platformio/system/prune.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/system/prune.py b/platformio/system/prune.py index e0ef2dd8..e00da69c 100644 --- a/platformio/system/prune.py +++ b/platformio/system/prune.py @@ -66,7 +66,7 @@ def _prune_packages(force, dry_run, silent, handler): for pkg in handler(dry_run=True) ] items = sorted(items, key=itemgetter(1), reverse=True) - reclaimed_space = sum([item[1] for item in items]) + reclaimed_space = sum(item[1] for item in items) if items and not silent: click.echo( tabulate( From 1b17234c41a998bc51512e6cd6e3bd990c47d8c7 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 2 Jun 2022 18:05:41 +0300 Subject: [PATCH 03/93] Fixed an issue when a custom "pio test --project-config" was not handled properly // Resolve #4299 --- HISTORY.rst | 5 +++++ platformio/builder/main.py | 4 +++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index e37671e5..a3a0661b 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -13,6 +13,11 @@ PlatformIO Core 6 **A professional collaborative platform for declarative, safety-critical, and test-driven embedded development.** +6.0.3 (2022-??-??) +~~~~~~~~~~~~~~~~~~ + +- Fixed an issue when a custom `pio test --project-config `__ was not handled properly (`issue #4299 `_) + 6.0.2 (2022-06-01) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/builder/main.py b/platformio/builder/main.py index 6aeee4f4..9cbd387b 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -28,7 +28,7 @@ from SCons.Script import DefaultEnvironment # pylint: disable=import-error from SCons.Script import Import # pylint: disable=import-error from SCons.Script import Variables # pylint: disable=import-error -from platformio import compat, fs +from platformio import app, compat, fs from platformio.platform.base import PlatformBase from platformio.proc import get_pythonexe_path from platformio.project.helpers import get_project_dir @@ -110,6 +110,8 @@ env.Replace( # Setup project optional directories config = env.GetProjectConfig() +app.set_session_var("custom_project_conf", config.path) + env.Replace( PROJECT_DIR=get_project_dir(), PROJECT_CORE_DIR=config.get("platformio", "core_dir"), From b15a4e746a0a731e74a12d9865160a7d74539534 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 2 Jun 2022 18:09:11 +0300 Subject: [PATCH 04/93] Bump version to 6.0.3a1 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 7fe4bec7..6f1c57a7 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (6, 0, 2) +VERSION = (6, 0, "3a1") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 30937df4e64f0fdcc75fe2eed44fb539d481a8b5 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 10 Jun 2022 12:46:52 +0300 Subject: [PATCH 05/93] Lock "requests" dependency for Python 3.6 --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 51853205..a49c1bc9 100644 --- a/setup.py +++ b/setup.py @@ -34,6 +34,7 @@ minimal_requirements = [ "pyelftools>=0.27,<1", "pyserial==3.*", "requests==2.*", + "requests==%s" % ("2.*" if sys.version_info >= (3, 7) else "2.27.1"), "semantic_version==2.10.*", "tabulate==0.8.*", "zeroconf<1", From 51ab0bbd3c56e16d077a2ee32c3c30083f5ff1fc Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 10 Jun 2022 13:32:48 +0300 Subject: [PATCH 06/93] Update tests --- tests/commands/pkg/test_install.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/commands/pkg/test_install.py b/tests/commands/pkg/test_install.py index 3f3fcc96..449783d1 100644 --- a/tests/commands/pkg/test_install.py +++ b/tests/commands/pkg/test_install.py @@ -79,7 +79,7 @@ def test_global_packages( assert pkgs_to_specs(LibraryPackageManager().get_installed()) == [ PackageSpec("ArduinoJson@5.13.4"), PackageSpec("DallasTemperature@3.9.0+sha.964939d"), - PackageSpec("OneWire@2.3.6"), + PackageSpec("OneWire@2.3.7"), ] # custom storage storage_dir = tmp_path / "custom_lib_storage" @@ -134,7 +134,7 @@ def test_skip_dependencies(clirunner, validate_cliresult, isolated_pio_core, tmp os.path.join(ProjectConfig().get("platformio", "libdeps_dir"), "devkit") ).get_installed() assert pkgs_to_specs(installed_lib_pkgs) == [ - PackageSpec("DallasTemperature@3.9.1") + PackageSpec("DallasTemperature@3.10.0") ] assert len(ToolPackageManager().get_installed()) == 0 @@ -153,8 +153,8 @@ def test_baremetal_project(clirunner, validate_cliresult, isolated_pio_core, tmp os.path.join(ProjectConfig().get("platformio", "libdeps_dir"), "baremetal") ).get_installed() assert pkgs_to_specs(installed_lib_pkgs) == [ - PackageSpec("DallasTemperature@3.9.1"), - PackageSpec("OneWire@2.3.6"), + PackageSpec("DallasTemperature@3.10.0"), + PackageSpec("OneWire@2.3.7"), ] assert pkgs_to_specs(ToolPackageManager().get_installed()) == [ PackageSpec("toolchain-atmelavr@1.70300.191015"), @@ -176,8 +176,8 @@ def test_project(clirunner, validate_cliresult, isolated_pio_core, tmp_path): os.path.join(config.get("platformio", "libdeps_dir"), "devkit") ) assert pkgs_to_specs(lm.get_installed()) == [ - PackageSpec("DallasTemperature@3.9.1"), - PackageSpec("OneWire@2.3.6"), + PackageSpec("DallasTemperature@3.10.0"), + PackageSpec("OneWire@2.3.7"), ] assert pkgs_to_specs(ToolPackageManager().get_installed()) == [ PackageSpec("framework-arduino-avr-attiny@1.5.2"), @@ -241,7 +241,7 @@ platform = native config.get("platformio", "lib_dir") ).get_installed() assert pkgs_to_specs(installed_private_pkgs) == [ - PackageSpec("OneWire@2.3.6"), + PackageSpec("OneWire@2.3.7"), PackageSpec("My Private Lib@1.0.0"), ] installed_env_pkgs = LibraryPackageManager( @@ -249,7 +249,7 @@ platform = native ).get_installed() assert pkgs_to_specs(installed_env_pkgs) == [ PackageSpec("ArduinoJson@5.13.4"), - PackageSpec("DallasTemperature@3.9.1"), + PackageSpec("DallasTemperature@3.10.0"), ] @@ -269,8 +269,8 @@ def test_remove_project_unused_libdeps( storage_dir = os.path.join(config.get("platformio", "libdeps_dir"), "baremetal") lm = LibraryPackageManager(storage_dir) assert pkgs_to_specs(lm.get_installed()) == [ - PackageSpec("DallasTemperature@3.9.1"), - PackageSpec("OneWire@2.3.6"), + PackageSpec("DallasTemperature@3.10.0"), + PackageSpec("OneWire@2.3.7"), ] # add new deps @@ -285,8 +285,8 @@ def test_remove_project_unused_libdeps( lm = LibraryPackageManager(storage_dir) assert pkgs_to_specs(lm.get_installed()) == [ PackageSpec("ArduinoJson@5.13.4"), - PackageSpec("DallasTemperature@3.9.1"), - PackageSpec("OneWire@2.3.6"), + PackageSpec("DallasTemperature@3.10.0"), + PackageSpec("OneWire@2.3.7"), ] # manually remove from cofiguration file From 4a7a8b8b6873ad454c0c631c582fd0775330a60d Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 10 Jun 2022 14:00:46 +0300 Subject: [PATCH 07/93] Update tests --- tests/commands/pkg/test_update.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/commands/pkg/test_update.py b/tests/commands/pkg/test_update.py index c70e2b25..e49423fd 100644 --- a/tests/commands/pkg/test_update.py +++ b/tests/commands/pkg/test_update.py @@ -161,7 +161,7 @@ def test_project(clirunner, validate_cliresult, isolated_pio_core, tmp_path): ) assert pkgs_to_specs(lm.get_installed()) == [ PackageSpec("DallasTemperature@3.8.1"), - PackageSpec("OneWire@2.3.6"), + PackageSpec("OneWire@2.3.7"), ] assert pkgs_to_specs(PlatformPackageManager().get_installed()) == [ PackageSpec("atmelavr@2.2.0") @@ -187,8 +187,8 @@ def test_project(clirunner, validate_cliresult, isolated_pio_core, tmp_path): assert pkgs[0].metadata.name == "atmelavr" assert pkgs[0].metadata.version.major == 3 assert pkgs_to_specs(lm.get_installed()) == [ - PackageSpec("DallasTemperature@3.9.1"), - PackageSpec("OneWire@2.3.6"), + PackageSpec("DallasTemperature@3.10.0"), + PackageSpec("OneWire@2.3.7"), ] assert pkgs_to_specs(ToolPackageManager().get_installed()) == [ PackageSpec("framework-arduino-avr-attiny@1.3.2"), @@ -230,7 +230,7 @@ def test_custom_project_libraries( ) assert pkgs_to_specs(lm.get_installed()) == [ PackageSpec("DallasTemperature@3.8.1"), - PackageSpec("OneWire@2.3.6"), + PackageSpec("OneWire@2.3.7"), ] # update package result = clirunner.invoke( @@ -260,8 +260,8 @@ def test_custom_project_libraries( os.path.join(config.get("platformio", "libdeps_dir"), "devkit") ) assert pkgs_to_specs(lm.get_installed()) == [ - PackageSpec("DallasTemperature@3.9.1"), - PackageSpec("OneWire@2.3.6"), + PackageSpec("DallasTemperature@3.10.0"), + PackageSpec("OneWire@2.3.7"), ] assert config.get("env:devkit", "lib_deps") == [ "milesburton/DallasTemperature@^3.8.0" From 4bc3e3cf95f6c4a97be93b7a04800bc6d34866d3 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 10 Jun 2022 14:01:42 +0300 Subject: [PATCH 08/93] Restructure "device" module --- platformio/builder/tools/pioupload.py | 2 +- platformio/commands/device/__init__.py | 2 +- platformio/debug/helpers.py | 2 +- platformio/device/commands/__init__.py | 13 -- platformio/device/commands/list.py | 99 ----------- platformio/device/commands/monitor.py | 184 --------------------- platformio/device/filters/__init__.py | 13 -- platformio/device/filters/base.py | 104 ------------ platformio/device/filters/hexlify.py | 38 ----- platformio/device/filters/log2file.py | 45 ----- platformio/device/filters/send_on_enter.py | 38 ----- platformio/device/filters/time.py | 37 ----- platformio/device/list.py | 154 ----------------- platformio/home/rpc/handlers/os.py | 2 +- platformio/public.py | 4 +- platformio/remote/cli.py | 2 +- platformio/remote/client/agent_service.py | 2 +- platformio/run/cli.py | 2 +- platformio/util.py | 2 +- 19 files changed, 10 insertions(+), 735 deletions(-) delete mode 100644 platformio/device/commands/__init__.py delete mode 100644 platformio/device/commands/list.py delete mode 100644 platformio/device/commands/monitor.py delete mode 100644 platformio/device/filters/__init__.py delete mode 100644 platformio/device/filters/base.py delete mode 100644 platformio/device/filters/hexlify.py delete mode 100644 platformio/device/filters/log2file.py delete mode 100644 platformio/device/filters/send_on_enter.py delete mode 100644 platformio/device/filters/time.py delete mode 100644 platformio/device/list.py diff --git a/platformio/builder/tools/pioupload.py b/platformio/builder/tools/pioupload.py index 14ee59e1..46d2ca49 100644 --- a/platformio/builder/tools/pioupload.py +++ b/platformio/builder/tools/pioupload.py @@ -27,7 +27,7 @@ from serial import Serial, SerialException from platformio import exception, fs from platformio.device.finder import find_mbed_disk, find_serial_port, is_pattern_port -from platformio.device.list import list_serial_ports +from platformio.device.list.util import list_serial_ports from platformio.proc import exec_command diff --git a/platformio/commands/device/__init__.py b/platformio/commands/device/__init__.py index 1af0f8d4..d78b0035 100644 --- a/platformio/commands/device/__init__.py +++ b/platformio/commands/device/__init__.py @@ -13,6 +13,6 @@ # limitations under the License. # pylint: disable=unused-import -from platformio.device.filters.base import ( +from platformio.device.monitor.filters.base import ( DeviceMonitorFilterBase as DeviceMonitorFilter, ) diff --git a/platformio/debug/helpers.py b/platformio/debug/helpers.py index f156cf45..35f0d483 100644 --- a/platformio/debug/helpers.py +++ b/platformio/debug/helpers.py @@ -23,7 +23,7 @@ from io import BytesIO from platformio.cli import PlatformioCLI from platformio.compat import IS_WINDOWS, is_bytes from platformio.debug.exception import DebugInvalidOptionsError -from platformio.device.list import list_serial_ports +from platformio.device.list.util import list_serial_ports from platformio.run.cli import cli as cmd_run from platformio.run.cli import print_processing_header from platformio.test.helpers import list_test_names diff --git a/platformio/device/commands/__init__.py b/platformio/device/commands/__init__.py deleted file mode 100644 index b0514903..00000000 --- a/platformio/device/commands/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright (c) 2014-present PlatformIO -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/platformio/device/commands/list.py b/platformio/device/commands/list.py deleted file mode 100644 index 9cd3364f..00000000 --- a/platformio/device/commands/list.py +++ /dev/null @@ -1,99 +0,0 @@ -# Copyright (c) 2014-present PlatformIO -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import json - -import click - -from platformio.device.list import ( - list_logical_devices, - list_mdns_services, - list_serial_ports, -) - - -@click.command("list", short_help="List devices") -@click.option("--serial", is_flag=True, help="List serial ports, default") -@click.option("--logical", is_flag=True, help="List logical devices") -@click.option("--mdns", is_flag=True, help="List multicast DNS services") -@click.option("--json-output", is_flag=True) -def device_list_cmd( # pylint: disable=too-many-branches - serial, logical, mdns, json_output -): - if not logical and not mdns: - serial = True - data = {} - if serial: - data["serial"] = list_serial_ports() - if logical: - data["logical"] = list_logical_devices() - if mdns: - data["mdns"] = list_mdns_services() - - single_key = list(data)[0] if len(list(data)) == 1 else None - - if json_output: - return click.echo(json.dumps(data[single_key] if single_key else data)) - - titles = { - "serial": "Serial Ports", - "logical": "Logical Devices", - "mdns": "Multicast DNS Services", - } - - for key, value in data.items(): - if not single_key: - click.secho(titles[key], bold=True) - click.echo("=" * len(titles[key])) - - if key == "serial": - for item in value: - click.secho(item["port"], fg="cyan") - click.echo("-" * len(item["port"])) - click.echo("Hardware ID: %s" % item["hwid"]) - click.echo("Description: %s" % item["description"]) - click.echo("") - - if key == "logical": - for item in value: - click.secho(item["path"], fg="cyan") - click.echo("-" * len(item["path"])) - click.echo("Name: %s" % item["name"]) - click.echo("") - - if key == "mdns": - for item in value: - click.secho(item["name"], fg="cyan") - click.echo("-" * len(item["name"])) - click.echo("Type: %s" % item["type"]) - click.echo("IP: %s" % item["ip"]) - click.echo("Port: %s" % item["port"]) - if item["properties"]: - click.echo( - "Properties: %s" - % ( - "; ".join( - [ - "%s=%s" % (k, v) - for k, v in item["properties"].items() - ] - ) - ) - ) - click.echo("") - - if single_key: - click.echo("") - - return True diff --git a/platformio/device/commands/monitor.py b/platformio/device/commands/monitor.py deleted file mode 100644 index f3d4ace5..00000000 --- a/platformio/device/commands/monitor.py +++ /dev/null @@ -1,184 +0,0 @@ -# Copyright (c) 2014-present PlatformIO -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import sys - -import click -from serial.tools import miniterm - -from platformio import exception, fs -from platformio.device.filters.base import register_filters -from platformio.device.finder import find_serial_port -from platformio.platform.factory import PlatformFactory -from platformio.project.config import ProjectConfig -from platformio.project.exception import NotPlatformIOProjectError -from platformio.project.options import ProjectOptions - - -@click.command("monitor", short_help="Monitor device (Serial/Socket)") -@click.option("--port", "-p", help="Port, a number or a device name") -@click.option( - "--baud", - "-b", - type=int, - help="Set baud rate, default=%d" % ProjectOptions["env.monitor_speed"].default, -) -@click.option( - "--parity", - default="N", - type=click.Choice(["N", "E", "O", "S", "M"]), - help="Set parity, default=N", -) -@click.option("--rtscts", is_flag=True, help="Enable RTS/CTS flow control, default=Off") -@click.option( - "--xonxoff", is_flag=True, help="Enable software flow control, default=Off" -) -@click.option( - "--rts", default=None, type=click.IntRange(0, 1), help="Set initial RTS line state" -) -@click.option( - "--dtr", default=None, type=click.IntRange(0, 1), help="Set initial DTR line state" -) -@click.option("--echo", is_flag=True, help="Enable local echo, default=Off") -@click.option( - "--encoding", - default="UTF-8", - help="Set the encoding for the serial port (e.g. hexlify, " - "Latin1, UTF-8), default: UTF-8", -) -@click.option("--filter", "-f", multiple=True, help="Add filters/text transformations") -@click.option( - "--eol", - default="CRLF", - type=click.Choice(["CR", "LF", "CRLF"]), - help="End of line mode, default=CRLF", -) -@click.option("--raw", is_flag=True, help="Do not apply any encodings/transformations") -@click.option( - "--exit-char", - type=int, - default=3, - help="ASCII code of special character that is used to exit " - "the application, default=3 (Ctrl+C)", -) -@click.option( - "--menu-char", - type=int, - default=20, - help="ASCII code of special character that is used to " - "control miniterm (menu), default=20 (DEC)", -) -@click.option( - "--quiet", - is_flag=True, - help="Diagnostics: suppress non-error messages, default=Off", -) -@click.option( - "-d", - "--project-dir", - default=os.getcwd, - type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True), -) -@click.option( - "-e", - "--environment", - help="Load configuration from `platformio.ini` and specified environment", -) -def device_monitor_cmd(**kwargs): # pylint: disable=too-many-branches - project_options = {} - platform = None - with fs.cd(kwargs["project_dir"]): - try: - project_options = get_project_options(kwargs["environment"]) - kwargs = apply_project_monitor_options(kwargs, project_options) - if "platform" in project_options: - platform = PlatformFactory.new(project_options["platform"]) - except NotPlatformIOProjectError: - pass - register_filters(platform=platform, options=kwargs) - kwargs["port"] = find_serial_port( - initial_port=kwargs["port"], - board_config=platform.board_config(project_options.get("board")) - if platform and project_options.get("board") - else None, - upload_protocol=project_options.get("upload_port"), - ) - - # override system argv with patched options - sys.argv = ["monitor"] + project_options_to_monitor_argv( - kwargs, - project_options, - ignore=("port", "baud", "rts", "dtr", "environment", "project_dir"), - ) - - if not kwargs["quiet"]: - click.echo( - "--- Available filters and text transformations: %s" - % ", ".join(sorted(miniterm.TRANSFORMATIONS.keys())) - ) - click.echo("--- More details at https://bit.ly/pio-monitor-filters") - try: - miniterm.main( - default_port=kwargs["port"], - default_baudrate=kwargs["baud"] - or ProjectOptions["env.monitor_speed"].default, - default_rts=kwargs["rts"], - default_dtr=kwargs["dtr"], - ) - except Exception as e: - raise exception.MinitermException(e) - - -def get_project_options(environment=None): - config = ProjectConfig.get_instance() - config.validate(envs=[environment] if environment else None) - environment = environment or config.get_default_env() - return config.items(env=environment, as_dict=True) - - -def apply_project_monitor_options(cli_options, project_options): - for k in ("port", "speed", "rts", "dtr"): - k2 = "monitor_%s" % k - if k == "speed": - k = "baud" - if cli_options[k] is None and k2 in project_options: - cli_options[k] = project_options[k2] - if k != "port": - cli_options[k] = int(cli_options[k]) - return cli_options - - -def project_options_to_monitor_argv(cli_options, project_options, ignore=None): - confmon_flags = project_options.get("monitor_flags", []) - result = confmon_flags[::] - - for f in project_options.get("monitor_filters", []): - result.extend(["--filter", f]) - - for k, v in cli_options.items(): - if v is None or (ignore and k in ignore): - continue - k = "--" + k.replace("_", "-") - if k in confmon_flags: - continue - if isinstance(v, bool): - if v: - result.append(k) - elif isinstance(v, tuple): - for i in v: - result.extend([k, i]) - else: - result.extend([k, str(v)]) - return result diff --git a/platformio/device/filters/__init__.py b/platformio/device/filters/__init__.py deleted file mode 100644 index b0514903..00000000 --- a/platformio/device/filters/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright (c) 2014-present PlatformIO -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/platformio/device/filters/base.py b/platformio/device/filters/base.py deleted file mode 100644 index 203cdb77..00000000 --- a/platformio/device/filters/base.py +++ /dev/null @@ -1,104 +0,0 @@ -# Copyright (c) 2014-present PlatformIO -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import inspect -import os - -from serial.tools import miniterm - -from platformio import fs -from platformio.compat import get_object_members, load_python_module -from platformio.package.manager.tool import ToolPackageManager -from platformio.project.config import ProjectConfig - - -class DeviceMonitorFilterBase(miniterm.Transform): - def __init__(self, options=None): - """Called by PlatformIO to pass context""" - miniterm.Transform.__init__(self) - - self.options = options or {} - self.project_dir = self.options.get("project_dir") - self.environment = self.options.get("environment") - - self.config = ProjectConfig.get_instance() - if not self.environment: - default_envs = self.config.default_envs() - if default_envs: - self.environment = default_envs[0] - elif self.config.envs(): - self.environment = self.config.envs()[0] - - def __call__(self): - """Called by the miniterm library when the filter is actually used""" - return self - - @property - def NAME(self): - raise NotImplementedError("Please declare NAME attribute for the filter class") - - -def register_filters(platform=None, options=None): - # project filters - load_monitor_filters( - ProjectConfig.get_instance().get("platformio", "monitor_dir"), - prefix="filter_", - options=options, - ) - # platform filters - if platform: - load_monitor_filters( - os.path.join(platform.get_dir(), "monitor"), - prefix="filter_", - options=options, - ) - # load package filters - pm = ToolPackageManager() - for pkg in pm.get_installed(): - load_monitor_filters( - os.path.join(pkg.path, "monitor"), prefix="filter_", options=options - ) - # default filters - load_monitor_filters( - os.path.join(fs.get_source_dir(), "device", "filters"), - options=options, - ) - - -def load_monitor_filters(monitor_dir, prefix=None, options=None): - if not os.path.isdir(monitor_dir): - return - for name in os.listdir(monitor_dir): - if (prefix and not name.startswith(prefix)) or not name.endswith(".py"): - continue - path = os.path.join(monitor_dir, name) - if not os.path.isfile(path): - continue - load_monitor_filter(path, options) - - -def load_monitor_filter(path, options=None): - name = os.path.basename(path) - name = name[: name.find(".")] - module = load_python_module("platformio.device.filters.%s" % name, path) - for cls in get_object_members(module).values(): - if ( - not inspect.isclass(cls) - or not issubclass(cls, DeviceMonitorFilterBase) - or cls == DeviceMonitorFilterBase - ): - continue - obj = cls(options) - miniterm.TRANSFORMATIONS[obj.NAME] = obj - return True diff --git a/platformio/device/filters/hexlify.py b/platformio/device/filters/hexlify.py deleted file mode 100644 index 045f637e..00000000 --- a/platformio/device/filters/hexlify.py +++ /dev/null @@ -1,38 +0,0 @@ -# Copyright (c) 2014-present PlatformIO -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import serial - -from platformio.device.filters.base import DeviceMonitorFilterBase - - -class Hexlify(DeviceMonitorFilterBase): - NAME = "hexlify" - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self._counter = 0 - - def rx(self, text): - result = "" - for b in serial.iterbytes(text): - if (self._counter % 16) == 0: - result += "\n{:04X} | ".format(self._counter) - asciicode = ord(b) - if asciicode <= 255: - result += "{:02X} ".format(asciicode) - else: - result += "?? " - self._counter += 1 - return result diff --git a/platformio/device/filters/log2file.py b/platformio/device/filters/log2file.py deleted file mode 100644 index e4c622d1..00000000 --- a/platformio/device/filters/log2file.py +++ /dev/null @@ -1,45 +0,0 @@ -# Copyright (c) 2014-present PlatformIO -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import io -import os.path -from datetime import datetime - -from platformio.device.filters.base import DeviceMonitorFilterBase - - -class LogToFile(DeviceMonitorFilterBase): - NAME = "log2file" - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self._log_fp = None - - def __call__(self): - log_file_name = "platformio-device-monitor-%s.log" % datetime.now().strftime( - "%y%m%d-%H%M%S" - ) - print("--- Logging an output to %s" % os.path.abspath(log_file_name)) - # pylint: disable=consider-using-with - self._log_fp = io.open(log_file_name, "w", encoding="utf-8") - return self - - def __del__(self): - if self._log_fp: - self._log_fp.close() - - def rx(self, text): - self._log_fp.write(text) - self._log_fp.flush() - return text diff --git a/platformio/device/filters/send_on_enter.py b/platformio/device/filters/send_on_enter.py deleted file mode 100644 index ec002295..00000000 --- a/platformio/device/filters/send_on_enter.py +++ /dev/null @@ -1,38 +0,0 @@ -# Copyright (c) 2014-present PlatformIO -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from platformio.device.filters.base import DeviceMonitorFilterBase - - -class SendOnEnter(DeviceMonitorFilterBase): - NAME = "send_on_enter" - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self._buffer = "" - - if self.options.get("eol") == "CR": - self._eol = "\r" - elif self.options.get("eol") == "LF": - self._eol = "\n" - else: - self._eol = "\r\n" - - def tx(self, text): - self._buffer += text - if self._buffer.endswith(self._eol): - text = self._buffer - self._buffer = "" - return text - return "" diff --git a/platformio/device/filters/time.py b/platformio/device/filters/time.py deleted file mode 100644 index d7ba1c7f..00000000 --- a/platformio/device/filters/time.py +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright (c) 2014-present PlatformIO -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from datetime import datetime - -from platformio.device.filters.base import DeviceMonitorFilterBase - - -class Timestamp(DeviceMonitorFilterBase): - NAME = "time" - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self._line_started = False - - def rx(self, text): - if self._line_started and "\n" not in text: - return text - timestamp = datetime.now().strftime("%H:%M:%S.%f")[:-3] - if not self._line_started: - self._line_started = True - text = "%s > %s" % (timestamp, text) - if text.endswith("\n"): - self._line_started = False - return text[:-1].replace("\n", "\n%s > " % timestamp) + "\n" - return text.replace("\n", "\n%s > " % timestamp) diff --git a/platformio/device/list.py b/platformio/device/list.py deleted file mode 100644 index 3695f760..00000000 --- a/platformio/device/list.py +++ /dev/null @@ -1,154 +0,0 @@ -# Copyright (c) 2014-present PlatformIO -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import json -import os -import re -import time -from glob import glob - -import zeroconf - -from platformio import __version__, exception, proc -from platformio.compat import IS_MACOS, IS_WINDOWS - - -def list_serial_ports(filter_hwid=False): - try: - # pylint: disable=import-outside-toplevel - from serial.tools.list_ports import comports - except ImportError: - raise exception.GetSerialPortsError(os.name) - - result = [] - for p, d, h in comports(): - if not p: - continue - if not filter_hwid or "VID:PID" in h: - result.append({"port": p, "description": d, "hwid": h}) - - if filter_hwid: - return result - - # fix for PySerial - if not result and IS_MACOS: - for p in glob("/dev/tty.*"): - result.append({"port": p, "description": "n/a", "hwid": "n/a"}) - return result - - -def list_logical_devices(): - items = [] - if IS_WINDOWS: - try: - result = proc.exec_command( - ["wmic", "logicaldisk", "get", "name,VolumeName"] - ).get("out", "") - devicenamere = re.compile(r"^([A-Z]{1}\:)\s*(\S+)?") - for line in result.split("\n"): - match = devicenamere.match(line.strip()) - if not match: - continue - items.append({"path": match.group(1) + "\\", "name": match.group(2)}) - return items - except WindowsError: # pylint: disable=undefined-variable - pass - # try "fsutil" - result = proc.exec_command(["fsutil", "fsinfo", "drives"]).get("out", "") - for device in re.findall(r"[A-Z]:\\", result): - items.append({"path": device, "name": None}) - return items - - result = proc.exec_command(["df"]).get("out") - devicenamere = re.compile(r"^/.+\d+\%\s+([a-z\d\-_/]+)$", flags=re.I) - for line in result.split("\n"): - match = devicenamere.match(line.strip()) - if not match: - continue - items.append({"path": match.group(1), "name": os.path.basename(match.group(1))}) - return items - - -def list_mdns_services(): - class mDNSListener(object): - def __init__(self): - self._zc = zeroconf.Zeroconf(interfaces=zeroconf.InterfaceChoice.All) - self._found_types = [] - self._found_services = [] - - def __enter__(self): - zeroconf.ServiceBrowser( - self._zc, - [ - "_http._tcp.local.", - "_hap._tcp.local.", - "_services._dns-sd._udp.local.", - ], - self, - ) - return self - - def __exit__(self, etype, value, traceback): - self._zc.close() - - def add_service(self, zc, type_, name): - try: - assert zeroconf.service_type_name(name) - assert str(name) - except (AssertionError, UnicodeError, zeroconf.BadTypeInNameException): - return - if name not in self._found_types: - self._found_types.append(name) - zeroconf.ServiceBrowser(self._zc, name, self) - if type_ in self._found_types: - s = zc.get_service_info(type_, name) - if s: - self._found_services.append(s) - - def remove_service(self, zc, type_, name): - pass - - def update_service(self, zc, type_, name): - pass - - def get_services(self): - return self._found_services - - items = [] - with mDNSListener() as mdns: - time.sleep(3) - for service in mdns.get_services(): - properties = None - if service.properties: - try: - properties = { - k.decode("utf8"): v.decode("utf8") - if isinstance(v, bytes) - else v - for k, v in service.properties.items() - } - json.dumps(properties) - except UnicodeDecodeError: - properties = None - - items.append( - { - "type": service.type, - "name": service.name, - "ip": ", ".join(service.parsed_addresses()), - "port": service.port, - "properties": properties, - } - ) - return items diff --git a/platformio/home/rpc/handlers/os.py b/platformio/home/rpc/handlers/os.py index 1aaf6d63..fad11318 100644 --- a/platformio/home/rpc/handlers/os.py +++ b/platformio/home/rpc/handlers/os.py @@ -24,7 +24,7 @@ import click from platformio import __default_requests_timeout__, fs from platformio.cache import ContentCache -from platformio.device.list import list_logical_devices +from platformio.device.list.util import list_logical_devices from platformio.home import helpers from platformio.http import ensure_internet_on diff --git a/platformio/public.py b/platformio/public.py index 51eecc52..681ab126 100644 --- a/platformio/public.py +++ b/platformio/public.py @@ -14,8 +14,8 @@ # pylint: disable=unused-import -from platformio.device.filters.base import DeviceMonitorFilterBase -from platformio.device.list import list_serial_ports +from platformio.device.list.util import list_serial_ports +from platformio.device.monitor.filters.base import DeviceMonitorFilterBase from platformio.fs import to_unix_path from platformio.platform.base import PlatformBase from platformio.project.config import ProjectConfig diff --git a/platformio/remote/cli.py b/platformio/remote/cli.py index 1a32c549..40bf0b8f 100644 --- a/platformio/remote/cli.py +++ b/platformio/remote/cli.py @@ -24,7 +24,7 @@ from time import sleep import click from platformio import fs, proc -from platformio.device.commands.monitor import ( +from platformio.device.monitor.command import ( apply_project_monitor_options, device_monitor_cmd, get_project_options, diff --git a/platformio/remote/client/agent_service.py b/platformio/remote/client/agent_service.py index a2f0a05a..ac227795 100644 --- a/platformio/remote/client/agent_service.py +++ b/platformio/remote/client/agent_service.py @@ -18,7 +18,7 @@ from twisted.logger import LogLevel # pylint: disable=import-error from twisted.spread import pb # pylint: disable=import-error from platformio import proc -from platformio.device.list import list_serial_ports +from platformio.device.list.util import list_serial_ports from platformio.project.config import ProjectConfig from platformio.project.exception import NotPlatformIOProjectError from platformio.remote.ac.process import ProcessAsyncCmd diff --git a/platformio/run/cli.py b/platformio/run/cli.py index aa1c0d6c..29c6812d 100644 --- a/platformio/run/cli.py +++ b/platformio/run/cli.py @@ -22,7 +22,7 @@ import click from tabulate import tabulate from platformio import app, exception, fs, util -from platformio.device.commands.monitor import device_monitor_cmd +from platformio.device.monitor.command import device_monitor_cmd from platformio.project.config import ProjectConfig from platformio.project.helpers import find_project_dir_above, load_build_metadata from platformio.run.helpers import clean_build_dir, handle_legacy_libdeps diff --git a/platformio/util.py b/platformio/util.py index 00158d45..1dbac429 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -27,7 +27,7 @@ import click from platformio import __version__ # pylint: disable=unused-import -from platformio.device.list import list_serial_ports as get_serial_ports +from platformio.device.list.util import list_serial_ports as get_serial_ports from platformio.fs import cd, load_json from platformio.proc import exec_command From a6e61a7a5a24cf73d2866b9817e8b9a7e99852d0 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 10 Jun 2022 14:01:55 +0300 Subject: [PATCH 09/93] Restructure "device" module --- platformio/device/cli.py | 4 +- platformio/device/finder.py | 2 +- platformio/device/list/__init__.py | 13 ++ platformio/device/list/command.py | 99 ++++++++++ platformio/device/list/util.py | 154 +++++++++++++++ platformio/device/monitor/__init__.py | 13 ++ platformio/device/monitor/command.py | 184 ++++++++++++++++++ platformio/device/monitor/filters/__init__.py | 13 ++ platformio/device/monitor/filters/base.py | 100 ++++++++++ platformio/device/monitor/filters/hexlify.py | 38 ++++ platformio/device/monitor/filters/log2file.py | 45 +++++ .../device/monitor/filters/send_on_enter.py | 38 ++++ platformio/device/monitor/filters/time.py | 37 ++++ 13 files changed, 737 insertions(+), 3 deletions(-) create mode 100644 platformio/device/list/__init__.py create mode 100644 platformio/device/list/command.py create mode 100644 platformio/device/list/util.py create mode 100644 platformio/device/monitor/__init__.py create mode 100644 platformio/device/monitor/command.py create mode 100644 platformio/device/monitor/filters/__init__.py create mode 100644 platformio/device/monitor/filters/base.py create mode 100644 platformio/device/monitor/filters/hexlify.py create mode 100644 platformio/device/monitor/filters/log2file.py create mode 100644 platformio/device/monitor/filters/send_on_enter.py create mode 100644 platformio/device/monitor/filters/time.py diff --git a/platformio/device/cli.py b/platformio/device/cli.py index 5865c43e..13dec793 100644 --- a/platformio/device/cli.py +++ b/platformio/device/cli.py @@ -14,8 +14,8 @@ import click -from platformio.device.commands.list import device_list_cmd -from platformio.device.commands.monitor import device_monitor_cmd +from platformio.device.list.command import device_list_cmd +from platformio.device.monitor.command import device_monitor_cmd @click.group( diff --git a/platformio/device/finder.py b/platformio/device/finder.py index 0fe98baa..382fe000 100644 --- a/platformio/device/finder.py +++ b/platformio/device/finder.py @@ -18,7 +18,7 @@ from fnmatch import fnmatch import serial from platformio.compat import IS_WINDOWS -from platformio.device.list import list_logical_devices, list_serial_ports +from platformio.device.list.util import list_logical_devices, list_serial_ports def is_pattern_port(port): diff --git a/platformio/device/list/__init__.py b/platformio/device/list/__init__.py new file mode 100644 index 00000000..b0514903 --- /dev/null +++ b/platformio/device/list/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/device/list/command.py b/platformio/device/list/command.py new file mode 100644 index 00000000..078371cf --- /dev/null +++ b/platformio/device/list/command.py @@ -0,0 +1,99 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json + +import click + +from platformio.device.list.util import ( + list_logical_devices, + list_mdns_services, + list_serial_ports, +) + + +@click.command("list", short_help="List devices") +@click.option("--serial", is_flag=True, help="List serial ports, default") +@click.option("--logical", is_flag=True, help="List logical devices") +@click.option("--mdns", is_flag=True, help="List multicast DNS services") +@click.option("--json-output", is_flag=True) +def device_list_cmd( # pylint: disable=too-many-branches + serial, logical, mdns, json_output +): + if not logical and not mdns: + serial = True + data = {} + if serial: + data["serial"] = list_serial_ports() + if logical: + data["logical"] = list_logical_devices() + if mdns: + data["mdns"] = list_mdns_services() + + single_key = list(data)[0] if len(list(data)) == 1 else None + + if json_output: + return click.echo(json.dumps(data[single_key] if single_key else data)) + + titles = { + "serial": "Serial Ports", + "logical": "Logical Devices", + "mdns": "Multicast DNS Services", + } + + for key, value in data.items(): + if not single_key: + click.secho(titles[key], bold=True) + click.echo("=" * len(titles[key])) + + if key == "serial": + for item in value: + click.secho(item["port"], fg="cyan") + click.echo("-" * len(item["port"])) + click.echo("Hardware ID: %s" % item["hwid"]) + click.echo("Description: %s" % item["description"]) + click.echo("") + + if key == "logical": + for item in value: + click.secho(item["path"], fg="cyan") + click.echo("-" * len(item["path"])) + click.echo("Name: %s" % item["name"]) + click.echo("") + + if key == "mdns": + for item in value: + click.secho(item["name"], fg="cyan") + click.echo("-" * len(item["name"])) + click.echo("Type: %s" % item["type"]) + click.echo("IP: %s" % item["ip"]) + click.echo("Port: %s" % item["port"]) + if item["properties"]: + click.echo( + "Properties: %s" + % ( + "; ".join( + [ + "%s=%s" % (k, v) + for k, v in item["properties"].items() + ] + ) + ) + ) + click.echo("") + + if single_key: + click.echo("") + + return True diff --git a/platformio/device/list/util.py b/platformio/device/list/util.py new file mode 100644 index 00000000..3695f760 --- /dev/null +++ b/platformio/device/list/util.py @@ -0,0 +1,154 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import os +import re +import time +from glob import glob + +import zeroconf + +from platformio import __version__, exception, proc +from platformio.compat import IS_MACOS, IS_WINDOWS + + +def list_serial_ports(filter_hwid=False): + try: + # pylint: disable=import-outside-toplevel + from serial.tools.list_ports import comports + except ImportError: + raise exception.GetSerialPortsError(os.name) + + result = [] + for p, d, h in comports(): + if not p: + continue + if not filter_hwid or "VID:PID" in h: + result.append({"port": p, "description": d, "hwid": h}) + + if filter_hwid: + return result + + # fix for PySerial + if not result and IS_MACOS: + for p in glob("/dev/tty.*"): + result.append({"port": p, "description": "n/a", "hwid": "n/a"}) + return result + + +def list_logical_devices(): + items = [] + if IS_WINDOWS: + try: + result = proc.exec_command( + ["wmic", "logicaldisk", "get", "name,VolumeName"] + ).get("out", "") + devicenamere = re.compile(r"^([A-Z]{1}\:)\s*(\S+)?") + for line in result.split("\n"): + match = devicenamere.match(line.strip()) + if not match: + continue + items.append({"path": match.group(1) + "\\", "name": match.group(2)}) + return items + except WindowsError: # pylint: disable=undefined-variable + pass + # try "fsutil" + result = proc.exec_command(["fsutil", "fsinfo", "drives"]).get("out", "") + for device in re.findall(r"[A-Z]:\\", result): + items.append({"path": device, "name": None}) + return items + + result = proc.exec_command(["df"]).get("out") + devicenamere = re.compile(r"^/.+\d+\%\s+([a-z\d\-_/]+)$", flags=re.I) + for line in result.split("\n"): + match = devicenamere.match(line.strip()) + if not match: + continue + items.append({"path": match.group(1), "name": os.path.basename(match.group(1))}) + return items + + +def list_mdns_services(): + class mDNSListener(object): + def __init__(self): + self._zc = zeroconf.Zeroconf(interfaces=zeroconf.InterfaceChoice.All) + self._found_types = [] + self._found_services = [] + + def __enter__(self): + zeroconf.ServiceBrowser( + self._zc, + [ + "_http._tcp.local.", + "_hap._tcp.local.", + "_services._dns-sd._udp.local.", + ], + self, + ) + return self + + def __exit__(self, etype, value, traceback): + self._zc.close() + + def add_service(self, zc, type_, name): + try: + assert zeroconf.service_type_name(name) + assert str(name) + except (AssertionError, UnicodeError, zeroconf.BadTypeInNameException): + return + if name not in self._found_types: + self._found_types.append(name) + zeroconf.ServiceBrowser(self._zc, name, self) + if type_ in self._found_types: + s = zc.get_service_info(type_, name) + if s: + self._found_services.append(s) + + def remove_service(self, zc, type_, name): + pass + + def update_service(self, zc, type_, name): + pass + + def get_services(self): + return self._found_services + + items = [] + with mDNSListener() as mdns: + time.sleep(3) + for service in mdns.get_services(): + properties = None + if service.properties: + try: + properties = { + k.decode("utf8"): v.decode("utf8") + if isinstance(v, bytes) + else v + for k, v in service.properties.items() + } + json.dumps(properties) + except UnicodeDecodeError: + properties = None + + items.append( + { + "type": service.type, + "name": service.name, + "ip": ", ".join(service.parsed_addresses()), + "port": service.port, + "properties": properties, + } + ) + return items diff --git a/platformio/device/monitor/__init__.py b/platformio/device/monitor/__init__.py new file mode 100644 index 00000000..b0514903 --- /dev/null +++ b/platformio/device/monitor/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/device/monitor/command.py b/platformio/device/monitor/command.py new file mode 100644 index 00000000..a36b1243 --- /dev/null +++ b/platformio/device/monitor/command.py @@ -0,0 +1,184 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys + +import click +from serial.tools import miniterm + +from platformio import exception, fs +from platformio.device.finder import find_serial_port +from platformio.device.monitor.filters.base import register_filters +from platformio.platform.factory import PlatformFactory +from platformio.project.config import ProjectConfig +from platformio.project.exception import NotPlatformIOProjectError +from platformio.project.options import ProjectOptions + + +@click.command("monitor", short_help="Monitor device (Serial/Socket)") +@click.option("--port", "-p", help="Port, a number or a device name") +@click.option( + "--baud", + "-b", + type=int, + help="Set baud rate, default=%d" % ProjectOptions["env.monitor_speed"].default, +) +@click.option( + "--parity", + default="N", + type=click.Choice(["N", "E", "O", "S", "M"]), + help="Set parity, default=N", +) +@click.option("--rtscts", is_flag=True, help="Enable RTS/CTS flow control, default=Off") +@click.option( + "--xonxoff", is_flag=True, help="Enable software flow control, default=Off" +) +@click.option( + "--rts", default=None, type=click.IntRange(0, 1), help="Set initial RTS line state" +) +@click.option( + "--dtr", default=None, type=click.IntRange(0, 1), help="Set initial DTR line state" +) +@click.option("--echo", is_flag=True, help="Enable local echo, default=Off") +@click.option( + "--encoding", + default="UTF-8", + help="Set the encoding for the serial port (e.g. hexlify, " + "Latin1, UTF-8), default: UTF-8", +) +@click.option("--filter", "-f", multiple=True, help="Add filters/text transformations") +@click.option( + "--eol", + default="CRLF", + type=click.Choice(["CR", "LF", "CRLF"]), + help="End of line mode, default=CRLF", +) +@click.option("--raw", is_flag=True, help="Do not apply any encodings/transformations") +@click.option( + "--exit-char", + type=int, + default=3, + help="ASCII code of special character that is used to exit " + "the application, default=3 (Ctrl+C)", +) +@click.option( + "--menu-char", + type=int, + default=20, + help="ASCII code of special character that is used to " + "control miniterm (menu), default=20 (DEC)", +) +@click.option( + "--quiet", + is_flag=True, + help="Diagnostics: suppress non-error messages, default=Off", +) +@click.option( + "-d", + "--project-dir", + default=os.getcwd, + type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True), +) +@click.option( + "-e", + "--environment", + help="Load configuration from `platformio.ini` and specified environment", +) +def device_monitor_cmd(**kwargs): # pylint: disable=too-many-branches + project_options = {} + platform = None + with fs.cd(kwargs["project_dir"]): + try: + project_options = get_project_options(kwargs["environment"]) + kwargs = apply_project_monitor_options(kwargs, project_options) + if "platform" in project_options: + platform = PlatformFactory.new(project_options["platform"]) + except NotPlatformIOProjectError: + pass + register_filters(platform=platform, options=kwargs) + kwargs["port"] = find_serial_port( + initial_port=kwargs["port"], + board_config=platform.board_config(project_options.get("board")) + if platform and project_options.get("board") + else None, + upload_protocol=project_options.get("upload_port"), + ) + + # override system argv with patched options + sys.argv = ["monitor"] + project_options_to_monitor_argv( + kwargs, + project_options, + ignore=("port", "baud", "rts", "dtr", "environment", "project_dir"), + ) + + if not kwargs["quiet"]: + click.echo( + "--- Available filters and text transformations: %s" + % ", ".join(sorted(miniterm.TRANSFORMATIONS.keys())) + ) + click.echo("--- More details at https://bit.ly/pio-monitor-filters") + try: + miniterm.main( + default_port=kwargs["port"], + default_baudrate=kwargs["baud"] + or ProjectOptions["env.monitor_speed"].default, + default_rts=kwargs["rts"], + default_dtr=kwargs["dtr"], + ) + except Exception as e: + raise exception.MinitermException(e) + + +def get_project_options(environment=None): + config = ProjectConfig.get_instance() + config.validate(envs=[environment] if environment else None) + environment = environment or config.get_default_env() + return config.items(env=environment, as_dict=True) + + +def apply_project_monitor_options(cli_options, project_options): + for k in ("port", "speed", "rts", "dtr"): + k2 = "monitor_%s" % k + if k == "speed": + k = "baud" + if cli_options[k] is None and k2 in project_options: + cli_options[k] = project_options[k2] + if k != "port": + cli_options[k] = int(cli_options[k]) + return cli_options + + +def project_options_to_monitor_argv(cli_options, project_options, ignore=None): + confmon_flags = project_options.get("monitor_flags", []) + result = confmon_flags[::] + + for f in project_options.get("monitor_filters", []): + result.extend(["--filter", f]) + + for k, v in cli_options.items(): + if v is None or (ignore and k in ignore): + continue + k = "--" + k.replace("_", "-") + if k in confmon_flags: + continue + if isinstance(v, bool): + if v: + result.append(k) + elif isinstance(v, tuple): + for i in v: + result.extend([k, i]) + else: + result.extend([k, str(v)]) + return result diff --git a/platformio/device/monitor/filters/__init__.py b/platformio/device/monitor/filters/__init__.py new file mode 100644 index 00000000..b0514903 --- /dev/null +++ b/platformio/device/monitor/filters/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/device/monitor/filters/base.py b/platformio/device/monitor/filters/base.py new file mode 100644 index 00000000..e773ed65 --- /dev/null +++ b/platformio/device/monitor/filters/base.py @@ -0,0 +1,100 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +import os + +from serial.tools import miniterm + +from platformio.compat import get_object_members, load_python_module +from platformio.package.manager.tool import ToolPackageManager +from platformio.project.config import ProjectConfig + + +class DeviceMonitorFilterBase(miniterm.Transform): + def __init__(self, options=None): + """Called by PlatformIO to pass context""" + miniterm.Transform.__init__(self) + + self.options = options or {} + self.project_dir = self.options.get("project_dir") + self.environment = self.options.get("environment") + + self.config = ProjectConfig.get_instance() + if not self.environment: + default_envs = self.config.default_envs() + if default_envs: + self.environment = default_envs[0] + elif self.config.envs(): + self.environment = self.config.envs()[0] + + def __call__(self): + """Called by the miniterm library when the filter is actually used""" + return self + + @property + def NAME(self): + raise NotImplementedError("Please declare NAME attribute for the filter class") + + +def register_filters(platform=None, options=None): + # project filters + load_monitor_filters( + ProjectConfig.get_instance().get("platformio", "monitor_dir"), + prefix="filter_", + options=options, + ) + # platform filters + if platform: + load_monitor_filters( + os.path.join(platform.get_dir(), "monitor"), + prefix="filter_", + options=options, + ) + # load package filters + pm = ToolPackageManager() + for pkg in pm.get_installed(): + load_monitor_filters( + os.path.join(pkg.path, "monitor"), prefix="filter_", options=options + ) + # default filters + load_monitor_filters(os.path.dirname(__file__), options=options) + + +def load_monitor_filters(monitor_dir, prefix=None, options=None): + if not os.path.isdir(monitor_dir): + return + for name in os.listdir(monitor_dir): + if (prefix and not name.startswith(prefix)) or not name.endswith(".py"): + continue + path = os.path.join(monitor_dir, name) + if not os.path.isfile(path): + continue + load_monitor_filter(path, options) + + +def load_monitor_filter(path, options=None): + name = os.path.basename(path) + name = name[: name.find(".")] + module = load_python_module("platformio.device.monitor.filters.%s" % name, path) + for cls in get_object_members(module).values(): + if ( + not inspect.isclass(cls) + or not issubclass(cls, DeviceMonitorFilterBase) + or cls == DeviceMonitorFilterBase + ): + continue + obj = cls(options) + miniterm.TRANSFORMATIONS[obj.NAME] = obj + return True diff --git a/platformio/device/monitor/filters/hexlify.py b/platformio/device/monitor/filters/hexlify.py new file mode 100644 index 00000000..28e83bfb --- /dev/null +++ b/platformio/device/monitor/filters/hexlify.py @@ -0,0 +1,38 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import serial + +from platformio.device.monitor.filters.base import DeviceMonitorFilterBase + + +class Hexlify(DeviceMonitorFilterBase): + NAME = "hexlify" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._counter = 0 + + def rx(self, text): + result = "" + for b in serial.iterbytes(text): + if (self._counter % 16) == 0: + result += "\n{:04X} | ".format(self._counter) + asciicode = ord(b) + if asciicode <= 255: + result += "{:02X} ".format(asciicode) + else: + result += "?? " + self._counter += 1 + return result diff --git a/platformio/device/monitor/filters/log2file.py b/platformio/device/monitor/filters/log2file.py new file mode 100644 index 00000000..bf97b551 --- /dev/null +++ b/platformio/device/monitor/filters/log2file.py @@ -0,0 +1,45 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import io +import os.path +from datetime import datetime + +from platformio.device.monitor.filters.base import DeviceMonitorFilterBase + + +class LogToFile(DeviceMonitorFilterBase): + NAME = "log2file" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._log_fp = None + + def __call__(self): + log_file_name = "platformio-device-monitor-%s.log" % datetime.now().strftime( + "%y%m%d-%H%M%S" + ) + print("--- Logging an output to %s" % os.path.abspath(log_file_name)) + # pylint: disable=consider-using-with + self._log_fp = io.open(log_file_name, "w", encoding="utf-8") + return self + + def __del__(self): + if self._log_fp: + self._log_fp.close() + + def rx(self, text): + self._log_fp.write(text) + self._log_fp.flush() + return text diff --git a/platformio/device/monitor/filters/send_on_enter.py b/platformio/device/monitor/filters/send_on_enter.py new file mode 100644 index 00000000..c28888cd --- /dev/null +++ b/platformio/device/monitor/filters/send_on_enter.py @@ -0,0 +1,38 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from platformio.device.monitor.filters.base import DeviceMonitorFilterBase + + +class SendOnEnter(DeviceMonitorFilterBase): + NAME = "send_on_enter" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._buffer = "" + + if self.options.get("eol") == "CR": + self._eol = "\r" + elif self.options.get("eol") == "LF": + self._eol = "\n" + else: + self._eol = "\r\n" + + def tx(self, text): + self._buffer += text + if self._buffer.endswith(self._eol): + text = self._buffer + self._buffer = "" + return text + return "" diff --git a/platformio/device/monitor/filters/time.py b/platformio/device/monitor/filters/time.py new file mode 100644 index 00000000..cde4e772 --- /dev/null +++ b/platformio/device/monitor/filters/time.py @@ -0,0 +1,37 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from datetime import datetime + +from platformio.device.monitor.filters.base import DeviceMonitorFilterBase + + +class Timestamp(DeviceMonitorFilterBase): + NAME = "time" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._line_started = False + + def rx(self, text): + if self._line_started and "\n" not in text: + return text + timestamp = datetime.now().strftime("%H:%M:%S.%f")[:-3] + if not self._line_started: + self._line_started = True + text = "%s > %s" % (timestamp, text) + if text.endswith("\n"): + self._line_started = False + return text[:-1].replace("\n", "\n%s > " % timestamp) + "\n" + return text.replace("\n", "\n%s > " % timestamp) From c42fe3297228922f04ce597d02978703451d7f8d Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 10 Jun 2022 17:30:36 +0300 Subject: [PATCH 10/93] Fix typo in upload_protocol --- platformio/device/monitor/command.py | 2 +- platformio/test/runners/readers/serial.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/platformio/device/monitor/command.py b/platformio/device/monitor/command.py index a36b1243..a4abd767 100644 --- a/platformio/device/monitor/command.py +++ b/platformio/device/monitor/command.py @@ -113,7 +113,7 @@ def device_monitor_cmd(**kwargs): # pylint: disable=too-many-branches board_config=platform.board_config(project_options.get("board")) if platform and project_options.get("board") else None, - upload_protocol=project_options.get("upload_port"), + upload_protocol=project_options.get("upload_protocol"), ) # override system argv with patched options diff --git a/platformio/test/runners/readers/serial.py b/platformio/test/runners/readers/serial.py index 195298a0..5ac5e0f3 100644 --- a/platformio/test/runners/readers/serial.py +++ b/platformio/test/runners/readers/serial.py @@ -71,7 +71,7 @@ class SerialTestOutputReader: board_config=self.test_runner.platform.board_config( project_options["board"] ), - upload_protocol=project_options.get("upload_port"), + upload_protocol=project_options.get("upload_protocol"), ensure_ready=True, ) From 7f351bc7c8840aa4150d0ebc9000f17b3c0f2af4 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 11 Jun 2022 20:55:52 +0300 Subject: [PATCH 11/93] Automatically reconnect device monitor if a connection fails // Resolve #3939 --- HISTORY.rst | 8 +- docs | 2 +- platformio/device/monitor/command.py | 99 +++++---------- platformio/device/monitor/terminal.py | 174 ++++++++++++++++++++++++++ platformio/exception.py | 4 - platformio/remote/cli.py | 26 +++- setup.py | 2 +- 7 files changed, 242 insertions(+), 73 deletions(-) create mode 100644 platformio/device/monitor/terminal.py diff --git a/HISTORY.rst b/HISTORY.rst index a3a0661b..50efce56 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -16,7 +16,13 @@ PlatformIO Core 6 6.0.3 (2022-??-??) ~~~~~~~~~~~~~~~~~~ -- Fixed an issue when a custom `pio test --project-config `__ was not handled properly (`issue #4299 `_) +* **Device Monitor** + + - Automatically reconnect if a connection fails + - Added new `pio device monitor --no-reconnect `__ option to disable automatic reconnection + - Handle disconnects more gracefully (`issue #3939 `_) + +* Fixed an issue when a custom `pio test --project-config `__ was not handled properly (`issue #4299 `_) 6.0.2 (2022-06-01) ~~~~~~~~~~~~~~~~~~ diff --git a/docs b/docs index 300060ea..c86b25dd 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 300060ea08be494465b03b427186bee66eda1766 +Subproject commit c86b25dd81f4bfec721a66415f052d2c20d9e044 diff --git a/platformio/device/monitor/command.py b/platformio/device/monitor/command.py index a4abd767..ae63c1a6 100644 --- a/platformio/device/monitor/command.py +++ b/platformio/device/monitor/command.py @@ -13,14 +13,13 @@ # limitations under the License. import os -import sys import click -from serial.tools import miniterm from platformio import exception, fs from platformio.device.finder import find_serial_port from platformio.device.monitor.filters.base import register_filters +from platformio.device.monitor.terminal import start_terminal from platformio.platform.factory import PlatformFactory from platformio.project.config import ProjectConfig from platformio.project.exception import NotPlatformIOProjectError @@ -30,10 +29,11 @@ from platformio.project.options import ProjectOptions @click.command("monitor", short_help="Monitor device (Serial/Socket)") @click.option("--port", "-p", help="Port, a number or a device name") @click.option( - "--baud", "-b", + "--baud", type=int, - help="Set baud rate, default=%d" % ProjectOptions["env.monitor_speed"].default, + default=ProjectOptions["env.monitor_speed"].default, + help="Set baud/speed, default=%d" % ProjectOptions["env.monitor_speed"].default, ) @click.option( "--parity", @@ -58,7 +58,9 @@ from platformio.project.options import ProjectOptions help="Set the encoding for the serial port (e.g. hexlify, " "Latin1, UTF-8), default: UTF-8", ) -@click.option("--filter", "-f", multiple=True, help="Add filters/text transformations") +@click.option( + "-f", "--filter", "filters", multiple=True, help="Add filters/text transformations" +) @click.option( "--eol", default="CRLF", @@ -78,13 +80,21 @@ from platformio.project.options import ProjectOptions type=int, default=20, help="ASCII code of special character that is used to " - "control miniterm (menu), default=20 (DEC)", + "control terminal (menu), default=20 (DEC)", ) @click.option( "--quiet", is_flag=True, help="Diagnostics: suppress non-error messages, default=Off", ) +@click.option( + "--reconnect/--no-reconnect", + default=True, + help=( + "If established connection fails, " + "silently retry on the same port, default=True" + ), +) @click.option( "-d", "--project-dir", @@ -96,49 +106,32 @@ from platformio.project.options import ProjectOptions "--environment", help="Load configuration from `platformio.ini` and specified environment", ) -def device_monitor_cmd(**kwargs): # pylint: disable=too-many-branches - project_options = {} +def device_monitor_cmd(**options): platform = None - with fs.cd(kwargs["project_dir"]): + project_options = {} + with fs.cd(options["project_dir"]): try: - project_options = get_project_options(kwargs["environment"]) - kwargs = apply_project_monitor_options(kwargs, project_options) + project_options = get_project_options(options["environment"]) + options = apply_project_monitor_options(options, project_options) if "platform" in project_options: platform = PlatformFactory.new(project_options["platform"]) except NotPlatformIOProjectError: pass - register_filters(platform=platform, options=kwargs) - kwargs["port"] = find_serial_port( - initial_port=kwargs["port"], + register_filters(platform=platform, options=options) + options["port"] = find_serial_port( + initial_port=options["port"], board_config=platform.board_config(project_options.get("board")) if platform and project_options.get("board") else None, upload_protocol=project_options.get("upload_protocol"), ) - # override system argv with patched options - sys.argv = ["monitor"] + project_options_to_monitor_argv( - kwargs, - project_options, - ignore=("port", "baud", "rts", "dtr", "environment", "project_dir"), - ) + if options["menu_char"] == options["exit_char"]: + raise exception.UserSideException( + "--exit-char can not be the same as --menu-char" + ) - if not kwargs["quiet"]: - click.echo( - "--- Available filters and text transformations: %s" - % ", ".join(sorted(miniterm.TRANSFORMATIONS.keys())) - ) - click.echo("--- More details at https://bit.ly/pio-monitor-filters") - try: - miniterm.main( - default_port=kwargs["port"], - default_baudrate=kwargs["baud"] - or ProjectOptions["env.monitor_speed"].default, - default_rts=kwargs["rts"], - default_dtr=kwargs["dtr"], - ) - except Exception as e: - raise exception.MinitermException(e) + start_terminal(options) def get_project_options(environment=None): @@ -148,37 +141,13 @@ def get_project_options(environment=None): return config.items(env=environment, as_dict=True) -def apply_project_monitor_options(cli_options, project_options): +def apply_project_monitor_options(initial_options, project_options): for k in ("port", "speed", "rts", "dtr"): k2 = "monitor_%s" % k if k == "speed": k = "baud" - if cli_options[k] is None and k2 in project_options: - cli_options[k] = project_options[k2] + if initial_options[k] is None and k2 in project_options: + initial_options[k] = project_options[k2] if k != "port": - cli_options[k] = int(cli_options[k]) - return cli_options - - -def project_options_to_monitor_argv(cli_options, project_options, ignore=None): - confmon_flags = project_options.get("monitor_flags", []) - result = confmon_flags[::] - - for f in project_options.get("monitor_filters", []): - result.extend(["--filter", f]) - - for k, v in cli_options.items(): - if v is None or (ignore and k in ignore): - continue - k = "--" + k.replace("_", "-") - if k in confmon_flags: - continue - if isinstance(v, bool): - if v: - result.append(k) - elif isinstance(v, tuple): - for i in v: - result.extend([k, i]) - else: - result.extend([k, str(v)]) - return result + initial_options[k] = int(initial_options[k]) + return initial_options diff --git a/platformio/device/monitor/terminal.py b/platformio/device/monitor/terminal.py new file mode 100644 index 00000000..8da37f7a --- /dev/null +++ b/platformio/device/monitor/terminal.py @@ -0,0 +1,174 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import signal +import threading + +import click +import serial +from serial.tools import miniterm + +from platformio.exception import UserSideException + + +class Terminal(miniterm.Miniterm): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.pio_unexpected_exception = None + + def reader(self): + try: + super().reader() + except Exception as exc: # pylint: disable=broad-except + self.pio_unexpected_exception = exc + + def writer(self): + try: + super().writer() + except Exception as exc: # pylint: disable=broad-except + self.pio_unexpected_exception = exc + + +def start_terminal(options): + retries = 0 + is_port_valid = False + while True: + term = None + try: + term = new_terminal(options) + is_port_valid = True + options["port"] = term.serial.name + if retries: + click.echo("\t Connected!", err=True) + elif not options["quiet"]: + print_terminal_settings(term) + retries = 0 # reset + term.start() + try: + term.join(True) + except KeyboardInterrupt: + pass + term.join() + term.console.cleanup() + term.close() + if term.pio_unexpected_exception: + click.secho( + "Disconnected (%s)" % term.pio_unexpected_exception, + fg="red", + err=True, + ) + if options["reconnect"]: + raise UserSideException(term.pio_unexpected_exception) + return + except UserSideException as exc: + if not is_port_valid: + raise exc + if not retries: + click.echo("Reconnecting to %s " % options["port"], err=True, nl=False) + signal.signal(signal.SIGINT, signal.SIG_DFL) + else: + click.echo(".", err=True, nl=False) + retries += 1 + threading.Event().wait(retries / 2) + + +def new_terminal(options): + term = Terminal( + new_serial_instance(options), + echo=options["echo"], + eol=options["eol"].lower(), + filters=options["filters"] or ["default"], + ) + term.exit_character = chr(options["exit_char"]) + term.menu_character = chr(options["menu_char"]) + term.raw = options["raw"] + term.set_rx_encoding(options["encoding"]) + term.set_tx_encoding(options["encoding"]) + return term + + +def print_terminal_settings(terminal): + click.echo( + "--- Terminal on {p.name} | " + "{p.baudrate} {p.bytesize}-{p.parity}-{p.stopbits}".format(p=terminal.serial) + ) + click.echo( + "--- Available filters and text transformations: %s" + % ", ".join(sorted(miniterm.TRANSFORMATIONS.keys())) + ) + click.echo("--- More details at https://bit.ly/pio-monitor-filters") + click.echo( + "--- Quit: {} | Menu: {} | Help: {} followed by {}".format( + miniterm.key_description(terminal.exit_character), + miniterm.key_description(terminal.menu_character), + miniterm.key_description(terminal.menu_character), + miniterm.key_description("\x08"), + ) + ) + + +def new_serial_instance(options): # pylint: disable=too-many-branches + serial_instance = None + port = options["port"] + while serial_instance is None: + # no port given on command line -> ask user now + if port is None or port == "-": + try: + port = miniterm.ask_for_port() + except KeyboardInterrupt: + click.echo("", err=True) + raise UserSideException("User aborted and port is not given") + else: + if not port: + raise UserSideException("Port is not given") + try: + serial_instance = serial.serial_for_url( + port, + options["baud"], + parity=options["parity"], + rtscts=options["rtscts"], + xonxoff=options["xonxoff"], + do_not_open=True, + ) + + if not hasattr(serial_instance, "cancel_read"): + # enable timeout for alive flag polling if cancel_read is not available + serial_instance.timeout = 1 + + if options["dtr"] is not None: + if not options["quiet"]: + click.echo( + "--- forcing DTR {}".format( + "active" if options["dtr"] else "inactive" + ) + ) + serial_instance.dtr = options["dtr"] + + if options["rts"] is not None: + if not options["quiet"]: + click.echo( + "--- forcing RTS {}".format( + "active" if options["rts"] else "inactive" + ) + ) + serial_instance.rts = options["rts"] + + if isinstance(serial_instance, serial.Serial): + serial_instance.exclusive = True + + serial_instance.open() + except serial.SerialException as exc: + raise UserSideException(exc) + + return serial_instance diff --git a/platformio/exception.py b/platformio/exception.py index 5c0b44ea..a8287c04 100644 --- a/platformio/exception.py +++ b/platformio/exception.py @@ -30,10 +30,6 @@ class ReturnErrorCode(PlatformioException): MESSAGE = "{0}" -class MinitermException(PlatformioException): - pass - - class UserSideException(PlatformioException): pass diff --git a/platformio/remote/cli.py b/platformio/remote/cli.py index 40bf0b8f..25f53800 100644 --- a/platformio/remote/cli.py +++ b/platformio/remote/cli.py @@ -28,7 +28,6 @@ from platformio.device.monitor.command import ( apply_project_monitor_options, device_monitor_cmd, get_project_options, - project_options_to_monitor_argv, ) from platformio.package.manager.core import inject_contrib_pysite from platformio.project.exception import NotPlatformIOProjectError @@ -360,6 +359,7 @@ def device_monitor(ctx, agents, **kwargs): pass kwargs["baud"] = kwargs["baud"] or ProjectOptions["env.monitor_speed"].default + kwargs["reconnect"] = False def _tx_target(sock_dir): subcmd_argv = ["remote"] @@ -387,3 +387,27 @@ def device_monitor(ctx, agents, **kwargs): fs.rmtree(sock_dir) return True + + +def project_options_to_monitor_argv(cli_options, project_options, ignore=None): + confmon_flags = project_options.get("monitor_flags", []) + result = confmon_flags[::] + + for f in project_options.get("monitor_filters", []): + result.extend(["--filter", f]) + + for k, v in cli_options.items(): + if v is None or (ignore and k in ignore): + continue + k = "--" + k.replace("_", "-") + if k in confmon_flags: + continue + if isinstance(v, bool): + if v: + result.append(k) + elif isinstance(v, tuple): + for i in v: + result.extend([k, i]) + else: + result.extend([k, str(v)]) + return result diff --git a/setup.py b/setup.py index a49c1bc9..eea95656 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ minimal_requirements = [ "colorama", "marshmallow==%s" % ("3.*" if sys.version_info >= (3, 7) else "3.14.1"), "pyelftools>=0.27,<1", - "pyserial==3.*", + "pyserial==3.5.*", # keep in sync "device/monitor/terminal.py" "requests==2.*", "requests==%s" % ("2.*" if sys.version_info >= (3, 7) else "2.27.1"), "semantic_version==2.10.*", From 7aaa9c028bcb1c1af650b4a81096764a1de0ac57 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 11 Jun 2022 20:56:24 +0300 Subject: [PATCH 12/93] Bump version to 6.0.3a2 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 6f1c57a7..b615d048 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (6, 0, "3a1") +VERSION = (6, 0, "3a2") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 375c396b7b01cc6902928e4fa2b2c8f2317b24e0 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sun, 12 Jun 2022 12:14:28 +0300 Subject: [PATCH 13/93] Minor improvements for `pio device monitor` command --- HISTORY.rst | 2 +- docs | 2 +- platformio/device/monitor/command.py | 40 +++++++++++++-------------- platformio/device/monitor/terminal.py | 2 +- 4 files changed, 23 insertions(+), 23 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 50efce56..d194ee84 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -19,7 +19,7 @@ PlatformIO Core 6 * **Device Monitor** - Automatically reconnect if a connection fails - - Added new `pio device monitor --no-reconnect `__ option to disable automatic reconnection + - Added new `pio device monitor --no-reconnect `__ option to disable automatic reconnection - Handle disconnects more gracefully (`issue #3939 `_) * Fixed an issue when a custom `pio test --project-config `__ was not handled properly (`issue #4299 `_) diff --git a/docs b/docs index c86b25dd..f0acc304 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit c86b25dd81f4bfec721a66415f052d2c20d9e044 +Subproject commit f0acc3040fa8132d59ef7b930052a36d4c6e0bd6 diff --git a/platformio/device/monitor/command.py b/platformio/device/monitor/command.py index ae63c1a6..0a9a518a 100644 --- a/platformio/device/monitor/command.py +++ b/platformio/device/monitor/command.py @@ -33,30 +33,30 @@ from platformio.project.options import ProjectOptions "--baud", type=int, default=ProjectOptions["env.monitor_speed"].default, - help="Set baud/speed, default=%d" % ProjectOptions["env.monitor_speed"].default, + show_default=True, + help="Set baud/speed", ) @click.option( "--parity", default="N", + show_default=True, type=click.Choice(["N", "E", "O", "S", "M"]), - help="Set parity, default=N", -) -@click.option("--rtscts", is_flag=True, help="Enable RTS/CTS flow control, default=Off") -@click.option( - "--xonxoff", is_flag=True, help="Enable software flow control, default=Off" + help="Set parity", ) +@click.option("--rtscts", is_flag=True, help="Enable RTS/CTS flow control") +@click.option("--xonxoff", is_flag=True, help="Enable software flow control") @click.option( "--rts", default=None, type=click.IntRange(0, 1), help="Set initial RTS line state" ) @click.option( "--dtr", default=None, type=click.IntRange(0, 1), help="Set initial DTR line state" ) -@click.option("--echo", is_flag=True, help="Enable local echo, default=Off") +@click.option("--echo", is_flag=True, help="Enable local echo") @click.option( "--encoding", default="UTF-8", - help="Set the encoding for the serial port (e.g. hexlify, " - "Latin1, UTF-8), default: UTF-8", + show_default=True, + help="Set the encoding for the serial port (e.g. hexlify, Latin1, UTF-8)", ) @click.option( "-f", "--filter", "filters", multiple=True, help="Add filters/text transformations" @@ -64,36 +64,35 @@ from platformio.project.options import ProjectOptions @click.option( "--eol", default="CRLF", + show_default=True, type=click.Choice(["CR", "LF", "CRLF"]), - help="End of line mode, default=CRLF", + help="End of line mode", ) @click.option("--raw", is_flag=True, help="Do not apply any encodings/transformations") @click.option( "--exit-char", type=int, default=3, + show_default=True, help="ASCII code of special character that is used to exit " - "the application, default=3 (Ctrl+C)", + "the application [default=3 (Ctrl+C)]", ) @click.option( "--menu-char", type=int, default=20, help="ASCII code of special character that is used to " - "control terminal (menu), default=20 (DEC)", + "control terminal (menu) [default=20 (DEC)]", ) @click.option( "--quiet", is_flag=True, - help="Diagnostics: suppress non-error messages, default=Off", + help="Diagnostics: suppress non-error messages", ) @click.option( - "--reconnect/--no-reconnect", - default=True, - help=( - "If established connection fails, " - "silently retry on the same port, default=True" - ), + "--no-reconnect", + is_flag=True, + help="Disable automatic reconnection if the established connection fails", ) @click.option( "-d", @@ -104,7 +103,7 @@ from platformio.project.options import ProjectOptions @click.option( "-e", "--environment", - help="Load configuration from `platformio.ini` and specified environment", + help="Load configuration from `platformio.ini` and the specified environment", ) def device_monitor_cmd(**options): platform = None @@ -131,6 +130,7 @@ def device_monitor_cmd(**options): "--exit-char can not be the same as --menu-char" ) + print(options) start_terminal(options) diff --git a/platformio/device/monitor/terminal.py b/platformio/device/monitor/terminal.py index 8da37f7a..6cb78e20 100644 --- a/platformio/device/monitor/terminal.py +++ b/platformio/device/monitor/terminal.py @@ -68,7 +68,7 @@ def start_terminal(options): fg="red", err=True, ) - if options["reconnect"]: + if not options["no_reconnect"]: raise UserSideException(term.pio_unexpected_exception) return except UserSideException as exc: From 4b5bc91abb743d8858ef08e814ea2c72da7866de Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sun, 12 Jun 2022 12:33:02 +0300 Subject: [PATCH 14/93] =?UTF-8?q?Merged=20the=20Unit=20Testing=20=E2=80=9C?= =?UTF-8?q?building=E2=80=9D=20stage=20with=20=E2=80=9Cuploading=E2=80=9D?= =?UTF-8?q?=20for=20the=20embedded=20target=20//=20Resolve=20#4307?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- HISTORY.rst | 1 + platformio/test/runners/base.py | 10 ++++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index d194ee84..2b2fa4cb 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -22,6 +22,7 @@ PlatformIO Core 6 - Added new `pio device monitor --no-reconnect `__ option to disable automatic reconnection - Handle disconnects more gracefully (`issue #3939 `_) +* Merged the |UNITTESTING| "building" stage with "uploading" for the embedded target (`issue #4307 `_) * Fixed an issue when a custom `pio test --project-config `__ was not handled properly (`issue #4299 `_) 6.0.2 (2022-06-01) diff --git a/platformio/test/runners/base.py b/platformio/test/runners/base.py index 2a0e6cec..218e0c11 100644 --- a/platformio/test/runners/base.py +++ b/platformio/test/runners/base.py @@ -117,6 +117,9 @@ class TestRunnerBase: def stage_building(self): if self.options.without_building: return None + # run "building" once at the "uploading" stage for the embedded target + if not self.options.without_uploading and self.platform.is_embedded(): + return None click.secho("Building...", bold=True) targets = ["__test"] if not self.options.without_debugging: @@ -132,9 +135,12 @@ class TestRunnerBase: ) def stage_uploading(self): - if self.options.without_uploading or not self.platform.is_embedded(): + is_embedded = self.platform.is_embedded() + if self.options.without_uploading or not is_embedded: return None - click.secho("Uploading...", bold=True) + click.secho( + "Building & Uploading..." if is_embedded else "Uploading...", bold=True + ) targets = ["upload"] if self.options.without_building: targets.append("nobuild") From 854c549e1ce8c1c48d8e2b95371e347b5124b31b Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 13 Jun 2022 20:19:09 +0300 Subject: [PATCH 15/93] Restore original standard streams for device monitor // Issue #3939 --- platformio/device/monitor/command.py | 1 - platformio/device/monitor/terminal.py | 11 +++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/platformio/device/monitor/command.py b/platformio/device/monitor/command.py index 0a9a518a..24216b6c 100644 --- a/platformio/device/monitor/command.py +++ b/platformio/device/monitor/command.py @@ -130,7 +130,6 @@ def device_monitor_cmd(**options): "--exit-char can not be the same as --menu-char" ) - print(options) start_terminal(options) diff --git a/platformio/device/monitor/terminal.py b/platformio/device/monitor/terminal.py index 6cb78e20..9a043b6e 100644 --- a/platformio/device/monitor/terminal.py +++ b/platformio/device/monitor/terminal.py @@ -13,6 +13,7 @@ # limitations under the License. import signal +import sys import threading import click @@ -60,8 +61,17 @@ def start_terminal(options): except KeyboardInterrupt: pass term.join() + + # cleanup term.console.cleanup() + + # restore original standard streams + sys.stdin = sys.__stdin__ + sys.stdout = sys.__stdout__ + sys.stderr = sys.__stderr__ + term.close() + if term.pio_unexpected_exception: click.secho( "Disconnected (%s)" % term.pio_unexpected_exception, @@ -70,6 +80,7 @@ def start_terminal(options): ) if not options["no_reconnect"]: raise UserSideException(term.pio_unexpected_exception) + return except UserSideException as exc: if not is_port_valid: From 46858fff3926acba0da60bc014b44cb7d90e3ab1 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 13 Jun 2022 20:19:28 +0300 Subject: [PATCH 16/93] Bump version to 6.0.3a3 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index b615d048..14b1458b 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (6, 0, "3a2") +VERSION = (6, 0, "3a3") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From bd21ff0d3e4b61a7c2cef6f1cc335c696ccb7c05 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 14 Jun 2022 10:02:23 +0300 Subject: [PATCH 17/93] Fixed an issue when monitor_speed was ignored in configuration file // Resolve #4319 --- platformio/device/monitor/command.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/platformio/device/monitor/command.py b/platformio/device/monitor/command.py index 24216b6c..ac74b336 100644 --- a/platformio/device/monitor/command.py +++ b/platformio/device/monitor/command.py @@ -32,9 +32,8 @@ from platformio.project.options import ProjectOptions "-b", "--baud", type=int, - default=ProjectOptions["env.monitor_speed"].default, show_default=True, - help="Set baud/speed", + help="Set baud/speed [default=%d]" % ProjectOptions["env.monitor_speed"].default, ) @click.option( "--parity", From 743a3e2c02f3d54cb7f2aaa12181e405206c2605 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 15 Jun 2022 12:51:06 +0300 Subject: [PATCH 18/93] Improved a serial port finder for a board with predefined HWIDs // Resolve #4308 --- HISTORY.rst | 1 + platformio/device/finder.py | 63 ++++++++++++++--------- platformio/device/monitor/command.py | 3 ++ platformio/test/runners/readers/serial.py | 13 ++--- platformio/util.py | 44 ++++++++++++++++ 5 files changed, 90 insertions(+), 34 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 2b2fa4cb..b3f0490e 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -22,6 +22,7 @@ PlatformIO Core 6 - Added new `pio device monitor --no-reconnect `__ option to disable automatic reconnection - Handle disconnects more gracefully (`issue #3939 `_) +* Improved a serial port finder for a board with predefined HWIDs * Merged the |UNITTESTING| "building" stage with "uploading" for the embedded target (`issue #4307 `_) * Fixed an issue when a custom `pio test --project-config `__ was not handled properly (`issue #4299 `_) diff --git a/platformio/device/finder.py b/platformio/device/finder.py index 382fe000..126df554 100644 --- a/platformio/device/finder.py +++ b/platformio/device/finder.py @@ -19,6 +19,7 @@ import serial from platformio.compat import IS_WINDOWS from platformio.device.list.util import list_logical_devices, list_serial_ports +from platformio.util import retry def is_pattern_port(port): @@ -44,21 +45,20 @@ def is_serial_port_ready(port, timeout=1): def find_serial_port( - initial_port, board_config=None, upload_protocol=None, ensure_ready=False + initial_port, board_config=None, upload_protocol=None, ensure_ready=False, timeout=3 ): if initial_port: if not is_pattern_port(initial_port): return initial_port return match_serial_port(initial_port) - port = None + if upload_protocol and upload_protocol.startswith("blackmagic"): - port = find_blackmagic_serial_port() - if not port and board_config: - port = find_board_serial_port(board_config) - if port: - return port + return find_blackmagic_serial_port(timeout=timeout) + if board_config and board_config.get("build.hwids"): + return find_board_serial_port(board_config.get("build.hwids"), timeout=timeout) # pick the last PID:VID USB device + port = None usb_port = None for item in list_serial_ports(): if ensure_ready and not is_serial_port_ready(item["port"]): @@ -69,26 +69,41 @@ def find_serial_port( return usb_port or port -def find_blackmagic_serial_port(): - for item in list_serial_ports(): - port = item["port"] - if IS_WINDOWS and port.startswith("COM") and len(port) > 4: - port = "\\\\.\\%s" % port - if "GDB" in item["description"]: - return port +def find_blackmagic_serial_port(timeout=0): + try: + + @retry(timeout=timeout) + def wrapper(): + for item in list_serial_ports(): + port = item["port"] + if IS_WINDOWS and port.startswith("COM") and len(port) > 4: + port = "\\\\.\\%s" % port + if "GDB" in item["description"]: + return port + raise retry.RetryNextException() + + return wrapper() + except retry.RetryStopException: + pass return None -def find_board_serial_port(board_config): - board_hwids = board_config.get("build.hwids", []) - if not board_hwids: - return None - for item in list_serial_ports(filter_hwid=True): - port = item["port"] - for hwid in board_hwids: - hwid_str = ("%s:%s" % (hwid[0], hwid[1])).replace("0x", "") - if hwid_str in item["hwid"]: - return port +def find_board_serial_port(hwids, timeout=0): + try: + + @retry(timeout=timeout) + def wrapper(): + for item in list_serial_ports(filter_hwid=True): + port = item["port"] + for hwid in hwids: + hwid_str = ("%s:%s" % (hwid[0], hwid[1])).replace("0x", "") + if hwid_str in item["hwid"]: + return port + raise retry.RetryNextException() + + return wrapper() + except retry.RetryStopException: + pass return None diff --git a/platformio/device/monitor/command.py b/platformio/device/monitor/command.py index ac74b336..16f3e8df 100644 --- a/platformio/device/monitor/command.py +++ b/platformio/device/monitor/command.py @@ -122,8 +122,11 @@ def device_monitor_cmd(**options): if platform and project_options.get("board") else None, upload_protocol=project_options.get("upload_protocol"), + ensure_ready=True, ) + options["baud"] = options["baud"] or ProjectOptions["env.monitor_speed"].default + if options["menu_char"] == options["exit_char"]: raise exception.UserSideException( "--exit-char can not be the same as --menu-char" diff --git a/platformio/test/runners/readers/serial.py b/platformio/test/runners/readers/serial.py index 5ac5e0f3..e2a7eddb 100644 --- a/platformio/test/runners/readers/serial.py +++ b/platformio/test/runners/readers/serial.py @@ -66,7 +66,7 @@ class SerialTestOutputReader: project_options = self.test_runner.project_config.items( env=self.test_runner.test_suite.env_name, as_dict=True ) - scan_options = dict( + port = find_serial_port( initial_port=self.test_runner.get_test_port(), board_config=self.test_runner.platform.board_config( project_options["board"] @@ -74,15 +74,8 @@ class SerialTestOutputReader: upload_protocol=project_options.get("upload_protocol"), ensure_ready=True, ) - - elapsed = 0 - while elapsed < 5: - port = find_serial_port(**scan_options) - if port: - return port - sleep(0.25) - elapsed += 0.25 - + if port: + return port raise UserSideException( "Please specify `test_port` for environment or use " "global `--test-port` option." diff --git a/platformio/util.py b/platformio/util.py index 1dbac429..56e95d2f 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -78,6 +78,50 @@ class throttle(object): return wrapper +# Retry: Begin + + +class RetryException(Exception): + pass + + +class RetryNextException(RetryException): + pass + + +class RetryStopException(RetryException): + pass + + +class retry(object): + + RetryNextException = RetryNextException + RetryStopException = RetryStopException + + def __init__(self, timeout=0, step=0.25): + self.timeout = timeout + self.step = step + + def __call__(self, func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + elapsed = 0 + while True: + try: + return func(*args, **kwargs) + except self.RetryNextException: + pass + if elapsed >= self.timeout: + raise self.RetryStopException() + elapsed += self.step + time.sleep(self.step) + + return wrapper + + +# Retry: End + + def singleton(cls): """From PEP-318 http://www.python.org/dev/peps/pep-0318/#examples""" _instances = {} From 6f9985125df9f35252354c0f4e2938a17010f066 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 15 Jun 2022 14:30:05 +0300 Subject: [PATCH 19/93] Fixed an issue when the monitor filters were not applied in their order // Resolve #4320 --- platformio/device/monitor/terminal.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/device/monitor/terminal.py b/platformio/device/monitor/terminal.py index 9a043b6e..05c241c9 100644 --- a/platformio/device/monitor/terminal.py +++ b/platformio/device/monitor/terminal.py @@ -99,7 +99,7 @@ def new_terminal(options): new_serial_instance(options), echo=options["echo"], eol=options["eol"].lower(), - filters=options["filters"] or ["default"], + filters=list(reversed(options["filters"] or ["default"])), ) term.exit_character = chr(options["exit_char"]) term.menu_character = chr(options["menu_char"]) From 30709fd0b36e135642565cdd3e89f2a8eb9331c6 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 15 Jun 2022 15:36:17 +0300 Subject: [PATCH 20/93] Replaced monitor_flags with independent project configuration options: monitor_parity, monitor_eol, monitor_raw, monitor_echo --- HISTORY.rst | 2 + docs | 2 +- platformio/device/monitor/command.py | 64 ++++++++++++++---------- platformio/project/options.py | 31 +++++++++--- platformio/remote/cli.py | 75 ++++++++++++++-------------- 5 files changed, 103 insertions(+), 71 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index b3f0490e..3a423748 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -21,6 +21,8 @@ PlatformIO Core 6 - Automatically reconnect if a connection fails - Added new `pio device monitor --no-reconnect `__ option to disable automatic reconnection - Handle disconnects more gracefully (`issue #3939 `_) + - Replaced ``monitor_flags`` with independent project configuration options: `monitor_parity `__, `monitor_eol `__, `monitor_raw `__, `monitor_echo `__ + - Fixed an issue when the monitor filters were not applied in their order (`issue #4320 `_) * Improved a serial port finder for a board with predefined HWIDs * Merged the |UNITTESTING| "building" stage with "uploading" for the embedded target (`issue #4307 `_) diff --git a/docs b/docs index f0acc304..a3c98fe6 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit f0acc3040fa8132d59ef7b930052a36d4c6e0bd6 +Subproject commit a3c98fe6f3cf13738d16b497f0396c7bbe20ae0c diff --git a/platformio/device/monitor/command.py b/platformio/device/monitor/command.py index 16f3e8df..44df4043 100644 --- a/platformio/device/monitor/command.py +++ b/platformio/device/monitor/command.py @@ -13,6 +13,7 @@ # limitations under the License. import os +import sys import click @@ -31,24 +32,26 @@ from platformio.project.options import ProjectOptions @click.option( "-b", "--baud", - type=int, - show_default=True, + type=ProjectOptions["env.monitor_speed"].type, help="Set baud/speed [default=%d]" % ProjectOptions["env.monitor_speed"].default, ) @click.option( "--parity", - default="N", - show_default=True, - type=click.Choice(["N", "E", "O", "S", "M"]), - help="Set parity", + type=ProjectOptions["env.monitor_parity"].type, + help="Enable parity checking [default=%s]" + % ProjectOptions["env.monitor_parity"].default, ) @click.option("--rtscts", is_flag=True, help="Enable RTS/CTS flow control") @click.option("--xonxoff", is_flag=True, help="Enable software flow control") @click.option( - "--rts", default=None, type=click.IntRange(0, 1), help="Set initial RTS line state" + "--rts", + type=ProjectOptions["env.monitor_rts"].type, + help="Set initial RTS line state", ) @click.option( - "--dtr", default=None, type=click.IntRange(0, 1), help="Set initial DTR line state" + "--dtr", + type=ProjectOptions["env.monitor_dtr"].type, + help="Set initial DTR line state", ) @click.option("--echo", is_flag=True, help="Enable local echo") @click.option( @@ -58,16 +61,18 @@ from platformio.project.options import ProjectOptions help="Set the encoding for the serial port (e.g. hexlify, Latin1, UTF-8)", ) @click.option( - "-f", "--filter", "filters", multiple=True, help="Add filters/text transformations" + "-f", + "--filter", + "filters", + multiple=True, + help="Apply filters/text transformations", ) @click.option( "--eol", - default="CRLF", - show_default=True, - type=click.Choice(["CR", "LF", "CRLF"]), - help="End of line mode", + type=ProjectOptions["env.monitor_eol"].type, + help="End of line mode [default=%s]" % ProjectOptions["env.monitor_eol"].default, ) -@click.option("--raw", is_flag=True, help="Do not apply any encodings/transformations") +@click.option("--raw", is_flag=True, help=ProjectOptions["env.monitor_raw"].description) @click.option( "--exit-char", type=int, @@ -105,16 +110,17 @@ from platformio.project.options import ProjectOptions help="Load configuration from `platformio.ini` and the specified environment", ) def device_monitor_cmd(**options): - platform = None - project_options = {} with fs.cd(options["project_dir"]): + platform = None + project_options = {} try: project_options = get_project_options(options["environment"]) - options = apply_project_monitor_options(options, project_options) if "platform" in project_options: platform = PlatformFactory.new(project_options["platform"]) except NotPlatformIOProjectError: pass + + options = apply_project_monitor_options(options, project_options) register_filters(platform=platform, options=options) options["port"] = find_serial_port( initial_port=options["port"], @@ -125,8 +131,6 @@ def device_monitor_cmd(**options): ensure_ready=True, ) - options["baud"] = options["baud"] or ProjectOptions["env.monitor_speed"].default - if options["menu_char"] == options["exit_char"]: raise exception.UserSideException( "--exit-char can not be the same as --menu-char" @@ -143,12 +147,18 @@ def get_project_options(environment=None): def apply_project_monitor_options(initial_options, project_options): - for k in ("port", "speed", "rts", "dtr"): - k2 = "monitor_%s" % k - if k == "speed": - k = "baud" - if initial_options[k] is None and k2 in project_options: - initial_options[k] = project_options[k2] - if k != "port": - initial_options[k] = int(initial_options[k]) + for option_meta in ProjectOptions.values(): + if option_meta.group != "monitor": + continue + cli_key = option_meta.name.split("_", 1)[1] + if cli_key == "speed": + cli_key = "baud" + # value set from CLI, skip overriding + if initial_options[cli_key] not in (None, (), []) and ( + option_meta.type != click.BOOL or f"--{cli_key}" in sys.argv[1:] + ): + continue + initial_options[cli_key] = project_options.get( + option_meta.name, option_meta.default + ) return initial_options diff --git a/platformio/project/options.py b/platformio/project/options.py index ad96b24c..21a9a130 100644 --- a/platformio/project/options.py +++ b/platformio/project/options.py @@ -519,6 +519,13 @@ ProjectOptions = OrderedDict( oldnames=["monitor_baud"], default=9600, ), + ConfigEnvOption( + group="monitor", + name="monitor_parity", + description="A monitor parity checking", + type=click.Choice(["N", "E", "O", "S", "M"]), + default="N", + ), ConfigEnvOption( group="monitor", name="monitor_filters", @@ -541,12 +548,24 @@ ProjectOptions = OrderedDict( ), ConfigEnvOption( group="monitor", - name="monitor_flags", - description=( - "The extra flags and options for `platformio device monitor` " - "command" - ), - multiple=True, + name="monitor_eol", + description="A monitor end of line mode", + type=click.Choice(["CR", "LF", "CRLF"]), + default="CRLF", + ), + ConfigEnvOption( + group="monitor", + name="monitor_raw", + description="Disable encodings/transformations of device output", + type=click.BOOL, + default=False, + ), + ConfigEnvOption( + group="monitor", + name="monitor_echo", + description="Enable a monitor local echo", + type=click.BOOL, + default=False, ), # Library ConfigEnvOption( diff --git a/platformio/remote/cli.py b/platformio/remote/cli.py index 25f53800..bede41d1 100644 --- a/platformio/remote/cli.py +++ b/platformio/remote/cli.py @@ -270,60 +270,67 @@ def device_list(agents, json_output): @remote_device.command("monitor", short_help="Monitor remote device") @click.option("--port", "-p", help="Port, a number or a device name") @click.option( - "--baud", "-b", - type=int, - help="Set baud rate, default=%d" % ProjectOptions["env.monitor_speed"].default, + "--baud", + type=ProjectOptions["env.monitor_speed"].type, + help="Set baud/speed [default=%d]" % ProjectOptions["env.monitor_speed"].default, ) @click.option( "--parity", - default="N", - type=click.Choice(["N", "E", "O", "S", "M"]), - help="Set parity, default=N", + type=ProjectOptions["env.monitor_parity"].type, + help="Set parity [default=%s]" % ProjectOptions["env.monitor_parity"].default, ) -@click.option("--rtscts", is_flag=True, help="Enable RTS/CTS flow control, default=Off") +@click.option("--rtscts", is_flag=True, help="Enable RTS/CTS flow control") +@click.option("--xonxoff", is_flag=True, help="Enable software flow control") @click.option( - "--xonxoff", is_flag=True, help="Enable software flow control, default=Off" + "--rts", + type=ProjectOptions["env.monitor_rts"].type, + help="Set initial RTS line state", ) @click.option( - "--rts", default=None, type=click.IntRange(0, 1), help="Set initial RTS line state" + "--dtr", + type=ProjectOptions["env.monitor_dtr"].type, + help="Set initial DTR line state", ) -@click.option( - "--dtr", default=None, type=click.IntRange(0, 1), help="Set initial DTR line state" -) -@click.option("--echo", is_flag=True, help="Enable local echo, default=Off") +@click.option("--echo", is_flag=True, help="Enable local echo") @click.option( "--encoding", default="UTF-8", - help="Set the encoding for the serial port (e.g. hexlify, " - "Latin1, UTF-8), default: UTF-8", + show_default=True, + help="Set the encoding for the serial port (e.g. hexlify, Latin1, UTF-8)", +) +@click.option( + "-f", + "--filter", + "filters", + multiple=True, + help="Apply filters/text transformations", ) -@click.option("--filter", "-f", multiple=True, help="Add text transformation") @click.option( "--eol", - default="CRLF", - type=click.Choice(["CR", "LF", "CRLF"]), - help="End of line mode, default=CRLF", + type=ProjectOptions["env.monitor_eol"].type, + help="End of line mode [default=%s]" % ProjectOptions["env.monitor_eol"].default, ) -@click.option("--raw", is_flag=True, help="Do not apply any encodings/transformations") +@click.option("--raw", is_flag=True, help=ProjectOptions["env.monitor_raw"].description) @click.option( "--exit-char", type=int, default=3, + show_default=True, help="ASCII code of special character that is used to exit " - "the application, default=3 (Ctrl+C)", + "the application [default=3 (Ctrl+C)]", ) @click.option( "--menu-char", type=int, default=20, help="ASCII code of special character that is used to " - "control miniterm (menu), default=20 (DEC)", + "control terminal (menu) [default=20 (DEC)]", ) @click.option( "--quiet", is_flag=True, - help="Diagnostics: suppress non-error messages, default=Off", + help="Diagnostics: suppress non-error messages", ) @click.option( "-d", @@ -354,19 +361,17 @@ def device_monitor(ctx, agents, **kwargs): try: with fs.cd(kwargs["project_dir"]): project_options = get_project_options(kwargs["environment"]) - kwargs = apply_project_monitor_options(kwargs, project_options) except NotPlatformIOProjectError: pass - kwargs["baud"] = kwargs["baud"] or ProjectOptions["env.monitor_speed"].default - kwargs["reconnect"] = False + kwargs = apply_project_monitor_options(kwargs, project_options) def _tx_target(sock_dir): subcmd_argv = ["remote"] for agent in agents: subcmd_argv.extend(["--agent", agent]) subcmd_argv.extend(["device", "monitor"]) - subcmd_argv.extend(project_options_to_monitor_argv(kwargs, project_options)) + subcmd_argv.extend(project_options_to_monitor_argv(kwargs)) subcmd_argv.extend(["--sock", sock_dir]) subprocess.call([proc.where_is_program("platformio")] + subcmd_argv) @@ -381,6 +386,7 @@ def device_monitor(ctx, agents, **kwargs): return with open(sock_file, encoding="utf8") as fp: kwargs["port"] = fp.read() + kwargs["no_reconnect"] = True ctx.invoke(device_monitor_cmd, **kwargs) t.join(2) finally: @@ -389,19 +395,14 @@ def device_monitor(ctx, agents, **kwargs): return True -def project_options_to_monitor_argv(cli_options, project_options, ignore=None): - confmon_flags = project_options.get("monitor_flags", []) - result = confmon_flags[::] - - for f in project_options.get("monitor_filters", []): - result.extend(["--filter", f]) - +def project_options_to_monitor_argv(cli_options): + result = [] + for item in cli_options["filters"] or []: + result.extend(["--filter", item]) for k, v in cli_options.items(): - if v is None or (ignore and k in ignore): + if v is None or k == "filters": continue k = "--" + k.replace("_", "-") - if k in confmon_flags: - continue if isinstance(v, bool): if v: result.append(k) From ac63cf02407ec0e85bf5533d95dda4391cd843b8 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 15 Jun 2022 15:37:20 +0300 Subject: [PATCH 21/93] Bump version to 6.0.3a4 --- platformio/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 14b1458b..98339fa8 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (6, 0, "3a3") +VERSION = (6, 0, "3a4") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" @@ -47,7 +47,7 @@ __pioremote_endpoint__ = "ssl:host=remote.platformio.org:port=4413" __default_requests_timeout__ = (10, None) # (connect, read) __core_packages__ = { - "contrib-piohome": "~3.4.1", + "contrib-piohome": "~3.4.2", "contrib-pysite": "~2.%d%d.0" % (sys.version_info.major, sys.version_info.minor), "tool-scons": "~4.40300.0", "tool-cppcheck": "~1.270.0", From e12bc9fe5f0b6020f3cfafc8477f2e754759387d Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 15 Jun 2022 18:05:55 +0300 Subject: [PATCH 22/93] Do not resolve dependencies from the project "src" folder when the `test_build_src` option is not enabled --- HISTORY.rst | 8 ++++++-- platformio/builder/tools/piolib.py | 7 +++++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 3a423748..c68e7f6f 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -24,9 +24,13 @@ PlatformIO Core 6 - Replaced ``monitor_flags`` with independent project configuration options: `monitor_parity `__, `monitor_eol `__, `monitor_raw `__, `monitor_echo `__ - Fixed an issue when the monitor filters were not applied in their order (`issue #4320 `_) +* **Unit Testing** + + - Merged the |UNITTESTING| "building" stage with "uploading" for the embedded target (`issue #4307 `_) + - Do not resolve dependencies from the project "src" folder when the `test_build_src `__ option is not enabled + - Fixed an issue when a custom `pio test --project-config `__ was not handled properly (`issue #4299 `_) + * Improved a serial port finder for a board with predefined HWIDs -* Merged the |UNITTESTING| "building" stage with "uploading" for the embedded target (`issue #4307 `_) -* Fixed an issue when a custom `pio test --project-config `__ was not handled properly (`issue #4299 `_) 6.0.2 (2022-06-01) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index 104e8496..f868ed2d 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -889,10 +889,13 @@ class ProjectAsLibBuilder(LibBuilderBase): return include_dirs def get_search_files(self): + items = [] + build_type = self.env.GetBuildType() # project files - items = super().get_search_files() + if "test" not in build_type or self.env.GetProjectOption("test_build_src"): + items.extend(super().get_search_files()) # test files - if "test" in self.env.GetBuildType(): + if "test" in build_type: items.extend( [ os.path.join("$PROJECT_TEST_DIR", item) From 4bccaae94512b89c6e76d38f1ff06330785dff85 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 16 Jun 2022 09:45:20 +0300 Subject: [PATCH 23/93] Fix an issue with serial port finding when board does not have HWIDs // Issue #4323 --- platformio/device/finder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/device/finder.py b/platformio/device/finder.py index 126df554..d26e959e 100644 --- a/platformio/device/finder.py +++ b/platformio/device/finder.py @@ -54,7 +54,7 @@ def find_serial_port( if upload_protocol and upload_protocol.startswith("blackmagic"): return find_blackmagic_serial_port(timeout=timeout) - if board_config and board_config.get("build.hwids"): + if board_config and board_config.get("build.hwids", []): return find_board_serial_port(board_config.get("build.hwids"), timeout=timeout) # pick the last PID:VID USB device From 63c2278a83ad88cce88f2e4f524fec55b400e902 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 16 Jun 2022 09:46:04 +0300 Subject: [PATCH 24/93] Bump version to 6.0.3a5 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 98339fa8..9f5464a0 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (6, 0, "3a4") +VERSION = (6, 0, "3a5") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 50cbc4d4e2fb69780768f333730db40fcd9c7d06 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 16 Jun 2022 13:04:38 +0300 Subject: [PATCH 25/93] Allowed to Import("projenv") in a library extra script // Resolve #4305 --- HISTORY.rst | 1 + docs | 2 +- platformio/builder/tools/piolib.py | 2 ++ platformio/builder/tools/platformio.py | 3 --- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index c68e7f6f..f2c17c6d 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -30,6 +30,7 @@ PlatformIO Core 6 - Do not resolve dependencies from the project "src" folder when the `test_build_src `__ option is not enabled - Fixed an issue when a custom `pio test --project-config `__ was not handled properly (`issue #4299 `_) +* Allowed to ``Import("projenv")`` in a library extra script (`issue #4305 `_) * Improved a serial port finder for a board with predefined HWIDs 6.0.2 (2022-06-01) diff --git a/docs b/docs index a3c98fe6..b3bb4bfe 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit a3c98fe6f3cf13738d16b497f0396c7bbe20ae0c +Subproject commit b3bb4bfe67f98edbf5acfbdbb2cf61d7912dec29 diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index f868ed2d..3a0072c2 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -1142,6 +1142,8 @@ def ConfigureProjectLibBuilder(env): _print_deps_tree(lb, level + 1) project = ProjectAsLibBuilder(env, "$PROJECT_DIR") + env.Export(dict(projenv=project.env)) + ldf_mode = LibBuilderBase.lib_ldf_mode.fget(project) # pylint: disable=no-member click.echo("LDF: Library Dependency Finder -> https://bit.ly/configure-pio-ldf") diff --git a/platformio/builder/tools/platformio.py b/platformio/builder/tools/platformio.py index 69875a70..9b2aedb8 100644 --- a/platformio/builder/tools/platformio.py +++ b/platformio/builder/tools/platformio.py @@ -23,7 +23,6 @@ from SCons.Node import FS # pylint: disable=import-error from SCons.Script import COMMAND_LINE_TARGETS # pylint: disable=import-error from SCons.Script import AlwaysBuild # pylint: disable=import-error from SCons.Script import DefaultEnvironment # pylint: disable=import-error -from SCons.Script import Export # pylint: disable=import-error from SCons.Script import SConscript # pylint: disable=import-error from platformio import __version__, fs @@ -180,8 +179,6 @@ def ProcessProjectDeps(env): ) env.Exit(1) - Export("projenv") - def ParseFlagsExtended(env, flags): # pylint: disable=too-many-branches if not isinstance(flags, list): From 42690d3fa76d17fc76b6042731e533cb6f1e9fbf Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 16 Jun 2022 16:58:03 +0300 Subject: [PATCH 26/93] Find serial port using known device HWIDs --- platformio/device/finder.py | 93 ++++++++++++++++++++++++++++++++----- 1 file changed, 81 insertions(+), 12 deletions(-) diff --git a/platformio/device/finder.py b/platformio/device/finder.py index d26e959e..89d99a94 100644 --- a/platformio/device/finder.py +++ b/platformio/device/finder.py @@ -15,12 +15,32 @@ import os from fnmatch import fnmatch +import click import serial from platformio.compat import IS_WINDOWS from platformio.device.list.util import list_logical_devices, list_serial_ports +from platformio.package.manager.platform import PlatformPackageManager +from platformio.platform.factory import PlatformFactory from platformio.util import retry +KNOWN_UART_HWIDS = ( + # Silicon Labs + "10C4:EA60", # CP210X + "10C4:EA61", # CP210X + "10C4:EA63", # CP210X + "10C4:EA70", # CP2105 + "10C4:EA71", # CP2108 + "10C4:EA80", # CP2110 + "10C4:80A9", # CP210X +) + + +def normalize_board_hwid(value): + if isinstance(value, (list, tuple)): + value = ("%s:%s" % (value[0], value[1])).replace("0x", "") + return value.upper() + def is_pattern_port(port): if not port: @@ -53,20 +73,22 @@ def find_serial_port( return match_serial_port(initial_port) if upload_protocol and upload_protocol.startswith("blackmagic"): - return find_blackmagic_serial_port(timeout=timeout) + return find_blackmagic_serial_port(timeout) if board_config and board_config.get("build.hwids", []): - return find_board_serial_port(board_config.get("build.hwids"), timeout=timeout) + return find_board_serial_port(board_config, timeout) + port = find_known_uart_port(ensure_ready, timeout) + if port: + return port - # pick the last PID:VID USB device - port = None - usb_port = None + # pick the best PID:VID USB device + best_port = None for item in list_serial_ports(): if ensure_ready and not is_serial_port_ready(item["port"]): continue port = item["port"] if "VID:PID" in item["hwid"]: - usb_port = port - return usb_port or port + best_port = port + return best_port or port def find_blackmagic_serial_port(timeout=0): @@ -88,22 +110,69 @@ def find_blackmagic_serial_port(timeout=0): return None -def find_board_serial_port(hwids, timeout=0): +def find_board_serial_port(board_config, timeout=0): + hwids = board_config.get("build.hwids", []) + try: + + @retry(timeout=timeout) + def wrapper(): + for item in list_serial_ports(filter_hwid=True): + hwid = item["hwid"].upper() + for board_hwid in hwids: + if normalize_board_hwid(board_hwid) in hwid: + return item["port"] + raise retry.RetryNextException() + + return wrapper() + except retry.RetryStopException: + pass + + click.secho( + "TimeoutError: Could not automatically find serial port " + "for the `%s` board based on the declared HWIDs=%s" + % (board_config.get("name", "unknown"), hwids), + fg="yellow", + err=True, + ) + + return None + + +def find_known_uart_port(ensure_ready=False, timeout=0): + known_hwids = list(KNOWN_UART_HWIDS) + # load HWIDs from installed dev-platforms + for platform in PlatformPackageManager().get_installed(): + p = PlatformFactory.new(platform) + for board_config in p.get_boards().values(): + for board_hwid in board_config.get("build.hwids", []): + board_hwid = normalize_board_hwid(board_hwid) + if board_hwid not in known_hwids: + known_hwids.append(board_hwid) + try: @retry(timeout=timeout) def wrapper(): for item in list_serial_ports(filter_hwid=True): port = item["port"] - for hwid in hwids: - hwid_str = ("%s:%s" % (hwid[0], hwid[1])).replace("0x", "") - if hwid_str in item["hwid"]: - return port + hwid = item["hwid"].upper() + if not any(item in hwid for item in known_hwids): + continue + if not ensure_ready or is_serial_port_ready(port): + return port raise retry.RetryNextException() return wrapper() except retry.RetryStopException: pass + + click.secho( + "TimeoutError: Could not automatically find serial port " + "based on the known UART bridges", + fg="yellow", + err=True, + ) + return None From 7a01da70391c32513f5d57bffcecfec0625de9db Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 17 Jun 2022 12:25:52 +0300 Subject: [PATCH 27/93] Added ``env.StringifyMacro(value)`` helper function for the Advanced Scripting --- HISTORY.rst | 1 + platformio/builder/tools/platformio.py | 5 +++ tests/commands/test_run.py | 48 +++++++++++++++++++++++++- 3 files changed, 53 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index f2c17c6d..8f36ba47 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -32,6 +32,7 @@ PlatformIO Core 6 * Allowed to ``Import("projenv")`` in a library extra script (`issue #4305 `_) * Improved a serial port finder for a board with predefined HWIDs +* Added ``env.StringifyMacro(value)`` helper function for the `Advanced Scripting `__ 6.0.2 (2022-06-01) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/builder/tools/platformio.py b/platformio/builder/tools/platformio.py index 9b2aedb8..bd4dd53c 100644 --- a/platformio/builder/tools/platformio.py +++ b/platformio/builder/tools/platformio.py @@ -264,6 +264,10 @@ def ProcessUnFlags(env, flags): env[key].remove(current) +def StringifyMacro(env, value): + return '\\"%s\\"' % value.replace('"', '\\\\\\"') + + def MatchSourceFiles(env, src_dir, src_filter=None): src_filter = env.subst(src_filter) if src_filter else None src_filter = src_filter or SRC_FILTER_DEFAULT @@ -368,6 +372,7 @@ def generate(env): env.AddMethod(ParseFlagsExtended) env.AddMethod(ProcessFlags) env.AddMethod(ProcessUnFlags) + env.AddMethod(StringifyMacro) env.AddMethod(MatchSourceFiles) env.AddMethod(CollectBuildFiles) env.AddMethod(AddBuildMiddleware) diff --git a/tests/commands/test_run.py b/tests/commands/test_run.py index 5a2617fe..b4321a10 100644 --- a/tests/commands/test_run.py +++ b/tests/commands/test_run.py @@ -252,5 +252,51 @@ platform = native lib_deps = symlink://../External """ ) - result = clirunner.invoke(cmd_run, ["--project-dir", str(project_dir), "--verbose"]) + result = clirunner.invoke(cmd_run, ["--project-dir", str(project_dir)]) validate_cliresult(result) + + +def test_stringification(clirunner, validate_cliresult, tmp_path: Path): + project_dir = tmp_path / "project" + src_dir = project_dir / "src" + src_dir.mkdir(parents=True) + (src_dir / "main.c").write_text( + """ +#include +int main(void) { + printf("MACRO_1=<%s>\\n", MACRO_1); + printf("MACRO_2=<%s>\\n", MACRO_2); + printf("MACRO_3=<%s>\\n", MACRO_3); + printf("MACRO_4=<%s>\\n", MACRO_4); + return(0); +} +""" + ) + (project_dir / "platformio.ini").write_text( + """ +[env:native] +platform = native +extra_scripts = script.py +build_flags = + '-DMACRO_1="Hello World!"' + '-DMACRO_2="Text is \\\\"Quoted\\\\""' + """ + ) + (project_dir / "script.py").write_text( + """ +Import("projenv") + +projenv.Append(CPPDEFINES=[ + ("MACRO_3", projenv.StringifyMacro('Hello "World"! Isn\\'t true?')), + ("MACRO_4", projenv.StringifyMacro("Special chars: ',(,),[,],:")) +]) + """ + ) + result = clirunner.invoke( + cmd_run, ["--project-dir", str(project_dir), "-t", "exec"] + ) + validate_cliresult(result) + assert "MACRO_1=" in result.output + assert 'MACRO_2=' in result.output + assert 'MACRO_3=' in result.output + assert 'MACRO_4=' in result.output From d4784c05f5c9be08d5d20b89ac76a7534186f596 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 17 Jun 2022 12:29:09 +0300 Subject: [PATCH 28/93] =?UTF-8?q?Documented=20Stringification=20=E2=80=93?= =?UTF-8?q?=20converting=20a=20macro=20argument=20into=20a=20string=20cons?= =?UTF-8?q?tant=20=20//=20Resolve=20#4310?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- HISTORY.rst | 1 + docs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index 8f36ba47..33737527 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -32,6 +32,7 @@ PlatformIO Core 6 * Allowed to ``Import("projenv")`` in a library extra script (`issue #4305 `_) * Improved a serial port finder for a board with predefined HWIDs +* Documented `Stringification `__ – converting a macro argument into a string constant (`issue #4310 `_) * Added ``env.StringifyMacro(value)`` helper function for the `Advanced Scripting `__ 6.0.2 (2022-06-01) diff --git a/docs b/docs index b3bb4bfe..6bfc42b2 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit b3bb4bfe67f98edbf5acfbdbb2cf61d7912dec29 +Subproject commit 6bfc42b22668cea6210f70feee8dcf427de55c7f From ca3b3717d3b30195b71d95863191c6eba0a5a8e2 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 17 Jun 2022 12:31:51 +0300 Subject: [PATCH 29/93] Bump version to 6.0.3a6 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 9f5464a0..9eac4901 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (6, 0, "3a5") +VERSION = (6, 0, "3a6") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 5afa0a955e980c87270614ffc839514a377d586b Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 17 Jun 2022 13:58:26 +0300 Subject: [PATCH 30/93] PyLint fix --- platformio/builder/tools/platformio.py | 2 +- tests/commands/test_run.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/platformio/builder/tools/platformio.py b/platformio/builder/tools/platformio.py index bd4dd53c..3334e978 100644 --- a/platformio/builder/tools/platformio.py +++ b/platformio/builder/tools/platformio.py @@ -264,7 +264,7 @@ def ProcessUnFlags(env, flags): env[key].remove(current) -def StringifyMacro(env, value): +def StringifyMacro(env, value): # pylint: disable=unused-argument return '\\"%s\\"' % value.replace('"', '\\\\\\"') diff --git a/tests/commands/test_run.py b/tests/commands/test_run.py index b4321a10..b747e427 100644 --- a/tests/commands/test_run.py +++ b/tests/commands/test_run.py @@ -299,4 +299,4 @@ projenv.Append(CPPDEFINES=[ assert "MACRO_1=" in result.output assert 'MACRO_2=' in result.output assert 'MACRO_3=' in result.output - assert 'MACRO_4=' in result.output + assert "MACRO_4=" in result.output From db6b8a6dbc3ba0b23d44ffb4b595a749739fb99e Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 17 Jun 2022 14:25:39 +0300 Subject: [PATCH 31/93] Fixed an issue when `build_unflags operation ignores a flag value // Resolve #4309 --- HISTORY.rst | 5 +++-- platformio/builder/tools/platformio.py | 4 +++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 33737527..ab73f480 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -30,10 +30,11 @@ PlatformIO Core 6 - Do not resolve dependencies from the project "src" folder when the `test_build_src `__ option is not enabled - Fixed an issue when a custom `pio test --project-config `__ was not handled properly (`issue #4299 `_) -* Allowed to ``Import("projenv")`` in a library extra script (`issue #4305 `_) -* Improved a serial port finder for a board with predefined HWIDs * Documented `Stringification `__ – converting a macro argument into a string constant (`issue #4310 `_) * Added ``env.StringifyMacro(value)`` helper function for the `Advanced Scripting `__ +* Allowed to ``Import("projenv")`` in a library extra script (`issue #4305 `_) +* Improved a serial port finder for a board with predefined HWIDs +* Fixed an issue when `build_unflags `__ operation ignores a flag value (`issue #4309 `_) 6.0.2 (2022-06-01) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/builder/tools/platformio.py b/platformio/builder/tools/platformio.py index 3334e978..59eb6387 100644 --- a/platformio/builder/tools/platformio.py +++ b/platformio/builder/tools/platformio.py @@ -258,7 +258,9 @@ def ProcessUnFlags(env, flags): for current in env.get(key, []): conditions = [ unflag == current, - isinstance(current, (tuple, list)) and unflag[0] == current[0], + not isinstance(unflag, (tuple, list)) + and isinstance(current, (tuple, list)) + and unflag == current[0], ] if any(conditions): env[key].remove(current) From f68c18d1e57357c0af2891dda5bf79a512c362f5 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 17 Jun 2022 18:58:21 +0300 Subject: [PATCH 32/93] Fixed an issue when the build_unflags option was not applied to the ASPPFLAGS scope --- HISTORY.rst | 3 ++- platformio/builder/tools/platformio.py | 34 ++++++++++---------------- tests/commands/test_run.py | 34 +++++++++++++++++++++----- 3 files changed, 43 insertions(+), 28 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index ab73f480..d9663c0b 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -34,7 +34,8 @@ PlatformIO Core 6 * Added ``env.StringifyMacro(value)`` helper function for the `Advanced Scripting `__ * Allowed to ``Import("projenv")`` in a library extra script (`issue #4305 `_) * Improved a serial port finder for a board with predefined HWIDs -* Fixed an issue when `build_unflags `__ operation ignores a flag value (`issue #4309 `_) +* Fixed an issue when the `build_unflags `__ operation ignores a flag value (`issue #4309 `_) +* Fixed an issue when the `build_unflags `__ option was not applied to the ``ASPPFLAGS`` scope 6.0.2 (2022-06-01) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/builder/tools/platformio.py b/platformio/builder/tools/platformio.py index 59eb6387..c828a943 100644 --- a/platformio/builder/tools/platformio.py +++ b/platformio/builder/tools/platformio.py @@ -243,27 +243,19 @@ def ProcessUnFlags(env, flags): if not flags: return parsed = env.ParseFlagsExtended(flags) - - # get all flags and copy them to each "*FLAGS" variable - all_flags = [] - for key, unflags in parsed.items(): - if key.endswith("FLAGS"): - all_flags.extend(unflags) - for key, unflags in parsed.items(): - if key.endswith("FLAGS"): - parsed[key].extend(all_flags) - - for key, unflags in parsed.items(): - for unflag in unflags: - for current in env.get(key, []): - conditions = [ - unflag == current, - not isinstance(unflag, (tuple, list)) - and isinstance(current, (tuple, list)) - and unflag == current[0], - ] - if any(conditions): - env[key].remove(current) + unflag_scopes = tuple(set(["ASPPFLAGS"] + list(parsed.keys()))) + for scope in unflag_scopes: + for unflags in parsed.values(): + for unflag in unflags: + for current in env.get(scope, []): + conditions = [ + unflag == current, + not isinstance(unflag, (tuple, list)) + and isinstance(current, (tuple, list)) + and unflag == current[0], + ] + if any(conditions): + env[scope].remove(current) def StringifyMacro(env, value): # pylint: disable=unused-argument diff --git a/tests/commands/test_run.py b/tests/commands/test_run.py index b747e427..3a2817ba 100644 --- a/tests/commands/test_run.py +++ b/tests/commands/test_run.py @@ -112,7 +112,16 @@ def test_build_unflags(clirunner, validate_cliresult, tmpdir): """ [env:native] platform = native -build_unflags = -DTMP_MACRO1=45 -I. -DNON_EXISTING_MACRO -lunknownLib -Os +build_unflags = + -DTMP_MACRO_1=45 + -DTMP_MACRO_3=13 + -DTMP_MACRO_4 + -DNON_EXISTING_MACRO + -I. + -lunknownLib + -Os +build_flags = + -DTMP_MACRO_3=10 extra_scripts = pre:extra.py """ ) @@ -121,9 +130,10 @@ extra_scripts = pre:extra.py """ Import("env") env.Append(CPPPATH="%s") -env.Append(CPPDEFINES="TMP_MACRO1") -env.Append(CPPDEFINES=["TMP_MACRO2"]) -env.Append(CPPDEFINES=("TMP_MACRO3", 13)) +env.Append(CPPDEFINES="TMP_MACRO_1") +env.Append(CPPDEFINES=["TMP_MACRO_2"]) +env.Append(CPPDEFINES=[("TMP_MACRO_3", 13)]) +env.Append(CPPDEFINES=[("TMP_MACRO_4", 4)]) env.Append(CCFLAGS=["-Os"]) env.Append(LIBS=["unknownLib"]) """ @@ -132,8 +142,20 @@ env.Append(LIBS=["unknownLib"]) tmpdir.mkdir("src").join("main.c").write( """ -#ifdef TMP_MACRO1 -#error "TMP_MACRO1 should be removed" +#ifndef TMP_MACRO_1 +#error "TMP_MACRO_1 should be defined" +#endif + +#ifndef TMP_MACRO_2 +#error "TMP_MACRO_2 should be defined" +#endif + +#if TMP_MACRO_3 != 10 +#error "TMP_MACRO_3 should be 10" +#endif + +#ifdef TMP_MACRO_4 +#error "TMP_MACRO_4 should not be defined" #endif int main() { From b72c1636f708abe61348d4dd3a40a6da57685c96 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 17 Jun 2022 23:05:20 +0300 Subject: [PATCH 33/93] Improved a serial port finder for Black Magic Probe // Resolve #4023 --- HISTORY.rst | 1 + platformio/debug/config/base.py | 16 +++++++-- platformio/debug/helpers.py | 45 +---------------------- platformio/device/finder.py | 63 +++++++++++++++++++++++++++------ 4 files changed, 67 insertions(+), 58 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index d9663c0b..85709451 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -33,6 +33,7 @@ PlatformIO Core 6 * Documented `Stringification `__ – converting a macro argument into a string constant (`issue #4310 `_) * Added ``env.StringifyMacro(value)`` helper function for the `Advanced Scripting `__ * Allowed to ``Import("projenv")`` in a library extra script (`issue #4305 `_) +* Improved a serial port finder for `Black Magic Probe `__ (`issue #4023 `_) * Improved a serial port finder for a board with predefined HWIDs * Fixed an issue when the `build_unflags `__ operation ignores a flag value (`issue #4309 `_) * Fixed an issue when the `build_unflags `__ option was not applied to the ``ASPPFLAGS`` scope diff --git a/platformio/debug/config/base.py b/platformio/debug/config/base.py index c53c7d0f..fa85cd55 100644 --- a/platformio/debug/config/base.py +++ b/platformio/debug/config/base.py @@ -18,7 +18,7 @@ import os from platformio import fs, proc, util from platformio.compat import string_types from platformio.debug.exception import DebugInvalidOptionsError -from platformio.debug.helpers import reveal_debug_port +from platformio.device.finder import find_debug_port from platformio.project.config import ProjectConfig from platformio.project.helpers import load_build_metadata from platformio.project.options import ProjectOptions @@ -119,12 +119,22 @@ class DebugConfigBase: # pylint: disable=too-many-instance-attributes @property def port(self): - return reveal_debug_port( + initial_port = ( self.env_options.get("debug_port", self.tool_settings.get("port")) - or self._port, + or self._port + ) + port = find_debug_port( + initial_port, self.tool_name, self.tool_settings, ) + if port: + return port + if not self.tool_settings.get("require_debug_port"): + return None + raise DebugInvalidOptionsError( + "Please specify `debug_port` for the working environment" + ) @port.setter def port(self, value): diff --git a/platformio/debug/helpers.py b/platformio/debug/helpers.py index 35f0d483..a2d89de3 100644 --- a/platformio/debug/helpers.py +++ b/platformio/debug/helpers.py @@ -16,14 +16,12 @@ import os import re import sys import time -from fnmatch import fnmatch from hashlib import sha1 from io import BytesIO from platformio.cli import PlatformioCLI -from platformio.compat import IS_WINDOWS, is_bytes +from platformio.compat import is_bytes from platformio.debug.exception import DebugInvalidOptionsError -from platformio.device.list.util import list_serial_ports from platformio.run.cli import cli as cmd_run from platformio.run.cli import print_processing_header from platformio.test.helpers import list_test_names @@ -161,44 +159,3 @@ def is_prog_obsolete(prog_path): with open(prog_hash_path, mode="w", encoding="utf8") as fp: fp.write(new_digest) return True - - -def reveal_debug_port(env_debug_port, tool_name, tool_settings): - def _get_pattern(): - if not env_debug_port: - return None - if set(["*", "?", "[", "]"]) & set(env_debug_port): - return env_debug_port - return None - - def _is_match_pattern(port): - pattern = _get_pattern() - if not pattern: - return True - return fnmatch(port, pattern) - - def _look_for_serial_port(hwids): - for item in list_serial_ports(filter_hwid=True): - if not _is_match_pattern(item["port"]): - continue - port = item["port"] - if tool_name.startswith("blackmagic"): - if IS_WINDOWS and port.startswith("COM") and len(port) > 4: - port = "\\\\.\\%s" % port - if "GDB" in item["description"]: - return port - for hwid in hwids: - hwid_str = ("%s:%s" % (hwid[0], hwid[1])).replace("0x", "") - if hwid_str in item["hwid"]: - return port - return None - - if env_debug_port and not _get_pattern(): - return env_debug_port - if not tool_settings.get("require_debug_port"): - return None - - debug_port = _look_for_serial_port(tool_settings.get("hwids", [])) - if not debug_port: - raise DebugInvalidOptionsError("Please specify `debug_port` for environment") - return debug_port diff --git a/platformio/device/finder.py b/platformio/device/finder.py index 89d99a94..65d641c0 100644 --- a/platformio/device/finder.py +++ b/platformio/device/finder.py @@ -18,13 +18,16 @@ from fnmatch import fnmatch import click import serial -from platformio.compat import IS_WINDOWS +from platformio.compat import IS_MACOS, IS_WINDOWS from platformio.device.list.util import list_logical_devices, list_serial_ports from platformio.package.manager.platform import PlatformPackageManager from platformio.platform.factory import PlatformFactory from platformio.util import retry -KNOWN_UART_HWIDS = ( +BLACK_MAGIC_HWIDS = [ + "1D50:6018", +] +KNOWN_UART_HWIDS = BLACK_MAGIC_HWIDS + [ # Silicon Labs "10C4:EA60", # CP210X "10C4:EA61", # CP210X @@ -33,7 +36,7 @@ KNOWN_UART_HWIDS = ( "10C4:EA71", # CP2108 "10C4:EA80", # CP2110 "10C4:80A9", # CP210X -) +] def normalize_board_hwid(value): @@ -91,18 +94,44 @@ def find_serial_port( return best_port or port -def find_blackmagic_serial_port(timeout=0): +def find_blackmagic_serial_port(timeout=0, only_gdb_port=False): try: @retry(timeout=timeout) def wrapper(): - for item in list_serial_ports(): - port = item["port"] - if IS_WINDOWS and port.startswith("COM") and len(port) > 4: - port = "\\\\.\\%s" % port - if "GDB" in item["description"]: - return port - raise retry.RetryNextException() + candidates = [] + for item in list_serial_ports(filter_hwid=True): + if ( + not any(hwid in item["hwid"].upper() for hwid in BLACK_MAGIC_HWIDS) + and not "Black Magic" in item["description"] + ): + continue + if ( + IS_WINDOWS + and item["port"].startswith("COM") + and len(item["port"]) > 4 + ): + item["port"] = "\\\\.\\%s" % item["port"] + candidates.append(item) + + if not candidates: + raise retry.RetryNextException() + + for item in candidates: + if ("GDB" if only_gdb_port else "UART") in item["description"]: + return item["port"] + if IS_MACOS: + # 1 - GDB, 3 - UART + for item in candidates: + if item["port"].endswith("1" if only_gdb_port else "3"): + return item["port"] + + candidates = sorted(candidates, key=lambda item: item["port"]) + return ( + candidates[0] # first port is GDB? + if len(candidates) == 1 or only_gdb_port + else candidates[1] + )["port"] return wrapper() except retry.RetryStopException: @@ -193,3 +222,15 @@ def find_mbed_disk(initial_port): if item["name"] and any(l in item["name"].lower() for l in msdlabels): return item["path"] return None + + +def find_debug_port(initial_port, tool_name, tool_settings): + if initial_port: + if not is_pattern_port(initial_port): + return initial_port + return match_serial_port(initial_port) + if not tool_settings.get("require_debug_port"): + return None + if tool_name.startswith("blackmagic"): + return find_blackmagic_serial_port(timeout=0, only_gdb_port=True) + return None From 18e130fd120d17e30e8f7fc426e0eab0672cec59 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 17 Jun 2022 23:05:47 +0300 Subject: [PATCH 34/93] Bump version to 6.0.3a7 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 9eac4901..43e87f1b 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (6, 0, "3a6") +VERSION = (6, 0, "3a7") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 58d533a3bbf4d4744496e28dc5b702b773b13717 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 18 Jun 2022 14:06:24 +0300 Subject: [PATCH 35/93] Prefer Black Magic GDB for uploading // Issue #4023 --- platformio/builder/tools/pioupload.py | 1 + platformio/debug/config/base.py | 11 ++++++---- platformio/device/finder.py | 31 +++++++++++---------------- 3 files changed, 20 insertions(+), 23 deletions(-) diff --git a/platformio/builder/tools/pioupload.py b/platformio/builder/tools/pioupload.py index 46d2ca49..f85dc735 100644 --- a/platformio/builder/tools/pioupload.py +++ b/platformio/builder/tools/pioupload.py @@ -116,6 +116,7 @@ def AutodetectUploadPort(*args, **kwargs): initial_port=initial_port, board_config=env.BoardConfig() if "BOARD" in env else None, upload_protocol=upload_protocol, + prefer_gdb_port="blackmagic" in upload_protocol, ) ) diff --git a/platformio/debug/config/base.py b/platformio/debug/config/base.py index fa85cd55..9787cc8c 100644 --- a/platformio/debug/config/base.py +++ b/platformio/debug/config/base.py @@ -18,7 +18,7 @@ import os from platformio import fs, proc, util from platformio.compat import string_types from platformio.debug.exception import DebugInvalidOptionsError -from platformio.device.finder import find_debug_port +from platformio.device.finder import find_serial_port, is_pattern_port from platformio.project.config import ProjectConfig from platformio.project.helpers import load_build_metadata from platformio.project.options import ProjectOptions @@ -123,10 +123,13 @@ class DebugConfigBase: # pylint: disable=too-many-instance-attributes self.env_options.get("debug_port", self.tool_settings.get("port")) or self._port ) - port = find_debug_port( + if initial_port and not is_pattern_port(initial_port): + return initial_port + port = find_serial_port( initial_port, - self.tool_name, - self.tool_settings, + board_config=self.board_config, + upload_protocol=self.tool_name, + prefer_gdb_port=True, ) if port: return port diff --git a/platformio/device/finder.py b/platformio/device/finder.py index 65d641c0..6102b2f0 100644 --- a/platformio/device/finder.py +++ b/platformio/device/finder.py @@ -67,8 +67,13 @@ def is_serial_port_ready(port, timeout=1): return False -def find_serial_port( - initial_port, board_config=None, upload_protocol=None, ensure_ready=False, timeout=3 +def find_serial_port( # pylint: disable=too-many-arguments + initial_port, + board_config=None, + upload_protocol=None, + ensure_ready=False, + prefer_gdb_port=False, + timeout=3, ): if initial_port: if not is_pattern_port(initial_port): @@ -76,7 +81,7 @@ def find_serial_port( return match_serial_port(initial_port) if upload_protocol and upload_protocol.startswith("blackmagic"): - return find_blackmagic_serial_port(timeout) + return find_blackmagic_serial_port(prefer_gdb_port, timeout) if board_config and board_config.get("build.hwids", []): return find_board_serial_port(board_config, timeout) port = find_known_uart_port(ensure_ready, timeout) @@ -94,7 +99,7 @@ def find_serial_port( return best_port or port -def find_blackmagic_serial_port(timeout=0, only_gdb_port=False): +def find_blackmagic_serial_port(prefer_gdb_port=False, timeout=0): try: @retry(timeout=timeout) @@ -118,18 +123,18 @@ def find_blackmagic_serial_port(timeout=0, only_gdb_port=False): raise retry.RetryNextException() for item in candidates: - if ("GDB" if only_gdb_port else "UART") in item["description"]: + if ("GDB" if prefer_gdb_port else "UART") in item["description"]: return item["port"] if IS_MACOS: # 1 - GDB, 3 - UART for item in candidates: - if item["port"].endswith("1" if only_gdb_port else "3"): + if item["port"].endswith("1" if prefer_gdb_port else "3"): return item["port"] candidates = sorted(candidates, key=lambda item: item["port"]) return ( candidates[0] # first port is GDB? - if len(candidates) == 1 or only_gdb_port + if len(candidates) == 1 or prefer_gdb_port else candidates[1] )["port"] @@ -222,15 +227,3 @@ def find_mbed_disk(initial_port): if item["name"] and any(l in item["name"].lower() for l in msdlabels): return item["path"] return None - - -def find_debug_port(initial_port, tool_name, tool_settings): - if initial_port: - if not is_pattern_port(initial_port): - return initial_port - return match_serial_port(initial_port) - if not tool_settings.get("require_debug_port"): - return None - if tool_name.startswith("blackmagic"): - return find_blackmagic_serial_port(timeout=0, only_gdb_port=True) - return None From d8f36b6534bca369b4f02ebb68010e0ade153743 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 18 Jun 2022 16:59:49 +0300 Subject: [PATCH 36/93] Minor improvements to pkg show layout --- platformio/package/commands/show.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/platformio/package/commands/show.py b/platformio/package/commands/show.py index 82b6e47c..bf2d1f65 100644 --- a/platformio/package/commands/show.py +++ b/platformio/package/commands/show.py @@ -59,14 +59,6 @@ def package_show_cmd(spec, pkg_type): ) ) - click.echo() - type_plural = "libraries" if data["type"] == "library" else (data["type"] + "s") - click.secho( - "https://registry.platformio.org/%s/%s/%s" - % (type_plural, data["owner"]["username"], quote(data["name"])), - fg="blue", - ) - # Description click.echo() click.echo(data["description"]) @@ -87,7 +79,17 @@ def package_show_cmd(spec, pkg_type): ("frameworks", "Compatible Frameworks"), ("keywords", "Keywords"), ] - extra = [] + type_plural = "libraries" if data["type"] == "library" else (data["type"] + "s") + extra = [ + ( + "Registry", + click.style( + "https://registry.platformio.org/%s/%s/%s" + % (type_plural, data["owner"]["username"], quote(data["name"])), + fg="blue", + ), + ) + ] for key, title in fields: if "." in key: k1, k2 = key.split(".") From 8cbe7bc7a6eb40412b3e66d0a4f24f6906767535 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 18 Jun 2022 17:25:38 +0300 Subject: [PATCH 37/93] Use new unified package API for deprecated `pio platform` command // Issue #4198 --- platformio/commands/platform.py | 298 ++++++++++---------------------- tests/commands/test_platform.py | 15 +- 2 files changed, 103 insertions(+), 210 deletions(-) diff --git a/platformio/commands/platform.py b/platformio/commands/platform.py index d9e8af2d..a02476c4 100644 --- a/platformio/commands/platform.py +++ b/platformio/commands/platform.py @@ -18,9 +18,13 @@ import os import click -from platformio.commands.boards import print_boards from platformio.exception import UserSideException -from platformio.package.exception import UnknownPackageError +from platformio.package.commands.install import package_install_cmd +from platformio.package.commands.list import package_list_cmd +from platformio.package.commands.search import package_search_cmd +from platformio.package.commands.show import package_show_cmd +from platformio.package.commands.uninstall import package_uninstall_cmd +from platformio.package.commands.update import package_update_cmd from platformio.package.manager.platform import PlatformPackageManager from platformio.package.meta import PackageItem, PackageSpec from platformio.package.version import get_original_version @@ -36,13 +40,17 @@ def cli(): @cli.command("search", short_help="Search for development platform") @click.argument("query", required=False) @click.option("--json-output", is_flag=True) -def platform_search(query, json_output): +@click.pass_context +def platform_search(ctx, query, json_output): if not json_output: click.secho( "\nWARNING: This command is deprecated and will be removed in " "the next releases. \nPlease use `pio pkg search` instead.\n", fg="yellow", ) + query = query or "" + return ctx.invoke(package_search_cmd, query=f"type:platform {query}".strip()) + platforms = [] for platform in _get_registry_platforms(): if query == "all": @@ -55,17 +63,23 @@ def platform_search(query, json_output): platform["name"], with_boards=False, expose_packages=False ) ) - - if json_output: - click.echo(json.dumps(platforms)) - else: - _print_platforms(platforms) + click.echo(json.dumps(platforms)) + return None @cli.command("frameworks", short_help="List supported frameworks, SDKs") @click.argument("query", required=False) @click.option("--json-output", is_flag=True) def platform_frameworks(query, json_output): + if not json_output: + click.secho( + "\nWARNING: This command is deprecated and will be removed in " + "the next releases. \nPlease https://docs.platformio.org" + "/en/latest/frameworks/index.html.\n", + fg="yellow", + ) + return + regclient = PlatformPackageManager().get_registry_client_instance() frameworks = [] for framework in regclient.fetch_json_data( @@ -85,21 +99,21 @@ def platform_frameworks(query, json_output): frameworks.append(framework) frameworks = sorted(frameworks, key=lambda manifest: manifest["name"]) - if json_output: - click.echo(json.dumps(frameworks)) - else: - _print_platforms(frameworks) + click.echo(json.dumps(frameworks)) @cli.command("list", short_help="List installed development platforms") @click.option("--json-output", is_flag=True) -def platform_list(json_output): +@click.pass_context +def platform_list(ctx, json_output): if not json_output: click.secho( "\nWARNING: This command is deprecated and will be removed in " "the next releases. \nPlease use `pio pkg list` instead.\n", fg="yellow", ) + return ctx.invoke(package_list_cmd, **{"global": True, "only_platforms": True}) + platforms = [] pm = PlatformPackageManager() for pkg in pm.get_installed(): @@ -108,81 +122,27 @@ def platform_list(json_output): ) platforms = sorted(platforms, key=lambda manifest: manifest["name"]) - if json_output: - click.echo(json.dumps(platforms)) - else: - _print_platforms(platforms) + click.echo(json.dumps(platforms)) + return None @cli.command("show", short_help="Show details about development platform") @click.argument("platform") @click.option("--json-output", is_flag=True) -def platform_show(platform, json_output): # pylint: disable=too-many-branches +@click.pass_context +def platform_show(ctx, platform, json_output): # pylint: disable=too-many-branches if not json_output: click.secho( "\nWARNING: This command is deprecated and will be removed in " "the next releases. \nPlease use `pio pkg show` instead.\n", fg="yellow", ) + return ctx.invoke(package_show_cmd, pkg_type="platform", spec=platform) + data = _get_platform_data(platform) if not data: raise UnknownPlatform(platform) - if json_output: - return click.echo(json.dumps(data)) - - dep = "{ownername}/{name}".format(**data) if "ownername" in data else data["name"] - click.echo( - "{dep} ~ {title}".format(dep=click.style(dep, fg="cyan"), title=data["title"]) - ) - click.echo("=" * (3 + len(dep + data["title"]))) - click.echo(data["description"]) - click.echo() - if "version" in data: - click.echo("Version: %s" % data["version"]) - if data["homepage"]: - click.echo("Home: %s" % data["homepage"]) - if data["repository"]: - click.echo("Repository: %s" % data["repository"]) - if data["url"]: - click.echo("Vendor: %s" % data["url"]) - if data["license"]: - click.echo("License: %s" % data["license"]) - if data["frameworks"]: - click.echo("Frameworks: %s" % ", ".join(data["frameworks"])) - - if not data["packages"]: - return None - - if not isinstance(data["packages"][0], dict): - click.echo("Packages: %s" % ", ".join(data["packages"])) - else: - click.echo() - click.secho("Packages", bold=True) - click.echo("--------") - for item in data["packages"]: - click.echo() - click.echo("Package %s" % click.style(item["name"], fg="yellow")) - click.echo("-" * (8 + len(item["name"]))) - if item["type"]: - click.echo("Type: %s" % item["type"]) - click.echo("Requirements: %s" % item["requirements"]) - click.echo( - "Installed: %s" % ("Yes" if item.get("version") else "No (optional)") - ) - if "version" in item: - click.echo("Version: %s" % item["version"]) - if "originalVersion" in item: - click.echo("Original version: %s" % item["originalVersion"]) - if "description" in item: - click.echo("Description: %s" % item["description"]) - - if data["boards"]: - click.echo() - click.secho("Boards", bold=True) - click.echo("------") - print_boards(data["boards"]) - - return True + return click.echo(json.dumps(data)) @cli.command("install", short_help="Install new development platform") @@ -198,7 +158,9 @@ def platform_show(platform, json_output): # pylint: disable=too-many-branches is_flag=True, help="Reinstall/redownload dev/platform and its packages if exist", ) -def platform_install( # pylint: disable=too-many-arguments,too-many-locals +@click.pass_context +def platform_install( # pylint: disable=too-many-arguments + ctx, platforms, with_package, without_package, @@ -212,76 +174,37 @@ def platform_install( # pylint: disable=too-many-arguments,too-many-locals "the next releases. \nPlease use `pio pkg install` instead.\n", fg="yellow", ) - - def _find_pkg_names(p, candidates): - result = [] - for candidate in candidates: - found = False - # lookup by package types - for _name, _opts in p.packages.items(): - if _opts.get("type") == candidate: - result.append(_name) - found = True - if ( - p.frameworks - and candidate.startswith("framework-") - and candidate[10:] in p.frameworks - ): - result.append(p.frameworks[candidate[10:]]["package"]) - found = True - if not found: - result.append(candidate) - return result - - pm = PlatformPackageManager() - pm.set_log_level(logging.WARN if silent else logging.DEBUG) - for platform in platforms: - if with_package or without_package or with_all_packages: - pkg = pm.install(platform, skip_dependencies=True) - p = PlatformFactory.new(pkg) - if with_all_packages: - with_package = list(p.packages) - with_package = set(_find_pkg_names(p, with_package or [])) - without_package = set(_find_pkg_names(p, without_package or [])) - upkgs = with_package | without_package - ppkgs = set(p.packages) - if not upkgs.issubset(ppkgs): - raise UnknownPackageError(", ".join(upkgs - ppkgs)) - for name, options in p.packages.items(): - if name in without_package: - continue - if name in with_package or not ( - skip_default_package or options.get("optional", False) - ): - p.pm.install(p.get_package_spec(name), force=force) - else: - pkg = pm.install(platform, skip_dependencies=skip_default_package) - - if pkg and not silent: - click.secho( - "The platform '%s' has been successfully installed!\n" - "The rest of the packages will be installed later " - "depending on your build environment." % platform, - fg="green", - ) + ctx.invoke( + package_install_cmd, + **{ + "global": True, + "platforms": platforms, + "skip_dependencies": ( + not with_all_packages + and (with_package or without_package or skip_default_package) + ), + "silent": silent, + "force": force, + }, + ) @cli.command("uninstall", short_help="Uninstall development platform") @click.argument("platforms", nargs=-1, required=True, metavar="[PLATFORM...]") -def platform_uninstall(platforms): +@click.pass_context +def platform_uninstall(ctx, platforms): click.secho( "\nWARNING: This command is deprecated and will be removed in " "the next releases. \nPlease use `pio pkg uninstall` instead.\n", fg="yellow", ) - pm = PlatformPackageManager() - pm.set_log_level(logging.DEBUG) - for platform in platforms: - if pm.uninstall(platform): - click.secho( - "The platform '%s' has been successfully removed!" % platform, - fg="green", - ) + ctx.invoke( + package_uninstall_cmd, + **{ + "global": True, + "platforms": platforms, + }, + ) @cli.command("update", short_help="Update installed development platforms") @@ -300,9 +223,12 @@ def platform_uninstall(platforms): ) @click.option("-s", "--silent", is_flag=True, help="Suppress progress reporting") @click.option("--json-output", is_flag=True) +@click.pass_context def platform_update( # pylint: disable=too-many-locals, too-many-arguments - platforms, only_check, dry_run, silent, json_output, **_ + ctx, platforms, only_check, dry_run, silent, json_output, **_ ): + only_check = dry_run or only_check + if only_check and not json_output: raise UserSideException( "This command is deprecated, please use `pio pkg outdated` instead" @@ -314,54 +240,42 @@ def platform_update( # pylint: disable=too-many-locals, too-many-arguments "the next releases. \nPlease use `pio pkg update` instead.\n", fg="yellow", ) + return ctx.invoke( + package_update_cmd, + **{ + "global": True, + "platforms": platforms, + "silent": silent, + }, + ) pm = PlatformPackageManager() pm.set_log_level(logging.WARN if silent else logging.DEBUG) platforms = platforms or pm.get_installed() - only_check = dry_run or only_check - - if only_check and json_output: - result = [] - for platform in platforms: - spec = None - pkg = None - if isinstance(platform, PackageItem): - pkg = platform - else: - spec = PackageSpec(platform) - pkg = pm.get_package(spec) - if not pkg: - continue - outdated = pm.outdated(pkg, spec) - if ( - not outdated.is_outdated(allow_incompatible=True) - and not PlatformFactory.new(pkg).are_outdated_packages() - ): - continue - data = _get_installed_platform_data( - pkg, with_boards=False, expose_packages=False - ) - if outdated.is_outdated(allow_incompatible=True): - data["versionLatest"] = ( - str(outdated.latest) if outdated.latest else None - ) - result.append(data) - return click.echo(json.dumps(result)) - + result = [] for platform in platforms: - click.echo( - "Platform %s" - % click.style( - platform.metadata.name - if isinstance(platform, PackageItem) - else platform, - fg="cyan", - ) + spec = None + pkg = None + if isinstance(platform, PackageItem): + pkg = platform + else: + spec = PackageSpec(platform) + pkg = pm.get_package(spec) + if not pkg: + continue + outdated = pm.outdated(pkg, spec) + if ( + not outdated.is_outdated(allow_incompatible=True) + and not PlatformFactory.new(pkg).are_outdated_packages() + ): + continue + data = _get_installed_platform_data( + pkg, with_boards=False, expose_packages=False ) - click.echo("--------") - pm.update(platform) - click.echo() - + if outdated.is_outdated(allow_incompatible=True): + data["versionLatest"] = str(outdated.latest) if outdated.latest else None + result.append(data) + click.echo(json.dumps(result)) return True @@ -370,32 +284,6 @@ def platform_update( # pylint: disable=too-many-locals, too-many-arguments # -def _print_platforms(platforms): - for platform in platforms: - click.echo( - "{name} ~ {title}".format( - name=click.style(platform["name"], fg="cyan"), title=platform["title"] - ) - ) - click.echo("=" * (3 + len(platform["name"] + platform["title"]))) - click.echo(platform["description"]) - click.echo() - if "homepage" in platform: - click.echo("Home: %s" % platform["homepage"]) - if "frameworks" in platform and platform["frameworks"]: - click.echo("Frameworks: %s" % ", ".join(platform["frameworks"])) - if "packages" in platform: - click.echo("Packages: %s" % ", ".join(platform["packages"])) - if "version" in platform: - if "__src_url" in platform: - click.echo( - "Version: %s (%s)" % (platform["version"], platform["__src_url"]) - ) - else: - click.echo("Version: " + platform["version"]) - click.echo() - - def _get_registry_platforms(): regclient = PlatformPackageManager().get_registry_client_instance() return regclient.fetch_json_data("get", "/v2/platforms", x_cache_valid="1d") diff --git a/tests/commands/test_platform.py b/tests/commands/test_platform.py index 604e392f..967bf51e 100644 --- a/tests/commands/test_platform.py +++ b/tests/commands/test_platform.py @@ -15,6 +15,7 @@ # pylint: disable=unused-argument import json +import os from platformio.commands import platform as cli_platform from platformio.package.exception import UnknownPackageError @@ -36,7 +37,7 @@ def test_search_json_output(clirunner, validate_cliresult, isolated_pio_core): def test_search_raw_output(clirunner, validate_cliresult): result = clirunner.invoke(cli_platform.platform_search, ["arduino"]) validate_cliresult(result) - assert "teensy" in result.output + assert "atmelavr" in result.output def test_install_unknown_version(clirunner): @@ -75,8 +76,7 @@ def test_install_known_version(clirunner, validate_cliresult, isolated_pio_core) validate_cliresult(result) output = strip_ansi_codes(result.output) assert "atmelavr @ 2.0.0" in output - assert "Installing tool-avrdude @" in output - assert len(isolated_pio_core.join("packages").listdir()) == 1 + assert not os.path.isdir(str(isolated_pio_core.join("packages"))) def test_install_from_vcs(clirunner, validate_cliresult, isolated_pio_core): @@ -89,7 +89,7 @@ def test_install_from_vcs(clirunner, validate_cliresult, isolated_pio_core): ) validate_cliresult(result) assert "espressif8266" in result.output - assert len(isolated_pio_core.join("packages").listdir()) == 1 + assert not os.path.isdir(str(isolated_pio_core.join("packages"))) def test_list_json_output(clirunner, validate_cliresult): @@ -109,6 +109,11 @@ def test_list_raw_output(clirunner, validate_cliresult): def test_update_check(clirunner, validate_cliresult, isolated_pio_core): + result = clirunner.invoke( + cli_platform.package_install_cmd, + ["--global", "--tool", "platformio/tool-avrdude@~1.60300.0"], + ) + validate_cliresult(result) result = clirunner.invoke( cli_platform.platform_update, ["--dry-run", "--json-output"] ) @@ -120,7 +125,7 @@ def test_update_check(clirunner, validate_cliresult, isolated_pio_core): def test_update_raw(clirunner, validate_cliresult, isolated_pio_core): - result = clirunner.invoke(cli_platform.platform_update) + result = clirunner.invoke(cli_platform.platform_update, ["atmelavr"]) validate_cliresult(result) output = strip_ansi_codes(result.output) assert "Removing atmelavr @ 2.0.0" in output From 5142feba7ac2fc1f9af567c3f92b5821b9352e1f Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 20 Jun 2022 14:24:09 +0300 Subject: [PATCH 38/93] Use new unified package API for deprecated `pio lib` command // Resolve #4198 --- .../commands/{lib/command.py => lib.py} | 446 +++++------------- platformio/commands/lib/__init__.py | 13 - platformio/commands/lib/helpers.py | 104 ---- platformio/commands/platform.py | 4 +- platformio/package/commands/show.py | 6 +- platformio/package/manager/library.py | 37 +- tests/commands/test_ci.py | 7 +- tests/commands/test_lib.py | 8 +- tests/commands/test_lib_complex.py | 32 +- 9 files changed, 170 insertions(+), 487 deletions(-) rename platformio/commands/{lib/command.py => lib.py} (54%) delete mode 100644 platformio/commands/lib/__init__.py delete mode 100644 platformio/commands/lib/helpers.py diff --git a/platformio/commands/lib/command.py b/platformio/commands/lib.py similarity index 54% rename from platformio/commands/lib/command.py rename to platformio/commands/lib.py index 3b9af87e..ed3c9e8d 100644 --- a/platformio/commands/lib/command.py +++ b/platformio/commands/lib.py @@ -17,16 +17,18 @@ import json import logging import os -import time -from urllib.parse import quote import click -from tabulate import tabulate -from platformio import exception, fs, util +from platformio import exception, fs from platformio.cli import PlatformioCLI -from platformio.commands.lib.helpers import get_builtin_libs, save_project_libdeps -from platformio.package.exception import NotGlobalLibDir, UnknownPackageError +from platformio.package.commands.install import package_install_cmd +from platformio.package.commands.list import package_list_cmd +from platformio.package.commands.search import package_search_cmd +from platformio.package.commands.show import package_show_cmd +from platformio.package.commands.uninstall import package_uninstall_cmd +from platformio.package.commands.update import package_update_cmd +from platformio.package.exception import NotGlobalLibDir from platformio.package.manager.library import LibraryPackageManager from platformio.package.meta import PackageItem, PackageSpec from platformio.proc import is_ci @@ -43,6 +45,20 @@ def get_project_global_lib_dir(): return ProjectConfig.get_instance().get("platformio", "globallib_dir") +def invoke_command(ctx, cmd, **kwargs): + input_dirs = ctx.meta.get(CTX_META_INPUT_DIRS_KEY, []) + project_environments = ctx.meta[CTX_META_PROJECT_ENVIRONMENTS_KEY] + for input_dir in input_dirs: + cmd_kwargs = kwargs.copy() + if is_platformio_project(input_dir): + cmd_kwargs["project_dir"] = input_dir + cmd_kwargs["environments"] = project_environments + else: + cmd_kwargs["global"] = True + cmd_kwargs["storage_dir"] = input_dir + ctx.invoke(cmd, **cmd_kwargs) + + @click.group(short_help="Library manager", hidden=True) @click.option( "-d", @@ -146,55 +162,14 @@ def lib_install( # pylint: disable=too-many-arguments,unused-argument "the next releases. \nPlease use `pio pkg install` instead.\n", fg="yellow", ) - storage_dirs = ctx.meta[CTX_META_STORAGE_DIRS_KEY] - storage_libdeps = ctx.meta.get(CTX_META_STORAGE_LIBDEPS_KEY, []) - - installed_pkgs = {} - for storage_dir in storage_dirs: - if not silent and (libraries or storage_dir in storage_libdeps): - print_storage_header(storage_dirs, storage_dir) - lm = LibraryPackageManager(storage_dir) - lm.set_log_level(logging.WARN if silent else logging.DEBUG) - - if libraries: - installed_pkgs = { - library: lm.install(library, force=force) for library in libraries - } - - elif storage_dir in storage_libdeps: - for library in storage_libdeps[storage_dir]: - lm.install(library, force=force) - - if save and installed_pkgs: - _save_deps(ctx, installed_pkgs) - - -def _save_deps(ctx, pkgs, action="add"): - specs = [] - for library, pkg in pkgs.items(): - spec = PackageSpec(library) - if spec.external: - specs.append(spec) - else: - specs.append( - PackageSpec( - owner=pkg.metadata.spec.owner, - name=pkg.metadata.spec.name, - requirements=spec.requirements - or ( - ("^%s" % pkg.metadata.version) - if not pkg.metadata.version.build - else pkg.metadata.version - ), - ) - ) - - input_dirs = ctx.meta.get(CTX_META_INPUT_DIRS_KEY, []) - project_environments = ctx.meta[CTX_META_PROJECT_ENVIRONMENTS_KEY] - for input_dir in input_dirs: - if not is_platformio_project(input_dir): - continue - save_project_libdeps(input_dir, specs, project_environments, action=action) + return invoke_command( + ctx, + package_install_cmd, + libraries=libraries, + no_save=not save, + force=force, + silent=silent, + ) @cli.command("uninstall", short_help="Remove libraries") @@ -214,16 +189,13 @@ def lib_uninstall(ctx, libraries, save, silent): "the next releases. \nPlease use `pio pkg uninstall` instead.\n", fg="yellow", ) - storage_dirs = ctx.meta[CTX_META_STORAGE_DIRS_KEY] - uninstalled_pkgs = {} - for storage_dir in storage_dirs: - print_storage_header(storage_dirs, storage_dir) - lm = LibraryPackageManager(storage_dir) - lm.set_log_level(logging.WARN if silent else logging.DEBUG) - uninstalled_pkgs = {library: lm.uninstall(library) for library in libraries} - - if save and uninstalled_pkgs: - _save_deps(ctx, uninstalled_pkgs, action="remove") + invoke_command( + ctx, + package_uninstall_cmd, + libraries=libraries, + no_save=not save, + silent=silent, + ) @cli.command("update", short_help="Update installed libraries") @@ -255,60 +227,51 @@ def lib_update( # pylint: disable=too-many-arguments "the next releases. \nPlease use `pio pkg update` instead.\n", fg="yellow", ) + return invoke_command( + ctx, + package_update_cmd, + libraries=libraries, + silent=silent, + ) storage_dirs = ctx.meta[CTX_META_STORAGE_DIRS_KEY] json_result = {} for storage_dir in storage_dirs: - if not json_output: - print_storage_header(storage_dirs, storage_dir) lib_deps = ctx.meta.get(CTX_META_STORAGE_LIBDEPS_KEY, {}).get(storage_dir, []) lm = LibraryPackageManager(storage_dir) lm.set_log_level(logging.WARN if silent else logging.DEBUG) _libraries = libraries or lib_deps or lm.get_installed() - if only_check and json_output: - result = [] - for library in _libraries: - spec = None - pkg = None - if isinstance(library, PackageItem): - pkg = library - else: - spec = PackageSpec(library) - pkg = lm.get_package(spec) - if not pkg: - continue - outdated = lm.outdated(pkg, spec) - if not outdated.is_outdated(allow_incompatible=True): - continue - manifest = lm.legacy_load_manifest(pkg) - manifest["versionWanted"] = ( - str(outdated.wanted) if outdated.wanted else None - ) - manifest["versionLatest"] = ( - str(outdated.latest) if outdated.latest else None - ) - result.append(manifest) - json_result[storage_dir] = result - else: - for library in _libraries: - to_spec = ( - None if isinstance(library, PackageItem) else PackageSpec(library) - ) - try: - lm.update(library, to_spec=to_spec) - except UnknownPackageError as e: - if library not in lib_deps: - raise e - - if json_output: - return click.echo( - json.dumps( - json_result[storage_dirs[0]] if len(storage_dirs) == 1 else json_result + result = [] + for library in _libraries: + spec = None + pkg = None + if isinstance(library, PackageItem): + pkg = library + else: + spec = PackageSpec(library) + pkg = lm.get_package(spec) + if not pkg: + continue + outdated = lm.outdated(pkg, spec) + if not outdated.is_outdated(allow_incompatible=True): + continue + manifest = lm.legacy_load_manifest(pkg) + manifest["versionWanted"] = ( + str(outdated.wanted) if outdated.wanted else None ) - ) + manifest["versionLatest"] = ( + str(outdated.latest) if outdated.latest else None + ) + result.append(manifest) - return True + json_result[storage_dir] = result + + return click.echo( + json.dumps( + json_result[storage_dirs[0]] if len(storage_dirs) == 1 else json_result + ) + ) @cli.command("list", short_help="List installed libraries") @@ -321,29 +284,18 @@ def lib_list(ctx, json_output): "the next releases. \nPlease use `pio pkg list` instead.\n", fg="yellow", ) + return invoke_command(ctx, package_list_cmd, only_libraries=True) + storage_dirs = ctx.meta[CTX_META_STORAGE_DIRS_KEY] json_result = {} for storage_dir in storage_dirs: - if not json_output: - print_storage_header(storage_dirs, storage_dir) lm = LibraryPackageManager(storage_dir) - items = lm.legacy_get_installed() - if json_output: - json_result[storage_dir] = items - elif items: - for item in sorted(items, key=lambda i: i["name"]): - print_lib_item(item) - else: - click.echo("No items found") - - if json_output: - return click.echo( - json.dumps( - json_result[storage_dirs[0]] if len(storage_dirs) == 1 else json_result - ) + json_result[storage_dir] = lm.legacy_get_installed() + return click.echo( + json.dumps( + json_result[storage_dirs[0]] if len(storage_dirs) == 1 else json_result ) - - return True + ) @cli.command("search", short_help="Search for a library") @@ -363,14 +315,10 @@ def lib_list(ctx, json_output): is_flag=True, help="Do not prompt, automatically paginate with delay", ) -def lib_search(query, json_output, page, noninteractive, **filters): - if not json_output: - click.secho( - "\nWARNING: This command is deprecated and will be removed in " - "the next releases. \nPlease use `pio pkg search` instead.\n", - fg="yellow", - ) - regclient = LibraryPackageManager().get_registry_client_instance() +@click.pass_context +def lib_search( # pylint: disable=unused-argument + ctx, query, json_output, page, noninteractive, **filters +): if not query: query = [] if not isinstance(query, list): @@ -380,72 +328,30 @@ def lib_search(query, json_output, page, noninteractive, **filters): for value in values: query.append('%s:"%s"' % (key, value)) + if not json_output: + click.secho( + "\nWARNING: This command is deprecated and will be removed in " + "the next releases. \nPlease use `pio pkg search` instead.\n", + fg="yellow", + ) + query.append("type:library") + return ctx.invoke(package_search_cmd, query=" ".join(query), page=page) + + regclient = LibraryPackageManager().get_registry_client_instance() result = regclient.fetch_json_data( "get", "/v2/lib/search", params=dict(query=" ".join(query), page=page), x_cache_valid="1d", ) - - if json_output: - click.echo(json.dumps(result)) - return - - if result["total"] == 0: - click.secho( - "Nothing has been found by your request\n" - "Try a less-specific search or use truncation (or wildcard) " - "operator", - fg="yellow", - nl=False, - ) - click.secho(" *", fg="green") - click.secho("For example: DS*, PCA*, DHT* and etc.\n", fg="yellow") - click.echo( - "For more examples and advanced search syntax, please use documentation:" - ) - click.secho( - "https://docs.platformio.org/page/userguide/lib/cmd_search.html\n", - fg="cyan", - ) - return - - click.secho( - "Found %d libraries:\n" % result["total"], - fg="green" if result["total"] else "yellow", - ) - - while True: - for item in result["items"]: - print_lib_item(item) - - if int(result["page"]) * int(result["perpage"]) >= int(result["total"]): - break - - if noninteractive: - click.echo() - click.secho( - "Loading next %d libraries... Press Ctrl+C to stop!" - % result["perpage"], - fg="yellow", - ) - click.echo() - time.sleep(5) - elif not click.confirm("Show next libraries?"): - break - result = regclient.fetch_json_data( - "get", - "/v2/lib/search", - params=dict(query=" ".join(query), page=int(result["page"]) + 1), - x_cache_valid="1d", - ) + return click.echo(json.dumps(result)) @cli.command("builtin", short_help="List built-in libraries") @click.option("--storage", multiple=True) @click.option("--json-output", is_flag=True) def lib_builtin(storage, json_output): - items = get_builtin_libs(storage) + items = LibraryPackageManager.get_builtin_libs(storage) if json_output: return click.echo(json.dumps(items)) @@ -465,13 +371,16 @@ def lib_builtin(storage, json_output): @cli.command("show", short_help="Show detailed info about a library") @click.argument("library", metavar="[LIBRARY]") @click.option("--json-output", is_flag=True) -def lib_show(library, json_output): +@click.pass_context +def lib_show(ctx, library, json_output): if not json_output: click.secho( "\nWARNING: This command is deprecated and will be removed in " "the next releases. \nPlease use `pio pkg show` instead.\n", fg="yellow", ) + return ctx.invoke(package_show_cmd, pkg_type="library", spec=library) + lm = LibraryPackageManager() lm.set_log_level(logging.ERROR if json_output else logging.DEBUG) lib_id = lm.reveal_registry_package_id(library) @@ -479,86 +388,7 @@ def lib_show(library, json_output): lib = regclient.fetch_json_data( "get", "/v2/lib/info/%d" % lib_id, x_cache_valid="1h" ) - if json_output: - return click.echo(json.dumps(lib)) - - title = "{ownername}/{name}".format(**lib) - click.secho(title, fg="cyan") - click.echo("=" * len(title)) - click.echo(lib["description"]) - click.echo() - - click.secho("ID: %d" % lib["id"]) - click.echo( - "Version: %s, released %s" - % ( - lib["version"]["name"], - util.parse_datetime(lib["version"]["released"]).strftime("%c"), - ) - ) - click.echo("Manifest: %s" % lib["confurl"]) - for key in ("homepage", "repository", "license"): - if key not in lib or not lib[key]: - continue - if isinstance(lib[key], list): - click.echo("%s: %s" % (key.capitalize(), ", ".join(lib[key]))) - else: - click.echo("%s: %s" % (key.capitalize(), lib[key])) - - blocks = [] - - _authors = [] - for author in lib.get("authors", []): - _data = [] - for key in ("name", "email", "url", "maintainer"): - if not author.get(key): - continue - if key == "email": - _data.append("<%s>" % author[key]) - elif key == "maintainer": - _data.append("(maintainer)") - else: - _data.append(author[key]) - _authors.append(" ".join(_data)) - if _authors: - blocks.append(("Authors", _authors)) - - blocks.append(("Keywords", lib["keywords"])) - for key in ("frameworks", "platforms"): - if key not in lib or not lib[key]: - continue - blocks.append(("Compatible %s" % key, [i["title"] for i in lib[key]])) - blocks.append(("Headers", lib["headers"])) - blocks.append(("Examples", lib["examples"])) - blocks.append( - ( - "Versions", - [ - "%s, released %s" - % (v["name"], util.parse_datetime(v["released"]).strftime("%c")) - for v in lib["versions"] - ], - ) - ) - blocks.append( - ( - "Unique Downloads", - [ - "Today: %s" % lib["dlstats"]["day"], - "Week: %s" % lib["dlstats"]["week"], - "Month: %s" % lib["dlstats"]["month"], - ], - ) - ) - - for (title, rows) in blocks: - click.echo() - click.secho(title, bold=True) - click.echo("-" * len(title)) - for row in rows: - click.echo(row) - - return True + return click.echo(json.dumps(lib)) @cli.command("register", short_help="Deprecated") @@ -572,76 +402,18 @@ def lib_register(config_url): # pylint: disable=unused-argument @cli.command("stats", short_help="Library Registry Statistics") @click.option("--json-output", is_flag=True) def lib_stats(json_output): + if not json_output: + click.secho( + "\nWARNING: This command is deprecated and will be removed in " + "the next releases. \nPlease visit " + "https://registry.platformio.org\n", + fg="yellow", + ) + return None + regclient = LibraryPackageManager().get_registry_client_instance() result = regclient.fetch_json_data("get", "/v2/lib/stats", x_cache_valid="1h") - - if json_output: - return click.echo(json.dumps(result)) - - for key in ("updated", "added"): - tabular_data = [ - ( - click.style(item["name"], fg="cyan"), - util.parse_datetime(item["date"]).strftime("%c"), - "https://platformio.org/lib/show/%s/%s" - % (item["id"], quote(item["name"])), - ) - for item in result.get(key, []) - ] - table = tabulate( - tabular_data, - headers=[click.style("RECENTLY " + key.upper(), bold=True), "Date", "URL"], - ) - click.echo(table) - click.echo() - - for key in ("lastkeywords", "topkeywords"): - tabular_data = [ - ( - click.style(name, fg="cyan"), - "https://platformio.org/lib/search?query=" + quote("keyword:%s" % name), - ) - for name in result.get(key, []) - ] - table = tabulate( - tabular_data, - headers=[ - click.style( - ("RECENT" if key == "lastkeywords" else "POPULAR") + " KEYWORDS", - bold=True, - ), - "URL", - ], - ) - click.echo(table) - click.echo() - - for key, title in (("dlday", "Today"), ("dlweek", "Week"), ("dlmonth", "Month")): - tabular_data = [ - ( - click.style(item["name"], fg="cyan"), - "https://platformio.org/lib/show/%s/%s" - % (item["id"], quote(item["name"])), - ) - for item in result.get(key, []) - ] - table = tabulate( - tabular_data, - headers=[click.style("FEATURED: " + title.upper(), bold=True), "URL"], - ) - click.echo(table) - click.echo() - - return True - - -def print_storage_header(storage_dirs, storage_dir): - if storage_dirs and storage_dirs[0] != storage_dir: - click.echo("") - click.echo( - click.style("Library Storage: ", bold=True) - + click.style(storage_dir, fg="blue") - ) + return click.echo(json.dumps(result)) def print_lib_item(item): diff --git a/platformio/commands/lib/__init__.py b/platformio/commands/lib/__init__.py deleted file mode 100644 index b0514903..00000000 --- a/platformio/commands/lib/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright (c) 2014-present PlatformIO -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/platformio/commands/lib/helpers.py b/platformio/commands/lib/helpers.py deleted file mode 100644 index 9b99cd6c..00000000 --- a/platformio/commands/lib/helpers.py +++ /dev/null @@ -1,104 +0,0 @@ -# Copyright (c) 2014-present PlatformIO -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os - -from platformio import util -from platformio.compat import ci_strings_are_equal -from platformio.package.manager.platform import PlatformPackageManager -from platformio.package.meta import PackageSpec -from platformio.platform.factory import PlatformFactory -from platformio.project.config import ProjectConfig -from platformio.project.exception import InvalidProjectConfError - - -@util.memoized(expire="60s") -def get_builtin_libs(storage_names=None): - # pylint: disable=import-outside-toplevel - from platformio.package.manager.library import LibraryPackageManager - - items = [] - storage_names = storage_names or [] - pm = PlatformPackageManager() - for pkg in pm.get_installed(): - p = PlatformFactory.new(pkg) - for storage in p.get_lib_storages(): - if storage_names and storage["name"] not in storage_names: - continue - lm = LibraryPackageManager(storage["path"]) - items.append( - { - "name": storage["name"], - "path": storage["path"], - "items": lm.legacy_get_installed(), - } - ) - return items - - -def is_builtin_lib(name): - for storage in get_builtin_libs(): - for lib in storage["items"]: - if lib.get("name") == name: - return True - return False - - -def ignore_deps_by_specs(deps, specs): - result = [] - for dep in deps: - depspec = PackageSpec(dep) - if depspec.external: - result.append(dep) - continue - ignore_conditions = [] - for spec in specs: - if depspec.owner: - ignore_conditions.append( - ci_strings_are_equal(depspec.owner, spec.owner) - and ci_strings_are_equal(depspec.name, spec.name) - ) - else: - ignore_conditions.append(ci_strings_are_equal(depspec.name, spec.name)) - if not any(ignore_conditions): - result.append(dep) - return result - - -def save_project_libdeps(project_dir, specs, environments=None, action="add"): - config = ProjectConfig.get_instance(os.path.join(project_dir, "platformio.ini")) - config.validate(environments) - for env in config.envs(): - if environments and env not in environments: - continue - config.expand_interpolations = False - candidates = [] - try: - candidates = ignore_deps_by_specs( - config.get("env:" + env, "lib_deps"), specs - ) - except InvalidProjectConfError: - pass - if action == "add": - candidates.extend(spec.as_dependency() for spec in specs) - if candidates: - result = [] - for item in candidates: - item = item.strip() - if item and item not in result: - result.append(item) - config.set("env:" + env, "lib_deps", result) - elif config.has_option("env:" + env, "lib_deps"): - config.remove_option("env:" + env, "lib_deps") - config.save() diff --git a/platformio/commands/platform.py b/platformio/commands/platform.py index a02476c4..2f2ae47c 100644 --- a/platformio/commands/platform.py +++ b/platformio/commands/platform.py @@ -74,8 +74,8 @@ def platform_frameworks(query, json_output): if not json_output: click.secho( "\nWARNING: This command is deprecated and will be removed in " - "the next releases. \nPlease https://docs.platformio.org" - "/en/latest/frameworks/index.html.\n", + "the next releases. \nPlease visit https://docs.platformio.org" + "/en/latest/frameworks/index.html\n", fg="yellow", ) return diff --git a/platformio/package/commands/show.py b/platformio/package/commands/show.py index bf2d1f65..5be6790c 100644 --- a/platformio/package/commands/show.py +++ b/platformio/package/commands/show.py @@ -129,7 +129,11 @@ def fetch_package_data(spec, pkg_type=None): return client.get_package( pkg_type, spec.owner, spec.name, version=spec.requirements ) - qualifiers = dict(names=spec.name.lower()) + qualifiers = {} + if spec.id: + qualifiers["ids"] = str(spec.id) + if spec.name: + qualifiers["names"] = spec.name.lower() if pkg_type: qualifiers["types"] = pkg_type if spec.owner: diff --git a/platformio/package/manager/library.py b/platformio/package/manager/library.py index 802e0cfd..f17e5eb8 100644 --- a/platformio/package/manager/library.py +++ b/platformio/package/manager/library.py @@ -15,10 +15,11 @@ import json import os -from platformio.commands.lib.helpers import is_builtin_lib +from platformio import util from platformio.package.exception import MissingPackageManifestError from platformio.package.manager.base import BasePackageManager from platformio.package.meta import PackageSpec, PackageType +from platformio.platform.factory import PlatformFactory from platformio.project.config import ProjectConfig @@ -84,7 +85,39 @@ class LibraryPackageManager(BasePackageManager): # pylint: disable=too-many-anc # skip built-in dependencies not_builtin_conds = [spec.external, spec.owner] if not any(not_builtin_conds): - not_builtin_conds.append(not is_builtin_lib(spec.name)) + not_builtin_conds.append(not self.is_builtin_lib(spec.name)) if any(not_builtin_conds): return super().install_dependency(dependency) return None + + @staticmethod + @util.memoized(expire="60s") + def get_builtin_libs(storage_names=None): + # pylint: disable=import-outside-toplevel + from platformio.package.manager.platform import PlatformPackageManager + + items = [] + storage_names = storage_names or [] + pm = PlatformPackageManager() + for pkg in pm.get_installed(): + p = PlatformFactory.new(pkg) + for storage in p.get_lib_storages(): + if storage_names and storage["name"] not in storage_names: + continue + lm = LibraryPackageManager(storage["path"]) + items.append( + { + "name": storage["name"], + "path": storage["path"], + "items": lm.legacy_get_installed(), + } + ) + return items + + @classmethod + def is_builtin_lib(cls, name): + for storage in cls.get_builtin_libs(): + for lib in storage["items"]: + if lib.get("name") == name: + return True + return False diff --git a/tests/commands/test_ci.py b/tests/commands/test_ci.py index 01ac9c37..ae632280 100644 --- a/tests/commands/test_ci.py +++ b/tests/commands/test_ci.py @@ -15,7 +15,7 @@ from os.path import isfile, join from platformio.commands.ci import cli as cmd_ci -from platformio.commands.lib.command import cli as cmd_lib +from platformio.package.commands.install import package_install_cmd def test_ci_empty(clirunner): @@ -170,7 +170,8 @@ def test_ci_project_conf(clirunner, validate_cliresult): def test_ci_lib_and_board(clirunner, tmpdir_factory, validate_cliresult): storage_dir = str(tmpdir_factory.mktemp("lib")) result = clirunner.invoke( - cmd_lib, ["--storage-dir", storage_dir, "install", "1@2.3.2"] + package_install_cmd, + ["--global", "--storage-dir", storage_dir, "--library", "1"], ) validate_cliresult(result) @@ -182,7 +183,7 @@ def test_ci_lib_and_board(clirunner, tmpdir_factory, validate_cliresult): "OneWire", "examples", "DS2408_Switch", - "DS2408_Switch.pde", + "DS2408_Switch.ino", ), "-l", join(storage_dir, "OneWire"), diff --git a/tests/commands/test_lib.py b/tests/commands/test_lib.py index cf1e53b6..8f47ad61 100644 --- a/tests/commands/test_lib.py +++ b/tests/commands/test_lib.py @@ -20,7 +20,7 @@ import os import pytest import semantic_version -from platformio.commands.lib.command import cli as cmd_lib +from platformio.commands.lib import cli as cmd_lib from platformio.package.meta import PackageType from platformio.package.vcsclient import VCSClientFactory from platformio.project.config import ProjectConfig @@ -137,11 +137,7 @@ lib_deps = # test list result = clirunner.invoke(cmd_lib, ["-d", str(project_dir), "list"]) validate_cliresult(result) - assert "Version: 0.8.3+sha." in result.stdout - assert ( - "Source: git+https://github.com/OttoWinter/async-mqtt-client.git#v0.8.3" - in result.stdout - ) + assert "AsyncMqttClient-esphome @ 0.8.3+sha.f5aa899" in result.stdout result = clirunner.invoke( cmd_lib, ["-d", str(project_dir), "list", "--json-output"] ) diff --git a/tests/commands/test_lib_complex.py b/tests/commands/test_lib_complex.py index 31f61fc6..6ae585a2 100644 --- a/tests/commands/test_lib_complex.py +++ b/tests/commands/test_lib_complex.py @@ -18,7 +18,7 @@ import json import re from platformio.cli import PlatformioCLI -from platformio.commands.lib.command import cli as cmd_lib +from platformio.commands.lib import cli as cmd_lib from platformio.package.exception import UnknownPackageError from platformio.util import strip_ansi_codes @@ -28,12 +28,12 @@ PlatformioCLI.leftover_args = ["--json-output"] # hook for click def test_search(clirunner, validate_cliresult): result = clirunner.invoke(cmd_lib, ["search", "DHT22"]) validate_cliresult(result) - match = re.search(r"Found\s+(\d+)\slibraries:", result.output) + match = re.search(r"Found\s+(\d+)\spackages", result.output) assert int(match.group(1)) > 2 result = clirunner.invoke(cmd_lib, ["search", "DHT22", "--platform=timsp430"]) validate_cliresult(result) - match = re.search(r"Found\s+(\d+)\slibraries:", result.output) + match = re.search(r"Found\s+(\d+)\spackages", result.output) assert int(match.group(1)) > 1 @@ -175,10 +175,10 @@ def test_global_lib_list(clirunner, validate_cliresult): assert all( n in result.output for n in ( - "Source: https://github.com/Pedroalbuquerque/ESP32WebServer/archive/master.zip", - "Version: 5.10.1", - "Source: git+https://github.com/gioblu/PJON.git#3.0", - "Version: 3.0.0+sha.1fb26fd", + "required: https://github.com/Pedroalbuquerque/ESP32WebServer/archive/master.zip", + "ArduinoJson @ 5.10.1", + "required: git+https://github.com/gioblu/PJON.git#3.0", + "PJON @ 3.0.0+sha.1fb26f", ) ) @@ -251,11 +251,12 @@ def test_global_lib_update(clirunner, validate_cliresult): validate_cliresult(result) assert "Removing NeoPixelBus @ 2.2.4" in strip_ansi_codes(result.output) - # update rest libraries - result = clirunner.invoke(cmd_lib, ["-g", "update"]) + # update all libraries + result = clirunner.invoke( + cmd_lib, + ["-g", "update", "adafruit/Adafruit PN532", "marvinroger/AsyncMqttClient"], + ) validate_cliresult(result) - assert result.output.count("+sha.") == 4 - assert result.output.count("already up-to-date") == 14 # update unknown library result = clirunner.invoke(cmd_lib, ["-g", "update", "Unknown"]) @@ -314,7 +315,7 @@ def test_global_lib_uninstall(clirunner, validate_cliresult, isolated_pio_core): def test_lib_show(clirunner, validate_cliresult): result = clirunner.invoke(cmd_lib, ["show", "64"]) validate_cliresult(result) - assert all(s in result.output for s in ("ArduinoJson", "Arduino", "Atmel AVR")) + assert all(s in result.output for s in ("ArduinoJson", "Arduino")) result = clirunner.invoke(cmd_lib, ["show", "OneWire", "--json-output"]) validate_cliresult(result) assert "OneWire" in result.output @@ -328,13 +329,6 @@ def test_lib_builtin(clirunner, validate_cliresult): def test_lib_stats(clirunner, validate_cliresult): - result = clirunner.invoke(cmd_lib, ["stats"]) - validate_cliresult(result) - assert all( - s in result.output - for s in ("UPDATED", "POPULAR", "https://platformio.org/lib/show") - ) - result = clirunner.invoke(cmd_lib, ["stats", "--json-output"]) validate_cliresult(result) assert set( From 9ae67fdad933a32061fa288e7241741cdae5e7b4 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 20 Jun 2022 14:49:39 +0300 Subject: [PATCH 39/93] Fixed an issue on Windows OS when flags were wrapped to the temporary file while generating compilation database --- HISTORY.rst | 1 + platformio/builder/main.py | 2 +- platformio/builder/tools/piomaxlen.py | 7 +++++-- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 85709451..a2c8f297 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -37,6 +37,7 @@ PlatformIO Core 6 * Improved a serial port finder for a board with predefined HWIDs * Fixed an issue when the `build_unflags `__ operation ignores a flag value (`issue #4309 `_) * Fixed an issue when the `build_unflags `__ option was not applied to the ``ASPPFLAGS`` scope +* Fixed an issue on Windows OS when flags were wrapped to the temporary file while generating the `Compilation database "compile_commands.json" `__ 6.0.2 (2022-06-01) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/builder/main.py b/platformio/builder/main.py index 9cbd387b..e46dd079 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -59,13 +59,13 @@ DEFAULT_ENV_OPTIONS = dict( "pioplatform", "piotest", "piotarget", - "piomaxlen", "piolib", "pioupload", "piosize", "pioino", "piomisc", "piointegration", + "piomaxlen", ], toolpath=[os.path.join(fs.get_source_dir(), "builder", "tools")], variables=clivars, diff --git a/platformio/builder/tools/piomaxlen.py b/platformio/builder/tools/piomaxlen.py index c7360418..b6a0ebb0 100644 --- a/platformio/builder/tools/piomaxlen.py +++ b/platformio/builder/tools/piomaxlen.py @@ -19,6 +19,7 @@ import os import re from SCons.Platform import TempFileMunge # pylint: disable=import-error +from SCons.Script import COMMAND_LINE_TARGETS # pylint: disable=import-error from SCons.Subst import quote_spaces # pylint: disable=import-error from platformio.compat import IS_WINDOWS, hashlib_encode_data @@ -70,11 +71,13 @@ def _file_long_data(env, data): return tmp_file -def exists(_): - return True +def exists(env): + return "compiledb" not in COMMAND_LINE_TARGETS and not env.IsIntegrationDump() def generate(env): + if not exists(env): + return env kwargs = dict( _long_sources_hook=long_sources_hook, TEMPFILE=TempFileMunge, From 4b2f0eb1d5431dc70b1271b4908026ab77bad3b4 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 20 Jun 2022 22:20:47 +0300 Subject: [PATCH 40/93] Extend CP210X rules --- scripts/99-platformio-udev.rules | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/scripts/99-platformio-udev.rules b/scripts/99-platformio-udev.rules index 574b6fe6..05c6ab45 100644 --- a/scripts/99-platformio-udev.rules +++ b/scripts/99-platformio-udev.rules @@ -25,7 +25,8 @@ # # CP210X USB UART -ATTRS{idVendor}=="10c4", ATTRS{idProduct}=="ea60", MODE:="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" +ATTRS{idVendor}=="10c4", ATTRS{idProduct}=="ea[67][013]", MODE:="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" +ATTRS{idVendor}=="10c4", ATTRS{idProduct}=="80a9", MODE:="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # FT231XS USB UART ATTRS{idVendor}=="0403", ATTRS{idProduct}=="6015", MODE:="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" @@ -61,13 +62,13 @@ ATTRS{idVendor}=="16c0", ATTRS{idProduct}=="04[789A]?", ENV{MTP_NO_PROBE}="1" SUBSYSTEMS=="usb", ATTRS{idVendor}=="16c0", ATTRS{idProduct}=="04[789ABCD]?", MODE:="0666" KERNEL=="ttyACM*", ATTRS{idVendor}=="16c0", ATTRS{idProduct}=="04[789B]?", MODE:="0666" -#TI Stellaris Launchpad +# TI Stellaris Launchpad ATTRS{idVendor}=="1cbe", ATTRS{idProduct}=="00fd", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" -#TI MSP430 Launchpad +# TI MSP430 Launchpad ATTRS{idVendor}=="0451", ATTRS{idProduct}=="f432", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" -#GD32V DFU Bootloader +# GD32V DFU Bootloader ATTRS{idVendor}=="28e9", ATTRS{idProduct}=="0189", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # FireBeetle-ESP32 From 92a5c1bac62444398978332087753a49b7a77d9a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 20 Jun 2022 22:21:28 +0300 Subject: [PATCH 41/93] Handle UDEV rules and their PID/VID data when searching for serial port --- platformio/device/finder.py | 60 +++++++++++++++++++++++----------- platformio/device/list/util.py | 5 ++- platformio/fs.py | 10 ++++-- 3 files changed, 52 insertions(+), 23 deletions(-) diff --git a/platformio/device/finder.py b/platformio/device/finder.py index 6102b2f0..98aef4f6 100644 --- a/platformio/device/finder.py +++ b/platformio/device/finder.py @@ -20,6 +20,7 @@ import serial from platformio.compat import IS_MACOS, IS_WINDOWS from platformio.device.list.util import list_logical_devices, list_serial_ports +from platformio.fs import get_platformio_udev_rules_path from platformio.package.manager.platform import PlatformPackageManager from platformio.platform.factory import PlatformFactory from platformio.util import retry @@ -27,16 +28,29 @@ from platformio.util import retry BLACK_MAGIC_HWIDS = [ "1D50:6018", ] -KNOWN_UART_HWIDS = BLACK_MAGIC_HWIDS + [ - # Silicon Labs - "10C4:EA60", # CP210X - "10C4:EA61", # CP210X - "10C4:EA63", # CP210X - "10C4:EA70", # CP2105 - "10C4:EA71", # CP2108 - "10C4:EA80", # CP2110 - "10C4:80A9", # CP210X -] + + +def parse_udev_rules_hwids(path): + result = [] + with open(path, mode="r", encoding="utf8") as fp: + for line in fp.readlines(): + line = line.strip() + if not line or line.startswith("#"): + continue + attrs = {} + for attr in line.split(","): + attr = attr.replace("==", "=").replace('"', "").strip() + if "=" not in attr: + continue + name, value = attr.split("=", 1) + attrs[name] = value + hwid = "%s:%s" % ( + attrs.get("ATTRS{idVendor}", "*"), + attrs.get("ATTRS{idProduct}", "*"), + ) + if hwid != "*:*": + result.append(hwid.upper()) + return result def normalize_board_hwid(value): @@ -73,7 +87,7 @@ def find_serial_port( # pylint: disable=too-many-arguments upload_protocol=None, ensure_ready=False, prefer_gdb_port=False, - timeout=3, + timeout=2, ): if initial_port: if not is_pattern_port(initial_port): @@ -173,8 +187,14 @@ def find_board_serial_port(board_config, timeout=0): def find_known_uart_port(ensure_ready=False, timeout=0): - known_hwids = list(KNOWN_UART_HWIDS) - # load HWIDs from installed dev-platforms + known_hwids = list(BLACK_MAGIC_HWIDS) + + # load from UDEV rules + udev_rules_path = get_platformio_udev_rules_path() + if os.path.isfile(udev_rules_path): + known_hwids.extend(parse_udev_rules_hwids(udev_rules_path)) + + # load from installed dev-platforms for platform in PlatformPackageManager().get_installed(): p = PlatformFactory.new(platform) for board_config in p.get_boards().values(): @@ -187,13 +207,15 @@ def find_known_uart_port(ensure_ready=False, timeout=0): @retry(timeout=timeout) def wrapper(): - for item in list_serial_ports(filter_hwid=True): - port = item["port"] - hwid = item["hwid"].upper() - if not any(item in hwid for item in known_hwids): + for item in list_serial_ports(as_objects=True): + if not item.vid or not item.pid: continue - if not ensure_ready or is_serial_port_ready(port): - return port + hwid = "{:04X}:{:04X}".format(item.vid, item.pid) + for pattern in known_hwids: + if fnmatch(hwid, pattern) and ( + not ensure_ready or is_serial_port_ready(item.device) + ): + return item.device raise retry.RetryNextException() return wrapper() diff --git a/platformio/device/list/util.py b/platformio/device/list/util.py index 3695f760..25387484 100644 --- a/platformio/device/list/util.py +++ b/platformio/device/list/util.py @@ -24,13 +24,16 @@ from platformio import __version__, exception, proc from platformio.compat import IS_MACOS, IS_WINDOWS -def list_serial_ports(filter_hwid=False): +def list_serial_ports(filter_hwid=False, as_objects=False): try: # pylint: disable=import-outside-toplevel from serial.tools.list_ports import comports except ImportError: raise exception.GetSerialPortsError(os.name) + if as_objects: + return comports() + result = [] for p, d, h in comports(): if not p: diff --git a/platformio/fs.py b/platformio/fs.py index 2ede27b7..29dd118f 100644 --- a/platformio/fs.py +++ b/platformio/fs.py @@ -97,6 +97,12 @@ def calculate_folder_size(path): return result +def get_platformio_udev_rules_path(): + return os.path.abspath( + os.path.join(get_source_dir(), "..", "scripts", "99-platformio-udev.rules") + ) + + def ensure_udev_rules(): from platformio.util import get_systype # pylint: disable=import-outside-toplevel @@ -119,9 +125,7 @@ def ensure_udev_rules(): if not any(os.path.isfile(p) for p in installed_rules): raise exception.MissedUdevRules - origin_path = os.path.abspath( - os.path.join(get_source_dir(), "..", "scripts", "99-platformio-udev.rules") - ) + origin_path = get_platformio_udev_rules_path() if not os.path.isfile(origin_path): return None From 092326cb9127d47b461cada7f666a2206763f8d4 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 20 Jun 2022 22:40:36 +0300 Subject: [PATCH 42/93] Warn about incompatible Bash version for the Shell Completion // Resolve #4326 --- HISTORY.rst | 1 + platformio/system/completion.py | 12 ++++++++++++ 2 files changed, 13 insertions(+) diff --git a/HISTORY.rst b/HISTORY.rst index a2c8f297..6c313133 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -35,6 +35,7 @@ PlatformIO Core 6 * Allowed to ``Import("projenv")`` in a library extra script (`issue #4305 `_) * Improved a serial port finder for `Black Magic Probe `__ (`issue #4023 `_) * Improved a serial port finder for a board with predefined HWIDs +* Warn about incompatible Bash version for the `Shell Completion `__ (`issue #4326 `_) * Fixed an issue when the `build_unflags `__ operation ignores a flag value (`issue #4309 `_) * Fixed an issue when the `build_unflags `__ option was not applied to the ``ASPPFLAGS`` scope * Fixed an issue on Windows OS when flags were wrapped to the temporary file while generating the `Compilation database "compile_commands.json" `__ diff --git a/platformio/system/completion.py b/platformio/system/completion.py index 3ea80b99..3192021c 100644 --- a/platformio/system/completion.py +++ b/platformio/system/completion.py @@ -13,6 +13,8 @@ # limitations under the License. import os +import re +import subprocess from enum import Enum import click @@ -26,6 +28,14 @@ class ShellType(Enum): BASH = "bash" +def get_bash_version(): + result = subprocess.run(["bash", "--version"], capture_output=True, check=True) + match = re.search(r"version\s+(\d+)\.(\d+)", result.stdout.decode()) + if match: + return (int(match.group(1)), int(match.group(2))) + return (0, 0) + + def get_completion_install_path(shell): home_dir = os.path.expanduser("~") prog_name = click.get_current_context().find_root().info_name @@ -59,6 +69,8 @@ def is_completion_code_installed(shell, path): def install_completion_code(shell, path): + if shell == ShellType.BASH and get_bash_version() < (4, 4): + raise click.ClickException("The minimal supported Bash version is 4.4") if is_completion_code_installed(shell, path): return None append = shell != ShellType.FISH From f893fcf1351c1318c8f594171655b126ca6bbfbf Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 20 Jun 2022 22:41:02 +0300 Subject: [PATCH 43/93] Bump version to 6.0.3b1 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 43e87f1b..1b0acd65 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (6, 0, "3a7") +VERSION = (6, 0, "3b1") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From c557473cfbb1ae94bbbaf42e352107719d492e03 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 22 Jun 2022 13:15:32 +0300 Subject: [PATCH 44/93] Docs: Fix redirect URL --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 6bfc42b2..519b5db4 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 6bfc42b22668cea6210f70feee8dcf427de55c7f +Subproject commit 519b5db43440c219e6388e6d64f53850bc6882ca From cee3f4d90fe4ad8486f0cf4f6a1c0ecb238f379a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 22 Jun 2022 19:03:50 +0300 Subject: [PATCH 45/93] Do not resolve debugging serial port by default --- platformio/debug/config/base.py | 21 +++------------------ platformio/debug/config/blackmagic.py | 24 ++++++++++++++++++++++++ platformio/debug/config/generic.py | 3 ++- platformio/debug/config/jlink.py | 3 ++- platformio/debug/config/mspdebug.py | 3 ++- platformio/debug/config/qemu.py | 3 ++- platformio/debug/config/renode.py | 3 ++- 7 files changed, 37 insertions(+), 23 deletions(-) diff --git a/platformio/debug/config/base.py b/platformio/debug/config/base.py index 9787cc8c..46a91673 100644 --- a/platformio/debug/config/base.py +++ b/platformio/debug/config/base.py @@ -18,14 +18,13 @@ import os from platformio import fs, proc, util from platformio.compat import string_types from platformio.debug.exception import DebugInvalidOptionsError -from platformio.device.finder import find_serial_port, is_pattern_port from platformio.project.config import ProjectConfig from platformio.project.helpers import load_build_metadata from platformio.project.options import ProjectOptions class DebugConfigBase: # pylint: disable=too-many-instance-attributes - def __init__(self, platform, project_config, env_name): + def __init__(self, platform, project_config, env_name, port=None): self.platform = platform self.project_config = project_config self.env_name = env_name @@ -49,6 +48,7 @@ class DebugConfigBase: # pylint: disable=too-many-instance-attributes self._load_cmds = None self._port = None + self.port = port self.server = self._configure_server() try: @@ -119,25 +119,10 @@ class DebugConfigBase: # pylint: disable=too-many-instance-attributes @property def port(self): - initial_port = ( + return ( self.env_options.get("debug_port", self.tool_settings.get("port")) or self._port ) - if initial_port and not is_pattern_port(initial_port): - return initial_port - port = find_serial_port( - initial_port, - board_config=self.board_config, - upload_protocol=self.tool_name, - prefer_gdb_port=True, - ) - if port: - return port - if not self.tool_settings.get("require_debug_port"): - return None - raise DebugInvalidOptionsError( - "Please specify `debug_port` for the working environment" - ) @port.setter def port(self, value): diff --git a/platformio/debug/config/blackmagic.py b/platformio/debug/config/blackmagic.py index bfc16246..5a89c2f9 100644 --- a/platformio/debug/config/blackmagic.py +++ b/platformio/debug/config/blackmagic.py @@ -13,6 +13,8 @@ # limitations under the License. from platformio.debug.config.base import DebugConfigBase +from platformio.debug.exception import DebugInvalidOptionsError +from platformio.device.finder import find_serial_port, is_pattern_port class BlackmagicDebugConfig(DebugConfigBase): @@ -47,3 +49,25 @@ while ($busy) end set language auto """ + + @property + def port(self): + # pylint: disable=assignment-from-no-return + initial_port = DebugConfigBase.port.fget(self) + if initial_port and not is_pattern_port(initial_port): + return initial_port + port = find_serial_port( + initial_port, + board_config=self.board_config, + upload_protocol=self.tool_name, + prefer_gdb_port=True, + ) + if port: + return port + raise DebugInvalidOptionsError( + "Please specify `debug_port` for the working environment" + ) + + @port.setter + def port(self, value): + self._port = value diff --git a/platformio/debug/config/generic.py b/platformio/debug/config/generic.py index 870aad7b..1f155ecb 100644 --- a/platformio/debug/config/generic.py +++ b/platformio/debug/config/generic.py @@ -34,5 +34,6 @@ $INIT_BREAK """ def __init__(self, *args, **kwargs): + if "port" not in kwargs: + kwargs["port"] = ":3333" super().__init__(*args, **kwargs) - self.port = ":3333" diff --git a/platformio/debug/config/jlink.py b/platformio/debug/config/jlink.py index ed5f9966..03a1bd3a 100644 --- a/platformio/debug/config/jlink.py +++ b/platformio/debug/config/jlink.py @@ -38,8 +38,9 @@ $INIT_BREAK """ def __init__(self, *args, **kwargs): + if "port" not in kwargs: + kwargs["port"] = ":2331" super().__init__(*args, **kwargs) - self.port = ":2331" @property def server_ready_pattern(self): diff --git a/platformio/debug/config/mspdebug.py b/platformio/debug/config/mspdebug.py index 86ee8d6a..09266b3b 100644 --- a/platformio/debug/config/mspdebug.py +++ b/platformio/debug/config/mspdebug.py @@ -32,5 +32,6 @@ $INIT_BREAK """ def __init__(self, *args, **kwargs): + if "port" not in kwargs: + kwargs["port"] = ":2000" super().__init__(*args, **kwargs) - self.port = ":2000" diff --git a/platformio/debug/config/qemu.py b/platformio/debug/config/qemu.py index e272a373..e9a57409 100644 --- a/platformio/debug/config/qemu.py +++ b/platformio/debug/config/qemu.py @@ -33,5 +33,6 @@ $INIT_BREAK """ def __init__(self, *args, **kwargs): + if "port" not in kwargs: + kwargs["port"] = ":1234" super().__init__(*args, **kwargs) - self.port = ":1234" diff --git a/platformio/debug/config/renode.py b/platformio/debug/config/renode.py index 0a4164de..724be407 100644 --- a/platformio/debug/config/renode.py +++ b/platformio/debug/config/renode.py @@ -35,8 +35,9 @@ monitor start """ def __init__(self, *args, **kwargs): + if "port" not in kwargs: + kwargs["port"] = ":3333" super().__init__(*args, **kwargs) - self.port = ":3333" @property def server_ready_pattern(self): From 7e3e394707c9e5ef4e1299a23f1e8a27bb37f92c Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 22 Jun 2022 19:05:05 +0300 Subject: [PATCH 46/93] Bump version to 6.0.3rc1 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 1b0acd65..bb90beb5 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (6, 0, "3b1") +VERSION = (6, 0, "3rc1") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From a76933990c3863a0d8bc79a5d896b74f7b8e64de Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 23 Jun 2022 14:20:27 +0300 Subject: [PATCH 47/93] Remove generic targets --- platformio/builder/tools/piotarget.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/platformio/builder/tools/piotarget.py b/platformio/builder/tools/piotarget.py index 6ebe1f99..7436acaf 100644 --- a/platformio/builder/tools/piotarget.py +++ b/platformio/builder/tools/piotarget.py @@ -104,19 +104,6 @@ def DumpTargets(env): t["group"] == "Platform" for t in targets.values() ): targets["upload"] = dict(name="upload", group="Platform", title="Upload") - targets["compiledb"] = dict( - name="compiledb", - title="Compilation Database", - description="Generate compilation database `compile_commands.json`", - group="Advanced", - ) - targets["clean"] = dict(name="clean", title="Clean", group="General") - targets["cleanall"] = dict( - name="cleanall", - title="Clean All", - group="General", - description="Clean a build environment and installed library dependencies", - ) return list(targets.values()) From 13fc8508b3ff67faac19a3f792050e67ed61a90f Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 23 Jun 2022 18:58:53 +0300 Subject: [PATCH 48/93] Bump "uvicorn" dependency to 0.18.* --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index eea95656..52693e06 100644 --- a/setup.py +++ b/setup.py @@ -44,7 +44,7 @@ home_requirements = [ "aiofiles==0.8.*", "ajsonrpc==1.*", "starlette==%s" % ("0.20.*" if sys.version_info >= (3, 7) else "0.19.1"), - "uvicorn==%s" % ("0.17.*" if sys.version_info >= (3, 7) else "0.16.0"), + "uvicorn==%s" % ("0.18.*" if sys.version_info >= (3, 7) else "0.16.0"), "wsproto==%s" % ("1.1.*" if sys.version_info >= (3, 7) else "1.0.0"), ] From dd63c8002aabc1e400bcc27123da84c18e2336f0 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 24 Jun 2022 21:09:44 +0300 Subject: [PATCH 49/93] Make "MatchSourceFiles' configurable for source extensions --- platformio/builder/tools/platformio.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/platformio/builder/tools/platformio.py b/platformio/builder/tools/platformio.py index c828a943..803669f8 100644 --- a/platformio/builder/tools/platformio.py +++ b/platformio/builder/tools/platformio.py @@ -262,12 +262,11 @@ def StringifyMacro(env, value): # pylint: disable=unused-argument return '\\"%s\\"' % value.replace('"', '\\\\\\"') -def MatchSourceFiles(env, src_dir, src_filter=None): +def MatchSourceFiles(env, src_dir, src_filter=None, src_exts=None): src_filter = env.subst(src_filter) if src_filter else None src_filter = src_filter or SRC_FILTER_DEFAULT - return fs.match_src_files( - env.subst(src_dir), src_filter, SRC_BUILD_EXT + SRC_HEADER_EXT - ) + src_exts = src_exts or (SRC_BUILD_EXT + SRC_HEADER_EXT) + return fs.match_src_files(env.subst(src_dir), src_filter, src_exts) def CollectBuildFiles( @@ -280,7 +279,7 @@ def CollectBuildFiles( if src_dir.endswith(os.sep): src_dir = src_dir[:-1] - for item in env.MatchSourceFiles(src_dir, src_filter): + for item in env.MatchSourceFiles(src_dir, src_filter, SRC_BUILD_EXT): _reldir = os.path.dirname(item) _src_dir = os.path.join(src_dir, _reldir) if _reldir else src_dir _var_dir = os.path.join(variant_dir, _reldir) if _reldir else variant_dir @@ -289,8 +288,7 @@ def CollectBuildFiles( variants.append(_var_dir) env.VariantDir(_var_dir, _src_dir, duplicate) - if fs.path_endswith_ext(item, SRC_BUILD_EXT): - sources.append(env.File(os.path.join(_var_dir, os.path.basename(item)))) + sources.append(env.File(os.path.join(_var_dir, os.path.basename(item)))) middlewares = env.get("__PIO_BUILD_MIDDLEWARES") if not middlewares: From 86c4bd69d2869297ec12c114d4b7799c5accc68f Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 24 Jun 2022 21:17:26 +0300 Subject: [PATCH 50/93] Fixed an issue with the LDF when recursively scanning dependencies in the "chain" mode --- HISTORY.rst | 1 + platformio/builder/tools/piolib.py | 54 ++++++++++++------------------ tests/commands/test_test.py | 7 +++- 3 files changed, 28 insertions(+), 34 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 6c313133..ef4e815d 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -39,6 +39,7 @@ PlatformIO Core 6 * Fixed an issue when the `build_unflags `__ operation ignores a flag value (`issue #4309 `_) * Fixed an issue when the `build_unflags `__ option was not applied to the ``ASPPFLAGS`` scope * Fixed an issue on Windows OS when flags were wrapped to the temporary file while generating the `Compilation database "compile_commands.json" `__ +* Fixed an issue with the `LDF `__ when recursively scanning dependencies in the ``chain`` mode 6.0.2 (2022-06-01) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index 3a0072c2..75a977a0 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -318,19 +318,12 @@ class LibBuilderBase: ) def get_search_files(self): - items = [ + return [ os.path.join(self.src_dir, item) - for item in self.env.MatchSourceFiles(self.src_dir, self.src_filter) - ] - include_dir = self.include_dir - if include_dir: - items.extend( - [ - os.path.join(include_dir, item) - for item in self.env.MatchSourceFiles(include_dir) - ] + for item in self.env.MatchSourceFiles( + self.src_dir, self.src_filter, piotool.SRC_BUILD_EXT ) - return items + ] def _get_found_includes( # pylint: disable=too-many-branches self, search_files=None @@ -366,24 +359,28 @@ class LibBuilderBase: tuple(include_dirs), depth=self.CCONDITIONAL_SCANNER_DEPTH, ) - # mark candidates already processed via Conditional Scanner - self._processed_files.extend( - [ - c.get_abspath() - for c in candidates - if c.get_abspath() not in self._processed_files - ] - ) + except Exception as e: # pylint: disable=broad-except if self.verbose and "+" in self.lib_ldf_mode: sys.stderr.write( "Warning! Classic Pre Processor is used for `%s`, " "advanced has failed with `%s`\n" % (path, e) ) - candidates = LibBuilderBase.CLASSIC_SCANNER( - self.env.File(path), self.env, tuple(include_dirs) + candidates = self.env.File(path).get_implicit_deps( + self.env, + LibBuilderBase.CLASSIC_SCANNER, + lambda _: tuple(include_dirs), ) + # mark candidates already processed + self._processed_files.extend( + [ + c.get_abspath() + for c in candidates + if c.get_abspath() not in self._processed_files + ] + ) + # print(path, [c.get_abspath() for c in candidates]) for item in candidates: if item not in result: @@ -415,11 +412,12 @@ class LibBuilderBase: lib_inc_map = {} for inc in self._get_found_includes(search_files): + inc_path = inc.get_abspath() for lb in self.env.GetLibBuilders(): - if inc.get_abspath() in lb: + if inc_path in lb: if lb not in lib_inc_map: lib_inc_map[lb] = [] - lib_inc_map[lb].append(inc.get_abspath()) + lib_inc_map[lb].append(inc_path) break for lb, lb_search_files in lib_inc_map.items(): @@ -878,16 +876,6 @@ class ProjectAsLibBuilder(LibBuilderBase): def src_dir(self): return self.env.subst("$PROJECT_SRC_DIR") - def get_include_dirs(self): - include_dirs = [] - project_include_dir = self.env.subst("$PROJECT_INCLUDE_DIR") - if os.path.isdir(project_include_dir): - include_dirs.append(project_include_dir) - for include_dir in super().get_include_dirs(): - if include_dir not in include_dirs: - include_dirs.append(include_dir) - return include_dirs - def get_search_files(self): items = [] build_type = self.env.GetBuildType() diff --git a/tests/commands/test_test.py b/tests/commands/test_test.py index 2a6c7c56..9d7c1dd5 100644 --- a/tests/commands/test_test.py +++ b/tests/commands/test_test.py @@ -309,10 +309,15 @@ platform = native """ ) test_dir = project_dir.mkdir("test") - test_dir.join("test_main.c").write( + test_dir.join("test_main.h").write( """ #include #include + """ + ) + test_dir.join("test_main.c").write( + """ +#include "test_main.h" void setUp(){ printf("setUp called"); From 300b7b21387c1fb2e6ae22489c5ed498db8593ba Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 25 Jun 2022 20:10:18 +0300 Subject: [PATCH 51/93] Minor improvements to the ProjectAsLibBuilder --- platformio/builder/tools/piolib.py | 51 +++++++++++++++----------- platformio/builder/tools/platformio.py | 13 +++---- 2 files changed, 35 insertions(+), 29 deletions(-) diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index 75a977a0..7269c708 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -332,9 +332,7 @@ class LibBuilderBase: if not LibBuilderBase._INCLUDE_DIRS_CACHE: LibBuilderBase._INCLUDE_DIRS_CACHE = [ self.env.Dir(d) - for d in ProjectAsLibBuilder( - self.envorigin, "$PROJECT_DIR" - ).get_include_dirs() + for d in ProjectAsLibBuilder.get_instance().get_include_dirs() ] for lb in self.env.GetLibBuilders(): LibBuilderBase._INCLUDE_DIRS_CACHE.extend( @@ -766,6 +764,24 @@ class PlatformIOLibBuilder(LibBuilderBase): return os.path.abspath(self._manifest.get("build").get("includeDir")) return LibBuilderBase.include_dir.fget(self) # pylint: disable=no-member + def get_include_dirs(self): + include_dirs = super().get_include_dirs() + + # backwards compatibility with PlatformIO 2.0 + if ( + "build" not in self._manifest + and self._has_arduino_manifest() + and not os.path.isdir(os.path.join(self.path, "src")) + and os.path.isdir(os.path.join(self.path, "utility")) + ): + include_dirs.append(os.path.join(self.path, "utility")) + + for path in self.env.get("CPPPATH", []): + if path not in self.envorigin.get("CPPPATH", []): + include_dirs.append(self.env.subst(path)) + + return include_dirs + @property def src_dir(self): if "srcDir" in self._manifest.get("build", {}): @@ -841,32 +857,23 @@ class PlatformIOLibBuilder(LibBuilderBase): def is_frameworks_compatible(self, frameworks): return util.items_in_list(frameworks, self._manifest.get("frameworks") or ["*"]) - def get_include_dirs(self): - include_dirs = super().get_include_dirs() - - # backwards compatibility with PlatformIO 2.0 - if ( - "build" not in self._manifest - and self._has_arduino_manifest() - and not os.path.isdir(os.path.join(self.path, "src")) - and os.path.isdir(os.path.join(self.path, "utility")) - ): - include_dirs.append(os.path.join(self.path, "utility")) - - for path in self.env.get("CPPPATH", []): - if path not in self.envorigin.get("CPPPATH", []): - include_dirs.append(self.env.subst(path)) - - return include_dirs - class ProjectAsLibBuilder(LibBuilderBase): + + _INSTANCE = None + def __init__(self, env, *args, **kwargs): # backup original value, will be reset in base.__init__ project_src_filter = env.get("SRC_FILTER") super().__init__(env, *args, **kwargs) self.env["SRC_FILTER"] = project_src_filter + @classmethod + def get_instance(cls, *args, **kwargs): + if not cls._INSTANCE: + cls._INSTANCE = ProjectAsLibBuilder(*args, **kwargs) + return cls._INSTANCE + @property def include_dir(self): include_dir = self.env.subst("$PROJECT_INCLUDE_DIR") @@ -1129,7 +1136,7 @@ def ConfigureProjectLibBuilder(env): if lb.depbuilders: _print_deps_tree(lb, level + 1) - project = ProjectAsLibBuilder(env, "$PROJECT_DIR") + project = ProjectAsLibBuilder.get_instance(env, "$PROJECT_DIR") env.Export(dict(projenv=project.env)) ldf_mode = LibBuilderBase.lib_ldf_mode.fget(project) # pylint: disable=no-member diff --git a/platformio/builder/tools/platformio.py b/platformio/builder/tools/platformio.py index 803669f8..e6c9d7ba 100644 --- a/platformio/builder/tools/platformio.py +++ b/platformio/builder/tools/platformio.py @@ -141,23 +141,22 @@ def ProcessProgramDeps(env): def ProcessProjectDeps(env): - project_lib_builder = env.ConfigureProjectLibBuilder() - projenv = project_lib_builder.env + plb = env.ConfigureProjectLibBuilder() # prepend project libs to the beginning of list - env.Prepend(LIBS=project_lib_builder.build()) + env.Prepend(LIBS=plb.build()) # prepend extra linker related options from libs env.PrependUnique( **{ - key: project_lib_builder.env.get(key) + key: plb.env.get(key) for key in ("LIBS", "LIBPATH", "LINKFLAGS") - if project_lib_builder.env.get(key) + if plb.env.get(key) } ) if "test" in env.GetBuildType(): build_files_before_nums = len(env.get("PIOBUILDFILES", [])) - projenv.BuildSources( + plb.env.BuildSources( "$BUILD_TEST_DIR", "$PROJECT_TEST_DIR", "$PIOTEST_SRC_FILTER" ) if len(env.get("PIOBUILDFILES", [])) - build_files_before_nums < 1: @@ -168,7 +167,7 @@ def ProcessProjectDeps(env): env.Exit(1) if "test" not in env.GetBuildType() or env.GetProjectOption("test_build_src"): - projenv.BuildSources( + plb.env.BuildSources( "$BUILD_SRC_DIR", "$PROJECT_SRC_DIR", env.get("SRC_FILTER") ) From bb6490d6f2b62342ba8062dc5fe4bd908830f93d Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 25 Jun 2022 20:11:17 +0300 Subject: [PATCH 52/93] Do not automatically configure any flags for the testing frameworks --- platformio/test/runners/doctest.py | 6 ------ platformio/test/runners/googletest.py | 8 -------- 2 files changed, 14 deletions(-) diff --git a/platformio/test/runners/doctest.py b/platformio/test/runners/doctest.py index d0fc931f..9a6df7cf 100644 --- a/platformio/test/runners/doctest.py +++ b/platformio/test/runners/doctest.py @@ -108,12 +108,6 @@ class DoctestTestRunner(TestRunnerBase): super().__init__(*args, **kwargs) self._tc_parser = DoctestTestCaseParser() - def configure_build_env(self, env): - env.Append(CPPDEFINES=["DOCTEST_CONFIG_COLORS_NONE"]) - if self.platform.is_embedded(): - return - env.Append(CXXFLAGS=["-std=c++11"]) - def on_testing_line_output(self, line): if self.options.verbose: click.echo(line, nl=False) diff --git a/platformio/test/runners/googletest.py b/platformio/test/runners/googletest.py index 11e51ff6..973b16fa 100644 --- a/platformio/test/runners/googletest.py +++ b/platformio/test/runners/googletest.py @@ -17,7 +17,6 @@ import re import click -from platformio.compat import IS_WINDOWS from platformio.test.result import TestCase, TestCaseSource, TestStatus from platformio.test.runners.base import TestRunnerBase @@ -98,13 +97,6 @@ class GoogletestTestRunner(TestRunnerBase): self._tc_parser = GoogletestTestCaseParser() os.environ["GTEST_COLOR"] = "no" # disable ANSI symbols - def configure_build_env(self, env): - if self.platform.is_embedded(): - return - env.Append(CXXFLAGS=["-std=c++11"]) - if not IS_WINDOWS: - env.Append(CCFLAGS=["-pthread"], LINKFLAGS=["-pthread"]) - def on_testing_line_output(self, line): if self.options.verbose: click.echo(line, nl=False) From 655eedd7b060ad5c08a1a30a0225c40f8397bee8 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 25 Jun 2022 20:20:13 +0300 Subject: [PATCH 53/93] Export Unit Testing flags only to the project build environment --- HISTORY.rst | 3 ++- platformio/builder/tools/piolib.py | 2 ++ platformio/builder/tools/platformio.py | 2 -- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index ef4e815d..2b307a33 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -26,7 +26,8 @@ PlatformIO Core 6 * **Unit Testing** - - Merged the |UNITTESTING| "building" stage with "uploading" for the embedded target (`issue #4307 `_) + - Export |UNITTESTING| flags only to the project build environment (``projenv``, files in "src" folder) + - Merged the "building" stage with "uploading" for the embedded target (`issue #4307 `_) - Do not resolve dependencies from the project "src" folder when the `test_build_src `__ option is not enabled - Fixed an issue when a custom `pio test --project-config `__ was not handled properly (`issue #4299 `_) diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index 7269c708..addc3ce8 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -867,6 +867,8 @@ class ProjectAsLibBuilder(LibBuilderBase): project_src_filter = env.get("SRC_FILTER") super().__init__(env, *args, **kwargs) self.env["SRC_FILTER"] = project_src_filter + if "test" in self.env.GetBuildType(): + self.env.ConfigureTestTarget() @classmethod def get_instance(cls, *args, **kwargs): diff --git a/platformio/builder/tools/platformio.py b/platformio/builder/tools/platformio.py index e6c9d7ba..c08c7438 100644 --- a/platformio/builder/tools/platformio.py +++ b/platformio/builder/tools/platformio.py @@ -126,8 +126,6 @@ def ProcessProgramDeps(env): if "debug" in env.GetBuildType(): env.ConfigureDebugTarget() - if "test" in env.GetBuildType(): - env.ConfigureTestTarget() # remove specified flags env.ProcessUnFlags(env.get("BUILD_UNFLAGS")) From 284ccc9e8ada1bf3034228b7d4f4b7c9e0355226 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 25 Jun 2022 20:53:08 +0300 Subject: [PATCH 54/93] Remove ProjectAsLibBuilder's singleton --- platformio/builder/tools/piolib.py | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index addc3ce8..32907106 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -332,7 +332,9 @@ class LibBuilderBase: if not LibBuilderBase._INCLUDE_DIRS_CACHE: LibBuilderBase._INCLUDE_DIRS_CACHE = [ self.env.Dir(d) - for d in ProjectAsLibBuilder.get_instance().get_include_dirs() + for d in ProjectAsLibBuilder( + self.envorigin, "$PROJECT_DIR" + ).get_include_dirs() ] for lb in self.env.GetLibBuilders(): LibBuilderBase._INCLUDE_DIRS_CACHE.extend( @@ -859,9 +861,6 @@ class PlatformIOLibBuilder(LibBuilderBase): class ProjectAsLibBuilder(LibBuilderBase): - - _INSTANCE = None - def __init__(self, env, *args, **kwargs): # backup original value, will be reset in base.__init__ project_src_filter = env.get("SRC_FILTER") @@ -870,12 +869,6 @@ class ProjectAsLibBuilder(LibBuilderBase): if "test" in self.env.GetBuildType(): self.env.ConfigureTestTarget() - @classmethod - def get_instance(cls, *args, **kwargs): - if not cls._INSTANCE: - cls._INSTANCE = ProjectAsLibBuilder(*args, **kwargs) - return cls._INSTANCE - @property def include_dir(self): include_dir = self.env.subst("$PROJECT_INCLUDE_DIR") @@ -1138,7 +1131,7 @@ def ConfigureProjectLibBuilder(env): if lb.depbuilders: _print_deps_tree(lb, level + 1) - project = ProjectAsLibBuilder.get_instance(env, "$PROJECT_DIR") + project = ProjectAsLibBuilder(env, "$PROJECT_DIR") env.Export(dict(projenv=project.env)) ldf_mode = LibBuilderBase.lib_ldf_mode.fget(project) # pylint: disable=no-member From 1ea6d471105dddff6aa1a77b10e8905a4cf7afab Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 25 Jun 2022 22:44:35 +0300 Subject: [PATCH 55/93] Initialize unit testing target once per project builder --- platformio/builder/tools/piolib.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index 32907106..e728216e 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -866,8 +866,6 @@ class ProjectAsLibBuilder(LibBuilderBase): project_src_filter = env.get("SRC_FILTER") super().__init__(env, *args, **kwargs) self.env["SRC_FILTER"] = project_src_filter - if "test" in self.env.GetBuildType(): - self.env.ConfigureTestTarget() @property def include_dir(self): @@ -1134,6 +1132,9 @@ def ConfigureProjectLibBuilder(env): project = ProjectAsLibBuilder(env, "$PROJECT_DIR") env.Export(dict(projenv=project.env)) + if "test" in env.GetBuildType(): + project.env.ConfigureTestTarget() + ldf_mode = LibBuilderBase.lib_ldf_mode.fget(project) # pylint: disable=no-member click.echo("LDF: Library Dependency Finder -> https://bit.ly/configure-pio-ldf") From b9a9fd4f436b84096756a815e93caf5b73e989f3 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sun, 26 Jun 2022 14:47:25 +0300 Subject: [PATCH 56/93] Updated "Getting Started" documentation for `GoogleTest` --- HISTORY.rst | 1 + docs | 2 +- examples | 2 +- tests/commands/test_test.py | 20 ++++++++++++++++---- 4 files changed, 19 insertions(+), 6 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 2b307a33..7a910971 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -26,6 +26,7 @@ PlatformIO Core 6 * **Unit Testing** + - Updated "Getting Started" documentation for `GoogleTest `__ testing and mocking framework - Export |UNITTESTING| flags only to the project build environment (``projenv``, files in "src" folder) - Merged the "building" stage with "uploading" for the embedded target (`issue #4307 `_) - Do not resolve dependencies from the project "src" folder when the `test_build_src `__ option is not enabled diff --git a/docs b/docs index 519b5db4..03cb09aa 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 519b5db43440c219e6388e6d64f53850bc6882ca +Subproject commit 03cb09aa4c3541c0367e07842aa20e88ba93863e diff --git a/examples b/examples index 6c52fd32..7376c622 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit 6c52fd327753f2ca14b575bd8719674b479e1181 +Subproject commit 7376c6225bccebf5b5f33d6cc099442fad79b30d diff --git a/tests/commands/test_test.py b/tests/commands/test_test.py index 9d7c1dd5..7965d75b 100644 --- a/tests/commands/test_test.py +++ b/tests/commands/test_test.py @@ -13,6 +13,7 @@ # limitations under the License. import os +import shutil import sys import xml.etree.ElementTree as ET from pathlib import Path @@ -26,12 +27,16 @@ from platformio.test.cli import cli as pio_test_cmd def test_calculator_example(tmp_path: Path): junit_output_path = tmp_path / "junit.xml" + project_dir = tmp_path / "project" + shutil.copytree( + os.path.join("examples", "unit-testing", "calculator"), str(project_dir) + ) result = proc.exec_command( [ "platformio", "test", "-d", - os.path.join("examples", "unit-testing", "calculator"), + str(project_dir), "-e", "uno", "-e", @@ -67,11 +72,15 @@ def test_calculator_example(tmp_path: Path): def test_list_tests(clirunner, validate_cliresult, tmp_path: Path): json_output_path = tmp_path / "report.json" + project_dir = tmp_path / "project" + shutil.copytree( + os.path.join("examples", "unit-testing", "calculator"), str(project_dir) + ) result = clirunner.invoke( pio_test_cmd, [ "-d", - os.path.join("examples", "unit-testing", "calculator"), + str(project_dir), "--list-tests", "--json-output-path", str(json_output_path), @@ -593,13 +602,16 @@ int main(int argc, char **argv) def test_googletest_framework(clirunner, tmp_path: Path): - project_dir = os.path.join("examples", "unit-testing", "googletest") + project_dir = tmp_path / "project" + shutil.copytree( + os.path.join("examples", "unit-testing", "googletest"), str(project_dir) + ) junit_output_path = tmp_path / "junit.xml" result = clirunner.invoke( pio_test_cmd, [ "-d", - project_dir, + str(project_dir), "-e", "native", "--junit-output-path", From ca2622b7a60e4b120435c8bfa93bc1a3c9df15d2 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sun, 26 Jun 2022 16:06:25 +0300 Subject: [PATCH 57/93] Skip GoogleTest for CI on Windows --- HISTORY.rst | 1 - docs | 2 +- examples | 2 +- tests/commands/test_test.py | 4 ++++ 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 7a910971..2b307a33 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -26,7 +26,6 @@ PlatformIO Core 6 * **Unit Testing** - - Updated "Getting Started" documentation for `GoogleTest `__ testing and mocking framework - Export |UNITTESTING| flags only to the project build environment (``projenv``, files in "src" folder) - Merged the "building" stage with "uploading" for the embedded target (`issue #4307 `_) - Do not resolve dependencies from the project "src" folder when the `test_build_src `__ option is not enabled diff --git a/docs b/docs index 03cb09aa..148072a2 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 03cb09aa4c3541c0367e07842aa20e88ba93863e +Subproject commit 148072a28e6b135049b1ee54911a8f58ce7393c9 diff --git a/examples b/examples index 7376c622..b2657021 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit 7376c6225bccebf5b5f33d6cc099442fad79b30d +Subproject commit b26570214771dde4db03f04ff37988eef7effbf8 diff --git a/tests/commands/test_test.py b/tests/commands/test_test.py index 7965d75b..26089500 100644 --- a/tests/commands/test_test.py +++ b/tests/commands/test_test.py @@ -601,6 +601,10 @@ int main(int argc, char **argv) assert json_report["failure_nums"] == 1 +@pytest.mark.skipif( + sys.platform == "win32" and os.environ.get("GITHUB_ACTIONS") == "true", + reason="skip Github Actions on Windows (MinGW issue)", +) def test_googletest_framework(clirunner, tmp_path: Path): project_dir = tmp_path / "project" shutil.copytree( From 6a3b6f0d4476663e4fc539e730a480a60aa32581 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sun, 26 Jun 2022 17:21:09 +0300 Subject: [PATCH 58/93] Bump GoogleTest to ^1.12.0 and Doctest to ^2.4.9 --- platformio/test/runners/doctest.py | 2 +- platformio/test/runners/googletest.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/platformio/test/runners/doctest.py b/platformio/test/runners/doctest.py index 9a6df7cf..2360c4e6 100644 --- a/platformio/test/runners/doctest.py +++ b/platformio/test/runners/doctest.py @@ -102,7 +102,7 @@ class DoctestTestCaseParser: class DoctestTestRunner(TestRunnerBase): - EXTRA_LIB_DEPS = ["doctest/doctest@^2.4.8"] + EXTRA_LIB_DEPS = ["doctest/doctest@^2.4.9"] def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) diff --git a/platformio/test/runners/googletest.py b/platformio/test/runners/googletest.py index 973b16fa..9c8beba2 100644 --- a/platformio/test/runners/googletest.py +++ b/platformio/test/runners/googletest.py @@ -90,7 +90,7 @@ class GoogletestTestCaseParser: class GoogletestTestRunner(TestRunnerBase): - EXTRA_LIB_DEPS = ["google/googletest@^1.11.0"] + EXTRA_LIB_DEPS = ["google/googletest@^1.12.0"] def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) From c42db2ec22315b9dc3e75ad3b752e4f57685e7dc Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sun, 26 Jun 2022 17:32:47 +0300 Subject: [PATCH 59/93] Rename `pio pkg publish --non-interactive` option to the `--no-interactive` --- docs | 2 +- platformio/package/commands/publish.py | 13 ++++++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/docs b/docs index 148072a2..d24f9a64 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 148072a28e6b135049b1ee54911a8f58ce7393c9 +Subproject commit d24f9a649c849448e96fb7ee4aaa3aa6ffa7ba55 diff --git a/platformio/package/commands/publish.py b/platformio/package/commands/publish.py index c1c12a5a..6d2c73ed 100644 --- a/platformio/package/commands/publish.py +++ b/platformio/package/commands/publish.py @@ -71,14 +71,21 @@ def validate_datetime(ctx, param, value): # pylint: disable=unused-argument help="Notify by email when package is processed", ) @click.option( - "--non-interactive", + "--no-interactive", is_flag=True, help="Do not show interactive prompt", ) +@click.option( + "--non-interactive", + is_flag=True, + help="Do not show interactive prompt", + hidden=True, +) def package_publish_cmd( # pylint: disable=too-many-arguments, too-many-locals - package, owner, type_, released_at, private, notify, non_interactive + package, owner, type_, released_at, private, notify, no_interactive, non_interactive ): click.secho("Preparing a package...", fg="cyan") + no_interactive = no_interactive or non_interactive owner = owner or AccountClient().get_logged_username() do_not_pack = ( not os.path.isdir(package) @@ -118,7 +125,7 @@ def package_publish_cmd( # pylint: disable=too-many-arguments, too-many-locals # look for duplicates check_package_duplicates(owner, type_, name, version, manifest.get("system")) - if not non_interactive: + if not no_interactive: click.confirm( "Are you sure you want to publish the %s %s to the registry?\n" % ( From 89a80f158e0acf901c642e267b39dfe305291bfc Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sun, 26 Jun 2022 17:33:22 +0300 Subject: [PATCH 60/93] Bump version to 6.0.3rc2 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index bb90beb5..61b09714 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (6, 0, "3rc1") +VERSION = (6, 0, "3rc2") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 3c17b31d5e402d2b4456550381c54b1dc06083e8 Mon Sep 17 00:00:00 2001 From: Ollie Tedder Date: Mon, 27 Jun 2022 11:24:42 +0100 Subject: [PATCH 61/93] Export twice to maintain backward compatability (#4333) * Export twice to maintain backward compattability * Removed double import and just use another variable (after running make before-commit) * Improve comment --- platformio/util.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/platformio/util.py b/platformio/util.py index 56e95d2f..45af8b12 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -33,6 +33,10 @@ from platformio.proc import exec_command # pylint: enable=unused-import +# also export list_serial_ports as get_serialports to be +# backward compatiblty with arduinosam versions 3.9.0 to 3.5.0 (and possibly others) +get_serialports = get_serial_ports + class memoized(object): def __init__(self, expire=0): From 99b5204802a2cdb7f916268ef63854bb7b2f5e62 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 27 Jun 2022 20:12:04 +0300 Subject: [PATCH 62/93] Fixed a "PermissionError" on Windows when running "clean" or "cleanall" targets // Resolve #4331 --- HISTORY.rst | 1 + platformio/builder/tools/piotarget.py | 13 +++---------- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 2b307a33..22461dad 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -41,6 +41,7 @@ PlatformIO Core 6 * Fixed an issue when the `build_unflags `__ option was not applied to the ``ASPPFLAGS`` scope * Fixed an issue on Windows OS when flags were wrapped to the temporary file while generating the `Compilation database "compile_commands.json" `__ * Fixed an issue with the `LDF `__ when recursively scanning dependencies in the ``chain`` mode +* Fixed a "PermissionError" on Windows when running "clean" or "cleanall" targets (`issue #4331 `_) 6.0.2 (2022-06-01) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/builder/tools/piotarget.py b/platformio/builder/tools/piotarget.py index 7436acaf..2dfdf3d8 100644 --- a/platformio/builder/tools/piotarget.py +++ b/platformio/builder/tools/piotarget.py @@ -43,26 +43,19 @@ def PioClean(env, clean_all=False): def _clean_dir(path): clean_rel_path = _relpath(path) - for root, _, files in os.walk(path): - for f in files: - dst = os.path.join(root, f) - os.remove(dst) - print( - "Removed %s" - % (dst if not clean_rel_path.startswith(".") else _relpath(dst)) - ) + print(f"Removing `{clean_rel_path}` folder...", end="") + fs.rmtree(path) + print(" done!") build_dir = env.subst("$BUILD_DIR") libdeps_dir = env.subst("$PROJECT_LIBDEPS_DIR") if os.path.isdir(build_dir): _clean_dir(build_dir) - fs.rmtree(build_dir) else: print("Build environment is clean") if clean_all and os.path.isdir(libdeps_dir): _clean_dir(libdeps_dir) - fs.rmtree(libdeps_dir) print("Done cleaning") From 32e440bec7f019ac4cf6d2ccf67e805acf07e399 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 28 Jun 2022 12:59:23 +0300 Subject: [PATCH 63/93] Fixed an issue when testing results were wrong in the verbose mode // Resolve #4336 --- HISTORY.rst | 34 ++++++++++++++++----------- platformio/test/runners/doctest.py | 5 ++-- platformio/test/runners/googletest.py | 5 ++-- platformio/test/runners/unity.py | 10 ++++---- 4 files changed, 31 insertions(+), 23 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 22461dad..be4dcdc0 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -16,11 +16,13 @@ PlatformIO Core 6 6.0.3 (2022-??-??) ~~~~~~~~~~~~~~~~~~ -* **Device Monitor** +* **Device Manager** - - Automatically reconnect if a connection fails + - Automatically reconnect device monitor if a connection fails - Added new `pio device monitor --no-reconnect `__ option to disable automatic reconnection - - Handle disconnects more gracefully (`issue #3939 `_) + - Handle device monitor disconnects more gracefully (`issue #3939 `_) + - Improved a serial port finder for `Black Magic Probe `__ (`issue #4023 `_) + - Improved a serial port finder for a board with predefined HWIDs - Replaced ``monitor_flags`` with independent project configuration options: `monitor_parity `__, `monitor_eol `__, `monitor_raw `__, `monitor_echo `__ - Fixed an issue when the monitor filters were not applied in their order (`issue #4320 `_) @@ -30,18 +32,22 @@ PlatformIO Core 6 - Merged the "building" stage with "uploading" for the embedded target (`issue #4307 `_) - Do not resolve dependencies from the project "src" folder when the `test_build_src `__ option is not enabled - Fixed an issue when a custom `pio test --project-config `__ was not handled properly (`issue #4299 `_) + - Fixed an issue when testing results were wrong in the verbose mode (`issue #4336 `_) -* Documented `Stringification `__ – converting a macro argument into a string constant (`issue #4310 `_) -* Added ``env.StringifyMacro(value)`` helper function for the `Advanced Scripting `__ -* Allowed to ``Import("projenv")`` in a library extra script (`issue #4305 `_) -* Improved a serial port finder for `Black Magic Probe `__ (`issue #4023 `_) -* Improved a serial port finder for a board with predefined HWIDs -* Warn about incompatible Bash version for the `Shell Completion `__ (`issue #4326 `_) -* Fixed an issue when the `build_unflags `__ operation ignores a flag value (`issue #4309 `_) -* Fixed an issue when the `build_unflags `__ option was not applied to the ``ASPPFLAGS`` scope -* Fixed an issue on Windows OS when flags were wrapped to the temporary file while generating the `Compilation database "compile_commands.json" `__ -* Fixed an issue with the `LDF `__ when recursively scanning dependencies in the ``chain`` mode -* Fixed a "PermissionError" on Windows when running "clean" or "cleanall" targets (`issue #4331 `_) +* **Build System** + + - Documented `Stringification `__ – converting a macro argument into a string constant (`issue #4310 `_) + - Added ``env.StringifyMacro(value)`` helper function for the `Advanced Scripting `__ + - Allowed to ``Import("projenv")`` in a library extra script (`issue #4305 `_) + - Fixed an issue when the `build_unflags `__ operation ignores a flag value (`issue #4309 `_) + - Fixed an issue when the `build_unflags `__ option was not applied to the ``ASPPFLAGS`` scope + - Fixed an issue on Windows OS when flags were wrapped to the temporary file while generating the `Compilation database "compile_commands.json" `__ + - Fixed an issue with the `LDF `__ when recursively scanning dependencies in the ``chain`` mode + - Fixed a "PermissionError" on Windows when running "clean" or "cleanall" targets (`issue #4331 `_) + +* **Miscellaneous** + + - Warn about incompatible Bash version for the `Shell Completion `__ (`issue #4326 `_) 6.0.2 (2022-06-01) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/test/runners/doctest.py b/platformio/test/runners/doctest.py index 2360c4e6..52a0916f 100644 --- a/platformio/test/runners/doctest.py +++ b/platformio/test/runners/doctest.py @@ -113,9 +113,10 @@ class DoctestTestRunner(TestRunnerBase): click.echo(line, nl=False) test_case = self._tc_parser.parse(line) - if test_case and not self.options.verbose: - click.echo(test_case.humanize()) + if test_case: self.test_suite.add_case(test_case) + if not self.options.verbose: + click.echo(test_case.humanize()) if "[doctest] Status:" in line: self.test_suite.on_finish() diff --git a/platformio/test/runners/googletest.py b/platformio/test/runners/googletest.py index 9c8beba2..5268d72f 100644 --- a/platformio/test/runners/googletest.py +++ b/platformio/test/runners/googletest.py @@ -102,9 +102,10 @@ class GoogletestTestRunner(TestRunnerBase): click.echo(line, nl=False) test_case = self._tc_parser.parse(line) - if test_case and not self.options.verbose: - click.echo(test_case.humanize()) + if test_case: self.test_suite.add_case(test_case) + if not self.options.verbose: + click.echo(test_case.humanize()) if "Global test environment tear-down" in line: self.test_suite.on_finish() diff --git a/platformio/test/runners/unity.py b/platformio/test/runners/unity.py index e048c1ed..39418857 100644 --- a/platformio/test/runners/unity.py +++ b/platformio/test/runners/unity.py @@ -265,8 +265,10 @@ void unityOutputComplete(void) { unittest_uart_end(); } return test_case = self.parse_test_case(line) - if test_case and not self.options.verbose: - click.echo(test_case.humanize()) + if test_case: + self.test_suite.add_case(test_case) + if not self.options.verbose: + click.echo(test_case.humanize()) if all(s in line for s in ("Tests", "Failures", "Ignored")): self.test_suite.on_finish() @@ -286,12 +288,10 @@ void unityOutputComplete(void) { unittest_uart_end(); } source = TestCaseSource( filename=data["source_file"], line=int(data.get("source_line")) ) - test_case = TestCase( + return TestCase( name=data.get("name").strip(), status=TestStatus.from_string(data.get("status")), message=(data.get("message") or "").strip() or None, stdout=line, source=source, ) - self.test_suite.add_case(test_case) - return test_case From 1f096fe03f3e4c1b663924f2c0e5488265e8e446 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 28 Jun 2022 14:28:52 +0300 Subject: [PATCH 64/93] Fixed "non-existent variable ''projenv''" when ESP-IDF is used // Resolve #4329 --- platformio/builder/tools/piolib.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index e728216e..bab464ba 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -333,7 +333,7 @@ class LibBuilderBase: LibBuilderBase._INCLUDE_DIRS_CACHE = [ self.env.Dir(d) for d in ProjectAsLibBuilder( - self.envorigin, "$PROJECT_DIR" + self.envorigin, "$PROJECT_DIR", export_projenv=False ).get_include_dirs() ] for lb in self.env.GetLibBuilders(): @@ -862,10 +862,15 @@ class PlatformIOLibBuilder(LibBuilderBase): class ProjectAsLibBuilder(LibBuilderBase): def __init__(self, env, *args, **kwargs): + export_projenv = kwargs.get("export_projenv", True) + if "export_projenv" in kwargs: + del kwargs["export_projenv"] # backup original value, will be reset in base.__init__ project_src_filter = env.get("SRC_FILTER") super().__init__(env, *args, **kwargs) self.env["SRC_FILTER"] = project_src_filter + if export_projenv: + env.Export(dict(projenv=self.env)) @property def include_dir(self): @@ -1130,7 +1135,6 @@ def ConfigureProjectLibBuilder(env): _print_deps_tree(lb, level + 1) project = ProjectAsLibBuilder(env, "$PROJECT_DIR") - env.Export(dict(projenv=project.env)) if "test" in env.GetBuildType(): project.env.ConfigureTestTarget() From 3363b3a516b70066a4cbe0c26b674cbffda91516 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 28 Jun 2022 19:36:49 +0300 Subject: [PATCH 65/93] Significantly improved support for `Pre & Post Actions` --- HISTORY.rst | 9 ++++-- docs | 2 +- platformio/builder/main.py | 3 ++ platformio/builder/tools/piohooks.py | 42 ++++++++++++++++++++++++++++ tests/commands/test_run.py | 19 +++++++++++-- 5 files changed, 69 insertions(+), 6 deletions(-) create mode 100644 platformio/builder/tools/piohooks.py diff --git a/HISTORY.rst b/HISTORY.rst index be4dcdc0..6b9aacb6 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -13,7 +13,7 @@ PlatformIO Core 6 **A professional collaborative platform for declarative, safety-critical, and test-driven embedded development.** -6.0.3 (2022-??-??) +6.1.0 (2022-??-??) ~~~~~~~~~~~~~~~~~~ * **Device Manager** @@ -36,9 +36,14 @@ PlatformIO Core 6 * **Build System** + - Significantly improved support for `Pre & Post Actions `__ + + * Allowed to declare actions in the `PRE-type scripts `__ even if the target is not ready yet + * Allowed library maintainers to use Pre & Post Actions in the library `extraScript `__ + + - Allowed to ``Import("projenv")`` in a library extra script (`issue #4305 `_) - Documented `Stringification `__ – converting a macro argument into a string constant (`issue #4310 `_) - Added ``env.StringifyMacro(value)`` helper function for the `Advanced Scripting `__ - - Allowed to ``Import("projenv")`` in a library extra script (`issue #4305 `_) - Fixed an issue when the `build_unflags `__ operation ignores a flag value (`issue #4309 `_) - Fixed an issue when the `build_unflags `__ option was not applied to the ``ASPPFLAGS`` scope - Fixed an issue on Windows OS when flags were wrapped to the temporary file while generating the `Compilation database "compile_commands.json" `__ diff --git a/docs b/docs index d24f9a64..54d25a92 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit d24f9a649c849448e96fb7ee4aaa3aa6ffa7ba55 +Subproject commit 54d25a92c921f9e2e000898cfbfcd8637154bff3 diff --git a/platformio/builder/main.py b/platformio/builder/main.py index e46dd079..e2482625 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -53,6 +53,7 @@ DEFAULT_ENV_OPTIONS = dict( "cc", "c++", "link", + "piohooks", "pioasm", "platformio", "pioproject", @@ -221,6 +222,8 @@ env.AddPreAction( AlwaysBuild(env.Alias("__debug", DEFAULT_TARGETS)) AlwaysBuild(env.Alias("__test", DEFAULT_TARGETS)) +env.ProcessDelayedActions() + ############################################################################## if "envdump" in COMMAND_LINE_TARGETS: diff --git a/platformio/builder/tools/piohooks.py b/platformio/builder/tools/piohooks.py new file mode 100644 index 00000000..653a7f30 --- /dev/null +++ b/platformio/builder/tools/piohooks.py @@ -0,0 +1,42 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import + + +def AddActionWrapper(handler): + def wraps(env, files, action): + nodes = env.arg2nodes(files, env.fs.Entry) + unknown_nodes = [node for node in nodes if not node.exists()] + if unknown_nodes: + env.Append(**{"_PIO_DELAYED_ACTIONS": [(handler, unknown_nodes, action)]}) + return handler([node for node in nodes if node.exists()], action) + + return wraps + + +def ProcessDelayedActions(env): + for func, nodes, action in env.get("_PIO_DELAYED_ACTIONS", []): + func(nodes, action) + + +def generate(env): + env.Replace(**{"_PIO_DELAYED_ACTIONS": []}) + env.AddMethod(AddActionWrapper(env.AddPreAction), "AddPreAction") + env.AddMethod(AddActionWrapper(env.AddPostAction), "AddPostAction") + env.AddMethod(ProcessDelayedActions) + + +def exists(_): + return True diff --git a/tests/commands/test_run.py b/tests/commands/test_run.py index 3a2817ba..c91594e2 100644 --- a/tests/commands/test_run.py +++ b/tests/commands/test_run.py @@ -17,7 +17,7 @@ from pathlib import Path from platformio.run.cli import cli as cmd_run -def test_build_flags(clirunner, validate_cliresult, tmpdir): +def test_generic_build(clirunner, validate_cliresult, tmpdir): build_flags = [ ("-D TEST_INT=13", "-DTEST_INT=13"), ("-DTEST_SINGLE_MACRO", "-DTEST_SINGLE_MACRO"), @@ -28,7 +28,9 @@ def test_build_flags(clirunner, validate_cliresult, tmpdir): """ [env:native] platform = native -extra_scripts = extra.py +extra_scripts = + pre:pre_script.py + post_script.py lib_ldf_mode = deep+ build_src_flags = -DI_AM_ONLY_SRC_FLAG build_flags = @@ -38,7 +40,17 @@ build_flags = % " ".join([f[0] for f in build_flags]) ) - tmpdir.join("extra.py").write( + tmpdir.join("pre_script.py").write( + """ +Import("env") + +def post_prog_action(source, target, env): + print("post_prog_action is called") + +env.AddPostAction("$PROG_PATH", post_prog_action) + """ + ) + tmpdir.join("post_script.py").write( """ Import("projenv") @@ -102,6 +114,7 @@ void dummy(void ) {}; result = clirunner.invoke(cmd_run, ["--project-dir", str(tmpdir), "--verbose"]) validate_cliresult(result) + assert "post_prog_action is called" in result.output build_output = result.output[result.output.find("Scanning dependencies...") :] for flag in build_flags: assert flag[1] in build_output, flag From 772e25df49181e90bd39d2b2e8253c33876a7990 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 28 Jun 2022 19:37:16 +0300 Subject: [PATCH 66/93] Bump version to 6.1.0b1 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 61b09714..84a0da80 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (6, 0, "3rc2") +VERSION = (6, 1, "0b1") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From aef49a8bff73be2a3e60bebd6a63590082183ecf Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 29 Jun 2022 14:34:49 +0300 Subject: [PATCH 67/93] Docs: Improve advanced scripting docs --- HISTORY.rst | 2 +- docs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 6b9aacb6..c5ecd654 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -47,7 +47,7 @@ PlatformIO Core 6 - Fixed an issue when the `build_unflags `__ operation ignores a flag value (`issue #4309 `_) - Fixed an issue when the `build_unflags `__ option was not applied to the ``ASPPFLAGS`` scope - Fixed an issue on Windows OS when flags were wrapped to the temporary file while generating the `Compilation database "compile_commands.json" `__ - - Fixed an issue with the `LDF `__ when recursively scanning dependencies in the ``chain`` mode + - Fixed an issue with the |LDF| when recursively scanning dependencies in the ``chain`` mode - Fixed a "PermissionError" on Windows when running "clean" or "cleanall" targets (`issue #4331 `_) * **Miscellaneous** diff --git a/docs b/docs index 54d25a92..bee67b44 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 54d25a92c921f9e2e000898cfbfcd8637154bff3 +Subproject commit bee67b449de38dce2b964a6e254d0ebdba2475bd From 6bec593b9355606b24cea6390e93a879ea92e0a3 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 29 Jun 2022 22:17:30 +0300 Subject: [PATCH 68/93] Simplify output from the clean target --- platformio/builder/tools/piotarget.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/platformio/builder/tools/piotarget.py b/platformio/builder/tools/piotarget.py index 2dfdf3d8..4d7c3a74 100644 --- a/platformio/builder/tools/piotarget.py +++ b/platformio/builder/tools/piotarget.py @@ -43,9 +43,8 @@ def PioClean(env, clean_all=False): def _clean_dir(path): clean_rel_path = _relpath(path) - print(f"Removing `{clean_rel_path}` folder...", end="") + print(f"Removing {clean_rel_path}") fs.rmtree(path) - print(" done!") build_dir = env.subst("$BUILD_DIR") libdeps_dir = env.subst("$PROJECT_LIBDEPS_DIR") From 401f8a4891d546071d4d3a689002edebd3c3b42e Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 29 Jun 2022 22:46:53 +0300 Subject: [PATCH 69/93] Added new "pio run --monitor-port" option to specify custom device monitor port to the "monitor" target // Resolve #4337 --- HISTORY.rst | 5 +++-- docs | 2 +- platformio/run/cli.py | 8 +++++++- platformio/test/exception.py | 2 +- 4 files changed, 12 insertions(+), 5 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index c5ecd654..d17de658 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -41,9 +41,10 @@ PlatformIO Core 6 * Allowed to declare actions in the `PRE-type scripts `__ even if the target is not ready yet * Allowed library maintainers to use Pre & Post Actions in the library `extraScript `__ - - Allowed to ``Import("projenv")`` in a library extra script (`issue #4305 `_) - - Documented `Stringification `__ – converting a macro argument into a string constant (`issue #4310 `_) + - Documented `Stringification `__ – converting a macro argument into a string constant (`issue #4310 `_) + - Added new `pio run --monitor-port `__ option to specify custom device monitor port to the ``monitor`` target (`issue #4337 `_) - Added ``env.StringifyMacro(value)`` helper function for the `Advanced Scripting `__ + - Allowed to ``Import("projenv")`` in a library extra script (`issue #4305 `_) - Fixed an issue when the `build_unflags `__ operation ignores a flag value (`issue #4309 `_) - Fixed an issue when the `build_unflags `__ option was not applied to the ``ASPPFLAGS`` scope - Fixed an issue on Windows OS when flags were wrapped to the temporary file while generating the `Compilation database "compile_commands.json" `__ diff --git a/docs b/docs index bee67b44..b6ae993e 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit bee67b449de38dce2b964a6e254d0ebdba2475bd +Subproject commit b6ae993eda09b83110cf420ca1130182b395a826 diff --git a/platformio/run/cli.py b/platformio/run/cli.py index 29c6812d..eb85b10c 100644 --- a/platformio/run/cli.py +++ b/platformio/run/cli.py @@ -41,6 +41,7 @@ except NotImplementedError: @click.option("-e", "--environment", multiple=True) @click.option("-t", "--target", multiple=True) @click.option("--upload-port") +@click.option("--monitor-port") @click.option( "-d", "--project-dir", @@ -83,6 +84,7 @@ def cli( environment, target, upload_port, + monitor_port, project_dir, project_conf, jobs, @@ -146,6 +148,7 @@ def cli( environment, target, upload_port, + monitor_port, jobs, program_args, is_test_running, @@ -174,6 +177,7 @@ def process_env( environments, targets, upload_port, + monitor_port, jobs, program_args, is_test_running, @@ -207,7 +211,9 @@ def process_env( and "nobuild" not in ep.get_build_targets() ): ctx.invoke( - device_monitor_cmd, environment=environments[0] if environments else None + device_monitor_cmd, + port=monitor_port, + environment=environments[0] if environments else None, ) return result diff --git a/platformio/test/exception.py b/platformio/test/exception.py index 048e7399..12b76ea4 100644 --- a/platformio/test/exception.py +++ b/platformio/test/exception.py @@ -23,7 +23,7 @@ class TestDirNotExistsError(UnitTestError, UserSideException): MESSAGE = ( "A test folder '{0}' does not exist.\nPlease create 'test' " - "directory in the project root and put a test set.\n" + "directory in the project root and put a test suite.\n" "More details about Unit " "Testing: https://docs.platformio.org/en/latest/advanced/" "unit-testing/index.html" From 3b878747f23b2c10aed7ada77ed07fab410c2fe2 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 1 Jul 2022 18:03:48 +0300 Subject: [PATCH 70/93] Bump GoogleTest dependency to 1.12.1 --- platformio/test/runners/googletest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/test/runners/googletest.py b/platformio/test/runners/googletest.py index 5268d72f..b299002e 100644 --- a/platformio/test/runners/googletest.py +++ b/platformio/test/runners/googletest.py @@ -90,7 +90,7 @@ class GoogletestTestCaseParser: class GoogletestTestRunner(TestRunnerBase): - EXTRA_LIB_DEPS = ["google/googletest@^1.12.0"] + EXTRA_LIB_DEPS = ["google/googletest@^1.12.1"] def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) From 1445a91fab5b6d6d9eb532079a58900bd46d6fde Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 1 Jul 2022 18:43:08 +0300 Subject: [PATCH 71/93] Docs: Refactor "Custom firmware/program name" scripting example --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index b6ae993e..c2b86cce 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit b6ae993eda09b83110cf420ca1130182b395a826 +Subproject commit c2b86cce5018dfe260b7c1bdd23a46696b2da752 From d85bc0f7f859b2e6d23ee7119c9423bf04c983b7 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 1 Jul 2022 19:29:07 +0300 Subject: [PATCH 72/93] Make $PROGPATH configurable --- docs | 2 +- platformio/builder/tools/platformio.py | 5 +---- tests/commands/test_run.py | 2 +- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/docs b/docs index c2b86cce..ad5bd94f 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit c2b86cce5018dfe260b7c1bdd23a46696b2da752 +Subproject commit ad5bd94f7ca41235fe183c445bb4680cea1e0f63 diff --git a/platformio/builder/tools/platformio.py b/platformio/builder/tools/platformio.py index c08c7438..b4b0d56a 100644 --- a/platformio/builder/tools/platformio.py +++ b/platformio/builder/tools/platformio.py @@ -75,10 +75,7 @@ def BuildProgram(env): env.Prepend(_LIBFLAGS="-Wl,--start-group ") env.Append(_LIBFLAGS=" -Wl,--end-group") - program = env.Program( - os.path.join("$BUILD_DIR", env.subst("$PROGNAME$PROGSUFFIX")), - env["PIOBUILDFILES"], - ) + program = env.Program("$PROGPATH", env["PIOBUILDFILES"]) env.Replace(PIOMAINPROG=program) AlwaysBuild( diff --git a/tests/commands/test_run.py b/tests/commands/test_run.py index c91594e2..23d1e096 100644 --- a/tests/commands/test_run.py +++ b/tests/commands/test_run.py @@ -47,7 +47,7 @@ Import("env") def post_prog_action(source, target, env): print("post_prog_action is called") -env.AddPostAction("$PROG_PATH", post_prog_action) +env.AddPostAction("$PROGPATH", post_prog_action) """ ) tmpdir.join("post_script.py").write( From e4264a6a51b3bc45d84d69e6e4503fa5a303a93e Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 1 Jul 2022 19:38:51 +0300 Subject: [PATCH 73/93] Make $PROGPATH configurable --- platformio/builder/tools/platformio.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/builder/tools/platformio.py b/platformio/builder/tools/platformio.py index b4b0d56a..04b1b1d7 100644 --- a/platformio/builder/tools/platformio.py +++ b/platformio/builder/tools/platformio.py @@ -75,7 +75,7 @@ def BuildProgram(env): env.Prepend(_LIBFLAGS="-Wl,--start-group ") env.Append(_LIBFLAGS=" -Wl,--end-group") - program = env.Program("$PROGPATH", env["PIOBUILDFILES"]) + program = env.Program(env.subst("$PROGPATH"), env["PIOBUILDFILES"]) env.Replace(PIOMAINPROG=program) AlwaysBuild( From ffebfd437622f4fd55317e8baee63141eb630c80 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 1 Jul 2022 19:44:14 +0300 Subject: [PATCH 74/93] Drop deprecated "program" target --- platformio/builder/main.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/platformio/builder/main.py b/platformio/builder/main.py index e2482625..74f6acd3 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -79,7 +79,8 @@ DEFAULT_ENV_OPTIONS = dict( COMPILATIONDB_PATH=os.path.join("$PROJECT_DIR", "compile_commands.json"), LIBPATH=["$BUILD_DIR"], PROGNAME="program", - PROG_PATH=os.path.join("$BUILD_DIR", "$PROGNAME$PROGSUFFIX"), + PROGPATH=os.path.join("$BUILD_DIR", "$PROGNAME$PROGSUFFIX"), + PROG_PATH="$PROGPATH", # deprecated PYTHONEXE=get_pythonexe_path(), IDE_EXTRA_DATA={}, ) @@ -200,7 +201,7 @@ for item in env.GetExtraScripts("post"): if env.get("SIZETOOL") and not ( set(["nobuild", "sizedata"]) & set(COMMAND_LINE_TARGETS) ): - env.Depends(["upload", "program"], "checkprogsize") + env.Depends("upload", "checkprogsize") # Replace platform's "size" target with our _new_targets = [t for t in DEFAULT_TARGETS if str(t) != "size"] Default(None) @@ -212,7 +213,7 @@ if "compiledb" in COMMAND_LINE_TARGETS: # Print configured protocols env.AddPreAction( - ["upload", "program"], + "upload", env.VerboseAction( lambda source, target, env: env.PrintUploadInfo(), "Configuring upload protocol...", From f01cd7570c7ee223324528adca31aa04d4371615 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 1 Jul 2022 20:14:58 +0300 Subject: [PATCH 75/93] Fix handling of unknown targets when processing pre/post actions --- platformio/builder/tools/piohooks.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/platformio/builder/tools/piohooks.py b/platformio/builder/tools/piohooks.py index 653a7f30..acd43fbf 100644 --- a/platformio/builder/tools/piohooks.py +++ b/platformio/builder/tools/piohooks.py @@ -17,11 +17,21 @@ from __future__ import absolute_import def AddActionWrapper(handler): def wraps(env, files, action): - nodes = env.arg2nodes(files, env.fs.Entry) - unknown_nodes = [node for node in nodes if not node.exists()] - if unknown_nodes: - env.Append(**{"_PIO_DELAYED_ACTIONS": [(handler, unknown_nodes, action)]}) - return handler([node for node in nodes if node.exists()], action) + if not isinstance(files, (list, tuple, set)): + files = [files] + known_nodes = [] + unknown_files = [] + for item in files: + nodes = env.arg2nodes(item, env.fs.Entry) + if nodes and nodes[0].exists(): + known_nodes.extend(nodes) + else: + unknown_files.append(item) + if unknown_files: + env.Append(**{"_PIO_DELAYED_ACTIONS": [(handler, unknown_files, action)]}) + if known_nodes: + return handler(known_nodes, action) + return [] return wraps From 19d518fc4c4585e7ff5570bbcb23a6db6d640ea3 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 2 Jul 2022 18:37:57 +0300 Subject: [PATCH 76/93] Fix PyLint: Consider explicitly re-raising --- .pylintrc | 6 +----- platformio/__main__.py | 18 +++++++++--------- platformio/account/client.py | 4 ++-- platformio/app.py | 14 ++++++++------ platformio/builder/tools/piolib.py | 8 ++++---- platformio/builder/tools/pioplatform.py | 4 ++-- platformio/builder/tools/pioupload.py | 4 ++-- platformio/commands/ci.py | 4 ++-- platformio/commands/upgrade.py | 8 ++++---- platformio/debug/cli.py | 6 +++--- platformio/debug/config/base.py | 4 ++-- platformio/device/list/util.py | 4 ++-- platformio/device/monitor/terminal.py | 6 +++--- platformio/fs.py | 8 ++++---- platformio/home/rpc/handlers/account.py | 6 +++--- platformio/home/rpc/handlers/piocore.py | 12 ++++++------ platformio/http.py | 6 +++--- platformio/package/commands/exec.py | 2 +- platformio/package/commands/pack.py | 4 ++-- platformio/package/commands/publish.py | 4 ++-- platformio/package/lockfile.py | 4 ++-- platformio/package/manager/_download.py | 4 ++-- platformio/package/manager/_install.py | 4 ++-- platformio/package/manager/_registry.py | 4 ++-- platformio/package/manager/base.py | 4 ++-- platformio/package/manager/core.py | 2 +- platformio/package/manager/platform.py | 4 ++-- platformio/package/manifest/parser.py | 10 +++++----- platformio/package/manifest/schema.py | 8 ++++---- platformio/package/vcsclient.py | 16 ++++++++-------- platformio/platform/board.py | 4 ++-- platformio/platform/factory.py | 4 ++-- platformio/proc.py | 4 ++-- platformio/project/commands/init.py | 4 ++-- platformio/project/config.py | 16 ++++++++-------- platformio/registry/client.py | 6 +++--- platformio/remote/ac/base.py | 4 ++-- platformio/remote/client/agent_service.py | 4 ++-- platformio/remote/factory/client.py | 4 ++-- platformio/telemetry.py | 6 +++--- platformio/test/runners/base.py | 8 ++++---- platformio/test/runners/factory.py | 4 ++-- platformio/test/runners/readers/program.py | 4 ++-- platformio/test/runners/readers/serial.py | 4 ++-- tests/package/test_manifest.py | 6 +++--- 45 files changed, 136 insertions(+), 138 deletions(-) diff --git a/.pylintrc b/.pylintrc index 3943bac6..8f80370c 100644 --- a/.pylintrc +++ b/.pylintrc @@ -16,8 +16,4 @@ disable= useless-import-alias, bad-option-value, consider-using-dict-items, - consider-using-f-string, - - ; PY2 Compat - super-with-arguments, - raise-missing-from + consider-using-f-string diff --git a/platformio/__main__.py b/platformio/__main__.py index c7f8e245..6a1cda94 100644 --- a/platformio/__main__.py +++ b/platformio/__main__.py @@ -100,15 +100,15 @@ def main(argv=None): ensure_python3(raise_exception=True) configure() cli() # pylint: disable=no-value-for-parameter - except SystemExit as e: - if e.code and str(e.code).isdigit(): - exit_code = int(e.code) - except Exception as e: # pylint: disable=broad-except - if not isinstance(e, exception.ReturnErrorCode): - maintenance.on_platformio_exception(e) + except SystemExit as exc: + if exc.code and str(exc.code).isdigit(): + exit_code = int(exc.code) + except Exception as exc: # pylint: disable=broad-except + if not isinstance(exc, exception.ReturnErrorCode): + maintenance.on_platformio_exception(exc) error_str = "Error: " - if isinstance(e, exception.PlatformioException): - error_str += str(e) + if isinstance(exc, exception.PlatformioException): + error_str += str(exc) else: error_str += format_exc() error_str += """ @@ -128,7 +128,7 @@ An unexpected error occurred. Further steps: ============================================================ """ click.secho(error_str, fg="red", err=True) - exit_code = int(str(e)) if str(e).isdigit() else 1 + exit_code = int(str(exc)) if str(exc).isdigit() else 1 sys.argv = prev_sys_argv return exit_code diff --git a/platformio/account/client.py b/platformio/account/client.py index a2eb0c28..7aabb24d 100644 --- a/platformio/account/client.py +++ b/platformio/account/client.py @@ -46,8 +46,8 @@ class AccountClient(HTTPClient): # pylint:disable=too-many-public-methods def get_refresh_token(): try: return app.get_state_item("account").get("auth").get("refresh_token") - except: # pylint:disable=bare-except - raise AccountNotAuthorized() + except Exception as exc: + raise AccountNotAuthorized() from exc @staticmethod def delete_local_session(): diff --git a/platformio/app.py b/platformio/app.py index 26b4c80b..c9af8190 100644 --- a/platformio/app.py +++ b/platformio/app.py @@ -103,8 +103,10 @@ class State(object): try: with open(self.path, mode="w", encoding="utf8") as fp: fp.write(json.dumps(self._storage)) - except IOError: - raise exception.HomeDirPermissionsError(os.path.dirname(self.path)) + except IOError as exc: + raise exception.HomeDirPermissionsError( + os.path.dirname(self.path) + ) from exc self._unlock_state_file() def _lock_state_file(self): @@ -113,8 +115,8 @@ class State(object): self._lockfile = LockFile(self.path) try: self._lockfile.acquire() - except IOError: - raise exception.HomeDirPermissionsError(os.path.dirname(self.path)) + except IOError as exc: + raise exception.HomeDirPermissionsError(os.path.dirname(self.path)) from exc def _unlock_state_file(self): if hasattr(self, "_lockfile") and self._lockfile: @@ -169,8 +171,8 @@ def sanitize_setting(name, value): value = str(value).lower() in ("true", "yes", "y", "1") elif isinstance(defdata["value"], int): value = int(value) - except Exception: - raise exception.InvalidSettingValue(value, name) + except Exception as exc: + raise exception.InvalidSettingValue(value, name) from exc return value diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index bab464ba..137692b8 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -360,11 +360,11 @@ class LibBuilderBase: depth=self.CCONDITIONAL_SCANNER_DEPTH, ) - except Exception as e: # pylint: disable=broad-except + except Exception as exc: # pylint: disable=broad-except if self.verbose and "+" in self.lib_ldf_mode: sys.stderr.write( "Warning! Classic Pre Processor is used for `%s`, " - "advanced has failed with `%s`\n" % (path, e) + "advanced has failed with `%s`\n" % (path, exc) ) candidates = self.env.File(path).get_implicit_deps( self.env, @@ -956,8 +956,8 @@ class ProjectAsLibBuilder(LibBuilderBase): try: lm.install(spec) did_install = True - except (HTTPClientError, UnknownPackageError, InternetIsOffline) as e: - click.secho("Warning! %s" % e, fg="yellow") + except (HTTPClientError, UnknownPackageError, InternetIsOffline) as exc: + click.secho("Warning! %s" % exc, fg="yellow") # reset cache if did_install: diff --git a/platformio/builder/tools/pioplatform.py b/platformio/builder/tools/pioplatform.py index c8c2785f..d6e71e58 100644 --- a/platformio/builder/tools/pioplatform.py +++ b/platformio/builder/tools/pioplatform.py @@ -51,8 +51,8 @@ def BoardConfig(env, board=None): board = board or env.get("BOARD") assert board, "BoardConfig: Board is not defined" return p.board_config(board) - except (AssertionError, UnknownBoard) as e: - sys.stderr.write("Error: %s\n" % str(e)) + except (AssertionError, UnknownBoard) as exc: + sys.stderr.write("Error: %s\n" % str(exc)) env.Exit(1) return None diff --git a/platformio/builder/tools/pioupload.py b/platformio/builder/tools/pioupload.py index f85dc735..2229a9fb 100644 --- a/platformio/builder/tools/pioupload.py +++ b/platformio/builder/tools/pioupload.py @@ -109,8 +109,8 @@ def AutodetectUploadPort(*args, **kwargs): else: try: fs.ensure_udev_rules() - except exception.InvalidUdevRules as e: - sys.stderr.write("\n%s\n\n" % e) + except exception.InvalidUdevRules as exc: + sys.stderr.write("\n%s\n\n" % exc) env.Replace( UPLOAD_PORT=find_serial_port( initial_port=initial_port, diff --git a/platformio/commands/ci.py b/platformio/commands/ci.py index f1875b23..9364d1e9 100644 --- a/platformio/commands/ci.py +++ b/platformio/commands/ci.py @@ -39,8 +39,8 @@ def validate_path(ctx, param, value): # pylint: disable=unused-argument try: assert invalid_path is None return value - except AssertionError: - raise click.BadParameter("Found invalid path: %s" % invalid_path) + except AssertionError as exc: + raise click.BadParameter("Found invalid path: %s" % invalid_path) from exc @click.command("ci", short_help="Continuous Integration") diff --git a/platformio/commands/upgrade.py b/platformio/commands/upgrade.py index 7664732a..2766dbd5 100644 --- a/platformio/commands/upgrade.py +++ b/platformio/commands/upgrade.py @@ -71,9 +71,9 @@ def cli(dev): click.secho( "Warning! Please restart IDE to affect PIO Home changes", fg="yellow" ) - except Exception as e: # pylint: disable=broad-except + except Exception as exc: if not r: - raise exception.UpgradeError("\n".join([str(cmd), str(e)])) + raise exception.UpgradeError("\n".join([str(cmd), str(exc)])) from exc permission_errors = ("permission denied", "not permitted") if any(m in r["err"].lower() for m in permission_errors) and not IS_WINDOWS: click.secho( @@ -127,8 +127,8 @@ def get_latest_version(): except: # pylint: disable=bare-except pass return get_pypi_latest_version() - except: - raise exception.GetLatestVersionError() + except Exception as exc: + raise exception.GetLatestVersionError() from exc def get_develop_latest_version(): diff --git a/platformio/debug/cli.py b/platformio/debug/cli.py index 01e81f36..64bd1b43 100644 --- a/platformio/debug/cli.py +++ b/platformio/debug/cli.py @@ -129,11 +129,11 @@ def _debug_in_project_dir( try: fs.ensure_udev_rules() - except exception.InvalidUdevRules as e: + except exception.InvalidUdevRules as exc: click.echo( - helpers.escape_gdbmi_stream("~", str(e) + "\n") + helpers.escape_gdbmi_stream("~", str(exc) + "\n") if helpers.is_gdbmi_mode() - else str(e) + "\n", + else str(exc) + "\n", nl=False, ) diff --git a/platformio/debug/config/base.py b/platformio/debug/config/base.py index 46a91673..a5867340 100644 --- a/platformio/debug/config/base.py +++ b/platformio/debug/config/base.py @@ -203,8 +203,8 @@ class DebugConfigBase: # pylint: disable=too-many-instance-attributes def get_init_script(self, debugger): try: return getattr(self, "%s_INIT_SCRIPT" % debugger.upper()) - except AttributeError: - raise NotImplementedError + except AttributeError as exc: + raise NotImplementedError from exc def reveal_patterns(self, source, recursive=True): program_path = self.program_path or "" diff --git a/platformio/device/list/util.py b/platformio/device/list/util.py index 25387484..e82dc601 100644 --- a/platformio/device/list/util.py +++ b/platformio/device/list/util.py @@ -28,8 +28,8 @@ def list_serial_ports(filter_hwid=False, as_objects=False): try: # pylint: disable=import-outside-toplevel from serial.tools.list_ports import comports - except ImportError: - raise exception.GetSerialPortsError(os.name) + except ImportError as exc: + raise exception.GetSerialPortsError(os.name) from exc if as_objects: return comports() diff --git a/platformio/device/monitor/terminal.py b/platformio/device/monitor/terminal.py index 05c241c9..38c18390 100644 --- a/platformio/device/monitor/terminal.py +++ b/platformio/device/monitor/terminal.py @@ -137,9 +137,9 @@ def new_serial_instance(options): # pylint: disable=too-many-branches if port is None or port == "-": try: port = miniterm.ask_for_port() - except KeyboardInterrupt: + except KeyboardInterrupt as exc: click.echo("", err=True) - raise UserSideException("User aborted and port is not given") + raise UserSideException("User aborted and port is not given") from exc else: if not port: raise UserSideException("Port is not given") @@ -180,6 +180,6 @@ def new_serial_instance(options): # pylint: disable=too-many-branches serial_instance.open() except serial.SerialException as exc: - raise UserSideException(exc) + raise UserSideException(exc) from exc return serial_instance diff --git a/platformio/fs.py b/platformio/fs.py index 29dd118f..b6678cc7 100644 --- a/platformio/fs.py +++ b/platformio/fs.py @@ -54,8 +54,8 @@ def load_json(file_path): try: with open(file_path, mode="r", encoding="utf8") as f: return json.load(f) - except ValueError: - raise exception.InvalidJSONFile(file_path) + except ValueError as exc: + raise exception.InvalidJSONFile(file_path) from exc def humanize_file_size(filesize): @@ -231,9 +231,9 @@ def rmtree(path): if st_mode & stat.S_IREAD: os.chmod(path, st_mode | stat.S_IWRITE) func(path) - except Exception as e: # pylint: disable=broad-except + except Exception as exc: # pylint: disable=broad-except click.secho( - "%s \nPlease manually remove the file `%s`" % (str(e), path), + "%s \nPlease manually remove the file `%s`" % (str(exc), path), fg="red", err=True, ) diff --git a/platformio/home/rpc/handlers/account.py b/platformio/home/rpc/handlers/account.py index 23777437..d857d587 100644 --- a/platformio/home/rpc/handlers/account.py +++ b/platformio/home/rpc/handlers/account.py @@ -23,7 +23,7 @@ class AccountRPC: try: client = AccountClient() return getattr(client, method)(*args, **kwargs) - except Exception as e: # pylint: disable=bare-except + except Exception as exc: # pylint: disable=bare-except raise JSONRPC20DispatchException( - code=4003, message="PIO Account Call Error", data=str(e) - ) + code=4003, message="PIO Account Call Error", data=str(exc) + ) from exc diff --git a/platformio/home/rpc/handlers/piocore.py b/platformio/home/rpc/handlers/piocore.py index add05a31..a1098f3a 100644 --- a/platformio/home/rpc/handlers/piocore.py +++ b/platformio/home/rpc/handlers/piocore.py @@ -94,10 +94,10 @@ class PIOCoreRPC: # fall-back to subprocess method result = await PIOCoreRPC._call_subprocess(args, options) return PIOCoreRPC._process_result(result, to_json) - except Exception as e: # pylint: disable=bare-except + except Exception as exc: # pylint: disable=bare-except raise JSONRPC20DispatchException( - code=4003, message="PIO Core Call Error", data=str(e) - ) + code=4003, message="PIO Core Call Error", data=str(exc) + ) from exc @staticmethod async def _call_subprocess(args, options): @@ -139,8 +139,8 @@ class PIOCoreRPC: return text try: return json.loads(out) - except ValueError as e: - click.secho("%s => `%s`" % (e, out), fg="red", err=True) + except ValueError as exc: + click.secho("%s => `%s`" % (exc, out), fg="red", err=True) # if PIO Core prints unhandled warnings for line in out.split("\n"): line = line.strip() @@ -150,4 +150,4 @@ class PIOCoreRPC: return json.loads(line) except ValueError: pass - raise e + raise exc diff --git a/platformio/http.py b/platformio/http.py index 5f5fbdd2..2b31411f 100644 --- a/platformio/http.py +++ b/platformio/http.py @@ -132,11 +132,11 @@ class HTTPClient(object): except ( requests.exceptions.ConnectionError, requests.exceptions.Timeout, - ) as e: + ) as exc: try: self._next_session() - except: # pylint: disable=bare-except - raise HTTPClientError(str(e)) + except Exception as exc2: + raise HTTPClientError(str(exc2)) from exc def fetch_json_data(self, method, path, **kwargs): if method not in ("get", "head", "options"): diff --git a/platformio/package/commands/exec.py b/platformio/package/commands/exec.py index b5484f73..fc100f19 100644 --- a/platformio/package/commands/exec.py +++ b/platformio/package/commands/exec.py @@ -67,7 +67,7 @@ def package_exec_cmd(obj, package, call, args): if force_click_stream: click.echo(result.stdout.decode().strip(), err=result.returncode != 0) except Exception as exc: - raise UserSideException(exc) + raise UserSideException(exc) from exc if result and result.returncode != 0: raise ReturnErrorCode(result.returncode) diff --git a/platformio/package/commands/pack.py b/platformio/package/commands/pack.py index c80995e4..038ffc15 100644 --- a/platformio/package/commands/pack.py +++ b/platformio/package/commands/pack.py @@ -39,7 +39,7 @@ def package_pack_cmd(package, output): ManifestSchema().load_manifest( ManifestParserFactory.new_from_archive(archive_path).as_dict() ) - except ManifestValidationError as e: + except ManifestValidationError as exc: os.remove(archive_path) - raise e + raise exc click.secho('Wrote a tarball to "%s"' % archive_path, fg="green") diff --git a/platformio/package/commands/publish.py b/platformio/package/commands/publish.py index 6d2c73ed..8d282fa0 100644 --- a/platformio/package/commands/publish.py +++ b/platformio/package/commands/publish.py @@ -36,8 +36,8 @@ def validate_datetime(ctx, param, value): # pylint: disable=unused-argument return value try: datetime.strptime(value, "%Y-%m-%d %H:%M:%S") - except ValueError as e: - raise click.BadParameter(e) + except ValueError as exc: + raise click.BadParameter(exc) return value diff --git a/platformio/package/lockfile.py b/platformio/package/lockfile.py index 296036aa..3a410a60 100644 --- a/platformio/package/lockfile.py +++ b/platformio/package/lockfile.py @@ -72,10 +72,10 @@ class LockFile(object): msvcrt.locking( # pylint: disable=used-before-assignment self._fp.fileno(), msvcrt.LK_NBLCK, 1 ) - except (BlockingIOError, IOError): + except (BlockingIOError, IOError) as exc: self._fp.close() self._fp = None - raise LockFileExists + raise LockFileExists from exc return True def _unlock(self): diff --git a/platformio/package/manager/_download.py b/platformio/package/manager/_download.py index e408908a..0c073e11 100644 --- a/platformio/package/manager/_download.py +++ b/platformio/package/manager/_download.py @@ -70,7 +70,7 @@ class PackageManagerDownloadMixin(object): fd = FileDownloader(url) fd.set_destination(tmp_path) fd.start(with_progress=with_progress, silent=silent) - except IOError as e: + except IOError as exc: raise_error = not with_progress if with_progress: try: @@ -86,7 +86,7 @@ class PackageManagerDownloadMixin(object): fg="red", ) ) - raise e + raise exc if checksum: fd.verify(checksum) os.close(tmp_fd) diff --git a/platformio/package/manager/_install.py b/platformio/package/manager/_install.py index 8129d9b0..a885b68f 100644 --- a/platformio/package/manager/_install.py +++ b/platformio/package/manager/_install.py @@ -36,9 +36,9 @@ class PackageManagerInstallMixin(object): try: with FileUnpacker(src) as fu: return fu.unpack(dst, with_progress=with_progress) - except IOError as e: + except IOError as exc: if not with_progress: - raise e + raise exc with FileUnpacker(src) as fu: return fu.unpack(dst, with_progress=False) diff --git a/platformio/package/manager/_registry.py b/platformio/package/manager/_registry.py index 0a9f39c5..f861ddf4 100644 --- a/platformio/package/manager/_registry.py +++ b/platformio/package/manager/_registry.py @@ -56,9 +56,9 @@ class PackageManagerRegistryMixin(object): ), checksum or pkgfile["checksum"]["sha256"], ) - except Exception as e: # pylint: disable=broad-except + except Exception as exc: # pylint: disable=broad-except self.log.warning( - click.style("Warning! Package Mirror: %s" % e, fg="yellow") + click.style("Warning! Package Mirror: %s" % exc, fg="yellow") ) self.log.warning( click.style("Looking for another mirror...", fg="yellow") diff --git a/platformio/package/manager/base.py b/platformio/package/manager/base.py index b692140f..c90b940f 100644 --- a/platformio/package/manager/base.py +++ b/platformio/package/manager/base.py @@ -187,9 +187,9 @@ class BasePackageManager( # pylint: disable=too-many-public-methods,too-many-in result = ManifestParserFactory.new_from_file(item).as_dict() self.memcache_set(cache_key, result) return result - except ManifestException as e: + except ManifestException as exc: if not PlatformioCLI.in_silence(): - self.log.warning(click.style(str(e), fg="yellow")) + self.log.warning(click.style(str(exc), fg="yellow")) raise MissingPackageManifestError(", ".join(self.manifest_names)) @staticmethod diff --git a/platformio/package/manager/core.py b/platformio/package/manager/core.py index d9a05cb5..28fab07b 100644 --- a/platformio/package/manager/core.py +++ b/platformio/package/manager/core.py @@ -156,7 +156,7 @@ def build_contrib_pysite_package(target_dir, with_metadata=True): raise UserSideException( "\n\nPlease ensure that the next packages are installed:\n\n" "sudo apt install python3-dev libffi-dev libssl-dev\n" - ) + ) from exc raise exc # build manifests diff --git a/platformio/package/manager/platform.py b/platformio/package/manager/platform.py index 41f7b41e..e5e948ba 100644 --- a/platformio/package/manager/platform.py +++ b/platformio/package/manager/platform.py @@ -53,9 +53,9 @@ class PlatformPackageManager(BasePackageManager): # pylint: disable=too-many-an # set logging level for underlying tool manager p.pm.set_log_level(self.log.getEffectiveLevel()) p.ensure_engine_compatible() - except IncompatiblePlatform as e: + except IncompatiblePlatform as exc: super().uninstall(pkg, skip_dependencies=True) - raise e + raise exc if project_env: p.configure_project_packages(project_env, project_targets) if not skip_dependencies: diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py index 8735bcff..8b0c4e87 100644 --- a/platformio/package/manifest/parser.py +++ b/platformio/package/manifest/parser.py @@ -61,9 +61,9 @@ class ManifestParserFactory(object): try: with io.open(path, encoding=encoding) as fp: return fp.read() - except UnicodeDecodeError as e: - last_err = e - raise last_err # pylint: disable=raising-bad-type + except UnicodeDecodeError as exc: + last_err = exc + raise last_err @classmethod def new_from_file(cls, path, remote_url=False): @@ -145,8 +145,8 @@ class BaseManifestParser(object): self.package_dir = package_dir try: self._data = self.parse(contents) - except Exception as e: - raise ManifestParserError("Could not parse manifest -> %s" % e) + except Exception as exc: + raise ManifestParserError("Could not parse manifest -> %s" % exc) from exc self._data = self.normalize_repository(self._data) self._data = self.parse_examples(self._data) diff --git a/platformio/package/manifest/schema.py b/platformio/package/manifest/schema.py index 8258dfe8..8f8a7d16 100644 --- a/platformio/package/manifest/schema.py +++ b/platformio/package/manifest/schema.py @@ -243,17 +243,17 @@ class ManifestSchema(BaseSchema): if "Invalid leading zero" in str(exc): raise exc semantic_version.Version.coerce(value) - except (AssertionError, ValueError): + except (AssertionError, ValueError) as exc: raise ValidationError( "Invalid semantic versioning format, see https://semver.org/" - ) + ) from exc @validates("license") def validate_license(self, value): try: spdx = self.load_spdx_licenses() - except requests.exceptions.RequestException: - raise ValidationError("Could not load SPDX licenses for validation") + except requests.exceptions.RequestException as exc: + raise ValidationError("Could not load SPDX licenses for validation") from exc known_ids = set(item.get("licenseId") for item in spdx.get("licenses", [])) if value in known_ids: return True diff --git a/platformio/package/vcsclient.py b/platformio/package/vcsclient.py index dc4c090d..9299ada9 100644 --- a/platformio/package/vcsclient.py +++ b/platformio/package/vcsclient.py @@ -51,8 +51,8 @@ class VCSClientFactory(object): ) assert isinstance(obj, VCSClientBase) return obj - except (KeyError, AssertionError): - raise VCSBaseException("VCS: Unknown repository type %s" % remote_url) + except (KeyError, AssertionError) as exc: + raise VCSBaseException("VCS: Unknown repository type %s" % remote_url) from exc class VCSClientBase(object): @@ -73,10 +73,10 @@ class VCSClientBase(object): self.get_cmd_output(["--version"]) else: assert self.run_cmd(["--version"]) - except (AssertionError, OSError, PlatformioException): + except (AssertionError, OSError, PlatformioException) as exc: raise UserSideException( "VCS: `%s` client is not installed in your system" % self.command - ) + ) from exc return True @property @@ -108,8 +108,8 @@ class VCSClientBase(object): try: subprocess.check_call(args, **kwargs) return True - except subprocess.CalledProcessError as e: - raise VCSBaseException("VCS: Could not process command %s" % e.cmd) + except subprocess.CalledProcessError as exc: + raise VCSBaseException("VCS: Could not process command %s" % exc.cmd) from exc def get_cmd_output(self, args, **kwargs): args = [self.command] + args @@ -152,10 +152,10 @@ class GitClient(VCSClientBase): def check_client(self): try: return VCSClientBase.check_client(self) - except UserSideException: + except UserSideException as exc: raise UserSideException( "Please install Git client from https://git-scm.com/downloads" - ) + ) from exc def get_branches(self): output = self.get_cmd_output(["branch"]) diff --git a/platformio/platform/board.py b/platformio/platform/board.py index 2e998ac0..fa0cdf9c 100644 --- a/platformio/platform/board.py +++ b/platformio/platform/board.py @@ -28,8 +28,8 @@ class PlatformBoardConfig(object): self.manifest_path = manifest_path try: self._manifest = fs.load_json(manifest_path) - except ValueError: - raise InvalidBoardManifest(manifest_path) + except ValueError as exc: + raise InvalidBoardManifest(manifest_path) from exc if not set(["name", "url", "vendor"]) <= set(self._manifest): raise UserSideException( "Please specify name, url and vendor fields for " + manifest_path diff --git a/platformio/platform/factory.py b/platformio/platform/factory.py index 0345cb2a..72b44790 100644 --- a/platformio/platform/factory.py +++ b/platformio/platform/factory.py @@ -35,8 +35,8 @@ class PlatformFactory(object): sys.modules["platformio.managers.platform"] = base try: return load_python_module("platformio.platform.%s" % name, path) - except ImportError: - raise UnknownPlatform(name) + except ImportError as exc: + raise UnknownPlatform(name) from exc @classmethod def new(cls, pkg_or_spec, autoinstall=False) -> base.PlatformBase: diff --git a/platformio/proc.py b/platformio/proc.py index 83943273..f32fe715 100644 --- a/platformio/proc.py +++ b/platformio/proc.py @@ -115,8 +115,8 @@ def exec_command(*args, **kwargs): try: result["out"], result["err"] = p.communicate() result["returncode"] = p.returncode - except KeyboardInterrupt: - raise exception.AbortedByUser() + except KeyboardInterrupt as exc: + raise exception.AbortedByUser() from exc finally: for s in ("stdout", "stderr"): if isinstance(kwargs[s], AsyncPipeBase): diff --git a/platformio/project/commands/init.py b/platformio/project/commands/init.py index 7c3b77cb..0f3b719b 100644 --- a/platformio/project/commands/init.py +++ b/platformio/project/commands/init.py @@ -34,11 +34,11 @@ def validate_boards(ctx, param, value): # pylint: disable=W0613 for id_ in value: try: pm.board_config(id_) - except UnknownBoard: + except UnknownBoard as exc: raise click.BadParameter( "`%s`. Please search for board ID using `platformio boards` " "command" % id_ - ) + ) from exc return value diff --git a/platformio/project/config.py b/platformio/project/config.py index 999827d5..e9440bc9 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -97,8 +97,8 @@ class ProjectConfigBase(object): self._parsed.append(path) try: self._parser.read(path, "utf-8") - except configparser.Error as e: - raise exception.InvalidProjectConfError(path, str(e)) + except configparser.Error as exc: + raise exception.InvalidProjectConfError(path, str(exc)) if not parse_extra: return @@ -324,10 +324,10 @@ class ProjectConfigBase(object): # handle nested calls try: value = self.get(section, option) - except RecursionError: + except RecursionError as exc: raise exception.ProjectOptionValueError( "Infinite recursion has been detected", option, section - ) + ) from exc if isinstance(value, list): return "\n".join(value) return str(value) @@ -336,8 +336,8 @@ class ProjectConfigBase(object): value = None try: value = self.getraw(section, option, default) - except configparser.Error as e: - raise exception.InvalidProjectConfError(self.path, str(e)) + except configparser.Error as exc: + raise exception.InvalidProjectConfError(self.path, str(exc)) option_meta = self.find_option_meta(section, option) if not option_meta: @@ -349,10 +349,10 @@ class ProjectConfigBase(object): value = self.parse_multi_values(value or []) try: return self.cast_to(value, option_meta.type) - except click.BadParameter as e: + except click.BadParameter as exc: if not self.expand_interpolations: return value - raise exception.ProjectOptionValueError(e.format_message(), option, section) + raise exception.ProjectOptionValueError(exc.format_message(), option, section) @staticmethod def cast_to(value, to_type): diff --git a/platformio/registry/client.py b/platformio/registry/client.py index 2c465e0c..a682db57 100644 --- a/platformio/registry/client.py +++ b/platformio/registry/client.py @@ -153,7 +153,7 @@ class RegistryClient(HTTPClient): x_cache_valid="1h", x_with_authorization=self.allowed_private_packages(), ) - except HTTPClientError as e: - if e.response is not None and e.response.status_code == 404: + except HTTPClientError as exc: + if exc.response is not None and exc.response.status_code == 404: return None - raise e + raise exc diff --git a/platformio/remote/ac/base.py b/platformio/remote/ac/base.py index 7b76a327..e92118c9 100644 --- a/platformio/remote/ac/base.py +++ b/platformio/remote/ac/base.py @@ -30,8 +30,8 @@ class AsyncCommandBase(object): try: self.start() - except Exception as e: - raise pb.Error(str(e)) + except Exception as exc: + raise pb.Error(str(exc)) from exc @property def id(self): diff --git a/platformio/remote/client/agent_service.py b/platformio/remote/client/agent_service.py index ac227795..3a884e6e 100644 --- a/platformio/remote/client/agent_service.py +++ b/platformio/remote/client/agent_service.py @@ -164,8 +164,8 @@ class RemoteAgentService(RemoteClientBase): origin_pio_ini, (os.path.getatime(back_pio_ini), os.path.getmtime(back_pio_ini)), ) - except NotPlatformIOProjectError as e: - raise pb.Error(str(e)) + except NotPlatformIOProjectError as exc: + raise pb.Error(str(exc)) from exc cmd_args = ["platformio", "--force", command, "-d", project_dir] for env in options.get("environment", []): diff --git a/platformio/remote/factory/client.py b/platformio/remote/factory/client.py index 2565e6ad..b9e3cd5a 100644 --- a/platformio/remote/factory/client.py +++ b/platformio/remote/factory/client.py @@ -38,10 +38,10 @@ class RemoteClientFactory(pb.PBClientFactory, protocol.ReconnectingClientFactory auth_token = None try: auth_token = AccountClient().fetch_authentication_token() - except Exception as e: # pylint:disable=broad-except + except Exception as exc: # pylint:disable=broad-except d = defer.Deferred() d.addErrback(self.clientAuthorizationFailed) - d.errback(pb.Error(e)) + d.errback(pb.Error(exc)) return d d = self.login( diff --git a/platformio/telemetry.py b/platformio/telemetry.py index c0fbf97d..8606e48e 100644 --- a/platformio/telemetry.py +++ b/platformio/telemetry.py @@ -274,11 +274,11 @@ class MPDataPusher(object): ) r.raise_for_status() return True - except requests.exceptions.HTTPError as e: + except requests.exceptions.HTTPError as exc: # skip Bad Request - if 400 >= e.response.status_code < 500: + if 400 >= exc.response.status_code < 500: return True - except: # pylint: disable=W0702 + except: # pylint: disable=bare-except pass self._http_offline = True return False diff --git a/platformio/test/runners/base.py b/platformio/test/runners/base.py index 218e0c11..73d3b731 100644 --- a/platformio/test/runners/base.py +++ b/platformio/test/runners/base.py @@ -128,11 +128,11 @@ class TestRunnerBase: targets.append("checkprogsize") try: return self.run_project_targets(targets) - except ReturnErrorCode: + except ReturnErrorCode as exc: raise UnitTestSuiteError( "Building stage has failed, see errors above. " "Use `pio test -vvv` option to enable verbose output." - ) + ) from exc def stage_uploading(self): is_embedded = self.platform.is_embedded() @@ -150,11 +150,11 @@ class TestRunnerBase: targets.append("__debug") try: return self.run_project_targets(targets) - except ReturnErrorCode: + except ReturnErrorCode as exc: raise UnitTestSuiteError( "Uploading stage has failed, see errors above. " "Use `pio test -vvv` option to enable verbose output." - ) + ) from exc def stage_testing(self): if self.options.without_testing: diff --git a/platformio/test/runners/factory.py b/platformio/test/runners/factory.py index 6c428316..64381807 100644 --- a/platformio/test/runners/factory.py +++ b/platformio/test/runners/factory.py @@ -56,11 +56,11 @@ class TestRunnerFactory(object): try: mod = load_python_module(module_name, custom_runner_path) - except (FileNotFoundError, ImportError): + except (FileNotFoundError, ImportError) as exc: raise UserSideException( "Could not find custom test runner " f"by this path -> {custom_runner_path}" - ) + ) from exc else: mod = importlib.import_module(module_name) runner_cls = getattr(mod, cls.get_clsname(test_framework)) diff --git a/platformio/test/runners/readers/program.py b/platformio/test/runners/readers/program.py index b0b33130..82cc6a2a 100644 --- a/platformio/test/runners/readers/program.py +++ b/platformio/test/runners/readers/program.py @@ -108,8 +108,8 @@ class ProgramTestOutputReader: raise UnitTestError( f"Program received signal {sig.name} ({signal_description})" ) - except ValueError: - raise UnitTestError("Program errored with %d code" % return_code) + except ValueError as exc: + raise UnitTestError("Program errored with %d code" % return_code) from exc def begin(self): try: diff --git a/platformio/test/runners/readers/serial.py b/platformio/test/runners/readers/serial.py index e2a7eddb..da4b1a05 100644 --- a/platformio/test/runners/readers/serial.py +++ b/platformio/test/runners/readers/serial.py @@ -45,8 +45,8 @@ class SerialTestOutputReader: ser.rts = self.test_runner.options.monitor_rts ser.dtr = self.test_runner.options.monitor_dtr ser.open() - except serial.SerialException as e: - click.secho(str(e), fg="red", err=True) + except serial.SerialException as exc: + click.secho(str(exc), fg="red", err=True) return None if not self.test_runner.options.no_reset: diff --git a/tests/package/test_manifest.py b/tests/package/test_manifest.py index c16ce04b..828412f4 100644 --- a/tests/package/test_manifest.py +++ b/tests/package/test_manifest.py @@ -531,9 +531,9 @@ includes=MozziGuts.h errors = None try: ManifestSchema().load_manifest(raw_data) - except ManifestValidationError as e: - data = e.valid_data - errors = e.messages + except ManifestValidationError as exc: + data = exc.valid_data + errors = exc.messages assert errors["authors"] From b006f5301094f91595634bbb929b76f7b39a56d8 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 2 Jul 2022 19:03:25 +0300 Subject: [PATCH 77/93] PyLint: Fix "useless-object-inheritance" --- .pylintrc | 8 +------- platformio/app.py | 2 +- platformio/builder/tools/pioino.py | 2 +- platformio/builder/tools/piolib.py | 2 +- platformio/cache.py | 2 +- platformio/check/defect.py | 2 +- platformio/check/tools/__init__.py | 2 +- platformio/check/tools/base.py | 2 +- platformio/debug/config/factory.py | 2 +- platformio/device/list/util.py | 2 +- platformio/fs.py | 2 +- platformio/home/rpc/handlers/piocore.py | 2 +- platformio/http.py | 4 ++-- platformio/maintenance.py | 2 +- platformio/package/download.py | 2 +- platformio/package/lockfile.py | 2 +- platformio/package/manager/_download.py | 2 +- platformio/package/manager/_install.py | 2 +- platformio/package/manager/_legacy.py | 2 +- platformio/package/manager/_registry.py | 2 +- platformio/package/manager/_symlink.py | 2 +- platformio/package/manager/_uninstall.py | 2 +- platformio/package/manager/_update.py | 2 +- platformio/package/manifest/parser.py | 6 +++--- platformio/package/manifest/schema.py | 6 ++++-- platformio/package/meta.py | 10 +++++----- platformio/package/pack.py | 2 +- platformio/package/unpack.py | 4 ++-- platformio/package/vcsclient.py | 12 ++++++++---- platformio/platform/_packages.py | 2 +- platformio/platform/_run.py | 2 +- platformio/platform/board.py | 2 +- platformio/platform/factory.py | 2 +- platformio/proc.py | 2 +- platformio/project/config.py | 8 +++++--- platformio/project/generator.py | 2 +- platformio/project/options.py | 2 +- platformio/registry/mirror.py | 2 +- platformio/remote/ac/base.py | 2 +- platformio/remote/projectsync.py | 2 +- platformio/run/processor.py | 2 +- platformio/telemetry.py | 4 ++-- platformio/test/runners/factory.py | 2 +- platformio/util.py | 6 +++--- 44 files changed, 69 insertions(+), 67 deletions(-) diff --git a/.pylintrc b/.pylintrc index 8f80370c..c2f85aa2 100644 --- a/.pylintrc +++ b/.pylintrc @@ -3,16 +3,10 @@ output-format=colorized [MESSAGES CONTROL] disable= - bad-continuation, - bad-whitespace, missing-docstring, - ungrouped-imports, invalid-name, - cyclic-import, - duplicate-code, - superfluous-parens, too-few-public-methods, - useless-object-inheritance, + superfluous-parens, useless-import-alias, bad-option-value, consider-using-dict-items, diff --git a/platformio/app.py b/platformio/app.py index c9af8190..b2c78bc1 100644 --- a/platformio/app.py +++ b/platformio/app.py @@ -70,7 +70,7 @@ SESSION_VARS = { } -class State(object): +class State: def __init__(self, path=None, lock=False): self.path = path self.lock = lock diff --git a/platformio/builder/tools/pioino.py b/platformio/builder/tools/pioino.py index 0c1f59a2..453d1f51 100644 --- a/platformio/builder/tools/pioino.py +++ b/platformio/builder/tools/pioino.py @@ -26,7 +26,7 @@ import click from platformio.compat import get_filesystem_encoding, get_locale_encoding -class InoToCPPConverter(object): +class InoToCPPConverter: PROTOTYPE_RE = re.compile( r"""^( diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index 137692b8..ea16b3b3 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -46,7 +46,7 @@ from platformio.package.meta import PackageItem from platformio.project.options import ProjectOptions -class LibBuilderFactory(object): +class LibBuilderFactory: @staticmethod def new(env, path, verbose=int(ARGUMENTS.get("PIOVERBOSE", 0))): clsname = "UnknownLibBuilder" diff --git a/platformio/cache.py b/platformio/cache.py index e8b7982d..20901545 100644 --- a/platformio/cache.py +++ b/platformio/cache.py @@ -23,7 +23,7 @@ from platformio.package.lockfile import LockFile from platformio.project.helpers import get_project_cache_dir -class ContentCache(object): +class ContentCache: def __init__(self, namespace=None): self.cache_dir = os.path.join(get_project_cache_dir(), namespace or "content") self._db_path = os.path.join(self.cache_dir, "db.data") diff --git a/platformio/check/defect.py b/platformio/check/defect.py index d271ab48..15f9df70 100644 --- a/platformio/check/defect.py +++ b/platformio/check/defect.py @@ -22,7 +22,7 @@ from platformio.project.helpers import get_project_dir # pylint: disable=too-many-arguments -class DefectItem(object): +class DefectItem: SEVERITY_HIGH = 1 SEVERITY_MEDIUM = 2 diff --git a/platformio/check/tools/__init__.py b/platformio/check/tools/__init__.py index 824df5bb..58a9263d 100644 --- a/platformio/check/tools/__init__.py +++ b/platformio/check/tools/__init__.py @@ -18,7 +18,7 @@ from platformio.check.tools.cppcheck import CppcheckCheckTool from platformio.check.tools.pvsstudio import PvsStudioCheckTool -class CheckToolFactory(object): +class CheckToolFactory: @staticmethod def new(tool, project_dir, config, envname, options): cls = None diff --git a/platformio/check/tools/base.py b/platformio/check/tools/base.py index 90c58bc1..d51b3158 100644 --- a/platformio/check/tools/base.py +++ b/platformio/check/tools/base.py @@ -25,7 +25,7 @@ from platformio.package.meta import PackageSpec from platformio.project.helpers import load_build_metadata -class CheckToolBase(object): # pylint: disable=too-many-instance-attributes +class CheckToolBase: # pylint: disable=too-many-instance-attributes def __init__(self, project_dir, config, envname, options): self.config = config self.envname = envname diff --git a/platformio/debug/config/factory.py b/platformio/debug/config/factory.py index 4741b800..d7358b92 100644 --- a/platformio/debug/config/factory.py +++ b/platformio/debug/config/factory.py @@ -19,7 +19,7 @@ from platformio.debug.config.generic import GenericDebugConfig from platformio.debug.config.native import NativeDebugConfig -class DebugConfigFactory(object): +class DebugConfigFactory: @staticmethod def get_clsname(name): name = re.sub(r"[^\da-z\_\-]+", "", name, flags=re.I) diff --git a/platformio/device/list/util.py b/platformio/device/list/util.py index e82dc601..2c1d0385 100644 --- a/platformio/device/list/util.py +++ b/platformio/device/list/util.py @@ -84,7 +84,7 @@ def list_logical_devices(): def list_mdns_services(): - class mDNSListener(object): + class mDNSListener: def __init__(self): self._zc = zeroconf.Zeroconf(interfaces=zeroconf.InterfaceChoice.All) self._found_types = [] diff --git a/platformio/fs.py b/platformio/fs.py index b6678cc7..60111694 100644 --- a/platformio/fs.py +++ b/platformio/fs.py @@ -28,7 +28,7 @@ from platformio import exception, proc from platformio.compat import IS_WINDOWS -class cd(object): +class cd: def __init__(self, new_path): self.new_path = new_path self.prev_path = os.getcwd() diff --git a/platformio/home/rpc/handlers/piocore.py b/platformio/home/rpc/handlers/piocore.py index a1098f3a..7a0052d4 100644 --- a/platformio/home/rpc/handlers/piocore.py +++ b/platformio/home/rpc/handlers/piocore.py @@ -29,7 +29,7 @@ from platformio.compat import get_locale_encoding, is_bytes from platformio.home import helpers -class MultiThreadingStdStream(object): +class MultiThreadingStdStream: def __init__(self, parent_stream): self._buffers = {threading.get_ident(): parent_stream} diff --git a/platformio/http.py b/platformio/http.py index 2b31411f..ff88a85b 100644 --- a/platformio/http.py +++ b/platformio/http.py @@ -57,7 +57,7 @@ class EndpointSession(requests.Session): return super().request(method, urljoin(self.base_url, url), *args, **kwargs) -class EndpointSessionIterator(object): +class EndpointSessionIterator: def __init__(self, endpoints): if not isinstance(endpoints, list): endpoints = [endpoints] @@ -82,7 +82,7 @@ class EndpointSessionIterator(object): return session -class HTTPClient(object): +class HTTPClient: def __init__(self, endpoints): self._session_iter = EndpointSessionIterator(endpoints) self._session = None diff --git a/platformio/maintenance.py b/platformio/maintenance.py index 925d2f3a..bd9ccd9f 100644 --- a/platformio/maintenance.py +++ b/platformio/maintenance.py @@ -79,7 +79,7 @@ def set_caller(caller=None): return app.set_session_var("caller_id", caller) -class Upgrader(object): +class Upgrader: def __init__(self, from_version, to_version): self.from_version = pepver_to_semver(from_version) self.to_version = pepver_to_semver(to_version) diff --git a/platformio/package/download.py b/platformio/package/download.py index ffc57d50..f8b67ba8 100644 --- a/platformio/package/download.py +++ b/platformio/package/download.py @@ -25,7 +25,7 @@ from platformio import __default_requests_timeout__, app, fs from platformio.package.exception import PackageException -class FileDownloader(object): +class FileDownloader: def __init__(self, url, dest_dir=None): self._request = None # make connection diff --git a/platformio/package/lockfile.py b/platformio/package/lockfile.py index 3a410a60..3c6b2047 100644 --- a/platformio/package/lockfile.py +++ b/platformio/package/lockfile.py @@ -44,7 +44,7 @@ class LockFileTimeoutError(PlatformioException): pass -class LockFile(object): +class LockFile: def __init__(self, path, timeout=LOCKFILE_TIMEOUT, delay=LOCKFILE_DELAY): self.timeout = timeout self.delay = delay diff --git a/platformio/package/manager/_download.py b/platformio/package/manager/_download.py index 0c073e11..9d9c6118 100644 --- a/platformio/package/manager/_download.py +++ b/platformio/package/manager/_download.py @@ -25,7 +25,7 @@ from platformio.package.download import FileDownloader from platformio.package.lockfile import LockFile -class PackageManagerDownloadMixin(object): +class PackageManagerDownloadMixin: DOWNLOAD_CACHE_EXPIRE = 86400 * 30 # keep package in a local cache for 1 month diff --git a/platformio/package/manager/_install.py b/platformio/package/manager/_install.py index a885b68f..ae442bd0 100644 --- a/platformio/package/manager/_install.py +++ b/platformio/package/manager/_install.py @@ -26,7 +26,7 @@ from platformio.package.unpack import FileUnpacker from platformio.package.vcsclient import VCSClientFactory -class PackageManagerInstallMixin(object): +class PackageManagerInstallMixin: _INSTALL_HISTORY = None # avoid circle dependencies diff --git a/platformio/package/manager/_legacy.py b/platformio/package/manager/_legacy.py index 978efc9c..4b02a1df 100644 --- a/platformio/package/manager/_legacy.py +++ b/platformio/package/manager/_legacy.py @@ -18,7 +18,7 @@ from platformio import fs from platformio.package.meta import PackageItem, PackageSpec -class PackageManagerLegacyMixin(object): +class PackageManagerLegacyMixin: def build_legacy_spec(self, pkg_dir): # find src manifest src_manifest_name = ".piopkgmanager.json" diff --git a/platformio/package/manager/_registry.py b/platformio/package/manager/_registry.py index f861ddf4..4015cffb 100644 --- a/platformio/package/manager/_registry.py +++ b/platformio/package/manager/_registry.py @@ -23,7 +23,7 @@ from platformio.registry.client import RegistryClient from platformio.registry.mirror import RegistryFileMirrorIterator -class PackageManagerRegistryMixin(object): +class PackageManagerRegistryMixin: def install_from_registry(self, spec, search_qualifiers=None): if spec.owner and spec.name and not search_qualifiers: package = self.fetch_registry_package(spec) diff --git a/platformio/package/manager/_symlink.py b/platformio/package/manager/_symlink.py index 8c5eae38..33ff03f1 100644 --- a/platformio/package/manager/_symlink.py +++ b/platformio/package/manager/_symlink.py @@ -20,7 +20,7 @@ from platformio.package.exception import PackageException from platformio.package.meta import PackageItem, PackageSpec -class PackageManagerSymlinkMixin(object): +class PackageManagerSymlinkMixin: @staticmethod def is_symlink(path): return path and path.endswith(".pio-link") and os.path.isfile(path) diff --git a/platformio/package/manager/_uninstall.py b/platformio/package/manager/_uninstall.py index 9c6b5772..c845145a 100644 --- a/platformio/package/manager/_uninstall.py +++ b/platformio/package/manager/_uninstall.py @@ -22,7 +22,7 @@ from platformio.package.exception import UnknownPackageError from platformio.package.meta import PackageItem, PackageSpec -class PackageManagerUninstallMixin(object): +class PackageManagerUninstallMixin: def uninstall(self, spec, skip_dependencies=False): try: self.lock() diff --git a/platformio/package/manager/_update.py b/platformio/package/manager/_update.py index 5d689ba6..353e9e45 100644 --- a/platformio/package/manager/_update.py +++ b/platformio/package/manager/_update.py @@ -21,7 +21,7 @@ from platformio.package.meta import PackageItem, PackageOutdatedResult, PackageS from platformio.package.vcsclient import VCSBaseException, VCSClientFactory -class PackageManagerUpdateMixin(object): +class PackageManagerUpdateMixin: def outdated(self, pkg, spec=None): assert isinstance(pkg, PackageItem) assert pkg.metadata diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py index 8b0c4e87..a70af7ea 100644 --- a/platformio/package/manifest/parser.py +++ b/platformio/package/manifest/parser.py @@ -27,7 +27,7 @@ from platformio.package.exception import ManifestParserError, UnknownManifestErr from platformio.project.helpers import is_platformio_project -class ManifestFileType(object): +class ManifestFileType: PLATFORM_JSON = "platform.json" LIBRARY_JSON = "library.json" LIBRARY_PROPERTIES = "library.properties" @@ -53,7 +53,7 @@ class ManifestFileType(object): return None -class ManifestParserFactory(object): +class ManifestParserFactory: @staticmethod def read_manifest_contents(path): last_err = None @@ -139,7 +139,7 @@ class ManifestParserFactory(object): raise UnknownManifestError("Unknown manifest file type %s" % type) -class BaseManifestParser(object): +class BaseManifestParser: def __init__(self, contents, remote_url=None, package_dir=None): self.remote_url = remote_url self.package_dir = package_dir diff --git a/platformio/package/manifest/schema.py b/platformio/package/manifest/schema.py index 8f8a7d16..d9eeba14 100644 --- a/platformio/package/manifest/schema.py +++ b/platformio/package/manifest/schema.py @@ -28,7 +28,7 @@ from platformio.util import memoized class BaseSchema(Schema): - class Meta(object): # pylint: disable=no-init + class Meta: # pylint: disable=no-init unknown = marshmallow.EXCLUDE # pylint: disable=no-member def load_manifest(self, data): @@ -253,7 +253,9 @@ class ManifestSchema(BaseSchema): try: spdx = self.load_spdx_licenses() except requests.exceptions.RequestException as exc: - raise ValidationError("Could not load SPDX licenses for validation") from exc + raise ValidationError( + "Could not load SPDX licenses for validation" + ) from exc known_ids = set(item.get("licenseId") for item in spdx.get("licenses", [])) if value in known_ids: return True diff --git a/platformio/package/meta.py b/platformio/package/meta.py index 20199527..34475a9c 100644 --- a/platformio/package/meta.py +++ b/platformio/package/meta.py @@ -27,7 +27,7 @@ from platformio.package.manifest.parser import ManifestFileType from platformio.package.version import cast_version_to_semver -class PackageType(object): +class PackageType: LIBRARY = "library" PLATFORM = "platform" TOOL = "tool" @@ -63,7 +63,7 @@ class PackageType(object): return None -class PackageOutdatedResult(object): +class PackageOutdatedResult: UPDATE_INCREMENT_MAJOR = "major" UPDATE_INCREMENT_MINOR = "minor" UPDATE_INCREMENT_PATCH = "patch" @@ -122,7 +122,7 @@ class PackageOutdatedResult(object): return True -class PackageSpec(object): # pylint: disable=too-many-instance-attributes +class PackageSpec: # pylint: disable=too-many-instance-attributes def __init__( # pylint: disable=redefined-builtin,too-many-arguments self, raw=None, owner=None, id=None, name=None, requirements=None, uri=None ): @@ -358,7 +358,7 @@ class PackageSpec(object): # pylint: disable=too-many-instance-attributes return name -class PackageMetaData(object): +class PackageMetaData: def __init__( # pylint: disable=redefined-builtin self, type, name, version, spec=None ): @@ -426,7 +426,7 @@ class PackageMetaData(object): return PackageMetaData(**data) -class PackageItem(object): +class PackageItem: METAFILE_NAME = ".piopm" diff --git a/platformio/package/pack.py b/platformio/package/pack.py index ed9a9a21..0170ee3d 100644 --- a/platformio/package/pack.py +++ b/platformio/package/pack.py @@ -32,7 +32,7 @@ from platformio.package.meta import PackageItem from platformio.package.unpack import FileUnpacker -class PackagePacker(object): +class PackagePacker: INCLUDE_DEFAULT = list(ManifestFileType.items().values()) + [ "README", "README.md", diff --git a/platformio/package/unpack.py b/platformio/package/unpack.py index d8544a25..e39222c2 100644 --- a/platformio/package/unpack.py +++ b/platformio/package/unpack.py @@ -31,7 +31,7 @@ class ExtractArchiveItemError(PackageException): ) -class BaseArchiver(object): +class BaseArchiver: def __init__(self, arhfileobj): self._afo = arhfileobj @@ -129,7 +129,7 @@ class ZIPArchiver(BaseArchiver): self.preserve_mtime(item, dest_dir) -class FileUnpacker(object): +class FileUnpacker: def __init__(self, path): self.path = path self._archiver = None diff --git a/platformio/package/vcsclient.py b/platformio/package/vcsclient.py index 9299ada9..b4dafc03 100644 --- a/platformio/package/vcsclient.py +++ b/platformio/package/vcsclient.py @@ -29,7 +29,7 @@ class VCSBaseException(PackageException): pass -class VCSClientFactory(object): +class VCSClientFactory: @staticmethod def new(src_dir, remote_url, silent=False): result = urlparse(remote_url) @@ -52,10 +52,12 @@ class VCSClientFactory(object): assert isinstance(obj, VCSClientBase) return obj except (KeyError, AssertionError) as exc: - raise VCSBaseException("VCS: Unknown repository type %s" % remote_url) from exc + raise VCSBaseException( + "VCS: Unknown repository type %s" % remote_url + ) from exc -class VCSClientBase(object): +class VCSClientBase: command = None @@ -109,7 +111,9 @@ class VCSClientBase(object): subprocess.check_call(args, **kwargs) return True except subprocess.CalledProcessError as exc: - raise VCSBaseException("VCS: Could not process command %s" % exc.cmd) from exc + raise VCSBaseException( + "VCS: Could not process command %s" % exc.cmd + ) from exc def get_cmd_output(self, args, **kwargs): args = [self.command] + args diff --git a/platformio/platform/_packages.py b/platformio/platform/_packages.py index c741c790..2a3aa490 100644 --- a/platformio/platform/_packages.py +++ b/platformio/platform/_packages.py @@ -15,7 +15,7 @@ from platformio.package.meta import PackageSpec -class PlatformPackagesMixin(object): +class PlatformPackagesMixin: def get_package_spec(self, name, version=None): return PackageSpec( owner=self.packages[name].get("owner"), diff --git a/platformio/platform/_run.py b/platformio/platform/_run.py index 5bac13ae..79afff94 100644 --- a/platformio/platform/_run.py +++ b/platformio/platform/_run.py @@ -27,7 +27,7 @@ from platformio.package.manager.core import get_core_package_dir from platformio.platform.exception import BuildScriptNotFound -class PlatformRunMixin(object): +class PlatformRunMixin: LINE_ERROR_RE = re.compile(r"(^|\s+)error:?\s+", re.I) diff --git a/platformio/platform/board.py b/platformio/platform/board.py index fa0cdf9c..73a3ebd7 100644 --- a/platformio/platform/board.py +++ b/platformio/platform/board.py @@ -21,7 +21,7 @@ from platformio.exception import UserSideException from platformio.platform.exception import InvalidBoardManifest -class PlatformBoardConfig(object): +class PlatformBoardConfig: def __init__(self, manifest_path): self._id = os.path.basename(manifest_path)[:-5] assert os.path.isfile(manifest_path) diff --git a/platformio/platform/factory.py b/platformio/platform/factory.py index 72b44790..b628590c 100644 --- a/platformio/platform/factory.py +++ b/platformio/platform/factory.py @@ -23,7 +23,7 @@ from platformio.platform import base from platformio.platform.exception import UnknownPlatform -class PlatformFactory(object): +class PlatformFactory: @staticmethod def get_clsname(name): name = re.sub(r"[^\da-z\_]+", "", name, flags=re.I) diff --git a/platformio/proc.py b/platformio/proc.py index f32fe715..fb66bc7f 100644 --- a/platformio/proc.py +++ b/platformio/proc.py @@ -27,7 +27,7 @@ from platformio.compat import ( ) -class AsyncPipeBase(object): +class AsyncPipeBase: def __init__(self): self._fd_read, self._fd_write = os.pipe() self._pipe_reader = os.fdopen( diff --git a/platformio/project/config.py b/platformio/project/config.py index e9440bc9..3443863a 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -38,7 +38,7 @@ CONFIG_HEADER = """ """ -class ProjectConfigBase(object): +class ProjectConfigBase: INLINE_COMMENT_RE = re.compile(r"\s+;.*$") VARTPL_RE = re.compile(r"\$\{([^\.\}\()]+)\.([^\}]+)\}") @@ -352,7 +352,9 @@ class ProjectConfigBase(object): except click.BadParameter as exc: if not self.expand_interpolations: return value - raise exception.ProjectOptionValueError(exc.format_message(), option, section) + raise exception.ProjectOptionValueError( + exc.format_message(), option, section + ) @staticmethod def cast_to(value, to_type): @@ -394,7 +396,7 @@ class ProjectConfigBase(object): return True -class ProjectConfigDirsMixin(object): +class ProjectConfigDirsMixin: def get_optional_dir(self, name): """ Deprecated, used by platformio-node-helpers.project.observer.fetchLibDirs diff --git a/platformio/project/generator.py b/platformio/project/generator.py index 10efba7d..cca74dbf 100644 --- a/platformio/project/generator.py +++ b/platformio/project/generator.py @@ -23,7 +23,7 @@ from platformio.proc import where_is_program from platformio.project.helpers import load_build_metadata -class ProjectGenerator(object): +class ProjectGenerator: def __init__(self, config, env_name, ide, board_ids=None): self.config = config self.project_dir = os.path.dirname(config.path) diff --git a/platformio/project/options.py b/platformio/project/options.py index 21a9a130..4c29b925 100644 --- a/platformio/project/options.py +++ b/platformio/project/options.py @@ -24,7 +24,7 @@ from platformio import fs from platformio.compat import IS_WINDOWS, hashlib_encode_data -class ConfigOption(object): # pylint: disable=too-many-instance-attributes +class ConfigOption: # pylint: disable=too-many-instance-attributes def __init__( self, scope, diff --git a/platformio/registry/mirror.py b/platformio/registry/mirror.py index d967838e..4235e88d 100644 --- a/platformio/registry/mirror.py +++ b/platformio/registry/mirror.py @@ -21,7 +21,7 @@ from platformio.http import HTTPClient from platformio.registry.client import RegistryClient -class RegistryFileMirrorIterator(object): +class RegistryFileMirrorIterator: HTTP_CLIENT_INSTANCES = {} diff --git a/platformio/remote/ac/base.py b/platformio/remote/ac/base.py index e92118c9..8105062c 100644 --- a/platformio/remote/ac/base.py +++ b/platformio/remote/ac/base.py @@ -16,7 +16,7 @@ from twisted.internet import defer # pylint: disable=import-error from twisted.spread import pb # pylint: disable=import-error -class AsyncCommandBase(object): +class AsyncCommandBase: MAX_BUFFER_SIZE = 1024 * 1024 # 1Mb diff --git a/platformio/remote/projectsync.py b/platformio/remote/projectsync.py index 867922bd..820034ae 100644 --- a/platformio/remote/projectsync.py +++ b/platformio/remote/projectsync.py @@ -31,7 +31,7 @@ class PROJECT_SYNC_STAGE(constants.Flags): COMPLETED = constants.FlagConstant() -class ProjectSync(object): +class ProjectSync: def __init__(self, path): self.path = path if not isdir(self.path): diff --git a/platformio/run/processor.py b/platformio/run/processor.py index 4751d352..00a96f02 100644 --- a/platformio/run/processor.py +++ b/platformio/run/processor.py @@ -20,7 +20,7 @@ from platformio.test.runners.base import CTX_META_TEST_RUNNING_NAME # pylint: disable=too-many-instance-attributes -class EnvironmentProcessor(object): +class EnvironmentProcessor: def __init__( # pylint: disable=too-many-arguments self, cmd_ctx, diff --git a/platformio/telemetry.py b/platformio/telemetry.py index 8606e48e..a3aded5a 100644 --- a/platformio/telemetry.py +++ b/platformio/telemetry.py @@ -34,7 +34,7 @@ from platformio.proc import is_ci, is_container from platformio.project.helpers import is_platformio_project -class TelemetryBase(object): +class TelemetryBase: def __init__(self): self._params = {} @@ -198,7 +198,7 @@ class MeasurementProtocol(TelemetryBase): @util.singleton -class MPDataPusher(object): +class MPDataPusher: MAX_WORKERS = 5 diff --git a/platformio/test/runners/factory.py b/platformio/test/runners/factory.py index 64381807..ecb6f884 100644 --- a/platformio/test/runners/factory.py +++ b/platformio/test/runners/factory.py @@ -23,7 +23,7 @@ from platformio.test.result import TestSuite from platformio.test.runners.base import TestRunnerBase, TestRunnerOptions -class TestRunnerFactory(object): +class TestRunnerFactory: @staticmethod def get_clsname(name): name = re.sub(r"[^\da-z\_\-]+", "", name, flags=re.I) diff --git a/platformio/util.py b/platformio/util.py index 45af8b12..5a8b1923 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -38,7 +38,7 @@ from platformio.proc import exec_command get_serialports = get_serial_ports -class memoized(object): +class memoized: def __init__(self, expire=0): expire = str(expire) if expire.isdigit(): @@ -65,7 +65,7 @@ class memoized(object): self.cache.clear() -class throttle(object): +class throttle: def __init__(self, threshhold): self.threshhold = threshhold # milliseconds self.last = 0 @@ -97,7 +97,7 @@ class RetryStopException(RetryException): pass -class retry(object): +class retry: RetryNextException = RetryNextException RetryStopException = RetryStopException From 96fb8c74f9a6dbd1ee42bac645680ce002c36052 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 2 Jul 2022 19:05:32 +0300 Subject: [PATCH 78/93] PyLint: fix "superfluous-parens" --- .pylintrc | 1 - platformio/home/rpc/handlers/project.py | 2 +- tests/commands/test_lib_complex.py | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.pylintrc b/.pylintrc index c2f85aa2..bf49cf40 100644 --- a/.pylintrc +++ b/.pylintrc @@ -6,7 +6,6 @@ disable= missing-docstring, invalid-name, too-few-public-methods, - superfluous-parens, useless-import-alias, bad-option-value, consider-using-dict-items, diff --git a/platformio/home/rpc/handlers/project.py b/platformio/home/rpc/handlers/project.py index 13a9ebbb..2ca3496b 100644 --- a/platformio/home/rpc/handlers/project.py +++ b/platformio/home/rpc/handlers/project.py @@ -247,7 +247,7 @@ class ProjectRPC: if not isinstance(platforms, list): platforms = [platforms] c_based_platforms = ["intel_mcs51", "ststm8"] - is_cpp_project = not (set(platforms) & set(c_based_platforms)) + is_cpp_project = not set(platforms) & set(c_based_platforms) except exception.PlatformioException: pass diff --git a/tests/commands/test_lib_complex.py b/tests/commands/test_lib_complex.py index 6ae585a2..eb56c5b7 100644 --- a/tests/commands/test_lib_complex.py +++ b/tests/commands/test_lib_complex.py @@ -272,7 +272,7 @@ def test_global_lib_uninstall(clirunner, validate_cliresult, isolated_pio_core): items = sorted(items, key=lambda item: item["__pkg_dir"]) result = clirunner.invoke(cmd_lib, ["-g", "uninstall", items[0]["__pkg_dir"]]) validate_cliresult(result) - assert ("Removing %s" % items[0]["name"]) in strip_ansi_codes(result.output) + assert "Removing %s" % items[0]["name"] in strip_ansi_codes(result.output) # uninstall the rest libraries result = clirunner.invoke( From 029e66cd06a5048d53cea8cde2f2d949be063b44 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 2 Jul 2022 19:19:48 +0300 Subject: [PATCH 79/93] PyLint fixes --- .pylintrc | 3 --- platformio/account/team/commands/list.py | 4 ++-- platformio/builder/tools/piolib.py | 3 +-- platformio/check/tools/clangtidy.py | 4 ++-- platformio/package/manifest/schema.py | 4 ++-- platformio/remote/client/device_monitor.py | 2 +- platformio/test/helpers.py | 6 +++--- platformio/test/runners/base.py | 4 ++-- 8 files changed, 13 insertions(+), 17 deletions(-) diff --git a/.pylintrc b/.pylintrc index bf49cf40..4236ced1 100644 --- a/.pylintrc +++ b/.pylintrc @@ -6,7 +6,4 @@ disable= missing-docstring, invalid-name, too-few-public-methods, - useless-import-alias, - bad-option-value, - consider-using-dict-items, consider-using-f-string diff --git a/platformio/account/team/commands/list.py b/platformio/account/team/commands/list.py index e41e872d..d0395d7c 100644 --- a/platformio/account/team/commands/list.py +++ b/platformio/account/team/commands/list.py @@ -37,8 +37,8 @@ def team_list_cmd(orgname, json_output): return click.echo(json.dumps(data[orgname] if orgname else data)) if not any(data.values()): return click.secho("You do not have any teams.", fg="yellow") - for org_name in data: - for team in data[org_name]: + for org_name, teams in data.items(): + for team in teams: click.echo() click.secho("%s:%s" % (org_name, team.get("name")), fg="cyan") click.echo("-" * len("%s:%s" % (org_name, team.get("name")))) diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index ea16b3b3..9059a25e 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -12,9 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -# pylint: disable=no-self-use, unused-argument, too-many-lines # pylint: disable=too-many-instance-attributes, too-many-public-methods -# pylint: disable=assignment-from-no-return +# pylint: disable=assignment-from-no-return, unused-argument, too-many-lines from __future__ import absolute_import diff --git a/platformio/check/tools/clangtidy.py b/platformio/check/tools/clangtidy.py index 682a2cb7..1dd4165e 100644 --- a/platformio/check/tools/clangtidy.py +++ b/platformio/check/tools/clangtidy.py @@ -67,8 +67,8 @@ class ClangtidyCheckTool(CheckToolBase): project_files = self.get_project_target_files(self.options["patterns"]) src_files = [] - for scope in project_files: - src_files.extend(project_files[scope]) + for items in project_files.values(): + src_files.extend(items) cmd.extend(flags + src_files + ["--"]) cmd.extend( diff --git a/platformio/package/manifest/schema.py b/platformio/package/manifest/schema.py index d9eeba14..5b74033c 100644 --- a/platformio/package/manifest/schema.py +++ b/platformio/package/manifest/schema.py @@ -28,7 +28,7 @@ from platformio.util import memoized class BaseSchema(Schema): - class Meta: # pylint: disable=no-init + class Meta: unknown = marshmallow.EXCLUDE # pylint: disable=no-member def load_manifest(self, data): @@ -232,7 +232,7 @@ class ManifestSchema(BaseSchema): ) @validates("version") - def validate_version(self, value): # pylint: disable=no-self-use + def validate_version(self, value): try: value = str(value) assert "." in value diff --git a/platformio/remote/client/device_monitor.py b/platformio/remote/client/device_monitor.py index 7ddc0048..46ca19e4 100644 --- a/platformio/remote/client/device_monitor.py +++ b/platformio/remote/client/device_monitor.py @@ -22,7 +22,7 @@ from twisted.spread import pb # pylint: disable=import-error from platformio.remote.client.base import RemoteClientBase -class SMBridgeProtocol(protocol.Protocol): # pylint: disable=no-init +class SMBridgeProtocol(protocol.Protocol): def connectionMade(self): self.factory.add_client(self) diff --git a/platformio/test/helpers.py b/platformio/test/helpers.py index a789a6ea..d4364ee5 100644 --- a/platformio/test/helpers.py +++ b/platformio/test/helpers.py @@ -42,10 +42,10 @@ def list_test_suites(project_config, environments, filters, ignores): # filter and ignore patterns patterns = dict(filter=list(filters), ignore=list(ignores)) - for key in patterns: - if patterns[key]: # overriden from CLI + for key, value in patterns.items(): + if value: # overriden from CLI continue - patterns[key].extend( + patterns[key].extend( # pylint: disable=unnecessary-dict-index-lookup project_config.get(f"env:{env_name}", f"test_{key}", []) ) diff --git a/platformio/test/runners/base.py b/platformio/test/runners/base.py index 73d3b731..8f9136f1 100644 --- a/platformio/test/runners/base.py +++ b/platformio/test/runners/base.py @@ -194,7 +194,7 @@ class TestRunnerBase: target=targets, ) - def configure_build_env(self, env): # pylint: disable=no-self-use + def configure_build_env(self, env): """ Configure SCons build environment Called in "builder/tools/piotest" tool @@ -212,5 +212,5 @@ class TestRunnerBase: self._testing_output_buffer = self._testing_output_buffer[nl_pos + 1 :] self.on_testing_line_output(line) - def on_testing_line_output(self, line): # pylint: disable=no-self-use + def on_testing_line_output(self, line): click.echo(line, nl=False) From 2a5de4396449b1e2367af150bc1a53e8c5c1cae5 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 2 Jul 2022 20:16:56 +0300 Subject: [PATCH 80/93] Refactor project IDE integration --- platformio/home/rpc/handlers/project.py | 2 +- platformio/project/commands/init.py | 2 +- platformio/project/integration/__init__.py | 13 +++++++++++++ platformio/project/{ => integration}/generator.py | 10 +++++----- .../{ => integration}/tpls/atom/.clang_complete.tpl | 0 .../{ => integration}/tpls/atom/.gcc-flags.json.tpl | 0 .../{ => integration}/tpls/atom/.gitignore.tpl | 0 .../{ => integration}/tpls/clion/.gitignore.tpl | 0 .../{ => integration}/tpls/clion/CMakeLists.txt.tpl | 0 .../tpls/clion/CMakeListsPrivate.txt.tpl | 0 .../tpls/codeblocks/platformio.cbp.tpl | 0 .../{ => integration}/tpls/eclipse/.cproject.tpl | 0 .../{ => integration}/tpls/eclipse/.project.tpl | 0 .../.settings/PlatformIO Debugger.launch.tpl | 0 .../eclipse/.settings/language.settings.xml.tpl | 0 .../.settings/org.eclipse.cdt.core.prefs.tpl | 0 .../project/{ => integration}/tpls/emacs/.ccls.tpl | 0 .../{ => integration}/tpls/emacs/.gitignore.tpl | 0 .../tpls/netbeans/nbproject/configurations.xml.tpl | 0 .../nbproject/private/configurations.xml.tpl | 0 .../nbproject/private/launcher.properties.tpl | 0 .../tpls/netbeans/nbproject/private/private.xml.tpl | 0 .../tpls/netbeans/nbproject/project.xml.tpl | 0 .../{ => integration}/tpls/qtcreator/.gitignore.tpl | 0 .../{ => integration}/tpls/qtcreator/Makefile.tpl | 0 .../tpls/qtcreator/platformio.cflags.tpl | 0 .../tpls/qtcreator/platformio.config.tpl | 0 .../tpls/qtcreator/platformio.creator.tpl | 0 .../tpls/qtcreator/platformio.cxxflags.tpl | 0 .../tpls/qtcreator/platformio.files.tpl | 0 .../tpls/qtcreator/platformio.includes.tpl | 0 .../{ => integration}/tpls/sublimetext/.ccls.tpl | 0 .../tpls/sublimetext/platformio.sublime-project.tpl | 0 .../project/{ => integration}/tpls/vim/.ccls.tpl | 0 .../{ => integration}/tpls/vim/.gitignore.tpl | 0 .../visualstudio/platformio.vcxproj.filters.tpl | 0 .../tpls/visualstudio/platformio.vcxproj.tpl | 0 .../{ => integration}/tpls/vscode/.gitignore.tpl | 0 .../tpls/vscode/.vscode/c_cpp_properties.json.tpl | 0 .../tpls/vscode/.vscode/extensions.json.tpl | 0 .../tpls/vscode/.vscode/launch.json.tpl | 0 41 files changed, 20 insertions(+), 7 deletions(-) create mode 100644 platformio/project/integration/__init__.py rename platformio/project/{ => integration}/generator.py (95%) rename platformio/project/{ => integration}/tpls/atom/.clang_complete.tpl (100%) rename platformio/project/{ => integration}/tpls/atom/.gcc-flags.json.tpl (100%) rename platformio/project/{ => integration}/tpls/atom/.gitignore.tpl (100%) rename platformio/project/{ => integration}/tpls/clion/.gitignore.tpl (100%) rename platformio/project/{ => integration}/tpls/clion/CMakeLists.txt.tpl (100%) rename platformio/project/{ => integration}/tpls/clion/CMakeListsPrivate.txt.tpl (100%) rename platformio/project/{ => integration}/tpls/codeblocks/platformio.cbp.tpl (100%) rename platformio/project/{ => integration}/tpls/eclipse/.cproject.tpl (100%) rename platformio/project/{ => integration}/tpls/eclipse/.project.tpl (100%) rename platformio/project/{ => integration}/tpls/eclipse/.settings/PlatformIO Debugger.launch.tpl (100%) rename platformio/project/{ => integration}/tpls/eclipse/.settings/language.settings.xml.tpl (100%) rename platformio/project/{ => integration}/tpls/eclipse/.settings/org.eclipse.cdt.core.prefs.tpl (100%) rename platformio/project/{ => integration}/tpls/emacs/.ccls.tpl (100%) rename platformio/project/{ => integration}/tpls/emacs/.gitignore.tpl (100%) rename platformio/project/{ => integration}/tpls/netbeans/nbproject/configurations.xml.tpl (100%) rename platformio/project/{ => integration}/tpls/netbeans/nbproject/private/configurations.xml.tpl (100%) rename platformio/project/{ => integration}/tpls/netbeans/nbproject/private/launcher.properties.tpl (100%) rename platformio/project/{ => integration}/tpls/netbeans/nbproject/private/private.xml.tpl (100%) rename platformio/project/{ => integration}/tpls/netbeans/nbproject/project.xml.tpl (100%) rename platformio/project/{ => integration}/tpls/qtcreator/.gitignore.tpl (100%) rename platformio/project/{ => integration}/tpls/qtcreator/Makefile.tpl (100%) rename platformio/project/{ => integration}/tpls/qtcreator/platformio.cflags.tpl (100%) rename platformio/project/{ => integration}/tpls/qtcreator/platformio.config.tpl (100%) rename platformio/project/{ => integration}/tpls/qtcreator/platformio.creator.tpl (100%) rename platformio/project/{ => integration}/tpls/qtcreator/platformio.cxxflags.tpl (100%) rename platformio/project/{ => integration}/tpls/qtcreator/platformio.files.tpl (100%) rename platformio/project/{ => integration}/tpls/qtcreator/platformio.includes.tpl (100%) rename platformio/project/{ => integration}/tpls/sublimetext/.ccls.tpl (100%) rename platformio/project/{ => integration}/tpls/sublimetext/platformio.sublime-project.tpl (100%) rename platformio/project/{ => integration}/tpls/vim/.ccls.tpl (100%) rename platformio/project/{ => integration}/tpls/vim/.gitignore.tpl (100%) rename platformio/project/{ => integration}/tpls/visualstudio/platformio.vcxproj.filters.tpl (100%) rename platformio/project/{ => integration}/tpls/visualstudio/platformio.vcxproj.tpl (100%) rename platformio/project/{ => integration}/tpls/vscode/.gitignore.tpl (100%) rename platformio/project/{ => integration}/tpls/vscode/.vscode/c_cpp_properties.json.tpl (100%) rename platformio/project/{ => integration}/tpls/vscode/.vscode/extensions.json.tpl (100%) rename platformio/project/{ => integration}/tpls/vscode/.vscode/launch.json.tpl (100%) diff --git a/platformio/home/rpc/handlers/project.py b/platformio/home/rpc/handlers/project.py index 2ca3496b..146629ca 100644 --- a/platformio/home/rpc/handlers/project.py +++ b/platformio/home/rpc/handlers/project.py @@ -26,8 +26,8 @@ from platformio.home.rpc.handlers.piocore import PIOCoreRPC from platformio.package.manager.platform import PlatformPackageManager from platformio.project.config import ProjectConfig from platformio.project.exception import ProjectError -from platformio.project.generator import ProjectGenerator from platformio.project.helpers import get_project_dir, is_platformio_project +from platformio.project.integration.generator import ProjectGenerator from platformio.project.options import get_config_options_schema diff --git a/platformio/project/commands/init.py b/platformio/project/commands/init.py index 0f3b719b..baae29ec 100644 --- a/platformio/project/commands/init.py +++ b/platformio/project/commands/init.py @@ -25,8 +25,8 @@ from platformio.package.commands.install import install_project_dependencies from platformio.package.manager.platform import PlatformPackageManager from platformio.platform.exception import UnknownBoard from platformio.project.config import ProjectConfig -from platformio.project.generator import ProjectGenerator from platformio.project.helpers import is_platformio_project +from platformio.project.integration.generator import ProjectGenerator def validate_boards(ctx, param, value): # pylint: disable=W0613 diff --git a/platformio/project/integration/__init__.py b/platformio/project/integration/__init__.py new file mode 100644 index 00000000..b0514903 --- /dev/null +++ b/platformio/project/integration/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/project/generator.py b/platformio/project/integration/generator.py similarity index 95% rename from platformio/project/generator.py rename to platformio/project/integration/generator.py index cca74dbf..7ffd05e3 100644 --- a/platformio/project/generator.py +++ b/platformio/project/integration/generator.py @@ -15,6 +15,7 @@ import codecs import os import sys +from pathlib import Path import bottle @@ -51,12 +52,11 @@ class ProjectGenerator: @staticmethod def get_supported_ides(): - tpls_dir = os.path.join(fs.get_source_dir(), "project", "tpls") return sorted( [ - d - for d in os.listdir(tpls_dir) - if os.path.isdir(os.path.join(tpls_dir, d)) + item.name + for item in (Path(__file__).parent / "tpls").iterdir() + if item.is_dir() ] ) @@ -132,7 +132,7 @@ class ProjectGenerator: def get_tpls(self): tpls = [] - tpls_dir = os.path.join(fs.get_source_dir(), "project", "tpls", self.ide) + tpls_dir = str(Path(__file__).parent / "tpls" / self.ide) for root, _, files in os.walk(tpls_dir): for f in files: if not f.endswith(".tpl"): diff --git a/platformio/project/tpls/atom/.clang_complete.tpl b/platformio/project/integration/tpls/atom/.clang_complete.tpl similarity index 100% rename from platformio/project/tpls/atom/.clang_complete.tpl rename to platformio/project/integration/tpls/atom/.clang_complete.tpl diff --git a/platformio/project/tpls/atom/.gcc-flags.json.tpl b/platformio/project/integration/tpls/atom/.gcc-flags.json.tpl similarity index 100% rename from platformio/project/tpls/atom/.gcc-flags.json.tpl rename to platformio/project/integration/tpls/atom/.gcc-flags.json.tpl diff --git a/platformio/project/tpls/atom/.gitignore.tpl b/platformio/project/integration/tpls/atom/.gitignore.tpl similarity index 100% rename from platformio/project/tpls/atom/.gitignore.tpl rename to platformio/project/integration/tpls/atom/.gitignore.tpl diff --git a/platformio/project/tpls/clion/.gitignore.tpl b/platformio/project/integration/tpls/clion/.gitignore.tpl similarity index 100% rename from platformio/project/tpls/clion/.gitignore.tpl rename to platformio/project/integration/tpls/clion/.gitignore.tpl diff --git a/platformio/project/tpls/clion/CMakeLists.txt.tpl b/platformio/project/integration/tpls/clion/CMakeLists.txt.tpl similarity index 100% rename from platformio/project/tpls/clion/CMakeLists.txt.tpl rename to platformio/project/integration/tpls/clion/CMakeLists.txt.tpl diff --git a/platformio/project/tpls/clion/CMakeListsPrivate.txt.tpl b/platformio/project/integration/tpls/clion/CMakeListsPrivate.txt.tpl similarity index 100% rename from platformio/project/tpls/clion/CMakeListsPrivate.txt.tpl rename to platformio/project/integration/tpls/clion/CMakeListsPrivate.txt.tpl diff --git a/platformio/project/tpls/codeblocks/platformio.cbp.tpl b/platformio/project/integration/tpls/codeblocks/platformio.cbp.tpl similarity index 100% rename from platformio/project/tpls/codeblocks/platformio.cbp.tpl rename to platformio/project/integration/tpls/codeblocks/platformio.cbp.tpl diff --git a/platformio/project/tpls/eclipse/.cproject.tpl b/platformio/project/integration/tpls/eclipse/.cproject.tpl similarity index 100% rename from platformio/project/tpls/eclipse/.cproject.tpl rename to platformio/project/integration/tpls/eclipse/.cproject.tpl diff --git a/platformio/project/tpls/eclipse/.project.tpl b/platformio/project/integration/tpls/eclipse/.project.tpl similarity index 100% rename from platformio/project/tpls/eclipse/.project.tpl rename to platformio/project/integration/tpls/eclipse/.project.tpl diff --git a/platformio/project/tpls/eclipse/.settings/PlatformIO Debugger.launch.tpl b/platformio/project/integration/tpls/eclipse/.settings/PlatformIO Debugger.launch.tpl similarity index 100% rename from platformio/project/tpls/eclipse/.settings/PlatformIO Debugger.launch.tpl rename to platformio/project/integration/tpls/eclipse/.settings/PlatformIO Debugger.launch.tpl diff --git a/platformio/project/tpls/eclipse/.settings/language.settings.xml.tpl b/platformio/project/integration/tpls/eclipse/.settings/language.settings.xml.tpl similarity index 100% rename from platformio/project/tpls/eclipse/.settings/language.settings.xml.tpl rename to platformio/project/integration/tpls/eclipse/.settings/language.settings.xml.tpl diff --git a/platformio/project/tpls/eclipse/.settings/org.eclipse.cdt.core.prefs.tpl b/platformio/project/integration/tpls/eclipse/.settings/org.eclipse.cdt.core.prefs.tpl similarity index 100% rename from platformio/project/tpls/eclipse/.settings/org.eclipse.cdt.core.prefs.tpl rename to platformio/project/integration/tpls/eclipse/.settings/org.eclipse.cdt.core.prefs.tpl diff --git a/platformio/project/tpls/emacs/.ccls.tpl b/platformio/project/integration/tpls/emacs/.ccls.tpl similarity index 100% rename from platformio/project/tpls/emacs/.ccls.tpl rename to platformio/project/integration/tpls/emacs/.ccls.tpl diff --git a/platformio/project/tpls/emacs/.gitignore.tpl b/platformio/project/integration/tpls/emacs/.gitignore.tpl similarity index 100% rename from platformio/project/tpls/emacs/.gitignore.tpl rename to platformio/project/integration/tpls/emacs/.gitignore.tpl diff --git a/platformio/project/tpls/netbeans/nbproject/configurations.xml.tpl b/platformio/project/integration/tpls/netbeans/nbproject/configurations.xml.tpl similarity index 100% rename from platformio/project/tpls/netbeans/nbproject/configurations.xml.tpl rename to platformio/project/integration/tpls/netbeans/nbproject/configurations.xml.tpl diff --git a/platformio/project/tpls/netbeans/nbproject/private/configurations.xml.tpl b/platformio/project/integration/tpls/netbeans/nbproject/private/configurations.xml.tpl similarity index 100% rename from platformio/project/tpls/netbeans/nbproject/private/configurations.xml.tpl rename to platformio/project/integration/tpls/netbeans/nbproject/private/configurations.xml.tpl diff --git a/platformio/project/tpls/netbeans/nbproject/private/launcher.properties.tpl b/platformio/project/integration/tpls/netbeans/nbproject/private/launcher.properties.tpl similarity index 100% rename from platformio/project/tpls/netbeans/nbproject/private/launcher.properties.tpl rename to platformio/project/integration/tpls/netbeans/nbproject/private/launcher.properties.tpl diff --git a/platformio/project/tpls/netbeans/nbproject/private/private.xml.tpl b/platformio/project/integration/tpls/netbeans/nbproject/private/private.xml.tpl similarity index 100% rename from platformio/project/tpls/netbeans/nbproject/private/private.xml.tpl rename to platformio/project/integration/tpls/netbeans/nbproject/private/private.xml.tpl diff --git a/platformio/project/tpls/netbeans/nbproject/project.xml.tpl b/platformio/project/integration/tpls/netbeans/nbproject/project.xml.tpl similarity index 100% rename from platformio/project/tpls/netbeans/nbproject/project.xml.tpl rename to platformio/project/integration/tpls/netbeans/nbproject/project.xml.tpl diff --git a/platformio/project/tpls/qtcreator/.gitignore.tpl b/platformio/project/integration/tpls/qtcreator/.gitignore.tpl similarity index 100% rename from platformio/project/tpls/qtcreator/.gitignore.tpl rename to platformio/project/integration/tpls/qtcreator/.gitignore.tpl diff --git a/platformio/project/tpls/qtcreator/Makefile.tpl b/platformio/project/integration/tpls/qtcreator/Makefile.tpl similarity index 100% rename from platformio/project/tpls/qtcreator/Makefile.tpl rename to platformio/project/integration/tpls/qtcreator/Makefile.tpl diff --git a/platformio/project/tpls/qtcreator/platformio.cflags.tpl b/platformio/project/integration/tpls/qtcreator/platformio.cflags.tpl similarity index 100% rename from platformio/project/tpls/qtcreator/platformio.cflags.tpl rename to platformio/project/integration/tpls/qtcreator/platformio.cflags.tpl diff --git a/platformio/project/tpls/qtcreator/platformio.config.tpl b/platformio/project/integration/tpls/qtcreator/platformio.config.tpl similarity index 100% rename from platformio/project/tpls/qtcreator/platformio.config.tpl rename to platformio/project/integration/tpls/qtcreator/platformio.config.tpl diff --git a/platformio/project/tpls/qtcreator/platformio.creator.tpl b/platformio/project/integration/tpls/qtcreator/platformio.creator.tpl similarity index 100% rename from platformio/project/tpls/qtcreator/platformio.creator.tpl rename to platformio/project/integration/tpls/qtcreator/platformio.creator.tpl diff --git a/platformio/project/tpls/qtcreator/platformio.cxxflags.tpl b/platformio/project/integration/tpls/qtcreator/platformio.cxxflags.tpl similarity index 100% rename from platformio/project/tpls/qtcreator/platformio.cxxflags.tpl rename to platformio/project/integration/tpls/qtcreator/platformio.cxxflags.tpl diff --git a/platformio/project/tpls/qtcreator/platformio.files.tpl b/platformio/project/integration/tpls/qtcreator/platformio.files.tpl similarity index 100% rename from platformio/project/tpls/qtcreator/platformio.files.tpl rename to platformio/project/integration/tpls/qtcreator/platformio.files.tpl diff --git a/platformio/project/tpls/qtcreator/platformio.includes.tpl b/platformio/project/integration/tpls/qtcreator/platformio.includes.tpl similarity index 100% rename from platformio/project/tpls/qtcreator/platformio.includes.tpl rename to platformio/project/integration/tpls/qtcreator/platformio.includes.tpl diff --git a/platformio/project/tpls/sublimetext/.ccls.tpl b/platformio/project/integration/tpls/sublimetext/.ccls.tpl similarity index 100% rename from platformio/project/tpls/sublimetext/.ccls.tpl rename to platformio/project/integration/tpls/sublimetext/.ccls.tpl diff --git a/platformio/project/tpls/sublimetext/platformio.sublime-project.tpl b/platformio/project/integration/tpls/sublimetext/platformio.sublime-project.tpl similarity index 100% rename from platformio/project/tpls/sublimetext/platformio.sublime-project.tpl rename to platformio/project/integration/tpls/sublimetext/platformio.sublime-project.tpl diff --git a/platformio/project/tpls/vim/.ccls.tpl b/platformio/project/integration/tpls/vim/.ccls.tpl similarity index 100% rename from platformio/project/tpls/vim/.ccls.tpl rename to platformio/project/integration/tpls/vim/.ccls.tpl diff --git a/platformio/project/tpls/vim/.gitignore.tpl b/platformio/project/integration/tpls/vim/.gitignore.tpl similarity index 100% rename from platformio/project/tpls/vim/.gitignore.tpl rename to platformio/project/integration/tpls/vim/.gitignore.tpl diff --git a/platformio/project/tpls/visualstudio/platformio.vcxproj.filters.tpl b/platformio/project/integration/tpls/visualstudio/platformio.vcxproj.filters.tpl similarity index 100% rename from platformio/project/tpls/visualstudio/platformio.vcxproj.filters.tpl rename to platformio/project/integration/tpls/visualstudio/platformio.vcxproj.filters.tpl diff --git a/platformio/project/tpls/visualstudio/platformio.vcxproj.tpl b/platformio/project/integration/tpls/visualstudio/platformio.vcxproj.tpl similarity index 100% rename from platformio/project/tpls/visualstudio/platformio.vcxproj.tpl rename to platformio/project/integration/tpls/visualstudio/platformio.vcxproj.tpl diff --git a/platformio/project/tpls/vscode/.gitignore.tpl b/platformio/project/integration/tpls/vscode/.gitignore.tpl similarity index 100% rename from platformio/project/tpls/vscode/.gitignore.tpl rename to platformio/project/integration/tpls/vscode/.gitignore.tpl diff --git a/platformio/project/tpls/vscode/.vscode/c_cpp_properties.json.tpl b/platformio/project/integration/tpls/vscode/.vscode/c_cpp_properties.json.tpl similarity index 100% rename from platformio/project/tpls/vscode/.vscode/c_cpp_properties.json.tpl rename to platformio/project/integration/tpls/vscode/.vscode/c_cpp_properties.json.tpl diff --git a/platformio/project/tpls/vscode/.vscode/extensions.json.tpl b/platformio/project/integration/tpls/vscode/.vscode/extensions.json.tpl similarity index 100% rename from platformio/project/tpls/vscode/.vscode/extensions.json.tpl rename to platformio/project/integration/tpls/vscode/.vscode/extensions.json.tpl diff --git a/platformio/project/tpls/vscode/.vscode/launch.json.tpl b/platformio/project/integration/tpls/vscode/.vscode/launch.json.tpl similarity index 100% rename from platformio/project/tpls/vscode/.vscode/launch.json.tpl rename to platformio/project/integration/tpls/vscode/.vscode/launch.json.tpl From d2e27f538531fbfb9a4e8d527313cd5bec21cfda Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 2 Jul 2022 20:17:31 +0300 Subject: [PATCH 81/93] Better wording with dependency resolving --- platformio/package/commands/install.py | 4 +--- platformio/package/commands/list.py | 4 +--- platformio/package/commands/uninstall.py | 4 +--- platformio/package/commands/update.py | 4 +--- 4 files changed, 4 insertions(+), 12 deletions(-) diff --git a/platformio/package/commands/install.py b/platformio/package/commands/install.py index 8a6e3017..4aa6ec5d 100644 --- a/platformio/package/commands/install.py +++ b/platformio/package/commands/install.py @@ -100,9 +100,7 @@ def install_project_dependencies(options): if environments and env not in environments: continue if not options.get("silent"): - click.echo( - "Resolving %s environment packages..." % click.style(env, fg="cyan") - ) + click.echo("Resolving %s dependencies..." % click.style(env, fg="cyan")) already_up_to_date = not install_project_env_dependencies(env, options) if not options.get("silent") and already_up_to_date: click.secho("Already up-to-date.", fg="green") diff --git a/platformio/package/commands/list.py b/platformio/package/commands/list.py index 578ecd20..c92cc1b4 100644 --- a/platformio/package/commands/list.py +++ b/platformio/package/commands/list.py @@ -163,9 +163,7 @@ def list_project_packages(options): for env in config.envs(): if environments and env not in environments: continue - click.echo( - "Resolving %s environment packages..." % click.style(env, fg="cyan") - ) + click.echo("Resolving %s dependencies..." % click.style(env, fg="cyan")) found = False if not only_packages or only_platform_packages: _found = print_project_env_platform_packages(env, options) diff --git a/platformio/package/commands/uninstall.py b/platformio/package/commands/uninstall.py index 5fff8160..e393d2a8 100644 --- a/platformio/package/commands/uninstall.py +++ b/platformio/package/commands/uninstall.py @@ -92,9 +92,7 @@ def uninstall_project_dependencies(options): if environments and env not in environments: continue if not options["silent"]: - click.echo( - "Resolving %s environment packages..." % click.style(env, fg="cyan") - ) + click.echo("Resolving %s dependencies..." % click.style(env, fg="cyan")) already_up_to_date = not uninstall_project_env_dependencies(env, options) if not options["silent"] and already_up_to_date: click.secho("Already up-to-date.", fg="green") diff --git a/platformio/package/commands/update.py b/platformio/package/commands/update.py index a520b7b3..67fc5dd1 100644 --- a/platformio/package/commands/update.py +++ b/platformio/package/commands/update.py @@ -95,9 +95,7 @@ def update_project_dependencies(options): if environments and env not in environments: continue if not options["silent"]: - click.echo( - "Resolving %s environment packages..." % click.style(env, fg="cyan") - ) + click.echo("Resolving %s dependencies..." % click.style(env, fg="cyan")) already_up_to_date = not update_project_env_dependencies(env, options) if not options["silent"] and already_up_to_date: click.secho("Already up-to-date.", fg="green") From 6c71a3bea29ee023eb7acc4ad5df3e1efa63ed46 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 2 Jul 2022 20:18:26 +0300 Subject: [PATCH 82/93] Disable core linting with Python 3.6 --- .github/workflows/core.yml | 1 + .pylintrc | 4 +++- Makefile | 4 ++-- tox.ini | 4 ++-- 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml index df920e4e..e566e02c 100644 --- a/.github/workflows/core.yml +++ b/.github/workflows/core.yml @@ -33,6 +33,7 @@ jobs: pip install tox - name: Python Lint + if: ${{ matrix.python-version != '3.6' }} run: | tox -e lint diff --git a/.pylintrc b/.pylintrc index 4236ced1..0b9ff1f8 100644 --- a/.pylintrc +++ b/.pylintrc @@ -4,6 +4,8 @@ output-format=colorized [MESSAGES CONTROL] disable= missing-docstring, + duplicate-code, invalid-name, too-few-public-methods, - consider-using-f-string + consider-using-f-string, + cyclic-import diff --git a/Makefile b/Makefile index 8b280334..09c0b838 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ lint: - pylint -j 6 --rcfile=./.pylintrc ./tests - pylint -j 6 --rcfile=./.pylintrc ./platformio + pylint --rcfile=./.pylintrc ./tests + pylint --rcfile=./.pylintrc ./platformio isort: isort ./platformio diff --git a/tox.ini b/tox.ini index 92555cf5..8332f8d3 100644 --- a/tox.ini +++ b/tox.ini @@ -41,7 +41,8 @@ commands = [testenv:lint] commands = {envpython} --version - pylint --rcfile=./.pylintrc ./platformio ./tests + pylint --rcfile=./.pylintrc ./platformio + pylint --rcfile=./.pylintrc ./tests [testenv:testcore] commands = @@ -54,7 +55,6 @@ commands = py.test -v --basetemp="{envtmpdir}" tests/test_examples.py [testenv:docs] -; basepython = ~/.pyenv/versions/3.6.12/bin/python deps = sphinx sphinx_rtd_theme From d3b7508bd55e4a8b62e6ce70d3d9cb4cf1770dc1 Mon Sep 17 00:00:00 2001 From: Tim Gates Date: Sun, 3 Jul 2022 20:16:33 +1000 Subject: [PATCH 83/93] docs: Fix a few typos (#4340) There are small typos in: - platformio/test/helpers.py - platformio/util.py - tests/project/test_savedeps.py Fixes: - Should read `specified` rather than `sepcified`. - Should read `overridden` rather than `overriden`. - Should read `compatibility` rather than `compatiblty`. --- platformio/test/helpers.py | 2 +- platformio/util.py | 2 +- tests/project/test_savedeps.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/platformio/test/helpers.py b/platformio/test/helpers.py index d4364ee5..908173d0 100644 --- a/platformio/test/helpers.py +++ b/platformio/test/helpers.py @@ -43,7 +43,7 @@ def list_test_suites(project_config, environments, filters, ignores): # filter and ignore patterns patterns = dict(filter=list(filters), ignore=list(ignores)) for key, value in patterns.items(): - if value: # overriden from CLI + if value: # overridden from CLI continue patterns[key].extend( # pylint: disable=unnecessary-dict-index-lookup project_config.get(f"env:{env_name}", f"test_{key}", []) diff --git a/platformio/util.py b/platformio/util.py index 5a8b1923..f235c2ee 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -34,7 +34,7 @@ from platformio.proc import exec_command # pylint: enable=unused-import # also export list_serial_ports as get_serialports to be -# backward compatiblty with arduinosam versions 3.9.0 to 3.5.0 (and possibly others) +# backward compatibility with arduinosam versions 3.9.0 to 3.5.0 (and possibly others) get_serialports = get_serial_ports diff --git a/tests/project/test_savedeps.py b/tests/project/test_savedeps.py index 9740ce71..2a16511d 100644 --- a/tests/project/test_savedeps.py +++ b/tests/project/test_savedeps.py @@ -54,7 +54,7 @@ def test_save_libraries(tmp_path): PackageSpec("https://github.com/nanopb/nanopb.git"), ] - # add to the sepcified environment + # add to the specified environment save_project_dependencies( str(project_dir), specs, scope="lib_deps", action="add", environments=["debug"] ) @@ -140,7 +140,7 @@ def test_save_tools(tmp_path): PackageSpec("platformio/tool-esptoolpy"), ] - # add to the sepcified environment + # add to the specified environment save_project_dependencies( str(project_dir), specs, From de856ee730660b772d577a64b0274c520a711b61 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 4 Jul 2022 13:54:40 +0300 Subject: [PATCH 84/93] Export IDEs templates --- setup.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index 52693e06..2a41db90 100644 --- a/setup.py +++ b/setup.py @@ -62,10 +62,10 @@ setup( packages=find_packages(exclude=["tests.*", "tests"]) + ["scripts"], package_data={ "platformio": [ - "project/tpls/*/.*.tpl", - "project/tpls/*/*.tpl", - "project/tpls/*/*/*.tpl", - "project/tpls/*/.*/*.tpl", + "project/integration/tpls/*/.*.tpl", + "project/integration/tpls/*/*.tpl", + "project/integration/tpls/*/*/*.tpl", + "project/integration/tpls/*/.*/*.tpl", ], "scripts": ["99-platformio-udev.rules"], }, From 3928cb522e4563c8c59d27a601de1159d9515fdd Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 4 Jul 2022 13:55:11 +0300 Subject: [PATCH 85/93] Bump version to 6.1.0b2 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 84a0da80..5f8dbaeb 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (6, 1, "0b1") +VERSION = (6, 1, "0b2") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 523b6dfa98cd2a8e436f658e480eba249eae56a9 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 4 Jul 2022 17:32:11 +0300 Subject: [PATCH 86/93] Do not immediately terminate a testing program when results are received --- HISTORY.rst | 2 ++ docs | 2 +- examples | 2 +- platformio/test/runners/readers/program.py | 21 +++++++++++++++------ tests/commands/test_test.py | 8 -------- 5 files changed, 19 insertions(+), 16 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index d17de658..ff55d09d 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -28,9 +28,11 @@ PlatformIO Core 6 * **Unit Testing** + - Updated "Getting Started" documentation for `GoogleTest `__ testing and mocking framework - Export |UNITTESTING| flags only to the project build environment (``projenv``, files in "src" folder) - Merged the "building" stage with "uploading" for the embedded target (`issue #4307 `_) - Do not resolve dependencies from the project "src" folder when the `test_build_src `__ option is not enabled + - Do not immediately terminate a testing program when results are received - Fixed an issue when a custom `pio test --project-config `__ was not handled properly (`issue #4299 `_) - Fixed an issue when testing results were wrong in the verbose mode (`issue #4336 `_) diff --git a/docs b/docs index ad5bd94f..be6e3e87 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit ad5bd94f7ca41235fe183c445bb4680cea1e0f63 +Subproject commit be6e3e87ecdac12d6dc869fcac42b5cd481fa982 diff --git a/examples b/examples index b2657021..7fbb0ec1 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit b26570214771dde4db03f04ff37988eef7effbf8 +Subproject commit 7fbb0ec1532e98af213ffd242d725a8cde1061f8 diff --git a/platformio/test/runners/readers/program.py b/platformio/test/runners/readers/program.py index 82cc6a2a..d80d170d 100644 --- a/platformio/test/runners/readers/program.py +++ b/platformio/test/runners/readers/program.py @@ -18,14 +18,22 @@ import signal import subprocess import time -from platformio.compat import IS_WINDOWS, get_filesystem_encoding, get_locale_encoding +from platformio.compat import ( + IS_WINDOWS, + aio_get_running_loop, + get_filesystem_encoding, + get_locale_encoding, +) from platformio.test.exception import UnitTestError +EXITING_TIMEOUT = 5 # seconds + class ProgramProcessProtocol(asyncio.SubprocessProtocol): def __init__(self, test_runner, exit_future): self.test_runner = test_runner self.exit_future = exit_future + self._exit_timer = None def pipe_data_received(self, _, data): try: @@ -34,7 +42,9 @@ class ProgramProcessProtocol(asyncio.SubprocessProtocol): data = data.decode("latin-1") self.test_runner.on_testing_data_output(data) if self.test_runner.test_suite.is_finished(): - self._stop_testing() + self._exit_timer = aio_get_running_loop().call_later( + EXITING_TIMEOUT, self._stop_testing + ) def process_exited(self): self._stop_testing() @@ -42,12 +52,11 @@ class ProgramProcessProtocol(asyncio.SubprocessProtocol): def _stop_testing(self): if not self.exit_future.done(): self.exit_future.set_result(True) + if self._exit_timer: + self._exit_timer.cancel() class ProgramTestOutputReader: - - KILLING_TIMEOUT = 5 # seconds - def __init__(self, test_runner): self.test_runner = test_runner self.aio_loop = ( @@ -89,7 +98,7 @@ class ProgramTestOutputReader: # wait until subprocess will be killed start = time.time() while ( - start > (time.time() - self.KILLING_TIMEOUT) + start > (time.time() - EXITING_TIMEOUT) and transport.get_returncode() is None ): await asyncio.sleep(0.5) diff --git a/tests/commands/test_test.py b/tests/commands/test_test.py index 26089500..595d5398 100644 --- a/tests/commands/test_test.py +++ b/tests/commands/test_test.py @@ -471,10 +471,6 @@ void unittest_uart_end(){} validate_cliresult(result) -@pytest.mark.skipif( - sys.platform == "win32" and os.environ.get("GITHUB_ACTIONS") == "true", - reason="skip Github Actions on Windows (MinGW issue)", -) def test_doctest_framework(clirunner, tmp_path: Path): project_dir = tmp_path / "project" project_dir.mkdir() @@ -601,10 +597,6 @@ int main(int argc, char **argv) assert json_report["failure_nums"] == 1 -@pytest.mark.skipif( - sys.platform == "win32" and os.environ.get("GITHUB_ACTIONS") == "true", - reason="skip Github Actions on Windows (MinGW issue)", -) def test_googletest_framework(clirunner, tmp_path: Path): project_dir = tmp_path / "project" shutil.copytree( From 3cf62f8fa6328c3a7f14f03ccce27c1ebcff2c23 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 4 Jul 2022 18:04:45 +0300 Subject: [PATCH 87/93] Disable GoogleTest and Doctest frameworks on CI/Github Actions --- tests/commands/test_test.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/commands/test_test.py b/tests/commands/test_test.py index 595d5398..26089500 100644 --- a/tests/commands/test_test.py +++ b/tests/commands/test_test.py @@ -471,6 +471,10 @@ void unittest_uart_end(){} validate_cliresult(result) +@pytest.mark.skipif( + sys.platform == "win32" and os.environ.get("GITHUB_ACTIONS") == "true", + reason="skip Github Actions on Windows (MinGW issue)", +) def test_doctest_framework(clirunner, tmp_path: Path): project_dir = tmp_path / "project" project_dir.mkdir() @@ -597,6 +601,10 @@ int main(int argc, char **argv) assert json_report["failure_nums"] == 1 +@pytest.mark.skipif( + sys.platform == "win32" and os.environ.get("GITHUB_ACTIONS") == "true", + reason="skip Github Actions on Windows (MinGW issue)", +) def test_googletest_framework(clirunner, tmp_path: Path): project_dir = tmp_path / "project" shutil.copytree( From 6134db8e81ca38aea9b8b7784a0de84c7faa5216 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 4 Jul 2022 18:50:06 +0300 Subject: [PATCH 88/93] Fixed an issue when library dependencies were installed for the incompatible project environment // Resolve #4338 --- HISTORY.rst | 4 +++ platformio/builder/tools/piolib.py | 24 ++++++++++----- platformio/package/commands/install.py | 22 ++++++++++++-- platformio/package/manager/_install.py | 38 ++++++++++++++++-------- platformio/package/manager/base.py | 3 +- platformio/package/manager/library.py | 3 +- platformio/package/meta.py | 41 ++++++++++++++++++++++++++ tests/commands/pkg/test_install.py | 14 +++++++-- tests/package/test_meta.py | 23 +++++++++++++++ 9 files changed, 146 insertions(+), 26 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index ff55d09d..17987fe0 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -53,6 +53,10 @@ PlatformIO Core 6 - Fixed an issue with the |LDF| when recursively scanning dependencies in the ``chain`` mode - Fixed a "PermissionError" on Windows when running "clean" or "cleanall" targets (`issue #4331 `_) +* **Package Management** + + - Fixed an issue when library dependencies were installed for the incompatible project environment (`issue #4338 `_) + * **Miscellaneous** - Warn about incompatible Bash version for the `Shell Completion `__ (`issue #4326 `_) diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index 9059a25e..c830c881 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -28,7 +28,7 @@ import SCons.Scanner # pylint: disable=import-error from SCons.Script import ARGUMENTS # pylint: disable=import-error from SCons.Script import DefaultEnvironment # pylint: disable=import-error -from platformio import exception, fs, util +from platformio import exception, fs from platformio.builder.tools import platformio as piotool from platformio.compat import IS_WINDOWS, hashlib_encode_data, string_types from platformio.http import HTTPClientError, InternetIsOffline @@ -41,7 +41,7 @@ from platformio.package.manifest.parser import ( ManifestParserError, ManifestParserFactory, ) -from platformio.package.meta import PackageItem +from platformio.package.meta import PackageCompatibility, PackageItem from platformio.project.options import ProjectOptions @@ -582,10 +582,14 @@ class ArduinoLibBuilder(LibBuilderBase): return "chain+" def is_frameworks_compatible(self, frameworks): - return util.items_in_list(frameworks, ["arduino", "energia"]) + return PackageCompatibility(frameworks=frameworks).is_compatible( + PackageCompatibility(frameworks=["arduino", "energia"]) + ) def is_platforms_compatible(self, platforms): - return util.items_in_list(platforms, self._manifest.get("platforms") or ["*"]) + return PackageCompatibility(platforms=platforms).is_compatible( + PackageCompatibility(platforms=self._manifest.get("platforms")) + ) @property def build_flags(self): @@ -640,7 +644,9 @@ class MbedLibBuilder(LibBuilderBase): return include_dirs def is_frameworks_compatible(self, frameworks): - return util.items_in_list(frameworks, ["mbed"]) + return PackageCompatibility(frameworks=frameworks).is_compatible( + PackageCompatibility(frameworks=["mbed"]) + ) def process_extra_options(self): self._process_mbed_lib_confs() @@ -853,10 +859,14 @@ class PlatformIOLibBuilder(LibBuilderBase): ) def is_platforms_compatible(self, platforms): - return util.items_in_list(platforms, self._manifest.get("platforms") or ["*"]) + return PackageCompatibility(platforms=platforms).is_compatible( + PackageCompatibility(platforms=self._manifest.get("platforms")) + ) def is_frameworks_compatible(self, frameworks): - return util.items_in_list(frameworks, self._manifest.get("frameworks") or ["*"]) + return PackageCompatibility(frameworks=frameworks).is_compatible( + PackageCompatibility(frameworks=self._manifest.get("frameworks")) + ) class ProjectAsLibBuilder(LibBuilderBase): diff --git a/platformio/package/commands/install.py b/platformio/package/commands/install.py index 4aa6ec5d..ecfbe7bb 100644 --- a/platformio/package/commands/install.py +++ b/platformio/package/commands/install.py @@ -23,7 +23,9 @@ from platformio.package.exception import UnknownPackageError from platformio.package.manager.library import LibraryPackageManager from platformio.package.manager.platform import PlatformPackageManager from platformio.package.manager.tool import ToolPackageManager -from platformio.package.meta import PackageSpec +from platformio.package.meta import PackageCompatibility, PackageSpec +from platformio.platform.exception import UnknownPlatform +from platformio.platform.factory import PlatformFactory from platformio.project.config import ProjectConfig from platformio.project.savedeps import pkg_to_save_spec, save_project_dependencies from platformio.test.result import TestSuite @@ -202,8 +204,24 @@ def _install_project_env_libraries(project_env, options): _uninstall_project_unused_libdeps(project_env, options) already_up_to_date = not options.get("force") config = ProjectConfig.get_instance() + + compatibility_qualifiers = {} + if config.get(f"env:{project_env}", "platform"): + try: + p = PlatformFactory.new(config.get(f"env:{project_env}", "platform")) + compatibility_qualifiers["platforms"] = [p.name] + except UnknownPlatform: + pass + if config.get(f"env:{project_env}", "framework"): + compatibility_qualifiers["frameworks"] = config.get( + f"env:{project_env}", "framework" + ) + env_lm = LibraryPackageManager( - os.path.join(config.get("platformio", "libdeps_dir"), project_env) + os.path.join(config.get("platformio", "libdeps_dir"), project_env), + compatibility=PackageCompatibility(**compatibility_qualifiers) + if compatibility_qualifiers + else None, ) private_lm = LibraryPackageManager( os.path.join(config.get("platformio", "lib_dir")) diff --git a/platformio/package/manager/_install.py b/platformio/package/manager/_install.py index ae442bd0..c89d7b86 100644 --- a/platformio/package/manager/_install.py +++ b/platformio/package/manager/_install.py @@ -21,7 +21,7 @@ import click from platformio import app, compat, fs, util from platformio.package.exception import PackageException, UnknownPackageError -from platformio.package.meta import PackageItem +from platformio.package.meta import PackageCompatibility, PackageItem from platformio.package.unpack import FileUnpacker from platformio.package.vcsclient import VCSClientFactory @@ -55,9 +55,9 @@ class PackageManagerInstallMixin: def _install( self, spec, - search_qualifiers=None, skip_dependencies=False, force=False, + compatibility: PackageCompatibility = None, ): spec = self.ensure_spec(spec) @@ -97,7 +97,12 @@ class PackageManagerInstallMixin: if spec.external: pkg = self.install_from_uri(spec.uri, spec) else: - pkg = self.install_from_registry(spec, search_qualifiers) + pkg = self.install_from_registry( + spec, + search_qualifiers=compatibility.to_search_qualifiers() + if compatibility + else None, + ) if not pkg or not pkg.metadata: raise PackageException( @@ -137,20 +142,29 @@ class PackageManagerInstallMixin: if dependency.get("owner"): self.log.warning( click.style( - "Warning! Could not install dependency %s for package '%s'" - % (dependency, pkg.metadata.name), + "Warning! Could not install `%s` dependency " + "for the`%s` package" % (dependency, pkg.metadata.name), fg="yellow", ) ) def install_dependency(self, dependency): - spec = self.dependency_to_spec(dependency) - search_qualifiers = { - key: value - for key, value in dependency.items() - if key in ("authors", "platforms", "frameworks") - } - return self._install(spec, search_qualifiers=search_qualifiers or None) + dependency_compatibility = PackageCompatibility.from_dependency(dependency) + if self.compatibility and not dependency_compatibility.is_compatible( + self.compatibility + ): + self.log.debug( + click.style( + "Skip incompatible `%s` dependency with `%s`" + % (dependency, self.compatibility), + fg="yellow", + ) + ) + return None + return self._install( + spec=self.dependency_to_spec(dependency), + compatibility=dependency_compatibility, + ) def install_from_uri(self, uri, spec, checksum=None): spec = self.ensure_spec(spec) diff --git a/platformio/package/manager/base.py b/platformio/package/manager/base.py index c90b940f..8369f736 100644 --- a/platformio/package/manager/base.py +++ b/platformio/package/manager/base.py @@ -59,9 +59,10 @@ class BasePackageManager( # pylint: disable=too-many-public-methods,too-many-in ): _MEMORY_CACHE = {} - def __init__(self, pkg_type, package_dir): + def __init__(self, pkg_type, package_dir, compatibility=None): self.pkg_type = pkg_type self.package_dir = package_dir + self.compatibility = compatibility self.log = self._setup_logger() self._MEMORY_CACHE = {} diff --git a/platformio/package/manager/library.py b/platformio/package/manager/library.py index f17e5eb8..6babfc9c 100644 --- a/platformio/package/manager/library.py +++ b/platformio/package/manager/library.py @@ -24,11 +24,12 @@ from platformio.project.config import ProjectConfig class LibraryPackageManager(BasePackageManager): # pylint: disable=too-many-ancestors - def __init__(self, package_dir=None): + def __init__(self, package_dir=None, **kwargs): super().__init__( PackageType.LIBRARY, package_dir or ProjectConfig.get_instance().get("platformio", "globallib_dir"), + **kwargs ) @property diff --git a/platformio/package/meta.py b/platformio/package/meta.py index 34475a9c..fbd2b734 100644 --- a/platformio/package/meta.py +++ b/platformio/package/meta.py @@ -25,6 +25,7 @@ from platformio import fs from platformio.compat import get_object_members, hashlib_encode_data, string_types from platformio.package.manifest.parser import ManifestFileType from platformio.package.version import cast_version_to_semver +from platformio.util import items_in_list class PackageType: @@ -63,6 +64,46 @@ class PackageType: return None +class PackageCompatibility: + + KNOWN_QUALIFIERS = ("platforms", "frameworks", "authors") + + @classmethod + def from_dependency(cls, dependency): + assert isinstance(dependency, dict) + qualifiers = { + key: value + for key, value in dependency.items() + if key in cls.KNOWN_QUALIFIERS + } + return PackageCompatibility(**qualifiers) + + def __init__(self, **kwargs): + self.qualifiers = {} + for key, value in kwargs.items(): + if key not in self.KNOWN_QUALIFIERS: + raise ValueError( + "Unknown package compatibility qualifier -> `%s`" % key + ) + self.qualifiers[key] = value + + def __repr__(self): + return "PackageCompatibility <%s>" % self.qualifiers + + def to_search_qualifiers(self): + return self.qualifiers + + def is_compatible(self, other): + assert isinstance(other, PackageCompatibility) + for key, value in self.qualifiers.items(): + other_value = other.qualifiers.get(key) + if not value or not other_value: + continue + if not items_in_list(value, other_value): + return False + return True + + class PackageOutdatedResult: UPDATE_INCREMENT_MAJOR = "major" UPDATE_INCREMENT_MINOR = "minor" diff --git a/tests/commands/pkg/test_install.py b/tests/commands/pkg/test_install.py index 449783d1..415baea8 100644 --- a/tests/commands/pkg/test_install.py +++ b/tests/commands/pkg/test_install.py @@ -29,7 +29,9 @@ from platformio.project.config import ProjectConfig PROJECT_CONFIG_TPL = """ [env] platform = platformio/atmelavr@^3.4.0 -lib_deps = milesburton/DallasTemperature@^3.9.1 +lib_deps = + milesburton/DallasTemperature@^3.9.1 + https://github.com/esphome/ESPAsyncWebServer/archive/refs/tags/v2.1.0.zip [env:baremetal] board = uno @@ -134,7 +136,8 @@ def test_skip_dependencies(clirunner, validate_cliresult, isolated_pio_core, tmp os.path.join(ProjectConfig().get("platformio", "libdeps_dir"), "devkit") ).get_installed() assert pkgs_to_specs(installed_lib_pkgs) == [ - PackageSpec("DallasTemperature@3.10.0") + PackageSpec("DallasTemperature@3.10.0"), + PackageSpec("ESPAsyncWebServer-esphome@2.1.0"), ] assert len(ToolPackageManager().get_installed()) == 0 @@ -154,6 +157,7 @@ def test_baremetal_project(clirunner, validate_cliresult, isolated_pio_core, tmp ).get_installed() assert pkgs_to_specs(installed_lib_pkgs) == [ PackageSpec("DallasTemperature@3.10.0"), + PackageSpec("ESPAsyncWebServer-esphome@2.1.0"), PackageSpec("OneWire@2.3.7"), ] assert pkgs_to_specs(ToolPackageManager().get_installed()) == [ @@ -177,6 +181,7 @@ def test_project(clirunner, validate_cliresult, isolated_pio_core, tmp_path): ) assert pkgs_to_specs(lm.get_installed()) == [ PackageSpec("DallasTemperature@3.10.0"), + PackageSpec("ESPAsyncWebServer-esphome@2.1.0"), PackageSpec("OneWire@2.3.7"), ] assert pkgs_to_specs(ToolPackageManager().get_installed()) == [ @@ -184,7 +189,8 @@ def test_project(clirunner, validate_cliresult, isolated_pio_core, tmp_path): PackageSpec("toolchain-atmelavr@1.70300.191015"), ] assert config.get("env:devkit", "lib_deps") == [ - "milesburton/DallasTemperature@^3.9.1" + "milesburton/DallasTemperature@^3.9.1", + "https://github.com/esphome/ESPAsyncWebServer/archive/refs/tags/v2.1.0.zip", ] # test "Already up-to-date" @@ -270,6 +276,7 @@ def test_remove_project_unused_libdeps( lm = LibraryPackageManager(storage_dir) assert pkgs_to_specs(lm.get_installed()) == [ PackageSpec("DallasTemperature@3.10.0"), + PackageSpec("ESPAsyncWebServer-esphome@2.1.0"), PackageSpec("OneWire@2.3.7"), ] @@ -286,6 +293,7 @@ def test_remove_project_unused_libdeps( assert pkgs_to_specs(lm.get_installed()) == [ PackageSpec("ArduinoJson@5.13.4"), PackageSpec("DallasTemperature@3.10.0"), + PackageSpec("ESPAsyncWebServer-esphome@2.1.0"), PackageSpec("OneWire@2.3.7"), ] diff --git a/tests/package/test_meta.py b/tests/package/test_meta.py index 4faabeba..f7eae748 100644 --- a/tests/package/test_meta.py +++ b/tests/package/test_meta.py @@ -18,6 +18,7 @@ import jsondiff import semantic_version from platformio.package.meta import ( + PackageCompatibility, PackageMetaData, PackageOutdatedResult, PackageSpec, @@ -312,3 +313,25 @@ def test_metadata_load(tmpdir_factory): metadata.dump(str(piopm_path)) restored_metadata = PackageMetaData.load(str(piopm_path)) assert metadata == restored_metadata + + +def test_compatibility(): + assert PackageCompatibility().is_compatible(PackageCompatibility()) + assert PackageCompatibility().is_compatible( + PackageCompatibility(platforms=["espressif32"]) + ) + assert PackageCompatibility(frameworks=["arduino"]).is_compatible( + PackageCompatibility(platforms=["espressif32"]) + ) + assert PackageCompatibility(platforms="espressif32").is_compatible( + PackageCompatibility(platforms=["espressif32"]) + ) + assert PackageCompatibility( + platforms="espressif32", frameworks=["arduino"] + ).is_compatible(PackageCompatibility(platforms=None)) + assert PackageCompatibility( + platforms="espressif32", frameworks=["arduino"] + ).is_compatible(PackageCompatibility(platforms=["*"])) + assert not PackageCompatibility( + platforms="espressif32", frameworks=["arduino"] + ).is_compatible(PackageCompatibility(platforms=["atmelavr"])) From 6c18b37d54c1a5ee364412ad09db32dd59e0b927 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 4 Jul 2022 18:50:23 +0300 Subject: [PATCH 89/93] Bump version to 6.1.0rc1 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 5f8dbaeb..286123a8 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (6, 1, "0b2") +VERSION = (6, 1, "0rc1") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 6e16b435684f827fc4f421d731a30ab4e2698868 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 5 Jul 2022 18:57:06 +0300 Subject: [PATCH 90/93] Automatically upgrade outdated Unity library --- platformio/test/runners/unity.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/platformio/test/runners/unity.py b/platformio/test/runners/unity.py index 39418857..4b28a8ad 100644 --- a/platformio/test/runners/unity.py +++ b/platformio/test/runners/unity.py @@ -19,6 +19,7 @@ from pathlib import Path import click +from platformio.package.manager.library import LibraryPackageManager from platformio.test.exception import UnitTestSuiteError from platformio.test.result import TestCase, TestCaseSource, TestStatus from platformio.test.runners.base import TestRunnerBase @@ -184,6 +185,24 @@ void unityOutputComplete(void) { unittest_uart_end(); } ), ) + def __init__(self, *args, **kwargs): + """Delete when Unity > 2.5.2 is released""" + super().__init__(*args, **kwargs) + self._tmp_pre_upgrade() + + def _tmp_pre_upgrade(self): + """Delete when Unity > 2.5.2 is released""" + lm = LibraryPackageManager( + os.path.join( + self.project_config.get("platformio", "libdeps_dir"), + self.test_suite.env_name, + ), + ) + pkg = lm.get_package(self.EXTRA_LIB_DEPS[0]) + if not pkg or os.path.isfile(os.path.join(pkg.path, "platformio-build.py")): + return + lm.uninstall(pkg) + def get_unity_framework_config(self): if not self.platform.is_embedded(): return self.UNITY_FRAMEWORK_CONFIG["native"] From 1495e24e1e518d0a5a9c2b9672fdc72719dcc5b8 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 6 Jul 2022 16:22:13 +0300 Subject: [PATCH 91/93] Temporary disable ESPHome from CI --- .github/workflows/projects.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/projects.yml b/.github/workflows/projects.yml index 1e21bed1..ad62bc64 100644 --- a/.github/workflows/projects.yml +++ b/.github/workflows/projects.yml @@ -13,11 +13,11 @@ jobs: folder: "Marlin" config_dir: "Marlin" env_name: "mega2560" - - esphome: - repository: "esphome/esphome" - folder: "esphome" - config_dir: "esphome" - env_name: "esp32-arduino" + # - esphome: + # repository: "esphome/esphome" + # folder: "esphome" + # config_dir: "esphome" + # env_name: "esp32-arduino" - smartknob: repository: "scottbez1/smartknob" folder: "smartknob" From c2ddc89e46f684f1897db84a71c9b70e404d150a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 6 Jul 2022 16:24:03 +0300 Subject: [PATCH 92/93] Bump version to 6.1.0 --- HISTORY.rst | 2 +- docs | 2 +- platformio/__init__.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 17987fe0..7ad9d795 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -13,7 +13,7 @@ PlatformIO Core 6 **A professional collaborative platform for declarative, safety-critical, and test-driven embedded development.** -6.1.0 (2022-??-??) +6.1.0 (2022-07-06) ~~~~~~~~~~~~~~~~~~ * **Device Manager** diff --git a/docs b/docs index be6e3e87..f5958b87 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit be6e3e87ecdac12d6dc869fcac42b5cd481fa982 +Subproject commit f5958b875629eac7b9b95932d524952731e79480 diff --git a/platformio/__init__.py b/platformio/__init__.py index 286123a8..1ceb17a7 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (6, 1, "0rc1") +VERSION = (6, 1, 0) __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 84c2e0a3d6d7c6a76217f908bf99b48989196495 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 6 Jul 2022 18:41:00 +0300 Subject: [PATCH 93/93] Tests: fest latest package version in runtime --- tests/commands/pkg/test_install.py | 76 +++++++++++++++++++++++------- tests/commands/pkg/test_update.py | 32 +++++++++---- tests/conftest.py | 17 +++++++ 3 files changed, 99 insertions(+), 26 deletions(-) diff --git a/tests/commands/pkg/test_install.py b/tests/commands/pkg/test_install.py index 415baea8..dcbd2d98 100644 --- a/tests/commands/pkg/test_install.py +++ b/tests/commands/pkg/test_install.py @@ -50,7 +50,11 @@ def pkgs_to_specs(pkgs): def test_global_packages( - clirunner, validate_cliresult, func_isolated_pio_core, tmp_path + clirunner, + validate_cliresult, + func_isolated_pio_core, + get_pkg_latest_version, + tmp_path, ): # libraries result = clirunner.invoke( @@ -81,7 +85,7 @@ def test_global_packages( assert pkgs_to_specs(LibraryPackageManager().get_installed()) == [ PackageSpec("ArduinoJson@5.13.4"), PackageSpec("DallasTemperature@3.9.0+sha.964939d"), - PackageSpec("OneWire@2.3.7"), + PackageSpec("OneWire@%s" % get_pkg_latest_version("paulstoffregen/OneWire")), ] # custom storage storage_dir = tmp_path / "custom_lib_storage" @@ -122,7 +126,9 @@ def test_global_packages( ] -def test_skip_dependencies(clirunner, validate_cliresult, isolated_pio_core, tmp_path): +def test_skip_dependencies( + clirunner, validate_cliresult, isolated_pio_core, get_pkg_latest_version, tmp_path +): project_dir = tmp_path / "project" project_dir.mkdir() (project_dir / "platformio.ini").write_text(PROJECT_CONFIG_TPL) @@ -136,13 +142,18 @@ def test_skip_dependencies(clirunner, validate_cliresult, isolated_pio_core, tmp os.path.join(ProjectConfig().get("platformio", "libdeps_dir"), "devkit") ).get_installed() assert pkgs_to_specs(installed_lib_pkgs) == [ - PackageSpec("DallasTemperature@3.10.0"), + PackageSpec( + "DallasTemperature@%s" + % get_pkg_latest_version("milesburton/DallasTemperature") + ), PackageSpec("ESPAsyncWebServer-esphome@2.1.0"), ] assert len(ToolPackageManager().get_installed()) == 0 -def test_baremetal_project(clirunner, validate_cliresult, isolated_pio_core, tmp_path): +def test_baremetal_project( + clirunner, validate_cliresult, isolated_pio_core, get_pkg_latest_version, tmp_path +): project_dir = tmp_path / "project" project_dir.mkdir() (project_dir / "platformio.ini").write_text(PROJECT_CONFIG_TPL) @@ -156,16 +167,23 @@ def test_baremetal_project(clirunner, validate_cliresult, isolated_pio_core, tmp os.path.join(ProjectConfig().get("platformio", "libdeps_dir"), "baremetal") ).get_installed() assert pkgs_to_specs(installed_lib_pkgs) == [ - PackageSpec("DallasTemperature@3.10.0"), + PackageSpec( + "DallasTemperature@%s" + % get_pkg_latest_version("milesburton/DallasTemperature") + ), PackageSpec("ESPAsyncWebServer-esphome@2.1.0"), - PackageSpec("OneWire@2.3.7"), + PackageSpec( + "OneWire@%s" % get_pkg_latest_version("paulstoffregen/OneWire") + ), ] assert pkgs_to_specs(ToolPackageManager().get_installed()) == [ PackageSpec("toolchain-atmelavr@1.70300.191015"), ] -def test_project(clirunner, validate_cliresult, isolated_pio_core, tmp_path): +def test_project( + clirunner, validate_cliresult, isolated_pio_core, get_pkg_latest_version, tmp_path +): project_dir = tmp_path / "project" project_dir.mkdir() (project_dir / "platformio.ini").write_text(PROJECT_CONFIG_TPL) @@ -180,9 +198,14 @@ def test_project(clirunner, validate_cliresult, isolated_pio_core, tmp_path): os.path.join(config.get("platformio", "libdeps_dir"), "devkit") ) assert pkgs_to_specs(lm.get_installed()) == [ - PackageSpec("DallasTemperature@3.10.0"), + PackageSpec( + "DallasTemperature@%s" + % get_pkg_latest_version("milesburton/DallasTemperature") + ), PackageSpec("ESPAsyncWebServer-esphome@2.1.0"), - PackageSpec("OneWire@2.3.7"), + PackageSpec( + "OneWire@%s" % get_pkg_latest_version("paulstoffregen/OneWire") + ), ] assert pkgs_to_specs(ToolPackageManager().get_installed()) == [ PackageSpec("framework-arduino-avr-attiny@1.5.2"), @@ -202,7 +225,9 @@ def test_project(clirunner, validate_cliresult, isolated_pio_core, tmp_path): assert "Already up-to-date" in result.output -def test_private_lib_deps(clirunner, validate_cliresult, isolated_pio_core, tmp_path): +def test_private_lib_deps( + clirunner, validate_cliresult, isolated_pio_core, get_pkg_latest_version, tmp_path +): project_dir = tmp_path / "project" private_lib_dir = project_dir / "lib" / "private" private_lib_dir.mkdir(parents=True) @@ -247,7 +272,9 @@ platform = native config.get("platformio", "lib_dir") ).get_installed() assert pkgs_to_specs(installed_private_pkgs) == [ - PackageSpec("OneWire@2.3.7"), + PackageSpec( + "OneWire@%s" % get_pkg_latest_version("paulstoffregen/OneWire") + ), PackageSpec("My Private Lib@1.0.0"), ] installed_env_pkgs = LibraryPackageManager( @@ -255,12 +282,15 @@ platform = native ).get_installed() assert pkgs_to_specs(installed_env_pkgs) == [ PackageSpec("ArduinoJson@5.13.4"), - PackageSpec("DallasTemperature@3.10.0"), + PackageSpec( + "DallasTemperature@%s" + % get_pkg_latest_version("milesburton/DallasTemperature") + ), ] def test_remove_project_unused_libdeps( - clirunner, validate_cliresult, isolated_pio_core, tmp_path + clirunner, validate_cliresult, isolated_pio_core, get_pkg_latest_version, tmp_path ): project_dir = tmp_path / "project" project_dir.mkdir() @@ -275,9 +305,14 @@ def test_remove_project_unused_libdeps( storage_dir = os.path.join(config.get("platformio", "libdeps_dir"), "baremetal") lm = LibraryPackageManager(storage_dir) assert pkgs_to_specs(lm.get_installed()) == [ - PackageSpec("DallasTemperature@3.10.0"), + PackageSpec( + "DallasTemperature@%s" + % get_pkg_latest_version("milesburton/DallasTemperature") + ), PackageSpec("ESPAsyncWebServer-esphome@2.1.0"), - PackageSpec("OneWire@2.3.7"), + PackageSpec( + "OneWire@%s" % get_pkg_latest_version("paulstoffregen/OneWire") + ), ] # add new deps @@ -292,9 +327,14 @@ def test_remove_project_unused_libdeps( lm = LibraryPackageManager(storage_dir) assert pkgs_to_specs(lm.get_installed()) == [ PackageSpec("ArduinoJson@5.13.4"), - PackageSpec("DallasTemperature@3.10.0"), + PackageSpec( + "DallasTemperature@%s" + % get_pkg_latest_version("milesburton/DallasTemperature") + ), PackageSpec("ESPAsyncWebServer-esphome@2.1.0"), - PackageSpec("OneWire@2.3.7"), + PackageSpec( + "OneWire@%s" % get_pkg_latest_version("paulstoffregen/OneWire") + ), ] # manually remove from cofiguration file diff --git a/tests/commands/pkg/test_update.py b/tests/commands/pkg/test_update.py index e49423fd..06ab92a7 100644 --- a/tests/commands/pkg/test_update.py +++ b/tests/commands/pkg/test_update.py @@ -145,7 +145,9 @@ def test_global_packages( assert isinstance(result.exception, UnknownPackageError) -def test_project(clirunner, validate_cliresult, isolated_pio_core, tmp_path): +def test_project( + clirunner, validate_cliresult, isolated_pio_core, get_pkg_latest_version, tmp_path +): project_dir = tmp_path / "project" project_dir.mkdir() (project_dir / "platformio.ini").write_text(PROJECT_OUTDATED_CONFIG_TPL) @@ -161,7 +163,9 @@ def test_project(clirunner, validate_cliresult, isolated_pio_core, tmp_path): ) assert pkgs_to_specs(lm.get_installed()) == [ PackageSpec("DallasTemperature@3.8.1"), - PackageSpec("OneWire@2.3.7"), + PackageSpec( + "OneWire@%s" % get_pkg_latest_version("paulstoffregen/OneWire") + ), ] assert pkgs_to_specs(PlatformPackageManager().get_installed()) == [ PackageSpec("atmelavr@2.2.0") @@ -187,8 +191,13 @@ def test_project(clirunner, validate_cliresult, isolated_pio_core, tmp_path): assert pkgs[0].metadata.name == "atmelavr" assert pkgs[0].metadata.version.major == 3 assert pkgs_to_specs(lm.get_installed()) == [ - PackageSpec("DallasTemperature@3.10.0"), - PackageSpec("OneWire@2.3.7"), + PackageSpec( + "DallasTemperature@%s" + % get_pkg_latest_version("milesburton/DallasTemperature") + ), + PackageSpec( + "OneWire@%s" % get_pkg_latest_version("paulstoffregen/OneWire") + ), ] assert pkgs_to_specs(ToolPackageManager().get_installed()) == [ PackageSpec("framework-arduino-avr-attiny@1.3.2"), @@ -211,7 +220,7 @@ def test_project(clirunner, validate_cliresult, isolated_pio_core, tmp_path): def test_custom_project_libraries( - clirunner, validate_cliresult, isolated_pio_core, tmp_path + clirunner, validate_cliresult, isolated_pio_core, get_pkg_latest_version, tmp_path ): project_dir = tmp_path / "project" project_dir.mkdir() @@ -230,7 +239,9 @@ def test_custom_project_libraries( ) assert pkgs_to_specs(lm.get_installed()) == [ PackageSpec("DallasTemperature@3.8.1"), - PackageSpec("OneWire@2.3.7"), + PackageSpec( + "OneWire@%s" % get_pkg_latest_version("paulstoffregen/OneWire") + ), ] # update package result = clirunner.invoke( @@ -260,8 +271,13 @@ def test_custom_project_libraries( os.path.join(config.get("platformio", "libdeps_dir"), "devkit") ) assert pkgs_to_specs(lm.get_installed()) == [ - PackageSpec("DallasTemperature@3.10.0"), - PackageSpec("OneWire@2.3.7"), + PackageSpec( + "DallasTemperature@%s" + % get_pkg_latest_version("milesburton/DallasTemperature") + ), + PackageSpec( + "OneWire@%s" % get_pkg_latest_version("paulstoffregen/OneWire") + ), ] assert config.get("env:devkit", "lib_deps") == [ "milesburton/DallasTemperature@^3.8.0" diff --git a/tests/conftest.py b/tests/conftest.py index b26aaf49..eedf54c5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -13,6 +13,7 @@ # limitations under the License. import email +import functools import imaplib import os import time @@ -21,6 +22,8 @@ import pytest from click.testing import CliRunner from platformio import http +from platformio.package.meta import PackageSpec, PackageType +from platformio.registry.client import RegistryClient def pytest_configure(config): @@ -131,3 +134,17 @@ def receive_email(): # pylint:disable=redefined-outer-name, too-many-locals return result return _receive_email + + +@pytest.fixture(scope="session") +def get_pkg_latest_version(): + @functools.lru_cache() + def wrap(spec, pkg_type=None): + if not isinstance(spec, PackageSpec): + spec = PackageSpec(spec) + pkg_type = pkg_type or PackageType.LIBRARY + client = RegistryClient() + pkg = client.get_package(pkg_type, spec.owner, spec.name) + return pkg["version"]["name"] + + return wrap