From 2695690b345f9835f990a631ccf0797956653904 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 18 Jul 2022 21:06:32 +0300 Subject: [PATCH 01/39] Bump version to 6.1.4a1 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 703712d9..17dc9ecd 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (6, 1, 3) +VERSION = (6, 1, "4a1") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 9ff117b0fb07ac6ff4a5ef5dd5031df8bd181734 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 22 Jul 2022 18:17:44 +0300 Subject: [PATCH 02/39] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index beb6d196..511b06e4 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit beb6d196ead70c1025340005ba007b3dee11e92a +Subproject commit 511b06e4e95a69f6f87f0121db2dbe6464c5efac From e498119e0de3d1bc7694a85352048c07855da3a6 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 22 Jul 2022 18:18:26 +0300 Subject: [PATCH 03/39] Improved project dependency resolving when using the `pio project init --ide` command --- HISTORY.rst | 5 ++++ platformio/project/commands/init.py | 45 ++++++++++++++++------------- 2 files changed, 30 insertions(+), 20 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 33bf355d..be4ff36a 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -13,6 +13,11 @@ PlatformIO Core 6 **A professional collaborative platform for declarative, safety-critical, and test-driven embedded development.** +6.1.4 (2022-??-??) +~~~~~~~~~~~~~~~~~~ + +* Improved project dependency resolving when using the `pio project init --ide `__ command + 6.1.3 (2022-07-18) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/project/commands/init.py b/platformio/project/commands/init.py index baae29ec..ca57de34 100644 --- a/platformio/project/commands/init.py +++ b/platformio/project/commands/init.py @@ -79,27 +79,32 @@ def project_init_cmd( elif board: update_board_envs(project_dir, board, project_option, env_prefix) - # resolve project dependencies - if not no_install_dependencies and (environment or board): - install_project_dependencies( - options=dict( - project_dir=project_dir, - environments=[environment] if environment else [], - silent=silent, - ) - ) - - if ide: - if not silent: - click.echo( - "Updating metadata for the %s IDE..." % click.style(ide, fg="cyan") - ) - with fs.cd(project_dir): - config = ProjectConfig.get_instance( - os.path.join(project_dir, "platformio.ini") - ) + with fs.cd(project_dir): + generator = None + config = ProjectConfig.get_instance(os.path.join(project_dir, "platformio.ini")) + if ide: config.validate() - ProjectGenerator(config, environment, ide, board).generate() + # init generator and pick the best env if user didn't specify + generator = ProjectGenerator(config, environment, ide, board) + if not environment: + environment = generator.env_name + + # resolve project dependencies + if not no_install_dependencies and (environment or board): + install_project_dependencies( + options=dict( + project_dir=project_dir, + environments=[environment] if environment else [], + silent=silent, + ) + ) + + if generator: + if not silent: + click.echo( + "Updating metadata for the %s IDE..." % click.style(ide, fg="cyan") + ) + generator.generate() if is_new_project: init_cvs_ignore(project_dir) From 36d470279ce6aaa17f2a1b615d0749ff7cd37fd1 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 22 Jul 2022 18:18:51 +0300 Subject: [PATCH 04/39] Bump version to 6.1.4a2 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 17dc9ecd..57d79665 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (6, 1, "4a1") +VERSION = (6, 1, "4a2") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 410324b2c7b5f58a0ee0ac2881301511e177217b Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 25 Jul 2022 16:39:09 +0300 Subject: [PATCH 05/39] Move banners to the bottom --- README.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index 4bdda76f..cd3454f5 100644 --- a/README.rst +++ b/README.rst @@ -1,7 +1,3 @@ -.. image:: https://raw.githubusercontent.com/vshymanskyy/StandWithUkraine/main/banner-direct.svg - :target: https://github.com/vshymanskyy/StandWithUkraine/blob/main/docs/README.md - :alt: SWUbanner - PlatformIO Core =============== @@ -99,3 +95,7 @@ Copyright (c) 2014-present PlatformIO The PlatformIO is licensed under the permissive Apache 2.0 license, so you can use it in both commercial and personal projects with confidence. + +.. image:: https://raw.githubusercontent.com/vshymanskyy/StandWithUkraine/main/banner-direct.svg + :target: https://github.com/vshymanskyy/StandWithUkraine/blob/main/docs/README.md + :alt: SWUbanner \ No newline at end of file From fd8c9786c088520edafc2409029f8270a2929b18 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 25 Jul 2022 16:39:20 +0300 Subject: [PATCH 06/39] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 511b06e4..4522803c 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 511b06e4e95a69f6f87f0121db2dbe6464c5efac +Subproject commit 4522803cd65aef6c74df9c03f53c2a2e76922402 From 699da0a8fb5eafce345a08a683360bc0c30e6735 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 29 Jul 2022 14:58:11 +0300 Subject: [PATCH 07/39] Prefer global env when generating integration data --- platformio/builder/tools/piointegration.py | 61 +++++++++------------- 1 file changed, 25 insertions(+), 36 deletions(-) diff --git a/platformio/builder/tools/piointegration.py b/platformio/builder/tools/piointegration.py index f776bccf..900738c6 100644 --- a/platformio/builder/tools/piointegration.py +++ b/platformio/builder/tools/piointegration.py @@ -145,51 +145,40 @@ def _subst_cmd(env, cmd): return " ".join([SCons.Subst.quote_spaces(arg) for arg in args]) -def DumpIntegrationData(env, globalenv): - """env here is `projenv`""" - +def DumpIntegrationData(*args): + projenv, globalenv = args[0:2] # pylint: disable=unbalanced-tuple-unpacking data = { - "env_name": env["PIOENV"], - "libsource_dirs": [env.subst(item) for item in env.GetLibSourceDirs()], - "defines": dump_defines(env), - "includes": env.DumpIntegrationIncludes(), - "cc_path": where_is_program(env.subst("$CC"), env.subst("${ENV['PATH']}")), - "cxx_path": where_is_program(env.subst("$CXX"), env.subst("${ENV['PATH']}")), - "gdb_path": where_is_program(env.subst("$GDB"), env.subst("${ENV['PATH']}")), - "prog_path": env.subst("$PROG_PATH"), - "svd_path": dump_svd_path(env), - "compiler_type": env.GetCompilerType(), + "env_name": globalenv["PIOENV"], + "libsource_dirs": [ + globalenv.subst(item) for item in globalenv.GetLibSourceDirs() + ], + "defines": dump_defines(projenv), + "includes": projenv.DumpIntegrationIncludes(), + "cc_flags": _subst_cmd(projenv, "$CFLAGS $CCFLAGS $CPPFLAGS"), + "cxx_flags": _subst_cmd(projenv, "$CXXFLAGS $CCFLAGS $CPPFLAGS"), + "cc_path": where_is_program( + globalenv.subst("$CC"), globalenv.subst("${ENV['PATH']}") + ), + "cxx_path": where_is_program( + globalenv.subst("$CXX"), globalenv.subst("${ENV['PATH']}") + ), + "gdb_path": where_is_program( + globalenv.subst("$GDB"), globalenv.subst("${ENV['PATH']}") + ), + "prog_path": globalenv.subst("$PROG_PATH"), + "svd_path": dump_svd_path(globalenv), + "compiler_type": globalenv.GetCompilerType(), "targets": globalenv.DumpTargets(), "extra": dict( flash_images=[ - {"offset": item[0], "path": env.subst(item[1])} - for item in env.get("FLASH_EXTRA_IMAGES", []) + {"offset": item[0], "path": globalenv.subst(item[1])} + for item in globalenv.get("FLASH_EXTRA_IMAGES", []) ] ), } data["extra"].update( - env.get("INTEGRATION_EXTRA_DATA", env.get("IDE_EXTRA_DATA", {})) + globalenv.get("INTEGRATION_EXTRA_DATA", globalenv.get("IDE_EXTRA_DATA", {})) ) - - env_ = env.Clone() - # https://github.com/platformio/platformio-atom-ide/issues/34 - _new_defines = [] - for item in SCons.Defaults.processDefines(env_.get("CPPDEFINES", [])): - item = item.replace('\\"', '"') - if " " in item: - _new_defines.append(item.replace(" ", "\\\\ ")) - else: - _new_defines.append(item) - env_.Replace(CPPDEFINES=_new_defines) - - # export C/C++ build flags - data.update( - { - "cc_flags": _subst_cmd(env_, "$CFLAGS $CCFLAGS $CPPFLAGS"), - "cxx_flags": _subst_cmd(env_, "$CXXFLAGS $CCFLAGS $CPPFLAGS"), - } - ) - return data From 9ddf73baa65e197f3e5c05c33eb3e1fe82121791 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 29 Jul 2022 17:18:21 +0300 Subject: [PATCH 08/39] Fixed an issue when escaping macros/defines for IDE integration // Resolve #4360 --- HISTORY.rst | 1 + platformio/builder/tools/piointegration.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index be4ff36a..1ac233e5 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -17,6 +17,7 @@ PlatformIO Core 6 ~~~~~~~~~~~~~~~~~~ * Improved project dependency resolving when using the `pio project init --ide `__ command +* Fixed an issue when escaping macros/defines for IDE integration (`issue #4360 `_) 6.1.3 (2022-07-18) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/builder/tools/piointegration.py b/platformio/builder/tools/piointegration.py index 900738c6..99b76f0b 100644 --- a/platformio/builder/tools/piointegration.py +++ b/platformio/builder/tools/piointegration.py @@ -94,7 +94,7 @@ def dump_defines(env): for item in SCons.Defaults.processDefines(env.get("CPPDEFINES", [])): item = item.strip() if item: - defines.append(env.subst(item).replace("\\", "")) + defines.append(env.subst(item).replace('\\"', '"')) # special symbol for Atmel AVR MCU if env["PIOPLATFORM"] == "atmelavr": From 7c9e0393f8c4e5c27b6954448ce5e08fd3c6e0fb Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 29 Jul 2022 19:37:59 +0300 Subject: [PATCH 09/39] Improved device port finder when using dual channel UART converter // Resolve #4367 --- HISTORY.rst | 1 + platformio/builder/tools/pioupload.py | 7 +- platformio/debug/config/blackmagic.py | 7 +- platformio/device/finder.py | 350 +++++++++++----------- platformio/device/monitor/command.py | 7 +- platformio/test/runners/readers/serial.py | 7 +- 6 files changed, 184 insertions(+), 195 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 1ac233e5..821dbc4e 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -16,6 +16,7 @@ PlatformIO Core 6 6.1.4 (2022-??-??) ~~~~~~~~~~~~~~~~~~ +* Improved device port finder when using dual channel UART converter (`issue #4367 `_) * Improved project dependency resolving when using the `pio project init --ide `__ command * Fixed an issue when escaping macros/defines for IDE integration (`issue #4360 `_) diff --git a/platformio/builder/tools/pioupload.py b/platformio/builder/tools/pioupload.py index d46b1590..1f51ff5c 100644 --- a/platformio/builder/tools/pioupload.py +++ b/platformio/builder/tools/pioupload.py @@ -26,7 +26,7 @@ from SCons.Script import ARGUMENTS # pylint: disable=import-error from serial import Serial, SerialException from platformio import exception, fs -from platformio.device.finder import find_mbed_disk, find_serial_port, is_pattern_port +from platformio.device.finder import SerialPortFinder, find_mbed_disk, is_pattern_port from platformio.device.list.util import list_serial_ports from platformio.proc import exec_command @@ -112,13 +112,12 @@ def AutodetectUploadPort(*args, **kwargs): except exception.InvalidUdevRules as exc: sys.stderr.write("\n%s\n\n" % exc) env.Replace( - UPLOAD_PORT=find_serial_port( - initial_port=initial_port, + UPLOAD_PORT=SerialPortFinder( board_config=env.BoardConfig() if "BOARD" in env else None, upload_protocol=upload_protocol, prefer_gdb_port="blackmagic" in upload_protocol, verbose=int(ARGUMENTS.get("PIOVERBOSE", 0)), - ) + ).find(initial_port) ) if env.subst("$UPLOAD_PORT"): diff --git a/platformio/debug/config/blackmagic.py b/platformio/debug/config/blackmagic.py index 5a89c2f9..83d98d9c 100644 --- a/platformio/debug/config/blackmagic.py +++ b/platformio/debug/config/blackmagic.py @@ -14,7 +14,7 @@ from platformio.debug.config.base import DebugConfigBase from platformio.debug.exception import DebugInvalidOptionsError -from platformio.device.finder import find_serial_port, is_pattern_port +from platformio.device.finder import SerialPortFinder, is_pattern_port class BlackmagicDebugConfig(DebugConfigBase): @@ -56,12 +56,11 @@ set language auto initial_port = DebugConfigBase.port.fget(self) if initial_port and not is_pattern_port(initial_port): return initial_port - port = find_serial_port( - initial_port, + port = SerialPortFinder( board_config=self.board_config, upload_protocol=self.tool_name, prefer_gdb_port=True, - ) + ).find(initial_port) if port: return port raise DebugInvalidOptionsError( diff --git a/platformio/device/finder.py b/platformio/device/finder.py index 0d7eba90..bf35c542 100644 --- a/platformio/device/finder.py +++ b/platformio/device/finder.py @@ -18,7 +18,7 @@ from fnmatch import fnmatch import click import serial -from platformio.compat import IS_MACOS, IS_WINDOWS +from platformio.compat import IS_WINDOWS from platformio.device.list.util import list_logical_devices, list_serial_ports from platformio.fs import get_platformio_udev_rules_path from platformio.package.manager.platform import PlatformPackageManager @@ -53,190 +53,12 @@ def parse_udev_rules_hwids(path): return result -def normalize_board_hwid(value): - if isinstance(value, (list, tuple)): - value = ("%s:%s" % (value[0], value[1])).replace("0x", "") - return value.upper() - - def is_pattern_port(port): if not port: return False return set(["*", "?", "[", "]"]) & set(port) -def match_serial_port(pattern): - for item in list_serial_ports(): - if fnmatch(item["port"], pattern): - return item["port"] - return None - - -def is_serial_port_ready(port, timeout=1): - try: - serial.Serial(port, timeout=timeout).close() - return True - except: # pylint: disable=bare-except - pass - return False - - -def find_serial_port( # pylint: disable=too-many-arguments - initial_port, - board_config=None, - upload_protocol=None, - ensure_ready=False, - prefer_gdb_port=False, - timeout=2, - verbose=False, -): - if initial_port: - if not is_pattern_port(initial_port): - return initial_port - return match_serial_port(initial_port) - - if upload_protocol and upload_protocol.startswith("blackmagic"): - return find_blackmagic_serial_port(prefer_gdb_port, timeout) - port = None - if board_config and board_config.get("build.hwids", []): - port = find_board_serial_port(board_config, timeout, verbose) - if not port: - port = find_known_uart_port(ensure_ready, timeout, verbose) - if port: - return port - - # pick the best PID:VID USB device - best_port = None - for item in list_serial_ports(): - if ensure_ready and not is_serial_port_ready(item["port"]): - continue - port = item["port"] - if "VID:PID" in item["hwid"]: - best_port = port - return best_port or port - - -def find_blackmagic_serial_port(prefer_gdb_port=False, timeout=0): - try: - - @retry(timeout=timeout) - def wrapper(): - candidates = [] - for item in list_serial_ports(filter_hwid=True): - if ( - not any(hwid in item["hwid"].upper() for hwid in BLACK_MAGIC_HWIDS) - and not "Black Magic" in item["description"] - ): - continue - if ( - IS_WINDOWS - and item["port"].startswith("COM") - and len(item["port"]) > 4 - ): - item["port"] = "\\\\.\\%s" % item["port"] - candidates.append(item) - - if not candidates: - raise retry.RetryNextException() - - for item in candidates: - if ("GDB" if prefer_gdb_port else "UART") in item["description"]: - return item["port"] - if IS_MACOS: - # 1 - GDB, 3 - UART - for item in candidates: - if item["port"].endswith("1" if prefer_gdb_port else "3"): - return item["port"] - - candidates = sorted(candidates, key=lambda item: item["port"]) - return ( - candidates[0] # first port is GDB? - if len(candidates) == 1 or prefer_gdb_port - else candidates[1] - )["port"] - - return wrapper() - except retry.RetryStopException: - pass - return None - - -def find_board_serial_port(board_config, timeout=0, verbose=False): - hwids = board_config.get("build.hwids", []) - try: - - @retry(timeout=timeout) - def wrapper(): - for item in list_serial_ports(filter_hwid=True): - hwid = item["hwid"].upper() - for board_hwid in hwids: - if normalize_board_hwid(board_hwid) in hwid: - return item["port"] - raise retry.RetryNextException() - - return wrapper() - except retry.RetryStopException: - pass - - if verbose: - click.secho( - "TimeoutError: Could not automatically find serial port " - "for the `%s` board based on the declared HWIDs=%s" - % (board_config.get("name", "unknown"), hwids), - fg="yellow", - err=True, - ) - - return None - - -def find_known_uart_port(ensure_ready=False, timeout=0, verbose=False): - known_hwids = list(BLACK_MAGIC_HWIDS) - - # load from UDEV rules - udev_rules_path = get_platformio_udev_rules_path() - if os.path.isfile(udev_rules_path): - known_hwids.extend(parse_udev_rules_hwids(udev_rules_path)) - - # load from installed dev-platforms - for platform in PlatformPackageManager().get_installed(): - p = PlatformFactory.new(platform) - for board_config in p.get_boards().values(): - for board_hwid in board_config.get("build.hwids", []): - board_hwid = normalize_board_hwid(board_hwid) - if board_hwid not in known_hwids: - known_hwids.append(board_hwid) - - try: - - @retry(timeout=timeout) - def wrapper(): - for item in list_serial_ports(as_objects=True): - if not item.vid or not item.pid: - continue - hwid = "{:04X}:{:04X}".format(item.vid, item.pid) - for pattern in known_hwids: - if fnmatch(hwid, pattern) and ( - not ensure_ready or is_serial_port_ready(item.device) - ): - return item.device - raise retry.RetryNextException() - - return wrapper() - except retry.RetryStopException: - pass - - if verbose: - click.secho( - "TimeoutError: Could not automatically find serial port " - "based on the known UART bridges", - fg="yellow", - err=True, - ) - - return None - - def find_mbed_disk(initial_port): msdlabels = ("mbed", "nucleo", "frdm", "microbit") for item in list_logical_devices(): @@ -254,3 +76,173 @@ def find_mbed_disk(initial_port): if item["name"] and any(l in item["name"].lower() for l in msdlabels): return item["path"] return None + + +def is_serial_port_ready(port, timeout=1): + try: + serial.Serial(port, timeout=timeout).close() + return True + except: # pylint: disable=bare-except + pass + return False + + +class SerialPortFinder: + def __init__( # pylint: disable=too-many-arguments + self, + board_config=None, + upload_protocol=None, + ensure_ready=False, + prefer_gdb_port=False, + timeout=2, + verbose=False, + ): + self.board_config = board_config + self.upload_protocol = upload_protocol + self.ensure_ready = ensure_ready + self.prefer_gdb_port = prefer_gdb_port + self.timeout = timeout + self.verbose = verbose + + @staticmethod + def normalize_board_hwid(value): + if isinstance(value, (list, tuple)): + value = ("%s:%s" % (value[0], value[1])).replace("0x", "") + return value.upper() + + @staticmethod + def match_serial_port(pattern): + for item in list_serial_ports(): + if fnmatch(item["port"], pattern): + return item["port"] + return None + + @staticmethod + def match_device_hwid(patterns): + for item in list_serial_ports(as_objects=True): + if not item.vid or not item.pid: + continue + hwid = "{:04X}:{:04X}".format(item.vid, item.pid) + for pattern in patterns: + if fnmatch(hwid, pattern): + return item + return None + + def find(self, initial_port=None): + if initial_port: + if not is_pattern_port(initial_port): + return initial_port + return self.match_serial_port(initial_port) + + if self.upload_protocol and self.upload_protocol.startswith("blackmagic"): + return self._find_blackmagic_port() + + device = None + if self.board_config and self.board_config.get("build.hwids", []): + device = self._find_board_device() + if not device: + device = self._find_known_device() + if device: + port = self._reveal_device_port(device) + + # pick the best PID:VID USB device + best_port = None + for item in list_serial_ports(): + if self.ensure_ready and not is_serial_port_ready(item["port"]): + continue + port = item["port"] + if "VID:PID" in item["hwid"]: + best_port = port + return best_port or port + + def _reveal_device_port(self, device): + candidates = [] + for item in list_serial_ports(as_objects=True): + if item.vid == device.vid and item.pid == device.pid: + candidates.append(item) + if len(candidates) == 1: + return device.device + for item in candidates: + if ("GDB" if self.prefer_gdb_port else "UART") in item.description: + return item.device + candidates = sorted(candidates, key=lambda item: item.device) + # first port is GDB? BlackMagic, ESP-Prog + return candidates[0 if self.prefer_gdb_port else -1].device + + def _find_blackmagic_port(self): + device = self.match_device_hwid(BLACK_MAGIC_HWIDS) + if not device: + return None + port = self._reveal_device_port(device) + if IS_WINDOWS and port.startswith("COM") and len(port) > 4: + return "\\\\.\\%s" % port + return port + + def _find_board_device(self): + hwids = [ + self.normalize_board_hwid(hwid) + for hwid in self.board_config.get("build.hwids", []) + ] + try: + + @retry(timeout=self.timeout) + def wrapper(): + device = self.match_device_hwid(hwids) + if device: + return device + raise retry.RetryNextException() + + return wrapper() + except retry.RetryStopException: + pass + + if self.verbose: + click.secho( + "TimeoutError: Could not automatically find serial port " + "for the `%s` board based on the declared HWIDs=%s" + % (self.board_config.get("name", "unknown"), hwids), + fg="yellow", + err=True, + ) + + return None + + def _find_known_device(self): + hwids = list(BLACK_MAGIC_HWIDS) + + # load from UDEV rules + udev_rules_path = get_platformio_udev_rules_path() + if os.path.isfile(udev_rules_path): + hwids.extend(parse_udev_rules_hwids(udev_rules_path)) + + # load from installed dev-platforms + for platform in PlatformPackageManager().get_installed(): + p = PlatformFactory.new(platform) + for board_config in p.get_boards().values(): + for board_hwid in board_config.get("build.hwids", []): + board_hwid = self.normalize_board_hwid(board_hwid) + if board_hwid not in hwids: + hwids.append(board_hwid) + + try: + + @retry(timeout=self.timeout) + def wrapper(): + device = self.match_device_hwid(hwids) + if device: + return device + raise retry.RetryNextException() + + return wrapper() + except retry.RetryStopException: + pass + + if self.verbose: + click.secho( + "TimeoutError: Could not automatically find serial port " + "based on the known UART bridges", + fg="yellow", + err=True, + ) + + return None diff --git a/platformio/device/monitor/command.py b/platformio/device/monitor/command.py index 8ba61677..301de772 100644 --- a/platformio/device/monitor/command.py +++ b/platformio/device/monitor/command.py @@ -18,7 +18,7 @@ import sys import click from platformio import exception, fs -from platformio.device.finder import find_serial_port +from platformio.device.finder import SerialPortFinder from platformio.device.monitor.filters.base import register_filters from platformio.device.monitor.terminal import get_available_filters, start_terminal from platformio.platform.factory import PlatformFactory @@ -124,14 +124,13 @@ def device_monitor_cmd(**options): options = apply_project_monitor_options(options, project_options) register_filters(platform=platform, options=options) - options["port"] = find_serial_port( - initial_port=options["port"], + options["port"] = SerialPortFinder( board_config=platform.board_config(project_options.get("board")) if platform and project_options.get("board") else None, upload_protocol=project_options.get("upload_protocol"), ensure_ready=True, - ) + ).find(initial_port=options["port"]) if options["menu_char"] == options["exit_char"]: raise exception.UserSideException( diff --git a/platformio/test/runners/readers/serial.py b/platformio/test/runners/readers/serial.py index ac7765bc..ab35775b 100644 --- a/platformio/test/runners/readers/serial.py +++ b/platformio/test/runners/readers/serial.py @@ -17,7 +17,7 @@ from time import sleep import click import serial -from platformio.device.finder import find_serial_port +from platformio.device.finder import SerialPortFinder from platformio.exception import UserSideException @@ -66,15 +66,14 @@ class SerialTestOutputReader: project_options = self.test_runner.project_config.items( env=self.test_runner.test_suite.env_name, as_dict=True ) - port = find_serial_port( - initial_port=self.test_runner.get_test_port(), + port = SerialPortFinder( board_config=self.test_runner.platform.board_config( project_options["board"] ), upload_protocol=project_options.get("upload_protocol"), ensure_ready=True, verbose=self.test_runner.options.verbose, - ) + ).find(initial_port=self.test_runner.get_test_port()) if port: return port raise UserSideException( From 537558d4104d147d0a8f47c7e82f34d136995cc0 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 29 Jul 2022 20:03:14 +0300 Subject: [PATCH 10/39] Do not resolve project dependencies on for the "cleanall" target // Resolve #4344 --- HISTORY.rst | 1 + platformio/run/processor.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index 821dbc4e..971d34c3 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -18,6 +18,7 @@ PlatformIO Core 6 * Improved device port finder when using dual channel UART converter (`issue #4367 `_) * Improved project dependency resolving when using the `pio project init --ide `__ command +* Do not resolve project dependencies on for the ``cleanall`` target (`issue #4344 `_) * Fixed an issue when escaping macros/defines for IDE integration (`issue #4360 `_) 6.1.3 (2022-07-18) diff --git a/platformio/run/processor.py b/platformio/run/processor.py index 00a96f02..053478ac 100644 --- a/platformio/run/processor.py +++ b/platformio/run/processor.py @@ -79,7 +79,7 @@ class EnvironmentProcessor: if "monitor" in build_targets: build_targets.remove("monitor") - if "clean" not in build_targets: + if not set(["clean", "cleanall"]) & set(build_targets): install_project_env_dependencies( self.name, { From 0939b43899284b3547e2b730afa8af5b4e5de812 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 29 Jul 2022 20:03:55 +0300 Subject: [PATCH 11/39] Bump version to 6.1.4b1 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 57d79665..02cefd18 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (6, 1, "4a2") +VERSION = (6, 1, "4b1") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 6653c024873ea325addfe7fd115e7272cf4e5679 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 30 Jul 2022 12:15:49 +0300 Subject: [PATCH 12/39] Use the cached "BUILD_TYPE" from env --- platformio/builder/main.py | 1 + platformio/builder/tools/piolib.py | 6 +++--- platformio/builder/tools/platformio.py | 8 ++++---- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/platformio/builder/main.py b/platformio/builder/main.py index 74f6acd3..730af43a 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -127,6 +127,7 @@ env.Replace( PROJECT_DATA_DIR=config.get("platformio", "data_dir"), PROJECTDATA_DIR="$PROJECT_DATA_DIR", # legacy for dev/platform PROJECT_BUILD_DIR=config.get("platformio", "build_dir"), + BUILD_TYPE=env.GetBuildType(), BUILD_CACHE_DIR=config.get("platformio", "build_cache_dir"), LIBSOURCE_DIRS=[ config.get("platformio", "lib_dir"), diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index d375347f..c27fb116 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -146,7 +146,7 @@ class LibBuilderBase: self._processed_search_files = [] # pass a macro to the projenv + libs - if "test" in env.GetBuildType(): + if "test" in env["BUILD_TYPE"]: self.env.Append(CPPDEFINES=["PIO_UNIT_TESTING"]) # reset source filter, could be overridden with extra script @@ -911,7 +911,7 @@ class ProjectAsLibBuilder(LibBuilderBase): def get_search_files(self): items = [] - build_type = self.env.GetBuildType() + build_type = self.env["BUILD_TYPE"] # project files if "test" not in build_type or self.env.GetProjectOption("test_build_src"): items.extend(super().get_search_files()) @@ -1164,7 +1164,7 @@ def ConfigureProjectLibBuilder(env): project = ProjectAsLibBuilder(env, "$PROJECT_DIR") - if "test" in env.GetBuildType(): + if "test" in env["BUILD_TYPE"]: project.env.ConfigureTestTarget() ldf_mode = LibBuilderBase.lib_ldf_mode.fget(project) # pylint: disable=no-member diff --git a/platformio/builder/tools/platformio.py b/platformio/builder/tools/platformio.py index 04b1b1d7..d642a8f7 100644 --- a/platformio/builder/tools/platformio.py +++ b/platformio/builder/tools/platformio.py @@ -86,7 +86,7 @@ def BuildProgram(env): ) ) - print("Building in %s mode" % env.GetBuildType()) + print("Building in %s mode" % env["BUILD_TYPE"]) return program @@ -121,7 +121,7 @@ def ProcessProgramDeps(env): # process framework scripts env.BuildFrameworks(env.get("PIOFRAMEWORK")) - if "debug" in env.GetBuildType(): + if "debug" in env["BUILD_TYPE"]: env.ConfigureDebugTarget() # remove specified flags @@ -149,7 +149,7 @@ def ProcessProjectDeps(env): } ) - if "test" in env.GetBuildType(): + if "test" in env["BUILD_TYPE"]: build_files_before_nums = len(env.get("PIOBUILDFILES", [])) plb.env.BuildSources( "$BUILD_TEST_DIR", "$PROJECT_TEST_DIR", "$PIOTEST_SRC_FILTER" @@ -161,7 +161,7 @@ def ProcessProjectDeps(env): ) env.Exit(1) - if "test" not in env.GetBuildType() or env.GetProjectOption("test_build_src"): + if "test" not in env["BUILD_TYPE"] or env.GetProjectOption("test_build_src"): plb.env.BuildSources( "$BUILD_SRC_DIR", "$PROJECT_SRC_DIR", env.get("SRC_FILTER") ) From fa0de1dad438b3e0b444bda6b952485b1fe03698 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 30 Jul 2022 12:16:32 +0300 Subject: [PATCH 13/39] Switch to the official PROGPATH --- platformio/builder/tools/piointegration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/builder/tools/piointegration.py b/platformio/builder/tools/piointegration.py index 99b76f0b..0517774b 100644 --- a/platformio/builder/tools/piointegration.py +++ b/platformio/builder/tools/piointegration.py @@ -165,7 +165,7 @@ def DumpIntegrationData(*args): "gdb_path": where_is_program( globalenv.subst("$GDB"), globalenv.subst("${ENV['PATH']}") ), - "prog_path": globalenv.subst("$PROG_PATH"), + "prog_path": globalenv.subst("$PROGPATH"), "svd_path": dump_svd_path(globalenv), "compiler_type": globalenv.GetCompilerType(), "targets": globalenv.DumpTargets(), From fcb676abc67a496fee3a8d29a4940476f64b1d58 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 1 Aug 2022 13:06:12 +0300 Subject: [PATCH 14/39] Docs: Sync dev-platforms --- docs | 2 +- examples | 2 +- scripts/docspregen.py | 13 ++++++++++--- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/docs b/docs index 4522803c..2175b369 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 4522803cd65aef6c74df9c03f53c2a2e76922402 +Subproject commit 2175b369e206d000e1cca1f14ba2d309b358790a diff --git a/examples b/examples index 7fbb0ec1..f98cb5a9 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit 7fbb0ec1532e98af213ffd242d725a8cde1061f8 +Subproject commit f98cb5a9be74b9c87b6e5f3cd530d8d7f0548825 diff --git a/scripts/docspregen.py b/scripts/docspregen.py index 2846c7da..a9a96c7b 100644 --- a/scripts/docspregen.py +++ b/scripts/docspregen.py @@ -39,6 +39,8 @@ RST_COPYRIGHT = """.. Copyright (c) 2014-present PlatformIO Date: Mon, 1 Aug 2022 19:47:27 +0300 Subject: [PATCH 15/39] Remove deprecated "absolute_import" --- platformio/app.py | 2 -- platformio/builder/tools/compilation_db.py | 2 -- platformio/builder/tools/pioasm.py | 2 -- platformio/builder/tools/piohooks.py | 2 -- platformio/builder/tools/pioino.py | 2 -- platformio/builder/tools/piointegration.py | 1 - platformio/builder/tools/piolib.py | 2 -- platformio/builder/tools/piomaxlen.py | 2 -- platformio/builder/tools/piomisc.py | 2 -- platformio/builder/tools/pioplatform.py | 2 -- platformio/builder/tools/pioproject.py | 2 -- platformio/builder/tools/piosize.py | 2 -- platformio/builder/tools/piotarget.py | 2 -- platformio/builder/tools/piotest.py | 2 -- platformio/builder/tools/pioupload.py | 2 -- platformio/home/rpc/handlers/app.py | 2 -- platformio/home/rpc/handlers/os.py | 2 -- platformio/home/rpc/handlers/piocore.py | 2 -- platformio/home/rpc/handlers/project.py | 2 -- platformio/util.py | 2 -- 20 files changed, 39 deletions(-) diff --git a/platformio/app.py b/platformio/app.py index b2c78bc1..75035b57 100644 --- a/platformio/app.py +++ b/platformio/app.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import absolute_import - import getpass import hashlib import json diff --git a/platformio/builder/tools/compilation_db.py b/platformio/builder/tools/compilation_db.py index 6dff1746..23280dec 100644 --- a/platformio/builder/tools/compilation_db.py +++ b/platformio/builder/tools/compilation_db.py @@ -23,8 +23,6 @@ # pylint: disable=unused-argument, protected-access, unused-variable, import-error # Original: https://github.com/mongodb/mongo/blob/master/site_scons/site_tools/compilation_db.py -from __future__ import absolute_import - import itertools import json import os diff --git a/platformio/builder/tools/pioasm.py b/platformio/builder/tools/pioasm.py index 6af96bcd..4d5cfc13 100644 --- a/platformio/builder/tools/pioasm.py +++ b/platformio/builder/tools/pioasm.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import absolute_import - import SCons.Tool.asm # pylint: disable=import-error # diff --git a/platformio/builder/tools/piohooks.py b/platformio/builder/tools/piohooks.py index acd43fbf..9c67b006 100644 --- a/platformio/builder/tools/piohooks.py +++ b/platformio/builder/tools/piohooks.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import absolute_import - def AddActionWrapper(handler): def wraps(env, files, action): diff --git a/platformio/builder/tools/pioino.py b/platformio/builder/tools/pioino.py index 453d1f51..de1f8cf8 100644 --- a/platformio/builder/tools/pioino.py +++ b/platformio/builder/tools/pioino.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import absolute_import - import atexit import glob import io diff --git a/platformio/builder/tools/piointegration.py b/platformio/builder/tools/piointegration.py index 0517774b..ae8ee2ed 100644 --- a/platformio/builder/tools/piointegration.py +++ b/platformio/builder/tools/piointegration.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import absolute_import import glob import os diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index c27fb116..e2b98d91 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -15,8 +15,6 @@ # pylint: disable=too-many-instance-attributes, too-many-public-methods # pylint: disable=assignment-from-no-return, unused-argument, too-many-lines -from __future__ import absolute_import - import hashlib import io import os diff --git a/platformio/builder/tools/piomaxlen.py b/platformio/builder/tools/piomaxlen.py index b6a0ebb0..d20879eb 100644 --- a/platformio/builder/tools/piomaxlen.py +++ b/platformio/builder/tools/piomaxlen.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import absolute_import - import hashlib import os import re diff --git a/platformio/builder/tools/piomisc.py b/platformio/builder/tools/piomisc.py index 76ec2e37..47496fe5 100644 --- a/platformio/builder/tools/piomisc.py +++ b/platformio/builder/tools/piomisc.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import absolute_import - import os import sys diff --git a/platformio/builder/tools/pioplatform.py b/platformio/builder/tools/pioplatform.py index d6e71e58..d13c78ec 100644 --- a/platformio/builder/tools/pioplatform.py +++ b/platformio/builder/tools/pioplatform.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import absolute_import - import os import sys diff --git a/platformio/builder/tools/pioproject.py b/platformio/builder/tools/pioproject.py index bf59d918..9b7b1499 100644 --- a/platformio/builder/tools/pioproject.py +++ b/platformio/builder/tools/pioproject.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import absolute_import - from platformio.compat import MISSING from platformio.project.config import ProjectConfig diff --git a/platformio/builder/tools/piosize.py b/platformio/builder/tools/piosize.py index 7eece12a..f562fe64 100644 --- a/platformio/builder/tools/piosize.py +++ b/platformio/builder/tools/piosize.py @@ -14,8 +14,6 @@ # pylint: disable=too-many-locals -from __future__ import absolute_import - import json import sys from os import environ, makedirs, remove diff --git a/platformio/builder/tools/piotarget.py b/platformio/builder/tools/piotarget.py index 4d7c3a74..d9f616a5 100644 --- a/platformio/builder/tools/piotarget.py +++ b/platformio/builder/tools/piotarget.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import absolute_import - import os from SCons.Action import Action # pylint: disable=import-error diff --git a/platformio/builder/tools/piotest.py b/platformio/builder/tools/piotest.py index 07886efb..17431871 100644 --- a/platformio/builder/tools/piotest.py +++ b/platformio/builder/tools/piotest.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import absolute_import - import os from platformio.builder.tools import platformio as piotool diff --git a/platformio/builder/tools/pioupload.py b/platformio/builder/tools/pioupload.py index 1f51ff5c..77bf28d2 100644 --- a/platformio/builder/tools/pioupload.py +++ b/platformio/builder/tools/pioupload.py @@ -14,8 +14,6 @@ # pylint: disable=unused-argument -from __future__ import absolute_import - import os import re import sys diff --git a/platformio/home/rpc/handlers/app.py b/platformio/home/rpc/handlers/app.py index 5ef4afa8..9d7fc590 100644 --- a/platformio/home/rpc/handlers/app.py +++ b/platformio/home/rpc/handlers/app.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import absolute_import - import os from pathlib import Path diff --git a/platformio/home/rpc/handlers/os.py b/platformio/home/rpc/handlers/os.py index fad11318..9fe0198f 100644 --- a/platformio/home/rpc/handlers/os.py +++ b/platformio/home/rpc/handlers/os.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import absolute_import - import glob import io import os diff --git a/platformio/home/rpc/handlers/piocore.py b/platformio/home/rpc/handlers/piocore.py index 7a0052d4..df252114 100644 --- a/platformio/home/rpc/handlers/piocore.py +++ b/platformio/home/rpc/handlers/piocore.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import absolute_import - import io import json import os diff --git a/platformio/home/rpc/handlers/project.py b/platformio/home/rpc/handlers/project.py index 146629ca..6c12bcae 100644 --- a/platformio/home/rpc/handlers/project.py +++ b/platformio/home/rpc/handlers/project.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import absolute_import - import os import shutil import time diff --git a/platformio/util.py b/platformio/util.py index f6800b4f..6c22566c 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import absolute_import - import functools import math import platform From a0493e6ac40ddf7f55665d7368a8b82cfd716270 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 1 Aug 2022 19:48:27 +0300 Subject: [PATCH 16/39] Handle both extra IDE data: IDE_EXTRA_DATA & INTEGRATION_EXTRA_DATA --- platformio/builder/main.py | 1 - platformio/builder/tools/piointegration.py | 6 +++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/platformio/builder/main.py b/platformio/builder/main.py index 730af43a..07958308 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -82,7 +82,6 @@ DEFAULT_ENV_OPTIONS = dict( PROGPATH=os.path.join("$BUILD_DIR", "$PROGNAME$PROGSUFFIX"), PROG_PATH="$PROGPATH", # deprecated PYTHONEXE=get_pythonexe_path(), - IDE_EXTRA_DATA={}, ) # Declare command verbose messages diff --git a/platformio/builder/tools/piointegration.py b/platformio/builder/tools/piointegration.py index ae8ee2ed..acdc66f3 100644 --- a/platformio/builder/tools/piointegration.py +++ b/platformio/builder/tools/piointegration.py @@ -175,9 +175,8 @@ def DumpIntegrationData(*args): ] ), } - data["extra"].update( - globalenv.get("INTEGRATION_EXTRA_DATA", globalenv.get("IDE_EXTRA_DATA", {})) - ) + for key in ("IDE_EXTRA_DATA", "INTEGRATION_EXTRA_DATA"): + data["extra"].update(globalenv.get(key, {})) return data @@ -186,6 +185,7 @@ def exists(_): def generate(env): + env["INTEGRATION_EXTRA_DATA"] = {} env.AddMethod(IsIntegrationDump) env.AddMethod(DumpIntegrationIncludes) env.AddMethod(DumpIntegrationData) From 5e8eb7709044862e93b3937c2e86b3111ffb3f71 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 1 Aug 2022 19:49:24 +0300 Subject: [PATCH 17/39] Rename "platformio" conflicted tool/module name to "piobuild" --- platformio/builder/main.py | 2 +- platformio/builder/tools/compilation_db.py | 2 +- .../builder/tools/{platformio.py => piobuild.py} | 2 -- platformio/builder/tools/piolib.py | 14 +++++++------- platformio/builder/tools/piotest.py | 6 +++--- 5 files changed, 12 insertions(+), 14 deletions(-) rename platformio/builder/tools/{platformio.py => piobuild.py} (99%) diff --git a/platformio/builder/main.py b/platformio/builder/main.py index 07958308..36121743 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -55,7 +55,7 @@ DEFAULT_ENV_OPTIONS = dict( "link", "piohooks", "pioasm", - "platformio", + "piobuild", "pioproject", "pioplatform", "piotest", diff --git a/platformio/builder/tools/compilation_db.py b/platformio/builder/tools/compilation_db.py index 23280dec..9fa11c86 100644 --- a/platformio/builder/tools/compilation_db.py +++ b/platformio/builder/tools/compilation_db.py @@ -29,7 +29,7 @@ import os import SCons -from platformio.builder.tools.platformio import SRC_ASM_EXT, SRC_C_EXT, SRC_CXX_EXT +from platformio.builder.tools.piobuild import SRC_ASM_EXT, SRC_C_EXT, SRC_CXX_EXT from platformio.proc import where_is_program # Implements the ability for SCons to emit a compilation database for the MongoDB project. See diff --git a/platformio/builder/tools/platformio.py b/platformio/builder/tools/piobuild.py similarity index 99% rename from platformio/builder/tools/platformio.py rename to platformio/builder/tools/piobuild.py index d642a8f7..4a32a1d7 100644 --- a/platformio/builder/tools/platformio.py +++ b/platformio/builder/tools/piobuild.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import absolute_import - import fnmatch import os import sys diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index e2b98d91..8e1bec15 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -27,7 +27,7 @@ from SCons.Script import ARGUMENTS # pylint: disable=import-error from SCons.Script import DefaultEnvironment # pylint: disable=import-error from platformio import exception, fs -from platformio.builder.tools import platformio as piotool +from platformio.builder.tools import piobuild from platformio.compat import IS_WINDOWS, hashlib_encode_data, string_types from platformio.http import HTTPClientError, InternetIsOffline from platformio.package.exception import ( @@ -92,7 +92,7 @@ class LibBuilderFactory: return ["mbed"] for fname in files: if not fs.path_endswith_ext( - fname, piotool.SRC_BUILD_EXT + piotool.SRC_HEADER_EXT + fname, piobuild.SRC_BUILD_EXT + piobuild.SRC_HEADER_EXT ): continue with io.open( @@ -197,7 +197,7 @@ class LibBuilderBase: @property def src_filter(self): - return piotool.SRC_FILTER_DEFAULT + [ + return piobuild.SRC_FILTER_DEFAULT + [ "-" % os.sep, "-" % os.sep, "-" % os.sep, @@ -329,7 +329,7 @@ class LibBuilderBase: return [ os.path.join(self.src_dir, item) for item in self.env.MatchSourceFiles( - self.src_dir, self.src_filter, piotool.SRC_BUILD_EXT + self.src_dir, self.src_filter, piobuild.SRC_BUILD_EXT ) ] @@ -394,10 +394,10 @@ class LibBuilderBase: result.append(item) if not self.PARSE_SRC_BY_H_NAME: continue - if not fs.path_endswith_ext(item_path, piotool.SRC_HEADER_EXT): + if not fs.path_endswith_ext(item_path, piobuild.SRC_HEADER_EXT): continue item_fname = item_path[: item_path.rindex(".")] - for ext in piotool.SRC_C_EXT + piotool.SRC_CXX_EXT: + for ext in piobuild.SRC_C_EXT + piobuild.SRC_CXX_EXT: if not os.path.isfile("%s.%s" % (item_fname, ext)): continue item_c_node = self.env.File("%s.%s" % (item_fname, ext)) @@ -558,7 +558,7 @@ class ArduinoLibBuilder(LibBuilderBase): src_filter = [] is_utility = os.path.isdir(os.path.join(self.path, "utility")) - for ext in piotool.SRC_BUILD_EXT + piotool.SRC_HEADER_EXT: + for ext in piobuild.SRC_BUILD_EXT + piobuild.SRC_HEADER_EXT: # arduino ide ignores files with .asm or .ASM extensions if ext.lower() == "asm": continue diff --git a/platformio/builder/tools/piotest.py b/platformio/builder/tools/piotest.py index 17431871..b3b324b1 100644 --- a/platformio/builder/tools/piotest.py +++ b/platformio/builder/tools/piotest.py @@ -14,7 +14,7 @@ import os -from platformio.builder.tools import platformio as piotool +from platformio.builder.tools import piobuild from platformio.test.result import TestSuite from platformio.test.runners.factory import TestRunnerFactory @@ -22,7 +22,7 @@ from platformio.test.runners.factory import TestRunnerFactory def ConfigureTestTarget(env): env.Append( CPPDEFINES=["UNIT_TEST"], # deprecated, use PIO_UNIT_TESTING - PIOTEST_SRC_FILTER=[f"+<*.{ext}>" for ext in piotool.SRC_BUILD_EXT], + PIOTEST_SRC_FILTER=[f"+<*.{ext}>" for ext in piobuild.SRC_BUILD_EXT], ) env.Prepend(CPPPATH=["$PROJECT_TEST_DIR"]) @@ -36,7 +36,7 @@ def ConfigureTestTarget(env): env.Prepend( PIOTEST_SRC_FILTER=[ f"+<{test_name}{os.path.sep}*.{ext}>" - for ext in piotool.SRC_BUILD_EXT + for ext in piobuild.SRC_BUILD_EXT ], CPPPATH=[os.path.join("$PROJECT_TEST_DIR", test_name)], ) From 09f7ff2db3cd3820f45be4b336ae8baa8c7269a9 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 1 Aug 2022 20:08:25 +0300 Subject: [PATCH 18/39] Upgraded build engine to the SCons 4.4.0 --- HISTORY.rst | 1 + platformio/__init__.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index 971d34c3..7587f549 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -18,6 +18,7 @@ PlatformIO Core 6 * Improved device port finder when using dual channel UART converter (`issue #4367 `_) * Improved project dependency resolving when using the `pio project init --ide `__ command +* Upgraded build engine to the SCons 4.4.0 (`release notes `__) * Do not resolve project dependencies on for the ``cleanall`` target (`issue #4344 `_) * Fixed an issue when escaping macros/defines for IDE integration (`issue #4360 `_) diff --git a/platformio/__init__.py b/platformio/__init__.py index 02cefd18..afd7b969 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -49,7 +49,7 @@ __default_requests_timeout__ = (10, None) # (connect, read) __core_packages__ = { "contrib-piohome": "~3.4.2", "contrib-pysite": "~2.%d%d.0" % (sys.version_info.major, sys.version_info.minor), - "tool-scons": "~4.40300.0", + "tool-scons": "~4.40400.0", "tool-cppcheck": "~1.270.0", "tool-clangtidy": "~1.120001.0", "tool-pvs-studio": "~7.18.0", From 5c2673cd7128099143685092a36f9f9480a3bfa8 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 1 Aug 2022 20:08:46 +0300 Subject: [PATCH 19/39] Bump version to 6.1.4b2 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index afd7b969..b893a6dd 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (6, 1, "4b1") +VERSION = (6, 1, "4b2") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From b7b01dd6a0ca2b073ce349c7360661be05238efa Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 3 Aug 2022 16:35:37 +0300 Subject: [PATCH 20/39] Drop "pio --force" option --- platformio/__main__.py | 4 ++-- platformio/app.py | 2 -- platformio/commands/ci.py | 4 +--- platformio/maintenance.py | 3 +-- 4 files changed, 4 insertions(+), 9 deletions(-) diff --git a/platformio/__main__.py b/platformio/__main__.py index 6a1cda94..c0ad38ac 100644 --- a/platformio/__main__.py +++ b/platformio/__main__.py @@ -31,7 +31,7 @@ from platformio.compat import IS_CYGWIN, ensure_python3 @click.option("--caller", "-c", help="Caller ID (service)") @click.option("--no-ansi", is_flag=True, help="Do not print ANSI control characters") @click.pass_context -def cli(ctx, force, caller, no_ansi): +def cli(ctx, force, caller, no_ansi): # pylint: disable=unused-argument try: if ( no_ansi @@ -53,7 +53,7 @@ def cli(ctx, force, caller, no_ansi): except: # pylint: disable=bare-except pass - maintenance.on_platformio_start(ctx, force, caller) + maintenance.on_platformio_start(ctx, caller) @cli.result_callback() diff --git a/platformio/app.py b/platformio/app.py index 75035b57..8315be3c 100644 --- a/platformio/app.py +++ b/platformio/app.py @@ -62,7 +62,6 @@ DEFAULT_SETTINGS = { SESSION_VARS = { "command_ctx": None, - "force_option": False, "caller_id": None, "custom_project_conf": None, } @@ -229,7 +228,6 @@ def set_session_var(name, value): def is_disabled_progressbar(): return any( [ - get_session_var("force_option"), proc.is_ci(), os.getenv("PLATFORMIO_DISABLE_PROGRESSBAR") == "true", ] diff --git a/platformio/commands/ci.py b/platformio/commands/ci.py index e62d1ca9..aebd9e7b 100644 --- a/platformio/commands/ci.py +++ b/platformio/commands/ci.py @@ -19,7 +19,7 @@ import tempfile import click -from platformio import app, fs +from platformio import fs from platformio.exception import CIBuildEnvsEmpty from platformio.project.commands.init import project_init_cmd, validate_boards from platformio.project.config import ProjectConfig @@ -84,8 +84,6 @@ def cli( # pylint: disable=too-many-arguments, too-many-branches raise click.BadParameter("Missing argument 'src'") try: - app.set_session_var("force_option", True) - if not keep_build_dir and os.path.isdir(build_dir): fs.rmtree(build_dir) if not os.path.isdir(build_dir): diff --git a/platformio/maintenance.py b/platformio/maintenance.py index bd9ccd9f..304ebc4a 100644 --- a/platformio/maintenance.py +++ b/platformio/maintenance.py @@ -32,9 +32,8 @@ from platformio.package.version import pepver_to_semver from platformio.system.prune import calculate_unnecessary_system_data -def on_platformio_start(ctx, force, caller): +def on_platformio_start(ctx, caller): app.set_session_var("command_ctx", ctx) - app.set_session_var("force_option", force) set_caller(caller) telemetry.on_command() From 8ada1c2b349f65270b925ea52953772030bc9b6c Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 3 Aug 2022 16:56:21 +0300 Subject: [PATCH 21/39] Use "strip_ansi" from click module --- platformio/util.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/platformio/util.py b/platformio/util.py index 6c22566c..0bd26e40 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -205,4 +205,5 @@ def humanize_duration_time(duration): def strip_ansi_codes(text): - return re.sub(r"\x1B\[\d+(;\d+){0,2}m", "", text) + # pylint: disable=protected-access + return click._compat.strip_ansi(text) From 83c4e5f463b03ec38b9c0d26f4be749647c1da76 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 3 Aug 2022 17:32:40 +0300 Subject: [PATCH 22/39] Improve downloading progress for non-terminal stream --- platformio/app.py | 7 +------ platformio/package/download.py | 35 ++++++++++++++++++---------------- 2 files changed, 20 insertions(+), 22 deletions(-) diff --git a/platformio/app.py b/platformio/app.py index 8315be3c..bf7af870 100644 --- a/platformio/app.py +++ b/platformio/app.py @@ -226,12 +226,7 @@ def set_session_var(name, value): def is_disabled_progressbar(): - return any( - [ - proc.is_ci(), - os.getenv("PLATFORMIO_DISABLE_PROGRESSBAR") == "true", - ] - ) + return os.getenv("PLATFORMIO_DISABLE_PROGRESSBAR") == "true" def get_cid(): diff --git a/platformio/package/download.py b/platformio/package/download.py index f8b67ba8..a0638404 100644 --- a/platformio/package/download.py +++ b/platformio/package/download.py @@ -13,8 +13,7 @@ # limitations under the License. import io -import math -from email.utils import parsedate_tz +from email.utils import parsedate from os.path import getsize, join from time import mktime @@ -72,22 +71,27 @@ class FileDownloader: def start(self, with_progress=True, silent=False): label = "Downloading" + file_size = self.get_size() itercontent = self._request.iter_content(chunk_size=io.DEFAULT_BUFFER_SIZE) - fp = open(self._destination, "wb") # pylint: disable=consider-using-with try: - if not with_progress or self.get_size() == -1: - if not silent: - click.echo("%s..." % label) - for chunk in itercontent: - if chunk: + with open(self._destination, "wb") as fp: + if not with_progress or file_size == -1: + if not silent: + click.echo("%s..." % label) + for chunk in itercontent: fp.write(chunk) - else: - chunks = int(math.ceil(self.get_size() / float(io.DEFAULT_BUFFER_SIZE))) - with click.progressbar(length=chunks, label=label) as pb: - for _ in pb: - fp.write(next(itercontent)) + else: + pb = click.progressbar( + length=file_size, + iterable=itercontent, + label=label, + ) + pb.update_min_steps = file_size / (5 if pb.is_hidden else 100) + with pb: + for chunk in pb: + pb.update(len(chunk)) + fp.write(chunk) finally: - fp.close() self._request.close() if self.get_lmtime(): @@ -132,8 +136,7 @@ class FileDownloader: return True def _preserve_filemtime(self, lmdate): - timedata = parsedate_tz(lmdate) - lmtime = mktime(timedata[:9]) + lmtime = mktime(parsedate(lmdate)) fs.change_filemtime(self._destination, lmtime) def __del__(self): From b04c1591c2769ecb739ecb682b7f317e0ad6e54c Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 3 Aug 2022 20:31:56 +0300 Subject: [PATCH 23/39] Use "get_terminal_size().columns" directly --- platformio/check/cli.py | 2 +- platformio/commands/boards.py | 2 +- platformio/maintenance.py | 6 +++--- platformio/run/cli.py | 2 +- platformio/test/cli.py | 2 +- platformio/util.py | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/platformio/check/cli.py b/platformio/check/cli.py index 3033008e..e293e8e0 100644 --- a/platformio/check/cli.py +++ b/platformio/check/cli.py @@ -198,7 +198,7 @@ def print_processing_header(tool, envname, envdump): "Checking %s > %s (%s)" % (click.style(envname, fg="cyan", bold=True), tool, "; ".join(envdump)) ) - terminal_width, _ = shutil.get_terminal_size() + terminal_width = shutil.get_terminal_size().columns click.secho("-" * terminal_width, bold=True) diff --git a/platformio/commands/boards.py b/platformio/commands/boards.py index 2fe047e5..8ca008f1 100644 --- a/platformio/commands/boards.py +++ b/platformio/commands/boards.py @@ -41,7 +41,7 @@ def cli(query, installed, json_output): # pylint: disable=R0912 grpboards[board["platform"]] = [] grpboards[board["platform"]].append(board) - terminal_width, _ = shutil.get_terminal_size() + terminal_width = shutil.get_terminal_size().columns for (platform, boards) in sorted(grpboards.items()): click.echo("") click.echo("Platform: ", nl=False) diff --git a/platformio/maintenance.py b/platformio/maintenance.py index 304ebc4a..7893a23d 100644 --- a/platformio/maintenance.py +++ b/platformio/maintenance.py @@ -125,7 +125,7 @@ class Upgrader: def after_upgrade(ctx): - terminal_width, _ = shutil.get_terminal_size() + terminal_width = shutil.get_terminal_size().columns last_version = app.get_state_item("last_version", "0.0.0") if last_version == __version__: return @@ -221,7 +221,7 @@ def check_platformio_upgrade(): if pepver_to_semver(latest_version) <= pepver_to_semver(__version__): return - terminal_width, _ = shutil.get_terminal_size() + terminal_width = shutil.get_terminal_size().columns click.echo("") click.echo("*" * terminal_width) @@ -266,7 +266,7 @@ def check_prune_system(): if (unnecessary_size / 1024) < threshold_mb: return - terminal_width, _ = shutil.get_terminal_size() + terminal_width = shutil.get_terminal_size().columns click.echo() click.echo("*" * terminal_width) click.secho( diff --git a/platformio/run/cli.py b/platformio/run/cli.py index eb85b10c..adf0153f 100644 --- a/platformio/run/cli.py +++ b/platformio/run/cli.py @@ -228,7 +228,7 @@ def print_processing_header(env, config, verbose=False): "Processing %s (%s)" % (click.style(env, fg="cyan", bold=True), "; ".join(env_dump)) ) - terminal_width, _ = shutil.get_terminal_size() + terminal_width = shutil.get_terminal_size().columns click.secho("-" * terminal_width, bold=True) diff --git a/platformio/test/cli.py b/platformio/test/cli.py index bfb14fd6..1e456d4d 100644 --- a/platformio/test/cli.py +++ b/platformio/test/cli.py @@ -183,7 +183,7 @@ def print_suite_header(test_suite): click.style(test_suite.env_name, fg="cyan", bold=True), ) ) - terminal_width, _ = shutil.get_terminal_size() + terminal_width = shutil.get_terminal_size().columns click.secho("-" * terminal_width, bold=True) diff --git a/platformio/util.py b/platformio/util.py index 0bd26e40..63a16d6e 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -186,7 +186,7 @@ def merge_dicts(d1, d2, path=None): def print_labeled_bar(label, is_error=False, fg=None, sep="="): - terminal_width, _ = shutil.get_terminal_size() + terminal_width = shutil.get_terminal_size().columns width = len(click.unstyle(label)) half_line = sep * int((terminal_width - width - 2) / 2) click.secho("%s %s %s" % (half_line, label, half_line), fg=fg, err=is_error) From a4756987a42d515c62264bbb16af6bcb69054100 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 3 Aug 2022 20:33:24 +0300 Subject: [PATCH 24/39] Provide a simplified download progress in non-terminal environments --- platformio/compat.py | 7 +++++++ platformio/package/download.py | 30 ++++++++++++++++++++++++------ 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/platformio/compat.py b/platformio/compat.py index cdce3939..9cf9161e 100644 --- a/platformio/compat.py +++ b/platformio/compat.py @@ -50,6 +50,13 @@ def isascii(text): return True +def is_terminal(): + try: + return sys.stdout.isatty() + except Exception: # pylint: disable=broad-except + return False + + def ci_strings_are_equal(a, b): if a == b: return True diff --git a/platformio/package/download.py b/platformio/package/download.py index a0638404..b1800e4b 100644 --- a/platformio/package/download.py +++ b/platformio/package/download.py @@ -21,6 +21,7 @@ import click import requests from platformio import __default_requests_timeout__, app, fs +from platformio.compat import is_terminal from platformio.package.exception import PackageException @@ -75,19 +76,36 @@ class FileDownloader: itercontent = self._request.iter_content(chunk_size=io.DEFAULT_BUFFER_SIZE) try: with open(self._destination, "wb") as fp: - if not with_progress or file_size == -1: + if file_size == -1 or not with_progress or silent: if not silent: - click.echo("%s..." % label) + click.echo(f"{label}...") for chunk in itercontent: fp.write(chunk) + + elif not is_terminal(): + click.echo(f"{label} 0%", nl=False) + print_percent_step = 10 + printed_percents = 0 + downloaded_size = 0 + for chunk in itercontent: + fp.write(chunk) + downloaded_size += len(chunk) + if (downloaded_size / file_size * 100) >= ( + printed_percents + print_percent_step + ): + printed_percents += print_percent_step + click.echo(f" {printed_percents}%", nl=False) + click.echo("") + else: - pb = click.progressbar( + with click.progressbar( length=file_size, iterable=itercontent, label=label, - ) - pb.update_min_steps = file_size / (5 if pb.is_hidden else 100) - with pb: + update_min_steps=min( + 256 * 1024, file_size / 100 + ), # every 256Kb or less, + ) as pb: for chunk in pb: pb.update(len(chunk)) fp.write(chunk) From 59902abd092f7fc374862340bbd3ac22e582c5a8 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 4 Aug 2022 10:48:37 +0300 Subject: [PATCH 25/39] Keep support for legacy "IDE_EXTRA_DATA" // Resolve #4379 --- platformio/builder/tools/piointegration.py | 1 + 1 file changed, 1 insertion(+) diff --git a/platformio/builder/tools/piointegration.py b/platformio/builder/tools/piointegration.py index acdc66f3..d02d980f 100644 --- a/platformio/builder/tools/piointegration.py +++ b/platformio/builder/tools/piointegration.py @@ -185,6 +185,7 @@ def exists(_): def generate(env): + env["IDE_EXTRA_DATA"] = {} # legacy support env["INTEGRATION_EXTRA_DATA"] = {} env.AddMethod(IsIntegrationDump) env.AddMethod(DumpIntegrationIncludes) From 17a7293967eda4f366b3cb02f899e123bd249430 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 4 Aug 2022 10:48:58 +0300 Subject: [PATCH 26/39] Bump version to 6.1.4b3 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index b893a6dd..375b20a9 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (6, 1, "4b2") +VERSION = (6, 1, "4b3") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 32d501bed1faf839f0bbed96376cf58a705d9681 Mon Sep 17 00:00:00 2001 From: Benedikt Kleiner Date: Mon, 8 Aug 2022 14:22:42 +0200 Subject: [PATCH 27/39] optionally pass env to AddBuildMiddleware callback (#4380) --- platformio/builder/tools/piobuild.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/platformio/builder/tools/piobuild.py b/platformio/builder/tools/piobuild.py index 4a32a1d7..06937e58 100644 --- a/platformio/builder/tools/piobuild.py +++ b/platformio/builder/tools/piobuild.py @@ -292,7 +292,10 @@ def CollectBuildFiles( for callback, pattern in middlewares: if pattern and not fnmatch.fnmatch(node.srcnode().get_path(), pattern): continue - new_node = callback(new_node) + if callback.__code__.co_argcount == 2: + new_node = callback(env, new_node) + else: + new_node = callback(new_node) if new_node: new_sources.append(new_node) From 8c13d13f8067b371c0fbfc05c0ee24a7cfed82ee Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 8 Aug 2022 16:51:19 +0300 Subject: [PATCH 28/39] =?UTF-8?q?Added=20support=20for=20accepting=20the?= =?UTF-8?q?=20original=20FileNode=20environment=20in=20a=20=E2=80=9Ccallba?= =?UTF-8?q?ck=E2=80=9D=20function=20when=20using=20Build=20Middlewares=20/?= =?UTF-8?q?/=20Resolve=20#4380?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- HISTORY.rst | 1 + docs | 2 +- platformio/builder/tools/piobuild.py | 2 ++ 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index 7587f549..b6bd269b 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -16,6 +16,7 @@ PlatformIO Core 6 6.1.4 (2022-??-??) ~~~~~~~~~~~~~~~~~~ +* Added support for accepting the original FileNode environment in a "callback" function when using `Build Middlewares `__ (`pull #4380 `_) * Improved device port finder when using dual channel UART converter (`issue #4367 `_) * Improved project dependency resolving when using the `pio project init --ide `__ command * Upgraded build engine to the SCons 4.4.0 (`release notes `__) diff --git a/docs b/docs index 2175b369..f79c01f6 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 2175b369e206d000e1cca1f14ba2d309b358790a +Subproject commit f79c01f6f3825d413e9e6f92b9c3cba899860c8b diff --git a/platformio/builder/tools/piobuild.py b/platformio/builder/tools/piobuild.py index 06937e58..9652eabe 100644 --- a/platformio/builder/tools/piobuild.py +++ b/platformio/builder/tools/piobuild.py @@ -296,6 +296,8 @@ def CollectBuildFiles( new_node = callback(env, new_node) else: new_node = callback(new_node) + if not new_node: + break if new_node: new_sources.append(new_node) From dfffd5e97bb5370eeaea6ad035c9e268453e24c2 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 8 Aug 2022 17:39:36 +0300 Subject: [PATCH 29/39] =?UTF-8?q?Warn=20about=20calling=20=E2=80=9Cenv.Bui?= =?UTF-8?q?ldSources=E2=80=9D=20in=20a=20POST-type=20script=20//=20Resolve?= =?UTF-8?q?=20#4385?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- HISTORY.rst | 1 + platformio/builder/tools/piobuild.py | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/HISTORY.rst b/HISTORY.rst index b6bd269b..d51051d0 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -21,6 +21,7 @@ PlatformIO Core 6 * Improved project dependency resolving when using the `pio project init --ide `__ command * Upgraded build engine to the SCons 4.4.0 (`release notes `__) * Do not resolve project dependencies on for the ``cleanall`` target (`issue #4344 `_) +* Warn about calling "env.BuildSources" in a POST-type script (`issue #4385 `_) * Fixed an issue when escaping macros/defines for IDE integration (`issue #4360 `_) 6.1.3 (2022-07-18) diff --git a/platformio/builder/tools/piobuild.py b/platformio/builder/tools/piobuild.py index 9652eabe..69359a32 100644 --- a/platformio/builder/tools/piobuild.py +++ b/platformio/builder/tools/piobuild.py @@ -341,6 +341,14 @@ def BuildLibrary(env, variant_dir, src_dir, src_filter=None, nodes=None): def BuildSources(env, variant_dir, src_dir, src_filter=None): + if env.get("PIOMAINPROG"): + sys.stderr.write( + "Error: The main program is already constructed and the inline " + "source files are not allowed. Please use `env.BuildLibrary(...)` " + "or PRE-type script instead." + ) + env.Exit(1) + nodes = env.CollectBuildFiles(variant_dir, src_dir, src_filter) DefaultEnvironment().Append( PIOBUILDFILES=[ From 4e78c3ec40a07985a177eefa642fad1c9fed9ffd Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 9 Aug 2022 15:45:37 +0300 Subject: [PATCH 30/39] Improve docs for Github Actions --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index f79c01f6..1961fcd4 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit f79c01f6f3825d413e9e6f92b9c3cba899860c8b +Subproject commit 1961fcd4a38f8c6150470311f32b876efb85d3b0 From e2f0d96f09bb0993b3e3c1ee9dba439e209907cd Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 9 Aug 2022 15:46:19 +0300 Subject: [PATCH 31/39] Formatting --- platformio/builder/tools/piobuild.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/builder/tools/piobuild.py b/platformio/builder/tools/piobuild.py index 69359a32..32a744e3 100644 --- a/platformio/builder/tools/piobuild.py +++ b/platformio/builder/tools/piobuild.py @@ -346,7 +346,7 @@ def BuildSources(env, variant_dir, src_dir, src_filter=None): "Error: The main program is already constructed and the inline " "source files are not allowed. Please use `env.BuildLibrary(...)` " "or PRE-type script instead." - ) + ) env.Exit(1) nodes = env.CollectBuildFiles(variant_dir, src_dir, src_filter) From bc9d9ac2db09421dba84c323683dc4b5baecdf24 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 9 Aug 2022 15:47:04 +0300 Subject: [PATCH 32/39] Keep custom "unwantedRecommendations" when generating projects for VSCode // Resolve #4383 --- HISTORY.rst | 3 ++- .../tpls/vscode/.vscode/extensions.json.tpl | 13 +++++++++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index d51051d0..0acfa211 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -20,7 +20,8 @@ PlatformIO Core 6 * Improved device port finder when using dual channel UART converter (`issue #4367 `_) * Improved project dependency resolving when using the `pio project init --ide `__ command * Upgraded build engine to the SCons 4.4.0 (`release notes `__) -* Do not resolve project dependencies on for the ``cleanall`` target (`issue #4344 `_) +* Keep custom "unwantedRecommendations" when generating projects for VSCode (`issue #4383 `_) +* Do not resolve project dependencies for the ``cleanall`` target (`issue #4344 `_) * Warn about calling "env.BuildSources" in a POST-type script (`issue #4385 `_) * Fixed an issue when escaping macros/defines for IDE integration (`issue #4360 `_) diff --git a/platformio/project/integration/tpls/vscode/.vscode/extensions.json.tpl b/platformio/project/integration/tpls/vscode/.vscode/extensions.json.tpl index 26412ae7..8d8e8a02 100644 --- a/platformio/project/integration/tpls/vscode/.vscode/extensions.json.tpl +++ b/platformio/project/integration/tpls/vscode/.vscode/extensions.json.tpl @@ -3,13 +3,20 @@ % import re % % recommendations = set(["platformio.platformio-ide"]) +% unwantedRecommendations = set(["ms-vscode.cpptools-extension-pack"]) % previous_json = os.path.join(project_dir, ".vscode", "extensions.json") % if os.path.isfile(previous_json): % fp = open(previous_json) % contents = re.sub(r"^\s*//.*$", "", fp.read(), flags=re.M).strip() % fp.close() % if contents: -% recommendations |= set(json.loads(contents).get("recommendations", [])) +% try: +% data = json.loads(contents) +% recommendations |= set(data.get("recommendations", [])) +% unwantedRecommendations |= set(data.get("unwantedRecommendations", [])) +% except ValueError: +% pass +% end % end % end { @@ -21,6 +28,8 @@ % end ], "unwantedRecommendations": [ - "ms-vscode.cpptools-extension-pack" +% for i, item in enumerate(sorted(unwantedRecommendations)): + "{{ item }}"{{ ("," if (i + 1) < len(unwantedRecommendations) else "") }} +% end ] } From c56b35f5047d1a8c6837e30e6333ad22f595b783 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 9 Aug 2022 15:47:38 +0300 Subject: [PATCH 33/39] Bump version to 6.1.4b4 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 375b20a9..019bae5c 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (6, 1, "4b3") +VERSION = (6, 1, "4b4") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 847fdd4deb1c66cea74b28e8f1358314b3818958 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 10 Aug 2022 15:05:31 +0300 Subject: [PATCH 34/39] Fixed an issue when the "cleanall" target removes dependencies from all working environments // Resolve #4386 --- HISTORY.rst | 1 + platformio/builder/tools/piotarget.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index 0acfa211..c7b8f88e 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -24,6 +24,7 @@ PlatformIO Core 6 * Do not resolve project dependencies for the ``cleanall`` target (`issue #4344 `_) * Warn about calling "env.BuildSources" in a POST-type script (`issue #4385 `_) * Fixed an issue when escaping macros/defines for IDE integration (`issue #4360 `_) +* Fixed an issue when the "cleanall" target removes dependencies from all working environments (`issue #4386 `_) 6.1.3 (2022-07-18) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/builder/tools/piotarget.py b/platformio/builder/tools/piotarget.py index d9f616a5..51333e4d 100644 --- a/platformio/builder/tools/piotarget.py +++ b/platformio/builder/tools/piotarget.py @@ -45,7 +45,7 @@ def PioClean(env, clean_all=False): fs.rmtree(path) build_dir = env.subst("$BUILD_DIR") - libdeps_dir = env.subst("$PROJECT_LIBDEPS_DIR") + libdeps_dir = env.subst(os.path.join("$PROJECT_LIBDEPS_DIR", "$PIOENV")) if os.path.isdir(build_dir): _clean_dir(build_dir) else: From e3796cfda1d93c0e4974e212054ea1a6a55ca6bb Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 12 Aug 2022 12:46:55 +0300 Subject: [PATCH 35/39] Allow custom system for the package manager API --- platformio/package/manager/_registry.py | 19 ++++++++++++------- platformio/package/manager/base.py | 4 ++-- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/platformio/package/manager/_registry.py b/platformio/package/manager/_registry.py index 4015cffb..494e2d24 100644 --- a/platformio/package/manager/_registry.py +++ b/platformio/package/manager/_registry.py @@ -41,7 +41,7 @@ class PackageManagerRegistryMixin: if not package or not version: raise UnknownPackageError(spec.humanize()) - pkgfile = self._pick_compatible_pkg_file(version["files"]) if version else None + pkgfile = self.pick_compatible_pkg_file(version["files"]) if version else None if not pkgfile: raise UnknownPackageError(spec.humanize()) @@ -162,7 +162,7 @@ class PackageManagerRegistryMixin: time.sleep(1) return (None, None) - def filter_incompatible_registry_versions(self, versions, spec=None): + def get_compatible_registry_versions(self, versions, spec=None, custom_system=None): assert not spec or isinstance(spec, PackageSpec) result = [] for version in versions: @@ -170,22 +170,27 @@ class PackageManagerRegistryMixin: if spec and spec.requirements and semver not in spec.requirements: continue if not any( - self.is_system_compatible(f.get("system")) for f in version["files"] + self.is_system_compatible(f.get("system"), custom_system=custom_system) + for f in version["files"] ): continue result.append(version) return result - def pick_best_registry_version(self, versions, spec=None): + def pick_best_registry_version(self, versions, spec=None, custom_system=None): best = None - for version in self.filter_incompatible_registry_versions(versions, spec): + for version in self.get_compatible_registry_versions( + versions, spec, custom_system + ): semver = cast_version_to_semver(version["name"]) if not best or (semver > cast_version_to_semver(best["name"])): best = version return best - def _pick_compatible_pkg_file(self, version_files): + def pick_compatible_pkg_file(self, version_files, custom_system=None): for item in version_files: - if self.is_system_compatible(item.get("system")): + if self.is_system_compatible( + item.get("system"), custom_system=custom_system + ): return item return None diff --git a/platformio/package/manager/base.py b/platformio/package/manager/base.py index 8369f736..c4d8d5bd 100644 --- a/platformio/package/manager/base.py +++ b/platformio/package/manager/base.py @@ -116,10 +116,10 @@ class BasePackageManager( # pylint: disable=too-many-public-methods,too-many-in self._MEMORY_CACHE.clear() @staticmethod - def is_system_compatible(value): + def is_system_compatible(value, custom_system=None): if not value or "*" in value: return True - return util.items_in_list(value, util.get_systype()) + return util.items_in_list(value, custom_system or util.get_systype()) @staticmethod def ensure_dir_exists(path): From c5d7c4f88ea302519ef8599d029a8ef7f72323ed Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 12 Aug 2022 12:47:16 +0300 Subject: [PATCH 36/39] Update SPDX list to v3.18 --- platformio/package/manifest/schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/package/manifest/schema.py b/platformio/package/manifest/schema.py index 5b74033c..1e5f935a 100644 --- a/platformio/package/manifest/schema.py +++ b/platformio/package/manifest/schema.py @@ -276,7 +276,7 @@ class ManifestSchema(BaseSchema): @staticmethod @memoized(expire="1h") def load_spdx_licenses(): - version = "3.17" + version = "3.18" spdx_data_url = ( "https://raw.githubusercontent.com/spdx/license-list-data/" "v%s/json/licenses.json" % version From 10136729ab1bddc8b1b73fb9aeb5a2e08a0f4fc9 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 12 Aug 2022 14:59:37 +0300 Subject: [PATCH 37/39] Increase expire time --- tests/package/test_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/package/test_manager.py b/tests/package/test_manager.py index 896a7231..cd1fc1a2 100644 --- a/tests/package/test_manager.py +++ b/tests/package/test_manager.py @@ -44,7 +44,7 @@ def test_download(isolated_pio_core): lm.cleanup_expired_downloads(time.time()) assert os.path.isfile(archive_path) # test outdated downloads - lm.set_download_utime(archive_path, time.time() - lm.DOWNLOAD_CACHE_EXPIRE - 1) + lm.set_download_utime(archive_path, time.time() - lm.DOWNLOAD_CACHE_EXPIRE - 60) lm.cleanup_expired_downloads(time.time()) assert not os.path.isfile(archive_path) # check that key is deleted from DB From cd8179a41fe0f73a1f54bc1924c48274d8a545f2 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 12 Aug 2022 15:00:05 +0300 Subject: [PATCH 38/39] Bump version to 6.1.4 --- HISTORY.rst | 2 +- docs | 2 +- platformio/__init__.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index c7b8f88e..c59180be 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -13,7 +13,7 @@ PlatformIO Core 6 **A professional collaborative platform for declarative, safety-critical, and test-driven embedded development.** -6.1.4 (2022-??-??) +6.1.4 (2022-08-12) ~~~~~~~~~~~~~~~~~~ * Added support for accepting the original FileNode environment in a "callback" function when using `Build Middlewares `__ (`pull #4380 `_) diff --git a/docs b/docs index 1961fcd4..b38923e3 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 1961fcd4a38f8c6150470311f32b876efb85d3b0 +Subproject commit b38923e39bc9caa219d5816a41c65f821882c9c1 diff --git a/platformio/__init__.py b/platformio/__init__.py index 019bae5c..bce9bfd8 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (6, 1, "4b4") +VERSION = (6, 1, 4) __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 43dd429aa209b990a461b4401efe18846467bf49 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 12 Aug 2022 16:54:44 +0300 Subject: [PATCH 39/39] Use random to remove a cache --- tests/package/test_manager.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/package/test_manager.py b/tests/package/test_manager.py index cd1fc1a2..c1de90b1 100644 --- a/tests/package/test_manager.py +++ b/tests/package/test_manager.py @@ -18,6 +18,7 @@ import logging import os import time from pathlib import Path +from random import random import pytest import semantic_version @@ -41,11 +42,11 @@ def test_download(isolated_pio_core): lm.set_log_level(logging.ERROR) archive_path = lm.download(url, checksum) assert fs.calculate_file_hashsum("sha256", archive_path) == checksum - lm.cleanup_expired_downloads(time.time()) + lm.cleanup_expired_downloads(random()) assert os.path.isfile(archive_path) # test outdated downloads - lm.set_download_utime(archive_path, time.time() - lm.DOWNLOAD_CACHE_EXPIRE - 60) - lm.cleanup_expired_downloads(time.time()) + lm.set_download_utime(archive_path, time.time() - lm.DOWNLOAD_CACHE_EXPIRE - 1) + lm.cleanup_expired_downloads(random()) assert not os.path.isfile(archive_path) # check that key is deleted from DB with open(lm.get_download_usagedb_path(), encoding="utf8") as fp: