From 9768b860956d7470448a6bf27e91b09f33b81569 Mon Sep 17 00:00:00 2001 From: Alexey Gerenkov Date: Fri, 14 Mar 2025 18:47:56 +0300 Subject: [PATCH 1/3] feat(tools): Update Clang version to esp-19.1.2_20250312 Closes https://github.com/espressif/esp-idf/issues/15731 --- tools/tools.json | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/tools/tools.json b/tools/tools.json index a813f02257..afffee87f8 100644 --- a/tools/tools.json +++ b/tools/tools.json @@ -266,36 +266,36 @@ "versions": [ { "linux-amd64": { - "sha256": "aee15b8e02440f9ec6a8070f017621dc400dbd62a4701f9cf456dbe34d2a0c4d", - "size": 309829872, - "url": "https://github.com/espressif/llvm-project/releases/download/esp-18.1.2_20240912/clang-esp-18.1.2_20240912-x86_64-linux-gnu.tar.xz" + "sha256": "8546cd8ac0596835fbe3970d7c8ed6a842713ab948b759525756867f3bc0a5ef", + "size": 332799436, + "url": "https://github.com/espressif/llvm-project/releases/download/esp-19.1.2_20250312/clang-esp-19.1.2_20250312-x86_64-linux-gnu.tar.xz" }, "linux-arm64": { - "sha256": "14abbc368d9c153270aa4d22ce28d78633cb0f1ca83d4be70591d9e39ae9bc82", - "size": 295561044, - "url": "https://github.com/espressif/llvm-project/releases/download/esp-18.1.2_20240912/clang-esp-18.1.2_20240912-aarch64-linux-gnu.tar.xz" + "sha256": "02dcde2e9e71706f36f64a4dc551085eb155d08714aaefc251c05fa5c4e743ae", + "size": 313340976, + "url": "https://github.com/espressif/llvm-project/releases/download/esp-19.1.2_20250312/clang-esp-19.1.2_20250312-aarch64-linux-gnu.tar.xz" }, "linux-armhf": { - "sha256": "4133285303aabb1831c477536a13413319a569170b0aa54b92abe69cc0e7b938", - "size": 298186872, - "url": "https://github.com/espressif/llvm-project/releases/download/esp-18.1.2_20240912/clang-esp-18.1.2_20240912-arm-linux-gnueabihf.tar.xz" + "sha256": "cd79d1b92185adb91711909de96bf4a3ad1df214a3040013446ab36ccd533a18", + "size": 316011700, + "url": "https://github.com/espressif/llvm-project/releases/download/esp-19.1.2_20250312/clang-esp-19.1.2_20250312-arm-linux-gnueabihf.tar.xz" }, "macos": { - "sha256": "b4641ec4dd574b6b7d037aa1bb2e5ff5a8a4623c88e89668db656282eb1d9dc8", - "size": 282868024, - "url": "https://github.com/espressif/llvm-project/releases/download/esp-18.1.2_20240912/clang-esp-18.1.2_20240912-x86_64-apple-darwin.tar.xz" + "sha256": "d9832307626ebf04115b7b51caa059f8caff8875b35e68000ead7f02c690aa82", + "size": 297033276, + "url": "https://github.com/espressif/llvm-project/releases/download/esp-19.1.2_20250312/clang-esp-19.1.2_20250312-x86_64-apple-darwin.tar.xz" }, "macos-arm64": { - "sha256": "5d2e187ef40ecc9996630a7c6efcc19bdfd32ec4ce8cc4dd3014cd24e7016560", - "size": 261757404, - "url": "https://github.com/espressif/llvm-project/releases/download/esp-18.1.2_20240912/clang-esp-18.1.2_20240912-aarch64-apple-darwin.tar.xz" + "sha256": "f67ae3c07d0d32685ee05b6add063f37e799a7bd14af86cb3998da237709c57f", + "size": 272383664, + "url": "https://github.com/espressif/llvm-project/releases/download/esp-19.1.2_20250312/clang-esp-19.1.2_20250312-aarch64-apple-darwin.tar.xz" }, - "name": "esp-18.1.2_20240912", + "name": "esp-19.1.2_20250312", "status": "recommended", "win64": { - "sha256": "c4af15073b105dc174c0452dfd1875bab200412fa3151c0363cfc0d30abf5173", - "size": 429583224, - "url": "https://github.com/espressif/llvm-project/releases/download/esp-18.1.2_20240912/clang-esp-18.1.2_20240912-x86_64-w64-mingw32.tar.xz" + "sha256": "0fb20439a3d1d31088e646a6f2a8c9277ac6aed858188e2afa470bdb42e13afb", + "size": 465651960, + "url": "https://github.com/espressif/llvm-project/releases/download/esp-19.1.2_20250312/clang-esp-19.1.2_20250312-x86_64-w64-mingw32.tar.xz" } } ] From e7423ebbccc6aebaf520c3ee223be6e5f1faa612 Mon Sep 17 00:00:00 2001 From: Alexey Gerenkov Date: Fri, 14 Mar 2025 18:49:15 +0300 Subject: [PATCH 2/3] feat(tools): Add standalone Clang libraries to tools.json --- docs/en/api-guides/tools/idf-tools-notes.inc | 5 + .../api-guides/tools/idf-tools-notes.inc | 5 + tools/idf_tools.py | 94 ++++++++++++++----- tools/tools.json | 63 ++++++++++++- tools/tools_schema.json | 25 +++++ 5 files changed, 168 insertions(+), 24 deletions(-) diff --git a/docs/en/api-guides/tools/idf-tools-notes.inc b/docs/en/api-guides/tools/idf-tools-notes.inc index 4d0c863923..7baaca01a9 100644 --- a/docs/en/api-guides/tools/idf-tools-notes.inc +++ b/docs/en/api-guides/tools/idf-tools-notes.inc @@ -43,6 +43,11 @@ On Linux and macOS, it is recommended to install CMake using the OS-specific pac .. tool-esp-clang-notes +--- + +.. tool-esp-clang-libs-notes + + --- .. tool-ninja-notes diff --git a/docs/zh_CN/api-guides/tools/idf-tools-notes.inc b/docs/zh_CN/api-guides/tools/idf-tools-notes.inc index 9097786aaf..73582b6f98 100644 --- a/docs/zh_CN/api-guides/tools/idf-tools-notes.inc +++ b/docs/zh_CN/api-guides/tools/idf-tools-notes.inc @@ -43,6 +43,11 @@ On Linux and macOS, it is recommended to install CMake using the OS-specific pac .. tool-esp-clang-notes +--- + +.. tool-esp-clang-libs-notes + + --- .. tool-ninja-notes diff --git a/tools/idf_tools.py b/tools/idf_tools.py index 0e839f8746..e7667b5e06 100755 --- a/tools/idf_tools.py +++ b/tools/idf_tools.py @@ -82,7 +82,7 @@ TOOLS_FILE = 'tools/tools.json' TOOLS_SCHEMA_FILE = 'tools/tools_schema.json' TOOLS_FILE_NEW = 'tools/tools.new.json' IDF_ENV_FILE = 'idf-env.json' -TOOLS_FILE_VERSION = 2 +TOOLS_FILE_VERSION = 3 IDF_TOOLS_PATH_DEFAULT = os.path.join('~', '.espressif') UNKNOWN_VERSION = 'unknown' SUBST_TOOL_PATH_REGEX = re.compile(r'\${TOOL_PATH}') @@ -789,6 +789,7 @@ IDFToolOptions = namedtuple('IDFToolOptions', [ 'version_regex', 'version_regex_replace', 'is_executable', + 'tool_info_file', 'export_paths', 'export_vars', 'install', @@ -818,7 +819,8 @@ class IDFTool(object): supported_targets: List[str], version_regex_replace: Optional[str] = None, strip_container_dirs: int = 0, - is_executable: bool = True) -> None: + is_executable: bool = True, + tool_info_file: str = '') -> None: self.name = name self.description = description self.drop_versions() @@ -826,12 +828,13 @@ class IDFTool(object): self.versions_installed: List[str] = [] if version_regex_replace is None: version_regex_replace = VERSION_REGEX_REPLACE_DEFAULT - self.options = IDFToolOptions(version_cmd, version_regex, version_regex_replace, is_executable, + self.options = IDFToolOptions(version_cmd, version_regex, version_regex_replace, is_executable, tool_info_file, [], OrderedDict(), install, info_url, license, strip_container_dirs, supported_targets) # type: ignore self.platform_overrides: List[Dict[str, str]] = [] self._platform = CURRENT_PLATFORM self._update_current_options() self.is_executable = is_executable + self.tool_info_file = tool_info_file def copy_for_platform(self, platform: str) -> 'IDFTool': """ @@ -905,6 +908,16 @@ class IDFTool(object): result[k] = v_repl return result + def parse_tool_version(self, ver_str: str) -> str: + """ + Extract the version string from the provided input and return it as a result. + Returns 'unknown' if version string can not be extracted.. + """ + match = re.search(self._current_options.version_regex, ver_str) # type: ignore + if not match: + return UNKNOWN_VERSION + return re.sub(self._current_options.version_regex, self._current_options.version_regex_replace, match.group(0)) # type: ignore + def get_version(self, extra_paths: Optional[List[str]] = None, executable_path: Optional[str] = None) -> str: """ Execute the tool, optionally prepending extra_paths to PATH, @@ -934,17 +947,39 @@ class IDFTool(object): except subprocess.CalledProcessError as e: raise ToolExecError(f'non-zero exit code ({e.returncode}) with message: {e.stderr.decode("utf-8",errors="ignore")}') # type: ignore - in_str = version_cmd_result.decode('utf-8') - match = re.search(self._current_options.version_regex, in_str) # type: ignore - if not match: - return UNKNOWN_VERSION - return re.sub(self._current_options.version_regex, self._current_options.version_regex_replace, match.group(0)) # type: ignore + return self.parse_tool_version(version_cmd_result.decode('utf-8')) + + def get_version_from_file(self, version: str) -> str: + """ + Extract the version string from tool info file and return it as a result. + Returns 'unknown' if version string can not be extracted. + """ + # this function can not be called for a different platform + assert self._platform == CURRENT_PLATFORM + # Replace '/' with OS specific path separator + info_file_path = os.path.join(*self.tool_info_file.split('/')) + info_file_path = os.path.join(self.get_path_for_version(version), info_file_path) + if not os.path.exists(info_file_path): + raise ToolNotFoundError(f'Tool {self.name} not found: No info file.') + with open(info_file_path, 'r', encoding='utf-8') as f: + try: + tool_info = json.load(f) + except (json.JSONDecodeError, UnicodeDecodeError): + raise ToolNotFoundError(f'Tool {self.name} not found: Bad info file.') + + if 'version' not in tool_info: + raise ToolNotFoundError(f'Tool {self.name} not found: No version in info file.') + + return self.parse_tool_version(tool_info['version']) def check_binary_valid(self, version: str) -> bool: if not self.is_executable: return True try: - ver_str = self.get_version(self.get_export_paths(version)) + if self.tool_info_file: + ver_str = self.get_version_from_file(version) + else: + ver_str = self.get_version(self.get_export_paths(version)) except (ToolNotFoundError, ToolExecError) as e: fatal(f'tool {self.name} version {version} is installed, but getting error: {e}') return False @@ -1028,17 +1063,18 @@ class IDFTool(object): # this function can not be called for a different platform assert self._platform == CURRENT_PLATFORM tool_error = False - # First check if the tool is in system PATH - try: - ver_str = self.get_version() - except ToolNotFoundError: - # not in PATH - pass - except ToolExecError as e: - fatal(f'tool {self.name} is found in PATH, but has failed: {e}') - tool_error = True - else: - self.version_in_path = ver_str + if not self.tool_info_file: + # First check if the tool is in system PATH + try: + ver_str = self.get_version() + except ToolNotFoundError: + # not in PATH + pass + except ToolExecError as e: + fatal(f'tool {self.name} is found in PATH, but has failed: {e}') + tool_error = True + else: + self.version_in_path = ver_str # Now check all the versions installed in GlobalVarsStore.idf_tools_path self.versions_installed = [] @@ -1053,7 +1089,10 @@ class IDFTool(object): self.versions_installed.append(version) continue try: - ver_str = self.get_version(self.get_export_paths(version)) + if self.tool_info_file: + ver_str = self.get_version_from_file(version) + else: + ver_str = self.get_version(self.get_export_paths(version)) except ToolNotFoundError as e: warn(f'directory for tool {self.name} version {version} is present, but the tool has not been found: {e}') except ToolExecError as e: @@ -1083,7 +1122,10 @@ class IDFTool(object): # export paths list directly here. paths = [os.path.join(tool_path, version, *p) for p in self._current_options.export_paths] try: - ver_str = self.get_version(paths) + if self.tool_info_file: + ver_str = self.get_version_from_file(version) + else: + ver_str = self.get_version(paths) except (ToolNotFoundError, ToolExecError): continue if ver_str != version: @@ -1191,6 +1233,10 @@ class IDFTool(object): if not isinstance(is_executable, bool): raise RuntimeError(f'is_executable for tool {tool_name} is not a bool') + tool_info_file = tool_dict.get('tool_info_file', '') + if not isinstance(tool_info_file, str): + raise RuntimeError(f'tool_info_file for tool {tool_name} is not a string') + version_cmd = tool_dict.get('version_cmd') if type(version_cmd) is not list: raise RuntimeError(f'version_cmd for tool {tool_name} is not a list of strings') @@ -1242,7 +1288,7 @@ class IDFTool(object): # Create the object tool_obj: 'IDFTool' = cls(tool_name, description, install, info_url, license, # type: ignore version_cmd, version_regex, supported_targets, version_regex_replace, # type: ignore - strip_container_dirs, is_executable) # type: ignore + strip_container_dirs, is_executable, tool_info_file) # type: ignore for path in export_paths: # type: ignore tool_obj.options.export_paths.append(path) # type: ignore @@ -1371,6 +1417,8 @@ class IDFTool(object): tool_json['strip_container_dirs'] = self.options.strip_container_dirs if self.options.is_executable is False: tool_json['is_executable'] = self.options.is_executable + if self.options.tool_info_file: + tool_json['tool_info_file'] = self.options.tool_info_file return tool_json diff --git a/tools/tools.json b/tools/tools.json index afffee87f8..066a90c9c3 100644 --- a/tools/tools.json +++ b/tools/tools.json @@ -300,6 +300,67 @@ } ] }, + { + "description": "Standalone Clang shared libraries distribution", + "export_paths": [], + "export_vars": {}, + "info_url": "https://github.com/espressif/llvm-project", + "install": "on_request", + "license": "Apache-2.0", + "name": "esp-clang-libs", + "supported_targets": [ + "esp32", + "esp32s2", + "esp32s3", + "esp32c3", + "esp32c2", + "esp32c6", + "esp32c5", + "esp32h2", + "esp32p4", + "esp32c61", + "esp32h21" + ], + "tool_info_file": "esp-clang/esp-clang-libs.info", + "version_cmd": [], + "version_regex": "([0-9a-zA-Z\\.\\-_]+)", + "versions": [ + { + "linux-amd64": { + "sha256": "c13b8a731beec0c2af56f94073ca17202c9e5047de4c6e61f5e3ed319d231962", + "size": 76164612, + "url": "https://github.com/espressif/llvm-project/releases/download/esp-19.1.2_20250312/libs-clang-esp-19.1.2_20250312-x86_64-linux-gnu.tar.xz" + }, + "linux-arm64": { + "sha256": "dcf356d2a12eaf9660aa5e91cd70df14486d4bd45f47a8025327255b2fe3ece8", + "size": 69938392, + "url": "https://github.com/espressif/llvm-project/releases/download/esp-19.1.2_20250312/libs-clang-esp-19.1.2_20250312-aarch64-linux-gnu.tar.xz" + }, + "linux-armhf": { + "sha256": "e9e110841ffdd43c7fcc57de24f501eec917a02e54f6dbf3a2069fed1017f8a8", + "size": 72698812, + "url": "https://github.com/espressif/llvm-project/releases/download/esp-19.1.2_20250312/libs-clang-esp-19.1.2_20250312-arm-linux-gnueabihf.tar.xz" + }, + "macos": { + "sha256": "0ac4cf2340e766a240ae40519f404fceaea4262d5ab89292aa3a4118e6402492", + "size": 58104496, + "url": "https://github.com/espressif/llvm-project/releases/download/esp-19.1.2_20250312/libs-clang-esp-19.1.2_20250312-x86_64-apple-darwin.tar.xz" + }, + "macos-arm64": { + "sha256": "d723ce5ae431b44305c888c989e7f405446a23c13f179edc1b6939f9ba437498", + "size": 48707440, + "url": "https://github.com/espressif/llvm-project/releases/download/esp-19.1.2_20250312/libs-clang-esp-19.1.2_20250312-aarch64-apple-darwin.tar.xz" + }, + "name": "esp-19.1.2_20250312", + "status": "recommended", + "win64": { + "sha256": "29e5c945f601b85dd88c29bfabfd8c592d8064e3aba01d946e0ff475e5810c17", + "size": 59681772, + "url": "https://github.com/espressif/llvm-project/releases/download/esp-19.1.2_20250312/libs-clang-esp-19.1.2_20250312-x86_64-w64-mingw32.tar.xz" + } + } + ] + }, { "description": "Toolchain for 32-bit RISC-V based on GCC", "export_paths": [ @@ -985,5 +1046,5 @@ ] } ], - "version": 2 + "version": 3 } diff --git a/tools/tools_schema.json b/tools/tools_schema.json index 5c3efad012..c76eb6142d 100644 --- a/tools/tools_schema.json +++ b/tools/tools_schema.json @@ -72,6 +72,10 @@ "description": "If given, this will be used as substitute expression for the regex defined in version_regex, to obtain the version string. Not specifying this is equivalent to setting it to '\\1' (i.e. return the first capture group).", "type": "string" }, + "tool_info_file": { + "description": "Tools info file path. Relative to tool install folder. If present tool version will be read from it instead of invoking the binary.", + "type": "string" + }, "strip_container_dirs": { "type": "integer", "description": "If specified, this number of top directory levels will removed when extracting. E.g. if strip_container_dirs=2, archive path a/b/c/d.txt will be extracted as c/d.txt" @@ -286,5 +290,26 @@ } } } + }, + "$comment": "Condition verifying that version <=2 of tools.json must not contain 'tool_info_file' keyword in any of #/definitions/toolInfo properties.", + "if": { + "properties": { + "version": { + "maximum": 2 + } + } + }, + "then": { + "properties": { + "tools": { + "items": { + "not": { + "required": [ + "tool_info_file" + ] + } + } + } + } } } From 0cb7eb046551c409ab2bb9785dbe2c4ed5d20a5e Mon Sep 17 00:00:00 2001 From: Alexey Gerenkov Date: Fri, 4 Apr 2025 18:45:25 +0300 Subject: [PATCH 3/3] fix(tools): Fix linter errors in 'idf_tools.py' --- tools/idf_tools.py | 870 +++++++++++++++++++++++++++++---------------- 1 file changed, 566 insertions(+), 304 deletions(-) diff --git a/tools/idf_tools.py b/tools/idf_tools.py index e7667b5e06..7bd77d1e50 100755 --- a/tools/idf_tools.py +++ b/tools/idf_tools.py @@ -46,8 +46,8 @@ import sys import tarfile import tempfile import time -from collections import namedtuple from collections import OrderedDict +from collections import namedtuple from json import JSONEncoder from ssl import SSLContext from tarfile import TarFile @@ -64,9 +64,19 @@ except RuntimeError as e: print(e) raise SystemExit(1) -from typing import IO, Any, Callable, Dict, Iterator, List, Optional, Set, Tuple, Union +from typing import IO +from typing import Any +from typing import Callable +from typing import Dict +from typing import Iterator +from typing import List +from typing import Optional +from typing import Set +from typing import Tuple +from typing import Union from urllib.error import ContentTooShortError -from urllib.parse import urljoin, urlparse +from urllib.parse import urljoin +from urllib.parse import urlparse from urllib.request import urlopen from urllib.response import addinfourl @@ -106,6 +116,7 @@ class GlobalVarsStore: One global instance of this class is initialized and used as an entrypoint (store) It handles string and boolean properties. """ + _instance: Optional['GlobalVarsStore'] = None _bool_properties = ['quiet', 'non_interactive'] _string_properties = ['idf_path', 'idf_tools_path', 'tools_json'] @@ -158,7 +169,7 @@ def warn(text: str, *args: str) -> None: sys.stderr.write(f'WARNING: {text}\n', *args) -def info(text: str, f: Optional[IO[str]]=None, *args: str) -> None: +def info(text: str, f: Optional[IO[str]] = None, *args: str) -> None: """ Writes text to a stream specified by second arg, sys.stdout by default. """ @@ -186,8 +197,10 @@ def print_hints_on_download_error(err: str) -> None: # Certificate issue on Windows can be hidden under different errors which might be even translated, # e.g. "[WinError -2146881269] ASN1 valor de tag inválido encontrado" if sys.platform == 'win32': - info('By downloading and using the offline installer from https://dl.espressif.com/dl/esp-idf ' - 'you might be able to work around this issue.') + info( + 'By downloading and using the offline installer from https://dl.espressif.com/dl/esp-idf ' + 'you might be able to work around this issue.' + ) PYTHON_PLATFORM = f'{platform.system()}-{platform.machine()}' @@ -210,6 +223,7 @@ class Platforms: This includes strings produced from "platform.system() + '-' + platform.machine()", see PYTHON_PLATFORM definition above. """ + # Mappings from various other names these platforms are known as, to the identifiers above. # This includes strings produced from "platform.system() + '-' + platform.machine()", see PYTHON_PLATFORM # definition above. @@ -261,9 +275,7 @@ class Platforms: } # List of platforms that are not supported by ESP-IDF - UNSUPPORTED_PLATFORMS = [ - 'Linux-armv6l' - ] + UNSUPPORTED_PLATFORMS = ['Linux-armv6l'] @staticmethod def detect_linux_arm_platform(supposed_platform: Optional[str]) -> Optional[str]: @@ -283,7 +295,8 @@ class Platforms: # suppose that installed python was built with the right ABI with open(sys.executable, 'rb') as f: - # see ELF header description in https://man7.org/linux/man-pages/man5/elf.5.html, offsets depend on ElfN size + # see ELF header description in https://man7.org/linux/man-pages/man5/elf.5.html, + # offsets depend on ElfN size if int.from_bytes(f.read(4), sys.byteorder) != int.from_bytes(b'\x7fELF', sys.byteorder): return supposed_platform # ELF magic not found. Use the default platform name from PLATFORM_FROM_NAME f.seek(18) # seek to e_machine @@ -309,7 +322,7 @@ class Platforms: raise ValueError('System platform could not be identified.') if platform_alias in Platforms.UNSUPPORTED_PLATFORMS: - raise ValueError(f'Platform \'{platform_alias}\' is not supported by ESP-IDF.') + raise ValueError(f"Platform '{platform_alias}' is not supported by ESP-IDF.") if platform_alias == 'any' and CURRENT_PLATFORM: platform_alias = CURRENT_PLATFORM @@ -318,7 +331,7 @@ class Platforms: platform_name = Platforms.detect_linux_arm_platform(platform_name) if not platform_name: - raise ValueError(f'Support for platform \'{platform_alias}\' hasn\'t been added yet.') + raise ValueError(f"Support for platform '{platform_alias}' hasn't been added yet.") return platform_name @@ -407,11 +420,12 @@ MrY= -----END CERTIFICATE----- """ -DL_CERT_DICT = {'dl.espressif.com': DIGICERT_ROOT_G2_CERT, - 'github.com': DIGICERT_ROOT_CA_CERT} +DL_CERT_DICT = {'dl.espressif.com': DIGICERT_ROOT_G2_CERT, 'github.com': DIGICERT_ROOT_CA_CERT} -def run_cmd_check_output(cmd: List[str], input_text: Optional[str]=None, extra_paths: Optional[List[str]]=None) -> bytes: +def run_cmd_check_output( + cmd: List[str], input_text: Optional[str] = None, extra_paths: Optional[List[str]] = None +) -> bytes: """ Runs command and checks output for exceptions. If AttributeError or TypeError occurs, function re-runs the process. If return code was not 0, subprocess.CalledProcessError is raised, otherwise, the original error is masked. @@ -473,7 +487,7 @@ def get_env_for_extra_paths(extra_paths: List[str]) -> Dict[str, str]: return env_arg -def get_file_size_sha256(filename: str, block_size: int=65536) -> Tuple[int, str]: +def get_file_size_sha256(filename: str, block_size: int = 65536) -> Tuple[int, str]: """ Gets file size and its sha256. """ @@ -552,11 +566,13 @@ def splittype(url: str) -> Tuple[Optional[str], str]: return None, url -def urlretrieve_ctx(url: str, - filename: str, - reporthook: Optional[Callable[[int, int, int], None]]=None, - data: Optional[bytes]=None, - context: Optional[SSLContext]=None) -> Tuple[str, addinfourl]: +def urlretrieve_ctx( + url: str, + filename: str, + reporthook: Optional[Callable[[int, int, int], None]] = None, + data: Optional[bytes] = None, + context: Optional[SSLContext] = None, +) -> Tuple[str, addinfourl]: """ Retrieve data from given URL. An alternative version of urlretrieve which takes SSL context as an argument. """ @@ -598,9 +614,7 @@ def urlretrieve_ctx(url: str, reporthook(blocknum, bs, size) if size >= 0 and read < size: - raise ContentTooShortError( - 'retrieval incomplete: got only %i out of %i bytes' - % (read, size), result) + raise ContentTooShortError('retrieval incomplete: got only %i out of %i bytes' % (read, size), result) return result @@ -648,7 +662,9 @@ def rename_with_retry(path_from: str, path_to: str) -> None: except OSError: msg = f'Rename {path_from} to {path_to} failed' if retry == retry_count - 1: - fatal(f'{msg}. Antivirus software might be causing this. Disabling it temporarily could solve the issue.') + fatal( + f'{msg}. Antivirus software might be causing this. Disabling it temporarily could solve the issue.' + ) raise warn(f'{msg}, retrying...') # Sleep before the next try in order to pass the antivirus check on Windows @@ -689,6 +705,7 @@ class ToolNotFoundError(RuntimeError): """ Raise when the tool is not found (not present in the paths etc.). """ + pass @@ -696,13 +713,15 @@ class ToolExecError(RuntimeError): """ Raise when the tool returns with a non-zero exit code. """ + pass class ToolBinaryError(RuntimeError): - """" + """ " Raise when an error occurred when running any version of the tool. """ + pass @@ -710,6 +729,7 @@ class IDFToolDownload(object): """ Structure to store all the relevant information about particular download. """ + def __init__(self, platform_name: str, url: str, size: int, sha256: str, rename_dist: str) -> None: self.platform_name = platform_name self.url = url @@ -725,6 +745,7 @@ class IDFToolVersion(object): and easy way of comparing different versions. Also allows platform compatibility check and getting right download for given platform, if available. """ + STATUS_RECOMMENDED = 'recommended' STATUS_SUPPORTED = 'supported' STATUS_DEPRECATED = 'deprecated' @@ -741,8 +762,9 @@ class IDFToolVersion(object): if self.status != other.status: return self.status > other.status else: - assert not (self.status == IDFToolVersion.STATUS_RECOMMENDED - and other.status == IDFToolVersion.STATUS_RECOMMENDED) + assert not ( + self.status == IDFToolVersion.STATUS_RECOMMENDED and other.status == IDFToolVersion.STATUS_RECOMMENDED + ) return self.version < other.version def __eq__(self, other: object) -> bool: @@ -784,19 +806,23 @@ class IDFToolVersion(object): return set(self.downloads.keys()) -IDFToolOptions = namedtuple('IDFToolOptions', [ - 'version_cmd', - 'version_regex', - 'version_regex_replace', - 'is_executable', - 'tool_info_file', - 'export_paths', - 'export_vars', - 'install', - 'info_url', - 'license', - 'strip_container_dirs', - 'supported_targets']) +IDFToolOptions = namedtuple( + 'IDFToolOptions', + [ + 'version_cmd', + 'version_regex', + 'version_regex_replace', + 'is_executable', + 'tool_info_file', + 'export_paths', + 'export_vars', + 'install', + 'info_url', + 'license', + 'strip_container_dirs', + 'supported_targets', + ], +) class IDFTool(object): @@ -804,23 +830,27 @@ class IDFTool(object): Used to store info about IDF tools from tools.json file in a Python-accesible form. The term "IDF tool" is used for e.g. CMake, ninja, QUEMU and toolchains. """ + # possible values of 'install' field INSTALL_ALWAYS = 'always' INSTALL_ON_REQUEST = 'on_request' INSTALL_NEVER = 'never' - def __init__(self, name: str, - description: str, - install: str, - info_url: str, - license: str, - version_cmd: List[str], - version_regex: str, - supported_targets: List[str], - version_regex_replace: Optional[str] = None, - strip_container_dirs: int = 0, - is_executable: bool = True, - tool_info_file: str = '') -> None: + def __init__( + self, + name: str, + description: str, + install: str, + info_url: str, + lic: str, + version_cmd: List[str], + version_regex: str, + supported_targets: List[str], + version_regex_replace: Optional[str] = None, + strip_container_dirs: int = 0, + is_executable: bool = True, + tool_info_file: str = '', + ) -> None: self.name = name self.description = description self.drop_versions() @@ -828,8 +858,20 @@ class IDFTool(object): self.versions_installed: List[str] = [] if version_regex_replace is None: version_regex_replace = VERSION_REGEX_REPLACE_DEFAULT - self.options = IDFToolOptions(version_cmd, version_regex, version_regex_replace, is_executable, tool_info_file, - [], OrderedDict(), install, info_url, license, strip_container_dirs, supported_targets) # type: ignore + self.options = IDFToolOptions( + version_cmd, + version_regex, + version_regex_replace, + is_executable, + tool_info_file, + [], + OrderedDict(), + install, + info_url, + lic, + strip_container_dirs, + supported_targets, + ) # type: ignore self.platform_overrides: List[Dict[str, str]] = [] self._platform = CURRENT_PLATFORM self._update_current_options() @@ -945,7 +987,9 @@ class IDFTool(object): # tool is not on the path raise ToolNotFoundError(f'Tool {self.name} not found with error: {e}') except subprocess.CalledProcessError as e: - raise ToolExecError(f'non-zero exit code ({e.returncode}) with message: {e.stderr.decode("utf-8",errors="ignore")}') # type: ignore + raise ToolExecError( + f'non-zero exit code ({e.returncode}) with message: {e.stderr.decode("utf-8", errors="ignore")}' + ) # type: ignore return self.parse_tool_version(version_cmd_result.decode('utf-8')) @@ -1012,7 +1056,7 @@ class IDFTool(object): Checks whether the tool is suitable for at least one of the specified targets. """ supported_targets = self.get_supported_targets() - return (any(item in targets for item in supported_targets) or supported_targets == ['all']) + return any(item in targets for item in supported_targets) or supported_targets == ['all'] def compatible_with_platform(self) -> bool: """ @@ -1033,9 +1077,11 @@ class IDFTool(object): """ Get all recommended versions of the tool. If more versions are recommended, highest version is returned. """ - recommended_versions = [k for k, v in self.versions.items() - if v.status == IDFToolVersion.STATUS_RECOMMENDED - and v.compatible_with_platform(self._platform)] + recommended_versions = [ + k + for k, v in self.versions.items() + if v.status == IDFToolVersion.STATUS_RECOMMENDED and v.compatible_with_platform(self._platform) + ] assert len(recommended_versions) <= 1 if recommended_versions: return recommended_versions[0] @@ -1045,9 +1091,12 @@ class IDFTool(object): """ Get the preferred installed version of the tool. If more versions installed, return the highest. """ - recommended_versions = [k for k in self.versions_installed - if self.versions[k].status == IDFToolVersion.STATUS_RECOMMENDED - and self.versions[k].compatible_with_platform(self._platform)] + recommended_versions = [ + k + for k in self.versions_installed + if self.versions[k].status == IDFToolVersion.STATUS_RECOMMENDED + and self.versions[k].compatible_with_platform(self._platform) + ] assert len(recommended_versions) <= 1 if recommended_versions: return recommended_versions[0] @@ -1094,7 +1143,9 @@ class IDFTool(object): else: ver_str = self.get_version(self.get_export_paths(version)) except ToolNotFoundError as e: - warn(f'directory for tool {self.name} version {version} is present, but the tool has not been found: {e}') + warn( + f'directory for tool {self.name} version {version} is present, but the tool has not been found: {e}' + ) except ToolExecError as e: fatal(f'tool {self.name} version {version} is installed, but cannot be run: {e}') tool_error = True @@ -1269,8 +1320,8 @@ class IDFTool(object): if not isinstance(info_url, str): raise RuntimeError(f'info_url for tool {tool_name} is not a string') - license = tool_dict.get('license', False) # type: ignore - if not isinstance(license, str): + lic = tool_dict.get('license', False) # type: ignore + if not isinstance(lic, str): raise RuntimeError(f'license for tool {tool_name} is not a string') strip_container_dirs = tool_dict.get('strip_container_dirs', 0) @@ -1286,9 +1337,20 @@ class IDFTool(object): raise RuntimeError(f'supported_targets for tool {tool_name} is not a list of strings') # Create the object - tool_obj: 'IDFTool' = cls(tool_name, description, install, info_url, license, # type: ignore - version_cmd, version_regex, supported_targets, version_regex_replace, # type: ignore - strip_container_dirs, is_executable, tool_info_file) # type: ignore + tool_obj: 'IDFTool' = cls( + tool_name, + description, + install, + info_url, + lic, # type: ignore + version_cmd, + version_regex, + supported_targets, + version_regex_replace, # type: ignore + strip_container_dirs, # type: ignore + is_executable, + tool_info_file, + ) # type: ignore for path in export_paths: # type: ignore tool_obj.options.export_paths.append(path) # type: ignore @@ -1307,18 +1369,21 @@ class IDFTool(object): version_cmd = override.get('version_cmd') # type: ignore if version_cmd is not None and type(version_cmd) is not list: - raise RuntimeError('version_cmd for override %d of tool %s is not a list of strings' % - (index, tool_name)) + raise RuntimeError( + 'version_cmd for override %d of tool %s is not a list of strings' % (index, tool_name) + ) version_regex = override.get('version_regex') # type: ignore if version_regex is not None and (not isinstance(version_regex, str) or not version_regex): - raise RuntimeError('version_regex for override %d of tool %s is not a non-empty string' % - (index, tool_name)) + raise RuntimeError( + 'version_regex for override %d of tool %s is not a non-empty string' % (index, tool_name) + ) version_regex_replace = override.get('version_regex_replace') # type: ignore if version_regex_replace is not None and not isinstance(version_regex_replace, str): - raise RuntimeError('version_regex_replace for override %d of tool %s is not a string' % - (index, tool_name)) + raise RuntimeError( + 'version_regex_replace for override %d of tool %s is not a string' % (index, tool_name) + ) export_paths = override.get('export_paths') # type: ignore if export_paths is not None and type(export_paths) is not list: @@ -1329,7 +1394,7 @@ class IDFTool(object): raise RuntimeError('export_vars for override %d of tool %s is not a mapping' % (index, tool_name)) tool_obj.platform_overrides.append(override) # type: ignore - recommended_versions:Dict[str, List[str]] = {} + recommended_versions: Dict[str, List[str]] = {} for version_dict in versions: # type: ignore version = version_dict.get('name') # type: ignore if not isinstance(version, str): @@ -1337,8 +1402,9 @@ class IDFTool(object): version_status = version_dict.get('status') # type: ignore if not isinstance(version_status, str) and version_status not in IDFToolVersion.STATUS_VALUES: - raise RuntimeError('tool {} version {} status is not one of {}', tool_name, version, - IDFToolVersion.STATUS_VALUES) + raise RuntimeError( + 'tool {} version {} status is not one of {}', tool_name, version, IDFToolVersion.STATUS_VALUES + ) version_obj = IDFToolVersion(version, version_status) for platform_id, platform_dict in version_dict.items(): # type: ignore @@ -1347,12 +1413,15 @@ class IDFTool(object): try: Platforms.get(platform_id) except ValueError: - raise RuntimeError('invalid platform %s for tool %s version %s' % - (platform_id, tool_name, version)) + raise RuntimeError('invalid platform %s for tool %s version %s' % (platform_id, tool_name, version)) - version_obj.add_download(platform_id, - platform_dict['url'], platform_dict['size'], - platform_dict['sha256'], platform_dict.get('rename_dist', '')) + version_obj.add_download( + platform_id, + platform_dict['url'], + platform_dict['size'], + platform_dict['sha256'], + platform_dict.get('rename_dist', ''), + ) if version_status == IDFToolVersion.STATUS_RECOMMENDED: if platform_id not in recommended_versions: @@ -1362,9 +1431,13 @@ class IDFTool(object): tool_obj.add_version(version_obj) for platform_id, version_list in recommended_versions.items(): if len(version_list) > 1: - raise RuntimeError(f'tool {tool_name} for platform {platform_id} has {len(recommended_versions)} recommended versions') + raise RuntimeError( + f'tool {tool_name} for platform {platform_id} has {len(recommended_versions)} recommended versions' + ) if install != IDFTool.INSTALL_NEVER and len(recommended_versions) == 0: - raise RuntimeError(f'required/optional tool {tool_name} for platform {platform_id} has no recommended versions') + raise RuntimeError( + f'required/optional tool {tool_name} for platform {platform_id} has no recommended versions' + ) tool_obj._update_current_options() return tool_obj @@ -1375,24 +1448,17 @@ class IDFTool(object): """ versions_array = [] for version, version_obj in self.versions.items(): - version_json = { - 'name': version, - 'status': version_obj.status - } + version_json = {'name': version, 'status': version_obj.status} for platform_id, download in version_obj.downloads.items(): if download.rename_dist: version_json[platform_id] = { 'url': download.url, 'size': download.size, 'sha256': download.sha256, - 'rename_dist': download.rename_dist + 'rename_dist': download.rename_dist, } else: - version_json[platform_id] = { - 'url': download.url, - 'size': download.size, - 'sha256': download.sha256 - } + version_json[platform_id] = {'url': download.url, 'size': download.size, 'sha256': download.sha256} versions_array.append(version_json) overrides_array = self.platform_overrides @@ -1424,9 +1490,11 @@ class IDFTool(object): class IDFEnvEncoder(JSONEncoder): """ - IDFEnvEncoder is used for encoding IDFEnv, IDFRecord, SelectedIDFRecord classes to JSON in readable format. Not as (__main__.IDFRecord object at '0x7fcxx'). + IDFEnvEncoder is used for encoding IDFEnv, IDFRecord, SelectedIDFRecord classes to JSON in readable format. + Not as (__main__.IDFRecord object at '0x7fcxx'). Additionally remove first underscore with private properties when processing. """ + def default(self, obj): # type: ignore return {k.lstrip('_'): v for k, v in vars(obj).items()} @@ -1441,6 +1509,7 @@ class IDFRecord: * targets - ESP chips for which are installed needed toolchains (example ['esp32' , 'esp32s2']) - Default value is [], since user didn't define any targets yet """ + def __init__(self) -> None: self.version: str = '' self.path: str = '' @@ -1452,7 +1521,7 @@ class IDFRecord: 'version': self.version, 'path': self.path, 'features': self._features, - 'targets': self._targets + 'targets': self._targets, }.items() def __str__(self) -> str: @@ -1512,7 +1581,8 @@ class IDFRecord: idf_record_obj.version = record_dict['version'] idf_record_obj.path = record_dict['path'] except KeyError: - # When some of these key attributes, which are irreplaceable with default values, are not found, raise VallueError + # When some of these key attributes, which are irreplaceable with default values, are not found, + # raise VallueError raise ValueError('Inconsistent record') idf_record_obj.update_features(record_dict.get('features', [])) @@ -1523,11 +1593,13 @@ class IDFRecord: class IDFEnv: """ - IDFEnv represents ESP-IDF Environments installed on system and is responsible for loading and saving structured data. + IDFEnv represents ESP-IDF Environments installed on system and is responsible + for loading and saving structured data. All information is saved and loaded from IDF_ENV_FILE. Contains: * idf_installed - all installed environments of ESP-IDF on system. """ + def __init__(self) -> None: active_idf_id = active_repo_id() self.idf_installed: Dict[str, IDFRecord] = {active_idf_id: IDFRecord.get_active_idf_record()} @@ -1559,9 +1631,14 @@ class IDFEnv: json.dump(dict(self), w, cls=IDFEnvEncoder, ensure_ascii=False, indent=4) # type: ignore except (IOError, OSError): if not os.access(g.idf_tools_path, os.W_OK): - raise OSError(f'IDF_TOOLS_PATH {g.idf_tools_path} is not accessible to write. ' - 'Required changes have not been saved') - raise OSError(f'File {idf_env_file_path} is not accessible to write or corrupted. Required changes have not been saved') + raise OSError( + f'IDF_TOOLS_PATH {g.idf_tools_path} is not accessible to write. ' + 'Required changes have not been saved' + ) + raise OSError( + f'File {idf_env_file_path} is not accessible to write or corrupted. ' + 'Required changes have not been saved' + ) def get_active_idf_record(self) -> IDFRecord: return self.idf_installed[active_repo_id()] @@ -1569,7 +1646,8 @@ class IDFEnv: @classmethod def get_idf_env(cls) -> 'IDFEnv': """ - IDFEnv class is used to process IDF_ENV_FILE file. The constructor is therefore called only in this method that loads the file and checks its contents. + IDFEnv class is used to process IDF_ENV_FILE file. + The constructor is therefore called only in this method that loads the file and checks its contents. """ idf_env_obj = cls() try: @@ -1580,18 +1658,20 @@ class IDFEnv: try: idf_installed = idf_env_json['idfInstalled'] except KeyError: - # If no ESP-IDF record is found in loaded file, do not update and keep default value from constructor + # If no ESP-IDF record is found in loaded file, + # do not update and keep default value from constructor pass else: # Load and verify ESP-IDF records found in IDF_ENV_FILE idf_installed.pop('sha', None) - idf_installed_verified:Dict[str, IDFRecord] = {} + idf_installed_verified: Dict[str, IDFRecord] = {} for idf in idf_installed: try: idf_installed_verified[idf] = IDFRecord.get_idf_record_from_dict(idf_installed[idf]) except ValueError as err: warn(f'{err} "{idf}" found in {idf_env_file_path}, removing this record.') - # Combine ESP-IDF loaded records with the one in constructor, to be sure that there is an active ESP-IDF record in the idf_installed + # Combine ESP-IDF loaded records with the one in constructor, + # to be sure that there is an active ESP-IDF record in the idf_installed # If the active record is already in idf_installed, it is not overwritten idf_env_obj.idf_installed = dict(idf_env_obj.idf_installed, **idf_installed_verified) @@ -1604,14 +1684,17 @@ class IDFEnv: class ENVState: """ - ENVState is used to handle IDF global variables that are set in environment and need to be removed when switching between ESP-IDF versions in opened shell. + ENVState is used to handle IDF global variables that are set in environment and + need to be removed when switching between ESP-IDF versions in opened shell. Every opened shell/terminal has it's own temporary file to store these variables. - The temporary file's name is generated automatically with suffix 'idf_ + opened shell ID'. Path to this tmp file is stored as env global variable (env_key). + The temporary file's name is generated automatically with suffix 'idf_ + opened shell ID'. + Path to this tmp file is stored as env global variable (env_key). The shell ID is crucial, since in one terminal can be opened more shells. * env_key - global variable name/key * deactivate_file_path - global variable value (generated tmp file name) * idf_variables - loaded IDF variables from file """ + env_key = 'IDF_DEACTIVATE_FILE_PATH' deactivate_file_path = os.environ.get(env_key, '') @@ -1640,8 +1723,10 @@ class ENVState: self.deactivate_file_path = fp.name fp.write(json.dumps(self.idf_variables, ensure_ascii=False, indent=4).encode('utf-8')) except (IOError, OSError): - warn(f'File storing IDF env variables {self.deactivate_file_path} is not accessible to write. ' - 'Potentional switching ESP-IDF versions may cause problems') + warn( + f'File storing IDF env variables {self.deactivate_file_path} is not accessible to write. ' + 'Potentional switching ESP-IDF versions may cause problems' + ) return self.deactivate_file_path @@ -1723,9 +1808,14 @@ def get_idf_version() -> str: if idf_version is None: try: - with open(os.path.join(g.idf_path, 'components', 'esp_common', 'include', 'esp_idf_version.h'), encoding='utf-8') as f: - m = re.search(r'^#define\s+ESP_IDF_VERSION_MAJOR\s+(\d+).+?^#define\s+ESP_IDF_VERSION_MINOR\s+(\d+)', - f.read(), re.DOTALL | re.MULTILINE) + with open( + os.path.join(g.idf_path, 'components', 'esp_common', 'include', 'esp_idf_version.h'), encoding='utf-8' + ) as f: + m = re.search( + r'^#define\s+ESP_IDF_VERSION_MAJOR\s+(\d+).+?^#define\s+ESP_IDF_VERSION_MINOR\s+(\d+)', + f.read(), + re.DOTALL | re.MULTILINE, + ) if m: idf_version = '.'.join((m.group(1), m.group(2))) else: @@ -1739,13 +1829,13 @@ def get_idf_version() -> str: def get_python_env_path() -> Tuple[str, str, str, str]: """ - Returns tuple of Python environment path, Python env. path with subdir and full path from Python (i.e. with executable). + Returns tuple of Python environment path, Python env. path with subdir and + full path from Python (i.e. with executable). """ idf_version = get_idf_version() - idf_python_env_path = os.getenv('IDF_PYTHON_ENV_PATH') or os.path.join(g.idf_tools_path, - 'python_env', - PYTHON_VENV_DIR_TEMPLATE.format(idf_version, - PYTHON_VER_MAJOR_MINOR)) + idf_python_env_path = os.getenv('IDF_PYTHON_ENV_PATH') or os.path.join( + g.idf_tools_path, 'python_env', PYTHON_VENV_DIR_TEMPLATE.format(idf_version, PYTHON_VER_MAJOR_MINOR) + ) python_exe, subdir = get_python_exe_and_subdir() idf_python_export_path = os.path.join(idf_python_env_path, subdir) @@ -1776,14 +1866,18 @@ def expand_tools_arg(tools_spec: List[str], overall_tools: OrderedDict, targets: # Filtering tools if they are in overall_tools # Processing wildcards if possible for tool_pattern in tools_spec: - tools.extend([k for k, _ in overall_tools.items() if fnmatch.fnmatch(k,tool_pattern) and k not in tools]) + tools.extend([k for k, _ in overall_tools.items() if fnmatch.fnmatch(k, tool_pattern) and k not in tools]) # Processing "metapackage" if 'required' in tools_spec: - tools.extend([k for k, v in overall_tools.items() if v.get_install_type() == IDFTool.INSTALL_ALWAYS and k not in tools]) + tools.extend( + [k for k, v in overall_tools.items() if v.get_install_type() == IDFTool.INSTALL_ALWAYS and k not in tools] + ) elif 'all' in tools_spec: - tools.extend([k for k, v in overall_tools.items() if v.get_install_type() != IDFTool.INSTALL_NEVER and k not in tools]) + tools.extend( + [k for k, v in overall_tools.items() if v.get_install_type() != IDFTool.INSTALL_NEVER and k not in tools] + ) # Filtering by ESP_targets tools = [k for k in tools if overall_tools[k].is_supported_for_any_of_targets(targets)] @@ -1804,7 +1898,10 @@ def parse_targets_arg(targets_str: str) -> List[str]: else: invalid_targets = [t for t in targets if t not in targets_from_tools_json] if invalid_targets: - warn(f'Targets: \"{", ".join(invalid_targets)}\" are not supported. Only allowed options are: {", ".join(targets_from_tools_json)}.') + warn( + f'Targets: "{", ".join(invalid_targets)}" are not supported. ' + 'Only allowed options are: {", ".join(targets_from_tools_json)}.' + ) raise SystemExit(1) return targets @@ -1869,7 +1966,7 @@ def get_all_targets_from_tools_json() -> List[str]: return sorted(targets_from_tools_json) -def filter_tools_info(idf_env_obj: IDFEnv, tools_info: Dict[str, IDFTool]) -> Dict[str,IDFTool]: +def filter_tools_info(idf_env_obj: IDFEnv, tools_info: Dict[str, IDFTool]) -> Dict[str, IDFTool]: """ Filter tools info; return only those targets which: * are installable (install type is INSTALL_ALWAYS or INSTALL_ON_REQUEST) @@ -1879,22 +1976,28 @@ def filter_tools_info(idf_env_obj: IDFEnv, tools_info: Dict[str, IDFTool]) -> Di if not targets: return tools_info else: - filtered_tools_spec = {k:v for k, v in tools_info.items() if - (v.get_install_type() == IDFTool.INSTALL_ALWAYS or v.get_install_type() == IDFTool.INSTALL_ON_REQUEST) and - (any(item in targets for item in v.get_supported_targets()) or v.get_supported_targets() == ['all'])} + filtered_tools_spec = { + k: v + for k, v in tools_info.items() + if (v.get_install_type() == IDFTool.INSTALL_ALWAYS or v.get_install_type() == IDFTool.INSTALL_ON_REQUEST) + and (any(item in targets for item in v.get_supported_targets()) or v.get_supported_targets() == ['all']) + } return OrderedDict(filtered_tools_spec) -def add_variables_to_deactivate_file(args: List[str], new_idf_vars:Dict[str, Any]) -> str: +def add_variables_to_deactivate_file(args: List[str], new_idf_vars: Dict[str, Any]) -> str: """ Add IDF global variables that need to be removed when the active esp-idf environment is deactivated. """ if 'PATH' in new_idf_vars: - new_idf_vars['PATH'] = new_idf_vars['PATH'].split(':')[:-1] # PATH is stored as list of sub-paths without '$PATH' + # PATH is stored as list of sub-paths without '$PATH' + new_idf_vars['PATH'] = new_idf_vars['PATH'].split(':')[:-1] new_idf_vars['PATH'] = new_idf_vars.get('PATH', []) args_add_paths_extras = vars(args).get('add_paths_extras') # remove mypy error with args - new_idf_vars['PATH'] = new_idf_vars['PATH'] + args_add_paths_extras.split(':') if args_add_paths_extras else new_idf_vars['PATH'] + new_idf_vars['PATH'] = ( + new_idf_vars['PATH'] + args_add_paths_extras.split(':') if args_add_paths_extras else new_idf_vars['PATH'] + ) env_state_obj = ENVState.get_env_state() @@ -1985,8 +2088,11 @@ def list_default(args): # type: ignore versions_sorted = sorted(versions_for_platform.keys(), key=tool.versions.get, reverse=True) # type: ignore for version in versions_sorted: version_obj = tool.versions[version] - info(' - {} ({}{})'.format(version, version_obj.status, - ', installed' if version in tool.versions_installed else '')) + info( + ' - {} ({}{})'.format( + version, version_obj.status, ', installed' if version in tool.versions_installed else '' + ) + ) def list_outdated(args): # type: ignore @@ -2068,7 +2174,11 @@ def handle_recommended_version_to_use( tool_export_paths = tool.get_export_paths(version_to_use) tool_export_vars = tool.get_export_vars(version_to_use) if tool.version_in_path and tool.version_in_path not in tool.versions: - info(f'Not using an unsupported version of tool {tool.name} found in PATH: {tool.version_in_path}.' + prefer_system_hint, f=sys.stderr) + info( + f'Not using an unsupported version of tool {tool.name} found in PATH: {tool.version_in_path}.' + + prefer_system_hint, + f=sys.stderr, + ) return tool_export_paths, tool_export_vars @@ -2079,41 +2189,35 @@ def handle_supported_or_deprecated_version(tool: IDFTool, tool_name: str) -> Non """ version_obj: IDFToolVersion = tool.versions[tool.version_in_path] # type: ignore if version_obj.status == IDFToolVersion.STATUS_SUPPORTED: - info(f'Using a supported version of tool {tool_name} found in PATH: {tool.version_in_path}.', - f=sys.stderr) - info(f'However the recommended version is {tool.get_recommended_version()}.', - f=sys.stderr) + info(f'Using a supported version of tool {tool_name} found in PATH: {tool.version_in_path}.', f=sys.stderr) + info(f'However the recommended version is {tool.get_recommended_version()}.', f=sys.stderr) elif version_obj.status == IDFToolVersion.STATUS_DEPRECATED: warn(f'using a deprecated version of tool {tool_name} found in PATH: {tool.version_in_path}') # The following function is used in process_tool which is a part of the action_export. -def handle_missing_versions( - tool: IDFTool, - tool_name: str, - install_cmd: str, - prefer_system_hint: str -) -> None: +def handle_missing_versions(tool: IDFTool, tool_name: str, install_cmd: str, prefer_system_hint: str) -> None: """ Prints the info about missing tool to stderr if tool has no supported versions installed. """ msg = f'tool {tool.name} has no installed versions.' if 'NIX_PATH' in os.environ: - fatal(f'{msg} The environment indicates that you might be using NixOS. ' - 'Please see https://nixos.wiki/wiki/ESP-IDF for how to install tools for it.') + fatal( + f'{msg} The environment indicates that you might be using NixOS. ' + 'Please see https://nixos.wiki/wiki/ESP-IDF for how to install tools for it.' + ) else: - fatal(f'{msg} Please run \'{install_cmd}\' to install it.') + fatal(f"{msg} Please run '{install_cmd}' to install it.") if tool.version_in_path and tool.version_in_path not in tool.versions: - info(f'An unsupported version of tool {tool_name} was found in PATH: {tool.version_in_path}. ' + - prefer_system_hint, f=sys.stderr) + info( + f'An unsupported version of tool {tool_name} was found in PATH: {tool.version_in_path}. ' + + prefer_system_hint, + f=sys.stderr, + ) def process_tool( - tool: IDFTool, - tool_name: str, - args: argparse.Namespace, - install_cmd: str, - prefer_system_hint: str + tool: IDFTool, tool_name: str, args: argparse.Namespace, install_cmd: str, prefer_system_hint: str ) -> Tuple[list, dict, bool]: """ Helper function used only in action export. @@ -2159,8 +2263,9 @@ def process_tool( if not tool.versions_installed: if tool.get_install_type() == IDFTool.INSTALL_ALWAYS: if os.getenv('IDF_SKIP_TOOLS_CHECK', '0') == '1': - warn(f'Tool {tool_name} is not installed and IDF_SKIP_TOOLS_CHECK is set. ' - 'This may cause build failures.') + warn( + f'Tool {tool_name} is not installed and IDF_SKIP_TOOLS_CHECK is set. This may cause build failures.' + ) else: handle_missing_versions(tool, tool_name, install_cmd, prefer_system_hint) tool_found = False @@ -2175,17 +2280,21 @@ def check_python_venv_compatibility(idf_python_env_path: str, idf_version: str) with open(os.path.join(idf_python_env_path, VENV_VER_FILE), 'r', encoding='utf-8') as f: read_idf_version = f.read().strip() if read_idf_version != idf_version: - fatal(f'Python environment is set to {idf_python_env_path} which was generated for ' - f'ESP-IDF {read_idf_version} instead of the current {idf_version}. ' - 'The issue can be solved by (1) removing the directory and re-running the install script, ' - 'or (2) unsetting the IDF_PYTHON_ENV_PATH environment variable, or (3) ' - 're-runing the install script from a clean shell where an ESP-IDF environment is ' - 'not active.') + fatal( + f'Python environment is set to {idf_python_env_path} which was generated for ' + f'ESP-IDF {read_idf_version} instead of the current {idf_version}. ' + 'The issue can be solved by (1) removing the directory and re-running the install script, ' + 'or (2) unsetting the IDF_PYTHON_ENV_PATH environment variable, or (3) ' + 're-runing the install script from a clean shell where an ESP-IDF environment is ' + 'not active.' + ) raise SystemExit(1) except OSError as e: # perhaps the environment was generated before the support for VENV_VER_FILE was added - warn(f'The following issue occurred while accessing the ESP-IDF version file in the Python environment: {e}. ' - '(Diagnostic information. It can be ignored.)') + warn( + f'The following issue occurred while accessing the ESP-IDF version file in the Python environment: {e}. ' + '(Diagnostic information. It can be ignored.)' + ) def action_export(args: Any) -> None: @@ -2204,13 +2313,17 @@ def action_export(args: Any) -> None: self_restart_cmd = f'{sys.executable} {__file__}{(" --tools-json {args.tools_json}") if args.tools_json else ""}' self_restart_cmd = to_shell_specific_paths([self_restart_cmd])[0] - prefer_system_hint = '' if IDF_TOOLS_EXPORT_CMD else f' To use it, run \'{self_restart_cmd} export --prefer-system\'' - install_cmd = to_shell_specific_paths([IDF_TOOLS_INSTALL_CMD])[0] if IDF_TOOLS_INSTALL_CMD else f'{self_restart_cmd} install' + prefer_system_hint = '' if IDF_TOOLS_EXPORT_CMD else f" To use it, run '{self_restart_cmd} export --prefer-system'" + install_cmd = ( + to_shell_specific_paths([IDF_TOOLS_INSTALL_CMD])[0] if IDF_TOOLS_INSTALL_CMD else f'{self_restart_cmd} install' + ) for name, tool in tools_info.items(): if tool.get_install_type() == IDFTool.INSTALL_NEVER: continue - tool_export_paths, tool_export_vars, tool_found = process_tool(tool, name, args, install_cmd, prefer_system_hint) + tool_export_paths, tool_export_vars, tool_found = process_tool( + tool, name, args, install_cmd, prefer_system_hint + ) if not tool_found: all_tools_found = False paths_to_export += tool_export_paths @@ -2262,7 +2375,10 @@ def action_export(args: Any) -> None: if paths_to_check.index(r'\tools;') < paths_to_check.index(r'\idf-exe'): warn('The PATH is not in correct order (idf-exe should be before esp-idf\\tools)') except ValueError: - fatal(f'Both of the directories (..\\idf-exe\\.. and ..\\tools) has to be in the PATH:\n\n{paths_to_check}\n') + fatal( + 'Both of the directories (..\\idf-exe\\.. and ..\\tools) ' + 'has to be in the PATH:\n\n{paths_to_check}\n' + ) if export_vars: # if not copy of export_vars is given to function, it brekas the formatting string for 'export_statements' @@ -2294,13 +2410,15 @@ def apply_mirror_prefix_map(args: Any, idf_download_url: str) -> str: mirror_prefix_map = mirror_prefix_map_env.split(';') if IDF_MAINTAINER and args and args.mirror_prefix_map: if mirror_prefix_map: - warn('Both IDF_MIRROR_PREFIX_MAP environment variable and --mirror-prefix-map flag are specified, ' - 'will use the value from the command line.') + warn( + 'Both IDF_MIRROR_PREFIX_MAP environment variable and --mirror-prefix-map flag are specified, ' + 'will use the value from the command line.' + ) mirror_prefix_map = args.mirror_prefix_map if mirror_prefix_map: for item in mirror_prefix_map: if URL_PREFIX_MAP_SEPARATOR not in item: - warn(f'invalid mirror-prefix-map item (missing \'{URL_PREFIX_MAP_SEPARATOR}\') {item}') + warn(f"invalid mirror-prefix-map item (missing '{URL_PREFIX_MAP_SEPARATOR}') {item}") continue search, replace = item.split(URL_PREFIX_MAP_SEPARATOR, 1) replace = replace.replace('\\', '\\\\') # On windows replace single \ with double \\ @@ -2336,11 +2454,13 @@ def apply_github_assets_option(idf_download_url: str) -> str: return new_url -def get_tools_spec_and_platform_info(selected_platform: str, targets: List[str], tools_spec: List[str], - quiet: bool = False) -> Tuple[List[str], Dict[str, IDFTool]]: +def get_tools_spec_and_platform_info( + selected_platform: str, targets: List[str], tools_spec: List[str], quiet: bool = False +) -> Tuple[List[str], Dict[str, IDFTool]]: """ Returns tools_spec list and dict of tools for selected platform in form tool_name : IDFTool object. - NOTE: If this function is not called from action_download, but is used just for detecting active tools, info about downloading is unwanted. + NOTE: If this function is not called from action_download, but is used just for detecting active tools, + info about downloading is unwanted. """ global global_quiet try: @@ -2362,7 +2482,8 @@ def get_tools_spec_and_platform_info(selected_platform: str, targets: List[str], def action_download(args): # type: ignore """ - Saves current IDF environment and for every tools in tools_spec, downloads the right archive for tools version and target platform, if possible. + Saves current IDF environment and for every tools in tools_spec, + downloads the right archive for tools version and target platform, if possible. If not, prints appropriate message to stderr and raise SystemExit() exception. """ tools_spec = parse_tools_arg(args.tools) @@ -2414,7 +2535,8 @@ def action_download(args): # type: ignore def action_install(args): # type: ignore """ - Saves current IDF environment and for every tool in tools_spec, installs the tool from the archive downloaded before, if possible. + Saves current IDF environment and for every tool in tools_spec, + installs the tool from the archive downloaded before, if possible. If not, raises SystemExit. """ tools_spec = parse_tools_arg(args.tools) @@ -2523,8 +2645,10 @@ def get_constraints(idf_version: str, online: bool = True) -> str: if os.path.isfile(constraint_path): return constraint_path else: - fatal(f'{constraint_path} doesn\'t exist. Perhaps you\'ve forgotten to run the install scripts. ' - f'Please check the installation guide for more information.') + fatal( + f"{constraint_path} doesn't exist. Perhaps you've forgotten to run the install scripts. " + f'Please check the installation guide for more information.' + ) raise SystemExit(1) mkdir_p(os.path.dirname(temp_path)) @@ -2562,15 +2686,18 @@ def get_constraints(idf_version: str, online: bool = True) -> str: def install_legacy_python_virtualenv(path: str) -> None: """ - Checks if pip is installed (and installs it if not), checks whether virtualenv is already installed (and in which version), + Checks if pip is installed (and installs it if not), + checks whether virtualenv is already installed (and in which version), and finally creates virtual environment with python -m virtualenv . """ # Before creating the virtual environment, check if pip is installed. try: subprocess.check_call([sys.executable, '-m', 'pip', '--version']) except subprocess.CalledProcessError: - fatal(f'Python interpreter at {sys.executable} doesn\'t have pip installed. ' - 'Please check the Getting Started Guides for the steps to install prerequisites for your OS.') + fatal( + f"Python interpreter at {sys.executable} doesn't have pip installed. " + 'Please check the Getting Started Guides for the steps to install prerequisites for your OS.' + ) raise SystemExit(1) virtualenv_installed_via_pip = False @@ -2578,8 +2705,9 @@ def install_legacy_python_virtualenv(path: str) -> None: import virtualenv # noqa: F401 except ImportError: info('Installing virtualenv') - subprocess.check_call([sys.executable, '-m', 'pip', 'install', '--user', 'virtualenv'], - stdout=sys.stdout, stderr=sys.stderr) + subprocess.check_call( + [sys.executable, '-m', 'pip', 'install', '--user', 'virtualenv'], stdout=sys.stdout, stderr=sys.stderr + ) virtualenv_installed_via_pip = True # since we just installed virtualenv via pip, we know that version is recent enough # so the version check below is not necessary. @@ -2601,10 +2729,9 @@ def install_legacy_python_virtualenv(path: str) -> None: if with_seeder_option: virtualenv_options += ['--seeder', 'pip'] - subprocess.check_call([sys.executable, '-m', 'virtualenv', - *virtualenv_options, - path], - stdout=sys.stdout, stderr=sys.stderr) + subprocess.check_call( + [sys.executable, '-m', 'virtualenv', *virtualenv_options, path], stdout=sys.stdout, stderr=sys.stderr + ) def action_install_python_env(args): # type: ignore @@ -2620,7 +2747,9 @@ def action_install_python_env(args): # type: ignore nix_store = os.environ.get('NIX_STORE') is_nix = nix_store is not None and sys.base_prefix.startswith(nix_store) and sys.prefix.startswith(nix_store) - is_virtualenv = not is_nix and (hasattr(sys, 'real_prefix') or (hasattr(sys, 'base_prefix') and sys.base_prefix != sys.prefix)) + is_virtualenv = not is_nix and ( + hasattr(sys, 'real_prefix') or (hasattr(sys, 'base_prefix') and sys.base_prefix != sys.prefix) + ) if is_virtualenv and (not os.path.exists(idf_python_env_path) or reinstall): fatal('This script was called from a virtual environment, can not create a virtual environment again') raise SystemExit(1) @@ -2654,7 +2783,12 @@ def action_install_python_env(args): # type: ignore if os.path.exists(virtualenv_python): check_python_venv_compatibility(idf_python_env_path, idf_version) else: - if subprocess.run([sys.executable, '-m', 'venv', '-h'], check=False, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL).returncode == 0: + if ( + subprocess.run( + [sys.executable, '-m', 'venv', '-h'], check=False, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL + ).returncode + == 0 + ): # venv available virtualenv_options = [] @@ -2662,51 +2796,63 @@ def action_install_python_env(args): # type: ignore try: environ_idf_python_env_path = os.environ['IDF_PYTHON_ENV_PATH'] - correct_env_path = environ_idf_python_env_path.endswith(PYTHON_VENV_DIR_TEMPLATE.format(idf_version, - PYTHON_VER_MAJOR_MINOR)) - if not correct_env_path and re.search(PYTHON_VENV_DIR_TEMPLATE.format(r'\d+\.\d+', r'\d+\.\d+'), - environ_idf_python_env_path): - warn(f'IDF_PYTHON_ENV_PATH is set to {environ_idf_python_env_path} but it does not match ' - f'the detected {idf_version} ESP-IDF version and/or the used {PYTHON_VER_MAJOR_MINOR} ' - 'version of Python. If you have not set IDF_PYTHON_ENV_PATH intentionally then it is ' - 'recommended to re-run this script from a clean shell where an ESP-IDF environment is ' - 'not active.') + correct_env_path = environ_idf_python_env_path.endswith( + PYTHON_VENV_DIR_TEMPLATE.format(idf_version, PYTHON_VER_MAJOR_MINOR) + ) + if not correct_env_path and re.search( + PYTHON_VENV_DIR_TEMPLATE.format(r'\d+\.\d+', r'\d+\.\d+'), environ_idf_python_env_path + ): + warn( + f'IDF_PYTHON_ENV_PATH is set to {environ_idf_python_env_path} but it does not match ' + f'the detected {idf_version} ESP-IDF version and/or the used {PYTHON_VER_MAJOR_MINOR} ' + 'version of Python. If you have not set IDF_PYTHON_ENV_PATH intentionally then it is ' + 'recommended to re-run this script from a clean shell where an ESP-IDF environment is ' + 'not active.' + ) # Verify if IDF_PYTHON_ENV_PATH is a valid ESP-IDF Python virtual environment directory # to decide if content should be removed - if ( - os.path.exists(os.path.join(environ_idf_python_env_path, VENV_VER_FILE)) - or re.search(PYTHON_VENV_DIR_TEMPLATE.format(r'\d+\.\d+', r'\d+\.\d+'), environ_idf_python_env_path) + if os.path.exists(os.path.join(environ_idf_python_env_path, VENV_VER_FILE)) or re.search( + PYTHON_VENV_DIR_TEMPLATE.format(r'\d+\.\d+', r'\d+\.\d+'), environ_idf_python_env_path ): virtualenv_options.append('--clear') # delete environment if already exists elif os.listdir(environ_idf_python_env_path): # show the message only if the directory is not empty - info(f'IDF_PYTHON_ENV_PATH is set to {environ_idf_python_env_path}, ' - 'but it does not appear to be an ESP-IDF Python virtual environment directory. ' - 'Existing data in this folder will be preserved to prevent unintentional data loss.') + info( + f'IDF_PYTHON_ENV_PATH is set to {environ_idf_python_env_path}, ' + 'but it does not appear to be an ESP-IDF Python virtual environment directory. ' + 'Existing data in this folder will be preserved to prevent unintentional data loss.' + ) except KeyError: # if IDF_PYTHON_ENV_PATH not defined then the above checks can be skipped pass - subprocess.check_call([sys.executable, '-m', 'venv', - *virtualenv_options, - idf_python_env_path], - stdout=sys.stdout, stderr=sys.stderr) + subprocess.check_call( + [sys.executable, '-m', 'venv', *virtualenv_options, idf_python_env_path], + stdout=sys.stdout, + stderr=sys.stderr, + ) try: with open(os.path.join(idf_python_env_path, VENV_VER_FILE), 'w', encoding='utf-8') as f: f.write(idf_version) - except OSError as e: - warn(f'The following issue occurred while generating the ESP-IDF version file in the Python environment: {e}. ' - '(Diagnostic information. It can be ignored.)') + except OSError: + warn( + 'The following issue occurred while generating the ' + 'ESP-IDF version file in the Python environment: {e}. ' + '(Diagnostic information. It can be ignored.)' + ) else: # The embeddable Python for Windows doesn't have the built-in venv module install_legacy_python_virtualenv(idf_python_env_path) env_copy = os.environ.copy() - if env_copy.get('PIP_USER') == 'yes': - warn('Found PIP_USER="yes" in the environment. Disabling PIP_USER in this shell to install packages into a virtual environment.') + if env_copy.get('PIP_USER') == 'yes': + warn( + 'Found PIP_USER="yes" in the environment. ' + 'Disabling PIP_USER in this shell to install packages into a virtual environment.' + ) env_copy['PIP_USER'] = 'no' constraint_file = get_constraints(idf_version) if use_constraints else None @@ -2759,7 +2905,10 @@ def action_check_python_dependencies(args): # type: ignore _, _, virtualenv_python, idf_version = get_python_env_path() if not os.path.isfile(virtualenv_python): - fatal(f'{virtualenv_python} doesn\'t exist! Please run the install script or "idf_tools.py install-python-env" in order to create it') + fatal( + f"{virtualenv_python} doesn't exist! Please run the install script or " + '"idf_tools.py install-python-env" in order to create it' + ) raise SystemExit(1) if use_constraints: @@ -2773,10 +2922,7 @@ def action_check_python_dependencies(args): # type: ignore # The dependency checker will be invoked with virtualenv_python. idf_tools.py could have been invoked with a # different one, therefore, importing is not a suitable option. - dep_check_cmd = [virtualenv_python, - os.path.join(g.idf_path, - 'tools', - 'check_python_dependencies.py')] + dep_check_cmd = [virtualenv_python, os.path.join(g.idf_path, 'tools', 'check_python_dependencies.py')] if use_constraints: dep_check_cmd += ['-c', constr_path] @@ -2795,10 +2941,11 @@ def action_check_python_dependencies(args): # type: ignore raise SystemExit(1) -class ChecksumCalculator(): +class ChecksumCalculator: """ A class used to get size/checksum/basename of local artifact files. """ + def __init__(self, files: List[str]) -> None: self.files = files @@ -2811,16 +2958,18 @@ class ChecksumParsingError(RuntimeError): """ Raised when checksum cannot be parsed. """ + pass -class ChecksumFileParser(): +class ChecksumFileParser: """ A class used to get size/sha256/filename of artifact using checksum-file with format: # : bytes * ... (2 lines for every artifact) ... """ + def __init__(self, filename_prefix: str, url: str) -> None: self.filename_prefix = filename_prefix @@ -2877,9 +3026,16 @@ def action_add_version(args: Any) -> None: tool_obj = tools_info.get(tool_name) if not tool_obj: info(f'Creating new tool entry for {tool_name}') - tool_obj = IDFTool(tool_name, TODO_MESSAGE, IDFTool.INSTALL_ALWAYS, - TODO_MESSAGE, TODO_MESSAGE, [TODO_MESSAGE], TODO_MESSAGE, - [TODO_MESSAGE]) + tool_obj = IDFTool( + tool_name, + TODO_MESSAGE, + IDFTool.INSTALL_ALWAYS, + TODO_MESSAGE, + TODO_MESSAGE, + [TODO_MESSAGE], + TODO_MESSAGE, + [TODO_MESSAGE], + ) tools_info[tool_name] = tool_obj version = args.version version_status = IDFToolVersion.STATUS_SUPPORTED @@ -2893,9 +3049,11 @@ def action_add_version(args: Any) -> None: tool_obj.versions[version] = version_obj url_prefix = args.url_prefix or f'https://{TODO_MESSAGE}/' filename_prefix = args.dist_filename_prefix if len(args.dist_filename_prefix) else tool_name - checksum_info: ChecksumFileParser = (ChecksumFileParser(filename_prefix, args.checksum_file) - if args.checksum_file - else ChecksumCalculator(args.artifact_file)) # type: ignore + checksum_info: ChecksumFileParser = ( + ChecksumFileParser(filename_prefix, args.checksum_file) # type: ignore + if args.checksum_file + else ChecksumCalculator(args.artifact_file) + ) updated_tools = [] for file_size, file_sha256, file_name in checksum_info: xz_file = file_name.replace('.tar.gz', '.tar.xz') @@ -2950,10 +3108,14 @@ def action_uninstall(args: Any) -> None: unused_tools_versions = {} for tool in installed_tools: - tool_versions = os.listdir(os.path.join(tools_path, tool)) if os.path.isdir(os.path.join(tools_path, tool)) else [] + tool_versions = ( + os.listdir(os.path.join(tools_path, tool)) if os.path.isdir(os.path.join(tools_path, tool)) else [] + ) try: - unused_versions = ([x for x in tool_versions if x != tools_info[tool].get_recommended_version()]) - except KeyError: # When tool that is not supported by tools_info (tools.json) anymore, remove the whole tool file + unused_versions = [x for x in tool_versions if x != tools_info[tool].get_recommended_version()] + except ( + KeyError + ): # When tool that is not supported by tools_info (tools.json) anymore, remove the whole tool file unused_versions = [''] if unused_versions: unused_tools_versions[tool] = unused_versions @@ -2967,8 +3129,14 @@ def action_uninstall(args: Any) -> None: # Print unused tools. if args.dry_run: if unused_tools_versions: - print('For removing old versions of {} use command \'{} {} {}\''.format(', '.join(unused_tools_versions), get_python_exe_and_subdir()[0], - os.path.join(g.idf_path, 'tools', 'idf_tools.py'), 'uninstall')) + print( + "For removing old versions of {} use command '{} {} {}'".format( + ', '.join(unused_tools_versions), + get_python_exe_and_subdir()[0], + os.path.join(g.idf_path, 'tools', 'idf_tools.py'), + 'uninstall', + ) + ) return # Remove installed tools that are not used by current ESP-IDF version. @@ -2986,7 +3154,9 @@ def action_uninstall(args: Any) -> None: # Remove old archives versions and archives that are not used by the current ESP-IDF version. if args.remove_archives: - tools_spec, tools_info_for_platform = get_tools_spec_and_platform_info(CURRENT_PLATFORM, ['all'], ['all'], quiet=True) + tools_spec, tools_info_for_platform = get_tools_spec_and_platform_info( + CURRENT_PLATFORM, ['all'], ['all'], quiet=True + ) used_archives = [] # Detect used active archives @@ -3056,7 +3226,8 @@ def action_gen_doc(args): # type: ignore license_url = f'https://spdx.org/licenses/{tool_obj.options.license}' - print_out(""" + print_out( + """ .. _tool-{name}: {name} @@ -3079,12 +3250,15 @@ More info: {info_url} * - Platform - Required - Download -""".rstrip().format(name=tool_name, - underline=args.heading_underline_char * len(tool_name), - description=tool_obj.description, - license=tool_obj.options.license, - license_url=license_url, - info_url=info_url)) +""".rstrip().format( + name=tool_name, + underline=args.heading_underline_char * len(tool_name), + description=tool_obj.description, + license=tool_obj.options.license, + license_url=license_url, + info_url=info_url, + ) + ) for platform_name in sorted(tool_obj.get_supported_platforms()): platform_tool = tool_obj.copy_for_platform(platform_name) @@ -3104,7 +3278,8 @@ More info: {info_url} # Note: keep the list entries indented to the same number of columns # as the list header above. - print_out(""" + print_out( + """ * - {} - {} - {} @@ -3112,7 +3287,8 @@ More info: {info_url} .. rst-class:: tool-sha256 SHA256: {} -""".strip('\n').format(platform_name, install_type_str, download_obj.url, download_obj.sha256)) +""".strip('\n').format(platform_name, install_type_str, download_obj.url, download_obj.sha256) + ) print_out('') print_out('') @@ -3155,8 +3331,10 @@ def action_get_tool_supported_versions(args: Any) -> None: def main(argv: List[str]) -> None: parser = argparse.ArgumentParser() - parser.add_argument('--quiet', help='Don\'t output diagnostic messages to stdout/stderr', action='store_true') - parser.add_argument('--non-interactive', help='Don\'t output interactive messages and questions', action='store_true') + parser.add_argument('--quiet', help="Don't output diagnostic messages to stdout/stderr", action='store_true') + parser.add_argument( + '--non-interactive', help="Don't output interactive messages and questions", action='store_true' + ) parser.add_argument('--tools-json', help='Path to the tools.json file to use') parser.add_argument('--idf-path', help='ESP-IDF path to use') @@ -3165,64 +3343,129 @@ def main(argv: List[str]) -> None: list_parser.add_argument('--outdated', help='Print only outdated installed tools', action='store_true') subparsers.add_parser('check', help='Print summary of tools installed or found in PATH') export = subparsers.add_parser('export', help='Output command for setting tool paths, suitable for shell') - export.add_argument('--format', choices=[EXPORT_SHELL, EXPORT_KEY_VALUE], default=EXPORT_SHELL, - help=('Format of the output: shell (suitable for printing into shell), ' - 'or key-value (suitable for parsing by other tools')) - export.add_argument('--prefer-system', help=('Normally, if the tool is already present in PATH, ' - 'but has an unsupported version, a version from the tools directory ' - 'will be used instead. If this flag is given, the version in PATH ' - 'will be used.'), action='store_true') - export.add_argument('--deactivate', help='Output command for deactivate different ESP-IDF version, previously set with export', action='store_true') + export.add_argument( + '--format', + choices=[EXPORT_SHELL, EXPORT_KEY_VALUE], + default=EXPORT_SHELL, + help=( + 'Format of the output: shell (suitable for printing into shell), ' + 'or key-value (suitable for parsing by other tools' + ), + ) + export.add_argument( + '--prefer-system', + help=( + 'Normally, if the tool is already present in PATH, ' + 'but has an unsupported version, a version from the tools directory ' + 'will be used instead. If this flag is given, the version in PATH ' + 'will be used.' + ), + action='store_true', + ) + export.add_argument( + '--deactivate', + help='Output command for deactivate different ESP-IDF version, previously set with export', + action='store_true', + ) export.add_argument('--unset', help=argparse.SUPPRESS, action='store_true') export.add_argument('--add_paths_extras', help='Add idf-related path extras for deactivate option') install = subparsers.add_parser('install', help='Download and install tools into the tools directory') - install.add_argument('tools', metavar='TOOL', nargs='*', default=['required'], - help=('Tools to install.\n' - 'To install a specific version use @ syntax. ' - 'To install tools by pattern use wildcards in . ' - 'Use empty or \'required\' to install required tools, not optional ones. ' - 'Use \'all\' to install all tools, including the optional ones.')) - install.add_argument('--targets', default='all', help=('A comma separated list of desired chip targets for installing. ' - 'It defaults to installing all supported targets.')) + install.add_argument( + 'tools', + metavar='TOOL', + nargs='*', + default=['required'], + help=( + 'Tools to install.\n' + 'To install a specific version use @ syntax. ' + 'To install tools by pattern use wildcards in . ' + "Use empty or 'required' to install required tools, not optional ones. " + "Use 'all' to install all tools, including the optional ones." + ), + ) + install.add_argument( + '--targets', + default='all', + help=( + 'A comma separated list of desired chip targets for installing. ' + 'It defaults to installing all supported targets.' + ), + ) download = subparsers.add_parser('download', help='Download the tools into the dist directory') download.add_argument('--platform', default=CURRENT_PLATFORM, help='Platform to download the tools for') - download.add_argument('tools', metavar='TOOL', nargs='*', default=['required'], - help=('Tools to download. ' - 'To download a specific version use @ syntax. ' - 'To download tools by pattern use wildcards in . ' - 'Use empty or \'required\' to download required tools, not optional ones. ' - 'Use \'all\' to download all tools, including the optional ones.')) - download.add_argument('--targets', default='all', help=('A comma separated list of desired chip targets for installing. ' - ' It defaults to installing all supported targets.')) + download.add_argument( + 'tools', + metavar='TOOL', + nargs='*', + default=['required'], + help=( + 'Tools to download. ' + 'To download a specific version use @ syntax. ' + 'To download tools by pattern use wildcards in . ' + "Use empty or 'required' to download required tools, not optional ones. " + "Use 'all' to download all tools, including the optional ones." + ), + ) + download.add_argument( + '--targets', + default='all', + help=( + 'A comma separated list of desired chip targets for installing. ' + ' It defaults to installing all supported targets.' + ), + ) - uninstall = subparsers.add_parser('uninstall', help='Remove installed tools, that are not used by current version of ESP-IDF.') + uninstall = subparsers.add_parser( + 'uninstall', help='Remove installed tools, that are not used by current version of ESP-IDF.' + ) uninstall.add_argument('--dry-run', help='Print unused tools.', action='store_true') - uninstall.add_argument('--remove-archives', help='Remove old archive versions and archives from unused tools.', action='store_true') + uninstall.add_argument( + '--remove-archives', help='Remove old archive versions and archives from unused tools.', action='store_true' + ) no_constraints_default = os.environ.get('IDF_PYTHON_CHECK_CONSTRAINTS', '').lower() in ['0', 'n', 'no'] if IDF_MAINTAINER: for subparser in [download, install]: - subparser.add_argument('--mirror-prefix-map', nargs='*', - help=('Pattern to rewrite download URLs, with source and replacement separated by comma. ' - 'E.g. http://foo.com,http://test.foo.com')) + subparser.add_argument( + '--mirror-prefix-map', + nargs='*', + help=( + 'Pattern to rewrite download URLs, with source and replacement separated by comma. ' + 'E.g. http://foo.com,http://test.foo.com' + ), + ) - install_python_env = subparsers.add_parser('install-python-env', - help=('Create Python virtual environment and install the ' - 'required Python packages')) - install_python_env.add_argument('--reinstall', help='Discard the previously installed environment', - action='store_true') - install_python_env.add_argument('--extra-wheels-dir', help=('Additional directories with wheels ' - 'to use during installation')) + install_python_env = subparsers.add_parser( + 'install-python-env', help=('Create Python virtual environment and install the required Python packages') + ) + install_python_env.add_argument( + '--reinstall', help='Discard the previously installed environment', action='store_true' + ) + install_python_env.add_argument( + '--extra-wheels-dir', help=('Additional directories with wheels to use during installation') + ) install_python_env.add_argument('--extra-wheels-url', help='Additional URL with wheels', default=IDF_PIP_WHEELS_URL) install_python_env.add_argument('--no-index', help='Work offline without retrieving wheels index') - install_python_env.add_argument('--features', default='core', help=('A comma separated list of desired features for installing. ' - 'It defaults to installing just the core functionality.')) - install_python_env.add_argument('--no-constraints', action='store_true', default=no_constraints_default, - help=('Disable constraint settings. Use with care and only when you want to manage ' - 'package versions by yourself. It can be set with the IDF_PYTHON_CHECK_CONSTRAINTS ' - 'environment variable.')) + install_python_env.add_argument( + '--features', + default='core', + help=( + 'A comma separated list of desired features for installing. ' + 'It defaults to installing just the core functionality.' + ), + ) + install_python_env.add_argument( + '--no-constraints', + action='store_true', + default=no_constraints_default, + help=( + 'Disable constraint settings. Use with care and only when you want to manage ' + 'package versions by yourself. It can be set with the IDF_PYTHON_CHECK_CONSTRAINTS ' + 'environment variable.' + ), + ) if IDF_MAINTAINER: add_version = subparsers.add_parser('add-version', help='Add or update download info for a version') @@ -3231,11 +3474,15 @@ def main(argv: List[str]) -> None: add_version.add_argument('--version', help='Version identifier', required=True) add_version.add_argument('--url-prefix', help='String to prepend to file names to obtain download URLs') add_version.add_argument('--override', action='store_true', help='Override tool versions with new data') - add_version.add_argument('--dist-filename-prefix', - help='Prefix of distro archive name in checksum file. If skipped tool name is used.', - default='') + add_version.add_argument( + '--dist-filename-prefix', + help='Prefix of distro archive name in checksum file. If skipped tool name is used.', + default='', + ) add_version_files_group = add_version.add_mutually_exclusive_group(required=True) - add_version_files_group.add_argument('--checksum-file', help='URL or path to local file with checksum/size for artifacts') + add_version_files_group.add_argument( + '--checksum-file', help='URL or path to local file with checksum/size for artifacts' + ) add_version_files_group.add_argument('--artifact-file', help='File names of the download artifacts', nargs='*') rewrite = subparsers.add_parser('rewrite', help='Load tools.json, validate, and save the result back into JSON') @@ -3244,25 +3491,35 @@ def main(argv: List[str]) -> None: subparsers.add_parser('validate', help='Validate tools.json against schema file') gen_doc = subparsers.add_parser('gen-doc', help='Write the list of tools as a documentation page') - gen_doc.add_argument('--output', type=argparse.FileType('w'), default=sys.stdout, - help='Output file name') - gen_doc.add_argument('--heading-underline-char', help='Character to use when generating RST sections', default='~') + gen_doc.add_argument('--output', type=argparse.FileType('w'), default=sys.stdout, help='Output file name') + gen_doc.add_argument( + '--heading-underline-char', help='Character to use when generating RST sections', default='~' + ) - check_python_dependencies = subparsers.add_parser('check-python-dependencies', - help='Check that all required Python packages are installed.') - check_python_dependencies.add_argument('--no-constraints', action='store_true', default=no_constraints_default, - help='Disable constraint settings. Use with care and only when you want ' - 'to manage package versions by yourself. It can be set with the IDF_PYTHON_CHECK_CONSTRAINTS ' - 'environment variable.') + check_python_dependencies = subparsers.add_parser( + 'check-python-dependencies', help='Check that all required Python packages are installed.' + ) + check_python_dependencies.add_argument( + '--no-constraints', + action='store_true', + default=no_constraints_default, + help='Disable constraint settings. Use with care and only when you want ' + 'to manage package versions by yourself. It can be set with the IDF_PYTHON_CHECK_CONSTRAINTS ' + 'environment variable.', + ) if os.environ.get('IDF_TOOLS_VERSION_HELPER'): - check_tool_supported = subparsers.add_parser('check-tool-supported', - help='Check that selected tool is compatible with IDF. Writes "True"/"False" to stdout in success.') + check_tool_supported = subparsers.add_parser( + 'check-tool-supported', + help='Check that selected tool is compatible with IDF. Writes "True"/"False" to stdout in success.', + ) check_tool_supported.add_argument('--tool-name', required=True, help='Tool name (from tools.json)') check_tool_supported.add_argument('--exec-path', required=True, help='Full path to executable under the test') - get_tool_supported_versions = subparsers.add_parser('get-tool-supported-versions', help='Prints a list of tool\'s supported versions') - get_tool_supported_versions.add_argument('--tool-name', required=True, help='Tool name (from tools.json)') + get_tool_supported_versions = subparsers.add_parser( + 'get-tool-supported-versions', help="Prints a list of tool's supported versions" + ) + get_tool_supported_versions.add_argument('--tool-name', required=True, help='Tool name (from tools.json)') args = parser.parse_args(argv) @@ -3277,13 +3534,16 @@ def main(argv: List[str]) -> None: if 'unset' in args and args.unset: args.deactivate = True - g.idf_path = args.idf_path or os.environ.get('IDF_PATH') or os.path.realpath(os.path.join(os.path.dirname(__file__), '..')) + g.idf_path = ( + args.idf_path or os.environ.get('IDF_PATH') or os.path.realpath(os.path.join(os.path.dirname(__file__), '..')) + ) os.environ['IDF_PATH'] = g.idf_path g.idf_tools_path = os.environ.get('IDF_TOOLS_PATH') or os.path.expanduser(IDF_TOOLS_PATH_DEFAULT) # On macOS, unset __PYVENV_LAUNCHER__ variable if it is set. - # Otherwise sys.executable keeps pointing to the system Python, even when a python binary from a virtualenv is invoked. + # Otherwise sys.executable keeps pointing to the system Python, + # even when a python binary from a virtualenv is invoked. # See https://bugs.python.org/issue22490#msg283859. os.environ.pop('__PYVENV_LAUNCHER__', None) @@ -3304,7 +3564,9 @@ def main(argv: List[str]) -> None: if __name__ == '__main__': if 'MSYSTEM' in os.environ: - fatal('MSys/Mingw is not supported. Please follow the getting started guide of the documentation to set up ' - 'a supported environment') + fatal( + 'MSys/Mingw is not supported. Please follow the getting started guide of the documentation to set up ' + 'a supported environment' + ) raise SystemExit(1) main(sys.argv[1:])