diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 901d50d37d..2a459dfbed 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,7 +9,7 @@ repos: hooks: - id: ruff-format - id: ruff - args: [ "--fix" ] + args: [ "--fix", "--show-fixes" ] - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.5.0 hooks: diff --git a/tools/export_utils/shell_types.py b/tools/export_utils/shell_types.py index 921f13a6f5..d14e3a57c1 100644 --- a/tools/export_utils/shell_types.py +++ b/tools/export_utils/shell_types.py @@ -9,17 +9,14 @@ import sys import textwrap from datetime import datetime from datetime import timedelta +from importlib.metadata import version as importlib_version from pathlib import Path from subprocess import run from tempfile import NamedTemporaryFile from tempfile import TemporaryDirectory from tempfile import gettempdir -from typing import Dict -from typing import List from typing import TextIO -from typing import Union -import click from console_output import debug from console_output import status_message from console_output import warn @@ -28,7 +25,7 @@ from utils import run_cmd class Shell: - def __init__(self, shell: str, deactivate_cmd: str, new_esp_idf_env: Dict[str, str]): + def __init__(self, shell: str, deactivate_cmd: str, new_esp_idf_env: dict[str, str]): self.shell = shell self.deactivate_cmd = deactivate_cmd self.new_esp_idf_env = new_esp_idf_env @@ -65,7 +62,7 @@ class Shell: def export(self) -> None: raise NotImplementedError('Subclass must implement abstract method "export"') - def expanded_env(self) -> Dict[str, str]: + def expanded_env(self) -> dict[str, str]: expanded_env = self.new_esp_idf_env.copy() if 'PATH' not in expanded_env: @@ -88,7 +85,7 @@ class Shell: class UnixShell(Shell): - def __init__(self, shell: str, deactivate_cmd: str, new_esp_idf_env: Dict[str, str]): + def __init__(self, shell: str, deactivate_cmd: str, new_esp_idf_env: dict[str, str]): super().__init__(shell, deactivate_cmd, new_esp_idf_env) with NamedTemporaryFile(dir=self.tmp_dir_path, delete=False, prefix='activate_') as fd: @@ -110,10 +107,8 @@ class UnixShell(Shell): if stdout is not None: fd.write(f'{stdout}\n') fd.write( - ( - 'echo "\nDone! You can now compile ESP-IDF projects.\n' - 'Go to the project directory and run:\n\n idf.py build"\n' - ) + 'echo "\nDone! You can now compile ESP-IDF projects.\n' + 'Go to the project directory and run:\n\n idf.py build"\n' ) def export(self) -> None: @@ -122,7 +117,7 @@ class UnixShell(Shell): print(f'. {self.script_file_path}') def click_ver(self) -> int: - return int(click.__version__.split('.')[0]) + return int(importlib_version('click').split('.')[0]) class BashShell(UnixShell): @@ -208,7 +203,7 @@ class ZshShell(UnixShell): class FishShell(UnixShell): - def __init__(self, shell: str, deactivate_cmd: str, new_esp_idf_env: Dict[str, str]): + def __init__(self, shell: str, deactivate_cmd: str, new_esp_idf_env: dict[str, str]): super().__init__(shell, deactivate_cmd, new_esp_idf_env) self.new_esp_idf_env['IDF_TOOLS_INSTALL_CMD'] = os.path.join(conf.IDF_PATH, 'install.fish') self.new_esp_idf_env['IDF_TOOLS_EXPORT_CMD'] = os.path.join(conf.IDF_PATH, 'export.fish') @@ -232,7 +227,7 @@ class FishShell(UnixShell): class PowerShell(Shell): - def __init__(self, shell: str, deactivate_cmd: str, new_esp_idf_env: Dict[str, str]): + def __init__(self, shell: str, deactivate_cmd: str, new_esp_idf_env: dict[str, str]): super().__init__(shell, deactivate_cmd, new_esp_idf_env) with NamedTemporaryFile(dir=self.tmp_dir_path, delete=False, prefix='activate_', suffix='.ps1') as fd: @@ -270,22 +265,20 @@ class PowerShell(Shell): functions = self.get_functions() fd.write(f'{functions}\n') fd.write( - ( - 'echo "\nDone! You can now compile ESP-IDF projects.\n' - 'Go to the project directory and run:\n\n idf.py build\n"' - ) + 'echo "\nDone! You can now compile ESP-IDF projects.\n' + 'Go to the project directory and run:\n\n idf.py build\n"' ) def spawn(self) -> None: self.init_file() new_env = os.environ.copy() arguments = ['-NoExit', '-Command', f'{self.script_file_path}'] - cmd: Union[str, List[str]] = [self.shell] + arguments + cmd: str | list[str] = [self.shell] + arguments run(cmd, env=new_env) class WinCmd(Shell): - def __init__(self, shell: str, deactivate_cmd: str, new_esp_idf_env: Dict[str, str]): + def __init__(self, shell: str, deactivate_cmd: str, new_esp_idf_env: dict[str, str]): super().__init__(shell, deactivate_cmd, new_esp_idf_env) with NamedTemporaryFile(dir=self.tmp_dir_path, delete=False, prefix='activate_', suffix='.bat') as fd: @@ -338,7 +331,7 @@ class WinCmd(Shell): self.init_file() new_env = os.environ.copy() arguments = ['/k', f'{self.script_file_path}'] - cmd: Union[str, List[str]] = [self.shell] + arguments + cmd: str | list[str] = [self.shell] + arguments cmd = ' '.join(cmd) run(cmd, env=new_env) diff --git a/tools/idf.py b/tools/idf.py index 740713f7f4..ff13605392 100755 --- a/tools/idf.py +++ b/tools/idf.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# SPDX-FileCopyrightText: 2019-2024 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2019-2025 Espressif Systems (Shanghai) CO LTD # # SPDX-License-Identifier: Apache-2.0 # @@ -22,14 +22,11 @@ import subprocess import sys from collections import Counter from collections import OrderedDict +from collections.abc import Callable from collections.abc import KeysView from importlib import import_module from pkgutil import iter_modules from typing import Any -from typing import Callable -from typing import Dict -from typing import List -from typing import Optional from typing import Union # pyc files remain in the filesystem when switching between branches which might raise errors for incompatible @@ -39,19 +36,30 @@ sys.dont_write_bytecode = True import python_version_checker # noqa: E402 try: - from idf_py_actions.errors import FatalError # noqa: E402 - from idf_py_actions.tools import (PROG, SHELL_COMPLETE_RUN, SHELL_COMPLETE_VAR, PropertyDict, # noqa: E402 - debug_print_idf_version, get_target, merge_action_lists, print_warning) + from idf_py_actions.errors import FatalError + from idf_py_actions.tools import PROG + from idf_py_actions.tools import SHELL_COMPLETE_RUN + from idf_py_actions.tools import SHELL_COMPLETE_VAR + from idf_py_actions.tools import PropertyDict + from idf_py_actions.tools import debug_print_idf_version + from idf_py_actions.tools import get_target + from idf_py_actions.tools import merge_action_lists + from idf_py_actions.tools import print_warning + if os.getenv('IDF_COMPONENT_MANAGER') != '0': from idf_component_manager import idf_extensions except ImportError as e: - print((f'{e}\n' - f'This usually means that "idf.py" was not ' - f'spawned within an ESP-IDF shell environment or the python virtual ' - f'environment used by "idf.py" is corrupted.\n' - f'Please use idf.py only in an ESP-IDF shell environment. If problem persists, ' - f'please try to install ESP-IDF tools again as described in the Get Started guide.'), - file=sys.stderr) + print( + ( + f'{e}\n' + 'This usually means that "idf.py" was not ' + 'spawned within an ESP-IDF shell environment or the python virtual ' + 'environment used by "idf.py" is corrupted.\n' + 'Please use idf.py only in an ESP-IDF shell environment. If problem persists, ' + 'please try to install ESP-IDF tools again as described in the Get Started guide.' + ), + file=sys.stderr, + ) if e.name is None: # The ImportError or ModuleNotFoundError might be raised without # specifying a module name. In this not so common situation, re-raise @@ -69,7 +77,7 @@ PYTHON = sys.executable os.environ['PYTHON'] = sys.executable -def check_environment() -> List: +def check_environment() -> list: """ Verify the environment contains the top-level tools we need to operate @@ -84,11 +92,12 @@ def check_environment() -> List: set_idf_path = os.path.realpath(os.environ['IDF_PATH']) if set_idf_path != detected_idf_path: print_warning( - 'WARNING: IDF_PATH environment variable is set to %s but %s path indicates IDF directory %s. ' - 'Using the environment variable directory, but results may be unexpected...' % - (set_idf_path, PROG, detected_idf_path)) + f'WARNING: IDF_PATH environment variable is set to {set_idf_path}' + f' but {PROG} path indicates IDF directory {detected_idf_path}. ' + 'Using the environment variable directory, but results may be unexpected...' + ) else: - print_warning('Setting IDF_PATH environment variable: %s' % detected_idf_path) + print_warning(f'Setting IDF_PATH environment variable: {detected_idf_path}') os.environ['IDF_PATH'] = detected_idf_path try: @@ -122,15 +131,18 @@ def check_environment() -> List: try: python_venv_path = os.environ['IDF_PYTHON_ENV_PATH'] if python_venv_path and not sys.executable.startswith(python_venv_path): - print_warning(f'WARNING: Python interpreter "{sys.executable}" used to start idf.py is not from installed venv "{python_venv_path}"') + print_warning( + f'WARNING: Python interpreter "{sys.executable}" used to start idf.py' + f' is not from installed venv "{python_venv_path}"' + ) except KeyError: print_warning('WARNING: The IDF_PYTHON_ENV_PATH is missing in environmental variables!') return checks_output -def _safe_relpath(path: str, start: Optional[str]=None) -> str: - """ Return a relative path, same as os.path.relpath, but only if this is possible. +def _safe_relpath(path: str, start: str | None = None) -> str: + """Return a relative path, same as os.path.relpath, but only if this is possible. It is not possible on Windows, if the start directory and the path are on different drives. """ @@ -140,14 +152,15 @@ def _safe_relpath(path: str, start: Optional[str]=None) -> str: return os.path.abspath(path) -def init_cli(verbose_output: Optional[List]=None) -> Any: +def init_cli(verbose_output: list | None = None) -> Any: # Click is imported here to run it after check_environment() import click from click.shell_completion import CompletionItem - class Deprecation(object): + class Deprecation: """Construct deprecation notice for help messages""" - def __init__(self, deprecated: Union[Dict, str, bool]=False) -> None: + + def __init__(self, deprecated: dict | str | bool = False) -> None: self.deprecated = deprecated self.since = None self.removed = None @@ -162,25 +175,33 @@ def init_cli(verbose_output: Optional[List]=None) -> Any: elif isinstance(deprecated, str): self.custom_message = deprecated - def full_message(self, type: str='Option') -> str: + def full_message(self, msg_type: str = 'Option') -> str: if self.exit_with_error: - return '%s is deprecated %sand was removed%s.%s' % ( - type, - 'since %s ' % self.since if self.since else '', - ' in %s' % self.removed if self.removed else '', - ' %s' % self.custom_message if self.custom_message else '', - ) + msg = f'{msg_type} is deprecated' + if self.since: + msg += f' since {self.since}' + msg += ' and was removed' + if self.removed: + msg += f' in {self.removed}. ' + if self.custom_message: + msg += self.custom_message + return msg else: - return '%s is deprecated %sand will be removed in%s.%s' % ( - type, - 'since %s ' % self.since if self.since else '', - ' %s' % self.removed if self.removed else ' future versions', - ' %s' % self.custom_message if self.custom_message else '', - ) + msg = f'{msg_type} is deprecated' + if self.since: + msg += f' since {self.since}' + msg += ' and will be removed' + if self.removed: + msg += f' in {self.removed}. ' + else: + msg += ' in future versions. ' + if self.custom_message: + msg += self.custom_message + return msg - def help(self, text: str, type: str='Option', separator: str=' ') -> str: + def help(self, text: str, msg_type: str = 'Option', separator: str = ' ') -> str: text = text or '' - return self.full_message(type) + separator + text if self.deprecated else text + return self.full_message(msg_type) + separator + text if self.deprecated else text def short_help(self, text: str) -> str: text = text or '' @@ -193,13 +214,22 @@ def init_cli(verbose_output: Optional[List]=None) -> Any: if isinstance(option, Option) and option.deprecated and ctx.params[option.name] != default: deprecation = Deprecation(option.deprecated) if deprecation.exit_with_error: - raise FatalError('Error: %s' % deprecation.full_message('Option "%s"' % option.name)) + error = deprecation.full_message(f'Option "{option.name}"') + raise FatalError(f'Error: {error}') else: - print_warning('Warning: %s' % deprecation.full_message('Option "%s"' % option.name)) + error = deprecation.full_message(f'Option "{option.name}"') + print_warning(f'Warning: {error}') - class Task(object): - def __init__(self, callback: Callable, name: str, aliases: List, dependencies: Optional[List], - order_dependencies: Optional[List], action_args: Dict) -> None: + class Task: + def __init__( + self, + callback: Callable, + name: str, + aliases: list, + dependencies: list | None, + order_dependencies: list | None, + action_args: dict, + ) -> None: self.callback = callback self.name = name self.dependencies = dependencies @@ -207,7 +237,9 @@ def init_cli(verbose_output: Optional[List]=None) -> Any: self.action_args = action_args self.aliases = aliases - def __call__(self, context: click.core.Context, global_args: PropertyDict, action_args: Optional[Dict]=None) -> None: + def __call__( + self, context: click.core.Context, global_args: PropertyDict, action_args: dict | None = None + ) -> None: if action_args is None: action_args = self.action_args @@ -215,18 +247,19 @@ def init_cli(verbose_output: Optional[List]=None) -> Any: class Action(click.Command): def __init__( - self, - name: Optional[str]=None, - aliases: Optional[List]=None, - deprecated: Union[Dict, str, bool]=False, - dependencies: Optional[List]=None, - order_dependencies: Optional[List]=None, - hidden: bool=False, - **kwargs: Any) -> None: - super(Action, self).__init__(name, **kwargs) + self, + name: str | None = None, + aliases: list | None = None, + deprecated: dict | str | bool = False, + dependencies: list | None = None, + order_dependencies: list | None = None, + hidden: bool = False, + **kwargs: Any, + ) -> None: + super().__init__(name, **kwargs) self.name: str = self.name or self.callback.__name__ - self.deprecated: Union[Dict, str, bool] = deprecated + self.deprecated: dict | str | bool = deprecated self.hidden: bool = hidden if aliases is None: @@ -247,11 +280,11 @@ def init_cli(verbose_output: Optional[List]=None) -> Any: if deprecated: deprecation = Deprecation(deprecated) self.short_help = deprecation.short_help(self.short_help) - self.help = deprecation.help(self.help, type='Command', separator='\n') + self.help = deprecation.help(self.help, msg_type='Command', separator='\n') # Add aliases to help string if aliases: - aliases_help = 'Aliases: %s.' % ', '.join(aliases) + aliases_help = f'Aliases: {", ".join(aliases)}.' self.help = '\n'.join([self.help, aliases_help]) self.short_help = ' '.join([aliases_help, self.short_help]) @@ -269,23 +302,23 @@ def init_cli(verbose_output: Optional[List]=None) -> Any: aliases=self.aliases, ) - self.callback = wrapped_callback + self.callback: Callable = wrapped_callback def invoke(self, ctx: click.core.Context) -> click.core.Context: if self.deprecated: deprecation = Deprecation(self.deprecated) - message = deprecation.full_message('Command "%s"' % self.name) + message = deprecation.full_message(f'Command "{self.name}"') if deprecation.exit_with_error: - raise FatalError('Error: %s' % message) + raise FatalError(f'Error: {message}') else: - print_warning('Warning: %s' % message) + print_warning(f'Warning: {message}') self.deprecated = False # disable Click's built-in deprecation handling # Print warnings for options check_deprecation(ctx) - return super(Action, self).invoke(ctx) + return super().invoke(ctx) class Argument(click.Argument): """ @@ -293,22 +326,23 @@ def init_cli(verbose_output: Optional[List]=None) -> Any: names - alias of 'param_decls' """ + def __init__(self, **kwargs: str): names = kwargs.pop('names') - super(Argument, self).__init__(names, **kwargs) + super().__init__(names, **kwargs) - class Scope(object): + class Scope: """ - Scope for sub-command option. - possible values: - - default - only available on defined level (global/action) - - global - When defined for action, also available as global - - shared - Opposite to 'global': when defined in global scope, also available for all actions + Scope for sub-command option. + possible values: + - default - only available on defined level (global/action) + - global - When defined for action, also available as global + - shared - Opposite to 'global': when defined in global scope, also available for all actions """ SCOPES = ('default', 'global', 'shared') - def __init__(self, scope: Optional[Union['Scope', str]]=None) -> None: # noqa: F821 + def __init__(self, scope: Union['Scope', str] | None = None) -> None: # noqa: F821 if scope is None: self._scope = 'default' elif isinstance(scope, str) and scope in self.SCOPES: @@ -316,7 +350,7 @@ def init_cli(verbose_output: Optional[List]=None) -> Any: elif isinstance(scope, Scope): self._scope = str(scope) else: - raise FatalError('Unknown scope for option: %s' % scope) + raise FatalError(f'Unknown scope for option: {scope}') @property def is_global(self) -> bool: @@ -331,7 +365,14 @@ def init_cli(verbose_output: Optional[List]=None) -> Any: class Option(click.Option): """Option that knows whether it should be global""" - def __init__(self, scope: Optional[Union[Scope, str]]=None, deprecated: Union[Dict, str, bool]=False, hidden: bool=False, **kwargs: str) -> None: + + def __init__( + self, + scope: Scope | str | None = None, + deprecated: dict | str | bool = False, + hidden: bool = False, + **kwargs: str, + ) -> None: """ Keyword arguments additional to Click's Option class: @@ -344,7 +385,7 @@ def init_cli(verbose_output: Optional[List]=None) -> Any: """ kwargs['param_decls'] = kwargs.pop('names') - super(Option, self).__init__(**kwargs) + super().__init__(**kwargs) self.deprecated = deprecated self.scope = Scope(scope) @@ -355,7 +396,7 @@ def init_cli(verbose_output: Optional[List]=None) -> Any: self.help: str = deprecation.help(self.help) if self.envvar: - self.help += ' The default value can be set with the %s environment variable.' % self.envvar + self.help += f' The default value can be set with the {self.envvar} environment variable.' if self.scope.is_global: self.help += ' This option can be used at most once either globally, or for one subcommand.' @@ -365,18 +406,24 @@ def init_cli(verbose_output: Optional[List]=None) -> Any: if self.hidden: return None - return super(Option, self).get_help_record(ctx) + return super().get_help_record(ctx) - class CLI(click.MultiCommand): + class CLI(click.Group): """Action list contains all actions with options available for CLI""" - def __init__(self, all_actions: Optional[Dict]=None, verbose_output: Optional[List]=None, help: Optional[str]=None) -> None: - super(CLI, self).__init__( + + def __init__( + self, + all_actions: dict | None = None, + verbose_output: list | None = None, + cli_help: str | None = None, + ) -> None: + super().__init__( chain=True, invoke_without_command=True, result_callback=self.execute_tasks, no_args_is_help=True, context_settings={'max_content_width': 140}, - help=help, + help=cli_help, ) self._actions = {} self.global_action_callbacks = [] @@ -430,8 +477,9 @@ def init_cli(verbose_output: Optional[List]=None) -> Any: if option.scope.is_shared: raise FatalError( - '"%s" is defined for action "%s". ' - ' "shared" options can be declared only on global level' % (option.name, name)) + f'"{option.name}" is defined for action "{name}". ' + '"shared" options can be declared only on global level' + ) # Promote options to global if see for the first time if option.scope.is_global and option.name not in [o.name for o in self.params]: @@ -439,10 +487,10 @@ def init_cli(verbose_output: Optional[List]=None) -> Any: self._actions[name].params.append(option) - def list_commands(self, ctx: click.core.Context) -> List: + def list_commands(self, ctx: click.core.Context) -> list: return sorted(filter(lambda name: not self._actions[name].hidden, self._actions)) - def get_command(self, ctx: click.core.Context, name: str) -> Optional[Action]: + def get_command(self, ctx: click.core.Context, name: str) -> Action | None: if name in self.commands_with_aliases: return self._actions.get(self.commands_with_aliases.get(name)) @@ -453,19 +501,20 @@ def init_cli(verbose_output: Optional[List]=None) -> Any: return Action(name=name, callback=callback.unwrapped_callback) return None - def shell_complete(self, ctx: click.core.Context, incomplete: str) -> List[CompletionItem]: + def shell_complete(self, ctx: click.core.Context, incomplete: str) -> list[CompletionItem]: # Enable @-argument completion in bash only if @ is not present in # COMP_WORDBREAKS. When @ is included, the @-argument is not considered # part of the completion word, causing @-argument completion to function # unreliably in bash. - complete_file = ('bash' not in os.environ.get('_IDF.PY_COMPLETE', '') or - '@' not in os.environ.get('IDF_PY_COMP_WORDBREAKS', '')) + complete_file = 'bash' not in os.environ.get('_IDF.PY_COMPLETE', '') or '@' not in os.environ.get( + 'IDF_PY_COMP_WORDBREAKS', '' + ) if incomplete.startswith('@') and complete_file: path_prefix = incomplete[1:] candidates = glob.glob(path_prefix + '*') result = [CompletionItem(f'@{c}') for c in candidates] return result - return super(CLI, self).shell_complete(ctx, incomplete) # type: ignore + return super().shell_complete(ctx, incomplete) # type: ignore def _print_closing_message(self, args: PropertyDict, actions: KeysView) -> None: # print a closing message of some kind, @@ -482,7 +531,7 @@ def init_cli(verbose_output: Optional[List]=None) -> Any: # how to flash them def print_flashing_message(title: str, key: str) -> None: with open(os.path.join(args.build_dir, 'flasher_args.json'), encoding='utf-8') as file: - flasher_args: Dict[str, Any] = json.load(file) + flasher_args: dict[str, Any] = json.load(file) def flasher_path(f: Union[str, 'os.PathLike[str]']) -> str: if type(args.build_dir) is bytes: @@ -491,11 +540,12 @@ def init_cli(verbose_output: Optional[List]=None) -> Any: if key != 'project': # flashing a single item if key not in flasher_args: - # This is the case for 'idf.py bootloader' if Secure Boot is on, need to follow manual flashing steps - print('\n%s build complete.' % title) + # This is the case for 'idf.py bootloader' + # if Secure Boot is on, need to follow manual flashing steps + print(f'\n{title} build complete.') return cmd = '' - if (key == 'bootloader'): # bootloader needs --flash-mode, etc to be passed in + if key == 'bootloader': # bootloader needs --flash-mode, etc to be passed in cmd = ' '.join(flasher_args['write_flash_args']) + ' ' cmd += flasher_args[key]['offset'] + ' ' @@ -518,11 +568,13 @@ def init_cli(verbose_output: Optional[List]=None) -> Any: print('or') print(f' idf.py -p PORT {flash_target}') - esptool_cmd = ['python -m esptool', - '--chip {}'.format(flasher_args['extra_esptool_args']['chip']), - f'-b {args.baud}', - '--before {}'.format(flasher_args['extra_esptool_args']['before']), - '--after {}'.format(flasher_args['extra_esptool_args']['after'])] + esptool_cmd = [ + 'python -m esptool', + '--chip {}'.format(flasher_args['extra_esptool_args']['chip']), + f'-b {args.baud}', + '--before {}'.format(flasher_args['extra_esptool_args']['before']), + '--after {}'.format(flasher_args['extra_esptool_args']['after']), + ] if not flasher_args['extra_esptool_args']['stub']: esptool_cmd += ['--no-stub'] @@ -549,20 +601,24 @@ def init_cli(verbose_output: Optional[List]=None) -> Any: if 'bootloader' in actions: print_flashing_message('Bootloader', 'bootloader') - def execute_tasks(self, tasks: List, **kwargs: str) -> OrderedDict: + def execute_tasks(self, tasks: list, **kwargs: str) -> OrderedDict: ctx = click.get_current_context() global_args = PropertyDict(kwargs) # Show warning if some tasks are present several times in the list dupplicated_tasks = sorted( - [item for item, count in Counter(task.name for task in tasks).items() if count > 1]) + [item for item, count in Counter(task.name for task in tasks).items() if count > 1] + ) if dupplicated_tasks: - dupes = ', '.join('"%s"' % t for t in dupplicated_tasks) + print('----------------------------------------------------------------------------------------') + dupes = ', '.join(f'"{t}"' for t in dupplicated_tasks) print_warning( - 'WARNING: Command%s found in the list of commands more than once. ' % - ('s %s are' % dupes if len(dupplicated_tasks) > 1 else ' %s is' % dupes) + - 'Only first occurrence will be executed.') + f'WARNING: {"Commands" if len(dupplicated_tasks) > 1 else "Command"} {dupes} ' + f'{"are" if len(dupplicated_tasks) > 1 else "is"} ' + 'found in the list of commands more than once. ' + 'Only first occurrence will be executed.' + ) for task in tasks: # Set propagated global options. @@ -577,13 +633,17 @@ def init_cli(verbose_output: Optional[List]=None) -> Any: if global_value != default and local_value != default and global_value != local_value: if hasattr(option, 'envvar') and option.envvar and os.getenv(option.envvar) != default: - msg = (f'This option cannot be set in command line if the {option.envvar} ' - 'environment variable is set to a different value.') + msg = ( + f'This option cannot be set in command line if the {option.envvar} ' + 'environment variable is set to a different value.' + ) else: msg = 'This option can appear at most once in the command line.' - raise FatalError(f'Option "{key}" provided for "{task.name}" is already defined to ' - f'a different value. {msg}') + raise FatalError( + f'Option "{key}" provided for "{task.name}" is already defined to ' + f'a different value. {msg}' + ) if local_value != default: global_args[key] = local_value @@ -615,8 +675,9 @@ def init_cli(verbose_output: Optional[List]=None) -> Any: # and put to the front of the list of unprocessed tasks else: print( - 'Adding "%s"\'s dependency "%s" to list of commands with default set of options.' % - (task.name, dep)) + f'Adding "{task.name}"\'s dependency "{dep}" ' + 'to list of commands with default set of options.' + ) dep_task = ctx.invoke(ctx.command.get_command(ctx, dep)) # Remove options with global scope from invoke tasks because they are already in global_args @@ -648,11 +709,12 @@ def init_cli(verbose_output: Optional[List]=None) -> Any: for task in tasks_to_run.values(): name_with_aliases = task.name if task.aliases: - name_with_aliases += ' (aliases: %s)' % ', '.join(task.aliases) + name_with_aliases += f' (aliases: {", ".join(task.aliases)})' - # When machine-readable json format for help is printed, don't show info about executing action so the output is deserializable + # When machine-readable json format for help is printed, + # don't show info about executing action so the output is deserializable if name_with_aliases != 'help' or not task.action_args.get('json_option', False): - print('Executing action: %s' % name_with_aliases) + print(f'Executing action: {name_with_aliases}') task(ctx, global_args, task.action_args) self._print_closing_message(global_args, tasks_to_run.keys()) @@ -663,10 +725,7 @@ def init_cli(verbose_output: Optional[List]=None) -> Any: # fully featured click parser to be sure that extensions are loaded from the right place @click.command( add_help_option=False, - context_settings={ - 'allow_extra_args': True, - 'ignore_unknown_options': True - }, + context_settings={'allow_extra_args': True, 'ignore_unknown_options': True}, ) @click.option('-C', '--project-dir', default=os.getcwd(), type=click.Path()) def parse_project_dir(project_dir: str) -> Any: @@ -675,7 +734,7 @@ def init_cli(verbose_output: Optional[List]=None) -> Any: # Set `complete_var` to not existing environment variable name to prevent early cmd completion project_dir = parse_project_dir(standalone_mode=False, complete_var='_IDF.PY_COMPLETE_NOT_EXISTING') - all_actions: Dict = {} + all_actions: dict = {} # Load extensions from components dir idf_py_extensions_path = os.path.join(os.environ['IDF_PATH'], 'tools', 'idf_py_actions') extension_dirs = [os.path.realpath(idf_py_extensions_path)] @@ -689,7 +748,7 @@ def init_cli(verbose_output: Optional[List]=None) -> Any: extensions = [] for directory in extension_dirs: if directory and not os.path.exists(directory): - print_warning('WARNING: Directory with idf.py extensions doesn\'t exist:\n %s' % directory) + print_warning(f"WARNING: Directory with idf.py extensions doesn't exist:\n\t{directory}") continue sys.path.append(directory) @@ -713,7 +772,7 @@ def init_cli(verbose_output: Optional[List]=None) -> Any: try: all_actions = merge_action_lists(all_actions, extension.action_extensions(all_actions, project_dir)) except AttributeError: - print_warning('WARNING: Cannot load idf.py extension "%s"' % name) + print_warning(f'WARNING: Cannot load idf.py extension "{name}"') # Load extensions from project dir if os.path.exists(os.path.join(project_dir, 'idf_ext.py')): @@ -723,7 +782,8 @@ def init_cli(verbose_output: Optional[List]=None) -> Any: except ImportError: print_warning('Error importing extension file idf_ext.py. Skipping.') print_warning( - "Please make sure that it contains implementation (even if it's empty) of add_action_extensions") + "Please make sure that it contains implementation (even if it's empty) of add_action_extensions" + ) try: all_actions = merge_action_lists(all_actions, action_extensions(all_actions, project_dir)) @@ -733,12 +793,13 @@ def init_cli(verbose_output: Optional[List]=None) -> Any: cli_help = ( 'ESP-IDF CLI build management tool. ' 'For commands that are not known to idf.py an attempt to execute it as a build system target will be made. ' - 'Selected target: {}'.format(get_target(project_dir))) + f'Selected target: {get_target(project_dir)}' + ) - return CLI(help=cli_help, verbose_output=verbose_output, all_actions=all_actions) + return CLI(cli_help=cli_help, verbose_output=verbose_output, all_actions=all_actions) -def main(argv: Optional[List[Any]] = None) -> None: +def main(argv: list[Any] | None = None) -> None: # Check the environment only when idf.py is invoked regularly from command line. checks_output = None if SHELL_COMPLETE_RUN else check_environment() @@ -761,7 +822,7 @@ def main(argv: Optional[List[Any]] = None) -> None: cli(argv, prog_name=PROG, complete_var=SHELL_COMPLETE_VAR) -def expand_file_arguments(argv: List[Any]) -> List[Any]: +def expand_file_arguments(argv: list[Any]) -> list[Any]: """ Any argument starting with "@" gets replaced with all values read from a text file. Text file arguments can be split by newline or by space. @@ -771,7 +832,7 @@ def expand_file_arguments(argv: List[Any]) -> List[Any]: visited = set() expanded = False - def expand_args(args: List[Any], parent_path: str, file_stack: List[str]) -> List[str]: + def expand_args(args: list[Any], parent_path: str, file_stack: list[str]) -> list[str]: expanded_args = [] for arg in args: if not arg.startswith('@'): @@ -789,13 +850,17 @@ def expand_file_arguments(argv: List[Any]) -> List[Any]: visited.add(rel_path) try: - with open(rel_path, 'r', encoding='utf-8') as f: + with open(rel_path, encoding='utf-8') as f: for line in f: - expanded_args.extend(expand_args(shlex.split(line), os.path.dirname(rel_path), file_stack + [file_name])) - except IOError: + expanded_args.extend( + expand_args(shlex.split(line), os.path.dirname(rel_path), file_stack + [file_name]) + ) + except OSError: file_stack_str = ' -> '.join(['@' + f for f in file_stack + [file_name]]) - raise FatalError(f"File '{rel_path}' (expansion of {file_stack_str}) could not be opened. " - 'Please ensure the file exists and you have the necessary permissions to read it.') + raise FatalError( + f"File '{rel_path}' (expansion of {file_stack_str}) could not be opened. " + 'Please ensure the file exists and you have the necessary permissions to read it.' + ) return expanded_args argv = expand_args(argv, os.getcwd(), []) @@ -806,11 +871,7 @@ def expand_file_arguments(argv: List[Any]) -> List[Any]: return argv -def _valid_unicode_config() -> Union[codecs.CodecInfo, bool]: - # Python 2 is always good - if sys.version_info[0] == 2: - return True - +def _valid_unicode_config() -> codecs.CodecInfo | bool: # With python 3 unicode environment is required try: return codecs.lookup(locale.getpreferredencoding()).name != 'ascii' @@ -820,11 +881,15 @@ def _valid_unicode_config() -> Union[codecs.CodecInfo, bool]: def _find_usable_locale() -> str: try: - locales = subprocess.Popen(['locale', '-a'], stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0].decode('ascii', 'replace') + locales = ( + subprocess.Popen(['locale', '-a'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + .communicate()[0] + .decode('ascii', 'replace') + ) except OSError: locales = '' - usable_locales: List[str] = [] + usable_locales: list[str] = [] for line in locales.splitlines(): locale = line.strip() locale_name = locale.lower().replace('-', '') @@ -843,7 +908,8 @@ def _find_usable_locale() -> str: if not usable_locales: raise FatalError( 'Support for Unicode filenames is required, but no suitable UTF-8 locale was found on your system.' - ' Please refer to the manual for your operating system for details on locale reconfiguration.') + ' Please refer to the manual for your operating system for details on locale reconfiguration.' + ) return usable_locales[0] @@ -853,14 +919,16 @@ if __name__ == '__main__': if 'MSYSTEM' in os.environ: print_warning( 'MSys/Mingw is no longer supported. Please follow the getting started guide of the ' - 'documentation in order to set up a suitiable environment, or continue at your own risk.') + 'documentation in order to set up a suitiable environment, or continue at your own risk.' + ) elif os.name == 'posix' and not _valid_unicode_config(): # Trying to find best utf-8 locale available on the system and restart python with it best_locale = _find_usable_locale() print_warning( 'Your environment is not configured to handle unicode filenames outside of ASCII range.' - ' Environment variable LC_ALL is temporary set to %s for unicode support.' % best_locale) + f' Environment variable LC_ALL is temporary set to {best_locale} for unicode support.' + ) os.environ['LC_ALL'] = best_locale ret = subprocess.call([sys.executable] + sys.argv, env=os.environ) diff --git a/tools/test_build_system/test_common.py b/tools/test_build_system/test_common.py index 9da88ea787..d79ea42099 100644 --- a/tools/test_build_system/test_common.py +++ b/tools/test_build_system/test_common.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: 2022-2024 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2022-2025 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 import json import logging @@ -9,28 +9,23 @@ import subprocess import sys import textwrap from pathlib import Path -from typing import List import pytest -from test_build_system_helpers import append_to_file from test_build_system_helpers import EnvDict +from test_build_system_helpers import IdfPyFunc +from test_build_system_helpers import append_to_file from test_build_system_helpers import file_contains from test_build_system_helpers import find_python from test_build_system_helpers import get_snapshot -from test_build_system_helpers import IdfPyFunc from test_build_system_helpers import replace_in_file from test_build_system_helpers import run_idf_py -def get_subdirs_absolute_paths(path: Path) -> List[str]: +def get_subdirs_absolute_paths(path: Path) -> list[str]: """ Get a list of files with absolute path in a given `path` folder """ - return [ - '{}/{}'.format(dir_path, file_name) - for dir_path, _, file_names in os.walk(path) - for file_name in file_names - ] + return [f'{dir_path}/{file_name}' for dir_path, _, file_names in os.walk(path) for file_name in file_names] @pytest.mark.usefixtures('test_app_copy') @@ -51,10 +46,11 @@ def test_hints_no_color_output_when_noninteractive(idf_py: IdfPyFunc) -> None: """Check that idf.py hints don't include color escape codes in non-interactive builds""" # make the build fail in such a way that idf.py shows a hint - replace_in_file('main/build_test_app.c', '// placeholder_inside_main', - 'esp_chip_info_t chip_info; esp_chip_info(&chip_info);') + replace_in_file( + 'main/build_test_app.c', '// placeholder_inside_main', 'esp_chip_info_t chip_info; esp_chip_info(&chip_info);' + ) - with (pytest.raises(subprocess.CalledProcessError)) as exc_info: + with pytest.raises(subprocess.CalledProcessError) as exc_info: idf_py('build') # Should not actually include a color escape sequence! @@ -70,10 +66,7 @@ def test_idf_copy(idf_copy: Path, idf_py: IdfPyFunc) -> None: idf_py('build') -def test_idf_build_with_env_var_sdkconfig_defaults( - test_app_copy: Path, - default_idf_env: EnvDict -) -> None: +def test_idf_build_with_env_var_sdkconfig_defaults(test_app_copy: Path, default_idf_env: EnvDict) -> None: with open(test_app_copy / 'sdkconfig.test', 'w') as fw: fw.write('CONFIG_BT_ENABLED=y') @@ -86,9 +79,7 @@ def test_idf_build_with_env_var_sdkconfig_defaults( @pytest.mark.usefixtures('test_app_copy') @pytest.mark.test_app_copy('examples/system/efuse') -def test_efuse_summary_cmake_functions( - default_idf_env: EnvDict -) -> None: +def test_efuse_summary_cmake_functions(default_idf_env: EnvDict) -> None: default_idf_env['IDF_CI_BUILD'] = '1' output = run_idf_py('efuse-filter', env=default_idf_env) assert 'FROM_CMAKE: MAC: 00:00:00:00:00:00' in output.stdout @@ -117,16 +108,22 @@ def test_python_interpreter_unix(test_app_copy: Path) -> None: logging.info("Make sure idf.py never runs '/usr/bin/env python' or similar") env_dict = dict(**os.environ) python = find_python(env_dict['PATH']) - (test_app_copy / 'python').write_text(textwrap.dedent("""#!/bin/sh - echo "idf.py has executed '/usr/bin/env python' or similar" - exit 1 - """)) + (test_app_copy / 'python').write_text( + textwrap.dedent( + """ + #!/bin/sh + echo "idf.py has executed '/usr/bin/env python' or similar" + exit 1 + """ + ) + ) st = os.stat(test_app_copy / 'python') # equivalent to 'chmod +x ./python' os.chmod((test_app_copy / 'python'), st.st_mode | stat.S_IEXEC) env_dict['PATH'] = str(test_app_copy) + os.pathsep + env_dict['PATH'] - # python is loaded from env:$PATH, but since false interpreter is provided there, python needs to be specified as argument + # python is loaded from env:$PATH + # but since false interpreter is provided there, python needs to be specified as argument # if idf.py is reconfigured during it's execution, it would load a false interpreter run_idf_py('reconfigure', env=env_dict, python=python) @@ -140,7 +137,8 @@ def test_python_interpreter_win(test_app_copy: Path) -> None: # on windows python interpreter has compiled code '.exe' format, so this false file can be empty (test_app_copy / 'python.exe').write_text('') env_dict['PATH'] = str(test_app_copy) + os.pathsep + env_dict['PATH'] - # python is loaded from env:$PATH, but since false interpreter is provided there, python needs to be specified as argument + # python is loaded from env:$PATH + # but since false interpreter is provided there, python needs to be specified as argument # if idf.py is reconfigured during it's execution, it would load a false interpreter run_idf_py('reconfigure', env=env_dict, python=python) @@ -172,7 +170,7 @@ def test_ccache_used_to_build(test_app_copy: Path) -> None: def test_toolchain_prefix_in_description_file(idf_py: IdfPyFunc, test_app_copy: Path) -> None: logging.info('Toolchain prefix is set in project description file') idf_py('reconfigure') - data = json.load(open(test_app_copy / 'build' / 'project_description.json', 'r')) + data = json.load(open(test_app_copy / 'build' / 'project_description.json')) assert 'monitor_toolprefix' in data @@ -195,8 +193,10 @@ def test_subcommands_with_options(idf_py: IdfPyFunc, default_idf_env: EnvDict) - def test_fallback_to_build_system_target(idf_py: IdfPyFunc, test_app_copy: Path) -> None: logging.info('idf.py fallback to build system target') msg = 'Custom target is running' - append_to_file(test_app_copy / 'CMakeLists.txt', - 'add_custom_target(custom_target COMMAND ${{CMAKE_COMMAND}} -E echo "{}")'.format(msg)) + append_to_file( + test_app_copy / 'CMakeLists.txt', + f'add_custom_target(custom_target COMMAND ${{CMAKE_COMMAND}} -E echo "{msg}")', + ) ret = idf_py('custom_target') assert msg in ret.stdout, 'Custom target did not produce expected output' @@ -205,10 +205,16 @@ def test_create_component_project(idf_copy: Path) -> None: logging.info('Create project and component using idf.py and build it') run_idf_py('-C', 'projects', 'create-project', 'temp_test_project', workdir=idf_copy) run_idf_py('-C', 'components', 'create-component', 'temp_test_component', workdir=idf_copy) - replace_in_file(idf_copy / 'projects' / 'temp_test_project' / 'main' / 'temp_test_project.c', '{\n\n}', - '\n'.join(['{', '\tfunc();', '}'])) - replace_in_file(idf_copy / 'projects' / 'temp_test_project' / 'main' / 'temp_test_project.c', '#include ', - '\n'.join(['#include ', '#include "temp_test_component.h"'])) + replace_in_file( + idf_copy / 'projects' / 'temp_test_project' / 'main' / 'temp_test_project.c', + '{\n\n}', + '\n'.join(['{', '\tfunc();', '}']), + ) + replace_in_file( + idf_copy / 'projects' / 'temp_test_project' / 'main' / 'temp_test_project.c', + '#include ', + '\n'.join(['#include ', '#include "temp_test_component.h"']), + ) run_idf_py('build', workdir=(idf_copy / 'projects' / 'temp_test_project')) @@ -244,6 +250,7 @@ def test_create_project_with_idf_readonly(idf_copy: Path) -> None: if '/bin/' in path: continue # skip executables os.chmod(os.path.join(root, name), file_permission) + logging.info('Check that command for creating new project will success if the IDF itself is readonly.') change_file_permissions(idf_copy, write_permission=False) try: @@ -267,7 +274,18 @@ def test_docs_command(idf_py: IdfPyFunc) -> None: assert 'https://docs.espressif.com/projects/esp-idf/en/v4.2.1' in ret.stdout ret = idf_py('docs', '--no-browser', '--language', 'en', '--version', 'v4.2.1', '--target', 'esp32') assert 'https://docs.espressif.com/projects/esp-idf/en/v4.2.1/esp32' in ret.stdout - ret = idf_py('docs', '--no-browser', '--language', 'en', '--version', 'v4.2.1', '--target', 'esp32', '--starting-page', 'get-started') + ret = idf_py( + 'docs', + '--no-browser', + '--language', + 'en', + '--version', + 'v4.2.1', + '--target', + 'esp32', + '--starting-page', + 'get-started', + ) assert 'https://docs.espressif.com/projects/esp-idf/en/v4.2.1/esp32/get-started' in ret.stdout @@ -285,25 +303,31 @@ def test_deprecation_warning(idf_py: IdfPyFunc) -> None: def test_save_defconfig_check(idf_py: IdfPyFunc, test_app_copy: Path) -> None: logging.info('Save-defconfig checks') - (test_app_copy / 'sdkconfig').write_text('\n'.join(['CONFIG_COMPILER_OPTIMIZATION_SIZE=y', - 'CONFIG_ESPTOOLPY_FLASHFREQ_80M=y'])) + (test_app_copy / 'sdkconfig').write_text( + '\n'.join(['CONFIG_COMPILER_OPTIMIZATION_SIZE=y', 'CONFIG_ESPTOOLPY_FLASHFREQ_80M=y']) + ) idf_py('save-defconfig') - assert not file_contains(test_app_copy / 'sdkconfig.defaults', 'CONFIG_IDF_TARGET'), \ + assert not file_contains(test_app_copy / 'sdkconfig.defaults', 'CONFIG_IDF_TARGET'), ( 'CONFIG_IDF_TARGET should not be in sdkconfig.defaults' - assert file_contains(test_app_copy / 'sdkconfig.defaults', 'CONFIG_COMPILER_OPTIMIZATION_SIZE=y'), \ + ) + assert file_contains(test_app_copy / 'sdkconfig.defaults', 'CONFIG_COMPILER_OPTIMIZATION_SIZE=y'), ( 'Missing CONFIG_COMPILER_OPTIMIZATION_SIZE=y in sdkconfig.defaults' - assert file_contains(test_app_copy / 'sdkconfig.defaults', 'CONFIG_ESPTOOLPY_FLASHFREQ_80M=y'), \ + ) + assert file_contains(test_app_copy / 'sdkconfig.defaults', 'CONFIG_ESPTOOLPY_FLASHFREQ_80M=y'), ( 'Missing CONFIG_ESPTOOLPY_FLASHFREQ_80M=y in sdkconfig.defaults' + ) idf_py('fullclean') (test_app_copy / 'sdkconfig').unlink() (test_app_copy / 'sdkconfig.defaults').unlink() idf_py('set-target', 'esp32c3') (test_app_copy / 'sdkconfig').write_text('CONFIG_PARTITION_TABLE_OFFSET=0x8001') idf_py('save-defconfig') - assert file_contains(test_app_copy / 'sdkconfig.defaults', 'CONFIG_IDF_TARGET="esp32c3"'), \ + assert file_contains(test_app_copy / 'sdkconfig.defaults', 'CONFIG_IDF_TARGET="esp32c3"'), ( 'Missing CONFIG_IDF_TARGET="esp32c3" in sdkconfig.defaults' - assert file_contains(test_app_copy / 'sdkconfig.defaults', 'CONFIG_PARTITION_TABLE_OFFSET=0x8001'), \ + ) + assert file_contains(test_app_copy / 'sdkconfig.defaults', 'CONFIG_PARTITION_TABLE_OFFSET=0x8001'), ( 'Missing CONFIG_PARTITION_TABLE_OFFSET=0x8001 in sdkconfig.defaults' + ) def test_merge_bin_cmd(idf_py: IdfPyFunc, test_app_copy: Path) -> None: diff --git a/tools/test_idf_py/test_idf_py.py b/tools/test_idf_py/test_idf_py.py index b4efe0a917..fcb885b7e5 100755 --- a/tools/test_idf_py/test_idf_py.py +++ b/tools/test_idf_py/test_idf_py.py @@ -204,7 +204,8 @@ class TestDeprecations(TestWithoutExtensions): [sys.executable, idf_py_path, '-C', current_dir, 'test-2'], env=os.environ, stderr=subprocess.STDOUT ) except subprocess.CalledProcessError as e: - self.assertIn('Error: Command "test-2" is deprecated and was removed.', e.output.decode('utf-8', 'ignore')) + output = e.output.decode('utf-8', 'ignore').replace('\r\n', '\n') + self.assertIn('Error: Command "test-2" is deprecated and was removed\n', output) def test_exit_with_error_for_option(self): try: @@ -239,7 +240,6 @@ class TestDeprecations(TestWithoutExtensions): env=os.environ, stderr=subprocess.STDOUT, ).decode('utf-8', 'ignore') - self.assertIn('Warning: Option "test_sub_1" is deprecated and will be removed in future versions.', output) self.assertIn( 'Warning: Command "test-1" is deprecated and will be removed in future versions. ' @@ -374,8 +374,8 @@ class TestWrapperCommands(TestCase): class TestEFuseCommands(TestWrapperCommands): """ Test if wrapper commands for espefuse.py are working as expected. - The goal is NOT to test the functionality of espefuse.py, but to test if the wrapper commands - are working as expected. + The goal is NOT to test the functionality of espefuse.py + but to test if the wrapper commands are working as expected. """ def test_efuse_summary(self): @@ -438,8 +438,8 @@ class TestEFuseCommands(TestWrapperCommands): class TestSecureCommands(TestWrapperCommands): """ Test if wrapper commands for espsecure.py are working as expected. - The goal is NOT to test the functionality of espsecure.py, but to test if the wrapper commands are - working as expected. + The goal is NOT to test the functionality of espsecure.py + but to test if the wrapper commands are working as expected. """ @classmethod @@ -577,8 +577,8 @@ class TestSecureCommands(TestWrapperCommands): class TestMergeBinCommands(TestWrapperCommands): """ Test if merge-bin command is invoked as expected. - This test is not testing the functionality of esptool.py merge_bin command, but the invocation of - the command from idf.py. + This test is not testing the functionality of esptool.py merge_bin command + but the invocation of the command from idf.py. """ def test_merge_bin(self):