diff --git a/tools/export_utils/shell_types.py b/tools/export_utils/shell_types.py index df860d8486..ea88e032fa 100644 --- a/tools/export_utils/shell_types.py +++ b/tools/export_utils/shell_types.py @@ -9,6 +9,7 @@ import sys import textwrap from datetime import datetime from datetime import timedelta +from importlib.metadata import version as importlib_version from pathlib import Path from subprocess import run from tempfile import NamedTemporaryFile @@ -19,7 +20,6 @@ from typing import List from typing import TextIO from typing import Union -import click from console_output import debug from console_output import status_message from console_output import warn @@ -122,7 +122,7 @@ class UnixShell(Shell): print(f'. {self.script_file_path}') def click_ver(self) -> int: - return int(click.__version__.split('.')[0]) + return int(importlib_version('click').split('.')[0]) class BashShell(UnixShell): diff --git a/tools/idf.py b/tools/idf.py index 740713f7f4..7f1c2ee2ad 100755 --- a/tools/idf.py +++ b/tools/idf.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# SPDX-FileCopyrightText: 2019-2024 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2019-2025 Espressif Systems (Shanghai) CO LTD # # SPDX-License-Identifier: Apache-2.0 # @@ -39,19 +39,30 @@ sys.dont_write_bytecode = True import python_version_checker # noqa: E402 try: - from idf_py_actions.errors import FatalError # noqa: E402 - from idf_py_actions.tools import (PROG, SHELL_COMPLETE_RUN, SHELL_COMPLETE_VAR, PropertyDict, # noqa: E402 - debug_print_idf_version, get_target, merge_action_lists, print_warning) + from idf_py_actions.errors import FatalError + from idf_py_actions.tools import PROG + from idf_py_actions.tools import SHELL_COMPLETE_RUN + from idf_py_actions.tools import SHELL_COMPLETE_VAR + from idf_py_actions.tools import PropertyDict + from idf_py_actions.tools import debug_print_idf_version + from idf_py_actions.tools import get_target + from idf_py_actions.tools import merge_action_lists + from idf_py_actions.tools import print_warning + if os.getenv('IDF_COMPONENT_MANAGER') != '0': from idf_component_manager import idf_extensions except ImportError as e: - print((f'{e}\n' - f'This usually means that "idf.py" was not ' - f'spawned within an ESP-IDF shell environment or the python virtual ' - f'environment used by "idf.py" is corrupted.\n' - f'Please use idf.py only in an ESP-IDF shell environment. If problem persists, ' - f'please try to install ESP-IDF tools again as described in the Get Started guide.'), - file=sys.stderr) + print( + ( + f'{e}\n' + f'This usually means that "idf.py" was not ' + f'spawned within an ESP-IDF shell environment or the python virtual ' + f'environment used by "idf.py" is corrupted.\n' + f'Please use idf.py only in an ESP-IDF shell environment. If problem persists, ' + f'please try to install ESP-IDF tools again as described in the Get Started guide.' + ), + file=sys.stderr, + ) if e.name is None: # The ImportError or ModuleNotFoundError might be raised without # specifying a module name. In this not so common situation, re-raise @@ -85,8 +96,9 @@ def check_environment() -> List: if set_idf_path != detected_idf_path: print_warning( 'WARNING: IDF_PATH environment variable is set to %s but %s path indicates IDF directory %s. ' - 'Using the environment variable directory, but results may be unexpected...' % - (set_idf_path, PROG, detected_idf_path)) + 'Using the environment variable directory, but results may be unexpected...' + % (set_idf_path, PROG, detected_idf_path) + ) else: print_warning('Setting IDF_PATH environment variable: %s' % detected_idf_path) os.environ['IDF_PATH'] = detected_idf_path @@ -122,15 +134,18 @@ def check_environment() -> List: try: python_venv_path = os.environ['IDF_PYTHON_ENV_PATH'] if python_venv_path and not sys.executable.startswith(python_venv_path): - print_warning(f'WARNING: Python interpreter "{sys.executable}" used to start idf.py is not from installed venv "{python_venv_path}"') + print_warning( + f'WARNING: Python interpreter "{sys.executable}" used to start idf.py' + f' is not from installed venv "{python_venv_path}"' + ) except KeyError: print_warning('WARNING: The IDF_PYTHON_ENV_PATH is missing in environmental variables!') return checks_output -def _safe_relpath(path: str, start: Optional[str]=None) -> str: - """ Return a relative path, same as os.path.relpath, but only if this is possible. +def _safe_relpath(path: str, start: Optional[str] = None) -> str: + """Return a relative path, same as os.path.relpath, but only if this is possible. It is not possible on Windows, if the start directory and the path are on different drives. """ @@ -140,14 +155,15 @@ def _safe_relpath(path: str, start: Optional[str]=None) -> str: return os.path.abspath(path) -def init_cli(verbose_output: Optional[List]=None) -> Any: +def init_cli(verbose_output: Optional[List] = None) -> Any: # Click is imported here to run it after check_environment() import click from click.shell_completion import CompletionItem class Deprecation(object): """Construct deprecation notice for help messages""" - def __init__(self, deprecated: Union[Dict, str, bool]=False) -> None: + + def __init__(self, deprecated: Union[Dict, str, bool] = False) -> None: self.deprecated = deprecated self.since = None self.removed = None @@ -162,25 +178,25 @@ def init_cli(verbose_output: Optional[List]=None) -> Any: elif isinstance(deprecated, str): self.custom_message = deprecated - def full_message(self, type: str='Option') -> str: + def full_message(self, msg_type: str = 'Option') -> str: if self.exit_with_error: return '%s is deprecated %sand was removed%s.%s' % ( - type, + msg_type, 'since %s ' % self.since if self.since else '', ' in %s' % self.removed if self.removed else '', ' %s' % self.custom_message if self.custom_message else '', ) else: return '%s is deprecated %sand will be removed in%s.%s' % ( - type, + msg_type, 'since %s ' % self.since if self.since else '', ' %s' % self.removed if self.removed else ' future versions', ' %s' % self.custom_message if self.custom_message else '', ) - def help(self, text: str, type: str='Option', separator: str=' ') -> str: + def help(self, text: str, msg_type: str = 'Option', separator: str = ' ') -> str: text = text or '' - return self.full_message(type) + separator + text if self.deprecated else text + return self.full_message(msg_type) + separator + text if self.deprecated else text def short_help(self, text: str) -> str: text = text or '' @@ -198,8 +214,15 @@ def init_cli(verbose_output: Optional[List]=None) -> Any: print_warning('Warning: %s' % deprecation.full_message('Option "%s"' % option.name)) class Task(object): - def __init__(self, callback: Callable, name: str, aliases: List, dependencies: Optional[List], - order_dependencies: Optional[List], action_args: Dict) -> None: + def __init__( + self, + callback: Callable, + name: str, + aliases: List, + dependencies: Optional[List], + order_dependencies: Optional[List], + action_args: Dict, + ) -> None: self.callback = callback self.name = name self.dependencies = dependencies @@ -207,7 +230,9 @@ def init_cli(verbose_output: Optional[List]=None) -> Any: self.action_args = action_args self.aliases = aliases - def __call__(self, context: click.core.Context, global_args: PropertyDict, action_args: Optional[Dict]=None) -> None: + def __call__( + self, context: click.core.Context, global_args: PropertyDict, action_args: Optional[Dict] = None + ) -> None: if action_args is None: action_args = self.action_args @@ -215,14 +240,15 @@ def init_cli(verbose_output: Optional[List]=None) -> Any: class Action(click.Command): def __init__( - self, - name: Optional[str]=None, - aliases: Optional[List]=None, - deprecated: Union[Dict, str, bool]=False, - dependencies: Optional[List]=None, - order_dependencies: Optional[List]=None, - hidden: bool=False, - **kwargs: Any) -> None: + self, + name: Optional[str] = None, + aliases: Optional[List] = None, + deprecated: Union[Dict, str, bool] = False, + dependencies: Optional[List] = None, + order_dependencies: Optional[List] = None, + hidden: bool = False, + **kwargs: Any, + ) -> None: super(Action, self).__init__(name, **kwargs) self.name: str = self.name or self.callback.__name__ @@ -247,7 +273,7 @@ def init_cli(verbose_output: Optional[List]=None) -> Any: if deprecated: deprecation = Deprecation(deprecated) self.short_help = deprecation.short_help(self.short_help) - self.help = deprecation.help(self.help, type='Command', separator='\n') + self.help = deprecation.help(self.help, msg_type='Command', separator='\n') # Add aliases to help string if aliases: @@ -269,7 +295,7 @@ def init_cli(verbose_output: Optional[List]=None) -> Any: aliases=self.aliases, ) - self.callback = wrapped_callback + self.callback: Callable = wrapped_callback def invoke(self, ctx: click.core.Context) -> click.core.Context: if self.deprecated: @@ -293,22 +319,23 @@ def init_cli(verbose_output: Optional[List]=None) -> Any: names - alias of 'param_decls' """ + def __init__(self, **kwargs: str): names = kwargs.pop('names') super(Argument, self).__init__(names, **kwargs) class Scope(object): """ - Scope for sub-command option. - possible values: - - default - only available on defined level (global/action) - - global - When defined for action, also available as global - - shared - Opposite to 'global': when defined in global scope, also available for all actions + Scope for sub-command option. + possible values: + - default - only available on defined level (global/action) + - global - When defined for action, also available as global + - shared - Opposite to 'global': when defined in global scope, also available for all actions """ SCOPES = ('default', 'global', 'shared') - def __init__(self, scope: Optional[Union['Scope', str]]=None) -> None: # noqa: F821 + def __init__(self, scope: Optional[Union['Scope', str]] = None) -> None: # noqa: F821 if scope is None: self._scope = 'default' elif isinstance(scope, str) and scope in self.SCOPES: @@ -331,7 +358,14 @@ def init_cli(verbose_output: Optional[List]=None) -> Any: class Option(click.Option): """Option that knows whether it should be global""" - def __init__(self, scope: Optional[Union[Scope, str]]=None, deprecated: Union[Dict, str, bool]=False, hidden: bool=False, **kwargs: str) -> None: + + def __init__( + self, + scope: Optional[Union[Scope, str]] = None, + deprecated: Union[Dict, str, bool] = False, + hidden: bool = False, + **kwargs: str, + ) -> None: """ Keyword arguments additional to Click's Option class: @@ -367,16 +401,22 @@ def init_cli(verbose_output: Optional[List]=None) -> Any: return super(Option, self).get_help_record(ctx) - class CLI(click.MultiCommand): + class CLI(click.Group): """Action list contains all actions with options available for CLI""" - def __init__(self, all_actions: Optional[Dict]=None, verbose_output: Optional[List]=None, help: Optional[str]=None) -> None: + + def __init__( + self, + all_actions: Optional[Dict] = None, + verbose_output: Optional[List] = None, + cli_help: Optional[str] = None, + ) -> None: super(CLI, self).__init__( chain=True, invoke_without_command=True, result_callback=self.execute_tasks, no_args_is_help=True, context_settings={'max_content_width': 140}, - help=help, + help=cli_help, ) self._actions = {} self.global_action_callbacks = [] @@ -431,7 +471,8 @@ def init_cli(verbose_output: Optional[List]=None) -> Any: if option.scope.is_shared: raise FatalError( '"%s" is defined for action "%s". ' - ' "shared" options can be declared only on global level' % (option.name, name)) + ' "shared" options can be declared only on global level' % (option.name, name) + ) # Promote options to global if see for the first time if option.scope.is_global and option.name not in [o.name for o in self.params]: @@ -458,8 +499,9 @@ def init_cli(verbose_output: Optional[List]=None) -> Any: # COMP_WORDBREAKS. When @ is included, the @-argument is not considered # part of the completion word, causing @-argument completion to function # unreliably in bash. - complete_file = ('bash' not in os.environ.get('_IDF.PY_COMPLETE', '') or - '@' not in os.environ.get('IDF_PY_COMP_WORDBREAKS', '')) + complete_file = 'bash' not in os.environ.get('_IDF.PY_COMPLETE', '') or '@' not in os.environ.get( + 'IDF_PY_COMP_WORDBREAKS', '' + ) if incomplete.startswith('@') and complete_file: path_prefix = incomplete[1:] candidates = glob.glob(path_prefix + '*') @@ -491,11 +533,12 @@ def init_cli(verbose_output: Optional[List]=None) -> Any: if key != 'project': # flashing a single item if key not in flasher_args: - # This is the case for 'idf.py bootloader' if Secure Boot is on, need to follow manual flashing steps + # This is the case for 'idf.py bootloader' + # if Secure Boot is on, need to follow manual flashing steps print('\n%s build complete.' % title) return cmd = '' - if (key == 'bootloader'): # bootloader needs --flash-mode, etc to be passed in + if key == 'bootloader': # bootloader needs --flash-mode, etc to be passed in cmd = ' '.join(flasher_args['write_flash_args']) + ' ' cmd += flasher_args[key]['offset'] + ' ' @@ -518,11 +561,13 @@ def init_cli(verbose_output: Optional[List]=None) -> Any: print('or') print(f' idf.py -p PORT {flash_target}') - esptool_cmd = ['python -m esptool', - '--chip {}'.format(flasher_args['extra_esptool_args']['chip']), - f'-b {args.baud}', - '--before {}'.format(flasher_args['extra_esptool_args']['before']), - '--after {}'.format(flasher_args['extra_esptool_args']['after'])] + esptool_cmd = [ + 'python -m esptool', + '--chip {}'.format(flasher_args['extra_esptool_args']['chip']), + f'-b {args.baud}', + '--before {}'.format(flasher_args['extra_esptool_args']['before']), + '--after {}'.format(flasher_args['extra_esptool_args']['after']), + ] if not flasher_args['extra_esptool_args']['stub']: esptool_cmd += ['--no-stub'] @@ -555,14 +600,16 @@ def init_cli(verbose_output: Optional[List]=None) -> Any: # Show warning if some tasks are present several times in the list dupplicated_tasks = sorted( - [item for item, count in Counter(task.name for task in tasks).items() if count > 1]) + [item for item, count in Counter(task.name for task in tasks).items() if count > 1] + ) if dupplicated_tasks: dupes = ', '.join('"%s"' % t for t in dupplicated_tasks) print_warning( - 'WARNING: Command%s found in the list of commands more than once. ' % - ('s %s are' % dupes if len(dupplicated_tasks) > 1 else ' %s is' % dupes) + - 'Only first occurrence will be executed.') + 'WARNING: Command%s found in the list of commands more than once. ' + % ('s %s are' % dupes if len(dupplicated_tasks) > 1 else ' %s is' % dupes) + + 'Only first occurrence will be executed.' + ) for task in tasks: # Set propagated global options. @@ -577,13 +624,17 @@ def init_cli(verbose_output: Optional[List]=None) -> Any: if global_value != default and local_value != default and global_value != local_value: if hasattr(option, 'envvar') and option.envvar and os.getenv(option.envvar) != default: - msg = (f'This option cannot be set in command line if the {option.envvar} ' - 'environment variable is set to a different value.') + msg = ( + f'This option cannot be set in command line if the {option.envvar} ' + 'environment variable is set to a different value.' + ) else: msg = 'This option can appear at most once in the command line.' - raise FatalError(f'Option "{key}" provided for "{task.name}" is already defined to ' - f'a different value. {msg}') + raise FatalError( + f'Option "{key}" provided for "{task.name}" is already defined to ' + f'a different value. {msg}' + ) if local_value != default: global_args[key] = local_value @@ -615,8 +666,9 @@ def init_cli(verbose_output: Optional[List]=None) -> Any: # and put to the front of the list of unprocessed tasks else: print( - 'Adding "%s"\'s dependency "%s" to list of commands with default set of options.' % - (task.name, dep)) + 'Adding "%s"\'s dependency "%s" to list of commands with default set of options.' + % (task.name, dep) + ) dep_task = ctx.invoke(ctx.command.get_command(ctx, dep)) # Remove options with global scope from invoke tasks because they are already in global_args @@ -650,7 +702,8 @@ def init_cli(verbose_output: Optional[List]=None) -> Any: if task.aliases: name_with_aliases += ' (aliases: %s)' % ', '.join(task.aliases) - # When machine-readable json format for help is printed, don't show info about executing action so the output is deserializable + # When machine-readable json format for help is printed, + # don't show info about executing action so the output is deserializable if name_with_aliases != 'help' or not task.action_args.get('json_option', False): print('Executing action: %s' % name_with_aliases) task(ctx, global_args, task.action_args) @@ -663,10 +716,7 @@ def init_cli(verbose_output: Optional[List]=None) -> Any: # fully featured click parser to be sure that extensions are loaded from the right place @click.command( add_help_option=False, - context_settings={ - 'allow_extra_args': True, - 'ignore_unknown_options': True - }, + context_settings={'allow_extra_args': True, 'ignore_unknown_options': True}, ) @click.option('-C', '--project-dir', default=os.getcwd(), type=click.Path()) def parse_project_dir(project_dir: str) -> Any: @@ -689,7 +739,7 @@ def init_cli(verbose_output: Optional[List]=None) -> Any: extensions = [] for directory in extension_dirs: if directory and not os.path.exists(directory): - print_warning('WARNING: Directory with idf.py extensions doesn\'t exist:\n %s' % directory) + print_warning("WARNING: Directory with idf.py extensions doesn't exist:\n %s" % directory) continue sys.path.append(directory) @@ -723,7 +773,8 @@ def init_cli(verbose_output: Optional[List]=None) -> Any: except ImportError: print_warning('Error importing extension file idf_ext.py. Skipping.') print_warning( - "Please make sure that it contains implementation (even if it's empty) of add_action_extensions") + "Please make sure that it contains implementation (even if it's empty) of add_action_extensions" + ) try: all_actions = merge_action_lists(all_actions, action_extensions(all_actions, project_dir)) @@ -733,9 +784,10 @@ def init_cli(verbose_output: Optional[List]=None) -> Any: cli_help = ( 'ESP-IDF CLI build management tool. ' 'For commands that are not known to idf.py an attempt to execute it as a build system target will be made. ' - 'Selected target: {}'.format(get_target(project_dir))) + 'Selected target: {}'.format(get_target(project_dir)) + ) - return CLI(help=cli_help, verbose_output=verbose_output, all_actions=all_actions) + return CLI(cli_help=cli_help, verbose_output=verbose_output, all_actions=all_actions) def main(argv: Optional[List[Any]] = None) -> None: @@ -791,11 +843,15 @@ def expand_file_arguments(argv: List[Any]) -> List[Any]: try: with open(rel_path, 'r', encoding='utf-8') as f: for line in f: - expanded_args.extend(expand_args(shlex.split(line), os.path.dirname(rel_path), file_stack + [file_name])) + expanded_args.extend( + expand_args(shlex.split(line), os.path.dirname(rel_path), file_stack + [file_name]) + ) except IOError: file_stack_str = ' -> '.join(['@' + f for f in file_stack + [file_name]]) - raise FatalError(f"File '{rel_path}' (expansion of {file_stack_str}) could not be opened. " - 'Please ensure the file exists and you have the necessary permissions to read it.') + raise FatalError( + f"File '{rel_path}' (expansion of {file_stack_str}) could not be opened. " + 'Please ensure the file exists and you have the necessary permissions to read it.' + ) return expanded_args argv = expand_args(argv, os.getcwd(), []) @@ -820,7 +876,11 @@ def _valid_unicode_config() -> Union[codecs.CodecInfo, bool]: def _find_usable_locale() -> str: try: - locales = subprocess.Popen(['locale', '-a'], stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0].decode('ascii', 'replace') + locales = ( + subprocess.Popen(['locale', '-a'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + .communicate()[0] + .decode('ascii', 'replace') + ) except OSError: locales = '' @@ -843,7 +903,8 @@ def _find_usable_locale() -> str: if not usable_locales: raise FatalError( 'Support for Unicode filenames is required, but no suitable UTF-8 locale was found on your system.' - ' Please refer to the manual for your operating system for details on locale reconfiguration.') + ' Please refer to the manual for your operating system for details on locale reconfiguration.' + ) return usable_locales[0] @@ -853,14 +914,16 @@ if __name__ == '__main__': if 'MSYSTEM' in os.environ: print_warning( 'MSys/Mingw is no longer supported. Please follow the getting started guide of the ' - 'documentation in order to set up a suitiable environment, or continue at your own risk.') + 'documentation in order to set up a suitiable environment, or continue at your own risk.' + ) elif os.name == 'posix' and not _valid_unicode_config(): # Trying to find best utf-8 locale available on the system and restart python with it best_locale = _find_usable_locale() print_warning( 'Your environment is not configured to handle unicode filenames outside of ASCII range.' - ' Environment variable LC_ALL is temporary set to %s for unicode support.' % best_locale) + ' Environment variable LC_ALL is temporary set to %s for unicode support.' % best_locale + ) os.environ['LC_ALL'] = best_locale ret = subprocess.call([sys.executable] + sys.argv, env=os.environ) diff --git a/tools/legacy_exports/export_legacy.fish b/tools/legacy_exports/export_legacy.fish index dee51f5c2a..cf30dd6ef9 100644 --- a/tools/legacy_exports/export_legacy.fish +++ b/tools/legacy_exports/export_legacy.fish @@ -101,7 +101,7 @@ end __main -set click_version (python -c 'import click; print(click.__version__.split(".")[0])') +set click_version (python -c 'from importlib.metadata import version as importlib_version; print(importlib_version('click').split(".")[0])') if test $click_version -lt 8 eval (env _IDF.PY_COMPLETE=source_fish idf.py) else diff --git a/tools/legacy_exports/export_legacy.sh b/tools/legacy_exports/export_legacy.sh index 9ad0f7ff50..7c0fc14b4d 100644 --- a/tools/legacy_exports/export_legacy.sh +++ b/tools/legacy_exports/export_legacy.sh @@ -209,7 +209,7 @@ __cleanup() { __enable_autocomplete() { - click_version="$(python -c 'import click; print(click.__version__.split(".")[0])')" + click_version="$(python -c 'from importlib.metadata import version as importlib_version; print(importlib_version('click').split(".")[0])')" if [ "${click_version}" -lt 8 ] then SOURCE_ZSH=source_zsh