espressif/esp-idf
1
0
Fork 1
mirror of https://github.com/espressif/esp-idf.git synced 2025-10-03 10:30:58 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'fix/click_version' into 'master'

Browse Source 
Fixed click deprecation warnings

Closes IDF-13075 and IDF-13088

See merge request espressif/esp-idf!40765
This commit is contained in:
Jakub Kocka
2025-08-29 14:48:55 +08:00
parent 6c968cee04 6a8e47baf8
commit 117919d7d6
5 changed files with 297 additions and 212 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.pre-commit-config.yaml
View File

@@ -9,7 +9,7 @@ repos:
hooks:
- id: ruff-format
- id: ruff
args: [ "--fix" ]
args: [ "--fix", "--show-fixes" ]
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v4.5.0
hooks:

27
tools/export_utils/shell_types.py
View File

@@ -9,17 +9,14 @@ import sys
import textwrap
from datetime import datetime
from datetime import timedelta
from importlib.metadata import version as importlib_version
from pathlib import Path
from subprocess import run
from tempfile import NamedTemporaryFile
from tempfile import TemporaryDirectory
from tempfile import gettempdir
from typing import Dict
from typing import List
from typing import TextIO
from typing import Union
import click
from console_output import debug
from console_output import status_message
from console_output import warn
@@ -28,7 +25,7 @@ from utils import run_cmd
class Shell:
def __init__(self, shell: str, deactivate_cmd: str, new_esp_idf_env: Dict[str, str]):
def __init__(self, shell: str, deactivate_cmd: str, new_esp_idf_env: dict[str, str]):
self.shell = shell
self.deactivate_cmd = deactivate_cmd
self.new_esp_idf_env = new_esp_idf_env
@@ -65,7 +62,7 @@ class Shell:
def export(self) -> None:
raise NotImplementedError('Subclass must implement abstract method "export"')
def expanded_env(self) -> Dict[str, str]:
def expanded_env(self) -> dict[str, str]:
expanded_env = self.new_esp_idf_env.copy()
if 'PATH' not in expanded_env:
@@ -88,7 +85,7 @@ class Shell:
class UnixShell(Shell):
def __init__(self, shell: str, deactivate_cmd: str, new_esp_idf_env: Dict[str, str]):
def __init__(self, shell: str, deactivate_cmd: str, new_esp_idf_env: dict[str, str]):
super().__init__(shell, deactivate_cmd, new_esp_idf_env)
with NamedTemporaryFile(dir=self.tmp_dir_path, delete=False, prefix='activate_') as fd:
@@ -110,11 +107,9 @@ class UnixShell(Shell):
if stdout is not None:
fd.write(f'{stdout}\n')
fd.write(
(
'echo "\nDone! You can now compile ESP-IDF projects.\n'
'Go to the project directory and run:\n\n idf.py build"\n'
)
)
def export(self) -> None:
with open(self.script_file_path, 'w', encoding='utf-8') as fd:
@@ -122,7 +117,7 @@ class UnixShell(Shell):
print(f'. {self.script_file_path}')
def click_ver(self) -> int:
return int(click.__version__.split('.')[0])
return int(importlib_version('click').split('.')[0])
class BashShell(UnixShell):
@@ -208,7 +203,7 @@ class ZshShell(UnixShell):
class FishShell(UnixShell):
def __init__(self, shell: str, deactivate_cmd: str, new_esp_idf_env: Dict[str, str]):
def __init__(self, shell: str, deactivate_cmd: str, new_esp_idf_env: dict[str, str]):
super().__init__(shell, deactivate_cmd, new_esp_idf_env)
self.new_esp_idf_env['IDF_TOOLS_INSTALL_CMD'] = os.path.join(conf.IDF_PATH, 'install.fish')
self.new_esp_idf_env['IDF_TOOLS_EXPORT_CMD'] = os.path.join(conf.IDF_PATH, 'export.fish')
@@ -232,7 +227,7 @@ class FishShell(UnixShell):
class PowerShell(Shell):
def __init__(self, shell: str, deactivate_cmd: str, new_esp_idf_env: Dict[str, str]):
def __init__(self, shell: str, deactivate_cmd: str, new_esp_idf_env: dict[str, str]):
super().__init__(shell, deactivate_cmd, new_esp_idf_env)
with NamedTemporaryFile(dir=self.tmp_dir_path, delete=False, prefix='activate_', suffix='.ps1') as fd:
@@ -270,22 +265,20 @@ class PowerShell(Shell):
functions = self.get_functions()
fd.write(f'{functions}\n')
fd.write(
(
'echo "\nDone! You can now compile ESP-IDF projects.\n'
'Go to the project directory and run:\n\n idf.py build\n"'
)
)
def spawn(self) -> None:
self.init_file()
new_env = os.environ.copy()
arguments = ['-NoExit', '-Command', f'{self.script_file_path}']
cmd: Union[str, List[str]] = [self.shell] + arguments
cmd: str | list[str] = [self.shell] + arguments
run(cmd, env=new_env)
class WinCmd(Shell):
def __init__(self, shell: str, deactivate_cmd: str, new_esp_idf_env: Dict[str, str]):
def __init__(self, shell: str, deactivate_cmd: str, new_esp_idf_env: dict[str, str]):
super().__init__(shell, deactivate_cmd, new_esp_idf_env)
with NamedTemporaryFile(dir=self.tmp_dir_path, delete=False, prefix='activate_', suffix='.bat') as fd:
@@ -338,7 +331,7 @@ class WinCmd(Shell):
self.init_file()
new_env = os.environ.copy()
arguments = ['/k', f'{self.script_file_path}']
cmd: Union[str, List[str]] = [self.shell] + arguments
cmd: str | list[str] = [self.shell] + arguments
cmd = ' '.join(cmd)
run(cmd, env=new_env)

332
tools/idf.py
View File

@@ -1,6 +1,6 @@
#!/usr/bin/env python
#
# SPDX-FileCopyrightText: 2019-2024 Espressif Systems (Shanghai) CO LTD
# SPDX-FileCopyrightText: 2019-2025 Espressif Systems (Shanghai) CO LTD
#
# SPDX-License-Identifier: Apache-2.0
#
@@ -22,14 +22,11 @@ import subprocess
import sys
from collections import Counter
from collections import OrderedDict
from collections.abc import Callable
from collections.abc import KeysView
from importlib import import_module
from pkgutil import iter_modules
from typing import Any
from typing import Callable
from typing import Dict
from typing import List
from typing import Optional
from typing import Union
# pyc files remain in the filesystem when switching between branches which might raise errors for incompatible
@@ -39,19 +36,30 @@ sys.dont_write_bytecode = True
import python_version_checker # noqa: E402
try:
from idf_py_actions.errors import FatalError # noqa: E402
from idf_py_actions.tools import (PROG, SHELL_COMPLETE_RUN, SHELL_COMPLETE_VAR, PropertyDict, # noqa: E402
debug_print_idf_version, get_target, merge_action_lists, print_warning)
from idf_py_actions.errors import FatalError
from idf_py_actions.tools import PROG
from idf_py_actions.tools import SHELL_COMPLETE_RUN
from idf_py_actions.tools import SHELL_COMPLETE_VAR
from idf_py_actions.tools import PropertyDict
from idf_py_actions.tools import debug_print_idf_version
from idf_py_actions.tools import get_target
from idf_py_actions.tools import merge_action_lists
from idf_py_actions.tools import print_warning
if os.getenv('IDF_COMPONENT_MANAGER') != '0':
from idf_component_manager import idf_extensions
except ImportError as e:
print((f'{e}\n'
f'This usually means that "idf.py" was not '
f'spawned within an ESP-IDF shell environment or the python virtual '
f'environment used by "idf.py" is corrupted.\n'
f'Please use idf.py only in an ESP-IDF shell environment. If problem persists, '
f'please try to install ESP-IDF tools again as described in the Get Started guide.'),
file=sys.stderr)
print(
(
f'{e}\n'
'This usually means that "idf.py" was not '
'spawned within an ESP-IDF shell environment or the python virtual '
'environment used by "idf.py" is corrupted.\n'
'Please use idf.py only in an ESP-IDF shell environment. If problem persists, '
'please try to install ESP-IDF tools again as described in the Get Started guide.'
),
file=sys.stderr,
)
if e.name is None:
# The ImportError or ModuleNotFoundError might be raised without
# specifying a module name. In this not so common situation, re-raise
@@ -69,7 +77,7 @@ PYTHON = sys.executable
os.environ['PYTHON'] = sys.executable
def check_environment() -> List:
def check_environment() -> list:
"""
Verify the environment contains the top-level tools we need to operate
@@ -84,11 +92,12 @@ def check_environment() -> List:
set_idf_path = os.path.realpath(os.environ['IDF_PATH'])
if set_idf_path != detected_idf_path:
print_warning(
'WARNING: IDF_PATH environment variable is set to %s but %s path indicates IDF directory %s. '
'Using the environment variable directory, but results may be unexpected...' %
(set_idf_path, PROG, detected_idf_path))
f'WARNING: IDF_PATH environment variable is set to {set_idf_path}'
f' but {PROG} path indicates IDF directory {detected_idf_path}. '
'Using the environment variable directory, but results may be unexpected...'
)
else:
print_warning('Setting IDF_PATH environment variable: %s' % detected_idf_path)
print_warning(f'Setting IDF_PATH environment variable: {detected_idf_path}')
os.environ['IDF_PATH'] = detected_idf_path
try:
@@ -122,15 +131,18 @@ def check_environment() -> List:
try:
python_venv_path = os.environ['IDF_PYTHON_ENV_PATH']
if python_venv_path and not sys.executable.startswith(python_venv_path):
print_warning(f'WARNING: Python interpreter "{sys.executable}" used to start idf.py is not from installed venv "{python_venv_path}"')
print_warning(
f'WARNING: Python interpreter "{sys.executable}" used to start idf.py'
f' is not from installed venv "{python_venv_path}"'
)
except KeyError:
print_warning('WARNING: The IDF_PYTHON_ENV_PATH is missing in environmental variables!')
return checks_output
def _safe_relpath(path: str, start: Optional[str]=None) -> str:
""" Return a relative path, same as os.path.relpath, but only if this is possible.
def _safe_relpath(path: str, start: str | None = None) -> str:
"""Return a relative path, same as os.path.relpath, but only if this is possible.
It is not possible on Windows, if the start directory and the path are on different drives.
"""
@@ -140,14 +152,15 @@ def _safe_relpath(path: str, start: Optional[str]=None) -> str:
return os.path.abspath(path)
def init_cli(verbose_output: Optional[List]=None) -> Any:
def init_cli(verbose_output: list | None = None) -> Any:
# Click is imported here to run it after check_environment()
import click
from click.shell_completion import CompletionItem
class Deprecation(object):
class Deprecation:
"""Construct deprecation notice for help messages"""
def __init__(self, deprecated: Union[Dict, str, bool]=False) -> None:
def __init__(self, deprecated: dict | str | bool = False) -> None:
self.deprecated = deprecated
self.since = None
self.removed = None
@@ -162,25 +175,33 @@ def init_cli(verbose_output: Optional[List]=None) -> Any:
elif isinstance(deprecated, str):
self.custom_message = deprecated
def full_message(self, type: str='Option') -> str:
def full_message(self, msg_type: str = 'Option') -> str:
if self.exit_with_error:
return '%s is deprecated %sand was removed%s.%s' % (
type,
'since %s ' % self.since if self.since else '',
' in %s' % self.removed if self.removed else '',
' %s' % self.custom_message if self.custom_message else '',
)
msg = f'{msg_type} is deprecated'
if self.since:
msg += f' since {self.since}'
msg += ' and was removed'
if self.removed:
msg += f' in {self.removed}. '
if self.custom_message:
msg += self.custom_message
return msg
else:
return '%s is deprecated %sand will be removed in%s.%s' % (
type,
'since %s ' % self.since if self.since else '',
' %s' % self.removed if self.removed else ' future versions',
' %s' % self.custom_message if self.custom_message else '',
)
msg = f'{msg_type} is deprecated'
if self.since:
msg += f' since {self.since}'
msg += ' and will be removed'
if self.removed:
msg += f' in {self.removed}. '
else:
msg += ' in future versions. '
if self.custom_message:
msg += self.custom_message
return msg
def help(self, text: str, type: str='Option', separator: str=' ') -> str:
def help(self, text: str, msg_type: str = 'Option', separator: str = ' ') -> str:
text = text or ''
return self.full_message(type) + separator + text if self.deprecated else text
return self.full_message(msg_type) + separator + text if self.deprecated else text
def short_help(self, text: str) -> str:
text = text or ''
@@ -193,13 +214,22 @@ def init_cli(verbose_output: Optional[List]=None) -> Any:
if isinstance(option, Option) and option.deprecated and ctx.params[option.name] != default:
deprecation = Deprecation(option.deprecated)
if deprecation.exit_with_error:
raise FatalError('Error: %s' % deprecation.full_message('Option "%s"' % option.name))
error = deprecation.full_message(f'Option "{option.name}"')
raise FatalError(f'Error: {error}')
else:
print_warning('Warning: %s' % deprecation.full_message('Option "%s"' % option.name))
error = deprecation.full_message(f'Option "{option.name}"')
print_warning(f'Warning: {error}')
class Task(object):
def __init__(self, callback: Callable, name: str, aliases: List, dependencies: Optional[List],
order_dependencies: Optional[List], action_args: Dict) -> None:
class Task:
def __init__(
self,
callback: Callable,
name: str,
aliases: list,
dependencies: list | None,
order_dependencies: list | None,
action_args: dict,
) -> None:
self.callback = callback
self.name = name
self.dependencies = dependencies
@@ -207,7 +237,9 @@ def init_cli(verbose_output: Optional[List]=None) -> Any:
self.action_args = action_args
self.aliases = aliases
def __call__(self, context: click.core.Context, global_args: PropertyDict, action_args: Optional[Dict]=None) -> None:
def __call__(
self, context: click.core.Context, global_args: PropertyDict, action_args: dict | None = None
) -> None:
if action_args is None:
action_args = self.action_args
@@ -216,17 +248,18 @@ def init_cli(verbose_output: Optional[List]=None) -> Any:
class Action(click.Command):
def __init__(
self,
name: Optional[str]=None,
aliases: Optional[List]=None,
deprecated: Union[Dict, str, bool]=False,
dependencies: Optional[List]=None,
order_dependencies: Optional[List]=None,
hidden: bool=False,
**kwargs: Any) -> None:
super(Action, self).__init__(name, **kwargs)
name: str | None = None,
aliases: list | None = None,
deprecated: dict | str | bool = False,
dependencies: list | None = None,
order_dependencies: list | None = None,
hidden: bool = False,
**kwargs: Any,
) -> None:
super().__init__(name, **kwargs)
self.name: str = self.name or self.callback.__name__
self.deprecated: Union[Dict, str, bool] = deprecated
self.deprecated: dict | str | bool = deprecated
self.hidden: bool = hidden
if aliases is None:
@@ -247,11 +280,11 @@ def init_cli(verbose_output: Optional[List]=None) -> Any:
if deprecated:
deprecation = Deprecation(deprecated)
self.short_help = deprecation.short_help(self.short_help)
self.help = deprecation.help(self.help, type='Command', separator='\n')
self.help = deprecation.help(self.help, msg_type='Command', separator='\n')
# Add aliases to help string
if aliases:
aliases_help = 'Aliases: %s.' % ', '.join(aliases)
aliases_help = f'Aliases: {", ".join(aliases)}.'
self.help = '\n'.join([self.help, aliases_help])
self.short_help = ' '.join([aliases_help, self.short_help])
@@ -269,23 +302,23 @@ def init_cli(verbose_output: Optional[List]=None) -> Any:
aliases=self.aliases,
)
self.callback = wrapped_callback
self.callback: Callable = wrapped_callback
def invoke(self, ctx: click.core.Context) -> click.core.Context:
if self.deprecated:
deprecation = Deprecation(self.deprecated)
message = deprecation.full_message('Command "%s"' % self.name)
message = deprecation.full_message(f'Command "{self.name}"')
if deprecation.exit_with_error:
raise FatalError('Error: %s' % message)
raise FatalError(f'Error: {message}')
else:
print_warning('Warning: %s' % message)
print_warning(f'Warning: {message}')
self.deprecated = False # disable Click's built-in deprecation handling
# Print warnings for options
check_deprecation(ctx)
return super(Action, self).invoke(ctx)
return super().invoke(ctx)
class Argument(click.Argument):
"""
@@ -293,11 +326,12 @@ def init_cli(verbose_output: Optional[List]=None) -> Any:
names - alias of 'param_decls'
"""
def __init__(self, **kwargs: str):
names = kwargs.pop('names')
super(Argument, self).__init__(names, **kwargs)
super().__init__(names, **kwargs)
class Scope(object):
class Scope:
"""
Scope for sub-command option.
possible values:
@@ -308,7 +342,7 @@ def init_cli(verbose_output: Optional[List]=None) -> Any:
SCOPES = ('default', 'global', 'shared')
def __init__(self, scope: Optional[Union['Scope', str]]=None) -> None: # noqa: F821
def __init__(self, scope: Union['Scope', str] | None = None) -> None: # noqa: F821
if scope is None:
self._scope = 'default'
elif isinstance(scope, str) and scope in self.SCOPES:
@@ -316,7 +350,7 @@ def init_cli(verbose_output: Optional[List]=None) -> Any:
elif isinstance(scope, Scope):
self._scope = str(scope)
else:
raise FatalError('Unknown scope for option: %s' % scope)
raise FatalError(f'Unknown scope for option: {scope}')
@property
def is_global(self) -> bool:
@@ -331,7 +365,14 @@ def init_cli(verbose_output: Optional[List]=None) -> Any:
class Option(click.Option):
"""Option that knows whether it should be global"""
def __init__(self, scope: Optional[Union[Scope, str]]=None, deprecated: Union[Dict, str, bool]=False, hidden: bool=False, **kwargs: str) -> None:
def __init__(
self,
scope: Scope | str | None = None,
deprecated: dict | str | bool = False,
hidden: bool = False,
**kwargs: str,
) -> None:
"""
Keyword arguments additional to Click's Option class:
@@ -344,7 +385,7 @@ def init_cli(verbose_output: Optional[List]=None) -> Any:
"""
kwargs['param_decls'] = kwargs.pop('names')
super(Option, self).__init__(**kwargs)
super().__init__(**kwargs)
self.deprecated = deprecated
self.scope = Scope(scope)
@@ -355,7 +396,7 @@ def init_cli(verbose_output: Optional[List]=None) -> Any:
self.help: str = deprecation.help(self.help)
if self.envvar:
self.help += ' The default value can be set with the %s environment variable.' % self.envvar
self.help += f' The default value can be set with the {self.envvar} environment variable.'
if self.scope.is_global:
self.help += ' This option can be used at most once either globally, or for one subcommand.'
@@ -365,18 +406,24 @@ def init_cli(verbose_output: Optional[List]=None) -> Any:
if self.hidden:
return None
return super(Option, self).get_help_record(ctx)
return super().get_help_record(ctx)
class CLI(click.MultiCommand):
class CLI(click.Group):
"""Action list contains all actions with options available for CLI"""
def __init__(self, all_actions: Optional[Dict]=None, verbose_output: Optional[List]=None, help: Optional[str]=None) -> None:
super(CLI, self).__init__(
def __init__(
self,
all_actions: dict | None = None,
verbose_output: list | None = None,
cli_help: str | None = None,
) -> None:
super().__init__(
chain=True,
invoke_without_command=True,
result_callback=self.execute_tasks,
no_args_is_help=True,
context_settings={'max_content_width': 140},
help=help,
help=cli_help,
)
self._actions = {}
self.global_action_callbacks = []
@@ -430,8 +477,9 @@ def init_cli(verbose_output: Optional[List]=None) -> Any:
if option.scope.is_shared:
raise FatalError(
'"%s" is defined for action "%s". '
' "shared" options can be declared only on global level' % (option.name, name))
f'"{option.name}" is defined for action "{name}". '
'"shared" options can be declared only on global level'
)
# Promote options to global if see for the first time
if option.scope.is_global and option.name not in [o.name for o in self.params]:
@@ -439,10 +487,10 @@ def init_cli(verbose_output: Optional[List]=None) -> Any:
self._actions[name].params.append(option)
def list_commands(self, ctx: click.core.Context) -> List:
def list_commands(self, ctx: click.core.Context) -> list:
return sorted(filter(lambda name: not self._actions[name].hidden, self._actions))
def get_command(self, ctx: click.core.Context, name: str) -> Optional[Action]:
def get_command(self, ctx: click.core.Context, name: str) -> Action | None:
if name in self.commands_with_aliases:
return self._actions.get(self.commands_with_aliases.get(name))
@@ -453,19 +501,20 @@ def init_cli(verbose_output: Optional[List]=None) -> Any:
return Action(name=name, callback=callback.unwrapped_callback)
return None
def shell_complete(self, ctx: click.core.Context, incomplete: str) -> List[CompletionItem]:
def shell_complete(self, ctx: click.core.Context, incomplete: str) -> list[CompletionItem]:
# Enable @-argument completion in bash only if @ is not present in
# COMP_WORDBREAKS. When @ is included, the @-argument is not considered
# part of the completion word, causing @-argument completion to function
# unreliably in bash.
complete_file = ('bash' not in os.environ.get('_IDF.PY_COMPLETE', '') or
'@' not in os.environ.get('IDF_PY_COMP_WORDBREAKS', ''))
complete_file = 'bash' not in os.environ.get('_IDF.PY_COMPLETE', '') or '@' not in os.environ.get(
'IDF_PY_COMP_WORDBREAKS', ''
)
if incomplete.startswith('@') and complete_file:
path_prefix = incomplete[1:]
candidates = glob.glob(path_prefix + '*')
result = [CompletionItem(f'@{c}') for c in candidates]
return result
return super(CLI, self).shell_complete(ctx, incomplete) # type: ignore
return super().shell_complete(ctx, incomplete) # type: ignore
def _print_closing_message(self, args: PropertyDict, actions: KeysView) -> None:
# print a closing message of some kind,
@@ -482,7 +531,7 @@ def init_cli(verbose_output: Optional[List]=None) -> Any:
# how to flash them
def print_flashing_message(title: str, key: str) -> None:
with open(os.path.join(args.build_dir, 'flasher_args.json'), encoding='utf-8') as file:
flasher_args: Dict[str, Any] = json.load(file)
flasher_args: dict[str, Any] = json.load(file)
def flasher_path(f: Union[str, 'os.PathLike[str]']) -> str:
if type(args.build_dir) is bytes:
@@ -491,11 +540,12 @@ def init_cli(verbose_output: Optional[List]=None) -> Any:
if key != 'project': # flashing a single item
if key not in flasher_args:
# This is the case for 'idf.py bootloader' if Secure Boot is on, need to follow manual flashing steps
print('\n%s build complete.' % title)
# This is the case for 'idf.py bootloader'
# if Secure Boot is on, need to follow manual flashing steps
print(f'\n{title} build complete.')
return
cmd = ''
if (key == 'bootloader'): # bootloader needs --flash-mode, etc to be passed in
if key == 'bootloader': # bootloader needs --flash-mode, etc to be passed in
cmd = ' '.join(flasher_args['write_flash_args']) + ' '
cmd += flasher_args[key]['offset'] + ' '
@@ -518,11 +568,13 @@ def init_cli(verbose_output: Optional[List]=None) -> Any:
print('or')
print(f' idf.py -p PORT {flash_target}')
esptool_cmd = ['python -m esptool',
esptool_cmd = [
'python -m esptool',
'--chip {}'.format(flasher_args['extra_esptool_args']['chip']),
f'-b {args.baud}',
'--before {}'.format(flasher_args['extra_esptool_args']['before']),
'--after {}'.format(flasher_args['extra_esptool_args']['after'])]
'--after {}'.format(flasher_args['extra_esptool_args']['after']),
]
if not flasher_args['extra_esptool_args']['stub']:
esptool_cmd += ['--no-stub']
@@ -549,20 +601,24 @@ def init_cli(verbose_output: Optional[List]=None) -> Any:
if 'bootloader' in actions:
print_flashing_message('Bootloader', 'bootloader')
def execute_tasks(self, tasks: List, **kwargs: str) -> OrderedDict:
def execute_tasks(self, tasks: list, **kwargs: str) -> OrderedDict:
ctx = click.get_current_context()
global_args = PropertyDict(kwargs)
# Show warning if some tasks are present several times in the list
dupplicated_tasks = sorted(
[item for item, count in Counter(task.name for task in tasks).items() if count > 1])
[item for item, count in Counter(task.name for task in tasks).items() if count > 1]
)
if dupplicated_tasks:
dupes = ', '.join('"%s"' % t for t in dupplicated_tasks)
print('----------------------------------------------------------------------------------------')
dupes = ', '.join(f'"{t}"' for t in dupplicated_tasks)
print_warning(
'WARNING: Command%s found in the list of commands more than once. ' %
('s %s are' % dupes if len(dupplicated_tasks) > 1 else ' %s is' % dupes) +
'Only first occurrence will be executed.')
f'WARNING: {"Commands" if len(dupplicated_tasks) > 1 else "Command"} {dupes} '
f'{"are" if len(dupplicated_tasks) > 1 else "is"} '
'found in the list of commands more than once. '
'Only first occurrence will be executed.'
)
for task in tasks:
# Set propagated global options.
@@ -577,13 +633,17 @@ def init_cli(verbose_output: Optional[List]=None) -> Any:
if global_value != default and local_value != default and global_value != local_value:
if hasattr(option, 'envvar') and option.envvar and os.getenv(option.envvar) != default:
msg = (f'This option cannot be set in command line if the {option.envvar} '
'environment variable is set to a different value.')
msg = (
f'This option cannot be set in command line if the {option.envvar} '
'environment variable is set to a different value.'
)
else:
msg = 'This option can appear at most once in the command line.'
raise FatalError(f'Option "{key}" provided for "{task.name}" is already defined to '
f'a different value. {msg}')
raise FatalError(
f'Option "{key}" provided for "{task.name}" is already defined to '
f'a different value. {msg}'
)
if local_value != default:
global_args[key] = local_value
@@ -615,8 +675,9 @@ def init_cli(verbose_output: Optional[List]=None) -> Any:
# and put to the front of the list of unprocessed tasks
else:
print(
'Adding "%s"\'s dependency "%s" to list of commands with default set of options.' %
(task.name, dep))
f'Adding "{task.name}"\'s dependency "{dep}" '
'to list of commands with default set of options.'
)
dep_task = ctx.invoke(ctx.command.get_command(ctx, dep))
# Remove options with global scope from invoke tasks because they are already in global_args
@@ -648,11 +709,12 @@ def init_cli(verbose_output: Optional[List]=None) -> Any:
for task in tasks_to_run.values():
name_with_aliases = task.name
if task.aliases:
name_with_aliases += ' (aliases: %s)' % ', '.join(task.aliases)
name_with_aliases += f' (aliases: {", ".join(task.aliases)})'
# When machine-readable json format for help is printed, don't show info about executing action so the output is deserializable
# When machine-readable json format for help is printed,
# don't show info about executing action so the output is deserializable
if name_with_aliases != 'help' or not task.action_args.get('json_option', False):
print('Executing action: %s' % name_with_aliases)
print(f'Executing action: {name_with_aliases}')
task(ctx, global_args, task.action_args)
self._print_closing_message(global_args, tasks_to_run.keys())
@@ -663,10 +725,7 @@ def init_cli(verbose_output: Optional[List]=None) -> Any:
# fully featured click parser to be sure that extensions are loaded from the right place
@click.command(
add_help_option=False,
context_settings={
'allow_extra_args': True,
'ignore_unknown_options': True
},
context_settings={'allow_extra_args': True, 'ignore_unknown_options': True},
)
@click.option('-C', '--project-dir', default=os.getcwd(), type=click.Path())
def parse_project_dir(project_dir: str) -> Any:
@@ -675,7 +734,7 @@ def init_cli(verbose_output: Optional[List]=None) -> Any:
# Set `complete_var` to not existing environment variable name to prevent early cmd completion
project_dir = parse_project_dir(standalone_mode=False, complete_var='_IDF.PY_COMPLETE_NOT_EXISTING')
all_actions: Dict = {}
all_actions: dict = {}
# Load extensions from components dir
idf_py_extensions_path = os.path.join(os.environ['IDF_PATH'], 'tools', 'idf_py_actions')
extension_dirs = [os.path.realpath(idf_py_extensions_path)]
@@ -689,7 +748,7 @@ def init_cli(verbose_output: Optional[List]=None) -> Any:
extensions = []
for directory in extension_dirs:
if directory and not os.path.exists(directory):
print_warning('WARNING: Directory with idf.py extensions doesn\'t exist:\n %s' % directory)
print_warning(f"WARNING: Directory with idf.py extensions doesn't exist:\n\t{directory}")
continue
sys.path.append(directory)
@@ -713,7 +772,7 @@ def init_cli(verbose_output: Optional[List]=None) -> Any:
try:
all_actions = merge_action_lists(all_actions, extension.action_extensions(all_actions, project_dir))
except AttributeError:
print_warning('WARNING: Cannot load idf.py extension "%s"' % name)
print_warning(f'WARNING: Cannot load idf.py extension "{name}"')
# Load extensions from project dir
if os.path.exists(os.path.join(project_dir, 'idf_ext.py')):
@@ -723,7 +782,8 @@ def init_cli(verbose_output: Optional[List]=None) -> Any:
except ImportError:
print_warning('Error importing extension file idf_ext.py. Skipping.')
print_warning(
"Please make sure that it contains implementation (even if it's empty) of add_action_extensions")
"Please make sure that it contains implementation (even if it's empty) of add_action_extensions"
)
try:
all_actions = merge_action_lists(all_actions, action_extensions(all_actions, project_dir))
@@ -733,12 +793,13 @@ def init_cli(verbose_output: Optional[List]=None) -> Any:
cli_help = (
'ESP-IDF CLI build management tool. '
'For commands that are not known to idf.py an attempt to execute it as a build system target will be made. '
'Selected target: {}'.format(get_target(project_dir)))
f'Selected target: {get_target(project_dir)}'
)
return CLI(help=cli_help, verbose_output=verbose_output, all_actions=all_actions)
return CLI(cli_help=cli_help, verbose_output=verbose_output, all_actions=all_actions)
def main(argv: Optional[List[Any]] = None) -> None:
def main(argv: list[Any] | None = None) -> None:
# Check the environment only when idf.py is invoked regularly from command line.
checks_output = None if SHELL_COMPLETE_RUN else check_environment()
@@ -761,7 +822,7 @@ def main(argv: Optional[List[Any]] = None) -> None:
cli(argv, prog_name=PROG, complete_var=SHELL_COMPLETE_VAR)
def expand_file_arguments(argv: List[Any]) -> List[Any]:
def expand_file_arguments(argv: list[Any]) -> list[Any]:
"""
Any argument starting with "@" gets replaced with all values read from a text file.
Text file arguments can be split by newline or by space.
@@ -771,7 +832,7 @@ def expand_file_arguments(argv: List[Any]) -> List[Any]:
visited = set()
expanded = False
def expand_args(args: List[Any], parent_path: str, file_stack: List[str]) -> List[str]:
def expand_args(args: list[Any], parent_path: str, file_stack: list[str]) -> list[str]:
expanded_args = []
for arg in args:
if not arg.startswith('@'):
@@ -789,13 +850,17 @@ def expand_file_arguments(argv: List[Any]) -> List[Any]:
visited.add(rel_path)
try:
with open(rel_path, 'r', encoding='utf-8') as f:
with open(rel_path, encoding='utf-8') as f:
for line in f:
expanded_args.extend(expand_args(shlex.split(line), os.path.dirname(rel_path), file_stack + [file_name]))
except IOError:
expanded_args.extend(
expand_args(shlex.split(line), os.path.dirname(rel_path), file_stack + [file_name])
)
except OSError:
file_stack_str = ' -> '.join(['@' + f for f in file_stack + [file_name]])
raise FatalError(f"File '{rel_path}' (expansion of {file_stack_str}) could not be opened. "
'Please ensure the file exists and you have the necessary permissions to read it.')
raise FatalError(
f"File '{rel_path}' (expansion of {file_stack_str}) could not be opened. "
'Please ensure the file exists and you have the necessary permissions to read it.'
)
return expanded_args
argv = expand_args(argv, os.getcwd(), [])
@@ -806,11 +871,7 @@ def expand_file_arguments(argv: List[Any]) -> List[Any]:
return argv
def _valid_unicode_config() -> Union[codecs.CodecInfo, bool]:
# Python 2 is always good
if sys.version_info[0] == 2:
return True
def _valid_unicode_config() -> codecs.CodecInfo | bool:
# With python 3 unicode environment is required
try:
return codecs.lookup(locale.getpreferredencoding()).name != 'ascii'
@@ -820,11 +881,15 @@ def _valid_unicode_config() -> Union[codecs.CodecInfo, bool]:
def _find_usable_locale() -> str:
try:
locales = subprocess.Popen(['locale', '-a'], stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0].decode('ascii', 'replace')
locales = (
subprocess.Popen(['locale', '-a'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
.communicate()[0]
.decode('ascii', 'replace')
)
except OSError:
locales = ''
usable_locales: List[str] = []
usable_locales: list[str] = []
for line in locales.splitlines():
locale = line.strip()
locale_name = locale.lower().replace('-', '')
@@ -843,7 +908,8 @@ def _find_usable_locale() -> str:
if not usable_locales:
raise FatalError(
'Support for Unicode filenames is required, but no suitable UTF-8 locale was found on your system.'
' Please refer to the manual for your operating system for details on locale reconfiguration.')
' Please refer to the manual for your operating system for details on locale reconfiguration.'
)
return usable_locales[0]
@@ -853,14 +919,16 @@ if __name__ == '__main__':
if 'MSYSTEM' in os.environ:
print_warning(
'MSys/Mingw is no longer supported. Please follow the getting started guide of the '
'documentation in order to set up a suitiable environment, or continue at your own risk.')
'documentation in order to set up a suitiable environment, or continue at your own risk.'
)
elif os.name == 'posix' and not _valid_unicode_config():
# Trying to find best utf-8 locale available on the system and restart python with it
best_locale = _find_usable_locale()
print_warning(
'Your environment is not configured to handle unicode filenames outside of ASCII range.'
' Environment variable LC_ALL is temporary set to %s for unicode support.' % best_locale)
f' Environment variable LC_ALL is temporary set to {best_locale} for unicode support.'
)
os.environ['LC_ALL'] = best_locale
ret = subprocess.call([sys.executable] + sys.argv, env=os.environ)

102
tools/test_build_system/test_common.py
View File

@@ -1,4 +1,4 @@
# SPDX-FileCopyrightText: 2022-2024 Espressif Systems (Shanghai) CO LTD
# SPDX-FileCopyrightText: 2022-2025 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0
import json
import logging
@@ -9,28 +9,23 @@ import subprocess
import sys
import textwrap
from pathlib import Path
from typing import List
import pytest
from test_build_system_helpers import append_to_file
from test_build_system_helpers import EnvDict
from test_build_system_helpers import IdfPyFunc
from test_build_system_helpers import append_to_file
from test_build_system_helpers import file_contains
from test_build_system_helpers import find_python
from test_build_system_helpers import get_snapshot
from test_build_system_helpers import IdfPyFunc
from test_build_system_helpers import replace_in_file
from test_build_system_helpers import run_idf_py
def get_subdirs_absolute_paths(path: Path) -> List[str]:
def get_subdirs_absolute_paths(path: Path) -> list[str]:
"""
Get a list of files with absolute path in a given `path` folder
"""
return [
'{}/{}'.format(dir_path, file_name)
for dir_path, _, file_names in os.walk(path)
for file_name in file_names
]
return [f'{dir_path}/{file_name}' for dir_path, _, file_names in os.walk(path) for file_name in file_names]
@pytest.mark.usefixtures('test_app_copy')
@@ -51,10 +46,11 @@ def test_hints_no_color_output_when_noninteractive(idf_py: IdfPyFunc) -> None:
"""Check that idf.py hints don't include color escape codes in non-interactive builds"""
# make the build fail in such a way that idf.py shows a hint
replace_in_file('main/build_test_app.c', '// placeholder_inside_main',
'esp_chip_info_t chip_info; esp_chip_info(&chip_info);')
replace_in_file(
'main/build_test_app.c', '// placeholder_inside_main', 'esp_chip_info_t chip_info; esp_chip_info(&chip_info);'
)
with (pytest.raises(subprocess.CalledProcessError)) as exc_info:
with pytest.raises(subprocess.CalledProcessError) as exc_info:
idf_py('build')
# Should not actually include a color escape sequence!
@@ -70,10 +66,7 @@ def test_idf_copy(idf_copy: Path, idf_py: IdfPyFunc) -> None:
idf_py('build')
def test_idf_build_with_env_var_sdkconfig_defaults(
test_app_copy: Path,
default_idf_env: EnvDict
) -> None:
def test_idf_build_with_env_var_sdkconfig_defaults(test_app_copy: Path, default_idf_env: EnvDict) -> None:
with open(test_app_copy / 'sdkconfig.test', 'w') as fw:
fw.write('CONFIG_BT_ENABLED=y')
@@ -86,9 +79,7 @@ def test_idf_build_with_env_var_sdkconfig_defaults(
@pytest.mark.usefixtures('test_app_copy')
@pytest.mark.test_app_copy('examples/system/efuse')
def test_efuse_summary_cmake_functions(
default_idf_env: EnvDict
) -> None:
def test_efuse_summary_cmake_functions(default_idf_env: EnvDict) -> None:
default_idf_env['IDF_CI_BUILD'] = '1'
output = run_idf_py('efuse-filter', env=default_idf_env)
assert 'FROM_CMAKE: MAC: 00:00:00:00:00:00' in output.stdout
@@ -117,16 +108,22 @@ def test_python_interpreter_unix(test_app_copy: Path) -> None:
logging.info("Make sure idf.py never runs '/usr/bin/env python' or similar")
env_dict = dict(**os.environ)
python = find_python(env_dict['PATH'])
(test_app_copy / 'python').write_text(textwrap.dedent("""#!/bin/sh
(test_app_copy / 'python').write_text(
textwrap.dedent(
"""
#!/bin/sh
echo "idf.py has executed '/usr/bin/env python' or similar"
exit 1
"""))
"""
)
)
st = os.stat(test_app_copy / 'python')
# equivalent to 'chmod +x ./python'
os.chmod((test_app_copy / 'python'), st.st_mode | stat.S_IEXEC)
env_dict['PATH'] = str(test_app_copy) + os.pathsep + env_dict['PATH']
# python is loaded from env:$PATH, but since false interpreter is provided there, python needs to be specified as argument
# python is loaded from env:$PATH
# but since false interpreter is provided there, python needs to be specified as argument
# if idf.py is reconfigured during it's execution, it would load a false interpreter
run_idf_py('reconfigure', env=env_dict, python=python)
@@ -140,7 +137,8 @@ def test_python_interpreter_win(test_app_copy: Path) -> None:
# on windows python interpreter has compiled code '.exe' format, so this false file can be empty
(test_app_copy / 'python.exe').write_text('')
env_dict['PATH'] = str(test_app_copy) + os.pathsep + env_dict['PATH']
# python is loaded from env:$PATH, but since false interpreter is provided there, python needs to be specified as argument
# python is loaded from env:$PATH
# but since false interpreter is provided there, python needs to be specified as argument
# if idf.py is reconfigured during it's execution, it would load a false interpreter
run_idf_py('reconfigure', env=env_dict, python=python)
@@ -172,7 +170,7 @@ def test_ccache_used_to_build(test_app_copy: Path) -> None:
def test_toolchain_prefix_in_description_file(idf_py: IdfPyFunc, test_app_copy: Path) -> None:
logging.info('Toolchain prefix is set in project description file')
idf_py('reconfigure')
data = json.load(open(test_app_copy / 'build' / 'project_description.json', 'r'))
data = json.load(open(test_app_copy / 'build' / 'project_description.json'))
assert 'monitor_toolprefix' in data
@@ -195,8 +193,10 @@ def test_subcommands_with_options(idf_py: IdfPyFunc, default_idf_env: EnvDict) -
def test_fallback_to_build_system_target(idf_py: IdfPyFunc, test_app_copy: Path) -> None:
logging.info('idf.py fallback to build system target')
msg = 'Custom target is running'
append_to_file(test_app_copy / 'CMakeLists.txt',
'add_custom_target(custom_target COMMAND ${{CMAKE_COMMAND}} -E echo "{}")'.format(msg))
append_to_file(
test_app_copy / 'CMakeLists.txt',
f'add_custom_target(custom_target COMMAND ${{CMAKE_COMMAND}} -E echo "{msg}")',
)
ret = idf_py('custom_target')
assert msg in ret.stdout, 'Custom target did not produce expected output'
@@ -205,10 +205,16 @@ def test_create_component_project(idf_copy: Path) -> None:
logging.info('Create project and component using idf.py and build it')
run_idf_py('-C', 'projects', 'create-project', 'temp_test_project', workdir=idf_copy)
run_idf_py('-C', 'components', 'create-component', 'temp_test_component', workdir=idf_copy)
replace_in_file(idf_copy / 'projects' / 'temp_test_project' / 'main' / 'temp_test_project.c', '{\n\n}',
'\n'.join(['{', '\tfunc();', '}']))
replace_in_file(idf_copy / 'projects' / 'temp_test_project' / 'main' / 'temp_test_project.c', '#include <stdio.h>',
'\n'.join(['#include <stdio.h>', '#include "temp_test_component.h"']))
replace_in_file(
idf_copy / 'projects' / 'temp_test_project' / 'main' / 'temp_test_project.c',
'{\n\n}',
'\n'.join(['{', '\tfunc();', '}']),
)
replace_in_file(
idf_copy / 'projects' / 'temp_test_project' / 'main' / 'temp_test_project.c',
'#include <stdio.h>',
'\n'.join(['#include <stdio.h>', '#include "temp_test_component.h"']),
)
run_idf_py('build', workdir=(idf_copy / 'projects' / 'temp_test_project'))
@@ -244,6 +250,7 @@ def test_create_project_with_idf_readonly(idf_copy: Path) -> None:
if '/bin/' in path:
continue # skip executables
os.chmod(os.path.join(root, name), file_permission)
logging.info('Check that command for creating new project will success if the IDF itself is readonly.')
change_file_permissions(idf_copy, write_permission=False)
try:
@@ -267,7 +274,18 @@ def test_docs_command(idf_py: IdfPyFunc) -> None:
assert 'https://docs.espressif.com/projects/esp-idf/en/v4.2.1' in ret.stdout
ret = idf_py('docs', '--no-browser', '--language', 'en', '--version', 'v4.2.1', '--target', 'esp32')
assert 'https://docs.espressif.com/projects/esp-idf/en/v4.2.1/esp32' in ret.stdout
ret = idf_py('docs', '--no-browser', '--language', 'en', '--version', 'v4.2.1', '--target', 'esp32', '--starting-page', 'get-started')
ret = idf_py(
'docs',
'--no-browser',
'--language',
'en',
'--version',
'v4.2.1',
'--target',
'esp32',
'--starting-page',
'get-started',
)
assert 'https://docs.espressif.com/projects/esp-idf/en/v4.2.1/esp32/get-started' in ret.stdout
@@ -285,25 +303,31 @@ def test_deprecation_warning(idf_py: IdfPyFunc) -> None:
def test_save_defconfig_check(idf_py: IdfPyFunc, test_app_copy: Path) -> None:
logging.info('Save-defconfig checks')
(test_app_copy / 'sdkconfig').write_text('\n'.join(['CONFIG_COMPILER_OPTIMIZATION_SIZE=y',
'CONFIG_ESPTOOLPY_FLASHFREQ_80M=y']))
(test_app_copy / 'sdkconfig').write_text(
'\n'.join(['CONFIG_COMPILER_OPTIMIZATION_SIZE=y', 'CONFIG_ESPTOOLPY_FLASHFREQ_80M=y'])
)
idf_py('save-defconfig')
assert not file_contains(test_app_copy / 'sdkconfig.defaults', 'CONFIG_IDF_TARGET'), \
assert not file_contains(test_app_copy / 'sdkconfig.defaults', 'CONFIG_IDF_TARGET'), (
'CONFIG_IDF_TARGET should not be in sdkconfig.defaults'
assert file_contains(test_app_copy / 'sdkconfig.defaults', 'CONFIG_COMPILER_OPTIMIZATION_SIZE=y'), \
)
assert file_contains(test_app_copy / 'sdkconfig.defaults', 'CONFIG_COMPILER_OPTIMIZATION_SIZE=y'), (
'Missing CONFIG_COMPILER_OPTIMIZATION_SIZE=y in sdkconfig.defaults'
assert file_contains(test_app_copy / 'sdkconfig.defaults', 'CONFIG_ESPTOOLPY_FLASHFREQ_80M=y'), \
)
assert file_contains(test_app_copy / 'sdkconfig.defaults', 'CONFIG_ESPTOOLPY_FLASHFREQ_80M=y'), (
'Missing CONFIG_ESPTOOLPY_FLASHFREQ_80M=y in sdkconfig.defaults'
)
idf_py('fullclean')
(test_app_copy / 'sdkconfig').unlink()
(test_app_copy / 'sdkconfig.defaults').unlink()
idf_py('set-target', 'esp32c3')
(test_app_copy / 'sdkconfig').write_text('CONFIG_PARTITION_TABLE_OFFSET=0x8001')
idf_py('save-defconfig')
assert file_contains(test_app_copy / 'sdkconfig.defaults', 'CONFIG_IDF_TARGET="esp32c3"'), \
assert file_contains(test_app_copy / 'sdkconfig.defaults', 'CONFIG_IDF_TARGET="esp32c3"'), (
'Missing CONFIG_IDF_TARGET="esp32c3" in sdkconfig.defaults'
assert file_contains(test_app_copy / 'sdkconfig.defaults', 'CONFIG_PARTITION_TABLE_OFFSET=0x8001'), \
)
assert file_contains(test_app_copy / 'sdkconfig.defaults', 'CONFIG_PARTITION_TABLE_OFFSET=0x8001'), (
'Missing CONFIG_PARTITION_TABLE_OFFSET=0x8001 in sdkconfig.defaults'
)
def test_merge_bin_cmd(idf_py: IdfPyFunc, test_app_copy: Path) -> None:

16
tools/test_idf_py/test_idf_py.py
View File

@@ -204,7 +204,8 @@ class TestDeprecations(TestWithoutExtensions):
[sys.executable, idf_py_path, '-C', current_dir, 'test-2'], env=os.environ, stderr=subprocess.STDOUT
)
except subprocess.CalledProcessError as e:
self.assertIn('Error: Command "test-2" is deprecated and was removed.', e.output.decode('utf-8', 'ignore'))
output = e.output.decode('utf-8', 'ignore').replace('\r\n', '\n')
self.assertIn('Error: Command "test-2" is deprecated and was removed\n', output)
def test_exit_with_error_for_option(self):
try:
@@ -239,7 +240,6 @@ class TestDeprecations(TestWithoutExtensions):
env=os.environ,
stderr=subprocess.STDOUT,
).decode('utf-8', 'ignore')
self.assertIn('Warning: Option "test_sub_1" is deprecated and will be removed in future versions.', output)
self.assertIn(
'Warning: Command "test-1" is deprecated and will be removed in future versions. '
@@ -374,8 +374,8 @@ class TestWrapperCommands(TestCase):
class TestEFuseCommands(TestWrapperCommands):
"""
Test if wrapper commands for espefuse.py are working as expected.
The goal is NOT to test the functionality of espefuse.py, but to test if the wrapper commands
are working as expected.
The goal is NOT to test the functionality of espefuse.py
but to test if the wrapper commands are working as expected.
"""
def test_efuse_summary(self):
@@ -438,8 +438,8 @@ class TestEFuseCommands(TestWrapperCommands):
class TestSecureCommands(TestWrapperCommands):
"""
Test if wrapper commands for espsecure.py are working as expected.
The goal is NOT to test the functionality of espsecure.py, but to test if the wrapper commands are
working as expected.
The goal is NOT to test the functionality of espsecure.py
but to test if the wrapper commands are working as expected.
"""
@classmethod
@@ -577,8 +577,8 @@ class TestSecureCommands(TestWrapperCommands):
class TestMergeBinCommands(TestWrapperCommands):
"""
Test if merge-bin command is invoked as expected.
This test is not testing the functionality of esptool.py merge_bin command, but the invocation of
the command from idf.py.
This test is not testing the functionality of esptool.py merge_bin command
but the invocation of the command from idf.py.
"""
def test_merge_bin(self):