From 1c22f6c4e867d429b0a45430689959cddc7c84f2 Mon Sep 17 00:00:00 2001 From: Frantisek Hrbata Date: Wed, 8 Nov 2023 16:56:19 +0100 Subject: [PATCH 1/3] feat: add python script to activate ESP-IDF environment Signed-off-by: Frantisek Hrbata --- activate.py | 43 +++ activate_venv.py | 466 +++++++++++++++++++++++ export.fish | 104 +---- export.sh | 264 ++----------- tools/ci/executable-list.txt | 1 + tools/idf_tools.py | 7 +- tools/requirements/requirements.core.txt | 2 + 7 files changed, 557 insertions(+), 330 deletions(-) create mode 100755 activate.py create mode 100644 activate_venv.py diff --git a/activate.py b/activate.py new file mode 100755 index 0000000000..40e1e78156 --- /dev/null +++ b/activate.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python +# SPDX-FileCopyrightText: 2023-2024 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 +""" +Ensure that the Python version used to initiate this script is appropriate for +running the ESP-IDF shell activation. The primary goal is to perform the minimum +necessary checks to identify the virtual environment with the default user Python +and then launch activate.py using the ESP-IDF Python virtual environment. +""" +import os +import sys +from subprocess import run +from subprocess import SubprocessError + + +def die(msg: str) -> None: + sys.exit(f'error: {msg}') + + +idf_path = os.path.realpath(os.path.dirname(__file__)) +idf_tools_path = os.path.join(idf_path, 'tools') +sys.path.insert(0, idf_tools_path) + +try: + # The idf_tools module checks for Python version compatibility. + import idf_tools +except ImportError as e: + die(f'Unable to import the idf_tools module: {e}') + +# Get ESP-IDF venv python path +idf_tools.g.idf_path = idf_path +os.environ['IDF_PYTHON_ENV_PATH'] = '' # let idf_tools get the pyenv path +idf_tools.g.idf_tools_path = os.environ.get('IDF_TOOLS_PATH') or os.path.expanduser(idf_tools.IDF_TOOLS_PATH_DEFAULT) +idf_python_env_path, idf_python_export_path, virtualenv_python, idf_version = idf_tools.get_python_env_path() + +os.environ['IDF_PATH'] = idf_path +os.environ['IDF_PYTHON_ENV_PATH'] = idf_python_env_path +os.environ['ESP_IDF_VERSION'] = idf_version + +try: + run([virtualenv_python, os.path.join(idf_path, 'activate_venv.py')] + sys.argv[1:], check=True) +except (OSError, SubprocessError): + die(f'Activation script failed') diff --git a/activate_venv.py b/activate_venv.py new file mode 100644 index 0000000000..646f798428 --- /dev/null +++ b/activate_venv.py @@ -0,0 +1,466 @@ +# SPDX-FileCopyrightText: 2023-2024 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 +import argparse +import os +import shutil +import sys +from pathlib import Path +from subprocess import run +from subprocess import SubprocessError +from tempfile import gettempdir +from tempfile import NamedTemporaryFile +from tempfile import TemporaryDirectory +from textwrap import dedent +from typing import Any +from typing import Callable +from typing import Dict +from typing import List +from typing import Optional +from typing import TextIO + +try: + # The ESP-IDF virtual environment hasn't been verified yet, so see if the rich library + # can be imported to display error and status messages nicely. + from rich.console import Console +except ImportError as e: + sys.exit(f'error: Unable to import the rich module: {e}. Please execute the install script.') + + +def status_message(msg: str, rv_on_ok: bool=False, die_on_err: bool=True) -> Callable: + def inner(func: Callable) -> Callable: + def wrapper(*args: Any, **kwargs: Any) -> Any: + eprint(f'[dark_orange]*[/dark_orange] {msg} ... ', end='') + + try: + rv = func(*args, **kwargs) + except Exception as e: + eprint('[red]FAILED[/red]') + if ARGS.debug: + raise + if not die_on_err: + return None + die(str(e)) + + if rv_on_ok: + eprint(f'[green]{rv}[/green]') + else: + eprint('[green]OK[/green]') + + return rv + return wrapper + return inner + + +class Shell(): + def __init__(self, shell: str, deactivate_cmd: str, new_esp_idf_env: Dict[str,str]): + self.shell = shell + self.deactivate_cmd = deactivate_cmd + self.new_esp_idf_env = new_esp_idf_env + + # TODO We are not removing the temporary activation scripts. + self.tmp_dir_path = Path(gettempdir()) / 'esp_idf_activate' + self.tmp_dir_path.mkdir(parents=True, exist_ok=True) + with NamedTemporaryFile(dir=self.tmp_dir_path, delete=False, prefix='activate_') as fd: + self.script_file_path = Path(fd.name) + debug(f'Temporary script file path: {self.script_file_path}') + + def expanded_env(self) -> Dict[str, str]: + expanded_env = self.new_esp_idf_env.copy() + + if 'PATH' not in expanded_env: + return expanded_env + + # The PATH returned by idf_tools.py export is not expanded. + # Note that for the export script, the PATH should remain unexpanded + # to ensure proper deactivation. In the export script, + # the expansion should occur after deactivation, when the PATH is adjusted. + # But it has to be expanded for processes started with the new PATH. + expanded_env['PATH'] = os.path.expandvars(expanded_env['PATH']) + return expanded_env + + def spawn(self) -> None: + # This method should likely work for all shells because we are delegating the initialization + # purely to Python os.environ. + new_env = os.environ.copy() + new_env.update(self.expanded_env()) + run([self.shell], env=new_env) + + +class UnixShell(Shell): + def __init__(self, shell: str, deactivate_cmd: str, new_esp_idf_env: Dict[str,str]): + super().__init__(shell, deactivate_cmd, new_esp_idf_env) + self.new_esp_idf_env['IDF_TOOLS_INSTALL_CMD'] = os.path.join(IDF_PATH, 'install.sh') + self.new_esp_idf_env['IDF_TOOLS_EXPORT_CMD'] = os.path.join(IDF_PATH, 'export.sh') + + def export_file(self, fd: TextIO) -> None: + fd.write(f'{self.deactivate_cmd}\n') + for var, value in self.new_esp_idf_env.items(): + fd.write(f'export {var}="{value}"\n') + prompt = self.get_prompt() + fd.write(f'{prompt}\n') + + def get_prompt(self) -> str: + return f'PS1="(ESP-IDF {IDF_VERSION}) $PS1"' + + def export(self) -> None: + with open(self.script_file_path, 'w') as fd: + self.export_file(fd) + fd.write((f'echo "\nDone! You can now compile ESP-IDF projects.\n' + 'Go to the project directory and run:\n\n idf.py build\n"')) + + print(f'. {self.script_file_path}') + + def click_ver(self) -> int: + return int(click.__version__.split('.')[0]) + + +class BashShell(UnixShell): + def get_bash_major(self) -> int: + env = self.expanded_env() + stdout = run_cmd(['bash', '-c', 'echo ${BASH_VERSINFO[0]}'], env=env) + bash_maj = int(stdout) + return bash_maj + + @status_message('Shell completion', die_on_err=False) + def autocompletion(self) -> str: + bash_maj = self.get_bash_major() + if bash_maj < 4: + raise RuntimeError('Autocompletion not supported') + + env = self.expanded_env() + env['LANG'] = 'en' + env['_IDF.PY_COMPLETE'] = 'bash_source' if self.click_ver() >= 8 else 'source_bash' + stdout = run_cmd([IDF_PY], env=env) + return stdout + + def export_file(self, fd: TextIO) -> None: + super().export_file(fd) + stdout = self.autocompletion() + if stdout is not None: + fd.write(f'{stdout}\n') + + def init_file(self) -> None: + with open(self.script_file_path, 'w') as fd: + # We will use the --init-file option to pass a custom rc file, which will ignore .bashrc, + # so we need to source .bashrc first. + fd.write(f'source ~/.bashrc\n') + + stdout = self.autocompletion() + if stdout is not None: + fd.write(f'{stdout}\n') + + prompt = self.get_prompt() + fd.write(f'{prompt}\n') + + def spawn(self) -> None: + self.init_file() + new_env = os.environ.copy() + new_env.update(self.expanded_env()) + run([self.shell, '--init-file', str(self.script_file_path)], env=new_env) + + +class ZshShell(UnixShell): + @status_message('Shell completion', die_on_err=False) + def autocompletion(self) -> str: + env = self.expanded_env() + env['LANG'] = 'en' + env['_IDF.PY_COMPLETE'] = 'zsh_source' if self.click_ver() >= 8 else 'source_zsh' + stdout = run_cmd([IDF_PY], env=env) + return f'autoload -Uz compinit && compinit -u\n{stdout}' + + def export_file(self, fd: TextIO) -> None: + super().export_file(fd) + stdout = self.autocompletion() + # Add autocompletion + if stdout is not None: + fd.write(f'{stdout}\n') + + def init_file(self) -> None: + # If ZDOTDIR is unset, HOME is used instead. + # https://zsh.sourceforge.io/Doc/Release/Files.html#Startup_002fShutdown-Files + zdotdir = os.environ.get('ZDOTDIR', str(Path.home())) + with open(self.script_file_path, 'w') as fd: + # We will use the ZDOTDIR env variable to load our custom script in the newly spawned shell + # so we need to source .zshrc first. + zshrc_path = Path(zdotdir) / '.zshrc' + if zshrc_path.is_file(): + fd.write(f'source {zshrc_path}\n') + + # Add autocompletion + stdout = self.autocompletion() + if stdout is not None: + fd.write(f'{stdout}\n') + + prompt = self.get_prompt() + fd.write(f'{prompt}\n') + + # TODO This might not be needed, or consider resetting it to the original value + fd.write('unset ZDOTDIR\n') + + def spawn(self) -> None: + self.init_file() + + # Create a temporary directory to use as ZDOTDIR + tmpdir = TemporaryDirectory() + tmpdir_path = Path(tmpdir.name) + debug(f'Temporary ZDOTDIR {tmpdir_path} with .zshrc file') + + # Copy init script to the custom ZDOTDIR + zshrc_path = tmpdir_path / '.zshrc' + shutil.copy(str(self.script_file_path), str(zshrc_path)) + + new_env = os.environ.copy() + new_env.update(self.expanded_env()) + # Set new ZDOTDIR in the new environment + new_env['ZDOTDIR'] = str(tmpdir_path) + + run([self.shell], env=new_env) + + +class FishShell(UnixShell): + def __init__(self, shell: str, deactivate_cmd: str, new_esp_idf_env: Dict[str,str]): + super().__init__(shell, deactivate_cmd, new_esp_idf_env) + self.new_esp_idf_env['IDF_TOOLS_INSTALL_CMD'] = os.path.join(IDF_PATH, 'install.fish') + self.new_esp_idf_env['IDF_TOOLS_EXPORT_CMD'] = os.path.join(IDF_PATH, 'export.fish') + + @status_message('Shell completion', die_on_err=False) + def autocompletion(self) -> str: + env = self.expanded_env() + env['LANG'] = 'en' + env['_IDF.PY_COMPLETE'] = 'fish_source' if self.click_ver() >= 8 else 'source_fish' + stdout = run_cmd([IDF_PY], env=env) + return stdout + + def get_prompt(self) -> str: + prompt = dedent(f''' + functions -c fish_prompt _old_fish_prompt + function fish_prompt + printf "(ESP-IDF {IDF_VERSION}) " + _old_fish_prompt + end + ''') + return prompt + + def export_file(self, fd: TextIO) -> None: + fd.write(f'{self.deactivate_cmd}\n') + for var, value in self.new_esp_idf_env.items(): + fd.write(f'export {var}="{value}"\n') + + # Add autocompletion + stdout = self.autocompletion() + if stdout is not None: + fd.write(f'{stdout}\n') + + # Adjust fish prompt + prompt = self.get_prompt() + fd.write(f'{prompt}\n') + + def init_file(self) -> None: + with open(self.script_file_path, 'w') as fd: + # Add autocompletion + stdout = self.autocompletion() + if stdout is not None: + fd.write(f'{stdout}\n') + # Adjust fish prompt + prompt = self.get_prompt() + fd.write(f'{prompt}\n') + + def spawn(self) -> None: + self.init_file() + new_env = os.environ.copy() + new_env.update(self.expanded_env()) + run([self.shell, f'--init-command=source {self.script_file_path}'], env=new_env) + + +SHELL_CLASSES = { + 'bash': BashShell, + 'zsh': ZshShell, + 'fish': FishShell, + 'sh': UnixShell, + 'ksh': UnixShell, + 'dash': UnixShell, + 'nu': UnixShell, +} + +SUPPORTED_SHELLS = ' '.join(SHELL_CLASSES.keys()) + +CONSOLE_STDERR = None +CONSOLE_STDOUT = None + + +def err(*args: Any, **kwargs: Any) -> None: + CONSOLE_STDERR.print('[red]error[/red]: ', *args, **kwargs) # type: ignore + + +def warn(*args: Any, **kwargs: Any) -> None: + CONSOLE_STDERR.print('[yellow]warning[/yellow]: ', *args, **kwargs) # type: ignore + + +def debug(*args: Any, **kwargs: Any) -> None: + if not ARGS.debug: + return + CONSOLE_STDERR.print('[green_yellow]debug[/green_yellow]: ', *args, **kwargs) # type: ignore + + +def die(*args: Any, **kwargs: Any) -> None: + err(*args, **kwargs) + sys.exit(1) + + +def eprint(*args: Any, **kwargs: Any) -> None: + CONSOLE_STDERR.print(*args, **kwargs) # type: ignore + + +def oprint(*args: Any, **kwargs: Any) -> None: + CONSOLE_STDOUT.print(*args, **kwargs) # type: ignore + + +def run_cmd(cmd: List[str], env: Optional[Dict[str, Any]]=None) -> str: + new_env = os.environ.copy() + if env is not None: + new_env.update(env) + + cmd_str = '"{}"'.format(' '.join(cmd)) + try: + p = run(cmd, env=new_env, text=True, capture_output=True) + except (OSError, SubprocessError) as e: + raise RuntimeError(f'Command {cmd_str} failed: {e}') + + stdout = p.stdout.strip() + stderr = p.stderr.strip() + if p.returncode: + raise RuntimeError(f'Command {cmd_str} failed with error code {p.returncode}\n{stdout}\n{stderr}') + + return stdout + + +parser = argparse.ArgumentParser(prog='activate', + description='Activate ESP-IDF environment') +parser.add_argument('-s', '--shell', + metavar='SHELL', + default=os.environ.get('ESP_IDF_SHELL', None), + help='Explicitly specify shell to start. For example bash, zsh, powershell.exe, cmd.exe') +parser.add_argument('-l', '--list', + action='store_true', + help=('List supported shells.')) +parser.add_argument('-e', '--export', + action='store_true', + help=('Generate commands to run in the terminal.')) +parser.add_argument('-n', '--no-color', + action='store_true', + help=('Disable ANSI color escape sequences.')) +parser.add_argument('-d', '--debug', + action='store_true', + help=('Enable debug information.')) +parser.add_argument('-q', '--quiet', + action='store_true', + help=('Suppress all output.')) + +ARGS = parser.parse_args() + +CONSOLE_STDERR = Console(stderr=True, quiet=ARGS.quiet, no_color=ARGS.no_color) +CONSOLE_STDOUT = Console(quiet=ARGS.quiet, no_color=ARGS.no_color) + +if ARGS.list: + oprint(SUPPORTED_SHELLS) + sys.exit() + +# The activate.py script sets the following environment variables +IDF_PATH = os.environ['IDF_PATH'] +IDF_VERSION = os.environ['ESP_IDF_VERSION'] +IDF_PYTHON_ENV_PATH = os.environ['IDF_PYTHON_ENV_PATH'] +IDF_TOOLS_PY = os.path.join(IDF_PATH, 'tools', 'idf_tools.py') +IDF_PY = os.path.join(IDF_PATH, 'tools', 'idf.py') + +eprint(f'[dark_orange]Activating ESP-IDF {IDF_VERSION}') +debug(f'IDF_PATH {IDF_PATH}') +debug(f'IDF_PYTHON_ENV_PATH {IDF_PYTHON_ENV_PATH}') + + +@status_message('Checking python version', rv_on_ok=True) +def check_python_version() -> str: + # Check the Python version within a virtual environment + python_version_checker = os.path.join(IDF_PATH, 'tools', 'python_version_checker.py') + run_cmd([sys.executable, python_version_checker]) + ver = sys.version_info[:3] + return f'{ver[0]}.{ver[1]}.{ver[2]}' + + +@status_message('Checking python dependencies') +def check_python_dependencies() -> None: + # Check Python dependencies within the virtual environment + run_cmd([sys.executable, IDF_TOOLS_PY, 'check-python-dependencies']) + + +check_python_version() +check_python_dependencies() +# TODO Report installed tools that are not currently used by active ESP-IDF version + +# From this point forward, we are functioning within a fully validated ESP-IDF environment. + +# TODO Verify the architectures supported by psutils. We might need to create a wheel for it or +# substitute it with ps and tasklist commands. +import psutil # noqa: E402 +import click # noqa: E402 + + +@status_message('Deactivating the current ESP-IDF environment') +def get_deactivate_cmd() -> str: + # Get previous ESP-IDF system environment variables + cmd = [sys.executable, IDF_TOOLS_PY, 'export', '--deactivate'] + stdout = run_cmd(cmd) + return stdout + + +@status_message('Establishing a new ESP-IDF environment') +def get_idf_env() -> Dict[str,str]: + # Get ESP-IDF system environment variables + extra_paths_list = [os.path.join('components', 'espcoredump'), + os.path.join('components', 'partition_table'), + os.path.join('components', 'app_update')] + extra_paths = ':'.join([os.path.join(IDF_PATH, path) for path in extra_paths_list]) + cmd = [sys.executable, IDF_TOOLS_PY, 'export', '--format', 'key-value', '--add_paths_extras', extra_paths] + stdout = run_cmd(cmd) + + # idf_tools.py might not export certain environment variables if they are already set + idf_env: Dict[str, Any] = { + 'IDF_PATH': os.environ['IDF_PATH'], + 'ESP_IDF_VERSION': os.environ['ESP_IDF_VERSION'], + 'IDF_PYTHON_ENV_PATH': os.environ['IDF_PYTHON_ENV_PATH'], + } + + for line in stdout.splitlines(): + var, val = line.split('=') + idf_env[var] = val + + if 'PATH' in idf_env: + idf_env['PATH'] = ':'.join([extra_paths, idf_env['PATH']]) + + return idf_env + + +@status_message('Identifying shell', rv_on_ok=True) +def detect_shell() -> str: + if ARGS.shell is not None: + return str(ARGS.shell) + ppid = psutil.Process(os.getpid()).ppid() + # Look for grandparent, because we started from activate.py. + pppid = psutil.Process(ppid).ppid() + return str(psutil.Process(pppid).name()) + + +deactivate_cmd = get_deactivate_cmd() +new_esp_idf_env = get_idf_env() +detected_shell = detect_shell() + +if detected_shell not in SHELL_CLASSES: + die(f'"{detected_shell}" shell is not among the supported options: "{SUPPORTED_SHELLS}"') + +shell = SHELL_CLASSES[detected_shell](detected_shell, deactivate_cmd, new_esp_idf_env) + +if ARGS.export: + shell.export() + sys.exit() + +shell.spawn() +eprint(f'[dark_orange]ESP-IDF environment exited.') diff --git a/export.fish b/export.fish index 64ccdf7879..cdff5798e0 100644 --- a/export.fish +++ b/export.fish @@ -5,105 +5,5 @@ function unset set --erase $argv end -function __main - set script_dir (dirname (realpath (status -f))) - if not set -q IDF_PATH - set -gx IDF_PATH $script_dir - echo "Setting IDF_PATH to '$IDF_PATH'" - end - - if test "$IDF_PATH" != "$script_dir" - # Change IDF_PATH is important when there are 2 ESP-IDF versions in different directories. - # Sourcing this script without change, would cause sourcing wrong export script. - echo "Resetting IDF_PATH from '$IDF_PATH' to '$script_dir'" - set IDF_PATH "$script_dir" - end - - set oldpath = $PATH - - echo "Detecting the Python interpreter" - source "$IDF_PATH"/tools/detect_python.fish - - echo "Checking Python compatibility" - "$ESP_PYTHON" "$IDF_PATH"/tools/python_version_checker.py - - echo "Checking other ESP-IDF version." - set idf_deactivate ("$ESP_PYTHON" "$IDF_PATH"/tools/idf_tools.py export --deactivate) || return 1 - eval "$idf_deactivate" - - echo "Adding ESP-IDF tools to PATH..." - # Call idf_tools.py to export tool paths - set -gx IDF_TOOLS_EXPORT_CMD "$IDF_PATH"/export.fish - set -gx IDF_TOOLS_INSTALL_CMD "$IDF_PATH"/install.fish - # Allow calling some IDF python tools without specifying the full path - # "$IDF_PATH"/tools is already added by 'idf_tools.py export' - set IDF_ADD_PATHS_EXTRAS "$IDF_PATH"/components/espcoredump - set IDF_ADD_PATHS_EXTRAS "$IDF_ADD_PATHS_EXTRAS":"$IDF_PATH"/components/partition_table - set IDF_ADD_PATHS_EXTRAS "$IDF_ADD_PATHS_EXTRAS":"$IDF_PATH"/components/app_update - - set idf_exports ("$ESP_PYTHON" "$IDF_PATH"/tools/idf_tools.py export --add_paths_extras="$IDF_ADD_PATHS_EXTRAS") || return 1 - eval "$idf_exports" - set -x PATH "$IDF_ADD_PATHS_EXTRAS":"$PATH" - - echo "Checking if Python packages are up to date..." - "$ESP_PYTHON" "$IDF_PATH"/tools/idf_tools.py check-python-dependencies || return 1 - - set added_path_variables - for entry in $PATH; - if not contains $entry $oldpath - set -a added_path_variables $entry - end - end - if set -q added_path_variables[1] - echo "Added the following directories to PATH:" - for entry in $added_path_variables; - echo $entry - end - else - echo "All paths are already set." - end - - set uninstall ("$ESP_PYTHON" "$IDF_PATH"/tools/idf_tools.py uninstall --dry-run) || return 1 - if test -n "$uninstall" - echo "" - echo "Detected installed tools that are not currently used by active ESP-IDF version." - echo "$uninstall" - echo "For free up even more space, remove installation packages of those tools. Use option '$ESP_PYTHON $IDF_PATH/tools/idf_tools.py uninstall --remove-archives'." - echo "" - end - - # Clean up - set -e added_path_variables - set -e cmd - set -e old_path - set -e paths - set -e path_prefix - set -e path_entry - set -e IDF_ADD_PATHS_EXTRAS - set -e idf_exports - set -e ESP_PYTHON - set -e uninstall - set -e script_dir - set -e idf_deactivate - - - # Not unsetting IDF_PYTHON_ENV_PATH, it can be used by IDF build system - # to check whether we are using a private Python environment - - echo "Done! You can now compile ESP-IDF projects." - echo "Go to the project directory and run:" - echo "" - echo " idf.py build" - echo "" -end - -__main - -set click_version (python -c 'import click; print(click.__version__.split(".")[0])') -if test $click_version -lt 8 - eval (env _IDF.PY_COMPLETE=source_fish idf.py) -else - eval (env _IDF.PY_COMPLETE=fish_source idf.py) -end - -functions -e __main +set script_dir (dirname (realpath (status -f))) +eval ("$script_dir"/activate.py --export) diff --git a/export.sh b/export.sh index cd4afe3a74..02646293b5 100644 --- a/export.sh +++ b/export.sh @@ -1,233 +1,47 @@ # This script should be sourced, not executed. -__realpath() { - wdir="$PWD"; [ "$PWD" = "/" ] && wdir="" - arg=$1 - case "$arg" in - /*) scriptdir="${arg}";; - *) scriptdir="$wdir/${arg#./}";; - esac - scriptdir="${scriptdir%/*}" - echo "$scriptdir" -} +# shellcheck disable=SC2128,SC2169,SC2039,SC3054 # ignore array expansion warning +if [ -n "${BASH_SOURCE-}" ] && [ "${BASH_SOURCE[0]}" = "${0}" ] +then + echo "This script should be sourced, not executed:" + # shellcheck disable=SC2039,SC3054 # reachable only with bash + echo ". ${BASH_SOURCE[0]}" + exit 1 +fi +# Attempt to identify the ESP-IDF directory +idf_path="." -__verbose() { - [ -n "${IDF_EXPORT_QUIET-}" ] && return - echo "$@" -} - -__script_dir(){ - # shellcheck disable=SC2169,SC2169,SC2039,SC3010,SC3028 # unreachable with 'dash' - if [ "$(uname -s)" = "Darwin" ]; then - # convert possibly relative path to absolute - script_dir="$(__realpath "${self_path}")" - # resolve any ../ references to make the path shorter - script_dir="$(cd "${script_dir}" || exit 1; pwd)" - else - # convert to full path and get the directory name of that - script_name="$(readlink -f "${self_path}")" - script_dir="$(dirname "${script_name}")" - fi - if [ "$script_dir" = '.' ] - then - script_dir="$(pwd)" - fi - echo "$script_dir" -} - -__is_dir_esp_idf(){ - if [ ! -f "$1/tools/idf.py" ] || [ ! -f "$1/tools/idf_tools.py" ] - then - # Echo command here is not used for printing to the terminal, but as non-empty return value from function. - echo "THIS DIRECTORY IS NOT ESP-IDF" - fi -} - -__main() { - # The file doesn't have executable permissions, so this shouldn't really happen. - # Doing this in case someone tries to chmod +x it and execute... - - # shellcheck disable=SC2128,SC2169,SC2039,SC3054 # ignore array expansion warning - if [ -n "${BASH_SOURCE-}" ] && [ "${BASH_SOURCE[0]}" = "${0}" ] - then - echo "This script should be sourced, not executed:" - # shellcheck disable=SC2039,SC3054 # reachable only with bash - echo ". ${BASH_SOURCE[0]}" - return 1 - fi - - # If using bash or zsh, try to guess IDF_PATH from script location. - self_path="" - # shellcheck disable=SC2128 # ignore array expansion warning - if [ -n "${BASH_SOURCE-}" ] - then - self_path="${BASH_SOURCE}" - elif [ -n "${ZSH_VERSION-}" ] - then +# shellcheck disable=SC2128,SC2169,SC2039,SC3054,SC3028 # ignore array expansion warning +if test -n "${BASH_SOURCE-}" +then + # shellcheck disable=SC3028,SC3054 # unreachable with 'dash' + idf_path=$(dirname "${BASH_SOURCE[0]}") +elif test -n "${ZSH_VERSION-}" +then # shellcheck disable=SC2296 # ignore parameter starts with '{' because it's zsh - self_path="${(%):-%x}" - fi + idf_path=$(dirname "${(%):-%x}") +elif test -n "${IDF_PATH-}" +then + idf_path=$IDF_PATH +fi - script_dir="$(__script_dir)" - # Since sh or dash shells can't detect script_dir correctly, check if script_dir looks like an IDF directory - is_script_dir_esp_idf=$(__is_dir_esp_idf "${script_dir}") +if [ ! -f "${idf_path}/tools/idf.py" ] || + [ ! -f "${idf_path}/tools/idf_tools.py" ] || + [ ! -f "${idf_path}/activate.py" ] +then + # Echo command here is not used for printing to the terminal, but as non-empty return value from function. + echo "Could not detect IDF_PATH. Please set it before sourcing this script:" + echo " export IDF_PATH=(add path here)" + unset idf_path + return 1 +fi - if [ -z "${IDF_PATH-}" ] - then - # IDF_PATH not set in the environment. +# TODO Maybe we can use "command -v" to check just for python and python3 +. "${idf_path}/tools/detect_python.sh" - if [ -n "${is_script_dir_esp_idf}" ] - then - echo "Could not detect IDF_PATH. Please set it before sourcing this script:" - echo " export IDF_PATH=(add path here)" - return 1 - fi - export IDF_PATH="${script_dir}" - echo "Setting IDF_PATH to '${IDF_PATH}'" - else - # IDF_PATH came from the environment, check if the path is valid - # Set IDF_PATH to script_dir, if script_dir looks like an IDF directory - if [ ! "${IDF_PATH}" = "${script_dir}" ] && [ -z "${is_script_dir_esp_idf}" ] - then - # Change IDF_PATH is important when there are 2 ESP-IDF versions in different directories. - # Sourcing this script without change, would cause sourcing wrong export script. - echo "Resetting IDF_PATH from '${IDF_PATH}' to '${script_dir}' " - export IDF_PATH="${script_dir}" - fi - # Check if this path looks like an IDF directory - is_idf_path_esp_idf=$(__is_dir_esp_idf "${IDF_PATH}") - if [ -n "${is_idf_path_esp_idf}" ] - then - echo "IDF_PATH is set to '${IDF_PATH}', but it doesn't look like an ESP-IDF directory." - echo "If you have set IDF_PATH manually, check if the path is correct." - return 1 - fi - - # The varible might have been set (rather than exported), re-export it to be sure - export IDF_PATH="${IDF_PATH}" - fi - - old_path="$PATH" - - echo "Detecting the Python interpreter" - . "${IDF_PATH}/tools/detect_python.sh" - - echo "Checking Python compatibility" - "$ESP_PYTHON" "${IDF_PATH}/tools/python_version_checker.py" - - __verbose "Checking other ESP-IDF version." - idf_deactivate=$("$ESP_PYTHON" "${IDF_PATH}/tools/idf_tools.py" export --deactivate) || return 1 - eval "${idf_deactivate}" - - __verbose "Adding ESP-IDF tools to PATH..." - # Call idf_tools.py to export tool paths - export IDF_TOOLS_EXPORT_CMD=${IDF_PATH}/export.sh - export IDF_TOOLS_INSTALL_CMD=${IDF_PATH}/install.sh - # Allow calling some IDF python tools without specifying the full path - # ${IDF_PATH}/tools is already added by 'idf_tools.py export' - IDF_ADD_PATHS_EXTRAS="${IDF_PATH}/components/espcoredump" - IDF_ADD_PATHS_EXTRAS="${IDF_ADD_PATHS_EXTRAS}:${IDF_PATH}/components/partition_table" - IDF_ADD_PATHS_EXTRAS="${IDF_ADD_PATHS_EXTRAS}:${IDF_PATH}/components/app_update" - - idf_exports=$("$ESP_PYTHON" "${IDF_PATH}/tools/idf_tools.py" export "--add_paths_extras=${IDF_ADD_PATHS_EXTRAS}") || return 1 - eval "${idf_exports}" - export PATH="${IDF_ADD_PATHS_EXTRAS}:${PATH}" - - __verbose "Checking if Python packages are up to date..." - "$ESP_PYTHON" "${IDF_PATH}/tools/idf_tools.py" check-python-dependencies || return 1 - - if [ -n "$BASH" ] - then - path_prefix="${PATH%%"${old_path}"}" - # shellcheck disable=SC2169,SC2039 # unreachable with 'dash' - if [ -n "${path_prefix}" ]; then - __verbose "Added the following directories to PATH:" - else - __verbose "All paths are already set." - fi - old_ifs="$IFS" - IFS=":" - for path_entry in ${path_prefix} - do - __verbose " ${path_entry}" - done - IFS="$old_ifs" - unset old_ifs - else - __verbose "Updated PATH variable:" - __verbose " ${PATH}" - fi - - uninstall=$("$ESP_PYTHON" "${IDF_PATH}/tools/idf_tools.py" uninstall --dry-run) || return 1 - if [ -n "$uninstall" ] - then - __verbose "" - __verbose "Detected installed tools that are not currently used by active ESP-IDF version." - __verbose "${uninstall}" - __verbose "To free up even more space, remove installation packages of those tools. Use option '${ESP_PYTHON} ${IDF_PATH}/tools/idf_tools.py uninstall --remove-archives'." - __verbose "" - fi - - __verbose "Done! You can now compile ESP-IDF projects." - __verbose "Go to the project directory and run:" - __verbose "" - __verbose " idf.py build" - __verbose "" -} - -__cleanup() { - unset old_path - unset paths - unset path_prefix - unset path_entry - unset IDF_ADD_PATHS_EXTRAS - unset idf_exports - unset idf_deactivate - unset ESP_PYTHON - unset SOURCE_ZSH - unset SOURCE_BASH - unset WARNING_MSG - unset uninstall - unset is_idf_path_esp_idf - unset is_script_dir_esp_idf - - unset __realpath - unset __main - unset __verbose - unset __enable_autocomplete - unset __cleanup - unset __is_dir_esp_idf - - # Not unsetting IDF_PYTHON_ENV_PATH, it can be used by IDF build system - # to check whether we are using a private Python environment - - return "$1" -} - - -__enable_autocomplete() { - click_version="$(python -c 'import click; print(click.__version__.split(".")[0])')" - if [ "${click_version}" -lt 8 ] - then - SOURCE_ZSH=source_zsh - SOURCE_BASH=source_bash - else - SOURCE_ZSH=zsh_source - SOURCE_BASH=bash_source - fi - if [ -n "${ZSH_VERSION-}" ] - then - autoload -Uz compinit && compinit -u - eval "$(env _IDF.PY_COMPLETE=$SOURCE_ZSH idf.py)" || echo "WARNING: Failed to load shell autocompletion for zsh version: $ZSH_VERSION!" - elif [ -n "${BASH_SOURCE-}" ] - then - WARNING_MSG="WARNING: Failed to load shell autocompletion for bash version: $BASH_VERSION!" - # shellcheck disable=SC3028,SC3054,SC2086,SC2169 # code block for 'bash' only - [ ${BASH_VERSINFO[0]} -lt 4 ] && { echo "$WARNING_MSG"; return; } - eval "$(env LANG=en _IDF.PY_COMPLETE=$SOURCE_BASH idf.py)" || echo "$WARNING_MSG" - fi -} - -__main && __enable_autocomplete -__cleanup $? +# Evaluate the ESP-IDF environment set up by the activate.py script. +idf_exports=$("$ESP_PYTHON" "${idf_path}/activate.py" --export) +eval "${idf_exports}" +unset idf_path +return 0 diff --git a/tools/ci/executable-list.txt b/tools/ci/executable-list.txt index f6bbe453d3..d235422c16 100644 --- a/tools/ci/executable-list.txt +++ b/tools/ci/executable-list.txt @@ -1,3 +1,4 @@ +activate.py components/app_update/otatool.py components/efuse/efuse_table_gen.py components/efuse/test_efuse_host/efuse_tests.py diff --git a/tools/idf_tools.py b/tools/idf_tools.py index 6e155575a9..753e87e24a 100755 --- a/tools/idf_tools.py +++ b/tools/idf_tools.py @@ -1851,7 +1851,7 @@ def add_variables_to_deactivate_file(args: List[str], new_idf_vars:Dict[str, Any return deactivate_file_path -def deactivate_statement(args: List[str]) -> None: +def print_deactivate_statement(args: List[str]) -> None: """ Deactivate statement is sequence of commands, that remove IDF global variables from environment, so the environment gets to the state it was before calling export.{sh/fish} script. @@ -2152,8 +2152,9 @@ def action_export(args: Any) -> None: """ Exports all necessary environment variables and paths needed for tools used. """ - if args.deactivate and different_idf_detected(): - deactivate_statement(args) + if args.deactivate: + if different_idf_detected(): + print_deactivate_statement(args) return tools_info = load_tools_info() diff --git a/tools/requirements/requirements.core.txt b/tools/requirements/requirements.core.txt index e1bac0b056..9e5d4660bb 100644 --- a/tools/requirements/requirements.core.txt +++ b/tools/requirements/requirements.core.txt @@ -19,6 +19,8 @@ esp-idf-size esp-idf-panic-decoder pyclang construct +rich +psutil # gdb extensions dependencies freertos_gdb From 88527faff88edad0679f2adcc87afb1a66b6a296 Mon Sep 17 00:00:00 2001 From: Marek Fiala Date: Fri, 21 Jun 2024 17:13:18 +0200 Subject: [PATCH 2/3] feat(tools): Added Windows shells support + refactoring --- .gitlab/ci/rules.yml | 7 + activate_venv.py | 466 ---------------------- docs/en/api-guides/tools/idf-tools.rst | 14 +- docs/zh_CN/api-guides/tools/idf-tools.rst | 14 +- export.bat | 96 +---- export.fish | 17 +- export.ps1 | 96 +---- export.sh | 6 +- activate.py => tools/activate.py | 6 +- tools/ci/executable-list.txt | 2 +- tools/export_utils/activate_venv.py | 177 ++++++++ tools/export_utils/console_output.py | 69 ++++ tools/export_utils/shell_types.py | 316 +++++++++++++++ tools/export_utils/utils.py | 48 +++ 14 files changed, 694 insertions(+), 640 deletions(-) delete mode 100644 activate_venv.py rename activate.py => tools/activate.py (85%) create mode 100644 tools/export_utils/activate_venv.py create mode 100644 tools/export_utils/console_output.py create mode 100644 tools/export_utils/shell_types.py create mode 100644 tools/export_utils/utils.py diff --git a/.gitlab/ci/rules.yml b/.gitlab/ci/rules.yml index 3685149e7f..d16fbbc9c3 100644 --- a/.gitlab/ci/rules.yml +++ b/.gitlab/ci/rules.yml @@ -83,6 +83,8 @@ - "tools/idf_monitor.py" + - "tools/activate.py" + - "tools/idf.py" - "tools/idf_py_actions/**/*" - "tools/test_idf_py/**/*" @@ -96,6 +98,11 @@ - "tools/test_idf_tools/**/*" - "tools/install_util.py" + - "tools/export_utils/utils.py" + - "tools/export_utils/shell_types.py" + - "tools/export_utils/console_output.py" + - "tools/export_utils/activate_venv.py" + - "tools/requirements/*" - "tools/requirements.json" - "tools/requirements_schema.json" diff --git a/activate_venv.py b/activate_venv.py deleted file mode 100644 index 646f798428..0000000000 --- a/activate_venv.py +++ /dev/null @@ -1,466 +0,0 @@ -# SPDX-FileCopyrightText: 2023-2024 Espressif Systems (Shanghai) CO LTD -# SPDX-License-Identifier: Apache-2.0 -import argparse -import os -import shutil -import sys -from pathlib import Path -from subprocess import run -from subprocess import SubprocessError -from tempfile import gettempdir -from tempfile import NamedTemporaryFile -from tempfile import TemporaryDirectory -from textwrap import dedent -from typing import Any -from typing import Callable -from typing import Dict -from typing import List -from typing import Optional -from typing import TextIO - -try: - # The ESP-IDF virtual environment hasn't been verified yet, so see if the rich library - # can be imported to display error and status messages nicely. - from rich.console import Console -except ImportError as e: - sys.exit(f'error: Unable to import the rich module: {e}. Please execute the install script.') - - -def status_message(msg: str, rv_on_ok: bool=False, die_on_err: bool=True) -> Callable: - def inner(func: Callable) -> Callable: - def wrapper(*args: Any, **kwargs: Any) -> Any: - eprint(f'[dark_orange]*[/dark_orange] {msg} ... ', end='') - - try: - rv = func(*args, **kwargs) - except Exception as e: - eprint('[red]FAILED[/red]') - if ARGS.debug: - raise - if not die_on_err: - return None - die(str(e)) - - if rv_on_ok: - eprint(f'[green]{rv}[/green]') - else: - eprint('[green]OK[/green]') - - return rv - return wrapper - return inner - - -class Shell(): - def __init__(self, shell: str, deactivate_cmd: str, new_esp_idf_env: Dict[str,str]): - self.shell = shell - self.deactivate_cmd = deactivate_cmd - self.new_esp_idf_env = new_esp_idf_env - - # TODO We are not removing the temporary activation scripts. - self.tmp_dir_path = Path(gettempdir()) / 'esp_idf_activate' - self.tmp_dir_path.mkdir(parents=True, exist_ok=True) - with NamedTemporaryFile(dir=self.tmp_dir_path, delete=False, prefix='activate_') as fd: - self.script_file_path = Path(fd.name) - debug(f'Temporary script file path: {self.script_file_path}') - - def expanded_env(self) -> Dict[str, str]: - expanded_env = self.new_esp_idf_env.copy() - - if 'PATH' not in expanded_env: - return expanded_env - - # The PATH returned by idf_tools.py export is not expanded. - # Note that for the export script, the PATH should remain unexpanded - # to ensure proper deactivation. In the export script, - # the expansion should occur after deactivation, when the PATH is adjusted. - # But it has to be expanded for processes started with the new PATH. - expanded_env['PATH'] = os.path.expandvars(expanded_env['PATH']) - return expanded_env - - def spawn(self) -> None: - # This method should likely work for all shells because we are delegating the initialization - # purely to Python os.environ. - new_env = os.environ.copy() - new_env.update(self.expanded_env()) - run([self.shell], env=new_env) - - -class UnixShell(Shell): - def __init__(self, shell: str, deactivate_cmd: str, new_esp_idf_env: Dict[str,str]): - super().__init__(shell, deactivate_cmd, new_esp_idf_env) - self.new_esp_idf_env['IDF_TOOLS_INSTALL_CMD'] = os.path.join(IDF_PATH, 'install.sh') - self.new_esp_idf_env['IDF_TOOLS_EXPORT_CMD'] = os.path.join(IDF_PATH, 'export.sh') - - def export_file(self, fd: TextIO) -> None: - fd.write(f'{self.deactivate_cmd}\n') - for var, value in self.new_esp_idf_env.items(): - fd.write(f'export {var}="{value}"\n') - prompt = self.get_prompt() - fd.write(f'{prompt}\n') - - def get_prompt(self) -> str: - return f'PS1="(ESP-IDF {IDF_VERSION}) $PS1"' - - def export(self) -> None: - with open(self.script_file_path, 'w') as fd: - self.export_file(fd) - fd.write((f'echo "\nDone! You can now compile ESP-IDF projects.\n' - 'Go to the project directory and run:\n\n idf.py build\n"')) - - print(f'. {self.script_file_path}') - - def click_ver(self) -> int: - return int(click.__version__.split('.')[0]) - - -class BashShell(UnixShell): - def get_bash_major(self) -> int: - env = self.expanded_env() - stdout = run_cmd(['bash', '-c', 'echo ${BASH_VERSINFO[0]}'], env=env) - bash_maj = int(stdout) - return bash_maj - - @status_message('Shell completion', die_on_err=False) - def autocompletion(self) -> str: - bash_maj = self.get_bash_major() - if bash_maj < 4: - raise RuntimeError('Autocompletion not supported') - - env = self.expanded_env() - env['LANG'] = 'en' - env['_IDF.PY_COMPLETE'] = 'bash_source' if self.click_ver() >= 8 else 'source_bash' - stdout = run_cmd([IDF_PY], env=env) - return stdout - - def export_file(self, fd: TextIO) -> None: - super().export_file(fd) - stdout = self.autocompletion() - if stdout is not None: - fd.write(f'{stdout}\n') - - def init_file(self) -> None: - with open(self.script_file_path, 'w') as fd: - # We will use the --init-file option to pass a custom rc file, which will ignore .bashrc, - # so we need to source .bashrc first. - fd.write(f'source ~/.bashrc\n') - - stdout = self.autocompletion() - if stdout is not None: - fd.write(f'{stdout}\n') - - prompt = self.get_prompt() - fd.write(f'{prompt}\n') - - def spawn(self) -> None: - self.init_file() - new_env = os.environ.copy() - new_env.update(self.expanded_env()) - run([self.shell, '--init-file', str(self.script_file_path)], env=new_env) - - -class ZshShell(UnixShell): - @status_message('Shell completion', die_on_err=False) - def autocompletion(self) -> str: - env = self.expanded_env() - env['LANG'] = 'en' - env['_IDF.PY_COMPLETE'] = 'zsh_source' if self.click_ver() >= 8 else 'source_zsh' - stdout = run_cmd([IDF_PY], env=env) - return f'autoload -Uz compinit && compinit -u\n{stdout}' - - def export_file(self, fd: TextIO) -> None: - super().export_file(fd) - stdout = self.autocompletion() - # Add autocompletion - if stdout is not None: - fd.write(f'{stdout}\n') - - def init_file(self) -> None: - # If ZDOTDIR is unset, HOME is used instead. - # https://zsh.sourceforge.io/Doc/Release/Files.html#Startup_002fShutdown-Files - zdotdir = os.environ.get('ZDOTDIR', str(Path.home())) - with open(self.script_file_path, 'w') as fd: - # We will use the ZDOTDIR env variable to load our custom script in the newly spawned shell - # so we need to source .zshrc first. - zshrc_path = Path(zdotdir) / '.zshrc' - if zshrc_path.is_file(): - fd.write(f'source {zshrc_path}\n') - - # Add autocompletion - stdout = self.autocompletion() - if stdout is not None: - fd.write(f'{stdout}\n') - - prompt = self.get_prompt() - fd.write(f'{prompt}\n') - - # TODO This might not be needed, or consider resetting it to the original value - fd.write('unset ZDOTDIR\n') - - def spawn(self) -> None: - self.init_file() - - # Create a temporary directory to use as ZDOTDIR - tmpdir = TemporaryDirectory() - tmpdir_path = Path(tmpdir.name) - debug(f'Temporary ZDOTDIR {tmpdir_path} with .zshrc file') - - # Copy init script to the custom ZDOTDIR - zshrc_path = tmpdir_path / '.zshrc' - shutil.copy(str(self.script_file_path), str(zshrc_path)) - - new_env = os.environ.copy() - new_env.update(self.expanded_env()) - # Set new ZDOTDIR in the new environment - new_env['ZDOTDIR'] = str(tmpdir_path) - - run([self.shell], env=new_env) - - -class FishShell(UnixShell): - def __init__(self, shell: str, deactivate_cmd: str, new_esp_idf_env: Dict[str,str]): - super().__init__(shell, deactivate_cmd, new_esp_idf_env) - self.new_esp_idf_env['IDF_TOOLS_INSTALL_CMD'] = os.path.join(IDF_PATH, 'install.fish') - self.new_esp_idf_env['IDF_TOOLS_EXPORT_CMD'] = os.path.join(IDF_PATH, 'export.fish') - - @status_message('Shell completion', die_on_err=False) - def autocompletion(self) -> str: - env = self.expanded_env() - env['LANG'] = 'en' - env['_IDF.PY_COMPLETE'] = 'fish_source' if self.click_ver() >= 8 else 'source_fish' - stdout = run_cmd([IDF_PY], env=env) - return stdout - - def get_prompt(self) -> str: - prompt = dedent(f''' - functions -c fish_prompt _old_fish_prompt - function fish_prompt - printf "(ESP-IDF {IDF_VERSION}) " - _old_fish_prompt - end - ''') - return prompt - - def export_file(self, fd: TextIO) -> None: - fd.write(f'{self.deactivate_cmd}\n') - for var, value in self.new_esp_idf_env.items(): - fd.write(f'export {var}="{value}"\n') - - # Add autocompletion - stdout = self.autocompletion() - if stdout is not None: - fd.write(f'{stdout}\n') - - # Adjust fish prompt - prompt = self.get_prompt() - fd.write(f'{prompt}\n') - - def init_file(self) -> None: - with open(self.script_file_path, 'w') as fd: - # Add autocompletion - stdout = self.autocompletion() - if stdout is not None: - fd.write(f'{stdout}\n') - # Adjust fish prompt - prompt = self.get_prompt() - fd.write(f'{prompt}\n') - - def spawn(self) -> None: - self.init_file() - new_env = os.environ.copy() - new_env.update(self.expanded_env()) - run([self.shell, f'--init-command=source {self.script_file_path}'], env=new_env) - - -SHELL_CLASSES = { - 'bash': BashShell, - 'zsh': ZshShell, - 'fish': FishShell, - 'sh': UnixShell, - 'ksh': UnixShell, - 'dash': UnixShell, - 'nu': UnixShell, -} - -SUPPORTED_SHELLS = ' '.join(SHELL_CLASSES.keys()) - -CONSOLE_STDERR = None -CONSOLE_STDOUT = None - - -def err(*args: Any, **kwargs: Any) -> None: - CONSOLE_STDERR.print('[red]error[/red]: ', *args, **kwargs) # type: ignore - - -def warn(*args: Any, **kwargs: Any) -> None: - CONSOLE_STDERR.print('[yellow]warning[/yellow]: ', *args, **kwargs) # type: ignore - - -def debug(*args: Any, **kwargs: Any) -> None: - if not ARGS.debug: - return - CONSOLE_STDERR.print('[green_yellow]debug[/green_yellow]: ', *args, **kwargs) # type: ignore - - -def die(*args: Any, **kwargs: Any) -> None: - err(*args, **kwargs) - sys.exit(1) - - -def eprint(*args: Any, **kwargs: Any) -> None: - CONSOLE_STDERR.print(*args, **kwargs) # type: ignore - - -def oprint(*args: Any, **kwargs: Any) -> None: - CONSOLE_STDOUT.print(*args, **kwargs) # type: ignore - - -def run_cmd(cmd: List[str], env: Optional[Dict[str, Any]]=None) -> str: - new_env = os.environ.copy() - if env is not None: - new_env.update(env) - - cmd_str = '"{}"'.format(' '.join(cmd)) - try: - p = run(cmd, env=new_env, text=True, capture_output=True) - except (OSError, SubprocessError) as e: - raise RuntimeError(f'Command {cmd_str} failed: {e}') - - stdout = p.stdout.strip() - stderr = p.stderr.strip() - if p.returncode: - raise RuntimeError(f'Command {cmd_str} failed with error code {p.returncode}\n{stdout}\n{stderr}') - - return stdout - - -parser = argparse.ArgumentParser(prog='activate', - description='Activate ESP-IDF environment') -parser.add_argument('-s', '--shell', - metavar='SHELL', - default=os.environ.get('ESP_IDF_SHELL', None), - help='Explicitly specify shell to start. For example bash, zsh, powershell.exe, cmd.exe') -parser.add_argument('-l', '--list', - action='store_true', - help=('List supported shells.')) -parser.add_argument('-e', '--export', - action='store_true', - help=('Generate commands to run in the terminal.')) -parser.add_argument('-n', '--no-color', - action='store_true', - help=('Disable ANSI color escape sequences.')) -parser.add_argument('-d', '--debug', - action='store_true', - help=('Enable debug information.')) -parser.add_argument('-q', '--quiet', - action='store_true', - help=('Suppress all output.')) - -ARGS = parser.parse_args() - -CONSOLE_STDERR = Console(stderr=True, quiet=ARGS.quiet, no_color=ARGS.no_color) -CONSOLE_STDOUT = Console(quiet=ARGS.quiet, no_color=ARGS.no_color) - -if ARGS.list: - oprint(SUPPORTED_SHELLS) - sys.exit() - -# The activate.py script sets the following environment variables -IDF_PATH = os.environ['IDF_PATH'] -IDF_VERSION = os.environ['ESP_IDF_VERSION'] -IDF_PYTHON_ENV_PATH = os.environ['IDF_PYTHON_ENV_PATH'] -IDF_TOOLS_PY = os.path.join(IDF_PATH, 'tools', 'idf_tools.py') -IDF_PY = os.path.join(IDF_PATH, 'tools', 'idf.py') - -eprint(f'[dark_orange]Activating ESP-IDF {IDF_VERSION}') -debug(f'IDF_PATH {IDF_PATH}') -debug(f'IDF_PYTHON_ENV_PATH {IDF_PYTHON_ENV_PATH}') - - -@status_message('Checking python version', rv_on_ok=True) -def check_python_version() -> str: - # Check the Python version within a virtual environment - python_version_checker = os.path.join(IDF_PATH, 'tools', 'python_version_checker.py') - run_cmd([sys.executable, python_version_checker]) - ver = sys.version_info[:3] - return f'{ver[0]}.{ver[1]}.{ver[2]}' - - -@status_message('Checking python dependencies') -def check_python_dependencies() -> None: - # Check Python dependencies within the virtual environment - run_cmd([sys.executable, IDF_TOOLS_PY, 'check-python-dependencies']) - - -check_python_version() -check_python_dependencies() -# TODO Report installed tools that are not currently used by active ESP-IDF version - -# From this point forward, we are functioning within a fully validated ESP-IDF environment. - -# TODO Verify the architectures supported by psutils. We might need to create a wheel for it or -# substitute it with ps and tasklist commands. -import psutil # noqa: E402 -import click # noqa: E402 - - -@status_message('Deactivating the current ESP-IDF environment') -def get_deactivate_cmd() -> str: - # Get previous ESP-IDF system environment variables - cmd = [sys.executable, IDF_TOOLS_PY, 'export', '--deactivate'] - stdout = run_cmd(cmd) - return stdout - - -@status_message('Establishing a new ESP-IDF environment') -def get_idf_env() -> Dict[str,str]: - # Get ESP-IDF system environment variables - extra_paths_list = [os.path.join('components', 'espcoredump'), - os.path.join('components', 'partition_table'), - os.path.join('components', 'app_update')] - extra_paths = ':'.join([os.path.join(IDF_PATH, path) for path in extra_paths_list]) - cmd = [sys.executable, IDF_TOOLS_PY, 'export', '--format', 'key-value', '--add_paths_extras', extra_paths] - stdout = run_cmd(cmd) - - # idf_tools.py might not export certain environment variables if they are already set - idf_env: Dict[str, Any] = { - 'IDF_PATH': os.environ['IDF_PATH'], - 'ESP_IDF_VERSION': os.environ['ESP_IDF_VERSION'], - 'IDF_PYTHON_ENV_PATH': os.environ['IDF_PYTHON_ENV_PATH'], - } - - for line in stdout.splitlines(): - var, val = line.split('=') - idf_env[var] = val - - if 'PATH' in idf_env: - idf_env['PATH'] = ':'.join([extra_paths, idf_env['PATH']]) - - return idf_env - - -@status_message('Identifying shell', rv_on_ok=True) -def detect_shell() -> str: - if ARGS.shell is not None: - return str(ARGS.shell) - ppid = psutil.Process(os.getpid()).ppid() - # Look for grandparent, because we started from activate.py. - pppid = psutil.Process(ppid).ppid() - return str(psutil.Process(pppid).name()) - - -deactivate_cmd = get_deactivate_cmd() -new_esp_idf_env = get_idf_env() -detected_shell = detect_shell() - -if detected_shell not in SHELL_CLASSES: - die(f'"{detected_shell}" shell is not among the supported options: "{SUPPORTED_SHELLS}"') - -shell = SHELL_CLASSES[detected_shell](detected_shell, deactivate_cmd, new_esp_idf_env) - -if ARGS.export: - shell.export() - sys.exit() - -shell.spawn() -eprint(f'[dark_orange]ESP-IDF environment exited.') diff --git a/docs/en/api-guides/tools/idf-tools.rst b/docs/en/api-guides/tools/idf-tools.rst index 5d30f70316..425f0abb09 100644 --- a/docs/en/api-guides/tools/idf-tools.rst +++ b/docs/en/api-guides/tools/idf-tools.rst @@ -186,7 +186,19 @@ Since the installed tools are not permanently added to the user or system ``PATH ``export.sh`` may be used with shells other than Bash (such as zsh). However, in this case, it is required to set the ``IDF_PATH`` environment variable before running the script. When used in Bash, the script guesses the ``IDF_PATH`` value from its own location. -In addition to calling ``idf_tools.py``, these scripts list the directories that have been added to the ``PATH``. +activate.py +~~~~~~~~~~~ + +The environment setup is handled by the underlying ``tools/activate.py`` Python script. This script performs all necessary preparations and checks, generating a temporary file that is subsequently sourced by the export script. + +``activate.py`` can also function as a standalone command. When run, it launches a new child shell with an ESP-IDF environment, which can be utilized and then exited with the ``exit`` command. Upon exiting the child shell, you will return to the parent shell from which the script was initially executed. + +Additionally, the specific behavior of the ``activate.py`` script can be modified with various options, such as spawning a specific shell with ESP-IDF using the ``--shell`` option. For more information on available options, use the ``activate.py --help`` command. + +.. note:: + + When using ``activate.py`` on Windows, it should be executed with ``python activate.py``. This ensures the script runs in the current terminal window rather than launching a new one that closes immediately. + Other Installation Methods -------------------------- diff --git a/docs/zh_CN/api-guides/tools/idf-tools.rst b/docs/zh_CN/api-guides/tools/idf-tools.rst index 7cbb700966..f614ccae00 100644 --- a/docs/zh_CN/api-guides/tools/idf-tools.rst +++ b/docs/zh_CN/api-guides/tools/idf-tools.rst @@ -186,7 +186,19 @@ ESP-IDF 的根目录中提供了针对不同 shell 的用户安装脚本，包 ``export.sh`` 可以在除了 Bash 外的其他 shell（如 zsh）中使用。但在这种情况下，必须在运行脚本前设置 ``IDF_PATH`` 环境变量。在 Bash 中使用时，脚本会从当前目录猜测 ``IDF_PATH`` 的值。 -除了调用 ``idf_tools.py``，这些脚本还会列出已经添加到 ``PATH`` 的目录。 +activate.py +~~~~~~~~~~~ + +环境设置由底层的 ``tools/activate.py`` 脚本处理。该脚本用于执行所有必要的准备和检查，并生成一个临时文件，之后供导出脚本使用。 + +``activate.py`` 也可以作为独立命令运行。执行该脚本时，会启动一个新的子 shell 并加载 ESP-IDF 环境。使用 ``exit`` 命令可以退出子 shell，并退回至最初执行该脚本的父 shell。 + +此外，``activate.py`` 脚本的具体行为可以通过各种选项进行修改，例如使用 ``--shell`` 选项可以生成特定的 ESP-IDF shell。若想了解更多有关可用选项的详细信息，请使用 ``activate.py --help`` 命令。 + +.. note:: + + 在 Windows 系统中使用 ``activate.py`` 脚本时，应执行 ``python activate.py`` 命令。这可以确保脚本在当前终端窗口中运行，而不是启动一个立即关闭的新窗口。 + 其他安装方法 -------------------------- diff --git a/export.bat b/export.bat index c37c6060d0..e9cf9df0e1 100644 --- a/export.bat +++ b/export.bat @@ -25,73 +25,25 @@ if not "%MISSING_REQUIREMENTS%" == "" goto :__error_missing_requirements set IDF_PATH=%~dp0 set IDF_PATH=%IDF_PATH:~0,-1% -echo Checking Python compatibility -python.exe "%IDF_PATH%\tools\python_version_checker.py" - -set "IDF_TOOLS_PY_PATH=%IDF_PATH%\tools\idf_tools.py" -set "IDF_TOOLS_JSON_PATH=%IDF_PATH%\tools\tools.json" -set "IDF_TOOLS_EXPORT_CMD=%IDF_PATH%\export.bat" -set "IDF_TOOLS_INSTALL_CMD=%IDF_PATH%\install.bat" -echo Setting IDF_PATH: %IDF_PATH% -echo. - -set "OLD_PATH=%PATH%" -echo Adding ESP-IDF tools to PATH... -:: Export tool paths and environment variables. -:: It is possible to do this without a temporary file (running idf_tools.py from for /r command), -:: but that way it is impossible to get the exit code of idf_tools.py. -set "IDF_TOOLS_EXPORTS_FILE=%TEMP%\idf_export_vars.tmp" -python.exe "%IDF_PATH%\tools\idf_tools.py" export --format key-value >"%IDF_TOOLS_EXPORTS_FILE%" -if %errorlevel% neq 0 ( - set SCRIPT_EXIT_CODE=%errorlevel% - goto :__end +if not exist "%IDF_PATH%\tools\idf.py" ( + set SCRIPT_EXIT_CODE=1 + goto :__missing_file +) +if not exist "%IDF_PATH%\tools\idf_tools.py" ( + set SCRIPT_EXIT_CODE=1 + goto :__missing_file +) +if not exist "%IDF_PATH%\tools\activate.py" ( + set SCRIPT_EXIT_CODE=1 + goto :__missing_file ) -for /f "usebackq tokens=1,2 eol=# delims==" %%a in ("%IDF_TOOLS_EXPORTS_FILE%") do ( - call set "%%a=%%b" - ) -:: This removes OLD_PATH substring from PATH, leaving only the paths which have been added, -:: and prints semicolon-delimited components of the path on separate lines -call set PATH_ADDITIONS=%%PATH:%OLD_PATH%=%% -if "%PATH_ADDITIONS%"=="" call :__print_nothing_added -if not "%PATH_ADDITIONS%"=="" echo %PATH_ADDITIONS:;=&echo. % - -DOSKEY idf.py=python.exe "%IDF_PATH%\tools\idf.py" $* -DOSKEY esptool.py=python.exe "%IDF_PATH%\components\esptool_py\esptool\esptool.py" $* -DOSKEY espefuse.py=python.exe "%IDF_PATH%\components\esptool_py\esptool\espefuse.py" $* -DOSKEY espsecure.py=python.exe "%IDF_PATH%\components\esptool_py\esptool\espsecure.py" $* -DOSKEY otatool.py=python.exe "%IDF_PATH%\components\app_update\otatool.py" $* -DOSKEY parttool.py=python.exe "%IDF_PATH%\components\partition_table\parttool.py" $* - -echo Checking if Python packages are up to date... -python.exe "%IDF_PATH%\tools\idf_tools.py" check-python-dependencies -if %errorlevel% neq 0 ( - set SCRIPT_EXIT_CODE=%errorlevel% - goto :__end -) - -python.exe "%IDF_PATH%\tools\idf_tools.py" uninstall --dry-run > UNINSTALL_OUTPUT -SET /p UNINSTALL=nul 2>nul -) -set IDF_TOOLS_EXPORTS_FILE= -set IDF_TOOLS_EXPORT_CMD= -set IDF_TOOLS_INSTALL_CMD= -set IDF_TOOLS_PY_PATH= -set IDF_TOOLS_JSON_PATH= -set OLD_PATH= -set PATH_ADDITIONS= set MISSING_REQUIREMENTS= -set UNINSTALL= +set activate= exit /b %SCRIPT_EXIT_CODE% diff --git a/export.fish b/export.fish index cdff5798e0..283fa66471 100644 --- a/export.fish +++ b/export.fish @@ -5,5 +5,18 @@ function unset set --erase $argv end -set script_dir (dirname (realpath (status -f))) -eval ("$script_dir"/activate.py --export) +set idf_path (dirname (realpath (status -f))) + +if not test -f "$idf_path/tools/idf.py" + or not test -f "$idf_path/tools/idf_tools.py" + or not test -f "$idf_path/tools/activate.py" + echo "Could not detect IDF_PATH. Please set it before sourcing this script:" + echo " export IDF_PATH=(add path here)" + set -e idf_path + exit 1 +end + +source "$idf_path"/tools/detect_python.fish + +eval ("$idf_path"/tools/activate.py --export) +set -e idf_path diff --git a/export.ps1 b/export.ps1 index d519b2296d..76df9bd81b 100644 --- a/export.ps1 +++ b/export.ps1 @@ -1,92 +1,20 @@ #!/usr/bin/env pwsh -$S = [IO.Path]::PathSeparator # path separator. WIN:';', UNIX:":" -$IDF_PATH = "$PSScriptRoot" +$idf_path = "$PSScriptRoot" -Write-Output "Setting IDF_PATH: $IDF_PATH" -$env:IDF_PATH = "$IDF_PATH" +if (-not (Test-Path "$idf_path/tools/idf.py") -or + -not (Test-Path "$idf_path/tools/idf_tools.py") -or + -not (Test-Path "$idf_path/tools/activate.py")) { -Write-Output "Checking Python compatibility" -python "$IDF_PATH/tools/python_version_checker.py" + Write-Output "Could not detect IDF_PATH. Please set it before running this script:" + Write-Output ' $env:IDF_PATH=(add path here)' -Write-Output "Adding ESP-IDF tools to PATH..." -$OLD_PATH = $env:PATH.split($S) | Select-Object -Unique # array without duplicates -# using idf_tools.py to get $envars_array to set -$envars_raw = python "$IDF_PATH/tools/idf_tools.py" export --format key-value -if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } # if error + $env:IDF_PATH = "" -$envars_array = @() # will be filled like: -# [ -# [vname1, vval1], [vname2, vval2], ... -# ] -foreach ($line in $envars_raw) { - $pair = $line.split("=") # split in name, val - $var_name = $pair[0].Trim() # trim spaces on the ends of the name - $var_val = $pair[1].Trim() # trim spaces on the ends of the val - $envars_array += (, ($var_name, $var_val)) + exit 1 } -if ($null -eq $IsWindows) { - # $IsWindows was added in PowerShell Core 6 and PowerShell 7 together with multi-platform support. # I.E. if this - # internal variable is not set then PowerShell 5 is used and # the platform cannot be # anything else than Windows. - $Windows = $true -} - -foreach ($pair in $envars_array) { - # setting the values - $var_name = $pair[0].Trim() # trim spaces on the ends of the name - $var_val = $pair[1].Trim() # trim spaces on the ends of the val - if ($var_name -eq "PATH") { - # trim "%PATH%" or "`$PATH" - if ($IsWindows -or $Windows) { - $var_val = $var_val.Trim($S + "%PATH%") - } else { - $var_val = $var_val.Trim($S + "`$PATH") - } - # apply - $env:PATH = $var_val + $S + $env:PATH - } else { - New-Item -Path "env:$var_name" -Value "$var_val" -Force - } -} - -# Allow calling some IDF python tools without specifying the full path -function idf.py { &python "$IDF_PATH\tools\idf.py" $args } -function espefuse.py { &python "$IDF_PATH\components\esptool_py\esptool\espefuse.py" $args } -function espsecure.py { &python "$IDF_PATH\components\esptool_py\esptool\espsecure.py" $args } -function otatool.py { &python "$IDF_PATH\components\app_update\otatool.py" $args } -function parttool.py { &python "$IDF_PATH\components\partition_table\parttool.py" $args } - -#Compare Path's OLD vs. NEW -$NEW_PATH = $env:PATH.split($S) | Select-Object -Unique # array without duplicates -$dif_Path = Compare-Object -ReferenceObject $OLD_PATH -DifferenceObject $NEW_PATH -PassThru -if ($null -ne $dif_Path) { - Write-Output "`nAdded to PATH`n-------------" - Write-Output $dif_Path -} else { - Write-Output "No directories added to PATH:" - Write-Output $OLD_PATH -} - - -Write-Output "Checking if Python packages are up to date..." - -Start-Process -Wait -NoNewWindow -FilePath "python" -Args "`"$IDF_PATH/tools/idf_tools.py`" check-python-dependencies" -if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } # if error - -$uninstall = python "$IDF_PATH/tools/idf_tools.py" uninstall --dry-run - -if (![string]::IsNullOrEmpty($uninstall)){ - Write-Output "" - Write-Output "Detected installed tools that are not currently used by active ESP-IDF version." - Write-Output "$uninstall" - Write-Output "For free up even more space, remove installation packages of those tools. Use option 'python.exe $IDF_PATH\tools\idf_tools.py uninstall --remove-archives'." - Write-Output "" -} - -Write-Output " -Done! You can now compile ESP-IDF projects. -Go to the project directory and run: - idf.py build - -" +$idf_exports = python "$idf_path/tools/activate.py" --export +# The dot sourcing is added here in PowerShell since +# Win PSAnalyzer complains about using `Invoke-Expression` command +. $idf_exports diff --git a/export.sh b/export.sh index 02646293b5..f4eb6334e3 100644 --- a/export.sh +++ b/export.sh @@ -28,20 +28,18 @@ fi if [ ! -f "${idf_path}/tools/idf.py" ] || [ ! -f "${idf_path}/tools/idf_tools.py" ] || - [ ! -f "${idf_path}/activate.py" ] + [ ! -f "${idf_path}/tools/activate.py" ] then - # Echo command here is not used for printing to the terminal, but as non-empty return value from function. echo "Could not detect IDF_PATH. Please set it before sourcing this script:" echo " export IDF_PATH=(add path here)" unset idf_path return 1 fi -# TODO Maybe we can use "command -v" to check just for python and python3 . "${idf_path}/tools/detect_python.sh" # Evaluate the ESP-IDF environment set up by the activate.py script. -idf_exports=$("$ESP_PYTHON" "${idf_path}/activate.py" --export) +idf_exports=$("$ESP_PYTHON" "${idf_path}/tools/activate.py" --export) eval "${idf_exports}" unset idf_path return 0 diff --git a/activate.py b/tools/activate.py similarity index 85% rename from activate.py rename to tools/activate.py index 40e1e78156..8c0c021a5d 100755 --- a/activate.py +++ b/tools/activate.py @@ -17,8 +17,8 @@ def die(msg: str) -> None: sys.exit(f'error: {msg}') -idf_path = os.path.realpath(os.path.dirname(__file__)) -idf_tools_path = os.path.join(idf_path, 'tools') +idf_tools_path = os.path.realpath(os.path.dirname(__file__)) +idf_path = os.path.dirname(idf_tools_path) sys.path.insert(0, idf_tools_path) try: @@ -38,6 +38,6 @@ os.environ['IDF_PYTHON_ENV_PATH'] = idf_python_env_path os.environ['ESP_IDF_VERSION'] = idf_version try: - run([virtualenv_python, os.path.join(idf_path, 'activate_venv.py')] + sys.argv[1:], check=True) + run([virtualenv_python, os.path.join(idf_path, 'tools', 'export_utils', 'activate_venv.py')] + sys.argv[1:], check=True) except (OSError, SubprocessError): die(f'Activation script failed') diff --git a/tools/ci/executable-list.txt b/tools/ci/executable-list.txt index d235422c16..6524d2b249 100644 --- a/tools/ci/executable-list.txt +++ b/tools/ci/executable-list.txt @@ -1,4 +1,3 @@ -activate.py components/app_update/otatool.py components/efuse/efuse_table_gen.py components/efuse/test_efuse_host/efuse_tests.py @@ -48,6 +47,7 @@ examples/system/ota/otatool/otatool_example.py examples/system/ota/otatool/otatool_example.sh install.fish install.sh +tools/activate.py tools/check_python_dependencies.py tools/ci/build_template_app.sh tools/ci/check_api_violation.sh diff --git a/tools/export_utils/activate_venv.py b/tools/export_utils/activate_venv.py new file mode 100644 index 0000000000..6014cb5fe3 --- /dev/null +++ b/tools/export_utils/activate_venv.py @@ -0,0 +1,177 @@ +# SPDX-FileCopyrightText: 2023-2024 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 +import argparse +import os +import sys +from typing import Any +from typing import Dict + +from console_output import CONSOLE_STDERR +from console_output import CONSOLE_STDOUT +from console_output import debug +from console_output import die +from console_output import eprint +from console_output import oprint +from console_output import status_message +from shell_types import SHELL_CLASSES +from shell_types import SUPPORTED_SHELLS +from utils import conf +from utils import run_cmd + + +def parse_arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser(prog='activate', + description='Activate ESP-IDF environment', + epilog='On Windows, run `python activate.py` to execute this script in the current terminal window.') + parser.add_argument('-s', '--shell', + metavar='SHELL', + default=os.environ.get('ESP_IDF_SHELL', None), + help='Explicitly specify shell to start. For example bash, zsh, powershell.exe, cmd.exe') + parser.add_argument('-l', '--list', + action='store_true', + help=('List supported shells.')) + parser.add_argument('-e', '--export', + action='store_true', + help=('Generate commands to run in the terminal.')) + parser.add_argument('-n', '--no-color', + action='store_true', + help=('Disable ANSI color escape sequences.')) + parser.add_argument('-d', '--debug', + action='store_true', + help=('Enable debug information.')) + parser.add_argument('-q', '--quiet', + action='store_true', + help=('Suppress all output.')) + + return parser.parse_args() + + +@status_message('Checking python version', rv_on_ok=True) +def check_python_version() -> str: + # Check the Python version within a virtual environment + python_version_checker = os.path.join(conf.IDF_PATH, 'tools', 'python_version_checker.py') + run_cmd([sys.executable, python_version_checker]) + ver = sys.version_info + return f'{ver[0]}.{ver[1]}.{ver[2]}' + + +@status_message('Checking python dependencies') +def check_python_dependencies() -> None: + # Check Python dependencies within the virtual environment + run_cmd([sys.executable, conf.IDF_TOOLS_PY, 'check-python-dependencies']) + + +@status_message('Deactivating the current ESP-IDF environment (if any)') +def get_deactivate_cmd() -> str: + # Get previous ESP-IDF system environment variables + cmd = [sys.executable, conf.IDF_TOOLS_PY, 'export', '--deactivate'] + stdout: str = run_cmd(cmd) + return stdout + + +@status_message('Establishing a new ESP-IDF environment') +def get_idf_env() -> Dict[str,str]: + # Get ESP-IDF system environment variables + extra_paths_list = [os.path.join('components', 'espcoredump'), + os.path.join('components', 'partition_table'), + os.path.join('components', 'app_update')] + extra_paths = os.pathsep.join([os.path.join(conf.IDF_PATH, path) for path in extra_paths_list]) + cmd = [sys.executable, conf.IDF_TOOLS_PY, 'export', '--format', 'key-value', '--add_paths_extras', extra_paths] + stdout = run_cmd(cmd) + + # idf_tools.py might not export certain environment variables if they are already set + idf_env: Dict[str, Any] = { + 'IDF_PATH': os.environ['IDF_PATH'], + 'ESP_IDF_VERSION': os.environ['ESP_IDF_VERSION'], + 'IDF_PYTHON_ENV_PATH': os.environ['IDF_PYTHON_ENV_PATH'], + } + + for line in stdout.splitlines(): + var, val = line.split('=') + idf_env[var] = val + + if 'PATH' in idf_env: + idf_env['PATH'] = os.pathsep.join([extra_paths, idf_env['PATH']]) + + return idf_env + + +@status_message('Identifying shell', rv_on_ok=True) +def detect_shell(args: Any) -> str: + import psutil + + if args.shell is not None: + return str(args.shell) + + current_pid = os.getpid() + detected_shell_name = '' + while True: + parent_pid = psutil.Process(current_pid).ppid() + parent_name = psutil.Process(parent_pid).name() + if not parent_name.startswith('python'): + detected_shell_name = parent_name + conf.DETECTED_SHELL_PATH = psutil.Process(parent_pid).exe() + break + current_pid = parent_pid + + return detected_shell_name + + +@status_message('Detecting outdated tools in system', rv_on_ok=True) +def print_uninstall_msg() -> Any: + stdout = run_cmd([sys.executable, conf.IDF_TOOLS_PY, 'uninstall', '--dry-run']) + if stdout: + python_cmd = 'python.exe' if sys.platform == 'win32' else 'python' + msg = (f'Found tools that are not used by active ESP-IDF version.\n' + f'[bright_cyan]{stdout}\n' + f'To free up even more space, remove installation packages of those tools.\n' + f'Use option {python_cmd} {conf.IDF_TOOLS_PY} uninstall --remove-archives.') + else: + msg = 'OK - no outdated tools found' + + return msg + + +def main() -> None: + args = parse_arguments() + + # Setup parsed arguments + CONSOLE_STDERR.no_color = args.no_color + CONSOLE_STDOUT.no_color = args.no_color + CONSOLE_STDERR.quiet = args.quiet + CONSOLE_STDOUT.quiet = args.quiet + # Fill config global holder + conf.ARGS = args + + if conf.ARGS.list: + oprint(SUPPORTED_SHELLS) + sys.exit() + + eprint(f'[dark_orange]Activating ESP-IDF {conf.IDF_VERSION}') + debug(f'IDF_PATH {conf.IDF_PATH}') + debug(f'IDF_PYTHON_ENV_PATH {conf.IDF_PYTHON_ENV_PATH}') + + check_python_version() + check_python_dependencies() + + deactivate_cmd = get_deactivate_cmd() + new_esp_idf_env = get_idf_env() + detected_shell = detect_shell(conf.ARGS) + print_uninstall_msg() + + if detected_shell not in SHELL_CLASSES: + die(f'"{detected_shell}" shell is not among the supported options: "{SUPPORTED_SHELLS}"') + + shell = SHELL_CLASSES[detected_shell](detected_shell, deactivate_cmd, new_esp_idf_env) + + if conf.ARGS.export: + shell.export() + sys.exit() + + eprint(f'[dark_orange]Starting new \'{shell.shell}\' shell with ESP-IDF environment... (use "exit" command to quit)') + shell.spawn() + eprint(f'[dark_orange]ESP-IDF environment exited.') + + +if __name__ == '__main__': + main() diff --git a/tools/export_utils/console_output.py b/tools/export_utils/console_output.py new file mode 100644 index 0000000000..03d8cb8453 --- /dev/null +++ b/tools/export_utils/console_output.py @@ -0,0 +1,69 @@ +# SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 +import sys +from typing import Any +from typing import Callable + +from utils import conf + +try: + # The ESP-IDF virtual environment hasn't been verified yet, so see if the rich library + # can be imported to display error and status messages nicely. + from rich.console import Console +except ImportError as e: + sys.exit(f'error: Unable to import the rich module: {e}. Please execute the install script.') + +CONSOLE_STDERR = Console(stderr=True, width=255) +CONSOLE_STDOUT = Console(width=255) + + +def status_message(msg: str, rv_on_ok: bool=False, die_on_err: bool=True) -> Callable: + def inner(func: Callable) -> Callable: + def wrapper(*args: Any, **kwargs: Any) -> Any: + eprint(f'[dark_orange]*[/dark_orange] {msg} ... ', end='') + + try: + rv = func(*args, **kwargs) + except Exception as e: + eprint('[red]FAILED[/red]') + if conf.ARGS.debug: + raise + if not die_on_err: + return None + die(str(e)) + + if rv_on_ok: + eprint(f'[green]{rv}[/green]') + else: + eprint('[green]OK[/green]') + + return rv + return wrapper + return inner + + +def err(*args: Any, **kwargs: Any) -> None: + CONSOLE_STDERR.print('[red]error[/red]: ', *args, **kwargs) # type: ignore + + +def warn(*args: Any, **kwargs: Any) -> None: + CONSOLE_STDERR.print('[yellow]warning[/yellow]: ', *args, **kwargs) # type: ignore + + +def debug(*args: Any, **kwargs: Any) -> None: + if not conf.ARGS.debug: + return + CONSOLE_STDERR.print('[green_yellow]debug[/green_yellow]: ', *args, **kwargs) # type: ignore + + +def die(*args: Any, **kwargs: Any) -> None: + err(*args, **kwargs) + sys.exit(1) + + +def eprint(*args: Any, **kwargs: Any) -> None: + CONSOLE_STDERR.print(*args, **kwargs) # type: ignore + + +def oprint(*args: Any, **kwargs: Any) -> None: + CONSOLE_STDOUT.print(*args, **kwargs) # type: ignore diff --git a/tools/export_utils/shell_types.py b/tools/export_utils/shell_types.py new file mode 100644 index 0000000000..c1f0c4e690 --- /dev/null +++ b/tools/export_utils/shell_types.py @@ -0,0 +1,316 @@ +# SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 +import os +import re +import shutil +import sys +from pathlib import Path +from subprocess import run +from tempfile import gettempdir +from tempfile import NamedTemporaryFile +from tempfile import TemporaryDirectory +from typing import Dict +from typing import List +from typing import TextIO +from typing import Union + +import click +from console_output import debug +from console_output import status_message +from utils import conf +from utils import run_cmd + + +class Shell(): + def __init__(self, shell: str, deactivate_cmd: str, new_esp_idf_env: Dict[str,str]): + self.shell = shell + self.deactivate_cmd = deactivate_cmd + self.new_esp_idf_env = new_esp_idf_env + self.tmp_dir_path = Path(gettempdir()) / 'esp_idf_activate' + if not conf.ARGS.debug and os.path.exists(self.tmp_dir_path): + # Do not cleanup temporary directory when debugging + shutil.rmtree(self.tmp_dir_path) + self.tmp_dir_path.mkdir(parents=True, exist_ok=True) + + def export(self) -> None: + raise NotImplementedError('Subclass must implement abstract method "export"') + + def expanded_env(self) -> Dict[str, str]: + expanded_env = self.new_esp_idf_env.copy() + + if 'PATH' not in expanded_env: + return expanded_env + + # The PATH returned by idf_tools.py export is not expanded. + # Note that for the export script, the PATH should remain unexpanded + # to ensure proper deactivation. In the export script, + # the expansion should occur after deactivation, when the PATH is adjusted. + # But it has to be expanded for processes started with the new PATH. + expanded_env['PATH'] = os.path.expandvars(expanded_env['PATH']) + return expanded_env + + def spawn(self) -> None: + # This method should likely work for all shells because we are delegating the initialization + # purely to Python os.environ. + new_env = os.environ.copy() + new_env.update(self.expanded_env()) + run([self.shell], env=new_env) + + +class UnixShell(Shell): + def __init__(self, shell: str, deactivate_cmd: str, new_esp_idf_env: Dict[str,str]): + super().__init__(shell, deactivate_cmd, new_esp_idf_env) + + with NamedTemporaryFile(dir=self.tmp_dir_path, delete=False, prefix='activate_') as fd: + self.script_file_path = Path(fd.name) + debug(f'Temporary script file path: {self.script_file_path}') + + self.new_esp_idf_env['IDF_TOOLS_INSTALL_CMD'] = os.path.join(conf.IDF_PATH, 'install.sh') + self.new_esp_idf_env['IDF_TOOLS_EXPORT_CMD'] = os.path.join(conf.IDF_PATH, 'export.sh') + + def autocompletion(self) -> None: + # Basic POSIX shells does not support autocompletion + return None + + def init_file(self) -> None: + with open(self.script_file_path, 'w') as fd: + self.export_file(fd) + + def export_file(self, fd: TextIO) -> None: + fd.write(f'{self.deactivate_cmd}\n') + for var, value in self.new_esp_idf_env.items(): + fd.write(f'export {var}="{value}"\n') + stdout = self.autocompletion() # type: ignore + if stdout is not None: + fd.write(f'{stdout}\n') + fd.write((f'echo "\nDone! You can now compile ESP-IDF projects.\n' + 'Go to the project directory and run:\n\n idf.py build"\n')) + + def export(self) -> None: + self.init_file() + print(f'. {self.script_file_path}') + + def click_ver(self) -> int: + return int(click.__version__.split('.')[0]) + + +class BashShell(UnixShell): + def get_bash_major_minor(self) -> float: + env = self.expanded_env() + bash_interpreter = conf.DETECTED_SHELL_PATH if conf.DETECTED_SHELL_PATH else 'bash' + stdout = run_cmd([bash_interpreter, '-c', 'echo ${BASH_VERSINFO[0]}.${BASH_VERSINFO[1]}'], env=env) + bash_maj_min = float(stdout) + return bash_maj_min + + @status_message('Shell completion', die_on_err=False) + def autocompletion(self) -> str: + bash_maj_min = self.get_bash_major_minor() + # Click supports bash version >= 4.4 + # https://click.palletsprojects.com/en/8.1.x/changes/#version-8-0-0 + if bash_maj_min < 4.4: + raise RuntimeError('Autocompletion not supported') + + env = self.expanded_env() + env['LANG'] = 'en' + env['_IDF.PY_COMPLETE'] = 'bash_source' if self.click_ver() >= 8 else 'source_bash' + stdout: str = run_cmd([sys.executable, conf.IDF_PY], env=env) + return stdout + + def init_file(self) -> None: + with open(self.script_file_path, 'w') as fd: + # We will use the --init-file option to pass a custom rc file, which will ignore .bashrc, + # so we need to source .bashrc first. + bashrc_path = os.path.expanduser('~/.bashrc') + if os.path.isfile(bashrc_path): + fd.write(f'source {bashrc_path}\n') + self.export_file(fd) + + def spawn(self) -> None: + self.init_file() + new_env = os.environ.copy() + new_env.update(self.expanded_env()) + run([self.shell, '--init-file', str(self.script_file_path)], env=new_env) + + +class ZshShell(UnixShell): + @status_message('Shell completion', die_on_err=False) + def autocompletion(self) -> str: + env = self.expanded_env() + env['LANG'] = 'en' + env['_IDF.PY_COMPLETE'] = 'zsh_source' if self.click_ver() >= 8 else 'source_zsh' + stdout = run_cmd([sys.executable, conf.IDF_PY], env=env) + return f'autoload -Uz compinit && compinit -u\n{stdout}' + + def init_file(self) -> None: + # If ZDOTDIR is unset, HOME is used instead. + # https://zsh.sourceforge.io/Doc/Release/Files.html#Startup_002fShutdown-Files + zdotdir = os.environ.get('ZDOTDIR', str(Path.home())) + with open(self.script_file_path, 'w') as fd: + # We will use the ZDOTDIR env variable to load our custom script in the newly spawned shell + # so we need to source .zshrc first. + zshrc_path = Path(zdotdir) / '.zshrc' + if zshrc_path.is_file(): + fd.write(f'source {zshrc_path}\n') + + self.export_file(fd) + + def spawn(self) -> None: + self.init_file() + + # Create a temporary directory to use as ZDOTDIR + tmpdir = TemporaryDirectory() + tmpdir_path = Path(tmpdir.name) + debug(f'Temporary ZDOTDIR {tmpdir_path} with .zshrc file') + + # Copy init script to the custom ZDOTDIR + zshrc_path = tmpdir_path / '.zshrc' + shutil.copy(str(self.script_file_path), str(zshrc_path)) + + new_env = os.environ.copy() + new_env.update(self.expanded_env()) + # Set new ZDOTDIR in the new environment + new_env['ZDOTDIR'] = str(tmpdir_path) + + run([self.shell], env=new_env) + + +class FishShell(UnixShell): + def __init__(self, shell: str, deactivate_cmd: str, new_esp_idf_env: Dict[str,str]): + super().__init__(shell, deactivate_cmd, new_esp_idf_env) + self.new_esp_idf_env['IDF_TOOLS_INSTALL_CMD'] = os.path.join(conf.IDF_PATH, 'install.fish') + self.new_esp_idf_env['IDF_TOOLS_EXPORT_CMD'] = os.path.join(conf.IDF_PATH, 'export.fish') + + @status_message('Shell completion', die_on_err=False) + def autocompletion(self) -> str: + env = self.expanded_env() + env['LANG'] = 'en' + env['_IDF.PY_COMPLETE'] = 'fish_source' if self.click_ver() >= 8 else 'source_fish' + stdout: str = run_cmd([sys.executable, conf.IDF_PY], env=env) + return stdout + + def spawn(self) -> None: + self.init_file() + new_env = os.environ.copy() + new_env.update(self.expanded_env()) + run([self.shell, f'--init-command=source {self.script_file_path}'], env=new_env) + + +class PowerShell(Shell): + def __init__(self, shell: str, deactivate_cmd: str, new_esp_idf_env: Dict[str,str]): + super().__init__(shell, deactivate_cmd, new_esp_idf_env) + + with NamedTemporaryFile(dir=self.tmp_dir_path, delete=False, prefix='activate_', suffix='.ps1') as fd: + self.script_file_path = Path(fd.name) + debug(f'Temporary script file path: {self.script_file_path}') + + self.new_esp_idf_env['IDF_TOOLS_INSTALL_CMD'] = os.path.join(conf.IDF_PATH, 'install.ps1') + self.new_esp_idf_env['IDF_TOOLS_EXPORT_CMD'] = os.path.join(conf.IDF_PATH, 'export.ps1') + + def get_functions(self) -> str: + return '\n'.join([ + r'function idf.py { &python "$Env:IDF_PATH\tools\idf.py" $args }', + r'function global:esptool.py { &python -m esptool $args }', + r'function global:espefuse.py { &python -m espefuse $args }', + r'function global:espsecure.py { &python -m espsecure $args }', + r'function global:otatool.py { &python "$Env:IDF_PATH\components\app_update\otatool.py" $args }', + r'function global:parttool.py { &python "$Env:IDF_PATH\components\partition_table\parttool.py" $args }', + ]) + + def export(self) -> None: + self.init_file() + # Powershell is the only Shell class that does not return the script name in dot sourcing style + # since PSAnalyzer complains about using `InvokeExpression` command + print(f'{self.script_file_path}') + + def init_file(self) -> None: + with open(self.script_file_path, 'w') as fd: + # fd.write(f'{self.deactivate_cmd}\n') TODO in upcoming task IDF-10292 + for var, value in self.new_esp_idf_env.items(): + if var == 'PATH': + value = re.sub(r'(%PATH%|\$PATH)', r'$Env:PATH', value) + fd.write(f'$Env:{var}="{value}"\n') + functions = self.get_functions() + fd.write(f'{functions}\n') + fd.write((f'echo "\nDone! You can now compile ESP-IDF projects.\n' + 'Go to the project directory and run:\n\n idf.py build\n"')) + + def spawn(self) -> None: + self.init_file() + new_env = os.environ.copy() + new_env.update(self.expanded_env()) + arguments = ['-NoExit', '-Command', f'{self.script_file_path}'] + cmd: Union[str, List[str]] = [self.shell] + arguments + run(cmd, env=new_env) + + +class WinCmd(Shell): + def __init__(self, shell: str, deactivate_cmd: str, new_esp_idf_env: Dict[str,str]): + super().__init__(shell, deactivate_cmd, new_esp_idf_env) + + with NamedTemporaryFile(dir=self.tmp_dir_path, delete=False, prefix='activate_', suffix='.bat') as fd: + self.script_file_path = Path(fd.name) + debug(f'Temporary script file path: {self.script_file_path}') + + self.new_esp_idf_env['IDF_TOOLS_INSTALL_CMD'] = os.path.join(conf.IDF_PATH, 'install.bat') + self.new_esp_idf_env['IDF_TOOLS_EXPORT_CMD'] = os.path.join(conf.IDF_PATH, 'export.bat') + self.new_esp_idf_env['IDF_TOOLS_JSON_PATH'] = os.path.join(conf.IDF_PATH, 'tools', 'tools.json') + self.new_esp_idf_env['IDF_TOOLS_PY_PATH'] = conf.IDF_TOOLS_PY + + def get_functions(self) -> str: + return '\n'.join([ + r'DOSKEY idf.py=python.exe "%IDF_PATH%\tools\idf.py" $*', + r'DOSKEY esptool.py=python.exe -m esptool $*', + r'DOSKEY espefuse.py=python.exe -m espefuse $*', + r'DOSKEY espsecure.py=python.exe -m espsecure $*', + r'DOSKEY otatool.py=python.exe "%IDF_PATH%\components\app_update\otatool.py" $*', + r'DOSKEY parttool.py=python.exe "%IDF_PATH%\components\partition_table\parttool.py" $*', + ]) + + def export(self) -> None: + self.init_file() + print(f'call {self.script_file_path}') + + def init_file(self) -> None: + with open(self.script_file_path, 'w') as fd: + fd.write('@echo off\n') + # fd.write(f'{self.deactivate_cmd}\n') TODO in upcoming task IDF-10292 + for var, value in self.new_esp_idf_env.items(): + fd.write(f'set {var}={value}\n') + functions = self.get_functions() + fd.write(f'{functions}\n') + fd.write('\n'.join([ + 'echo.', + 'echo Done! You can now compile ESP-IDF projects.', + 'echo Go to the project directory and run:', + 'echo.', + 'echo idf.py build', + 'echo.', + ])) + + def spawn(self) -> None: + self.init_file() + new_env = os.environ.copy() + new_env.update(self.expanded_env()) + arguments = ['/k', f'{self.script_file_path}'] + cmd: Union[str, List[str]] = [self.shell] + arguments + cmd = ' '.join(cmd) + run(cmd, env=new_env) + + +SHELL_CLASSES = { + 'bash': BashShell, + 'zsh': ZshShell, + 'fish': FishShell, + 'sh': UnixShell, + 'ksh': UnixShell, + 'dash': UnixShell, + 'nu': UnixShell, + 'pwsh.exe': PowerShell, + 'pwsh': PowerShell, + 'powershell.exe': PowerShell, + 'powershell': PowerShell, + 'cmd.exe': WinCmd, + 'cmd': WinCmd +} + +SUPPORTED_SHELLS = ' '.join(SHELL_CLASSES.keys()) diff --git a/tools/export_utils/utils.py b/tools/export_utils/utils.py new file mode 100644 index 0000000000..53ba07a852 --- /dev/null +++ b/tools/export_utils/utils.py @@ -0,0 +1,48 @@ +# SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 +import argparse +import os +from subprocess import run +from subprocess import SubprocessError +from typing import Any +from typing import Dict +from typing import List +from typing import Optional + + +class Config: + """ + Config serves as global hodler for variables used across modules + It holds also arguments from command line + """ + def __init__(self) -> None: + self.IDF_PATH = os.environ['IDF_PATH'] + self.IDF_VERSION = os.environ['ESP_IDF_VERSION'] + self.IDF_PYTHON_ENV_PATH = os.environ['IDF_PYTHON_ENV_PATH'] + self.IDF_TOOLS_PY = os.path.join(self.IDF_PATH, 'tools', 'idf_tools.py') + self.IDF_PY = os.path.join(self.IDF_PATH, 'tools', 'idf.py') + self.ARGS: Optional[argparse.Namespace] = None + self.DETECTED_SHELL_PATH: str = '' + + +# Global variable instance +conf = Config() + + +def run_cmd(cmd: List[str], env: Optional[Dict[str, Any]]=None) -> str: + new_env = os.environ.copy() + if env is not None: + new_env.update(env) + + cmd_str = '"{}"'.format(' '.join(cmd)) + try: + p = run(cmd, env=new_env, text=True, capture_output=True) + except (OSError, SubprocessError) as e: + raise RuntimeError(f'Command {cmd_str} failed: {e}') + + stdout: str = p.stdout.strip() + stderr: str = p.stderr.strip() + if p.returncode: + raise RuntimeError(f'Command {cmd_str} failed with error code {p.returncode}\n{stdout}\n{stderr}') + + return stdout From 7b417fc3f2df0f953f3449db67fc1cf803ee1d7c Mon Sep 17 00:00:00 2001 From: Marek Fiala Date: Wed, 21 Aug 2024 16:34:19 +0200 Subject: [PATCH 3/3] feat(tools): Add backup option to use legacy export script This feature serves as an immediate fix in case the new export approach causes issues. The user can simply set the environment variable `ESP_IDF_LEGACY_EXPORT`, and the old export script will be used. --- export.bat | 8 + export.fish | 7 + export.ps1 | 7 + export.sh | 7 + tools/ci/exclude_check_tools_files.txt | 4 + tools/legacy_exports/export_legacy.bat | 129 +++++++++++++ tools/legacy_exports/export_legacy.fish | 111 +++++++++++ tools/legacy_exports/export_legacy.ps1 | 94 ++++++++++ tools/legacy_exports/export_legacy.sh | 235 ++++++++++++++++++++++++ 9 files changed, 602 insertions(+) create mode 100644 tools/legacy_exports/export_legacy.bat create mode 100644 tools/legacy_exports/export_legacy.fish create mode 100644 tools/legacy_exports/export_legacy.ps1 create mode 100644 tools/legacy_exports/export_legacy.sh diff --git a/export.bat b/export.bat index e9cf9df0e1..f15ce3f977 100644 --- a/export.bat +++ b/export.bat @@ -6,6 +6,14 @@ if defined MSYSTEM ( set SCRIPT_EXIT_CODE=0 +:: Emergency backup option to use previous export.bat (export_legacy.bat) if the new export approach fails. +:: To use it, set environmental variable like: set ESP_IDF_LEGACY_EXPORT=1 +if not "%ESP_IDF_LEGACY_EXPORT%"=="" ( + tools\legacy_exports\export_legacy.bat + set SCRIPT_EXIT_CODE=%errorlevel% + goto :eof +) + :: Missing requirements check set MISSING_REQUIREMENTS= python.exe --version >NUL 2>NUL diff --git a/export.fish b/export.fish index 283fa66471..9c8dccdb8b 100644 --- a/export.fish +++ b/export.fish @@ -5,6 +5,13 @@ function unset set --erase $argv end +# Emergency backup option to use previous export.fish (export_legacy.fish) if the new export approach fails. +# To use it, set environmental variable like: export ESP_IDF_LEGACY_EXPORT=1 +if test -n "$ESP_IDF_LEGACY_EXPORT" + source tools/legacy_exports/export_legacy.fish + exit $status +end + set idf_path (dirname (realpath (status -f))) if not test -f "$idf_path/tools/idf.py" diff --git a/export.ps1 b/export.ps1 index 76df9bd81b..90295563bf 100644 --- a/export.ps1 +++ b/export.ps1 @@ -1,5 +1,12 @@ #!/usr/bin/env pwsh +# Emergency backup option to use previous export.ps1 (export_legacy.ps1) if the new export approach fails. +# To use it, set environmental variable like: $Env:ESP_IDF_LEGACY_EXPORT=1 +if ($env:ESP_IDF_LEGACY_EXPORT) { + . ./tools/legacy_exports/export_legacy.ps1 + exit $LASTEXITCODE +} + $idf_path = "$PSScriptRoot" if (-not (Test-Path "$idf_path/tools/idf.py") -or diff --git a/export.sh b/export.sh index f4eb6334e3..38a8cece0e 100644 --- a/export.sh +++ b/export.sh @@ -1,5 +1,12 @@ # This script should be sourced, not executed. +# Emergency backup option to use previous export.sh (export_legacy.sh) if the new export approach fails. +# To use it, set environmental variable like: export ESP_IDF_LEGACY_EXPORT=1 +if [ -n "${ESP_IDF_LEGACY_EXPORT-}" ]; then + . ./tools/legacy_exports/export_legacy.sh + return $? +fi + # shellcheck disable=SC2128,SC2169,SC2039,SC3054 # ignore array expansion warning if [ -n "${BASH_SOURCE-}" ] && [ "${BASH_SOURCE[0]}" = "${0}" ] then diff --git a/tools/ci/exclude_check_tools_files.txt b/tools/ci/exclude_check_tools_files.txt index b5326f7e41..614bb15c63 100644 --- a/tools/ci/exclude_check_tools_files.txt +++ b/tools/ci/exclude_check_tools_files.txt @@ -51,3 +51,7 @@ tools/esp_prov/**/* tools/ci/sort_yaml.py tools/ci/sg_rules/* tools/ci/previous_stage_job_status.py +tools/legacy_exports/export_legacy.fish +tools/legacy_exports/export_legacy.sh +tools/legacy_exports/export_legacy.ps1 +tools/legacy_exports/export_legacy.bat diff --git a/tools/legacy_exports/export_legacy.bat b/tools/legacy_exports/export_legacy.bat new file mode 100644 index 0000000000..74971a290d --- /dev/null +++ b/tools/legacy_exports/export_legacy.bat @@ -0,0 +1,129 @@ +@echo off +if defined MSYSTEM ( + echo This .bat file is for Windows CMD.EXE shell only. + goto :eof +) + +set SCRIPT_EXIT_CODE=0 + +:: Missing requirements check +set MISSING_REQUIREMENTS= +python.exe --version >NUL 2>NUL +if %errorlevel% neq 0 ( + set SCRIPT_EXIT_CODE=%errorlevel% + set "MISSING_REQUIREMENTS= python &echo\" +) +git.exe --version >NUL 2>NUL +if %errorlevel% neq 0 ( + set SCRIPT_EXIT_CODE=%errorlevel% + set "MISSING_REQUIREMENTS=%MISSING_REQUIREMENTS% git" +) + +if not "%MISSING_REQUIREMENTS%" == "" goto :__error_missing_requirements + +:: Infer IDF_PATH from script location +set IDF_PATH=%~dp0 +set IDF_PATH=%IDF_PATH:~0,-1% +:: As export_legacy got moved, remove the trailing 'tools\legacy_exports' to detect IDF_PATH +set "IDF_PATH=%IDF_PATH:\tools\legacy_exports=%" + +echo Checking Python compatibility +python.exe "%IDF_PATH%\tools\python_version_checker.py" + +set "IDF_TOOLS_PY_PATH=%IDF_PATH%\tools\idf_tools.py" +set "IDF_TOOLS_JSON_PATH=%IDF_PATH%\tools\tools.json" +set "IDF_TOOLS_EXPORT_CMD=%IDF_PATH%\export.bat" +set "IDF_TOOLS_INSTALL_CMD=%IDF_PATH%\install.bat" +echo Setting IDF_PATH: %IDF_PATH% +echo. + +set "OLD_PATH=%PATH%" +echo Adding ESP-IDF tools to PATH... +:: Export tool paths and environment variables. +:: It is possible to do this without a temporary file (running idf_tools.py from for /r command), +:: but that way it is impossible to get the exit code of idf_tools.py. +set "IDF_TOOLS_EXPORTS_FILE=%TEMP%\idf_export_vars.tmp" +python.exe "%IDF_PATH%\tools\idf_tools.py" export --format key-value >"%IDF_TOOLS_EXPORTS_FILE%" +if %errorlevel% neq 0 ( + set SCRIPT_EXIT_CODE=%errorlevel% + goto :__end +) + +for /f "usebackq tokens=1,2 eol=# delims==" %%a in ("%IDF_TOOLS_EXPORTS_FILE%") do ( + call set "%%a=%%b" + ) + +:: This removes OLD_PATH substring from PATH, leaving only the paths which have been added, +:: and prints semicolon-delimited components of the path on separate lines +call set PATH_ADDITIONS=%%PATH:%OLD_PATH%=%% +if "%PATH_ADDITIONS%"=="" call :__print_nothing_added +if not "%PATH_ADDITIONS%"=="" echo %PATH_ADDITIONS:;=&echo. % + +DOSKEY idf.py=python.exe "%IDF_PATH%\tools\idf.py" $* +DOSKEY esptool.py=python.exe "%IDF_PATH%\components\esptool_py\esptool\esptool.py" $* +DOSKEY espefuse.py=python.exe "%IDF_PATH%\components\esptool_py\esptool\espefuse.py" $* +DOSKEY espsecure.py=python.exe "%IDF_PATH%\components\esptool_py\esptool\espsecure.py" $* +DOSKEY otatool.py=python.exe "%IDF_PATH%\components\app_update\otatool.py" $* +DOSKEY parttool.py=python.exe "%IDF_PATH%\components\partition_table\parttool.py" $* + +echo Checking if Python packages are up to date... +python.exe "%IDF_PATH%\tools\idf_tools.py" check-python-dependencies +if %errorlevel% neq 0 ( + set SCRIPT_EXIT_CODE=%errorlevel% + goto :__end +) + +python.exe "%IDF_PATH%\tools\idf_tools.py" uninstall --dry-run > UNINSTALL_OUTPUT +SET /p UNINSTALL=nul 2>nul +) +set IDF_TOOLS_EXPORTS_FILE= +set IDF_TOOLS_EXPORT_CMD= +set IDF_TOOLS_INSTALL_CMD= +set IDF_TOOLS_PY_PATH= +set IDF_TOOLS_JSON_PATH= +set OLD_PATH= +set PATH_ADDITIONS= +set MISSING_REQUIREMENTS= +set UNINSTALL= +exit /b %SCRIPT_EXIT_CODE% diff --git a/tools/legacy_exports/export_legacy.fish b/tools/legacy_exports/export_legacy.fish new file mode 100644 index 0000000000..dee51f5c2a --- /dev/null +++ b/tools/legacy_exports/export_legacy.fish @@ -0,0 +1,111 @@ +# This script should be sourced, not executed. + +# `idf_tools.py export --deactivate` create statement, with keyword unset, but fish shell support only `set --erase variable` +function unset + set --erase $argv +end + +function __main + set script_dir (dirname (realpath (status -f))) + # As export_legacy got moved, remove the trailing 'tools\legacy_exports' to detect IDF_PATH + set script_dir (string replace -r '/tools/legacy_exports$' '' $script_dir) + if not set -q IDF_PATH + set -gx IDF_PATH $script_dir + echo "Setting IDF_PATH to '$IDF_PATH'" + end + + if test "$IDF_PATH" != "$script_dir" + # Change IDF_PATH is important when there are 2 ESP-IDF versions in different directories. + # Sourcing this script without change, would cause sourcing wrong export script. + echo "Resetting IDF_PATH from '$IDF_PATH' to '$script_dir'" + set IDF_PATH "$script_dir" + end + + set oldpath = $PATH + + echo "Detecting the Python interpreter" + source "$IDF_PATH"/tools/detect_python.fish + + echo "Checking Python compatibility" + "$ESP_PYTHON" "$IDF_PATH"/tools/python_version_checker.py + + echo "Checking other ESP-IDF version." + set idf_deactivate ("$ESP_PYTHON" "$IDF_PATH"/tools/idf_tools.py export --deactivate) || return 1 + eval "$idf_deactivate" + + echo "Adding ESP-IDF tools to PATH..." + # Call idf_tools.py to export tool paths + set -gx IDF_TOOLS_EXPORT_CMD "$IDF_PATH"/export.fish + set -gx IDF_TOOLS_INSTALL_CMD "$IDF_PATH"/install.fish + # Allow calling some IDF python tools without specifying the full path + # "$IDF_PATH"/tools is already added by 'idf_tools.py export' + set IDF_ADD_PATHS_EXTRAS "$IDF_PATH"/components/espcoredump + set IDF_ADD_PATHS_EXTRAS "$IDF_ADD_PATHS_EXTRAS":"$IDF_PATH"/components/partition_table + set IDF_ADD_PATHS_EXTRAS "$IDF_ADD_PATHS_EXTRAS":"$IDF_PATH"/components/app_update + + set idf_exports ("$ESP_PYTHON" "$IDF_PATH"/tools/idf_tools.py export --add_paths_extras="$IDF_ADD_PATHS_EXTRAS") || return 1 + eval "$idf_exports" + set -x PATH "$IDF_ADD_PATHS_EXTRAS":"$PATH" + + echo "Checking if Python packages are up to date..." + "$ESP_PYTHON" "$IDF_PATH"/tools/idf_tools.py check-python-dependencies || return 1 + + set added_path_variables + for entry in $PATH; + if not contains $entry $oldpath + set -a added_path_variables $entry + end + end + if set -q added_path_variables[1] + echo "Added the following directories to PATH:" + for entry in $added_path_variables; + echo $entry + end + else + echo "All paths are already set." + end + + set uninstall ("$ESP_PYTHON" "$IDF_PATH"/tools/idf_tools.py uninstall --dry-run) || return 1 + if test -n "$uninstall" + echo "" + echo "Detected installed tools that are not currently used by active ESP-IDF version." + echo "$uninstall" + echo "For free up even more space, remove installation packages of those tools. Use option '$ESP_PYTHON $IDF_PATH/tools/idf_tools.py uninstall --remove-archives'." + echo "" + end + + # Clean up + set -e added_path_variables + set -e cmd + set -e old_path + set -e paths + set -e path_prefix + set -e path_entry + set -e IDF_ADD_PATHS_EXTRAS + set -e idf_exports + set -e ESP_PYTHON + set -e uninstall + set -e script_dir + set -e idf_deactivate + + + # Not unsetting IDF_PYTHON_ENV_PATH, it can be used by IDF build system + # to check whether we are using a private Python environment + + echo "Done! You can now compile ESP-IDF projects." + echo "Go to the project directory and run:" + echo "" + echo " idf.py build" + echo "" +end + +__main + +set click_version (python -c 'import click; print(click.__version__.split(".")[0])') +if test $click_version -lt 8 + eval (env _IDF.PY_COMPLETE=source_fish idf.py) +else + eval (env _IDF.PY_COMPLETE=fish_source idf.py) +end + +functions -e __main diff --git a/tools/legacy_exports/export_legacy.ps1 b/tools/legacy_exports/export_legacy.ps1 new file mode 100644 index 0000000000..e3ffa97096 --- /dev/null +++ b/tools/legacy_exports/export_legacy.ps1 @@ -0,0 +1,94 @@ +#!/usr/bin/env pwsh +$S = [IO.Path]::PathSeparator # path separator. WIN:';', UNIX:":" + +$IDF_PATH = "$PSScriptRoot" +# As export_legacy got moved, remove the trailing 'tools\legacy_exports' to detect IDF_PATH +$IDF_PATH = $IDF_PATH -replace "\\tools\\legacy_exports$", "" + +Write-Output "Setting IDF_PATH: $IDF_PATH" +$env:IDF_PATH = "$IDF_PATH" + +Write-Output "Checking Python compatibility" +python "$IDF_PATH/tools/python_version_checker.py" + +Write-Output "Adding ESP-IDF tools to PATH..." +$OLD_PATH = $env:PATH.split($S) | Select-Object -Unique # array without duplicates +# using idf_tools.py to get $envars_array to set +$envars_raw = python "$IDF_PATH/tools/idf_tools.py" export --format key-value +if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } # if error + +$envars_array = @() # will be filled like: +# [ +# [vname1, vval1], [vname2, vval2], ... +# ] +foreach ($line in $envars_raw) { + $pair = $line.split("=") # split in name, val + $var_name = $pair[0].Trim() # trim spaces on the ends of the name + $var_val = $pair[1].Trim() # trim spaces on the ends of the val + $envars_array += (, ($var_name, $var_val)) +} + +if ($null -eq $IsWindows) { + # $IsWindows was added in PowerShell Core 6 and PowerShell 7 together with multi-platform support. # I.E. if this + # internal variable is not set then PowerShell 5 is used and # the platform cannot be # anything else than Windows. + $Windows = $true +} + +foreach ($pair in $envars_array) { + # setting the values + $var_name = $pair[0].Trim() # trim spaces on the ends of the name + $var_val = $pair[1].Trim() # trim spaces on the ends of the val + if ($var_name -eq "PATH") { + # trim "%PATH%" or "`$PATH" + if ($IsWindows -or $Windows) { + $var_val = $var_val.Trim($S + "%PATH%") + } else { + $var_val = $var_val.Trim($S + "`$PATH") + } + # apply + $env:PATH = $var_val + $S + $env:PATH + } else { + New-Item -Path "env:$var_name" -Value "$var_val" -Force + } +} + +# Allow calling some IDF python tools without specifying the full path +function idf.py { &python "$IDF_PATH\tools\idf.py" $args } +function espefuse.py { &python "$IDF_PATH\components\esptool_py\esptool\espefuse.py" $args } +function espsecure.py { &python "$IDF_PATH\components\esptool_py\esptool\espsecure.py" $args } +function otatool.py { &python "$IDF_PATH\components\app_update\otatool.py" $args } +function parttool.py { &python "$IDF_PATH\components\partition_table\parttool.py" $args } + +#Compare Path's OLD vs. NEW +$NEW_PATH = $env:PATH.split($S) | Select-Object -Unique # array without duplicates +$dif_Path = Compare-Object -ReferenceObject $OLD_PATH -DifferenceObject $NEW_PATH -PassThru +if ($null -ne $dif_Path) { + Write-Output "`nAdded to PATH`n-------------" + Write-Output $dif_Path +} else { + Write-Output "No directories added to PATH:" + Write-Output $OLD_PATH +} + + +Write-Output "Checking if Python packages are up to date..." + +Start-Process -Wait -NoNewWindow -FilePath "python" -Args "`"$IDF_PATH/tools/idf_tools.py`" check-python-dependencies" +if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } # if error + +$uninstall = python "$IDF_PATH/tools/idf_tools.py" uninstall --dry-run + +if (![string]::IsNullOrEmpty($uninstall)){ + Write-Output "" + Write-Output "Detected installed tools that are not currently used by active ESP-IDF version." + Write-Output "$uninstall" + Write-Output "For free up even more space, remove installation packages of those tools. Use option 'python.exe $IDF_PATH\tools\idf_tools.py uninstall --remove-archives'." + Write-Output "" +} + +Write-Output " +Done! You can now compile ESP-IDF projects. +Go to the project directory and run: + idf.py build + +" diff --git a/tools/legacy_exports/export_legacy.sh b/tools/legacy_exports/export_legacy.sh new file mode 100644 index 0000000000..9ad0f7ff50 --- /dev/null +++ b/tools/legacy_exports/export_legacy.sh @@ -0,0 +1,235 @@ +# This script should be sourced, not executed. + +__realpath() { + wdir="$PWD"; [ "$PWD" = "/" ] && wdir="" + arg=$1 + case "$arg" in + /*) scriptdir="${arg}";; + *) scriptdir="$wdir/${arg#./}";; + esac + scriptdir="${scriptdir%/*}" + echo "$scriptdir" +} + + +__verbose() { + [ -n "${IDF_EXPORT_QUIET-}" ] && return + echo "$@" +} + +__script_dir(){ + # shellcheck disable=SC2169,SC2169,SC2039,SC3010,SC3028 # unreachable with 'dash' + if [ "$(uname -s)" = "Darwin" ]; then + # convert possibly relative path to absolute + script_dir="$(__realpath "${self_path}")" + # resolve any ../ references to make the path shorter + script_dir="$(cd "${script_dir}" || exit 1; pwd)" + else + # convert to full path and get the directory name of that + script_name="$(readlink -f "${self_path}")" + script_dir="$(dirname "${script_name}")" + fi + if [ "$script_dir" = '.' ] + then + script_dir="$(pwd)" + fi + echo "$script_dir" +} + +__is_dir_esp_idf(){ + if [ ! -f "$1/tools/idf.py" ] || [ ! -f "$1/tools/idf_tools.py" ] + then + # Echo command here is not used for printing to the terminal, but as non-empty return value from function. + echo "THIS DIRECTORY IS NOT ESP-IDF" + fi +} + +__main() { + # The file doesn't have executable permissions, so this shouldn't really happen. + # Doing this in case someone tries to chmod +x it and execute... + + # shellcheck disable=SC2128,SC2169,SC2039,SC3054 # ignore array expansion warning + if [ -n "${BASH_SOURCE-}" ] && [ "${BASH_SOURCE[0]}" = "${0}" ] + then + echo "This script should be sourced, not executed:" + # shellcheck disable=SC2039,SC3054 # reachable only with bash + echo ". ${BASH_SOURCE[0]}" + return 1 + fi + + # If using bash or zsh, try to guess IDF_PATH from script location. + self_path="" + # shellcheck disable=SC2128 # ignore array expansion warning + if [ -n "${BASH_SOURCE-}" ] + then + self_path="${BASH_SOURCE}" + elif [ -n "${ZSH_VERSION-}" ] + then + # shellcheck disable=SC2296 # ignore parameter starts with '{' because it's zsh + self_path="${(%):-%x}" + fi + + script_dir="$(__script_dir)" + # As export_legacy got moved, remove the trailing 'tools\legacy_exports' to detect IDF_PATH + script_dir="${script_dir%/tools/legacy_exports}" + # Since sh or dash shells can't detect script_dir correctly, check if script_dir looks like an IDF directory + is_script_dir_esp_idf=$(__is_dir_esp_idf "${script_dir}") + + if [ -z "${IDF_PATH-}" ] + then + # IDF_PATH not set in the environment. + + if [ -n "${is_script_dir_esp_idf}" ] + then + echo "Could not detect IDF_PATH. Please set it before sourcing this script:" + echo " export IDF_PATH=(add path here)" + return 1 + fi + export IDF_PATH="${script_dir}" + echo "Setting IDF_PATH to '${IDF_PATH}'" + else + # IDF_PATH came from the environment, check if the path is valid + # Set IDF_PATH to script_dir, if script_dir looks like an IDF directory + if [ ! "${IDF_PATH}" = "${script_dir}" ] && [ -z "${is_script_dir_esp_idf}" ] + then + # Change IDF_PATH is important when there are 2 ESP-IDF versions in different directories. + # Sourcing this script without change, would cause sourcing wrong export script. + echo "Resetting IDF_PATH from '${IDF_PATH}' to '${script_dir}' " + export IDF_PATH="${script_dir}" + fi + # Check if this path looks like an IDF directory + is_idf_path_esp_idf=$(__is_dir_esp_idf "${IDF_PATH}") + if [ -n "${is_idf_path_esp_idf}" ] + then + echo "IDF_PATH is set to '${IDF_PATH}', but it doesn't look like an ESP-IDF directory." + echo "If you have set IDF_PATH manually, check if the path is correct." + return 1 + fi + + # The variable might have been set (rather than exported), re-export it to be sure + export IDF_PATH="${IDF_PATH}" + fi + + old_path="$PATH" + + echo "Detecting the Python interpreter" + . "${IDF_PATH}/tools/detect_python.sh" + + echo "Checking Python compatibility" + "$ESP_PYTHON" "${IDF_PATH}/tools/python_version_checker.py" + + __verbose "Checking other ESP-IDF version." + idf_deactivate=$("$ESP_PYTHON" "${IDF_PATH}/tools/idf_tools.py" export --deactivate) || return 1 + eval "${idf_deactivate}" + + __verbose "Adding ESP-IDF tools to PATH..." + # Call idf_tools.py to export tool paths + export IDF_TOOLS_EXPORT_CMD=${IDF_PATH}/export.sh + export IDF_TOOLS_INSTALL_CMD=${IDF_PATH}/install.sh + # Allow calling some IDF python tools without specifying the full path + # ${IDF_PATH}/tools is already added by 'idf_tools.py export' + IDF_ADD_PATHS_EXTRAS="${IDF_PATH}/components/espcoredump" + IDF_ADD_PATHS_EXTRAS="${IDF_ADD_PATHS_EXTRAS}:${IDF_PATH}/components/partition_table" + IDF_ADD_PATHS_EXTRAS="${IDF_ADD_PATHS_EXTRAS}:${IDF_PATH}/components/app_update" + + idf_exports=$("$ESP_PYTHON" "${IDF_PATH}/tools/idf_tools.py" export "--add_paths_extras=${IDF_ADD_PATHS_EXTRAS}") || return 1 + eval "${idf_exports}" + export PATH="${IDF_ADD_PATHS_EXTRAS}:${PATH}" + + __verbose "Checking if Python packages are up to date..." + "$ESP_PYTHON" "${IDF_PATH}/tools/idf_tools.py" check-python-dependencies || return 1 + + if [ -n "$BASH" ] + then + path_prefix="${PATH%%"${old_path}"}" + # shellcheck disable=SC2169,SC2039 # unreachable with 'dash' + if [ -n "${path_prefix}" ]; then + __verbose "Added the following directories to PATH:" + else + __verbose "All paths are already set." + fi + old_ifs="$IFS" + IFS=":" + for path_entry in ${path_prefix} + do + __verbose " ${path_entry}" + done + IFS="$old_ifs" + unset old_ifs + else + __verbose "Updated PATH variable:" + __verbose " ${PATH}" + fi + + uninstall=$("$ESP_PYTHON" "${IDF_PATH}/tools/idf_tools.py" uninstall --dry-run) || return 1 + if [ -n "$uninstall" ] + then + __verbose "" + __verbose "Detected installed tools that are not currently used by active ESP-IDF version." + __verbose "${uninstall}" + __verbose "To free up even more space, remove installation packages of those tools. Use option '${ESP_PYTHON} ${IDF_PATH}/tools/idf_tools.py uninstall --remove-archives'." + __verbose "" + fi + + __verbose "Done! You can now compile ESP-IDF projects." + __verbose "Go to the project directory and run:" + __verbose "" + __verbose " idf.py build" + __verbose "" +} + +__cleanup() { + unset old_path + unset paths + unset path_prefix + unset path_entry + unset IDF_ADD_PATHS_EXTRAS + unset idf_exports + unset idf_deactivate + unset ESP_PYTHON + unset SOURCE_ZSH + unset SOURCE_BASH + unset WARNING_MSG + unset uninstall + unset is_idf_path_esp_idf + unset is_script_dir_esp_idf + + unset __realpath + unset __main + unset __verbose + unset __enable_autocomplete + unset __cleanup + unset __is_dir_esp_idf + + # Not unsetting IDF_PYTHON_ENV_PATH, it can be used by IDF build system + # to check whether we are using a private Python environment + + return "$1" +} + + +__enable_autocomplete() { + click_version="$(python -c 'import click; print(click.__version__.split(".")[0])')" + if [ "${click_version}" -lt 8 ] + then + SOURCE_ZSH=source_zsh + SOURCE_BASH=source_bash + else + SOURCE_ZSH=zsh_source + SOURCE_BASH=bash_source + fi + if [ -n "${ZSH_VERSION-}" ] + then + autoload -Uz compinit && compinit -u + eval "$(env _IDF.PY_COMPLETE=$SOURCE_ZSH idf.py)" || echo "WARNING: Failed to load shell autocompletion for zsh version: $ZSH_VERSION!" + elif [ -n "${BASH_SOURCE-}" ] + then + WARNING_MSG="WARNING: Failed to load shell autocompletion for bash version: $BASH_VERSION!" + # shellcheck disable=SC3028,SC3054,SC2086,SC2169 # code block for 'bash' only + [ ${BASH_VERSINFO[0]} -lt 4 ] && { echo "$WARNING_MSG"; return; } + eval "$(env LANG=en _IDF.PY_COMPLETE=$SOURCE_BASH idf.py)" || echo "$WARNING_MSG" + fi +} + +__main && __enable_autocomplete +__cleanup $?