From 39ebccb7fb24d1eb5b5b7ea67f76d454ff802014 Mon Sep 17 00:00:00 2001 From: Frantisek Hrbata Date: Wed, 19 Mar 2025 10:11:43 +0100 Subject: [PATCH] fix(tools): handle packages with dots in their names during dependency checks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The `setuptools` package starting with `v70.1.0`[1] contains built-in `bdist_wheel` command. Before this version `setuptools` relied on the `bdist_wheel` command implementation from the `wheel` package. Starting with `setuptools` `v75.8.1` the `PEP 491`[3] restrictions on the distribution name of a wheel package are enforced[4], replacting also `.` with `_`. Note that `PEP 491` actually allows `.` in the distribution name, but for some reason the latest packaging docs[10][11] does not, stating that `.` should be replaced with `_`. This was discussion here[12]. Also the `wheel` package starting with `v0.45.0`[5] is using the `bdist_wheel` command from `setuptools`. This means that any package which has `.` in its distribution name, like `ruamel.yaml.clib`, can have different wheel name, depending on which version of the `bdist_wheel` command was used. The `bdist_wheel` command from setuptools prior `v75.8.1` or `wheel` prior `v0.45.0` will keep the dots in distribution name preserved. For exaple the `ruamel.yaml.clib` package will have distribution name `ruamel.yaml.clib-0.2.12.dist-info. Newer versions will replace the dots with `_` according to [10][11], creating distribution like `ruamel_yaml_clib-0.2.12.dist-info`. From packaging point of view `ruamel.yaml.clib-0.2.12.dist-info` and `ruamel_yaml_clib-0.2.12.dist-info` are the same packages, but this is not reflected in `importlib.metadata` prior python 3.10[9], which does not perform name normalization prior the distribution search. This causes the `version` from `importlib.metadata` to fail on python prior the 3.10 version if the package with dots in distribution name was generated with normalized paths with newer `setuptools`. Note that the distribution name normalization was backported to some later 3.9 python version. Let's demonstrate this behavior on a simple package with the `my.minimal.package` name. ``` my_minimal_package/ ├── pkg │   └── __init__.py └── setup.py from setuptools import setup, find_packages setup( name='my.minimal.package', version='0.1.0', packages=find_packages(), install_requires=[], entry_points={}, ) ``` With python 3.9.0 search for `my.minimal.package` fails because of the missing name normalization. ``` docker run --rm -it --platform linux/x86_64 python:3.9.0 bash python -m venv venv . venv/bin/activate pip install setuptools==v75.8.1 python setup.py bdist_wheel pip install dist/my_minimal_package-0.1.0-py3-none-any.whl python Python 3.9.0 (default, Nov 18 2020, 13:28:38) [GCC 8.3.0] on linux Type "help", "copyright", "credits" or "license" for more information. >>> from importlib.metadata import version as get_version >>> get_version('my.minimal.package') Traceback (most recent call last): File "", line 1, in File "/usr/local/lib/python3.9/importlib/metadata.py", line 551, in version return distribution(distribution_name).version File "/usr/local/lib/python3.9/importlib/metadata.py", line 524, in distribution return Distribution.from_name(distribution_name) File "/usr/local/lib/python3.9/importlib/metadata.py", line 187, in from_name raise PackageNotFoundError(name) importlib.metadata.PackageNotFoundError: my.minimal.package >>> get_version('my_minimal_package') '0.1.0' ``` With python 3.10.0 search for both `my.minimal.package` and `my_minimal_package` succeeds. ``` docker run --rm -it --platform linux/x86_64 python:3.10.0 bash python Python 3.10.0 (default, Dec 3 2021, 00:21:30) [GCC 10.2.1 20210110] on linux Type "help", "copyright", "credits" or "license" for more information. >>> from importlib.metadata import version as get_version >>> get_version('my.minimal.package') '0.1.0' >>> get_version('my_minimal_package') '0.1.0' ``` In our `tools/check_python_dependencies.py` we cannot relay on the default distribution finder, used in the `version` function from `importlib.metadata`, to do name normalization on older python versions. To cope with this, implement a fallback version search. If `version` fails with `PackageNotFoundError`, do the name normalization according to [10][11] and try again. Note: There is also a `wheel`[6][7] `v0.43.0` package embeded in `setuptools` along with the new implementation[8]. This one seems to be used if the external `wheel` package is not available but imported. TBH this is all kinda messy and I may have overlooked something. * [1] https://setuptools.pypa.io/en/stable/history.html#v70-1-0 * [2] https://setuptools.pypa.io/en/stable/history.html#v75-8-1 * [3] https://peps.python.org/pep-0491/#escaping-and-unicode * [4] https://github.com/pypa/setuptools/pull/4766/files * [5] https://wheel.readthedocs.io/en/stable/news.html * [6] https://github.com/pypa/setuptools/blob/main/setuptools/_vendor/wheel/__init__.py * [7] https://github.com/pypa/setuptools/issues/1386 * [8] https://github.com/pypa/setuptools/blob/main/setuptools/command/bdist_wheel.py * [9] https://github.com/python/cpython/commit/c6ca368867bd68d44f333df840aa85d425a51410 * [10] https://packaging.python.org/en/latest/specifications/name-normalization/#name-normalization * [11] https://packaging.python.org/en/latest/specifications/binary-distribution-format/ #escaping-and-unicode * [12] https://github.com/pypa/setuptools/issues/3777 Signed-off-by: Frantisek Hrbata --- tools/check_python_dependencies.py | 43 +++++++++++++++++++++++++----- 1 file changed, 37 insertions(+), 6 deletions(-) diff --git a/tools/check_python_dependencies.py b/tools/check_python_dependencies.py index 6621b03314..1c290b4b7c 100755 --- a/tools/check_python_dependencies.py +++ b/tools/check_python_dependencies.py @@ -1,11 +1,12 @@ #!/usr/bin/env python # -# SPDX-FileCopyrightText: 2018-2024 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2018-2025 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 import argparse import os import re import sys +from typing import Optional try: from packaging.requirements import Requirement @@ -17,12 +18,14 @@ except ImportError: sys.exit(1) try: - from importlib.metadata import requires - from importlib.metadata import version as get_version + from importlib.metadata import requires as _requires + from importlib.metadata import version as _version + from importlib.metadata import PackageNotFoundError except ImportError: # compatibility for python <=3.7 - from importlib_metadata import requires # type: ignore - from importlib_metadata import version as get_version # type: ignore + from importlib_metadata import requires as _requires # type: ignore + from importlib_metadata import version as _version # type: ignore + from importlib_metadata import PackageNotFoundError # type: ignore try: from typing import Set @@ -33,6 +36,34 @@ except ImportError: PYTHON_PACKAGE_RE = re.compile(r'[^<>=~]+') + +# The version and requires function from importlib.metadata in python prior +# 3.10 does perform distribution name normalization before searching for +# package distribution. This might cause problems for package with dot in its +# name as the wheel build backend(e.g. setuptools >= 75.8.1), may perform +# distribution name normalization. If the package name is not found, try again +# with normalized name. +# https://packaging.python.org/en/latest/specifications/binary-distribution-format/#escaping-and-unicode +def normalize_name(name: str) -> str: + return re.sub(r'[-_.]+', '-', name).lower().replace('-', '_') + + +def get_version(name: str) -> str: + try: + version = _version(name) + except PackageNotFoundError: + version = _version(normalize_name(name)) + return version + + +def get_requires(name: str) -> Optional[list]: + try: + requires = _requires(name) + except PackageNotFoundError: + requires = _requires(normalize_name(name)) + return requires + + if __name__ == '__main__': parser = argparse.ArgumentParser(description='ESP-IDF Python package dependency checker') parser.add_argument('--requirements', '-r', @@ -108,7 +139,7 @@ if __name__ == '__main__': dependency_requirements = set() extras = list(requirement.extras) or [''] # `requires` returns all sub-requirements including all extras - we need to filter out just required ones - for name in requires(requirement.name) or []: + for name in get_requires(requirement.name) or []: sub_req = Requirement(name) # check extras e.g. esptool[hsm] for extra in extras: