Merge branch 'fix/py_dep_check_v5.1' into 'release/v5.1'

fix(tools): catch more general errors in python dependency checker (v5.1)

See merge request espressif/esp-idf!29164

.gitlab/ci/rules.yml

@@ -152,6 +152,8 @@
 
   - "tools/split_paths_by_spaces.py"
 
+  - "tools/check_python_dependencies.py"
+
 .patterns-windows: &patterns-windows
   - "tools/windows/**/*"
 

tools/check_python_dependencies.py

@@ -1,8 +1,7 @@
 #!/usr/bin/env python
 #
-# SPDX-FileCopyrightText: 2018-2023 Espressif Systems (Shanghai) CO LTD
+# SPDX-FileCopyrightText: 2018-2024 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
-
 import argparse
 import os
 import re
@@ -18,11 +17,11 @@ except ImportError:
     sys.exit(1)
 
 try:
-    from importlib.metadata import PackageNotFoundError, requires
+    from importlib.metadata import requires
     from importlib.metadata import version as get_version
 except ImportError:
     # compatibility for python <=3.7
-    from importlib_metadata import PackageNotFoundError, requires  # type: ignore
+    from importlib_metadata import requires  # type: ignore
     from importlib_metadata import version as get_version  # type: ignore
 
 try:
@@ -92,8 +91,10 @@ if __name__ == '__main__':
         if not req.marker or req.marker.evaluate():
             try:
                 version_check(req)
-            except PackageNotFoundError as e:
+            except Exception as e:
-                not_satisfied.append(f"'{e}' - was not found and is required by the application")
+                # Catch general exception, because get_version may return None (https://github.com/python/cpython/issues/91216)
+                # log package name alongside the error message for easier debugging
+                not_satisfied.append(f"Error while checking requirement '{req}'. Package was not found and is required by the application: {e}")
                 new_req_list.remove(req)
         else:
             new_req_list.remove(req)
@@ -120,8 +121,10 @@ if __name__ == '__main__':
                 # version is taken into account as well because packages can have different requirements for a given
                 # Python package. The dependencies need to be checked for all of them because they can be different.
                 new_req_list.extend(dependency_requirements - already_checked)
-            except PackageNotFoundError as e:
+            except Exception as e:
-                not_satisfied.append(f"'{e}' - was not found and is required by the application")
+                # Catch general exception, because get_version may return None (https://github.com/python/cpython/issues/91216)
+                # log package name alongside the error message for easier debugging
+                not_satisfied.append(f"Error while checking requirement '{req}'. Package was not found and is required by the application: {e}")
 
     if len(not_satisfied) > 0:
         print('The following Python requirements are not satisfied:')

tools/ci/exclude_check_tools_files.txt

@@ -9,7 +9,6 @@ tools/ci/get_all_test_results.py
 tools/ci/idf_unity_tester.py
 tools/gdb_panic_server.py
 tools/check_term.py
-tools/check_python_dependencies.py
 tools/python_version_checker.py
 tools/generate_debug_prefix_map.py
 tools/ci/checkout_project_ref.py

tools/test_idf_tools/test_idf_tools_python_env.py

@@ -1,6 +1,5 @@
-# SPDX-FileCopyrightText: 2022-2023 Espressif Systems (Shanghai) CO LTD
+# SPDX-FileCopyrightText: 2022-2024 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
-
 # NOTE: unittest is by default sorting tests based on their names,
 # so the order if which the tests are started may be different from
 # the order in which they are defined. Please make sure all tests
@@ -9,7 +8,6 @@
 # If test needs to change global state, it should return it to the
 # original state after it's finished. For more information please see
 # https://docs.python.org/3/library/unittest.html#organizing-test-code
-
 import inspect
 import os
 import shutil
@@ -30,7 +28,8 @@ TOOLS_DIR = os.environ.get('IDF_TOOLS_PATH') or os.path.expanduser(idf_tools.IDF
 PYTHON_DIR = os.path.join(TOOLS_DIR, 'python_env')
 PYTHON_DIR_BACKUP = tempfile.mkdtemp()
 REQ_SATISFIED = 'Python requirements are satisfied'
-REQ_MISSING = "{}' - was not found and is required by the application"
+# Python3.7 has a different error message that includes also "No package metadata was found for"
+REQ_MISSING = r'.*Package was not found and is required by the application: (No package metadata was found for )?{}.*'
 REQ_CORE = '- {}'.format(os.path.join(IDF_PATH, 'tools', 'requirements', 'requirements.core.txt'))
 REQ_GDBGUI = '- {}'.format(os.path.join(IDF_PATH, 'tools', 'requirements', 'requirements.gdbgui.txt'))
 CONSTR = 'Constraint file: {}/espidf.constraints'.format(TOOLS_DIR)
@@ -347,7 +346,7 @@ class TestCheckPythonDependencies(BasePythonInstall):
 
         # check-python-dependencies should fail as the package was not installed yet
         output = self.run_idf_tools(['check-python-dependencies'])
-        self.assertIn(REQ_MISSING.format('foopackage'), output)
+        self.assertRegex(output, REQ_MISSING.format('foopackage'))
         self.assertNotIn(REQ_SATISFIED, output)
 
     def test_dev_version(self):  # type: () -> None