Merge branch 'fix/gdbgui_v4.4' into 'release/v4.4'

Tools: gdbgui is not supported on Python 3.11

See merge request espressif/esp-idf!21054

requirements.txt

@@ -19,15 +19,18 @@ pyparsing>=2.0.3,<2.4.0
 pyelftools>=0.22
 idf-component-manager~=1.0
 
-gdbgui==0.13.2.0
+gdbgui==0.13.2.0; python_version < "3.11"
 # 0.13.2.1 supports Python 3.6+ only
 # Windows is not supported since 0.14.0.0. See https://github.com/cs01/gdbgui/issues/348
-pygdbmi<=0.9.0.2
+pygdbmi<=0.9.0.2; python_version < "3.11"
 # The pygdbmi required max version 0.9.0.2 since 0.9.0.3 is not compatible with latest gdbgui (>=0.13.2.0)
 # A compatible Socket.IO should be used. See https://github.com/miguelgrinberg/python-socketio/issues/578
-python-socketio<5
+python-socketio<5; python_version < "3.11"
-jinja2<3.1  # See https://github.com/espressif/esp-idf/issues/8760
+jinja2<3.1; python_version < "3.11"   # See https://github.com/espressif/esp-idf/issues/8760
-itsdangerous<2.1
+itsdangerous<2.1; python_version < "3.11"
+
+# gdbgui is not supported on Python 3.11. See https://github.com/cs01/gdbgui/issues/447
+pygdbmi<=0.9.0.2; python_version > "3.10"
 
 kconfiglib==13.7.1
 

tools/ci/python_packages/ttfw_idf/DebugUtils.py

@@ -1,16 +1,5 @@
-# Copyright 2020 Espressif Systems (Shanghai) PTE LTD
+# SPDX-FileCopyrightText: 2020-2022 Espressif Systems (Shanghai) CO LTD
-#
+# SPDX-License-Identifier: Apache-2.0
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
 
 from __future__ import unicode_literals
 
@@ -18,7 +7,6 @@ import logging
 from io import open
 
 import pexpect
-import pygdbmi.gdbcontroller
 from tiny_test_fw import Utility
 
 try:
@@ -27,7 +15,7 @@ except ImportError:
     # Exception is ignored so the package is not required for those who don't use debug utils.
     err_msg = 'Please install py_debug_backend for debug utils to work properly!'
 
-    class debug_backend(object):
+    class debug_backend(object):  # type: ignore
         @staticmethod
         def create_oocd(*args, **kwargs):
             raise RuntimeError(err_msg)
@@ -36,6 +24,17 @@ except ImportError:
         def create_gdb(*args, **kwargs):
             raise RuntimeError(err_msg)
 
+try:
+    from debug_backend.defs import NoGdbProcessError
+except ImportError:
+    # fallback for pygdbmi<0.10.0.0 which is used in the previous version of debug_backend
+    try:
+        from pygdbmi.gdbcontroller import NoGdbProcessError
+    except ImportError:
+        # If debug_backend is not installed, the exception is ignored.
+        class NoGdbProcessError(ValueError):  # type: ignore
+            pass
+
 
 class CustomProcess(object):
     def __init__(self, cmd, logfile, verbose=True):
@@ -125,8 +124,9 @@ class GDBBackend(object):
     def __exit__(self, type, value, traceback):
         try:
             self.gdb.gdb_exit()
-        except pygdbmi.gdbcontroller.NoGdbProcessError as e:
+        except NoGdbProcessError as e:
-            #  the debug backend can fail on gdb exit when it tries to read the response after issuing the exit command.
+            #  the debug backend can fail on gdb exit
+            #  when it tries to read the response after issuing the exit command.
             Utility.console_log('Ignoring exception: {}'.format(e), 'O')
         except debug_backend.defs.DebuggerTargetStateTimeoutError:
             Utility.console_log('Ignoring timeout exception for GDB exit', 'O')

tools/ci/python_packages/ttfw_idf/requirements.txt

@@ -1,4 +1,4 @@
 -r ../tiny_test_fw/requirements.txt
 pexpect
 python-gitlab
-pygdbmi<=0.9.0.2
+pygdbmi>=0.9.0.0

tools/idf_py_actions/debug_ext.py

@@ -210,8 +210,12 @@ def action_extensions(base_actions, project_path):
         env['PURE_PYTHON'] = '1'
         try:
             process = subprocess.Popen(args, stdout=gdbgui_out, stderr=subprocess.STDOUT, bufsize=1, env=env)
-        except Exception as e:
+        except (OSError, subprocess.CalledProcessError) as e:
             print(e)
+            if sys.version_info[:2] >= (3, 11):
+                raise SystemExit('Unfortunately, gdbgui is supported only with Python 3.10 or older. '
+                                 'See: https://github.com/espressif/esp-idf/issues/10116. '
+                                 'Please use "idf.py gdb" or debug in Eclipse/Vscode instead.')
             raise FatalError('Error starting gdbgui. Please make sure gdbgui can be started', ctx)
 
         processes['gdbgui'] = process

tools/test_apps/system/panic/test_panic_util/test_panic_util.py

@@ -3,9 +3,10 @@ import os
 import re
 import subprocess
 import sys
+from subprocess import Popen
 
 import ttfw_idf
-from pygdbmi.gdbcontroller import GdbController, GdbTimeoutError, NoGdbProcessError
+from pygdbmi.gdbcontroller import GdbController
 from tiny_test_fw import DUT, TinyFW, Utility
 from tiny_test_fw.Utility import CaseConfig, SearchCases
 
@@ -14,6 +15,26 @@ TEST_PATH = os.path.relpath(os.path.join(os.path.dirname(os.path.abspath(__file_
 TEST_SUITE = 'Panic'
 
 
+class NoGdbProcessError(ValueError):
+    """Raise when trying to interact with gdb subprocess, but it does not exist.
+    It may have been killed and removed, or failed to initialize for some reason."""
+
+    pass
+
+
+def verify_valid_gdb_subprocess(gdb_process: Popen) -> None:
+    """Verify there is a process object, and that it is still running.
+    Raise NoGdbProcessError if either of the above are not true."""
+    if not gdb_process:
+        raise NoGdbProcessError('gdb process is not attached')
+
+    elif gdb_process.poll() is not None:
+        raise NoGdbProcessError(
+            'gdb process has already finished with return code: %s'
+            % str(gdb_process.poll())
+        )
+
+
 def ok(data):
     """ Helper function used with dut.expect_any """
     pass
@@ -160,14 +181,33 @@ class PanicTestMixin(object):
         self._port_close()
 
         Utility.console_log('Starting GDB...', 'orange')
-        self.gdb = GdbController(gdb_path=self.TOOLCHAIN_PREFIX + 'gdb')
+        gdb_path = self.TOOLCHAIN_PREFIX + 'gdb'
-        Utility.console_log('Running command: {}'.format(self.gdb.get_subprocess_cmd()), 'orange')
+        try:
+            from pygdbmi.constants import GdbTimeoutError
+            default_gdb_args = ['--nx', '--quiet', '--interpreter=mi2']
+            gdb_command = [gdb_path] + default_gdb_args
+            self.gdb = GdbController(command=gdb_command)
+        except ImportError:
+            # fallback for pygdbmi<0.10.0.0.
+            from pygdbmi.gdbcontroller import GdbTimeoutError
+            self.gdb = GdbController(gdb_path=gdb_path)
+
+        try:
+            gdb_command = self.gdb.command
+        except AttributeError:
+            # fallback for pygdbmi < 0.10
+            gdb_command = self.gdb.cmd
+
+        Utility.console_log('Running command: {}'.format(gdb_command), 'orange')
 
         for _ in range(10):
             try:
                 # GdbController creates a process with subprocess.Popen(). Is it really running? It is probable that
                 # an RPI under high load will get non-responsive during creating a lot of processes.
-                resp = self.gdb.get_gdb_response(timeout_sec=10)  # calls verify_valid_gdb_subprocess() internally
+                if not hasattr(self.gdb, 'verify_valid_gdb_subprocess'):
+                    # for pygdbmi >= 0.10.0.0
+                    verify_valid_gdb_subprocess(self.gdb.gdb_process)
+                resp = self.gdb.get_gdb_response(timeout_sec=10)  # calls verify_valid_gdb_subprocess() internally (pygdbmi < 0.10)
                 # it will be interesting to look up this response if the next GDB command fails (times out)
                 Utility.console_log('GDB response: {}'.format(resp), 'orange')
                 break  # success