From 3eec946f607eaf33a49a36f62b8ea9b38b567340 Mon Sep 17 00:00:00 2001 From: Alexey Lapshin Date: Mon, 15 Aug 2022 18:47:24 +0400 Subject: [PATCH 1/4] tools: add esp-rom-elfs version '20220823' --- docs/en/api-guides/tools/idf-tools-notes.inc | 5 ++ .../api-guides/tools/idf-tools-notes.inc | 5 ++ tools/idf_tools.py | 89 ++++++++++--------- tools/test_idf_tools/test_idf_tools.py | 32 +++++-- tools/tools.json | 34 +++++++ tools/tools_schema.json | 4 + 6 files changed, 123 insertions(+), 46 deletions(-) diff --git a/docs/en/api-guides/tools/idf-tools-notes.inc b/docs/en/api-guides/tools/idf-tools-notes.inc index 3f5fa241a0..4aee6e2d39 100644 --- a/docs/en/api-guides/tools/idf-tools-notes.inc +++ b/docs/en/api-guides/tools/idf-tools-notes.inc @@ -72,6 +72,11 @@ On Linux and macOS, it is recommended to install ninja using the OS-specific pac .. tool-dfu-util-notes +--- + +.. tool-esp-rom-elfs-notes + + --- .. tool-idf-python-notes diff --git a/docs/zh_CN/api-guides/tools/idf-tools-notes.inc b/docs/zh_CN/api-guides/tools/idf-tools-notes.inc index 6f03f311af..ec7bb478c0 100644 --- a/docs/zh_CN/api-guides/tools/idf-tools-notes.inc +++ b/docs/zh_CN/api-guides/tools/idf-tools-notes.inc @@ -74,6 +74,11 @@ On Linux and macOS, it is recommended to install ninja using the OS package mana .. tool-dfu-util-notes +--- + +.. tool-esp-rom-elfs-notes + + --- .. tool-idf-python-notes diff --git a/tools/idf_tools.py b/tools/idf_tools.py index cdaf963742..ddaa6d5ed9 100755 --- a/tools/idf_tools.py +++ b/tools/idf_tools.py @@ -162,6 +162,9 @@ class Platforms: if platform_alias is None: return None + if platform_alias == 'any' and CURRENT_PLATFORM: + platform_alias = CURRENT_PLATFORM + platform_name = Platforms.PLATFORM_FROM_NAME.get(platform_alias, None) # ARM platform may run on armhf hardware but having armel installed packages. @@ -447,7 +450,7 @@ def rename_with_retry(path_from, path_to): # type: (str, str) -> None time.sleep(0.5) -def strip_container_dirs(path, levels): # type: (str, int) -> None +def do_strip_container_dirs(path, levels): # type: (str, int) -> None assert levels > 0 # move the original directory out of the way (add a .tmp suffix) tmp_path = path + '.tmp' @@ -544,6 +547,7 @@ IDFToolOptions = namedtuple('IDFToolOptions', [ 'version_cmd', 'version_regex', 'version_regex_replace', + 'is_executable', 'export_paths', 'export_vars', 'install', @@ -560,8 +564,8 @@ class IDFTool(object): INSTALL_NEVER = 'never' def __init__(self, name, description, install, info_url, license, version_cmd, version_regex, supported_targets, version_regex_replace=None, - strip_container_dirs=0): - # type: (str, str, str, str, str, List[str], str, List[str], Optional[str], int) -> None + strip_container_dirs=0, is_executable=True): + # type: (str, str, str, str, str, List[str], str, List[str], Optional[str], int, bool) -> None self.name = name self.description = description self.drop_versions() @@ -569,11 +573,12 @@ class IDFTool(object): self.versions_installed = [] # type: List[str] if version_regex_replace is None: version_regex_replace = VERSION_REGEX_REPLACE_DEFAULT - self.options = IDFToolOptions(version_cmd, version_regex, version_regex_replace, + self.options = IDFToolOptions(version_cmd, version_regex, version_regex_replace, is_executable, [], OrderedDict(), install, info_url, license, strip_container_dirs, supported_targets) # type: ignore self.platform_overrides = [] # type: List[Dict[str, str]] self._platform = CURRENT_PLATFORM self._update_current_options() + self.is_executable = is_executable def copy_for_platform(self, platform): # type: (str) -> IDFTool result = copy.deepcopy(self) @@ -620,7 +625,9 @@ class IDFTool(object): v_repl = re.sub(SUBST_TOOL_PATH_REGEX, replace_path, v) if v_repl != v: v_repl = to_shell_specific_paths([v_repl])[0] - result[k] = v_repl + old_v = os.environ.get(k) + if old_v is None or old_v != v_repl: + result[k] = v_repl return result def check_version(self, extra_paths=None): # type: (Optional[List[str]]) -> str @@ -711,6 +718,9 @@ class IDFTool(object): if not os.path.exists(tool_path): # version not installed continue + if not self.is_executable: + self.versions_installed.append(version) + continue try: ver_str = self.check_version(self.get_export_paths(version)) except ToolNotFound: @@ -776,7 +786,7 @@ class IDFTool(object): mkdir_p(dest_dir) unpack(archive_path, dest_dir) if self._current_options.strip_container_dirs: # type: ignore - strip_container_dirs(dest_dir, self._current_options.strip_container_dirs) # type: ignore + do_strip_container_dirs(dest_dir, self._current_options.strip_container_dirs) # type: ignore @staticmethod def check_download_file(download_obj, local_path): # type: (IDFToolDownload, str) -> bool @@ -793,28 +803,29 @@ class IDFTool(object): @classmethod def from_json(cls, tool_dict): # type: (Dict[str, Union[str, List[str], Dict[str, str]]]) -> IDFTool - # json.load will return 'str' types in Python 3 and 'unicode' in Python 2 - expected_str_type = type(u'') - # Validate json fields tool_name = tool_dict.get('name') # type: ignore - if type(tool_name) is not expected_str_type: + if not isinstance(tool_name, str): raise RuntimeError('tool_name is not a string') description = tool_dict.get('description') # type: ignore - if type(description) is not expected_str_type: + if not isinstance(description, str): raise RuntimeError('description is not a string') + is_executable = tool_dict.get('is_executable', True) # type: ignore + if not isinstance(is_executable, bool): + raise RuntimeError('is_executable for tool %s is not a bool' % tool_name) + version_cmd = tool_dict.get('version_cmd') if type(version_cmd) is not list: raise RuntimeError('version_cmd for tool %s is not a list of strings' % tool_name) version_regex = tool_dict.get('version_regex') - if type(version_regex) is not expected_str_type or not version_regex: + if not isinstance(version_regex, str) or (not version_regex and is_executable): raise RuntimeError('version_regex for tool %s is not a non-empty string' % tool_name) version_regex_replace = tool_dict.get('version_regex_replace') - if version_regex_replace and type(version_regex_replace) is not expected_str_type: + if version_regex_replace and not isinstance(version_regex_replace, str): raise RuntimeError('version_regex_replace for tool %s is not a string' % tool_name) export_paths = tool_dict.get('export_paths') @@ -830,15 +841,15 @@ class IDFTool(object): raise RuntimeError('versions for tool %s is not an array' % tool_name) install = tool_dict.get('install', False) # type: ignore - if type(install) is not expected_str_type: + if not isinstance(install, str): raise RuntimeError('install for tool %s is not a string' % tool_name) info_url = tool_dict.get('info_url', False) # type: ignore - if type(info_url) is not expected_str_type: + if not isinstance(info_url, str): raise RuntimeError('info_url for tool %s is not a string' % tool_name) license = tool_dict.get('license', False) # type: ignore - if type(license) is not expected_str_type: + if not isinstance(license, str): raise RuntimeError('license for tool %s is not a string' % tool_name) strip_container_dirs = tool_dict.get('strip_container_dirs', 0) @@ -856,7 +867,7 @@ class IDFTool(object): # Create the object tool_obj = cls(tool_name, description, install, info_url, license, # type: ignore version_cmd, version_regex, supported_targets, version_regex_replace, # type: ignore - strip_container_dirs) # type: ignore + strip_container_dirs, is_executable) # type: ignore for path in export_paths: # type: ignore tool_obj.options.export_paths.append(path) # type: ignore @@ -870,7 +881,7 @@ class IDFTool(object): raise RuntimeError('platforms for override %d of tool %s is not a list' % (index, tool_name)) install = override.get('install') # type: ignore - if install is not None and type(install) is not expected_str_type: + if install is not None and not isinstance(install, str): raise RuntimeError('install for override %d of tool %s is not a string' % (index, tool_name)) version_cmd = override.get('version_cmd') # type: ignore @@ -879,12 +890,12 @@ class IDFTool(object): (index, tool_name)) version_regex = override.get('version_regex') # type: ignore - if version_regex is not None and (type(version_regex) is not expected_str_type or not version_regex): + if version_regex is not None and (not isinstance(version_regex, str) or not version_regex): raise RuntimeError('version_regex for override %d of tool %s is not a non-empty string' % (index, tool_name)) version_regex_replace = override.get('version_regex_replace') # type: ignore - if version_regex_replace is not None and type(version_regex_replace) is not expected_str_type: + if version_regex_replace is not None and not isinstance(version_regex_replace, str): raise RuntimeError('version_regex_replace for override %d of tool %s is not a string' % (index, tool_name)) @@ -900,11 +911,11 @@ class IDFTool(object): recommended_versions = {} # type: dict[str, list[str]] for version_dict in versions: # type: ignore version = version_dict.get('name') # type: ignore - if type(version) is not expected_str_type: + if not isinstance(version, str): raise RuntimeError('version name for tool {} is not a string'.format(tool_name)) version_status = version_dict.get('status') # type: ignore - if type(version_status) is not expected_str_type and version_status not in IDFToolVersion.STATUS_VALUES: + if not isinstance(version_status, str) and version_status not in IDFToolVersion.STATUS_VALUES: raise RuntimeError('tool {} version {} status is not one of {}', tool_name, version, IDFToolVersion.STATUS_VALUES) @@ -971,6 +982,8 @@ class IDFTool(object): tool_json['platform_overrides'] = overrides_array if self.options.strip_container_dirs: tool_json['strip_container_dirs'] = self.options.strip_container_dirs + if self.options.is_executable is False: + tool_json['is_executable'] = self.options.is_executable return tool_json @@ -1531,10 +1544,22 @@ def action_export(args): # type: ignore all_tools_found = True export_vars = {} paths_to_export = [] + + self_restart_cmd = f'{sys.executable} {__file__}{(" --tools-json " + args.tools_json) if args.tools_json else ""}' + self_restart_cmd = to_shell_specific_paths([self_restart_cmd])[0] + prefer_system_hint = '' if IDF_TOOLS_EXPORT_CMD else f' To use it, run \'{self_restart_cmd} export --prefer-system\'' + install_cmd = to_shell_specific_paths([IDF_TOOLS_INSTALL_CMD])[0] if IDF_TOOLS_INSTALL_CMD else self_restart_cmd + ' install' + for name, tool in tools_info.items(): if tool.get_install_type() == IDFTool.INSTALL_NEVER: continue tool.find_installed_versions() + version_to_use = tool.get_preferred_installed_version() + + if not tool.is_executable and version_to_use: + tool_export_vars = tool.get_export_vars(version_to_use) + export_vars = {**export_vars, **tool_export_vars} + continue if tool.version_in_path: if tool.version_in_path not in tool.versions: @@ -1558,20 +1583,6 @@ def action_export(args): # type: ignore warn('using a deprecated version of tool {} found in PATH: {}'.format(name, tool.version_in_path)) continue - self_restart_cmd = '{} {}{}'.format(sys.executable, __file__, - (' --tools-json ' + args.tools_json) if args.tools_json else '') - self_restart_cmd = to_shell_specific_paths([self_restart_cmd])[0] - - if IDF_TOOLS_EXPORT_CMD: - prefer_system_hint = '' - else: - prefer_system_hint = ' To use it, run \'{} export --prefer-system\''.format(self_restart_cmd) - - if IDF_TOOLS_INSTALL_CMD: - install_cmd = to_shell_specific_paths([IDF_TOOLS_INSTALL_CMD])[0] - else: - install_cmd = self_restart_cmd + ' install' - if not tool.versions_installed: if tool.get_install_type() == IDFTool.INSTALL_ALWAYS: all_tools_found = False @@ -1590,15 +1601,11 @@ def action_export(args): # type: ignore info('Not using an unsupported version of tool {} found in PATH: {}.'.format( tool.name, tool.version_in_path) + prefer_system_hint, f=sys.stderr) - version_to_use = tool.get_preferred_installed_version() export_paths = tool.get_export_paths(version_to_use) if export_paths: paths_to_export += export_paths tool_export_vars = tool.get_export_vars(version_to_use) - for k, v in tool_export_vars.items(): - old_v = os.environ.get(k) - if old_v is None or old_v != v: - export_vars[k] = v + export_vars = {**export_vars, **tool_export_vars} current_path = os.getenv('PATH') idf_python_env_path, idf_python_export_path, virtualenv_python, _ = get_python_env_path() diff --git a/tools/test_idf_tools/test_idf_tools.py b/tools/test_idf_tools/test_idf_tools.py index 44a6eb0ac2..daa689f1e9 100755 --- a/tools/test_idf_tools/test_idf_tools.py +++ b/tools/test_idf_tools/test_idf_tools.py @@ -45,6 +45,7 @@ XTENSA_ESP32S2_ELF = 'xtensa-esp32s2-elf' XTENSA_ESP32S3_ELF = 'xtensa-esp32s3-elf' XTENSA_ESP_GDB = 'xtensa-esp-elf-gdb' RISCV_ESP_GDB = 'riscv32-esp-elf-gdb' +ESP_ROM_ELFS = 'esp-rom-elfs' def get_version_dict(): @@ -70,6 +71,7 @@ XTENSA_ESP32S2_ELF_VERSION = version_dict[XTENSA_ESP32S2_ELF] XTENSA_ESP32S3_ELF_VERSION = version_dict[XTENSA_ESP32S3_ELF] XTENSA_ESP_GDB_VERSION = version_dict[XTENSA_ESP_GDB] RISCV_ESP_GDB_VERSION = version_dict[RISCV_ESP_GDB] +ESP_ROM_ELFS_VERSION = version_dict[ESP_ROM_ELFS] class TestUsage(unittest.TestCase): @@ -143,7 +145,7 @@ class TestUsage(unittest.TestCase): self.assertIn('* %s:' % XTENSA_ESP32S3_ELF, output) self.assertIn('- %s (recommended)' % XTENSA_ESP32S3_ELF_VERSION, output) - required_tools_installed = 8 + required_tools_installed = 9 output = self.run_idf_tools_with_action(['install']) self.assert_tool_installed(output, OPENOCD, OPENOCD_VERSION) self.assert_tool_installed(output, RISCV_ELF, RISCV_ELF_VERSION) @@ -153,6 +155,7 @@ class TestUsage(unittest.TestCase): self.assert_tool_installed(output, ESP32ULP, ESP32ULP_VERSION) self.assert_tool_installed(output, XTENSA_ESP_GDB, XTENSA_ESP_GDB_VERSION) self.assert_tool_installed(output, RISCV_ESP_GDB, RISCV_ESP_GDB_VERSION) + self.assert_tool_installed(output, ESP_ROM_ELFS, ESP_ROM_ELFS_VERSION) self.assertIn('Destination: {}'.format(os.path.join(self.temp_tools_dir, 'dist')), output) self.assertEqual(required_tools_installed, output.count('Done')) @@ -165,6 +168,7 @@ class TestUsage(unittest.TestCase): self.assertIn('version installed in tools directory: ' + XTENSA_ESP32S3_ELF_VERSION, output) self.assertIn('version installed in tools directory: ' + XTENSA_ESP_GDB_VERSION, output) self.assertIn('version installed in tools directory: ' + RISCV_ESP_GDB_VERSION, output) + self.assertIn('version installed in tools directory: ' + ESP_ROM_ELFS_VERSION, output) output = self.run_idf_tools_with_action(['export']) self.assertIn('%s/tools/esp32ulp-elf/%s/esp32ulp-elf/bin' % @@ -183,9 +187,11 @@ class TestUsage(unittest.TestCase): (self.temp_tools_dir, XTENSA_ESP_GDB_VERSION), output) self.assertIn('%s/tools/riscv32-esp-elf-gdb/%s/riscv32-esp-elf-gdb/bin' % (self.temp_tools_dir, RISCV_ESP_GDB_VERSION), output) + self.assertIn('%s/tools/esp-rom-elfs/%s/' % + (self.temp_tools_dir, ESP_ROM_ELFS_VERSION), output) def test_tools_for_esp32(self): - required_tools_installed = 4 + required_tools_installed = 5 output = self.run_idf_tools_with_action(['install', '--targets=esp32']) self.assert_tool_installed(output, XTENSA_ESP32_ELF, XTENSA_ESP32_ELF_VERSION) self.assert_tool_installed(output, OPENOCD, OPENOCD_VERSION) @@ -195,6 +201,7 @@ class TestUsage(unittest.TestCase): self.assert_tool_not_installed(output, XTENSA_ESP32S2_ELF, XTENSA_ESP32S2_ELF_VERSION) self.assert_tool_not_installed(output, XTENSA_ESP32S3_ELF, XTENSA_ESP32S3_ELF_VERSION) self.assert_tool_not_installed(output, RISCV_ESP_GDB, RISCV_ESP_GDB_VERSION) + self.assert_tool_installed(output, ESP_ROM_ELFS, ESP_ROM_ELFS_VERSION) self.assertIn('Destination: {}'.format(os.path.join(self.temp_tools_dir, 'dist')), output) self.assertEqual(required_tools_installed, output.count('Done')) @@ -203,6 +210,7 @@ class TestUsage(unittest.TestCase): self.assertIn('version installed in tools directory: ' + XTENSA_ESP32_ELF_VERSION, output) self.assertIn('version installed in tools directory: ' + OPENOCD_VERSION, output) self.assertIn('version installed in tools directory: ' + XTENSA_ESP_GDB_VERSION, output) + self.assertIn('version installed in tools directory: ' + ESP_ROM_ELFS_VERSION, output) output = self.run_idf_tools_with_action(['export']) self.assertIn('%s/tools/esp32ulp-elf/%s/esp32ulp-elf/bin' % @@ -221,9 +229,11 @@ class TestUsage(unittest.TestCase): (self.temp_tools_dir, XTENSA_ESP32S3_ELF_VERSION), output) self.assertNotIn('%s/tools/riscv32-esp-elf-gdb/%s/riscv32-esp-elf-gdb/bin' % (self.temp_tools_dir, RISCV_ESP_GDB_VERSION), output) + self.assertIn('%s/tools/esp-rom-elfs/%s/' % + (self.temp_tools_dir, ESP_ROM_ELFS_VERSION), output) def test_tools_for_esp32c3(self): - required_tools_installed = 3 + required_tools_installed = 4 output = self.run_idf_tools_with_action(['install', '--targets=esp32c3']) self.assert_tool_installed(output, OPENOCD, OPENOCD_VERSION) self.assert_tool_installed(output, RISCV_ELF, RISCV_ELF_VERSION) @@ -233,6 +243,7 @@ class TestUsage(unittest.TestCase): self.assert_tool_not_installed(output, XTENSA_ESP32S3_ELF, XTENSA_ESP32S3_ELF_VERSION) self.assert_tool_not_installed(output, ESP32ULP, ESP32ULP_VERSION) self.assert_tool_not_installed(output, XTENSA_ESP_GDB_VERSION, XTENSA_ESP_GDB_VERSION) + self.assert_tool_installed(output, ESP_ROM_ELFS, ESP_ROM_ELFS_VERSION) self.assertIn('Destination: {}'.format(os.path.join(self.temp_tools_dir, 'dist')), output) self.assertEqual(required_tools_installed, output.count('Done')) @@ -240,6 +251,7 @@ class TestUsage(unittest.TestCase): self.assertIn('version installed in tools directory: ' + OPENOCD_VERSION, output) self.assertIn('version installed in tools directory: ' + RISCV_ELF_VERSION, output) self.assertIn('version installed in tools directory: ' + RISCV_ESP_GDB_VERSION, output) + self.assertIn('version installed in tools directory: ' + ESP_ROM_ELFS_VERSION, output) output = self.run_idf_tools_with_action(['export']) self.assertIn('%s/tools/openocd-esp32/%s/openocd-esp32/bin' % @@ -256,15 +268,18 @@ class TestUsage(unittest.TestCase): (self.temp_tools_dir, XTENSA_ESP32S3_ELF_VERSION), output) self.assertNotIn('%s/tools/xtensa-esp-elf-gdb/%s/xtensa-esp-elf-gdb/bin' % (self.temp_tools_dir, XTENSA_ESP_GDB_VERSION), output) + self.assertIn('%s/tools/esp-rom-elfs/%s/' % + (self.temp_tools_dir, ESP_ROM_ELFS_VERSION), output) def test_tools_for_esp32s2(self): - required_tools_installed = 5 + required_tools_installed = 6 output = self.run_idf_tools_with_action(['install', '--targets=esp32s2']) self.assert_tool_installed(output, XTENSA_ESP32S2_ELF, XTENSA_ESP32S2_ELF_VERSION) self.assert_tool_installed(output, OPENOCD, OPENOCD_VERSION) self.assert_tool_installed(output, RISCV_ELF, RISCV_ELF_VERSION) self.assert_tool_installed(output, ESP32ULP, ESP32ULP_VERSION) self.assert_tool_installed(output, XTENSA_ESP_GDB, XTENSA_ESP_GDB_VERSION) + self.assert_tool_installed(output, ESP_ROM_ELFS, ESP_ROM_ELFS_VERSION) self.assert_tool_not_installed(output, RISCV_ESP_GDB, RISCV_ESP_GDB_VERSION) self.assert_tool_not_installed(output, XTENSA_ESP32_ELF, XTENSA_ESP32_ELF_VERSION) self.assert_tool_not_installed(output, XTENSA_ESP32S3_ELF, XTENSA_ESP32S3_ELF_VERSION) @@ -275,6 +290,7 @@ class TestUsage(unittest.TestCase): self.assertIn('version installed in tools directory: ' + OPENOCD_VERSION, output) self.assertIn('version installed in tools directory: ' + XTENSA_ESP32S2_ELF_VERSION, output) self.assertIn('version installed in tools directory: ' + XTENSA_ESP_GDB_VERSION, output) + self.assertIn('version installed in tools directory: ' + ESP_ROM_ELFS_VERSION, output) output = self.run_idf_tools_with_action(['export']) self.assertIn('%s/tools/xtensa-esp32s2-elf/%s/xtensa-esp32s2-elf/bin' % @@ -293,15 +309,18 @@ class TestUsage(unittest.TestCase): (self.temp_tools_dir, XTENSA_ESP_GDB_VERSION), output) self.assertNotIn('%s/tools/riscv32-esp-elf-gdb/%s/riscv32-esp-elf-gdb/bin' % (self.temp_tools_dir, RISCV_ESP_GDB_VERSION), output) + self.assertIn('%s/tools/esp-rom-elfs/%s/' % + (self.temp_tools_dir, ESP_ROM_ELFS_VERSION), output) def test_tools_for_esp32s3(self): - required_tools_installed = 5 + required_tools_installed = 6 output = self.run_idf_tools_with_action(['install', '--targets=esp32s3']) self.assert_tool_installed(output, XTENSA_ESP32S3_ELF, XTENSA_ESP32S3_ELF_VERSION) self.assert_tool_installed(output, OPENOCD, OPENOCD_VERSION) self.assert_tool_installed(output, RISCV_ELF, RISCV_ELF_VERSION) self.assert_tool_installed(output, ESP32ULP, ESP32ULP_VERSION) self.assert_tool_installed(output, XTENSA_ESP_GDB, XTENSA_ESP_GDB_VERSION) + self.assert_tool_installed(output, ESP_ROM_ELFS, ESP_ROM_ELFS_VERSION) self.assert_tool_not_installed(output, RISCV_ESP_GDB, RISCV_ESP_GDB_VERSION) self.assert_tool_not_installed(output, XTENSA_ESP32_ELF, XTENSA_ESP32_ELF_VERSION) self.assert_tool_not_installed(output, XTENSA_ESP32S2_ELF, XTENSA_ESP32S2_ELF_VERSION) @@ -313,6 +332,7 @@ class TestUsage(unittest.TestCase): self.assertIn('version installed in tools directory: ' + XTENSA_ESP32S3_ELF_VERSION, output) self.assertIn('version installed in tools directory: ' + XTENSA_ESP_GDB_VERSION, output) self.assertIn('version installed in tools directory: ' + RISCV_ESP_GDB_VERSION, output) + self.assertIn('version installed in tools directory: ' + ESP_ROM_ELFS_VERSION, output) output = self.run_idf_tools_with_action(['export']) self.assertIn('%s/tools/openocd-esp32/%s/openocd-esp32/bin' % @@ -331,6 +351,8 @@ class TestUsage(unittest.TestCase): (self.temp_tools_dir, XTENSA_ESP_GDB_VERSION), output) self.assertNotIn('%s/tools/riscv32-esp-elf-gdb/%s/riscv32-esp-elf-gdb/bin' % (self.temp_tools_dir, RISCV_ESP_GDB_VERSION), output) + self.assertIn('%s/tools/esp-rom-elfs/%s/' % + (self.temp_tools_dir, ESP_ROM_ELFS_VERSION), output) def test_uninstall_option(self): self.run_idf_tools_with_action(['install', '--targets=esp32,esp32c3']) diff --git a/tools/tools.json b/tools/tools.json index 1b7dcbeec3..cb0bd2b8ba 100644 --- a/tools/tools.json +++ b/tools/tools.json @@ -949,6 +949,40 @@ } } ] + }, + { + "description": "ESP ROM ELFs", + "export_paths": [ + [ + "" + ] + ], + "export_vars": { + "ESP_ROM_ELF_DIR": "${TOOL_PATH}/" + }, + "info_url": "https://github.com/espressif/esp-rom-elfs", + "install": "always", + "is_executable": false, + "license": "Apache-2.0", + "name": "esp-rom-elfs", + "supported_targets": [ + "all" + ], + "version_cmd": [ + "" + ], + "version_regex": "", + "versions": [ + { + "any": { + "sha256": "add4bedbdd950c8409ff45bbf5610316e7d14c4635ea6906f057f2183ab3e3e9", + "size": 2454730, + "url": "https://github.com/espressif/esp-rom-elfs/releases/download/20220823/esp-rom-elfs-20220823.tar.gz" + }, + "name": "20220823", + "status": "recommended" + } + ] } ], "version": 1 diff --git a/tools/tools_schema.json b/tools/tools_schema.json index f09044d622..51775970f6 100644 --- a/tools/tools_schema.json +++ b/tools/tools_schema.json @@ -47,6 +47,10 @@ "$ref": "#/definitions/installRequirementInfo", "description": "If 'always', the tool will be installed by default. If 'on_request', tool will be installed when specifically requested. If 'never', tool will not be considered for installation." }, + "is_executable": { + "description": "If false - tool does not contain executables. The version will not be checked but export_vars applied.", + "type": "boolean" + }, "license": { "description": "License name. Use SPDX license identifier if it exists, short name of the license otherwise.", "type": "string" From 2e9f175ae538bff946b49d61718e46bff1792b39 Mon Sep 17 00:00:00 2001 From: Alexey Lapshin Date: Sat, 20 Aug 2022 13:36:14 +0400 Subject: [PATCH 2/4] tools: fixed elf symbols load if gdbinit specified ROM and bootloader symbols add to use in GDB (via 'idf.py gdb') --- components/bootloader/project_include.cmake | 3 +- tools/cmake/project_description.json.in | 1 + tools/idf_py_actions/debug_ext.py | 180 ++++++++++++++++---- tools/idf_py_actions/roms.json | 47 +++++ tools/idf_py_actions/roms_schema.json | 29 ++++ tools/test_idf_py/test_idf_py.py | 47 ++++- 6 files changed, 276 insertions(+), 31 deletions(-) create mode 100644 tools/idf_py_actions/roms.json create mode 100644 tools/idf_py_actions/roms_schema.json diff --git a/components/bootloader/project_include.cmake b/components/bootloader/project_include.cmake index 5dcb18cd48..2f3b3eb8a8 100644 --- a/components/bootloader/project_include.cmake +++ b/components/bootloader/project_include.cmake @@ -11,8 +11,9 @@ endif() # idf_build_get_property(build_dir BUILD_DIR) set(BOOTLOADER_BUILD_DIR "${build_dir}/bootloader") +set(BOOTLOADER_ELF_FILE "${BOOTLOADER_BUILD_DIR}/bootloader.elf") set(bootloader_binary_files - "${BOOTLOADER_BUILD_DIR}/bootloader.elf" + "${BOOTLOADER_ELF_FILE}" "${BOOTLOADER_BUILD_DIR}/bootloader.bin" "${BOOTLOADER_BUILD_DIR}/bootloader.map" ) diff --git a/tools/cmake/project_description.json.in b/tools/cmake/project_description.json.in index 2145d0fa5f..8ed6c197ce 100644 --- a/tools/cmake/project_description.json.in +++ b/tools/cmake/project_description.json.in @@ -4,6 +4,7 @@ "build_dir": "${BUILD_DIR}", "config_file": "${SDKCONFIG}", "config_defaults": "${SDKCONFIG_DEFAULTS}", + "bootloader_elf": "${BOOTLOADER_ELF_FILE}", "app_elf": "${PROJECT_EXECUTABLE}", "app_bin": "${PROJECT_BIN}", "git_revision": "${IDF_VER}", diff --git a/tools/idf_py_actions/debug_ext.py b/tools/idf_py_actions/debug_ext.py index 53bdfb3d36..734b19fa16 100644 --- a/tools/idf_py_actions/debug_ext.py +++ b/tools/idf_py_actions/debug_ext.py @@ -4,10 +4,12 @@ import json import os import re import shlex +import shutil import subprocess import sys import threading import time +from textwrap import indent from threading import Thread from typing import Any, Dict, List, Optional @@ -16,6 +18,45 @@ from idf_py_actions.errors import FatalError from idf_py_actions.tools import PropertyDict, ensure_build_directory PYTHON = sys.executable +ESP_ROM_INFO_FILE = 'roms.json' +GDBINIT_PYTHON_TEMPLATE = ''' +# Add Python GDB extensions +python +import sys +sys.path = {sys_path} +import freertos_gdb +end +''' +GDBINIT_PYTHON_NOT_SUPPORTED = ''' +# Python scripting is not supported in this copy of GDB. +# Please make sure that your Python distribution contains Python shared library. +''' +GDBINIT_BOOTLOADER_ADD_SYMBOLS = ''' +# Load bootloader symbols +set confirm off + add-symbol-file {boot_elf} +set confirm on +''' +GDBINIT_BOOTLOADER_NOT_FOUND = ''' +# Bootloader elf was not found +''' +GDBINIT_APP_ADD_SYMBOLS = ''' +# Load application file +file {app_elf} +''' +GDBINIT_CONNECT = ''' +# Connect to the default openocd-esp port and break on app_main() +target remote :3333 +monitor reset halt +flushregs +thbreak app_main +continue +''' +GDBINIT_MAIN = ''' +source {py_extensions} +source {symbols} +source {connect} +''' def action_extensions(base_actions: Dict, project_path: str) -> Dict: @@ -91,22 +132,111 @@ def action_extensions(base_actions: Dict, project_path: str) -> Dict: # execute simple python command to check is it supported return subprocess.run([gdb, '--batch-silent', '--ex', 'python import os'], stderr=subprocess.DEVNULL).returncode == 0 - def create_local_gdbinit(gdb: str, gdbinit: str, elf_file: str) -> None: - with open(gdbinit, 'w') as f: + def get_normalized_path(path: str) -> str: + if os.name == 'nt': + return os.path.normpath(path).replace('\\','\\\\') + return path + + def get_rom_if_condition_str(date_addr: int, date_str: str) -> str: + r = [] + for i in range(0, len(date_str), 4): + value = hex(int.from_bytes(bytes(date_str[i:i + 4], 'utf-8'), 'little')) + r.append(f'(*(int*) {hex(date_addr + i)}) == {value}') + return 'if ' + ' && '.join(r) + + def generate_gdbinit_rom_add_symbols(target: str) -> str: + base_ident = ' ' + rom_elfs_dir = os.getenv('ESP_ROM_ELF_DIR') + if not rom_elfs_dir: + raise FatalError('ESP_ROM_ELF_DIR environment variable is not defined. Please try to run IDF "install" and "export" scripts.') + with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), ESP_ROM_INFO_FILE), 'r') as f: + roms = json.load(f) + if target not in roms: + msg_body = f'Target "{target}" was not found in "{ESP_ROM_INFO_FILE}". Please check IDF integrity.' + if os.getenv('ESP_IDF_GDB_TESTING'): + raise FatalError(msg_body) + print(f'Warning: {msg_body}') + return f'# {msg_body}' + r = ['', f'# Load {target} ROM ELF symbols'] + is_one_revision = len(roms[target]) == 1 + if not is_one_revision: + r.append('define target hookpost-remote') + r.append('set confirm off') + # Workaround for reading ROM data on xtensa chips + # This should be deleted after the new openocd-esp release (newer than v0.11.0-esp32-20220706) + xtensa_chips = ['esp32', 'esp32s2', 'esp32s3'] + if target in xtensa_chips: + r.append('monitor xtensa set_permissive 1') + # Since GDB does not have 'else if' statement than we use nested 'if..else' instead. + for i, k in enumerate(roms[target], 1): + indent_str = base_ident * i + rom_file = get_normalized_path(os.path.join(rom_elfs_dir, f'{target}_rev{k["rev"]}_rom.elf')) + build_date_addr = int(k['build_date_str_addr'], base=16) + r.append(indent(f'# if $_streq((char *) {hex(build_date_addr)}, "{k["build_date_str"]}")', indent_str)) + r.append(indent(get_rom_if_condition_str(build_date_addr, k['build_date_str']), indent_str)) + r.append(indent(f'add-symbol-file {rom_file}', indent_str + base_ident)) + r.append(indent('else', indent_str)) + if i == len(roms[target]): + # In case no one known ROM ELF fits - print error and exit with error code 1 + indent_str += base_ident + msg_body = f'unknown {target} ROM revision.' + if os.getenv('ESP_IDF_GDB_TESTING'): + r.append(indent(f'echo Error: {msg_body}\\n', indent_str)) + r.append(indent('quit 1', indent_str)) + else: + r.append(indent(f'echo Warning: {msg_body}\\n', indent_str)) + # Close 'else' operators + for i in range(len(roms[target]), 0, -1): + r.append(indent('end', base_ident * i)) + if target in xtensa_chips: + r.append('monitor xtensa set_permissive 0') + r.append('set confirm on') + if not is_one_revision: + r.append('end') + r.append('') + return os.linesep.join(r) + raise FatalError(f'{ESP_ROM_INFO_FILE} file not found. Please check IDF integrity.') + + def generate_gdbinit_files(gdb: str, gdbinit: Optional[str], project_desc: Dict[str, Any]) -> None: + app_elf = get_normalized_path(os.path.join(project_desc['build_dir'], project_desc['app_elf'])) + if not os.path.exists(app_elf): + raise FatalError('ELF file not found. You need to build & flash the project before running debug targets') + + # Recreate empty 'gdbinit' directory + gdbinit_dir = os.path.join(project_desc['build_dir'], 'gdbinit') + if os.path.isfile(gdbinit_dir): + os.remove(gdbinit_dir) + elif os.path.isdir(gdbinit_dir): + shutil.rmtree(gdbinit_dir) + os.mkdir(gdbinit_dir) + + # Prepare gdbinit for Python GDB extensions import + py_extensions = os.path.join(gdbinit_dir, 'py_extensions') + with open(py_extensions, 'w') as f: if is_gdb_with_python(gdb): - f.write('python\n') - f.write('import sys\n') - f.write(f'sys.path = {sys.path}\n') - f.write('import freertos_gdb\n') - f.write('end\n') - if os.name == 'nt': - elf_file = elf_file.replace('\\','\\\\') - f.write('file {}\n'.format(elf_file)) - f.write('target remote :3333\n') - f.write('mon reset halt\n') - f.write('flushregs\n') - f.write('thb app_main\n') - f.write('c\n') + f.write(GDBINIT_PYTHON_TEMPLATE.format(sys_path=sys.path)) + else: + f.write(GDBINIT_PYTHON_NOT_SUPPORTED) + + # Prepare gdbinit for related ELFs symbols load + symbols = os.path.join(gdbinit_dir, 'symbols') + with open(symbols, 'w') as f: + boot_elf = get_normalized_path(project_desc['bootloader_elf']) if 'bootloader_elf' in project_desc else None + if boot_elf and os.path.exists(boot_elf): + f.write(GDBINIT_BOOTLOADER_ADD_SYMBOLS.format(boot_elf=boot_elf)) + else: + f.write(GDBINIT_BOOTLOADER_NOT_FOUND) + f.write(generate_gdbinit_rom_add_symbols(project_desc['target'])) + f.write(GDBINIT_APP_ADD_SYMBOLS.format(app_elf=app_elf)) + + # Generate the gdbinit for target connect if no custom gdbinit is present + if not gdbinit: + gdbinit = os.path.join(gdbinit_dir, 'connect') + with open(gdbinit, 'w') as f: + f.write(GDBINIT_CONNECT) + + with open(os.path.join(gdbinit_dir, 'gdbinit'), 'w') as f: + f.write(GDBINIT_MAIN.format(py_extensions=py_extensions, symbols=symbols, connect=gdbinit)) def debug_cleanup() -> None: print('cleaning up debug targets') @@ -191,7 +321,8 @@ def action_extensions(base_actions: Dict, project_path: str) -> Dict: processes['openocd_outfile_name'] = openocd_out_name print('OpenOCD started as a background task {}'.format(process.pid)) - def get_gdb_args(gdbinit: str, project_desc: Dict[str, Any]) -> List: + def get_gdb_args(project_desc: Dict[str, Any]) -> List: + gdbinit = os.path.join(project_desc['build_dir'], 'gdbinit', 'gdbinit') args = ['-x={}'.format(gdbinit)] debug_prefix_gdbinit = project_desc.get('debug_prefix_map_gdbinit') if debug_prefix_gdbinit: @@ -205,16 +336,14 @@ def action_extensions(base_actions: Dict, project_path: str) -> Dict: project_desc = get_project_desc(args, ctx) local_dir = project_desc['build_dir'] gdb = project_desc['monitor_toolprefix'] + 'gdb' - if gdbinit is None: - gdbinit = os.path.join(local_dir, 'gdbinit') - create_local_gdbinit(gdb, gdbinit, os.path.join(args.build_dir, project_desc['app_elf'])) + generate_gdbinit_files(gdb, gdbinit, project_desc) # this is a workaround for gdbgui # gdbgui is using shlex.split for the --gdb-args option. When the input is: # - '"-x=foo -x=bar"', would return ['foo bar'] # - '-x=foo', would return ['-x', 'foo'] and mess up the former option '--gdb-args' # so for one item, use extra double quotes. for more items, use no extra double quotes. - gdb_args_list = get_gdb_args(gdbinit, project_desc) + gdb_args_list = get_gdb_args(project_desc) gdb_args = '"{}"'.format(' '.join(gdb_args_list)) if len(gdb_args_list) == 1 else ' '.join(gdb_args_list) args = ['gdbgui', '-g', gdb, '--gdb-args', gdb_args] print(args) @@ -286,16 +415,9 @@ def action_extensions(base_actions: Dict, project_path: str) -> Dict: watch_openocd.start() processes['threads_to_join'].append(watch_openocd) project_desc = get_project_desc(args, ctx) - - elf_file = os.path.join(args.build_dir, project_desc['app_elf']) - if not os.path.exists(elf_file): - raise FatalError('ELF file not found. You need to build & flash the project before running debug targets', ctx) gdb = project_desc['monitor_toolprefix'] + 'gdb' - local_dir = project_desc['build_dir'] - if gdbinit is None: - gdbinit = os.path.join(local_dir, 'gdbinit') - create_local_gdbinit(gdb, gdbinit, elf_file) - args = [gdb, *get_gdb_args(gdbinit, project_desc)] + generate_gdbinit_files(gdb, gdbinit, project_desc) + args = [gdb, *get_gdb_args(project_desc)] if gdb_tui is not None: args += ['-tui'] t = Thread(target=run_gdb, args=(args,)) diff --git a/tools/idf_py_actions/roms.json b/tools/idf_py_actions/roms.json new file mode 100644 index 0000000000..16df8b7c18 --- /dev/null +++ b/tools/idf_py_actions/roms.json @@ -0,0 +1,47 @@ +{ + "esp32": [ + { + "rev": 0, + "build_date_str_addr": "0x3ff9ea80", + "build_date_str": "Jun 8 2016" + }, + { + "rev": 3, + "build_date_str_addr": "0x3ff9e986", + "build_date_str": "Jul 29 2019" + } + ], + "esp32s2": [ + { + "rev": 0, + "build_date_str_addr": "0x3ffaf34b", + "build_date_str": "Oct 25 2019" + } + ], + "esp32s3": [ + { + "rev": 0, + "build_date_str_addr": "0x3ff194ad", + "build_date_str": "Mar 1 2021" + } + ], + "esp32c2": [ + { + "rev": 0, + "build_date_str_addr": "0x3ff47874", + "build_date_str": "Jan 27 2022" + } + ], + "esp32c3": [ + { + "rev": 0, + "build_date_str_addr": "0x3ff1b878", + "build_date_str": "Sep 18 2020" + }, + { + "rev": 3, + "build_date_str_addr": "0x3ff1a374", + "build_date_str": "Feb 7 2021" + } + ] +} diff --git a/tools/idf_py_actions/roms_schema.json b/tools/idf_py_actions/roms_schema.json new file mode 100644 index 0000000000..b4ab4c5f47 --- /dev/null +++ b/tools/idf_py_actions/roms_schema.json @@ -0,0 +1,29 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "patternProperties": { + "^esp32.*$": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "object", + "properties": { + "rev": { + "type": "integer", + "minimum": 0, + "description": "Chip revision/ROM revision number" + }, + "build_date_str_addr": { + "type": "string", + "description": "The ROM build date string address to compare between ROM elf file and chip ROM memory", + "pattern": "^0x[0-9a-fA-F]{8}$" + } + }, + "required": ["rev", "build_date_str_addr"] + } + } + ] + } + } +} diff --git a/tools/test_idf_py/test_idf_py.py b/tools/test_idf_py/test_idf_py.py index 4a14faa687..edc63bfec7 100755 --- a/tools/test_idf_py/test_idf_py.py +++ b/tools/test_idf_py/test_idf_py.py @@ -9,7 +9,9 @@ import subprocess import sys from unittest import TestCase, main, mock +import elftools.common.utils as ecu import jsonschema +from elftools.elf.elffile import ELFFile try: from StringIO import StringIO @@ -25,7 +27,8 @@ except ImportError: current_dir = os.path.dirname(os.path.realpath(__file__)) idf_py_path = os.path.join(current_dir, '..', 'idf.py') extension_path = os.path.join(current_dir, 'test_idf_extensions', 'test_ext') -link_path = os.path.join(current_dir, '..', 'idf_py_actions', 'test_ext') +py_actions_path = os.path.join(current_dir, '..', 'idf_py_actions') +link_path = os.path.join(py_actions_path, 'test_ext') class TestWithoutExtensions(TestCase): @@ -246,5 +249,47 @@ class TestHelpOutput(TestWithoutExtensions): action_test(['idf.py', 'help', '--json', '--add-options'], schema_json) +class TestROMs(TestWithoutExtensions): + def get_string_from_elf_by_addr(self, filename: str, address: int) -> str: + result = '' + with open(filename, 'rb') as stream: + elf_file = ELFFile(stream) + ro = elf_file.get_section_by_name('.rodata') + ro_addr_delta = ro['sh_addr'] - ro['sh_offset'] + cstring = ecu.parse_cstring_from_stream(ro.stream, address - ro_addr_delta) + if cstring: + result = str(cstring.decode('utf-8')) + return result + + def test_roms_validate_json(self): + with open(os.path.join(py_actions_path, 'roms.json'), 'r') as f: + roms_json = json.load(f) + + with open(os.path.join(py_actions_path, 'roms_schema.json'), 'r') as f: + schema_json = json.load(f) + jsonschema.validate(roms_json, schema_json) + + def test_roms_check_supported_chips(self): + from idf_py_actions.constants import SUPPORTED_TARGETS + with open(os.path.join(py_actions_path, 'roms.json'), 'r') as f: + roms_json = json.load(f) + for chip in SUPPORTED_TARGETS: + self.assertTrue(chip in roms_json, msg=f'Have no ROM data for chip {chip}') + + def test_roms_validate_build_date(self): + sys.path.append(py_actions_path) + + rom_elfs_dir = os.getenv('ESP_ROM_ELF_DIR') + with open(os.path.join(py_actions_path, 'roms.json'), 'r') as f: + roms_json = json.load(f) + + for chip in roms_json: + for k in roms_json[chip]: + rom_file = os.path.join(rom_elfs_dir, f'{chip}_rev{k["rev"]}_rom.elf') + build_date_str = self.get_string_from_elf_by_addr(rom_file, int(k['build_date_str_addr'], base=16)) + self.assertTrue(len(build_date_str) == 11) + self.assertTrue(build_date_str == k['build_date_str']) + + if __name__ == '__main__': main() From 75613678086a90a518d7260e37a7c2dfa16fd1bb Mon Sep 17 00:00:00 2001 From: Alexey Lapshin Date: Thu, 8 Sep 2022 00:45:04 +0400 Subject: [PATCH 3/4] tools: add test for ROM symbols in GDB --- .gitlab/ci/build.yml | 1 + tools/idf_py_actions/constants.py | 8 +++ tools/idf_py_actions/debug_ext.py | 34 +++++++----- tools/test_apps/system/gdb/CMakeLists.txt | 6 +++ tools/test_apps/system/gdb/README.md | 6 +++ .../test_apps/system/gdb/main/CMakeLists.txt | 3 ++ .../system/gdb/main/hello_world_main.c | 46 ++++++++++++++++ tools/test_apps/system/gdb/pytest_gdb.py | 53 +++++++++++++++++++ 8 files changed, 143 insertions(+), 14 deletions(-) create mode 100644 tools/test_apps/system/gdb/CMakeLists.txt create mode 100644 tools/test_apps/system/gdb/README.md create mode 100644 tools/test_apps/system/gdb/main/CMakeLists.txt create mode 100644 tools/test_apps/system/gdb/main/hello_world_main.c create mode 100644 tools/test_apps/system/gdb/pytest_gdb.py diff --git a/.gitlab/ci/build.yml b/.gitlab/ci/build.yml index 80b3ae9fec..db483fc73c 100644 --- a/.gitlab/ci/build.yml +++ b/.gitlab/ci/build.yml @@ -31,6 +31,7 @@ - "**/build*/flash_project_args" - "**/build*/config/sdkconfig.json" - "**/build*/bootloader/*.bin" + - "**/build*/bootloader/*.elf" - "**/build*/partition_table/*.bin" - $SIZE_INFO_LOCATION when: always diff --git a/tools/idf_py_actions/constants.py b/tools/idf_py_actions/constants.py index 18ae271592..f225534da8 100644 --- a/tools/idf_py_actions/constants.py +++ b/tools/idf_py_actions/constants.py @@ -34,3 +34,11 @@ URL_TO_DOC = 'https://docs.espressif.com/projects/esp-idf' SUPPORTED_TARGETS = ['esp32', 'esp32s2', 'esp32c3', 'esp32s3', 'esp32c2'] PREVIEW_TARGETS = ['linux', 'esp32h2'] + +OPENOCD_TAGET_CONFIG_DEFAULT = '-f interface/ftdi/esp32_devkitj_v1.cfg -f target/{target}.cfg' +OPENOCD_TAGET_CONFIG: Dict[str, str] = { + 'esp32': '-f board/esp32-wrover-kit-3.3v.cfg', + 'esp32s2': '-f board/esp32s2-kaluga-1.cfg', + 'esp32c3': '-f board/esp32c3-builtin.cfg', + 'esp32s3': '-f board/esp32s3-builtin.cfg', +} diff --git a/tools/idf_py_actions/debug_ext.py b/tools/idf_py_actions/debug_ext.py index 734b19fa16..313adb18db 100644 --- a/tools/idf_py_actions/debug_ext.py +++ b/tools/idf_py_actions/debug_ext.py @@ -14,6 +14,7 @@ from threading import Thread from typing import Any, Dict, List, Optional from click.core import Context +from idf_py_actions.constants import OPENOCD_TAGET_CONFIG, OPENOCD_TAGET_CONFIG_DEFAULT from idf_py_actions.errors import FatalError from idf_py_actions.tools import PropertyDict, ensure_build_directory @@ -59,6 +60,11 @@ source {connect} ''' +def get_openocd_arguments(target: str) -> str: + default_args = OPENOCD_TAGET_CONFIG_DEFAULT.format(target=target) + return str(OPENOCD_TAGET_CONFIG.get(target, default_args)) + + def action_extensions(base_actions: Dict, project_path: str) -> Dict: OPENOCD_OUT_FILE = 'openocd_out.txt' GDBGUI_OUT_FILE = 'gdbgui_out.txt' @@ -286,12 +292,6 @@ def action_extensions(base_actions: Dict, project_path: str) -> Dict: """ Execute openocd as external tool """ - OPENOCD_TAGET_CONFIG = { - 'esp32': '-f board/esp32-wrover-kit-3.3v.cfg', - 'esp32s2': '-f board/esp32s2-kaluga-1.cfg', - 'esp32c3': '-f board/esp32c3-builtin.cfg', - 'esp32s3': '-f board/esp32s3-builtin.cfg', - } if os.getenv('OPENOCD_SCRIPTS') is None: raise FatalError('OPENOCD_SCRIPTS not found in the environment: Please run export.sh/export.bat', ctx) openocd_arguments = os.getenv('OPENOCD_COMMANDS') if openocd_commands is None else openocd_commands @@ -299,8 +299,7 @@ def action_extensions(base_actions: Dict, project_path: str) -> Dict: if openocd_arguments is None: # use default value if commands not defined in the environment nor command line target = project_desc['target'] - default_args = '-f interface/ftdi/esp32_devkitj_v1.cfg -f target/{}.cfg'.format(target) - openocd_arguments = OPENOCD_TAGET_CONFIG.get(target, default_args) + openocd_arguments = get_openocd_arguments(target) print('Note: OpenOCD cfg not found (via env variable OPENOCD_COMMANDS nor as a --openocd-commands argument)\n' 'OpenOCD arguments default to: "{}"'.format(openocd_arguments)) # script directory is taken from the environment by OpenOCD, update only if command line arguments to override @@ -405,9 +404,9 @@ def action_extensions(base_actions: Dict, project_path: str) -> Dict: """ Synchronous GDB target with text ui mode """ - gdb(action, ctx, args, 1, gdbinit, require_openocd) + gdb(action, ctx, args, False, 1, gdbinit, require_openocd) - def gdb(action: str, ctx: Context, args: PropertyDict, gdb_tui: Optional[int], gdbinit: Optional[str], require_openocd: bool) -> None: + def gdb(action: str, ctx: Context, args: PropertyDict, batch: bool, gdb_tui: Optional[int], gdbinit: Optional[str], require_openocd: bool) -> None: """ Synchronous GDB target """ @@ -420,6 +419,8 @@ def action_extensions(base_actions: Dict, project_path: str) -> Dict: args = [gdb, *get_gdb_args(project_desc)] if gdb_tui is not None: args += ['-tui'] + if batch: + args += ['--batch'] t = Thread(target=run_gdb, args=(args,)) t.start() while True: @@ -476,12 +477,17 @@ def action_extensions(base_actions: Dict, project_path: str) -> Dict: 'callback': gdb, 'help': 'Run the GDB.', 'options': [ + { + 'names': ['--batch'], + 'help': ('exit after processing gdbinit.\n'), + 'hidden': True, + 'is_flag': True, + 'default': False, + }, { 'names': ['--gdb-tui', '--gdb_tui'], - 'help': - ('run gdb in TUI mode\n'), - 'default': - None, + 'help': ('run gdb in TUI mode\n'), + 'default': None, }, gdbinit, fail_if_openocd_failed ], 'order_dependencies': ['all', 'flash'], diff --git a/tools/test_apps/system/gdb/CMakeLists.txt b/tools/test_apps/system/gdb/CMakeLists.txt new file mode 100644 index 0000000000..1c0bd70903 --- /dev/null +++ b/tools/test_apps/system/gdb/CMakeLists.txt @@ -0,0 +1,6 @@ +# The following lines of boilerplate have to be in your project's +# CMakeLists in this exact order for cmake to work correctly +cmake_minimum_required(VERSION 3.16) + +include($ENV{IDF_PATH}/tools/cmake/project.cmake) +project(gdb) diff --git a/tools/test_apps/system/gdb/README.md b/tools/test_apps/system/gdb/README.md new file mode 100644 index 0000000000..0b4b22e8c3 --- /dev/null +++ b/tools/test_apps/system/gdb/README.md @@ -0,0 +1,6 @@ +| Supported Targets | ESP32 | ESP32-C2 | ESP32-C3 | ESP32-S2 | ESP32-S3 | +| ----------------- | ----- | -------- | -------- | -------- | -------- | + +# IDF GDB test application + +This project tests if `idf.py gdb` works correct diff --git a/tools/test_apps/system/gdb/main/CMakeLists.txt b/tools/test_apps/system/gdb/main/CMakeLists.txt new file mode 100644 index 0000000000..69def7bc1d --- /dev/null +++ b/tools/test_apps/system/gdb/main/CMakeLists.txt @@ -0,0 +1,3 @@ +idf_component_register(SRCS "hello_world_main.c" + INCLUDE_DIRS "") +target_compile_options(${COMPONENT_LIB} PRIVATE "-Wno-format") diff --git a/tools/test_apps/system/gdb/main/hello_world_main.c b/tools/test_apps/system/gdb/main/hello_world_main.c new file mode 100644 index 0000000000..915866a4c2 --- /dev/null +++ b/tools/test_apps/system/gdb/main/hello_world_main.c @@ -0,0 +1,46 @@ +/* + * SPDX-FileCopyrightText: 2010-2022 Espressif Systems (Shanghai) CO LTD + * + * SPDX-License-Identifier: CC0-1.0 + */ + +#include +#include "sdkconfig.h" +#include "freertos/FreeRTOS.h" +#include "freertos/task.h" +#include "esp_chip_info.h" +#include "esp_flash.h" + +void app_main(void) +{ + printf("Hello world!\n"); + + /* Print chip information */ + esp_chip_info_t chip_info; + uint32_t flash_size; + esp_chip_info(&chip_info); + printf("This is %s chip with %d CPU core(s), WiFi%s%s, ", + CONFIG_IDF_TARGET, + chip_info.cores, + (chip_info.features & CHIP_FEATURE_BT) ? "/BT" : "", + (chip_info.features & CHIP_FEATURE_BLE) ? "/BLE" : ""); + + printf("silicon revision %d, ", chip_info.revision); + if(esp_flash_get_size(NULL, &flash_size) != ESP_OK) { + printf("Get flash size failed"); + return; + } + + printf("%uMB %s flash\n", flash_size / (1024 * 1024), + (chip_info.features & CHIP_FEATURE_EMB_FLASH) ? "embedded" : "external"); + + printf("Minimum free heap size: %d bytes\n", esp_get_minimum_free_heap_size()); + + for (int i = 10; i >= 0; i--) { + printf("Restarting in %d seconds...\n", i); + vTaskDelay(1000 / portTICK_PERIOD_MS); + } + printf("Restarting now.\n"); + fflush(stdout); + esp_restart(); +} diff --git a/tools/test_apps/system/gdb/pytest_gdb.py b/tools/test_apps/system/gdb/pytest_gdb.py new file mode 100644 index 0000000000..0bb16d8e81 --- /dev/null +++ b/tools/test_apps/system/gdb/pytest_gdb.py @@ -0,0 +1,53 @@ +# SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Unlicense OR CC0-1.0 +import os +import re +import signal +import subprocess +import sys + +import pexpect +import pytest +from pytest_embedded import Dut + +try: + from idf_py_actions.debug_ext import get_openocd_arguments +except ModuleNotFoundError: + sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', '..')) + from idf_py_actions.debug_ext import get_openocd_arguments + + +@pytest.mark.supported_targets +@pytest.mark.test_jtag_arm +def test_idf_gdb(dut: Dut) -> None: + # Need to wait a moment to connect via OpenOCD after the hard reset happened. + # Along with this check that app runs ok + dut.expect('Hello world!') + + # Don't need to have output from UART any more + dut.serial.stop_redirect_thread() + + with open(os.path.join(dut.logdir, 'ocd.log'), 'w') as ocd_log: + ocd = subprocess.Popen(f'openocd {get_openocd_arguments(dut.target)}', stdout=ocd_log, stderr=ocd_log, shell=True) + + try: + gdb_env = os.environ.copy() + gdb_env['ESP_IDF_GDB_TESTING'] = '1' + + with open(os.path.join(dut.logdir, 'gdb.log'), 'w') as gdb_log, \ + pexpect.spawn(f'idf.py -B {dut.app.binary_path} gdb --batch', + env=gdb_env, + timeout=60, + logfile=gdb_log, + encoding='utf-8', + codec_errors='ignore') as p: + p.expect(re.compile(r'add symbol table from file.*bootloader.elf')) + p.expect(re.compile(r'add symbol table from file.*rom.elf')) + p.expect_exact('hit Temporary breakpoint 1, app_main ()') + finally: + try: + ocd.send_signal(signal.SIGINT) + ocd.communicate(timeout=15) + except subprocess.TimeoutExpired: + ocd.kill() + ocd.communicate() From 2d61c9ca792862b9623584de7abc8a02c1f16ade Mon Sep 17 00:00:00 2001 From: Alexey Lapshin Date: Sat, 10 Sep 2022 00:59:45 +0400 Subject: [PATCH 4/4] tools: move cmake executable check into function uses it Before this change idf.py could exit with reason cmake does not exist in PATH even cmake will not be executed by idf.py (e.g., 'idf.py gdb'). --- tools/idf.py | 35 +++-------------------------------- tools/idf_py_actions/tools.py | 26 ++++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 32 deletions(-) diff --git a/tools/idf.py b/tools/idf.py index 0b91e13ff0..1d05593aed 100755 --- a/tools/idf.py +++ b/tools/idf.py @@ -27,7 +27,7 @@ from collections import Counter, OrderedDict, _OrderedDictKeysView from importlib import import_module from pkgutil import iter_modules from types import FrameType -from typing import Any, Callable, Dict, List, Optional, TextIO, Union +from typing import Any, Callable, Dict, List, Optional, Union # pyc files remain in the filesystem when switching between branches which might raise errors for incompatible # idf.py extensions. Therefore, pyc file generation is turned off: @@ -37,8 +37,8 @@ import python_version_checker # noqa: E402 try: from idf_py_actions.errors import FatalError # noqa: E402 - from idf_py_actions.tools import (PropertyDict, executable_exists, get_target, idf_version, # noqa: E402 - merge_action_lists, realpath) + from idf_py_actions.tools import (PROG, SHELL_COMPLETE_RUN, SHELL_COMPLETE_VAR, PropertyDict, # noqa: E402 + debug_print_idf_version, get_target, merge_action_lists, print_warning, realpath) if os.getenv('IDF_COMPONENT_MANAGER') != '0': from idf_component_manager import idf_extensions except ImportError: @@ -53,23 +53,6 @@ PYTHON = sys.executable # you have to pass env=os.environ explicitly anywhere that we create a process os.environ['PYTHON'] = sys.executable -# Name of the program, normally 'idf.py'. -# Can be overridden from idf.bat using IDF_PY_PROGRAM_NAME -PROG = os.getenv('IDF_PY_PROGRAM_NAME', 'idf.py') - -# environment variable used during click shell completion run -SHELL_COMPLETE_VAR = '_IDF.PY_COMPLETE' - -# was shell completion invoked? -SHELL_COMPLETE_RUN = SHELL_COMPLETE_VAR in os.environ - - -# function prints warning when autocompletion is not being performed -# set argument stream to sys.stderr for errors and exceptions -def print_warning(message: str, stream: TextIO=None) -> None: - if not SHELL_COMPLETE_RUN: - print(message, file=stream or sys.stderr) - def check_environment() -> List: """ @@ -79,10 +62,6 @@ def check_environment() -> List: """ checks_output = [] - if not executable_exists(['cmake', '--version']): - debug_print_idf_version() - raise FatalError("'cmake' must be available on the PATH to use %s" % PROG) - # verify that IDF_PATH env variable is set # find the directory idf.py is in, then the parent directory of this, and assume this is IDF_PATH detected_idf_path = realpath(os.path.join(os.path.dirname(__file__), '..')) @@ -137,14 +116,6 @@ def _safe_relpath(path: str, start: Optional[str]=None) -> str: return os.path.abspath(path) -def debug_print_idf_version() -> None: - version = idf_version() - if version: - print_warning('ESP-IDF %s' % version) - else: - print_warning('ESP-IDF version unknown') - - def init_cli(verbose_output: List=None) -> Any: # Click is imported here to run it after check_environment() import click diff --git a/tools/idf_py_actions/tools.py b/tools/idf_py_actions/tools.py index cfacc7239c..4e8f7ec919 100644 --- a/tools/idf_py_actions/tools.py +++ b/tools/idf_py_actions/tools.py @@ -16,6 +16,16 @@ import yaml from .constants import GENERATORS from .errors import FatalError +# Name of the program, normally 'idf.py'. +# Can be overridden from idf.bat using IDF_PY_PROGRAM_NAME +PROG = os.getenv('IDF_PY_PROGRAM_NAME', 'idf.py') + +# environment variable used during click shell completion run +SHELL_COMPLETE_VAR = '_IDF.PY_COMPLETE' + +# was shell completion invoked? +SHELL_COMPLETE_RUN = SHELL_COMPLETE_VAR in os.environ + def executable_exists(args: List) -> bool: try: @@ -78,6 +88,13 @@ def idf_version() -> Optional[str]: return version +# function prints warning when autocompletion is not being performed +# set argument stream to sys.stderr for errors and exceptions +def print_warning(message: str, stream: TextIO=None) -> None: + if not SHELL_COMPLETE_RUN: + print(message, file=stream or sys.stderr) + + def color_print(message: str, color: str, newline: Optional[str]='\n') -> None: """ Print a message to stderr with colored highlighting """ ansi_normal = '\033[0m' @@ -95,6 +112,10 @@ def red_print(message: str, newline: Optional[str]='\n') -> None: color_print(message, ansi_red, newline) +def debug_print_idf_version() -> None: + print_warning(f'ESP-IDF {idf_version() or "version unknown"}') + + def print_hints(*filenames: str) -> None: """Getting output files and printing hints on how to resolve errors based on the output.""" with open(os.path.join(os.path.dirname(__file__), 'hints.yml'), 'r') as file: @@ -383,6 +404,11 @@ def ensure_build_directory(args: 'PropertyDict', prog_name: str, always_run_cmak the build directory, an error is raised. If the parameter is None, this function will set it to an auto-detected default generator or to the value already configured in the build directory. """ + + if not executable_exists(['cmake', '--version']): + debug_print_idf_version() + raise FatalError(f'"cmake" must be available on the PATH to use {PROG}') + project_dir = args.project_dir # Verify the project directory if not os.path.isdir(project_dir):