From 92ef2a4c835ac5d6043ee14678ce0c5c941d3c63 Mon Sep 17 00:00:00 2001 From: "simon.chupin" Date: Tue, 9 Aug 2022 15:39:23 +0200 Subject: [PATCH] Tools: Add unit tests for idf.py hints --- .gitlab/ci/host-test.yml | 1 + tools/ci/executable-list.txt | 1 + tools/idf_py_actions/core_ext.py | 7 +- tools/idf_py_actions/hints.yml | 16 +--- tools/idf_py_actions/tools.py | 16 ++-- tools/test_idf_py/error_output.yml | 113 +++++++++++++++++++++++++++++ tools/test_idf_py/test_hints.py | 35 +++++++++ 7 files changed, 166 insertions(+), 23 deletions(-) create mode 100644 tools/test_idf_py/error_output.yml create mode 100755 tools/test_idf_py/test_hints.py diff --git a/.gitlab/ci/host-test.yml b/.gitlab/ci/host-test.yml index 5ba295e389..43c28abc7e 100644 --- a/.gitlab/ci/host-test.yml +++ b/.gitlab/ci/host-test.yml @@ -203,6 +203,7 @@ test_idf_py: script: - cd ${IDF_PATH}/tools/test_idf_py - ./test_idf_py.py + - ./test_hints.py # Test for create virtualenv. It must be invoked from Python, not from virtualenv. # Use docker image system python without any extra dependencies diff --git a/tools/ci/executable-list.txt b/tools/ci/executable-list.txt index 4c51965632..67708ec6ba 100644 --- a/tools/ci/executable-list.txt +++ b/tools/ci/executable-list.txt @@ -113,6 +113,7 @@ tools/python_version_checker.py tools/set-submodules-to-github.sh tools/test_apps/system/no_embedded_paths/check_for_file_paths.py tools/test_idf_monitor/run_test_idf_monitor.py +tools/test_idf_py/test_hints.py tools/test_idf_py/test_idf_py.py tools/test_idf_size/test.sh tools/test_idf_tools/test_idf_tools.py diff --git a/tools/idf_py_actions/core_ext.py b/tools/idf_py_actions/core_ext.py index 0198247c91..2b0e734ec9 100644 --- a/tools/idf_py_actions/core_ext.py +++ b/tools/idf_py_actions/core_ext.py @@ -18,8 +18,8 @@ from click.core import Context from idf_py_actions.constants import GENERATORS, PREVIEW_TARGETS, SUPPORTED_TARGETS, URL_TO_DOC from idf_py_actions.errors import FatalError from idf_py_actions.global_options import global_options -from idf_py_actions.tools import (PropertyDict, TargetChoice, ensure_build_directory, get_target, idf_version, - merge_action_lists, print_hints, realpath, run_target) +from idf_py_actions.tools import (PropertyDict, TargetChoice, ensure_build_directory, generate_hints, get_target, + idf_version, merge_action_lists, realpath, run_target, yellow_print) def action_extensions(base_actions: Dict, project_path: str) -> Any: @@ -41,7 +41,8 @@ def action_extensions(base_actions: Dict, project_path: str) -> Any: """ def tool_error_handler(e: int, stdout: str, stderr: str) -> None: - print_hints(stdout, stderr) + for hint in generate_hints(stdout, stderr): + yellow_print(hint) if output_format: os.environ['SIZE_OUTPUT_FORMAT'] = output_format diff --git a/tools/idf_py_actions/hints.yml b/tools/idf_py_actions/hints.yml index 4fce4d5395..096db95400 100644 --- a/tools/idf_py_actions/hints.yml +++ b/tools/idf_py_actions/hints.yml @@ -124,15 +124,15 @@ - re: "error: unknown type name '(portTickType|xTaskHandle|xQueueHandle|xSemaphoreHandle|xQueueSetHandle|xQueueSetMemberHandle|xTimeOutType|xMemoryRegion|xTaskParameters|xTaskStatusType|xTimerHandle|xCoRoutineHandle|pdTASK_HOOK_CODE|tmrTIMER_CALLBACK|pdTASK_CODE|xListItem|xList)'" - hint: "You maybe using pre FreeRTOS V8.0.0 data types. The backward compatibility of such data types is no longer enabled by default. Please turn on CONFIG_FREERTOS_ENABLE_BACKWARD_COMPATIBILITY explicitly to use such data types." + hint: "You are maybe using pre FreeRTOS V8.0.0 data types. The backward compatibility of such data types is no longer enabled by default. Please turn on CONFIG_FREERTOS_ENABLE_BACKWARD_COMPATIBILITY explicitly to use such data types." match_to_output: True - re: "error: 'portTICK_RATE_MS' undeclared" - hint: "You maybe using pre FreeRTOS V8.0.0 APIs. The backward compatibility of such APIs is no longer enabled by default. Please turn on CONFIG_FREERTOS_ENABLE_BACKWARD_COMPATIBILITY explicitly to use such APIs." + hint: "You are maybe using pre FreeRTOS V8.0.0 APIs. The backward compatibility of such APIs is no longer enabled by default. Please turn on CONFIG_FREERTOS_ENABLE_BACKWARD_COMPATIBILITY explicitly to use such APIs." match_to_output: True - re: "error: implicit declaration of function '(eTaskStateGet|pcTaskGetTaskName|pcTimerGetTimerName|pcQueueGetQueueName|vTaskGetTaskInfo|xTaskGetIdleRunTimeCounter)'" - hint: "You maybe using pre FreeRTOS V8.0.0 APIs. The backward compatibility of such APIs is no longer enabled by default. Please turn on CONFIG_FREERTOS_ENABLE_BACKWARD_COMPATIBILITY explicitly to use such APIs." + hint: "You are maybe using pre FreeRTOS V8.0.0 APIs. The backward compatibility of such APIs is no longer enabled by default. Please turn on CONFIG_FREERTOS_ENABLE_BACKWARD_COMPATIBILITY explicitly to use such APIs." match_to_output: True - re: "error: unknown type name 'TaskSnapshot_t'" @@ -146,10 +146,6 @@ re: "error: implicit declaration of function '(portENTER_CRITICAL_NESTED|portEXIT_CRITICAL_NESTED|vPortCPUInitializeMutex|vPortCPUAcquireMutex|vPortCPUAcquireMutexTimeout|vPortCPUReleaseMutex)'" hint: "The header file portmacro_deprecated.h has been removed. Users should refer the migration guide for alternative functions." match_to_output: True -- - re: "error: implicit declaration of function '(\\w+)'" - hint: "Please check that the function name is correct. Also it is possible that you've forgot to import {} library(s) in header file or add the necessary REQURIES component. Try to add missing libraries to your project header file or check idf_component_register(REQUIRES ...) section in your component CmakeList.txt file. For more information run 'idf.py docs -sp api-guides/build-system.html'.\nAlso, please check if the function has been removed, renamed or replaced by an alternative function - refer to the migration guide for more information." - match_to_output: True - re: "fatal error: {}.h: No such file or directory" @@ -179,10 +175,6 @@ re: "fatal error: .*atca_mbedtls_wrap\\.h: No such file or directory" hint: "To use CONFIG_ESP_TLS_USE_SECURE_ELEMENT option, please install `esp-cryptoauthlib` using 'idf.py add-dependency espressif/esp-cryptoauthlib'" -- - re: "fatal error: [\\w/]+\\.h: No such file or directory" - hint: "Please make sure that the header name is correct. Also please check if you've specified all component dependencies with 'idf_component_register(REQUIRES ...)'. If the component is not present then it should be added by the IDF Component Manager. For more information run 'idf.py docs -sp api-guides/build-system.html'.\nAlso, please check if the header file has been removed, renamed or relocated - refer to the migration guide for more information." - - re: "The CMAKE_[A-Z]+_COMPILER: [\\w+-]+ is not a full path and was not found in the PATH\\." hint: "Try to reinstall the toolchain for the chip that you trying to use. \nFor more information run 'idf.py docs -sp get-started/#installation' and follow the instructions for your system" @@ -196,7 +188,7 @@ hint: "Run 'idf.py fullclean' and try the build again." - - re: "CMake Error at .* \\(message\\): Directory specified in EXTRA_COMPONENT_DIRS doesn't exist: \\/.*\\/examples\\/common_components\\/.*" + re: "CMake Error at .* \\(message\\): Directory specified in EXTRA_COMPONENT_DIRS doesn't exist: \\/?.*\\/examples\\/common_components\\/.*" hint: "The component with path specified in the EXTRA_COMPONENT_DIRS variable has been moved to IDF component manager (or has been removed).\nPlease look out for component in 'https://components.espressif.com' and add using 'idf.py add-dependency' command.\nRefer to the migration guide for more details." - diff --git a/tools/idf_py_actions/tools.py b/tools/idf_py_actions/tools.py index 8822befae8..9a24ff4d9e 100644 --- a/tools/idf_py_actions/tools.py +++ b/tools/idf_py_actions/tools.py @@ -8,7 +8,7 @@ import sys from asyncio.subprocess import Process from io import open from types import FunctionType -from typing import Any, Dict, List, Match, Optional, TextIO, Tuple, Union +from typing import Any, Dict, Generator, List, Match, Optional, TextIO, Tuple, Union import click import yaml @@ -95,7 +95,7 @@ def red_print(message: str, newline: Optional[str]='\n') -> None: color_print(message, ansi_red, newline) -def print_hints(*filenames: str) -> None: +def generate_hints(*filenames: str) -> Generator: """Getting output files and printing hints on how to resolve errors based on the output.""" with open(os.path.join(os.path.dirname(__file__), 'hints.yml'), 'r') as file: hints = yaml.safe_load(file) @@ -128,14 +128,13 @@ def print_hints(*filenames: str) -> None: sys.exit(1) if hint_list: for message in hint_list: - yellow_print('HINT:', message) + yield ' '.join(['HINT:', message]) elif match: extra_info = ', '.join(match.groups()) if hint.get('match_to_output', '') else '' try: - yellow_print(' '.join(['HINT:', hint['hint'].format(extra_info)])) - except KeyError as e: - red_print('Argument {} missing in {}. Check hints.yml file.'.format(e, hint)) - sys.exit(1) + yield ' '.join(['HINT:', hint['hint'].format(extra_info)]) + except KeyError: + raise KeyError("Argument 'hint' missing in {}. Check hints.yml file.".format(hint)) def fit_text_in_terminal(out: str) -> str: @@ -195,7 +194,8 @@ class RunTool: return if stderr_output_file and stdout_output_file: - print_hints(stderr_output_file, stdout_output_file) + for hint in generate_hints(stderr_output_file, stdout_output_file): + yellow_print(hint) raise FatalError('{} failed with exit code {}, output of the command is in the {} and {}'.format(self.tool_name, process.returncode, stderr_output_file, stdout_output_file)) diff --git a/tools/test_idf_py/error_output.yml b/tools/test_idf_py/error_output.yml new file mode 100644 index 0000000000..0bc84a2a2b --- /dev/null +++ b/tools/test_idf_py/error_output.yml @@ -0,0 +1,113 @@ +'ccache error: Failed to create temporary file\n': + "HINT: On Windows, you should enable long path support in the installer, or disable ccache temporarily. See 'idf.py --help' or the documentation how to achieve this." + +"CMake Error at C:/Users/esp-idf/esp-idf/ (message): Directory specified in EXTRA_COMPONENT_DIRS doesn't exist: C:/Users/esp-idf/esp-idf/examples/common_components/component\n": + "HINT: The component with path specified in the EXTRA_COMPONENT_DIRS variable has been moved to IDF component manager (or has been removed).\nPlease look out for component in 'https://components.espressif.com' and add using 'idf.py add-dependency' command.\nRefer to the migration guide for more details." + +'CMake Error: The current CMakeCache.txt directory C:/Users/esp-idf/examples/get-started/hello_world/build is different than the directory /mnt/c/Users/esp-idf/examples/get-started/hello_world/build where CMakeCache.txt was created.\n': + "HINT: Run 'idf.py fullclean' and try the build again." + +"error: 'portTICK_RATE_MS' undeclared\n": + 'HINT: You are maybe using pre FreeRTOS V8.0.0 APIs. The backward compatibility of such APIs is no longer enabled by default. Please turn on CONFIG_FREERTOS_ENABLE_BACKWARD_COMPATIBILITY explicitly to use such APIs.' + +"error: enumeration value 'HTTP_EVENT_REDIRECT' not handled in switch\n": + "HINT: The event handler, specified in the 'event_handler' element, of the 'esp_http_client_config_t' struct now needs to handle the new 'HTTP_EVENT_REDIRECT' event case." + +"error: implicit declaration of function 'portENTER_CRITICAL_NESTED'\n": + 'HINT: The header file portmacro_deprecated.h has been removed. Users should refer the migration guide for alternative functions.' + +"error: implicit declaration of function 'bootloader_common_get_reset_reason'\n": + "HINT: Function 'bootloader_common_get_reset_reason()' has been removed. Please use the function 'esp_rom_get_reset_reason()' in the ROM component." + +"error: implicit declaration of function 'esp_base_mac_addr_get'\n": + 'HINT: esp_mac.h header file is not included by esp_system.h anymore. It shall then be manually included with #include "esp_mac.h"' + +"error: implicit declaration of function 'esp_chip_info'\n": + 'HINT: esp_chip_info.h header file is not included by esp_system.h anymore. It shall then be manually included with #include "esp_chip_info.h"' + +"error: implicit declaration of function 'esp_cpu_ccount_t'\n": + 'HINT: Use esp_cpu_cycle_count_t defined in esp_cpu.h instead of esp_cpu_ccount_t.' + +"error: implicit declaration of function 'esp_cpu_get_ccount'\n": + 'HINT: Use esp_cpu_get_cycle_count() defined in esp_cpu.h instead of esp_cpu_get_ccount.' + +"error: implicit declaration of function 'esp_cpu_set_ccount'\n": + 'HINT: Use esp_cpu_set_cycle_count() defined in esp_cpu.h instead of esp_cpu_set_ccount.' + +"error: implicit declaration of function 'esp_efuse_get_chip_ver'\n": + "HINT: Function 'esp_efuse_get_chip_ver()' has been removed. Please use the function efuse_hal_get_major_chip_version()." + +"error: implicit declaration of function 'esp_fill_random'\n": + 'HINT: esp_random.h header file is not included by esp_system.h anymore. It shall then be manually included with #include "esp_random.h"' + +"error: implicit declaration of function 'esp_spiram_get_chip_size'\n": + "HINT: Function 'esp_spiram_get_chip_size and esp_spiram_get_size' has been removed. Please use the function esp_psram_get_size()." + +"error: invalid use of incomplete typedef 'esp_tls_t'\n": + "HINT: The struct 'esp_tls_t' has now been made private - its elements can be only be accessed/modified through respective getter/setter functions. Please refer to the migration guide for more information." + +"error: unknown type name 'TaskSnapshot_t'\n": + 'HINT: The header file task_snapshot.h is no longer included as part of freertos/task.h. Users need to include freertos/task_snapshot.h explicitly.' + +"Failed to resolve component 'component'\n": + "HINT: The component component has been moved to the IDF component manager or has been removed and refactored into some other component.\nPlease look out for component in 'https://components.espressif.com' and add using 'idf.py add-dependency' command.\nRefer to the migration guide for more details." + +'fatal error: tmp/atca_mbedtls_wrap.h: No such file or directory\n': + "HINT: To use CONFIG_ESP_TLS_USE_SECURE_ELEMENT option, please install `esp-cryptoauthlib` using 'idf.py add-dependency espressif/esp-cryptoauthlib'" + +'fatal error: brownout.h: No such file or directory\n': + 'HINT: The Brownout API (functions/types/macros prefixed with "esp_brownout") has been made into a private API. If users still require usage of the Brownout API (though this is not recommended), it can be included via #include "esp_private/brownout.h".' + +'fatal error: compare_set.h: No such file or directory\n': + 'HINT: compare_set.h was removed. Include and use the API function provided by esp_cpu.h instead.' + +'fatal error: eh_frame_parser.h: No such file or directory\n': + 'HINT: The Backtrace Parser API (functions/types/macros prefixed with "eh_frame_parser") has been made into a private API. If users still require usage of the Backtrace Parser API (though this is not recommended), it can be included via #include "esp_private/eh_frame_parser.h".' + +'fatal error: esp32/cache_err_int.h: No such file or directory\n': + 'HINT: The Cache Error Interrupt API (functions/types/macros prefixed with "esp_cache_err") has been made into a private API. If users still require usage of the Cache Error Interrupt API (though this is not recommended), it can be included via #include "esp_private/cache_err_int.h".' + +'fatal error: esp32/clk.h: No such file or directory\n': + 'HINT: The ESP Clock API (functions/types/macros prefixed with "esp_clk") has been made into a private API. If users still require usage of the ESP Clock API (though this is not recommended), it can be included via #include "esp_private/esp_clk.h".' + +'fatal error: esp_adc_cal.h: No such file or directory\n': + "HINT: ``esp_adc_cal`` component is no longer supported. New adc calibration driver is in ``esp_adc``. Legacy adc calibration driver has been moved into ``esp_adc`` component. To use legacy ``esp_adc_cal`` driver APIs, you should add ``esp_adc`` component to the list of component requirements in CMakeLists.txt. For more information run 'idf.py docs -sp migration-guides/release-5.x/peripherals.html'." + +'fatal error: esp_intr.h: No such file or directory\n': + 'HINT: esp_intr.h was removed. Include esp_intr_alloc.h instead.' + +'fatal error: esp_panic.h: No such file or directory\n': + 'HINT: esp_panic.h was removed. Include use functionalities provided in esp_debug_helpers.h instead.' + +'fatal error: esp_rom_tjpgd.h: No such file or directory\n': + "HINT: esp_rom_tjpgd.h was removed. Please use esp_jpeg component from IDF component manager instead.\nPlease look out for component in 'https://components.espressif.com' and add using 'idf.py add-dependency' command.\nRefer to the migration guide for more details." + +'fatal error: soc/cpu.h: No such file or directory\n': + 'HINT: soc/cpu.h was removed. Include and use the API function provided by esp_cpu.h instead.' + +'fatal error: soc/spinlock.h: No such file or directory\n': + "HINT: spinlock.h must be included without the 'soc' part." + +'fatal error: soc_log.h: No such file or directory\n': + 'HINT: soc_log.h was renamed and made private. Consider using the logging APIs provided under esp_log.h instead.' + +'fatal error: spiram.h: No such file or directory\n': + 'HINT: spiram.h was removed. Include esp_psram.h instead. Make sure to also add esp_psram as a dependency in your CMakeLists.txt file.' + +'fatal error: trax.h: No such file or directory\n': + 'HINT: The Trax API (functions/types/macros prefixed with "trax_") has been made into a private API. If users still require usage of the Trax API (though this is not recommended), it can be included via #include "esp_private/trax.h".' + +"format 'format' expects argument of type 'unsigned int', but argument arg has type 'uint32_t' {aka 'int'}\n": + "HINT: The issue is better to resolve by replacing format specifiers to 'PRI'-family macros (include header file)." + +"ImportError: bad magic number in 'kconfiglib':\n": + "HINT: Run 'idf.py python-clean', and try again" + +'The CMAKE_CXX_COMPILER: xtensa-esp32s3-elf-g++ is not a full path and was not found in the PATH.\n': + "HINT: Try to reinstall the toolchain for the chip that you trying to use. \nFor more information run 'idf.py docs -sp get-started/#installation' and follow the instructions for your system" + +'The keyword signature for target_link_libraries has already been used\n': + 'HINT: Projects using target_link_libraries with project_elf explicitly and custom CMake projects must specify PRIVATE, PUBLIC or INTERFACE arguments.' + +"warning: passing argument 1 of 'esp_secure_boot_read_key_digests' from incompatible pointer type\n": + 'HINT: The parameter type of the function esp_secure_boot_read_key_digests() has been changed from ets_secure_boot_key_digests_t* to esp_secure_boot_key_digests_t*.' diff --git a/tools/test_idf_py/test_hints.py b/tools/test_idf_py/test_hints.py new file mode 100755 index 0000000000..31699801a0 --- /dev/null +++ b/tools/test_idf_py/test_hints.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python +# +# SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 +import os +import sys +import tempfile +import unittest + +import yaml + +CWD = os.path.join(os.path.dirname(__file__)) +ERR_OUT_YML = os.path.join(CWD, 'error_output.yml') + +try: + from idf_py_actions.tools import generate_hints +except ImportError: + sys.path.append(os.path.join(CWD, '..')) + from idf_py_actions.tools import generate_hints + + +class TestHintsMassages(unittest.TestCase): + def test_output(self) -> None: + with open(ERR_OUT_YML) as f: + error_output = yaml.safe_load(f) + for error, hint in error_output.items(): + with tempfile.NamedTemporaryFile(mode='w') as f: + f.write(error) + f.flush() + for generated_hint in generate_hints(f.name): + self.assertEqual(generated_hint, hint) + + +if __name__ == '__main__': + unittest.main()