From fb0dc461836f4be88188b895d5ce8018d40c0b13 Mon Sep 17 00:00:00 2001 From: Marek Fiala Date: Wed, 29 Mar 2023 17:22:15 +0200 Subject: [PATCH] Tools: Rewrite build system unit tests to python - sdkconfig, bootloader, components --- tools/test_build_system/MIGRATION.md | 26 +++---- tools/test_build_system/conftest.py | 4 +- tools/test_build_system/test_bootloader.py | 53 +++++++++++++++ tools/test_build_system/test_build.py | 12 ++++ .../test_build_system_helpers/idf_utils.py | 17 ++--- tools/test_build_system/test_common.py | 67 ++++++++++++++++--- tools/test_build_system/test_components.py | 55 ++++++++++++++- tools/test_build_system/test_sdkconfig.py | 26 +++++++ 8 files changed, 225 insertions(+), 35 deletions(-) create mode 100644 tools/test_build_system/test_bootloader.py create mode 100644 tools/test_build_system/test_sdkconfig.py diff --git a/tools/test_build_system/MIGRATION.md b/tools/test_build_system/MIGRATION.md index 3fbb5a1760..dd08ad1c64 100644 --- a/tools/test_build_system/MIGRATION.md +++ b/tools/test_build_system/MIGRATION.md @@ -50,7 +50,7 @@ idf.py fails if IDF_TARGET settings don't match in sdkconfig, CMakeCache.txt, an Setting EXTRA_COMPONENT_DIRS works | test_components.py::test_component_extra_dirs | Non-existent paths in EXTRA_COMPONENT_DIRS are not allowed | test_components.py::test_component_nonexistent_extra_dirs_not_allowed | Component names may contain spaces | test_components.py::test_component_names_contain_spaces | -sdkconfig should have contents of all files: sdkconfig, sdkconfig.defaults, sdkconfig.defaults.IDF_TARGET | | +sdkconfig should have contents of all files: sdkconfig, sdkconfig.defaults, sdkconfig.defaults.IDF_TARGET | test_sdkconfig.py::test_sdkconfig_contains_all_files | Test if it can build the example to run on host | | Test build ESP-IDF as a library to a custom CMake projects for all targets | test_cmake.py::test_build_custom_cmake_project | Building a project with CMake library imported and PSRAM workaround, all files compile with workaround | test_cmake.py::test_build_cmake_library_psram_workaround | @@ -63,22 +63,22 @@ Handling deprecated Kconfig options | test_kconfig.py::test_kconfig_deprecated_o Handling deprecated Kconfig options in sdkconfig.defaults | test_kconfig.py::test_kconfig_deprecated_options | Can have multiple deprecated Kconfig options map to a single new option | test_kconfig.py::test_kconfig_multiple_and_target_specific_options | Can have target specific deprecated Kconfig options | test_kconfig.py::test_kconfig_multiple_and_target_specific_options | -Confserver can be invoked by idf.py | | -Check ccache is used to build | | -Custom bootloader overrides original | | -Empty directory not treated as a component | | -If a component directory is added to COMPONENT_DIRS, its subdirectories are not added | | -If a component directory is added to COMPONENT_DIRS, its sibling directories are not added | | -toolchain prefix is set in project description file | | -Can set options to subcommands: print_filter for monitor | | -Fail on build time works | | -Component properties are set | | -should be able to specify multiple sdkconfig default files | | +Confserver can be invoked by idf.py | test_common.py::test_invoke_confserver | +Check ccache is used to build | test_common.py::test_ccache_used_to_build | +Custom bootloader overrides original | test_bootloader.py::test_bootloader_custom_overrides_original | +Empty directory not treated as a component | test_components.py::test_component_can_not_be_empty_dir | +If a component directory is added to COMPONENT_DIRS, its subdirectories are not added | test_components.py::test_component_subdirs_not_added_to_component_dirs | +If a component directory is added to COMPONENT_DIRS, its sibling directories are not added | test_components.py::test_component_sibling_dirs_not_added_to_component_dirs | +toolchain prefix is set in project description file | test_common.py::test_toolchain_prefix_in_description_file | +Can set options to subcommands: print_filter for monitor | test_common.py::test_subcommands_with_options | +Fail on build time works | test_build.py::test_build_fail_on_build_time | +Component properties are set | test_components.py::test_component_properties_are_set | +should be able to specify multiple sdkconfig default files | test_sdkconfig.py::test_sdkconfig_multiple_default_files | Supports git worktree | | idf.py fallback to build system target | | Build fails if partitions don't fit in flash | | Warning is given if smallest partition is nearly full | | -Flash size is correctly set in the bootloader image header | | +Flash size is correctly set in the bootloader image header | test_bootloader.py::test_bootloader_correctly_set_image_header | DFU build works | | UF2 build works | | Loadable ELF build works | | diff --git a/tools/test_build_system/conftest.py b/tools/test_build_system/conftest.py index 928adb5388..398db038e9 100644 --- a/tools/test_build_system/conftest.py +++ b/tools/test_build_system/conftest.py @@ -139,6 +139,6 @@ def fixture_default_idf_env() -> EnvDict: @pytest.fixture def idf_py(default_idf_env: EnvDict) -> IdfPyFunc: - def result(*args: str, check: bool = True) -> subprocess.CompletedProcess: - return run_idf_py(*args, env=default_idf_env, workdir=os.getcwd(), check=check) # type: ignore + def result(*args: str, check: bool = True, input_str: typing.Optional[str] = None) -> subprocess.CompletedProcess: + return run_idf_py(*args, env=default_idf_env, workdir=os.getcwd(), check=check, input_str=input_str) # type: ignore return result diff --git a/tools/test_build_system/test_bootloader.py b/tools/test_build_system/test_bootloader.py new file mode 100644 index 0000000000..991e5e119c --- /dev/null +++ b/tools/test_build_system/test_bootloader.py @@ -0,0 +1,53 @@ +# SPDX-FileCopyrightText: 2022-2023 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 +import logging +import shutil +from pathlib import Path +from typing import Union + +from test_build_system_helpers import EnvDict, IdfPyFunc, file_contains + + +def get_two_header_bytes(file_path: Union[str, Path]) -> str: + ''' + get the bytes 3-4 of the given file + https://docs.espressif.com/projects/esptool/en/latest/esp32/advanced-topics/firmware-image-format.html + ''' + data = b'' + with open(file_path, 'rb') as f: + data = f.read(4) + extracted_bytes = data[2:4] + return extracted_bytes.hex() + + +def test_bootloader_custom_overrides_original(test_app_copy: Path, idf_py: IdfPyFunc, default_idf_env: EnvDict) -> None: + logging.info('Custom bootloader overrides original') + idf_path = Path(default_idf_env.get('IDF_PATH')) + shutil.copytree(idf_path / 'components' / 'bootloader', test_app_copy / 'components' / 'bootloader') + # Because of relative include of Kconfig, also esp_bootloader_format needs to be copied. + shutil.copytree(idf_path / 'components' / 'esp_bootloader_format', test_app_copy / 'components' / 'esp_bootloader_format') + idf_py('bootloader') + assert file_contains(test_app_copy / 'build' / 'bootloader' / 'compile_commands.json', + str(test_app_copy / 'components' / 'bootloader' / 'subproject' / 'main' / 'bootloader_start.c')) + + +def test_bootloader_correctly_set_image_header(test_app_copy: Path, idf_py: IdfPyFunc) -> None: + logging.info('Flash size is correctly set in the bootloader image header') + # Build with the default 2MB setting + idf_py('bootloader') + assert get_two_header_bytes(test_app_copy / 'build' / 'bootloader' / 'bootloader.bin') == '0210' + + # Change to 4MB + (test_app_copy / 'sdkconfig').write_text('CONFIG_ESPTOOLPY_FLASHSIZE_4MB=y') + idf_py('reconfigure', 'bootloader') + assert get_two_header_bytes(test_app_copy / 'build' / 'bootloader' / 'bootloader.bin') == '0220' + + # Change to QIO, bootloader should still be DIO (will change to QIO in 2nd stage bootloader) + (test_app_copy / 'sdkconfig').write_text('CONFIG_FLASHMODE_QIO=y') + idf_py('reconfigure', 'bootloader') + assert get_two_header_bytes(test_app_copy / 'build' / 'bootloader' / 'bootloader.bin') == '0210' + + # Change to 80 MHz + (test_app_copy / 'sdkconfig').write_text('CONFIG_ESPTOOLPY_FLASHFREQ_80M=y') + idf_py('reconfigure', 'bootloader') + assert get_two_header_bytes(test_app_copy / 'build' / 'bootloader' / 'bootloader.bin') == '021f' diff --git a/tools/test_build_system/test_build.py b/tools/test_build_system/test_build.py index 104e0d34f8..8cb4164ce7 100644 --- a/tools/test_build_system/test_build.py +++ b/tools/test_build_system/test_build.py @@ -128,3 +128,15 @@ def test_build_with_sdkconfig_build_abspath(idf_py: IdfPyFunc, test_app_copy: Pa build_path = test_app_copy / 'build_tmp' sdkconfig_path = build_path / 'sdkconfig' idf_py('-D', f'SDKCONFIG={sdkconfig_path}', '-B', str(build_path), 'build') + + +def test_build_fail_on_build_time(idf_py: IdfPyFunc, test_app_copy: Path) -> None: + logging.info('Fail on build time works') + append_to_file(test_app_copy / 'CMakeLists.txt', '\n'.join(['', + 'if(NOT EXISTS "${CMAKE_CURRENT_LIST_DIR}/hello.txt")', + 'fail_at_build_time(test_file "hello.txt does not exists")', + 'endif()'])) + ret = idf_py('build', check=False) + assert ret.returncode != 0, 'Build should fail if requirements are not satisfied' + (test_app_copy / 'hello.txt').touch() + idf_py('build') diff --git a/tools/test_build_system/test_build_system_helpers/idf_utils.py b/tools/test_build_system/test_build_system_helpers/idf_utils.py index 201c5a899c..6161bf35a4 100644 --- a/tools/test_build_system/test_build_system_helpers/idf_utils.py +++ b/tools/test_build_system/test_build_system_helpers/idf_utils.py @@ -61,7 +61,8 @@ def run_idf_py(*args: str, idf_path: typing.Optional[typing.Union[str,Path]] = None, workdir: typing.Optional[str] = None, check: bool = True, - python: typing.Optional[str] = None) -> subprocess.CompletedProcess: + python: typing.Optional[str] = None, + input_str: typing.Optional[str] = None) -> subprocess.CompletedProcess: """ Run idf.py command with given arguments, raise an exception on failure :param args: arguments to pass to idf.py @@ -70,19 +71,19 @@ def run_idf_py(*args: str, :param workdir: directory where to run the build; if not set, the current directory is used :param check: check process exits with a zero exit code, if false all retvals are accepted without failing the test :param python: absolute path to python interpreter + :param input_str: input to idf.py """ - env_dict = dict(**os.environ) - if env is not None: - env_dict.update(env) + if not env: + env = dict(**os.environ) if not workdir: workdir = os.getcwd() # order: function argument -> value in env dictionary -> system environment if idf_path is None: - idf_path = env_dict.get('IDF_PATH') + idf_path = env.get('IDF_PATH') if not idf_path: raise ValueError('IDF_PATH must be set in the env array if idf_path argument is not set') if python is None: - python = find_python(env_dict['PATH']) + python = find_python(env['PATH']) cmd = [ python, @@ -91,9 +92,9 @@ def run_idf_py(*args: str, cmd += args # type: ignore logging.debug('running {} in {}'.format(' '.join(cmd), workdir)) return subprocess.run( - cmd, env=env_dict, cwd=workdir, + cmd, env=env, cwd=workdir, check=check, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - text=True, encoding='utf-8', errors='backslashreplace') + text=True, encoding='utf-8', errors='backslashreplace', input=input_str) def run_cmake(*cmake_args: str, env: typing.Optional[EnvDict] = None, diff --git a/tools/test_build_system/test_common.py b/tools/test_build_system/test_common.py index e2ac7ff32c..041b714ba4 100644 --- a/tools/test_build_system/test_common.py +++ b/tools/test_build_system/test_common.py @@ -1,5 +1,6 @@ # SPDX-FileCopyrightText: 2022-2023 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 +import json import logging import os import re @@ -12,8 +13,7 @@ from pathlib import Path from typing import List import pytest -from _pytest.monkeypatch import MonkeyPatch -from test_build_system_helpers import IdfPyFunc, find_python, get_snapshot, replace_in_file, run_idf_py +from test_build_system_helpers import EnvDict, IdfPyFunc, find_python, get_snapshot, replace_in_file, run_idf_py def get_subdirs_absolute_paths(path: Path) -> List[str]: @@ -78,14 +78,13 @@ def test_idf_copy(idf_copy: Path, idf_py: IdfPyFunc) -> None: def test_idf_build_with_env_var_sdkconfig_defaults( test_app_copy: Path, - idf_py: IdfPyFunc, - monkeypatch: MonkeyPatch, + default_idf_env: EnvDict ) -> None: with open(test_app_copy / 'sdkconfig.test', 'w') as fw: fw.write('CONFIG_BT_ENABLED=y') - monkeypatch.setenv('SDKCONFIG_DEFAULTS', 'sdkconfig.test') - idf_py('build') + default_idf_env['SDKCONFIG_DEFAULTS'] = 'sdkconfig.test' + run_idf_py('build', env=default_idf_env) with open(test_app_copy / 'sdkconfig') as fr: assert 'CONFIG_BT_ENABLED=y' in fr.read() @@ -93,12 +92,11 @@ def test_idf_build_with_env_var_sdkconfig_defaults( @pytest.mark.usefixtures('test_app_copy') @pytest.mark.test_app_copy('examples/system/efuse') -def test_efuse_symmary_cmake_functions( - idf_py: IdfPyFunc, - monkeypatch: MonkeyPatch +def test_efuse_summary_cmake_functions( + default_idf_env: EnvDict ) -> None: - monkeypatch.setenv('IDF_CI_BUILD', '1') - output = idf_py('efuse-summary') + default_idf_env['IDF_CI_BUILD'] = '1' + output = run_idf_py('efuse-summary', env=default_idf_env) assert 'FROM_CMAKE: MAC: 00:00:00:00:00:00' in output.stdout assert 'FROM_CMAKE: WR_DIS: 0' in output.stdout @@ -158,3 +156,50 @@ def test_python_interpreter_win(test_app_copy: Path) -> None: # python is loaded from env:$PATH, but since false interpreter is provided there, python needs to be specified as argument # if idf.py is reconfigured during it's execution, it would load a false interpreter run_idf_py('reconfigure', env=env_dict, python=python) + + +@pytest.mark.usefixtures('test_app_copy') +def test_invoke_confserver(idf_py: IdfPyFunc) -> None: + logging.info('Confserver can be invoked by idf.py') + idf_py('confserver', input_str='{"version": 1}') + + +def test_ccache_used_to_build(test_app_copy: Path) -> None: + logging.info('Check ccache is used to build') + (test_app_copy / 'ccache').touch(mode=0o755) + env_dict = dict(**os.environ) + env_dict['PATH'] = str(test_app_copy) + os.pathsep + env_dict['PATH'] + # Disable using ccache automatically + if 'IDF_CCACHE_ENABLE' in env_dict: + env_dict.pop('IDF_CCACHE_ENABLE') + + ret = run_idf_py('--ccache', 'reconfigure', env=env_dict) + assert 'ccache will be used' in ret.stdout + run_idf_py('fullclean', env=env_dict) + ret = run_idf_py('reconfigure', env=env_dict) + assert 'ccache will be used' not in ret.stdout + ret = run_idf_py('--no-ccache', 'reconfigure', env=env_dict) + assert 'ccache will be used' not in ret.stdout + + +def test_toolchain_prefix_in_description_file(idf_py: IdfPyFunc, test_app_copy: Path) -> None: + logging.info('Toolchain prefix is set in project description file') + idf_py('reconfigure') + data = json.load(open(test_app_copy / 'build' / 'project_description.json', 'r')) + assert 'monitor_toolprefix' in data + + +@pytest.mark.usefixtures('test_app_copy') +def test_subcommands_with_options(idf_py: IdfPyFunc, default_idf_env: EnvDict) -> None: + logging.info('Can set options to subcommands: print_filter for monitor') + idf_path = Path(default_idf_env.get('IDF_PATH')) + # try - finally block is here used to backup and restore idf_monitor.py + # since we need to handle only one file, this souluton is much faster than using idf_copy fixture + monitor_backup = (idf_path / 'tools' / 'idf_monitor.py').read_text() + try: + (idf_path / 'tools' / 'idf_monitor.py').write_text('import sys;print(sys.argv[1:])') + idf_py('build') + ret = idf_py('monitor', '--print-filter=*:I', '-p', 'tty.fake') + assert "'--print_filter', '*:I'" in ret.stdout + finally: + (idf_path / 'tools' / 'idf_monitor.py').write_text(monitor_backup) diff --git a/tools/test_build_system/test_components.py b/tools/test_build_system/test_components.py index 06856735d5..67e0f035c8 100644 --- a/tools/test_build_system/test_components.py +++ b/tools/test_build_system/test_components.py @@ -1,12 +1,13 @@ # SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 +import json import logging import shutil from pathlib import Path import pytest -from test_build_system_helpers import IdfPyFunc, replace_in_file +from test_build_system_helpers import IdfPyFunc, append_to_file, replace_in_file def test_component_extra_dirs(idf_py: IdfPyFunc, test_app_copy: Path) -> None: @@ -30,3 +31,55 @@ def test_component_names_contain_spaces(idf_py: IdfPyFunc, test_app_copy: Path) (test_app_copy / 'extra component').mkdir() (test_app_copy / 'extra component' / 'CMakeLists.txt').write_text('idf_component_register') idf_py('-DEXTRA_COMPONENT_DIRS="extra component;main"') + + +def test_component_can_not_be_empty_dir(idf_py: IdfPyFunc, test_app_copy: Path) -> None: + logging.info('Empty directory not treated as a component') + empty_component_dir = (test_app_copy / 'components' / 'esp32') + empty_component_dir.mkdir(parents=True) + idf_py('reconfigure') + data = json.load(open(test_app_copy / 'build' / 'project_description.json', 'r')) + assert str(empty_component_dir) not in data.get('build_component_paths') + + +def test_component_subdirs_not_added_to_component_dirs(idf_py: IdfPyFunc, test_app_copy: Path) -> None: + logging.info('If a component directory is added to COMPONENT_DIRS, its subdirectories are not added') + (test_app_copy / 'main' / 'test').mkdir(parents=True) + (test_app_copy / 'main' / 'test' / 'CMakeLists.txt').write_text('idf_component_register()') + idf_py('reconfigure') + data = json.load(open(test_app_copy / 'build' / 'project_description.json', 'r')) + assert str(test_app_copy / 'main' / 'test') not in data.get('build_component_paths') + assert str(test_app_copy / 'main') in data.get('build_component_paths') + + +def test_component_sibling_dirs_not_added_to_component_dirs(idf_py: IdfPyFunc, test_app_copy: Path) -> None: + logging.info('If a component directory is added to COMPONENT_DIRS, its sibling directories are not added') + mycomponents_subdir = (test_app_copy / 'mycomponents') + (mycomponents_subdir / 'mycomponent').mkdir(parents=True) + (mycomponents_subdir / 'mycomponent' / 'CMakeLists.txt').write_text('idf_component_register()') + + # first test by adding single component directory to EXTRA_COMPONENT_DIRS + (mycomponents_subdir / 'esp32').mkdir(parents=True) + (mycomponents_subdir / 'esp32' / 'CMakeLists.txt').write_text('idf_component_register()') + idf_py('-DEXTRA_COMPONENT_DIRS={}'.format(str(mycomponents_subdir / 'mycomponent')), 'reconfigure') + data = json.load(open(test_app_copy / 'build' / 'project_description.json', 'r')) + assert str(mycomponents_subdir / 'esp32') not in data.get('build_component_paths') + assert str(mycomponents_subdir / 'mycomponent') in data.get('build_component_paths') + shutil.rmtree(mycomponents_subdir / 'esp32') + + # now the same thing, but add a components directory + (test_app_copy / 'esp32').mkdir() + (test_app_copy / 'esp32' / 'CMakeLists.txt').write_text('idf_component_register()') + idf_py('-DEXTRA_COMPONENT_DIRS={}'.format(str(mycomponents_subdir)), 'reconfigure') + data = json.load(open(test_app_copy / 'build' / 'project_description.json', 'r')) + assert str(test_app_copy / 'esp32') not in data.get('build_component_paths') + assert str(mycomponents_subdir / 'mycomponent') in data.get('build_component_paths') + + +def test_component_properties_are_set(idf_py: IdfPyFunc, test_app_copy: Path) -> None: + logging.info('Component properties are set') + append_to_file(test_app_copy / 'CMakeLists.txt', '\n'.join(['', + 'idf_component_get_property(srcs main SRCS)', + 'message(STATUS SRCS:${srcs})'])) + ret = idf_py('reconfigure') + assert 'SRCS:{}'.format(test_app_copy / 'main' / 'build_test_app.c') in ret.stdout, 'Component properties should be set' diff --git a/tools/test_build_system/test_sdkconfig.py b/tools/test_build_system/test_sdkconfig.py new file mode 100644 index 0000000000..1b16ff213b --- /dev/null +++ b/tools/test_build_system/test_sdkconfig.py @@ -0,0 +1,26 @@ +# SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 +import logging +from pathlib import Path + +from test_build_system_helpers import IdfPyFunc, file_contains + + +def test_sdkconfig_contains_all_files(idf_py: IdfPyFunc, test_app_copy: Path) -> None: + logging.info('sdkconfig should have contents of all files: sdkconfig, sdkconfig.defaults, sdkconfig.defaults.IDF_TARGET') + (test_app_copy / 'sdkconfig').write_text('CONFIG_PARTITION_TABLE_TWO_OTA=y') + (test_app_copy / 'sdkconfig.defaults').write_text('CONFIG_PARTITION_TABLE_OFFSET=0x10000') + (test_app_copy / 'sdkconfig.defaults.esp32').write_text('CONFIG_ESP_DEFAULT_CPU_FREQ_MHZ_240=y') + idf_py('reconfigure') + assert all([file_contains((test_app_copy / 'sdkconfig'), x) for x in ['CONFIG_PARTITION_TABLE_TWO_OTA=y', + 'CONFIG_PARTITION_TABLE_OFFSET=0x10000', + 'CONFIG_ESP_DEFAULT_CPU_FREQ_MHZ_240=y']]) + + +def test_sdkconfig_multiple_default_files(idf_py: IdfPyFunc, test_app_copy: Path) -> None: + logging.info('should be able to specify multiple sdkconfig default files') + (test_app_copy / 'sdkconfig.defaults1').write_text('CONFIG_PARTITION_TABLE_OFFSET=0x10000') + (test_app_copy / 'sdkconfig.defaults2').write_text('CONFIG_PARTITION_TABLE_TWO_OTA=y') + idf_py('-DSDKCONFIG_DEFAULTS=sdkconfig.defaults1;sdkconfig.defaults2', 'reconfigure') + assert all([file_contains((test_app_copy / 'sdkconfig'), x) for x in ['CONFIG_PARTITION_TABLE_TWO_OTA=y', + 'CONFIG_PARTITION_TABLE_OFFSET=0x10000']])