Merge branch 'feature/rewrite_build_sys_tests_v4' into 'master'

Tools: Rewrite build system unit tests to python - idf.py sdkconfig, bootloader, components

Closes IDF-7164

See merge request espressif/esp-idf!22325

tools/test_build_system/MIGRATION.md

@@ -50,7 +50,7 @@ idf.py fails if IDF_TARGET settings don't match in sdkconfig, CMakeCache.txt, an
 Setting EXTRA_COMPONENT_DIRS works | test_components.py::test_component_extra_dirs |
 Non-existent paths in EXTRA_COMPONENT_DIRS are not allowed | test_components.py::test_component_nonexistent_extra_dirs_not_allowed |
 Component names may contain spaces | test_components.py::test_component_names_contain_spaces |
-sdkconfig should have contents of all files: sdkconfig, sdkconfig.defaults, sdkconfig.defaults.IDF_TARGET | |
+sdkconfig should have contents of all files: sdkconfig, sdkconfig.defaults, sdkconfig.defaults.IDF_TARGET | test_sdkconfig.py::test_sdkconfig_contains_all_files |
 Test if it can build the example to run on host | |
 Test build ESP-IDF as a library to a custom CMake projects for all targets | test_cmake.py::test_build_custom_cmake_project |
 Building a project with CMake library imported and PSRAM workaround, all files compile with workaround | test_cmake.py::test_build_cmake_library_psram_workaround |
@@ -63,22 +63,22 @@ Handling deprecated Kconfig options | test_kconfig.py::test_kconfig_deprecated_o
 Handling deprecated Kconfig options in sdkconfig.defaults | test_kconfig.py::test_kconfig_deprecated_options |
 Can have multiple deprecated Kconfig options map to a single new option | test_kconfig.py::test_kconfig_multiple_and_target_specific_options |
 Can have target specific deprecated Kconfig options | test_kconfig.py::test_kconfig_multiple_and_target_specific_options |
-Confserver can be invoked by idf.py | |
+Confserver can be invoked by idf.py | test_common.py::test_invoke_confserver |
-Check ccache is used to build | |
+Check ccache is used to build | test_common.py::test_ccache_used_to_build |
-Custom bootloader overrides original | |
+Custom bootloader overrides original | test_bootloader.py::test_bootloader_custom_overrides_original |
-Empty directory not treated as a component | |
+Empty directory not treated as a component | test_components.py::test_component_can_not_be_empty_dir |
-If a component directory is added to COMPONENT_DIRS, its subdirectories are not added | |
+If a component directory is added to COMPONENT_DIRS, its subdirectories are not added | test_components.py::test_component_subdirs_not_added_to_component_dirs |
-If a component directory is added to COMPONENT_DIRS, its sibling directories are not added | |
+If a component directory is added to COMPONENT_DIRS, its sibling directories are not added | test_components.py::test_component_sibling_dirs_not_added_to_component_dirs |
-toolchain prefix is set in project description file | |
+toolchain prefix is set in project description file | test_common.py::test_toolchain_prefix_in_description_file |
-Can set options to subcommands: print_filter for monitor | |
+Can set options to subcommands: print_filter for monitor | test_common.py::test_subcommands_with_options |
-Fail on build time works | |
+Fail on build time works | test_build.py::test_build_fail_on_build_time |
-Component properties are set | |
+Component properties are set | test_components.py::test_component_properties_are_set |
-should be able to specify multiple sdkconfig default files | |
+should be able to specify multiple sdkconfig default files | test_sdkconfig.py::test_sdkconfig_multiple_default_files |
 Supports git worktree | |
 idf.py fallback to build system target | |
 Build fails if partitions don't fit in flash | |
 Warning is given if smallest partition is nearly full | |
-Flash size is correctly set in the bootloader image header | |
+Flash size is correctly set in the bootloader image header | test_bootloader.py::test_bootloader_correctly_set_image_header |
 DFU build works | |
 UF2 build works | |
 Loadable ELF build works | |

tools/test_build_system/conftest.py

@@ -139,6 +139,6 @@ def fixture_default_idf_env() -> EnvDict:
 
 @pytest.fixture
 def idf_py(default_idf_env: EnvDict) -> IdfPyFunc:
-    def result(*args: str, check: bool = True) -> subprocess.CompletedProcess:
+    def result(*args: str, check: bool = True, input_str: typing.Optional[str] = None) -> subprocess.CompletedProcess:
-        return run_idf_py(*args, env=default_idf_env, workdir=os.getcwd(), check=check)  # type: ignore
+        return run_idf_py(*args, env=default_idf_env, workdir=os.getcwd(), check=check, input_str=input_str)  # type: ignore
     return result

tools/test_build_system/test_bootloader.py

@@ -0,0 +1,53 @@
+# SPDX-FileCopyrightText: 2022-2023 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Apache-2.0
+import logging
+import shutil
+from pathlib import Path
+from typing import Union
+
+from test_build_system_helpers import EnvDict, IdfPyFunc, file_contains
+
+
+def get_two_header_bytes(file_path: Union[str, Path]) -> str:
+    '''
+    get the bytes 3-4 of the given file
+    https://docs.espressif.com/projects/esptool/en/latest/esp32/advanced-topics/firmware-image-format.html
+    '''
+    data = b''
+    with open(file_path, 'rb') as f:
+        data = f.read(4)
+    extracted_bytes = data[2:4]
+    return extracted_bytes.hex()
+
+
+def test_bootloader_custom_overrides_original(test_app_copy: Path, idf_py: IdfPyFunc, default_idf_env: EnvDict) -> None:
+    logging.info('Custom bootloader overrides original')
+    idf_path = Path(default_idf_env.get('IDF_PATH'))
+    shutil.copytree(idf_path / 'components' / 'bootloader', test_app_copy / 'components' / 'bootloader')
+    # Because of relative include of Kconfig, also esp_bootloader_format needs to be copied.
+    shutil.copytree(idf_path / 'components' / 'esp_bootloader_format', test_app_copy / 'components' / 'esp_bootloader_format')
+    idf_py('bootloader')
+    assert file_contains(test_app_copy / 'build' / 'bootloader' / 'compile_commands.json',
+                         str(test_app_copy / 'components' / 'bootloader' / 'subproject' / 'main' / 'bootloader_start.c'))
+
+
+def test_bootloader_correctly_set_image_header(test_app_copy: Path, idf_py: IdfPyFunc) -> None:
+    logging.info('Flash size is correctly set in the bootloader image header')
+    # Build with the default 2MB setting
+    idf_py('bootloader')
+    assert get_two_header_bytes(test_app_copy / 'build' / 'bootloader' / 'bootloader.bin') == '0210'
+
+    # Change to 4MB
+    (test_app_copy / 'sdkconfig').write_text('CONFIG_ESPTOOLPY_FLASHSIZE_4MB=y')
+    idf_py('reconfigure', 'bootloader')
+    assert get_two_header_bytes(test_app_copy / 'build' / 'bootloader' / 'bootloader.bin') == '0220'
+
+    # Change to QIO, bootloader should still be DIO (will change to QIO in 2nd stage bootloader)
+    (test_app_copy / 'sdkconfig').write_text('CONFIG_FLASHMODE_QIO=y')
+    idf_py('reconfigure', 'bootloader')
+    assert get_two_header_bytes(test_app_copy / 'build' / 'bootloader' / 'bootloader.bin') == '0210'
+
+    # Change to 80 MHz
+    (test_app_copy / 'sdkconfig').write_text('CONFIG_ESPTOOLPY_FLASHFREQ_80M=y')
+    idf_py('reconfigure', 'bootloader')
+    assert get_two_header_bytes(test_app_copy / 'build' / 'bootloader' / 'bootloader.bin') == '021f'

tools/test_build_system/test_build.py

@@ -128,3 +128,15 @@ def test_build_with_sdkconfig_build_abspath(idf_py: IdfPyFunc, test_app_copy: Pa
     build_path = test_app_copy / 'build_tmp'
     sdkconfig_path = build_path / 'sdkconfig'
     idf_py('-D', f'SDKCONFIG={sdkconfig_path}', '-B', str(build_path), 'build')
+
+
+def test_build_fail_on_build_time(idf_py: IdfPyFunc, test_app_copy: Path) -> None:
+    logging.info('Fail on build time works')
+    append_to_file(test_app_copy / 'CMakeLists.txt', '\n'.join(['',
+                                                                'if(NOT EXISTS "${CMAKE_CURRENT_LIST_DIR}/hello.txt")',
+                                                                'fail_at_build_time(test_file "hello.txt does not exists")',
+                                                                'endif()']))
+    ret = idf_py('build', check=False)
+    assert ret.returncode != 0, 'Build should fail if requirements are not satisfied'
+    (test_app_copy / 'hello.txt').touch()
+    idf_py('build')

tools/test_build_system/test_build_system_helpers/idf_utils.py

@@ -61,7 +61,8 @@ def run_idf_py(*args: str,
                idf_path: typing.Optional[typing.Union[str,Path]] = None,
                workdir: typing.Optional[str] = None,
                check: bool = True,
-               python: typing.Optional[str] = None) -> subprocess.CompletedProcess:
+               python: typing.Optional[str] = None,
+               input_str: typing.Optional[str] = None) -> subprocess.CompletedProcess:
     """
     Run idf.py command with given arguments, raise an exception on failure
     :param args: arguments to pass to idf.py
@@ -70,19 +71,19 @@ def run_idf_py(*args: str,
     :param workdir: directory where to run the build; if not set, the current directory is used
     :param check: check process exits with a zero exit code, if false all retvals are accepted without failing the test
     :param python: absolute path to python interpreter
+    :param input_str: input to idf.py
     """
-    env_dict = dict(**os.environ)
+    if not env:
-    if env is not None:
+        env = dict(**os.environ)
-        env_dict.update(env)
     if not workdir:
         workdir = os.getcwd()
     # order: function argument -> value in env dictionary -> system environment
     if idf_path is None:
-        idf_path = env_dict.get('IDF_PATH')
+        idf_path = env.get('IDF_PATH')
         if not idf_path:
             raise ValueError('IDF_PATH must be set in the env array if idf_path argument is not set')
     if python is None:
-        python = find_python(env_dict['PATH'])
+        python = find_python(env['PATH'])
 
     cmd = [
         python,
@@ -91,9 +92,9 @@ def run_idf_py(*args: str,
     cmd += args  # type: ignore
     logging.debug('running {} in {}'.format(' '.join(cmd), workdir))
     return subprocess.run(
-        cmd, env=env_dict, cwd=workdir,
+        cmd, env=env, cwd=workdir,
         check=check, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
-        text=True, encoding='utf-8', errors='backslashreplace')
+        text=True, encoding='utf-8', errors='backslashreplace', input=input_str)
 
 
 def run_cmake(*cmake_args: str, env: typing.Optional[EnvDict] = None,

tools/test_build_system/test_common.py

@@ -1,5 +1,6 @@
 # SPDX-FileCopyrightText: 2022-2023 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
+import json
 import logging
 import os
 import re
@@ -12,8 +13,7 @@ from pathlib import Path
 from typing import List
 
 import pytest
-from _pytest.monkeypatch import MonkeyPatch
+from test_build_system_helpers import EnvDict, IdfPyFunc, find_python, get_snapshot, replace_in_file, run_idf_py
-from test_build_system_helpers import IdfPyFunc, find_python, get_snapshot, replace_in_file, run_idf_py
 
 
 def get_subdirs_absolute_paths(path: Path) -> List[str]:
@@ -78,14 +78,13 @@ def test_idf_copy(idf_copy: Path, idf_py: IdfPyFunc) -> None:
 
 def test_idf_build_with_env_var_sdkconfig_defaults(
     test_app_copy: Path,
-    idf_py: IdfPyFunc,
+    default_idf_env: EnvDict
-    monkeypatch: MonkeyPatch,
 ) -> None:
     with open(test_app_copy / 'sdkconfig.test', 'w') as fw:
         fw.write('CONFIG_BT_ENABLED=y')
 
-    monkeypatch.setenv('SDKCONFIG_DEFAULTS', 'sdkconfig.test')
+    default_idf_env['SDKCONFIG_DEFAULTS'] = 'sdkconfig.test'
-    idf_py('build')
+    run_idf_py('build', env=default_idf_env)
 
     with open(test_app_copy / 'sdkconfig') as fr:
         assert 'CONFIG_BT_ENABLED=y' in fr.read()
@@ -93,12 +92,11 @@ def test_idf_build_with_env_var_sdkconfig_defaults(
 
 @pytest.mark.usefixtures('test_app_copy')
 @pytest.mark.test_app_copy('examples/system/efuse')
-def test_efuse_symmary_cmake_functions(
+def test_efuse_summary_cmake_functions(
-    idf_py: IdfPyFunc,
+    default_idf_env: EnvDict
-    monkeypatch: MonkeyPatch
 ) -> None:
-    monkeypatch.setenv('IDF_CI_BUILD', '1')
+    default_idf_env['IDF_CI_BUILD'] = '1'
-    output = idf_py('efuse-summary')
+    output = run_idf_py('efuse-summary', env=default_idf_env)
     assert 'FROM_CMAKE: MAC: 00:00:00:00:00:00' in output.stdout
     assert 'FROM_CMAKE: WR_DIS: 0' in output.stdout
 
@@ -158,3 +156,50 @@ def test_python_interpreter_win(test_app_copy: Path) -> None:
     # python is loaded from env:$PATH, but since false interpreter is provided there, python needs to be specified as argument
     # if idf.py is reconfigured during it's execution, it would load a false interpreter
     run_idf_py('reconfigure', env=env_dict, python=python)
+
+
+@pytest.mark.usefixtures('test_app_copy')
+def test_invoke_confserver(idf_py: IdfPyFunc) -> None:
+    logging.info('Confserver can be invoked by idf.py')
+    idf_py('confserver', input_str='{"version": 1}')
+
+
+def test_ccache_used_to_build(test_app_copy: Path) -> None:
+    logging.info('Check ccache is used to build')
+    (test_app_copy / 'ccache').touch(mode=0o755)
+    env_dict = dict(**os.environ)
+    env_dict['PATH'] = str(test_app_copy) + os.pathsep + env_dict['PATH']
+    # Disable using ccache automatically
+    if 'IDF_CCACHE_ENABLE' in env_dict:
+        env_dict.pop('IDF_CCACHE_ENABLE')
+
+    ret = run_idf_py('--ccache', 'reconfigure', env=env_dict)
+    assert 'ccache will be used' in ret.stdout
+    run_idf_py('fullclean', env=env_dict)
+    ret = run_idf_py('reconfigure', env=env_dict)
+    assert 'ccache will be used' not in ret.stdout
+    ret = run_idf_py('--no-ccache', 'reconfigure', env=env_dict)
+    assert 'ccache will be used' not in ret.stdout
+
+
+def test_toolchain_prefix_in_description_file(idf_py: IdfPyFunc, test_app_copy: Path) -> None:
+    logging.info('Toolchain prefix is set in project description file')
+    idf_py('reconfigure')
+    data = json.load(open(test_app_copy / 'build' / 'project_description.json', 'r'))
+    assert 'monitor_toolprefix' in data
+
+
+@pytest.mark.usefixtures('test_app_copy')
+def test_subcommands_with_options(idf_py: IdfPyFunc, default_idf_env: EnvDict) -> None:
+    logging.info('Can set options to subcommands: print_filter for monitor')
+    idf_path = Path(default_idf_env.get('IDF_PATH'))
+    # try - finally block is here used to backup and restore idf_monitor.py
+    # since we need to handle only one file, this souluton is much faster than using idf_copy fixture
+    monitor_backup = (idf_path / 'tools' / 'idf_monitor.py').read_text()
+    try:
+        (idf_path / 'tools' / 'idf_monitor.py').write_text('import sys;print(sys.argv[1:])')
+        idf_py('build')
+        ret = idf_py('monitor', '--print-filter=*:I', '-p', 'tty.fake')
+        assert "'--print_filter', '*:I'" in ret.stdout
+    finally:
+        (idf_path / 'tools' / 'idf_monitor.py').write_text(monitor_backup)

tools/test_build_system/test_components.py

@@ -1,12 +1,13 @@
 # SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
 
+import json
 import logging
 import shutil
 from pathlib import Path
 
 import pytest
-from test_build_system_helpers import IdfPyFunc, replace_in_file
+from test_build_system_helpers import IdfPyFunc, append_to_file, replace_in_file
 
 
 def test_component_extra_dirs(idf_py: IdfPyFunc, test_app_copy: Path) -> None:
@@ -30,3 +31,55 @@ def test_component_names_contain_spaces(idf_py: IdfPyFunc, test_app_copy: Path)
     (test_app_copy / 'extra component').mkdir()
     (test_app_copy / 'extra component' / 'CMakeLists.txt').write_text('idf_component_register')
     idf_py('-DEXTRA_COMPONENT_DIRS="extra component;main"')
+
+
+def test_component_can_not_be_empty_dir(idf_py: IdfPyFunc, test_app_copy: Path) -> None:
+    logging.info('Empty directory not treated as a component')
+    empty_component_dir = (test_app_copy / 'components' / 'esp32')
+    empty_component_dir.mkdir(parents=True)
+    idf_py('reconfigure')
+    data = json.load(open(test_app_copy / 'build' / 'project_description.json', 'r'))
+    assert str(empty_component_dir) not in data.get('build_component_paths')
+
+
+def test_component_subdirs_not_added_to_component_dirs(idf_py: IdfPyFunc, test_app_copy: Path) -> None:
+    logging.info('If a component directory is added to COMPONENT_DIRS, its subdirectories are not added')
+    (test_app_copy / 'main' / 'test').mkdir(parents=True)
+    (test_app_copy / 'main' / 'test' / 'CMakeLists.txt').write_text('idf_component_register()')
+    idf_py('reconfigure')
+    data = json.load(open(test_app_copy / 'build' / 'project_description.json', 'r'))
+    assert str(test_app_copy / 'main' / 'test') not in data.get('build_component_paths')
+    assert str(test_app_copy / 'main') in data.get('build_component_paths')
+
+
+def test_component_sibling_dirs_not_added_to_component_dirs(idf_py: IdfPyFunc, test_app_copy: Path) -> None:
+    logging.info('If a component directory is added to COMPONENT_DIRS, its sibling directories are not added')
+    mycomponents_subdir = (test_app_copy / 'mycomponents')
+    (mycomponents_subdir / 'mycomponent').mkdir(parents=True)
+    (mycomponents_subdir / 'mycomponent' / 'CMakeLists.txt').write_text('idf_component_register()')
+
+    # first test by adding single component directory to EXTRA_COMPONENT_DIRS
+    (mycomponents_subdir / 'esp32').mkdir(parents=True)
+    (mycomponents_subdir / 'esp32' / 'CMakeLists.txt').write_text('idf_component_register()')
+    idf_py('-DEXTRA_COMPONENT_DIRS={}'.format(str(mycomponents_subdir / 'mycomponent')), 'reconfigure')
+    data = json.load(open(test_app_copy / 'build' / 'project_description.json', 'r'))
+    assert str(mycomponents_subdir / 'esp32') not in data.get('build_component_paths')
+    assert str(mycomponents_subdir / 'mycomponent') in data.get('build_component_paths')
+    shutil.rmtree(mycomponents_subdir / 'esp32')
+
+    # now the same thing, but add a components directory
+    (test_app_copy / 'esp32').mkdir()
+    (test_app_copy / 'esp32' / 'CMakeLists.txt').write_text('idf_component_register()')
+    idf_py('-DEXTRA_COMPONENT_DIRS={}'.format(str(mycomponents_subdir)), 'reconfigure')
+    data = json.load(open(test_app_copy / 'build' / 'project_description.json', 'r'))
+    assert str(test_app_copy / 'esp32') not in data.get('build_component_paths')
+    assert str(mycomponents_subdir / 'mycomponent') in data.get('build_component_paths')
+
+
+def test_component_properties_are_set(idf_py: IdfPyFunc, test_app_copy: Path) -> None:
+    logging.info('Component properties are set')
+    append_to_file(test_app_copy / 'CMakeLists.txt', '\n'.join(['',
+                                                                'idf_component_get_property(srcs main SRCS)',
+                                                                'message(STATUS SRCS:${srcs})']))
+    ret = idf_py('reconfigure')
+    assert 'SRCS:{}'.format(test_app_copy / 'main' / 'build_test_app.c') in ret.stdout, 'Component properties should be set'

tools/test_build_system/test_sdkconfig.py

@@ -0,0 +1,26 @@
+# SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Apache-2.0
+import logging
+from pathlib import Path
+
+from test_build_system_helpers import IdfPyFunc, file_contains
+
+
+def test_sdkconfig_contains_all_files(idf_py: IdfPyFunc, test_app_copy: Path) -> None:
+    logging.info('sdkconfig should have contents of all files: sdkconfig, sdkconfig.defaults, sdkconfig.defaults.IDF_TARGET')
+    (test_app_copy / 'sdkconfig').write_text('CONFIG_PARTITION_TABLE_TWO_OTA=y')
+    (test_app_copy / 'sdkconfig.defaults').write_text('CONFIG_PARTITION_TABLE_OFFSET=0x10000')
+    (test_app_copy / 'sdkconfig.defaults.esp32').write_text('CONFIG_ESP_DEFAULT_CPU_FREQ_MHZ_240=y')
+    idf_py('reconfigure')
+    assert all([file_contains((test_app_copy / 'sdkconfig'), x) for x in ['CONFIG_PARTITION_TABLE_TWO_OTA=y',
+                                                                          'CONFIG_PARTITION_TABLE_OFFSET=0x10000',
+                                                                          'CONFIG_ESP_DEFAULT_CPU_FREQ_MHZ_240=y']])
+
+
+def test_sdkconfig_multiple_default_files(idf_py: IdfPyFunc, test_app_copy: Path) -> None:
+    logging.info('should be able to specify multiple sdkconfig default files')
+    (test_app_copy / 'sdkconfig.defaults1').write_text('CONFIG_PARTITION_TABLE_OFFSET=0x10000')
+    (test_app_copy / 'sdkconfig.defaults2').write_text('CONFIG_PARTITION_TABLE_TWO_OTA=y')
+    idf_py('-DSDKCONFIG_DEFAULTS=sdkconfig.defaults1;sdkconfig.defaults2', 'reconfigure')
+    assert all([file_contains((test_app_copy / 'sdkconfig'), x) for x in ['CONFIG_PARTITION_TABLE_TWO_OTA=y',
+                                                                          'CONFIG_PARTITION_TABLE_OFFSET=0x10000']])