Merge branch 'feature/rewrite_build_sys_tests_v5' into 'master'

Feature/rewrite build sys tests - partitions, components, docs

Closes IDF-7257

See merge request espressif/esp-idf!23296

tools/test_build_system/MIGRATION.md

@@ -57,7 +57,7 @@ Building a project with CMake library imported and PSRAM workaround, all files c
 Test for external libraries in custom CMake projects with ESP-IDF components linked | test_cmake.py::test_build_custom_cmake_project |
 Test for external libraries in custom CMake projects with PSRAM strategy $strat | test_cmake.py::test_build_cmake_library_psram_strategies |
 Cleaning Python bytecode | test_common.py::test_python_clean |
-Displays partition table when executing target partition_table | test_common.py::test_partition_table |
+Displays partition table when executing target partition_table | test_partition.py::test_partition_table |
 Make sure a full build never runs '/usr/bin/env python' or similar | test_common.py::test_python_interpreter_unix, test_common.py::test_python_interpreter_win |
 Handling deprecated Kconfig options | test_kconfig.py::test_kconfig_deprecated_options |
 Handling deprecated Kconfig options in sdkconfig.defaults | test_kconfig.py::test_kconfig_deprecated_options |
@@ -75,25 +75,25 @@ Fail on build time works | test_build.py::test_build_fail_on_build_time |
 Component properties are set | test_components.py::test_component_properties_are_set |
 should be able to specify multiple sdkconfig default files | test_sdkconfig.py::test_sdkconfig_multiple_default_files |
 Supports git worktree | |
-idf.py fallback to build system target | |
+idf.py fallback to build system target | test_common.py::test_fallback_to_build_system_target |
-Build fails if partitions don't fit in flash | |
+Build fails if partitions don't fit in flash | test_partition.py::test_partitions_dont_fit_in_flash |
-Warning is given if smallest partition is nearly full | |
+Warning is given if smallest partition is nearly full | test_partition.py::test_partition_nearly_full_warning |
 Flash size is correctly set in the bootloader image header | test_bootloader.py::test_bootloader_correctly_set_image_header |
-DFU build works | |
+DFU build works | test_build.py::test_build_dfu |
-UF2 build works | |
+UF2 build works | test_build.py::test_build_uf2 |
-Loadable ELF build works | |
+Loadable ELF build works | test_build.py::test_build_loadable_elf |
-Defaults set properly for unspecified idf_build_process args | |
+Defaults set properly for unspecified idf_build_process args | test_cmake.py::test_defaults_for_unspecified_idf_build_process_args |
-Getting component overriden dir | |
+Getting component overriden dir | test_components.py::test_component_overriden_dir |
-Overriding Kconfig | |
+Overriding Kconfig | test_components.py::test_component_overriden_dir |
-Project components prioritized over EXTRA_COMPONENT_DIRS | |
+Project components prioritized over EXTRA_COMPONENT_DIRS | test_components.py::test_components_prioritizer_over_extra_components_dir |
-Components in EXCLUDE_COMPONENTS not passed to idf_component_manager | |
+Components in EXCLUDE_COMPONENTS not passed to idf_component_manager | test_components.py::test_exclude_components_not_passed |
-Create project using idf.py and build it | |
+Create project using idf.py and build it | test_common.py::test_create_component_and_project_plus_build |
-Create component using idf.py, create project using idf.py. | |
+Create component using idf.py, create project using idf.py. | test_common.py::test_create_component_and_project_plus_build |
-Add the component to the created project and build the project. | |
+Add the component to the created project and build the project. | test_common.py::test_create_component_and_project_plus_build |
-Check that command for creating new project will fail if the target folder is not empty. | |
+Check that command for creating new project will fail if the target folder is not empty. | test_common.py::test_create_project |
-Check that command for creating new project will fail if the target path is file. | |
+Check that command for creating new project will fail if the target path is file. | test_common.py::test_create_project |
-Check docs command | |
+Check docs command | test_common.py::test_docs_command |
-Deprecation warning check | |
+Deprecation warning check | test_common.py::test_deprecation_warning |
 Save-defconfig checks | |
 test_build | |
 test_build_ulp_fsm | |

tools/test_build_system/test_build.py

@@ -140,3 +140,41 @@ def test_build_fail_on_build_time(idf_py: IdfPyFunc, test_app_copy: Path) -> Non
     assert ret.returncode != 0, 'Build should fail if requirements are not satisfied'
     (test_app_copy / 'hello.txt').touch()
     idf_py('build')
+
+
+@pytest.mark.usefixtures('test_app_copy')
+def test_build_dfu(idf_py: IdfPyFunc) -> None:
+    logging.info('DFU build works')
+    ret = idf_py('dfu', check=False)
+    assert 'command "dfu" is not known to idf.py and is not a Ninja target' in ret.stderr, 'DFU build should fail for default chip target'
+    idf_py('set-target', 'esp32s2')
+    ret = idf_py('dfu')
+    assert 'build/dfu.bin" has been written. You may proceed with DFU flashing.' in ret.stdout, 'DFU build should succeed for esp32s2'
+    assert_built(BOOTLOADER_BINS + APP_BINS + PARTITION_BIN + ['build/dfu.bin'])
+
+
+@pytest.mark.usefixtures('test_app_copy')
+def test_build_uf2(idf_py: IdfPyFunc) -> None:
+    logging.info('UF2 build works')
+    ret = idf_py('uf2')
+    assert 'build/uf2.bin" has been written.' in ret.stdout, 'UF2 build should work for esp32'
+    assert_built(BOOTLOADER_BINS + APP_BINS + PARTITION_BIN + ['build/uf2.bin'])
+    ret = idf_py('uf2-app')
+    assert 'build/uf2-app.bin" has been written.' in ret.stdout, 'UF2 build should work for application binary'
+    assert_built(['build/uf2-app.bin'])
+    idf_py('set-target', 'esp32s2')
+    ret = idf_py('uf2')
+    assert 'build/uf2.bin" has been written.' in ret.stdout, 'UF2 build should work for esp32s2'
+    assert_built(BOOTLOADER_BINS + APP_BINS + PARTITION_BIN + ['build/uf2.bin'])
+
+
+def test_build_loadable_elf(idf_py: IdfPyFunc, test_app_copy: Path) -> None:
+    logging.info('Loadable ELF build works')
+    (test_app_copy / 'sdkconfig').write_text('\n'.join(['CONFIG_APP_BUILD_TYPE_RAM=y',
+                                                        'CONFIG_VFS_SUPPORT_TERMIOS=n',
+                                                        'CONFIG_NEWLIB_NANO_FORMAT=y',
+                                                        'CONFIG_ESP_SYSTEM_PANIC_PRINT_HALT=y',
+                                                        'CONFIG_ESP_ERR_TO_NAME_LOOKUP=n']))
+    idf_py('reconfigure')
+    assert (test_app_copy / 'build' / 'flasher_args.json').exists(), 'flasher_args.json should be generated in a loadable ELF build'
+    idf_py('build')

tools/test_build_system/test_build_system_helpers/idf_utils.py

@@ -97,23 +97,32 @@ def run_idf_py(*args: str,
         text=True, encoding='utf-8', errors='backslashreplace', input=input_str)
 
 
-def run_cmake(*cmake_args: str, env: typing.Optional[EnvDict] = None,
+def run_cmake(*cmake_args: str,
-              check: bool = True) -> subprocess.CompletedProcess:
+              env: typing.Optional[EnvDict] = None,
+              check: bool = True,
+              workdir: typing.Optional[Union[Path,str]] = None) -> subprocess.CompletedProcess:
     """
     Run cmake command with given arguments, raise an exception on failure
     :param cmake_args: arguments to pass cmake
     :param env: environment variables to run the cmake with; if not set, the default environment is used
+    :param check: check process exits with a zero exit code, if false all retvals are accepted without failing the test
+    :param workdir: directory where to run cmake; if not set, the current directory is used
     """
     if not env:
         env = dict(**os.environ)
-    workdir = (Path(os.getcwd()) / 'build')
+
-    workdir.mkdir(parents=True, exist_ok=True)
+    if workdir:
+        build_dir = Path(workdir, 'build')
+    else:
+        build_dir = (Path(os.getcwd()) / 'build')
+
+    build_dir.mkdir(parents=True, exist_ok=True)
 
     cmd = ['cmake'] + list(cmake_args)
 
-    logging.debug('running {} in {}'.format(' '.join(cmd), workdir))
+    logging.debug('running {} in {}'.format(' '.join(cmd), build_dir))
     return subprocess.run(
-        cmd, env=env, cwd=workdir,
+        cmd, env=env, cwd=build_dir,
         check=check, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
         text=True, encoding='utf-8', errors='backslashreplace')
 

tools/test_build_system/test_cmake.py

@@ -6,7 +6,8 @@ import re
 import shutil
 from pathlib import Path
 
-from test_build_system_helpers import IdfPyFunc, file_contains, run_cmake, run_cmake_and_build
+import pytest
+from test_build_system_helpers import EnvDict, IdfPyFunc, append_to_file, file_contains, run_cmake, run_cmake_and_build
 
 
 def test_build_custom_cmake_project(test_app_copy: Path) -> None:
@@ -47,3 +48,18 @@ def test_build_cmake_library_psram_strategies(idf_py: IdfPyFunc, test_app_copy:
                 assert f'mfix-esp32-psram-cache-strategy={strategy.lower()}' in r, ('All commands in compile_commands.json '
                                                                                     'should use PSRAM cache workaround strategy')
         (test_app_copy / 'sdkconfig').unlink()
+
+
+@pytest.mark.usefixtures('test_app_copy')
+@pytest.mark.usefixtures('idf_copy')
+def test_defaults_for_unspecified_idf_build_process_args(default_idf_env: EnvDict) -> None:
+    logging.info('Defaults set properly for unspecified idf_build_process args')
+    idf_path = Path(default_idf_env.get('IDF_PATH'))
+    idf_as_lib_path = idf_path / 'examples' / 'build_system' / 'cmake' / 'idf_as_lib'
+    append_to_file(idf_as_lib_path / 'CMakeLists.txt', '\n'.join(['idf_build_get_property(project_dir PROJECT_DIR)',
+                                                                  'message("Project directory: ${project_dir}")']))
+    ret = run_cmake('..',
+                    '-DCMAKE_TOOLCHAIN_FILE={}'.format(str(idf_path / 'tools' / 'cmake' / 'toolchain-esp32.cmake')),
+                    '-DTARGET=esp32',
+                    workdir=idf_as_lib_path)
+    assert 'Project directory: {}'.format(str(idf_as_lib_path)) in ret.stderr

tools/test_build_system/test_common.py

@@ -3,7 +3,6 @@
 import json
 import logging
 import os
-import re
 import shutil
 import stat
 import subprocess
@@ -13,7 +12,8 @@ from pathlib import Path
 from typing import List
 
 import pytest
-from test_build_system_helpers import EnvDict, IdfPyFunc, find_python, get_snapshot, replace_in_file, run_idf_py
+from test_build_system_helpers import (EnvDict, IdfPyFunc, append_to_file, find_python, get_snapshot, replace_in_file,
+                                       run_idf_py)
 
 
 def get_subdirs_absolute_paths(path: Path) -> List[str]:
@@ -118,13 +118,6 @@ def test_python_clean(idf_py: IdfPyFunc) -> None:
     assert len(abs_paths_suffix) == 0
 
 
-@pytest.mark.usefixtures('test_app_copy')
-def test_partition_table(idf_py: IdfPyFunc) -> None:
-    logging.info('Displays partition table when executing target partition_table')
-    output = idf_py('partition-table')
-    assert re.search('# ESP-IDF.+Partition Table', output.stdout)
-
-
 @pytest.mark.skipif(sys.platform == 'win32', reason='Windows does not support executing bash script')
 def test_python_interpreter_unix(test_app_copy: Path) -> None:
     logging.info("Make sure idf.py never runs '/usr/bin/env python' or similar")
@@ -203,3 +196,70 @@ def test_subcommands_with_options(idf_py: IdfPyFunc, default_idf_env: EnvDict) -
         assert "'--print_filter', '*:I'" in ret.stdout
     finally:
         (idf_path / 'tools' / 'idf_monitor.py').write_text(monitor_backup)
+
+
+def test_fallback_to_build_system_target(idf_py: IdfPyFunc, test_app_copy: Path) -> None:
+    logging.info('idf.py fallback to build system target')
+    msg = 'Custom target is running'
+    append_to_file(test_app_copy / 'CMakeLists.txt',
+                   'add_custom_target(custom_target COMMAND ${{CMAKE_COMMAND}} -E echo "{}")'.format(msg))
+    ret = idf_py('custom_target')
+    assert msg in ret.stdout, 'Custom target did not produce expected output'
+
+
+def test_create_component_and_project_plus_build(idf_copy: Path) -> None:
+    logging.info('Create project and component using idf.py and build it')
+    run_idf_py('-C', 'projects', 'create-project', 'temp_test_project', workdir=idf_copy)
+    run_idf_py('-C', 'components', 'create-component', 'temp_test_component', workdir=idf_copy)
+    replace_in_file(idf_copy / 'projects' / 'temp_test_project' / 'main' / 'temp_test_project.c', '{\n\n}',
+                    '\n'.join(['{', '\tfunc();', '}']))
+    replace_in_file(idf_copy / 'projects' / 'temp_test_project' / 'main' / 'temp_test_project.c', '#include <stdio.h>',
+                    '\n'.join(['#include <stdio.h>', '#include "temp_test_component.h"']))
+    run_idf_py('build', workdir=(idf_copy / 'projects' / 'temp_test_project'))
+
+
+# In this test function, there are actually two logical tests in one test function.
+# It would be better to have every check in a separate
+# test case, but that would mean doing idf_copy each time, and copying takes most of the time
+def test_create_project(idf_py: IdfPyFunc, idf_copy: Path) -> None:
+    logging.info('Check that command for creating new project will fail if the target folder is not empty.')
+    (idf_copy / 'example_proj').mkdir()
+    (idf_copy / 'example_proj' / 'tmp_1').touch()
+    ret = idf_py('create-project', '--path', str(idf_copy / 'example_proj'), 'temp_test_project', check=False)
+    assert ret.returncode == 3, 'Command create-project exit value is wrong.'
+
+    # cleanup for the following test
+    shutil.rmtree(idf_copy / 'example_proj')
+
+    logging.info('Check that command for creating new project will fail if the target path is file.')
+    (idf_copy / 'example_proj_file').touch()
+    ret = idf_py('create-project', '--path', str(idf_copy / 'example_proj_file'), 'temp_test_project', check=False)
+    assert ret.returncode == 4, 'Command create-project exit value is wrong.'
+
+
+@pytest.mark.usefixtures('test_app_copy')
+def test_docs_command(idf_py: IdfPyFunc) -> None:
+    logging.info('Check docs command')
+    idf_py('set-target', 'esp32')
+    ret = idf_py('docs', '--no-browser')
+    assert 'https://docs.espressif.com/projects/esp-idf/en' in ret.stdout
+    ret = idf_py('docs', '--no-browser', '--language', 'en')
+    assert 'https://docs.espressif.com/projects/esp-idf/en' in ret.stdout
+    ret = idf_py('docs', '--no-browser', '--language', 'en', '--version', 'v4.2.1')
+    assert 'https://docs.espressif.com/projects/esp-idf/en/v4.2.1' in ret.stdout
+    ret = idf_py('docs', '--no-browser', '--language', 'en', '--version', 'v4.2.1', '--target', 'esp32')
+    assert 'https://docs.espressif.com/projects/esp-idf/en/v4.2.1/esp32' in ret.stdout
+    ret = idf_py('docs', '--no-browser', '--language', 'en', '--version', 'v4.2.1', '--target', 'esp32', '--starting-page', 'get-started')
+    assert 'https://docs.espressif.com/projects/esp-idf/en/v4.2.1/esp32/get-started' in ret.stdout
+
+
+@pytest.mark.usefixtures('test_app_copy')
+def test_deprecation_warning(idf_py: IdfPyFunc) -> None:
+    logging.info('Deprecation warning check')
+    ret = idf_py('post_debug', check=False)
+    # click warning
+    assert 'Error: Command "post_debug" is deprecated since v4.4 and was removed in v5.0.' in ret.stderr
+
+    ret = idf_py('efuse_common_table', check=False)
+    # cmake warning
+    assert 'Have you wanted to run "efuse-common-table" instead?' in ret.stdout

tools/test_build_system/test_components.py

@@ -7,7 +7,7 @@ import shutil
 from pathlib import Path
 
 import pytest
-from test_build_system_helpers import IdfPyFunc, append_to_file, replace_in_file
+from test_build_system_helpers import EnvDict, IdfPyFunc, append_to_file, replace_in_file
 
 
 def test_component_extra_dirs(idf_py: IdfPyFunc, test_app_copy: Path) -> None:
@@ -83,3 +83,46 @@ def test_component_properties_are_set(idf_py: IdfPyFunc, test_app_copy: Path) ->
                                                                 'message(STATUS SRCS:${srcs})']))
     ret = idf_py('reconfigure')
     assert 'SRCS:{}'.format(test_app_copy / 'main' / 'build_test_app.c') in ret.stdout, 'Component properties should be set'
+
+
+def test_component_overriden_dir(idf_py: IdfPyFunc, test_app_copy: Path, default_idf_env: EnvDict) -> None:
+    logging.info('Getting component overriden dir')
+    (test_app_copy / 'components' / 'hal').mkdir(parents=True)
+    (test_app_copy / 'components' / 'hal' / 'CMakeLists.txt').write_text('\n'.join([
+        'idf_component_get_property(overriden_dir ${COMPONENT_NAME} COMPONENT_OVERRIDEN_DIR)',
+        'message(STATUS overriden_dir:${overriden_dir})']))
+    ret = idf_py('reconfigure')
+    idf_path = Path(default_idf_env.get('IDF_PATH'))
+    # no registration, overrides registration as well
+    assert 'overriden_dir:{}'.format(idf_path / 'components' / 'hal') in ret.stdout, 'Failed to get overriden dir'
+    append_to_file((test_app_copy / 'components' / 'hal' / 'CMakeLists.txt'), '\n'.join([
+        '',
+        'idf_component_register(KCONFIG ${overriden_dir}/Kconfig)',
+        'idf_component_get_property(kconfig ${COMPONENT_NAME} KCONFIG)',
+        'message(STATUS kconfig:${overriden_dir}/Kconfig)']))
+    ret = idf_py('reconfigure', check=False)
+    assert 'kconfig:{}'.format(idf_path / 'components' / 'hal') in ret.stdout, 'Failed to verify original `main` directory'
+
+
+def test_components_prioritizer_over_extra_components_dir(idf_py: IdfPyFunc, test_app_copy: Path) -> None:
+    logging.info('Project components prioritized over EXTRA_COMPONENT_DIRS')
+    (test_app_copy / 'extra_dir' / 'my_component').mkdir(parents=True)
+    (test_app_copy / 'extra_dir' / 'my_component' / 'CMakeLists.txt').write_text('idf_component_register()')
+    replace_in_file(test_app_copy / 'CMakeLists.txt',
+                    '# placeholder_before_include_project_cmake',
+                    'set(EXTRA_COMPONENT_DIRS extra_dir)')
+    ret = idf_py('reconfigure')
+    assert str(test_app_copy / 'extra_dir' / 'my_component') in ret.stdout, 'Unable to find component specified in EXTRA_COMPONENT_DIRS'
+    (test_app_copy / 'components' / 'my_component').mkdir(parents=True)
+    (test_app_copy / 'components' / 'my_component' / 'CMakeLists.txt').write_text('idf_component_register()')
+    ret = idf_py('reconfigure')
+    assert str(test_app_copy / 'components' / 'my_component') in ret.stdout, 'Project components should be prioritized over EXTRA_COMPONENT_DIRS'
+
+
+def test_exclude_components_not_passed(idf_py: IdfPyFunc, test_app_copy: Path) -> None:
+    logging.info('Components in EXCLUDE_COMPONENTS not passed to idf_component_manager')
+    idf_py('create-component', '-C', 'components', 'to_be_excluded')
+    (test_app_copy / 'components' / 'to_be_excluded' / 'idf_component.yml').write_text('invalid syntax..')
+    ret = idf_py('reconfigure', check=False)
+    assert ret.returncode == 2, 'Reconfigure should have failed due to invalid syntax in idf_component.yml'
+    idf_py('-DEXCLUDE_COMPONENTS=to_be_excluded', 'reconfigure')

tools/test_build_system/test_partition.py

@@ -0,0 +1,44 @@
+# SPDX-FileCopyrightText: 2022-2023 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Apache-2.0
+
+import logging
+import os
+import re
+import shutil
+from pathlib import Path
+
+import pytest
+from test_build_system_helpers import EnvDict, IdfPyFunc, append_to_file, replace_in_file
+
+
+@pytest.mark.usefixtures('test_app_copy')
+def test_partition_table(idf_py: IdfPyFunc) -> None:
+    logging.info('Displays partition table when executing target partition_table')
+    ret = idf_py('partition-table')
+    assert re.search('# ESP-IDF.+Partition Table', ret.stdout)
+
+
+def test_partitions_dont_fit_in_flash(idf_py: IdfPyFunc, test_app_copy: Path) -> None:
+    logging.info("Build fails if partitions don't fit in flash")
+    append_to_file(test_app_copy / 'sdkconfig', 'CONFIG_ESPTOOLPY_FLASHSIZE_1MB=y')
+    ret = idf_py('build', check=False)
+    assert ret.returncode == 2
+    assert 'does not fit in configured flash size 1MB' in ret.stdout
+
+
+def test_partition_nearly_full_warning(idf_py: IdfPyFunc, test_app_copy: Path, default_idf_env: EnvDict) -> None:
+    logging.info('Warning is given if smallest partition is nearly full')
+    ret = idf_py('build')
+    # Build a first time to get the binary size and to check that no warning is issued.
+    assert 'partition is nearly full' not in ret.stdout, 'Warning for nearly full smallest partition was given when the condition is not fulfilled'
+    # Get the size of the binary, in KB. Add 1 to the total.
+    # The goal is to create an app partition which is slightly bigger than the binary itself
+    updated_file_size = int(os.stat(test_app_copy / 'build' / 'build_test_app.bin').st_size / 1024) + 1
+    idf_path = Path(default_idf_env['IDF_PATH'])
+    shutil.copy2(idf_path / 'components' / 'partition_table' / 'partitions_singleapp.csv', test_app_copy / 'partitions.csv')
+    replace_in_file(test_app_copy / 'partitions.csv',
+                    'factory,  app,  factory, ,        1M',
+                    f'factory,  app,  factory, ,        {updated_file_size}K')
+    (test_app_copy / 'sdkconfig').write_text('\n'.join(['CONFIG_PARTITION_TABLE_CUSTOM=y', 'CONFIG_FREERTOS_SMP=n']))
+    ret = idf_py('build', check=False)
+    assert 'partition is nearly full' in ret.stdout