Merge branch 'feature/rewrite_build_sys_tests_v1' into 'master'

Rewrite build system unit tests to python v1

Closes IDF-6736

See merge request espressif/esp-idf!21639

tools/test_build_system/MIGRATION.md

@@ -11,15 +11,15 @@ Updating component source file rebuilds component | test_rebuild::test_rebuild_s
 Bootloader source file rebuilds bootloader | test_rebuild::test_rebuild_source_files |
 Partition CSV file rebuilds partitions | test_rebuild::test_rebuild_source_files |
 Partial build doesn't compile anything by default | test_rebuild::test_rebuild_no_changes |
-Rebuild when app version was changed | |
+Rebuild when app version was changed | test_rebuild.py::test_rebuild_version_change |
-Change app version | |
+Change app version | test_rebuild.py::test_rebuild_version_change |
-Re-building does not change app.bin | |
+Re-building does not change app.bin | test_rebuild.py::test_rebuild_version_change |
 Get the version of app from git describe. Project is not inside IDF and do not have a tag only a hash commit. | |
 Get the version of app from Kconfig option | |
 Use IDF version variables in component CMakeLists.txt file | |
 Project is in ESP-IDF which has a custom tag | |
-Moving BUILD_DIR_BASE out of tree | |
+Moving BUILD_DIR_BASE out of tree | test_build.py::test_build_alternative_directories |
-BUILD_DIR_BASE inside default build directory | |
+BUILD_DIR_BASE inside default build directory | test_build.py::test_build_alternative_directories |
 Can still clean build if all text files are CRLFs | |
 Updating rom ld file should re-link app and bootloader | test_rebuild::test_rebuild_linker |
 Updating app-only ld file should only re-link app | test_rebuild::test_rebuild_linker |
@@ -27,13 +27,13 @@ Updating ld file should only re-link app | test_rebuild::test_rebuild_linker |
 Updating fragment file should only re-link app | test_rebuild::test_rebuild_linker |
 sdkconfig update triggers full recompile | test_rebuild::test_rebuild_source_files |
 Updating project CMakeLists.txt triggers full recompile | test_rebuild::test_rebuild_source_files |
-Can build with Ninja (no idf.py) | |
+Can build with Ninja (no idf.py) | test_build.py::test_build_cmake_ninja |
-Can build with GNU Make (no idf.py) | |
+Can build with GNU Make (no idf.py) | test_build.py::test_build_cmake_makefile |
-idf.py can build with Ninja | |
+idf.py can build with Ninja | test_build.py::test_build_with_generator_ninja |
-idf.py can build with Unix Makefiles | |
+idf.py can build with Unix Makefiles | test_build.py::test_build_with_generator_makefile |
-Can build with IDF_PATH set via cmake cache not environment | |
+Can build with IDF_PATH set via cmake cache not environment | test_build.py::test_build_with_cmake_and_idf_path_unset |
-Can build with IDF_PATH unset and inferred by build system | |
+Can build with IDF_PATH unset and inferred by build system | test_build.py::test_build_with_cmake_and_idf_path_unset |
-Can build with IDF_PATH unset and inferred by cmake when Kconfig needs it to be set | |
+Can build with IDF_PATH unset and inferred by cmake when Kconfig needs it to be set | test_build.py::test_build_with_cmake_and_idf_path_unset |
 can build with phy_init_data | |
 can build with ethernet component disabled | |
 Compiler flags on build command line are taken into account | |

tools/test_build_system/test_build.py

@@ -0,0 +1,116 @@
+# SPDX-FileCopyrightText: 2022-2023 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Apache-2.0
+
+import logging
+import os
+import re
+import sys
+import typing
+from pathlib import Path
+from typing import List, Pattern, Union
+
+import pytest
+from test_build_system_helpers import (APP_BINS, BOOTLOADER_BINS, PARTITION_BIN, EnvDict, IdfPyFunc, get_idf_build_env,
+                                       replace_in_file, run_cmake)
+
+
+def run_cmake_and_build(*cmake_args: str, env: typing.Optional[EnvDict] = None) -> None:
+    """
+    Run cmake command with given arguments and build afterwards, raise an exception on failure
+    :param cmake_args: arguments to pass cmake
+    :param env: environment variables to run the cmake with; if not set, the default environment is used
+    """
+    run_cmake(*cmake_args, env=env)
+    run_cmake('--build', '.')
+
+
+def assert_built(paths: Union[List[str], List[Path]]) -> None:
+    for path in paths:
+        assert os.path.exists(path)
+
+
+def check_file_contains(filename: Union[str, Path], what: Union[str, Pattern]) -> None:
+    with open(filename, 'r', encoding='utf-8') as f:
+        data = f.read()
+        if isinstance(what, str):
+            assert what in data
+        else:
+            assert re.search(what, data) is not None
+
+
+def test_build_alternative_directories(idf_py: IdfPyFunc, session_work_dir: Path, test_app_copy: Path) -> None:
+    logging.info('Moving BUILD_DIR_BASE out of tree')
+    alt_build_dir = session_work_dir / 'alt_build'
+    idf_py('-B', str(alt_build_dir), 'build')
+    assert os.listdir(alt_build_dir) != [], 'No files found in new build directory!'
+    default_build_dir = test_app_copy / 'build'
+    if default_build_dir.exists():
+        assert os.listdir(default_build_dir) == [], f'Some files were incorrectly put into the default build directory: {default_build_dir}'
+
+    logging.info('BUILD_DIR_BASE inside default build directory')
+    build_subdir_inside_build_dir = default_build_dir / 'subdirectory'
+    idf_py('-B', str(build_subdir_inside_build_dir), 'build')
+    assert os.listdir(build_subdir_inside_build_dir) != [], 'No files found in new build directory!'
+
+
+@pytest.mark.usefixtures('test_app_copy')
+def test_build_cmake_ninja() -> None:
+    logging.info('Can build with Ninja (no idf.py)')
+    run_cmake_and_build('-G', 'Ninja', '..')
+    assert_built(BOOTLOADER_BINS + APP_BINS + PARTITION_BIN)
+
+
+@pytest.mark.skipif(sys.platform == 'win32', reason="GNU Make doesn't work on Windows")
+@pytest.mark.usefixtures('test_app_copy')
+def test_build_cmake_makefile() -> None:
+    logging.info('Can build with GNU Make (no idf.py)')
+    run_cmake_and_build('-G', 'Unix Makefiles', '..')
+    assert_built(BOOTLOADER_BINS + APP_BINS + PARTITION_BIN)
+
+
+@pytest.mark.usefixtures('test_app_copy')
+def test_build_with_generator_ninja(idf_py: IdfPyFunc) -> None:
+    logging.info('idf.py can build with Ninja')
+    idf_py('-G', 'Ninja', 'build')
+    cmake_cache_file = Path('build', 'CMakeCache.txt')
+    assert_built([cmake_cache_file])
+    check_file_contains(cmake_cache_file, 'CMAKE_GENERATOR:INTERNAL=Ninja')
+    assert_built(BOOTLOADER_BINS + APP_BINS + PARTITION_BIN)
+
+
+@pytest.mark.skipif(sys.platform == 'win32', reason="GNU Make doesn't work on Windows")
+@pytest.mark.usefixtures('test_app_copy')
+def test_build_with_generator_makefile(idf_py: IdfPyFunc) -> None:
+    logging.info('idf.py can build with Unix Makefiles')
+    idf_py('-G', 'Unix Makefiles', 'build')
+    cmake_cache_file = Path('build', 'CMakeCache.txt')
+    assert_built([cmake_cache_file])
+    check_file_contains(cmake_cache_file, 'CMAKE_GENERATOR:INTERNAL=Unix Makefiles')
+    assert_built(BOOTLOADER_BINS + APP_BINS + PARTITION_BIN)
+
+
+def test_build_with_cmake_and_idf_path_unset(idf_py: IdfPyFunc, test_app_copy: Path) -> None:
+    idf_path = Path(os.environ['IDF_PATH'])
+    env = get_idf_build_env(idf_path)
+    env.pop('IDF_PATH')
+
+    logging.info('Can build with IDF_PATH set via cmake cache not environment')
+    replace_in_file('CMakeLists.txt', 'ENV{IDF_PATH}', '{IDF_PATH}')
+    run_cmake_and_build('-G', 'Ninja', '..', '-DIDF_PATH={}'.format(idf_path), env=env)
+    assert_built(BOOTLOADER_BINS + APP_BINS + PARTITION_BIN)
+    idf_py('fullclean')
+
+    logging.info('Can build with IDF_PATH unset and inferred by cmake when Kconfig needs it to be set')
+    # working with already changed CMakeLists.txt
+    kconfig_file = test_app_copy / 'main' / 'Kconfig.projbuild'
+    kconfig_file.write_text('source "$IDF_PATH/examples/wifi/getting_started/station/main/Kconfig.projbuild"')
+    run_cmake_and_build('-G', 'Ninja', '..', '-DIDF_PATH={}'.format(idf_path), env=env)
+    assert_built(BOOTLOADER_BINS + APP_BINS + PARTITION_BIN)
+    kconfig_file.unlink()  # remove file to not affect following sub-test
+    idf_py('fullclean')
+
+    logging.info('Can build with IDF_PATH unset and inferred by build system')
+    # replacing {IDF_PATH} not ENV{IDF_PATH} since CMakeLists.txt was already changed in this test
+    replace_in_file('CMakeLists.txt', '{IDF_PATH}', '{ci_idf_path}')
+    run_cmake_and_build('-G', 'Ninja', '-D', 'ci_idf_path={}'.format(idf_path), '..', env=env)
+    assert_built(BOOTLOADER_BINS + APP_BINS + PARTITION_BIN)

tools/test_build_system/test_build_system_helpers/__init__.py

@@ -1,11 +1,13 @@
-# SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD
+# SPDX-FileCopyrightText: 2022-2023 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
+from .build_constants import ALL_ARTIFACTS, APP_BINS, BOOTLOADER_BINS, JSON_METADATA, PARTITION_BIN
 from .editing import append_to_file, replace_in_file
-from .idf_utils import EXT_IDF_PATH, EnvDict, IdfPyFunc, get_idf_build_env, run_idf_py
+from .idf_utils import EXT_IDF_PATH, EnvDict, IdfPyFunc, get_idf_build_env, run_cmake, run_idf_py
 from .snapshot import Snapshot, get_snapshot
 
 __all__ = [
     'append_to_file', 'replace_in_file',
     'get_idf_build_env', 'run_idf_py', 'EXT_IDF_PATH', 'EnvDict', 'IdfPyFunc',
-    'Snapshot', 'get_snapshot'
+    'Snapshot', 'get_snapshot', 'run_cmake', 'APP_BINS', 'BOOTLOADER_BINS',
+    'PARTITION_BIN', 'JSON_METADATA', 'ALL_ARTIFACTS'
 ]

tools/test_build_system/test_build_system_helpers/build_constants.py

@@ -0,0 +1,12 @@
+# SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Apache-2.0
+BOOTLOADER_BINS = ['build/bootloader/bootloader.elf', 'build/bootloader/bootloader.bin']
+APP_BINS = ['build/build_test_app.elf', 'build/build_test_app.bin']
+PARTITION_BIN = ['build/partition_table/partition-table.bin']
+JSON_METADATA = ['build/project_description.json', 'build/flasher_args.json', 'build/config/kconfig_menus.json', 'build/config/sdkconfig.json']
+ALL_ARTIFACTS = [
+    *BOOTLOADER_BINS,
+    *APP_BINS,
+    *PARTITION_BIN,
+    *JSON_METADATA
+]

tools/test_build_system/test_build_system_helpers/idf_utils.py

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD
+# SPDX-FileCopyrightText: 2022-2023 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
 import logging
 import os
@@ -88,3 +88,22 @@ def run_idf_py(*args: str,
         cmd, env=env_dict, cwd=workdir,
         check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
         text=True, encoding='utf-8', errors='backslashreplace')
+
+
+def run_cmake(*cmake_args: str, env: typing.Optional[EnvDict] = None) -> None:
+    """
+    Run cmake command with given arguments, raise an exception on failure
+    :param cmake_args: arguments to pass cmake
+    :param env: environment variables to run the cmake with; if not set, the default environment is used
+    """
+    if not env:
+        env = dict(**os.environ)
+    workdir = (Path(os.getcwd()) / 'build')
+    workdir.mkdir(parents=True, exist_ok=True)
+
+    cmd = ['cmake'] + list(cmake_args)
+
+    logging.debug('running {} in {}'.format(' '.join(cmd), workdir))
+    subprocess.check_call(
+        cmd, env=env, cwd=workdir,
+        stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)

tools/test_build_system/test_rebuild.py

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD
+# SPDX-FileCopyrightText: 2022-2023 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
 # These tests check whether the build system rebuilds some files or not
 # depending on the changes to the project.
@@ -8,18 +8,8 @@ from pathlib import Path
 from typing import List, Union
 
 import pytest
-from test_build_system_helpers import IdfPyFunc, get_snapshot, replace_in_file
+from test_build_system_helpers import (ALL_ARTIFACTS, APP_BINS, BOOTLOADER_BINS, PARTITION_BIN, IdfPyFunc,
-
+                                       get_snapshot, replace_in_file)
-BOOTLOADER_BINS = ['build/bootloader/bootloader.elf', 'build/bootloader/bootloader.bin']
-APP_BINS = ['build/build_test_app.elf', 'build/build_test_app.bin']
-PARTITION_BIN = ['build/partition_table/partition-table.bin']
-JSON_METADATA = ['build/project_description.json', 'build/flasher_args.json', 'build/config/kconfig_menus.json', 'build/config/sdkconfig.json']
-ALL_ARTIFACTS = [
-    *BOOTLOADER_BINS,
-    *APP_BINS,
-    *PARTITION_BIN,
-    *JSON_METADATA
-]
 
 
 @pytest.mark.usefixtures('test_app_copy')
@@ -137,3 +127,26 @@ def test_rebuild_linker(idf_py: IdfPyFunc) -> None:
     (idf_path / 'components/esp_common/common.lf').touch()
     rebuild_and_check(idf_py,
                       APP_BINS, BOOTLOADER_BINS + PARTITION_BIN)
+
+
+@pytest.mark.usefixtures('idf_copy')
+def test_rebuild_version_change(idf_py: IdfPyFunc, test_app_copy: Path) -> None:
+    idf_path = Path(os.environ['IDF_PATH'])
+
+    logging.info('Creating version files')
+    version_idf_file = idf_path / 'version.txt'
+    version_idf_file.write_text('IDF_VER_0123456789_0123456789_0123456789')
+
+    version_tmp_file = test_app_copy / 'version.txt'
+    version_tmp_file.write_text('project-version-1.0')
+
+    logging.info('Build first version')
+    idf_py('build')
+
+    logging.info('Changing app version')
+    version_tmp_file.write_text('project-version-2.0(012345678901234567890123456789)')
+
+    logging.info('rebuild should update only app files, not bootloader')
+    rebuild_and_check(idf_py, APP_BINS, BOOTLOADER_BINS)
+    logging.info('new rebuild should not change anything')
+    rebuild_and_check(idf_py, [], APP_BINS + BOOTLOADER_BINS)