From d6edcba3b6ba8dca3a67ebb2c92de75294e85155 Mon Sep 17 00:00:00 2001 From: Ivan Grokhotkov Date: Fri, 9 Aug 2024 12:49:10 +0200 Subject: [PATCH 1/5] test(build_system): extract file helpers, add 'bin_files_differ' - Move file-related functions bin_file_contains and file_contains from idf_utils.py and existing functions from editing.py into a new file file_utils.py - Add a function 'bin_files_differ' to compare binary files --- .../test_build_system_helpers/__init__.py | 29 +++++++-- .../test_build_system_helpers/editing.py | 17 ----- .../test_build_system_helpers/file_utils.py | 64 +++++++++++++++++++ .../test_build_system_helpers/idf_utils.py | 38 +---------- 4 files changed, 89 insertions(+), 59 deletions(-) delete mode 100644 tools/test_build_system/test_build_system_helpers/editing.py create mode 100644 tools/test_build_system/test_build_system_helpers/file_utils.py diff --git a/tools/test_build_system/test_build_system_helpers/__init__.py b/tools/test_build_system/test_build_system_helpers/__init__.py index 9f7616cdc0..57c6493e74 100644 --- a/tools/test_build_system/test_build_system_helpers/__init__.py +++ b/tools/test_build_system/test_build_system_helpers/__init__.py @@ -1,15 +1,30 @@ -# SPDX-FileCopyrightText: 2022-2023 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2022-2024 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 -from .build_constants import ALL_ARTIFACTS, APP_BINS, BOOTLOADER_BINS, JSON_METADATA, PARTITION_BIN -from .editing import append_to_file, replace_in_file -from .idf_utils import (EXT_IDF_PATH, EnvDict, IdfPyFunc, bin_file_contains, file_contains, find_python, - get_idf_build_env, run_cmake, run_cmake_and_build, run_idf_py) -from .snapshot import Snapshot, get_snapshot +from .build_constants import ALL_ARTIFACTS +from .build_constants import APP_BINS +from .build_constants import BOOTLOADER_BINS +from .build_constants import JSON_METADATA +from .build_constants import PARTITION_BIN +from .file_utils import append_to_file +from .file_utils import bin_file_contains +from .file_utils import bin_files_differ +from .file_utils import file_contains +from .file_utils import replace_in_file +from .idf_utils import EnvDict +from .idf_utils import EXT_IDF_PATH +from .idf_utils import find_python +from .idf_utils import get_idf_build_env +from .idf_utils import IdfPyFunc +from .idf_utils import run_cmake +from .idf_utils import run_cmake_and_build +from .idf_utils import run_idf_py +from .snapshot import get_snapshot +from .snapshot import Snapshot __all__ = [ 'append_to_file', 'replace_in_file', 'get_idf_build_env', 'run_idf_py', 'EXT_IDF_PATH', 'EnvDict', 'IdfPyFunc', 'Snapshot', 'get_snapshot', 'run_cmake', 'APP_BINS', 'BOOTLOADER_BINS', 'PARTITION_BIN', 'JSON_METADATA', 'ALL_ARTIFACTS', - 'run_cmake_and_build', 'find_python', 'file_contains', 'bin_file_contains' + 'run_cmake_and_build', 'find_python', 'file_contains', 'bin_file_contains', 'bin_files_differ' ] diff --git a/tools/test_build_system/test_build_system_helpers/editing.py b/tools/test_build_system/test_build_system_helpers/editing.py deleted file mode 100644 index 7f21d52822..0000000000 --- a/tools/test_build_system/test_build_system_helpers/editing.py +++ /dev/null @@ -1,17 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD -# SPDX-License-Identifier: Apache-2.0 -import typing -from pathlib import Path - - -def append_to_file(filename: typing.Union[str, Path], what: str) -> None: - with open(filename, 'a', encoding='utf-8') as f: - f.write(what) - - -def replace_in_file(filename: typing.Union[str, Path], search: str, replace: str) -> None: - with open(filename, 'r', encoding='utf-8') as f: - data = f.read() - result = data.replace(search, replace) - with open(filename, 'w', encoding='utf-8') as f: - f.write(result) diff --git a/tools/test_build_system/test_build_system_helpers/file_utils.py b/tools/test_build_system/test_build_system_helpers/file_utils.py new file mode 100644 index 0000000000..e79a8b62f4 --- /dev/null +++ b/tools/test_build_system/test_build_system_helpers/file_utils.py @@ -0,0 +1,64 @@ +# SPDX-FileCopyrightText: 2022-2024 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 +import re +import typing as t +from pathlib import Path +from pathlib import WindowsPath + + +def append_to_file(filename: t.Union[str, Path], what: str) -> None: + with open(filename, 'a', encoding='utf-8') as f: + f.write(what) + + +def replace_in_file(filename: t.Union[str, Path], search: str, replace: str) -> None: + with open(filename, 'r', encoding='utf-8') as f: + data = f.read() + result = data.replace(search, replace) + with open(filename, 'w', encoding='utf-8') as f: + f.write(result) + + +def file_contains(filename: t.Union[str, Path], what: t.Union[t.Union[str, Path], t.Pattern]) -> bool: + """ + Returns true if file contains required object + :param filename: path to file where lookup is executed + :param what: searched substring or regex object + """ + with open(filename, 'r', encoding='utf-8') as f: + data = f.read() + if isinstance(what, t.Pattern): + return re.search(what, data) is not None + else: + what_str = str(what) + # In case of windows path, try both single-slash `\` and double-slash '\\' paths + if isinstance(what, WindowsPath): + what_double_slash = what_str.replace('\\', '\\\\') + return what_str in data or what_double_slash in data + + return what_str in data + + +def bin_file_contains(filename: t.Union[str, Path], what: bytearray) -> bool: + """ + Returns true if the binary file contains the given string + :param filename: path to file where lookup is executed + :param what: searched bytes + """ + with open(filename, 'rb') as f: + data = f.read() + return data.find(what) != -1 + + +def bin_files_differ(filename1: t.Union[str, Path], filename2: t.Union[str, Path]) -> bool: + """ + Checks if two binary files are different + :param filename1: path to first file + :param filename2: path to second file + :return: True if files have different content, False if the content is the same + """ + with open(filename1, 'rb') as f1: + data1 = f1.read() + with open(filename2, 'rb') as f2: + data2 = f2.read() + return data1 != data2 diff --git a/tools/test_build_system/test_build_system_helpers/idf_utils.py b/tools/test_build_system/test_build_system_helpers/idf_utils.py index bec2df14ea..638789535e 100644 --- a/tools/test_build_system/test_build_system_helpers/idf_utils.py +++ b/tools/test_build_system/test_build_system_helpers/idf_utils.py @@ -1,14 +1,13 @@ -# SPDX-FileCopyrightText: 2022-2023 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2022-2024 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 import logging import os -import re import shutil import subprocess import sys import typing -from pathlib import Path, WindowsPath -from typing import Pattern, Union +from pathlib import Path +from typing import Union try: EXT_IDF_PATH = os.environ['IDF_PATH'] # type: str @@ -135,34 +134,3 @@ def run_cmake_and_build(*cmake_args: str, env: typing.Optional[EnvDict] = None) """ run_cmake(*cmake_args, env=env) run_cmake('--build', '.') - - -def file_contains(filename: Union[str, Path], what: Union[Union[str, Path], Pattern]) -> bool: - """ - Returns true if file contains required object - :param filename: path to file where lookup is executed - :param what: searched substring or regex object - """ - with open(filename, 'r', encoding='utf-8') as f: - data = f.read() - if isinstance(what, Pattern): - return re.search(what, data) is not None - else: - what_str = str(what) - # In case of windows path, try both single-slash `\` and double-slash '\\' paths - if isinstance(what, WindowsPath): - what_double_slash = what_str.replace('\\', '\\\\') - return what_str in data or what_double_slash in data - - return what_str in data - - -def bin_file_contains(filename: Union[str, Path], what: bytearray) -> bool: - """ - Returns true if the binary file contains the given string - :param filename: path to file where lookup is executed - :param what: searched bytes - """ - with open(filename, 'rb') as f: - data = f.read() - return data.find(what) != -1 From fc4c4456431cc40b3f647a061320cc5632fa02e8 Mon Sep 17 00:00:00 2001 From: Ivan Grokhotkov Date: Fri, 9 Aug 2024 13:26:42 +0200 Subject: [PATCH 2/5] test(build_system): move reproducible build test to pytest --- .gitlab/ci/host-test.yml | 16 ----- .gitlab/ci/rules.yml | 2 - tools/ci/executable-list.txt | 1 - tools/ci/test_reproducible_build.sh | 33 --------- tools/test_build_system/conftest.py | 4 +- .../test_reproducible_build.py | 67 +++++++++++++++++++ 6 files changed, 70 insertions(+), 53 deletions(-) delete mode 100755 tools/ci/test_reproducible_build.sh create mode 100644 tools/test_build_system/test_reproducible_build.py diff --git a/.gitlab/ci/host-test.yml b/.gitlab/ci/host-test.yml index 186671117e..109020a66c 100644 --- a/.gitlab/ci/host-test.yml +++ b/.gitlab/ci/host-test.yml @@ -68,22 +68,6 @@ test_ldgen_on_host: variables: LC_ALL: C.UTF-8 -test_reproducible_build: - extends: .host_test_template - script: - - ./tools/ci/test_reproducible_build.sh - artifacts: - when: on_failure - paths: - - "**/sdkconfig" - - "**/build*/*.bin" - - "**/build*/*.elf" - - "**/build*/*.map" - - "**/build*/flasher_args.json" - - "**/build*/*.bin" - - "**/build*/bootloader/*.bin" - - "**/build*/partition_table/*.bin" - test_spiffs_on_host: extends: .host_test_template script: diff --git a/.gitlab/ci/rules.yml b/.gitlab/ci/rules.yml index 9cf84cf871..3685149e7f 100644 --- a/.gitlab/ci/rules.yml +++ b/.gitlab/ci/rules.yml @@ -108,8 +108,6 @@ - "tools/detect_python.sh" - "tools/detect_python.fish" - - "tools/ci/test_reproducible_build.sh" - - "tools/gen_soc_caps_kconfig/*" - "tools/gen_soc_caps_kconfig/test/test_gen_soc_caps_kconfig.py" diff --git a/tools/ci/executable-list.txt b/tools/ci/executable-list.txt index 0ba2bd2cfe..f6bbe453d3 100644 --- a/tools/ci/executable-list.txt +++ b/tools/ci/executable-list.txt @@ -80,7 +80,6 @@ tools/ci/push_to_github.sh tools/ci/sort_yaml.py tools/ci/test_autocomplete/test_autocomplete.py tools/ci/test_configure_ci_environment.sh -tools/ci/test_reproducible_build.sh tools/docker/entrypoint.sh tools/esp_app_trace/logtrace_proc.py tools/esp_app_trace/sysviewtrace_proc.py diff --git a/tools/ci/test_reproducible_build.sh b/tools/ci/test_reproducible_build.sh deleted file mode 100755 index 143972f441..0000000000 --- a/tools/ci/test_reproducible_build.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash - -set -euo - -for path in \ - "examples/get-started/hello_world" \ - "examples/bluetooth/nimble/blecent"; do - cd "${IDF_PATH}/${path}" - - echo "CONFIG_APP_REPRODUCIBLE_BUILD=y" >sdkconfig - - idf.py -B build_first fullclean build - idf.py -B build_second fullclean build - - for item in \ - "partition_table/partition-table.bin" \ - "bootloader/bootloader.bin" \ - "bootloader/bootloader.elf" \ - "bootloader/bootloader.map" \ - "*.bin" \ - "*.elf" \ - "*.map"; do - diff -s build_first/${item} build_second/${item} # use glob, don't use double quotes - done - - # test gdb - rm -f gdb.txt - elf_file=$(find build_first -maxdepth 1 -iname '*.elf') - xtensa-esp32-elf-gdb -x build_first/prefix_map_gdbinit -ex 'set logging enabled' -ex 'set pagination off' -ex 'list app_main' -ex 'quit' "$elf_file" - if grep "No such file or directory" gdb.txt; then - exit 1 - fi -done diff --git a/tools/test_build_system/conftest.py b/tools/test_build_system/conftest.py index fbfc97c053..2607f06938 100644 --- a/tools/test_build_system/conftest.py +++ b/tools/test_build_system/conftest.py @@ -95,7 +95,9 @@ def test_app_copy(func_work_dir: Path, request: FixtureRequest) -> typing.Genera # by default, use hello_world app and copy it to a temporary directory with # the name resembling that of the test copy_from = 'tools/test_build_system/build_test_app' - copy_to = request.node.name + '_app' + # sanitize test name in case pytest.mark.parametrize was used + test_name_sanitized = request.node.name.replace('[', '_').replace(']', '') + copy_to = test_name_sanitized + '_app' # allow overriding source and destination via pytest.mark.test_app_copy() mark = request.node.get_closest_marker('test_app_copy') diff --git a/tools/test_build_system/test_reproducible_build.py b/tools/test_build_system/test_reproducible_build.py new file mode 100644 index 0000000000..a19a20174d --- /dev/null +++ b/tools/test_build_system/test_reproducible_build.py @@ -0,0 +1,67 @@ +# SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 +# This test checks the behavior of reproducible builds option. +import logging +import os +import subprocess +from pathlib import Path + +import pytest +from test_build_system_helpers import append_to_file +from test_build_system_helpers import bin_files_differ +from test_build_system_helpers import BOOTLOADER_BINS +from test_build_system_helpers import IdfPyFunc + + +@pytest.mark.parametrize( + 'app_name', [ + pytest.param('blink', marks=[pytest.mark.test_app_copy('examples/get-started/blink')]), + pytest.param('blecent', marks=[pytest.mark.test_app_copy('examples/bluetooth/nimble/blecent')]), + ] +) +def test_reproducible_builds(app_name: str, idf_py: IdfPyFunc, test_app_copy: Path) -> None: + append_to_file(test_app_copy / 'sdkconfig', 'CONFIG_APP_REPRODUCIBLE_BUILD=y') + build_first = test_app_copy / 'build_first' + build_second = test_app_copy / 'build_second' + + logging.info(f'Building in {build_first} directory') + idf_py('-B', str(build_first), 'build') + + elf_file = build_first / f'{app_name}.elf' + logging.info(f'Checking that various paths are not included in the ELF file') + strings_output = subprocess.check_output( + ['xtensa-esp32-elf-strings', str(elf_file)], + encoding='utf-8' + ) + idf_path = os.environ['IDF_PATH'] + assert str(idf_path) not in strings_output, f'{idf_path} found in {elf_file}' + assert str(test_app_copy) not in strings_output, f'{test_app_copy} found in {elf_file}' + + logging.info(f'Building in {build_second} directory') + idf_py('-B', str(build_second), 'build') + + logging.info(f'Comparing build artifacts') + artifacts_to_check = [ + f'build/{app_name}.map', + f'build/{app_name}.elf', + f'build/{app_name}.bin', + ] + BOOTLOADER_BINS + + for artifact in artifacts_to_check: + path_first = artifact.replace('build/', f'{build_first}/') + path_second = artifact.replace('build/', f'{build_second}/') + + assert not bin_files_differ(path_first, path_second), f'{path_first} and {path_second} differ' + + logging.info(f'Checking that GDB works with CONFIG_APP_REPRODUCIBLE_BUILD=y') + gdb_output = subprocess.check_output([ + 'xtensa-esp32-elf-gdb', + '--batch', '--quiet', + '-x', f'{build_first}/prefix_map_gdbinit', + '-ex', 'set logging enabled', + '-ex', 'set pagination off', + '-ex', 'list app_main', + str(elf_file) + ], encoding='utf-8', stderr=subprocess.STDOUT, cwd=str(build_first)) + + assert 'No such file or directory' not in gdb_output, f'GDB failed to find app_main in {elf_file}:\n{gdb_output}' From 1c343d892331b40c79ece93353d7bbd25d2a437e Mon Sep 17 00:00:00 2001 From: Ivan Grokhotkov Date: Fri, 9 Aug 2024 14:45:58 +0200 Subject: [PATCH 3/5] change(build_system): refactor reproducible build handling - remove generate_debug_prefix_map.py, move its logic into CMake - move all reproducible builds logic into a new file, prefix_map.cmake --- CMakeLists.txt | 42 ++---------------------- tools/ci/exclude_check_tools_files.txt | 1 - tools/cmake/idf.cmake | 1 + tools/cmake/prefix_map.cmake | 41 +++++++++++++++++++++++ tools/generate_debug_prefix_map.py | 45 -------------------------- 5 files changed, 44 insertions(+), 86 deletions(-) create mode 100644 tools/cmake/prefix_map.cmake delete mode 100644 tools/generate_debug_prefix_map.py diff --git a/CMakeLists.txt b/CMakeLists.txt index 6cedae7de3..5501c1e4cb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -152,46 +152,8 @@ if(CONFIG_COMPILER_DUMP_RTL_FILES) list(APPEND compile_options "-fdump-rtl-expand") endif() -if(NOT ${CMAKE_C_COMPILER_VERSION} VERSION_LESS 8.0.0) - if(CONFIG_COMPILER_HIDE_PATHS_MACROS) - list(APPEND compile_options "-fmacro-prefix-map=${CMAKE_SOURCE_DIR}=.") - list(APPEND compile_options "-fmacro-prefix-map=${IDF_PATH}=/IDF") - endif() - - if(CONFIG_APP_REPRODUCIBLE_BUILD) - idf_build_set_property(DEBUG_PREFIX_MAP_GDBINIT "${BUILD_DIR}/prefix_map_gdbinit") - - list(APPEND compile_options "-fdebug-prefix-map=${IDF_PATH}=/IDF") - list(APPEND compile_options "-fdebug-prefix-map=${PROJECT_DIR}=/IDF_PROJECT") - list(APPEND compile_options "-fdebug-prefix-map=${BUILD_DIR}=/IDF_BUILD") - - # component dirs - idf_build_get_property(python PYTHON) - idf_build_get_property(idf_path IDF_PATH) - idf_build_get_property(component_dirs BUILD_COMPONENT_DIRS) - - execute_process( - COMMAND ${python} - "${idf_path}/tools/generate_debug_prefix_map.py" - "${BUILD_DIR}" - "${component_dirs}" - OUTPUT_VARIABLE result - RESULT_VARIABLE ret - ) - if(NOT ret EQUAL 0) - message(FATAL_ERROR "This is a bug. Please report to https://github.com/espressif/esp-idf/issues") - endif() - - spaces2list(result) - list(LENGTH component_dirs length) - math(EXPR max_index "${length} - 1") - foreach(index RANGE ${max_index}) - list(GET component_dirs ${index} folder) - list(GET result ${index} after) - list(APPEND compile_options "-fdebug-prefix-map=${folder}=${after}") - endforeach() - endif() -endif() +__generate_prefix_map(prefix_map_compile_options) +list(APPEND compile_options ${prefix_map_compile_options}) if(CONFIG_COMPILER_DISABLE_GCC12_WARNINGS) list(APPEND compile_options "-Wno-address" diff --git a/tools/ci/exclude_check_tools_files.txt b/tools/ci/exclude_check_tools_files.txt index c5d24295e0..b5326f7e41 100644 --- a/tools/ci/exclude_check_tools_files.txt +++ b/tools/ci/exclude_check_tools_files.txt @@ -8,7 +8,6 @@ tools/ci/get_all_test_results.py tools/gdb_panic_server.py tools/check_term.py tools/python_version_checker.py -tools/generate_debug_prefix_map.py tools/ci/astyle-rules.yml tools/ci/checkout_project_ref.py tools/ci/ci_fetch_submodule.py diff --git a/tools/cmake/idf.cmake b/tools/cmake/idf.cmake index 3d2e96af1b..28c2b1d060 100644 --- a/tools/cmake/idf.cmake +++ b/tools/cmake/idf.cmake @@ -48,6 +48,7 @@ if(NOT __idf_env_set) include(ldgen) include(dfu) include(version) + include(prefix_map) __build_init("${idf_path}") diff --git a/tools/cmake/prefix_map.cmake b/tools/cmake/prefix_map.cmake new file mode 100644 index 0000000000..440d412b43 --- /dev/null +++ b/tools/cmake/prefix_map.cmake @@ -0,0 +1,41 @@ +# Utilities for remapping path prefixes +# +# __generate_prefix_map +# Prepares the list of compiler flags for remapping various paths +# to fixed names. This is used when reproducible builds are required. +# This function also creates a gdbinit file for the debugger to +# remap the substituted paths back to the real paths in the filesystem. +function(__generate_prefix_map compile_options_var) + set(compile_options) + idf_build_get_property(idf_path IDF_PATH) + idf_build_get_property(build_components BUILD_COMPONENTS) + + if(CONFIG_COMPILER_HIDE_PATHS_MACROS) + list(APPEND compile_options "-fmacro-prefix-map=${CMAKE_SOURCE_DIR}=.") + list(APPEND compile_options "-fmacro-prefix-map=${idf_path}=/IDF") + endif() + + if(CONFIG_APP_REPRODUCIBLE_BUILD) + list(APPEND compile_options "-fdebug-prefix-map=${idf_path}=/IDF") + list(APPEND compile_options "-fdebug-prefix-map=${PROJECT_DIR}=/IDF_PROJECT") + list(APPEND compile_options "-fdebug-prefix-map=${BUILD_DIR}=/IDF_BUILD") + + # Generate mapping for component paths + set(gdbinit_file_lines) + foreach(component_name ${build_components}) + idf_component_get_property(component_dir ${component_name} COMPONENT_DIR) + + string(TOUPPER ${component_name} component_name_uppercase) + set(substituted_path "/COMPONENT_${component_name_uppercase}_DIR") + list(APPEND compile_options "-fdebug-prefix-map=${component_dir}=${substituted_path}") + string(APPEND gdbinit_file_lines "set substitute-path ${substituted_path} ${component_dir}\n") + endforeach() + + # Write the final gdbinit file + set(gdbinit_path "${BUILD_DIR}/prefix_map_gdbinit") + file(WRITE "${gdbinit_path}" "${gdbinit_file_lines}") + idf_build_set_property(DEBUG_PREFIX_MAP_GDBINIT "${gdbinit_path}") + endif() + + set(${compile_options_var} ${compile_options} PARENT_SCOPE) +endfunction() diff --git a/tools/generate_debug_prefix_map.py b/tools/generate_debug_prefix_map.py deleted file mode 100644 index d9d46b4e2f..0000000000 --- a/tools/generate_debug_prefix_map.py +++ /dev/null @@ -1,45 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Espressif Systems (Shanghai) CO LTD -# SPDX-License-Identifier: Apache-2.0 - -# General Workflow: -# 1. read all components dirs, a semicolon-separated string (cmake list) -# 2. map the component dir with a unique prefix /COMPONENT__DIR -# 2. write the prefix mapping file to $BUILD_DIR/prefix_map_gdbinit -# 3. print the unique prefix out, a space-separated string, will be used by the build system to add compile options. - -import argparse -import os -from typing import List - - -def component_name(component_dir: str) -> str: - return '/COMPONENT_{}_DIR'.format(os.path.basename(component_dir).upper()) - - -GDB_SUBSTITUTE_PATH_FMT = 'set substitute-path {} {}\n' - - -def write_gdbinit(build_dir: str, folders: List[str]) -> None: - gdb_init_filepath = os.path.join(build_dir, 'prefix_map_gdbinit') - - with open(gdb_init_filepath, 'w') as fw: - for folder in folders: - fw.write(f'{GDB_SUBSTITUTE_PATH_FMT.format(component_name(folder), folder)}') - - -def main(build_dir: str, folders: List[str]) -> None: - write_gdbinit(build_dir, folders) - print(' '.join([component_name(folder) for folder in folders]), end='') - - -if __name__ == '__main__': - parser = argparse.ArgumentParser(description='print the debug-prefix-map and write to ' - '$BUILD_DIR/prefix_map_gdbinit file') - - parser.add_argument('build_dir', - help='build dir') - parser.add_argument('folders', - help='component folders, semicolon separated string') - args = parser.parse_args() - - main(args.build_dir, args.folders.split(';')) From bef4cb05ff1cf8439b501ce99db75b468d31c273 Mon Sep 17 00:00:00 2001 From: Ivan Grokhotkov Date: Mon, 12 Aug 2024 13:28:15 +0200 Subject: [PATCH 4/5] fix(build_system): fix toolchain sysroot directory affecting builds Closes https://github.com/espressif/esp-idf/issues/13680 --- docs/en/api-guides/reproducible-builds.rst | 2 ++ docs/zh_CN/api-guides/reproducible-builds.rst | 2 ++ tools/cmake/prefix_map.cmake | 13 +++++++++++++ tools/test_build_system/test_reproducible_build.py | 5 +++++ 4 files changed, 22 insertions(+) diff --git a/docs/en/api-guides/reproducible-builds.rst b/docs/en/api-guides/reproducible-builds.rst index 5919ef8d89..93ef078732 100644 --- a/docs/en/api-guides/reproducible-builds.rst +++ b/docs/en/api-guides/reproducible-builds.rst @@ -13,6 +13,7 @@ When reproducible builds are enabled, the application built with ESP-IDF does no - Directory where the project is located - Directory where ESP-IDF is located (``IDF_PATH``) - Build time +- Toolchain installation path Reasons for Non-Reproducible Builds ----------------------------------- @@ -46,6 +47,7 @@ ESP-IDF achieves reproducible builds using the following measures: - Path to the project is replaced with ``/IDF_PROJECT`` - Path to the build directory is replaced with ``/IDF_BUILD`` - Paths to components are replaced with ``/COMPONENT_NAME_DIR`` (where ``NAME`` is the name of the component) + - Path to the toolchain is replaced with ``/TOOLCHAIN`` - Build date and time are not included into the :ref:`application metadata structure ` and :ref:`bootloader metadata structure ` if :ref:`CONFIG_APP_REPRODUCIBLE_BUILD` is enabled. - ESP-IDF build system ensures that source file lists, component lists and other sequences are sorted before passing them to CMake. Various other parts of the build system, such as the linker script generator also perform sorting to ensure that same output is produced regardless of the environment. diff --git a/docs/zh_CN/api-guides/reproducible-builds.rst b/docs/zh_CN/api-guides/reproducible-builds.rst index ec922e3f55..74d568b0e4 100644 --- a/docs/zh_CN/api-guides/reproducible-builds.rst +++ b/docs/zh_CN/api-guides/reproducible-builds.rst @@ -13,6 +13,7 @@ ESP-IDF 构建系统支持 `可重复构建 ` 和 :ref:`引导加载程序元数据结构 ` 中。 - ESP-IDF 构建系统在将源文件列表、组件列表和其他序列传递给 CMake 之前会对其进行排序。构建系统的其他各个部分，如链接器脚本生成器，也会先排序，从而确保无论环境如何，输出都一致。 diff --git a/tools/cmake/prefix_map.cmake b/tools/cmake/prefix_map.cmake index 440d412b43..6373b6b087 100644 --- a/tools/cmake/prefix_map.cmake +++ b/tools/cmake/prefix_map.cmake @@ -31,6 +31,19 @@ function(__generate_prefix_map compile_options_var) string(APPEND gdbinit_file_lines "set substitute-path ${substituted_path} ${component_dir}\n") endforeach() + # Mapping for toolchain path + execute_process( + COMMAND ${CMAKE_C_COMPILER} -print-sysroot + OUTPUT_VARIABLE compiler_sysroot + ) + if(compiler_sysroot STREQUAL "") + message(FATAL_ERROR "Failed to determine toolchain sysroot") + endif() + string(STRIP "${compiler_sysroot}" compiler_sysroot) + get_filename_component(compiler_sysroot "${compiler_sysroot}/.." REALPATH) + list(APPEND compile_options "-fdebug-prefix-map=${compiler_sysroot}=/TOOLCHAIN") + string(APPEND gdbinit_file_lines "set substitute-path /TOOLCHAIN ${compiler_sysroot}\n") + # Write the final gdbinit file set(gdbinit_path "${BUILD_DIR}/prefix_map_gdbinit") file(WRITE "${gdbinit_path}" "${gdbinit_file_lines}") diff --git a/tools/test_build_system/test_reproducible_build.py b/tools/test_build_system/test_reproducible_build.py index a19a20174d..c5ba093b55 100644 --- a/tools/test_build_system/test_reproducible_build.py +++ b/tools/test_build_system/test_reproducible_build.py @@ -3,6 +3,7 @@ # This test checks the behavior of reproducible builds option. import logging import os +import shutil import subprocess from pathlib import Path @@ -36,6 +37,10 @@ def test_reproducible_builds(app_name: str, idf_py: IdfPyFunc, test_app_copy: Pa idf_path = os.environ['IDF_PATH'] assert str(idf_path) not in strings_output, f'{idf_path} found in {elf_file}' assert str(test_app_copy) not in strings_output, f'{test_app_copy} found in {elf_file}' + compiler_path = shutil.which('xtensa-esp32-elf-gcc') + assert compiler_path is not None, 'failed to determine compiler path' + toolchain_path = Path(compiler_path).parent.parent + assert str(toolchain_path) not in strings_output, f'{toolchain_path} found in {elf_file}' logging.info(f'Building in {build_second} directory') idf_py('-B', str(build_second), 'build') From cbc52e7d300a24762f959e2b155545b03cea47e2 Mon Sep 17 00:00:00 2001 From: Ivan Grokhotkov Date: Fri, 9 Aug 2024 15:38:55 +0200 Subject: [PATCH 5/5] test(build_system): check reproducible builds with spaces in paths --- tools/test_build_system/test_spaces.py | 29 +++++++++++++++++++------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/tools/test_build_system/test_spaces.py b/tools/test_build_system/test_spaces.py index 7635f00a98..5a21214997 100644 --- a/tools/test_build_system/test_spaces.py +++ b/tools/test_build_system/test_spaces.py @@ -8,6 +8,7 @@ import sys from pathlib import Path import pytest +from test_build_system_helpers import IdfPyFunc from test_build_system_helpers import run_idf_py # In this test file the test are grouped into 3 bundles @@ -23,8 +24,6 @@ def clean_app_dir(app_path: Path) -> None: @pytest.mark.idf_copy('esp idf with spaces') def test_spaces_bundle1(idf_copy: Path) -> None: logging.info('Running test spaces bundle 1') - # test_build - run_idf_py('build', workdir=(idf_copy / 'examples' / 'get-started' / 'hello_world')) # test spiffsgen run_idf_py('build', workdir=(idf_copy / 'examples' / 'storage' / 'spiffsgen')) # test build ulp_fsm @@ -40,12 +39,6 @@ def test_spaces_bundle2(idf_copy: Path) -> None: run_idf_py('build', workdir=(idf_copy / 'examples' / 'security' / 'flash_encryption')) # test_x509_cert_bundle run_idf_py('build', workdir=(idf_copy / 'examples' / 'protocols' / 'https_x509_bundle')) - # test dfu - hello_world_app_path = (idf_copy / 'examples' / 'get-started' / 'hello_world') - run_idf_py('-DIDF_TARGET=esp32s2', 'dfu', workdir=hello_world_app_path) - clean_app_dir(hello_world_app_path) - # test uf2 - run_idf_py('uf2', workdir=hello_world_app_path) @pytest.mark.idf_copy('esp idf with spaces') @@ -69,6 +62,26 @@ def test_spaces_bundle3(idf_copy: Path) -> None: workdir=secure_boot_app_path) +# Use this bundle for tests which can be done with the default build_test_app +@pytest.mark.parametrize('dummy_', [ + # Dummy parameter with a space in it, used so that the test directory name contains a space + pytest.param('test spaces') +]) +@pytest.mark.idf_copy('esp idf with spaces') +@pytest.mark.usefixtures('idf_copy') +def test_spaces_bundle4(dummy_: str, idf_py: IdfPyFunc, test_app_copy: Path) -> None: + logging.info(f'Running test spaces bundle 4 in {test_app_copy}') + (test_app_copy / 'sdkconfig').write_text('CONFIG_APP_REPRODUCIBLE_BUILD=y') + idf_py('build') + (test_app_copy / 'sdkconfig').unlink() + + idf_py('set-target', 'esp32s2') + + idf_py('dfu') + + idf_py('uf2') + + @pytest.mark.skipif(sys.platform == 'win32', reason='Unix test') @pytest.mark.idf_copy('esp idf with spaces') def test_install_export_unix(idf_copy: Path) -> None: