From d595ea9b2be9a470b2d17e95fc60ba5fa8b81e4d Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Mon, 9 Dec 2024 15:21:50 +0100 Subject: [PATCH 1/3] feat: support kconfig in component manager --- tools/cmake/build.cmake | 40 ++++-- tools/cmake/kconfig.cmake | 8 +- tools/cmake/project.cmake | 35 ++++-- .../test_component_manager.py | 114 +++++++++++++++++- 4 files changed, 175 insertions(+), 22 deletions(-) diff --git a/tools/cmake/build.cmake b/tools/cmake/build.cmake index e3589c4891..64a37bf694 100644 --- a/tools/cmake/build.cmake +++ b/tools/cmake/build.cmake @@ -540,11 +540,15 @@ macro(idf_build_process target) # Call for component manager to download dependencies for all components idf_build_get_property(idf_component_manager IDF_COMPONENT_MANAGER) + + set(result 0) if(idf_component_manager EQUAL 1) idf_build_get_property(build_dir BUILD_DIR) set(managed_components_list_file ${build_dir}/managed_components_list.temp.cmake) set(local_components_list_file ${build_dir}/local_components_list.temp.yml) + set(__RERUN_EXITCODE 10) # missing kconfig + set(__contents "components:\n") idf_build_get_property(build_component_targets BUILD_COMPONENT_TARGETS) foreach(__build_component_target ${build_component_targets}) @@ -571,6 +575,7 @@ macro(idf_build_process target) "idf_component_manager.prepare_components" "--project_dir=${project_dir}" "--lock_path=${dependencies_lock_file}" + "--sdkconfig_json_file=${build_dir}/config/sdkconfig.json" "--interface_version=${component_manager_interface_version}" "prepare_dependencies" "--local_components_list_file=${local_components_list_file}" @@ -579,7 +584,11 @@ macro(idf_build_process target) ERROR_VARIABLE error) if(NOT result EQUAL 0) - message(FATAL_ERROR "${error}") + if(result EQUAL ${__RERUN_EXITCODE}) + message(WARNING "${error}") + else() + message(FATAL_ERROR "${error}") + endif() endif() include(${managed_components_list_file}) @@ -650,21 +659,30 @@ macro(idf_build_process target) # Generate config values in different formats idf_build_get_property(sdkconfig SDKCONFIG) idf_build_get_property(sdkconfig_defaults SDKCONFIG_DEFAULTS) - __kconfig_generate_config("${sdkconfig}" "${sdkconfig_defaults}") + + # add target here since we have all components + if(result EQUAL 0) + __kconfig_generate_config("${sdkconfig}" "${sdkconfig_defaults}" CREATE_MENUCONFIG_TARGET) + else() + __kconfig_generate_config("${sdkconfig}" "${sdkconfig_defaults}") + endif() + __build_import_configs() - # All targets built under this scope is with the ESP-IDF build system - set(ESP_PLATFORM 1) - idf_build_set_property(COMPILE_DEFINITIONS "ESP_PLATFORM" APPEND) + if(result EQUAL 0) + # All targets built under this scope is with the ESP-IDF build system + set(ESP_PLATFORM 1) + idf_build_set_property(COMPILE_DEFINITIONS "ESP_PLATFORM" APPEND) - # Perform component processing (inclusion of project_include.cmake, adding component - # subdirectories, creating library targets, linking libraries, etc.) - __build_process_project_includes() + # Perform component processing (inclusion of project_include.cmake, adding component + # subdirectories, creating library targets, linking libraries, etc.) + __build_process_project_includes() - idf_build_get_property(idf_path IDF_PATH) - add_subdirectory(${idf_path} ${build_dir}/esp-idf) + idf_build_get_property(idf_path IDF_PATH) + add_subdirectory(${idf_path} ${build_dir}/esp-idf) - unset(ESP_PLATFORM) + unset(ESP_PLATFORM) + endif() endmacro() # idf_build_executable diff --git a/tools/cmake/kconfig.cmake b/tools/cmake/kconfig.cmake index a768e7df58..7d6f3c3767 100644 --- a/tools/cmake/kconfig.cmake +++ b/tools/cmake/kconfig.cmake @@ -97,12 +97,14 @@ function(__get_init_config_version config version_out) endforeach() endfunction() - # # Generate the config files and create config related targets and configure # dependencies. # function(__kconfig_generate_config sdkconfig sdkconfig_defaults) + set(options OPTIONAL CREATE_MENUCONFIG_TARGET) + cmake_parse_arguments(PARSE_ARGV 2 "ARG" "${options}" "" "") + # List all Kconfig and Kconfig.projbuild in known components idf_build_get_property(component_targets __COMPONENT_TARGETS) idf_build_get_property(build_component_targets __BUILD_COMPONENT_TARGETS) @@ -253,6 +255,10 @@ function(__kconfig_generate_config sdkconfig sdkconfig_defaults) set(MENUCONFIG_CMD ${python} -m menuconfig) set(TERM_CHECK_CMD ${python} ${idf_path}/tools/check_term.py) + if(NOT ${ARG_CREATE_MENUCONFIG_TARGET}) + return() + endif() + # Generate the menuconfig target add_custom_target(menuconfig ${menuconfig_depends} diff --git a/tools/cmake/project.cmake b/tools/cmake/project.cmake index 45ad85ebb2..323b7d7880 100644 --- a/tools/cmake/project.cmake +++ b/tools/cmake/project.cmake @@ -61,7 +61,7 @@ if(NOT "$ENV{IDF_COMPONENT_MANAGER}" EQUAL "0") idf_build_set_property(IDF_COMPONENT_MANAGER 1) endif() # Set component manager interface version -idf_build_set_property(__COMPONENT_MANAGER_INTERFACE_VERSION 3) +idf_build_set_property(__COMPONENT_MANAGER_INTERFACE_VERSION 4) # # Parse and store the VERSION argument provided to the project() command. @@ -715,14 +715,31 @@ macro(project project_name) message(STATUS "Building ESP-IDF components for target ${IDF_TARGET}") - idf_build_process(${IDF_TARGET} - SDKCONFIG_DEFAULTS "${sdkconfig_defaults}" - SDKCONFIG ${sdkconfig} - BUILD_DIR ${build_dir} - PROJECT_NAME ${CMAKE_PROJECT_NAME} - PROJECT_DIR ${CMAKE_CURRENT_LIST_DIR} - PROJECT_VER "${project_ver}" - COMPONENTS "${components};${test_components}") + set(result 0) + set(retried 0) + + while(true) + idf_build_process(${IDF_TARGET} + SDKCONFIG_DEFAULTS "${sdkconfig_defaults}" + SDKCONFIG ${sdkconfig} + BUILD_DIR ${build_dir} + PROJECT_NAME ${CMAKE_PROJECT_NAME} + PROJECT_DIR ${CMAKE_CURRENT_LIST_DIR} + PROJECT_VER "${project_ver}" + COMPONENTS "${components};${test_components}" + ) + + if(result EQUAL 0) + break() + elseif(result EQUAL 10 AND retried EQUAL 0) + message(WARNING "Missing kconfig option. Re-run the build process...") + set(retried 1) + elseif(result EQUAL 10 AND retried EQUAL 1) + message(FATAL_ERROR "Missing required kconfig option after retry.") + else() + message(FATAL_ERROR "idf_build_process failed with exit code ${result}") + endif() + endwhile() # Special treatment for 'main' component for standard projects (not part of core build system). # Have it depend on every other component in the build. This is diff --git a/tools/test_build_system/test_component_manager.py b/tools/test_build_system/test_component_manager.py index ace7f85524..e0b0d10ff1 100644 --- a/tools/test_build_system/test_component_manager.py +++ b/tools/test_build_system/test_component_manager.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: 2022-2024 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2022-2025 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 import os.path from pathlib import Path @@ -44,3 +44,115 @@ def test_trimmed_components_still_passed_to_cmake(idf_py: IdfPyFunc, test_app_co fs = f.read() assert ' example/cmp:' in fs + + +class TestOptionalDependencyWithKconfig: + def test_direct_kconfig(self, idf_py: IdfPyFunc, test_app_copy: Path) -> None: + (test_app_copy / 'main' / 'idf_component.yml').write_text(""" + dependencies: + example/cmp: + version: "*" + rules: + - if: $CONFIG{BOOTLOADER_PROJECT_VER} == 1 + """) + + idf_py('reconfigure') + + with open('dependencies.lock', 'r') as f: + fs = f.read() + assert ' example/cmp:' in fs + + (test_app_copy / 'main' / 'idf_component.yml').write_text(""" + dependencies: + example/cmp: + version: "*" + rules: + - if: $CONFIG{BOOTLOADER_PROJECT_VER} != 1 + """) + + idf_py('reconfigure') + + with open('dependencies.lock', 'r') as f: + fs = f.read() + assert ' example/cmp:' not in fs + + def test_missing_kconfig_first_round(self, idf_py: IdfPyFunc, test_app_copy: Path) -> None: + (test_app_copy / 'main' / 'idf_component.yml').write_text(""" + dependencies: + espressif/mdns: + version: "*" + + example/cmp: + version: "*" + rules: + - if: $CONFIG{MDNS_MAX_SERVICES} == 10 # mdns kconfig CONFIG_MDNS_MAX_SERVICES default to 10 + """) + + idf_py('reconfigure') + + with open('dependencies.lock', 'r') as f: + fs = f.read() + assert ' example/cmp:' in fs + + def test_kconfig_load_existing_sdkconfig_defaults(self, idf_py: IdfPyFunc, test_app_copy: Path) -> None: + (test_app_copy / 'main' / 'idf_component.yml').write_text(""" + dependencies: + example/cmp: + version: "*" + rules: + - if: $CONFIG{BOOTLOADER_LOG_LEVEL_WARN} == True + """) + + (test_app_copy / 'sdkconfig.defaults').write_text(""" + CONFIG_BOOTLOADER_LOG_LEVEL_WARN=y + CONFIG_LOG_DEFAULT_LEVEL_WARN=y + """) + + idf_py('reconfigure') + + with open('dependencies.lock', 'r') as f: + fs = f.read() + assert ' example/cmp:' in fs + + def test_kconfig_load_missing_sdkconfig_defaults(self, idf_py: IdfPyFunc, test_app_copy: Path) -> None: + (test_app_copy / 'main' / 'idf_component.yml').write_text(""" + dependencies: + espressif/mdns: + version: "*" + + example/cmp: + version: "*" + rules: + - if: $CONFIG{MDNS_MAX_SERVICES} == 30 # mdns kconfig CONFIG_MDNS_MAX_SERVICES default to 10 + """) + + (test_app_copy / 'sdkconfig.defaults').write_text(""" + CONFIG_MDNS_MAX_SERVICES=30 + """) + + idf_py('reconfigure') + + with open('dependencies.lock', 'r') as f: + fs = f.read() + assert ' example/cmp:' in fs + + def test_missing_kconfig_second_round(self, idf_py: IdfPyFunc, test_app_copy: Path) -> None: + (test_app_copy / 'main' / 'idf_component.yml').write_text(""" + dependencies: + espressif/mdns: + version: "*" + + example/cmp: + version: "*" + rules: + - if: $CONFIG{OF_COURSE_NO_ONE_USE_FOO} == "bar" + """) + + res = idf_py('reconfigure', check=False) + + assert res.returncode != 0 + assert ( + f'OF_COURSE_NO_ONE_USE_FOO, introduced by example/cmp, ' + f'defined in {str(test_app_copy / "main" / "idf_component.yml")}' in res.stderr + ) + assert 'Missing required kconfig option after retry.' in res.stderr From c16fd0da037c4134f9ebb5f7805cf6b86794ec77 Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Fri, 22 Aug 2025 15:11:14 +0200 Subject: [PATCH 2/3] fix: kconfig optional dependency in transitive dependency --- tools/cmake/component.cmake | 1 + .../test_component_manager.py | 48 ++++++++++++++++--- 2 files changed, 43 insertions(+), 6 deletions(-) diff --git a/tools/cmake/component.cmake b/tools/cmake/component.cmake index c674b7703c..e4d445cf65 100644 --- a/tools/cmake/component.cmake +++ b/tools/cmake/component.cmake @@ -242,6 +242,7 @@ function(__component_get_requirements) "idf_component_manager.prepare_components" "--project_dir=${project_dir}" "--lock_path=${DEPENDENCIES_LOCK}" + "--sdkconfig_json_file=${build_dir}/config/sdkconfig.json" "--interface_version=${component_manager_interface_version}" "inject_requirements" "--idf_path=${idf_path}" diff --git a/tools/test_build_system/test_component_manager.py b/tools/test_build_system/test_component_manager.py index e0b0d10ff1..5434bc6745 100644 --- a/tools/test_build_system/test_component_manager.py +++ b/tools/test_build_system/test_component_manager.py @@ -1,5 +1,6 @@ # SPDX-FileCopyrightText: 2022-2025 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 +import json import os.path from pathlib import Path @@ -40,7 +41,7 @@ def test_trimmed_components_still_passed_to_cmake(idf_py: IdfPyFunc, test_app_co idf_py('reconfigure') - with open('dependencies.lock', 'r') as f: + with open('dependencies.lock') as f: fs = f.read() assert ' example/cmp:' in fs @@ -58,7 +59,7 @@ class TestOptionalDependencyWithKconfig: idf_py('reconfigure') - with open('dependencies.lock', 'r') as f: + with open('dependencies.lock') as f: fs = f.read() assert ' example/cmp:' in fs @@ -72,7 +73,7 @@ class TestOptionalDependencyWithKconfig: idf_py('reconfigure') - with open('dependencies.lock', 'r') as f: + with open('dependencies.lock') as f: fs = f.read() assert ' example/cmp:' not in fs @@ -90,7 +91,7 @@ class TestOptionalDependencyWithKconfig: idf_py('reconfigure') - with open('dependencies.lock', 'r') as f: + with open('dependencies.lock') as f: fs = f.read() assert ' example/cmp:' in fs @@ -110,7 +111,7 @@ class TestOptionalDependencyWithKconfig: idf_py('reconfigure') - with open('dependencies.lock', 'r') as f: + with open('dependencies.lock') as f: fs = f.read() assert ' example/cmp:' in fs @@ -132,7 +133,7 @@ class TestOptionalDependencyWithKconfig: idf_py('reconfigure') - with open('dependencies.lock', 'r') as f: + with open('dependencies.lock') as f: fs = f.read() assert ' example/cmp:' in fs @@ -156,3 +157,38 @@ class TestOptionalDependencyWithKconfig: f'defined in {str(test_app_copy / "main" / "idf_component.yml")}' in res.stderr ) assert 'Missing required kconfig option after retry.' in res.stderr + + def test_kconfig_in_transitive_dependency(self, idf_py: IdfPyFunc, test_app_copy: Path) -> None: + idf_py('create-component', 'foo') + (test_app_copy / 'foo' / 'idf_component.yml').write_text(""" + dependencies: + example/cmp: + version: "*" + rules: + - if: $CONFIG{WHO_AM_I} == "foo" + + espressif/mdns: + version: "*" + require: public + rules: + - if: $CONFIG{WHO_AM_I} == "foo" + """) + (test_app_copy / 'foo' / 'Kconfig').write_text(""" + menu "foo component config" + config WHO_AM_I + string "Who am I" + default "foo" + endmenu + """) + + replace_in_file( + (test_app_copy / 'CMakeLists.txt'), + '# placeholder_before_include_project_cmake', + 'set(EXTRA_COMPONENT_DIRS foo)', + ) + + idf_py('reconfigure') + + data = json.load(open(test_app_copy / 'build' / 'project_description.json')) + assert ['example__cmp'] == data['build_component_info']['foo']['priv_reqs'] + assert ['espressif__mdns'] == data['build_component_info']['foo']['reqs'] From 879494b7ac5bde5aaf4a76970f07f245efeb1802 Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Fri, 22 Aug 2025 15:12:30 +0200 Subject: [PATCH 3/3] fix: make sure sdkconfig.json is updated after running the menuconfig --- tools/cmake/kconfig.cmake | 37 +++++++++++++++---------------------- 1 file changed, 15 insertions(+), 22 deletions(-) diff --git a/tools/cmake/kconfig.cmake b/tools/cmake/kconfig.cmake index 7d6f3c3767..22bb1b9063 100644 --- a/tools/cmake/kconfig.cmake +++ b/tools/cmake/kconfig.cmake @@ -201,30 +201,23 @@ function(__kconfig_generate_config sdkconfig sdkconfig_defaults) set(sdkconfig_json ${config_dir}/sdkconfig.json) set(sdkconfig_json_menus ${config_dir}/kconfig_menus.json) + set(kconfgen_output_options + --output header ${sdkconfig_header} + --output cmake ${sdkconfig_cmake} + --output json ${sdkconfig_json} + --output json_menus ${sdkconfig_json_menus}) idf_build_get_property(output_sdkconfig __OUTPUT_SDKCONFIG) if(output_sdkconfig) - execute_process( - COMMAND ${prepare_kconfig_files_command}) - execute_process( - COMMAND ${kconfgen_basecommand} - --output header ${sdkconfig_header} - --output cmake ${sdkconfig_cmake} - --output json ${sdkconfig_json} - --output json_menus ${sdkconfig_json_menus} - --output config ${sdkconfig} - RESULT_VARIABLE config_result) - else() - execute_process( - COMMAND ${prepare_kconfig_files_command}) - execute_process( - COMMAND ${kconfgen_basecommand} - --output header ${sdkconfig_header} - --output cmake ${sdkconfig_cmake} - --output json ${sdkconfig_json} - --output json_menus ${sdkconfig_json_menus} - RESULT_VARIABLE config_result) + list(APPEND kconfgen_output_options --output config ${sdkconfig}) endif() + execute_process( + COMMAND ${prepare_kconfig_files_command}) + execute_process( + COMMAND ${kconfgen_basecommand} + ${kconfgen_output_options} + RESULT_VARIABLE config_result) + if(config_result) message(FATAL_ERROR "Failed to run kconfgen (${kconfgen_basecommand}). Error ${config_result}") endif() @@ -270,7 +263,7 @@ function(__kconfig_generate_config sdkconfig sdkconfig_defaults) --env "IDF_ENV_FPGA=${idf_env_fpga}" --env "IDF_INIT_VERSION=${idf_init_version}" --dont-write-deprecated - --output config ${sdkconfig} + ${kconfgen_output_options} COMMAND ${TERM_CHECK_CMD} COMMAND ${CMAKE_COMMAND} -E env "COMPONENT_KCONFIGS_SOURCE_FILE=${kconfigs_path}" @@ -289,7 +282,7 @@ function(__kconfig_generate_config sdkconfig sdkconfig_defaults) --env "IDF_TOOLCHAIN=${idf_toolchain}" --env "IDF_ENV_FPGA=${idf_env_fpga}" --env "IDF_INIT_VERSION=${idf_init_version}" - --output config ${sdkconfig} + ${kconfgen_output_options} ) # Custom target to run kconfserver from the build tool