Merge branch 'bugfix/ldgen_prebuilt_library_placements' into 'master'

build: pass pre-built libraries to ldgen, clean up how blobs are added Closes IDF-12049 and IDF-12736 See merge request espressif/esp-idf!40353
2025-10-03 10:30:58 +02:00 · 2025-08-19 10:50:31 +02:00
parent 6af87b6287 f45154b347
commit a939f410dd
14 changed files with 153 additions and 62 deletions
--- a/components/bt/CMakeLists.txt
+++ b/components/bt/CMakeLists.txt
@@ -937,29 +937,30 @@ idf_component_register(SRCS "${srcs}"
 if(CONFIG_BT_ENABLED)
    target_compile_options(${COMPONENT_LIB} PRIVATE -Wno-implicit-fallthrough -Wno-unused-const-variable)
    if(CONFIG_IDF_TARGET_ESP32)
-        target_link_directories(${COMPONENT_LIB} INTERFACE "${CMAKE_CURRENT_LIST_DIR}/controller/lib_esp32/esp32")
-        target_link_libraries(${COMPONENT_LIB} PUBLIC btdm_app)
-
-        target_link_libraries(${COMPONENT_LIB} INTERFACE "-u ld_include_hli_vectors_bt")
+        add_prebuilt_library(bt_btdm_app "${CMAKE_CURRENT_LIST_DIR}/controller/lib_esp32/esp32/libbtdm_app.a")
+        target_link_libraries(${COMPONENT_LIB} PRIVATE bt_btdm_app)
+        target_link_options(${COMPONENT_LIB} INTERFACE "SHELL:-u ld_include_hli_vectors_bt")
    elseif(CONFIG_IDF_TARGET_ESP32C3)
-        target_link_directories(${COMPONENT_LIB} INTERFACE
-                "${CMAKE_CURRENT_LIST_DIR}/controller/lib_esp32c3_family/esp32c3")
        if(CONFIG_BT_CTRL_RUN_IN_FLASH_ONLY)
-            target_link_libraries(${COMPONENT_LIB} PUBLIC btdm_app_flash)
+            set(lib_name "btdm_app_flash")
        else()
-            target_link_libraries(${COMPONENT_LIB} PUBLIC btdm_app)
+            set(lib_name "btdm_app")
        endif()
+        add_prebuilt_library(bt_btdm_app
+                             "${CMAKE_CURRENT_LIST_DIR}/controller/lib_esp32c3_family/esp32c3/lib${lib_name}.a")
+        target_link_libraries(${COMPONENT_LIB} PRIVATE bt_btdm_app)
    elseif(CONFIG_IDF_TARGET_ESP32S3)
-        target_link_directories(${COMPONENT_LIB} INTERFACE
-                "${CMAKE_CURRENT_LIST_DIR}/controller/lib_esp32c3_family/esp32s3")
        if(CONFIG_BT_CTRL_RUN_IN_FLASH_ONLY)
-            target_link_libraries(${COMPONENT_LIB} PUBLIC btdm_app_flash)
+            set(lib_name "btdm_app_flash")
        else()
-            target_link_libraries(${COMPONENT_LIB} PUBLIC btdm_app)
+            set(lib_name "btdm_app")
        endif()
+        add_prebuilt_library(bt_btdm_app
+                             "${CMAKE_CURRENT_LIST_DIR}/controller/lib_esp32c3_family/esp32s3/lib${lib_name}.a")
+        target_link_libraries(${COMPONENT_LIB} PRIVATE bt_btdm_app)
    elseif(CONFIG_BT_CONTROLLER_ENABLED)
        if(CONFIG_BT_LE_CONTROLLER_LOG_WRAP_PANIC_HANDLER_ENABLE)
-            target_link_libraries(${COMPONENT_LIB} INTERFACE "-Wl,--wrap=esp_panic_handler")
+            target_link_options(${COMPONENT_LIB} INTERFACE "-Wl,--wrap=esp_panic_handler")
        endif()
        if(CONFIG_BT_BLE_LOG_UHCI_OUT_ENABLED)
            target_link_libraries(${COMPONENT_LIB} INTERFACE "-Wl,--wrap=uart_tx_chars")
--- a/components/esp_coex/CMakeLists.txt
+++ b/components/esp_coex/CMakeLists.txt
@@ -45,7 +45,7 @@ if(CONFIG_ESP_COEX_ENABLED)
                                    REQUIRES ${COMPONENT_NAME})
            target_link_libraries(${COMPONENT_LIB} PUBLIC ${blob})
            if(CONFIG_IDF_TARGET_ESP32)
-                target_link_libraries(${COMPONENT_LIB} PRIVATE btdm_app)
+                target_link_libraries(${COMPONENT_LIB} PRIVATE bt_btdm_app)
            endif()
        endif()

--- a/components/esp_phy/CMakeLists.txt
+++ b/components/esp_phy/CMakeLists.txt
@@ -61,42 +61,84 @@ idf_component_register(SRCS "${srcs}"
                    )

 if(CONFIG_ESP_PHY_ENABLED)
-    set(target_name "${idf_target}")
-    target_link_directories(${COMPONENT_LIB} PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/lib/${target_name}")
+    set(lib_dir "${CMAKE_CURRENT_SOURCE_DIR}/lib/${idf_target}")

    # Override functions in PHY lib with the functions in 'phy_override.c'
-    target_link_libraries(${COMPONENT_LIB} INTERFACE "-u include_esp_phy_override")
+    target_link_options(${COMPONENT_LIB} INTERFACE "SHELL:-u include_esp_phy_override")

    if(link_binary_libs)
-        target_link_libraries(${COMPONENT_LIB} PUBLIC phy)

-        idf_component_get_property(esp_phy_lib esp_phy COMPONENT_LIB)
+        # Decide which libraries to link based on the configuration and the target
+        set(has_libbttestmode 0)
+        set(has_librftest 0)
+        set(has_librfate 0)
+        set(has_libphy 0)
+        set(has_libbtbb 0)
+        set(has_librtc 0)

        if(CONFIG_ESP_PHY_ENABLE_CERT_TEST)
            if(CONFIG_IDF_TARGET_ESP32)
-                target_link_libraries(${COMPONENT_LIB} INTERFACE $<TARGET_FILE:${esp_phy_lib}> libbttestmode.a
-                librftest.a $<TARGET_FILE:${esp_phy_lib}>)
+                set(has_libbttestmode 1)
+                set(has_librftest 1)
            elseif(CONFIG_IDF_TARGET_ESP32S2)
-                target_link_libraries(${COMPONENT_LIB} INTERFACE $<TARGET_FILE:${esp_phy_lib}> librftest.a
-                librfate.a $<TARGET_FILE:${esp_phy_lib}>)
+                set(has_librftest 1)
+                set(has_librfate 1)
            else()
-                target_link_libraries(${COMPONENT_LIB} INTERFACE $<TARGET_FILE:${esp_phy_lib}> libbttestmode.a
-                librfate.a librftest.a $<TARGET_FILE:${esp_phy_lib}>)
+                set(has_libbttestmode 1)
+                set(has_librftest 1)
+                set(has_librfate 1)
            endif()
        endif()

        if(CONFIG_IDF_TARGET_ESP32)
-            target_link_libraries(${COMPONENT_LIB} PUBLIC rtc)
-            target_link_libraries(${COMPONENT_LIB} INTERFACE $<TARGET_FILE:${esp_phy_lib}> libphy.a librtc.a
-            $<TARGET_FILE:${esp_phy_lib}>)
+            set(has_librtc 1)
+            set(has_libphy 1)
        elseif(CONFIG_SOC_BT_SUPPORTED OR CONFIG_SOC_IEEE802154_SUPPORTED)
-            target_link_libraries(${COMPONENT_LIB} PUBLIC btbb)
-            target_link_libraries(${COMPONENT_LIB} INTERFACE $<TARGET_FILE:${esp_phy_lib}> libphy.a libbtbb.a
-            $<TARGET_FILE:${esp_phy_lib}>)
+            set(has_libbtbb 1)
+            set(has_libphy 1)
        elseif(CONFIG_SOC_WIFI_SUPPORTED)
-            target_link_libraries(${COMPONENT_LIB} INTERFACE $<TARGET_FILE:${esp_phy_lib}> libphy.a
-            $<TARGET_FILE:${esp_phy_lib}>)
+            set(has_libphy 1)
        endif()
+
+        # Add the required library targets and add them to the list of libraries to link
+        if(has_libbttestmode)
+            add_prebuilt_library(esp_phy_libbttestmode "${lib_dir}/libbttestmode.a"
+                                REQUIRES ${COMPONENT_NAME})
+            list(APPEND libs_to_link esp_phy_libbttestmode)
+        endif()
+
+        if(has_librftest)
+            add_prebuilt_library(esp_phy_librftest "${lib_dir}/librftest.a"
+                                REQUIRES ${COMPONENT_NAME})
+            list(APPEND libs_to_link esp_phy_librftest)
+        endif()
+
+        if(has_librfate)
+            add_prebuilt_library(esp_phy_librfate "${lib_dir}/librfate.a"
+                                REQUIRES ${COMPONENT_NAME})
+            list(APPEND libs_to_link esp_phy_librfate)
+        endif()
+
+        if(has_libbtbb)
+            add_prebuilt_library(esp_phy_libbtbb "${lib_dir}/libbtbb.a"
+                                 REQUIRES ${COMPONENT_NAME})
+            list(APPEND libs_to_link esp_phy_libbtbb)
+        endif()
+
+        if(has_librtc)
+            add_prebuilt_library(esp_phy_librtc "${lib_dir}/librtc.a"
+                                 REQUIRES ${COMPONENT_NAME})
+            list(APPEND libs_to_link esp_phy_librtc)
+        endif()
+
+        if(has_libphy)
+            add_prebuilt_library(esp_phy_libphy "${lib_dir}/libphy.a"
+                                 REQUIRES ${COMPONENT_NAME})
+            list(APPEND libs_to_link esp_phy_libphy)
+        endif()
+
+        # Finally, link the libraries to the component
+        target_link_libraries(${COMPONENT_LIB} INTERFACE ${libs_to_link})
    endif()

   if(CONFIG_ESP_PHY_INIT_DATA_IN_PARTITION)
--- a/components/esp_wifi/CMakeLists.txt
+++ b/components/esp_wifi/CMakeLists.txt
@@ -88,13 +88,18 @@ if(CONFIG_ESP_WIFI_ENABLED OR CONFIG_ESP_HOST_WIFI_ENABLED)
            endif()
        endif()

+        # Since some blob names are very generic, add an "esp_wifi_" prefix when creating target names,
+        # to avoid conflicts with other libraries.
+        set(blob_targets ${blobs})
+        list(TRANSFORM blob_targets PREPEND "esp_wifi_")
        foreach(blob ${blobs})
-            add_prebuilt_library(${blob} "${CMAKE_CURRENT_SOURCE_DIR}/lib/${target_name}/lib${blob}.a"
+            set(blob_target "esp_wifi_${blob}")
+            add_prebuilt_library(${blob_target} "${CMAKE_CURRENT_SOURCE_DIR}/lib/${target_name}/lib${blob}.a"
                                REQUIRES ${COMPONENT_NAME})
-            set(blob_reqs ${blobs})
-            list(REMOVE_ITEM blob_reqs ${blob}) # remove itself from requirements
-            set_property(TARGET ${blob} APPEND PROPERTY INTERFACE_LINK_LIBRARIES ${blob_reqs})
-            target_link_libraries(${COMPONENT_LIB} PUBLIC ${blob})
+            set(blob_reqs ${blob_targets})
+            list(REMOVE_ITEM blob_reqs ${blob_target}) # remove itself from requirements
+            set_property(TARGET ${blob_target} APPEND PROPERTY INTERFACE_LINK_LIBRARIES ${blob_reqs})
+            target_link_libraries(${COMPONENT_LIB} PUBLIC ${blob_target})
        endforeach()
    endif()
    # TODO IDF-13365: remove the following lines
--- a/components/xtensa/CMakeLists.txt
+++ b/components/xtensa/CMakeLists.txt
@@ -13,7 +13,7 @@
 # The porting layer sources files are OS agnostic, thus are common across multiple Xtensa RTOS ports (e.g., FreeRTOS,
 # ThreadX). The Xtensa RTOS Porting Layer...
 #   - interfaces with an RTOS port via the "xtensa_rtos.h" header provided by the RTOS port
-#   - expected `#include <...h>` as the incldue path to the porting layer's headers
+#   - expected `#include <...h>` as the include path to the porting layer's headers

 idf_build_get_property(target IDF_TARGET)
 idf_build_get_property(arch IDF_TARGET_ARCH)
@@ -63,4 +63,6 @@ idf_component_register(SRCS ${srcs}
                    INCLUDE_DIRS ${include_dirs}
                    LDFRAGMENTS linker.lf)

-target_link_libraries(${COMPONENT_LIB} PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/${target}/libxt_hal.a")
+add_prebuilt_library(xtensa_xt_hal "${CMAKE_CURRENT_SOURCE_DIR}/${target}/libxt_hal.a"
+                                REQUIRES ${COMPONENT_NAME})
+target_link_libraries(${COMPONENT_LIB} PUBLIC xtensa_xt_hal)
--- a/tools/cmake/ldgen.cmake
+++ b/tools/cmake/ldgen.cmake
@@ -135,16 +135,14 @@ function(__ldgen_create_target exe_target)
            # Here we have two cases:
            #
            # 1. ${lib} is actually a target, but the target is outside the current scope.
-            #    This is the case for imported library targets (such as those added by
-            #    add_prebuilt_library), since by default imported targets are not
-            #    visible outside the directory where they are defined, unless they are
-            #    marked as GLOBAL.
-            #    This case covers many (but not all) of IDF's prebuilt libraries.
+            #    This is the case for imported library targets which are not defined as GLOBAL.
+            #    Libraries added using `add_prebuilt_library` are marked as GLOBAL, so they should
+            #    be handled in the if block above.
            #
            # 2. ${lib} is the name of a library, which the linker can find in its built-in
            #    or specified search paths.
-            #    This is the case for toolchain libraries (m, c, gcc, stdc++) as well
-            #    as for some prebuilt libraries which have been added using `-lname -Lpath`
+            #    This is the case for toolchain libraries (m, c, gcc, stdc++), as well as
+            #    any prebuilt libraries which may have been added using `-lname -Lpath`
            #    style flags.
            #
            # If we can successfully find the absolute path of each such library, this
--- a/tools/cmake/utilities.cmake
+++ b/tools/cmake/utilities.cmake
@@ -157,7 +157,7 @@ function(target_linker_script target deptype scriptfiles)
        # https://cmake.org/cmake/help/latest/command/target_link_options.html#option-de-duplication
        target_link_options("${target}" "${deptype}" "SHELL:-T ${scriptname}")

-        # Note: In ESP-IDF, most targets are libraries and libary LINK_DEPENDS don't propagate to
+        # Note: In ESP-IDF, most targets are libraries and library LINK_DEPENDS don't propagate to
        # executable(s) the library is linked to. Since CMake 3.13, INTERFACE_LINK_DEPENDS is
        # available to solve this. However, when GNU Make generator is used, this property also
        # propagates INTERFACE_LINK_DEPENDS dependencies to other static libraries.
@@ -225,7 +225,7 @@ endfunction()
 # fail_target
 #
 # Creates a phony target which fails when invoked. This is used when the necessary conditions
-# for a target are not met, such as configuration. Rather than ommitting the target altogether,
+# for a target are not met, such as configuration. Rather than omitting the target altogether,
 # we fail execution with a helpful message.
 function(fail_target target_name message_line0)
    idf_build_get_property(idf_path IDF_PATH)
@@ -303,7 +303,7 @@ function(add_prebuilt_library target_name lib_path)
    get_filename_component(lib_path "${lib_path}"
                ABSOLUTE BASE_DIR "${CMAKE_CURRENT_LIST_DIR}")

-    add_library(${target_name} STATIC IMPORTED)
+    add_library(${target_name} STATIC IMPORTED GLOBAL)
    set_property(TARGET ${target_name} PROPERTY IMPORTED_LOCATION ${lib_path})

    foreach(req ${__REQUIRES})
@@ -354,7 +354,7 @@ endfunction()

 # add_deprecated_target_alias
 #
-# Creates an alias for exising target and shows deprectation warning
+# Creates an alias for existing target and shows deprecation warning
 function(add_deprecated_target_alias old_target new_target)
    add_custom_target(${old_target}
     # `COMMAND` is important to print the `COMMENT` message at the end of the target action.
--- a/tools/test_apps/build_system/ldgen_test/check_placements.py
+++ b/tools/test_apps/build_system/ldgen_test/check_placements.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 #
-# SPDX-FileCopyrightText: 2020-2024 Espressif Systems (Shanghai) CO LTD
+# SPDX-FileCopyrightText: 2020-2025 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
 #
 # Check placements in this test app for main
@@ -8,13 +8,13 @@
 import argparse
 import subprocess

-from pyparsing import alphanums
-from pyparsing import hexnums
 from pyparsing import LineEnd
 from pyparsing import LineStart
 from pyparsing import Literal
 from pyparsing import Optional
 from pyparsing import Word
+from pyparsing import alphanums
+from pyparsing import hexnums

 argparser = argparse.ArgumentParser()

@@ -29,13 +29,16 @@ contents = subprocess.check_output([args.objdump, '-t', args.elf]).decode()


 def check_location(symbol, expected):
-    pattern = (LineStart() + Word(hexnums).setResultsName('address')
-               + Optional(Word(alphanums, exact=1))
-               + Optional(Word(alphanums,exact=1))
-               + Word(alphanums + '._*').setResultsName('actual')
-               + Word(hexnums)
-               + Literal(symbol)
-               + LineEnd())
+    pattern = (
+        LineStart()
+        + Word(hexnums).setResultsName('address')
+        + Optional(Word(alphanums, exact=1))
+        + Optional(Word(alphanums, exact=1))
+        + Word(alphanums + '._*').setResultsName('actual')
+        + Word(hexnums)
+        + Literal(symbol)
+        + LineEnd()
+    )

    try:
        results = pattern.searchString(contents)[0]
@@ -43,7 +46,9 @@ def check_location(symbol, expected):
        raise Exception("check placement fail: '%s' was not found" % (symbol))

    if results.actual != expected:
-        raise Exception("check placement fail: '%s' was placed in '%s', not in '%s'" % (symbol, results.actual, expected))
+        raise Exception(
+            "check placement fail: '%s' was placed in '%s', not in '%s'" % (symbol, results.actual, expected)
+        )

    print("check placement pass: '%s' was successfully placed in '%s'" % (symbol, results.actual))
    return int(results.address, 16)
@@ -82,3 +87,5 @@ check_location('func3', '.flash.text')
 check_location('func4', '.iram0.text')

 check_location('const_array', '.dram0.data')
+
+check_location('prebuilt_func', '.iram0.text')
--- a/tools/test_apps/build_system/ldgen_test/components/prebuilt/CMakeLists.txt
+++ b/tools/test_apps/build_system/ldgen_test/components/prebuilt/CMakeLists.txt
@@ -0,0 +1,14 @@
+idf_component_register()
+
+include(ExternalProject)
+externalproject_add(subproject
+    SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/subproject
+    CMAKE_ARGS -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}
+    BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/subproject"
+    INSTALL_COMMAND ""
+    BUILD_BYPRODUCTS "${CMAKE_CURRENT_BINARY_DIR}/subproject/libprebuilt.a"
+)
+
+add_prebuilt_library(prebuilt "${CMAKE_CURRENT_BINARY_DIR}/subproject/libprebuilt.a")
+
+target_link_libraries(${COMPONENT_LIB} INTERFACE prebuilt)
--- a/tools/test_apps/build_system/ldgen_test/components/prebuilt/subproject/CMakeLists.txt
+++ b/tools/test_apps/build_system/ldgen_test/components/prebuilt/subproject/CMakeLists.txt
@@ -0,0 +1,5 @@
+cmake_minimum_required(VERSION 3.12)
+
+project(prebuilt)
+add_library(prebuilt STATIC prebuilt.c)
+target_compile_options(prebuilt PRIVATE "-ffunction-sections" "-fdata-sections")
--- a/tools/test_apps/build_system/ldgen_test/components/prebuilt/subproject/prebuilt.c
+++ b/tools/test_apps/build_system/ldgen_test/components/prebuilt/subproject/prebuilt.c
@@ -0,0 +1,9 @@
+/*
+ * SPDX-FileCopyrightText: 2025 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Unlicense OR CC0-1.0
+ */
+int prebuilt_func(void)
+{
+    return 42;
+}
--- a/tools/test_apps/build_system/ldgen_test/main/CMakeLists.txt
+++ b/tools/test_apps/build_system/ldgen_test/main/CMakeLists.txt
@@ -1,3 +1,4 @@
 idf_component_register(SRCS "src1.c" "src2.c" "test_main.c" "consts.c"
                       INCLUDE_DIRS "."
+                       PRIV_REQUIRES prebuilt
                       LDFRAGMENTS "linker.lf")
--- a/tools/test_apps/build_system/ldgen_test/main/linker.lf
+++ b/tools/test_apps/build_system/ldgen_test/main/linker.lf
@@ -10,3 +10,8 @@ entries:
    if SOC_RTC_MEM_SUPPORTED = y:
        src1:func2 (rtc)
    consts : const_array (noflash)
+
+[mapping:prebuilt]
+archive: libprebuilt.a
+entries:
+    prebuilt:prebuilt_func (noflash)
--- a/tools/test_apps/build_system/ldgen_test/main/test_main.c
+++ b/tools/test_apps/build_system/ldgen_test/main/test_main.c
@@ -11,12 +11,14 @@ extern void func3(void);
 extern void func4(void);

 extern const int const_array[];
+extern int prebuilt_func(void);

 void app_main(void)
 {
    func2();
    func3();
    func4();
+    prebuilt_func();
    if (esp_ptr_in_dram(const_array)) {
        printf("const_array placed in dram\n");
    } else {