diff --git a/components/bt/CMakeLists.txt b/components/bt/CMakeLists.txt
index c52392971c..7adaa0248a 100644
--- a/components/bt/CMakeLists.txt
+++ b/components/bt/CMakeLists.txt
@@ -837,13 +837,13 @@ if(CONFIG_BT_ENABLED)
         target_link_directories(${COMPONENT_LIB} INTERFACE
                 "${CMAKE_CURRENT_LIST_DIR}/controller/lib_esp32c3_family/esp32s3")
         target_link_libraries(${COMPONENT_LIB} PUBLIC btdm_app)
-    elseif(CONFIG_IDF_TARGET_ESP32C2)
+    elseif(CONFIG_IDF_TARGET_ESP32C2 AND CONFIG_BT_CONTROLLER_ENABLED)
         add_prebuilt_library(libble_app "controller/lib_esp32c2/esp32c2-bt-lib/libble_app.a")
         target_link_libraries(${COMPONENT_LIB} PRIVATE libble_app)
-    elseif(CONFIG_IDF_TARGET_ESP32C6)
+    elseif(CONFIG_IDF_TARGET_ESP32C6 AND CONFIG_BT_CONTROLLER_ENABLED)
         add_prebuilt_library(libble_app "controller/lib_esp32c6/esp32c6-bt-lib/libble_app.a")
         target_link_libraries(${COMPONENT_LIB} PRIVATE libble_app)
-    elseif(CONFIG_IDF_TARGET_ESP32H2)
+    elseif(CONFIG_IDF_TARGET_ESP32H2 AND CONFIG_BT_CONTROLLER_ENABLED)
         add_prebuilt_library(libble_app "controller/lib_esp32h2/esp32h2-bt-lib/libble_app.a")
         target_link_libraries(${COMPONENT_LIB} PRIVATE libble_app)
     endif()