diff --git a/components/esp_phy/CMakeLists.txt b/components/esp_phy/CMakeLists.txt
index 24defd87b1..984fb3685a 100644
--- a/components/esp_phy/CMakeLists.txt
+++ b/components/esp_phy/CMakeLists.txt
@@ -61,42 +61,84 @@ idf_component_register(SRCS "${srcs}"
                     )
 
 if(CONFIG_ESP_PHY_ENABLED)
-    set(target_name "${idf_target}")
-    target_link_directories(${COMPONENT_LIB} PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/lib/${target_name}")
+    set(lib_dir "${CMAKE_CURRENT_SOURCE_DIR}/lib/${idf_target}")
 
     # Override functions in PHY lib with the functions in 'phy_override.c'
-    target_link_libraries(${COMPONENT_LIB} INTERFACE "-u include_esp_phy_override")
+    target_link_options(${COMPONENT_LIB} INTERFACE "SHELL:-u include_esp_phy_override")
 
     if(link_binary_libs)
-        target_link_libraries(${COMPONENT_LIB} PUBLIC phy)
 
-        idf_component_get_property(esp_phy_lib esp_phy COMPONENT_LIB)
+        # Decide which libraries to link based on the configuration and the target
+        set(has_libbttestmode 0)
+        set(has_librftest 0)
+        set(has_librfate 0)
+        set(has_libphy 0)
+        set(has_libbtbb 0)
+        set(has_librtc 0)
 
         if(CONFIG_ESP_PHY_ENABLE_CERT_TEST)
             if(CONFIG_IDF_TARGET_ESP32)
-                target_link_libraries(${COMPONENT_LIB} INTERFACE $<TARGET_FILE:${esp_phy_lib}> libbttestmode.a
-                librftest.a $<TARGET_FILE:${esp_phy_lib}>)
+                set(has_libbttestmode 1)
+                set(has_librftest 1)
             elseif(CONFIG_IDF_TARGET_ESP32S2)
-                target_link_libraries(${COMPONENT_LIB} INTERFACE $<TARGET_FILE:${esp_phy_lib}> librftest.a
-                librfate.a $<TARGET_FILE:${esp_phy_lib}>)
+                set(has_librftest 1)
+                set(has_librfate 1)
             else()
-                target_link_libraries(${COMPONENT_LIB} INTERFACE $<TARGET_FILE:${esp_phy_lib}> libbttestmode.a
-                librfate.a librftest.a $<TARGET_FILE:${esp_phy_lib}>)
+                set(has_libbttestmode 1)
+                set(has_librftest 1)
+                set(has_librfate 1)
             endif()
         endif()
 
         if(CONFIG_IDF_TARGET_ESP32)
-            target_link_libraries(${COMPONENT_LIB} PUBLIC rtc)
-            target_link_libraries(${COMPONENT_LIB} INTERFACE $<TARGET_FILE:${esp_phy_lib}> libphy.a librtc.a
-            $<TARGET_FILE:${esp_phy_lib}>)
+            set(has_librtc 1)
+            set(has_libphy 1)
         elseif(CONFIG_SOC_BT_SUPPORTED OR CONFIG_SOC_IEEE802154_SUPPORTED)
-            target_link_libraries(${COMPONENT_LIB} PUBLIC btbb)
-            target_link_libraries(${COMPONENT_LIB} INTERFACE $<TARGET_FILE:${esp_phy_lib}> libphy.a libbtbb.a
-            $<TARGET_FILE:${esp_phy_lib}>)
+            set(has_libbtbb 1)
+            set(has_libphy 1)
         elseif(CONFIG_SOC_WIFI_SUPPORTED)
-            target_link_libraries(${COMPONENT_LIB} INTERFACE $<TARGET_FILE:${esp_phy_lib}> libphy.a
-            $<TARGET_FILE:${esp_phy_lib}>)
+            set(has_libphy 1)
         endif()
+
+        # Add the required library targets and add them to the list of libraries to link
+        if(has_libbttestmode)
+            add_prebuilt_library(esp_phy_libbttestmode "${lib_dir}/libbttestmode.a"
+                                REQUIRES ${COMPONENT_NAME})
+            list(APPEND libs_to_link esp_phy_libbttestmode)
+        endif()
+
+        if(has_librftest)
+            add_prebuilt_library(esp_phy_librftest "${lib_dir}/librftest.a"
+                                REQUIRES ${COMPONENT_NAME})
+            list(APPEND libs_to_link esp_phy_librftest)
+        endif()
+
+        if(has_librfate)
+            add_prebuilt_library(esp_phy_librfate "${lib_dir}/librfate.a"
+                                REQUIRES ${COMPONENT_NAME})
+            list(APPEND libs_to_link esp_phy_librfate)
+        endif()
+
+        if(has_libbtbb)
+            add_prebuilt_library(esp_phy_libbtbb "${lib_dir}/libbtbb.a"
+                                 REQUIRES ${COMPONENT_NAME})
+            list(APPEND libs_to_link esp_phy_libbtbb)
+        endif()
+
+        if(has_librtc)
+            add_prebuilt_library(esp_phy_librtc "${lib_dir}/librtc.a"
+                                 REQUIRES ${COMPONENT_NAME})
+            list(APPEND libs_to_link esp_phy_librtc)
+        endif()
+
+        if(has_libphy)
+            add_prebuilt_library(esp_phy_libphy "${lib_dir}/libphy.a"
+                                 REQUIRES ${COMPONENT_NAME})
+            list(APPEND libs_to_link esp_phy_libphy)
+        endif()
+
+        # Finally, link the libraries to the component
+        target_link_libraries(${COMPONENT_LIB} INTERFACE ${libs_to_link})
     endif()
 
    if(CONFIG_ESP_PHY_INIT_DATA_IN_PARTITION)