diff --git a/components/esp32/CMakeLists.txt b/components/esp32/CMakeLists.txt
index 61b119d822..03b6385ead 100644
--- a/components/esp32/CMakeLists.txt
+++ b/components/esp32/CMakeLists.txt
@@ -26,11 +26,9 @@ else()
     # esp_timer is added here because cpu_start.c uses esp_timer
     set(priv_requires app_trace app_update bootloader_support esp_system log mbedtls nvs_flash pthread
         spi_flash vfs espcoredump esp_common perfmon esp_timer esp_ipc esp_pm)
-    set(fragments linker.lf)
 
     idf_component_register(SRCS "${srcs}"
                         INCLUDE_DIRS "${include_dirs}"
-                        LDFRAGMENTS "${fragments}"
                         REQUIRES "${requires}"
                         PRIV_REQUIRES "${priv_requires}"
                         REQUIRED_IDF_TARGETS esp32)
diff --git a/components/esp32/component.mk b/components/esp32/component.mk
index 7c65594384..779221c83f 100644
--- a/components/esp32/component.mk
+++ b/components/esp32/component.mk
@@ -17,8 +17,6 @@ COMPONENT_ADD_LDFLAGS += -L $(COMPONENT_PATH)/ld \
                          -u ld_include_panic_highint_hdl \
                          $(addprefix -T ,$(LINKER_SCRIPTS)) \
 
-COMPONENT_ADD_LDFRAGMENTS += linker.lf
-
 # final linking of project ELF depends on all binary libraries, and
 # all linker scripts (except esp32_out.ld, as this is code generated here.)
 COMPONENT_ADD_LINKER_DEPS := $(addprefix ld/, $(filter-out $(COMPONENT_BUILD_DIR)/esp32.project.ld, $(LINKER_SCRIPTS))) \
diff --git a/components/esp32c3/CMakeLists.txt b/components/esp32c3/CMakeLists.txt
index 1bcac4332b..59dc113a24 100644
--- a/components/esp32c3/CMakeLists.txt
+++ b/components/esp32c3/CMakeLists.txt
@@ -27,11 +27,8 @@ else()
         app_trace app_update bootloader_support log mbedtls nvs_flash
         pthread spi_flash vfs espcoredump esp_common esp_timer)
 
-    set(fragments linker.lf)
-
     idf_component_register(SRCS "${srcs}"
                         INCLUDE_DIRS "${include_dirs}"
-                        LDFRAGMENTS "${fragments}"
                         REQUIRES "${requires}"
                         PRIV_REQUIRES "${priv_requires}"
                         REQUIRED_IDF_TARGETS esp32c3)
diff --git a/components/esp32c3/linker.lf b/components/esp32c3/linker.lf
deleted file mode 100644
index 87e068ccd4..0000000000
--- a/components/esp32c3/linker.lf
+++ /dev/null
@@ -1,9 +0,0 @@
-[mapping:gcc]
-archive: libgcc.a
-entries:
-    lib2funcs (noflash_text)
-
-[mapping:gcov]
-archive: libgcov.a
-entries:
-    * (noflash)
diff --git a/components/esp32s2/CMakeLists.txt b/components/esp32s2/CMakeLists.txt
index 82afdc8392..737d0ed696 100644
--- a/components/esp32s2/CMakeLists.txt
+++ b/components/esp32s2/CMakeLists.txt
@@ -29,11 +29,8 @@ else()
         app_trace app_update bootloader_support esp_system log mbedtls nvs_flash
         pthread spi_flash vfs espcoredump esp_common esp_timer)
 
-    set(fragments linker.lf)
-
     idf_component_register(SRCS "${srcs}"
                         INCLUDE_DIRS "${include_dirs}"
-                        LDFRAGMENTS "${fragments}"
                         REQUIRES "${requires}"
                         PRIV_REQUIRES "${priv_requires}"
                         REQUIRED_IDF_TARGETS esp32s2)
diff --git a/components/esp32s2/linker.lf b/components/esp32s2/linker.lf
deleted file mode 100644
index 87e068ccd4..0000000000
--- a/components/esp32s2/linker.lf
+++ /dev/null
@@ -1,9 +0,0 @@
-[mapping:gcc]
-archive: libgcc.a
-entries:
-    lib2funcs (noflash_text)
-
-[mapping:gcov]
-archive: libgcov.a
-entries:
-    * (noflash)
diff --git a/components/esp32s3/CMakeLists.txt b/components/esp32s3/CMakeLists.txt
index 20d35aef25..882bcd6e5f 100644
--- a/components/esp32s3/CMakeLists.txt
+++ b/components/esp32s3/CMakeLists.txt
@@ -26,11 +26,9 @@ else()
     # esp_timer is added here because cpu_start.c uses esp_timer
     set(priv_requires app_trace app_update bootloader_support log mbedtls nvs_flash pthread
         spi_flash vfs espcoredump esp_common perfmon esp_timer esp_ipc)
-    set(fragments linker.lf)
 
     idf_component_register(SRCS "${srcs}"
                         INCLUDE_DIRS "${include_dirs}"
-                        LDFRAGMENTS "${fragments}"
                         REQUIRES "${requires}"
                         PRIV_REQUIRES "${priv_requires}"
                         REQUIRED_IDF_TARGETS esp32s3)
diff --git a/components/esp32s3/linker.lf b/components/esp32s3/linker.lf
deleted file mode 100644
index 87e068ccd4..0000000000
--- a/components/esp32s3/linker.lf
+++ /dev/null
@@ -1,9 +0,0 @@
-[mapping:gcc]
-archive: libgcc.a
-entries:
-    lib2funcs (noflash_text)
-
-[mapping:gcov]
-archive: libgcov.a
-entries:
-    * (noflash)
diff --git a/components/newlib/CMakeLists.txt b/components/newlib/CMakeLists.txt
index 4a9f680220..4fa4856879 100644
--- a/components/newlib/CMakeLists.txt
+++ b/components/newlib/CMakeLists.txt
@@ -20,10 +20,10 @@ set(srcs
 set(include_dirs platform_include)
 
 if(CONFIG_SPIRAM_CACHE_WORKAROUND)
-    set(ldfragments esp32-spiram-rom-functions-c.lf)
+    set(ldfragments "esp32-spiram-rom-functions-c.lf")
 endif()
 
-list(APPEND ldfragments newlib.lf)
+list(APPEND ldfragments "newlib.lf" "system_libs.lf")
 
 idf_component_register(SRCS "${srcs}"
                     INCLUDE_DIRS "${include_dirs}"
diff --git a/components/newlib/component.mk b/components/newlib/component.mk
index 5f79c910e7..3841d7b8e6 100644
--- a/components/newlib/component.mk
+++ b/components/newlib/component.mk
@@ -20,6 +20,6 @@ COMPONENT_SRCDIRS := . port
 COMPONENT_ADD_LDFLAGS += -u newlib_include_heap_impl
 COMPONENT_ADD_LDFLAGS += -u newlib_include_syscalls_impl
 
-COMPONENT_ADD_LDFRAGMENTS += newlib.lf
+COMPONENT_ADD_LDFRAGMENTS += newlib.lf system_libs.lf
 
 heap.o: CFLAGS += -fno-builtin
diff --git a/components/esp32/linker.lf b/components/newlib/system_libs.lf
similarity index 70%
rename from components/esp32/linker.lf
rename to components/newlib/system_libs.lf
index cd39081acd..f3b35f89ca 100644
--- a/components/esp32/linker.lf
+++ b/components/newlib/system_libs.lf
@@ -2,7 +2,8 @@
 archive: libgcc.a
 entries:
     lib2funcs (noflash_text)
-    _divsf3 (noflash)
+    if IDF_TARGET_ESP32 = n:
+        _divsf3 (noflash)
 
 [mapping:gcov]
 archive: libgcov.a