diff --git a/components/esp_system/ld/esp32/memory.ld.in b/components/esp_system/ld/esp32/memory.ld.in
index 34f0aa7b5c..04a8f4a868 100644
--- a/components/esp_system/ld/esp32/memory.ld.in
+++ b/components/esp_system/ld/esp32/memory.ld.in
@@ -91,7 +91,7 @@ MEMORY
 
   /* RTC fast memory (executable). Persists over deep sleep.
    */
-  rtc_iram_seg(RWX) :                org = 0x400C0000, len = 0x2000
+  rtc_iram_seg(RWX) :                org = 0x400C0000, len = 0x2000 - ESP_BOOTLOADER_RESERVE_RTC
 
   /* RTC fast memory (same block as above), viewed from data bus */
   rtc_data_seg(RW) :                 org = 0x3ff80000, len = 0x2000 - ESP_BOOTLOADER_RESERVE_RTC
diff --git a/components/esp_system/ld/esp32s2/memory.ld.in b/components/esp_system/ld/esp32s2/memory.ld.in
index f40fca87a4..5bd6f46878 100644
--- a/components/esp_system/ld/esp32s2/memory.ld.in
+++ b/components/esp_system/ld/esp32s2/memory.ld.in
@@ -97,7 +97,7 @@ MEMORY
 
   /* RTC fast memory (executable). Persists over deep sleep.
    */
-  rtc_iram_seg(RWX) :                org = 0x40070000, len = 0x2000
+  rtc_iram_seg(RWX) :                org = 0x40070000, len = 0x2000 - ESP_BOOTLOADER_RESERVE_RTC
 
   /* RTC slow memory (data accessible). Persists over deep sleep.