diff --git a/components/esp_hw_support/linker.lf b/components/esp_hw_support/linker.lf
index 468f03d4ba..dfc2509bf6 100644
--- a/components/esp_hw_support/linker.lf
+++ b/components/esp_hw_support/linker.lf
@@ -15,7 +15,8 @@ entries:
     rtc_clk (noflash)
     esp_gpio_reserve: esp_gpio_reserve_pins (noflash)
     esp_gpio_reserve: esp_gpio_is_pin_reserved (noflash)
-    if SOC_CONFIGURABLE_VDDSDIO_SUPPORTED:
+    if SOC_CONFIGURABLE_VDDSDIO_SUPPORTED = y && (PM_SLP_IRAM_OPT = y || ESP_SLEEP_POWER_DOWN_FLASH = y):
+        rtc_init:rtc_vddsdio_get_config (noflash)
         rtc_init:rtc_vddsdio_set_config (noflash)
     if IDF_TARGET_ESP32C6 = n && IDF_TARGET_ESP32H2 = n: # TODO: IDF-5645
         rtc_sleep (noflash_text)
diff --git a/components/esp_pm/linker.lf b/components/esp_pm/linker.lf
index f17e24e8a2..12d9282a40 100644
--- a/components/esp_pm/linker.lf
+++ b/components/esp_pm/linker.lf
@@ -13,8 +13,6 @@ entries:
         sleep_modes:esp_sleep_enable_timer_wakeup (noflash)
         sleep_modes:timer_wakeup_prepare (noflash)
         sleep_modes:get_power_down_flags (noflash)
-        if SOC_CONFIGURABLE_VDDSDIO_SUPPORTED:
-            rtc_init:rtc_vddsdio_get_config (noflash)
         esp_clk:esp_clk_slowclk_cal_set (noflash)
         esp_clk:esp_clk_slowclk_cal_get (noflash)
         esp_clk:esp_rtc_get_time_us (noflash)