diff --git a/components/esp_hw_support/include/esp_private/rtc_clk.h b/components/esp_hw_support/include/esp_private/rtc_clk.h
index 3a592a4780..9a27820d96 100644
--- a/components/esp_hw_support/include/esp_private/rtc_clk.h
+++ b/components/esp_hw_support/include/esp_private/rtc_clk.h
@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: 2022-2024 Espressif Systems (Shanghai) CO LTD
+ * SPDX-FileCopyrightText: 2022-2025 Espressif Systems (Shanghai) CO LTD
  *
  * SPDX-License-Identifier: Apache-2.0
  */
@@ -22,11 +22,24 @@ extern "C" {
  * source to XTAL (except for S2).
  *
  * Currently, this function is only called in `esp_restart_noos` and `esp_restart_noos_dig` to switch the CPU
- * clock source back to XTAL (by default) before reset, and in `esp_sleep_start` to switch CPU clock source to XTAL
- * before entering sleep for PMU supported chips.
+ * clock source back to XTAL (by default) before reset.
  */
 void rtc_clk_cpu_set_to_default_config(void);
 
+/**
+ * @brief Switch CPU clock source to XTAL, the PLL has different processing methods for different chips.
+ *        1. For earlier chips without PMU, there is no PMU module that can turn off the CPU's PLL, so it has to be
+ *           disabled at here to save the power consumption. Though ESP32C3/S3 has USB CDC device, it can not function
+ *           properly during sleep due to the lack of APB clock (before C6, USJ relies on APB clock to work). Therefore,
+ *           we will always disable CPU's PLL (i.e. BBPLL).
+ *        2. For PMU supported chips, CPU's PLL power can be turned off by PMU, so no need to disable the PLL at here.
+ *           Leaving PLL on at this stage also helps USJ keep connection and retention operation (if they rely on this PLL).
+ *           For ESP32P4, if the APB frequency is configured as the hardware default value (10MHz), this will cause the
+ *           regdma backup/restore to not achieve optimal performance. The MEM/APB frequency divider needs to be configured
+ *           to 40MHz to speed up the retention speed.
+ */
+void rtc_clk_cpu_freq_set_xtal_for_sleep(void);
+
 /**
  * @brief Notify that the BBPLL has a new in-use consumer
  *
diff --git a/components/esp_hw_support/port/esp32/rtc_clk.c b/components/esp_hw_support/port/esp32/rtc_clk.c
index 55c9aef6af..90b2ed3c50 100644
--- a/components/esp_hw_support/port/esp32/rtc_clk.c
+++ b/components/esp_hw_support/port/esp32/rtc_clk.c
@@ -417,6 +417,11 @@ void rtc_clk_cpu_set_to_default_config(void)
     rtc_clk_wait_for_slow_cycle();
 }
 
+void rtc_clk_cpu_freq_set_xtal_for_sleep(void)
+{
+    rtc_clk_cpu_freq_set_xtal();
+}
+
 bool rtc_clk_cpu_freq_mhz_to_config(uint32_t freq_mhz, rtc_cpu_freq_config_t* out_config)
 {
     uint32_t source_freq_mhz;
diff --git a/components/esp_hw_support/port/esp32c2/rtc_clk.c b/components/esp_hw_support/port/esp32c2/rtc_clk.c
index e9f20e6118..9aac2ad5e4 100644
--- a/components/esp_hw_support/port/esp32c2/rtc_clk.c
+++ b/components/esp_hw_support/port/esp32c2/rtc_clk.c
@@ -286,6 +286,11 @@ void rtc_clk_cpu_set_to_default_config(void)
     rtc_clk_cpu_freq_to_xtal(freq_mhz, 1);
 }
 
+void rtc_clk_cpu_freq_set_xtal_for_sleep(void)
+{
+    rtc_clk_cpu_freq_set_xtal();
+}
+
 /**
  * Switch to use XTAL as the CPU clock source.
  * Must satisfy: cpu_freq = XTAL_FREQ / div.
diff --git a/components/esp_hw_support/port/esp32c3/rtc_clk.c b/components/esp_hw_support/port/esp32c3/rtc_clk.c
index 19b0876d7a..73a41697d0 100644
--- a/components/esp_hw_support/port/esp32c3/rtc_clk.c
+++ b/components/esp_hw_support/port/esp32c3/rtc_clk.c
@@ -316,6 +316,11 @@ void rtc_clk_cpu_set_to_default_config(void)
     rtc_clk_cpu_freq_to_xtal(freq_mhz, 1);
 }
 
+void rtc_clk_cpu_freq_set_xtal_for_sleep(void)
+{
+    rtc_clk_cpu_freq_set_xtal();
+}
+
 /**
  * Switch to use XTAL as the CPU clock source.
  * Must satisfy: cpu_freq = XTAL_FREQ / div.
diff --git a/components/esp_hw_support/port/esp32c5/rtc_clk.c b/components/esp_hw_support/port/esp32c5/rtc_clk.c
index dc0da4c92a..5b61efe071 100644
--- a/components/esp_hw_support/port/esp32c5/rtc_clk.c
+++ b/components/esp_hw_support/port/esp32c5/rtc_clk.c
@@ -473,6 +473,11 @@ void rtc_clk_cpu_set_to_default_config(void)
     s_cur_pll_freq = 0; // no disable PLL, but set freq to 0 to trigger a PLL calibration after wake-up from sleep
 }
 
+void rtc_clk_cpu_freq_set_xtal_for_sleep(void)
+{
+    rtc_clk_cpu_set_to_default_config();
+}
+
 void rtc_clk_cpu_freq_to_pll_and_pll_lock_release(int cpu_freq_mhz)
 {
     rtc_clk_cpu_freq_to_pll_mhz(cpu_freq_mhz);
diff --git a/components/esp_hw_support/port/esp32c6/rtc_clk.c b/components/esp_hw_support/port/esp32c6/rtc_clk.c
index 1fdbdf6178..0565b12763 100644
--- a/components/esp_hw_support/port/esp32c6/rtc_clk.c
+++ b/components/esp_hw_support/port/esp32c6/rtc_clk.c
@@ -356,6 +356,11 @@ void rtc_clk_cpu_set_to_default_config(void)
     s_cur_pll_freq = 0; // no disable PLL, but set freq to 0 to trigger a PLL calibration after wake-up from sleep
 }
 
+void rtc_clk_cpu_freq_set_xtal_for_sleep(void)
+{
+    rtc_clk_cpu_set_to_default_config();
+}
+
 void rtc_clk_cpu_freq_to_pll_and_pll_lock_release(int cpu_freq_mhz)
 {
     rtc_clk_cpu_freq_to_pll_mhz(cpu_freq_mhz);
diff --git a/components/esp_hw_support/port/esp32c61/rtc_clk.c b/components/esp_hw_support/port/esp32c61/rtc_clk.c
index 435f970433..d5ed0f82a4 100644
--- a/components/esp_hw_support/port/esp32c61/rtc_clk.c
+++ b/components/esp_hw_support/port/esp32c61/rtc_clk.c
@@ -354,6 +354,11 @@ void rtc_clk_cpu_set_to_default_config(void)
     s_cur_pll_freq = 0; // no disable PLL, but set freq to 0 to trigger a PLL calibration after wake-up from sleep
 }
 
+void rtc_clk_cpu_freq_set_xtal_for_sleep(void)
+{
+    rtc_clk_cpu_set_to_default_config();
+}
+
 void rtc_clk_cpu_freq_to_pll_and_pll_lock_release(int cpu_freq_mhz)
 {
     rtc_clk_cpu_freq_to_pll_mhz(cpu_freq_mhz);
diff --git a/components/esp_hw_support/port/esp32h2/rtc_clk.c b/components/esp_hw_support/port/esp32h2/rtc_clk.c
index b251889190..06f2a95cbe 100644
--- a/components/esp_hw_support/port/esp32h2/rtc_clk.c
+++ b/components/esp_hw_support/port/esp32h2/rtc_clk.c
@@ -416,6 +416,11 @@ void rtc_clk_cpu_set_to_default_config(void)
     s_cur_pll_freq = 0; // no disable PLL, but set freq to 0 to trigger a PLL calibration after wake-up from sleep
 }
 
+void rtc_clk_cpu_freq_set_xtal_for_sleep(void)
+{
+    rtc_clk_cpu_set_to_default_config();
+}
+
 soc_xtal_freq_t rtc_clk_xtal_freq_get(void)
 {
     uint32_t xtal_freq_mhz = clk_ll_xtal_load_freq_mhz();
diff --git a/components/esp_hw_support/port/esp32p4/rtc_clk.c b/components/esp_hw_support/port/esp32p4/rtc_clk.c
index c727f7bd4b..709feb2ec3 100644
--- a/components/esp_hw_support/port/esp32p4/rtc_clk.c
+++ b/components/esp_hw_support/port/esp32p4/rtc_clk.c
@@ -426,6 +426,13 @@ void rtc_clk_cpu_set_to_default_config(void)
     int freq_mhz = (int)rtc_clk_xtal_freq_get();
 
     rtc_clk_cpu_freq_to_xtal(freq_mhz, 1, true);
+}
+
+void rtc_clk_cpu_freq_set_xtal_for_sleep(void)
+{
+    int freq_mhz = (int)rtc_clk_xtal_freq_get();
+
+    rtc_clk_cpu_freq_to_xtal(freq_mhz, 1, false);
     s_cur_cpll_freq = 0; // no disable PLL, but set freq to 0 to trigger a PLL calibration after wake-up from sleep
 }
 
diff --git a/components/esp_hw_support/port/esp32s2/rtc_clk.c b/components/esp_hw_support/port/esp32s2/rtc_clk.c
index 50fa74c697..1560492758 100644
--- a/components/esp_hw_support/port/esp32s2/rtc_clk.c
+++ b/components/esp_hw_support/port/esp32s2/rtc_clk.c
@@ -437,6 +437,11 @@ void rtc_clk_cpu_set_to_default_config(void)
     rtc_clk_cpu_freq_to_xtal(CLK_LL_XTAL_FREQ_MHZ, 1);
 }
 
+void rtc_clk_cpu_freq_set_xtal_for_sleep(void)
+{
+    rtc_clk_cpu_freq_set_xtal();
+}
+
 /**
  * Switch to use XTAL as the CPU clock source.
  * Must satisfy: cpu_freq = XTAL_FREQ / div.
diff --git a/components/esp_hw_support/port/esp32s3/rtc_clk.c b/components/esp_hw_support/port/esp32s3/rtc_clk.c
index 3f6e06d922..8423db84eb 100644
--- a/components/esp_hw_support/port/esp32s3/rtc_clk.c
+++ b/components/esp_hw_support/port/esp32s3/rtc_clk.c
@@ -378,6 +378,11 @@ void rtc_clk_cpu_set_to_default_config(void)
     rtc_clk_cpu_freq_to_xtal(freq_mhz, 1);
 }
 
+void rtc_clk_cpu_freq_set_xtal_for_sleep(void)
+{
+    rtc_clk_cpu_freq_set_xtal();
+}
+
 /**
  * Switch to use XTAL as the CPU clock source.
  * Must satisfy: cpu_freq = XTAL_FREQ / div.
diff --git a/components/esp_hw_support/sleep_modes.c b/components/esp_hw_support/sleep_modes.c
index 4d77d67445..5b81d0abc9 100644
--- a/components/esp_hw_support/sleep_modes.c
+++ b/components/esp_hw_support/sleep_modes.c
@@ -826,16 +826,7 @@ static esp_err_t IRAM_ATTR esp_sleep_start(uint32_t pd_flags, esp_sleep_mode_t m
     // Save current frequency and switch to XTAL
     rtc_cpu_freq_config_t cpu_freq_config;
     rtc_clk_cpu_freq_get_config(&cpu_freq_config);
-#if SOC_PMU_SUPPORTED
-    // For PMU supported chips, CPU's PLL power can be turned off by PMU, so no need to disable the PLL at here.
-    // Leaving PLL on at this stage also helps USJ keep connection and retention operation (if they rely on this PLL).
-    rtc_clk_cpu_set_to_default_config();
-#else
-    // For earlier chips, there is no PMU module that can turn off the CPU's PLL, so it has to be disabled at here to save the power consumption.
-    // Though ESP32C3/S3 has USB CDC device, it can not function properly during sleep due to the lack of APB clock (before C6, USJ relies on APB clock to work).
-    // Therefore, we will always disable CPU's PLL (i.e. BBPLL).
-    rtc_clk_cpu_freq_set_xtal();
-#endif
+    rtc_clk_cpu_freq_set_xtal_for_sleep();
 
 #if SOC_PM_SUPPORT_EXT0_WAKEUP
     // Configure pins for external wakeup