diff --git a/components/esp_hw_support/include/esp_private/rtc_clk.h b/components/esp_hw_support/include/esp_private/rtc_clk.h
index 52610fa0d1..0933f00706 100644
--- a/components/esp_hw_support/include/esp_private/rtc_clk.h
+++ b/components/esp_hw_support/include/esp_private/rtc_clk.h
@@ -18,8 +18,8 @@ extern "C" {
  * @brief Switch CPU clock source to XTAL, and let cpu frequency equal to main XTAL frequency.
  *
  * This function does not disable CPU's source PLL. If the PLL requires to be disabled to save power, please call
- * `rtc_clk_cpu_freq_set_xtal` instead. It does one extra check (if necessary) to see whether can disable the
- * corresponding PLL after switching the CPU clock source to XTAL.
+ * `rtc_clk_cpu_freq_set_xtal` instead. It will always disable the corresponding PLL after switching the CPU clock
+ * source to XTAL (except for S2).
  *
  * Currently, this function should only be called in `esp_restart_noos` and `esp_restart_noos_dig` to switch the CPU
  * clock source back to XTAL (by default) before reset.
@@ -32,12 +32,6 @@ void rtc_clk_cpu_set_to_default_config(void);
  * Currently, this function is only used for tracking whether USB Serial/JTAG is using the 48MHz PHY clock
  *
  * Note: Calling this function only helps to not disable the BBPLL clock in `rtc_clk_cpu_freq_set_config`.
- *       For light and deep sleep, whether to disable the BBPLL in the internal call to `rtc_clk_cpu_freq_set_xtal`
- *       varies for targets.
- *       On ESP32C3/S3, USB CDC device can not function properly during sleep due to the lack of APB clock. Therefore.
- *       `rtc_clk_cpu_freq_set_xtal` will always disable BBPLL, no matter whether BBPLL has any consumer.
- *       On ESP32C6/H2, USB CDC device can maintain the minimum connection with the host during sleep, so
- *       `rtc_clk_cpu_freq_set_xtal` will check for BBPLL consumers, and keep BBPLL if USB Serial/JTAG is in use.
  */
 void rtc_clk_bbpll_add_consumer(void);
 
diff --git a/components/esp_hw_support/port/esp32c5/include/soc/rtc.h b/components/esp_hw_support/port/esp32c5/include/soc/rtc.h
index b066b5032a..a0bbb0302d 100644
--- a/components/esp_hw_support/port/esp32c5/include/soc/rtc.h
+++ b/components/esp_hw_support/port/esp32c5/include/soc/rtc.h
@@ -340,8 +340,9 @@ void rtc_clk_cpu_freq_get_config(rtc_cpu_freq_config_t *out_config);
  * rtc_clk_cpu_freq_set_config when a switch to XTAL is needed.
  * Assumes that XTAL frequency has been determined — don't call in startup code.
  *
- * @note On ESP32C5, this function will check whether BBPLL can be disabled. If there is no consumer, then BBPLL will be
- * turned off. The behaviour is the same as using rtc_clk_cpu_freq_set_config to switch cpu clock source to XTAL.
+ * @note This function always disables BBPLL after switching the CPU clock source to XTAL for power saving purpose.
+ * If this is unwanted, please use rtc_clk_cpu_freq_set_config. It helps to check whether USB Serial JTAG is in use,
+ * if so, then BBPLL will not be turned off.
  */
 void rtc_clk_cpu_freq_set_xtal(void);
 
diff --git a/components/esp_hw_support/port/esp32c5/rtc_clk.c b/components/esp_hw_support/port/esp32c5/rtc_clk.c
index c44ccb9b31..51a6ea22ef 100644
--- a/components/esp_hw_support/port/esp32c5/rtc_clk.c
+++ b/components/esp_hw_support/port/esp32c5/rtc_clk.c
@@ -455,10 +455,7 @@ void rtc_clk_cpu_freq_set_config_fast(const rtc_cpu_freq_config_t *config)
 void rtc_clk_cpu_freq_set_xtal(void)
 {
     rtc_clk_cpu_set_to_default_config();
-    // We don't turn off the bbpll if some consumers depend on bbpll
-    if (!s_bbpll_digi_consumers_ref_count) {
-        rtc_clk_bbpll_disable();
-    }
+    rtc_clk_bbpll_disable();
 }
 
 void rtc_clk_cpu_set_to_default_config(void)
@@ -466,6 +463,7 @@ void rtc_clk_cpu_set_to_default_config(void)
     int freq_mhz = (int)rtc_clk_xtal_freq_get();
 
     rtc_clk_cpu_freq_to_xtal(freq_mhz, 1);
+    s_cur_pll_freq = 0; // no disable PLL, but set freq to 0 to trigger a PLL calibration after wake-up from sleep
 }
 
 void rtc_clk_cpu_freq_to_pll_and_pll_lock_release(int cpu_freq_mhz)
diff --git a/components/esp_hw_support/port/esp32c6/include/soc/rtc.h b/components/esp_hw_support/port/esp32c6/include/soc/rtc.h
index 81a3fa9711..0682c80833 100644
--- a/components/esp_hw_support/port/esp32c6/include/soc/rtc.h
+++ b/components/esp_hw_support/port/esp32c6/include/soc/rtc.h
@@ -338,8 +338,9 @@ void rtc_clk_cpu_freq_get_config(rtc_cpu_freq_config_t *out_config);
  * rtc_clk_cpu_freq_set_config when a switch to XTAL is needed.
  * Assumes that XTAL frequency has been determined — don't call in startup code.
  *
- * @note On ESP32C6, this function will check whether BBPLL can be disabled. If there is no consumer, then BBPLL will be
- * turned off. The behaviour is the same as using rtc_clk_cpu_freq_set_config to switch cpu clock source to XTAL.
+ * @note This function always disables BBPLL after switching the CPU clock source to XTAL for power saving purpose.
+ * If this is unwanted, please use rtc_clk_cpu_freq_set_config. It helps to check whether USB Serial JTAG is in use,
+ * if so, then BBPLL will not be turned off.
  */
 void rtc_clk_cpu_freq_set_xtal(void);
 
diff --git a/components/esp_hw_support/port/esp32c6/rtc_clk.c b/components/esp_hw_support/port/esp32c6/rtc_clk.c
index 228a713ad9..4cccf6593a 100644
--- a/components/esp_hw_support/port/esp32c6/rtc_clk.c
+++ b/components/esp_hw_support/port/esp32c6/rtc_clk.c
@@ -338,10 +338,7 @@ void rtc_clk_cpu_freq_set_config_fast(const rtc_cpu_freq_config_t *config)
 void rtc_clk_cpu_freq_set_xtal(void)
 {
     rtc_clk_cpu_set_to_default_config();
-    // We don't turn off the bbpll if some consumers depend on bbpll
-    if (!s_bbpll_digi_consumers_ref_count) {
-        rtc_clk_bbpll_disable();
-    }
+    rtc_clk_bbpll_disable();
 }
 
 void rtc_clk_cpu_set_to_default_config(void)
@@ -349,6 +346,7 @@ void rtc_clk_cpu_set_to_default_config(void)
     int freq_mhz = (int)rtc_clk_xtal_freq_get();
 
     rtc_clk_cpu_freq_to_xtal(freq_mhz, 1);
+    s_cur_pll_freq = 0; // no disable PLL, but set freq to 0 to trigger a PLL calibration after wake-up from sleep
 }
 
 void rtc_clk_cpu_freq_to_pll_and_pll_lock_release(int cpu_freq_mhz)
diff --git a/components/esp_hw_support/port/esp32c61/include/soc/rtc.h b/components/esp_hw_support/port/esp32c61/include/soc/rtc.h
index 50a428daf8..2f8ac74bfd 100644
--- a/components/esp_hw_support/port/esp32c61/include/soc/rtc.h
+++ b/components/esp_hw_support/port/esp32c61/include/soc/rtc.h
@@ -338,8 +338,9 @@ void rtc_clk_cpu_freq_get_config(rtc_cpu_freq_config_t *out_config);
  * rtc_clk_cpu_freq_set_config when a switch to XTAL is needed.
  * Assumes that XTAL frequency has been determined — don't call in startup code.
  *
- * @note On ESP32C61, this function will check whether BBPLL can be disabled. If there is no consumer, then BBPLL will be
- * turned off. The behaviour is the same as using rtc_clk_cpu_freq_set_config to switch cpu clock source to XTAL.
+ * @note This function always disables BBPLL after switching the CPU clock source to XTAL for power saving purpose.
+ * If this is unwanted, please use rtc_clk_cpu_freq_set_config. It helps to check whether USB Serial JTAG is in use,
+ * if so, then BBPLL will not be turned off.
  */
 void rtc_clk_cpu_freq_set_xtal(void);
 
diff --git a/components/esp_hw_support/port/esp32c61/rtc_clk.c b/components/esp_hw_support/port/esp32c61/rtc_clk.c
index 3a20ede1b9..435f970433 100644
--- a/components/esp_hw_support/port/esp32c61/rtc_clk.c
+++ b/components/esp_hw_support/port/esp32c61/rtc_clk.c
@@ -343,10 +343,7 @@ void rtc_clk_cpu_freq_set_config_fast(const rtc_cpu_freq_config_t *config)
 void rtc_clk_cpu_freq_set_xtal(void)
 {
     rtc_clk_cpu_set_to_default_config();
-    // We don't turn off the bbpll if some consumers depend on bbpll
-    if (!s_bbpll_digi_consumers_ref_count) {
-        rtc_clk_bbpll_disable();
-    }
+    rtc_clk_bbpll_disable();
 }
 
 void rtc_clk_cpu_set_to_default_config(void)
@@ -354,6 +351,7 @@ void rtc_clk_cpu_set_to_default_config(void)
     int freq_mhz = (int)rtc_clk_xtal_freq_get();
 
     rtc_clk_cpu_freq_to_xtal(freq_mhz, 1);
+    s_cur_pll_freq = 0; // no disable PLL, but set freq to 0 to trigger a PLL calibration after wake-up from sleep
 }
 
 void rtc_clk_cpu_freq_to_pll_and_pll_lock_release(int cpu_freq_mhz)
diff --git a/components/esp_hw_support/port/esp32h2/include/soc/rtc.h b/components/esp_hw_support/port/esp32h2/include/soc/rtc.h
index ae763dc4a8..8c5139942b 100644
--- a/components/esp_hw_support/port/esp32h2/include/soc/rtc.h
+++ b/components/esp_hw_support/port/esp32h2/include/soc/rtc.h
@@ -354,8 +354,9 @@ void rtc_clk_cpu_freq_get_config(rtc_cpu_freq_config_t *out_config);
  * rtc_clk_cpu_freq_set_config when a switch to XTAL is needed.
  * Assumes that XTAL frequency has been determined — don't call in startup code.
  *
- * @note On ESP32H2, this function will check whether BBPLL can be disabled. If there is no consumer, then BBPLL will be
- * turned off. The behaviour is the same as using rtc_clk_cpu_freq_set_config to switch cpu clock source to XTAL.
+ * @note This function always disables BBPLL after switching the CPU clock source to XTAL for power saving purpose.
+ * If this is unwanted, please use rtc_clk_cpu_freq_set_config. It helps to check whether USB Serial JTAG is in use,
+ * if so, then BBPLL will not be turned off.
  */
 void rtc_clk_cpu_freq_set_xtal(void);
 
diff --git a/components/esp_hw_support/port/esp32h2/rtc_clk.c b/components/esp_hw_support/port/esp32h2/rtc_clk.c
index 27898a95b4..b3de484dd1 100644
--- a/components/esp_hw_support/port/esp32h2/rtc_clk.c
+++ b/components/esp_hw_support/port/esp32h2/rtc_clk.c
@@ -398,10 +398,7 @@ void rtc_clk_cpu_freq_set_config_fast(const rtc_cpu_freq_config_t *config)
 void rtc_clk_cpu_freq_set_xtal(void)
 {
     rtc_clk_cpu_set_to_default_config();
-    // We don't turn off the bbpll if some consumers only depends on bbpll
-    if (!s_bbpll_digi_consumers_ref_count) {
-        rtc_clk_bbpll_disable();
-    }
+    rtc_clk_bbpll_disable();
 }
 
 void rtc_clk_cpu_set_to_default_config(void)
@@ -409,6 +406,7 @@ void rtc_clk_cpu_set_to_default_config(void)
     int freq_mhz = (int)rtc_clk_xtal_freq_get();
 
     rtc_clk_cpu_freq_to_xtal(freq_mhz, 1);
+    s_cur_pll_freq = 0; // no disable PLL, but set freq to 0 to trigger a PLL calibration after wake-up from sleep
 }
 
 soc_xtal_freq_t rtc_clk_xtal_freq_get(void)
diff --git a/components/esp_hw_support/port/esp32p4/include/soc/rtc.h b/components/esp_hw_support/port/esp32p4/include/soc/rtc.h
index ba2ede21c1..df86bffa5a 100644
--- a/components/esp_hw_support/port/esp32p4/include/soc/rtc.h
+++ b/components/esp_hw_support/port/esp32p4/include/soc/rtc.h
@@ -350,8 +350,8 @@ void rtc_clk_cpu_freq_get_config(rtc_cpu_freq_config_t *out_config);
  * rtc_clk_cpu_freq_set_config when a switch to XTAL is needed.
  * Assumes that XTAL frequency has been determined — don't call in startup code.
  *
- * @note On ESP32C6, this function will check whether BBPLL can be disabled. If there is no consumer, then BBPLL will be
- * turned off. The behaviour is the same as using rtc_clk_cpu_freq_set_config to switch cpu clock source to XTAL.
+ * @note On ESP32P4, this function always disables CPLL after switching the CPU clock source to XTAL,
+ * since there is no peripheral relies on CPLL clock (except Flash/PSRAM if their clock source selects CPLL).
  */
 void rtc_clk_cpu_freq_set_xtal(void);
 
@@ -494,7 +494,7 @@ void rtc_clk_apll_enable(bool enable);
  *
  * @return
  *      - 0 Failed
- *      - else Sucess
+ *      - else Success
  */
 uint32_t rtc_clk_apll_coeff_calc(uint32_t freq, uint32_t *_o_div, uint32_t *_sdm0, uint32_t *_sdm1, uint32_t *_sdm2);
 
diff --git a/components/esp_hw_support/port/esp32p4/rtc_clk.c b/components/esp_hw_support/port/esp32p4/rtc_clk.c
index aa3bfd9675..f9e19ce08c 100644
--- a/components/esp_hw_support/port/esp32p4/rtc_clk.c
+++ b/components/esp_hw_support/port/esp32p4/rtc_clk.c
@@ -426,6 +426,7 @@ void rtc_clk_cpu_set_to_default_config(void)
     int freq_mhz = (int)rtc_clk_xtal_freq_get();
 
     rtc_clk_cpu_freq_to_xtal(freq_mhz, 1, true);
+    s_cur_cpll_freq = 0; // no disable PLL, but set freq to 0 to trigger a PLL calibration after wake-up from sleep
 }
 
 soc_xtal_freq_t rtc_clk_xtal_freq_get(void)