diff --git a/components/esp_driver_ledc/src/ledc.c b/components/esp_driver_ledc/src/ledc.c
index a9d93f904a..4836fe422e 100644
--- a/components/esp_driver_ledc/src/ledc.c
+++ b/components/esp_driver_ledc/src/ledc.c
@@ -19,6 +19,7 @@
 #include "driver/ledc.h"
 #include "esp_rom_gpio.h"
 #include "clk_ctrl_os.h"
+#include "esp_private/esp_sleep_internal.h"
 #include "esp_private/periph_ctrl.h"
 #include "esp_private/gpio.h"
 #include "esp_private/esp_gpio_reserve.h"
@@ -561,6 +562,16 @@ static esp_err_t ledc_set_timer_div(ledc_mode_t speed_mode, ledc_timer_t timer_n
         }
         p_ledc_obj[speed_mode]->glb_clk_is_acquired[timer_num] = true;
         if (p_ledc_obj[speed_mode]->glb_clk != glb_clk) {
+#if SOC_LIGHT_SLEEP_SUPPORTED
+            /* keep ESP_PD_DOMAIN_RC_FAST on during light sleep */
+            if (glb_clk == LEDC_SLOW_CLK_RC_FAST) {
+                /* Keep ESP_PD_DOMAIN_RC_FAST on during light sleep */
+                esp_sleep_sub_mode_config(ESP_SLEEP_DIG_USE_RC_FAST_MODE, true);
+            } else if (p_ledc_obj[speed_mode]->glb_clk == LEDC_SLOW_CLK_RC_FAST) {
+                /* No need to keep ESP_PD_DOMAIN_RC_FAST on during light sleep anymore */
+                esp_sleep_sub_mode_config(ESP_SLEEP_DIG_USE_RC_FAST_MODE, false);
+            }
+#endif
             // TODO: release old glb_clk (if not UNINIT), and acquire new glb_clk [clk_tree]
             p_ledc_obj[speed_mode]->glb_clk = glb_clk;
             LEDC_FUNC_CLOCK_ATOMIC() {
@@ -571,12 +582,6 @@ static esp_err_t ledc_set_timer_div(ledc_mode_t speed_mode, ledc_timer_t timer_n
         portEXIT_CRITICAL(&ledc_spinlock);
 
         ESP_LOGD(LEDC_TAG, "In slow speed mode, global clk set: %d", glb_clk);
-
-        /* keep ESP_PD_DOMAIN_RC_FAST on during light sleep */
-#if SOC_LIGHT_SLEEP_SUPPORTED
-        extern void esp_sleep_periph_use_8m(bool use_or_not);
-        esp_sleep_periph_use_8m(glb_clk == LEDC_SLOW_CLK_RC_FAST);
-#endif
     }
 
     /* The divisor is correct, we can write in the hardware. */
diff --git a/components/esp_hw_support/port/esp32/rtc_clk.c b/components/esp_hw_support/port/esp32/rtc_clk.c
index 8749102291..6a6e3fa3d5 100644
--- a/components/esp_hw_support/port/esp32/rtc_clk.c
+++ b/components/esp_hw_support/port/esp32/rtc_clk.c
@@ -10,6 +10,7 @@
 #include <stdlib.h>
 #include "soc/rtc.h"
 #include "esp_private/rtc_clk.h"
+#include "esp_private/esp_sleep_internal.h"
 #include "soc/rtc_periph.h"
 #include "soc/sens_reg.h"
 #include "soc/soc_caps.h"
@@ -273,15 +274,23 @@ void rtc_clk_apll_coeff_set(uint32_t o_div, uint32_t sdm0, uint32_t sdm1, uint32
 
 void rtc_clk_slow_src_set(soc_rtc_slow_clk_src_t clk_src)
 {
-    clk_ll_rtc_slow_set_src(clk_src);
+#ifndef BOOTLOADER_BUILD
+    soc_rtc_slow_clk_src_t clk_src_before_switch = clk_ll_rtc_slow_get_src();
+    // Keep the RTC8M_CLK on in sleep if RTC clock is rc_fast_d256.
+    if (clk_src == SOC_RTC_SLOW_CLK_SRC_RC_FAST_D256 && clk_src_before_switch != SOC_RTC_SLOW_CLK_SRC_RC_FAST_D256) {       // Switch to RC_FAST_D256
+        esp_sleep_sub_mode_config(ESP_SLEEP_RTC_USE_RC_FAST_MODE, true);
+    } else if (clk_src == SOC_RTC_SLOW_CLK_SRC_RC_FAST_D256 && clk_src_before_switch == SOC_RTC_SLOW_CLK_SRC_RC_FAST_D256) { // Switch away from RC_FAST_D256
+        esp_sleep_sub_mode_config(ESP_SLEEP_RTC_USE_RC_FAST_MODE, false);
+    }
+#endif
 
+    clk_ll_rtc_slow_set_src(clk_src);
     // The logic should be moved to BT driver
     if (clk_src == SOC_RTC_SLOW_CLK_SRC_XTAL32K) {
         clk_ll_xtal32k_digi_enable();
     } else {
         clk_ll_xtal32k_digi_disable();
     }
-
     esp_rom_delay_us(SOC_DELAY_RTC_SLOW_CLK_SWITCH);
 }
 
diff --git a/components/esp_hw_support/port/esp32c2/rtc_clk.c b/components/esp_hw_support/port/esp32c2/rtc_clk.c
index 425b72b17f..59a6f63bff 100644
--- a/components/esp_hw_support/port/esp32c2/rtc_clk.c
+++ b/components/esp_hw_support/port/esp32c2/rtc_clk.c
@@ -13,6 +13,7 @@
 #include "esp32c2/rom/rtc.h"
 #include "esp32c2/rom/uart.h"
 #include "soc/rtc.h"
+#include "esp_private/esp_sleep_internal.h"
 #include "esp_private/rtc_clk.h"
 #include "soc/io_mux_reg.h"
 #include "soc/soc.h"
@@ -66,6 +67,16 @@ bool rtc_clk_8md256_enabled(void)
 
 void rtc_clk_slow_src_set(soc_rtc_slow_clk_src_t clk_src)
 {
+#ifndef BOOTLOADER_BUILD
+    soc_rtc_slow_clk_src_t clk_src_before_switch = clk_ll_rtc_slow_get_src();
+    // Keep the RTC8M_CLK on in sleep if RTC clock is rc_fast_d256.
+    if (clk_src == SOC_RTC_SLOW_CLK_SRC_RC_FAST_D256 && clk_src_before_switch != SOC_RTC_SLOW_CLK_SRC_RC_FAST_D256) {       // Switch to RC_FAST_D256
+        esp_sleep_sub_mode_config(ESP_SLEEP_RTC_USE_RC_FAST_MODE, true);
+    } else if (clk_src == SOC_RTC_SLOW_CLK_SRC_RC_FAST_D256 && clk_src_before_switch == SOC_RTC_SLOW_CLK_SRC_RC_FAST_D256) { // Switch away from RC_FAST_D256
+        esp_sleep_sub_mode_config(ESP_SLEEP_RTC_USE_RC_FAST_MODE, false);
+    }
+#endif
+
     clk_ll_rtc_slow_set_src(clk_src);
 
     /* Why we need to connect this clock to digital?
@@ -76,7 +87,6 @@ void rtc_clk_slow_src_set(soc_rtc_slow_clk_src_t clk_src)
     } else {
         clk_ll_xtal32k_digi_disable();
     }
-
     esp_rom_delay_us(SOC_DELAY_RTC_SLOW_CLK_SWITCH);
 }
 
diff --git a/components/esp_hw_support/port/esp32c3/rtc_clk.c b/components/esp_hw_support/port/esp32c3/rtc_clk.c
index 41ad7a967f..cec3673dbe 100644
--- a/components/esp_hw_support/port/esp32c3/rtc_clk.c
+++ b/components/esp_hw_support/port/esp32c3/rtc_clk.c
@@ -13,6 +13,7 @@
 #include "esp32c3/rom/rtc.h"
 #include "soc/rtc.h"
 #include "soc/io_mux_reg.h"
+#include "esp_private/esp_sleep_internal.h"
 #include "esp_private/rtc_clk.h"
 #include "esp_hw_log.h"
 #include "esp_rom_sys.h"
@@ -99,8 +100,17 @@ bool rtc_clk_8md256_enabled(void)
 
 void rtc_clk_slow_src_set(soc_rtc_slow_clk_src_t clk_src)
 {
-    clk_ll_rtc_slow_set_src(clk_src);
+#ifndef BOOTLOADER_BUILD
+    soc_rtc_slow_clk_src_t clk_src_before_switch = clk_ll_rtc_slow_get_src();
+    // Keep the RTC8M_CLK on in sleep if RTC clock is rc_fast_d256.
+    if (clk_src == SOC_RTC_SLOW_CLK_SRC_RC_FAST_D256 && clk_src_before_switch != SOC_RTC_SLOW_CLK_SRC_RC_FAST_D256) {       // Switch to RC_FAST_D256
+        esp_sleep_sub_mode_config(ESP_SLEEP_RTC_USE_RC_FAST_MODE, true);
+    } else if (clk_src == SOC_RTC_SLOW_CLK_SRC_RC_FAST_D256 && clk_src_before_switch == SOC_RTC_SLOW_CLK_SRC_RC_FAST_D256) { // Switch away from RC_FAST_D256
+        esp_sleep_sub_mode_config(ESP_SLEEP_RTC_USE_RC_FAST_MODE, false);
+    }
+#endif
 
+    clk_ll_rtc_slow_set_src(clk_src);
     /* Why we need to connect this clock to digital?
      * Or maybe this clock should be connected to digital when xtal 32k clock is enabled instead?
      */
@@ -109,7 +119,6 @@ void rtc_clk_slow_src_set(soc_rtc_slow_clk_src_t clk_src)
     } else {
         clk_ll_xtal32k_digi_disable();
     }
-
     esp_rom_delay_us(SOC_DELAY_RTC_SLOW_CLK_SWITCH);
 }
 
diff --git a/components/esp_hw_support/port/esp32s2/rtc_clk.c b/components/esp_hw_support/port/esp32s2/rtc_clk.c
index 31836515ce..91637bf300 100644
--- a/components/esp_hw_support/port/esp32s2/rtc_clk.c
+++ b/components/esp_hw_support/port/esp32s2/rtc_clk.c
@@ -13,6 +13,7 @@
 #include "esp32s2/rom/rtc.h"
 #include "soc/rtc.h"
 #include "esp_private/rtc_clk.h"
+#include "esp_private/esp_sleep_internal.h"
 #include "soc/rtc_cntl_reg.h"
 #include "soc/rtc_io_reg.h"
 #include "soc/soc_caps.h"
@@ -174,8 +175,17 @@ void rtc_clk_apll_coeff_set(uint32_t o_div, uint32_t sdm0, uint32_t sdm1, uint32
 
 void rtc_clk_slow_src_set(soc_rtc_slow_clk_src_t clk_src)
 {
-    clk_ll_rtc_slow_set_src(clk_src);
+#ifndef BOOTLOADER_BUILD
+    soc_rtc_slow_clk_src_t clk_src_before_switch = clk_ll_rtc_slow_get_src();
+    // Keep the RTC8M_CLK on in sleep if RTC clock is rc_fast_d256.
+    if (clk_src == SOC_RTC_SLOW_CLK_SRC_RC_FAST_D256 && clk_src_before_switch != SOC_RTC_SLOW_CLK_SRC_RC_FAST_D256) {       // Switch to RC_FAST_D256
+        esp_sleep_sub_mode_config(ESP_SLEEP_RTC_USE_RC_FAST_MODE, true);
+    } else if (clk_src == SOC_RTC_SLOW_CLK_SRC_RC_FAST_D256 && clk_src_before_switch == SOC_RTC_SLOW_CLK_SRC_RC_FAST_D256) { // Switch away from RC_FAST_D256
+        esp_sleep_sub_mode_config(ESP_SLEEP_RTC_USE_RC_FAST_MODE, false);
+    }
+#endif
 
+    clk_ll_rtc_slow_set_src(clk_src);
     /* Why we need to connect this clock to digital?
      * Or maybe this clock should be connected to digital when xtal 32k clock is enabled instead?
      */
@@ -184,7 +194,6 @@ void rtc_clk_slow_src_set(soc_rtc_slow_clk_src_t clk_src)
     } else {
         clk_ll_xtal32k_digi_disable();
     }
-
     esp_rom_delay_us(SOC_DELAY_RTC_SLOW_CLK_SWITCH);
 }
 
diff --git a/components/esp_hw_support/port/esp32s3/rtc_clk.c b/components/esp_hw_support/port/esp32s3/rtc_clk.c
index 3085c15f9d..4ad2f30239 100644
--- a/components/esp_hw_support/port/esp32s3/rtc_clk.c
+++ b/components/esp_hw_support/port/esp32s3/rtc_clk.c
@@ -13,6 +13,7 @@
 #include "esp32s3/rom/rtc.h"
 #include "soc/rtc.h"
 #include "soc/io_mux_reg.h"
+#include "esp_private/esp_sleep_internal.h"
 #include "esp_private/rtc_clk.h"
 #include "soc/rtc_io_reg.h"
 #include "esp_rom_sys.h"
@@ -114,8 +115,17 @@ bool rtc_clk_8md256_enabled(void)
 
 void rtc_clk_slow_src_set(soc_rtc_slow_clk_src_t clk_src)
 {
-    clk_ll_rtc_slow_set_src(clk_src);
+#ifndef BOOTLOADER_BUILD
+    soc_rtc_slow_clk_src_t clk_src_before_switch = clk_ll_rtc_slow_get_src();
+    // Keep the RTC8M_CLK on in sleep if RTC clock is rc_fast_d256.
+    if (clk_src == SOC_RTC_SLOW_CLK_SRC_RC_FAST_D256 && clk_src_before_switch != SOC_RTC_SLOW_CLK_SRC_RC_FAST_D256) {       // Switch to RC_FAST_D256
+        esp_sleep_sub_mode_config(ESP_SLEEP_RTC_USE_RC_FAST_MODE, true);
+    } else if (clk_src == SOC_RTC_SLOW_CLK_SRC_RC_FAST_D256 && clk_src_before_switch == SOC_RTC_SLOW_CLK_SRC_RC_FAST_D256) { // Switch away from RC_FAST_D256
+        esp_sleep_sub_mode_config(ESP_SLEEP_RTC_USE_RC_FAST_MODE, false);
+    }
+#endif
 
+    clk_ll_rtc_slow_set_src(clk_src);
     /* Why we need to connect this clock to digital?
      * Or maybe this clock should be connected to digital when xtal 32k clock is enabled instead?
      */
@@ -124,7 +134,6 @@ void rtc_clk_slow_src_set(soc_rtc_slow_clk_src_t clk_src)
     } else {
         clk_ll_xtal32k_digi_disable();
     }
-
     esp_rom_delay_us(SOC_DELAY_RTC_SLOW_CLK_SWITCH);
 }
 
diff --git a/components/esp_hw_support/sleep_modes.c b/components/esp_hw_support/sleep_modes.c
index 01d82d9694..145b6201a9 100644
--- a/components/esp_hw_support/sleep_modes.c
+++ b/components/esp_hw_support/sleep_modes.c
@@ -275,12 +275,6 @@ static const char *TAG = "sleep";
 static RTC_FAST_ATTR int32_t s_sleep_sub_mode_ref_cnt[ESP_SLEEP_MODE_MAX] = { 0 };
 //in this mode, 2uA is saved, but RTC memory can't use at high temperature, and RTCIO can't be used as INPUT.
 
-static bool s_periph_use_8m_flag = false;
-
-void esp_sleep_periph_use_8m(bool use_or_not)
-{
-    s_periph_use_8m_flag = use_or_not;
-}
 
 static uint32_t get_power_down_flags(void);
 #if SOC_PM_SUPPORT_EXT0_WAKEUP
@@ -768,20 +762,6 @@ static esp_err_t IRAM_ATTR esp_sleep_start(uint32_t pd_flags, esp_sleep_mode_t m
 
     int64_t sleep_duration = (int64_t) s_config.sleep_duration - (int64_t) s_config.sleep_time_adjustment;
 
-#if SOC_RTC_SLOW_CLK_SUPPORT_RC_FAST_D256
-    //Keep the RTC8M_CLK on if RTC clock is rc_fast_d256.
-    bool rtc_using_8md256 = (rtc_clk_slow_src_get() == SOC_RTC_SLOW_CLK_SRC_RC_FAST_D256);
-#else
-    bool rtc_using_8md256 = false;
-#endif
-    //Keep the RTC8M_CLK on if the ledc low-speed channel is clocked by RTC8M_CLK in lightsleep mode
-    bool periph_using_8m = !deep_sleep && s_periph_use_8m_flag;
-
-    //Override user-configured power modes.
-    if (rtc_using_8md256 || periph_using_8m) {
-        pd_flags &= ~RTC_SLEEP_PD_INT_8M;
-    }
-
     // Sleep UART prepare
     if (deep_sleep) {
         flush_uarts();
@@ -893,6 +873,10 @@ static esp_err_t IRAM_ATTR esp_sleep_start(uint32_t pd_flags, esp_sleep_mode_t m
         reject_triggers |= sleep_modem_reject_triggers();
     }
 
+    // Override user-configured FOSC power modes.
+    if (s_sleep_sub_mode_ref_cnt[ESP_SLEEP_RTC_USE_RC_FAST_MODE]) {
+        pd_flags &= ~RTC_SLEEP_PD_INT_8M;
+    }
 
     // Override user-configured XTAL power modes.
     if (s_sleep_sub_mode_ref_cnt[ESP_SLEEP_DIG_USE_XTAL_MODE] && !deep_sleep) {
@@ -901,16 +885,19 @@ static esp_err_t IRAM_ATTR esp_sleep_start(uint32_t pd_flags, esp_sleep_mode_t m
 
     //Append some flags in addition to power domains
     uint32_t sleep_flags = pd_flags;
-    if (s_sleep_sub_mode_ref_cnt[ESP_SLEEP_USE_ADC_TESEN_MONITOR_MODE]) {
+
+    if (s_sleep_sub_mode_ref_cnt[ESP_SLEEP_DIG_USE_RC_FAST_MODE] && !deep_sleep) {
+        pd_flags &= ~RTC_SLEEP_PD_INT_8M;
+        sleep_flags |= RTC_SLEEP_DIG_USE_8M;
+    }
+
+    if (s_sleep_sub_mode_ref_cnt[ESP_SLEEP_USE_ADC_TSEN_MONITOR_MODE]) {
         sleep_flags |= RTC_SLEEP_USE_ADC_TESEN_MONITOR;
     }
 
     if (s_sleep_sub_mode_ref_cnt[ESP_SLEEP_ULTRA_LOW_MODE] == 0) {
         sleep_flags |= RTC_SLEEP_NO_ULTRA_LOW;
     }
-    if (periph_using_8m) {
-        sleep_flags |= RTC_SLEEP_DIG_USE_8M;
-    }
 
 #if CONFIG_ESP_SLEEP_DEBUG
     if (s_sleep_ctx != NULL) {
@@ -2417,7 +2404,7 @@ esp_deep_sleep_disable_rom_logging(void)
 
 __attribute__((deprecated("Please use esp_sleep_sub_mode_config instead"))) void esp_sleep_periph_use_8m(bool use_or_not)
 {
-    esp_sleep_sub_mode_config(ESP_SLEEP_USE_RC_FAST_MODE, use_or_not);
+    esp_sleep_sub_mode_config(ESP_SLEEP_DIG_USE_RC_FAST_MODE, use_or_not);
 }
 
 __attribute__((deprecated("Please use esp_sleep_sub_mode_config instead"))) void esp_sleep_enable_adc_tsens_monitor(bool enable)