diff --git a/components/esp32/cpu_freq.c b/components/esp32/cpu_freq.c
index 0e9dc23044..257639cf90 100644
--- a/components/esp32/cpu_freq.c
+++ b/components/esp32/cpu_freq.c
@@ -18,6 +18,8 @@
 #include "sdkconfig.h"
 #include "phy.h"
 #include "rtc.h"
+#include "soc/soc.h"
+#include "soc/rtc_cntl_reg.h"
 
 /*
  * This function is not exposed as an API at this point,
@@ -36,6 +38,10 @@ void esp_set_cpu_freq(void)
     uart_tx_wait_idle(CONFIG_CONSOLE_UART_NUM);
 
     rtc_init_lite(XTAL_AUTO);
+    // work around a bug that RTC fast memory may be isolated
+    // from the system after rtc_init_lite
+    SET_PERI_REG_MASK(RTC_CNTL_PWC_REG, RTC_CNTL_FASTMEM_FORCE_NOISO_M);
+
     cpu_freq_t freq = CPU_80M;
     switch(freq_mhz) {
         case 240: