diff --git a/components/soc/esp32/include/soc/rtc.h b/components/soc/esp32/include/soc/rtc.h
index b47e39e1e7..2c4b4d320c 100644
--- a/components/soc/esp32/include/soc/rtc.h
+++ b/components/soc/esp32/include/soc/rtc.h
@@ -276,6 +276,25 @@ rtc_fast_freq_t rtc_clk_fast_freq_get();
  */
 void rtc_clk_cpu_freq_set(rtc_cpu_freq_t cpu_freq);
 
+/**
+ * @brief Switch CPU frequency
+ *
+ * This is a faster version of rtc_clk_cpu_freq_set, which can handle some of
+ * the frequency switch paths (XTAL -> PLL, PLL -> XTAL).
+ * When switching from PLL to XTAL, PLL is not disabled (unlike rtc_clk_cpu_freq_set).
+ * When switching back from XTAL to PLL, only the same PLL can be used.
+ * Therefore it is not possible to switch 240 -> XTAL -> (80 or 160) using this
+ * function.
+ *
+ * For unsupported cases, this function falls back to rtc_clk_cpu_freq_set.
+ *
+ * Unlike rtc_clk_cpu_freq_set, this function relies on static data, so it is
+ * less safe to use it e.g. from a panic handler (when memory might be corrupted).
+ *
+ * @param cpu_freq  new CPU frequency
+ */
+void rtc_clk_cpu_freq_set_fast(rtc_cpu_freq_t cpu_freq);
+
 /**
  * @brief Get the currently selected CPU frequency
  *
diff --git a/components/soc/esp32/rtc_clk.c b/components/soc/esp32/rtc_clk.c
index e3e007898d..0b1f902517 100644
--- a/components/soc/esp32/rtc_clk.c
+++ b/components/soc/esp32/rtc_clk.c
@@ -72,9 +72,9 @@ static const char* TAG = "rtc_clk";
  * All values are in microseconds.
  * TODO: some of these are excessive, and should be reduced.
  */
-#define DELAY_CPU_FREQ_SWITCH_TO_XTAL_WITH_150K  80
+#define DELAY_CPU_FREQ_SWITCH_TO_XTAL_WITH_150K  20
 #define DELAY_CPU_FREQ_SWITCH_TO_XTAL_WITH_32K   160
-#define DELAY_CPU_FREQ_SWITCH_TO_PLL    10
+#define DELAY_CPU_FREQ_SWITCH_TO_PLL    20
 #define DELAY_PLL_DBIAS_RAISE           3
 #define DELAY_PLL_ENABLE_WITH_150K      80
 #define DELAY_PLL_ENABLE_WITH_32K       160
@@ -87,6 +87,8 @@ static const char* TAG = "rtc_clk";
  */
 #define XTAL_FREQ_EST_CYCLES            10
 
+static rtc_cpu_freq_t s_cur_freq = RTC_CPU_FREQ_XTAL;
+static int s_pll_freq = 0;
 
 static void rtc_clk_32k_enable_internal(int dac, int dres, int dbias)
 {
@@ -323,6 +325,70 @@ void rtc_clk_bbpll_set(rtc_xtal_freq_t xtal_freq, rtc_cpu_freq_t cpu_freq)
     ets_delay_us(delay_pll_en);
 }
 
+/**
+ * Switch to XTAL frequency. Does not disable the PLL.
+ */
+static void rtc_clk_cpu_freq_to_xtal()
+{
+    rtc_xtal_freq_t xtal_freq = rtc_clk_xtal_freq_get();
+    ets_update_cpu_frequency(xtal_freq);
+    REG_SET_FIELD(RTC_CNTL_REG, RTC_CNTL_DIG_DBIAS_WAK, RTC_CNTL_DBIAS_1V10);
+    REG_SET_FIELD(APB_CTRL_SYSCLK_CONF_REG, APB_CTRL_PRE_DIV_CNT, 0);
+    REG_SET_FIELD(RTC_CNTL_CLK_CONF_REG, RTC_CNTL_SOC_CLK_SEL, RTC_CNTL_SOC_CLK_SEL_XTL);
+    DPORT_REG_WRITE(DPORT_CPU_PER_CONF_REG, 0); // clear DPORT_CPUPERIOD_SEL
+
+    rtc_clk_apb_freq_update(xtal_freq * MHZ);
+    s_cur_freq = RTC_CPU_FREQ_XTAL;
+}
+
+/**
+ * Switch to one of PLL-based frequencies. Current frequency can be XTAL or PLL.
+ * PLL must already be enabled.
+ * If switching between frequencies derived from different PLLs (320M and 480M),
+ * fall back to rtc_clk_cpu_freq_set.
+ * @param cpu_freq new CPU frequency
+ */
+static void rtc_clk_cpu_freq_to_pll(rtc_cpu_freq_t cpu_freq)
+{
+    int freq = 0;
+    if ((cpu_freq == RTC_CPU_FREQ_240M && s_pll_freq == 320) ||
+        (cpu_freq != RTC_CPU_FREQ_240M && s_pll_freq == 240)) {
+        /* need to switch PLLs, fall back to full implementation */
+        rtc_clk_cpu_freq_set(cpu_freq);
+        return;
+    }
+
+    if (cpu_freq == RTC_CPU_FREQ_80M) {
+        DPORT_REG_WRITE(DPORT_CPU_PER_CONF_REG, 0);
+        freq = 80;
+    } else if (cpu_freq == RTC_CPU_FREQ_160M) {
+        DPORT_REG_WRITE(DPORT_CPU_PER_CONF_REG, 1);
+        freq = 160;
+    } else if (cpu_freq == RTC_CPU_FREQ_240M) {
+        REG_SET_FIELD(RTC_CNTL_REG, RTC_CNTL_DIG_DBIAS_WAK, RTC_CNTL_DBIAS_1V25);
+        DPORT_REG_WRITE(DPORT_CPU_PER_CONF_REG, 2);
+        freq = 240;
+    }
+    REG_SET_FIELD(RTC_CNTL_CLK_CONF_REG, RTC_CNTL_SOC_CLK_SEL, RTC_CNTL_SOC_CLK_SEL_PLL);
+    rtc_clk_apb_freq_update(80 * MHZ);
+    ets_update_cpu_frequency(freq);
+    s_cur_freq = cpu_freq;
+}
+
+void rtc_clk_cpu_freq_set_fast(rtc_cpu_freq_t cpu_freq)
+{
+    if (cpu_freq == s_cur_freq) {
+        return;
+    } else if (cpu_freq == RTC_CPU_FREQ_2M || s_cur_freq == RTC_CPU_FREQ_2M) {
+        /* fall back to full implementation if switch to/from 2M is needed */
+        rtc_clk_cpu_freq_set(cpu_freq);
+    } else if (cpu_freq == RTC_CPU_FREQ_XTAL) {
+        rtc_clk_cpu_freq_to_xtal();
+    } else if (cpu_freq > RTC_CPU_FREQ_XTAL) {
+        rtc_clk_cpu_freq_to_pll(cpu_freq);
+    }
+}
+
 void rtc_clk_cpu_freq_set(rtc_cpu_freq_t cpu_freq)
 {
     rtc_xtal_freq_t xtal_freq = rtc_clk_xtal_freq_get();
@@ -338,6 +404,7 @@ void rtc_clk_cpu_freq_set(rtc_cpu_freq_t cpu_freq)
     SET_PERI_REG_MASK(RTC_CNTL_OPTIONS0_REG,
             RTC_CNTL_BB_I2C_FORCE_PD | RTC_CNTL_BBPLL_FORCE_PD |
             RTC_CNTL_BBPLL_I2C_FORCE_PD);
+    s_pll_freq = 0;
     rtc_clk_apb_freq_update(xtal_freq * MHZ);
 
     /* is APLL under force power down? */
@@ -365,17 +432,21 @@ void rtc_clk_cpu_freq_set(rtc_cpu_freq_t cpu_freq)
         if (cpu_freq == RTC_CPU_FREQ_80M) {
             DPORT_REG_SET_FIELD(DPORT_CPU_PER_CONF_REG, DPORT_CPUPERIOD_SEL, 0);
             ets_update_cpu_frequency(80);
+            s_pll_freq = 320;
         } else if (cpu_freq == RTC_CPU_FREQ_160M) {
             DPORT_REG_SET_FIELD(DPORT_CPU_PER_CONF_REG, DPORT_CPUPERIOD_SEL, 1);
             ets_update_cpu_frequency(160);
+            s_pll_freq = 320;
         } else if (cpu_freq == RTC_CPU_FREQ_240M) {
             DPORT_REG_SET_FIELD(DPORT_CPU_PER_CONF_REG, DPORT_CPUPERIOD_SEL, 2);
             ets_update_cpu_frequency(240);
+            s_pll_freq = 480;
         }
         REG_SET_FIELD(RTC_CNTL_CLK_CONF_REG, RTC_CNTL_SOC_CLK_SEL, RTC_CNTL_SOC_CLK_SEL_PLL);
         ets_delay_us(DELAY_CPU_FREQ_SWITCH_TO_PLL);
         rtc_clk_apb_freq_update(80 * MHZ);
     }
+    s_cur_freq = cpu_freq;
 }
 
 rtc_cpu_freq_t rtc_clk_cpu_freq_get()