diff --git a/components/driver/spi/gpspi/spi_master.c b/components/driver/spi/gpspi/spi_master.c
index 9daa87bd26..c7ca5c059c 100644
--- a/components/driver/spi/gpspi/spi_master.c
+++ b/components/driver/spi/gpspi/spi_master.c
@@ -513,6 +513,15 @@ esp_err_t spi_bus_remove_device(spi_device_handle_t handle)
 
 #if SOC_SPI_SUPPORT_CLK_RC_FAST
     if (handle->cfg.clock_source == SPI_CLK_SRC_RC_FAST) {
+        // If no transactions from other device, acquire the bus to switch module clock to `SPI_CLK_SRC_DEFAULT`
+        // because `SPI_CLK_SRC_RC_FAST` will be disabled then, which block following transactions
+        if (handle->host->cur_cs == DEV_NUM_MAX) {
+            spi_device_acquire_bus(handle, portMAX_DELAY);
+            SPI_MASTER_PERI_CLOCK_ATOMIC() {
+                spi_ll_set_clk_source(handle->host->hal.hw, SPI_CLK_SRC_DEFAULT);
+            }
+            spi_device_release_bus(handle);
+        }
         periph_rtc_dig_clk8m_disable();
     }
 #endif
diff --git a/components/driver/test_apps/spi/slave/sdkconfig.ci.iram_safe b/components/driver/test_apps/spi/slave/sdkconfig.ci.iram_safe
index d1f389ff26..bae0f4c016 100644
--- a/components/driver/test_apps/spi/slave/sdkconfig.ci.iram_safe
+++ b/components/driver/test_apps/spi/slave/sdkconfig.ci.iram_safe
@@ -1,5 +1,5 @@
 CONFIG_COMPILER_DUMP_RTL_FILES=y
-CONFIG_SPI_MASTER_ISR_IN_IRAM=n
+CONFIG_SPI_MASTER_ISR_IN_IRAM=y
 CONFIG_SPI_SLAVE_ISR_IN_IRAM=y
 CONFIG_COMPILER_OPTIMIZATION_ASSERTIONS_DISABLE=y
 CONFIG_COMPILER_OPTIMIZATION_NONE=y
diff --git a/components/soc/esp32c6/include/soc/clk_tree_defs.h b/components/soc/esp32c6/include/soc/clk_tree_defs.h
index c65dfda315..69c9619162 100644
--- a/components/soc/esp32c6/include/soc/clk_tree_defs.h
+++ b/components/soc/esp32c6/include/soc/clk_tree_defs.h
@@ -344,7 +344,7 @@ typedef enum {
 /**
  * @brief Array initializer for all supported clock sources of SPI
  */
-#define SOC_SPI_CLKS {SOC_MOD_CLK_PLL_F80M, SOC_MOD_CLK_XTAL, SOC_MOD_CLK_RC_FAST}
+#define SOC_SPI_CLKS {SOC_MOD_CLK_PLL_F80M, SOC_MOD_CLK_RC_FAST, SOC_MOD_CLK_XTAL}
 
 /**
  * @brief Type of SPI clock source.