diff --git a/components/bt/controller/esp32c3/bt.c b/components/bt/controller/esp32c3/bt.c
index 9ca583255f..cc6037abab 100644
--- a/components/bt/controller/esp32c3/bt.c
+++ b/components/bt/controller/esp32c3/bt.c
@@ -246,6 +246,7 @@ extern bool btdm_deep_sleep_mem_init(void);
 extern void btdm_deep_sleep_mem_deinit(void);
 extern void btdm_ble_power_down_dma_copy(bool copy);
 extern uint8_t btdm_sleep_clock_sync(void);
+extern void sdk_config_extend_set_pll_track(bool enable);
 
 #if CONFIG_MAC_BB_PD
 extern void esp_mac_bb_power_down(void);
@@ -1464,6 +1465,9 @@ esp_err_t esp_bt_controller_enable(esp_bt_mode_t mode)
         }
     } while (0);
 
+    // Disable pll track by default in BLE controller on ESP32-C3 and ESP32-S3
+    sdk_config_extend_set_pll_track(false);
+
     if (btdm_controller_enable(mode) != 0) {
         ret = ESP_ERR_INVALID_STATE;
         goto error;