diff --git a/components/esp_phy/include/esp_phy_init.h b/components/esp_phy/include/esp_phy_init.h
index 557b48823a..b9c3c99d2a 100644
--- a/components/esp_phy/include/esp_phy_init.h
+++ b/components/esp_phy/include/esp_phy_init.h
@@ -26,6 +26,15 @@ typedef struct {
 	uint8_t params[128];                    /*!< opaque PHY initialization parameters */
 } esp_phy_init_data_t;
 
+/**
+ * @brief PHY enable or disable modem
+ */
+typedef enum {
+    PHY_MODEM_WIFI       = 1,       /*!< PHY modem WIFI       */
+    PHY_MODEM_BT         = 2,       /*!< PHY modem BT         */
+    PHY_MODEM_IEEE802154 = 4,       /*!< PHY modem IEEE802154 */
+} esp_phy_modem_t;
+
 /**
  * @brief Opaque PHY calibration data
  */
@@ -144,8 +153,9 @@ esp_err_t esp_phy_erase_cal_data_in_nvs(void);
  * Now PHY and RF enabling job is done automatically when start WiFi or BT. Users should not
  * call this API in their application.
  *
+ * @param modem the modem to call the phy enable.
  */
-void esp_phy_enable(void);
+void esp_phy_enable(esp_phy_modem_t modem);
 
 /**
  * @brief Disable PHY and RF module
@@ -154,8 +164,9 @@ void esp_phy_enable(void);
  * Now PHY and RF disabling job is done automatically when stop WiFi or BT. Users should not
  * call this API in their application.
  *
+ * @param modem the modem to call the phy disable.
  */
-void esp_phy_disable(void);
+void esp_phy_disable(esp_phy_modem_t modem);
 
 /**
  * @brief Enable BTBB module
@@ -258,17 +269,6 @@ esp_err_t esp_phy_apply_phy_init_data(uint8_t *init_data);
  */
 char * get_phy_version_str(void);
 
-/**
- * @brief Enable phy track pll
- *
- */
-void phy_track_pll_init(void);
-
-/**
- * @brief Disable phy track pll
- *
- */
-void phy_track_pll_deinit(void);
 #ifdef __cplusplus
 }
 #endif
diff --git a/components/esp_phy/include/esp_private/phy.h b/components/esp_phy/include/esp_private/phy.h
index 858847d129..8abb2961bc 100644
--- a/components/esp_phy/include/esp_private/phy.h
+++ b/components/esp_phy/include/esp_private/phy.h
@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: 2015-2022 Espressif Systems (Shanghai) CO LTD
+ * SPDX-FileCopyrightText: 2015-2023 Espressif Systems (Shanghai) CO LTD
  *
  * SPDX-License-Identifier: Apache-2.0
  */
@@ -140,6 +140,34 @@ void phy_eco_version_sel(uint8_t chip_ver);
 void phy_improve_rx_special(bool enable);
 #endif
 
+/**
+ * @brief Enable phy track pll
+ *
+ */
+void phy_track_pll_init(void);
+
+/**
+ * @brief Disable phy track pll
+ *
+ */
+void phy_track_pll_deinit(void);
+
+/**
+ * @brief Set the flag recorded which modem has already enabled phy
+ *
+ */
+void phy_set_modem_flag(esp_phy_modem_t modem);
+
+/**
+ * @brief Clear the flag to record which modem calls phy disenable
+ */
+void phy_clr_modem_flag(esp_phy_modem_t modem);
+
+/**
+ * @brief Get the flag recorded which modem has already enabled phy
+ *
+ */
+esp_phy_modem_t phy_get_modem_flag(void);
 #ifdef __cplusplus
 }
 #endif
diff --git a/components/esp_phy/lib b/components/esp_phy/lib
index 95c3700899..f1d9b9b5cb 160000
--- a/components/esp_phy/lib
+++ b/components/esp_phy/lib
@@ -1 +1 @@
-Subproject commit 95c370089907f74805eac72975b0e85c62e175ed
+Subproject commit f1d9b9b5cb63dac81b9027f50f7a46b1d840ce5c
diff --git a/components/esp_phy/src/phy_common.c b/components/esp_phy/src/phy_common.c
index c4824841c7..d514227799 100644
--- a/components/esp_phy/src/phy_common.c
+++ b/components/esp_phy/src/phy_common.c
@@ -5,20 +5,42 @@
  */
 
 #include "esp_timer.h"
+#include "esp_phy_init.h"
 #include <stdint.h>
 
+static volatile uint16_t s_phy_modem_flag = 0;
 
-extern void bt_track_pll_cap(void);
+extern void phy_param_track_tot(bool en_wifi, bool en_ble_154);
 static esp_timer_handle_t phy_track_pll_timer;
-static volatile int64_t s_previous_timestamp;
+static volatile int64_t s_wifi_prev_timestamp;
+static volatile int64_t s_bt_154_prev_timestamp;
 #define PHY_TRACK_PLL_PERIOD_IN_US 1000000
 
+#if CONFIG_IEEE802154_ENABLED || CONFIG_BT_ENABLED || CONFIG_WIFI_ENABLED
+bool phy_enabled_modem_contains(esp_phy_modem_t modem)
+{
+    return (s_phy_modem_flag & modem) != 0;
+}
+#endif
+
 static void phy_track_pll_timer_callback(void* arg)
 {
-#if CONFIG_IEEE802154_ENABLED || CONFIG_BT_ENABLED
-    bt_track_pll_cap();
+    bool wifi_track_pll = false;
+    bool ble_154_track_pll = false;
+#if CONFIG_WIFI_ENABLED
+    if (phy_enabled_modem_contains(PHY_MODEM_WIFI)) {
+        wifi_track_pll = true;
+        s_wifi_prev_timestamp = esp_timer_get_time();
+    }
 #endif
-    s_previous_timestamp = esp_timer_get_time();
+
+#if CONFIG_IEEE802154_ENABLED || CONFIG_BT_ENABLED
+    if (phy_enabled_modem_contains(PHY_MODEM_BT | PHY_MODEM_IEEE802154)) {
+        ble_154_track_pll = true;
+        s_bt_154_prev_timestamp = esp_timer_get_time();
+    }
+#endif
+    phy_param_track_tot(wifi_track_pll, ble_154_track_pll);
 }
 
 void phy_track_pll_init(void)
@@ -26,8 +48,14 @@ void phy_track_pll_init(void)
     // Light sleep scenario: enabling and disabling PHY frequently, the timer will not get triggered.
     // Using a variable to record the previously tracked time when PLL was last called.
     // If the duration is larger than PHY_TRACK_PLL_PERIOD_IN_US, then track PLL.
-    int64_t now = esp_timer_get_time();
-    if (now - s_previous_timestamp > PHY_TRACK_PLL_PERIOD_IN_US) {
+    bool need_track_pll = false;
+#if CONFIG_WIFI_ENABLED
+    need_track_pll = need_track_pll || ((esp_timer_get_time() - s_wifi_prev_timestamp) > PHY_TRACK_PLL_PERIOD_IN_US);
+#endif
+#if CONFIG_IEEE802154_ENABLED || CONFIG_BT_ENABLED
+    need_track_pll = need_track_pll || ((esp_timer_get_time() - s_bt_154_prev_timestamp) > PHY_TRACK_PLL_PERIOD_IN_US);
+#endif
+    if (need_track_pll) {
         phy_track_pll_timer_callback((void* )0);
     }
 
@@ -44,3 +72,18 @@ void phy_track_pll_deinit(void)
     ESP_ERROR_CHECK(esp_timer_stop(phy_track_pll_timer));
     ESP_ERROR_CHECK(esp_timer_delete(phy_track_pll_timer));
 }
+
+void phy_set_modem_flag(esp_phy_modem_t modem)
+{
+    s_phy_modem_flag |= modem;
+}
+
+void phy_clr_modem_flag(esp_phy_modem_t modem)
+{
+    s_phy_modem_flag &= ~modem;
+}
+
+esp_phy_modem_t phy_get_modem_flag(void)
+{
+    return s_phy_modem_flag;
+}
diff --git a/components/esp_phy/src/phy_init.c b/components/esp_phy/src/phy_init.c
index 467c12a67a..23eab9c622 100644
--- a/components/esp_phy/src/phy_init.c
+++ b/components/esp_phy/src/phy_init.c
@@ -66,9 +66,6 @@ static DRAM_ATTR struct {
 #endif // !SOC_PMU_SUPPORTED
 #endif // SOC_PM_SUPPORT_MODEM_PD || SOC_PM_SUPPORT_WIFI_PD
 
-/* Reference count of enabling PHY */
-static uint8_t s_phy_access_ref = 0;
-
 #if CONFIG_IDF_TARGET_ESP32
 /* time stamp updated when the PHY/RF is turned on */
 static int64_t s_phy_rf_en_ts = 0;
@@ -233,11 +230,10 @@ static inline void phy_digital_regs_load(void)
 }
 #endif // SOC_PM_MODEM_RETENTION_BY_BACKUPDMA
 
-void esp_phy_enable(void)
+void esp_phy_enable(esp_phy_modem_t modem)
 {
     _lock_acquire(&s_phy_access_lock);
-
-    if (s_phy_access_ref == 0) {
+    if (phy_get_modem_flag() == 0) {
 #if CONFIG_IDF_TARGET_ESP32
         // Update time stamp
         s_phy_rf_en_ts = esp_timer_get_time();
@@ -275,18 +271,27 @@ void esp_phy_enable(void)
 #if CONFIG_IDF_TARGET_ESP32
         coex_bt_high_prio();
 #endif
+
+// ESP32 will track pll in the wifi/BT modem interrupt handler.
+#if !CONFIG_IDF_TARGET_ESP32
+        phy_track_pll_init();
+#endif
     }
-    s_phy_access_ref++;
+    phy_set_modem_flag(modem);
 
     _lock_release(&s_phy_access_lock);
 }
 
-void esp_phy_disable(void)
+void esp_phy_disable(esp_phy_modem_t modem)
 {
     _lock_acquire(&s_phy_access_lock);
 
-    s_phy_access_ref--;
-    if (s_phy_access_ref == 0) {
+    phy_clr_modem_flag(modem);
+    if (phy_get_modem_flag() == 0) {
+// ESP32 will track pll in the wifi/BT modem interrupt handler.
+#if !CONFIG_IDF_TARGET_ESP32
+        phy_track_pll_deinit();
+#endif
 #if SOC_PM_MODEM_RETENTION_BY_BACKUPDMA
         phy_digital_regs_store();
 #endif
@@ -310,7 +315,6 @@ void esp_phy_disable(void)
         // Disable WiFi/BT common peripheral clock. Do not disable clock for hardware RNG
         esp_phy_common_clock_disable();
     }
-
     _lock_release(&s_phy_access_lock);
 }
 
diff --git a/components/esp_phy/src/phy_init_esp32hxx.c b/components/esp_phy/src/phy_init_esp32hxx.c
index ea874b28b6..9387af78af 100644
--- a/components/esp_phy/src/phy_init_esp32hxx.c
+++ b/components/esp_phy/src/phy_init_esp32hxx.c
@@ -46,9 +46,10 @@ void IRAM_ATTR phy_exit_critical(uint32_t level)
     }
 }
 
-void esp_phy_enable(void)
+void esp_phy_enable(esp_phy_modem_t modem)
 {
     _lock_acquire(&s_phy_access_lock);
+    phy_set_modem_flag(modem);
     if (s_phy_access_ref == 0) {
 #if SOC_MODEM_CLOCK_IS_INDEPENDENT
         modem_clock_module_enable(PERIPH_PHY_MODULE);
@@ -68,7 +69,7 @@ void esp_phy_enable(void)
     _lock_release(&s_phy_access_lock);
 }
 
-void esp_phy_disable(void)
+void esp_phy_disable(esp_phy_modem_t modem)
 {
     _lock_acquire(&s_phy_access_lock);
 
@@ -85,5 +86,6 @@ void esp_phy_disable(void)
 #endif
     }
 
+    phy_clr_modem_flag(modem);
     _lock_release(&s_phy_access_lock);
 }
diff --git a/components/ieee802154/driver/esp_ieee802154_dev.c b/components/ieee802154/driver/esp_ieee802154_dev.c
index 7760c049ff..f0db360374 100644
--- a/components/ieee802154/driver/esp_ieee802154_dev.c
+++ b/components/ieee802154/driver/esp_ieee802154_dev.c
@@ -796,7 +796,7 @@ IRAM_ATTR static void ieee802154_rf_disable(void)
 {
 #if CONFIG_FREERTOS_USE_TICKLESS_IDLE
     if (s_rf_closed == false) {
-        esp_phy_disable();
+        esp_phy_disable(PHY_MODEM_IEEE802154);
         s_rf_closed = true;
     }
 #endif // CONFIG_FREERTOS_USE_TICKLESS_IDLE
@@ -806,7 +806,7 @@ IRAM_ATTR static void ieee802154_rf_enable(void)
 {
 #if CONFIG_FREERTOS_USE_TICKLESS_IDLE
     if (s_rf_closed) {
-        esp_phy_enable();
+        esp_phy_enable(PHY_MODEM_IEEE802154);
         s_rf_closed = false;
     }
 #endif //CONFIG_FREERTOS_USE_TICKLESS_IDLE
diff --git a/components/ieee802154/esp_ieee802154.c b/components/ieee802154/esp_ieee802154.c
index 64aef9ecea..83900d2b02 100644
--- a/components/ieee802154/esp_ieee802154.c
+++ b/components/ieee802154/esp_ieee802154.c
@@ -22,7 +22,7 @@
 esp_err_t esp_ieee802154_enable(void)
 {
     ieee802154_enable();
-    esp_phy_enable();
+    esp_phy_enable(PHY_MODEM_IEEE802154);
     esp_btbb_enable();
     return ieee802154_mac_init();
 }