diff --git a/components/bt/controller/esp32/bt.c b/components/bt/controller/esp32/bt.c
index 502f233733..6112b31815 100644
--- a/components/bt/controller/esp32/bt.c
+++ b/components/bt/controller/esp32/bt.c
@@ -1215,14 +1215,16 @@ esp_err_t esp_bt_controller_init(esp_bt_controller_config_t *cfg)
 
     ESP_LOGI(BTDM_LOG_TAG, "BT controller compile version [%s]", btdm_controller_get_compile_version());
 
+    esp_phy_modem_init();
+
+    esp_bt_power_domain_on();
+
     s_wakeup_req_sem = semphr_create_wrapper(1, 0);
     if (s_wakeup_req_sem == NULL) {
         err = ESP_ERR_NO_MEM;
         goto error;
     }
 
-    esp_bt_power_domain_on();
-
     btdm_controller_mem_init();
 
     periph_module_enable(PERIPH_BT_MODULE);
@@ -1335,6 +1337,10 @@ error:
         semphr_delete_wrapper(s_wakeup_req_sem);
         s_wakeup_req_sem = NULL;
     }
+
+    esp_bt_power_domain_off();
+
+    esp_phy_modem_deinit();
     return err;
 }
 
@@ -1380,6 +1386,8 @@ esp_err_t esp_bt_controller_deinit(void)
 
     esp_bt_power_domain_off();
 
+    esp_phy_modem_deinit();
+
     return ESP_OK;
 }
 
diff --git a/components/bt/controller/esp32c3/bt.c b/components/bt/controller/esp32c3/bt.c
index b6c21c639d..137e670b66 100644
--- a/components/bt/controller/esp32c3/bt.c
+++ b/components/bt/controller/esp32c3/bt.c
@@ -1093,15 +1093,11 @@ esp_err_t esp_bt_controller_init(esp_bt_controller_config_t *cfg)
 #if CONFIG_MAC_BB_PD
     esp_mac_bb_pd_mem_init();
 #endif
+    esp_phy_modem_init();
     esp_bt_power_domain_on();
 
     sdk_config_extend_set_pll_track(false);
 
-#if CONFIG_MAC_BB_PD
-    esp_mac_bb_pd_mem_init();
-#endif
-    esp_phy_pd_mem_init();
-
     btdm_controller_mem_init();
 
 #if CONFIG_MAC_BB_PD
@@ -1339,11 +1335,12 @@ error:
     esp_unregister_mac_bb_pu_callback(btdm_mac_bb_power_up_cb);
 #endif
 
-
+    esp_bt_power_domain_off();
 #if CONFIG_MAC_BB_PD
     esp_mac_bb_pd_mem_deinit();
 #endif
-    esp_phy_pd_mem_deinit();
+    esp_phy_modem_deinit();
+
     if (osi_funcs_p != NULL) {
         free(osi_funcs_p);
         osi_funcs_p = NULL;
@@ -1420,12 +1417,11 @@ esp_err_t esp_bt_controller_deinit(void)
     esp_unregister_mac_bb_pu_callback(btdm_mac_bb_power_up_cb);
 #endif
 
-    /* Fix the issue caused by the power off the bt power domain.
-     * This issue is only on ESP32C3.
-     */
-    phy_init_flag();
-
     esp_bt_power_domain_off();
+#if CONFIG_MAC_BB_PD
+    esp_mac_bb_pd_mem_deinit();
+#endif
+    esp_phy_modem_deinit();
 
     free(osi_funcs_p);
     osi_funcs_p = NULL;
diff --git a/components/esp_wifi/include/esp_phy_init.h b/components/esp_wifi/include/esp_phy_init.h
index a22ac25a00..9c25ce6826 100644
--- a/components/esp_wifi/include/esp_phy_init.h
+++ b/components/esp_wifi/include/esp_phy_init.h
@@ -181,12 +181,12 @@ void esp_phy_load_cal_and_init(void);
 /**
  * @brief Initialize backup memory for Phy power up/down
  */
-void esp_phy_pd_mem_init(void);
+void esp_phy_modem_init(void);
 
 /**
  * @brief Deinitialize backup memory for Phy power up/down
  */
-void esp_phy_pd_mem_deinit(void);
+void esp_phy_modem_deinit(void);
 
 #if CONFIG_MAC_BB_PD
 /**
diff --git a/components/esp_wifi/src/phy_init.c b/components/esp_wifi/src/phy_init.c
index 72e354a1b9..d46b818952 100644
--- a/components/esp_wifi/src/phy_init.c
+++ b/components/esp_wifi/src/phy_init.c
@@ -85,7 +85,7 @@ static DRAM_ATTR portMUX_TYPE s_phy_int_mux = portMUX_INITIALIZER_UNLOCKED;
 
 /* Memory to store PHY digital registers */
 static uint32_t* s_phy_digital_regs_mem = NULL;
-static uint8_t s_phy_backup_mem_ref = 0;
+static uint8_t s_phy_modem_init_ref = 0;
 
 #if CONFIG_MAC_BB_PD
 uint32_t* s_mac_bb_pd_mem = NULL;
@@ -311,11 +311,11 @@ void esp_wifi_bt_power_domain_off(void)
     _lock_release(&s_wifi_bt_pd_controller.lock);
 }
 
-void esp_phy_pd_mem_init(void)
+void esp_phy_modem_init(void)
 {
     _lock_acquire(&s_phy_access_lock);
 
-    s_phy_backup_mem_ref++;
+    s_phy_modem_init_ref++;
     if (s_phy_digital_regs_mem == NULL) {
         s_phy_digital_regs_mem = (uint32_t *)heap_caps_malloc(SOC_PHY_DIG_REGS_MEM_SIZE, MALLOC_CAP_DMA|MALLOC_CAP_INTERNAL);
     }
@@ -324,15 +324,21 @@ void esp_phy_pd_mem_init(void)
 
 }
 
-void esp_phy_pd_mem_deinit(void)
+void esp_phy_modem_deinit(void)
 {
     _lock_acquire(&s_phy_access_lock);
 
-    s_phy_backup_mem_ref--;
-    if (s_phy_backup_mem_ref == 0) {
+    s_phy_modem_init_ref--;
+    if (s_phy_modem_init_ref == 0) {
         s_is_phy_reg_stored = false;
         free(s_phy_digital_regs_mem);
         s_phy_digital_regs_mem = NULL;
+        /* Fix the issue caused by the power domain off.
+        * This issue is only on ESP32C3.
+        */
+#if CONFIG_IDF_TARGET_ESP32C3
+        phy_init_flag();
+#endif
     }
 
     _lock_release(&s_phy_access_lock);
diff --git a/components/esp_wifi/src/wifi_init.c b/components/esp_wifi/src/wifi_init.c
index c7ffcc55b5..37446c0293 100644
--- a/components/esp_wifi/src/wifi_init.c
+++ b/components/esp_wifi/src/wifi_init.c
@@ -140,15 +140,12 @@ esp_err_t esp_wifi_deinit(void)
 #if CONFIG_MAC_BB_PD
     esp_unregister_mac_bb_pd_callback(pm_mac_sleep);
     esp_unregister_mac_bb_pu_callback(pm_mac_wakeup);
-#endif
-#if CONFIG_IDF_TARGET_ESP32C3
-    phy_init_flag();
 #endif
     esp_wifi_power_domain_off();
 #if CONFIG_MAC_BB_PD
     esp_mac_bb_pd_mem_deinit();
 #endif
-    esp_phy_pd_mem_deinit();
+    esp_phy_modem_deinit();
 
     return err;
 }
@@ -268,7 +265,7 @@ esp_err_t esp_wifi_init(const wifi_init_config_t *config)
         esp_mac_bb_pd_mem_init();
         esp_wifi_internal_set_mac_sleep(true);
 #endif
-        esp_phy_pd_mem_init();
+        esp_phy_modem_init();
 #if CONFIG_IDF_TARGET_ESP32
         s_wifi_mac_time_update_cb = esp_wifi_internal_update_mac_time;
 #endif