diff --git a/components/bt/controller/esp32/bt.c b/components/bt/controller/esp32/bt.c
index 7c147de215..763dc4cdde 100644
--- a/components/bt/controller/esp32/bt.c
+++ b/components/bt/controller/esp32/bt.c
@@ -1628,6 +1628,8 @@ esp_err_t esp_bt_controller_init(esp_bt_controller_config_t *cfg)
         goto error;
     }
 
+    esp_phy_pd_mem_init();
+
     esp_bt_power_domain_on();
 
     btdm_controller_mem_init();
@@ -1794,6 +1796,8 @@ esp_err_t esp_bt_controller_deinit(void)
 
     esp_bt_power_domain_off();
 
+    esp_phy_pd_mem_deinit();
+
     return ESP_OK;
 }
 
diff --git a/components/bt/controller/esp32c3/bt.c b/components/bt/controller/esp32c3/bt.c
index 73f82aba22..57fcc349a0 100644
--- a/components/bt/controller/esp32c3/bt.c
+++ b/components/bt/controller/esp32c3/bt.c
@@ -967,6 +967,7 @@ esp_err_t esp_bt_controller_init(esp_bt_controller_config_t *cfg)
 #if CONFIG_MAC_BB_PD
     esp_mac_bb_pd_mem_init();
 #endif
+    esp_phy_pd_mem_init();
     esp_bt_power_domain_on();
 
     btdm_controller_mem_init();
@@ -1246,6 +1247,7 @@ esp_err_t esp_bt_controller_deinit(void)
 #if CONFIG_MAC_BB_PD
     esp_mac_bb_pd_mem_deinit();
 #endif
+    esp_phy_pd_mem_deinit();
 
     free(osi_funcs_p);
     osi_funcs_p = NULL;
diff --git a/components/bt/controller/esp32s3/bt.c b/components/bt/controller/esp32s3/bt.c
index fac79ac92a..06cb5eb848 100644
--- a/components/bt/controller/esp32s3/bt.c
+++ b/components/bt/controller/esp32s3/bt.c
@@ -934,6 +934,7 @@ esp_err_t esp_bt_controller_init(esp_bt_controller_config_t *cfg)
 #if CONFIG_MAC_BB_PD
     esp_mac_bb_pd_mem_init();
 #endif
+    esp_phy_pd_mem_init();
     esp_bt_power_domain_on();
 
     btdm_controller_mem_init();
@@ -1205,6 +1206,10 @@ esp_err_t esp_bt_controller_deinit(void)
 #endif
 
     esp_bt_power_domain_off();
+#if CONFIG_MAC_BB_PD
+    esp_mac_bb_pd_mem_deinit();
+#endif
+    esp_phy_pd_mem_deinit();
 
     free(osi_funcs_p);
     osi_funcs_p = NULL;
diff --git a/components/esp_phy/include/esp_phy_init.h b/components/esp_phy/include/esp_phy_init.h
index 6d94e51527..ec68213950 100644
--- a/components/esp_phy/include/esp_phy_init.h
+++ b/components/esp_phy/include/esp_phy_init.h
@@ -170,6 +170,16 @@ void esp_phy_disable(void);
  */
 void esp_phy_load_cal_and_init(void);
 
+/**
+ * @brief Initialize backup memory for Phy power up/down
+ */
+void esp_phy_pd_mem_init(void);
+
+/**
+ * @brief Deinitialize backup memory for Phy power up/down
+ */
+void esp_phy_pd_mem_deinit(void);
+
 #if CONFIG_MAC_BB_PD
 /**
  * @brief Initialize backup memory for MAC and Baseband power up/down
diff --git a/components/esp_phy/src/phy_init.c b/components/esp_phy/src/phy_init.c
index cc50a750c2..58ea6b9a24 100644
--- a/components/esp_phy/src/phy_init.c
+++ b/components/esp_phy/src/phy_init.c
@@ -70,11 +70,12 @@ static DRAM_ATTR portMUX_TYPE s_phy_int_mux = portMUX_INITIALIZER_UNLOCKED;
 
 /* Memory to store PHY digital registers */
 static uint32_t* s_phy_digital_regs_mem = NULL;
+static uint8_t s_phy_backup_mem_ref = 0;
 
 #if CONFIG_MAC_BB_PD
 uint32_t* s_mac_bb_pd_mem = NULL;
 /* Reference count of MAC BB backup memory */
-static uint8_t s_backup_mem_ref = 0;
+static uint8_t s_macbb_backup_mem_ref = 0;
 #endif
 
 #if CONFIG_ESP_PHY_MULTIPLE_INIT_DATA_BIN
@@ -203,10 +204,6 @@ IRAM_ATTR void esp_phy_common_clock_disable(void)
 
 static inline void phy_digital_regs_store(void)
 {
-    if (s_phy_digital_regs_mem == NULL) {
-        s_phy_digital_regs_mem = (uint32_t *)malloc(SOC_PHY_DIG_REGS_MEM_SIZE);
-    }
-
     if (s_phy_digital_regs_mem != NULL) {
         phy_dig_reg_backup(true, s_phy_digital_regs_mem);
     }
@@ -304,12 +301,38 @@ void esp_wifi_bt_power_domain_off(void)
     _lock_release(&s_wifi_bt_pd_controller.lock);
 }
 
+void esp_phy_pd_mem_init(void)
+{
+    _lock_acquire(&s_phy_access_lock);
+
+    s_phy_backup_mem_ref++;
+    if (s_phy_digital_regs_mem == NULL) {
+        s_phy_digital_regs_mem = (uint32_t *)heap_caps_malloc(SOC_PHY_DIG_REGS_MEM_SIZE, MALLOC_CAP_DMA|MALLOC_CAP_INTERNAL);
+    }
+
+    _lock_release(&s_phy_access_lock);
+
+}
+
+void esp_phy_pd_mem_deinit(void)
+{
+    _lock_acquire(&s_phy_access_lock);
+
+    s_phy_backup_mem_ref--;
+    if (s_phy_backup_mem_ref == 0) {
+        free(s_phy_digital_regs_mem);
+        s_phy_digital_regs_mem = NULL;
+    }
+
+    _lock_release(&s_phy_access_lock);
+}
+
 #if CONFIG_MAC_BB_PD
 void esp_mac_bb_pd_mem_init(void)
 {
     _lock_acquire(&s_phy_access_lock);
 
-    s_backup_mem_ref++;
+    s_macbb_backup_mem_ref++;
     if (s_mac_bb_pd_mem == NULL) {
         s_mac_bb_pd_mem = (uint32_t *)heap_caps_malloc(SOC_MAC_BB_PD_MEM_SIZE, MALLOC_CAP_DMA|MALLOC_CAP_INTERNAL);
     }
@@ -321,8 +344,8 @@ void esp_mac_bb_pd_mem_deinit(void)
 {
     _lock_acquire(&s_phy_access_lock);
 
-    s_backup_mem_ref--;
-    if (s_backup_mem_ref == 0) {
+    s_macbb_backup_mem_ref--;
+    if (s_macbb_backup_mem_ref == 0) {
         free(s_mac_bb_pd_mem);
         s_mac_bb_pd_mem = NULL;
     }
diff --git a/components/esp_wifi/src/wifi_init.c b/components/esp_wifi/src/wifi_init.c
index bf7ed36040..b4da70a602 100644
--- a/components/esp_wifi/src/wifi_init.c
+++ b/components/esp_wifi/src/wifi_init.c
@@ -136,6 +136,8 @@ esp_err_t esp_wifi_deinit(void)
 #if CONFIG_MAC_BB_PD
     esp_mac_bb_pd_mem_deinit();
 #endif
+    esp_phy_pd_mem_deinit();
+
     return err;
 }
 
@@ -251,6 +253,7 @@ esp_err_t esp_wifi_init(const wifi_init_config_t *config)
         esp_mac_bb_pd_mem_init();
         esp_wifi_internal_set_mac_sleep(true);
 #endif
+        esp_phy_pd_mem_init();
 #if CONFIG_IDF_TARGET_ESP32
         s_wifi_mac_time_update_cb = esp_wifi_internal_update_mac_time;
 #endif