diff --git a/components/esp_wifi/Kconfig b/components/esp_wifi/Kconfig
index bc73ea8168..e32158072a 100644
--- a/components/esp_wifi/Kconfig
+++ b/components/esp_wifi/Kconfig
@@ -415,10 +415,11 @@ menu "PHY"
 
     config ESP_PHY_ENABLE_USB
         bool "Enable USB when phy init"
-        depends on ESP_CONSOLE_USB_SERIAL_JTAG
-        default y
+        depends on USB_OTG_SUPPORTED || ESP_CONSOLE_USB_SERIAL_JTAG || ESP_CONSOLE_SECONDARY_USB_SERIAL_JTAG
+        default y if IDF_TARGET_ESP32C3 || IDF_TARGET_ESP32S3
+        default n
         help
-            When using USB Serial/JTAG controller, PHY should enable USB, otherwise log will
-            not be displayed. But working with USB, the RF performance may be affected.
+            When using USB Serial/JTAG/OTG/CDC, PHY should enable USB, otherwise USB module
+            can not work properly. Notice: Enabling this configuration option will slightly impact wifi performance.
 
 endmenu  # PHY
diff --git a/components/esp_wifi/src/phy_init.c b/components/esp_wifi/src/phy_init.c
index 9e1cf4aaf5..38fd96cbe3 100644
--- a/components/esp_wifi/src/phy_init.c
+++ b/components/esp_wifi/src/phy_init.c
@@ -613,7 +613,13 @@ void esp_phy_load_cal_and_init(void)
 #endif
 
 #if CONFIG_ESP_PHY_ENABLE_USB
-    phy_bbpll_en_usb(true);
+#if CONFIG_ESP_CONSOLE_USB_SERIAL_JTAG || CONFIG_ESP_CONSOLE_SECONDARY_USB_SERIAL_JTAG
+    if (usb_serial_jtag_ll_txfifo_writable() == 1)
+#endif // Only check usb_jtag status with usb_jtag related config options enabled.
+    {
+        // If the USB_SEIRAL_JTAG is really in use.
+        phy_bbpll_en_usb(true);
+    }
 #endif
 
 #ifdef CONFIG_ESP32_PHY_CALIBRATION_AND_DATA_STORAGE