diff --git a/components/hal/CMakeLists.txt b/components/hal/CMakeLists.txt
index 58698a7219..d4a21d3612 100644
--- a/components/hal/CMakeLists.txt
+++ b/components/hal/CMakeLists.txt
@@ -238,6 +238,13 @@ if(NOT BOOTLOADER_BUILD)
         list(APPEND srcs "huk_hal.c")
     endif()
 
+    if(CONFIG_SOC_USB_OTG_SUPPORTED)
+        list(APPEND srcs
+                    "usb_hal.c"
+                    "usb_dwc_hal.c"
+                    "usb_phy_hal.c")
+    endif()
+
     if(${target} STREQUAL "esp32")
         list(APPEND srcs
             "touch_sensor_hal.c"
@@ -248,23 +255,17 @@ if(NOT BOOTLOADER_BUILD)
     if(${target} STREQUAL "esp32s2")
         list(APPEND srcs
                     "touch_sensor_hal.c"
-                    "usb_hal.c"
-                    "usb_phy_hal.c"
                     "xt_wdt_hal.c"
                     "esp32s2/cp_dma_hal.c"
-                    "esp32s2/touch_sensor_hal.c"
-                    "usb_dwc_hal.c")
+                    "esp32s2/touch_sensor_hal.c")
     endif()
 
     if(${target} STREQUAL "esp32s3")
         list(APPEND srcs
             "touch_sensor_hal.c"
-            "usb_hal.c"
-            "usb_phy_hal.c"
             "xt_wdt_hal.c"
             "esp32s3/touch_sensor_hal.c"
-            "esp32s3/rtc_cntl_hal.c"
-            "usb_dwc_hal.c")
+            "esp32s3/rtc_cntl_hal.c")
     endif()
 
     if(${target} STREQUAL "esp32c3")