diff --git a/components/esp_system/port/CMakeLists.txt b/components/esp_system/port/CMakeLists.txt
index 3e3d88b6a7..7a323b9ea2 100644
--- a/components/esp_system/port/CMakeLists.txt
+++ b/components/esp_system/port/CMakeLists.txt
@@ -16,6 +16,29 @@ if(CONFIG_ESP_CONSOLE_USB_CDC)
     list(APPEND srcs "usb_console.c")
 endif()
 
+if(CONFIG_ESP_IPC_ISR_ENABLE)
+if(CONFIG_IDF_TARGET_ARCH_XTENSA)
+    list(APPEND srcs "arch/xtensa/esp_ipc_isr.c"
+                     "arch/xtensa/esp_ipc_isr_handler.S"
+                     "arch/xtensa/esp_ipc_isr_routines.S")
+endif()
+endif()
+
+if(CONFIG_IDF_TARGET_ARCH_XTENSA)
+    list(APPEND srcs "arch/xtensa/panic_arch.c"
+                     "arch/xtensa/panic_handler_asm.S"
+                     "arch/xtensa/expression_with_stack.c"
+                     "arch/xtensa/expression_with_stack_asm.S"
+                     "arch/xtensa/debug_helpers.c"
+                     "arch/xtensa/debug_helpers_asm.S"
+                     "arch/xtensa/debug_stubs.c"
+                     "arch/xtensa/trax.c")
+elseif(CONFIG_IDF_TARGET_ARCH_RISCV)
+    list(APPEND srcs "arch/riscv/expression_with_stack.c"
+                     "arch/riscv/panic_arch.c"
+                     "arch/riscv/debug_stubs.c")
+endif()
+
 add_prefix(srcs "${CMAKE_CURRENT_LIST_DIR}/" ${srcs})
 
 target_sources(${COMPONENT_LIB} PRIVATE ${srcs})
diff --git a/components/esp_system/port/soc/esp32/CMakeLists.txt b/components/esp_system/port/soc/esp32/CMakeLists.txt
index 0d58fd283a..1b6758a33a 100644
--- a/components/esp_system/port/soc/esp32/CMakeLists.txt
+++ b/components/esp_system/port/soc/esp32/CMakeLists.txt
@@ -2,22 +2,7 @@ set(srcs "highint_hdl.S"
          "clk.c"
          "reset_reason.c"
          "system_internal.c"
-         "cache_err_int.c"
-         "../../arch/xtensa/panic_arch.c"
-         "../../arch/xtensa/panic_handler_asm.S"
-         "../../arch/xtensa/expression_with_stack.c"
-         "../../arch/xtensa/expression_with_stack_asm.S"
-         "../../arch/xtensa/debug_helpers.c"
-         "../../arch/xtensa/debug_helpers_asm.S"
-         "../../arch/xtensa/debug_stubs.c"
-         "../../arch/xtensa/trax.c"
-         )
-
-if(CONFIG_ESP_IPC_ISR_ENABLE)
-    list(APPEND srcs "../../arch/xtensa/esp_ipc_isr.c"
-                     "../../arch/xtensa/esp_ipc_isr_handler.S"
-                     "../../arch/xtensa/esp_ipc_isr_routines.S")
-endif()
+         "cache_err_int.c")
 
 add_prefix(srcs "${CMAKE_CURRENT_LIST_DIR}/" ${srcs})
 
diff --git a/components/esp_system/port/soc/esp32c2/CMakeLists.txt b/components/esp_system/port/soc/esp32c2/CMakeLists.txt
index 09b915cd7e..4764e7e4d4 100644
--- a/components/esp_system/port/soc/esp32c2/CMakeLists.txt
+++ b/components/esp_system/port/soc/esp32c2/CMakeLists.txt
@@ -1,10 +1,7 @@
 set(srcs "clk.c"
          "reset_reason.c"
          "system_internal.c"
-         "cache_err_int.c"
-         "../../arch/riscv/expression_with_stack.c"
-         "../../arch/riscv/panic_arch.c"
-         "../../arch/riscv/debug_stubs.c")
+         "cache_err_int.c")
 
 add_prefix(srcs "${CMAKE_CURRENT_LIST_DIR}/" ${srcs})
 
diff --git a/components/esp_system/port/soc/esp32c3/CMakeLists.txt b/components/esp_system/port/soc/esp32c3/CMakeLists.txt
index 69579319fb..320b457e6c 100644
--- a/components/esp_system/port/soc/esp32c3/CMakeLists.txt
+++ b/components/esp_system/port/soc/esp32c3/CMakeLists.txt
@@ -2,10 +2,7 @@ set(srcs "clk.c"
          "reset_reason.c"
          "system_internal.c"
          "cache_err_int.c"
-         "apb_backup_dma.c"
-         "../../arch/riscv/expression_with_stack.c"
-         "../../arch/riscv/panic_arch.c"
-         "../../arch/riscv/debug_stubs.c")
+         "apb_backup_dma.c")
 
 add_prefix(srcs "${CMAKE_CURRENT_LIST_DIR}/" ${srcs})
 
diff --git a/components/esp_system/port/soc/esp32c6/CMakeLists.txt b/components/esp_system/port/soc/esp32c6/CMakeLists.txt
index 09b915cd7e..4764e7e4d4 100644
--- a/components/esp_system/port/soc/esp32c6/CMakeLists.txt
+++ b/components/esp_system/port/soc/esp32c6/CMakeLists.txt
@@ -1,10 +1,7 @@
 set(srcs "clk.c"
          "reset_reason.c"
          "system_internal.c"
-         "cache_err_int.c"
-         "../../arch/riscv/expression_with_stack.c"
-         "../../arch/riscv/panic_arch.c"
-         "../../arch/riscv/debug_stubs.c")
+         "cache_err_int.c")
 
 add_prefix(srcs "${CMAKE_CURRENT_LIST_DIR}/" ${srcs})
 
diff --git a/components/esp_system/port/soc/esp32h2/CMakeLists.txt b/components/esp_system/port/soc/esp32h2/CMakeLists.txt
index 890af3fd42..57a1a7af19 100644
--- a/components/esp_system/port/soc/esp32h2/CMakeLists.txt
+++ b/components/esp_system/port/soc/esp32h2/CMakeLists.txt
@@ -2,10 +2,7 @@ set(srcs
          "clk.c"
          "reset_reason.c"
          "system_internal.c"
-         "cache_err_int.c"
-         "../../arch/riscv/expression_with_stack.c"
-         "../../arch/riscv/panic_arch.c"
-         "../../arch/riscv/debug_stubs.c")
+         "cache_err_int.c")
 
 add_prefix(srcs "${CMAKE_CURRENT_LIST_DIR}/" ${srcs})
 
diff --git a/components/esp_system/port/soc/esp32p4/CMakeLists.txt b/components/esp_system/port/soc/esp32p4/CMakeLists.txt
index 09b915cd7e..4764e7e4d4 100644
--- a/components/esp_system/port/soc/esp32p4/CMakeLists.txt
+++ b/components/esp_system/port/soc/esp32p4/CMakeLists.txt
@@ -1,10 +1,7 @@
 set(srcs "clk.c"
          "reset_reason.c"
          "system_internal.c"
-         "cache_err_int.c"
-         "../../arch/riscv/expression_with_stack.c"
-         "../../arch/riscv/panic_arch.c"
-         "../../arch/riscv/debug_stubs.c")
+         "cache_err_int.c")
 
 add_prefix(srcs "${CMAKE_CURRENT_LIST_DIR}/" ${srcs})
 
diff --git a/components/esp_system/port/soc/esp32s2/CMakeLists.txt b/components/esp_system/port/soc/esp32s2/CMakeLists.txt
index 2adf51698a..ebac7a455f 100644
--- a/components/esp_system/port/soc/esp32s2/CMakeLists.txt
+++ b/components/esp_system/port/soc/esp32s2/CMakeLists.txt
@@ -2,22 +2,7 @@ set(srcs "highint_hdl.S"
          "clk.c"
          "reset_reason.c"
          "system_internal.c"
-         "cache_err_int.c"
-         "../../arch/xtensa/panic_arch.c"
-         "../../arch/xtensa/panic_handler_asm.S"
-         "../../arch/xtensa/expression_with_stack.c"
-         "../../arch/xtensa/expression_with_stack_asm.S"
-         "../../arch/xtensa/debug_helpers.c"
-         "../../arch/xtensa/debug_helpers_asm.S"
-         "../../arch/xtensa/debug_stubs.c"
-         "../../arch/xtensa/trax.c"
-         )
-
-if(CONFIG_ESP_IPC_ISR_ENABLE)
-    list(APPEND srcs "../../arch/xtensa/esp_ipc_isr.c"
-                     "../../arch/xtensa/esp_ipc_isr_handler.S"
-                     "../../arch/xtensa/esp_ipc_isr_routines.S")
-endif()
+         "cache_err_int.c")
 
 add_prefix(srcs "${CMAKE_CURRENT_LIST_DIR}/" ${srcs})
 
diff --git a/components/esp_system/port/soc/esp32s3/CMakeLists.txt b/components/esp_system/port/soc/esp32s3/CMakeLists.txt
index c910e6d3ea..b7e5fd025c 100644
--- a/components/esp_system/port/soc/esp32s3/CMakeLists.txt
+++ b/components/esp_system/port/soc/esp32s3/CMakeLists.txt
@@ -3,22 +3,7 @@ set(srcs "highint_hdl.S"
          "reset_reason.c"
          "system_internal.c"
          "cache_err_int.c"
-         "apb_backup_dma.c"
-         "../../arch/xtensa/panic_arch.c"
-         "../../arch/xtensa/panic_handler_asm.S"
-         "../../arch/xtensa/expression_with_stack.c"
-         "../../arch/xtensa/expression_with_stack_asm.S"
-         "../../arch/xtensa/debug_helpers.c"
-         "../../arch/xtensa/debug_helpers_asm.S"
-         "../../arch/xtensa/debug_stubs.c"
-         "../../arch/xtensa/trax.c"
-         )
-
-if(CONFIG_ESP_IPC_ISR_ENABLE)
-    list(APPEND srcs "../../arch/xtensa/esp_ipc_isr.c"
-                     "../../arch/xtensa/esp_ipc_isr_handler.S"
-                     "../../arch/xtensa/esp_ipc_isr_routines.S")
-endif()
+         "apb_backup_dma.c")
 
 add_prefix(srcs "${CMAKE_CURRENT_LIST_DIR}/" ${srcs})