diff --git a/components/esp_security/CMakeLists.txt b/components/esp_security/CMakeLists.txt
index e57ed3fc71..59bd2ea3ee 100644
--- a/components/esp_security/CMakeLists.txt
+++ b/components/esp_security/CMakeLists.txt
@@ -1,5 +1,14 @@
+set(srcs "")
+set(priv_requires "")
+
 if(NOT BOOTLOADER_BUILD)
-    set(src_dirs "src")
+    if(CONFIG_IDF_TARGET_ESP32H2 OR CONFIG_IDF_TARGET_ESP32P4 OR CONFIG_IDF_TARGET_ESP32C5)
+        list(APPEND srcs "src/crypto/${IDF_TARGET}/clk.c")
+    endif()
+
+    list(APPEND priv_requires "soc")
 endif()
-idf_component_register(SRC_DIRS ${src_dirs}
-                    INCLUDE_DIRS "include")
+
+idf_component_register(SRCS ${srcs}
+                    INCLUDE_DIRS "include"
+                    PRIV_REQUIRES ${priv_requires})
diff --git a/components/esp_security/src/crypto/esp32c5/clk.c b/components/esp_security/src/crypto/esp32c5/clk.c
new file mode 100644
index 0000000000..8ebf65d0e2
--- /dev/null
+++ b/components/esp_security/src/crypto/esp32c5/clk.c
@@ -0,0 +1,13 @@
+/*
+ * SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+#include "soc/soc.h"
+#include "soc/pcr_reg.h"
+
+__attribute__((weak)) void esp_crypto_clk_init(void)
+{
+    // Set crypto clock (`clk_sec`) to use 480M SPLL clock
+    REG_SET_FIELD(PCR_SEC_CONF_REG, PCR_SEC_CLK_SEL, 0x2);
+}
diff --git a/components/esp_security/src/crypto/esp32h2/clk.c b/components/esp_security/src/crypto/esp32h2/clk.c
new file mode 100644
index 0000000000..44abf80715
--- /dev/null
+++ b/components/esp_security/src/crypto/esp32h2/clk.c
@@ -0,0 +1,13 @@
+/*
+ * SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+#include "soc/soc.h"
+#include "soc/pcr_reg.h"
+
+__attribute__((weak)) void esp_crypto_clk_init(void)
+{
+    // Set crypto clock (`clk_sec`) to use 96M PLL clock
+    REG_SET_FIELD(PCR_SEC_CONF_REG, PCR_SEC_CLK_SEL, 0x3);
+}
diff --git a/components/esp_security/src/crypto/esp32p4/clk.c b/components/esp_security/src/crypto/esp32p4/clk.c
new file mode 100644
index 0000000000..3e159fdbdc
--- /dev/null
+++ b/components/esp_security/src/crypto/esp32p4/clk.c
@@ -0,0 +1,13 @@
+/*
+ * SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+#include "soc/soc.h"
+#include "soc/hp_sys_clkrst_reg.h"
+
+__attribute__((weak)) void esp_crypto_clk_init(void)
+{
+    // Set crypto clock (`clk_sec`) to use 240M PLL clock
+    REG_SET_FIELD(HP_SYS_CLKRST_PERI_CLK_CTRL25_REG, HP_SYS_CLKRST_REG_CRYPTO_CLK_SRC_SEL, 0x2);
+}
diff --git a/components/esp_system/CMakeLists.txt b/components/esp_system/CMakeLists.txt
index d2f97850ec..38f1032ed8 100644
--- a/components/esp_system/CMakeLists.txt
+++ b/components/esp_system/CMakeLists.txt
@@ -65,7 +65,7 @@ else()
 
     idf_component_register(SRCS "${srcs}"
                         INCLUDE_DIRS include
-                        PRIV_REQUIRES spi_flash esp_timer esp_mm
+                        PRIV_REQUIRES spi_flash esp_timer esp_mm esp_security
                                     # [refactor-todo] requirements due to init code,
                                     # should be removable once using component init functions
                                     # link-time registration is used.
diff --git a/components/esp_system/fpga_overrides_clk.c b/components/esp_system/fpga_overrides_clk.c
index 1b1f9efad2..cd2502cc26 100644
--- a/components/esp_system/fpga_overrides_clk.c
+++ b/components/esp_system/fpga_overrides_clk.c
@@ -78,6 +78,10 @@ void esp_perip_clk_init(void)
 {
 }
 
+void esp_crypto_clk_init(void)
+{
+}
+
 /**
  * @brief No-op function, used to force linking this file
  *
diff --git a/components/esp_system/port/include/esp_clk_internal.h b/components/esp_system/port/include/esp_clk_internal.h
index a3829368b3..63cd3a28ff 100644
--- a/components/esp_system/port/include/esp_clk_internal.h
+++ b/components/esp_system/port/include/esp_clk_internal.h
@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: 2015-2021 Espressif Systems (Shanghai) CO LTD
+ * SPDX-FileCopyrightText: 2015-2024 Espressif Systems (Shanghai) CO LTD
  *
  * SPDX-License-Identifier: Apache-2.0
  */
@@ -44,6 +44,14 @@ void esp_clk_init(void);
  */
 void esp_perip_clk_init(void);
 
+/**
+ * @brief Initialize the crypto clock
+ *
+ * Called from cpu_start.c, not intended to be called from other places.
+ * This function configures the crypto clock.
+ */
+void esp_crypto_clk_init(void);
+
 #if !CONFIG_IDF_TARGET_ESP32C2
 /* Selects an external clock source (32 kHz) for RTC.
  * Only internal use in unit test.
diff --git a/components/esp_system/port/soc/esp32h2/clk.c b/components/esp_system/port/soc/esp32h2/clk.c
index 87e2140cf3..d012e3e00a 100644
--- a/components/esp_system/port/soc/esp32h2/clk.c
+++ b/components/esp_system/port/soc/esp32h2/clk.c
@@ -140,9 +140,6 @@ __attribute__((weak)) void esp_clk_init(void)
 
     // Re calculate the ccount to make time calculation correct.
     esp_cpu_set_cycle_count((uint64_t)esp_cpu_get_cycle_count() * new_freq_mhz / old_freq_mhz);
-
-    // Set crypto clock (`clk_sec`) to use 96M PLL clock
-    REG_SET_FIELD(PCR_SEC_CONF_REG, PCR_SEC_CLK_SEL, 0x3);
 }
 
 static void select_rtc_slow_clk(soc_rtc_slow_clk_src_t rtc_slow_clk_src)
diff --git a/components/esp_system/port/soc/esp32p4/clk.c b/components/esp_system/port/soc/esp32p4/clk.c
index 49f9737e8e..a526cef6dc 100644
--- a/components/esp_system/port/soc/esp32p4/clk.c
+++ b/components/esp_system/port/soc/esp32p4/clk.c
@@ -107,9 +107,6 @@ __attribute__((weak)) void esp_clk_init(void)
 
     // Re calculate the ccount to make time calculation correct.
     esp_cpu_set_cycle_count((uint64_t)esp_cpu_get_cycle_count() * new_freq_mhz / old_freq_mhz);
-
-    // Set crypto clock (`clk_sec`) to use 240M PLL clock
-    REG_SET_FIELD(HP_SYS_CLKRST_PERI_CLK_CTRL25_REG, HP_SYS_CLKRST_REG_CRYPTO_CLK_SRC_SEL, 0x2);
 }
 
 static void select_rtc_slow_clk(soc_rtc_slow_clk_src_t rtc_slow_clk_src)