diff --git a/components/esp_security/src/init.c b/components/esp_security/src/init.c
index 989e6e2890..d00d0e9647 100644
--- a/components/esp_security/src/init.c
+++ b/components/esp_security/src/init.c
@@ -6,17 +6,44 @@
 
 #include "esp_private/startup_internal.h"
 #include "sdkconfig.h"
+#include "soc/soc_caps.h"
 #include "esp_crypto_clk.h"
 #include "esp_efuse.h"
 #include "esp_efuse_table.h"
 #include "esp_security_priv.h"
 #include "esp_err.h"
+#if SOC_KEY_MANAGER_ECDSA_KEY_DEPLOY || SOC_KEY_MANAGER_FE_KEY_DEPLOY
+#include "hal/key_mgr_ll.h"
+#endif
 
 __attribute__((unused)) static const char *TAG = "esp_security";
 
+static void esp_key_mgr_init(void)
+{
+    // The following operation makes the Key Manager to use eFuse key for ECDSA and XTS-AES operation by default
+    // This is to keep the default behavior same as the other chips
+    // If the Key Manager configuration is already locked then following operation does not have any effect
+#if SOC_KEY_MANAGER_ECDSA_KEY_DEPLOY || SOC_KEY_MANAGER_FE_KEY_DEPLOY
+    // Enable key manager clock
+    // Using ll APIs which do not require critical section
+    _key_mgr_ll_enable_bus_clock(true);
+    _key_mgr_ll_enable_peripheral_clock(true);
+
+    while (key_mgr_ll_get_state() != ESP_KEY_MGR_STATE_IDLE) {
+    };
+#if SOC_KEY_MANAGER_ECDSA_KEY_DEPLOY
+    key_mgr_ll_set_key_usage(ESP_KEY_MGR_ECDSA_KEY, ESP_KEY_MGR_USE_EFUSE_KEY);
+#endif
+#if SOC_KEY_MANAGER_FE_KEY_DEPLOY
+    key_mgr_ll_set_key_usage(ESP_KEY_MGR_XTS_AES_128_KEY, ESP_KEY_MGR_USE_EFUSE_KEY);
+#endif
+#endif /* SOC_KEY_MANAGER_ECDSA_KEY_DEPLOY || SOC_KEY_MANAGER_FE_KEY_DEPLOY */
+}
+
 ESP_SYSTEM_INIT_FN(esp_security_init, SECONDARY, BIT(0), 103)
 {
     esp_crypto_clk_init();
+    esp_key_mgr_init();
 #if CONFIG_ESP_CRYPTO_DPA_PROTECTION_AT_STARTUP
     esp_crypto_dpa_protection_startup();
 #endif
diff --git a/components/esp_system/port/cpu_start.c b/components/esp_system/port/cpu_start.c
index 1955320698..0911e8abd5 100644
--- a/components/esp_system/port/cpu_start.c
+++ b/components/esp_system/port/cpu_start.c
@@ -71,10 +71,6 @@
 #include "soc/hp_sys_clkrst_reg.h"
 #endif
 
-#if SOC_KEY_MANAGER_ECDSA_KEY_DEPLOY || SOC_KEY_MANAGER_FE_KEY_DEPLOY
-#include "hal/key_mgr_ll.h"
-#endif
-
 #include "esp_private/rtc_clk.h"
 
 #if SOC_INT_CLIC_SUPPORTED
@@ -319,22 +315,6 @@ static void start_other_core(void)
     }
 #endif
 
-    // The following operation makes the Key Manager to use eFuse key for ECDSA and XTS-AES operation by default
-    // This is to keep the default behavior same as the other chips
-    // If the Key Manager configuration is already locked then following operation does not have any effect
-#if SOC_KEY_MANAGER_ECDSA_KEY_DEPLOY || SOC_KEY_MANAGER_FE_KEY_DEPLOY
-    // Enable key manager clock
-    // Using ll APIs which do not require critical section
-    _key_mgr_ll_enable_bus_clock(true);
-    _key_mgr_ll_enable_peripheral_clock(true);
-#if SOC_KEY_MANAGER_ECDSA_KEY_DEPLOY
-    key_mgr_ll_set_key_usage(ESP_KEY_MGR_ECDSA_KEY, ESP_KEY_MGR_USE_EFUSE_KEY);
-#endif
-#if SOC_KEY_MANAGER_FE_KEY_DEPLOY
-    key_mgr_ll_set_key_usage(ESP_KEY_MGR_XTS_AES_128_KEY, ESP_KEY_MGR_USE_EFUSE_KEY);
-#endif
-#endif /* SOC_KEY_MANAGER_ECDSA_KEY_DEPLOY || SOC_KEY_MANAGER_FE_KEY_DEPLOY */
-
     ets_set_appcpu_boot_addr((uint32_t)call_start_cpu1);
 
     bool cpus_up = false;