diff --git a/components/bootloader_support/src/esp32/secure_boot_signatures.c b/components/bootloader_support/src/esp32/secure_boot_signatures.c
index 30cebbcf58..57dc55ba10 100644
--- a/components/bootloader_support/src/esp32/secure_boot_signatures.c
+++ b/components/bootloader_support/src/esp32/secure_boot_signatures.c
@@ -152,7 +152,18 @@ esp_err_t esp_secure_boot_verify_rsa_signature_block(const ets_secure_boot_signa
         bootloader_sha256_finish(sig_block_sha, (unsigned char *)sig_block_trusted_digest);
 
         if (memcmp(efuse_trusted_digest, sig_block_trusted_digest, DIGEST_LEN) != 0) {
-            ESP_LOGW(TAG, "Public key digest in eFuse BLK2 and the signature block don't match.");
+            /* Most likely explanation for this is that BLK2 is empty, and we're going to burn it
+               after we verify that the signature is valid. However, if BLK2 is not empty then we need to
+               fail here.
+            */
+            bool all_zeroes = true;
+            for (int i = 0; i < DIGEST_LEN; i++) {
+                all_zeroes = all_zeroes && (efuse_trusted_digest[i] == 0);
+            }
+            if (!all_zeroes) {
+                ESP_LOGE(TAG, "Different public key digest burned to eFuse BLK2");
+                return ESP_ERR_INVALID_STATE;
+            }
         }
 
         memcpy(efuse_trusted_digest, sig_block_trusted_digest, DIGEST_LEN);
diff --git a/components/bootloader_support/src/idf/secure_boot_signatures.c b/components/bootloader_support/src/idf/secure_boot_signatures.c
index d2021cf1a5..27b1f7af33 100644
--- a/components/bootloader_support/src/idf/secure_boot_signatures.c
+++ b/components/bootloader_support/src/idf/secure_boot_signatures.c
@@ -187,11 +187,14 @@ esp_err_t esp_secure_boot_verify_rsa_signature_block(const ets_secure_boot_signa
     bootloader_sha256_finish(sig_block_sha, (unsigned char *)sig_block_trusted_digest);
 
     if (memcmp(efuse_trusted_digest, sig_block_trusted_digest, DIGEST_LEN) != 0) {
-        if (esp_secure_boot_enabled()) {
+        const uint8_t zeroes[DIGEST_LEN] = {0};
+        /* Can't continue if secure boot is enabled, OR if a different digest is already written in efuse BLK2
+
+           (If BLK2 is empty and Secure Boot is disabled then we assume that it will be enabled later.)
+         */
+        if (esp_secure_boot_enabled() || memcmp(efuse_trusted_digest, zeroes, DIGEST_LEN) != 0) {
             ESP_LOGE(TAG, "Public key digest in eFuse BLK2 and the signature block don't match.");
             return ESP_FAIL;
-        } else {
-            ESP_LOGW(TAG, "Public key digest in eFuse BLK2 and the signature block don't match.");
         }
     }