From 2f01c9d715085ed77a5fc47b6dcd122b23253e78 Mon Sep 17 00:00:00 2001
From: Koji Takeda <takeda@wolfssl.com>
Date: Thu, 3 Apr 2025 16:18:16 +0900
Subject: [PATCH 1/2] Detect unknown key format

---
 src/ssl_load.c | 22 ++++++++++++++++++++--
 1 file changed, 20 insertions(+), 2 deletions(-)

diff --git a/src/ssl_load.c b/src/ssl_load.c
index f9bacaba0..051464564 100644
--- a/src/ssl_load.c
+++ b/src/ssl_load.c
@@ -1090,6 +1090,7 @@ static int ProcessBufferTryDecode(WOLFSSL_CTX* ctx, WOLFSSL* ssl,
     int devId = wolfSSL_CTX_GetDevId(ctx, ssl);
     byte* keyType = NULL;
     int* keySz = NULL;
+    int matchAnyKey = 0;
 
     (void)heap;
     (void)devId;
@@ -1141,8 +1142,19 @@ static int ProcessBufferTryDecode(WOLFSSL_CTX* ctx, WOLFSSL* ssl,
         ret = ProcessBufferTryDecodeRsa(ctx, ssl, der, keyFormat, heap, devId,
             keyType, keySz);
 #endif
+        matchAnyKey = 1;
     }
-#endif
+#ifdef WC_RSA_PSS
+    if(*keyFormat == RSAPSSk) {
+        /*
+            Require logic to verify that the der is RSAPSSk (when *keyFormat == RSAPSSK),
+            and to detect that the der is RSAPSSk (when *keyFormat == 0).
+        */
+
+        matchAnyKey = 1;
+    }
+#endif /* WC_RSA_PSS */
+#endif /* NO_RSA */
 #ifdef HAVE_ECC
     /* Try ECC if key format is ECDSA or SM2, or yet unknown. */
     if ((ret == 0) && ((*keyFormat == 0) || (*keyFormat == ECDSAk)
@@ -1152,6 +1164,7 @@ static int ProcessBufferTryDecode(WOLFSSL_CTX* ctx, WOLFSSL* ssl,
         )) {
         ret = ProcessBufferTryDecodeEcc(ctx, ssl, der, keyFormat, heap, devId,
             keyType, keySz);
+        matchAnyKey = 1;
     }
 #endif /* HAVE_ECC */
 #if defined(HAVE_ED25519) && defined(HAVE_ED25519_KEY_IMPORT)
@@ -1159,6 +1172,7 @@ static int ProcessBufferTryDecode(WOLFSSL_CTX* ctx, WOLFSSL* ssl,
     if ((ret == 0) && ((*keyFormat == 0 || *keyFormat == ED25519k))) {
         ret = ProcessBufferTryDecodeEd25519(ctx, ssl, der, keyFormat, heap,
             devId, keyType, keySz);
+        matchAnyKey = 1;
     }
 #endif /* HAVE_ED25519 && HAVE_ED25519_KEY_IMPORT */
 #if defined(HAVE_ED448) && defined(HAVE_ED448_KEY_IMPORT)
@@ -1166,6 +1180,7 @@ static int ProcessBufferTryDecode(WOLFSSL_CTX* ctx, WOLFSSL* ssl,
     if ((ret == 0) && ((*keyFormat == 0 || *keyFormat == ED448k))) {
         ret = ProcessBufferTryDecodeEd448(ctx, ssl, der, keyFormat, heap, devId,
             keyType, keySz);
+        matchAnyKey = 1;
     }
 #endif /* HAVE_ED448 && HAVE_ED448_KEY_IMPORT */
 #if defined(HAVE_FALCON)
@@ -1174,6 +1189,7 @@ static int ProcessBufferTryDecode(WOLFSSL_CTX* ctx, WOLFSSL* ssl,
             (*keyFormat == FALCON_LEVEL5k))) {
         ret = ProcessBufferTryDecodeFalcon(ctx, ssl, der, keyFormat, heap,
             keyType, keySz);
+        matchAnyKey = 1;
     }
 #endif /* HAVE_FALCON */
 #if defined(HAVE_DILITHIUM) && !defined(WOLFSSL_DILITHIUM_NO_SIGN) && \
@@ -1193,11 +1209,13 @@ static int ProcessBufferTryDecode(WOLFSSL_CTX* ctx, WOLFSSL* ssl,
         )) {
         ret = ProcessBufferTryDecodeDilithium(ctx, ssl, der, keyFormat, heap,
             keyType, keySz);
+        matchAnyKey = 1;
     }
 #endif /* HAVE_DILITHIUM */
 
     /* Check we know the format. */
-    if ((ret == 0) && (*keyFormat == 0)) {
+    if ((ret == 0) &&
+        ((*keyFormat == 0) || ((*keyFormat != 0) && (matchAnyKey == 0)))) {
         WOLFSSL_MSG("Not a supported key type");
         /* Not supported key format. */
         ret = WOLFSSL_BAD_FILE;

From 71ebad1fc7dc8b9a2d523d64584a56425cd30b27 Mon Sep 17 00:00:00 2001
From: Koji Takeda <takeda@wolfssl.com>
Date: Thu, 3 Apr 2025 19:10:32 +0900
Subject: [PATCH 2/2] Add test

---
 tests/api.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tests/api.c b/tests/api.c
index c0ebce887..2abcb912f 100644
--- a/tests/api.c
+++ b/tests/api.c
@@ -2312,6 +2312,10 @@ static int test_wolfSSL_CTX_use_PrivateKey_file(void)
     /* invalid key type */
     ExpectFalse(wolfSSL_CTX_use_PrivateKey_file(ctx, svrKeyFile, 9999));
 
+    /* invalid key format */
+    ExpectFalse(wolfSSL_CTX_use_PrivateKey_file(ctx, "./certs/dh-priv-2048.pem",
+                                                         WOLFSSL_FILETYPE_PEM));
+
     /* success */
 #ifdef NO_RSA
     /* rsa needed */