From c8fa1e915b1fba3c1dd940d03760139bb66ccc70 Mon Sep 17 00:00:00 2001 From: David Garske Date: Mon, 26 Jan 2026 16:06:05 -0800 Subject: [PATCH] Fix for RSA private key parsing (allowing public) and RSA keygen no malloc support. --- .github/workflows/os-check.yml | 1 + tests/api.c | 5 +++-- tests/api/test_ossl_rsa.c | 9 ++++---- wolfcrypt/src/asn.c | 38 +++++++++++++++++++++++++++++++--- wolfcrypt/src/rsa.c | 10 ++++++++- wolfcrypt/test/test.c | 16 ++++++++++++++ 6 files changed, 68 insertions(+), 11 deletions(-) diff --git a/.github/workflows/os-check.yml b/.github/workflows/os-check.yml index 02ff88e32..57b12de66 100644 --- a/.github/workflows/os-check.yml +++ b/.github/workflows/os-check.yml @@ -70,6 +70,7 @@ jobs: '--enable-all --enable-certgencache', '--enable-sessionexport --enable-dtls --enable-dtls13', '--enable-sessionexport', + '--disable-examples CPPFLAGS=-DWOLFSSL_NO_MALLOC', ] name: make check if: github.repository_owner == 'wolfssl' diff --git a/tests/api.c b/tests/api.c index 4f257c5dc..c4823b105 100644 --- a/tests/api.c +++ b/tests/api.c @@ -16971,12 +16971,13 @@ static int test_wolfSSL_d2i_PrivateKeys_bio(void) RSA_free(rsa); rsa = NULL; + /* d2i_RSA_PUBKEY_bio should fail when given private key DER. + * wc_RsaPublicKeyDecode correctly rejects private key format. */ ExpectIntGT(BIO_write(bio, client_key_der_2048, sizeof_client_key_der_2048), 0); - ExpectNotNull(d2i_RSA_PUBKEY_bio(bio, &rsa)); + ExpectNull(d2i_RSA_PUBKEY_bio(bio, &rsa)); (void)BIO_reset(bio); - RSA_free(rsa); rsa = RSA_new(); ExpectIntEQ(wolfSSL_i2d_RSAPrivateKey(rsa, NULL), 0); #endif /* USE_CERT_BUFFERS_2048 WOLFSSL_KEY_GEN */ diff --git a/tests/api/test_ossl_rsa.c b/tests/api/test_ossl_rsa.c index 7adbe126a..bb3c934f6 100644 --- a/tests/api/test_ossl_rsa.c +++ b/tests/api/test_ossl_rsa.c @@ -402,12 +402,11 @@ int test_wolfSSL_RSA_DER(void) for (i = 0; tbl[i].der != NULL; i++) { - /* Passing in pointer results in pointer moving. */ + /* d2i_RSAPublicKey should fail when given private key DER. + * wc_RsaPublicKeyDecode correctly rejects private key format. */ buff = tbl[i].der; - ExpectNotNull(d2i_RSAPublicKey(&rsa, &buff, tbl[i].sz)); - ExpectNotNull(rsa); - RSA_free(rsa); - rsa = NULL; + ExpectNull(d2i_RSAPublicKey(&rsa, &buff, tbl[i].sz)); + ExpectNull(rsa); } for (i = 0; tbl[i].der != NULL; i++) { diff --git a/wolfcrypt/src/asn.c b/wolfcrypt/src/asn.c index 31466ca84..a4870a763 100644 --- a/wolfcrypt/src/asn.c +++ b/wolfcrypt/src/asn.c @@ -11009,6 +11009,8 @@ int wc_RsaPublicKeyDecode_ex(const byte* input, word32* inOutIdx, word32 inSz, #ifndef WOLFSSL_ASN_TEMPLATE int ret = 0; int length = 0; + int firstLen = 0; + word32 seqEndIdx = 0; #if defined(OPENSSL_EXTRA) || defined(RSA_DECODE_EXTRA) word32 localIdx; byte tag; @@ -11066,16 +11068,19 @@ int wc_RsaPublicKeyDecode_ex(const byte* input, word32* inOutIdx, word32 inSz, } #endif /* OPENSSL_EXTRA */ + /* Calculate where the sequence should end for public key validation */ + seqEndIdx = *inOutIdx + (word32)length; + /* Get modulus */ - ret = GetASNInt(input, inOutIdx, &length, inSz); + ret = GetASNInt(input, inOutIdx, &firstLen, inSz); if (ret < 0) { return ASN_RSA_KEY_E; } if (nSz) - *nSz = (word32)length; + *nSz = (word32)firstLen; if (n) *n = &input[*inOutIdx]; - *inOutIdx += (word32)length; + *inOutIdx += (word32)firstLen; /* Get exponent */ ret = GetASNInt(input, inOutIdx, &length, inSz); @@ -11088,6 +11093,18 @@ int wc_RsaPublicKeyDecode_ex(const byte* input, word32* inOutIdx, word32 inSz, *e = &input[*inOutIdx]; *inOutIdx += (word32)length; + /* Detect if this is an RSA private key being passed as public key. + * An RSA private key has: version (small), modulus (large), exponent, + * followed by more integers (d, p, q, etc.). + * An RSA public key has: modulus (large), exponent, and nothing more. + * If the first integer is small (like version 0) AND there is more data + * remaining in the sequence, this is likely a private key. */ + if (firstLen <= MAX_VERSION_SZ && *inOutIdx < seqEndIdx) { + /* First integer is small and there's more data - looks like + * version field of a private key, not a modulus */ + return ASN_RSA_KEY_E; + } + return ret; #else DECL_ASNGETDATA(dataASN, rsaPublicKeyASN_Length); @@ -11157,6 +11174,21 @@ int wc_RsaPublicKeyDecode_ex(const byte* input, word32* inOutIdx, word32 inSz, } } #endif + if (ret == 0) { + /* Detect if this is an RSA private key being passed as public key. + * An RSA private key has: version (small int), modulus, exponent, ... + * An RSA public key has: modulus (large int), exponent, nothing more. + * If the first integer is small (like version 0) and there's more data + * after what we consumed, this is likely a private key. */ + word32 nLen = dataASN[RSAPUBLICKEYASN_IDX_PUBKEY_RSA_N].data.ref.length; + if (nLen <= MAX_VERSION_SZ && *inOutIdx < inSz) { + /* Check if next byte could be an INTEGER tag - indicating more + * fields like in a private key structure */ + if (input[*inOutIdx] == ASN_INTEGER) { + ret = ASN_RSA_KEY_E; + } + } + } if (ret == 0) { /* Return the buffers and lengths asked for. */ if (n != NULL) { diff --git a/wolfcrypt/src/rsa.c b/wolfcrypt/src/rsa.c index 6767231c1..269b39d27 100644 --- a/wolfcrypt/src/rsa.c +++ b/wolfcrypt/src/rsa.c @@ -628,7 +628,11 @@ int wc_FreeRsaKey(RsaKey* key) static int _ifc_pairwise_consistency_test(RsaKey* key, WC_RNG* rng) { static const char* msg = "Everyone gets Friday off."; - byte* sig; +#ifndef WOLFSSL_NO_MALLOC + byte* sig = NULL; +#else + byte sig[RSA_MAX_SIZE/8]; +#endif byte* plain; int ret = 0; word32 msgLen, plainLen, sigLen; @@ -643,11 +647,13 @@ static int _ifc_pairwise_consistency_test(RsaKey* key, WC_RNG* rng) WOLFSSL_MSG("Doing RSA consistency test"); +#ifndef WOLFSSL_NO_MALLOC /* Sign and verify. */ sig = (byte*)XMALLOC(sigLen, key->heap, DYNAMIC_TYPE_RSA); if (sig == NULL) { return MEMORY_E; } +#endif XMEMSET(sig, 0, sigLen); #ifdef WOLFSSL_CHECK_MEM_ZERO wc_MemZero_Add("Pairwise CT sig", sig, sigLen); @@ -690,7 +696,9 @@ static int _ifc_pairwise_consistency_test(RsaKey* key, WC_RNG* rng) ret = RSA_KEY_PAIR_E; ForceZero(sig, sigLen); +#ifndef WOLFSSL_NO_MALLOC XFREE(sig, key->heap, DYNAMIC_TYPE_RSA); +#endif return ret; } diff --git a/wolfcrypt/test/test.c b/wolfcrypt/test/test.c index d536907a4..8274174d6 100644 --- a/wolfcrypt/test/test.c +++ b/wolfcrypt/test/test.c @@ -22717,6 +22717,22 @@ static wc_test_ret_t rsa_decode_test(RsaKey* keyPub) goto done; } +#ifdef USE_CERT_BUFFERS_2048 + /* Test that public key decode rejects a private key */ + wc_FreeRsaKey(keyPub); + ret = wc_InitRsaKey(keyPub, NULL); + if (ret != 0) + return WC_TEST_RET_ENC_EC(ret); + inOutIdx = 0; + ret = wc_RsaPublicKeyDecode(client_key_der_2048, &inOutIdx, keyPub, + sizeof_client_key_der_2048); + if (ret != WC_NO_ERR_TRACE(ASN_RSA_KEY_E)) { + ret = WC_TEST_RET_ENC_EC(ret); + goto done; + } + ret = 0; /* success - public key decode correctly rejected private key */ +#endif + done: wc_FreeRsaKey(keyPub); return ret;