Fix potential mismatch in size between DECL_MP_INT_SIZE_DYN and NEW_MP_INT_SIZE. In some configurations DECL_MP_INT_SIZE_DYN will set the size to max while NEW_MP_INT_SIZE will memset bits.

tests/api/test_rsa.c

@@ -1211,6 +1211,56 @@ int test_wc_RsaDecrypt_BoundsCheck(void)
     return EXPECT_RESULT();
 } /* END test_wc_RsaDecryptBoundsCheck */
 
+/*
+ * Oversized RSA modulus (mp_bitsused(n) > RSA_MAX_SIZE) must not overflow the
+ * static stack buffer used by RsaFunctionCheckIn (DECL_MP_INT_SIZE_DYN).
+ */
+int test_wc_RsaFunctionCheckIn_OversizedModulus(void)
+{
+    EXPECT_DECLS;
+#if !defined(NO_RSA) && defined(WC_RSA_NO_PADDING) && defined(WC_RSA_DIRECT) && \
+    defined(WOLFSSL_PUBLIC_MP) && !defined(NO_RSA_BOUNDS_CHECK) && \
+    (defined(WOLFSSL_SP_MATH) || defined(WOLFSSL_SP_MATH_ALL)) && \
+    !defined(WOLFSSL_SMALL_STACK) && \
+    (defined(USE_CERT_BUFFERS_1024) || defined(USE_CERT_BUFFERS_2048))
+    WC_RNG rng;
+    RsaKey key;
+    const byte* derKey;
+    word32 derKeySz;
+    word32 idx = 0;
+    byte flatC[256];
+    word32 flatCSz;
+    byte out[256];
+    word32 outSz = sizeof(out);
+
+    #ifdef USE_CERT_BUFFERS_1024
+        derKey = server_key_der_1024;
+        derKeySz = (word32)sizeof_server_key_der_1024;
+        flatCSz = 128;
+    #else
+        derKey = server_key_der_2048;
+        derKeySz = (word32)sizeof_server_key_der_2048;
+        flatCSz = 256;
+    #endif
+
+    XMEMSET(&key, 0, sizeof(RsaKey));
+    XMEMSET(&rng, 0, sizeof(WC_RNG));
+
+    ExpectIntEQ(wc_InitRsaKey(&key, HEAP_HINT), 0);
+    ExpectIntEQ(wc_InitRng(&rng), 0);
+    ExpectIntEQ(wc_RsaPrivateKeyDecode(derKey, &idx, &key, derKeySz), 0);
+    /* Force modulus bit count above RSA_MAX_SIZE. */
+    ExpectIntEQ(mp_set_bit(&key.n, RSA_MAX_SIZE), 0);
+    XMEMSET(flatC, 0, flatCSz);
+    ExpectIntEQ(wc_RsaDirect(flatC, flatCSz, out, &outSz, &key,
+        RSA_PRIVATE_DECRYPT, &rng), WC_NO_ERR_TRACE(WC_KEY_SIZE_E));
+
+    DoExpectIntEQ(wc_FreeRsaKey(&key), 0);
+    DoExpectIntEQ(wc_FreeRng(&rng), 0);
+#endif
+    return EXPECT_RESULT();
+} /* END test_wc_RsaFunctionCheckIn_OversizedModulus */
+
 /*
  * Test wc_RsaKeyToDer with an mp_int large enough to wrap size calculations.
  */

tests/api/test_rsa.h

@@ -43,6 +43,7 @@ int test_wc_RsaEncryptSize(void);
 int test_wc_RsaSSL_SignVerify(void);
 int test_wc_RsaFlattenPublicKey(void);
 int test_wc_RsaDecrypt_BoundsCheck(void);
+int test_wc_RsaFunctionCheckIn_OversizedModulus(void);
 int test_wc_RsaKeyToDer_SizeOverflow(void);
 
 #define TEST_RSA_DECLS                                          \
@@ -65,6 +66,7 @@ int test_wc_RsaKeyToDer_SizeOverflow(void);
     TEST_DECL_GROUP("rsa", test_wc_RsaSSL_SignVerify),          \
     TEST_DECL_GROUP("rsa", test_wc_RsaFlattenPublicKey),        \
     TEST_DECL_GROUP("rsa", test_wc_RsaDecrypt_BoundsCheck),             \
+    TEST_DECL_GROUP("rsa", test_wc_RsaFunctionCheckIn_OversizedModulus), \
     TEST_DECL_GROUP("rsa", test_wc_RsaKeyToDer_SizeOverflow)
 
 #endif /* WOLFCRYPT_TEST_RSA_H */

wolfcrypt/src/ecc.c

@@ -2012,6 +2012,10 @@ static int _ecc_projective_add_point(ecc_point* P, ecc_point* Q, ecc_point* R,
    mp_int  *x, *y, *z;
    int     err;
 
+   if (mp_bitsused(modulus) > MAX_ECC_BITS_USE) {
+       return WC_KEY_SIZE_E;
+   }
+
    /* if Q == R then swap P and Q, so we don't require a local x,y,z */
    if (Q == R) {
       ecc_point* tPt  = P;
@@ -2412,6 +2416,10 @@ static int _ecc_projective_dbl_point(ecc_point *P, ecc_point *R, mp_int* a,
    mp_int *x, *y, *z;
    int    err;
 
+   if (mp_bitsused(modulus) > MAX_ECC_BITS_USE) {
+       return WC_KEY_SIZE_E;
+   }
+
 #ifdef WOLFSSL_SMALL_STACK
 #ifdef WOLFSSL_SMALL_STACK_CACHE
    if (R->key != NULL) {
@@ -2762,6 +2770,10 @@ int ecc_map_ex(ecc_point* P, mp_int* modulus, mp_digit mp, int ct)
         #endif
         mp_int *x, *y, *z;
 
+        if (mp_bitsused(modulus) > MAX_ECC_BITS_USE) {
+            return WC_KEY_SIZE_E;
+        }
+
         /* special case for point at infinity */
         if (mp_cmp_d(P->z, 0) == MP_EQ) {
             err = mp_set(P->x, 0);
@@ -3580,8 +3592,13 @@ static int ecc_point_to_mont(ecc_point* p, ecc_point* r, mp_int* modulus,
                              void* heap)
 {
    int err = MP_OKAY;
+
    DECL_MP_INT_SIZE_DYN(mu, mp_bitsused(modulus), MAX_ECC_BITS_USE);
 
+   if (mp_bitsused(modulus) > MAX_ECC_BITS_USE) {
+       return WC_KEY_SIZE_E;
+   }
+
    (void)heap;
 
    NEW_MP_INT_SIZE(mu, mp_bitsused(modulus), heap, DYNAMIC_TYPE_ECC);
@@ -3886,6 +3903,11 @@ static int ecc_check_order_minus_1(const mp_int* k, ecc_point* tG, ecc_point* R,
     int err;
     DECL_MP_INT_SIZE_DYN(t, mp_bitsused(order), MAX_ECC_BITS_USE);
 
+    if (mp_bitsused(order) > MAX_ECC_BITS_USE ||
+            mp_bitsused(modulus) > MAX_ECC_BITS_USE) {
+        return WC_KEY_SIZE_E;
+    }
+
     NEW_MP_INT_SIZE(t, mp_bitsused(modulus), NULL, DYNAMIC_TYPE_ECC);
 #ifdef MP_INT_SIZE_CHECK_NULL
     if (t == NULL) {
@@ -6836,6 +6858,10 @@ int wc_ecc_sign_hash(const byte* in, word32 inlen, byte* out, word32 *outlen,
     word32 keySz;
 #endif
 
+    if (ECC_KEY_MAX_BITS(key) > MAX_ECC_BITS_USE) {
+        return WC_KEY_SIZE_E;
+    }
+
     if (in == NULL || out == NULL || outlen == NULL || key == NULL) {
         return ECC_BAD_ARG_E;
     }
@@ -7034,8 +7060,13 @@ static int ecc_sign_hash_sw(ecc_key* key, ecc_key* pubkey, WC_RNG* rng,
 {
     int err = MP_OKAY;
     int loop_check = 0;
+
     DECL_MP_INT_SIZE_DYN(b, ECC_KEY_MAX_BITS_NONULLCHECK(key), MAX_ECC_BITS_USE);
 
+    if (ECC_KEY_MAX_BITS_NONULLCHECK(key) > MAX_ECC_BITS_USE) {
+        return WC_KEY_SIZE_E;
+    }
+
     NEW_MP_INT_SIZE(b, ECC_KEY_MAX_BITS_NONULLCHECK(key), key->heap, DYNAMIC_TYPE_ECC);
 #ifdef MP_INT_SIZE_CHECK_NULL
     if (b == NULL)
@@ -7363,6 +7394,9 @@ int wc_ecc_sign_hash_ex(const byte* in, word32 inlen, WC_RNG* rng,
 #else
    DECLARE_CURVE_SPECS(1);
 #endif
+   if (ECC_KEY_MAX_BITS(key) > MAX_ECC_BITS_USE) {
+       return WC_KEY_SIZE_E;
+   }
 #endif /* !WOLFSSL_SP_MATH */
 
    if (in == NULL || r == NULL || s == NULL || key == NULL || rng == NULL) {
@@ -8280,8 +8314,13 @@ static int ecc_mont_norm_points(ecc_point* A, ecc_point* Am, ecc_point* B,
     ecc_point* Bm, mp_int* modulus, void* heap)
 {
     int err = MP_OKAY;
+
     DECL_MP_INT_SIZE_DYN(mu, mp_bitsused(modulus), MAX_ECC_BITS_USE);
 
+    if (mp_bitsused(modulus) > MAX_ECC_BITS_USE) {
+        return WC_KEY_SIZE_E;
+    }
+
     (void)heap;
 
     NEW_MP_INT_SIZE(mu, mp_bitsused(modulus), heap, DYNAMIC_TYPE_ECC);
@@ -8668,6 +8707,10 @@ int wc_ecc_verify_hash(const byte* sig, word32 siglen, const byte* hash,
     word32 keySz;
 #endif
 
+    if (ECC_KEY_MAX_BITS(key) > MAX_ECC_BITS_USE) {
+        return WC_KEY_SIZE_E;
+    }
+
     if (sig == NULL || hash == NULL || res == NULL || key == NULL) {
         return ECC_BAD_ARG_E;
     }
@@ -9038,6 +9081,7 @@ static int ecc_verify_hash(mp_int *r, mp_int *s, const byte* hash,
    ecc_point  lcl_mG;
    ecc_point  lcl_mQ;
 #endif
+
    DECL_MP_INT_SIZE_DYN(w, ECC_KEY_MAX_BITS_NONULLCHECK(key), MAX_ECC_BITS_USE);
 #if !defined(WOLFSSL_ASYNC_CRYPT) || !defined(HAVE_CAVIUM_V)
    DECL_MP_INT_SIZE_DYN(e_lcl, ECC_KEY_MAX_BITS_NONULLCHECK(key), MAX_ECC_BITS_USE);
@@ -9050,6 +9094,11 @@ static int ecc_verify_hash(mp_int *r, mp_int *s, const byte* hash,
 #endif
    mp_int*    u1 = NULL;     /* Will be e. */
    mp_int*    u2 = NULL;     /* Will be w. */
+
+   if (ECC_KEY_MAX_BITS_NONULLCHECK(key) > MAX_ECC_BITS_USE) {
+       return WC_KEY_SIZE_E;
+   }
+
 #if defined(WOLFSSL_ASYNC_CRYPT) && defined(HAVE_CAVIUM_V)
    err = wc_ecc_alloc_mpint(key, &key->e);
    if (err != 0) {

wolfcrypt/src/rsa.c

@@ -839,6 +839,10 @@ int wc_CheckRsaKey(RsaKey* key)
     }
 #endif
 
+    if (mp_bitsused(&key->n) > RSA_MAX_SIZE) {
+        return WC_KEY_SIZE_E;
+    }
+
     NEW_MP_INT_SIZE(tmp, mp_bitsused(&key->n), NULL, DYNAMIC_TYPE_RSA);
 #ifdef MP_INT_SIZE_CHECK_NULL
     if (tmp == NULL) {
@@ -2875,6 +2879,10 @@ static int RsaFunctionPrivate(mp_int* tmp, RsaKey* key, WC_RNG* rng)
     DECL_MP_INT_SIZE_DYN(rndi, mp_bitsused(&key->n), RSA_MAX_SIZE);
 #endif /* WC_RSA_BLINDING && !WC_NO_RNG */
 
+    if (mp_bitsused(&key->n) > RSA_MAX_SIZE) {
+        return WC_KEY_SIZE_E;
+    }
+
     (void)rng;
 
 #if defined(WC_RSA_BLINDING) && !defined(WC_NO_RNG)
@@ -3054,6 +3062,10 @@ static int RsaFunctionSync(const byte* in, word32 inLen, byte* out,
     DECL_MP_INT_SIZE_DYN(tmp, mp_bitsused(&key->n), RSA_MAX_SIZE);
     int    ret = 0;
 
+    if (mp_bitsused(&key->n) > RSA_MAX_SIZE) {
+        return WC_KEY_SIZE_E;
+    }
+
     (void)rng;
 
     NEW_MP_INT_SIZE(tmp, mp_bitsused(&key->n), key->heap, DYNAMIC_TYPE_RSA);
@@ -3316,7 +3328,7 @@ int wc_RsaDirect(const byte* in, word32 inLen, byte* out, word32* outSz,
     }
 
     if ((ret = wc_RsaEncryptSize(key)) < 0) {
-        return BAD_FUNC_ARG;
+        return ret;
     }
 
     if (inLen != (word32)ret) {
@@ -3481,8 +3493,13 @@ int RsaFunctionCheckIn(const byte* in, word32 inLen, RsaKey* key,
     int checkSmallCt)
 {
     int ret = 0;
+
     DECL_MP_INT_SIZE_DYN(c, mp_bitsused(&key->n), RSA_MAX_SIZE);
 
+    if (mp_bitsused(&key->n) > RSA_MAX_SIZE) {
+        return WC_KEY_SIZE_E;
+    }
+
     NEW_MP_INT_SIZE(c, mp_bitsused(&key->n), key->heap, DYNAMIC_TYPE_RSA);
 #ifdef MP_INT_SIZE_CHECK_NULL
     if (c == NULL)
@@ -4823,6 +4840,10 @@ int wc_RsaEncryptSize(const RsaKey* key)
         return BAD_FUNC_ARG;
     }
 
+    if (mp_bitsused(&key->n) > RSA_MAX_SIZE) {
+        return WC_KEY_SIZE_E;
+    }
+
     ret = mp_unsigned_bin_size(&key->n);
 
 #if defined(WOLFSSL_MICROCHIP_TA100)

wolfssl/wolfcrypt/sp_int.h

@@ -895,9 +895,11 @@ while (0)
 #define DECL_MP_INT_SIZE(name, bits)                               \
     sp_int_digit name##d[MP_INT_SIZEOF_DIGITS(MP_BITS_CNT(bits))]; \
     sp_int* (name) = (sp_int*)name##d
-/* Zero out mp_int of minimal size. */
+/* Zero out mp_int of minimal size.
+ * Use the declared digit array size, not bits, so memset cannot exceed the
+ * buffer allocated by DECL_MP_INT_SIZE_DYN(..., bits, max). */
 #define NEW_MP_INT_SIZE(name, bits, heap, type) \
-    XMEMSET(name, 0, MP_INT_SIZEOF(MP_BITS_CNT(bits)))
+    XMEMSET(name, 0, sizeof(name##d))
 /* Dispose of static mp_int. */
 #define FREE_MP_INT_SIZE(name, heap, type) WC_DO_NOTHING
 /* Type to force compiler to not complain about size. */