From 4c3ddac23cf92cb2d3df7da6a78bc201ccc2742d Mon Sep 17 00:00:00 2001
From: Jacob Barthelmeh <jacob@wolfssl.com>
Date: Wed, 16 Mar 2016 14:51:25 -0600
Subject: [PATCH 1/2] catch invalid test case of RSA-OAEP and fix cast

---
 wolfcrypt/src/rsa.c   | 12 ++++++++++--
 wolfcrypt/test/test.c | 36 +++++++++++++++++++++---------------
 2 files changed, 31 insertions(+), 17 deletions(-)

diff --git a/wolfcrypt/src/rsa.c b/wolfcrypt/src/rsa.c
index 1d7c85232..12c444fed 100644
--- a/wolfcrypt/src/rsa.c
+++ b/wolfcrypt/src/rsa.c
@@ -421,8 +421,16 @@ static int wc_RsaPad_OAEP(const byte* input, word32 inputLen, byte* pkcsBlock,
         return ret;
     }
 
-    /* handles check of location for idx as well as psLen */
-    if (inputLen > (pkcsBlockLen - 2 * hLen - 2)) {
+    /* handles check of location for idx as well as psLen, cast to int to check
+       for pkcsBlockLen(k) - 2 * hLen - 2 being negative
+       This check is similar to decryption where k > 2 * hLen + 2 as msg
+       size aproaches 0. In decryption if k is less than or equal -- then there
+       is no possible room for msg.
+       k = RSA key size
+       hLen = hash digest size
+     */
+    if ((int)inputLen > ((int)pkcsBlockLen - 2 * hLen - 2)) {
+        WOLFSSL_MSG("OAEP pad error, message too long or hash to big for RSA key size");
         #ifdef WOLFSSL_SMALL_STACK
             XFREE(lHash, NULL, DYNAMIC_TYPE_TMP_BUFFER);
             XFREE(seed,  NULL, DYNAMIC_TYPE_TMP_BUFFER);
diff --git a/wolfcrypt/test/test.c b/wolfcrypt/test/test.c
index 922bfda4c..fb4febe0b 100644
--- a/wolfcrypt/test/test.c
+++ b/wolfcrypt/test/test.c
@@ -4146,24 +4146,30 @@ int rsa_test(void)
     #endif /* NO_SHA256 */
 
     #ifdef WOLFSSL_SHA512
-    XMEMSET(plain, 0, sizeof(plain));
-    ret = wc_RsaPublicEncrypt_ex(in, inLen, out, sizeof(out), &key, &rng,
+    /* Check valid RSA key size is used while using hash length of SHA512
+       If key size is less than (hash length * 2) + 2 then is invalid use
+       and test, since OAEP padding requires this.
+       BAD_FUNC_ARG is returned when this case is not met */
+    if (wc_RsaEncryptSize(&key) > ((int)SHA512_DIGEST_SIZE * 2) + 2) {
+        XMEMSET(plain, 0, sizeof(plain));
+        ret = wc_RsaPublicEncrypt_ex(in, inLen, out, sizeof(out), &key, &rng,
                   WC_RSA_OAEP_PAD, WC_HASH_TYPE_SHA512, WC_MGF1SHA512, NULL, 0);
-    if (ret < 0) {
-        free(tmp);
-        return -343;
-    }
-    ret = wc_RsaPrivateDecrypt_ex(out, ret, plain, sizeof(plain), &key,
+        if (ret < 0) {
+            free(tmp);
+            return -343;
+        }
+        ret = wc_RsaPrivateDecrypt_ex(out, ret, plain, sizeof(plain), &key,
                   WC_RSA_OAEP_PAD, WC_HASH_TYPE_SHA512, WC_MGF1SHA512, NULL, 0);
-    if (ret < 0) {
-        free(tmp);
-        return -344;
+        if (ret < 0) {
+            free(tmp);
+            return -344;
+        }
+        if (XMEMCMP(plain, in, inLen)) {
+            free(tmp);
+            return -345;
+        }
     }
-    if (XMEMCMP(plain, in, inLen)) {
-        free(tmp);
-        return -345;
-    }
-    #endif /* NO_SHA */
+    #endif /* WOLFSSL_SHA512 */
 
     /* check using pkcsv15 padding with _ex API */
     XMEMSET(plain, 0, sizeof(plain));

From 2dd5efd96972c7ed0167907664130814ca9258b3 Mon Sep 17 00:00:00 2001
From: Jacob Barthelmeh <jacob@wolfssl.com>
Date: Wed, 16 Mar 2016 15:25:52 -0600
Subject: [PATCH 2/2] sanity check for RSA key size and hash digest size

---
 wolfcrypt/src/rsa.c | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/wolfcrypt/src/rsa.c b/wolfcrypt/src/rsa.c
index 12c444fed..c1cd2f7a7 100644
--- a/wolfcrypt/src/rsa.c
+++ b/wolfcrypt/src/rsa.c
@@ -427,10 +427,19 @@ static int wc_RsaPad_OAEP(const byte* input, word32 inputLen, byte* pkcsBlock,
        size aproaches 0. In decryption if k is less than or equal -- then there
        is no possible room for msg.
        k = RSA key size
-       hLen = hash digest size
+       hLen = hash digest size -- will always be >= 0 at this point
      */
-    if ((int)inputLen > ((int)pkcsBlockLen - 2 * hLen - 2)) {
-        WOLFSSL_MSG("OAEP pad error, message too long or hash to big for RSA key size");
+    if ((word32)(2 * hLen + 2) > pkcsBlockLen) {
+        WOLFSSL_MSG("OAEP pad error hash to big for RSA key size");
+        #ifdef WOLFSSL_SMALL_STACK
+            XFREE(lHash, NULL, DYNAMIC_TYPE_TMP_BUFFER);
+            XFREE(seed,  NULL, DYNAMIC_TYPE_TMP_BUFFER);
+        #endif
+        return BAD_FUNC_ARG;
+    }
+
+    if (inputLen > (pkcsBlockLen - 2 * hLen - 2)) {
+        WOLFSSL_MSG("OAEP pad error message too long");
         #ifdef WOLFSSL_SMALL_STACK
             XFREE(lHash, NULL, DYNAMIC_TYPE_TMP_BUFFER);
             XFREE(seed,  NULL, DYNAMIC_TYPE_TMP_BUFFER);