diff --git a/wolfcrypt/src/rsa.c b/wolfcrypt/src/rsa.c
index 1d7c85232..12c444fed 100644
--- a/wolfcrypt/src/rsa.c
+++ b/wolfcrypt/src/rsa.c
@@ -421,8 +421,16 @@ static int wc_RsaPad_OAEP(const byte* input, word32 inputLen, byte* pkcsBlock,
         return ret;
     }
 
-    /* handles check of location for idx as well as psLen */
-    if (inputLen > (pkcsBlockLen - 2 * hLen - 2)) {
+    /* handles check of location for idx as well as psLen, cast to int to check
+       for pkcsBlockLen(k) - 2 * hLen - 2 being negative
+       This check is similar to decryption where k > 2 * hLen + 2 as msg
+       size aproaches 0. In decryption if k is less than or equal -- then there
+       is no possible room for msg.
+       k = RSA key size
+       hLen = hash digest size
+     */
+    if ((int)inputLen > ((int)pkcsBlockLen - 2 * hLen - 2)) {
+        WOLFSSL_MSG("OAEP pad error, message too long or hash to big for RSA key size");
         #ifdef WOLFSSL_SMALL_STACK
             XFREE(lHash, NULL, DYNAMIC_TYPE_TMP_BUFFER);
             XFREE(seed,  NULL, DYNAMIC_TYPE_TMP_BUFFER);
diff --git a/wolfcrypt/test/test.c b/wolfcrypt/test/test.c
index 922bfda4c..fb4febe0b 100644
--- a/wolfcrypt/test/test.c
+++ b/wolfcrypt/test/test.c
@@ -4146,24 +4146,30 @@ int rsa_test(void)
     #endif /* NO_SHA256 */
 
     #ifdef WOLFSSL_SHA512
-    XMEMSET(plain, 0, sizeof(plain));
-    ret = wc_RsaPublicEncrypt_ex(in, inLen, out, sizeof(out), &key, &rng,
+    /* Check valid RSA key size is used while using hash length of SHA512
+       If key size is less than (hash length * 2) + 2 then is invalid use
+       and test, since OAEP padding requires this.
+       BAD_FUNC_ARG is returned when this case is not met */
+    if (wc_RsaEncryptSize(&key) > ((int)SHA512_DIGEST_SIZE * 2) + 2) {
+        XMEMSET(plain, 0, sizeof(plain));
+        ret = wc_RsaPublicEncrypt_ex(in, inLen, out, sizeof(out), &key, &rng,
                   WC_RSA_OAEP_PAD, WC_HASH_TYPE_SHA512, WC_MGF1SHA512, NULL, 0);
-    if (ret < 0) {
-        free(tmp);
-        return -343;
-    }
-    ret = wc_RsaPrivateDecrypt_ex(out, ret, plain, sizeof(plain), &key,
+        if (ret < 0) {
+            free(tmp);
+            return -343;
+        }
+        ret = wc_RsaPrivateDecrypt_ex(out, ret, plain, sizeof(plain), &key,
                   WC_RSA_OAEP_PAD, WC_HASH_TYPE_SHA512, WC_MGF1SHA512, NULL, 0);
-    if (ret < 0) {
-        free(tmp);
-        return -344;
+        if (ret < 0) {
+            free(tmp);
+            return -344;
+        }
+        if (XMEMCMP(plain, in, inLen)) {
+            free(tmp);
+            return -345;
+        }
     }
-    if (XMEMCMP(plain, in, inLen)) {
-        free(tmp);
-        return -345;
-    }
-    #endif /* NO_SHA */
+    #endif /* WOLFSSL_SHA512 */
 
     /* check using pkcsv15 padding with _ex API */
     XMEMSET(plain, 0, sizeof(plain));