Refactor wolfSSL_RSA_To_Der

src/ssl.c

@@ -32667,45 +32667,67 @@ static int EncryptDerKey(byte *der, int *derSz, const EVP_CIPHER* cipher,
 #if defined(WOLFSSL_KEY_GEN) && !defined(NO_RSA) && !defined(HAVE_USER_RSA)
 static int wolfSSL_RSA_To_Der(WOLFSSL_RSA* rsa, byte** outBuf, int publicKey)
 {
-    int derMax = 0;
     int derSz  = 0;
+    int ret;
     byte* derBuf;
 
     WOLFSSL_ENTER("wolfSSL_RSA_To_Der");
 
-    if (!rsa || !outBuf || (publicKey != 0 && publicKey != 1)) {
+    if (!rsa || (publicKey != 0 && publicKey != 1)) {
         WOLFSSL_LEAVE("wolfSSL_RSA_To_Der", BAD_FUNC_ARG);
         return BAD_FUNC_ARG;
     }
-    /* 5 > size of n, d, p, q, d%(p-1), d(q-1), 1/q%p, e + ASN.1 additional
+
-        *  informations
+    if (rsa->inSet == 0) {
-        */
+        if ((ret = SetRsaInternal(rsa)) != WOLFSSL_SUCCESS) {
-    derMax = 5 * wolfSSL_RSA_size(rsa) + AES_BLOCK_SIZE;
+            WOLFSSL_MSG("SetRsaInternal() Failed");
-    derBuf = (byte*)XMALLOC(derMax, NULL, DYNAMIC_TYPE_TMP_BUFFER);
+            WOLFSSL_LEAVE("wolfSSL_RSA_To_Der", ret);
-    if (derBuf == NULL) {
+            return ret;
-        WOLFSSL_MSG("malloc failed");
+        }
-        WOLFSSL_LEAVE("wolfSSL_RSA_To_Der", MEMORY_ERROR);
-        return MEMORY_ERROR;
     }
-    /* Key to DER */
+
     if (publicKey) {
-        derSz = wc_RsaKeyToPublicDer((RsaKey*)rsa->internal, derBuf, derMax);
+        if ((derSz = wc_RsaPublicKeyDerSize((RsaKey *)rsa->internal, 1)) < 0) {
+            WOLFSSL_MSG("wc_RsaPublicKeyDerSize failed");
+            WOLFSSL_LEAVE("wolfSSL_RSA_To_Der", derSz);
+            return derSz;
+        }
+    }
+    else {
+        if ((derSz = wc_RsaKeyToDer((RsaKey*)rsa->internal, NULL, 0)) < 0) {
+            WOLFSSL_MSG("wc_RsaKeyToDer failed");
+            WOLFSSL_LEAVE("wolfSSL_RSA_To_Der", derSz);
+            return derSz;
+        }
+    }
+
+    if (outBuf) {
+        if (!(derBuf = (byte*)XMALLOC(derSz, NULL, DYNAMIC_TYPE_TMP_BUFFER))) {
+            WOLFSSL_MSG("malloc failed");
+            WOLFSSL_LEAVE("wolfSSL_RSA_To_Der", MEMORY_ERROR);
+            return MEMORY_ERROR;
+        }
+
+        /* Key to DER */
+        if (publicKey) {
+            derSz = wc_RsaKeyToPublicDer((RsaKey*)rsa->internal, derBuf, derSz);
+        }
+        else {
+            derSz = wc_RsaKeyToDer((RsaKey*)rsa->internal, derBuf, derSz);
+        }
+
         if (derSz < 0) {
             WOLFSSL_MSG("wc_RsaKeyToPublicDer failed");
             XFREE(derBuf, NULL, DYNAMIC_TYPE_TMP_BUFFER);
         }
         else {
-            *outBuf = derBuf;
+            if (*outBuf) {
-        }
+                XMEMCPY(*outBuf, derBuf, derSz);
-    }
+                XFREE(derBuf, NULL, DYNAMIC_TYPE_TMP_BUFFER);
-    else {
+            }
-        derSz = wc_RsaKeyToDer((RsaKey*)rsa->internal, derBuf, derMax);
+            else {
-        if (derSz < 0) {
+                *outBuf = derBuf;
-            WOLFSSL_MSG("wc_RsaKeyToDer failed");
+            }
-            XFREE(derBuf, NULL, DYNAMIC_TYPE_TMP_BUFFER);
-        }
-        else {
-            *outBuf = derBuf;
         }
     }
 
@@ -32755,7 +32777,7 @@ int wolfSSL_PEM_write_bio_RSAPrivateKey(WOLFSSL_BIO* bio, WOLFSSL_RSA* key,
     /* similar to how wolfSSL_PEM_write_mem_RSAPrivateKey finds DER of key */
     {
         int derSz;
-        byte* derBuf;
+        byte* derBuf = NULL;
 
         if ((derSz = wolfSSL_RSA_To_Der(key, &derBuf, 0)) < 0) {
             WOLFSSL_MSG("wolfSSL_RSA_To_Der failed");
@@ -33011,7 +33033,7 @@ int wolfSSL_PEM_write_mem_RSAPrivateKey(RSA* rsa, const EVP_CIPHER* cipher,
                                         unsigned char* passwd, int passwdSz,
                                         unsigned char **pem, int *plen)
 {
-    byte *derBuf, *tmp, *cipherInfo = NULL;
+    byte *derBuf = NULL, *tmp, *cipherInfo = NULL;
     int  derSz = 0;
     const int type = PRIVATEKEY_TYPE;
     const char* header = NULL;
@@ -36579,7 +36601,8 @@ WOLFSSL_RSA *wolfSSL_d2i_RSAPrivateKey(WOLFSSL_RSA **r,
     return rsa;
 }
 
-#if !defined(HAVE_FAST_RSA)
+#if !defined(HAVE_FAST_RSA) && defined(WOLFSSL_KEY_GEN) && \
+    !defined(NO_RSA) && !defined(HAVE_USER_RSA)
 /* Converts an internal RSA structure to DER format.
  * If "pp" is null then buffer size only is returned.
  * If "*pp" is null then a created buffer is set in *pp and the caller is
@@ -36588,10 +36611,7 @@ WOLFSSL_RSA *wolfSSL_d2i_RSAPrivateKey(WOLFSSL_RSA **r,
  */
 int wolfSSL_i2d_RSAPrivateKey(WOLFSSL_RSA *rsa, unsigned char **pp)
 {
-#if defined(WOLFSSL_KEY_GEN) && !defined(NO_RSA) && !defined(HAVE_USER_RSA)
-    byte* der = NULL;
     int ret;
-    int i;
 
     WOLFSSL_ENTER("wolfSSL_i2d_RSAPrivateKey");
 
@@ -36601,82 +36621,34 @@ int wolfSSL_i2d_RSAPrivateKey(WOLFSSL_RSA *rsa, unsigned char **pp)
         return BAD_FUNC_ARG;
     }
 
-    if (rsa->inSet == 0) {
+    if ((ret = wolfSSL_RSA_To_Der(rsa, pp, 0)) < 0) {
-        if ((ret = SetRsaInternal(rsa)) != WOLFSSL_SUCCESS) {
-            WOLFSSL_MSG("SetRsaInternal() Failed");
-            return ret;
-        }
-    }
-
-    if ((ret = wolfSSL_RSA_To_Der(rsa, &der, 0)) < 0) {
         WOLFSSL_MSG("wolfSSL_RSA_To_Der failed");
         return WOLFSSL_FAILURE;
     }
 
-    if (pp != NULL) {
-        if (*pp == NULL) {
-            /* create buffer and return it */
-            *pp = (unsigned char*)XMALLOC(ret, NULL, DYNAMIC_TYPE_OPENSSL);
-            if (*pp == NULL) {
-                return WOLFSSL_FATAL_ERROR;
-            }
-            XMEMCPY(*pp, der, ret);
-        }
-        else {
-            /* ret is the size of the DER buffer */
-            for (i = 0; i < ret; i++) {
-                *(*pp + i) = *(der + i);
-            }
-            *pp += ret;
-        }
-    }
-    XFREE(der, NULL, DYNAMIC_TYPE_TMP_BUFFER);
     return ret; /* returns size of DER if successful */
-#else
-    (void)rsa;
-    (void)pp;
-    WOLFSSL_MSG("Error, wolfSSL_i2d_RSAPrivateKey missing defines");
-    return WOLFSSL_FAILURE;
-#endif /* WOLFSSL_KEY_GEN */
 }
 
 
 int wolfSSL_i2d_RSAPublicKey(WOLFSSL_RSA *rsa, const unsigned char **pp)
 {
-    byte *der;
-    int derLen;
     int ret;
 
-    WOLFSSL_ENTER("i2d_RSAPublicKey");
+    /* check for bad functions arguments */
-    if (rsa == NULL)
+    if (rsa == NULL) {
-        return WOLFSSL_FATAL_ERROR;
+        WOLFSSL_MSG("Bad Function Arguments");
+        return BAD_FUNC_ARG;
+    }
 
-    if (rsa->inSet == 0) {
+    if ((ret = wolfSSL_RSA_To_Der(rsa, (byte**)pp, 1)) < 0) {
-        if ((ret = SetRsaInternal(rsa)) != WOLFSSL_SUCCESS) {
+        WOLFSSL_MSG("wolfSSL_RSA_To_Der failed");
-            WOLFSSL_MSG("SetRsaInternal Failed");
+        return WOLFSSL_FAILURE;
-            return ret;
-        }
     }
-    if ((derLen = wc_RsaPublicKeyDerSize((RsaKey *)rsa->internal, 1)) < 0)
-        return WOLFSSL_FATAL_ERROR;
-    der = (byte*)XMALLOC(derLen, NULL, DYNAMIC_TYPE_TMP_BUFFER);
-    if (der == NULL) {
-        return WOLFSSL_FATAL_ERROR;
-    }
-    if ((ret = wc_RsaKeyToPublicDer((RsaKey *)rsa->internal, der, derLen)) < 0){
-        WOLFSSL_MSG("RsaKeyToPublicDer failed");
-        if(der != NULL)
-            XFREE(der, NULL, DYNAMIC_TYPE_TMP_BUFFER);
-        return ret;
-    }
-    if ((pp != NULL) && (ret >= 0))
-        *pp = der;
-    else
-        XFREE(der, NULL, DYNAMIC_TYPE_TMP_BUFFER);
 
     return ret;
 }
-#endif /* #if !defined(HAVE_FAST_RSA) */
+#endif /* !defined(HAVE_FAST_RSA) && defined(WOLFSSL_KEY_GEN) && \
+        * !defined(NO_RSA) && !defined(HAVE_USER_RSA) */
 
 #endif /* !NO_RSA */
 #endif /* OPENSSL_EXTRA */

tests/api.c

@@ -12291,7 +12291,7 @@ static int test_wc_RsaKeyToDer (void)
             if (ret == BAD_FUNC_ARG) {
                 ret = wc_RsaKeyToDer(&genKey, NULL, FOURK_BUF);
             }
-            if (ret == BAD_FUNC_ARG) {
+            if (ret > 0) {
                 /* Try Public Key. */
                 genKey.type = 0;
                 ret = wc_RsaKeyToDer(&genKey, der, FOURK_BUF);
@@ -24202,8 +24202,7 @@ static void test_wolfSSL_d2i_PrivateKeys_bio(void)
 
 #if defined(WOLFSSL_KEY_GEN)
     unsigned char buff[4096];
-    unsigned char* bufPtr;
+    unsigned char* bufPtr = buff;
-    bufPtr = buff;
 #endif
 
     printf(testingFmt, "wolfSSL_d2i_PrivateKeys_bio()");
@@ -24432,11 +24431,12 @@ static void test_wolfSSL_RSA(void)
 
 static void test_wolfSSL_RSA_DER(void)
 {
-#if defined(OPENSSL_EXTRA) && !defined(NO_RSA) && !defined(HAVE_FAST_RSA)
+#if !defined(HAVE_FAST_RSA) && defined(WOLFSSL_KEY_GEN) && \
+    !defined(NO_RSA) && !defined(HAVE_USER_RSA) && defined(OPENSSL_EXTRA)
 
     RSA *rsa;
     int i;
-    const unsigned char *buff;
+    const unsigned char *buff = NULL;
 
     struct tbl_s
     {

wolfcrypt/src/asn.c

@@ -11089,7 +11089,7 @@ int wc_RsaKeyToDer(RsaKey* key, byte* output, word32 inLen)
     byte  ver[MAX_VERSION_SZ];
     byte* tmps[RSA_INTS];
 
-    if (!key || !output)
+    if (!key)
         return BAD_FUNC_ARG;
 
     if (key->type != RSA_PRIVATE)
@@ -11128,20 +11128,22 @@ int wc_RsaKeyToDer(RsaKey* key, byte* output, word32 inLen)
     seqSz = SetSequence(verSz + intTotalLen, seq);
 
     outLen = seqSz + verSz + intTotalLen;
-    if (outLen > (int)inLen) {
+    if (output) {
-        FreeTmpRsas(tmps, key->heap);
+        if (outLen > (int)inLen) {
-        return BAD_FUNC_ARG;
+            FreeTmpRsas(tmps, key->heap);
-    }
+            return BAD_FUNC_ARG;
+        }
 
-    /* write to output */
+        /* write to output */
-    XMEMCPY(output, seq, seqSz);
+        XMEMCPY(output, seq, seqSz);
-    j = seqSz;
+        j = seqSz;
-    XMEMCPY(output + j, ver, verSz);
+        XMEMCPY(output + j, ver, verSz);
-    j += verSz;
+        j += verSz;
 
-    for (i = 0; i < RSA_INTS; i++) {
+        for (i = 0; i < RSA_INTS; i++) {
-        XMEMCPY(output + j, tmps[i], sizes[i]);
+            XMEMCPY(output + j, tmps[i], sizes[i]);
-        j += sizes[i];
+            j += sizes[i];
+        }
     }
     FreeTmpRsas(tmps, key->heap);
 

wolfcrypt/src/evp.c

@@ -1247,7 +1247,8 @@ int wolfSSL_EVP_PKEY_keygen(WOLFSSL_EVP_PKEY_CTX *ctx,
     }
 
     switch (pkey->type) {
-#if !defined(NO_RSA) && !defined(HAVE_USER_RSA)
+#if !defined(HAVE_FAST_RSA) && defined(WOLFSSL_KEY_GEN) && \
+    !defined(NO_RSA) && !defined(HAVE_USER_RSA)
         case EVP_PKEY_RSA:
             pkey->rsa = wolfSSL_RSA_generate_key(ctx->nbits, WC_RSA_EXPONENT,
                 NULL, NULL);

wolfcrypt/user-crypto/src/rsa.c

@@ -2663,7 +2663,7 @@ int wc_RsaKeyToDer(RsaKey* key, byte* output, word32 inLen)
 
     USER_DEBUG(("Entering RsaKeyToDer\n"));
 
-    if (!key || !output)
+    if (!key)
         return USER_CRYPTO_ERROR;
 
     if (key->type != RSA_PRIVATE)
@@ -2739,19 +2739,21 @@ int wc_RsaKeyToDer(RsaKey* key, byte* output, word32 inLen)
     seqSz = SetSequence(verSz + intTotalLen, seq);
 
     outLen = seqSz + verSz + intTotalLen;
-    if (outLen > (int)inLen) {
+    if (output) {
-        return USER_CRYPTO_ERROR;
+        if (outLen > (int)inLen) {
-    }
+            return USER_CRYPTO_ERROR;
+        }
 
-    /* write to output */
+        /* write to output */
-    XMEMCPY(output, seq, seqSz);
+        XMEMCPY(output, seq, seqSz);
-    j = seqSz;
+        j = seqSz;
-    XMEMCPY(output + j, ver, verSz);
+        XMEMCPY(output + j, ver, verSz);
-    j += verSz;
+        j += verSz;
 
-    for (i = 0; i < RSA_INTS; i++) {
+        for (i = 0; i < RSA_INTS; i++) {
-        XMEMCPY(output + j, tmps[i], sizes[i]);
+            XMEMCPY(output + j, tmps[i], sizes[i]);
-        j += sizes[i];
+            j += sizes[i];
+        }
     }
     FreeTmpRsas(tmps, key->heap);
 

wolfssl/openssl/crypto.h

@@ -46,6 +46,7 @@ WOLFSSL_API void *wolfSSL_OPENSSL_malloc(size_t a);
 
 #define SSLeay_version wolfSSLeay_version
 #define SSLeay wolfSSLeay
+#define OpenSSL_version_num wolfSSL_OpenSSL_version_num
 
 #ifdef WOLFSSL_QT
     #define SSLEAY_VERSION 0x10001000L
@@ -85,8 +86,6 @@ typedef void (CRYPTO_free_func)(void*parent, void*ptr, CRYPTO_EX_DATA *ad, int i
 #define CRYPTO_THREAD_r_lock wc_LockMutex
 #define CRYPTO_THREAD_unlock wc_UnLockMutex
 
-#define OpenSSL_version_num wolfSSL_OpenSSL_version_num
-
 #endif /* OPENSSL_ALL || HAVE_STUNNEL || WOLFSSL_NGINX || WOLFSSL_HAPROXY */
 
 #endif /* header */