get derLen by RsaPublicKeyDerSize

src/ssl.c

@@ -27526,6 +27526,10 @@ WOLFSSL_RSA *wolfSSL_d2i_RSAPublicKey(WOLFSSL_RSA **r, const unsigned char **pp,
     WOLFSSL_RSA *rsa = NULL;
 
     WOLFSSL_ENTER("d2i_RSAPublicKey");
+    if(pp == NULL){
+        WOLFSSL_MSG("Bad argument");
+        return NULL;
+    }
     if((rsa = wolfSSL_RSA_new()) == NULL){
         WOLFSSL_MSG("RSA_new failed");
         return NULL;
@@ -27545,22 +27549,22 @@ WOLFSSL_RSA *wolfSSL_d2i_RSAPublicKey(WOLFSSL_RSA **r, const unsigned char **pp,
 int wolfSSL_i2d_RSAPublicKey(WOLFSSL_RSA *rsa, const unsigned char **pp)
 {
     byte *der;
-    word32 derLen = 165;
+    int derLen;
     int ret;
 
     WOLFSSL_ENTER("i2d_RSAPublicKey");
-    if(pp == NULL)
+    if((rsa == NULL) || (pp == NULL))
+        return WOLFSSL_FATAL_ERROR;
+    if((ret = SetRsaInternal(rsa)) != WOLFSSL_SUCCESS) {
+        WOLFSSL_MSG("SetRsaInternal Failed");
+        return ret;
+    }
+    if((derLen = RsaPublicKeyDerSize((RsaKey *)rsa->internal, 1)) < 0)
         return WOLFSSL_FATAL_ERROR;
     der = (byte*)XMALLOC(derLen, NULL, DYNAMIC_TYPE_TMP_BUFFER);
     if (der == NULL) {
         return WOLFSSL_FATAL_ERROR;
     }
-    if((ret = SetRsaInternal(rsa)) != WOLFSSL_SUCCESS) {
-        WOLFSSL_MSG("SetRsaInternal Failed");
-        XFREE(der, NULL, DYNAMIC_TYPE_TMP_BUFFER);
-        return ret;
-    }
-
     if((ret = wc_RsaKeyToPublicDer((RsaKey *)rsa->internal, der, derLen)) < 0){
         WOLFSSL_MSG("RsaKeyToPublicDer failed");
         XFREE(der, NULL, DYNAMIC_TYPE_TMP_BUFFER);

tests/api.c

@@ -18717,7 +18717,7 @@ static void test_wolfSSL_X509_CRL(void)
         ""};
 #endif
 
-    FILE * fp;
+    XFILE fp;
     int i;
 
     printf(testingFmt, "test_wolfSSL_X509_CRL");

wolfcrypt/src/asn.c

@@ -8326,6 +8326,85 @@ static int SetRsaPublicKey(byte* output, RsaKey* key,
 
     return idx;
 }
+
+int RsaPublicKeyDerSize(RsaKey* key, int with_header)
+{
+    byte* dummy = NULL;
+    byte seq[MAX_SEQ_SZ];
+    byte bitString[1 + MAX_LENGTH_SZ + 1];
+    int  nSz;
+    int  eSz;
+    int  seqSz;
+    int  bitStringSz;
+    int  idx;
+
+    if (key == NULL)
+        return BAD_FUNC_ARG;
+
+    /* n */
+    dummy = (byte*)XMALLOC(MAX_RSA_INT_SZ, key->heap, DYNAMIC_TYPE_TMP_BUFFER);
+    if (dummy == NULL)
+        return MEMORY_E;
+
+#ifdef HAVE_USER_RSA
+    nSz = SetASNIntRSA(key->n, dummy);
+#else
+    nSz = SetASNIntMP(&key->n, MAX_RSA_INT_SZ, dummy);
+#endif
+    XFREE(dummy, key->heap, DYNAMIC_TYPE_TMP_BUFFER);
+    if (nSz < 0) {
+        return nSz;
+    }
+
+    /* e */
+    dummy = (byte*)XMALLOC(MAX_RSA_E_SZ, key->heap, DYNAMIC_TYPE_TMP_BUFFER);
+    if (dummy == NULL) {
+        XFREE(dummy, key->heap, DYNAMIC_TYPE_TMP_BUFFER);
+        return MEMORY_E;
+    }
+
+#ifdef HAVE_USER_RSA
+    eSz = SetASNIntRSA(key->e, dummy);
+#else
+    eSz = SetASNIntMP(&key->e, MAX_RSA_INT_SZ, dummy);
+#endif
+    XFREE(dummy, key->heap, DYNAMIC_TYPE_TMP_BUFFER);
+    if (eSz < 0) {
+        return eSz;
+    }
+
+    seqSz  = SetSequence(nSz + eSz, seq);
+
+    /* headers */
+    if (with_header) {
+        int  algoSz;
+        dummy = (byte*)XMALLOC(MAX_RSA_INT_SZ, key->heap, DYNAMIC_TYPE_TMP_BUFFER);
+        if (dummy == NULL)
+            return MEMORY_E;
+
+        algoSz = SetAlgoID(RSAk, dummy, oidKeyType, 0);
+        bitStringSz  = SetBitString(seqSz + nSz + eSz, 0, bitString);
+
+        idx = SetSequence(nSz + eSz + seqSz + bitStringSz + algoSz, dummy);
+        XFREE(dummy, key->heap, DYNAMIC_TYPE_TMP_BUFFER);
+
+        /* algo */
+        idx += algoSz;
+        /* bit string */
+        idx += bitStringSz;
+    }
+    else
+        idx = 0;
+
+    /* seq */
+    idx += seqSz;
+    /* n */
+    idx += nSz;
+    /* e */
+    idx += eSz;
+
+    return idx;
+}
 #endif /* !NO_RSA && (WOLFSSL_CERT_GEN || (WOLFSSL_KEY_GEN &&
                                            !HAVE_USER_RSA))) */
 

wolfssl/wolfcrypt/asn.h

@@ -860,6 +860,7 @@ WOLFSSL_LOCAL int GetSerialNumber(const byte* input, word32* inOutIdx,
 WOLFSSL_LOCAL int GetNameHash(const byte* source, word32* idx, byte* hash,
                              int maxIdx);
 WOLFSSL_LOCAL int wc_CheckPrivateKey(byte* key, word32 keySz, DecodedCert* der);
+WOLFSSL_LOCAL int RsaPublicKeyDerSize(RsaKey* key, int with_header);
 
 #ifdef HAVE_ECC
     /* ASN sig helpers */