From c33ae4c24538662162a48092e17aa30ccaceb994 Mon Sep 17 00:00:00 2001
From: Hayden Roche <hayden@wolfssl.com>
Date: Fri, 18 Feb 2022 17:04:06 -0800
Subject: [PATCH] Improve wolfSSL_i2d_X509_NAME and
 wolfSSL_i2d_X509_NAME_canon.

Like other i2d functions, these functions should be able to take a NULL output
parameter and return the necessary output buffer size. This commit adds this
ability. This commit also removes some redundant code in wolfSSL_i2d_X509_NAME.
---
 src/ssl.c   | 37 ++++++++++++++++++-------------------
 tests/api.c |  5 +++++
 2 files changed, 23 insertions(+), 19 deletions(-)

diff --git a/src/ssl.c b/src/ssl.c
index e22f1130f..88118f6f0 100644
--- a/src/ssl.c
+++ b/src/ssl.c
@@ -44169,9 +44169,9 @@ static int wolfSSL_ASN1_STRING_canon(WOLFSSL_ASN1_STRING* asn_out,
 /*   convert to UTF8                                                        */
 /*   convert to lower case                                                  */
 /*   multi-spaces collapsed                                                 */
-/*   leading SEQUENCE hader is skipped                                      */
+/*   leading SEQUENCE header is skipped                                     */
 /* @param  name a pointer to X509_NAME that is to be converted              */
-/* @param  out  a pointer to conveted data                                  */
+/* @param  out  a pointer to converted data                                 */
 /* @return a number of converted bytes, otherwise <=0 error code            */
 int wolfSSL_i2d_X509_NAME_canon(WOLFSSL_X509_NAME* name, unsigned char** out)
 {
@@ -44183,7 +44183,7 @@ int wolfSSL_i2d_X509_NAME_canon(WOLFSSL_X509_NAME* name, unsigned char** out)
     EncodedName  names[MAX_NAME_ENTRIES];
 #endif
 
-    if (out == NULL || name == NULL)
+    if (name == NULL)
         return BAD_FUNC_ARG;
 
 #ifdef WOLFSSL_SMALL_STACK
@@ -44243,6 +44243,14 @@ int wolfSSL_i2d_X509_NAME_canon(WOLFSSL_X509_NAME* name, unsigned char** out)
         }
     }
 
+    if (out == NULL) {
+        /* If out is NULL, caller just wants length. */
+#ifdef WOLFSSL_SMALL_STACK
+        XFREE(names, NULL, DYNAMIC_TYPE_TMP_BUFFER);
+#endif
+        return totalBytes;
+    }
+
     /* skip header */
     /* check if using buffer passed in */
     if (*out == NULL) {
@@ -44291,7 +44299,8 @@ int wolfSSL_i2d_X509_NAME_canon(WOLFSSL_X509_NAME* name, unsigned char** out)
  *
  * out  pointer to either a pre setup buffer or a pointer to null for
  *      creating a dynamic buffer. In the case that a pre-existing buffer is
- *      used out will be incremented the size of the DER buffer on success.
+ *      used out will be incremented the size of the DER buffer on success. If
+ *      out is NULL, the function returns the necessary output buffer length.
  *
  * returns the size of the buffer on success, or negative value with failure
  */
@@ -44306,7 +44315,7 @@ int wolfSSL_i2d_X509_NAME(WOLFSSL_X509_NAME* name, unsigned char** out)
     EncodedName  names[MAX_NAME_ENTRIES];
 #endif
 
-    if (out == NULL || name == NULL)
+    if (name == NULL)
         return BAD_FUNC_ARG;
 
 #ifdef WOLFSSL_SMALL_STACK
@@ -44374,23 +44383,13 @@ int wolfSSL_i2d_X509_NAME(WOLFSSL_X509_NAME* name, unsigned char** out)
         return BUFFER_E;
     }
 
-    /* check if using buffer passed in */
-    if (*out == NULL) {
-        *out = local = (unsigned char*)XMALLOC(totalBytes + idx, NULL,
-                DYNAMIC_TYPE_OPENSSL);
-        if (*out == NULL) {
-            return MEMORY_E;
-        }
-    }
-
-    /* header */
-    idx = SetSequence(totalBytes, temp);
-    if (totalBytes + idx > ASN_NAME_MAX) {
+    if (out == NULL) {
+        /* If out is NULL, caller just wants length. */
+        totalBytes += idx;
 #ifdef WOLFSSL_SMALL_STACK
         XFREE(names, NULL, DYNAMIC_TYPE_TMP_BUFFER);
 #endif
-        WOLFSSL_MSG("Total Bytes is greater than ASN_NAME_MAX");
-        return BUFFER_E;
+        return totalBytes;
     }
 
     /* check if using buffer passed in */
diff --git a/tests/api.c b/tests/api.c
index bfeb66441..2da8085a7 100644
--- a/tests/api.c
+++ b/tests/api.c
@@ -29330,6 +29330,8 @@ static void test_wolfSSL_X509_NAME(void)
     AssertNotNull(d2i_name = d2i_X509_NAME(NULL, &tmp, sz));
 #endif
 
+    /* if output parameter is NULL, should still return required size. */
+    AssertIntGT((sz = i2d_X509_NAME((X509_NAME*)b, NULL)), 0);
     /* retry but with the function creating a buffer */
     tmp = NULL;
     AssertIntGT((sz = i2d_X509_NAME((X509_NAME*)b, &tmp)), 0);
@@ -32356,6 +32358,9 @@ static void test_wolfSSL_X509_Name_canon(void)
     AssertNotNull(x509 = PEM_read_X509(file, NULL, NULL, NULL));
     AssertNotNull(name = X509_get_issuer_name(x509));
 
+    /* When output buffer is NULL, should return necessary output buffer
+     * length.*/
+    AssertIntGT(wolfSSL_i2d_X509_NAME_canon(name, NULL), 0);
     AssertIntGT((len = wolfSSL_i2d_X509_NAME_canon(name, &pbuf)), 0);
     AssertIntEQ(wc_ShaHash((const byte*)pbuf, (word32)len, digest), 0);