Fixes and additional tests for compatibility function BN_bn2hex. In the DEBUG_WOLFSSL case it was returning a (char*)"", which was trying to be free'd. We cannot return const char* here, since its assumed to be an allocated pointer. Fix the dynamic type for XMALLOC/XFREE to match, since OPENSSL_free is used to free returned value. Fix to add room for null term. Added missing API unit test for BN_print_fp. Exposed these functions for OPENSSL_EXTRA.

src/ssl.c

@@ -23100,7 +23100,6 @@ WOLFSSL_BN_ULONG wolfSSL_BN_mod_word(const WOLFSSL_BIGNUM *bn,
 
 char *wolfSSL_BN_bn2hex(const WOLFSSL_BIGNUM *bn)
 {
-#if defined(WOLFSSL_KEY_GEN) || defined(HAVE_COMP_KEY) || defined(DEBUG_WOLFSSL)
     int len = 0;
     char *buf;
 
@@ -23115,24 +23114,20 @@ char *wolfSSL_BN_bn2hex(const WOLFSSL_BIGNUM *bn)
         WOLFSSL_MSG("mp_radix_size failure");
         return NULL;
     }
+    len += 1; /* add one for null terminator */
 
-    buf = (char*) XMALLOC(len, NULL, DYNAMIC_TYPE_ECC);
+    buf = (char*)XMALLOC(len, NULL, DYNAMIC_TYPE_OPENSSL);
     if (buf == NULL) {
         WOLFSSL_MSG("BN_bn2hex malloc buffer failure");
         return NULL;
     }
 
     if (mp_tohex((mp_int*)bn->internal, buf) != MP_OKAY) {
-        XFREE(buf, NULL, DYNAMIC_TYPE_ECC);
+        XFREE(buf, NULL, DYNAMIC_TYPE_OPENSSL);
         return NULL;
     }
 
     return buf;
-#else
-    (void)bn;
-    WOLFSSL_MSG("wolfSSL_BN_bn2hex not compiled in");
-    return (char*)"";
-#endif
 }
 
 #ifndef NO_FILESYSTEM
@@ -23141,7 +23136,6 @@ char *wolfSSL_BN_bn2hex(const WOLFSSL_BIGNUM *bn)
  */
 int wolfSSL_BN_print_fp(XFILE fp, const WOLFSSL_BIGNUM *bn)
 {
-#if defined(WOLFSSL_KEY_GEN) || defined(HAVE_COMP_KEY) || defined(DEBUG_WOLFSSL)
     char *buf;
 
     WOLFSSL_ENTER("wolfSSL_BN_print_fp");
@@ -23158,17 +23152,9 @@ int wolfSSL_BN_print_fp(XFILE fp, const WOLFSSL_BIGNUM *bn)
     }
 
     fprintf(fp, "%s", buf);
-    XFREE(buf, NULL, DYNAMIC_TYPE_ECC);
+    XFREE(buf, NULL, DYNAMIC_TYPE_OPENSSL);
 
     return WOLFSSL_SUCCESS;
-#else
-    (void)fp;
-    (void)bn;
-
-    WOLFSSL_MSG("wolfSSL_BN_print_fp not compiled in");
-
-    return WOLFSSL_SUCCESS;
-#endif
 }
 #endif /* !NO_FILESYSTEM */
 

tests/api.c

@@ -1420,9 +1420,8 @@ static void test_wolfSSL_EC(void)
     EC_POINT *Gxy, *new_point;
     BIGNUM *k = NULL, *Gx = NULL, *Gy = NULL, *Gz = NULL;
     BIGNUM *X, *Y;
-#if defined(WOLFSSL_KEY_GEN) || defined(HAVE_COMP_KEY) || defined(DEBUG_WOLFSSL)
     char* hexStr;
-#endif
+
     const char* kTest = "F4F8338AFCC562C5C3F3E1E46A7EFECD17AF381913FF7A96314EA47055EA0FD0";
     /* NISTP256R1 Gx/Gy */
     const char* kGx   = "6B17D1F2E12C4247F8BCE6E563A440F277037D812DEB33A0F4A13945D898C296";
@@ -1459,19 +1458,29 @@ static void test_wolfSSL_EC(void)
     AssertIntEQ(BN_is_zero(X), WOLFSSL_FAILURE);
 
     /* check bx2hex */
-#if defined(WOLFSSL_KEY_GEN) || defined(HAVE_COMP_KEY) || defined(DEBUG_WOLFSSL)
     hexStr = BN_bn2hex(k);
     AssertStrEQ(hexStr, kTest);
+#ifndef NO_FILESYSTEM
+    BN_print_fp(stdout, k);
+    printf("\n");
+#endif
     XFREE(hexStr, NULL, DYNAMIC_TYPE_ECC);
 
     hexStr = BN_bn2hex(Gx);
     AssertStrEQ(hexStr, kGx);
+#ifndef NO_FILESYSTEM
+    BN_print_fp(stdout, Gx);
+    printf("\n");
+#endif
     XFREE(hexStr, NULL, DYNAMIC_TYPE_ECC);
 
     hexStr = BN_bn2hex(Gy);
     AssertStrEQ(hexStr, kGy);
-    XFREE(hexStr, NULL, DYNAMIC_TYPE_ECC);
+#ifndef NO_FILESYSTEM
+    BN_print_fp(stdout, Gy);
+    printf("\n");
 #endif
+    XFREE(hexStr, NULL, DYNAMIC_TYPE_ECC);
 
     /* cleanup */
     BN_free(X);
@@ -20132,7 +20141,7 @@ static void test_wolfSSL_X509_get_serialNumber(void)
     ASN1_INTEGER* a;
     BIGNUM* bn;
     X509*   x509;
-
+    char *serialHex;
 
     printf(testingFmt, "wolfSSL_X509_get_serialNumber()");
 
@@ -20143,6 +20152,11 @@ static void test_wolfSSL_X509_get_serialNumber(void)
 
     /* check on value of ASN1 Integer */
     AssertNotNull(bn = ASN1_INTEGER_to_BN(a, NULL));
+
+    AssertNotNull(serialHex = BN_bn2hex(bn));
+    AssertStrEQ(serialHex, "1");
+    OPENSSL_free(serialHex);
+
     AssertIntEQ(BN_get_word(bn), 1);
 
     BN_free(bn);