From 3fbaccc8a11628e42465ca3f79cba136344e5131 Mon Sep 17 00:00:00 2001
From: David Garske <david@wolfssl.com>
Date: Thu, 20 Aug 2020 15:33:28 -0700
Subject: [PATCH] Fix for API unit test `test_wolfSSL_X509_sign`, which can
 have a varying length depending on if MSB is set. About 1 in 200 tests would
 fail.

---
 tests/api.c | 47 ++++++++++++++++++++++++-----------------------
 1 file changed, 24 insertions(+), 23 deletions(-)

diff --git a/tests/api.c b/tests/api.c
index d9c84548b..774a33296 100644
--- a/tests/api.c
+++ b/tests/api.c
@@ -27673,17 +27673,19 @@ static void test_wolfSSL_X509_sign(void)
 #endif
 #endif /* WOLFSSL_ALT_NAMES */
 
-    /* Test invalid parameters */
-    AssertIntEQ(X509_sign(NULL, priv, EVP_sha256()), 0);
-    AssertIntEQ(X509_sign(x509, NULL, EVP_sha256()), 0);
-    AssertIntEQ(X509_sign(x509, priv, NULL), 0);
-
+    /* test valid sign case */
     ret = X509_sign(x509, priv, EVP_sha256());
 
+#if defined(OPENSSL_ALL) && defined(WOLFSSL_ALT_NAMES)
+    AssertIntEQ(X509_get_ext_count(x509), 1);
+#endif
 #if defined(WOLFSSL_ALT_NAMES) && (defined(OPENSSL_ALL) || defined(WOLFSSL_IP_ALT_NAME))
     AssertIntEQ(wolfSSL_X509_check_ip_asc(x509, "127.0.0.1", 0), 1);
 #endif
 
+    AssertIntEQ(wolfSSL_X509_get_serial_number(x509, sn, &snSz),
+                WOLFSSL_SUCCESS);
+
 #if 0
     /* example for writing to file */
     XFILE tmpFile = XFOPEN("./signed.der", "wb");
@@ -27694,12 +27696,26 @@ static void test_wolfSSL_X509_sign(void)
     }
     XFCLOSE(tmpFile);
 #endif
+    
+    /* Variation in size depends on ASN.1 encoding when MSB is set */
+#ifndef WOLFSSL_ALT_NAMES
+    /* Valid case - size should be 798-797 with 16 byte serial number */
+    AssertTrue((ret == 781 + snSz) || (ret == 782 + snSz));
+#elif defined(OPENSSL_ALL) || defined(WOLFSSL_IP_ALT_NAME)
+    /* Valid case - size should be 935-936 with 16 byte serial number */
+    AssertTrue((ret == 919 + snSz) || (ret == 920 + snSz));
+#else
+    /* Valid case - size should be 926-927 with 16 byte serial number */
+    AssertTrue((ret == 910 + snSz) || (ret == 911 + snSz));
+#endif
+
+    /* Test invalid parameters */
+    AssertIntEQ(X509_sign(NULL, priv, EVP_sha256()), 0);
+    AssertIntEQ(X509_sign(x509, NULL, EVP_sha256()), 0);
+    AssertIntEQ(X509_sign(x509, priv, NULL), 0);
 
     /* test invalid version number */
 #if defined(OPENSSL_ALL)
-    #ifdef WOLFSSL_ALT_NAMES
-    AssertIntEQ(X509_get_ext_count(x509), 1);
-    #endif
     AssertIntNE(X509_set_version(x509, 6L), 0);
     AssertIntGT(X509_sign(x509, priv, EVP_sha256()), 0);
 
@@ -27707,21 +27723,6 @@ static void test_wolfSSL_X509_sign(void)
     AssertIntEQ(X509_get_ext_count(x509), SSL_FAILURE);
 #endif
 
-    AssertIntEQ(wolfSSL_X509_get_serial_number(x509, sn, &snSz),
-                WOLFSSL_SUCCESS);
-#ifndef WOLFSSL_ALT_NAMES
-    /* Valid case - size should be 798 with 16 byte serial number */
-    AssertIntEQ(ret, 782 + snSz);
-#else /* WOLFSSL_ALT_NAMES */
-    #if defined(OPENSSL_ALL) || defined(WOLFSSL_IP_ALT_NAME)
-    /* Valid case - size should be 936 with 16 byte serial number */
-    AssertIntEQ(ret, 920 + snSz);
-    #else
-    /* Valid case - size should be 927 with 16 byte serial number */
-    AssertIntEQ(ret, 911 + snSz);
-    #endif
-#endif /* WOLFSSL_ALT_NAMES */
-
     X509_NAME_free(name);
     EVP_PKEY_free(priv);
     EVP_PKEY_free(pub);