Fix X509 subject and issuer name_hash mismatch

Fix api test and cleanup.

src/x509.c

@@ -4781,96 +4781,98 @@ WOLFSSL_X509_NAME* wolfSSL_X509_get_subject_name(WOLFSSL_X509* cert)
 */
 unsigned long wolfSSL_X509_subject_name_hash(const WOLFSSL_X509* x509)
 {
-    unsigned long ret = 0;
+    unsigned long      hash = 0;
-    WOLFSSL_X509_NAME *subjectName = NULL;
+    WOLFSSL_X509_NAME* subjectName = NULL;
-    unsigned char* canonName = NULL;
+    unsigned char*     canonName = NULL;
-    byte digest[WC_MAX_DIGEST_SIZE];
+    byte               digest[WC_MAX_DIGEST_SIZE];
-    int size = 0;
+    int                size = 0;
 
     if (x509 == NULL) {
-        return ret;
+        return 0;
     }
 
     subjectName = wolfSSL_X509_get_subject_name((WOLFSSL_X509*)x509);
 
     if (subjectName == NULL) {
-        return ret;
+        WOLFSSL_MSG("wolfSSL_X509_get_subject_name error");
+        return 0;
     }
 
     size = wolfSSL_i2d_X509_NAME_canon(subjectName, &canonName);
 
-    if (size <= 0){
+    if (size <= 0 || canonName == NULL){
         WOLFSSL_MSG("wolfSSL_i2d_X509_NAME_canon error");
-        return ret;
+        return 0;
     }
 
     #ifndef NO_SHA
     if (wc_ShaHash((const byte*)canonName, (word32)size, digest) != 0) {
         WOLFSSL_MSG("wc_ShaHash error");
-        return ret;
+        return 0;
     }
     #elif !defined(NO_SHA256)
     if (wc_Sha256Hash((const byte*)canonName, (word32)size, digest) != 0) {
         WOLFSSL_MSG("wc_Sha256Hash error");
-        return ret;
+        return 0;
     }
     #endif
 
-    ret  =  (unsigned long) digest[0];
+    hash  =  (unsigned long) digest[0];
-    ret |= ((unsigned long) digest[1]) << 8;
+    hash |= ((unsigned long) digest[1]) << 8;
-    ret |= ((unsigned long) digest[2]) << 16;
+    hash |= ((unsigned long) digest[2]) << 16;
-    ret |= ((unsigned long) digest[3]) << 24;
+    hash |= ((unsigned long) digest[3]) << 24;
 
     XFREE(canonName, NULL, DYNAMIC_TYPE_OPENSSL);
 
-    return ret;
+    return hash;
 }
 
 unsigned long wolfSSL_X509_issuer_name_hash(const WOLFSSL_X509* x509)
 {
-    unsigned long ret = 0;
+    unsigned long      hash = 0;
-    WOLFSSL_X509_NAME *issuerName = NULL;
+    WOLFSSL_X509_NAME* issuerName = NULL;
-    unsigned char* canonName = NULL;
+    unsigned char*     canonName = NULL;
-    byte digest[WC_MAX_DIGEST_SIZE];
+    byte               digest[WC_MAX_DIGEST_SIZE];
-    int size = 0;
+    int                size = 0;
 
     if (x509 == NULL) {
-        return ret;
+        return 0;
     }
 
     issuerName = wolfSSL_X509_get_issuer_name((WOLFSSL_X509*)x509);
 
     if (issuerName == NULL) {
-        return ret;
+        WOLFSSL_MSG("wolfSSL_X509_get_issuer_name error");
+        return 0;
     }
 
     size = wolfSSL_i2d_X509_NAME_canon(issuerName, &canonName);
 
-    if (size <= 0){
+    if (size <= 0 || canonName == NULL){
         WOLFSSL_MSG("wolfSSL_i2d_X509_NAME_canon error");
-        return ret;
+        return 0;
     }
 
     #ifndef NO_SHA
     if (wc_ShaHash((const byte*)canonName, (word32)size, digest) != 0) {
         WOLFSSL_MSG("wc_ShaHash error");
-        return ret;
+        return 0;
     }
     #elif !defined(NO_SHA256)
     if (wc_Sha256Hash((const byte*)canonName, (word32)size, digest) != 0) {
         WOLFSSL_MSG("wc_ShaHash error");
-        return ret;
+        return 0;
     }
     #endif
 
-    ret  =  (unsigned long) digest[0];
+    hash  =  (unsigned long) digest[0];
-    ret |= ((unsigned long) digest[1]) << 8;
+    hash |= ((unsigned long) digest[1]) << 8;
-    ret |= ((unsigned long) digest[2]) << 16;
+    hash |= ((unsigned long) digest[2]) << 16;
-    ret |= ((unsigned long) digest[3]) << 24;
+    hash |= ((unsigned long) digest[3]) << 24;
 
     XFREE(canonName, NULL, DYNAMIC_TYPE_OPENSSL);
 
-    return ret;
+    return hash;
 }
 #endif /* OPENSSL_EXTRA && (!NO_SHA || !NO_SHA256) */
 

tests/api.c

@@ -31406,11 +31406,13 @@ static int test_wolfSSL_X509_subject_name_hash(void)
     ret1 = X509_subject_name_hash(x509);
     AssertIntNE(ret1, 0);
 
+#if !defined(NO_SHA)
     ret2 = X509_NAME_hash(X509_get_subject_name(x509));
     AssertIntNE(ret2, 0);
 
-#if !defined(NO_SHA)
     AssertIntEQ(ret1, ret2);
+#else
+    (void) ret2;
 #endif
 
     X509_free(x509);
@@ -31443,11 +31445,13 @@ static int test_wolfSSL_X509_issuer_name_hash(void)
     ret1 = X509_issuer_name_hash(x509);
     AssertIntNE(ret1, 0);
 
+#if !defined(NO_SHA)
     ret2 = X509_NAME_hash(X509_get_issuer_name(x509));
     AssertIntNE(ret2, 0);
 
-#if !defined(NO_SHA)
     AssertIntEQ(ret1, ret2);
+#else
+    (void) ret2;
 #endif
 
     X509_free(x509);