Fix CheckHostName matching

src/internal.c

@@ -9346,7 +9346,7 @@ int CheckForAltNames(DecodedCert* dCert, const char* domain, int* checkCN)
         altName = dCert->altNames;
 
     if (checkCN != NULL) {
-        *checkCN = altName == NULL;
+        *checkCN = (altName == NULL) ? 1 : 0;
     }
 
     while (altName) {
@@ -9415,23 +9415,29 @@ int CheckForAltNames(DecodedCert* dCert, const char* domain, int* checkCN)
 int CheckHostName(DecodedCert* dCert, const char *domainName, size_t domainNameLen)
 {
     int checkCN;
+    int ret = DOMAIN_NAME_MISMATCH;
 
     /* Assume name is NUL terminated. */
     (void)domainNameLen;
 
     if (CheckForAltNames(dCert, domainName, &checkCN) != 1) {
-        WOLFSSL_MSG("DomainName match on alt names failed too");
-        return DOMAIN_NAME_MISMATCH;
+        WOLFSSL_MSG("DomainName match on alt names failed");
     }
+    else {
+        ret = 0;
+    }
+
     if (checkCN == 1) {
         if (MatchDomainName(dCert->subjectCN, dCert->subjectCNLen,
-                            domainName) == 0) {
+                            domainName) == 1) {
+            ret = 0;
+        }
+        else {
             WOLFSSL_MSG("DomainName match on common name failed");
-            return DOMAIN_NAME_MISMATCH;
         }
     }
 
-    return 0;
+    return ret;
 }
 
 int CheckIPAddr(DecodedCert* dCert, const char* ipasc)

src/ssl.c

@@ -43661,6 +43661,11 @@ int wolfSSL_X509_check_host(WOLFSSL_X509 *x, const char *chk, size_t chklen,
     (void)flags;
     (void)peername;
 
+    if ((x == NULL) || (chk == NULL)) {
+        WOLFSSL_MSG("Invalid parameter");
+        return WOLFSSL_FAILURE;
+    }
+
     if (flags == WOLFSSL_NO_WILDCARDS) {
         WOLFSSL_MSG("X509_CHECK_FLAG_NO_WILDCARDS not yet implemented");
         return WOLFSSL_FAILURE;

tests/api.c

@@ -23875,6 +23875,35 @@ static void test_wolfSSL_X509_issuer_name_hash(void)
 #endif
 }
 
+static void test_wolfSSL_X509_check_host(void)
+{
+#if defined(OPENSSL_EXTRA) && !defined(NO_CERTS) && !defined(NO_FILESYSTEM) \
+    && !defined(NO_SHA) && !defined(NO_RSA)
+
+    X509* x509;
+    const char altName[] = "example.com";
+
+    printf(testingFmt, "wolfSSL_X509_check_host()");
+
+    AssertNotNull(x509 = wolfSSL_X509_load_certificate_file(cliCertFile,
+                SSL_FILETYPE_PEM));
+
+    AssertIntEQ(X509_check_host(x509, altName, XSTRLEN(altName), 0, NULL),
+            WOLFSSL_SUCCESS);
+
+    AssertIntEQ(X509_check_host(x509, NULL, 0, 0, NULL),
+            WOLFSSL_FAILURE);
+
+    X509_free(x509);
+
+    AssertIntEQ(X509_check_host(NULL, altName, XSTRLEN(altName), 0, NULL),
+            WOLFSSL_FAILURE);
+
+    printf(resultFmt, passed);
+
+#endif
+}
+
 static void test_wolfSSL_DES(void)
 {
     #if defined(OPENSSL_EXTRA) && !defined(NO_DES3)
@@ -36407,6 +36436,7 @@ void ApiTest(void)
     test_wolfSSL_X509_INFO();
     test_wolfSSL_X509_subject_name_hash();
     test_wolfSSL_X509_issuer_name_hash();
+    test_wolfSSL_X509_check_host();
     test_wolfSSL_DES();
     test_wolfSSL_certs();
     test_wolfSSL_ASN1_TIME_print();