Match OpenSSL self signed error return.

OpenSSL compat expects ASN_SELF_SIGNED_E when a self signed cert can't be verified. This is useful when translating the error with GetX509Error into a X509_V_ERR_* error.

src/internal.c

@@ -13142,7 +13142,7 @@ int ProcessPeerCerts(WOLFSSL* ssl, byte* input, word32* inOutIdx,
 #if defined(OPENSSL_ALL) && defined(WOLFSSL_CERT_GEN) && \
     (defined(WOLFSSL_CERT_REQ) || defined(WOLFSSL_CERT_EXT)) && \
     !defined(NO_FILESYSTEM) && !defined(NO_WOLFSSL_DIR)
-                    if (ret == ASN_NO_SIGNER_E) {
+                    if (ret == ASN_NO_SIGNER_E || ret == ASN_SELF_SIGNED_E) {
                         WOLFSSL_MSG("try to load certificate if hash dir is set");
                         ret = LoadCertByIssuer(SSL_STORE(ssl),
                            (WOLFSSL_X509_NAME*)args->dCert->issuerName,
@@ -13264,7 +13264,7 @@ int ProcessPeerCerts(WOLFSSL* ssl, byte* input, word32* inOutIdx,
                         chain mode only requires that the peer certificate
                         validate to a trusted CA */
                     if (ret != 0 && args->dCert->isCA) {
-                        if (ret == ASN_NO_SIGNER_E) {
+                        if (ret == ASN_NO_SIGNER_E || ret == ASN_SELF_SIGNED_E) {
                             if (!ssl->options.usingAltCertChain) {
                                 WOLFSSL_MSG("Trying alternate cert chain");
                                 ssl->options.usingAltCertChain = 1;
@@ -13369,7 +13369,7 @@ int ProcessPeerCerts(WOLFSSL* ssl, byte* input, word32* inOutIdx,
 #if defined(OPENSSL_ALL) && defined(WOLFSSL_CERT_GEN) && \
     (defined(WOLFSSL_CERT_REQ) || defined(WOLFSSL_CERT_EXT)) && \
     !defined(NO_FILESYSTEM) && !defined(NO_WOLFSSL_DIR)
-                    if (ret == ASN_NO_SIGNER_E) {
+                    if (ret == ASN_NO_SIGNER_E || ret == ASN_SELF_SIGNED_E) {
                         WOLFSSL_MSG("try to load certificate if hash dir is set");
                         ret = LoadCertByIssuer(SSL_STORE(ssl),
                            (WOLFSSL_X509_NAME*)args->dCert->issuerName,

src/ssl.c

@@ -7287,12 +7287,6 @@ int CM_VerifyBuffer_ex(WOLFSSL_CERT_MANAGER* cm, const byte* buff,
     if (ret == 0)
         ret = ParseCertRelative(cert, CERT_TYPE, 1, cm);
 
-#if defined(OPENSSL_ALL) || defined(WOLFSSL_QT)
-    /* ret needs to be self-singer error for Qt compat */
-    if (ret == ASN_NO_SIGNER_E && cert->selfSigned)
-        ret = ASN_SELF_SIGNED_E;
-#endif
-
 #ifdef HAVE_CRL
     if (ret == 0 && cm->crlEnabled)
         ret = CheckCertCRL(cm->crl, cert);

tests/api.c

@@ -1458,6 +1458,9 @@ static int test_wolfSSL_CertManagerGetCerts(void)
     WOLFSSL_BIO* bio = NULL;
 #endif
     int i = 0;
+    int ret = 0;
+    const byte* der;
+    int derSz = 0;
 
     printf(testingFmt, "wolfSSL_CertManagerGetCerts()");
     AssertNotNull(file1=fopen("./certs/ca-cert.pem", "rb"));
@@ -1467,6 +1470,17 @@ static int test_wolfSSL_CertManagerGetCerts(void)
 
     AssertNotNull(cm = wolfSSL_CertManagerNew_ex(NULL));
     AssertNull(sk = wolfSSL_CertManagerGetCerts(cm));
+
+    AssertNotNull(der = wolfSSL_X509_get_der(cert1, &derSz));
+    ret = wolfSSL_CertManagerVerifyBuffer(cm, der, derSz, WOLFSSL_FILETYPE_ASN1);
+#if defined(OPENSSL_ALL) || defined(WOLFSSL_QT)
+    /* Check that ASN_SELF_SIGNED_E is returned for a self-signed cert for QT
+     * and full OpenSSL compatibility */
+    AssertIntEQ(ret, ASN_SELF_SIGNED_E);
+#else
+    AssertIntEQ(ret, ASN_NO_SIGNER_E);
+#endif
+
     AssertIntEQ(WOLFSSL_SUCCESS, wolfSSL_CertManagerLoadCA(cm,
                 "./certs/ca-cert.pem", NULL));
 

wolfcrypt/src/asn.c

@@ -21531,8 +21531,18 @@ int ParseCertRelative(DecodedCert* cert, int type, int verify, void* cm)
         else {
             /* no signer */
             WOLFSSL_MSG("No CA signer to verify with");
-            WOLFSSL_ERROR_VERBOSE(ASN_NO_SIGNER_E);
+#if defined(OPENSSL_ALL) || defined(WOLFSSL_QT)
-            return ASN_NO_SIGNER_E;
+            /* ret needs to be self-signer error for Qt compat */
+            if (cert->selfSigned) {
+                WOLFSSL_ERROR_VERBOSE(ASN_SELF_SIGNED_E);
+                return ASN_SELF_SIGNED_E;
+            }
+            else
+#endif
+            {
+                WOLFSSL_ERROR_VERBOSE(ASN_NO_SIGNER_E);
+                return ASN_NO_SIGNER_E;
+            }
         }
     }
 

wolfssl/test.h

@@ -2783,6 +2783,14 @@ static WC_INLINE int myVerify(int preverify, WOLFSSL_X509_STORE_CTX* store)
                                       wolfSSL_X509_get_subject_name(peer), 0, 0);
         printf("\tPeer's cert info:\n issuer : %s\n subject: %s\n", issuer,
                                                                   subject);
+
+#if defined(OPENSSL_ALL) || defined(WOLFSSL_QT)
+        /* preverify needs to be self-signer error for Qt compat.
+         * Should be ASN_SELF_SIGNED_E */
+        if (XSTRCMP(issuer, subject) == 0 && preverify == ASN_NO_SIGNER_E)
+            return 0;
+#endif
+
         XFREE(subject, 0, DYNAMIC_TYPE_OPENSSL);
         XFREE(issuer,  0, DYNAMIC_TYPE_OPENSSL);
 #if defined(SHOW_CERTS) && !defined(NO_FILESYSTEM)