Fixes for verify callback failure override handling. Fixes the return codes in the failure cases.

src/internal.c

@@ -8448,9 +8448,7 @@ typedef struct ProcPeerCertArgs {
     word32 begin;
     int    totalCerts; /* number of certs in certs buffer */
     int    count;
-    int    dCertInit;
     int    certIdx;
-    int    fatal;
     int    lastErr;
 #ifdef WOLFSSL_ALT_CERT_CHAINS
     int    lastCaErr;
@@ -8458,11 +8456,14 @@ typedef struct ProcPeerCertArgs {
 #ifdef WOLFSSL_TLS13
     byte   ctxSz;
 #endif
-#ifdef WOLFSSL_TRUST_PEER_CERT
-    byte haveTrustPeer; /* was cert verified by loaded trusted peer cert */
-#endif
 #if defined(OPENSSL_EXTRA) || defined(OPENSSL_EXTRA_X509_SMALL)
     char   untrustedDepth;
+#endif
+    word16 fatal:1;
+    word16 verifyErr:1;
+    word16 dCertInit:1;
+#ifdef WOLFSSL_TRUST_PEER_CERT
+    word16 haveTrustPeer:1; /* was cert verified by loaded trusted peer cert */
 #endif
 } ProcPeerCertArgs;
 
@@ -8607,6 +8608,9 @@ static int DoVerifyCallback(WOLFSSL* ssl, int ret, ProcPeerCertArgs* args)
             if (ret == 0) {
                 ret = VERIFY_CERT_ERROR;
             }
+
+            /* mark as verify error */
+            args->verifyErr = 1;
         }
     #ifdef OPENSSL_EXTRA
         if (args->certIdx > 0)
@@ -9174,6 +9178,7 @@ int ProcessPeerCerts(WOLFSSL* ssl, byte* input, word32* inOutIdx,
                             ssl->options.usingAltCertChain = 1;
 
                             /* clear last CA fail since CA cert was validated */
+                            if (!args->verifyErr)
                                 args->lastCaErr = 0;
 
                         #ifdef SESSION_CERTS
@@ -9185,16 +9190,6 @@ int ProcessPeerCerts(WOLFSSL* ssl, byte* input, word32* inOutIdx,
                     }
                     else if (ret != 0) {
                         WOLFSSL_MSG("Failed to verify CA from chain");
-                    #ifdef WOLFSSL_ALT_CERT_CHAINS
-                        if (args->lastCaErr == 0) {
-                            /* store CA error and proceed to next cert */
-                            args->lastCaErr = ret;
-                            ret = 0;
-                        }
-                        else {
-                            args->lastErr = args->lastCaErr;
-                        }
-                    #endif
                     #ifdef OPENSSL_EXTRA
                         ssl->peerVerifyRet = X509_V_ERR_INVALID_CA;
                     #endif
@@ -9265,6 +9260,16 @@ int ProcessPeerCerts(WOLFSSL* ssl, byte* input, word32* inOutIdx,
                     ret = DoVerifyCallback(ssl, ret, args);
 
                     /* Handle error codes */
+                #ifdef WOLFSSL_ALT_CERT_CHAINS
+                    if (args->lastCaErr == 0) {
+                        /* capture CA error and proceed to next cert */
+                        args->lastCaErr = ret;
+                        ret = 0;
+                    }
+                    else {
+                        args->lastErr = args->lastCaErr;
+                    }
+                #endif
                     if (ret != 0 && args->lastErr == 0) {
                         args->lastErr = ret;   /* save error from last time */
                         ret = 0; /* reset error */
@@ -9337,8 +9342,23 @@ int ProcessPeerCerts(WOLFSSL* ssl, byte* input, word32* inOutIdx,
                             cert->buffer, cert->length);
                     }
                 #endif /* SESSION_CERTS && WOLFSSL_ALT_CERT_CHAINS */
+
+                    /* check if fatal error */
+                    if (args->verifyErr) {
+                        args->fatal = 1;
+                        if (ret == 0) {
+                            ret = args->lastErr;
+                        }
+                    #ifdef WOLFSSL_ALT_CERT_CHAINS
+                        if (ret == 0) {
+                            ret = args->lastCaErr;
+                        }
+                    #endif
+                    }
+                    else {
                         args->fatal = 0;
                     }
+                }
                 else if (ret == ASN_PARSE_E || ret == BUFFER_E) {
                     WOLFSSL_MSG("Got Peer cert ASN PARSE or BUFFER ERROR");
                 #ifdef OPENSSL_EXTRA
@@ -9355,7 +9375,8 @@ int ProcessPeerCerts(WOLFSSL* ssl, byte* input, word32* inOutIdx,
                     if (ssl->verifyCallback) {
                         WOLFSSL_MSG(
                             "\tCallback override available, will continue");
-                        args->fatal = 0;
+                        /* check if fatal error */
+                        args->fatal = (args->verifyErr) ? 1 : 0;
                     }
                     else {
                         WOLFSSL_MSG("\tNo callback override available, fatal");