F-4594: return non-zero from wolfSSL_get_verify_result on NULL ssl

WOLFSSL_FAILURE is 0, which equals X509_V_OK, so a NULL ssl was
indistinguishable from successful verification under the standard
"SSL_get_verify_result(ssl) \!= X509_V_OK" idiom. Return
WOLFSSL_X509_V_ERR_APPLICATION_VERIFICATION (50, matching the OpenSSL
compat value) instead, and add it to the X509 verify-error enum.

src/ssl.c

@@ -13025,7 +13025,10 @@ size_t wolfSSL_get_peer_finished(const WOLFSSL *ssl, void *buf, size_t count)
 long wolfSSL_get_verify_result(const WOLFSSL *ssl)
 {
     if (ssl == NULL) {
-        return WOLFSSL_FAILURE;
+        /* Return a non-zero error so the OpenSSL-idiomatic
+         * "!= X509_V_OK" check does not mistake a NULL ssl for a
+         * successful verification (X509_V_OK is 0). */
+        return WOLFSSL_X509_V_ERR_APPLICATION_VERIFICATION;
     }
 
     return (long)ssl->peerVerifyRet;

tests/api.c

@@ -19069,7 +19069,10 @@ static int test_wolfSSL_verify_result(void)
     WOLFSSL_CTX* ctx = NULL;
     long         result = 0xDEADBEEF;
 
-    ExpectIntEQ(WC_NO_ERR_TRACE(WOLFSSL_FAILURE), wolfSSL_get_verify_result(ssl));
+    /* A NULL ssl returns a non-zero verify error (not X509_V_OK) so the
+     * OpenSSL-idiomatic "!= X509_V_OK" check is not fooled. See F-4594. */
+    ExpectIntEQ(WOLFSSL_X509_V_ERR_APPLICATION_VERIFICATION,
+        wolfSSL_get_verify_result(ssl));
 
     ExpectNotNull(ctx = wolfSSL_CTX_new(wolfSSLv23_client_method()));
     ExpectNotNull(ssl = SSL_new(ctx));

wolfssl/ssl.h

@@ -2732,6 +2732,7 @@ enum {
     WOLFSSL_X509_V_ERR_PATH_LENGTH_EXCEEDED              = 25,
     WOLFSSL_X509_V_ERR_CERT_REJECTED                     = 28,
     WOLFSSL_X509_V_ERR_SUBJECT_ISSUER_MISMATCH           = 29,
+    WOLFSSL_X509_V_ERR_APPLICATION_VERIFICATION          = 50,
     WOLFSSL_X509_V_ERR_HOSTNAME_MISMATCH                 = 62,
     WOLFSSL_X509_V_ERR_IP_ADDRESS_MISMATCH               = 64,
     WOLFSSL_X509_V_ERR_INVALID_CA                        = 79,