Merge pull request #3882 from TakayukiMatsuo/tk11899

Return code differences in wolfSSL_EVP_PKEY_cmp et al.

src/ssl.c

@@ -16336,8 +16336,9 @@ int wolfSSL_set_compression(WOLFSSL* ssl)
             }
 
             XFREE(bio, 0, DYNAMIC_TYPE_OPENSSL);
+            return WOLFSSL_SUCCESS;
         }
-        return 1;
+        return WOLFSSL_FAILURE;
     }
 
     /* like BIO_free, but no return value */

tests/api.c

@@ -38278,6 +38278,7 @@ static void test_EVP_PKEY_cmp(void)
     EVP_PKEY *a, *b;
     const unsigned char *in;
 
+    printf(testingFmt, "wolfSSL_EVP_PKEY_cmp()");
 #if !defined(NO_RSA) && defined(USE_CERT_BUFFERS_2048)
     in = client_key_der_2048;
     AssertNotNull(a = wolfSSL_d2i_PrivateKey(EVP_PKEY_RSA, NULL,
@@ -38287,7 +38288,11 @@ static void test_EVP_PKEY_cmp(void)
         &in, (long)sizeof_client_key_der_2048));
 
     /* Test success case RSA */
+#if defined(WOLFSSL_ERROR_CODE_OPENSSL)
+    AssertIntEQ(EVP_PKEY_cmp(a, b), 1);
+#else
     AssertIntEQ(EVP_PKEY_cmp(a, b), 0);
+#endif /* WOLFSSL_ERROR_CODE_OPENSSL */
 
     EVP_PKEY_free(b);
     EVP_PKEY_free(a);
@@ -38302,7 +38307,11 @@ static void test_EVP_PKEY_cmp(void)
         &in, (long)sizeof_ecc_clikey_der_256));
 
     /* Test success case ECC */
+#if defined(WOLFSSL_ERROR_CODE_OPENSSL)
+    AssertIntEQ(EVP_PKEY_cmp(a, b), 1);
+#else
     AssertIntEQ(EVP_PKEY_cmp(a, b), 0);
+#endif /* WOLFSSL_ERROR_CODE_OPENSSL */
 
     EVP_PKEY_free(b);
     EVP_PKEY_free(a);
@@ -38319,8 +38328,11 @@ static void test_EVP_PKEY_cmp(void)
     AssertNotNull(b = wolfSSL_d2i_PrivateKey(EVP_PKEY_EC, NULL,
         &in, (long)sizeof_ecc_clikey_der_256));
 
+#if defined(WOLFSSL_ERROR_CODE_OPENSSL)
+    AssertIntEQ(EVP_PKEY_cmp(a, b), -1);
+#else
     AssertIntNE(EVP_PKEY_cmp(a, b), 0);
-
+#endif /* WOLFSSL_ERROR_CODE_OPENSSL */
     EVP_PKEY_free(b);
     EVP_PKEY_free(a);
 #endif
@@ -38328,10 +38340,17 @@ static void test_EVP_PKEY_cmp(void)
     /* invalid or empty failure cases */
     a = EVP_PKEY_new();
     b = EVP_PKEY_new();
+#if defined(WOLFSSL_ERROR_CODE_OPENSSL)
+    AssertIntEQ(EVP_PKEY_cmp(NULL, NULL), 0);
+    AssertIntEQ(EVP_PKEY_cmp(a, NULL), 0);
+    AssertIntEQ(EVP_PKEY_cmp(NULL, b), 0);
+    AssertIntEQ(EVP_PKEY_cmp(a, b), 0);
+#else
     AssertIntNE(EVP_PKEY_cmp(NULL, NULL), 0);
     AssertIntNE(EVP_PKEY_cmp(a, NULL), 0);
     AssertIntNE(EVP_PKEY_cmp(NULL, b), 0);
     AssertIntNE(EVP_PKEY_cmp(a, b), 0);
+#endif
     EVP_PKEY_free(b);
     EVP_PKEY_free(a);
 

wolfcrypt/src/evp.c

@@ -2031,17 +2031,30 @@ WOLFSSL_API int wolfSSL_EVP_PKEY_missing_parameters(WOLFSSL_EVP_PKEY *pkey)
 }
 #endif
 
+/* wolfSSL_EVP_PKEY_cmp
+ * returns 0 on success, -1 on failure.
+ *
+ * This behavior is different from openssl.
+ *  EVP_PKEY_cmp returns:
+ *    1 : two keys match
+ *    0 : do not match
+ *    -1: key types are different
+ *    -2: the operation is not supported
+ * If you want this function behave the same as openSSL,
+ * define WOLFSSL_ERROR_CODE_OPENSSL so that WS_RETURN_CODE translates return
+ * codes to match OpenSSL equivalent behavior.
+ */
 WOLFSSL_API int wolfSSL_EVP_PKEY_cmp(const WOLFSSL_EVP_PKEY *a, const WOLFSSL_EVP_PKEY *b)
 {
     int ret = -1; /* failure */
     int a_sz = 0, b_sz = 0;
 
     if (a == NULL || b == NULL)
-        return ret;
+        return WS_RETURN_CODE(ret, WOLFSSL_FAILURE);
 
     /* check its the same type of key */
     if (a->type != b->type)
-        return ret;
+        return WS_RETURN_CODE(ret, -1);
 
     /* get size based on key type */
     switch (a->type) {
@@ -2062,27 +2075,30 @@ WOLFSSL_API int wolfSSL_EVP_PKEY_cmp(const WOLFSSL_EVP_PKEY *a, const WOLFSSL_EV
         break;
 #endif /* HAVE_ECC */
     default:
-        return ret;
+        return WS_RETURN_CODE(ret, -2);
     } /* switch (a->type) */
 
     /* check size */
     if (a_sz <= 0 || b_sz <= 0 || a_sz != b_sz) {
-        return ret;
+        return WS_RETURN_CODE(ret, WOLFSSL_FAILURE);
     }
 
     /* check public key size */
     if (a->pkey_sz > 0 && b->pkey_sz > 0 && a->pkey_sz != b->pkey_sz) {
-        return ret;
+        return WS_RETURN_CODE(ret, WOLFSSL_FAILURE);
     }
 
     /* check public key */
     if (a->pkey.ptr && b->pkey.ptr) {
         if (XMEMCMP(a->pkey.ptr, b->pkey.ptr, a->pkey_sz) != 0) {
-            return ret;
+            return WS_RETURN_CODE(ret, WOLFSSL_FAILURE);
         }
     }
+#if defined(WOLFSSL_ERROR_CODE_OPENSSL)
+    ret = 1; /* the keys match */
+#else
     ret = 0; /* success */
-
+#endif
     return ret;
 }
 

wolfssl/ssl.h

@@ -719,6 +719,7 @@ enum AlertLevel {
  * Since wolfSSL 4.7.0, the following functions use this macro:
  * - wolfSSL_CTX_load_verify_locations
  * - wolfSSL_X509_LOOKUP_load_file
+ * - wolfSSL_EVP_PKEY_cmp
  */
 #if defined(WOLFSSL_ERROR_CODE_OPENSSL)
     #define WS_RETURN_CODE(item1,item2) \