Fix error codes for OpenSSL compatiblity

src/ssl.c

@@ -25617,6 +25617,23 @@ unsigned long wolfSSL_ERR_peek_error(void)
     return wolfSSL_ERR_peek_error_line_data(NULL, NULL, NULL, NULL);
 }
 
+int wolfSSL_ERR_GET_LIB(unsigned long err)
+{
+    switch (err) {
+    case PEM_R_NO_START_LINE:
+    case PEM_R_PROBLEMS_GETTING_PASSWORD:
+    case PEM_R_BAD_PASSWORD_READ:
+    case PEM_R_BAD_DECRYPT:
+        return ERR_LIB_PEM;
+    case EVP_R_BAD_DECRYPT:
+    case EVP_R_BN_DECODE_ERROR:
+    case EVP_R_DECODE_ERROR:
+    case EVP_R_PRIVATE_KEY_DECODE_ERROR:
+        return ERR_LIB_EVP;
+    default:
+        return 0;
+    }
+}
 
 /* This function is to find global error values that are the same through out
  * all library version. With wolfSSL having only one set of error codes the
@@ -25641,7 +25658,7 @@ int wolfSSL_ERR_GET_REASON(unsigned long err)
     ret = 0 - ret; /* setting as negative value */
     /* wolfCrypt range is less than MAX (-100)
        wolfSSL range is MIN (-300) and lower */
-    if (ret < MAX_CODE_E) {
+    if (ret < MAX_CODE_E && ret > MIN_CODE_E) {
         return ret;
     }
     else {
@@ -44218,7 +44235,8 @@ unsigned long wolfSSL_ERR_peek_error_line_data(const char **file, int *line,
     }
 
 #if defined(OPENSSL_ALL) || defined(WOLFSSL_NGINX) || \
-    defined(WOLFSSL_HAPROXY) || defined(WOLFSSL_MYSQL_COMPATIBLE)
+    defined(WOLFSSL_OPENSSH) || defined(WOLFSSL_HAPROXY) || \
+    defined(WOLFSSL_MYSQL_COMPATIBLE)
     {
         int ret = 0;
 
@@ -44227,7 +44245,10 @@ unsigned long wolfSSL_ERR_peek_error_line_data(const char **file, int *line,
                 WOLFSSL_MSG("Issue peeking at error node in queue");
                 return 0;
             }
-            ret = -ret;
+            /* OpenSSL uses positive error codes */
+            if (ret < 0) {
+                ret = -ret;
+            }
 
             if (ret == ASN_NO_PEM_HEADER)
                 return (ERR_LIB_PEM << 24) | PEM_R_NO_START_LINE;

wolfcrypt/src/asn.c

@@ -69,6 +69,10 @@ ASN Options:
 #include <wolfssl/wolfcrypt/wc_encrypt.h>
 #include <wolfssl/wolfcrypt/logging.h>
 
+#ifdef OPENSSL_EXTRA
+#include <wolfssl/openssl/ssl.h>
+#endif
+
 #include <wolfssl/wolfcrypt/random.h>
 #include <wolfssl/wolfcrypt/hash.h>
 #ifdef NO_INLINE
@@ -10510,8 +10514,18 @@ int PemToDer(const unsigned char* buff, long longSz, int type,
 #endif /* !NO_WOLFSSL_SKIP_TRAILING_PAD */
 
             }
+#ifdef OPENSSL_EXTRA
+            if (ret) {
+                PEMerr(0, PEM_R_BAD_DECRYPT);
+            }
+#endif
             ForceZero(password, passwordSz);
         }
+#ifdef OPENSSL_EXTRA
+        else {
+            PEMerr(0, PEM_R_BAD_PASSWORD_READ);
+        }
+#endif
 
     #ifdef WOLFSSL_SMALL_STACK
         XFREE(password, heap, DYNAMIC_TYPE_STRING);

wolfssl/openssl/ssl.h

@@ -47,6 +47,9 @@
 #include <wolfssl/openssl/objects.h>
 #endif
 
+/* need MIN_CODE_E to determine wolfSSL error range */
+#include <wolfssl/wolfcrypt/error-crypt.h>
+
 /* all NID_* values are in asn.h */
 #include <wolfssl/wolfcrypt/asn.h>
 
@@ -749,6 +752,7 @@ wolfSSL_X509_STORE_set_verify_cb((WOLFSSL_X509_STORE *)(s), (WOLFSSL_X509_STORE_
 #define SYS_F_IOCTLSOCKET    WOLFSSL_SYS_IOCTLSOCKET
 #define SYS_F_LISTEN         WOLFSSL_SYS_LISTEN
 
+#define ERR_GET_LIB                     wolfSSL_ERR_GET_LIB
 #define ERR_GET_REASON                  wolfSSL_ERR_GET_REASON
 
 #define ERR_put_error                   wolfSSL_ERR_put_error
@@ -1088,15 +1092,21 @@ enum {
  * PEM_read_bio_X509 is called and the return error is lost.
  * The error that needs to be detected is: SSL_NO_PEM_HEADER.
  */
-#define ERR_GET_LIB(l)  (int)((((unsigned long)l) >> 24L) & 0xffL)
 #define ERR_GET_FUNC(l) (int)((((unsigned long)l) >> 12L) & 0xfffL)
 
 #define PEM_F_PEM_DEF_CALLBACK  100
 
-#define PEM_R_NO_START_LINE     108
+/* Avoid wolfSSL error code range */
-#define PEM_R_PROBLEMS_GETTING_PASSWORD 109
+#define PEM_R_NO_START_LINE             (-MIN_CODE_E + 1)
-#define PEM_R_BAD_PASSWORD_READ 110
+#define PEM_R_PROBLEMS_GETTING_PASSWORD (-MIN_CODE_E + 2)
-#define PEM_R_BAD_DECRYPT       111
+#define PEM_R_BAD_PASSWORD_READ         (-MIN_CODE_E + 3)
+#define PEM_R_BAD_DECRYPT               (-MIN_CODE_E + 4)
+
+#define EVP_R_BAD_DECRYPT               (-MIN_CODE_E + 100 + 1)
+#define EVP_R_BN_DECODE_ERROR           (-MIN_CODE_E + 100 + 2)
+#define EVP_R_DECODE_ERROR              (-MIN_CODE_E + 100 + 3)
+#define EVP_R_PRIVATE_KEY_DECODE_ERROR  (-MIN_CODE_E + 100 + 4)
+
 #define ERR_LIB_PEM             9
 #define ERR_LIB_X509            10
 #define ERR_LIB_EVP             11

wolfssl/ssl.h

@@ -1045,6 +1045,7 @@ WOLFSSL_API int  wolfSSL_CTX_mcast_set_highwater_cb(WOLFSSL_CTX*,
                                                     CallbackMcastHighwater);
 WOLFSSL_API int  wolfSSL_mcast_set_highwater_ctx(WOLFSSL*, void*);
 
+WOLFSSL_API int   wolfSSL_ERR_GET_LIB(unsigned long err);
 WOLFSSL_API int   wolfSSL_ERR_GET_REASON(unsigned long err);
 WOLFSSL_API char* wolfSSL_ERR_error_string(unsigned long,char*);
 WOLFSSL_API void  wolfSSL_ERR_error_string_n(unsigned long e, char* buf,
@@ -1670,11 +1671,6 @@ enum {
     ASN1_GENERALIZEDTIME = 4,
     SSL_MAX_SSL_SESSION_ID_LENGTH = 32,
 
-    EVP_R_BAD_DECRYPT = 2,
-    EVP_R_BN_DECODE_ERROR = 3,
-    EVP_R_DECODE_ERROR = 4,
-    EVP_R_PRIVATE_KEY_DECODE_ERROR = 5,
-
     SSL_ST_CONNECT = 0x1000,
     SSL_ST_ACCEPT  = 0x2000,
     SSL_ST_MASK    = 0x0FFF,