Refactored async event intialization so its done prior to making possible async calls. Added threadId and removed volatile attribs in WOLF_EVENT. Fixed possible unitialized value in benchmark ret.

src/internal.c

@@ -3024,6 +3024,13 @@ int RsaSign(WOLFSSL* ssl, const byte* in, word32 inSz, byte* out,
 
     WOLFSSL_ENTER("RsaSign");
 
+#ifdef WOLFSSL_ASYNC_CRYPT
+    /* intialize event */
+    ret = wolfSSL_AsyncInit(ssl, &key->asyncDev, WC_ASYNC_FLAG_CALL_AGAIN);
+    if (ret != 0)
+        return ret;
+#endif
+
 #if defined(WC_RSA_PSS)
     if (sigAlgo == rsa_pss_sa_algo) {
         enum wc_HashType hashType = WC_HASH_TYPE_NONE;
@@ -3058,9 +3065,9 @@ int RsaSign(WOLFSSL* ssl, const byte* in, word32 inSz, byte* out,
         ret = wc_RsaSSL_Sign(in, inSz, out, *outSz, key, ssl->rng);
 
     /* Handle async pending response */
-#if defined(WOLFSSL_ASYNC_CRYPT)
+#ifdef WOLFSSL_ASYNC_CRYPT
     if (ret == WC_PENDING_E) {
-        ret = wolfSSL_AsyncPush(ssl, &key->asyncDev, WC_ASYNC_FLAG_CALL_AGAIN);
+        ret = wolfSSL_AsyncPush(ssl, &key->asyncDev);
     }
 #endif /* WOLFSSL_ASYNC_CRYPT */
 
@@ -3090,6 +3097,13 @@ int RsaVerify(WOLFSSL* ssl, byte* in, word32 inSz, byte** out, int sigAlgo,
 
     WOLFSSL_ENTER("RsaVerify");
 
+#ifdef WOLFSSL_ASYNC_CRYPT
+    /* intialize event */
+    ret = wolfSSL_AsyncInit(ssl, &key->asyncDev, WC_ASYNC_FLAG_CALL_AGAIN);
+    if (ret != 0)
+        return ret;
+#endif
+
 #if defined(WC_RSA_PSS)
     if (sigAlgo == rsa_pss_sa_algo) {
         enum wc_HashType hashType = WC_HASH_TYPE_NONE;
@@ -3121,9 +3135,9 @@ int RsaVerify(WOLFSSL* ssl, byte* in, word32 inSz, byte** out, int sigAlgo,
     }
 
     /* Handle async pending response */
-#if defined(WOLFSSL_ASYNC_CRYPT)
+#ifdef WOLFSSL_ASYNC_CRYPT
     if (ret == WC_PENDING_E) {
-        ret = wolfSSL_AsyncPush(ssl, &key->asyncDev, WC_ASYNC_FLAG_CALL_AGAIN);
+        ret = wolfSSL_AsyncPush(ssl, &key->asyncDev);
     }
 #endif /* WOLFSSL_ASYNC_CRYPT */
 
@@ -3154,6 +3168,13 @@ int VerifyRsaSign(WOLFSSL* ssl, byte* verifySig, word32 sigSz,
         return BUFFER_E;
     }
 
+#ifdef WOLFSSL_ASYNC_CRYPT
+    /* intialize event */
+    ret = wolfSSL_AsyncInit(ssl, &key->asyncDev, WC_ASYNC_FLAG_CALL_AGAIN);
+    if (ret != 0)
+        return ret;
+#endif
+
 #if defined(WC_RSA_PSS)
     if (sigAlgo == rsa_pss_sa_algo) {
         enum wc_HashType hashType = WC_HASH_TYPE_NONE;
@@ -3186,9 +3207,9 @@ int VerifyRsaSign(WOLFSSL* ssl, byte* verifySig, word32 sigSz,
     }
 
     /* Handle async pending response */
-#if defined(WOLFSSL_ASYNC_CRYPT)
+#ifdef WOLFSSL_ASYNC_CRYPT
     if (ret == WC_PENDING_E) {
-        ret = wolfSSL_AsyncPush(ssl, &key->asyncDev, WC_ASYNC_FLAG_CALL_AGAIN);
+        ret = wolfSSL_AsyncPush(ssl, &key->asyncDev);
     }
 #endif /* WOLFSSL_ASYNC_CRYPT */
 
@@ -3209,6 +3230,13 @@ int RsaDec(WOLFSSL* ssl, byte* in, word32 inSz, byte** out, word32* outSz,
 
     WOLFSSL_ENTER("RsaDec");
 
+#ifdef WOLFSSL_ASYNC_CRYPT
+    /* intialize event */
+    ret = wolfSSL_AsyncInit(ssl, &key->asyncDev, WC_ASYNC_FLAG_CALL_AGAIN);
+    if (ret != 0)
+        return ret;
+#endif
+
 #ifdef HAVE_PK_CALLBACKS
     if (ssl->ctx->RsaDecCb) {
             ret = ssl->ctx->RsaDecCb(ssl, in, inSz, out, keyBuf, keySz,
@@ -3226,9 +3254,9 @@ int RsaDec(WOLFSSL* ssl, byte* in, word32 inSz, byte** out, word32* outSz,
     }
 
     /* Handle async pending response */
-#if defined(WOLFSSL_ASYNC_CRYPT)
+#ifdef WOLFSSL_ASYNC_CRYPT
     if (ret == WC_PENDING_E) {
-        ret = wolfSSL_AsyncPush(ssl, &key->asyncDev, WC_ASYNC_FLAG_CALL_AGAIN);
+        ret = wolfSSL_AsyncPush(ssl, &key->asyncDev);
     }
 #endif /* WOLFSSL_ASYNC_CRYPT */
 
@@ -3255,6 +3283,13 @@ int RsaEnc(WOLFSSL* ssl, const byte* in, word32 inSz, byte* out, word32* outSz,
 
     WOLFSSL_ENTER("RsaEnc");
 
+#ifdef WOLFSSL_ASYNC_CRYPT
+    /* intialize event */
+    ret = wolfSSL_AsyncInit(ssl, &key->asyncDev, WC_ASYNC_FLAG_CALL_AGAIN);
+    if (ret != 0)
+        return ret;
+#endif
+
 #ifdef HAVE_PK_CALLBACKS
     if (ssl->ctx->RsaEncCb) {
             ret = ssl->ctx->RsaEncCb(ssl, in, inSz, out, outSz, keyBuf, keySz,
@@ -3267,9 +3302,9 @@ int RsaEnc(WOLFSSL* ssl, const byte* in, word32 inSz, byte* out, word32* outSz,
     }
 
     /* Handle async pending response */
-#if defined(WOLFSSL_ASYNC_CRYPT)
+#ifdef WOLFSSL_ASYNC_CRYPT
     if (ret == WC_PENDING_E) {
-        ret = wolfSSL_AsyncPush(ssl, &key->asyncDev, WC_ASYNC_FLAG_CALL_AGAIN);
+        ret = wolfSSL_AsyncPush(ssl, &key->asyncDev);
     }
 #endif /* WOLFSSL_ASYNC_CRYPT */
 
@@ -3300,6 +3335,13 @@ int EccSign(WOLFSSL* ssl, const byte* in, word32 inSz, byte* out,
 
     WOLFSSL_ENTER("EccSign");
 
+#ifdef WOLFSSL_ASYNC_CRYPT
+    /* intialize event */
+    ret = wolfSSL_AsyncInit(ssl, &key->asyncDev, WC_ASYNC_FLAG_CALL_AGAIN);
+    if (ret != 0)
+        return ret;
+#endif
+
 #if defined(HAVE_PK_CALLBACKS)
     if (ssl->ctx->EccSignCb) {
         ret = ssl->ctx->EccSignCb(ssl, in, inSz, out, outSz, keyBuf,
@@ -3312,9 +3354,9 @@ int EccSign(WOLFSSL* ssl, const byte* in, word32 inSz, byte* out,
     }
 
     /* Handle async pending response */
-#if defined(WOLFSSL_ASYNC_CRYPT)
+#ifdef WOLFSSL_ASYNC_CRYPT
     if (ret == WC_PENDING_E) {
-        ret = wolfSSL_AsyncPush(ssl, &key->asyncDev, WC_ASYNC_FLAG_CALL_AGAIN);
+        ret = wolfSSL_AsyncPush(ssl, &key->asyncDev);
     }
 #endif /* WOLFSSL_ASYNC_CRYPT */
 
@@ -3336,6 +3378,13 @@ int EccVerify(WOLFSSL* ssl, const byte* in, word32 inSz, const byte* out,
 
     WOLFSSL_ENTER("EccVerify");
 
+#ifdef WOLFSSL_ASYNC_CRYPT
+    /* intialize event */
+    ret = wolfSSL_AsyncInit(ssl, &key->asyncDev, WC_ASYNC_FLAG_CALL_AGAIN);
+    if (ret != 0)
+        return ret;
+#endif
+
 #ifdef HAVE_PK_CALLBACKS
     if (ssl->ctx->EccVerifyCb) {
         ret = ssl->ctx->EccVerifyCb(ssl, in, inSz, out, outSz, keyBuf, keySz,
@@ -3348,9 +3397,9 @@ int EccVerify(WOLFSSL* ssl, const byte* in, word32 inSz, const byte* out,
     }
 
     /* Handle async pending response */
-#if defined(WOLFSSL_ASYNC_CRYPT)
+#ifdef WOLFSSL_ASYNC_CRYPT
     if (ret == WC_PENDING_E) {
-        ret = wolfSSL_AsyncPush(ssl, &key->asyncDev, WC_ASYNC_FLAG_CALL_AGAIN);
+        ret = wolfSSL_AsyncPush(ssl, &key->asyncDev);
     }
     else
 #endif /* WOLFSSL_ASYNC_CRYPT */
@@ -3422,9 +3471,13 @@ int EccSharedSecret(WOLFSSL* ssl, ecc_key* priv_key, ecc_key* pub_key,
         int side, void* ctx)
 {
     int ret;
-#ifdef WOLFSSL_ASYNC_CRYPT
+#ifdef HAVE_PK_CALLBACKS
-    WC_ASYNC_DEV* asyncDev;
+    ecc_key* otherKey = NULL;
 #endif
+#ifdef WOLFSSL_ASYNC_CRYPT
+    WC_ASYNC_DEV* asyncDev = &priv_key->asyncDev;
+#endif
+
     (void)ssl;
     (void)pubKeyDer;
     (void)pubKeySz;
@@ -3435,30 +3488,37 @@ int EccSharedSecret(WOLFSSL* ssl, ecc_key* priv_key, ecc_key* pub_key,
 
 #ifdef HAVE_PK_CALLBACKS
     if (ssl->ctx->EccSharedSecretCb) {
-        ecc_key* otherKey = NULL;
-
         ret = EccGetKey(ssl, &otherKey);
-        if (ret == 0) {
+        if (ret != 0)
-            ret = ssl->ctx->EccSharedSecretCb(ssl, otherKey, pubKeyDer,
+            return ret;
-                pubKeySz, out, outlen, side, ctx);
-        }
     #ifdef WOLFSSL_ASYNC_CRYPT
         asyncDev = &otherKey->asyncDev;
     #endif
     }
+#endif
+
+#ifdef WOLFSSL_ASYNC_CRYPT
+    /* intialize event */
+    ret = wolfSSL_AsyncInit(ssl, asyncDev, WC_ASYNC_FLAG_CALL_AGAIN);
+    if (ret != 0)
+        return ret;
+#endif
+
+#ifdef HAVE_PK_CALLBACKS
+    if (ssl->ctx->EccSharedSecretCb) {
+        ret = ssl->ctx->EccSharedSecretCb(ssl, otherKey, pubKeyDer,
+            pubKeySz, out, outlen, side, ctx);
+    }
     else
 #endif
     {
         ret = wc_ecc_shared_secret(priv_key, pub_key, out, outlen);
-    #ifdef WOLFSSL_ASYNC_CRYPT
-        asyncDev = &priv_key->asyncDev;
-    #endif
     }
 
     /* Handle async pending response */
-#if defined(WOLFSSL_ASYNC_CRYPT)
+#ifdef WOLFSSL_ASYNC_CRYPT
     if (ret == WC_PENDING_E) {
-        ret = wolfSSL_AsyncPush(ssl, asyncDev, WC_ASYNC_FLAG_CALL_AGAIN);
+        ret = wolfSSL_AsyncPush(ssl, asyncDev);
     }
 #endif /* WOLFSSL_ASYNC_CRYPT */
 
@@ -3474,6 +3534,13 @@ int EccMakeKey(WOLFSSL* ssl, ecc_key* key, ecc_key* peer)
 
     WOLFSSL_ENTER("EccMakeKey");
 
+#ifdef WOLFSSL_ASYNC_CRYPT
+    /* intialize event */
+    ret = wolfSSL_AsyncInit(ssl, &key->asyncDev, WC_ASYNC_FLAG_NONE);
+    if (ret != 0)
+        return ret;
+#endif
+
     if (peer == NULL) {
         keySz = ssl->eccTempKeySz;
     }
@@ -3492,9 +3559,9 @@ int EccMakeKey(WOLFSSL* ssl, ecc_key* key, ecc_key* peer)
     }
 
     /* Handle async pending response */
-#if defined(WOLFSSL_ASYNC_CRYPT)
+#ifdef WOLFSSL_ASYNC_CRYPT
     if (ret == WC_PENDING_E) {
-        ret = wolfSSL_AsyncPush(ssl, &key->asyncDev, WC_ASYNC_FLAG_NONE);
+        ret = wolfSSL_AsyncPush(ssl, &key->asyncDev);
     }
 #endif /* WOLFSSL_ASYNC_CRYPT */
 
@@ -3530,6 +3597,13 @@ int Ed25519Sign(WOLFSSL* ssl, const byte* in, word32 inSz, byte* out,
 
     WOLFSSL_ENTER("Ed25519Sign");
 
+#ifdef WOLFSSL_ASYNC_CRYPT
+    /* intialize event */
+    ret = wolfSSL_AsyncInit(ssl, &key->asyncDev, WC_ASYNC_FLAG_CALL_AGAIN);
+    if (ret != 0)
+        return ret;
+#endif
+
 #if defined(HAVE_PK_CALLBACKS)
     if (ssl->ctx->Ed25519SignCb) {
         ret = ssl->ctx->Ed25519SignCb(ssl, in, inSz, out, outSz, keyBuf,
@@ -3542,9 +3616,9 @@ int Ed25519Sign(WOLFSSL* ssl, const byte* in, word32 inSz, byte* out,
     }
 
     /* Handle async pending response */
-#if defined(WOLFSSL_ASYNC_CRYPT)
+#ifdef WOLFSSL_ASYNC_CRYPT
     if (ret == WC_PENDING_E) {
-        ret = wolfSSL_AsyncPush(ssl, &key->asyncDev, WC_ASYNC_FLAG_CALL_AGAIN);
+        ret = wolfSSL_AsyncPush(ssl, &key->asyncDev);
     }
 #endif /* WOLFSSL_ASYNC_CRYPT */
 
@@ -3578,6 +3652,13 @@ int Ed25519Verify(WOLFSSL* ssl, const byte* in, word32 inSz, const byte* msg,
 
     WOLFSSL_ENTER("Ed25519Verify");
 
+#ifdef WOLFSSL_ASYNC_CRYPT
+    /* intialize event */
+    ret = wolfSSL_AsyncInit(ssl, &key->asyncDev, WC_ASYNC_FLAG_CALL_AGAIN);
+    if (ret != 0)
+        return ret;
+#endif
+
 #ifdef HAVE_PK_CALLBACKS
     if (ssl->ctx->Ed25519VerifyCb) {
         ret = ssl->ctx->Ed25519VerifyCb(ssl, in, inSz, msg, msgSz, keyBuf,
@@ -3591,9 +3672,9 @@ int Ed25519Verify(WOLFSSL* ssl, const byte* in, word32 inSz, const byte* msg,
     }
 
     /* Handle async pending response */
-#if defined(WOLFSSL_ASYNC_CRYPT)
+#ifdef WOLFSSL_ASYNC_CRYPT
     if (ret == WC_PENDING_E) {
-        ret = wolfSSL_AsyncPush(ssl, &key->asyncDev, WC_ASYNC_FLAG_CALL_AGAIN);
+        ret = wolfSSL_AsyncPush(ssl, &key->asyncDev);
     }
     else
 #endif /* WOLFSSL_ASYNC_CRYPT */
@@ -3659,6 +3740,13 @@ static int X25519SharedSecret(WOLFSSL* ssl, curve25519_key* priv_key,
 
     WOLFSSL_ENTER("X25519SharedSecret");
 
+#ifdef WOLFSSL_ASYNC_CRYPT
+    /* intialize event */
+    ret = wolfSSL_AsyncInit(ssl, &priv_key->asyncDev, WC_ASYNC_FLAG_CALL_AGAIN);
+    if (ret != 0)
+        return ret;
+#endif
+
 #ifdef HAVE_PK_CALLBACKS
     if (ssl->ctx->X25519SharedSecretCb) {
         curve25519_key* otherKey = NULL;
@@ -3677,10 +3765,9 @@ static int X25519SharedSecret(WOLFSSL* ssl, curve25519_key* priv_key,
     }
 
     /* Handle async pending response */
-#if defined(WOLFSSL_ASYNC_CRYPT)
+#ifdef WOLFSSL_ASYNC_CRYPT
     if (ret == WC_PENDING_E) {
-        ret = wolfSSL_AsyncPush(ssl, &priv_key->asyncDev,
+        ret = wolfSSL_AsyncPush(ssl, &priv_key->asyncDev);
-                                                    WC_ASYNC_FLAG_CALL_AGAIN);
     }
 #endif /* WOLFSSL_ASYNC_CRYPT */
 
@@ -3698,14 +3785,21 @@ static int X25519MakeKey(WOLFSSL* ssl, curve25519_key* key,
 
     WOLFSSL_ENTER("X25519MakeKey");
 
+#ifdef WOLFSSL_ASYNC_CRYPT
+    /* intialize event */
+    ret = wolfSSL_AsyncInit(ssl, &key->asyncDev, WC_ASYNC_FLAG_NONE);
+    if (ret != 0)
+        return ret;
+#endif
+
     ret = wc_curve25519_make_key(ssl->rng, CURVE25519_KEYSIZE, key);
     if (ret == 0)
         ssl->ecdhCurveOID = ECC_X25519_OID;
 
     /* Handle async pending response */
-#if defined(WOLFSSL_ASYNC_CRYPT)
+#ifdef WOLFSSL_ASYNC_CRYPT
     if (ret == WC_PENDING_E) {
-        ret = wolfSSL_AsyncPush(ssl, &key->asyncDev, WC_ASYNC_FLAG_NONE);
+        ret = wolfSSL_AsyncPush(ssl, &key->asyncDev);
     }
 #endif /* WOLFSSL_ASYNC_CRYPT */
 
@@ -3728,12 +3822,19 @@ int DhGenKeyPair(WOLFSSL* ssl, DhKey* dhKey,
 
     WOLFSSL_ENTER("DhGenKeyPair");
 
+#ifdef WOLFSSL_ASYNC_CRYPT
+    /* intialize event */
+    ret = wolfSSL_AsyncInit(ssl, &dhKey->asyncDev, WC_ASYNC_FLAG_NONE);
+    if (ret != 0)
+        return ret;
+#endif
+
     ret = wc_DhGenerateKeyPair(dhKey, ssl->rng, priv, privSz, pub, pubSz);
 
     /* Handle async pending response */
-#if defined(WOLFSSL_ASYNC_CRYPT)
+#ifdef WOLFSSL_ASYNC_CRYPT
     if (ret == WC_PENDING_E) {
-        ret = wolfSSL_AsyncPush(ssl, &dhKey->asyncDev, WC_ASYNC_FLAG_NONE);
+        ret = wolfSSL_AsyncPush(ssl, &dhKey->asyncDev);
     }
 #endif /* WOLFSSL_ASYNC_CRYPT */
 
@@ -3753,12 +3854,19 @@ int DhAgree(WOLFSSL* ssl, DhKey* dhKey,
 
     WOLFSSL_ENTER("DhAgree");
 
+#ifdef WOLFSSL_ASYNC_CRYPT
+    /* intialize event */
+    ret = wolfSSL_AsyncInit(ssl, &dhKey->asyncDev, WC_ASYNC_FLAG_NONE);
+    if (ret != 0)
+        return ret;
+#endif
+
     ret = wc_DhAgree(dhKey, agree, agreeSz, priv, privSz, otherPub, otherPubSz);
 
     /* Handle async pending response */
-#if defined(WOLFSSL_ASYNC_CRYPT)
+#ifdef WOLFSSL_ASYNC_CRYPT
     if (ret == WC_PENDING_E) {
-        ret = wolfSSL_AsyncPush(ssl, &dhKey->asyncDev, WC_ASYNC_FLAG_NONE);
+        ret = wolfSSL_AsyncPush(ssl, &dhKey->asyncDev);
     }
 #endif /* WOLFSSL_ASYNC_CRYPT */
 
@@ -7902,14 +8010,24 @@ int ProcessPeerCerts(WOLFSSL* ssl, byte* input, word32* inOutIdx,
                         args->dCertInit = 1;
                     }
 
+                #ifdef WOLFSSL_ASYNC_CRYPT
+                    /* intialize event */
+                    if (args->dCert->sigCtx.asyncDev) {
+                        ret = wolfSSL_AsyncInit(ssl,
+                            args->dCert->sigCtx.asyncDev,
+                            WC_ASYNC_FLAG_CALL_AGAIN);
+                        if (ret != 0)
+                            break;
+                    }
+                #endif
+
                     ret = ParseCertRelative(args->dCert, CERT_TYPE, 0,
                                                                 ssl->ctx->cm);
                     if (ret != 0) {
                     #ifdef WOLFSSL_ASYNC_CRYPT
                         if (ret == WC_PENDING_E) {
                             ret = wolfSSL_AsyncPush(ssl,
-                                args->dCert->sigCtx.asyncDev,
+                                args->dCert->sigCtx.asyncDev);
-                                WC_ASYNC_FLAG_CALL_AGAIN);
                         }
                     #endif
                         goto exit_ppc;
@@ -7967,14 +8085,24 @@ int ProcessPeerCerts(WOLFSSL* ssl, byte* input, word32* inOutIdx,
                         args->dCertInit = 1;
                     }
 
+                #ifdef WOLFSSL_ASYNC_CRYPT
+                    /* intialize event */
+                    if (args->dCert->sigCtx.asyncDev) {
+                        ret = wolfSSL_AsyncInit(ssl,
+                            args->dCert->sigCtx.asyncDev,
+                            WC_ASYNC_FLAG_CALL_AGAIN);
+                        if (ret != 0)
+                            break;
+                    }
+                #endif
+
                     ret = ParseCertRelative(args->dCert, CERT_TYPE, 0,
                                                                   ssl->ctx->cm);
                     if (ret != 0) {
                     #ifdef WOLFSSL_ASYNC_CRYPT
                         if (ret == WC_PENDING_E) {
                             ret = wolfSSL_AsyncPush(ssl,
-                                                   args->dCert->sigCtx.asyncDev,
+                                args->dCert->sigCtx.asyncDev);
-                                                   WC_ASYNC_FLAG_CALL_AGAIN);
                         }
                     #endif
                         goto exit_ppc;
@@ -8011,13 +8139,23 @@ int ProcessPeerCerts(WOLFSSL* ssl, byte* input, word32* inOutIdx,
                         args->dCertInit = 1;
                     }
 
+                #ifdef WOLFSSL_ASYNC_CRYPT
+                    /* intialize event */
+                    if (args->dCert->sigCtx.asyncDev) {
+                        ret = wolfSSL_AsyncInit(ssl,
+                            args->dCert->sigCtx.asyncDev,
+                            WC_ASYNC_FLAG_CALL_AGAIN);
+                        if (ret != 0)
+                            goto exit_ppc;
+                    }
+                #endif
+
                     ret = ParseCertRelative(args->dCert, CERT_TYPE,
                                     !ssl->options.verifyNone, ssl->ctx->cm);
                 #ifdef WOLFSSL_ASYNC_CRYPT
                     if (ret == WC_PENDING_E) {
                         ret = wolfSSL_AsyncPush(ssl,
-                            args->dCert->sigCtx.asyncDev,
+                            args->dCert->sigCtx.asyncDev);
-                            WC_ASYNC_FLAG_CALL_AGAIN);
                         goto exit_ppc;
                     }
                 #endif
@@ -8203,14 +8341,25 @@ int ProcessPeerCerts(WOLFSSL* ssl, byte* input, word32* inOutIdx,
             #ifdef WOLFSSL_TRUST_PEER_CERT
                 if (!haveTrustPeer)
             #endif
-                { /* only parse if not already present in dCert from above */
+                {
+                #ifdef WOLFSSL_ASYNC_CRYPT
+                    /* intialize event */
+                    if (args->dCert->sigCtx.asyncDev) {
+                        ret = wolfSSL_AsyncInit(ssl,
+                            args->dCert->sigCtx.asyncDev,
+                            WC_ASYNC_FLAG_CALL_AGAIN);
+                        if (ret != 0)
+                            goto exit_ppc;
+                    }
+                #endif
+
+                    /* only parse if not already present in dCert from above */
                     ret = ParseCertRelative(args->dCert, CERT_TYPE,
                                     !ssl->options.verifyNone, ssl->ctx->cm);
                 #ifdef WOLFSSL_ASYNC_CRYPT
                     if (ret == WC_PENDING_E) {
                         ret = wolfSSL_AsyncPush(ssl,
-                            args->dCert->sigCtx.asyncDev,
+                            args->dCert->sigCtx.asyncDev);
-                            WC_ASYNC_FLAG_CALL_AGAIN);
                         goto exit_ppc;
                     }
                 #endif
@@ -10464,12 +10613,18 @@ static INLINE int EncryptDo(WOLFSSL* ssl, byte* out, const byte* input,
 
     #ifdef BUILD_DES3
         case wolfssl_triple_des:
+        #ifdef WOLFSSL_ASYNC_CRYPT
+            /* intialize event */
+            asyncDev = &ssl->encrypt.des3->asyncDev;
+            ret = wolfSSL_AsyncInit(ssl, asyncDev, event_flags);
+            if (ret != 0)
+                break;
+        #endif
+
             ret = wc_Des3_CbcEncrypt(ssl->encrypt.des3, out, input, sz);
         #ifdef WOLFSSL_ASYNC_CRYPT
-            if (ret == WC_PENDING_E) {
+            if (ret == WC_PENDING_E && asyncOkay) {
-                asyncDev = &ssl->encrypt.des3->asyncDev;
+                ret = wolfSSL_AsyncPush(ssl, asyncDev);
-                if (asyncOkay)
-                    ret = wolfSSL_AsyncPush(ssl, asyncDev, event_flags);
             }
         #endif
             break;
@@ -10477,13 +10632,18 @@ static INLINE int EncryptDo(WOLFSSL* ssl, byte* out, const byte* input,
 
     #ifdef BUILD_AES
         case wolfssl_aes:
+        #ifdef WOLFSSL_ASYNC_CRYPT
+            /* intialize event */
+            asyncDev = &ssl->encrypt.aes->asyncDev;
+            ret = wolfSSL_AsyncInit(ssl, asyncDev, event_flags);
+            if (ret != 0)
+                break;
+        #endif
+
             ret = wc_AesCbcEncrypt(ssl->encrypt.aes, out, input, sz);
         #ifdef WOLFSSL_ASYNC_CRYPT
-            if (ret == WC_PENDING_E) {
+            if (ret == WC_PENDING_E && asyncOkay) {
-                asyncDev = &ssl->encrypt.aes->asyncDev;
+                ret = wolfSSL_AsyncPush(ssl, asyncDev);
-                if (asyncOkay)
-                    ret = wolfSSL_AsyncPush(ssl, asyncDev, event_flags);
-                break;
             }
         #endif
             break;
@@ -10495,6 +10655,15 @@ static INLINE int EncryptDo(WOLFSSL* ssl, byte* out, const byte* input,
         {
             wc_AesAuthEncryptFunc aes_auth_fn;
             const byte* additionalSrc;
+
+        #ifdef WOLFSSL_ASYNC_CRYPT
+            /* intialize event */
+            asyncDev = &ssl->encrypt.aes->asyncDev;
+            ret = wolfSSL_AsyncInit(ssl, asyncDev, event_flags);
+            if (ret != 0)
+                break;
+        #endif
+
         #if defined(BUILD_AESGCM) && defined(HAVE_AESCCM)
             aes_auth_fn = (ssl->specs.bulk_cipher_algorithm == wolfssl_aes_gcm)
                             ? wc_AesGcmEncrypt : wc_AesCcmEncrypt;
@@ -10536,10 +10705,8 @@ static INLINE int EncryptDo(WOLFSSL* ssl, byte* out, const byte* input,
                     ssl->specs.aead_mac_size,
                     ssl->encrypt.additional, AEAD_AUTH_DATA_SZ);
         #ifdef WOLFSSL_ASYNC_CRYPT
-            if (ret == WC_PENDING_E) {
+            if (ret == WC_PENDING_E && asyncOkay) {
-                asyncDev = &ssl->encrypt.aes->asyncDev;
+                ret = wolfSSL_AsyncPush(ssl, asyncDev);
-                if (asyncOkay)
-                    ret = wolfSSL_AsyncPush(ssl, asyncDev, event_flags);
             }
         #endif
         }
@@ -10710,11 +10877,18 @@ static INLINE int DecryptDo(WOLFSSL* ssl, byte* plain, const byte* input,
 
     #ifdef BUILD_DES3
         case wolfssl_triple_des:
+        #ifdef WOLFSSL_ASYNC_CRYPT
+            /* intialize event */
+            ret = wolfSSL_AsyncInit(ssl, &ssl->decrypt.des3->asyncDev,
+                WC_ASYNC_FLAG_CALL_AGAIN);
+            if (ret != 0)
+                break;
+        #endif
+
             ret = wc_Des3_CbcDecrypt(ssl->decrypt.des3, plain, input, sz);
         #ifdef WOLFSSL_ASYNC_CRYPT
             if (ret == WC_PENDING_E) {
-                ret = wolfSSL_AsyncPush(ssl, &ssl->decrypt.des3->asyncDev,
+                ret = wolfSSL_AsyncPush(ssl, &ssl->decrypt.des3->asyncDev);
-                                                    WC_ASYNC_FLAG_CALL_AGAIN);
             }
         #endif
             break;
@@ -10722,11 +10896,18 @@ static INLINE int DecryptDo(WOLFSSL* ssl, byte* plain, const byte* input,
 
     #ifdef BUILD_AES
         case wolfssl_aes:
+        #ifdef WOLFSSL_ASYNC_CRYPT
+            /* intialize event */
+            ret = wolfSSL_AsyncInit(ssl, &ssl->decrypt.aes->asyncDev,
+                WC_ASYNC_FLAG_CALL_AGAIN);
+            if (ret != 0)
+                break;
+        #endif
+
             ret = wc_AesCbcDecrypt(ssl->decrypt.aes, plain, input, sz);
         #ifdef WOLFSSL_ASYNC_CRYPT
             if (ret == WC_PENDING_E) {
-                ret = wolfSSL_AsyncPush(ssl, &ssl->decrypt.aes->asyncDev,
+                ret = wolfSSL_AsyncPush(ssl, &ssl->decrypt.aes->asyncDev);
-                                                    WC_ASYNC_FLAG_CALL_AGAIN);
             }
         #endif
             break;
@@ -10737,6 +10918,15 @@ static INLINE int DecryptDo(WOLFSSL* ssl, byte* plain, const byte* input,
         case wolfssl_aes_ccm: /* GCM AEAD macros use same size as CCM */
         {
             wc_AesAuthDecryptFunc aes_auth_fn;
+
+        #ifdef WOLFSSL_ASYNC_CRYPT
+            /* intialize event */
+            ret = wolfSSL_AsyncInit(ssl, &ssl->decrypt.aes->asyncDev,
+                WC_ASYNC_FLAG_CALL_AGAIN);
+            if (ret != 0)
+                break;
+        #endif
+
         #if defined(BUILD_AESGCM) && defined(HAVE_AESCCM)
             aes_auth_fn = (ssl->specs.bulk_cipher_algorithm == wolfssl_aes_gcm)
                             ? wc_AesGcmDecrypt : wc_AesCcmDecrypt;
@@ -10771,9 +10961,7 @@ static INLINE int DecryptDo(WOLFSSL* ssl, byte* plain, const byte* input,
                         ssl->decrypt.additional, AEAD_AUTH_DATA_SZ)) < 0) {
             #ifdef WOLFSSL_ASYNC_CRYPT
                 if (ret == WC_PENDING_E) {
-                    ret = wolfSSL_AsyncPush(ssl,
+                    ret = wolfSSL_AsyncPush(ssl, &ssl->decrypt.aes->asyncDev);
-                        &ssl->decrypt.aes->asyncDev, WC_ASYNC_FLAG_CALL_AGAIN);
-                    break;
                 }
             #endif
             }
@@ -24419,7 +24607,7 @@ int wolfSSL_AsyncPop(WOLFSSL* ssl, byte* state)
     return ret;
 }
 
-int wolfSSL_AsyncPush(WOLFSSL* ssl, WC_ASYNC_DEV* asyncDev, word32 flags)
+int wolfSSL_AsyncInit(WOLFSSL* ssl, WC_ASYNC_DEV* asyncDev, word32 flags)
 {
     int ret;
     WOLF_EVENT* event;
@@ -24433,13 +24621,30 @@ int wolfSSL_AsyncPush(WOLFSSL* ssl, WC_ASYNC_DEV* asyncDev, word32 flags)
 
     /* init event */
     ret = wolfAsync_EventInit(event, WOLF_EVENT_TYPE_ASYNC_WOLFSSL, ssl, flags);
-    if (ret == 0) {
-        ssl->async.dev = asyncDev;
 
-        /* place event into queue */
+    WOLFSSL_LEAVE("wolfSSL_AsyncInit", ret);
-        ret = wolfAsync_EventQueuePush(&ssl->ctx->event_queue, event);
+
+    return ret;
+}
+
+int wolfSSL_AsyncPush(WOLFSSL* ssl, WC_ASYNC_DEV* asyncDev)
+{
+    int ret;
+    WOLF_EVENT* event;
+
+    if (ssl == NULL || asyncDev == NULL) {
+        return BAD_FUNC_ARG;
     }
 
+    /* grab event pointer */
+    event = &asyncDev->event;
+
+    /* store reference to active async operation */
+    ssl->async.dev = asyncDev;
+
+    /* place event into queue */
+    ret = wolfAsync_EventQueuePush(&ssl->ctx->event_queue, event);
+
     /* success means return WC_PENDING_E */
     if (ret == 0) {
         ret = WC_PENDING_E;

src/tls13.c

@@ -1618,25 +1618,35 @@ static int EncryptTls13(WOLFSSL* ssl, byte* output, const byte* input,
             switch (ssl->specs.bulk_cipher_algorithm) {
             #ifdef BUILD_AESGCM
                 case wolfssl_aes_gcm:
+                #ifdef WOLFSSL_ASYNC_CRYPT
+                    /* intialize event */
+                    asyncDev = &ssl->encrypt.aes->asyncDev;
+                    ret = wolfSSL_AsyncInit(ssl, asyncDev, event_flags);
+                    if (ret != 0)
+                        break;
+                #endif
+
                     nonceSz = AESGCM_NONCE_SZ;
                     ret = wc_AesGcmEncrypt(ssl->encrypt.aes, output, input,
                         dataSz, ssl->encrypt.nonce, nonceSz,
                         output + dataSz, macSz, NULL, 0);
-                #ifdef WOLFSSL_ASYNC_CRYPT
-                    asyncDev = &ssl->encrypt.aes->asyncDev;
-                #endif
                     break;
             #endif
 
             #ifdef HAVE_AESCCM
                 case wolfssl_aes_ccm:
+                #ifdef WOLFSSL_ASYNC_CRYPT
+                    /* intialize event */
+                    asyncDev = &ssl->encrypt.aes->asyncDev;
+                    ret = wolfSSL_AsyncInit(ssl, asyncDev, event_flags);
+                    if (ret != 0)
+                        break;
+                #endif
+
                     nonceSz = AESCCM_NONCE_SZ;
                     ret = wc_AesCcmEncrypt(ssl->encrypt.aes, output, input,
                         dataSz, ssl->encrypt.nonce, nonceSz,
                         output + dataSz, macSz, NULL, 0);
-                #ifdef WOLFSSL_ASYNC_CRYPT
-                    asyncDev = &ssl->encrypt.aes->asyncDev;
-                #endif
                     break;
             #endif
 
@@ -1663,9 +1673,7 @@ static int EncryptTls13(WOLFSSL* ssl, byte* output, const byte* input,
                 }
                 else {
                     /* If pending, then leave and return will resume below */
-                    ret = wolfSSL_AsyncPush(ssl, asyncDev, event_flags);
+                    return wolfSSL_AsyncPush(ssl, asyncDev);
-
-                    return ret;
                 }
             }
         #endif
@@ -1824,14 +1832,21 @@ int DecryptTls13(WOLFSSL* ssl, byte* output, const byte* input, word16 sz)
             switch (ssl->specs.bulk_cipher_algorithm) {
             #ifdef BUILD_AESGCM
                 case wolfssl_aes_gcm:
+                #ifdef WOLFSSL_ASYNC_CRYPT
+                    /* intialize event */
+                    ret = wolfSSL_AsyncInit(ssl, &ssl->decrypt.aes->asyncDev,
+                        WC_ASYNC_FLAG_CALL_AGAIN);
+                    if (ret != 0)
+                        break;
+                #endif
+
                     nonceSz = AESGCM_NONCE_SZ;
                     ret = wc_AesGcmDecrypt(ssl->decrypt.aes, output, input,
                         dataSz, ssl->decrypt.nonce, nonceSz,
                         input + dataSz, macSz, NULL, 0);
                 #ifdef WOLFSSL_ASYNC_CRYPT
                     if (ret == WC_PENDING_E) {
-                        ret = wolfSSL_AsyncPush(ssl, &ssl->decrypt.aes->asyncDev,
+                        ret = wolfSSL_AsyncPush(ssl, &ssl->decrypt.aes->asyncDev);
-                                                        WC_ASYNC_FLAG_CALL_AGAIN);
                     }
                 #endif
                     break;
@@ -1839,14 +1854,21 @@ int DecryptTls13(WOLFSSL* ssl, byte* output, const byte* input, word16 sz)
 
             #ifdef HAVE_AESCCM
                 case wolfssl_aes_ccm:
+                #ifdef WOLFSSL_ASYNC_CRYPT
+                    /* intialize event */
+                    ret = wolfSSL_AsyncInit(ssl, &ssl->decrypt.aes->asyncDev,
+                        WC_ASYNC_FLAG_CALL_AGAIN);
+                    if (ret != 0)
+                        break;
+                #endif
+
                     nonceSz = AESCCM_NONCE_SZ;
                     ret = wc_AesCcmDecrypt(ssl->decrypt.aes, output, input,
                         dataSz, ssl->decrypt.nonce, nonceSz,
                         input + dataSz, macSz, NULL, 0);
                 #ifdef WOLFSSL_ASYNC_CRYPT
                     if (ret == WC_PENDING_E) {
-                        ret = wolfSSL_AsyncPush(ssl, &ssl->decrypt.aes->asyncDev,
+                        ret = wolfSSL_AsyncPush(ssl, &ssl->decrypt.aes->asyncDev);
-                                                        WC_ASYNC_FLAG_CALL_AGAIN);
                     }
                 #endif
                     break;

wolfcrypt/benchmark/benchmark.c

@@ -3329,7 +3329,7 @@ void bench_curve25519KeyGen(void)
 {
     curve25519_key genKey;
     double start;
-    int    ret, i, count;
+    int    ret = 0, i, count;
 
     /* Key Gen */
     bench_stats_start(&count, &start);

wolfcrypt/src/asn.c

@@ -4614,25 +4614,25 @@ static int ConfirmSignature(SignatureCtx* sigCtx,
                     if (sigCtx->key.rsa == NULL || sigCtx->plain == NULL) {
                         ERROR_OUT(MEMORY_E, exit_cs);
                     }
-
                     if ((ret = wc_InitRsaKey_ex(sigCtx->key.rsa, sigCtx->heap,
                                                         sigCtx->devId)) != 0) {
                         goto exit_cs;
                     }
-
                     if (sigSz > MAX_ENCODED_SIG_SZ) {
                         WOLFSSL_MSG("Verify Signature is too big");
                         ERROR_OUT(BUFFER_E, exit_cs);
                     }
-
                     if ((ret = wc_RsaPublicKeyDecode(key, &idx, sigCtx->key.rsa,
                                                                  keySz)) != 0) {
                         WOLFSSL_MSG("ASN Key decode error RSA");
                         goto exit_cs;
                     }
-
                     XMEMCPY(sigCtx->plain, sig, sigSz);
                     sigCtx->out = NULL;
+
+                #ifdef WOLFSSL_ASYNC_CRYPT
+                    sigCtx->asyncDev = &sigCtx->key.rsa->asyncDev;
+                #endif
                     break;
                 }
             #endif /* !NO_RSA */
@@ -4645,7 +4645,6 @@ static int ConfirmSignature(SignatureCtx* sigCtx,
                     if (sigCtx->key.ecc == NULL) {
                         ERROR_OUT(MEMORY_E, exit_cs);
                     }
-
                     if ((ret = wc_ecc_init_ex(sigCtx->key.ecc, sigCtx->heap,
                                                           sigCtx->devId)) < 0) {
                         goto exit_cs;
@@ -4655,6 +4654,9 @@ static int ConfirmSignature(SignatureCtx* sigCtx,
                         WOLFSSL_MSG("ASN Key import error ECC");
                         goto exit_cs;
                     }
+                #ifdef WOLFSSL_ASYNC_CRYPT
+                    sigCtx->asyncDev = &sigCtx->key.ecc->asyncDev;
+                #endif
                     break;
                 }
             #endif /* HAVE_ECC */
@@ -4668,7 +4670,6 @@ static int ConfirmSignature(SignatureCtx* sigCtx,
                     if (sigCtx->key.ed25519 == NULL) {
                         ERROR_OUT(MEMORY_E, exit_cs);
                     }
-
                     if ((ret = wc_ed25519_init(sigCtx->key.ed25519)) < 0) {
                         goto exit_cs;
                     }
@@ -4677,6 +4678,9 @@ static int ConfirmSignature(SignatureCtx* sigCtx,
                         WOLFSSL_MSG("ASN Key import error ED25519");
                         goto exit_cs;
                     }
+                #ifdef WOLFSSL_ASYNC_CRYPT
+                    sigCtx->asyncDev = &sigCtx->key.ecc->asyncDev;
+                #endif
                     break;
                 }
             #endif
@@ -4691,6 +4695,13 @@ static int ConfirmSignature(SignatureCtx* sigCtx,
             }
 
             sigCtx->state = SIG_STATE_DO;
+
+        #ifdef WOLFSSL_ASYNC_CRYPT
+            /* always return here, so we can properly init the async
+               context back in SSL world */
+            ret = WC_PENDING_E;
+            goto exit_cs;
+        #endif
         } /* SIG_STATE_KEY */
         FALL_THROUGH;
 
@@ -4702,10 +4713,6 @@ static int ConfirmSignature(SignatureCtx* sigCtx,
                 {
                     ret = wc_RsaSSL_VerifyInline(sigCtx->plain, sigSz,
                                                 &sigCtx->out, sigCtx->key.rsa);
-                #ifdef WOLFSSL_ASYNC_CRYPT
-                    if (ret == WC_PENDING_E)
-                        sigCtx->asyncDev = &sigCtx->key.rsa->asyncDev;
-                #endif
                     break;
                 }
             #endif /* !NO_RSA */
@@ -4714,10 +4721,6 @@ static int ConfirmSignature(SignatureCtx* sigCtx,
                 {
                     ret = wc_ecc_verify_hash(sig, sigSz, sigCtx->digest,
                         sigCtx->digestSz, &sigCtx->verify, sigCtx->key.ecc);
-                #ifdef WOLFSSL_ASYNC_CRYPT
-                    if (ret == WC_PENDING_E)
-                        sigCtx->asyncDev = &sigCtx->key.ecc->asyncDev;
-                #endif
                     break;
                 }
             #endif /* HAVE_ECC */
@@ -4726,10 +4729,6 @@ static int ConfirmSignature(SignatureCtx* sigCtx,
                 {
                     ret = wc_ed25519_verify_msg(sig, sigSz, buf, bufSz,
                                           &sigCtx->verify, sigCtx->key.ed25519);
-                #ifdef WOLFSSL_ASYNC_CRYPT
-                    if (ret == WC_PENDING_E)
-                        sigCtx->asyncDev = &sigCtx->key.ecc->asyncDev;
-                #endif
                     break;
                 }
             #endif

wolfssl/internal.h

@@ -3820,9 +3820,9 @@ WOLFSSL_LOCAL int AllocKey(WOLFSSL* ssl, int type, void** pKey);
 WOLFSSL_LOCAL void FreeKey(WOLFSSL* ssl, int type, void** pKey);
 
 #ifdef WOLFSSL_ASYNC_CRYPT
+    WOLFSSL_LOCAL int wolfSSL_AsyncInit(WOLFSSL* ssl, WC_ASYNC_DEV* asyncDev, word32 flags);
     WOLFSSL_LOCAL int wolfSSL_AsyncPop(WOLFSSL* ssl, byte* state);
-    WOLFSSL_LOCAL int wolfSSL_AsyncPush(WOLFSSL* ssl, WC_ASYNC_DEV* asyncDev,
+    WOLFSSL_LOCAL int wolfSSL_AsyncPush(WOLFSSL* ssl, WC_ASYNC_DEV* asyncDev);
-                                        word32 flags);
 #endif
 
 

wolfssl/wolfcrypt/wolfevent.h

@@ -76,10 +76,13 @@ struct WOLF_EVENT {
 #ifdef HAVE_CAVIUM
     CavReqId            reqId;
 #endif
-    volatile int        ret;    /* Async return code */
+#ifndef WC_NO_ASYNC_THREADING
+    pthread_t           threadId;
+#endif
+    int                 ret;    /* Async return code */
     unsigned int        flags;
     WOLF_EVENT_TYPE     type;
-    volatile WOLF_EVENT_STATE state;
+    WOLF_EVENT_STATE    state;
 };
 
 enum WOLF_POLL_FLAGS {