Memory usage improvements

TLS 1.3 Server: don't cache the messages for Ed25519/Ed448 when doing
TLS 1.3.

ASN DecodeCertInternal: Call GetCertName for issuer and subject after
freeing the dataASN.

src/tls13.c

@@ -11063,6 +11063,30 @@ int DoTls13HandShakeMsgType(WOLFSSL* ssl, byte* input, word32* inOutIdx,
         echInOutIdx = *inOutIdx;
 #endif
         ret = DoTls13ClientHello(ssl, input, inOutIdx, size);
+    #if !defined(WOLFSSL_NO_CLIENT_AUTH) && \
+               ((defined(HAVE_ED25519) && !defined(NO_ED25519_CLIENT_AUTH)) || \
+                (defined(HAVE_ED448) && !defined(NO_ED448_CLIENT_AUTH)))
+        if ((ssl->options.resuming || !ssl->options.verifyPeer ||
+               !IsAtLeastTLSv1_2(ssl) || IsAtLeastTLSv1_3(ssl->version))
+        #ifdef WOLFSSL_DTLS13
+               && (!ssl->options.dtls)
+        #endif
+               ) {
+        #if defined(WOLFSSL_ASYNC_CRYPT) || defined(WOLFSSL_NONBLOCK_OCSP)
+            if (ret != WC_PENDING_E && ret != OCSP_WANT_READ)
+        #endif
+            {
+                ssl->options.cacheMessages = 0;
+                if ((ssl->hsHashes != NULL) &&
+                        (ssl->hsHashes->messages != NULL)) {
+                    ForceZero(ssl->hsHashes->messages, ssl->hsHashes->length);
+                    XFREE(ssl->hsHashes->messages, ssl->heap,
+                        DYNAMIC_TYPE_HASHES);
+                    ssl->hsHashes->messages = NULL;
+                }
+            }
+        }
+    #endif
 #if defined(HAVE_ECH)
         if (ret == 0) {
             echX = TLSX_Find(ssl->extensions, TLSX_ECH);

wolfcrypt/src/asn.c

@@ -20084,6 +20084,12 @@ static int DecodeCertInternal(DecodedCert* cert, int verify, int* criticalExt,
     byte version;
     word32 idx;
     word32 serialSz;
+    const unsigned char* issuer = NULL;
+    word32 issuerSz = 0;
+    const unsigned char* subject = NULL;
+    word32 subjectSz = 0;
+    word32 pubKeyOffset = 0;
+    word32 pubKeyEnd = 0;
     int done = 0;
 
     CALLOC_ASNGETDATA(dataASN, x509CertASN_Length, ret, cert->heap);
@@ -20116,6 +20122,7 @@ static int DecodeCertInternal(DecodedCert* cert, int verify, int* criticalExt,
     if (ret == 0) {
         int i;
 
+        pubKeyOffset = dataASN[X509CERTASN_IDX_TBS_SPUBKEYINFO_SEQ].offset;
         /* Set fields extracted from data. */
         cert->version = version;
         cert->serialSz = (int)serialSz;
@@ -20147,41 +20154,26 @@ static int DecodeCertInternal(DecodedCert* cert, int verify, int* criticalExt,
         cert->afterDate = GetASNItem_Addr(dataASN[i], cert->source);
         cert->afterDateLen = (int)GetASNItem_Length(dataASN[i], cert->source);
 
-        /* Get the issuer name and calculate hash. */
+        /* Get the issuer name. */
-        idx = dataASN[X509CERTASN_IDX_TBS_ISSUER_SEQ].offset;
+        issuer = cert->source + dataASN[X509CERTASN_IDX_TBS_ISSUER_SEQ].offset;
-        ret = GetCertName(cert, cert->issuer, cert->issuerHash, ISSUER,
+        issuerSz = dataASN[X509CERTASN_IDX_TBS_VALIDITY_SEQ].offset -
-                          cert->source, &idx,
+            dataASN[X509CERTASN_IDX_TBS_ISSUER_SEQ].offset;
-                          dataASN[X509CERTASN_IDX_TBS_VALIDITY_SEQ].offset);
     }
     if (ret == 0) {
-        /* Get the subject name and calculate hash. */
+        /* Get the subject name. */
-        idx = dataASN[X509CERTASN_IDX_TBS_SUBJECT_SEQ].offset;
+        subject = cert->source +
-        ret = GetCertName(cert, cert->subject, cert->subjectHash, SUBJECT,
+            dataASN[X509CERTASN_IDX_TBS_SUBJECT_SEQ].offset;
-                          cert->source, &idx,
+        subjectSz = dataASN[X509CERTASN_IDX_TBS_SPUBKEYINFO_SEQ].offset -
-                          dataASN[X509CERTASN_IDX_TBS_SPUBKEYINFO_SEQ].offset);
+            dataASN[X509CERTASN_IDX_TBS_SUBJECT_SEQ].offset;
     }
-    if (ret == 0) {
+    if ((ret == 0) && (stopAtPubKey)) {
-        /* Determine if self signed by comparing issuer and subject hashes. */
+        /* Return any bad date error through badDateRet and return offset of
-    #ifdef WOLFSSL_CERT_REQ
+         * subjectPublicKeyInfo.
-        if (cert->isCSR)
+         */
-            cert->selfSigned = 1;
+        if (badDateRet != NULL) {
-        else
+            *badDateRet = badDate;
-    #endif
-        {
-            cert->selfSigned = XMEMCMP(cert->issuerHash, cert->subjectHash,
-                                       KEYID_SIZE) == 0 ? 1 : 0;
-        }
-
-        if (stopAtPubKey) {
-            /* Return any bad date error through badDateRet and return offset of
-             * subjectPublicKeyInfo.
-             */
-            if (badDateRet != NULL) {
-                *badDateRet = badDate;
-            }
-            ret = (int)dataASN[X509CERTASN_IDX_TBS_SPUBKEYINFO_SEQ].offset;
-            done = 1;
         }
+        done = 1;
     }
 
     if ((ret == 0) && (!done)) {
@@ -20251,11 +20243,8 @@ static int DecodeCertInternal(DecodedCert* cert, int verify, int* criticalExt,
         #endif
     }
     if ((ret == 0) && (!done)) {
-        /* Parse the public key. */
+        pubKeyEnd = dataASN[X509CERTASN_IDX_TBS_ISSUERUID].offset;
-        idx = dataASN[X509CERTASN_IDX_TBS_SPUBKEYINFO_SEQ].offset;
+        if (stopAfterPubKey) {
-        ret = GetCertKey(cert, cert->source, &idx,
-                dataASN[X509CERTASN_IDX_TBS_ISSUERUID].offset);
-        if ((ret == 0) && stopAfterPubKey) {
             /* Return any bad date error through badDateRed and return offset
              * after subjectPublicKeyInfo.
              */
@@ -20290,7 +20279,42 @@ static int DecodeCertInternal(DecodedCert* cert, int verify, int* criticalExt,
     /* Dispose of memory before allocating for extension decoding. */
     FREE_ASNGETDATA(dataASN, cert->heap);
 
-    if ((ret == 0) && (!done) && (cert->extensions != NULL)) {
+    if ((ret == 0) && (issuer != NULL)) {
+        idx = 0;
+        /* Put issuer into cert and calculate hash. */
+        ret = GetCertName(cert, cert->issuer, cert->issuerHash, ISSUER, issuer,
+            &idx, issuerSz);
+    }
+    if ((ret == 0) && (subject != NULL)) {
+        idx = 0;
+        /* Put subject into cert and calculate hash. */
+        ret = GetCertName(cert, cert->subject, cert->subjectHash, SUBJECT,
+            subject, &idx, subjectSz);
+    }
+    if (ret == 0) {
+        /* Determine if self signed by comparing issuer and subject hashes. */
+    #ifdef WOLFSSL_CERT_REQ
+        if (cert->isCSR) {
+            cert->selfSigned = 1;
+        }
+        else
+    #endif
+        {
+            cert->selfSigned = (XMEMCMP(cert->issuerHash, cert->subjectHash,
+                                        KEYID_SIZE) == 0);
+        }
+        if (stopAtPubKey) {
+            ret = pubKeyOffset;
+        }
+    }
+
+    if ((ret == 0) && (!stopAtPubKey)) {
+        /* Parse the public key. */
+        idx = pubKeyOffset;
+        ret = GetCertKey(cert, cert->source, &idx, pubKeyEnd);
+    }
+    if ((ret == 0) && (!stopAtPubKey) && (!stopAfterPubKey) &&
+            (cert->extensions != NULL)) {
         /* Decode the extension data starting at [3]. */
         ret = DecodeCertExtensions(cert);
         if (criticalExt != NULL) {
@@ -21260,6 +21284,10 @@ static int CheckCertSignature_ex(const byte* cert, word32 certSz, void* heap,
     word32 sigParamsSz = 0;
     const byte* caName = NULL;
     word32 caNameLen = 0;
+#ifndef NO_SKID
+    const byte* akiData = NULL;
+    word32 akiLen = 0;
+#endif
 
     (void)req;
     (void)heap;
@@ -21269,17 +21297,6 @@ static int CheckCertSignature_ex(const byte* cert, word32 certSz, void* heap,
     }
 
     ALLOC_ASNGETDATA(dataASN, x509CertASN_Length, ret, heap);
-#ifdef WOLFSSL_SMALL_STACK
-    if (ret == 0) {
-        sigCtx = (SignatureCtx*)XMALLOC(sizeof(*sigCtx), heap,
-                                                        DYNAMIC_TYPE_SIGNATURE);
-        if (sigCtx == NULL) {
-            ret = MEMORY_E;
-        }
-    }
-#endif
-
-    InitSignatureCtx(sigCtx, heap, INVALID_DEVID);
 
     if ((ret == 0) && (!req)) {
         /* Clear dynamic data for certificate items. */
@@ -21379,16 +21396,22 @@ static int CheckCertSignature_ex(const byte* cert, word32 certSz, void* heap,
 #endif
     }
 
+#ifndef NO_SKID
+    if ((ret == 0) && (pubKey == NULL) && !req) {
+        akiData = dataASN[X509CERTASN_IDX_TBS_EXT_SEQ].data.ref.data;
+        akiLen = dataASN[X509CERTASN_IDX_TBS_EXT_SEQ].data.ref.length;
+    }
+#endif
+
+    FREE_ASNGETDATA(dataASN, heap);
+
     /* If no public passed, then find the CA. */
     if ((ret == 0) && (pubKey == NULL)) {
 #ifndef NO_SKID
         /* Find the AKI extension in list of extensions and get hash. */
-        if ((!req) &&
+        if ((!req) && (akiData != NULL)) {
-                (dataASN[X509CERTASN_IDX_TBS_EXT_SEQ].data.ref.data != NULL)) {
             /* TODO: test case */
-            ret = GetAKIHash(dataASN[X509CERTASN_IDX_TBS_EXT_SEQ].data.ref.data,
+            ret = GetAKIHash(akiData, akiLen, hash, &extAuthKeyIdSet, heap);
-                             dataASN[X509CERTASN_IDX_TBS_EXT_SEQ].data.ref.length,
-                             hash, &extAuthKeyIdSet, heap);
         }
 
         /* Get the CA by hash one was found. */
@@ -21417,22 +21440,32 @@ static int CheckCertSignature_ex(const byte* cert, word32 certSz, void* heap,
         }
     }
 
-    FREE_ASNGETDATA(dataASN, heap);
-
     if (ret == 0) {
-        /* Check signature. */
+    #ifdef WOLFSSL_SMALL_STACK
-        ret = ConfirmSignature(sigCtx, tbs, tbsSz, pubKey, pubKeySz, pubKeyOID,
+        sigCtx = (SignatureCtx*)XMALLOC(sizeof(*sigCtx), heap,
-                sig, sigSz, sigOID, sigParams, sigParamsSz, NULL);
+            DYNAMIC_TYPE_SIGNATURE);
-        if (ret != 0) {
+        if (sigCtx == NULL) {
-            WOLFSSL_MSG("Confirm signature failed");
+            ret = MEMORY_E;
+        }
+        if (ret == 0)
+    #endif
+        {
+            InitSignatureCtx(sigCtx, heap, INVALID_DEVID);
+
+            /* Check signature. */
+            ret = ConfirmSignature(sigCtx, tbs, tbsSz, pubKey, pubKeySz,
+                pubKeyOID, sig, sigSz, sigOID, sigParams, sigParamsSz, NULL);
+            if (ret != 0) {
+                WOLFSSL_MSG("Confirm signature failed");
+            }
+
+            FreeSignatureCtx(sigCtx);
+        #ifdef WOLFSSL_SMALL_STACK
+            XFREE(sigCtx, heap, DYNAMIC_TYPE_SIGNATURE);
+        #endif
         }
     }
 
-    FreeSignatureCtx(sigCtx);
-#ifdef WOLFSSL_SMALL_STACK
-    if (sigCtx != NULL)
-        XFREE(sigCtx, heap, DYNAMIC_TYPE_SIGNATURE);
-#endif
     return ret;
 #endif /* WOLFSSL_ASN_TEMPLATE */
 }