add .github/workflows/smallStackSize.yml;

smallstack refactors in
* wolfcrypt/src/asn.c : wc_GetSubjectPubKeyInfoDerFromCert(),
* wolfcrypt/src/dilithium.c : dilithium_sign_with_seed_mu(),
* wolfcrypt/src/ecc.c : wc_ecc_mulmod_ex2(),
* wolfcrypt/src/wc_mlkem.c : mlkemkey_decapsulate(),
* and wolfcrypt/src/wc_mlkem_poly.c : mlkem_gen_matrix_k*_avx2() and mlkem_get_noise_k2_avx2();

wolfcrypt/test/test.c: in TEST_PASS(), fix STACK_SIZE_CHECKPOINT_WITH_MAX_CHECK to honor TEST_ALWAYS_RUN_TO_END.

.github/workflows/smallStackSize.yml

@@ -0,0 +1,56 @@
+name: Stack Size warnings
+
+# START OF COMMON SECTION
+on:
+  push:
+    branches: [ 'master', 'main', 'release/**' ]
+  pull_request:
+    branches: [ '*' ]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+# END OF COMMON SECTION
+
+jobs:
+  build_library:
+    strategy:
+      matrix:
+        config: [
+          # defaults, noasm
+          '--disable-asm',
+
+          # defaults + native PQ, no asm
+          '--disable-asm --enable-experimental --enable-kyber=yes,original --enable-lms --enable-xmss --enable-dilithium',
+
+          # all-crypto + native PQ, no asm
+          '--disable-asm --enable-all-crypto --enable-experimental --enable-kyber=yes,original --enable-lms --enable-xmss --enable-dilithium',
+
+          # defaults, intelasm + sp-asm
+          '--enable-intelasm --enable-sp-asm',
+
+          # defaults + native PQ, intelasm + sp-asm
+          '--enable-intelasm --enable-sp-asm --enable-experimental --enable-kyber=yes,original --enable-lms --enable-xmss --enable-dilithium',
+
+          # all-crypto + native PQ, intelasm + sp-asm
+          '--enable-intelasm --enable-sp-asm --enable-all-crypto --enable-experimental --enable-kyber=yes,original --enable-lms --enable-xmss --enable-dilithium'
+        ]
+    name: build library
+    if: github.repository_owner == 'wolfssl'
+    runs-on: ubuntu-22.04
+    # This should be a safe limit for the tests to run.
+    timeout-minutes: 6
+    steps:
+      - uses: actions/checkout@v4
+        name: Checkout wolfSSL
+
+      - name: install_multilib
+        run: sudo apt-get install -y gcc-multilib
+
+      - name: Build wolfCrypt with smallstack and stack depth warnings, and run testwolfcrypt
+        run: |
+          ./autogen.sh || $(exit 2)
+          echo "running ./configure ... ${{ matrix.config }}"
+          ./configure --enable-cryptonly --disable-cryptocb --disable-testcert --enable-smallstack --enable-smallstackcache --enable-crypttests --disable-benchmark --disable-examples --with-max-rsa-bits=16384 --enable-stacksize=verbose CFLAGS="-Wframe-larger-than=2048 -Wstack-usage=4096 -DWOLFSSL_TEST_MAX_RELATIVE_STACK_BYTES=8192 -DTEST_ALWAYS_RUN_TO_END" ${{ matrix.config }} || $(exit 3)
+          make -j 4 || $(exit 4)
+          ./wolfcrypt/test/testwolfcrypt

wolfcrypt/src/asn.c

@@ -24682,7 +24682,11 @@ WOLFSSL_API int wc_GetSubjectPubKeyInfoDerFromCert(const byte* certDer,
                                                    byte* pubKeyDer,
                                                    word32* pubKeyDerSz)
 {
-    DecodedCert cert;
+#ifdef WOLFSSL_SMALL_STACK
+    DecodedCert* cert;
+#else
+    DecodedCert cert[1];
+#endif
     int         ret;
     word32      startIdx;
     word32      idx;
@@ -24693,16 +24697,22 @@ WOLFSSL_API int wc_GetSubjectPubKeyInfoDerFromCert(const byte* certDer,
         return BAD_FUNC_ARG;
     }
 
+#ifdef WOLFSSL_SMALL_STACK
+    cert = (DecodedCert*)XMALLOC(sizeof(*cert), NULL, DYNAMIC_TYPE_TMP_BUFFER);
+    if (cert == NULL)
+        return MEMORY_E;
+#endif
+
     length = 0;
     badDate = 0;
 
-    wc_InitDecodedCert(&cert, certDer, certDerSz, NULL);
+    wc_InitDecodedCert(cert, certDer, certDerSz, NULL);
 
     /* Parse up to the SubjectPublicKeyInfo */
-    ret = wc_GetPubX509(&cert, 0, &badDate);
+    ret = wc_GetPubX509(cert, 0, &badDate);
     if (ret >= 0) {
         /* Save the starting index of SubjectPublicKeyInfo */
-        startIdx = cert.srcIdx;
+        startIdx = cert->srcIdx;
 
         /* Get the length of the SubjectPublicKeyInfo sequence */
         idx = startIdx;
@@ -24728,7 +24738,11 @@ WOLFSSL_API int wc_GetSubjectPubKeyInfoDerFromCert(const byte* certDer,
     }
 
     *pubKeyDerSz = length;
-    wc_FreeDecodedCert(&cert);
+    wc_FreeDecodedCert(cert);
+
+#ifdef WOLFSSL_SMALL_STACK
+    XFREE(cert, NULL, DYNAMIC_TYPE_TMP_BUFFER);
+#endif
 
     return ret;
 }

wolfcrypt/src/dilithium.c

@@ -6152,7 +6152,11 @@ static int dilithium_sign_with_seed_mu(dilithium_key* key,
 
         /* Step 11: Start rejection sampling loop */
         do {
+#ifdef WOLFSSL_SMALL_STACK
+            byte *w1e = NULL;
+#else
             byte w1e[DILITHIUM_MAX_W1_ENC_SZ];
+#endif
             sword32* w = w1;
             sword32* y_ntt = z;
             sword32* cs2 = ct0;
@@ -6182,11 +6186,20 @@ static int dilithium_sign_with_seed_mu(dilithium_key* key,
             if (valid) {
         #endif
                 /* Step 15: Encode w1. */
-                dilithium_vec_encode_w1(w1, params->k, params->gamma2, w1e);
+#ifdef WOLFSSL_SMALL_STACK
-                /* Step 15: Hash mu and encoded w1.
+                w1e = (byte *)XMALLOC(DILITHIUM_MAX_W1_ENC_SZ, key->heap,
-                 * Step 32: Hash is stored in signature. */
+                                      DYNAMIC_TYPE_DILITHIUM);
-                ret = dilithium_hash256(&key->shake, mu, DILITHIUM_MU_SZ,
+                if (w1e == NULL)
-                    w1e, params->w1EncSz, commit, params->lambda / 4);
+                    ret = MEMORY_E;
+                if (ret == 0)
+#endif
+                {
+                    dilithium_vec_encode_w1(w1, params->k, params->gamma2, w1e);
+                    /* Step 15: Hash mu and encoded w1.
+                     * Step 32: Hash is stored in signature. */
+                    ret = dilithium_hash256(&key->shake, mu, DILITHIUM_MU_SZ,
+                        w1e, params->w1EncSz, commit, params->lambda / 4);
+                }
                 if (ret == 0) {
                     /* Step 17: Compute c from first 256 bits of commit. */
                     ret = dilithium_sample_in_ball(params->level, &key->shake,
@@ -6237,6 +6250,10 @@ static int dilithium_sign_with_seed_mu(dilithium_key* key,
                             params->gamma2, params->omega, h) >= 0);
                     }
                 }
+
+#ifdef WOLFSSL_SMALL_STACK
+                XFREE(w1e, key->heap, DYNAMIC_TYPE_DILITHIUM);
+#endif
             }
 
             if (!valid) {

wolfcrypt/src/ecc.c

@@ -3915,7 +3915,7 @@ int wc_ecc_mulmod_ex2(const mp_int* k, ecc_point* G, ecc_point* R, mp_int* a,
 #endif
    int           i, err;
 #ifdef WOLFSSL_SMALL_STACK_CACHE
-   ecc_key       key;
+   ecc_key       *key = NULL;
 #endif
    mp_digit      mp;
 
@@ -3942,10 +3942,13 @@ int wc_ecc_mulmod_ex2(const mp_int* k, ecc_point* G, ecc_point* R, mp_int* a,
    XMEMSET(M, 0, sizeof(M));
 
 #ifdef WOLFSSL_SMALL_STACK_CACHE
-   err = ecc_key_tmp_init(&key, heap);
+   key = (ecc_key *)XMALLOC(sizeof(*key), heap, DYNAMIC_TYPE_ECC);
+   if (key == NULL)
+       return MEMORY_E;
+   err = ecc_key_tmp_init(key, heap);
    if (err != MP_OKAY)
       goto exit;
-   R->key = &key;
+   R->key = key;
 #endif /* WOLFSSL_SMALL_STACK_CACHE */
 
    /* alloc ram for window temps */
@@ -3958,7 +3961,7 @@ int wc_ecc_mulmod_ex2(const mp_int* k, ecc_point* G, ecc_point* R, mp_int* a,
          goto exit;
       }
 #ifdef WOLFSSL_SMALL_STACK_CACHE
-      M[i]->key = &key;
+      M[i]->key = key;
 #endif
   }
 
@@ -4000,7 +4003,8 @@ exit:
    }
 #ifdef WOLFSSL_SMALL_STACK_CACHE
    R->key = NULL;
-   ecc_key_tmp_final(&key, heap);
+   ecc_key_tmp_final(key, heap);
+   XFREE(key, heap, DYNAMIC_TYPE_ECC);
 #endif /* WOLFSSL_SMALL_STACK_CACHE */
 
    return err;

wolfcrypt/src/wc_mlkem.c

@@ -1144,7 +1144,8 @@ static MLKEM_NOINLINE int mlkemkey_decapsulate(MlKemKey* key, byte* m,
     sword16* w;
     unsigned int k = 0;
     unsigned int compVecSz;
-#if !defined(USE_INTEL_SPEEDUP) && !defined(WOLFSSL_NO_MALLOC)
+#if defined(WOLFSSL_SMALL_STACK) || \
+    (!defined(USE_INTEL_SPEEDUP) && !defined(WOLFSSL_NO_MALLOC))
     sword16* u = NULL;
 #else
     sword16 u[(WC_ML_KEM_MAX_K + 1) * MLKEM_N];
@@ -1198,7 +1199,8 @@ static MLKEM_NOINLINE int mlkemkey_decapsulate(MlKemKey* key, byte* m,
         break;
     }
 
-#if !defined(USE_INTEL_SPEEDUP) && !defined(WOLFSSL_NO_MALLOC)
+#if defined(WOLFSSL_SMALL_STACK) || \
+    (!defined(USE_INTEL_SPEEDUP) && !defined(WOLFSSL_NO_MALLOC))
     if (ret == 0) {
         /* Allocate dynamic memory for a vector and a polynomial. */
         u = (sword16*)XMALLOC((k + 1) * MLKEM_N * sizeof(sword16), key->heap,
@@ -1254,7 +1256,8 @@ static MLKEM_NOINLINE int mlkemkey_decapsulate(MlKemKey* key, byte* m,
         /* Step 8: return m */
     }
 
-#if !defined(USE_INTEL_SPEEDUP) && !defined(WOLFSSL_NO_MALLOC)
+#if defined(WOLFSSL_SMALL_STACK) || \
+    (!defined(USE_INTEL_SPEEDUP) && !defined(WOLFSSL_NO_MALLOC))
     /* Dispose of dynamically memory allocated in function. */
     XFREE(u, key->heap, DYNAMIC_TYPE_TMP_BUFFER);
 #endif

wolfcrypt/src/wc_mlkem_poly.c

@@ -2274,14 +2274,31 @@ void mlkem_decapsulate(const sword16* s, sword16* w, sword16* u,
 static int mlkem_gen_matrix_k2_avx2(sword16* a, byte* seed, int transposed)
 {
     int i;
+#ifdef WOLFSSL_SMALL_STACK
+    byte *rand = NULL;
+    word64 *state = NULL;
+#else
     byte rand[4 * GEN_MATRIX_SIZE + 2];
     word64 state[25 * 4];
+#endif
     unsigned int ctr0;
     unsigned int ctr1;
     unsigned int ctr2;
     unsigned int ctr3;
     byte* p;
 
+#ifdef WOLFSSL_SMALL_STACK
+    rand = (byte*)XMALLOC(4 * GEN_MATRIX_SIZE + 2, NULL,
+                          DYNAMIC_TYPE_TMP_BUFFER);
+    state = (word64*)XMALLOC(sizeof(word64) * 25 * 4, NULL,
+                          DYNAMIC_TYPE_TMP_BUFFER);
+    if ((rand == NULL) || (state == NULL)) {
+        XFREE(rand, NULL, DYNAMIC_TYPE_TMP_BUFFER);
+        XFREE(state, NULL, DYNAMIC_TYPE_TMP_BUFFER);
+        return MEMORY_E;
+    }
+#endif
+
     /* Loading 64 bits, only using 48 bits. Loading 2 bytes more than used. */
     rand[4 * GEN_MATRIX_SIZE + 0] = 0xff;
     rand[4 * GEN_MATRIX_SIZE + 1] = 0xff;
@@ -2345,6 +2362,11 @@ static int mlkem_gen_matrix_k2_avx2(sword16* a, byte* seed, int transposed)
             p, XOF_BLOCK_SIZE);
     }
 
+#ifdef WOLFSSL_SMALL_STACK
+    XFREE(rand, NULL, DYNAMIC_TYPE_TMP_BUFFER);
+    XFREE(state, NULL, DYNAMIC_TYPE_TMP_BUFFER);
+#endif
+
     return 0;
 }
 #endif
@@ -2365,14 +2387,31 @@ static int mlkem_gen_matrix_k3_avx2(sword16* a, byte* seed, int transposed)
 {
     int i;
     int k;
+#ifdef WOLFSSL_SMALL_STACK
+    byte *rand = NULL;
+    word64 *state = NULL;
+#else
     byte rand[4 * GEN_MATRIX_SIZE + 2];
     word64 state[25 * 4];
+#endif
     unsigned int ctr0;
     unsigned int ctr1;
     unsigned int ctr2;
     unsigned int ctr3;
     byte* p;
 
+#ifdef WOLFSSL_SMALL_STACK
+    rand = (byte*)XMALLOC(4 * GEN_MATRIX_SIZE + 2, NULL,
+                          DYNAMIC_TYPE_TMP_BUFFER);
+    state = (word64*)XMALLOC(sizeof(word64) * 25 * 4, NULL,
+                          DYNAMIC_TYPE_TMP_BUFFER);
+    if ((rand == NULL) || (state == NULL)) {
+        XFREE(rand, NULL, DYNAMIC_TYPE_TMP_BUFFER);
+        XFREE(state, NULL, DYNAMIC_TYPE_TMP_BUFFER);
+        return MEMORY_E;
+    }
+#endif
+
     /* Loading 64 bits, only using 48 bits. Loading 2 bytes more than used. */
     rand[4 * GEN_MATRIX_SIZE + 0] = 0xff;
     rand[4 * GEN_MATRIX_SIZE + 1] = 0xff;
@@ -2473,6 +2512,11 @@ static int mlkem_gen_matrix_k3_avx2(sword16* a, byte* seed, int transposed)
             XOF_BLOCK_SIZE);
     }
 
+#ifdef WOLFSSL_SMALL_STACK
+    XFREE(rand, NULL, DYNAMIC_TYPE_TMP_BUFFER);
+    XFREE(state, NULL, DYNAMIC_TYPE_TMP_BUFFER);
+#endif
+
     return 0;
 }
 #endif
@@ -2492,14 +2536,31 @@ static int mlkem_gen_matrix_k4_avx2(sword16* a, byte* seed, int transposed)
 {
     int i;
     int k;
+#ifdef WOLFSSL_SMALL_STACK
+    byte *rand = NULL;
+    word64 *state = NULL;
+#else
     byte rand[4 * GEN_MATRIX_SIZE + 2];
     word64 state[25 * 4];
+#endif
     unsigned int ctr0;
     unsigned int ctr1;
     unsigned int ctr2;
     unsigned int ctr3;
     byte* p;
 
+#ifdef WOLFSSL_SMALL_STACK
+    rand = (byte*)XMALLOC(4 * GEN_MATRIX_SIZE + 2, NULL,
+                          DYNAMIC_TYPE_TMP_BUFFER);
+    state = (word64*)XMALLOC(sizeof(word64) * 25 * 4, NULL,
+                          DYNAMIC_TYPE_TMP_BUFFER);
+    if ((rand == NULL) || (state == NULL)) {
+        XFREE(rand, NULL, DYNAMIC_TYPE_TMP_BUFFER);
+        XFREE(state, NULL, DYNAMIC_TYPE_TMP_BUFFER);
+        return MEMORY_E;
+    }
+#endif
+
     /* Loading 64 bits, only using 48 bits. Loading 2 bytes more than used. */
     rand[4 * GEN_MATRIX_SIZE + 0] = 0xff;
     rand[4 * GEN_MATRIX_SIZE + 1] = 0xff;
@@ -2563,6 +2624,11 @@ static int mlkem_gen_matrix_k4_avx2(sword16* a, byte* seed, int transposed)
         a += 4 * MLKEM_N;
     }
 
+#ifdef WOLFSSL_SMALL_STACK
+    XFREE(rand, NULL, DYNAMIC_TYPE_TMP_BUFFER);
+    XFREE(state, NULL, DYNAMIC_TYPE_TMP_BUFFER);
+#endif
+
     return 0;
 }
 #endif /* WOLFSSL_KYBER1024 || WOLFSSL_WC_ML_KEM_1024 */
@@ -4120,7 +4186,17 @@ static int mlkem_get_noise_k2_avx2(MLKEM_PRF_T* prf, sword16* vec1,
     sword16* vec2, sword16* poly, byte* seed)
 {
     int ret = 0;
+#ifdef WOLFSSL_SMALL_STACK
+    byte *rand;
+#else
     byte rand[4 * PRF_RAND_SZ];
+#endif
+
+#ifdef WOLFSSL_SMALL_STACK
+    rand = (byte*)XMALLOC(4 * PRF_RAND_SZ, NULL, DYNAMIC_TYPE_TMP_BUFFER);
+    if (rand == NULL)
+        return MEMORY_E;
+#endif
 
     mlkem_get_noise_x4_eta3_avx2(rand, seed);
     mlkem_cbd_eta3_avx2(vec1          , rand + 0 * PRF_RAND_SZ);
@@ -4137,6 +4213,10 @@ static int mlkem_get_noise_k2_avx2(MLKEM_PRF_T* prf, sword16* vec1,
         ret = mlkem_get_noise_eta2_avx2(prf, poly, seed);
     }
 
+#ifdef WOLFSSL_SMALL_STACK
+    XFREE(rand, NULL, DYNAMIC_TYPE_TMP_BUFFER);
+#endif
+
     return ret;
 }
 #endif

wolfcrypt/test/test.c

@@ -1469,15 +1469,23 @@ static WOLFSSL_TEST_SUBROUTINE wc_test_ret_t nist_sp80056c_kdf_test(void)
     }
 #endif
 
+#ifdef TEST_ALWAYS_RUN_TO_END
+    #define TEST_PASS_stack_size_fail_clause last_failed_test_ret = \
+        WC_TEST_RET_ENC_EC(MEMORY_E)
+#else
+    #define TEST_PASS_stack_size_fail_clause \
+        return err_sys("post-test check failed", WC_TEST_RET_ENC_NC)
+#endif
+
 /* set test pass output to printf if not overridden */
 #ifndef TEST_PASS
     /* redirect to printf */
     #define TEST_PASS(...) {                                    \
         if (STACK_SIZE_CHECKPOINT_WITH_MAX_CHECK                \
             (max_relative_stack, printf(__VA_ARGS__)) < 0) {    \
-            return err_sys("post-test check failed", WC_TEST_RET_ENC_NC);\
+            TEST_PASS_stack_size_fail_clause;                   \
         }                                                       \
-        PRINT_HEAP_CHECKPOINT("TEST_PASS", 0)                            \
+        PRINT_HEAP_CHECKPOINT("TEST_PASS", 0)                   \
         ASSERT_RESTORED_VECTOR_REGISTERS(exit(1););             \
     }
 #endif