fixes for defects identified by nightly testing:

* ecc.c: in wc_ecc_free(), fix gating around handling for key->sign_k to resolve memory leak, and in wc_ecc_gen_deterministic_k(), fix -Wconversion.

* test.c: add missing mp_free()s to ecdsa_test_deterministic_k_rs() and ecc521_test_deterministic_k().

* wc_HashType: change several occurrences of int to enum wc_HashType, including ecc_key.hashType and API wc_ecc_set_deterministic_ex(), to resolve C++ warnings.

* fixes for various C++ warnings/errors in crypto and TLS layers and test and benchmark code -- implicit casts, negative initializers for unsigned type, jumped initializers, and missing enums in switch()es.

src/tls.c

@@ -8705,7 +8705,7 @@ static int TLSX_KeyShare_ProcessPqc(WOLFSSL* ssl, KeyShareEntry* keyShareEntry)
     (!defined(HAVE_FIPS_VERSION) || (HAVE_FIPS_VERSION != 2))) && \
     !defined(HAVE_SELFTEST)
         if (ret == 0) {
-            ret = wc_ecc_set_rng(keyShareEntry->key, ssl->rng);
+            ret = wc_ecc_set_rng((ecc_key *)keyShareEntry->key, ssl->rng);
             if (ret != 0) {
                 WOLFSSL_MSG("Failure to set the ECC private key RNG.");
             }
@@ -8714,8 +8714,8 @@ static int TLSX_KeyShare_ProcessPqc(WOLFSSL* ssl, KeyShareEntry* keyShareEntry)
 
         if (ret == 0) {
             PRIVATE_KEY_UNLOCK();
-            ret = wc_ecc_shared_secret(keyShareEntry->key, &eccpubkey,
-                sharedSecret, &outlen);
+            ret = wc_ecc_shared_secret((ecc_key *)keyShareEntry->key,
+                &eccpubkey, sharedSecret, &outlen);
             PRIVATE_KEY_LOCK();
             if (outlen != sharedSecretLen - ssSz) {
                 WOLFSSL_MSG("ECC shared secret derivation error.");
@@ -9193,14 +9193,14 @@ static int server_generate_pqc_ciphertext(WOLFSSL* ssl,
     (!defined(HAVE_FIPS_VERSION) || (HAVE_FIPS_VERSION != 2))) && \
     !defined(HAVE_SELFTEST)
         if (ret == 0) {
-            ret = wc_ecc_set_rng(ecc_kse->key, ssl->rng);
+            ret = wc_ecc_set_rng((ecc_key *)ecc_kse->key, ssl->rng);
         }
 #endif
 
         if (ret == 0) {
             outlen = ecc_kse->keyLen;
             PRIVATE_KEY_UNLOCK();
-            ret = wc_ecc_shared_secret(ecc_kse->key, &eccpubkey,
+            ret = wc_ecc_shared_secret((ecc_key *)ecc_kse->key, &eccpubkey,
                                        sharedSecret,
                                        &outlen);
             PRIVATE_KEY_LOCK();

wolfcrypt/benchmark/benchmark.c

@@ -9586,7 +9586,7 @@ static int lms_read_key_mem(byte* priv, word32 privSz, void* context)
 }
 static byte lms_priv[HSS_MAX_PRIVATE_KEY_LEN];
 
-static void bench_lms_keygen(int parm, byte* pub)
+static void bench_lms_keygen(enum wc_LmsParm parm, byte* pub)
 {
     WC_RNG      rng;
     LmsKey      key;
@@ -9698,7 +9698,7 @@ exit_lms_keygen:
     wc_FreeRng(&rng);
 }
 
-static void bench_lms_sign_verify(int parm, byte* pub)
+static void bench_lms_sign_verify(enum wc_LmsParm parm, byte* pub)
 {
     LmsKey       key;
     int          ret = 0;
@@ -9762,6 +9762,31 @@ static void bench_lms_sign_verify(int parm, byte* pub)
     case WC_LMS_PARM_L1_H15_W4:
     case WC_LMS_PARM_L2_H10_W8:
     case WC_LMS_PARM_L3_H5_W2:
+    case WC_LMS_PARM_L1_H5_W1:
+    case WC_LMS_PARM_L1_H5_W2:
+    case WC_LMS_PARM_L1_H5_W4:
+    case WC_LMS_PARM_L1_H5_W8:
+    case WC_LMS_PARM_L1_H10_W2:
+    case WC_LMS_PARM_L1_H10_W4:
+    case WC_LMS_PARM_L1_H10_W8:
+    case WC_LMS_PARM_L1_H15_W8:
+    case WC_LMS_PARM_L1_H20_W2:
+    case WC_LMS_PARM_L1_H20_W4:
+    case WC_LMS_PARM_L1_H20_W8:
+    case WC_LMS_PARM_L2_H5_W2:
+    case WC_LMS_PARM_L2_H5_W4:
+    case WC_LMS_PARM_L2_H5_W8:
+    case WC_LMS_PARM_L2_H15_W2:
+    case WC_LMS_PARM_L2_H15_W4:
+    case WC_LMS_PARM_L2_H15_W8:
+    case WC_LMS_PARM_L2_H20_W2:
+    case WC_LMS_PARM_L2_H20_W4:
+    case WC_LMS_PARM_L2_H20_W8:
+    case WC_LMS_PARM_L3_H10_W8:
+    case WC_LMS_PARM_L4_H5_W2:
+    case WC_LMS_PARM_L4_H5_W4:
+    case WC_LMS_PARM_L4_H10_W4:
+    case WC_LMS_PARM_L4_H10_W8:
     default:
         XMEMCPY(key.pub, pub, HSS_MAX_PUBLIC_KEY_LEN);
         break;
@@ -9853,7 +9878,7 @@ static void bench_lms_sign_verify(int parm, byte* pub)
 
     loaded = 1;
 
-    sig = XMALLOC(sigSz, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER);
+    sig = (byte *)XMALLOC(sigSz, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER);
     if (sig == NULL) {
         printf("bench_lms_sign_verify malloc failed\n");
         goto exit_lms_sign_verify;

wolfcrypt/src/dilithium.c

@@ -2035,7 +2035,7 @@ static int dilithium_expand_a(wc_Shake* shake128, const byte* pub_seed, byte k,
 #define DILITHIUM_COEFF_S_VALID_ETA2(b) \
     ((b) < DILITHIUM_ETA_2_MOD)
 
-static const byte dilithium_coeff_eta2[] = {
+static const char dilithium_coeff_eta2[] = {
     2, 1, 0, -1, -2,
     2, 1, 0, -1, -2,
     2, 1, 0, -1, -2

wolfcrypt/src/ecc.c

@@ -7538,7 +7538,7 @@ int wc_ecc_gen_deterministic_k(const byte* hash, word32 hashSz,
 #endif
     word32 xSz, VSz, KSz, h1len, qLen;
     byte intOct;
-    word32 qbits = 0;
+    int qbits = 0;
 
     if (hash == NULL || k == NULL || order == NULL) {
         return BAD_FUNC_ARG;
@@ -7630,11 +7630,15 @@ int wc_ecc_gen_deterministic_k(const byte* hash, word32 hashSz,
         wc_MemZero_Add("wc_ecc_gen_deterministic_k x", x, qLen);
     #endif
         qbits = mp_count_bits(order);
+        if (qbits < 0)
+            ret = MP_VAL;
+    }
 
+    if (ret == 0) {
          /* hash truncate if too long */
-        if (((WOLFSSL_BIT_SIZE) * hashSz) > qbits) {
+        if (((WOLFSSL_BIT_SIZE) * hashSz) > (word32)qbits) {
             /* calculate truncated hash size using bits rounded up byte */
-            hashSz = (qbits + ((WOLFSSL_BIT_SIZE) - 1)) / (WOLFSSL_BIT_SIZE);
+            hashSz = ((word32)qbits + (WOLFSSL_BIT_SIZE - 1)) / WOLFSSL_BIT_SIZE;
         }
         ret = mp_read_unsigned_bin(z1, hash, hashSz);
     }
@@ -7726,7 +7730,7 @@ int wc_ecc_gen_deterministic_k(const byte* hash, word32 hashSz,
                 ret = mp_read_unsigned_bin(k, x, xSz);
             }
 
-            if ((ret == 0) && ((xSz * WOLFSSL_BIT_SIZE) != qbits)) {
+            if ((ret == 0) && ((xSz * WOLFSSL_BIT_SIZE) != (word32)qbits)) {
                 /* handle odd case where shift of 'k' is needed with RFC 6979
                  *  k = bits2int(T) in section 3.2 h.3 */
                 mp_rshb(k, ((int)xSz * WOLFSSL_BIT_SIZE) - qbits);
@@ -7779,7 +7783,8 @@ int wc_ecc_gen_deterministic_k(const byte* hash, word32 hashSz,
 /* Sets the deterministic flag for 'k' generation with sign.
  * returns 0 on success
  */
-int wc_ecc_set_deterministic_ex(ecc_key* key, byte flag, int hashType)
+int wc_ecc_set_deterministic_ex(ecc_key* key, byte flag,
+                                enum wc_HashType hashType)
 {
     if (key == NULL) {
         return BAD_FUNC_ARG;
@@ -7887,7 +7892,9 @@ int wc_ecc_free(ecc_key* key)
         return 0;
     }
 
-#if defined(WOLFSSL_ECDSA_SET_K) || defined(WOLFSSL_ECDSA_SET_K_ONE_LOOP)
+#if defined(WOLFSSL_ECDSA_SET_K) || defined(WOLFSSL_ECDSA_SET_K_ONE_LOOP) || \
+    defined(WOLFSSL_ECDSA_DETERMINISTIC_K) || \
+    defined(WOLFSSL_ECDSA_DETERMINISTIC_K_VARIANT)
 #ifndef WOLFSSL_NO_MALLOC
     if (key->sign_k != NULL)
 #endif

wolfcrypt/src/wc_lms.c

@@ -629,7 +629,7 @@ int wc_LmsKey_MakeKey(LmsKey* key, WC_RNG* rng)
         const LmsParams* params = key->params;
 
         /* Allocate memory for the private key data. */
-        key->priv_data = XMALLOC(LMS_PRIV_DATA_LEN(params->levels,
+        key->priv_data = (byte *)XMALLOC(LMS_PRIV_DATA_LEN(params->levels,
             params->height, params->p, params->rootLevels, params->cacheBits),
             key->heap, DYNAMIC_TYPE_LMS);
         /* Check pointer is valid. */
@@ -728,7 +728,7 @@ int wc_LmsKey_Reload(LmsKey* key)
         const LmsParams* params = key->params;
 
         /* Allocate memory for the private key data. */
-        key->priv_data = XMALLOC(LMS_PRIV_DATA_LEN(params->levels,
+        key->priv_data = (byte *)XMALLOC(LMS_PRIV_DATA_LEN(params->levels,
             params->height, params->p, params->rootLevels, params->cacheBits),
             key->heap, DYNAMIC_TYPE_LMS);
         /* Check pointer is valid. */

wolfcrypt/test/test.c

@@ -3829,23 +3829,26 @@ WOLFSSL_TEST_SUBROUTINE wc_test_ret_t sha256_test(void)
 #endif /* NO_LARGE_HASH_TEST */
 
 #if defined(WOLFSSL_HAVE_LMS) && !defined(WOLFSSL_LMS_FULL_HASH)
-    unsigned char data_hb[WC_SHA256_BLOCK_SIZE] = {
-        0x61, 0x62, 0x63, 0x80, 0x00, 0x00, 0x00, 0x00,
-        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18
-    };
+    {
+        WOLFSSL_SMALL_STACK_STATIC const unsigned char
+            data_hb[WC_SHA256_BLOCK_SIZE] = {
+            0x61, 0x62, 0x63, 0x80, 0x00, 0x00, 0x00, 0x00,
+            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18
+        };
 
-    ret = wc_Sha256HashBlock(&sha, data_hb, hash);
-    if (ret != 0) {
-        ERROR_OUT(WC_TEST_RET_ENC_EC(ret), exit);
-    }
-    if (XMEMCMP(hash, b.output, WC_SHA256_DIGEST_SIZE) != 0) {
-        ERROR_OUT(WC_TEST_RET_ENC_NC, exit);
+        ret = wc_Sha256HashBlock(&sha, data_hb, hash);
+        if (ret != 0) {
+            ERROR_OUT(WC_TEST_RET_ENC_EC(ret), exit);
+        }
+        if (XMEMCMP(hash, b.output, WC_SHA256_DIGEST_SIZE) != 0) {
+            ERROR_OUT(WC_TEST_RET_ENC_NC, exit);
+        }
     }
 #endif
 
@@ -29059,8 +29062,9 @@ static wc_test_ret_t ecc_test_vector(int keySize)
     (!defined(HAVE_FIPS) || FIPS_VERSION_GE(5,3))
 #if (!defined(NO_ECC256) || defined(HAVE_ALL_CURVES)) && ECC_MIN_KEY_SZ <= 256
 
-static wc_test_ret_t ecdsa_test_deterministic_k_sig(ecc_key *key, int hashType,
-    const char* msg, WC_RNG* rng, const byte* expSig, size_t expSigSz)
+static wc_test_ret_t ecdsa_test_deterministic_k_sig(ecc_key *key,
+    enum wc_HashType hashType, const char* msg, WC_RNG* rng, const byte* expSig,
+    size_t expSigSz)
 {
     wc_test_ret_t ret;
     int verify;
@@ -29246,8 +29250,8 @@ done:
 
 #ifdef WOLFSSL_PUBLIC_MP
 
-static wc_test_ret_t ecdsa_test_deterministic_k_rs(ecc_key *key, int hashType,
-    const char* msg, WC_RNG* rng,
+static wc_test_ret_t ecdsa_test_deterministic_k_rs(ecc_key *key,
+    enum wc_HashType hashType, const char* msg, WC_RNG* rng,
     mp_int* r, mp_int* s,
     mp_int* expR, mp_int* expS)
 {
@@ -29305,7 +29309,9 @@ static wc_test_ret_t ecc384_test_deterministic_k(WC_RNG* rng)
     ecc_key key[1];
     mp_int r[1], s[1], expR[1], expS[1];
 #endif
-    int key_inited = 0;
+    int key_inited = 0,
+        tmp_mp_ints_inited = 0;
+
     WOLFSSL_SMALL_STACK_STATIC const char* msg = "sample";
     WOLFSSL_SMALL_STACK_STATIC const char* dIUT =
         "6B9D3DAD2E1B8C1C05B19875B6659F4DE23C3B667BF297BA9AA47740787137D8"
@@ -29363,6 +29369,7 @@ static wc_test_ret_t ecc384_test_deterministic_k(WC_RNG* rng)
     if (ret != MP_OKAY) {
         goto done;
     }
+    tmp_mp_ints_inited = 1;
     ret = wc_ecc_init_ex(key, HEAP_HINT, devId);
     if (ret != 0) {
         goto done;
@@ -29415,6 +29422,12 @@ static wc_test_ret_t ecc384_test_deterministic_k(WC_RNG* rng)
 done:
     if (key_inited)
         wc_ecc_free(key);
+    if (tmp_mp_ints_inited) {
+        mp_free(r);
+        mp_free(s);
+        mp_free(expR);
+        mp_free(expS);
+    }
 #ifdef WOLFSSL_SMALL_STACK
     if (key != NULL)
         XFREE(key, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER);
@@ -29444,7 +29457,8 @@ static wc_test_ret_t ecc521_test_deterministic_k(WC_RNG* rng)
     ecc_key key[1];
     mp_int r[1], s[1], expR[1], expS[1];
 #endif
-    int key_inited = 0;
+    int key_inited = 0,
+        tmp_mp_ints_inited = 0;
     WOLFSSL_SMALL_STACK_STATIC const char* msg = "sample";
     WOLFSSL_SMALL_STACK_STATIC const char* dIUT =
        "0FAD06DAA62BA3B25D2FB40133DA757205DE67F5BB0018FEE8C86E1B68C7E75C"
@@ -29511,6 +29525,7 @@ static wc_test_ret_t ecc521_test_deterministic_k(WC_RNG* rng)
     if (ret != MP_OKAY) {
         goto done;
     }
+    tmp_mp_ints_inited = 1;
     ret = wc_ecc_init_ex(key, HEAP_HINT, devId);
     if (ret != 0) {
         return WC_TEST_RET_ENC_EC(ret);
@@ -29563,6 +29578,12 @@ static wc_test_ret_t ecc521_test_deterministic_k(WC_RNG* rng)
 done:
     if (key_inited)
         wc_ecc_free(key);
+    if (tmp_mp_ints_inited) {
+        mp_free(r);
+        mp_free(s);
+        mp_free(expR);
+        mp_free(expS);
+    }
 #ifdef WOLFSSL_SMALL_STACK
     if (key != NULL)
         XFREE(key, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER);

wolfssl/wolfcrypt/ecc.h

@@ -595,7 +595,7 @@ struct ecc_key {
 #if defined(WOLFSSL_ECDSA_DETERMINISTIC_K) || \
     defined(WOLFSSL_ECDSA_DETERMINISTIC_K_VARIANT)
     byte deterministic:1;
-    int hashType;
+    enum wc_HashType hashType;
 #endif
 
 #if defined(WOLFSSL_SMALL_STACK_CACHE) && !defined(WOLFSSL_ECC_NO_SMALL_STACK)
@@ -720,7 +720,8 @@ int wc_ecc_sign_hash_ex(const byte* in, word32 inlen, WC_RNG* rng,
 WOLFSSL_API
 int wc_ecc_set_deterministic(ecc_key* key, byte flag);
 WOLFSSL_API
-int wc_ecc_set_deterministic_ex(ecc_key* key, byte flag, int hashType);
+int wc_ecc_set_deterministic_ex(ecc_key* key, byte flag,
+        enum wc_HashType hashType);
 WOLFSSL_API
 int wc_ecc_gen_deterministic_k(const byte* hash, word32 hashSz,
         enum wc_HashType hashType, mp_int* priv, mp_int* k, mp_int* order,