F-6347 - Reject negative and oversized length in EVP_EncodeUpdate

wolfSSL_EVP_EncodeUpdate did not validate the input length. A large
inl caused the block loop and the residual copy to read far past the
caller's input buffer, and a negative inl was silently treated as
success. Reject negative lengths and lengths whose base64 output would
overflow a positive int before processing any data.

tests/api/test_evp.c

@@ -207,6 +207,21 @@ int test_wolfSSL_EVP_EncodeUpdate(void)
     XMEMSET( encOutBuff,0, sizeof(encOutBuff));
     ExpectIntEQ(EVP_EncodeBlock(encOutBuff, plain0, 0x7FFFFFFF), -1);
 
+    /* oversized length must be rejected by EVP_EncodeUpdate as well, rather
+     * than reading far past the caller's input buffer */
+    EVP_EncodeInit(ctx);
+    outl = 1;
+    XMEMSET( encOutBuff,0, sizeof(encOutBuff));
+    ExpectIntEQ(
+        EVP_EncodeUpdate(
+            ctx,
+            encOutBuff,
+            &outl,
+            plain0,
+            0x7FFFFFFF),
+        0);
+    ExpectIntEQ(outl, 0);
+
     /* pass small size( < 48bytes ) input, then make sure they are not
      * encoded  and just stored in ctx
      */

wolfcrypt/src/evp.c

@@ -13329,6 +13329,15 @@ int  wolfSSL_EVP_EncodeUpdate(WOLFSSL_EVP_ENCODE_CTX* ctx,
 
     *outl = 0;
 
+    if (inl < 0)
+        return 0;
+
+    /* Reject lengths whose base64 output would overflow a positive int. This
+     * also guards against reads far past the caller's input allocation. */
+    if (inl > (INT_MAX / (BASE64_ENCODE_RESULT_BLOCK_SIZE + 1)) *
+            BASE64_ENCODE_BLOCK_SIZE)
+        return 0;
+
     /* if the remaining data exists in the ctx, add input data to them
      * to create a block(48bytes) for encoding
      */