Added new build option --enable-memtest or WOLFSSL_FORCE_MALLOC_FAIL_TEST which enables random malloc failures for testing. This test supresses the abort() calls to detect seg faults. A new script ./scripts/memtest.sh starts the test. If an issue is found it can be reviewed with the ./scripts/memtest.txt log and reproduced using the seed printed at top of unit test as --- RNG MALLOC FAIL AT 295--- and rerun using ./tests/unit.test 295.

.gitignore

@@ -247,3 +247,4 @@ wolfcrypt/src/port/intel/qat_test
 
 # Arduino Generated Files
 /IDE/ARDUINO/wolfSSL
+scripts/memtest.txt

configure.ac

@@ -4009,6 +4009,19 @@ then
 fi
 
 
+# Memory Tests
+AC_ARG_ENABLE([memtest],
+    [AS_HELP_STRING([--enable-memtest],[Memory testing option, for internal use (default: disabled)])],
+    [ ENABLED_MEMTEST=$enableval ],
+    [ ENABLED_MEMTEST=no ]
+    )
+
+if test "x$ENABLED_MEMTEST" != "xno"
+then
+    AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_TRACK_MEMORY -DWOLFSSL_DEBUG_MEMORY -DWOLFSSL_FORCE_MALLOC_FAIL_TEST"
+fi
+
+
 # Default optimization CFLAGS enable
 AC_ARG_ENABLE([optflags],
     [AS_HELP_STRING([--enable-optflags],[Enable default optimization CFLAGS for the compiler (default: enabled)])],

examples/benchmark/tls_bench.c

@@ -778,6 +778,7 @@ static void ShowCiphers(void)
 
 int bench_tls(void* args)
 {
+    int ret = 0;
     info_t *theadInfo, *info;
     int i, doShutdown;
     char *cipher, *next_cipher, ciphers[4096];
@@ -793,7 +794,8 @@ int bench_tls(void* args)
     int argThreadPairs = THREAD_PAIRS;
     int argShowVerbose = SHOW_VERBOSE;
 
-    ((func_args*)args)->return_code = -1; /* error state */
+    if (args)
+        ((func_args*)args)->return_code = -1; /* error state */
 
     /* Initialize wolfSSL */
     wolfSSL_Init();
@@ -803,7 +805,7 @@ int bench_tls(void* args)
         switch (ch) {
             case '?' :
                 Usage();
-                exit(EXIT_SUCCESS);
+                XEXIT(EXIT_SUCCESS);
 
             case 'b' :
                 argTestSizeBytes = atoi(myoptarg);
@@ -817,7 +819,7 @@ int bench_tls(void* args)
 
             case 'e' :
                 ShowCiphers();
-                exit(EXIT_SUCCESS);
+                XEXIT(EXIT_SUCCESS);
 
             case 'i' :
                 argShowPeerInfo = 1;
@@ -845,7 +847,7 @@ int bench_tls(void* args)
 
             default:
                 Usage();
-                exit(MY_EX_USAGE);
+                XEXIT(MY_EX_USAGE);
         }
     }
 
@@ -985,7 +987,10 @@ int bench_tls(void* args)
     free(theadInfo);
 
     /* Return reporting a success */
-    return (((func_args*)args)->return_code = 0);
+    if (args)
+        ((func_args*)args)->return_code = ret;
+
+    return ret;
 }
 #endif /* !NO_WOLFSSL_CLIENT && !NO_WOLFSSL_SERVER */
 

examples/client/client.c

@@ -1019,7 +1019,7 @@ THREAD_RETURN WOLFSSL_THREAD client_test(void* args)
         switch (ch) {
             case '?' :
                 Usage();
-                exit(EXIT_SUCCESS);
+                XEXIT_T(EXIT_SUCCESS);
 
             case 'g' :
                 sendGET = 1;
@@ -1031,7 +1031,7 @@ THREAD_RETURN WOLFSSL_THREAD client_test(void* args)
 
             case 'e' :
                 ShowCiphers();
-                exit(EXIT_SUCCESS);
+                XEXIT_T(EXIT_SUCCESS);
 
             case 'D' :
                 overrideDateErrors = 1;
@@ -1114,13 +1114,13 @@ THREAD_RETURN WOLFSSL_THREAD client_test(void* args)
                 version = atoi(myoptarg);
                 if (version < 0 || version > 4) {
                     Usage();
-                    exit(MY_EX_USAGE);
+                    XEXIT_T(MY_EX_USAGE);
                 }
                 break;
 
             case 'V' :
                 ShowVersions();
-                exit(EXIT_SUCCESS);
+                XEXIT_T(EXIT_SUCCESS);
 
             case 'l' :
                 cipherList = myoptarg;
@@ -1137,7 +1137,7 @@ THREAD_RETURN WOLFSSL_THREAD client_test(void* args)
                 }
                 else {
                     Usage();
-                    exit(MY_EX_USAGE);
+                    XEXIT_T(MY_EX_USAGE);
                 }
                 break;
 
@@ -1158,7 +1158,7 @@ THREAD_RETURN WOLFSSL_THREAD client_test(void* args)
                     minDhKeyBits = atoi(myoptarg);
                     if (minDhKeyBits <= 0 || minDhKeyBits > 16000) {
                         Usage();
-                        exit(MY_EX_USAGE);
+                        XEXIT_T(MY_EX_USAGE);
                     }
                 #endif
                 break;
@@ -1167,7 +1167,7 @@ THREAD_RETURN WOLFSSL_THREAD client_test(void* args)
                 benchmark = atoi(myoptarg);
                 if (benchmark < 0 || benchmark > 1000000) {
                     Usage();
-                    exit(MY_EX_USAGE);
+                    XEXIT_T(MY_EX_USAGE);
                 }
                 break;
 
@@ -1181,7 +1181,7 @@ THREAD_RETURN WOLFSSL_THREAD client_test(void* args)
                 }
                 if (throughput <= 0 || block <= 0) {
                     Usage();
-                    exit(MY_EX_USAGE);
+                    XEXIT_T(MY_EX_USAGE);
                 }
                 break;
 
@@ -1228,7 +1228,7 @@ THREAD_RETURN WOLFSSL_THREAD client_test(void* args)
                     if (maxFragment < WOLFSSL_MFL_2_9 ||
                                                maxFragment > WOLFSSL_MFL_2_13) {
                         Usage();
-                        exit(MY_EX_USAGE);
+                        XEXIT_T(MY_EX_USAGE);
                     }
                 #endif
                 break;
@@ -1281,7 +1281,7 @@ THREAD_RETURN WOLFSSL_THREAD client_test(void* args)
                         alpn_opt = WOLFSSL_ALPN_FAILED_ON_MISMATCH;
                     else {
                         Usage();
-                        exit(MY_EX_USAGE);
+                        XEXIT_T(MY_EX_USAGE);
                     }
 
                     alpnList += 2;
@@ -1295,7 +1295,7 @@ THREAD_RETURN WOLFSSL_THREAD client_test(void* args)
 
                 if (XSTRNCMP(starttlsProt, "smtp", 4) != 0) {
                     Usage();
-                    exit(MY_EX_USAGE);
+                    XEXIT_T(MY_EX_USAGE);
                 }
 
                 break;
@@ -1371,7 +1371,7 @@ THREAD_RETURN WOLFSSL_THREAD client_test(void* args)
 
             default:
                 Usage();
-                exit(MY_EX_USAGE);
+                XEXIT_T(MY_EX_USAGE);
         }
     }
 
@@ -1457,7 +1457,7 @@ THREAD_RETURN WOLFSSL_THREAD client_test(void* args)
             printf("external test can't be run in this mode");
 
             ((func_args*)args)->return_code = 0;
-            exit(EXIT_SUCCESS);
+            XEXIT_T(EXIT_SUCCESS);
         }
     }
 
@@ -1849,7 +1849,7 @@ THREAD_RETURN WOLFSSL_THREAD client_test(void* args)
                                        benchmark, resumeSession, useX25519,
                                        helloRetry, onlyKeyShare, version);
         wolfSSL_CTX_free(ctx);
-        exit(EXIT_SUCCESS);
+        XEXIT_T(EXIT_SUCCESS);
     }
 
     if(throughput) {
@@ -1857,7 +1857,7 @@ THREAD_RETURN WOLFSSL_THREAD client_test(void* args)
             ClientBenchmarkThroughput(ctx, host, port, dtlsUDP, dtlsSCTP,
                                       block, throughput, useX25519);
         wolfSSL_CTX_free(ctx);
-        exit(EXIT_SUCCESS);
+        XEXIT_T(EXIT_SUCCESS);
     }
 
     #if defined(WOLFSSL_MDK_ARM)

examples/server/server.c

@@ -621,7 +621,7 @@ THREAD_RETURN WOLFSSL_THREAD server_test(void* args)
         switch (ch) {
             case '?' :
                 Usage();
-                exit(EXIT_SUCCESS);
+                XEXIT_T(EXIT_SUCCESS);
 
             case 'x' :
                 runWithErrors = 1;
@@ -701,7 +701,7 @@ THREAD_RETURN WOLFSSL_THREAD server_test(void* args)
                 version = atoi(myoptarg);
                 if (version < 0 || version > 4) {
                     Usage();
-                    exit(MY_EX_USAGE);
+                    XEXIT_T(MY_EX_USAGE);
                 }
                 break;
 
@@ -720,7 +720,7 @@ THREAD_RETURN WOLFSSL_THREAD server_test(void* args)
                 }
                 else {
                     Usage();
-                    exit(MY_EX_USAGE);
+                    XEXIT_T(MY_EX_USAGE);
                 }
                 break;
 
@@ -747,7 +747,7 @@ THREAD_RETURN WOLFSSL_THREAD server_test(void* args)
                     minDhKeyBits = atoi(myoptarg);
                     if (minDhKeyBits <= 0 || minDhKeyBits > 16000) {
                         Usage();
-                        exit(MY_EX_USAGE);
+                        XEXIT_T(MY_EX_USAGE);
                     }
                 #endif
                 break;
@@ -796,7 +796,7 @@ THREAD_RETURN WOLFSSL_THREAD server_test(void* args)
                         alpn_opt = WOLFSSL_ALPN_FAILED_ON_MISMATCH;
                     else {
                         Usage();
-                        exit(MY_EX_USAGE);
+                        XEXIT_T(MY_EX_USAGE);
                     }
 
                     alpnList += 2;
@@ -812,7 +812,7 @@ THREAD_RETURN WOLFSSL_THREAD server_test(void* args)
                 loops = atoi(myoptarg);
                 if (loops <= 0) {
                     Usage();
-                    exit(MY_EX_USAGE);
+                    XEXIT_T(MY_EX_USAGE);
                 }
                 break;
 
@@ -830,7 +830,7 @@ THREAD_RETURN WOLFSSL_THREAD server_test(void* args)
                 }
                 if (throughput <= 0 || block <= 0) {
                     Usage();
-                    exit(MY_EX_USAGE);
+                    XEXIT_T(MY_EX_USAGE);
                 }
                 break;
 
@@ -917,7 +917,7 @@ THREAD_RETURN WOLFSSL_THREAD server_test(void* args)
 
             default:
                 Usage();
-                exit(MY_EX_USAGE);
+                XEXIT_T(MY_EX_USAGE);
         }
     }
 

scripts/include.am

@@ -13,6 +13,7 @@ if BUILD_EXAMPLE_SERVERS
 dist_noinst_SCRIPTS+= scripts/resume.test
 
 EXTRA_DIST+= scripts/benchmark.test
+EXTRA_DIST+= scripts/memtest.sh
 
 if BUILD_CRL
 # make revoked test rely on completion of resume test

scripts/memtest.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+# Run this script from the wolfSSL root as `./scripts/memtest.sh`.
+
+./autogen.sh
+./configure --enable-debug --disable-shared --enable-memtest \
+	--enable-opensslextra --enable-des3 --enable-dh --enable-ecc --enable-aesgcm --enable-aesccm --enable-hc128 \
+	--enable-sniffer --enable-psk --enable-rabbit --enable-camellia --enable-sha512 --enable-crl --enable-ocsp --enable-savesession \
+	--enable-savecert --enable-atomicuser --enable-pkcallbacks --enable-scep;
+
+	#DTLS has issue with trapping client/server failure disconnect since its stateless. Need to find way to communicate failure through file system.
+	#--enable-dtls
+make
+
+for i in {1..1000}
+do
+    echo "Trying $i...\n"
+
+ 	./tests/unit.test > ./scripts/memtest.txt 2>&1
+
+ 	RESULT=$?
+ 	[ $RESULT -eq 139 ] && echo "Mem Seg Fault" && exit 1
+done
+echo "Loop SUCCESS"

tests/api.c

@@ -1349,6 +1349,10 @@ static THREAD_RETURN WOLFSSL_THREAD test_server_nofail(void* args)
     port = wolfSSLPort;
 #endif
 
+    /* do it here to detect failure */
+    tcp_accept(&sockfd, &clientfd, (func_args*)args, port, 0, 0, 0, 0, 1);
+    CloseSocket(sockfd);
+
     wolfSSL_CTX_set_verify(ctx,
                           WOLFSSL_VERIFY_PEER | WOLFSSL_VERIFY_FAIL_IF_NO_PEER_CERT, 0);
 
@@ -1382,8 +1386,6 @@ static THREAD_RETURN WOLFSSL_THREAD test_server_nofail(void* args)
     }
 
     ssl = wolfSSL_new(ctx);
-    tcp_accept(&sockfd, &clientfd, (func_args*)args, port, 0, 0, 0, 0, 1);
-    CloseSocket(sockfd);
 
     if (wolfSSL_set_fd(ssl, clientfd) != WOLFSSL_SUCCESS) {
         /*err_sys("SSL_set_fd failed");*/
@@ -1506,6 +1508,10 @@ static void test_client_nofail(void* args, void *cb)
     wolfSSL_CTX_set_default_passwd_cb(ctx, PasswordCallBack);
 #endif
 
+    /* Do connect here so server detects failures */
+    tcp_connect(&sockfd, wolfSSLIP, ((func_args*)args)->signal->port,
+                0, 0, NULL);
+
     if (wolfSSL_CTX_load_verify_locations(ctx, caCertFile, 0) != WOLFSSL_SUCCESS)
     {
         /* err_sys("can't load ca file, Please run from wolfSSL home dir");*/
@@ -1532,8 +1538,6 @@ static void test_client_nofail(void* args, void *cb)
     }
 
     ssl = wolfSSL_new(ctx);
-    tcp_connect(&sockfd, wolfSSLIP, ((func_args*)args)->signal->port,
-                0, 0, ssl);
     if (wolfSSL_set_fd(ssl, sockfd) != WOLFSSL_SUCCESS) {
         /*err_sys("SSL_set_fd failed");*/
         goto done2;

tests/suites.c

@@ -395,14 +395,14 @@ static int execute_test_case(int svr_argc, char** svr_argv,
     if ((cliArgs.return_code != 0 && testShouldFail == 0) ||
         (cliArgs.return_code == 0 && testShouldFail != 0)) {
         printf("client_test failed\n");
-        exit(EXIT_FAILURE);
+        XEXIT(EXIT_FAILURE);
     }
 
     join_thread(serverThread);
     if ((svrArgs.return_code != 0 && testShouldFail == 0) ||
         (svrArgs.return_code == 0 && testShouldFail != 0)) {
         printf("server_test failed\n");
-        exit(EXIT_FAILURE);
+        XEXIT(EXIT_FAILURE);
     }
 
 #ifdef WOLFSSL_TIRTOS
@@ -619,7 +619,8 @@ int SuiteTest(void)
     cipherSuiteCtx = wolfSSL_CTX_new(wolfSSLv23_client_method());
     if (cipherSuiteCtx == NULL) {
         printf("can't get cipher suite ctx\n");
-        exit(EXIT_FAILURE);
+        args.return_code = EXIT_FAILURE;
+        goto exit;
     }
 
     /* load in static memory buffer if enabled */
@@ -662,7 +663,8 @@ int SuiteTest(void)
     test_harness(&args);
     if (args.return_code != 0) {
         printf("error from script %d\n", args.return_code);
-        exit(EXIT_FAILURE);
+        args.return_code = EXIT_FAILURE;
+        goto exit;
     }
     #ifdef HAVE_ECC
     /* add TLSv13 ECC extra suites */
@@ -671,7 +673,8 @@ int SuiteTest(void)
     test_harness(&args);
     if (args.return_code != 0) {
         printf("error from script %d\n", args.return_code);
-        exit(EXIT_FAILURE);
+        args.return_code = EXIT_FAILURE;
+        goto exit;
     }
     #endif
     #ifndef WOLFSSL_NO_TLS12
@@ -681,7 +684,8 @@ int SuiteTest(void)
     test_harness(&args);
     if (args.return_code != 0) {
         printf("error from script %d\n", args.return_code);
-        exit(EXIT_FAILURE);
+        args.return_code = EXIT_FAILURE;
+        goto exit;
     }
     #endif
 #endif
@@ -692,7 +696,8 @@ int SuiteTest(void)
     test_harness(&args);
     if (args.return_code != 0) {
         printf("error from script %d\n", args.return_code);
-        exit(EXIT_FAILURE);
+        args.return_code = EXIT_FAILURE;
+        goto exit;
     }
 #endif
 #ifdef WOLFSSL_DTLS

tests/unit.c

@@ -49,6 +49,15 @@ int unit_test(int argc, char** argv)
 
     (void)argc;
     (void)argv;
+
+#ifdef WOLFSSL_FORCE_MALLOC_FAIL_TEST
+    if (argc > 1) {
+        word32 memFailCount = atoi(argv[1]);
+        printf("\n--- SET RNG MALLOC FAIL AT %d---\n", memFailCount);
+        wolfSSL_SetMemFailCount(memFailCount);
+    }
+#endif
+
     printf("starting unit tests...\n");
 
 #if defined(DEBUG_WOLFSSL) && !defined(HAVE_VALGRIND)

tests/unit.h

@@ -26,11 +26,17 @@
 #include <wolfssl/ssl.h>
 #include <wolfssl/test.h>    /* thread and tcp stuff */
 
+#ifdef WOLFSSL_FORCE_MALLOC_FAIL_TEST
+#define XABORT()
+#else
+#define XABORT() abort
+#endif
+
 #define Fail(description, result) do {                                         \
     printf("\nERROR - %s line %d failed with:", __FILE__, __LINE__);           \
     printf("\n    expected: "); printf description;                            \
     printf("\n    result:   "); printf result; printf("\n\n");                 \
-    abort();                                                                   \
+    XABORT();                                                                  \
 } while(0)
 
 #define Assert(test, description, result) if (!(test)) Fail(description, result)
@@ -62,7 +68,7 @@
 #define AssertStr(x, y, op, er) do {                                           \
     const char* _x = x;                                                        \
     const char* _y = y;                                                        \
-    int   _z = strcmp(_x, _y);                                                 \
+    int   _z = (_x && _y) ? strcmp(_x, _y) : -1;                               \
                                                                                \
     Assert(_z op 0, ("%s " #op " %s", #x, #y),                                 \
                                             ("\"%s\" " #er " \"%s\"", _x, _y));\

wolfcrypt/src/memory.c

@@ -44,6 +44,18 @@
     #include <stdio.h>
 #endif
 
+#ifdef WOLFSSL_FORCE_MALLOC_FAIL_TEST
+    static int gMemFailCountSeed;
+    static int gMemFailCount;
+    void wolfSSL_SetMemFailCount(int memFailCount)
+    {
+        if (gMemFailCountSeed == 0) {
+            gMemFailCountSeed = memFailCount;
+            gMemFailCount = memFailCount;
+        }
+    }
+#endif
+
 
 /* Set these to default values initially. */
 static wolfSSL_Malloc_cb  malloc_function = NULL;
@@ -123,6 +135,24 @@ void* wolfSSL_Malloc(size_t size)
             puts("wolfSSL_malloc failed");
     #endif
 
+#ifdef WOLFSSL_FORCE_MALLOC_FAIL_TEST
+    if (res && --gMemFailCount == 0) {
+        printf("\n---FORCED MEM FAIL TEST---\n");
+        if (free_function) {
+        #ifdef WOLFSSL_DEBUG_MEMORY
+            free_function(res, func, line);
+        #else
+            free_function(res);
+        #endif
+        }
+        else {
+            free(res); /* clear */
+        }
+        gMemFailCount = gMemFailCountSeed; /* reset */
+        return NULL;
+    }
+#endif
+
     return res;
 }
 

wolfcrypt/src/wc_port.c

@@ -86,6 +86,17 @@ int wolfCrypt_Init(void)
     if (initRefCount == 0) {
         WOLFSSL_ENTER("wolfCrypt_Init");
 
+    #ifdef WOLFSSL_FORCE_MALLOC_FAIL_TEST
+        {
+            word32 rngMallocFail;
+            time_t seed = time(NULL);
+            srand((word32)seed);
+            rngMallocFail = rand() % 2000; /* max 2000 */
+            printf("\n--- RNG MALLOC FAIL AT %d---\n", rngMallocFail);
+            wolfSSL_SetMemFailCount(rngMallocFail);
+        }
+    #endif
+
     #ifdef WOLF_CRYPTO_DEV
         wc_CryptoDev_Init();
     #endif

wolfssl/test.h

@@ -383,7 +383,31 @@ void join_thread(THREAD_TYPE);
 static const word16      wolfSSLPort = 11111;
 
 
-static WC_INLINE WC_NORETURN void err_sys(const char* msg)
+
+#ifndef MY_EX_USAGE
+#define MY_EX_USAGE 2
+#endif
+
+#ifndef EXIT_FAILURE
+#define EXIT_FAILURE 1
+#endif
+
+#ifdef WOLFSSL_FORCE_MALLOC_FAIL_TEST
+    #define XEXIT(rc)   return rc
+    #define XEXIT_T(rc) return (THREAD_RETURN)rc
+#else
+    #define XEXIT(rc)   exit((int)(rc))
+    #define XEXIT_T(rc) exit((int)(rc))
+#endif
+
+
+static WC_INLINE 
+#ifdef WOLFSSL_FORCE_MALLOC_FAIL_TEST
+THREAD_RETURN
+#else
+WC_NORETURN void 
+#endif
+err_sys(const char* msg)
 {
     printf("wolfSSL error: %s\n", msg);
 
@@ -397,13 +421,11 @@ static WC_INLINE WC_NORETURN void err_sys(const char* msg)
     if (msg)
 #endif
     {
-        exit(EXIT_FAILURE);
+        XEXIT((THREAD_RETURN)EXIT_FAILURE);
     }
 }
 
 
-#define MY_EX_USAGE 2
-
 extern int   myoptind;
 extern char* myoptarg;
 

wolfssl/wolfcrypt/memory.h

@@ -36,6 +36,10 @@
     extern "C" {
 #endif
 
+#ifdef WOLFSSL_FORCE_MALLOC_FAIL_TEST
+    WOLFSSL_API void wolfSSL_SetMemFailCount(int memFailCount);
+#endif
+
 #ifdef WOLFSSL_STATIC_MEMORY
     #ifdef WOLFSSL_DEBUG_MEMORY
         typedef void *(*wolfSSL_Malloc_cb)(size_t size, void* heap, int type, const char* func, unsigned int line);