Prime Number Testing

1. Update the function wolfSSL_BN_is_prime_ex to use mp_prime_is_prime_ex.
2. Modified fast and normal mp_prime_is_prime_ex() to use random numbers
that are in the range 2 < a < n-2.

src/ssl.c

@@ -22526,7 +22526,14 @@ int wolfSSL_BN_add(WOLFSSL_BIGNUM *r, WOLFSSL_BIGNUM *a, WOLFSSL_BIGNUM *b)
 int wolfSSL_BN_is_prime_ex(const WOLFSSL_BIGNUM *bn, int nbchecks,
                            WOLFSSL_BN_CTX *ctx, WOLFSSL_BN_GENCB *cb)
 {
-    int res;
+    WC_RNG*        rng    = NULL;
+#ifdef WOLFSSL_SMALL_STACK
+    WC_RNG*        tmpRNG = NULL;
+#else
+    WC_RNG         tmpRNG[1];
+#endif
+    int            initTmpRng = 0;
+    int            res = MP_NO;
 
     (void)ctx;
     (void)cb;
@@ -22538,10 +22545,38 @@ int wolfSSL_BN_is_prime_ex(const WOLFSSL_BIGNUM *bn, int nbchecks,
         return WOLFSSL_FATAL_ERROR;
     }
 
-    if (mp_prime_is_prime((mp_int*)bn->internal, nbchecks, &res) != MP_OKAY) {
+#ifdef WOLFSSL_SMALL_STACK
-        WOLFSSL_MSG("mp_prime_is_prime error");
+    tmpRNG = (WC_RNG*)XMALLOC(sizeof(WC_RNG), NULL, DYNAMIC_TYPE_RNG);
-        return WOLFSSL_FATAL_ERROR;
+    if (tmpRNG == NULL)
+        return WOLFSSL_FAILURE;
+#endif
+    if (wc_InitRng(tmpRNG) == 0) {
+        rng = tmpRNG;
+        initTmpRng = 1;
     }
+    else {
+        WOLFSSL_MSG("Bad RNG Init, trying global");
+        if (initGlobalRNG == 0) {
+            WOLFSSL_MSG("Global RNG no Init");
+        }
+        else
+            rng = &globalRNG;
+    }
+
+    if (rng) {
+        if (mp_prime_is_prime_ex((mp_int*)bn->internal,
+                                 nbchecks, &res, rng) != MP_OKAY) {
+            WOLFSSL_MSG("mp_prime_is_prime error");
+            res = MP_NO;
+        }
+    }
+
+    if (initTmpRng)
+        wc_FreeRng(tmpRNG);
+#ifdef WOLFSSL_SMALL_STACK
+    tmpRNG = (WC_RNG*)XMALLOC(sizeof(WC_RNG), NULL, DYNAMIC_TYPE_RNG);
+    XFREE(tmpRNG, NULL, DYNAMIC_TYPE_RNG);
+#endif
 
     if (res != MP_YES) {
         WOLFSSL_MSG("mp_prime_is_prime not prime");

wolfcrypt/src/integer.c

@@ -4609,9 +4609,10 @@ LBL_B:mp_clear (&b);
  */
 int mp_prime_is_prime_ex (mp_int * a, int t, int *result, WC_RNG *rng)
 {
-  mp_int  b;
+  mp_int  b, c;
   int     ix, err, res;
-  byte    scratch[16];
+  byte*   base = NULL;
+  word32  baseSz = 0;
 
   /* default to no */
   *result = MP_NO;
@@ -4643,19 +4644,39 @@ int mp_prime_is_prime_ex (mp_int * a, int t, int *result, WC_RNG *rng)
   if ((err = mp_init (&b)) != MP_OKAY) {
     return err;
   }
+  if ((err = mp_init (&c)) != MP_OKAY) {
+      mp_clear(&b);
+    return err;
+  }
+
+  baseSz = mp_count_bits(a);
+  baseSz = (baseSz / 8) + (baseSz % 8) ? 1 : 0;
+
+  base = (byte*)XMALLOC(baseSz, NULL, DYNAMIC_TYPE_TMP_BUFFER);
+  if (base == NULL) {
+      err = MP_MEM;
+      goto LBL_B;
+  }
+
+  if ((err = mp_copy(a, 2, &c)) != MP_OKAY) {
+      goto LBL_B;
+  }
 
  /* now do a miller rabin with up to t random numbers, this should
   * give a (1/4)^t chance of a false prime. */
   for (ix = 0; ix < t; ix++) {
     /* Set a test candidate. */
-    if ((err = wc_RNG_GenerateBlock(rng, scratch, sizeof(scratch))) != 0) {
+    if ((err = wc_RNG_GenerateBlock(rng, base, baseSz)) != 0) {
         goto LBL_B;
     }
 
-    if ((err = mp_read_unsigned_bin(&b, scratch, sizeof(scratch))) != MP_OKAY) {
+    if ((err = mp_read_unsigned_bin(&b, base, baseSz)) != MP_OKAY) {
         goto LBL_B;
     }
 
+    if (mp_cmp_d(&b, 2) != MP_GT || mp_cmp(&b, &c) != MP_LT)
+        continue;
+
     if ((err = mp_prime_miller_rabin (a, &b, &res)) != MP_OKAY) {
       goto LBL_B;
     }
@@ -4668,6 +4689,8 @@ int mp_prime_is_prime_ex (mp_int * a, int t, int *result, WC_RNG *rng)
   /* passed the test */
   *result = MP_YES;
 LBL_B:mp_clear (&b);
+      mp_clear (&c);
+      XFREE(base, NULL, DYNAMIC_TYPE_TMP_BUFFER);
   return err;
 }
 

wolfcrypt/src/tfm.c

@@ -2760,7 +2760,7 @@ int mp_mod_d(fp_int *a, fp_digit b, fp_digit *c)
 static void fp_gcd(fp_int *a, fp_int *b, fp_int *c);
 static void fp_lcm(fp_int *a, fp_int *b, fp_int *c);
 static int  fp_isprime_ex(fp_int *a, int t);
-static int  fp_isprime(fp_int *a);
+/* static int  fp_isprime(fp_int *a); */
 static int  fp_randprime(fp_int* N, int len, WC_RNG* rng, void* heap);
 
 int mp_gcd(fp_int *a, fp_int *b, fp_int *c)
@@ -2780,7 +2780,7 @@ int mp_lcm(fp_int *a, fp_int *b, fp_int *c)
 int mp_prime_is_prime(mp_int* a, int t, int* result)
 {
     (void)t;
-    *result = fp_isprime(a);
+    *result = fp_isprime_ex(a, t);
     return MP_OKAY;
 }
 
@@ -2940,11 +2940,13 @@ int fp_isprime_ex(fp_int *a, int t)
    return FP_YES;
 }
 
+#if 0
+/* Removed in favor of fp_isprime_ex(). */
 int fp_isprime(fp_int *a)
 {
   return fp_isprime_ex(a, 8);
 }
-
+#endif
 
 int mp_prime_is_prime_ex(mp_int* a, int t, int* result, WC_RNG* rng)
 {
@@ -2971,13 +2973,33 @@ int mp_prime_is_prime_ex(mp_int* a, int t, int* result, WC_RNG* rng)
     /* now do a miller rabin with up to t random numbers, this should
      * give a (1/4)^t chance of a false prime. */
     if (ret == FP_YES) {
-        byte scratch[16];
+        fp_int b, c;
-        fp_int b;
+        /* FP_MAX_BITS is 2 times the modulus size. The modulus size is
+         * 2 times the prime size. */
+        word32 baseSz;
+        #ifndef WOLFSSL_SMALL_STACK
+            byte base[FP_MAX_BITS/32];
+        #else
+            byte* base;
+        #endif
+
+        baseSz = fp_count_bits(a);
+        baseSz = (baseSz / 8) + (baseSz % 8) ? 1 : 0;
+
+        #ifdef WOLFSSL_SMALL_STACK
+            base = (byte*)XMALLOC(baseSz, NULL, DYNAMIC_TYPE_TMP_BUFFER);
+            if (base == NULL)
+                return FP_MEM;
+        #endif
 
         fp_init(&b);
+        fp_init(&c);
+        fp_sub_d(a, 2, &c);
         while (t > 0) {
-            wc_RNG_GenerateBlock(rng, scratch, sizeof(scratch));
+            wc_RNG_GenerateBlock(rng, base, baseSz);
-            fp_read_unsigned_bin(&b, scratch, sizeof(scratch));
+            fp_read_unsigned_bin(&b, base, baseSz);
+            if (fp_cmp_d(&b, 2) != FP_GT || fp_cmp(&b, &c) != FP_LT)
+                continue;
             fp_prime_miller_rabin(a, &b, &ret);
             if (ret == FP_NO)
                 break;
@@ -2985,6 +3007,10 @@ int mp_prime_is_prime_ex(mp_int* a, int t, int* result, WC_RNG* rng)
             t--;
         }
         fp_clear(&b);
+        fp_clear(&c);
+        #ifdef WOLFSSL_SMALL_STACK
+            XFREE(base, NULL, DYNAMIC_TYPE_TMP_BUFFER);
+        #endif
     }
 
     *result = ret;