Merge pull request #6468 from philljj/zd16236

Fix fastmath and heapmath invmod to be consistent with sp-math.

wolfcrypt/src/integer.c

@@ -1058,7 +1058,7 @@ int mp_invmod (mp_int * a, mp_int * b, mp_int * c)
 int fast_mp_invmod (mp_int * a, mp_int * b, mp_int * c)
 {
   mp_int  x, y, u, v, B, D;
-  int     res, neg, loop_check = 0;
+  int     res, loop_check = 0;
 
   /* 2. [modified] b must be odd   */
   if (mp_iseven (b) == MP_YES) {
@@ -1080,6 +1080,12 @@ int fast_mp_invmod (mp_int * a, mp_int * b, mp_int * c)
     goto LBL_ERR;
   }
 
+  if (mp_iszero (&y) == MP_YES) {
+    /* invmod doesn't exist for this a and b */
+    res = MP_VAL;
+    goto LBL_ERR;
+  }
+
   /* 3. u=x, v=y, A=1, B=0, C=0,D=1 */
   if ((res = mp_copy (&x, &u)) != MP_OKAY) {
     goto LBL_ERR;
@@ -1168,7 +1174,6 @@ top:
   }
 
   /* b is now the inverse */
-  neg = a->sign;
   while (D.sign == MP_NEG) {
     if ((res = mp_add (&D, b, &D)) != MP_OKAY) {
       goto LBL_ERR;
@@ -1181,7 +1186,6 @@ top:
       }
   }
   mp_exch (&D, c);
-  c->sign = neg;
   res = MP_OKAY;
 
 LBL_ERR:mp_clear(&x);
@@ -1226,6 +1230,13 @@ int mp_invmod_slow (mp_int * a, mp_int * b, mp_int * c)
   if ((res = mp_mod(a, b, &x)) != MP_OKAY) {
     goto LBL_ERR;
   }
+
+  if (mp_iszero (&x) == MP_YES) {
+    /* invmod doesn't exist for this a and b */
+    res = MP_VAL;
+    goto LBL_ERR;
+  }
+
   if (mp_isone(&x)) {
     res = mp_set(c, 1);
     goto LBL_ERR;

wolfcrypt/src/tfm.c

@@ -1086,6 +1086,14 @@ static int fp_invmod_slow (fp_int * a, fp_int * b, fp_int * c)
   }
   fp_copy(b, y);
 
+  if (fp_iszero(x) == FP_YES) {
+    /* invmod doesn't exist for this a and b */
+  #ifdef WOLFSSL_SMALL_STACK
+    XFREE(x, NULL, DYNAMIC_TYPE_BIGINT);
+  #endif
+    return FP_VAL;
+  }
+
   /* 2. [modified] if x,y are both even then return an error! */
   if (fp_iseven(x) == FP_YES && fp_iseven(y) == FP_YES) {
   #ifdef WOLFSSL_SMALL_STACK
@@ -1258,7 +1266,6 @@ int fp_invmod(fp_int *a, fp_int *b, fp_int *c)
 #else
   fp_int  *x, *y, *u, *v, *B, *D;
 #endif
-  int     neg;
   int     err;
 
   if (b->sign == FP_NEG || fp_iszero(b) == FP_YES) {
@@ -1288,17 +1295,6 @@ int fp_invmod(fp_int *a, fp_int *b, fp_int *c)
   fp_init(u);  fp_init(v);
   fp_init(B);  fp_init(D);
 
-  if (fp_cmp(a, b) != MP_LT) {
-    err = mp_mod(a, b, y);
-    if (err != FP_OKAY) {
-    #ifdef WOLFSSL_SMALL_STACK
-      XFREE(x, NULL, DYNAMIC_TYPE_BIGINT);
-    #endif
-      return err;
-    }
-    a = y;
-  }
-
   if (fp_iszero(a) == FP_YES) {
   #ifdef WOLFSSL_SMALL_STACK
     XFREE(x, NULL, DYNAMIC_TYPE_BIGINT);
@@ -1310,7 +1306,20 @@ int fp_invmod(fp_int *a, fp_int *b, fp_int *c)
   fp_copy(b, x);
 
   /* we need y = |a| */
-  fp_abs(a, y);
+  if ((err = mp_mod(a, b, y)) != FP_OKAY) {
+  #ifdef WOLFSSL_SMALL_STACK
+    XFREE(x, NULL, DYNAMIC_TYPE_BIGINT);
+  #endif
+    return err;
+  }
+
+  if (fp_iszero(y) == FP_YES) {
+    /* invmod doesn't exist for this a and b */
+  #ifdef WOLFSSL_SMALL_STACK
+    XFREE(x, NULL, DYNAMIC_TYPE_BIGINT);
+  #endif
+    return FP_VAL;
+  }
 
   /* 3. u=x, v=y, A=1, B=0, C=0,D=1 */
   fp_copy(x, u);
@@ -1408,7 +1417,6 @@ top:
   }
 
   /* b is now the inverse */
-  neg = a->sign;
   while (D->sign == FP_NEG) {
     err = fp_add (D, b, D);
     if (err != FP_OKAY) {
@@ -1429,7 +1437,6 @@ top:
     }
   }
   fp_copy (D, c);
-  c->sign = neg;
 #ifdef WOLFSSL_SMALL_STACK
   XFREE(x, NULL, DYNAMIC_TYPE_BIGINT);
 #endif