Merge pull request #2631 from SparkiDev/mp_invmod_fix

mp_invmod handles more inputs

wolfcrypt/src/integer.c

@@ -1163,6 +1163,11 @@ int mp_invmod_slow (mp_int * a, mp_int * b, mp_int * c)
   if ((res = mp_mod(a, b, &x)) != MP_OKAY) {
     goto LBL_ERR;
   }
+  if (mp_isone(&x)) {
+    mp_set(c, 1);
+    res = MP_OKAY;
+    goto LBL_ERR;
+  }
   if ((res = mp_copy (b, &y)) != MP_OKAY) {
     goto LBL_ERR;
   }

wolfcrypt/src/tfm.c

@@ -897,6 +897,9 @@ static int fp_invmod_slow (fp_int * a, fp_int * b, fp_int * c)
   if (b->sign == FP_NEG || fp_iszero(b) == FP_YES) {
     return FP_VAL;
   }
+  if (fp_iszero(a) == FP_YES) {
+    return FP_VAL;
+  }
 
 #ifdef WOLFSSL_SMALL_STACK
   x = (fp_int*)XMALLOC(sizeof(fp_int) * 8, NULL, DYNAMIC_TYPE_BIGINT);
@@ -1022,6 +1025,11 @@ int fp_invmod(fp_int *a, fp_int *b, fp_int *c)
   fp_int  *x, *y, *u, *v, *B, *D;
 #endif
   int     neg;
+  int     err;
+
+  if (b->sign == FP_NEG || fp_iszero(b) == FP_YES) {
+    return FP_VAL;
+  }
 
   /* [modified] sanity check on "a" */
   if (fp_iszero(a) == FP_YES) {
@@ -1046,6 +1054,24 @@ int fp_invmod(fp_int *a, fp_int *b, fp_int *c)
   fp_init(u);  fp_init(v);
   fp_init(B);  fp_init(D);
 
+  if (fp_cmp(a, b) != MP_LT) {
+    err = mp_mod(a, b, y);
+    if (err != FP_OKAY) {
+    #ifdef WOLFSSL_SMALL_STACK
+      XFREE(x, NULL, DYNAMIC_TYPE_BIGINT);
+    #endif
+      return err;
+    }
+    a = y;
+  }
+
+  if (fp_iszero(a) == FP_YES) {
+  #ifdef WOLFSSL_SMALL_STACK
+    XFREE(x, NULL, DYNAMIC_TYPE_BIGINT);
+  #endif
+    return FP_VAL;
+  }
+
   /* x == modulus, y == value to invert */
   fp_copy(b, x);