From d342e341d80ed5e1a6c83bf23808136affbae156 Mon Sep 17 00:00:00 2001
From: Sean Parkinson <sean@wolfssl.com>
Date: Fri, 5 May 2023 09:11:47 +1000
Subject: [PATCH] SP int; fix sp_gcd error checking

r can be as large as the smaller of a and b.
Fix sign check.
Add comments as to what GCD does.
---
 wolfcrypt/src/sp_int.c | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/wolfcrypt/src/sp_int.c b/wolfcrypt/src/sp_int.c
index d15a4e27e..74ded88aa 100644
--- a/wolfcrypt/src/sp_int.c
+++ b/wolfcrypt/src/sp_int.c
@@ -18697,6 +18697,9 @@ int sp_prime_is_prime_ex(const sp_int* a, int trials, int* result, WC_RNG* rng)
 #if !defined(NO_RSA) && defined(WOLFSSL_KEY_GEN)
 
 /* Calculates the Greatest Common Denominator (GCD) of a and b into r.
+ *
+ * Find the largest number that divides both a and b without remainder.
+ * r <= a, r <= b, a % r == 0, b % r == 0
  *
  * a and b are positive integers.
  *
@@ -18800,6 +18803,9 @@ static WC_INLINE int _sp_gcd(const sp_int* a, const sp_int* b, sp_int* r)
 }
 
 /* Calculates the Greatest Common Denominator (GCD) of a and b into r.
+ *
+ * Find the largest number that divides both a and b without remainder.
+ * r <= a, r <= b, a % r == 0, b % r == 0
  *
  * a and b are positive integers.
  *
@@ -18823,8 +18829,14 @@ int sp_gcd(const sp_int* a, const sp_int* b, sp_int* r)
     else if ((a->used >= SP_INT_DIGITS) || (b->used >= SP_INT_DIGITS)) {
         err = MP_VAL;
     }
+    /* Check that r is large enough to hold maximum sized result. */
+    else if (((a->used <= b->used) && (r->size < a->used)) ||
+             ((b->used < a->used) && (r->size < b->used))) {
+        err = MP_VAL;
+    }
 #ifdef WOLFSSL_SP_INT_NEGATIVE
-    else if ((a->sign == MP_NEG) || (b->sign >= MP_NEG)) {
+    /* Algorithm doesn't work with negative numbers. */
+    else if ((a->sign == MP_NEG) || (b->sign == MP_NEG)) {
         err = MP_VAL;
     }
 #endif