From cb9e0a1973854ab39ebaba0e3bf8473728a3ee79 Mon Sep 17 00:00:00 2001 From: Sean Parkinson Date: Wed, 22 May 2019 08:39:35 +1000 Subject: [PATCH] SP - div word in C when __div64 or __div128 not available Use WOLFSSL_SP_DIV_32 when using 32-bit C code. Use WOLFSSL_SP_DIV_64 when using 64-bit C code. --- wolfcrypt/src/sp_c32.c | 204 +++++++++++++++++++++++++++++++++-- wolfcrypt/src/sp_c64.c | 240 +++++++++++++++++++++++++++++++++++++++-- 2 files changed, 424 insertions(+), 20 deletions(-) diff --git a/wolfcrypt/src/sp_c32.c b/wolfcrypt/src/sp_c32.c index e53f2c158..a9a7c8bb3 100644 --- a/wolfcrypt/src/sp_c32.c +++ b/wolfcrypt/src/sp_c32.c @@ -1689,6 +1689,36 @@ SP_NOINLINE static void sp_2048_rshift_45(sp_digit* r, sp_digit* a, byte n) r[44] = a[44] >> n; } +#ifdef WOLFSSL_SP_DIV_32 +static WC_INLINE sp_digit sp_2048_div_word_45(sp_digit d1, sp_digit d0, + sp_digit div) +{ + sp_digit d, r, t; + + /* All 23 bits from d1 and top 8 bits from d0. */ + d = (d1 << 8) | (d0 >> 15); + r = d / div; + d -= r * div; + /* Up to 9 bits in r */ + /* Next 8 bits from d0. */ + r <<= 8; + d <<= 8; + d |= (d0 >> 7) & ((1 << 8) - 1); + t = d / div; + d -= t * div; + r += t; + /* Up to 17 bits in r */ + /* Remaining 7 bits from d0. */ + r <<= 7; + d <<= 7; + d |= d0 & ((1 << 7) - 1); + t = d / div; + r += t; + + return r; +} +#endif /* WOLFSSL_SP_DIV_32 */ + /* Divide d in a and put remainder into r (m*d + r = a) * m is not calculated as it is not needed at this time. * @@ -1702,7 +1732,9 @@ static int sp_2048_div_45(sp_digit* a, sp_digit* d, sp_digit* m, sp_digit* r) { int i; +#ifndef WOLFSSL_SP_DIV_32 int64_t d1; +#endif sp_digit div, r1; #if defined(WOLFSSL_SP_SMALL) || defined(WOLFSSL_SMALL_STACK) sp_digit* td; @@ -1739,10 +1771,14 @@ static int sp_2048_div_45(sp_digit* a, sp_digit* d, sp_digit* m, for (i=45; i>=0; i--) { t1[45 + i] += t1[45 + i - 1] >> 23; t1[45 + i - 1] &= 0x7fffff; +#ifndef WOLFSSL_SP_DIV_32 d1 = t1[45 + i]; d1 <<= 23; d1 += t1[45 + i - 1]; r1 = (sp_digit)(d1 / div); +#else + r1 = sp_2048_div_word_45(t1[45 + i], t1[45 + i - 1], div); +#endif sp_2048_mul_d_45(t2, sd, r1); sp_2048_sub_45(&t1[i], &t1[i], t2); @@ -1758,8 +1794,7 @@ static int sp_2048_div_45(sp_digit* a, sp_digit* d, sp_digit* m, } t1[45 - 1] += t1[45 - 2] >> 23; t1[45 - 2] &= 0x7fffff; - d1 = t1[45 - 1]; - r1 = (sp_digit)(d1 / div); + r1 = t1[45 - 1] / div; sp_2048_mul_d_45(t2, sd, r1); sp_2048_sub_45(t1, t1, t2); @@ -2556,6 +2591,36 @@ SP_NOINLINE static void sp_2048_rshift_90(sp_digit* r, sp_digit* a, byte n) r[89] = a[89] >> n; } +#ifdef WOLFSSL_SP_DIV_32 +static WC_INLINE sp_digit sp_2048_div_word_90(sp_digit d1, sp_digit d0, + sp_digit div) +{ + sp_digit d, r, t; + + /* All 23 bits from d1 and top 8 bits from d0. */ + d = (d1 << 8) | (d0 >> 15); + r = d / div; + d -= r * div; + /* Up to 9 bits in r */ + /* Next 8 bits from d0. */ + r <<= 8; + d <<= 8; + d |= (d0 >> 7) & ((1 << 8) - 1); + t = d / div; + d -= t * div; + r += t; + /* Up to 17 bits in r */ + /* Remaining 7 bits from d0. */ + r <<= 7; + d <<= 7; + d |= d0 & ((1 << 7) - 1); + t = d / div; + r += t; + + return r; +} +#endif /* WOLFSSL_SP_DIV_32 */ + /* Divide d in a and put remainder into r (m*d + r = a) * m is not calculated as it is not needed at this time. * @@ -2569,7 +2634,9 @@ static int sp_2048_div_90(sp_digit* a, sp_digit* d, sp_digit* m, sp_digit* r) { int i; +#ifndef WOLFSSL_SP_DIV_32 int64_t d1; +#endif sp_digit div, r1; #if defined(WOLFSSL_SP_SMALL) || defined(WOLFSSL_SMALL_STACK) sp_digit* td; @@ -2606,10 +2673,14 @@ static int sp_2048_div_90(sp_digit* a, sp_digit* d, sp_digit* m, for (i=90; i>=0; i--) { t1[90 + i] += t1[90 + i - 1] >> 23; t1[90 + i - 1] &= 0x7fffff; +#ifndef WOLFSSL_SP_DIV_32 d1 = t1[90 + i]; d1 <<= 23; d1 += t1[90 + i - 1]; r1 = (sp_digit)(d1 / div); +#else + r1 = sp_2048_div_word_90(t1[90 + i], t1[90 + i - 1], div); +#endif sp_2048_mul_d_90(t2, sd, r1); sp_2048_sub_90(&t1[i], &t1[i], t2); @@ -2625,8 +2696,7 @@ static int sp_2048_div_90(sp_digit* a, sp_digit* d, sp_digit* m, } t1[90 - 1] += t1[90 - 2] >> 23; t1[90 - 2] &= 0x7fffff; - d1 = t1[90 - 1]; - r1 = (sp_digit)(d1 / div); + r1 = t1[90 - 1] / div; sp_2048_mul_d_90(t2, sd, r1); sp_2048_sub_90(t1, t1, t2); @@ -4900,6 +4970,36 @@ SP_NOINLINE static int sp_3072_add_67(sp_digit* r, const sp_digit* a, return 0; } #endif +#ifdef WOLFSSL_SP_DIV_32 +static WC_INLINE sp_digit sp_3072_div_word_67(sp_digit d1, sp_digit d0, + sp_digit div) +{ + sp_digit d, r, t; + + /* All 23 bits from d1 and top 8 bits from d0. */ + d = (d1 << 8) | (d0 >> 15); + r = d / div; + d -= r * div; + /* Up to 9 bits in r */ + /* Next 8 bits from d0. */ + r <<= 8; + d <<= 8; + d |= (d0 >> 7) & ((1 << 8) - 1); + t = d / div; + d -= t * div; + r += t; + /* Up to 17 bits in r */ + /* Remaining 7 bits from d0. */ + r <<= 7; + d <<= 7; + d |= d0 & ((1 << 7) - 1); + t = d / div; + r += t; + + return r; +} +#endif /* WOLFSSL_SP_DIV_32 */ + /* Divide d in a and put remainder into r (m*d + r = a) * m is not calculated as it is not needed at this time. * @@ -4913,7 +5013,9 @@ static int sp_3072_div_67(sp_digit* a, sp_digit* d, sp_digit* m, sp_digit* r) { int i; +#ifndef WOLFSSL_SP_DIV_32 int64_t d1; +#endif sp_digit div, r1; #if defined(WOLFSSL_SP_SMALL) || defined(WOLFSSL_SMALL_STACK) sp_digit* td; @@ -4946,10 +5048,14 @@ static int sp_3072_div_67(sp_digit* a, sp_digit* d, sp_digit* m, for (i=66; i>=0; i--) { t1[67 + i] += t1[67 + i - 1] >> 23; t1[67 + i - 1] &= 0x7fffff; +#ifndef WOLFSSL_SP_DIV_32 d1 = t1[67 + i]; d1 <<= 23; d1 += t1[67 + i - 1]; r1 = (sp_digit)(d1 / div); +#else + r1 = sp_3072_div_word_67(t1[67 + i], t1[67 + i - 1], div); +#endif sp_3072_mul_d_67(t2, d, r1); sp_3072_sub_67(&t1[i], &t1[i], t2); @@ -4965,8 +5071,7 @@ static int sp_3072_div_67(sp_digit* a, sp_digit* d, sp_digit* m, } t1[67 - 1] += t1[67 - 2] >> 23; t1[67 - 2] &= 0x7fffff; - d1 = t1[67 - 1]; - r1 = (sp_digit)(d1 / div); + r1 = t1[67 - 1] / div; sp_3072_mul_d_67(t2, d, r1); sp_3072_sub_67(t1, t1, t2); @@ -5797,6 +5902,36 @@ SP_NOINLINE static void sp_3072_rshift_134(sp_digit* r, sp_digit* a, byte n) r[133] = a[133] >> n; } +#ifdef WOLFSSL_SP_DIV_32 +static WC_INLINE sp_digit sp_3072_div_word_134(sp_digit d1, sp_digit d0, + sp_digit div) +{ + sp_digit d, r, t; + + /* All 23 bits from d1 and top 8 bits from d0. */ + d = (d1 << 8) | (d0 >> 15); + r = d / div; + d -= r * div; + /* Up to 9 bits in r */ + /* Next 8 bits from d0. */ + r <<= 8; + d <<= 8; + d |= (d0 >> 7) & ((1 << 8) - 1); + t = d / div; + d -= t * div; + r += t; + /* Up to 17 bits in r */ + /* Remaining 7 bits from d0. */ + r <<= 7; + d <<= 7; + d |= d0 & ((1 << 7) - 1); + t = d / div; + r += t; + + return r; +} +#endif /* WOLFSSL_SP_DIV_32 */ + /* Divide d in a and put remainder into r (m*d + r = a) * m is not calculated as it is not needed at this time. * @@ -5810,7 +5945,9 @@ static int sp_3072_div_134(sp_digit* a, sp_digit* d, sp_digit* m, sp_digit* r) { int i; +#ifndef WOLFSSL_SP_DIV_32 int64_t d1; +#endif sp_digit div, r1; #if defined(WOLFSSL_SP_SMALL) || defined(WOLFSSL_SMALL_STACK) sp_digit* td; @@ -5847,10 +5984,14 @@ static int sp_3072_div_134(sp_digit* a, sp_digit* d, sp_digit* m, for (i=134; i>=0; i--) { t1[134 + i] += t1[134 + i - 1] >> 23; t1[134 + i - 1] &= 0x7fffff; +#ifndef WOLFSSL_SP_DIV_32 d1 = t1[134 + i]; d1 <<= 23; d1 += t1[134 + i - 1]; r1 = (sp_digit)(d1 / div); +#else + r1 = sp_3072_div_word_134(t1[134 + i], t1[134 + i - 1], div); +#endif sp_3072_mul_d_134(t2, sd, r1); sp_3072_sub_134(&t1[i], &t1[i], t2); @@ -5866,8 +6007,7 @@ static int sp_3072_div_134(sp_digit* a, sp_digit* d, sp_digit* m, } t1[134 - 1] += t1[134 - 2] >> 23; t1[134 - 2] &= 0x7fffff; - d1 = t1[134 - 1]; - r1 = (sp_digit)(d1 / div); + r1 = t1[134 - 1] / div; sp_3072_mul_d_134(t2, sd, r1); sp_3072_sub_134(t1, t1, t2); @@ -11346,6 +11486,45 @@ SP_NOINLINE static void sp_256_mul_d_10(sp_digit* r, const sp_digit* a, #endif /* WOLFSSL_SP_SMALL */ } +#ifdef WOLFSSL_SP_DIV_32 +static WC_INLINE sp_digit sp_256_div_word_10(sp_digit d1, sp_digit d0, + sp_digit div) +{ + sp_digit d, r, t, dv; + int64_t t0, t1; + + /* dv has 14 bits. */ + dv = (div >> 12) + 1; + /* All 26 bits from d1 and top 5 bits from d0. */ + d = (d1 << 5) | (d0 >> 21); + r = d / dv; + d -= r * dv; + /* Up to 17 bits in r */ + /* Next 9 bits from d0. */ + d <<= 9; + r <<= 9; + d |= (d0 >> 12) & ((1 << 9) - 1); + t = d / dv; + d -= t * dv; + r += t; + /* Up to 26 bits in r */ + + /* Handle rounding error with dv - top part */ + t0 = ((int64_t)d1 << 26) + d0; + t1 = (int64_t)r * div; + t1 = t0 - t1; + t = (sp_digit)(t1 >> 12) / dv; + r += t; + + /* Handle rounding error with dv - bottom 32 bits */ + t1 = (sp_digit)t0 - (r * div); + t = (sp_digit)t1 / div; + r += t; + + return r; +} +#endif /* WOLFSSL_SP_DIV_32 */ + /* Divide d in a and put remainder into r (m*d + r = a) * m is not calculated as it is not needed at this time. * @@ -11359,7 +11538,9 @@ static int sp_256_div_10(sp_digit* a, sp_digit* d, sp_digit* m, sp_digit* r) { int i; +#ifndef WOLFSSL_SP_DIV_32 int64_t d1; +#endif sp_digit div, r1; #if defined(WOLFSSL_SP_SMALL) || defined(WOLFSSL_SMALL_STACK) sp_digit* td; @@ -11392,10 +11573,14 @@ static int sp_256_div_10(sp_digit* a, sp_digit* d, sp_digit* m, for (i=9; i>=0; i--) { t1[10 + i] += t1[10 + i - 1] >> 26; t1[10 + i - 1] &= 0x3ffffff; +#ifndef WOLFSSL_SP_DIV_32 d1 = t1[10 + i]; d1 <<= 26; d1 += t1[10 + i - 1]; r1 = (sp_digit)(d1 / div); +#else + r1 = sp_256_div_word_10(t1[10 + i], t1[10 + i - 1], div); +#endif sp_256_mul_d_10(t2, d, r1); sp_256_sub_10(&t1[i], &t1[i], t2); @@ -11411,8 +11596,7 @@ static int sp_256_div_10(sp_digit* a, sp_digit* d, sp_digit* m, } t1[10 - 1] += t1[10 - 2] >> 26; t1[10 - 2] &= 0x3ffffff; - d1 = t1[10 - 1]; - r1 = (sp_digit)(d1 / div); + r1 = t1[10 - 1] / div; sp_256_mul_d_10(t2, d, r1); sp_256_sub_10(t1, t1, t2); diff --git a/wolfcrypt/src/sp_c64.c b/wolfcrypt/src/sp_c64.c index b0def7c94..046d432d2 100644 --- a/wolfcrypt/src/sp_c64.c +++ b/wolfcrypt/src/sp_c64.c @@ -1289,6 +1289,45 @@ SP_NOINLINE static int sp_2048_add_18(sp_digit* r, const sp_digit* a, return 0; } #endif +#ifdef WOLFSSL_SP_DIV_64 +static WC_INLINE sp_digit sp_2048_div_word_18(sp_digit d1, sp_digit d0, + sp_digit div) +{ + sp_digit d, r, t, dv; + int128_t t0, t1; + + /* dv has 29 bits. */ + dv = (div >> 28) + 1; + /* All 57 bits from d1 and top 6 bits from d0. */ + d = (d1 << 6) | (d0 >> 51); + r = d / dv; + d -= r * dv; + /* Up to 34 bits in r */ + /* Next 23 bits from d0. */ + d <<= 23; + r <<= 23; + d |= (d0 >> 28) & ((1 << 23) - 1); + t = d / dv; + d -= t * dv; + r += t; + /* Up to 57 bits in r */ + + /* Handle rounding error with dv - top part */ + t0 = ((int128_t)d1 << 57) + d0; + t1 = (int128_t)r * div; + t1 = t0 - t1; + t = (sp_digit)(t1 >> 28) / dv; + r += t; + + /* Handle rounding error with dv - bottom 64 bits */ + t1 = (sp_digit)t0 - (r * div); + t = (sp_digit)t1 / div; + r += t; + + return r; +} +#endif /* WOLFSSL_SP_DIV_64 */ + /* Divide d in a and put remainder into r (m*d + r = a) * m is not calculated as it is not needed at this time. * @@ -1302,7 +1341,9 @@ static int sp_2048_div_18(sp_digit* a, sp_digit* d, sp_digit* m, sp_digit* r) { int i; +#ifndef WOLFSSL_SP_DIV_64 int128_t d1; +#endif sp_digit div, r1; #if defined(WOLFSSL_SP_SMALL) || defined(WOLFSSL_SMALL_STACK) sp_digit* td; @@ -1335,10 +1376,14 @@ static int sp_2048_div_18(sp_digit* a, sp_digit* d, sp_digit* m, for (i=17; i>=0; i--) { t1[18 + i] += t1[18 + i - 1] >> 57; t1[18 + i - 1] &= 0x1ffffffffffffffl; +#ifndef WOLFSSL_SP_DIV_64 d1 = t1[18 + i]; d1 <<= 57; d1 += t1[18 + i - 1]; r1 = (sp_digit)(d1 / div); +#else + r1 = sp_2048_div_word_18(t1[18 + i], t1[18 + i - 1], div); +#endif sp_2048_mul_d_18(t2, d, r1); sp_2048_sub_18(&t1[i], &t1[i], t2); @@ -1354,8 +1399,7 @@ static int sp_2048_div_18(sp_digit* a, sp_digit* d, sp_digit* m, } t1[18 - 1] += t1[18 - 2] >> 57; t1[18 - 2] &= 0x1ffffffffffffffl; - d1 = t1[18 - 1]; - r1 = (sp_digit)(d1 / div); + r1 = t1[18 - 1] / div; sp_2048_mul_d_18(t2, d, r1); sp_2048_sub_18(t1, t1, t2); @@ -2096,6 +2140,45 @@ SP_NOINLINE static int sp_2048_add_36(sp_digit* r, const sp_digit* a, return 0; } #endif +#ifdef WOLFSSL_SP_DIV_64 +static WC_INLINE sp_digit sp_2048_div_word_36(sp_digit d1, sp_digit d0, + sp_digit div) +{ + sp_digit d, r, t, dv; + int128_t t0, t1; + + /* dv has 29 bits. */ + dv = (div >> 28) + 1; + /* All 57 bits from d1 and top 6 bits from d0. */ + d = (d1 << 6) | (d0 >> 51); + r = d / dv; + d -= r * dv; + /* Up to 34 bits in r */ + /* Next 23 bits from d0. */ + d <<= 23; + r <<= 23; + d |= (d0 >> 28) & ((1 << 23) - 1); + t = d / dv; + d -= t * dv; + r += t; + /* Up to 57 bits in r */ + + /* Handle rounding error with dv - top part */ + t0 = ((int128_t)d1 << 57) + d0; + t1 = (int128_t)r * div; + t1 = t0 - t1; + t = (sp_digit)(t1 >> 28) / dv; + r += t; + + /* Handle rounding error with dv - bottom 64 bits */ + t1 = (sp_digit)t0 - (r * div); + t = (sp_digit)t1 / div; + r += t; + + return r; +} +#endif /* WOLFSSL_SP_DIV_64 */ + /* Divide d in a and put remainder into r (m*d + r = a) * m is not calculated as it is not needed at this time. * @@ -2109,7 +2192,9 @@ static int sp_2048_div_36(sp_digit* a, sp_digit* d, sp_digit* m, sp_digit* r) { int i; +#ifndef WOLFSSL_SP_DIV_64 int128_t d1; +#endif sp_digit div, r1; #if defined(WOLFSSL_SP_SMALL) || defined(WOLFSSL_SMALL_STACK) sp_digit* td; @@ -2142,10 +2227,14 @@ static int sp_2048_div_36(sp_digit* a, sp_digit* d, sp_digit* m, for (i=35; i>=0; i--) { t1[36 + i] += t1[36 + i - 1] >> 57; t1[36 + i - 1] &= 0x1ffffffffffffffl; +#ifndef WOLFSSL_SP_DIV_64 d1 = t1[36 + i]; d1 <<= 57; d1 += t1[36 + i - 1]; r1 = (sp_digit)(d1 / div); +#else + r1 = sp_2048_div_word_36(t1[36 + i], t1[36 + i - 1], div); +#endif sp_2048_mul_d_36(t2, d, r1); sp_2048_sub_36(&t1[i], &t1[i], t2); @@ -2161,8 +2250,7 @@ static int sp_2048_div_36(sp_digit* a, sp_digit* d, sp_digit* m, } t1[36 - 1] += t1[36 - 2] >> 57; t1[36 - 2] &= 0x1ffffffffffffffl; - d1 = t1[36 - 1]; - r1 = (sp_digit)(d1 / div); + r1 = t1[36 - 1] / div; sp_2048_mul_d_36(t2, d, r1); sp_2048_sub_36(t1, t1, t2); @@ -4831,6 +4919,45 @@ static void sp_3072_cond_add_27(sp_digit* r, const sp_digit* a, #endif /* WOLFSSL_SP_SMALL */ } +#ifdef WOLFSSL_SP_DIV_64 +static WC_INLINE sp_digit sp_3072_div_word_27(sp_digit d1, sp_digit d0, + sp_digit div) +{ + sp_digit d, r, t, dv; + int128_t t0, t1; + + /* dv has 29 bits. */ + dv = (div >> 28) + 1; + /* All 57 bits from d1 and top 6 bits from d0. */ + d = (d1 << 6) | (d0 >> 51); + r = d / dv; + d -= r * dv; + /* Up to 34 bits in r */ + /* Next 23 bits from d0. */ + d <<= 23; + r <<= 23; + d |= (d0 >> 28) & ((1 << 23) - 1); + t = d / dv; + d -= t * dv; + r += t; + /* Up to 57 bits in r */ + + /* Handle rounding error with dv - top part */ + t0 = ((int128_t)d1 << 57) + d0; + t1 = (int128_t)r * div; + t1 = t0 - t1; + t = (sp_digit)(t1 >> 28) / dv; + r += t; + + /* Handle rounding error with dv - bottom 64 bits */ + t1 = (sp_digit)t0 - (r * div); + t = (sp_digit)t1 / div; + r += t; + + return r; +} +#endif /* WOLFSSL_SP_DIV_64 */ + /* Divide d in a and put remainder into r (m*d + r = a) * m is not calculated as it is not needed at this time. * @@ -4844,7 +4971,9 @@ static int sp_3072_div_27(sp_digit* a, sp_digit* d, sp_digit* m, sp_digit* r) { int i; +#ifndef WOLFSSL_SP_DIV_64 int128_t d1; +#endif sp_digit div, r1; #if defined(WOLFSSL_SP_SMALL) || defined(WOLFSSL_SMALL_STACK) sp_digit* td; @@ -4877,10 +5006,14 @@ static int sp_3072_div_27(sp_digit* a, sp_digit* d, sp_digit* m, for (i=26; i>=0; i--) { t1[27 + i] += t1[27 + i - 1] >> 57; t1[27 + i - 1] &= 0x1ffffffffffffffl; +#ifndef WOLFSSL_SP_DIV_64 d1 = t1[27 + i]; d1 <<= 57; d1 += t1[27 + i - 1]; r1 = (sp_digit)(d1 / div); +#else + r1 = sp_3072_div_word_27(t1[27 + i], t1[27 + i - 1], div); +#endif sp_3072_mul_d_27(t2, d, r1); sp_3072_sub_27(&t1[i], &t1[i], t2); @@ -4896,8 +5029,7 @@ static int sp_3072_div_27(sp_digit* a, sp_digit* d, sp_digit* m, } t1[27 - 1] += t1[27 - 2] >> 57; t1[27 - 2] &= 0x1ffffffffffffffl; - d1 = t1[27 - 1]; - r1 = (sp_digit)(d1 / div); + r1 = t1[27 - 1] / div; sp_3072_mul_d_27(t2, d, r1); sp_3072_sub_27(t1, t1, t2); @@ -5610,6 +5742,45 @@ static void sp_3072_cond_add_54(sp_digit* r, const sp_digit* a, #endif /* WOLFSSL_SP_SMALL */ } +#ifdef WOLFSSL_SP_DIV_64 +static WC_INLINE sp_digit sp_3072_div_word_54(sp_digit d1, sp_digit d0, + sp_digit div) +{ + sp_digit d, r, t, dv; + int128_t t0, t1; + + /* dv has 29 bits. */ + dv = (div >> 28) + 1; + /* All 57 bits from d1 and top 6 bits from d0. */ + d = (d1 << 6) | (d0 >> 51); + r = d / dv; + d -= r * dv; + /* Up to 34 bits in r */ + /* Next 23 bits from d0. */ + d <<= 23; + r <<= 23; + d |= (d0 >> 28) & ((1 << 23) - 1); + t = d / dv; + d -= t * dv; + r += t; + /* Up to 57 bits in r */ + + /* Handle rounding error with dv - top part */ + t0 = ((int128_t)d1 << 57) + d0; + t1 = (int128_t)r * div; + t1 = t0 - t1; + t = (sp_digit)(t1 >> 28) / dv; + r += t; + + /* Handle rounding error with dv - bottom 64 bits */ + t1 = (sp_digit)t0 - (r * div); + t = (sp_digit)t1 / div; + r += t; + + return r; +} +#endif /* WOLFSSL_SP_DIV_64 */ + /* Divide d in a and put remainder into r (m*d + r = a) * m is not calculated as it is not needed at this time. * @@ -5623,7 +5794,9 @@ static int sp_3072_div_54(sp_digit* a, sp_digit* d, sp_digit* m, sp_digit* r) { int i; +#ifndef WOLFSSL_SP_DIV_64 int128_t d1; +#endif sp_digit div, r1; #if defined(WOLFSSL_SP_SMALL) || defined(WOLFSSL_SMALL_STACK) sp_digit* td; @@ -5656,10 +5829,14 @@ static int sp_3072_div_54(sp_digit* a, sp_digit* d, sp_digit* m, for (i=53; i>=0; i--) { t1[54 + i] += t1[54 + i - 1] >> 57; t1[54 + i - 1] &= 0x1ffffffffffffffl; +#ifndef WOLFSSL_SP_DIV_64 d1 = t1[54 + i]; d1 <<= 57; d1 += t1[54 + i - 1]; r1 = (sp_digit)(d1 / div); +#else + r1 = sp_3072_div_word_54(t1[54 + i], t1[54 + i - 1], div); +#endif sp_3072_mul_d_54(t2, d, r1); sp_3072_sub_54(&t1[i], &t1[i], t2); @@ -5675,8 +5852,7 @@ static int sp_3072_div_54(sp_digit* a, sp_digit* d, sp_digit* m, } t1[54 - 1] += t1[54 - 2] >> 57; t1[54 - 2] &= 0x1ffffffffffffffl; - d1 = t1[54 - 1]; - r1 = (sp_digit)(d1 / div); + r1 = t1[54 - 1] / div; sp_3072_mul_d_54(t2, d, r1); sp_3072_sub_54(t1, t1, t2); @@ -10928,6 +11104,45 @@ SP_NOINLINE static void sp_256_mul_d_5(sp_digit* r, const sp_digit* a, #endif /* WOLFSSL_SP_SMALL */ } +#ifdef WOLFSSL_SP_DIV_64 +static WC_INLINE sp_digit sp_256_div_word_5(sp_digit d1, sp_digit d0, + sp_digit div) +{ + sp_digit d, r, t, dv; + int128_t t0, t1; + + /* dv has 27 bits. */ + dv = (div >> 25) + 1; + /* All 52 bits from d1 and top 11 bits from d0. */ + d = (d1 << 11) | (d0 >> 41); + r = d / dv; + d -= r * dv; + /* Up to 36 bits in r */ + /* Next 16 bits from d0. */ + d <<= 16; + r <<= 16; + d |= (d0 >> 25) & ((1 << 16) - 1); + t = d / dv; + d -= t * dv; + r += t; + /* Up to 52 bits in r */ + + /* Handle rounding error with dv - top part */ + t0 = ((int128_t)d1 << 52) + d0; + t1 = (int128_t)r * div; + t1 = t0 - t1; + t = (sp_digit)(t1 >> 25) / dv; + r += t; + + /* Handle rounding error with dv - bottom 64 bits */ + t1 = (sp_digit)t0 - (r * div); + t = (sp_digit)t1 / div; + r += t; + + return r; +} +#endif /* WOLFSSL_SP_DIV_64 */ + /* Divide d in a and put remainder into r (m*d + r = a) * m is not calculated as it is not needed at this time. * @@ -10941,7 +11156,9 @@ static int sp_256_div_5(sp_digit* a, sp_digit* d, sp_digit* m, sp_digit* r) { int i; +#ifndef WOLFSSL_SP_DIV_64 int128_t d1; +#endif sp_digit div, r1; #if defined(WOLFSSL_SP_SMALL) || defined(WOLFSSL_SMALL_STACK) sp_digit* td; @@ -10974,10 +11191,14 @@ static int sp_256_div_5(sp_digit* a, sp_digit* d, sp_digit* m, for (i=4; i>=0; i--) { t1[5 + i] += t1[5 + i - 1] >> 52; t1[5 + i - 1] &= 0xfffffffffffffl; +#ifndef WOLFSSL_SP_DIV_64 d1 = t1[5 + i]; d1 <<= 52; d1 += t1[5 + i - 1]; r1 = (sp_digit)(d1 / div); +#else + r1 = sp_256_div_word_5(t1[5 + i], t1[5 + i - 1], div); +#endif sp_256_mul_d_5(t2, d, r1); sp_256_sub_5(&t1[i], &t1[i], t2); @@ -10993,8 +11214,7 @@ static int sp_256_div_5(sp_digit* a, sp_digit* d, sp_digit* m, } t1[5 - 1] += t1[5 - 2] >> 52; t1[5 - 2] &= 0xfffffffffffffl; - d1 = t1[5 - 1]; - r1 = (sp_digit)(d1 / div); + r1 = t1[5 - 1] / div; sp_256_mul_d_5(t2, d, r1); sp_256_sub_5(t1, t1, t2);