From 5b6130889e7e4a4244c96d43045d6be61beb096a Mon Sep 17 00:00:00 2001
From: Sean Parkinson <sean@wolfssl.com>
Date: Wed, 23 Feb 2022 09:17:08 +1000
Subject: [PATCH] SP asm: fix for modexp corner case

When exponent bit length is a multiple of the window size and the top
word has only window bits in it, then n is shifted down by an undefined
value (size of a word). The n value is not used after this.
Check for this condition and don't attempt to shift n.
---
 wolfcrypt/src/sp_arm32.c    | 52 ++++++++++++++++++++++++++++++
 wolfcrypt/src/sp_arm64.c    | 52 ++++++++++++++++++++++++++++++
 wolfcrypt/src/sp_armthumb.c | 52 ++++++++++++++++++++++++++++++
 wolfcrypt/src/sp_cortexm.c  | 52 ++++++++++++++++++++++++++++++
 wolfcrypt/src/sp_x86_64.c   | 64 +++++++++++++++++++++++++++++++++++++
 5 files changed, 272 insertions(+)

diff --git a/wolfcrypt/src/sp_arm32.c b/wolfcrypt/src/sp_arm32.c
index 6a9cb09fe..b355642f2 100644
--- a/wolfcrypt/src/sp_arm32.c
+++ b/wolfcrypt/src/sp_arm32.c
@@ -4816,6 +4816,10 @@ static int sp_2048_mod_exp_32(sp_digit* r, const sp_digit* a, const sp_digit* e,
             n <<= c;
             c = 64 - c;
         }
+        else if (c == 0) {
+            /* All bits in top word used. */
+            y = (byte)n;
+        }
         else {
             y = (byte)(n >> c);
             n <<= 32 - c;
@@ -4983,6 +4987,10 @@ static int sp_2048_mod_exp_32(sp_digit* r, const sp_digit* a, const sp_digit* e,
             n <<= c;
             c = 64 - c;
         }
+        else if (c == 0) {
+            /* All bits in top word used. */
+            y = (byte)n;
+        }
         else {
             y = (byte)(n >> c);
             n <<= 32 - c;
@@ -7440,6 +7448,10 @@ static int sp_2048_mod_exp_64(sp_digit* r, const sp_digit* a, const sp_digit* e,
             n <<= c;
             c = 64 - c;
         }
+        else if (c == 0) {
+            /* All bits in top word used. */
+            y = (byte)n;
+        }
         else {
             y = (byte)(n >> c);
             n <<= 32 - c;
@@ -7590,6 +7602,10 @@ static int sp_2048_mod_exp_64(sp_digit* r, const sp_digit* a, const sp_digit* e,
             n <<= c;
             c = 64 - c;
         }
+        else if (c == 0) {
+            /* All bits in top word used. */
+            y = (byte)n;
+        }
         else {
             y = (byte)(n >> c);
             n <<= 32 - c;
@@ -8750,6 +8766,10 @@ static int sp_2048_mod_exp_2_64(sp_digit* r, const sp_digit* e, int bits,
             n <<= c;
             c = 64 - c;
         }
+        else if (c == 0) {
+            /* All bits in top word used. */
+            y = (byte)n;
+        }
         else {
             y = (byte)(n >> c);
             n <<= 32 - c;
@@ -15777,6 +15797,10 @@ static int sp_3072_mod_exp_48(sp_digit* r, const sp_digit* a, const sp_digit* e,
             n <<= c;
             c = 64 - c;
         }
+        else if (c == 0) {
+            /* All bits in top word used. */
+            y = (byte)n;
+        }
         else {
             y = (byte)(n >> c);
             n <<= 32 - c;
@@ -15944,6 +15968,10 @@ static int sp_3072_mod_exp_48(sp_digit* r, const sp_digit* a, const sp_digit* e,
             n <<= c;
             c = 64 - c;
         }
+        else if (c == 0) {
+            /* All bits in top word used. */
+            y = (byte)n;
+        }
         else {
             y = (byte)(n >> c);
             n <<= 32 - c;
@@ -19329,6 +19357,10 @@ static int sp_3072_mod_exp_96(sp_digit* r, const sp_digit* a, const sp_digit* e,
             n <<= c;
             c = 64 - c;
         }
+        else if (c == 0) {
+            /* All bits in top word used. */
+            y = (byte)n;
+        }
         else {
             y = (byte)(n >> c);
             n <<= 32 - c;
@@ -19479,6 +19511,10 @@ static int sp_3072_mod_exp_96(sp_digit* r, const sp_digit* a, const sp_digit* e,
             n <<= c;
             c = 64 - c;
         }
+        else if (c == 0) {
+            /* All bits in top word used. */
+            y = (byte)n;
+        }
         else {
             y = (byte)(n >> c);
             n <<= 32 - c;
@@ -20911,6 +20947,10 @@ static int sp_3072_mod_exp_2_96(sp_digit* r, const sp_digit* e, int bits,
             n <<= c;
             c = 64 - c;
         }
+        else if (c == 0) {
+            /* All bits in top word used. */
+            y = (byte)n;
+        }
         else {
             y = (byte)(n >> c);
             n <<= 32 - c;
@@ -27838,6 +27878,10 @@ static int sp_4096_mod_exp_128(sp_digit* r, const sp_digit* a, const sp_digit* e
             n <<= c;
             c = 64 - c;
         }
+        else if (c == 0) {
+            /* All bits in top word used. */
+            y = (byte)n;
+        }
         else {
             y = (byte)(n >> c);
             n <<= 32 - c;
@@ -27988,6 +28032,10 @@ static int sp_4096_mod_exp_128(sp_digit* r, const sp_digit* a, const sp_digit* e
             n <<= c;
             c = 64 - c;
         }
+        else if (c == 0) {
+            /* All bits in top word used. */
+            y = (byte)n;
+        }
         else {
             y = (byte)(n >> c);
             n <<= 32 - c;
@@ -29692,6 +29740,10 @@ static int sp_4096_mod_exp_2_128(sp_digit* r, const sp_digit* e, int bits,
             n <<= c;
             c = 64 - c;
         }
+        else if (c == 0) {
+            /* All bits in top word used. */
+            y = (byte)n;
+        }
         else {
             y = (byte)(n >> c);
             n <<= 32 - c;
diff --git a/wolfcrypt/src/sp_arm64.c b/wolfcrypt/src/sp_arm64.c
index adab94b4b..54f405f7e 100644
--- a/wolfcrypt/src/sp_arm64.c
+++ b/wolfcrypt/src/sp_arm64.c
@@ -3318,6 +3318,10 @@ static int sp_2048_mod_exp_16(sp_digit* r, const sp_digit* a, const sp_digit* e,
             n <<= c;
             c = 64 - c;
         }
+        else if (c == 0) {
+            /* All bits in top word used. */
+            y = (byte)n;
+        }
         else {
             y = (byte)(n >> c);
             n <<= 64 - c;
@@ -3485,6 +3489,10 @@ static int sp_2048_mod_exp_16(sp_digit* r, const sp_digit* a, const sp_digit* e,
             n <<= c;
             c = 64 - c;
         }
+        else if (c == 0) {
+            /* All bits in top word used. */
+            y = (byte)n;
+        }
         else {
             y = (byte)(n >> c);
             n <<= 64 - c;
@@ -4897,6 +4905,10 @@ static int sp_2048_mod_exp_32(sp_digit* r, const sp_digit* a, const sp_digit* e,
             n <<= c;
             c = 64 - c;
         }
+        else if (c == 0) {
+            /* All bits in top word used. */
+            y = (byte)n;
+        }
         else {
             y = (byte)(n >> c);
             n <<= 64 - c;
@@ -5047,6 +5059,10 @@ static int sp_2048_mod_exp_32(sp_digit* r, const sp_digit* a, const sp_digit* e,
             n <<= c;
             c = 64 - c;
         }
+        else if (c == 0) {
+            /* All bits in top word used. */
+            y = (byte)n;
+        }
         else {
             y = (byte)(n >> c);
             n <<= 64 - c;
@@ -5912,6 +5928,10 @@ static int sp_2048_mod_exp_2_32(sp_digit* r, const sp_digit* e, int bits,
             n <<= c;
             c = 64 - c;
         }
+        else if (c == 0) {
+            /* All bits in top word used. */
+            y = (byte)n;
+        }
         else {
             y = (byte)(n >> c);
             n <<= 64 - c;
@@ -10941,6 +10961,10 @@ static int sp_3072_mod_exp_24(sp_digit* r, const sp_digit* a, const sp_digit* e,
             n <<= c;
             c = 64 - c;
         }
+        else if (c == 0) {
+            /* All bits in top word used. */
+            y = (byte)n;
+        }
         else {
             y = (byte)(n >> c);
             n <<= 64 - c;
@@ -11108,6 +11132,10 @@ static int sp_3072_mod_exp_24(sp_digit* r, const sp_digit* a, const sp_digit* e,
             n <<= c;
             c = 64 - c;
         }
+        else if (c == 0) {
+            /* All bits in top word used. */
+            y = (byte)n;
+        }
         else {
             y = (byte)(n >> c);
             n <<= 64 - c;
@@ -12944,6 +12972,10 @@ static int sp_3072_mod_exp_48(sp_digit* r, const sp_digit* a, const sp_digit* e,
             n <<= c;
             c = 64 - c;
         }
+        else if (c == 0) {
+            /* All bits in top word used. */
+            y = (byte)n;
+        }
         else {
             y = (byte)(n >> c);
             n <<= 64 - c;
@@ -13094,6 +13126,10 @@ static int sp_3072_mod_exp_48(sp_digit* r, const sp_digit* a, const sp_digit* e,
             n <<= c;
             c = 64 - c;
         }
+        else if (c == 0) {
+            /* All bits in top word used. */
+            y = (byte)n;
+        }
         else {
             y = (byte)(n >> c);
             n <<= 64 - c;
@@ -14083,6 +14119,10 @@ static int sp_3072_mod_exp_2_48(sp_digit* r, const sp_digit* e, int bits,
             n <<= c;
             c = 64 - c;
         }
+        else if (c == 0) {
+            /* All bits in top word used. */
+            y = (byte)n;
+        }
         else {
             y = (byte)(n >> c);
             n <<= 64 - c;
@@ -18154,6 +18194,10 @@ static int sp_4096_mod_exp_64(sp_digit* r, const sp_digit* a, const sp_digit* e,
             n <<= c;
             c = 64 - c;
         }
+        else if (c == 0) {
+            /* All bits in top word used. */
+            y = (byte)n;
+        }
         else {
             y = (byte)(n >> c);
             n <<= 64 - c;
@@ -18304,6 +18348,10 @@ static int sp_4096_mod_exp_64(sp_digit* r, const sp_digit* a, const sp_digit* e,
             n <<= c;
             c = 64 - c;
         }
+        else if (c == 0) {
+            /* All bits in top word used. */
+            y = (byte)n;
+        }
         else {
             y = (byte)(n >> c);
             n <<= 64 - c;
@@ -19417,6 +19465,10 @@ static int sp_4096_mod_exp_2_64(sp_digit* r, const sp_digit* e, int bits,
             n <<= c;
             c = 64 - c;
         }
+        else if (c == 0) {
+            /* All bits in top word used. */
+            y = (byte)n;
+        }
         else {
             y = (byte)(n >> c);
             n <<= 64 - c;
diff --git a/wolfcrypt/src/sp_armthumb.c b/wolfcrypt/src/sp_armthumb.c
index 8a45d68d4..0a93de115 100644
--- a/wolfcrypt/src/sp_armthumb.c
+++ b/wolfcrypt/src/sp_armthumb.c
@@ -23875,6 +23875,10 @@ static int sp_2048_mod_exp_32(sp_digit* r, const sp_digit* a, const sp_digit* e,
             n <<= c;
             c = 64 - c;
         }
+        else if (c == 0) {
+            /* All bits in top word used. */
+            y = (byte)n;
+        }
         else {
             y = (byte)(n >> c);
             n <<= 32 - c;
@@ -24042,6 +24046,10 @@ static int sp_2048_mod_exp_32(sp_digit* r, const sp_digit* a, const sp_digit* e,
             n <<= c;
             c = 64 - c;
         }
+        else if (c == 0) {
+            /* All bits in top word used. */
+            y = (byte)n;
+        }
         else {
             y = (byte)(n >> c);
             n <<= 32 - c;
@@ -27320,6 +27328,10 @@ static int sp_2048_mod_exp_64(sp_digit* r, const sp_digit* a, const sp_digit* e,
             n <<= c;
             c = 64 - c;
         }
+        else if (c == 0) {
+            /* All bits in top word used. */
+            y = (byte)n;
+        }
         else {
             y = (byte)(n >> c);
             n <<= 32 - c;
@@ -27470,6 +27482,10 @@ static int sp_2048_mod_exp_64(sp_digit* r, const sp_digit* a, const sp_digit* e,
             n <<= c;
             c = 64 - c;
         }
+        else if (c == 0) {
+            /* All bits in top word used. */
+            y = (byte)n;
+        }
         else {
             y = (byte)(n >> c);
             n <<= 32 - c;
@@ -29689,6 +29705,10 @@ static int sp_2048_mod_exp_2_64(sp_digit* r, const sp_digit* e, int bits,
             n <<= c;
             c = 64 - c;
         }
+        else if (c == 0) {
+            /* All bits in top word used. */
+            y = (byte)n;
+        }
         else {
             y = (byte)(n >> c);
             n <<= 32 - c;
@@ -74927,6 +74947,10 @@ static int sp_3072_mod_exp_48(sp_digit* r, const sp_digit* a, const sp_digit* e,
             n <<= c;
             c = 64 - c;
         }
+        else if (c == 0) {
+            /* All bits in top word used. */
+            y = (byte)n;
+        }
         else {
             y = (byte)(n >> c);
             n <<= 32 - c;
@@ -75094,6 +75118,10 @@ static int sp_3072_mod_exp_48(sp_digit* r, const sp_digit* a, const sp_digit* e,
             n <<= c;
             c = 64 - c;
         }
+        else if (c == 0) {
+            /* All bits in top word used. */
+            y = (byte)n;
+        }
         else {
             y = (byte)(n >> c);
             n <<= 32 - c;
@@ -79203,6 +79231,10 @@ static int sp_3072_mod_exp_96(sp_digit* r, const sp_digit* a, const sp_digit* e,
             n <<= c;
             c = 64 - c;
         }
+        else if (c == 0) {
+            /* All bits in top word used. */
+            y = (byte)n;
+        }
         else {
             y = (byte)(n >> c);
             n <<= 32 - c;
@@ -79353,6 +79385,10 @@ static int sp_3072_mod_exp_96(sp_digit* r, const sp_digit* a, const sp_digit* e,
             n <<= c;
             c = 64 - c;
         }
+        else if (c == 0) {
+            /* All bits in top word used. */
+            y = (byte)n;
+        }
         else {
             y = (byte)(n >> c);
             n <<= 32 - c;
@@ -82370,6 +82406,10 @@ static int sp_3072_mod_exp_2_96(sp_digit* r, const sp_digit* e, int bits,
             n <<= c;
             c = 64 - c;
         }
+        else if (c == 0) {
+            /* All bits in top word used. */
+            y = (byte)n;
+        }
         else {
             y = (byte)(n >> c);
             n <<= 32 - c;
@@ -91577,6 +91617,10 @@ static int sp_4096_mod_exp_128(sp_digit* r, const sp_digit* a, const sp_digit* e
             n <<= c;
             c = 64 - c;
         }
+        else if (c == 0) {
+            /* All bits in top word used. */
+            y = (byte)n;
+        }
         else {
             y = (byte)(n >> c);
             n <<= 32 - c;
@@ -91727,6 +91771,10 @@ static int sp_4096_mod_exp_128(sp_digit* r, const sp_digit* a, const sp_digit* e
             n <<= c;
             c = 64 - c;
         }
+        else if (c == 0) {
+            /* All bits in top word used. */
+            y = (byte)n;
+        }
         else {
             y = (byte)(n >> c);
             n <<= 32 - c;
@@ -95537,6 +95585,10 @@ static int sp_4096_mod_exp_2_128(sp_digit* r, const sp_digit* e, int bits,
             n <<= c;
             c = 64 - c;
         }
+        else if (c == 0) {
+            /* All bits in top word used. */
+            y = (byte)n;
+        }
         else {
             y = (byte)(n >> c);
             n <<= 32 - c;
diff --git a/wolfcrypt/src/sp_cortexm.c b/wolfcrypt/src/sp_cortexm.c
index 6b1493148..af1005213 100644
--- a/wolfcrypt/src/sp_cortexm.c
+++ b/wolfcrypt/src/sp_cortexm.c
@@ -3278,6 +3278,10 @@ static int sp_2048_mod_exp_32(sp_digit* r, const sp_digit* a, const sp_digit* e,
             n <<= c;
             c = 64 - c;
         }
+        else if (c == 0) {
+            /* All bits in top word used. */
+            y = (byte)n;
+        }
         else {
             y = (byte)(n >> c);
             n <<= 32 - c;
@@ -3445,6 +3449,10 @@ static int sp_2048_mod_exp_32(sp_digit* r, const sp_digit* a, const sp_digit* e,
             n <<= c;
             c = 64 - c;
         }
+        else if (c == 0) {
+            /* All bits in top word used. */
+            y = (byte)n;
+        }
         else {
             y = (byte)(n >> c);
             n <<= 32 - c;
@@ -4353,6 +4361,10 @@ static int sp_2048_mod_exp_64(sp_digit* r, const sp_digit* a, const sp_digit* e,
             n <<= c;
             c = 64 - c;
         }
+        else if (c == 0) {
+            /* All bits in top word used. */
+            y = (byte)n;
+        }
         else {
             y = (byte)(n >> c);
             n <<= 32 - c;
@@ -4503,6 +4515,10 @@ static int sp_2048_mod_exp_64(sp_digit* r, const sp_digit* a, const sp_digit* e,
             n <<= c;
             c = 64 - c;
         }
+        else if (c == 0) {
+            /* All bits in top word used. */
+            y = (byte)n;
+        }
         else {
             y = (byte)(n >> c);
             n <<= 32 - c;
@@ -5506,6 +5522,10 @@ static int sp_2048_mod_exp_2_64(sp_digit* r, const sp_digit* e, int bits,
             n <<= c;
             c = 64 - c;
         }
+        else if (c == 0) {
+            /* All bits in top word used. */
+            y = (byte)n;
+        }
         else {
             y = (byte)(n >> c);
             n <<= 32 - c;
@@ -8549,6 +8569,10 @@ static int sp_3072_mod_exp_48(sp_digit* r, const sp_digit* a, const sp_digit* e,
             n <<= c;
             c = 64 - c;
         }
+        else if (c == 0) {
+            /* All bits in top word used. */
+            y = (byte)n;
+        }
         else {
             y = (byte)(n >> c);
             n <<= 32 - c;
@@ -8716,6 +8740,10 @@ static int sp_3072_mod_exp_48(sp_digit* r, const sp_digit* a, const sp_digit* e,
             n <<= c;
             c = 64 - c;
         }
+        else if (c == 0) {
+            /* All bits in top word used. */
+            y = (byte)n;
+        }
         else {
             y = (byte)(n >> c);
             n <<= 32 - c;
@@ -9762,6 +9790,10 @@ static int sp_3072_mod_exp_96(sp_digit* r, const sp_digit* a, const sp_digit* e,
             n <<= c;
             c = 64 - c;
         }
+        else if (c == 0) {
+            /* All bits in top word used. */
+            y = (byte)n;
+        }
         else {
             y = (byte)(n >> c);
             n <<= 32 - c;
@@ -9912,6 +9944,10 @@ static int sp_3072_mod_exp_96(sp_digit* r, const sp_digit* a, const sp_digit* e,
             n <<= c;
             c = 64 - c;
         }
+        else if (c == 0) {
+            /* All bits in top word used. */
+            y = (byte)n;
+        }
         else {
             y = (byte)(n >> c);
             n <<= 32 - c;
@@ -11111,6 +11147,10 @@ static int sp_3072_mod_exp_2_96(sp_digit* r, const sp_digit* e, int bits,
             n <<= c;
             c = 64 - c;
         }
+        else if (c == 0) {
+            /* All bits in top word used. */
+            y = (byte)n;
+        }
         else {
             y = (byte)(n >> c);
             n <<= 32 - c;
@@ -13707,6 +13747,10 @@ static int sp_4096_mod_exp_128(sp_digit* r, const sp_digit* a, const sp_digit* e
             n <<= c;
             c = 64 - c;
         }
+        else if (c == 0) {
+            /* All bits in top word used. */
+            y = (byte)n;
+        }
         else {
             y = (byte)(n >> c);
             n <<= 32 - c;
@@ -13857,6 +13901,10 @@ static int sp_4096_mod_exp_128(sp_digit* r, const sp_digit* a, const sp_digit* e
             n <<= c;
             c = 64 - c;
         }
+        else if (c == 0) {
+            /* All bits in top word used. */
+            y = (byte)n;
+        }
         else {
             y = (byte)(n >> c);
             n <<= 32 - c;
@@ -15253,6 +15301,10 @@ static int sp_4096_mod_exp_2_128(sp_digit* r, const sp_digit* e, int bits,
             n <<= c;
             c = 64 - c;
         }
+        else if (c == 0) {
+            /* All bits in top word used. */
+            y = (byte)n;
+        }
         else {
             y = (byte)(n >> c);
             n <<= 32 - c;
diff --git a/wolfcrypt/src/sp_x86_64.c b/wolfcrypt/src/sp_x86_64.c
index c3c7aee1f..a8c99789b 100644
--- a/wolfcrypt/src/sp_x86_64.c
+++ b/wolfcrypt/src/sp_x86_64.c
@@ -583,6 +583,10 @@ static int sp_2048_mod_exp_16(sp_digit* r, const sp_digit* a, const sp_digit* e,
             n <<= c;
             c = 64 - c;
         }
+        else if (c == 0) {
+            /* All bits in top word used. */
+            y = (byte)n;
+        }
         else {
             y = (byte)(n >> c);
             n <<= 64 - c;
@@ -806,6 +810,10 @@ static int sp_2048_mod_exp_avx2_16(sp_digit* r, const sp_digit* a, const sp_digi
             n <<= c;
             c = 64 - c;
         }
+        else if (c == 0) {
+            /* All bits in top word used. */
+            y = (byte)n;
+        }
         else {
             y = (byte)(n >> c);
             n <<= 64 - c;
@@ -1268,6 +1276,10 @@ static int sp_2048_mod_exp_32(sp_digit* r, const sp_digit* a, const sp_digit* e,
             n <<= c;
             c = 64 - c;
         }
+        else if (c == 0) {
+            /* All bits in top word used. */
+            y = (byte)n;
+        }
         else {
             y = (byte)(n >> c);
             n <<= 64 - c;
@@ -1475,6 +1487,10 @@ static int sp_2048_mod_exp_avx2_32(sp_digit* r, const sp_digit* a, const sp_digi
             n <<= c;
             c = 64 - c;
         }
+        else if (c == 0) {
+            /* All bits in top word used. */
+            y = (byte)n;
+        }
         else {
             y = (byte)(n >> c);
             n <<= 64 - c;
@@ -2186,6 +2202,10 @@ static int sp_2048_mod_exp_2_avx2_32(sp_digit* r, const sp_digit* e, int bits,
             n <<= c;
             c = 64 - c;
         }
+        else if (c == 0) {
+            /* All bits in top word used. */
+            y = (byte)n;
+        }
         else {
             y = (byte)(n >> c);
             n <<= 64 - c;
@@ -2321,6 +2341,10 @@ static int sp_2048_mod_exp_2_32(sp_digit* r, const sp_digit* e, int bits,
             n <<= c;
             c = 64 - c;
         }
+        else if (c == 0) {
+            /* All bits in top word used. */
+            y = (byte)n;
+        }
         else {
             y = (byte)(n >> c);
             n <<= 64 - c;
@@ -3108,6 +3132,10 @@ static int sp_3072_mod_exp_24(sp_digit* r, const sp_digit* a, const sp_digit* e,
             n <<= c;
             c = 64 - c;
         }
+        else if (c == 0) {
+            /* All bits in top word used. */
+            y = (byte)n;
+        }
         else {
             y = (byte)(n >> c);
             n <<= 64 - c;
@@ -3331,6 +3359,10 @@ static int sp_3072_mod_exp_avx2_24(sp_digit* r, const sp_digit* a, const sp_digi
             n <<= c;
             c = 64 - c;
         }
+        else if (c == 0) {
+            /* All bits in top word used. */
+            y = (byte)n;
+        }
         else {
             y = (byte)(n >> c);
             n <<= 64 - c;
@@ -3793,6 +3825,10 @@ static int sp_3072_mod_exp_48(sp_digit* r, const sp_digit* a, const sp_digit* e,
             n <<= c;
             c = 64 - c;
         }
+        else if (c == 0) {
+            /* All bits in top word used. */
+            y = (byte)n;
+        }
         else {
             y = (byte)(n >> c);
             n <<= 64 - c;
@@ -4000,6 +4036,10 @@ static int sp_3072_mod_exp_avx2_48(sp_digit* r, const sp_digit* a, const sp_digi
             n <<= c;
             c = 64 - c;
         }
+        else if (c == 0) {
+            /* All bits in top word used. */
+            y = (byte)n;
+        }
         else {
             y = (byte)(n >> c);
             n <<= 64 - c;
@@ -4711,6 +4751,10 @@ static int sp_3072_mod_exp_2_avx2_48(sp_digit* r, const sp_digit* e, int bits,
             n <<= c;
             c = 64 - c;
         }
+        else if (c == 0) {
+            /* All bits in top word used. */
+            y = (byte)n;
+        }
         else {
             y = (byte)(n >> c);
             n <<= 64 - c;
@@ -4846,6 +4890,10 @@ static int sp_3072_mod_exp_2_48(sp_digit* r, const sp_digit* e, int bits,
             n <<= c;
             c = 64 - c;
         }
+        else if (c == 0) {
+            /* All bits in top word used. */
+            y = (byte)n;
+        }
         else {
             y = (byte)(n >> c);
             n <<= 64 - c;
@@ -5683,6 +5731,10 @@ static int sp_4096_mod_exp_64(sp_digit* r, const sp_digit* a, const sp_digit* e,
             n <<= c;
             c = 64 - c;
         }
+        else if (c == 0) {
+            /* All bits in top word used. */
+            y = (byte)n;
+        }
         else {
             y = (byte)(n >> c);
             n <<= 64 - c;
@@ -5890,6 +5942,10 @@ static int sp_4096_mod_exp_avx2_64(sp_digit* r, const sp_digit* a, const sp_digi
             n <<= c;
             c = 64 - c;
         }
+        else if (c == 0) {
+            /* All bits in top word used. */
+            y = (byte)n;
+        }
         else {
             y = (byte)(n >> c);
             n <<= 64 - c;
@@ -6601,6 +6657,10 @@ static int sp_4096_mod_exp_2_avx2_64(sp_digit* r, const sp_digit* e, int bits,
             n <<= c;
             c = 64 - c;
         }
+        else if (c == 0) {
+            /* All bits in top word used. */
+            y = (byte)n;
+        }
         else {
             y = (byte)(n >> c);
             n <<= 64 - c;
@@ -6736,6 +6796,10 @@ static int sp_4096_mod_exp_2_64(sp_digit* r, const sp_digit* e, int bits,
             n <<= c;
             c = 64 - c;
         }
+        else if (c == 0) {
+            /* All bits in top word used. */
+            y = (byte)n;
+        }
         else {
             y = (byte)(n >> c);
             n <<= 64 - c;