From 1a6dd44d0302138565fdbc9465f4fb156db2055a Mon Sep 17 00:00:00 2001 From: Angus Gratton Date: Mon, 19 Sep 2016 18:00:03 +1000 Subject: [PATCH 01/10] hwcrypto bignum: Use mbedtls_mpi data structures for all bignum data Still doesn't solve the problem of multiplying two numbers where one is >2048 bits, needed for RSA support. --- components/mbedtls/port/esp_bignum.c | 135 ++++++++---------- .../mbedtls/port/include/mbedtls/esp_config.h | 4 +- 2 files changed, 58 insertions(+), 81 deletions(-) diff --git a/components/mbedtls/port/esp_bignum.c b/components/mbedtls/port/esp_bignum.c index 59bdc87260..caae0161f0 100644 --- a/components/mbedtls/port/esp_bignum.c +++ b/components/mbedtls/port/esp_bignum.c @@ -207,98 +207,80 @@ static int mpi_montred( mbedtls_mpi *A, const mbedtls_mpi *N, mbedtls_mpi_uint m return( mpi_montmul( A, &U, N, mm, T ) ); } +#if defined(MBEDTLS_MPI_MUL_MPI_ALT) /* MBEDTLS_MPI_MUL_MPI_ALT */ -/* Allocate parameters used by hardware MPI multiply, - and copy mbedtls_mpi structures into them */ -static int mul_pram_alloc(const mbedtls_mpi *A, const mbedtls_mpi *B, char **pA, char **pB, char **pX, size_t *bites) +/* Number of words used to hold 'mpi', rounded up to nearest + 16 words (512 bits) to match hardware support +*/ +static inline size_t hardware_words_needed(const mbedtls_mpi *mpi) { - char *sa, *sb, *sx; -// int algn; - int words, bytes; - int abytes, bbytes; - - if (A->n > B->n) - words = A->n; - else - words = B->n; - - bytes = (words / 16 + ((words % 16) ? 1 : 0 )) * 16 * 4 * 2; - - abytes = A->n * 4; - bbytes = B->n * 4; - - sa = malloc(bytes); - if (!sa) { - return -1; - } - - sb = malloc(bytes); - if (!sb) { - free(sa); - return -1; - } - - sx = malloc(bytes); - if (!sx) { - free(sa); - free(sb); - return -1; - } - - memcpy(sa, A->p, abytes); - memset(sa + abytes, 0, bytes - abytes); - - memcpy(sb, B->p, bbytes); - memset(sb + bbytes, 0, bytes - bbytes); - - *pA = sa; - *pB = sb; - - *pX = sx; - - *bites = bytes * 4; - - return 0; + size_t res; + for(res = mpi->n; res > 0; res-- ) { + if( mpi->p[res - 1] != 0 ) + break; + } + res = (res + 0xF) & ~0xF; + return res; } -#if defined(MBEDTLS_MPI_MUL_MPI_ALT) - int mbedtls_mpi_mul_mpi( mbedtls_mpi *X, const mbedtls_mpi *A, const mbedtls_mpi *B ) { int ret = -1; - size_t i, j; - char *s1 = NULL, *s2 = NULL, *dest = NULL; - size_t bites; + size_t words_a, words_b, words_x, words_mult; mbedtls_mpi TA, TB; mbedtls_mpi_init( &TA ); mbedtls_mpi_init( &TB ); - if( X == A ) { MBEDTLS_MPI_CHK( mbedtls_mpi_copy( &TA, A ) ); A = &TA; } - if( X == B ) { MBEDTLS_MPI_CHK( mbedtls_mpi_copy( &TB, B ) ); B = &TB; } + /* Count words needed for A & B in hardware */ + words_a = hardware_words_needed(A); + words_b = hardware_words_needed(B); - for( i = A->n; i > 0; i-- ) - if( A->p[i - 1] != 0 ) - break; + /* Take a copy of A if either X == A OR if A isn't long enough + to hold the number of words needed for hardware. - for( j = B->n; j > 0; j-- ) - if( B->p[j - 1] != 0 ) - break; + (can't grow A directly as it is const) - MBEDTLS_MPI_CHK( mbedtls_mpi_grow( X, i + j ) ); + TODO: growing the input operands is only necessary because the + ROM functions only take one length argument. It should be + possible for us to just copy the used data only into the + hardware buffers, and set the remaining bits to zero - saving + RAM. But we need to reimplement ets_bigint_mult_prepare() in + software for this. + */ + if( X == A || A->n < words_a) { + MBEDTLS_MPI_CHK( mbedtls_mpi_copy( &TA, A ) ); + MBEDTLS_MPI_CHK( mbedtls_mpi_grow( &TA, words_a) ); + A = &TA; + } + /* Same for B */ + if( X == B || B->n < words_b ) { + MBEDTLS_MPI_CHK( mbedtls_mpi_copy( &TB, B ) ); + MBEDTLS_MPI_CHK( mbedtls_mpi_grow( &TB, words_b) ); + B = &TB; + } + + /* Result X has to have room for double the larger operand */ + words_mult = (words_a > words_b ? words_a : words_b); + words_x = words_mult * 2; + MBEDTLS_MPI_CHK( mbedtls_mpi_grow( X, words_x ) ); + /* TODO: check if lset here is necessary, hardware should zero */ MBEDTLS_MPI_CHK( mbedtls_mpi_lset( X, 0 ) ); - if (mul_pram_alloc(A, B, &s1, &s2, &dest, &bites)) { - goto cleanup; - } - esp_mpi_acquire_hardware(); - if (ets_bigint_mult_prepare((uint32_t *)s1, (uint32_t *)s2, bites)){ - ets_bigint_wait_finish(); - if (ets_bigint_mult_getz((uint32_t *)dest, bites) == true) { - memcpy(X->p, dest, (i + j) * 4); - ret = 0; - } else { + + if(words_mult * 32 > 2048) { + printf("WARNING: %d bit operands (%d bits * %d bits) too large for hardware unit\n", words_mult * 32, mbedtls_mpi_bitlen(A), mbedtls_mpi_bitlen(B)); + } + + if (ets_bigint_mult_prepare(A->p, B->p, words_mult * 32)) { + ets_bigint_wait_finish(); + /* NB: argument to bigint_mult_getz is length of inputs, double this number (words_x) is + copied to output X->p. + */ + if (ets_bigint_mult_getz(X->p, words_mult * 32) == true) { + ret = 0; + } else { printf("ets_bigint_mult_getz failed\n"); } } else{ @@ -307,11 +289,6 @@ int mbedtls_mpi_mul_mpi( mbedtls_mpi *X, const mbedtls_mpi *A, const mbedtls_mpi esp_mpi_release_hardware(); X->s = A->s * B->s; - - free(s1); - free(s2); - free(dest); - cleanup: mbedtls_mpi_free( &TB ); mbedtls_mpi_free( &TA ); diff --git a/components/mbedtls/port/include/mbedtls/esp_config.h b/components/mbedtls/port/include/mbedtls/esp_config.h index 5a69ff78e4..4ddd9821c4 100644 --- a/components/mbedtls/port/include/mbedtls/esp_config.h +++ b/components/mbedtls/port/include/mbedtls/esp_config.h @@ -253,8 +253,8 @@ Disabled as number of limbs limited by bug. Internal TW#7112. */ -//#define MBEDTLS_MPI_EXP_MOD_ALT -//#define MBEDTLS_MPI_MUL_MPI_ALT +#define MBEDTLS_MPI_EXP_MOD_ALT +#define MBEDTLS_MPI_MUL_MPI_ALT /** * \def MBEDTLS_MD2_PROCESS_ALT From 6b3bc4d8c5bb3eacf1076417074bccdca73971d5 Mon Sep 17 00:00:00 2001 From: Angus Gratton Date: Tue, 20 Sep 2016 21:02:07 +1000 Subject: [PATCH 02/10] hwcrypto bignum: Implement multiplication modulo Fixes case where hardware bignum multiplication fails due to either operand >2048 bits. --- components/mbedtls/port/esp_bignum.c | 245 ++++++++++++++++-- .../mbedtls/port/include/mbedtls/esp_config.h | 4 +- 2 files changed, 223 insertions(+), 26 deletions(-) diff --git a/components/mbedtls/port/esp_bignum.c b/components/mbedtls/port/esp_bignum.c index caae0161f0..d6b79e32f5 100644 --- a/components/mbedtls/port/esp_bignum.c +++ b/components/mbedtls/port/esp_bignum.c @@ -52,6 +52,140 @@ static void esp_mpi_release_hardware( void ) _lock_release(&mpi_lock); } +/* Given a & b, determine u & v such that + + gcd(a,b) = d = au + bv + + Underlying algorithm comes from: + http://www.ucl.ac.uk/~ucahcjm/combopt/ext_gcd_python_programs.pdf + http://www.hackersdelight.org/hdcodetxt/mont64.c.txt + */ +static void extended_binary_gcd(const mbedtls_mpi *a, const mbedtls_mpi *b, + mbedtls_mpi *u, mbedtls_mpi *v) +{ + mbedtls_mpi ta, tb; + + mbedtls_mpi_init(&ta); + mbedtls_mpi_copy(&ta, a); + mbedtls_mpi_init(&tb); + mbedtls_mpi_copy(&tb, b); + + mbedtls_mpi_lset(u, 1); + mbedtls_mpi_lset(v, 0); + + /* Loop invariant: + ta = u*2*a - v*b. */ + while (mbedtls_mpi_cmp_int(&ta, 0) != 0) { + mbedtls_mpi_shift_r(&ta, 1); + if (mbedtls_mpi_get_bit(u, 0) == 0) { + // Remove common factor of 2 in u & v + mbedtls_mpi_shift_r(u, 1); + mbedtls_mpi_shift_r(v, 1); + } + else { + /* u = (u + b) >> 1 */ + mbedtls_mpi_add_mpi(u, u, b); + mbedtls_mpi_shift_r(u, 1); + /* v = (v >> 1) + a */ + mbedtls_mpi_shift_r(v, 1); + mbedtls_mpi_add_mpi(v, v, a); + } + } + mbedtls_mpi_free(&ta); + mbedtls_mpi_free(&tb); + + /* u = u * 2, so 1 = u*a - v*b */ + mbedtls_mpi_shift_l(u, 1); +} + +/* inner part of MPI modular multiply, after Rinv & Mprime are calculated */ +static int mpi_mul_mpi_mod_inner(mbedtls_mpi *X, const mbedtls_mpi *A, const mbedtls_mpi *B, const mbedtls_mpi *M, mbedtls_mpi *Rinv, uint32_t Mprime, size_t num_words) +{ + int ret; + mbedtls_mpi TA, TB; + size_t num_bits = num_words * 32; + + mbedtls_mpi_grow(Rinv, num_words); + + /* TODO: fill memory blocks directly so this isn't needed */ + mbedtls_mpi_init(&TA); + mbedtls_mpi_copy(&TA, A); + mbedtls_mpi_grow(&TA, num_words); + A = &TA; + mbedtls_mpi_init(&TB); + mbedtls_mpi_copy(&TB, B); + mbedtls_mpi_grow(&TB, num_words); + B = &TB; + + esp_mpi_acquire_hardware(); + + if(ets_bigint_mod_mult_prepare(A->p, B->p, M->p, Mprime, + Rinv->p, num_bits, false)) { + mbedtls_mpi_grow(X, num_words); + ets_bigint_wait_finish(); + if(ets_bigint_mod_mult_getz(M->p, X->p, num_bits)) { + X->s = A->s * B->s; + ret = 0; + } else { + printf("ets_bigint_mod_mult_getz failed\n"); + ret = MBEDTLS_ERR_MPI_BAD_INPUT_DATA; + } + } else { + printf("ets_bigint_mod_mult_prepare failed\n"); + ret = MBEDTLS_ERR_MPI_BAD_INPUT_DATA; + } + esp_mpi_release_hardware(); + + /* unclear why this is necessary, but the result seems + to come back rotated 32 bits to the right... */ + uint32_t last_word = X->p[num_words-1]; + X->p[num_words-1] = 0; + mbedtls_mpi_shift_l(X, 32); + X->p[0] = last_word; + + mbedtls_mpi_free(&TA); + mbedtls_mpi_free(&TB); + + return ret; +} + +/* X = (A * B) mod M + + Not an mbedTLS function + + num_bits guaranteed to be a multiple of 512 already. + + TODO: ensure M is odd + */ +int esp_mpi_mul_mpi_mod(mbedtls_mpi *X, const mbedtls_mpi *A, const mbedtls_mpi *B, const mbedtls_mpi *M, size_t num_bits) +{ + int ret = 0; + mbedtls_mpi RR, Rinv, Mprime; + uint32_t Mprime_int; + size_t num_words = num_bits / 32; + + /* Rinv & Mprime are calculated via extended binary gcd + algorithm, see references on extended_binary_gcd above. + */ + mbedtls_mpi_init(&Rinv); + mbedtls_mpi_init(&RR); + mbedtls_mpi_set_bit(&RR, num_bits+32, 1); + mbedtls_mpi_init(&Mprime); + extended_binary_gcd(&RR, M, &Rinv, &Mprime); + + /* M' is mod 2^32 */ + Mprime_int = Mprime.p[0]; + + ret = mpi_mul_mpi_mod_inner(X, A, B, M, &Rinv, Mprime_int, num_words); + + mbedtls_mpi_free(&RR); + mbedtls_mpi_free(&Mprime); + mbedtls_mpi_free(&Rinv); + + return ret; +} + + /* * Helper for mbedtls_mpi multiplication * copied/trimmed from mbedtls bignum.c @@ -223,6 +357,53 @@ static inline size_t hardware_words_needed(const mbedtls_mpi *mpi) return res; } + +/* Special-case multiply, where we use hardware montgomery mod + multiplication to solve the case where A or B are >2048 bits so + can't do standard multiplication. + + the modulus here is chosen with M=(2^num_bits-1) + to guarantee the output isn't actually modulo anything. This means + we don't need to calculate M' and Rinv, they are predictable + as follows: + M' = 1 + Rinv = (1 << (num_bits - 32) + + (See RSA Accelerator section in Technical Reference for derivation + of M', Rinv) +*/ +static int esp_mpi_mult_mpi_failover_mod_mult(mbedtls_mpi *X, const mbedtls_mpi *A, const mbedtls_mpi *B, size_t num_words) + { + mbedtls_mpi M, Rinv; + int ret; + size_t mprime; + size_t num_bits = num_words * 32; + + mbedtls_mpi_init(&M); + mbedtls_mpi_init(&Rinv); + + /* TODO: it may be faster to just use 4096-bit arithmetic every time, + and make these constants rather than runtime derived + derived. */ + /* M = (2^num_words)-1 */ + mbedtls_mpi_grow(&M, num_words); + for(int i = 0; i < num_words*32; i++) { + mbedtls_mpi_set_bit(&M, i, 1); + } + + /* Rinv = (2^num_words-32) */ + mbedtls_mpi_grow(&Rinv, num_words); + mbedtls_mpi_set_bit(&Rinv, num_bits - 32, 1); + + mprime = 1; + + ret = mpi_mul_mpi_mod_inner(X, A, B, &M, &Rinv, mprime, num_words); + + mbedtls_mpi_free(&M); + mbedtls_mpi_free(&Rinv); + return ret; + } + int mbedtls_mpi_mul_mpi( mbedtls_mpi *X, const mbedtls_mpi *A, const mbedtls_mpi *B ) { int ret = -1; @@ -236,6 +417,8 @@ int mbedtls_mpi_mul_mpi( mbedtls_mpi *X, const mbedtls_mpi *A, const mbedtls_mpi words_a = hardware_words_needed(A); words_b = hardware_words_needed(B); + words_mult = (words_a > words_b ? words_a : words_b); + /* Take a copy of A if either X == A OR if A isn't long enough to hold the number of words needed for hardware. @@ -248,47 +431,63 @@ int mbedtls_mpi_mul_mpi( mbedtls_mpi *X, const mbedtls_mpi *A, const mbedtls_mpi RAM. But we need to reimplement ets_bigint_mult_prepare() in software for this. */ - if( X == A || A->n < words_a) { + if( X == A || A->n < words_mult) { MBEDTLS_MPI_CHK( mbedtls_mpi_copy( &TA, A ) ); - MBEDTLS_MPI_CHK( mbedtls_mpi_grow( &TA, words_a) ); + MBEDTLS_MPI_CHK( mbedtls_mpi_grow( &TA, words_mult) ); A = &TA; } /* Same for B */ - if( X == B || B->n < words_b ) { + if( X == B || B->n < words_mult ) { MBEDTLS_MPI_CHK( mbedtls_mpi_copy( &TB, B ) ); - MBEDTLS_MPI_CHK( mbedtls_mpi_grow( &TB, words_b) ); + MBEDTLS_MPI_CHK( mbedtls_mpi_grow( &TB, words_mult) ); B = &TB; } /* Result X has to have room for double the larger operand */ - words_mult = (words_a > words_b ? words_a : words_b); words_x = words_mult * 2; MBEDTLS_MPI_CHK( mbedtls_mpi_grow( X, words_x ) ); /* TODO: check if lset here is necessary, hardware should zero */ MBEDTLS_MPI_CHK( mbedtls_mpi_lset( X, 0 ) ); - esp_mpi_acquire_hardware(); + /* If either operand is over 2048 bits, we can't use the standard hardware multiplier + (it assumes result is double longest operand, and result is max 4096 bits.) + However, we can fail over to mod_mult for up to 4096 bits. + */ if(words_mult * 32 > 2048) { - printf("WARNING: %d bit operands (%d bits * %d bits) too large for hardware unit\n", words_mult * 32, mbedtls_mpi_bitlen(A), mbedtls_mpi_bitlen(B)); - } - - if (ets_bigint_mult_prepare(A->p, B->p, words_mult * 32)) { - ets_bigint_wait_finish(); - /* NB: argument to bigint_mult_getz is length of inputs, double this number (words_x) is - copied to output X->p. + /* TODO: check if there's an overflow condition if words_a & words_b are both + the bit lengths of the operands, result could be 1 bit longer */ - if (ets_bigint_mult_getz(X->p, words_mult * 32) == true) { - ret = 0; - } else { - printf("ets_bigint_mult_getz failed\n"); - } - } else{ - printf("Baseline multiplication failed\n"); - } - esp_mpi_release_hardware(); + if((words_a + words_b) * 32 > 4096) { + printf("ERROR: %d bit operands (%d bits * %d bits) too large for hardware unit\n", words_mult * 32, mbedtls_mpi_bitlen(A), mbedtls_mpi_bitlen(B)); + ret = MBEDTLS_ERR_MPI_NOT_ACCEPTABLE; + } + else { + ret = esp_mpi_mult_mpi_failover_mod_mult(X, A, B, words_a + words_b); + } + } + else { - X->s = A->s * B->s; + /* normal mpi multiplication */ + esp_mpi_acquire_hardware(); + if (ets_bigint_mult_prepare(A->p, B->p, words_mult * 32)) { + ets_bigint_wait_finish(); + /* NB: argument to bigint_mult_getz is length of inputs, double this number (words_x) is + copied to output X->p. + */ + if (ets_bigint_mult_getz(X->p, words_mult * 32) == true) { + X->s = A->s * B->s; + ret = 0; + } else { + printf("ets_bigint_mult_getz failed\n"); + ret = MBEDTLS_ERR_MPI_NOT_ACCEPTABLE; + } + } else{ + printf("Baseline multiplication failed\n"); + ret = MBEDTLS_ERR_MPI_NOT_ACCEPTABLE; + } + esp_mpi_release_hardware(); + } cleanup: mbedtls_mpi_free( &TB ); mbedtls_mpi_free( &TA ); diff --git a/components/mbedtls/port/include/mbedtls/esp_config.h b/components/mbedtls/port/include/mbedtls/esp_config.h index 4ddd9821c4..e4f4af271a 100644 --- a/components/mbedtls/port/include/mbedtls/esp_config.h +++ b/components/mbedtls/port/include/mbedtls/esp_config.h @@ -250,10 +250,8 @@ /* The following MPI (bignum) functions have ESP32 hardware support, Uncommenting these macros will use the hardware-accelerated implementations. - - Disabled as number of limbs limited by bug. Internal TW#7112. */ -#define MBEDTLS_MPI_EXP_MOD_ALT +//#define MBEDTLS_MPI_EXP_MOD_ALT #define MBEDTLS_MPI_MUL_MPI_ALT /** From 9632c8e56c49db6351dd8a1b964d61ac67cff342 Mon Sep 17 00:00:00 2001 From: Angus Gratton Date: Tue, 20 Sep 2016 21:24:58 +1000 Subject: [PATCH 03/10] RSA Accelerator: Add mod_exp, refactor to avoid memory allocation & copying Not fully working at the moment, mod_exp has a bug. --- components/esp32/include/soc/hwcrypto_reg.h | 37 + components/esp32/include/soc/soc.h | 1 + components/mbedtls/port/esp_bignum.c | 1073 +++++++---------- .../mbedtls/port/include/mbedtls/esp_config.h | 2 +- 4 files changed, 502 insertions(+), 611 deletions(-) create mode 100644 components/esp32/include/soc/hwcrypto_reg.h diff --git a/components/esp32/include/soc/hwcrypto_reg.h b/components/esp32/include/soc/hwcrypto_reg.h new file mode 100644 index 0000000000..4f38b1ba93 --- /dev/null +++ b/components/esp32/include/soc/hwcrypto_reg.h @@ -0,0 +1,37 @@ +// Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#ifndef __HWCRYPTO_REG_H__ +#define __HWCRYPTO_REG_H__ + +#include "soc.h" + +/* registers for RSA acceleration via Multiple Precision Integer ops */ +#define RSA_MEM_M_BLOCK_BASE ((DR_REG_RSA_BASE)+0x000) +/* RB & Z use the same memory block, depending on phase of operation */ +#define RSA_MEM_RB_BLOCK_BASE ((DR_REG_RSA_BASE)+0x200) +#define RSA_MEM_Z_BLOCK_BASE ((DR_REG_RSA_BASE)+0x200) +#define RSA_MEM_Y_BLOCK_BASE ((DR_REG_RSA_BASE)+0x400) +#define RSA_MEM_X_BLOCK_BASE ((DR_REG_RSA_BASE)+0x600) + +#define RSA_M_DASH_REG (DR_REG_RSA_BASE + 0x800) +#define RSA_MODEXP_MODE_REG (DR_REG_RSA_BASE + 0x804) +#define RSA_START_MODEXP_REG (DR_REG_RSA_BASE + 0x808) +#define RSA_MULT_MODE_REG (DR_REG_RSA_BASE + 0x80c) +#define RSA_MULT_START_REG (DR_REG_RSA_BASE + 0x810) + +#define RSA_INTERRUPT_REG (DR_REG_RSA_BASE + 0X814) + +#define RSA_CLEAN_ADDR (DR_REG_RSA_BASE + 0X818) + +#endif diff --git a/components/esp32/include/soc/soc.h b/components/esp32/include/soc/soc.h index 4ffdfb069e..65698ec856 100755 --- a/components/esp32/include/soc/soc.h +++ b/components/esp32/include/soc/soc.h @@ -141,6 +141,7 @@ //}} #define DR_REG_DPORT_BASE 0x3ff00000 +#define DR_REG_RSA_BASE 0x3ff02000 #define DR_REG_UART_BASE 0x3ff40000 #define DR_REG_SPI1_BASE 0x3ff42000 #define DR_REG_SPI0_BASE 0x3ff43000 diff --git a/components/mbedtls/port/esp_bignum.c b/components/mbedtls/port/esp_bignum.c index d6b79e32f5..55e70c47ac 100644 --- a/components/mbedtls/port/esp_bignum.c +++ b/components/mbedtls/port/esp_bignum.c @@ -23,9 +23,19 @@ #include #include #include +#include +#include #include "mbedtls/bignum.h" #include "mbedtls/bn_mul.h" #include "rom/bigint.h" +#include "soc/hwcrypto_reg.h" +#include "esp_system.h" +#include "esp_log.h" + +#include "freertos/FreeRTOS.h" +#include "freertos/task.h" + +static const char *TAG = "bignum"; #if defined(MBEDTLS_MPI_MUL_MPI_ALT) || defined(MBEDTLS_MPI_EXP_MOD_ALT) @@ -35,6 +45,38 @@ static _lock_t mpi_lock; +/* Temporary debugging function to print an MPI number to + stdout. Happens to be in a format compatible with Python. +*/ +void mbedtls_mpi_printf(const char *name, const mbedtls_mpi *X) +{ + static char buf[1024]; + size_t n; + memset(buf, 0, sizeof(buf)); + printf("%s = 0x", name); + mbedtls_mpi_write_string(X, 16, buf, sizeof(buf)-1, &n); + if(n) { + puts(buf); + } else { + puts("TOOLONG"); + } +} + +/* Temporary debug function to dump a memory block's contents to stdout + TODO remove + */ +static void __attribute__((unused)) dump_memory_block(const char *label, uint32_t addr) +{ + printf("Dumping %s @ %08x\n", label, addr); + for(int i = 0; i < (4096 / 8); i += 4) { + if(i % 32 == 0) { + printf("\n %04x:", i); + } + printf("%08x ", REG_READ(addr + i)); + } + printf("Done\n"); +} + /* At the moment these hardware locking functions aren't exposed publically for MPI. If you want to use the ROM bigint functions and co-exist with mbedTLS, please raise a feature request. @@ -52,299 +94,11 @@ static void esp_mpi_release_hardware( void ) _lock_release(&mpi_lock); } -/* Given a & b, determine u & v such that - - gcd(a,b) = d = au + bv - - Underlying algorithm comes from: - http://www.ucl.ac.uk/~ucahcjm/combopt/ext_gcd_python_programs.pdf - http://www.hackersdelight.org/hdcodetxt/mont64.c.txt - */ -static void extended_binary_gcd(const mbedtls_mpi *a, const mbedtls_mpi *b, - mbedtls_mpi *u, mbedtls_mpi *v) -{ - mbedtls_mpi ta, tb; - - mbedtls_mpi_init(&ta); - mbedtls_mpi_copy(&ta, a); - mbedtls_mpi_init(&tb); - mbedtls_mpi_copy(&tb, b); - - mbedtls_mpi_lset(u, 1); - mbedtls_mpi_lset(v, 0); - - /* Loop invariant: - ta = u*2*a - v*b. */ - while (mbedtls_mpi_cmp_int(&ta, 0) != 0) { - mbedtls_mpi_shift_r(&ta, 1); - if (mbedtls_mpi_get_bit(u, 0) == 0) { - // Remove common factor of 2 in u & v - mbedtls_mpi_shift_r(u, 1); - mbedtls_mpi_shift_r(v, 1); - } - else { - /* u = (u + b) >> 1 */ - mbedtls_mpi_add_mpi(u, u, b); - mbedtls_mpi_shift_r(u, 1); - /* v = (v >> 1) + a */ - mbedtls_mpi_shift_r(v, 1); - mbedtls_mpi_add_mpi(v, v, a); - } - } - mbedtls_mpi_free(&ta); - mbedtls_mpi_free(&tb); - - /* u = u * 2, so 1 = u*a - v*b */ - mbedtls_mpi_shift_l(u, 1); -} - -/* inner part of MPI modular multiply, after Rinv & Mprime are calculated */ -static int mpi_mul_mpi_mod_inner(mbedtls_mpi *X, const mbedtls_mpi *A, const mbedtls_mpi *B, const mbedtls_mpi *M, mbedtls_mpi *Rinv, uint32_t Mprime, size_t num_words) -{ - int ret; - mbedtls_mpi TA, TB; - size_t num_bits = num_words * 32; - - mbedtls_mpi_grow(Rinv, num_words); - - /* TODO: fill memory blocks directly so this isn't needed */ - mbedtls_mpi_init(&TA); - mbedtls_mpi_copy(&TA, A); - mbedtls_mpi_grow(&TA, num_words); - A = &TA; - mbedtls_mpi_init(&TB); - mbedtls_mpi_copy(&TB, B); - mbedtls_mpi_grow(&TB, num_words); - B = &TB; - - esp_mpi_acquire_hardware(); - - if(ets_bigint_mod_mult_prepare(A->p, B->p, M->p, Mprime, - Rinv->p, num_bits, false)) { - mbedtls_mpi_grow(X, num_words); - ets_bigint_wait_finish(); - if(ets_bigint_mod_mult_getz(M->p, X->p, num_bits)) { - X->s = A->s * B->s; - ret = 0; - } else { - printf("ets_bigint_mod_mult_getz failed\n"); - ret = MBEDTLS_ERR_MPI_BAD_INPUT_DATA; - } - } else { - printf("ets_bigint_mod_mult_prepare failed\n"); - ret = MBEDTLS_ERR_MPI_BAD_INPUT_DATA; - } - esp_mpi_release_hardware(); - - /* unclear why this is necessary, but the result seems - to come back rotated 32 bits to the right... */ - uint32_t last_word = X->p[num_words-1]; - X->p[num_words-1] = 0; - mbedtls_mpi_shift_l(X, 32); - X->p[0] = last_word; - - mbedtls_mpi_free(&TA); - mbedtls_mpi_free(&TB); - - return ret; -} - -/* X = (A * B) mod M - - Not an mbedTLS function - - num_bits guaranteed to be a multiple of 512 already. - - TODO: ensure M is odd - */ -int esp_mpi_mul_mpi_mod(mbedtls_mpi *X, const mbedtls_mpi *A, const mbedtls_mpi *B, const mbedtls_mpi *M, size_t num_bits) -{ - int ret = 0; - mbedtls_mpi RR, Rinv, Mprime; - uint32_t Mprime_int; - size_t num_words = num_bits / 32; - - /* Rinv & Mprime are calculated via extended binary gcd - algorithm, see references on extended_binary_gcd above. - */ - mbedtls_mpi_init(&Rinv); - mbedtls_mpi_init(&RR); - mbedtls_mpi_set_bit(&RR, num_bits+32, 1); - mbedtls_mpi_init(&Mprime); - extended_binary_gcd(&RR, M, &Rinv, &Mprime); - - /* M' is mod 2^32 */ - Mprime_int = Mprime.p[0]; - - ret = mpi_mul_mpi_mod_inner(X, A, B, M, &Rinv, Mprime_int, num_words); - - mbedtls_mpi_free(&RR); - mbedtls_mpi_free(&Mprime); - mbedtls_mpi_free(&Rinv); - - return ret; -} - - -/* - * Helper for mbedtls_mpi multiplication - * copied/trimmed from mbedtls bignum.c - */ -static void mpi_mul_hlp( size_t i, mbedtls_mpi_uint *s, mbedtls_mpi_uint *d, mbedtls_mpi_uint b ) -{ - mbedtls_mpi_uint c = 0, t = 0; - - for( ; i >= 16; i -= 16 ) - { - MULADDC_INIT - MULADDC_CORE MULADDC_CORE - MULADDC_CORE MULADDC_CORE - MULADDC_CORE MULADDC_CORE - MULADDC_CORE MULADDC_CORE - - MULADDC_CORE MULADDC_CORE - MULADDC_CORE MULADDC_CORE - MULADDC_CORE MULADDC_CORE - MULADDC_CORE MULADDC_CORE - MULADDC_STOP - } - - for( ; i >= 8; i -= 8 ) - { - MULADDC_INIT - MULADDC_CORE MULADDC_CORE - MULADDC_CORE MULADDC_CORE - - MULADDC_CORE MULADDC_CORE - MULADDC_CORE MULADDC_CORE - MULADDC_STOP - } - - - for( ; i > 0; i-- ) - { - MULADDC_INIT - MULADDC_CORE - MULADDC_STOP - } - - t++; - - do { - *d += c; c = ( *d < c ); d++; - } - while( c != 0 ); -} - - -/* - * Helper for mbedtls_mpi subtraction - * Copied/adapter from mbedTLS bignum.c - */ -static void mpi_sub_hlp( size_t n, mbedtls_mpi_uint *s, mbedtls_mpi_uint *d ) -{ - size_t i; - mbedtls_mpi_uint c, z; - - for( i = c = 0; i < n; i++, s++, d++ ) - { - z = ( *d < c ); *d -= c; - c = ( *d < *s ) + z; *d -= *s; - } - - while( c != 0 ) - { - z = ( *d < c ); *d -= c; - c = z; i++; d++; - } -} - - -/* The following 3 Montgomery arithmetic function are - copied from mbedTLS bigint.c verbatim as they are static. - - TODO: find a way to support making the versions in mbedtls - non-static. -*/ - -/* - * Fast Montgomery initialization (thanks to Tom St Denis) - */ -static void mpi_montg_init( mbedtls_mpi_uint *mm, const mbedtls_mpi *N ) -{ - mbedtls_mpi_uint x, m0 = N->p[0]; - unsigned int i; - - x = m0; - x += ( ( m0 + 2 ) & 4 ) << 1; - - for( i = biL; i >= 8; i /= 2 ) - x *= ( 2 - ( m0 * x ) ); - - *mm = ~x + 1; -} - -/* - * Montgomery multiplication: A = A * B * R^-1 mod N (HAC 14.36) - */ -static int mpi_montmul( mbedtls_mpi *A, const mbedtls_mpi *B, const mbedtls_mpi *N, mbedtls_mpi_uint mm, - const mbedtls_mpi *T ) -{ - size_t i, n, m; - mbedtls_mpi_uint u0, u1, *d; - - if( T->n < N->n + 1 || T->p == NULL ) - return( MBEDTLS_ERR_MPI_BAD_INPUT_DATA ); - - memset( T->p, 0, T->n * ciL ); - - d = T->p; - n = N->n; - m = ( B->n < n ) ? B->n : n; - - for( i = 0; i < n; i++ ) - { - /* - * T = (T + u0*B + u1*N) / 2^biL - */ - u0 = A->p[i]; - u1 = ( d[0] + u0 * B->p[0] ) * mm; - - mpi_mul_hlp( m, B->p, d, u0 ); - mpi_mul_hlp( n, N->p, d, u1 ); - - *d++ = u0; d[n + 1] = 0; - } - - memcpy( A->p, d, ( n + 1 ) * ciL ); - - if( mbedtls_mpi_cmp_abs( A, N ) >= 0 ) - mpi_sub_hlp( n, N->p, A->p ); - else - /* prevent timing attacks */ - mpi_sub_hlp( n, A->p, T->p ); - - return( 0 ); -} - -/* - * Montgomery reduction: A = A * R^-1 mod N - */ -static int mpi_montred( mbedtls_mpi *A, const mbedtls_mpi *N, mbedtls_mpi_uint mm, const mbedtls_mpi *T ) -{ - mbedtls_mpi_uint z = 1; - mbedtls_mpi U; - - U.n = U.s = (int) z; - U.p = &z; - - return( mpi_montmul( A, &U, N, mm, T ) ); -} - -#if defined(MBEDTLS_MPI_MUL_MPI_ALT) /* MBEDTLS_MPI_MUL_MPI_ALT */ - /* Number of words used to hold 'mpi', rounded up to nearest - 16 words (512 bits) to match hardware support + 16 words (512 bits) to match hardware support. + + Note that mpi->N (size of memory buffer) may be higher than this + number, if the high bits are mostly zeroes. */ static inline size_t hardware_words_needed(const mbedtls_mpi *mpi) { @@ -357,356 +111,455 @@ static inline size_t hardware_words_needed(const mbedtls_mpi *mpi) return res; } +/* Copy mbedTLS MPI bignum 'mpi' to hardware memory block at 'mem_base'. -/* Special-case multiply, where we use hardware montgomery mod - multiplication to solve the case where A or B are >2048 bits so - can't do standard multiplication. - - the modulus here is chosen with M=(2^num_bits-1) - to guarantee the output isn't actually modulo anything. This means - we don't need to calculate M' and Rinv, they are predictable - as follows: - M' = 1 - Rinv = (1 << (num_bits - 32) - - (See RSA Accelerator section in Technical Reference for derivation - of M', Rinv) + If num_words is higher than the number of words in the bignum then + these additional words will be zeroed in the memory buffer. */ -static int esp_mpi_mult_mpi_failover_mod_mult(mbedtls_mpi *X, const mbedtls_mpi *A, const mbedtls_mpi *B, size_t num_words) - { - mbedtls_mpi M, Rinv; - int ret; - size_t mprime; - size_t num_bits = num_words * 32; - - mbedtls_mpi_init(&M); - mbedtls_mpi_init(&Rinv); - - /* TODO: it may be faster to just use 4096-bit arithmetic every time, - and make these constants rather than runtime derived - derived. */ - /* M = (2^num_words)-1 */ - mbedtls_mpi_grow(&M, num_words); - for(int i = 0; i < num_words*32; i++) { - mbedtls_mpi_set_bit(&M, i, 1); - } - - /* Rinv = (2^num_words-32) */ - mbedtls_mpi_grow(&Rinv, num_words); - mbedtls_mpi_set_bit(&Rinv, num_bits - 32, 1); - - mprime = 1; - - ret = mpi_mul_mpi_mod_inner(X, A, B, &M, &Rinv, mprime, num_words); - - mbedtls_mpi_free(&M); - mbedtls_mpi_free(&Rinv); - return ret; - } - -int mbedtls_mpi_mul_mpi( mbedtls_mpi *X, const mbedtls_mpi *A, const mbedtls_mpi *B ) +static inline void mpi_to_mem_block(uint32_t mem_base, const mbedtls_mpi *mpi, size_t num_words) { - int ret = -1; - size_t words_a, words_b, words_x, words_mult; - - mbedtls_mpi TA, TB; - - mbedtls_mpi_init( &TA ); mbedtls_mpi_init( &TB ); - - /* Count words needed for A & B in hardware */ - words_a = hardware_words_needed(A); - words_b = hardware_words_needed(B); - - words_mult = (words_a > words_b ? words_a : words_b); - - /* Take a copy of A if either X == A OR if A isn't long enough - to hold the number of words needed for hardware. - - (can't grow A directly as it is const) - - TODO: growing the input operands is only necessary because the - ROM functions only take one length argument. It should be - possible for us to just copy the used data only into the - hardware buffers, and set the remaining bits to zero - saving - RAM. But we need to reimplement ets_bigint_mult_prepare() in - software for this. - */ - if( X == A || A->n < words_mult) { - MBEDTLS_MPI_CHK( mbedtls_mpi_copy( &TA, A ) ); - MBEDTLS_MPI_CHK( mbedtls_mpi_grow( &TA, words_mult) ); - A = &TA; + for(size_t i = 0; i < mpi->n && i < num_words; i++) { + REG_WRITE(mem_base + i * 4, mpi->p[i]); } - /* Same for B */ - if( X == B || B->n < words_mult ) { - MBEDTLS_MPI_CHK( mbedtls_mpi_copy( &TB, B ) ); - MBEDTLS_MPI_CHK( mbedtls_mpi_grow( &TB, words_mult) ); - B = &TB; + for(size_t i = mpi->n; i < num_words; i++) { + REG_WRITE(mem_base + i * 4, 0); } - - /* Result X has to have room for double the larger operand */ - words_x = words_mult * 2; - MBEDTLS_MPI_CHK( mbedtls_mpi_grow( X, words_x ) ); - /* TODO: check if lset here is necessary, hardware should zero */ - MBEDTLS_MPI_CHK( mbedtls_mpi_lset( X, 0 ) ); - - /* If either operand is over 2048 bits, we can't use the standard hardware multiplier - (it assumes result is double longest operand, and result is max 4096 bits.) - - However, we can fail over to mod_mult for up to 4096 bits. - */ - if(words_mult * 32 > 2048) { - /* TODO: check if there's an overflow condition if words_a & words_b are both - the bit lengths of the operands, result could be 1 bit longer - */ - if((words_a + words_b) * 32 > 4096) { - printf("ERROR: %d bit operands (%d bits * %d bits) too large for hardware unit\n", words_mult * 32, mbedtls_mpi_bitlen(A), mbedtls_mpi_bitlen(B)); - ret = MBEDTLS_ERR_MPI_NOT_ACCEPTABLE; - } - else { - ret = esp_mpi_mult_mpi_failover_mod_mult(X, A, B, words_a + words_b); - } - } - else { - - /* normal mpi multiplication */ - esp_mpi_acquire_hardware(); - if (ets_bigint_mult_prepare(A->p, B->p, words_mult * 32)) { - ets_bigint_wait_finish(); - /* NB: argument to bigint_mult_getz is length of inputs, double this number (words_x) is - copied to output X->p. - */ - if (ets_bigint_mult_getz(X->p, words_mult * 32) == true) { - X->s = A->s * B->s; - ret = 0; - } else { - printf("ets_bigint_mult_getz failed\n"); - ret = MBEDTLS_ERR_MPI_NOT_ACCEPTABLE; - } - } else{ - printf("Baseline multiplication failed\n"); - ret = MBEDTLS_ERR_MPI_NOT_ACCEPTABLE; - } - esp_mpi_release_hardware(); - } -cleanup: - - mbedtls_mpi_free( &TB ); mbedtls_mpi_free( &TA ); - - return( ret ); } -#endif /* MBEDTLS_MPI_MUL_MPI_ALT */ +/* Read mbedTLS MPI bignum back from hardware memory block. -#if defined(MBEDTLS_MPI_EXP_MOD_ALT) -/* - * Sliding-window exponentiation: X = A^E mod N (HAC 14.85) + Reads num_words words from block. + + Can return a failure result if fails to grow the MPI result. +*/ +static inline int mem_block_to_mpi(mbedtls_mpi *x, uint32_t mem_base, int num_words) +{ + int ret = 0; + size_t x_n = x->n; + + /* this code is written in non-intuitive way, to only grow the + result if it is absolutely necessary - ie if all the high bits + are zero, the bignum won't be grown to fit them. */ + for(int i = num_words - 1; i >= 0; i--) { + uint32_t value = REG_READ(mem_base + i * 4); + if(value != 0 && x_n <= i) { + MBEDTLS_MPI_CHK( mbedtls_mpi_grow(x, i+1) ); + x_n = i+1; + } + if(x_n > i) { + x->p[i] = value; + } + } + /* Zero any remaining limbs in the bignum, if the buffer was + always bigger than num_words */ + for(size_t i = num_words; i < x->n; i++) { + x->p[i] = 0; + } + + cleanup: + return ret; +} + +/* Given a & b, determine u & v such that + + gcd(a,b) = d = au - bv + + This is suitable for calculating values for montgomery multiplication: + + gcd(R, M) = R * Rinv - M * Mprime = 1 + + Conditions which must be true: + - argument 'a' (R) is a power of 2. + - argument 'b' (M) is odd. + + Underlying algorithm comes from: + http://www.hackersdelight.org/hdcodetxt/mont64.c.txt + http://www.ucl.ac.uk/~ucahcjm/combopt/ext_gcd_python_programs.pdf */ -int mbedtls_mpi_exp_mod( mbedtls_mpi* X, const mbedtls_mpi* A, const mbedtls_mpi* E, const mbedtls_mpi* N, mbedtls_mpi* _RR ) +static void extended_binary_gcd(const mbedtls_mpi *a, const mbedtls_mpi *b, + mbedtls_mpi *u, mbedtls_mpi *v) +{ + mbedtls_mpi a_, ta; + + /* These checks degrade performance, TODO remove them... */ + assert(b->p[0] & 1); + assert(mbedtls_mpi_bitlen(a) == mbedtls_mpi_lsb(a)+1); + assert(mbedtls_mpi_cmp_mpi(a, b) > 0); + + mbedtls_mpi_lset(u, 1); + mbedtls_mpi_lset(v, 0); + + /* 'a' needs to be half its real value for this algorithm + TODO see if we can halve the number in the caller to avoid + allocating a bignum here. + */ + mbedtls_mpi_init(&a_); + mbedtls_mpi_copy(&a_, a); + mbedtls_mpi_shift_r(&a_, 1); + + mbedtls_mpi_init(&ta); + mbedtls_mpi_copy(&ta, &a_); + + //mbedtls_mpi_printf("a", &a_); + //mbedtls_mpi_printf("b", b); + + /* Loop invariant: + 2*ta = u*2*a - v*b. + + Loop until ta == 0 + */ + while (mbedtls_mpi_cmp_int(&ta, 0) != 0) { + //mbedtls_mpi_printf("ta", &ta); + //mbedtls_mpi_printf("u", u); + //mbedtls_mpi_printf("v", v); + //printf("2*ta == u*2*a - v*b\n"); + + mbedtls_mpi_shift_r(&ta, 1); + if (mbedtls_mpi_get_bit(u, 0) == 0) { + // Remove common factor of 2 in u & v + mbedtls_mpi_shift_r(u, 1); + mbedtls_mpi_shift_r(v, 1); + } + else { + /* u = (u + b) >> 1 */ + mbedtls_mpi_add_mpi(u, u, b); + mbedtls_mpi_shift_r(u, 1); + /* v = (v - a) >> 1 */ + mbedtls_mpi_shift_r(v, 1); + mbedtls_mpi_add_mpi(v, v, &a_); + } + } + mbedtls_mpi_free(&ta); + mbedtls_mpi_free(&a_); +} + +/* Execute RSA operation. op_reg specifies which 'START' register + to write to. +*/ +static inline void execute_op(uint32_t op_reg) +{ + /* Clear interrupt status, start operation */ + REG_WRITE(RSA_INTERRUPT_REG, 1); + REG_WRITE(op_reg, 1); + + /* TODO: use interrupt instead of busywaiting */ + while(REG_READ(RSA_INTERRUPT_REG) != 1) + { } + + /* clear the interrupt */ + REG_WRITE(RSA_INTERRUPT_REG, 1); +} + +/* Sub-stages of modulo multiplication/exponentiation operations */ +static int modular_op_prepare(const mbedtls_mpi *X, const mbedtls_mpi *M, size_t num_words); +inline static int modular_multiply_finish(mbedtls_mpi *Z, const mbedtls_mpi *X, const mbedtls_mpi *Y, size_t num_words); + +/* Z = (X * Y) mod M + + Not an mbedTLS function + */ +int esp_mpi_mul_mpi_mod(mbedtls_mpi *Z, const mbedtls_mpi *X, const mbedtls_mpi *Y, const mbedtls_mpi *M) { int ret; - size_t wbits, wsize, one = 1; - size_t i, j, nblimbs; - size_t bufsize, nbits; - mbedtls_mpi_uint ei, mm, state; - mbedtls_mpi RR, T, W[ 2 << MBEDTLS_MPI_WINDOW_SIZE ], Apos; - int neg; + size_t num_words = hardware_words_needed(M); - if( mbedtls_mpi_cmp_int( N, 0 ) < 0 || ( N->p[0] & 1 ) == 0 ) - return( MBEDTLS_ERR_MPI_BAD_INPUT_DATA ); + /* Calculate and load the first stage montgomery multiplication */ + MBEDTLS_MPI_CHK( modular_op_prepare(X, M, num_words) ); - if( mbedtls_mpi_cmp_int( E, 0 ) < 0 ) - return( MBEDTLS_ERR_MPI_BAD_INPUT_DATA ); + execute_op(RSA_MULT_START_REG); - /* - * Init temps and window size - */ - mpi_montg_init( &mm, N ); - mbedtls_mpi_init( &RR ); mbedtls_mpi_init( &T ); - mbedtls_mpi_init( &Apos ); - memset( W, 0, sizeof( W ) ); + MBEDTLS_MPI_CHK( modular_multiply_finish(Z, X, Y, num_words) ); - i = mbedtls_mpi_bitlen( E ); + esp_mpi_release_hardware(); - wsize = ( i > 671 ) ? 6 : ( i > 239 ) ? 5 : - ( i > 79 ) ? 4 : ( i > 23 ) ? 3 : 1; + cleanup: + return ret; +} - if( wsize > MBEDTLS_MPI_WINDOW_SIZE ) - wsize = MBEDTLS_MPI_WINDOW_SIZE; +#if defined(MBEDTLS_MPI_EXP_MOD_ALT) - j = N->n + 1; - MBEDTLS_MPI_CHK( mbedtls_mpi_grow( X, j ) ); - MBEDTLS_MPI_CHK( mbedtls_mpi_grow( &W[1], j ) ); - MBEDTLS_MPI_CHK( mbedtls_mpi_grow( &T, j * 2 ) ); +/* + * Sliding-window exponentiation: Z = X^Y mod M (HAC 14.85) + */ +int mbedtls_mpi_exp_mod( mbedtls_mpi* Z, const mbedtls_mpi* X, const mbedtls_mpi* Y, const mbedtls_mpi* M, mbedtls_mpi* _RR ) +{ + int ret; + size_t z_words = hardware_words_needed(Z); + size_t x_words = hardware_words_needed(X); + size_t y_words = hardware_words_needed(Y); + size_t m_words = hardware_words_needed(M); + size_t num_words; - /* - * Compensate for negative A (and correct at the end) - */ - neg = ( A->s == -1 ); - if( neg ) - { - MBEDTLS_MPI_CHK( mbedtls_mpi_copy( &Apos, A ) ); - Apos.s = 1; - A = &Apos; + mbedtls_mpi_printf("X",X); + mbedtls_mpi_printf("Y",Y); + mbedtls_mpi_printf("M",M); + + /* "all numbers must be the same length", so choose longest number + as cardinal length of operation... + */ + num_words = z_words; + if (x_words > num_words) { + num_words = x_words; + } + if (y_words > num_words) { + num_words = y_words; + } + if (m_words > num_words) { + num_words = m_words; + } + printf("num_words = %d # %d, %d, %d\n", num_words, x_words, y_words, m_words); + + /* TODO: _RR parameter currently ignored */ + + ret = modular_op_prepare(X, M, num_words); + if (ret != 0) { + return ret; } - /* - * If 1st call, pre-compute R^2 mod N - */ - if( _RR == NULL || _RR->p == NULL ) - { - MBEDTLS_MPI_CHK( mbedtls_mpi_lset( &RR, 1 ) ); - MBEDTLS_MPI_CHK( mbedtls_mpi_shift_l( &RR, N->n * 2 * biL ) ); - MBEDTLS_MPI_CHK( mbedtls_mpi_mod_mpi( &RR, &RR, N ) ); + mpi_to_mem_block(RSA_MEM_Y_BLOCK_BASE, Y, num_words); - if( _RR != NULL ) - memcpy( _RR, &RR, sizeof( mbedtls_mpi) ); - } - else - memcpy( &RR, _RR, sizeof( mbedtls_mpi) ); + //dump_memory_block("X_BLOCK", RSA_MEM_X_BLOCK_BASE); + //dump_memory_block("Y_BLOCK", RSA_MEM_Y_BLOCK_BASE); + //dump_memory_block("M_BLOCK", RSA_MEM_M_BLOCK_BASE); - /* - * W[1] = A * R^2 * R^-1 mod N = A * R mod N - */ - if( mbedtls_mpi_cmp_mpi( A, N ) >= 0 ) - MBEDTLS_MPI_CHK( mbedtls_mpi_mod_mpi( &W[1], A, N ) ); - else - MBEDTLS_MPI_CHK( mbedtls_mpi_copy( &W[1], A ) ); + REG_WRITE(RSA_MODEXP_MODE_REG, (num_words / 16) - 1); - mpi_montmul( &W[1], &RR, N, mm, &T ); + execute_op(RSA_START_MODEXP_REG); - /* - * X = R^2 * R^-1 mod N = R mod N - */ - MBEDTLS_MPI_CHK( mbedtls_mpi_copy( X, &RR ) ); - mpi_montred( X, N, mm, &T ); + //dump_memory_block("Z_BLOCK", RSA_MEM_Z_BLOCK_BASE); - if( wsize > 1 ) - { - /* - * W[1 << (wsize - 1)] = W[1] ^ (wsize - 1) - */ - j = one << ( wsize - 1 ); + /* TODO: only need to read m_words not num_words, provided result is correct... */ + ret = mem_block_to_mpi(Z, RSA_MEM_Z_BLOCK_BASE, num_words); - MBEDTLS_MPI_CHK( mbedtls_mpi_grow( &W[j], N->n + 1 ) ); - MBEDTLS_MPI_CHK( mbedtls_mpi_copy( &W[j], &W[1] ) ); + esp_mpi_release_hardware(); - for( i = 0; i < wsize - 1; i++ ) - mpi_montmul( &W[j], &W[j], N, mm, &T ); + mbedtls_mpi_printf("Z",Z); + printf("print (Z == (X ** Y) %% M)\n"); - /* - * W[i] = W[i - 1] * W[1] - */ - for( i = j + 1; i < ( one << wsize ); i++ ) - { - MBEDTLS_MPI_CHK( mbedtls_mpi_grow( &W[i], N->n + 1 ) ); - MBEDTLS_MPI_CHK( mbedtls_mpi_copy( &W[i], &W[i - 1] ) ); - - mpi_montmul( &W[i], &W[1], N, mm, &T ); - } - } - - nblimbs = E->n; - bufsize = 0; - nbits = 0; - wbits = 0; - state = 0; - - while( 1 ) - { - if( bufsize == 0 ) - { - if( nblimbs == 0 ) - break; - - nblimbs--; - - bufsize = sizeof( mbedtls_mpi_uint ) << 3; - } - - bufsize--; - - ei = (E->p[nblimbs] >> bufsize) & 1; - - /* - * skip leading 0s - */ - if( ei == 0 && state == 0 ) - continue; - - if( ei == 0 && state == 1 ) - { - /* - * out of window, square X - */ - mpi_montmul( X, X, N, mm, &T ); - continue; - } - - /* - * add ei to current window - */ - state = 2; - - nbits++; - wbits |= ( ei << ( wsize - nbits ) ); - - if( nbits == wsize ) - { - /* - * X = X^wsize R^-1 mod N - */ - for( i = 0; i < wsize; i++ ) - mpi_montmul( X, X, N, mm, &T ); - - /* - * X = X * W[wbits] R^-1 mod N - */ - mpi_montmul( X, &W[wbits], N, mm, &T ); - - state--; - nbits = 0; - wbits = 0; - } - } - - /* - * process the remaining bits - */ - for( i = 0; i < nbits; i++ ) - { - mpi_montmul( X, X, N, mm, &T ); - - wbits <<= 1; - - if( ( wbits & ( one << wsize ) ) != 0 ) - mpi_montmul( X, &W[1], N, mm, &T ); - } - - /* - * X = A^E * R * R^-1 mod N = A^E mod N - */ - mpi_montred( X, N, mm, &T ); - - if( neg ) - { - X->s = -1; - MBEDTLS_MPI_CHK( mbedtls_mpi_add_mpi( X, N, X ) ); - } - -cleanup: - - for( i = ( one << ( wsize - 1 ) ); i < ( one << wsize ); i++ ) - mbedtls_mpi_free( &W[i] ); - - mbedtls_mpi_free( &W[1] ); mbedtls_mpi_free( &T ); mbedtls_mpi_free( &Apos ); - - if( _RR == NULL || _RR->p == NULL ) - mbedtls_mpi_free( &RR ); - - return( ret ); + return ret; } #endif /* MBEDTLS_MPI_EXP_MOD_ALT */ + +/* The common parts of modulo multiplication and modular sliding + * window exponentiation: + * + * @param X first multiplication factor and/or base of exponent. + * @param M modulo value for result + * @param num_words size of modulo operation, in words (limbs). + * Should already be rounded up to a multiple of 16 words (512 bits) & range checked. + * + * Steps: + * Calculate Rinv & Mprime based on M & num_words + * Load all coefficients to memory + * Set mode register + * + * @note This function calls esp_mpi_acquire_hardware. If successful, + * returns 0 and it becomes the callers responsibility to call + * esp_mpi_release_hardware(). If failure is returned, the caller does + * not need to call esp_mpi_release_hardware(). + */ +static int modular_op_prepare(const mbedtls_mpi *X, const mbedtls_mpi *M, size_t num_words) +{ + int ret = 0; + mbedtls_mpi RR, Rinv, Mprime; + size_t num_bits; + + /* Calculate number of bits */ + num_bits = num_words * 32; + + if(num_bits > 4096) { + return MBEDTLS_ERR_MPI_NOT_ACCEPTABLE; + } + + /* Rinv & Mprime are calculated via extended binary gcd + algorithm, see references on extended_binary_gcd() above. + */ + mbedtls_mpi_init(&Rinv); + mbedtls_mpi_init(&RR); + mbedtls_mpi_init(&Mprime); + + mbedtls_mpi_set_bit(&RR, num_bits, 1); /* R = b^n where b = 2^32, n=num_words, + ie R = 2^N (where N=num_bits) */ + /* calculate Rinv & Mprime */ + extended_binary_gcd(&RR, M, &Rinv, &Mprime); + + /* Block of debugging data, output suitable to paste into Python + TODO remove + */ + mbedtls_mpi_printf("R", &RR); + mbedtls_mpi_printf("M", M); + mbedtls_mpi_printf("Rinv", &Rinv); + mbedtls_mpi_printf("Mprime", &Mprime); + printf("print (R * Rinv - M * Mprime == 1)\n"); + printf("print (Rinv == (R * R) %% M)\n"); + + esp_mpi_acquire_hardware(); + + /* Load M, X, Rinv, M-prime (M-prime is mod 2^32) */ + mpi_to_mem_block(RSA_MEM_M_BLOCK_BASE, M, num_words); + mpi_to_mem_block(RSA_MEM_X_BLOCK_BASE, X, num_words); + mpi_to_mem_block(RSA_MEM_RB_BLOCK_BASE, &Rinv, num_words); + REG_WRITE(RSA_M_DASH_REG, Mprime.p[0]); + + /* "mode" register loaded with number of 512-bit blocks, minus 1 */ + REG_WRITE(RSA_MULT_MODE_REG, (num_words / 16) - 1); + + mbedtls_mpi_free(&Rinv); + mbedtls_mpi_free(&RR); + mbedtls_mpi_free(&Mprime); + + return ret; +} + +/* Second & final step of a modular multiply - load second multiplication + * factor Y, run the multiply, read back the result into Z. + * + * @param Z result value + * @param X first multiplication factor (used to set sign of result). + * @param Y second multiplication factor. + * @param num_words size of modulo operation, in words (limbs). + * Should already be rounded up to a multiple of 16 words (512 bits) & range checked. + * + * Caller must have already called esp_mpi_acquire_hardware(). + */ +inline static int modular_multiply_finish(mbedtls_mpi *Z, const mbedtls_mpi *X, const mbedtls_mpi *Y, size_t num_words) +{ + int ret; + /* Load Y to X input memory block, rerun */ + mpi_to_mem_block(RSA_MEM_X_BLOCK_BASE, Y, num_words); + + execute_op(RSA_MULT_START_REG); + + /* Read result into Z */ + ret = mem_block_to_mpi(Z, RSA_MEM_Z_BLOCK_BASE, num_words); + + Z->s = X->s * Y->s; + + return ret; +} + +#if defined(MBEDTLS_MPI_MUL_MPI_ALT) /* MBEDTLS_MPI_MUL_MPI_ALT */ + +static int mpi_mult_mpi_failover_mod_mult(mbedtls_mpi *Z, const mbedtls_mpi *X, const mbedtls_mpi *Y, size_t num_words); + +/* Z = X * Y */ +int mbedtls_mpi_mul_mpi( mbedtls_mpi *Z, const mbedtls_mpi *X, const mbedtls_mpi *Y ) +{ + int ret; + size_t words_x, words_y, words_mult, words_z; + + /* Count words needed for X & Y in hardware */ + words_x = hardware_words_needed(X); + words_y = hardware_words_needed(Y); + + words_mult = (words_x > words_y ? words_x : words_y); + + /* Result Z has to have room for double the larger factor */ + words_z = words_mult * 2; + + /* If either factor is over 2048 bits, we can't use the standard hardware multiplier + (it assumes result is double longest factor, and result is max 4096 bits.) + + However, we can fail over to mod_mult for up to 4096 bits of result (modulo + multiplication doesn't have the same restriction, so result is simply the + number of bits in X plus number of bits in in Y.) + */ + if (words_mult * 32 > 2048) { + /* Calculate new length of Z */ + words_z = words_x + words_y; + if (words_z * 32 > 4096) { + ESP_LOGE(TAG, "ERROR: %d bit result (%d bits * %d bits) too large for hardware unit\n", words_z * 32, mbedtls_mpi_bitlen(X), mbedtls_mpi_bitlen(Y)); + return MBEDTLS_ERR_MPI_NOT_ACCEPTABLE; + } + else { + return mpi_mult_mpi_failover_mod_mult(Z, X, Y, words_z); + } + } + + /* Otherwise, we can use the (faster) multiply hardware unit */ + + esp_mpi_acquire_hardware(); + + /* Copy X (right-extended) & Y (left-extended) to memory block */ + mpi_to_mem_block(RSA_MEM_X_BLOCK_BASE, X, words_mult); + mpi_to_mem_block(RSA_MEM_Z_BLOCK_BASE + words_mult * 4, Y, words_mult); + /* NB: as Y is left-extended, we don't zero the bottom words_mult words of Y block. + This is OK for now because zeroing is done by hardware when we do esp_mpi_acquire_hardware(). + */ + + REG_WRITE(RSA_M_DASH_REG, 0); + + /* "mode" register loaded with number of 512-bit blocks in result, + plus 7 (for range 9-12). (this is ((N~ / 32) - 1) + 8)) + */ + REG_WRITE(RSA_MULT_MODE_REG, (words_z / 16) + 7); + + execute_op(RSA_MULT_START_REG); + + /* Read back the result */ + ret = mem_block_to_mpi(Z, RSA_MEM_Z_BLOCK_BASE, words_z); + + Z->s = X->s * Y->s; + + esp_mpi_release_hardware(); + + return ret; +} + +/* Special-case of mbedtls_mpi_mult_mpi(), where we use hardware montgomery mod + multiplication to solve the case where A or B are >2048 bits so + can't use the standard multiplication method. + + This case is simpler than esp_mpi_mul_mpi_mod() as we control the arguments: + + * Modulus is chosen with M=(2^num_bits - 1) (ie M=R-1), so output + isn't actually modulo anything. + * Therefore of of M' and Rinv are predictable as follows: + M' = 1 + Rinv = 1 + + (See RSA Accelerator section in Technical Reference * + extended_binary_gcd() function above for more about M', Rinv) +*/ +static int mpi_mult_mpi_failover_mod_mult(mbedtls_mpi *Z, const mbedtls_mpi *X, const mbedtls_mpi *Y, size_t num_words) + { + int ret = 0; + + /* Load coefficients to hardware */ + esp_mpi_acquire_hardware(); + + /* M = 2^num_words - 1, so block is entirely FF */ + for(int i = 0; i < num_words; i++) { + REG_WRITE(RSA_MEM_M_BLOCK_BASE + i * 4, UINT32_MAX); + } + /* Mprime = 1 */ + REG_WRITE(RSA_M_DASH_REG, 1); + + /* "mode" register loaded with number of 512-bit blocks, minus 1 */ + REG_WRITE(RSA_MULT_MODE_REG, (num_words / 16) - 1); + + /* Load X */ + mpi_to_mem_block(RSA_MEM_X_BLOCK_BASE, X, num_words); + + /* Rinv = 1 */ + REG_WRITE(RSA_MEM_RB_BLOCK_BASE, 1); + for(int i = 1; i < num_words; i++) { + REG_WRITE(RSA_MEM_RB_BLOCK_BASE + i * 4, 0); + } + + execute_op(RSA_MULT_START_REG); + + MBEDTLS_MPI_CHK( modular_multiply_finish(Z, X, Y, num_words) ); + + esp_mpi_release_hardware(); + + cleanup: + return ret; +} + +#endif /* MBEDTLS_MPI_MUL_MPI_ALT */ + #endif /* MBEDTLS_MPI_MUL_MPI_ALT || MBEDTLS_MPI_EXP_MOD_ALT */ diff --git a/components/mbedtls/port/include/mbedtls/esp_config.h b/components/mbedtls/port/include/mbedtls/esp_config.h index e4f4af271a..2b47d84ea4 100644 --- a/components/mbedtls/port/include/mbedtls/esp_config.h +++ b/components/mbedtls/port/include/mbedtls/esp_config.h @@ -251,7 +251,7 @@ Uncommenting these macros will use the hardware-accelerated implementations. */ -//#define MBEDTLS_MPI_EXP_MOD_ALT +#define MBEDTLS_MPI_EXP_MOD_ALT #define MBEDTLS_MPI_MUL_MPI_ALT /** From ce7b8059de18fd53a78848f2987716045c126f05 Mon Sep 17 00:00:00 2001 From: Angus Gratton Date: Wed, 12 Oct 2016 17:08:05 +1100 Subject: [PATCH 04/10] RSA Accelerator: Remove timing-sensitive optimisations Avoid potentially leaking timing information about number of bits set in MPI values. --- components/mbedtls/port/esp_bignum.c | 37 +++++++++++++--------------- 1 file changed, 17 insertions(+), 20 deletions(-) diff --git a/components/mbedtls/port/esp_bignum.c b/components/mbedtls/port/esp_bignum.c index 55e70c47ac..0a835c9e8d 100644 --- a/components/mbedtls/port/esp_bignum.c +++ b/components/mbedtls/port/esp_bignum.c @@ -97,15 +97,21 @@ static void esp_mpi_release_hardware( void ) /* Number of words used to hold 'mpi', rounded up to nearest 16 words (512 bits) to match hardware support. - Note that mpi->N (size of memory buffer) may be higher than this + Note that mpi->n (size of memory buffer) may be higher than this number, if the high bits are mostly zeroes. + + This implementation may cause the caller to leak a small amount of + timing information when an operation is performed (length of a + given mpi value, rounded to nearest 512 bits), but not all mbedTLS + RSA operations succeed if we use mpi->N as-is (buffers are too long). */ static inline size_t hardware_words_needed(const mbedtls_mpi *mpi) { - size_t res; - for(res = mpi->n; res > 0; res-- ) { - if( mpi->p[res - 1] != 0 ) - break; + size_t res = 1; + for(size_t i = 0; i < mpi->n; i++) { + if( mpi->p[i] != 0 ) { + res = i + 1; + } } res = (res + 0xF) & ~0xF; return res; @@ -135,23 +141,14 @@ static inline void mpi_to_mem_block(uint32_t mem_base, const mbedtls_mpi *mpi, s static inline int mem_block_to_mpi(mbedtls_mpi *x, uint32_t mem_base, int num_words) { int ret = 0; - size_t x_n = x->n; - /* this code is written in non-intuitive way, to only grow the - result if it is absolutely necessary - ie if all the high bits - are zero, the bignum won't be grown to fit them. */ - for(int i = num_words - 1; i >= 0; i--) { - uint32_t value = REG_READ(mem_base + i * 4); - if(value != 0 && x_n <= i) { - MBEDTLS_MPI_CHK( mbedtls_mpi_grow(x, i+1) ); - x_n = i+1; - } - if(x_n > i) { - x->p[i] = value; - } + MBEDTLS_MPI_CHK( mbedtls_mpi_grow(x, num_words) ); + + for(int i = 0; i < num_words; i++) { + x->p[i] = REG_READ(mem_base + i * 4); } - /* Zero any remaining limbs in the bignum, if the buffer was - always bigger than num_words */ + /* Zero any remaining limbs in the bignum, if the buffer is bigger + than num_words */ for(size_t i = num_words; i < x->n; i++) { x->p[i] = 0; } From 6b687b43f48252649af40f26ccca1e1599b65296 Mon Sep 17 00:00:00 2001 From: Dong Heng Date: Wed, 16 Nov 2016 20:37:51 +0800 Subject: [PATCH 05/10] mbedtls hardware RSA: Fix "mbedtls_mpi_exp_mod" hardware calculations --- components/mbedtls/port/esp_bignum.c | 158 ++++++++++++++++++++++++++- 1 file changed, 154 insertions(+), 4 deletions(-) diff --git a/components/mbedtls/port/esp_bignum.c b/components/mbedtls/port/esp_bignum.c index 0a835c9e8d..401d3fc24b 100644 --- a/components/mbedtls/port/esp_bignum.c +++ b/components/mbedtls/port/esp_bignum.c @@ -53,12 +53,11 @@ void mbedtls_mpi_printf(const char *name, const mbedtls_mpi *X) static char buf[1024]; size_t n; memset(buf, 0, sizeof(buf)); - printf("%s = 0x", name); mbedtls_mpi_write_string(X, 16, buf, sizeof(buf)-1, &n); if(n) { - puts(buf); + ESP_LOGI(TAG, "%s = 0x%s", name, buf); } else { - puts("TOOLONG"); + ESP_LOGI(TAG, "TOOLONG"); } } @@ -278,6 +277,7 @@ int esp_mpi_mul_mpi_mod(mbedtls_mpi *Z, const mbedtls_mpi *X, const mbedtls_mpi /* * Sliding-window exponentiation: Z = X^Y mod M (HAC 14.85) */ + #if 0 int mbedtls_mpi_exp_mod( mbedtls_mpi* Z, const mbedtls_mpi* X, const mbedtls_mpi* Y, const mbedtls_mpi* M, mbedtls_mpi* _RR ) { int ret; @@ -336,6 +336,155 @@ int mbedtls_mpi_exp_mod( mbedtls_mpi* Z, const mbedtls_mpi* X, const mbedtls_mpi return ret; } +#else + +/** + * There is a need for the value of integer N' such that B^-1(B-1)-N^-1N'=1, + * where B^-1(B-1) mod N=1. Actually, only the least significant part of + * N' is needed, hence the definition N0'=N' mod b. We reproduce below the + * simple algorithm from an article by Dusse and Kaliski to efficiently + * find N0' from N0 and b + */ +static mbedtls_mpi_uint modular_inverse(const mbedtls_mpi *M) +{ + int i; + uint64_t t = 1; + uint64_t two_2_i_minus_1 = 2; /* 2^(i-1) */ + uint64_t two_2_i = 4; /* 2^i */ + uint64_t N = M->p[0]; + + for (i = 2; i <= 32; i++) { + if ((mbedtls_mpi_uint) N * t % two_2_i >= two_2_i_minus_1) { + t += two_2_i_minus_1; + } + + two_2_i_minus_1 <<= 1; + two_2_i <<= 1; + } + + return (mbedtls_mpi_uint)(UINT32_MAX - t + 1); +} + +static int bignum_param_init(const mbedtls_mpi *M, mbedtls_mpi *_RR, mbedtls_mpi *r, mbedtls_mpi_uint *Mi, size_t num_words) +{ + int ret = 0; + size_t num_bits; + mbedtls_mpi RR; + + /* Calculate number of bits */ + num_bits = num_words * 32; + ESP_LOGI(TAG, "num_bits = %d\n", num_bits); + + /* + * R = b^n where b = 2^32, n=num_words, + * R = 2^N (where N=num_bits) + * RR(R^2) = 2^(2*N) (where N=num_bits) + * + * r = RR(R^2) mod M + * + * Get the RR(RR == r) value from up level if RR and RR->p is not NULL + */ + ESP_LOGI(TAG, "r = RR(R^2) mod M\n"); + if (_RR == NULL || _RR->p == NULL) { + ESP_LOGI(TAG, "RR(R^2) = 2^(2*N) (where N=num_bits)\n"); + mbedtls_mpi_init(&RR); + MBEDTLS_MPI_CHK(mbedtls_mpi_set_bit(&RR, num_bits * 2, 1)); + mbedtls_mpi_printf("RR", &RR); + + MBEDTLS_MPI_CHK(mbedtls_mpi_mod_mpi(r, &RR, M)); + + if (_RR != NULL) + memcpy(_RR, r, sizeof( mbedtls_mpi ) ); + } else { + memcpy(r, _RR, sizeof( mbedtls_mpi ) ); + } + mbedtls_mpi_printf("r", r); + + *Mi = modular_inverse(M); + +cleanup: + mbedtls_mpi_free(&RR); + + return ret; +} + +static void bignum_param_deinit(mbedtls_mpi *_RR, mbedtls_mpi *r) +{ + if (_RR == NULL || _RR->p == NULL) + mbedtls_mpi_free(r); +} + +/* + * Sliding-window exponentiation: Z = X^Y mod M (HAC 14.85) + */ +int mbedtls_mpi_exp_mod( mbedtls_mpi* Z, const mbedtls_mpi* X, const mbedtls_mpi* Y, const mbedtls_mpi* M, mbedtls_mpi* _RR ) +{ + int ret = 0; + size_t z_words = hardware_words_needed(Z); + size_t x_words = hardware_words_needed(X); + size_t y_words = hardware_words_needed(Y); + size_t m_words = hardware_words_needed(M); + size_t num_words; + + mbedtls_mpi r; + mbedtls_mpi_uint Mi = 0; + + /* "all numbers must be the same length", so choose longest number + as cardinal length of operation... + */ + num_words = z_words; + if (x_words > num_words) { + num_words = x_words; + } + if (y_words > num_words) { + num_words = y_words; + } + if (m_words > num_words) { + num_words = m_words; + } + ESP_LOGI(TAG, "num_words = %d # %d, %d, %d\n", num_words, x_words, y_words, m_words); + + if (num_words * 32 > 4096) + return MBEDTLS_ERR_MPI_NOT_ACCEPTABLE; + + mbedtls_mpi_init(&r); + ret = bignum_param_init(M, _RR, &r, &Mi, num_words); + if (ret != 0) { + return ret; + } + + mbedtls_mpi_printf("X",X); + mbedtls_mpi_printf("Y",Y); + + esp_mpi_acquire_hardware(); + + /* "mode" register loaded with number of 512-bit blocks, minus 1 */ + REG_WRITE(RSA_MODEXP_MODE_REG, (num_words / 16) - 1); + + /* Load M, X, Rinv, M-prime (M-prime is mod 2^32) */ + mpi_to_mem_block(RSA_MEM_X_BLOCK_BASE, X, num_words); + mpi_to_mem_block(RSA_MEM_Y_BLOCK_BASE, Y, num_words); + mpi_to_mem_block(RSA_MEM_M_BLOCK_BASE, M, num_words); + mpi_to_mem_block(RSA_MEM_RB_BLOCK_BASE, &r, num_words); + REG_WRITE(RSA_M_DASH_REG, Mi); + + execute_op(RSA_START_MODEXP_REG); + + ret = mem_block_to_mpi(Z, RSA_MEM_Z_BLOCK_BASE, num_words); + + esp_mpi_release_hardware(); + + mbedtls_mpi_printf("Z",Z); + ESP_LOGI(TAG, "print (Z == (X ** Y) %% M)\n"); + + bignum_param_deinit(_RR, &r); + + return ret; +} + + +#endif + #endif /* MBEDTLS_MPI_EXP_MOD_ALT */ @@ -385,7 +534,7 @@ static int modular_op_prepare(const mbedtls_mpi *X, const mbedtls_mpi *M, size_t /* Block of debugging data, output suitable to paste into Python TODO remove */ - mbedtls_mpi_printf("R", &RR); + mbedtls_mpi_printf("RR", &RR); mbedtls_mpi_printf("M", M); mbedtls_mpi_printf("Rinv", &Rinv); mbedtls_mpi_printf("Mprime", &Mprime); @@ -463,6 +612,7 @@ int mbedtls_mpi_mul_mpi( mbedtls_mpi *Z, const mbedtls_mpi *X, const mbedtls_mpi multiplication doesn't have the same restriction, so result is simply the number of bits in X plus number of bits in in Y.) */ + //ESP_LOGE(TAG, "INFO: %d bit result (%d bits * %d bits)\n", words_z * 32, mbedtls_mpi_bitlen(X), mbedtls_mpi_bitlen(Y)); if (words_mult * 32 > 2048) { /* Calculate new length of Z */ words_z = words_x + words_y; From f87be70d51e60ea62ed5d16cf93563e000b96702 Mon Sep 17 00:00:00 2001 From: Angus Gratton Date: Fri, 18 Nov 2016 13:44:37 +1100 Subject: [PATCH 06/10] mbedtls hardware RSA: Combine methods for calculating M' & r inverse Remove redundant gcd calculation, use consistent terminology. Also remove leftover debugging code --- components/mbedtls/port/esp_bignum.c | 586 +++++++++------------------ 1 file changed, 191 insertions(+), 395 deletions(-) diff --git a/components/mbedtls/port/esp_bignum.c b/components/mbedtls/port/esp_bignum.c index 401d3fc24b..076234b55b 100644 --- a/components/mbedtls/port/esp_bignum.c +++ b/components/mbedtls/port/esp_bignum.c @@ -39,46 +39,10 @@ static const char *TAG = "bignum"; #if defined(MBEDTLS_MPI_MUL_MPI_ALT) || defined(MBEDTLS_MPI_EXP_MOD_ALT) -/* Constants from mbedTLS bignum.c */ -#define ciL (sizeof(mbedtls_mpi_uint)) /* chars in limb */ -#define biL (ciL << 3) /* bits in limb */ - static _lock_t mpi_lock; -/* Temporary debugging function to print an MPI number to - stdout. Happens to be in a format compatible with Python. -*/ -void mbedtls_mpi_printf(const char *name, const mbedtls_mpi *X) -{ - static char buf[1024]; - size_t n; - memset(buf, 0, sizeof(buf)); - mbedtls_mpi_write_string(X, 16, buf, sizeof(buf)-1, &n); - if(n) { - ESP_LOGI(TAG, "%s = 0x%s", name, buf); - } else { - ESP_LOGI(TAG, "TOOLONG"); - } -} - -/* Temporary debug function to dump a memory block's contents to stdout - TODO remove - */ -static void __attribute__((unused)) dump_memory_block(const char *label, uint32_t addr) -{ - printf("Dumping %s @ %08x\n", label, addr); - for(int i = 0; i < (4096 / 8); i += 4) { - if(i % 32 == 0) { - printf("\n %04x:", i); - } - printf("%08x ", REG_READ(addr + i)); - } - printf("Done\n"); -} - /* At the moment these hardware locking functions aren't exposed publically - for MPI. If you want to use the ROM bigint functions and co-exist with mbedTLS, - please raise a feature request. + for MPI. If you want to use the ROM bigint functions and co-exist with mbedTLS, please raise a feature request. */ static void esp_mpi_acquire_hardware( void ) { @@ -116,6 +80,14 @@ static inline size_t hardware_words_needed(const mbedtls_mpi *mpi) return res; } +/* Convert number of bits to number of words, rounded up to nearest + 512 bit (16 word) block count. +*/ +static inline size_t bits_to_hardware_words(size_t num_bits) +{ + return ((num_bits + 511) / 512) * 16; +} + /* Copy mbedTLS MPI bignum 'mpi' to hardware memory block at 'mem_base'. If num_words is higher than the number of words in the bignum then @@ -156,194 +128,14 @@ static inline int mem_block_to_mpi(mbedtls_mpi *x, uint32_t mem_base, int num_wo return ret; } -/* Given a & b, determine u & v such that - - gcd(a,b) = d = au - bv - - This is suitable for calculating values for montgomery multiplication: - - gcd(R, M) = R * Rinv - M * Mprime = 1 - - Conditions which must be true: - - argument 'a' (R) is a power of 2. - - argument 'b' (M) is odd. - - Underlying algorithm comes from: - http://www.hackersdelight.org/hdcodetxt/mont64.c.txt - http://www.ucl.ac.uk/~ucahcjm/combopt/ext_gcd_python_programs.pdf - */ -static void extended_binary_gcd(const mbedtls_mpi *a, const mbedtls_mpi *b, - mbedtls_mpi *u, mbedtls_mpi *v) -{ - mbedtls_mpi a_, ta; - - /* These checks degrade performance, TODO remove them... */ - assert(b->p[0] & 1); - assert(mbedtls_mpi_bitlen(a) == mbedtls_mpi_lsb(a)+1); - assert(mbedtls_mpi_cmp_mpi(a, b) > 0); - - mbedtls_mpi_lset(u, 1); - mbedtls_mpi_lset(v, 0); - - /* 'a' needs to be half its real value for this algorithm - TODO see if we can halve the number in the caller to avoid - allocating a bignum here. - */ - mbedtls_mpi_init(&a_); - mbedtls_mpi_copy(&a_, a); - mbedtls_mpi_shift_r(&a_, 1); - - mbedtls_mpi_init(&ta); - mbedtls_mpi_copy(&ta, &a_); - - //mbedtls_mpi_printf("a", &a_); - //mbedtls_mpi_printf("b", b); - - /* Loop invariant: - 2*ta = u*2*a - v*b. - - Loop until ta == 0 - */ - while (mbedtls_mpi_cmp_int(&ta, 0) != 0) { - //mbedtls_mpi_printf("ta", &ta); - //mbedtls_mpi_printf("u", u); - //mbedtls_mpi_printf("v", v); - //printf("2*ta == u*2*a - v*b\n"); - - mbedtls_mpi_shift_r(&ta, 1); - if (mbedtls_mpi_get_bit(u, 0) == 0) { - // Remove common factor of 2 in u & v - mbedtls_mpi_shift_r(u, 1); - mbedtls_mpi_shift_r(v, 1); - } - else { - /* u = (u + b) >> 1 */ - mbedtls_mpi_add_mpi(u, u, b); - mbedtls_mpi_shift_r(u, 1); - /* v = (v - a) >> 1 */ - mbedtls_mpi_shift_r(v, 1); - mbedtls_mpi_add_mpi(v, v, &a_); - } - } - mbedtls_mpi_free(&ta); - mbedtls_mpi_free(&a_); -} - -/* Execute RSA operation. op_reg specifies which 'START' register - to write to. -*/ -static inline void execute_op(uint32_t op_reg) -{ - /* Clear interrupt status, start operation */ - REG_WRITE(RSA_INTERRUPT_REG, 1); - REG_WRITE(op_reg, 1); - - /* TODO: use interrupt instead of busywaiting */ - while(REG_READ(RSA_INTERRUPT_REG) != 1) - { } - - /* clear the interrupt */ - REG_WRITE(RSA_INTERRUPT_REG, 1); -} - -/* Sub-stages of modulo multiplication/exponentiation operations */ -static int modular_op_prepare(const mbedtls_mpi *X, const mbedtls_mpi *M, size_t num_words); -inline static int modular_multiply_finish(mbedtls_mpi *Z, const mbedtls_mpi *X, const mbedtls_mpi *Y, size_t num_words); - -/* Z = (X * Y) mod M - - Not an mbedTLS function - */ -int esp_mpi_mul_mpi_mod(mbedtls_mpi *Z, const mbedtls_mpi *X, const mbedtls_mpi *Y, const mbedtls_mpi *M) -{ - int ret; - size_t num_words = hardware_words_needed(M); - - /* Calculate and load the first stage montgomery multiplication */ - MBEDTLS_MPI_CHK( modular_op_prepare(X, M, num_words) ); - - execute_op(RSA_MULT_START_REG); - - MBEDTLS_MPI_CHK( modular_multiply_finish(Z, X, Y, num_words) ); - - esp_mpi_release_hardware(); - - cleanup: - return ret; -} - -#if defined(MBEDTLS_MPI_EXP_MOD_ALT) - -/* - * Sliding-window exponentiation: Z = X^Y mod M (HAC 14.85) - */ - #if 0 -int mbedtls_mpi_exp_mod( mbedtls_mpi* Z, const mbedtls_mpi* X, const mbedtls_mpi* Y, const mbedtls_mpi* M, mbedtls_mpi* _RR ) -{ - int ret; - size_t z_words = hardware_words_needed(Z); - size_t x_words = hardware_words_needed(X); - size_t y_words = hardware_words_needed(Y); - size_t m_words = hardware_words_needed(M); - size_t num_words; - - mbedtls_mpi_printf("X",X); - mbedtls_mpi_printf("Y",Y); - mbedtls_mpi_printf("M",M); - - /* "all numbers must be the same length", so choose longest number - as cardinal length of operation... - */ - num_words = z_words; - if (x_words > num_words) { - num_words = x_words; - } - if (y_words > num_words) { - num_words = y_words; - } - if (m_words > num_words) { - num_words = m_words; - } - printf("num_words = %d # %d, %d, %d\n", num_words, x_words, y_words, m_words); - - /* TODO: _RR parameter currently ignored */ - - ret = modular_op_prepare(X, M, num_words); - if (ret != 0) { - return ret; - } - - mpi_to_mem_block(RSA_MEM_Y_BLOCK_BASE, Y, num_words); - - //dump_memory_block("X_BLOCK", RSA_MEM_X_BLOCK_BASE); - //dump_memory_block("Y_BLOCK", RSA_MEM_Y_BLOCK_BASE); - //dump_memory_block("M_BLOCK", RSA_MEM_M_BLOCK_BASE); - - REG_WRITE(RSA_MODEXP_MODE_REG, (num_words / 16) - 1); - - execute_op(RSA_START_MODEXP_REG); - - //dump_memory_block("Z_BLOCK", RSA_MEM_Z_BLOCK_BASE); - - /* TODO: only need to read m_words not num_words, provided result is correct... */ - ret = mem_block_to_mpi(Z, RSA_MEM_Z_BLOCK_BASE, num_words); - - esp_mpi_release_hardware(); - - mbedtls_mpi_printf("Z",Z); - printf("print (Z == (X ** Y) %% M)\n"); - - return ret; -} - -#else /** - * There is a need for the value of integer N' such that B^-1(B-1)-N^-1N'=1, - * where B^-1(B-1) mod N=1. Actually, only the least significant part of - * N' is needed, hence the definition N0'=N' mod b. We reproduce below the - * simple algorithm from an article by Dusse and Kaliski to efficiently - * find N0' from N0 and b + * + * There is a need for the value of integer N' such that B^-1(B-1)-N^-1N'=1, + * where B^-1(B-1) mod N=1. Actually, only the least significant part of + * N' is needed, hence the definition N0'=N' mod b. We reproduce below the + * simple algorithm from an article by Dusse and Kaliski to efficiently + * find N0' from N0 and b */ static mbedtls_mpi_uint modular_inverse(const mbedtls_mpi *M) { @@ -365,59 +157,104 @@ static mbedtls_mpi_uint modular_inverse(const mbedtls_mpi *M) return (mbedtls_mpi_uint)(UINT32_MAX - t + 1); } -static int bignum_param_init(const mbedtls_mpi *M, mbedtls_mpi *_RR, mbedtls_mpi *r, mbedtls_mpi_uint *Mi, size_t num_words) +/* Calculate Rinv = RR^2 mod M, where: + * + * R = b^n where b = 2^32, n=num_words, + * R = 2^N (where N=num_bits) + * RR = R^2 = 2^(2*N) (where N=num_bits=num_words*32) + * + * This calculation is computationally expensive (mbedtls_mpi_mod_mpi) + * so caller should cache the result where possible. + * + * DO NOT call this function while holding esp_mpi_acquire_hardware(). + * + */ +static int calculate_rinv(mbedtls_mpi *Rinv, const mbedtls_mpi *M, int num_words) { - int ret = 0; - size_t num_bits; + int ret; + size_t num_bits = num_words * 32; mbedtls_mpi RR; + mbedtls_mpi_init(&RR); + MBEDTLS_MPI_CHK(mbedtls_mpi_set_bit(&RR, num_bits * 2, 1)); + MBEDTLS_MPI_CHK(mbedtls_mpi_mod_mpi(Rinv, &RR, M)); - /* Calculate number of bits */ - num_bits = num_words * 32; - ESP_LOGI(TAG, "num_bits = %d\n", num_bits); - - /* - * R = b^n where b = 2^32, n=num_words, - * R = 2^N (where N=num_bits) - * RR(R^2) = 2^(2*N) (where N=num_bits) - * - * r = RR(R^2) mod M - * - * Get the RR(RR == r) value from up level if RR and RR->p is not NULL - */ - ESP_LOGI(TAG, "r = RR(R^2) mod M\n"); - if (_RR == NULL || _RR->p == NULL) { - ESP_LOGI(TAG, "RR(R^2) = 2^(2*N) (where N=num_bits)\n"); - mbedtls_mpi_init(&RR); - MBEDTLS_MPI_CHK(mbedtls_mpi_set_bit(&RR, num_bits * 2, 1)); - mbedtls_mpi_printf("RR", &RR); - - MBEDTLS_MPI_CHK(mbedtls_mpi_mod_mpi(r, &RR, M)); - - if (_RR != NULL) - memcpy(_RR, r, sizeof( mbedtls_mpi ) ); - } else { - memcpy(r, _RR, sizeof( mbedtls_mpi ) ); - } - mbedtls_mpi_printf("r", r); - - *Mi = modular_inverse(M); - -cleanup: + cleanup: mbedtls_mpi_free(&RR); - return ret; } -static void bignum_param_deinit(mbedtls_mpi *_RR, mbedtls_mpi *r) + +/* Execute RSA operation. op_reg specifies which 'START' register + to write to. +*/ +static inline void execute_op(uint32_t op_reg) { - if (_RR == NULL || _RR->p == NULL) - mbedtls_mpi_free(r); + /* Clear interrupt status, start operation */ + REG_WRITE(RSA_INTERRUPT_REG, 1); + REG_WRITE(op_reg, 1); + + /* TODO: use interrupt instead of busywaiting */ + while(REG_READ(RSA_INTERRUPT_REG) != 1) + { } + + /* clear the interrupt */ + REG_WRITE(RSA_INTERRUPT_REG, 1); } +/* Sub-stages of modulo multiplication/exponentiation operations */ +inline static int modular_multiply_finish(mbedtls_mpi *Z, const mbedtls_mpi *X, const mbedtls_mpi *Y, size_t num_words); + +/* Z = (X * Y) mod M + + Not an mbedTLS function +*/ +int esp_mpi_mul_mpi_mod(mbedtls_mpi *Z, const mbedtls_mpi *X, const mbedtls_mpi *Y, const mbedtls_mpi *M) +{ + int ret; + size_t num_words = hardware_words_needed(M); + mbedtls_mpi Rinv; + mbedtls_mpi_uint Mprime; + + /* Calculate and load the first stage montgomery multiplication */ + mbedtls_mpi_init(&Rinv); + MBEDTLS_MPI_CHK(calculate_rinv(&Rinv, M, num_words)); + Mprime = modular_inverse(M); + + esp_mpi_acquire_hardware(); + + /* Load M, X, Rinv, Mprime (Mprime is mod 2^32) */ + mpi_to_mem_block(RSA_MEM_M_BLOCK_BASE, M, num_words); + mpi_to_mem_block(RSA_MEM_X_BLOCK_BASE, X, num_words); + mpi_to_mem_block(RSA_MEM_RB_BLOCK_BASE, &Rinv, num_words); + REG_WRITE(RSA_M_DASH_REG, (uint32_t)Mprime); + + /* "mode" register loaded with number of 512-bit blocks, minus 1 */ + REG_WRITE(RSA_MULT_MODE_REG, (num_words / 16) - 1); + + /* Execute first stage montgomery multiplication */ + execute_op(RSA_MULT_START_REG); + + /* execute second stage */ + MBEDTLS_MPI_CHK( modular_multiply_finish(Z, X, Y, num_words) ); + + esp_mpi_release_hardware(); + + cleanup: + mbedtls_mpi_free(&Rinv); + return ret; +} + +#if defined(MBEDTLS_MPI_EXP_MOD_ALT) + /* * Sliding-window exponentiation: Z = X^Y mod M (HAC 14.85) + * + * _Rinv is optional pre-calculated version of Rinv (via calculate_rinv()). + * + * (See RSA Accelerator section in Technical Reference for more about Mprime, Rinv) + * */ -int mbedtls_mpi_exp_mod( mbedtls_mpi* Z, const mbedtls_mpi* X, const mbedtls_mpi* Y, const mbedtls_mpi* M, mbedtls_mpi* _RR ) +int mbedtls_mpi_exp_mod( mbedtls_mpi* Z, const mbedtls_mpi* X, const mbedtls_mpi* Y, const mbedtls_mpi* M, mbedtls_mpi* _Rinv ) { int ret = 0; size_t z_words = hardware_words_needed(Z); @@ -426,8 +263,9 @@ int mbedtls_mpi_exp_mod( mbedtls_mpi* Z, const mbedtls_mpi* X, const mbedtls_mpi size_t m_words = hardware_words_needed(M); size_t num_words; - mbedtls_mpi r; - mbedtls_mpi_uint Mi = 0; + mbedtls_mpi Rinv_new; /* used if _Rinv == NULL */ + mbedtls_mpi *Rinv; /* points to _Rinv (if not NULL) othwerwise &RR_new */ + mbedtls_mpi_uint Mprime; /* "all numbers must be the same length", so choose longest number as cardinal length of operation... @@ -442,19 +280,24 @@ int mbedtls_mpi_exp_mod( mbedtls_mpi* Z, const mbedtls_mpi* X, const mbedtls_mpi if (m_words > num_words) { num_words = m_words; } - ESP_LOGI(TAG, "num_words = %d # %d, %d, %d\n", num_words, x_words, y_words, m_words); - if (num_words * 32 > 4096) + if (num_words * 32 > 4096) { return MBEDTLS_ERR_MPI_NOT_ACCEPTABLE; - - mbedtls_mpi_init(&r); - ret = bignum_param_init(M, _RR, &r, &Mi, num_words); - if (ret != 0) { - return ret; } - mbedtls_mpi_printf("X",X); - mbedtls_mpi_printf("Y",Y); + /* Determine RR pointer, either _RR for cached value + or local RR_new */ + if (_Rinv == NULL) { + mbedtls_mpi_init(&Rinv_new); + Rinv = &Rinv_new; + } else { + Rinv = _Rinv; + } + if (Rinv->p == NULL) { + MBEDTLS_MPI_CHK(calculate_rinv(Rinv, M, num_words)); + } + + Mprime = modular_inverse(M); esp_mpi_acquire_hardware(); @@ -465,8 +308,8 @@ int mbedtls_mpi_exp_mod( mbedtls_mpi* Z, const mbedtls_mpi* X, const mbedtls_mpi mpi_to_mem_block(RSA_MEM_X_BLOCK_BASE, X, num_words); mpi_to_mem_block(RSA_MEM_Y_BLOCK_BASE, Y, num_words); mpi_to_mem_block(RSA_MEM_M_BLOCK_BASE, M, num_words); - mpi_to_mem_block(RSA_MEM_RB_BLOCK_BASE, &r, num_words); - REG_WRITE(RSA_M_DASH_REG, Mi); + mpi_to_mem_block(RSA_MEM_RB_BLOCK_BASE, Rinv, num_words); + REG_WRITE(RSA_M_DASH_REG, Mprime); execute_op(RSA_START_MODEXP_REG); @@ -474,91 +317,16 @@ int mbedtls_mpi_exp_mod( mbedtls_mpi* Z, const mbedtls_mpi* X, const mbedtls_mpi esp_mpi_release_hardware(); - mbedtls_mpi_printf("Z",Z); - ESP_LOGI(TAG, "print (Z == (X ** Y) %% M)\n"); - - bignum_param_deinit(_RR, &r); - - return ret; -} - - -#endif - -#endif /* MBEDTLS_MPI_EXP_MOD_ALT */ - - -/* The common parts of modulo multiplication and modular sliding - * window exponentiation: - * - * @param X first multiplication factor and/or base of exponent. - * @param M modulo value for result - * @param num_words size of modulo operation, in words (limbs). - * Should already be rounded up to a multiple of 16 words (512 bits) & range checked. - * - * Steps: - * Calculate Rinv & Mprime based on M & num_words - * Load all coefficients to memory - * Set mode register - * - * @note This function calls esp_mpi_acquire_hardware. If successful, - * returns 0 and it becomes the callers responsibility to call - * esp_mpi_release_hardware(). If failure is returned, the caller does - * not need to call esp_mpi_release_hardware(). - */ -static int modular_op_prepare(const mbedtls_mpi *X, const mbedtls_mpi *M, size_t num_words) -{ - int ret = 0; - mbedtls_mpi RR, Rinv, Mprime; - size_t num_bits; - - /* Calculate number of bits */ - num_bits = num_words * 32; - - if(num_bits > 4096) { - return MBEDTLS_ERR_MPI_NOT_ACCEPTABLE; + cleanup: + if (_Rinv == NULL) { + mbedtls_mpi_free(&Rinv_new); } - /* Rinv & Mprime are calculated via extended binary gcd - algorithm, see references on extended_binary_gcd() above. - */ - mbedtls_mpi_init(&Rinv); - mbedtls_mpi_init(&RR); - mbedtls_mpi_init(&Mprime); - - mbedtls_mpi_set_bit(&RR, num_bits, 1); /* R = b^n where b = 2^32, n=num_words, - ie R = 2^N (where N=num_bits) */ - /* calculate Rinv & Mprime */ - extended_binary_gcd(&RR, M, &Rinv, &Mprime); - - /* Block of debugging data, output suitable to paste into Python - TODO remove - */ - mbedtls_mpi_printf("RR", &RR); - mbedtls_mpi_printf("M", M); - mbedtls_mpi_printf("Rinv", &Rinv); - mbedtls_mpi_printf("Mprime", &Mprime); - printf("print (R * Rinv - M * Mprime == 1)\n"); - printf("print (Rinv == (R * R) %% M)\n"); - - esp_mpi_acquire_hardware(); - - /* Load M, X, Rinv, M-prime (M-prime is mod 2^32) */ - mpi_to_mem_block(RSA_MEM_M_BLOCK_BASE, M, num_words); - mpi_to_mem_block(RSA_MEM_X_BLOCK_BASE, X, num_words); - mpi_to_mem_block(RSA_MEM_RB_BLOCK_BASE, &Rinv, num_words); - REG_WRITE(RSA_M_DASH_REG, Mprime.p[0]); - - /* "mode" register loaded with number of 512-bit blocks, minus 1 */ - REG_WRITE(RSA_MULT_MODE_REG, (num_words / 16) - 1); - - mbedtls_mpi_free(&Rinv); - mbedtls_mpi_free(&RR); - mbedtls_mpi_free(&Mprime); - return ret; } +#endif /* MBEDTLS_MPI_EXP_MOD_ALT */ + /* Second & final step of a modular multiply - load second multiplication * factor Y, run the multiply, read back the result into Z. * @@ -594,17 +362,43 @@ static int mpi_mult_mpi_failover_mod_mult(mbedtls_mpi *Z, const mbedtls_mpi *X, int mbedtls_mpi_mul_mpi( mbedtls_mpi *Z, const mbedtls_mpi *X, const mbedtls_mpi *Y ) { int ret; - size_t words_x, words_y, words_mult, words_z; + size_t bits_x, bits_y, words_x, words_y, words_mult, words_z; /* Count words needed for X & Y in hardware */ - words_x = hardware_words_needed(X); - words_y = hardware_words_needed(Y); + bits_x = mbedtls_mpi_bitlen(X); + bits_y = mbedtls_mpi_bitlen(Y); + /* Convert bit counts to words, rounded up to 512-bit + (16 word) blocks */ + words_x = bits_to_hardware_words(bits_x); + words_y = bits_to_hardware_words(bits_y); + + /* Short-circuit eval if either argument is 0 or 1. + + This is needed as the mpi modular division + argument will sometimes call in here when one + argument is too large for the hardware unit, but the other + argument is zero or one. + + This leaks some timing information, although overall there is a + lot less timing variation than a software MPI approach. + */ + if (bits_x == 0 || bits_y == 0) { + mbedtls_mpi_lset(Z, 0); + return 0; + } + if (bits_x == 1) { + return mbedtls_mpi_copy(Z, Y); + } + if (bits_y == 1) { + return mbedtls_mpi_copy(Z, X); + } words_mult = (words_x > words_y ? words_x : words_y); /* Result Z has to have room for double the larger factor */ words_z = words_mult * 2; + /* If either factor is over 2048 bits, we can't use the standard hardware multiplier (it assumes result is double longest factor, and result is max 4096 bits.) @@ -612,12 +406,11 @@ int mbedtls_mpi_mul_mpi( mbedtls_mpi *Z, const mbedtls_mpi *X, const mbedtls_mpi multiplication doesn't have the same restriction, so result is simply the number of bits in X plus number of bits in in Y.) */ - //ESP_LOGE(TAG, "INFO: %d bit result (%d bits * %d bits)\n", words_z * 32, mbedtls_mpi_bitlen(X), mbedtls_mpi_bitlen(Y)); if (words_mult * 32 > 2048) { /* Calculate new length of Z */ - words_z = words_x + words_y; + words_z = bits_to_hardware_words(bits_x + bits_y); if (words_z * 32 > 4096) { - ESP_LOGE(TAG, "ERROR: %d bit result (%d bits * %d bits) too large for hardware unit\n", words_z * 32, mbedtls_mpi_bitlen(X), mbedtls_mpi_bitlen(Y)); + ESP_LOGE(TAG, "ERROR: %d bit result %d bits * %d bits too large for hardware unit\n", words_z * 32, bits_x, bits_y); return MBEDTLS_ERR_MPI_NOT_ACCEPTABLE; } else { @@ -640,7 +433,7 @@ int mbedtls_mpi_mul_mpi( mbedtls_mpi *Z, const mbedtls_mpi *X, const mbedtls_mpi /* "mode" register loaded with number of 512-bit blocks in result, plus 7 (for range 9-12). (this is ((N~ / 32) - 1) + 8)) - */ + */ REG_WRITE(RSA_MULT_MODE_REG, (words_z / 16) + 7); execute_op(RSA_MULT_START_REG); @@ -656,54 +449,57 @@ int mbedtls_mpi_mul_mpi( mbedtls_mpi *Z, const mbedtls_mpi *X, const mbedtls_mpi } /* Special-case of mbedtls_mpi_mult_mpi(), where we use hardware montgomery mod - multiplication to solve the case where A or B are >2048 bits so - can't use the standard multiplication method. + multiplication to calculate an mbedtls_mpi_mult_mpi result where either + A or B are >2048 bits so can't use the standard multiplication method. - This case is simpler than esp_mpi_mul_mpi_mod() as we control the arguments: + Result (A bits + B bits) must still be less than 4096 bits. + + This case is simpler than the general case modulo multiply of + esp_mpi_mul_mpi_mod() because we can control the other arguments: * Modulus is chosen with M=(2^num_bits - 1) (ie M=R-1), so output - isn't actually modulo anything. - * Therefore of of M' and Rinv are predictable as follows: - M' = 1 - Rinv = 1 + isn't actually modulo anything. + * Mprime and Rinv are therefore predictable as follows: + Mprime = 1 + Rinv = 1 - (See RSA Accelerator section in Technical Reference * - extended_binary_gcd() function above for more about M', Rinv) + (See RSA Accelerator section in Technical Reference for more about Mprime, Rinv) */ static int mpi_mult_mpi_failover_mod_mult(mbedtls_mpi *Z, const mbedtls_mpi *X, const mbedtls_mpi *Y, size_t num_words) - { - int ret = 0; +{ + int ret = 0; - /* Load coefficients to hardware */ - esp_mpi_acquire_hardware(); + /* Load coefficients to hardware */ + esp_mpi_acquire_hardware(); - /* M = 2^num_words - 1, so block is entirely FF */ - for(int i = 0; i < num_words; i++) { - REG_WRITE(RSA_MEM_M_BLOCK_BASE + i * 4, UINT32_MAX); - } - /* Mprime = 1 */ - REG_WRITE(RSA_M_DASH_REG, 1); + /* M = 2^num_words - 1, so block is entirely FF */ + for(int i = 0; i < num_words; i++) { + REG_WRITE(RSA_MEM_M_BLOCK_BASE + i * 4, UINT32_MAX); + } + /* Mprime = 1 */ + REG_WRITE(RSA_M_DASH_REG, 1); - /* "mode" register loaded with number of 512-bit blocks, minus 1 */ - REG_WRITE(RSA_MULT_MODE_REG, (num_words / 16) - 1); + /* "mode" register loaded with number of 512-bit blocks, minus 1 */ + REG_WRITE(RSA_MULT_MODE_REG, (num_words / 16) - 1); - /* Load X */ - mpi_to_mem_block(RSA_MEM_X_BLOCK_BASE, X, num_words); + /* Load X */ + mpi_to_mem_block(RSA_MEM_X_BLOCK_BASE, X, num_words); - /* Rinv = 1 */ - REG_WRITE(RSA_MEM_RB_BLOCK_BASE, 1); - for(int i = 1; i < num_words; i++) { - REG_WRITE(RSA_MEM_RB_BLOCK_BASE + i * 4, 0); - } + /* Rinv = 1 */ + REG_WRITE(RSA_MEM_RB_BLOCK_BASE, 1); + for(int i = 1; i < num_words; i++) { + REG_WRITE(RSA_MEM_RB_BLOCK_BASE + i * 4, 0); + } - execute_op(RSA_MULT_START_REG); + execute_op(RSA_MULT_START_REG); - MBEDTLS_MPI_CHK( modular_multiply_finish(Z, X, Y, num_words) ); + /* finish the modular multiplication */ + MBEDTLS_MPI_CHK( modular_multiply_finish(Z, X, Y, num_words) ); - esp_mpi_release_hardware(); + esp_mpi_release_hardware(); cleanup: - return ret; + return ret; } #endif /* MBEDTLS_MPI_MUL_MPI_ALT */ From 68d370542a19c3d1b406a2ba6da3b8f088e541ae Mon Sep 17 00:00:00 2001 From: Angus Gratton Date: Fri, 18 Nov 2016 14:26:02 +1100 Subject: [PATCH 07/10] mbedtls hardware RSA: Put into menuconfig, squash warnings All combinations of enabling/disabling hardware acceleration no longer show unused warnings. --- components/mbedtls/Kconfig | 21 ++++++++++++++++++- components/mbedtls/library/bignum.c | 8 ++++++- components/mbedtls/port/esp_bignum.c | 8 ++++--- .../mbedtls/port/include/mbedtls/esp_config.h | 4 ++++ 4 files changed, 36 insertions(+), 5 deletions(-) diff --git a/components/mbedtls/Kconfig b/components/mbedtls/Kconfig index 60facc7d27..bb2aa8f6a3 100644 --- a/components/mbedtls/Kconfig +++ b/components/mbedtls/Kconfig @@ -22,7 +22,7 @@ config MBEDTLS_SSL_MAX_CONTENT_LEN config MBEDTLS_DEBUG bool "Enable mbedTLS debugging" - default "no" + default n help Enable mbedTLS debugging functions. @@ -34,4 +34,23 @@ config MBEDTLS_DEBUG functionality. See the "https_request_main" example for a sample function which connects the two together. +config MBEDTLS_HARDWARE_AES + bool "Enable hardware AES acceleration" + default y + help + Enable hardware accelerated AES encryption & decryption. + +config MBEDTLS_HARDWARE_MPI + bool "Enable hardware MPI (bignum) acceleration" + default y + help + Enable hardware accelerated multiple precision integer operations. + + Hardware accelerated multiplication, modulo multiplication, + and modular exponentiation for up to 4096 bit results. + + These operations are used by RSA. + + + endmenu diff --git a/components/mbedtls/library/bignum.c b/components/mbedtls/library/bignum.c index e739bc1d38..04ff9e07b2 100644 --- a/components/mbedtls/library/bignum.c +++ b/components/mbedtls/library/bignum.c @@ -1092,6 +1092,8 @@ int mbedtls_mpi_sub_int( mbedtls_mpi *X, const mbedtls_mpi *A, mbedtls_mpi_sint return( mbedtls_mpi_sub_mpi( X, A, &_B ) ); } +#if !defined(MBEDTLS_MPI_MUL_MPI_ALT) || !defined(MBEDTLS_MPI_EXP_MOD_ALT) + /* * Helper for mbedtls_mpi multiplication */ @@ -1103,6 +1105,7 @@ static */ __attribute__ ((noinline)) #endif + void mpi_mul_hlp( size_t i, mbedtls_mpi_uint *s, mbedtls_mpi_uint *d, mbedtls_mpi_uint b ) { mbedtls_mpi_uint c = 0, t = 0; @@ -1164,6 +1167,8 @@ void mpi_mul_hlp( size_t i, mbedtls_mpi_uint *s, mbedtls_mpi_uint *d, mbedtls_mp while( c != 0 ); } +#endif + #if !defined(MBEDTLS_MPI_MUL_MPI_ALT) /* * Baseline multiplication: X = A * B (HAC 14.12) @@ -1526,6 +1531,8 @@ int mbedtls_mpi_mod_int( mbedtls_mpi_uint *r, const mbedtls_mpi *A, mbedtls_mpi_ return( 0 ); } +#if !defined(MBEDTLS_MPI_EXP_MOD_ALT) + /* * Fast Montgomery initialization (thanks to Tom St Denis) */ @@ -1600,7 +1607,6 @@ static int mpi_montred( mbedtls_mpi *A, const mbedtls_mpi *N, mbedtls_mpi_uint m return( mpi_montmul( A, &U, N, mm, T ) ); } -#if !defined(MBEDTLS_MPI_EXP_MOD_ALT) /* * Sliding-window exponentiation: X = A^E mod N (HAC 14.85) */ diff --git a/components/mbedtls/port/esp_bignum.c b/components/mbedtls/port/esp_bignum.c index 076234b55b..11a1c63973 100644 --- a/components/mbedtls/port/esp_bignum.c +++ b/components/mbedtls/port/esp_bignum.c @@ -35,8 +35,6 @@ #include "freertos/FreeRTOS.h" #include "freertos/task.h" -static const char *TAG = "bignum"; - #if defined(MBEDTLS_MPI_MUL_MPI_ALT) || defined(MBEDTLS_MPI_EXP_MOD_ALT) static _lock_t mpi_lock; @@ -330,6 +328,8 @@ int mbedtls_mpi_exp_mod( mbedtls_mpi* Z, const mbedtls_mpi* X, const mbedtls_mpi /* Second & final step of a modular multiply - load second multiplication * factor Y, run the multiply, read back the result into Z. * + * Called from both mbedtls_mpi_exp_mod and mbedtls_mpi_mod_mpi. + * * @param Z result value * @param X first multiplication factor (used to set sign of result). * @param Y second multiplication factor. @@ -338,7 +338,7 @@ int mbedtls_mpi_exp_mod( mbedtls_mpi* Z, const mbedtls_mpi* X, const mbedtls_mpi * * Caller must have already called esp_mpi_acquire_hardware(). */ -inline static int modular_multiply_finish(mbedtls_mpi *Z, const mbedtls_mpi *X, const mbedtls_mpi *Y, size_t num_words) +static int modular_multiply_finish(mbedtls_mpi *Z, const mbedtls_mpi *X, const mbedtls_mpi *Y, size_t num_words) { int ret; /* Load Y to X input memory block, rerun */ @@ -356,6 +356,8 @@ inline static int modular_multiply_finish(mbedtls_mpi *Z, const mbedtls_mpi *X, #if defined(MBEDTLS_MPI_MUL_MPI_ALT) /* MBEDTLS_MPI_MUL_MPI_ALT */ +static const char *TAG = "bignum"; + static int mpi_mult_mpi_failover_mod_mult(mbedtls_mpi *Z, const mbedtls_mpi *X, const mbedtls_mpi *Y, size_t num_words); /* Z = X * Y */ diff --git a/components/mbedtls/port/include/mbedtls/esp_config.h b/components/mbedtls/port/include/mbedtls/esp_config.h index 2b47d84ea4..db87c6ef31 100644 --- a/components/mbedtls/port/include/mbedtls/esp_config.h +++ b/components/mbedtls/port/include/mbedtls/esp_config.h @@ -239,7 +239,9 @@ /* The following units have ESP32 hardware support, uncommenting each _ALT macro will use the hardware-accelerated implementation. */ +#ifdef CONFIG_MBEDTLS_HARDWARE_AES #define MBEDTLS_AES_ALT +#endif /* Currently hardware SHA does not work with TLS handshake, due to concurrency issue. Internal TW#7111. */ @@ -251,8 +253,10 @@ Uncommenting these macros will use the hardware-accelerated implementations. */ +#ifdef CONFIG_MBEDTLS_HARDWARE_MPI #define MBEDTLS_MPI_EXP_MOD_ALT #define MBEDTLS_MPI_MUL_MPI_ALT +#endif /** * \def MBEDTLS_MD2_PROCESS_ALT From 36f29017b62336ecf58d52b91f88b3c11ab36fe8 Mon Sep 17 00:00:00 2001 From: Angus Gratton Date: Fri, 18 Nov 2016 15:53:00 +1100 Subject: [PATCH 08/10] mbedtls hardware bignum: Support "RSA" interrupt for end of operation Allows CPU to do other things which bignum operation is in progress. --- components/mbedtls/Kconfig | 16 ++++++++++ components/mbedtls/port/esp_bignum.c | 48 +++++++++++++++++++++++++--- 2 files changed, 60 insertions(+), 4 deletions(-) diff --git a/components/mbedtls/Kconfig b/components/mbedtls/Kconfig index bb2aa8f6a3..d6e2a2dcb7 100644 --- a/components/mbedtls/Kconfig +++ b/components/mbedtls/Kconfig @@ -51,6 +51,22 @@ config MBEDTLS_HARDWARE_MPI These operations are used by RSA. +config MBEDTLS_MPI_USE_INTERRUPT + bool "Use interrupt for MPI operations" + depends on MBEDTLS_HARDWARE_MPI + default y + help + Use an interrupt to coordinate MPI operations. + This allows other code to run on the CPU while an MPI operation is pending. + Otherwise the CPU busy-waits. + +config MBEDTLS_MPI_INTERRUPT_NUM + int "MPI Interrupt number" + depends on MBEDTLS_MPI_USE_INTERRUPT + default 18 + help + CPU interrupt number for MPI interrupt to connect to. Must be otherwise unused. + Eventually this assignment will be handled automatically at runtime. endmenu diff --git a/components/mbedtls/port/esp_bignum.c b/components/mbedtls/port/esp_bignum.c index 11a1c63973..2133e7cae6 100644 --- a/components/mbedtls/port/esp_bignum.c +++ b/components/mbedtls/port/esp_bignum.c @@ -31,12 +31,43 @@ #include "soc/hwcrypto_reg.h" #include "esp_system.h" #include "esp_log.h" +#include "esp_intr.h" +#include "esp_attr.h" #include "freertos/FreeRTOS.h" #include "freertos/task.h" +#include "freertos/semphr.h" #if defined(MBEDTLS_MPI_MUL_MPI_ALT) || defined(MBEDTLS_MPI_EXP_MOD_ALT) +static const char *TAG = "bignum"; + +#if defined(CONFIG_MBEDTLS_MPI_USE_INTERRUPT) +static SemaphoreHandle_t op_complete_sem; + +static IRAM_ATTR void rsa_complete_isr(void *arg) +{ + BaseType_t higher_woken; + REG_WRITE(RSA_INTERRUPT_REG, 1); + xSemaphoreGiveFromISR(op_complete_sem, &higher_woken); + if (higher_woken) { + portYIELD_FROM_ISR(); + } +} + +static void rsa_isr_initialise() +{ + if (op_complete_sem == NULL) { + op_complete_sem = xSemaphoreCreateBinary(); + intr_matrix_set(xPortGetCoreID(), ETS_RSA_INTR_SOURCE, CONFIG_MBEDTLS_MPI_INTERRUPT_NUM); + xt_set_interrupt_handler(CONFIG_MBEDTLS_MPI_INTERRUPT_NUM, &rsa_complete_isr, NULL); + xthal_set_intclear(1 << CONFIG_MBEDTLS_MPI_INTERRUPT_NUM); + xt_ints_on(1 << CONFIG_MBEDTLS_MPI_INTERRUPT_NUM); + } +} + +#endif /* CONFIG_MBEDTLS_MPI_USE_INTERRUPT */ + static _lock_t mpi_lock; /* At the moment these hardware locking functions aren't exposed publically @@ -47,6 +78,9 @@ static void esp_mpi_acquire_hardware( void ) /* newlib locks lazy initialize on ESP-IDF */ _lock_acquire(&mpi_lock); ets_bigint_enable(); +#ifdef CONFIG_MBEDTLS_MPI_USE_INTERRUPT + rsa_isr_initialise(); +#endif } static void esp_mpi_release_hardware( void ) @@ -187,13 +221,21 @@ static int calculate_rinv(mbedtls_mpi *Rinv, const mbedtls_mpi *M, int num_words */ static inline void execute_op(uint32_t op_reg) { - /* Clear interrupt status, start operation */ + /* Clear interrupt status */ REG_WRITE(RSA_INTERRUPT_REG, 1); + REG_WRITE(op_reg, 1); - /* TODO: use interrupt instead of busywaiting */ +#ifdef CONFIG_MBEDTLS_MPI_USE_INTERRUPT + if (!xSemaphoreTake(op_complete_sem, 2000 / portTICK_PERIOD_MS)) { + ESP_LOGE(TAG, "Timed out waiting for RSA operation (op_reg 0x%x int_reg 0x%x)", + op_reg, REG_READ(RSA_INTERRUPT_REG)); + abort(); /* indicates a fundamental problem with driver */ + } +#else while(REG_READ(RSA_INTERRUPT_REG) != 1) { } +#endif /* clear the interrupt */ REG_WRITE(RSA_INTERRUPT_REG, 1); @@ -356,8 +398,6 @@ static int modular_multiply_finish(mbedtls_mpi *Z, const mbedtls_mpi *X, const m #if defined(MBEDTLS_MPI_MUL_MPI_ALT) /* MBEDTLS_MPI_MUL_MPI_ALT */ -static const char *TAG = "bignum"; - static int mpi_mult_mpi_failover_mod_mult(mbedtls_mpi *Z, const mbedtls_mpi *X, const mbedtls_mpi *Y, size_t num_words); /* Z = X * Y */ From 1cc0b3000b699e9ea451a18c9f255da1e590a62d Mon Sep 17 00:00:00 2001 From: Angus Gratton Date: Fri, 18 Nov 2016 16:38:22 +1100 Subject: [PATCH 09/10] mbedtls hardware bignum: Expose ESP-only bignum API in wrapper mbedtls/bignum.h --- components/mbedtls/port/esp_bignum.c | 14 +--- components/mbedtls/port/include/aes_alt.h | 3 +- .../mbedtls/port/include/mbedtls/bignum.h | 78 +++++++++++++++++++ components/mbedtls/port/include/sha1_alt.h | 17 +++- components/mbedtls/port/include/sha256_alt.h | 18 +++-- components/mbedtls/port/include/sha512_alt.h | 19 +++-- 6 files changed, 121 insertions(+), 28 deletions(-) create mode 100644 components/mbedtls/port/include/mbedtls/bignum.h diff --git a/components/mbedtls/port/esp_bignum.c b/components/mbedtls/port/esp_bignum.c index 2133e7cae6..68fd8bec98 100644 --- a/components/mbedtls/port/esp_bignum.c +++ b/components/mbedtls/port/esp_bignum.c @@ -26,7 +26,6 @@ #include #include #include "mbedtls/bignum.h" -#include "mbedtls/bn_mul.h" #include "rom/bigint.h" #include "soc/hwcrypto_reg.h" #include "esp_system.h" @@ -38,9 +37,7 @@ #include "freertos/task.h" #include "freertos/semphr.h" -#if defined(MBEDTLS_MPI_MUL_MPI_ALT) || defined(MBEDTLS_MPI_EXP_MOD_ALT) - -static const char *TAG = "bignum"; +static const __attribute__((unused)) char *TAG = "bignum"; #if defined(CONFIG_MBEDTLS_MPI_USE_INTERRUPT) static SemaphoreHandle_t op_complete_sem; @@ -70,10 +67,7 @@ static void rsa_isr_initialise() static _lock_t mpi_lock; -/* At the moment these hardware locking functions aren't exposed publically - for MPI. If you want to use the ROM bigint functions and co-exist with mbedTLS, please raise a feature request. -*/ -static void esp_mpi_acquire_hardware( void ) +void esp_mpi_acquire_hardware( void ) { /* newlib locks lazy initialize on ESP-IDF */ _lock_acquire(&mpi_lock); @@ -83,7 +77,7 @@ static void esp_mpi_acquire_hardware( void ) #endif } -static void esp_mpi_release_hardware( void ) +void esp_mpi_release_hardware( void ) { ets_bigint_disable(); _lock_release(&mpi_lock); @@ -546,5 +540,3 @@ static int mpi_mult_mpi_failover_mod_mult(mbedtls_mpi *Z, const mbedtls_mpi *X, #endif /* MBEDTLS_MPI_MUL_MPI_ALT */ -#endif /* MBEDTLS_MPI_MUL_MPI_ALT || MBEDTLS_MPI_EXP_MOD_ALT */ - diff --git a/components/mbedtls/port/include/aes_alt.h b/components/mbedtls/port/include/aes_alt.h index 7161b282c2..d4da6ca878 100644 --- a/components/mbedtls/port/include/aes_alt.h +++ b/components/mbedtls/port/include/aes_alt.h @@ -20,7 +20,6 @@ * * */ - #ifndef AES_ALT_H #define AES_ALT_H @@ -56,4 +55,4 @@ typedef esp_aes_context mbedtls_aes_context; } #endif -#endif /* aes.h */ +#endif diff --git a/components/mbedtls/port/include/mbedtls/bignum.h b/components/mbedtls/port/include/mbedtls/bignum.h new file mode 100644 index 0000000000..23cd56348a --- /dev/null +++ b/components/mbedtls/port/include/mbedtls/bignum.h @@ -0,0 +1,78 @@ +// Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#ifndef __ESP_MBEDTLS_BIGNUM_H__ +#define __ESP_MBEDTLS_BIGNUM_H__ + +#include_next "mbedtls/bignum.h" + +/** + * This is a wrapper for the main mbedtls/bignum.h. This wrapper + * provides a few additional ESP32-only functions. + * + * This is because we don't set MBEDTLS_BIGNUM_ALT in the same way we + * do for AES, SHA, etc. Because we still use most of the bignum.h + * implementation and just replace a few hardware accelerated + * functions (see MBEDTLS_MPI_EXP_MOD_ALT & MBEDTLS_MPI_MUL_MPI_ALT in + * esp_config.h). + * + * @note Unlike the other hardware accelerator support functions in esp32/hwcrypto, there is no + * generic "hwcrypto/bignum.h" header for using these functions without mbedTLS. The reason for this + * is that all of the function implementations depend strongly upon the mbedTLS MPI implementation. + */ + +/** + * @brief Lock access to RSA Accelerator (MPI/bignum operations) + * + * RSA Accelerator hardware unit can only be used by one + * consumer at a time. + * + * @note This function is non-recursive (do not call it twice from the + * same task.) + * + * @note You do not need to call this if you are using the mbedTLS bignum.h + * API or esp_mpi_xxx functions. This function is only needed if you + * want to call ROM RSA functions or access the registers directly. + * + */ +void esp_mpi_acquire_hardware(void); + +/** + * @brief Unlock access to RSA Accelerator (MPI/bignum operations) + * + * Has to be called once for each call to esp_mpi_acquire_hardware(). + * + * @note You do not need to call this if you are using the mbedTLS bignum.h + * API or esp_mpi_xxx functions. This function is only needed if you + * want to call ROM RSA functions or access the registers directly. + */ +void esp_mpi_release_hardware(void); + +/* @brief MPI modular mupltiplication function + * + * Calculates Z = (X * Y) mod M using MPI hardware acceleration. + * + * This is not part of the standard mbedTLS bignum API. + * + * @note All of X, Y & Z should be less than 4096 bit long or an error is returned. + * + * @param Z Result bignum, should be pre-initialised with mbedtls_mpi_init(). + * @param X First multiplication argument. + * @param Y Second multiplication argument. + * @param M Modulus value for result. + * + * @return 0 on success, mbedTLS MPI error codes on failure. + */ +int esp_mpi_mul_mpi_mod(mbedtls_mpi *Z, const mbedtls_mpi *X, const mbedtls_mpi *Y, const mbedtls_mpi *M); + +#endif diff --git a/components/mbedtls/port/include/sha1_alt.h b/components/mbedtls/port/include/sha1_alt.h index 60297b9fbf..f5e69b3f95 100644 --- a/components/mbedtls/port/include/sha1_alt.h +++ b/components/mbedtls/port/include/sha1_alt.h @@ -1,7 +1,16 @@ -/* - * copyright (c) 2010 - 2012 Espressif System - * - */ +// Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. #ifndef _SHA1_ALT_H_ #define _SHA1_ALT_H_ diff --git a/components/mbedtls/port/include/sha256_alt.h b/components/mbedtls/port/include/sha256_alt.h index 6d9986b3a1..143d8c75e1 100644 --- a/components/mbedtls/port/include/sha256_alt.h +++ b/components/mbedtls/port/include/sha256_alt.h @@ -1,8 +1,16 @@ -/* - * copyright (c) 2010 - 2012 Espressif System - * - */ +// Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. #ifndef _SHA256_ALT_H_ #define _SHA256_ALT_H_ @@ -30,4 +38,4 @@ typedef esp_sha_context mbedtls_sha256_context; } #endif -#endif /* sha256.h */ +#endif diff --git a/components/mbedtls/port/include/sha512_alt.h b/components/mbedtls/port/include/sha512_alt.h index 241f2be3b3..8044b42754 100644 --- a/components/mbedtls/port/include/sha512_alt.h +++ b/components/mbedtls/port/include/sha512_alt.h @@ -1,9 +1,16 @@ -/* - * copyright (c) 2010 - 2012 Espressif System - * - * esf Link List Descriptor - */ +// Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. #ifndef _SHA512_ALT_H_ #define _SHA512_ALT_H_ @@ -30,4 +37,4 @@ typedef esp_sha_context mbedtls_sha512_context; } #endif -#endif /* sha512.h */ +#endif From 1d4775558814b6673d59078cc37c8c65274b5d49 Mon Sep 17 00:00:00 2001 From: Angus Gratton Date: Mon, 21 Nov 2016 18:08:22 +1100 Subject: [PATCH 10/10] mbedtls hardware bignum: Use memcpy instead of REG_WRITE/REG_READ in a loop Removes memory barriers for better performance, thanks Ivan for pointing this out. Manually unrolling the loop further seemed like diminishing returns. --- components/mbedtls/port/esp_bignum.c | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/components/mbedtls/port/esp_bignum.c b/components/mbedtls/port/esp_bignum.c index 68fd8bec98..7570820e3b 100644 --- a/components/mbedtls/port/esp_bignum.c +++ b/components/mbedtls/port/esp_bignum.c @@ -121,12 +121,16 @@ static inline size_t bits_to_hardware_words(size_t num_bits) */ static inline void mpi_to_mem_block(uint32_t mem_base, const mbedtls_mpi *mpi, size_t num_words) { - for(size_t i = 0; i < mpi->n && i < num_words; i++) { - REG_WRITE(mem_base + i * 4, mpi->p[i]); - } - for(size_t i = mpi->n; i < num_words; i++) { - REG_WRITE(mem_base + i * 4, 0); - } + uint32_t *pbase = (uint32_t *)mem_base; + uint32_t copy_words = num_words < mpi->n ? num_words : mpi->n; + + /* Copy MPI data to memory block registers */ + memcpy(pbase, mpi->p, copy_words * 4); + + /* Zero any remaining memory block data */ + bzero(pbase + copy_words, (num_words - copy_words) * 4); + + /* Note: not executing memw here, can do it before we start a bignum operation */ } /* Read mbedTLS MPI bignum back from hardware memory block. @@ -141,15 +145,16 @@ static inline int mem_block_to_mpi(mbedtls_mpi *x, uint32_t mem_base, int num_wo MBEDTLS_MPI_CHK( mbedtls_mpi_grow(x, num_words) ); - for(int i = 0; i < num_words; i++) { - x->p[i] = REG_READ(mem_base + i * 4); - } + /* Copy data from memory block registers */ + memcpy(x->p, (uint32_t *)mem_base, num_words * 4); + /* Zero any remaining limbs in the bignum, if the buffer is bigger than num_words */ for(size_t i = num_words; i < x->n; i++) { x->p[i] = 0; } + asm volatile ("memw"); cleanup: return ret; } @@ -218,6 +223,9 @@ static inline void execute_op(uint32_t op_reg) /* Clear interrupt status */ REG_WRITE(RSA_INTERRUPT_REG, 1); + /* Note: above REG_WRITE includes a memw, so we know any writes + to the memory blocks are also complete. */ + REG_WRITE(op_reg, 1); #ifdef CONFIG_MBEDTLS_MPI_USE_INTERRUPT