From 97a64bcc7cdfe3928e62136e6c8a23b5b94b2a8d Mon Sep 17 00:00:00 2001 From: toddouska Date: Mon, 12 Sep 2016 12:03:37 -0700 Subject: [PATCH 1/3] remove unique aes last round Te table, pre fetch Te tables during software aes encrypt --- wolfcrypt/src/aes.c | 193 ++++++++++++++++++-------------------------- 1 file changed, 78 insertions(+), 115 deletions(-) diff --git a/wolfcrypt/src/aes.c b/wolfcrypt/src/aes.c index 501bd430e..8705c38b5 100644 --- a/wolfcrypt/src/aes.c +++ b/wolfcrypt/src/aes.c @@ -507,7 +507,7 @@ static const word32 rcon[] = { /* for 128-bit blocks, Rijndael never uses more than 10 rcon values */ }; -static const word32 Te[5][256] = { +static const word32 Te[4][256] = { { 0xc66363a5U, 0xf87c7c84U, 0xee777799U, 0xf67b7b8dU, 0xfff2f20dU, 0xd66b6bbdU, 0xde6f6fb1U, 0x91c5c554U, @@ -771,72 +771,6 @@ static const word32 Te[5][256] = { 0xbfbfda65U, 0xe6e631d7U, 0x4242c684U, 0x6868b8d0U, 0x4141c382U, 0x9999b029U, 0x2d2d775aU, 0x0f0f111eU, 0xb0b0cb7bU, 0x5454fca8U, 0xbbbbd66dU, 0x16163a2cU, -}, -{ - 0x63636363U, 0x7c7c7c7cU, 0x77777777U, 0x7b7b7b7bU, - 0xf2f2f2f2U, 0x6b6b6b6bU, 0x6f6f6f6fU, 0xc5c5c5c5U, - 0x30303030U, 0x01010101U, 0x67676767U, 0x2b2b2b2bU, - 0xfefefefeU, 0xd7d7d7d7U, 0xababababU, 0x76767676U, - 0xcacacacaU, 0x82828282U, 0xc9c9c9c9U, 0x7d7d7d7dU, - 0xfafafafaU, 0x59595959U, 0x47474747U, 0xf0f0f0f0U, - 0xadadadadU, 0xd4d4d4d4U, 0xa2a2a2a2U, 0xafafafafU, - 0x9c9c9c9cU, 0xa4a4a4a4U, 0x72727272U, 0xc0c0c0c0U, - 0xb7b7b7b7U, 0xfdfdfdfdU, 0x93939393U, 0x26262626U, - 0x36363636U, 0x3f3f3f3fU, 0xf7f7f7f7U, 0xccccccccU, - 0x34343434U, 0xa5a5a5a5U, 0xe5e5e5e5U, 0xf1f1f1f1U, - 0x71717171U, 0xd8d8d8d8U, 0x31313131U, 0x15151515U, - 0x04040404U, 0xc7c7c7c7U, 0x23232323U, 0xc3c3c3c3U, - 0x18181818U, 0x96969696U, 0x05050505U, 0x9a9a9a9aU, - 0x07070707U, 0x12121212U, 0x80808080U, 0xe2e2e2e2U, - 0xebebebebU, 0x27272727U, 0xb2b2b2b2U, 0x75757575U, - 0x09090909U, 0x83838383U, 0x2c2c2c2cU, 0x1a1a1a1aU, - 0x1b1b1b1bU, 0x6e6e6e6eU, 0x5a5a5a5aU, 0xa0a0a0a0U, - 0x52525252U, 0x3b3b3b3bU, 0xd6d6d6d6U, 0xb3b3b3b3U, - 0x29292929U, 0xe3e3e3e3U, 0x2f2f2f2fU, 0x84848484U, - 0x53535353U, 0xd1d1d1d1U, 0x00000000U, 0xededededU, - 0x20202020U, 0xfcfcfcfcU, 0xb1b1b1b1U, 0x5b5b5b5bU, - 0x6a6a6a6aU, 0xcbcbcbcbU, 0xbebebebeU, 0x39393939U, - 0x4a4a4a4aU, 0x4c4c4c4cU, 0x58585858U, 0xcfcfcfcfU, - 0xd0d0d0d0U, 0xefefefefU, 0xaaaaaaaaU, 0xfbfbfbfbU, - 0x43434343U, 0x4d4d4d4dU, 0x33333333U, 0x85858585U, - 0x45454545U, 0xf9f9f9f9U, 0x02020202U, 0x7f7f7f7fU, - 0x50505050U, 0x3c3c3c3cU, 0x9f9f9f9fU, 0xa8a8a8a8U, - 0x51515151U, 0xa3a3a3a3U, 0x40404040U, 0x8f8f8f8fU, - 0x92929292U, 0x9d9d9d9dU, 0x38383838U, 0xf5f5f5f5U, - 0xbcbcbcbcU, 0xb6b6b6b6U, 0xdadadadaU, 0x21212121U, - 0x10101010U, 0xffffffffU, 0xf3f3f3f3U, 0xd2d2d2d2U, - 0xcdcdcdcdU, 0x0c0c0c0cU, 0x13131313U, 0xececececU, - 0x5f5f5f5fU, 0x97979797U, 0x44444444U, 0x17171717U, - 0xc4c4c4c4U, 0xa7a7a7a7U, 0x7e7e7e7eU, 0x3d3d3d3dU, - 0x64646464U, 0x5d5d5d5dU, 0x19191919U, 0x73737373U, - 0x60606060U, 0x81818181U, 0x4f4f4f4fU, 0xdcdcdcdcU, - 0x22222222U, 0x2a2a2a2aU, 0x90909090U, 0x88888888U, - 0x46464646U, 0xeeeeeeeeU, 0xb8b8b8b8U, 0x14141414U, - 0xdedededeU, 0x5e5e5e5eU, 0x0b0b0b0bU, 0xdbdbdbdbU, - 0xe0e0e0e0U, 0x32323232U, 0x3a3a3a3aU, 0x0a0a0a0aU, - 0x49494949U, 0x06060606U, 0x24242424U, 0x5c5c5c5cU, - 0xc2c2c2c2U, 0xd3d3d3d3U, 0xacacacacU, 0x62626262U, - 0x91919191U, 0x95959595U, 0xe4e4e4e4U, 0x79797979U, - 0xe7e7e7e7U, 0xc8c8c8c8U, 0x37373737U, 0x6d6d6d6dU, - 0x8d8d8d8dU, 0xd5d5d5d5U, 0x4e4e4e4eU, 0xa9a9a9a9U, - 0x6c6c6c6cU, 0x56565656U, 0xf4f4f4f4U, 0xeaeaeaeaU, - 0x65656565U, 0x7a7a7a7aU, 0xaeaeaeaeU, 0x08080808U, - 0xbabababaU, 0x78787878U, 0x25252525U, 0x2e2e2e2eU, - 0x1c1c1c1cU, 0xa6a6a6a6U, 0xb4b4b4b4U, 0xc6c6c6c6U, - 0xe8e8e8e8U, 0xddddddddU, 0x74747474U, 0x1f1f1f1fU, - 0x4b4b4b4bU, 0xbdbdbdbdU, 0x8b8b8b8bU, 0x8a8a8a8aU, - 0x70707070U, 0x3e3e3e3eU, 0xb5b5b5b5U, 0x66666666U, - 0x48484848U, 0x03030303U, 0xf6f6f6f6U, 0x0e0e0e0eU, - 0x61616161U, 0x35353535U, 0x57575757U, 0xb9b9b9b9U, - 0x86868686U, 0xc1c1c1c1U, 0x1d1d1d1dU, 0x9e9e9e9eU, - 0xe1e1e1e1U, 0xf8f8f8f8U, 0x98989898U, 0x11111111U, - 0x69696969U, 0xd9d9d9d9U, 0x8e8e8e8eU, 0x94949494U, - 0x9b9b9b9bU, 0x1e1e1e1eU, 0x87878787U, 0xe9e9e9e9U, - 0xcecececeU, 0x55555555U, 0x28282828U, 0xdfdfdfdfU, - 0x8c8c8c8cU, 0xa1a1a1a1U, 0x89898989U, 0x0d0d0d0dU, - 0xbfbfbfbfU, 0xe6e6e6e6U, 0x42424242U, 0x68686868U, - 0x41414141U, 0x99999999U, 0x2d2d2d2dU, 0x0f0f0f0fU, - 0xb0b0b0b0U, 0x54545454U, 0xbbbbbbbbU, 0x16161616U, } }; @@ -1337,6 +1271,32 @@ static int AES_set_decrypt_key(const unsigned char* userKey, const int bits, #if defined(HAVE_AES_CBC) || defined(WOLFSSL_AES_DIRECT) ||\ defined(HAVE_AESGCM) + +#ifndef WC_CACHE_LINE_SZ + #if defined(__x86_64__) || defined(_M_X64) || (__ILP32__ >= 1) + #define WC_CACHE_LINE_SZ 64 + #else + /* default cache line size */ + #define WC_CACHE_LINE_SZ 32 + #endif +#endif + + +/* load 4 Te Tables into cache by cache line stride */ +static INLINE word32 PreFetchTe(void) +{ + word32 x = 0; + int i,j; + + for (i = 0; i < 4; i++) { + for (j = 0; j < 256; j += WC_CACHE_LINE_SZ) { + x &= Te[i][j]; + } + } + return x; +} + + static void wc_AesEncrypt(Aes* aes, const byte* inBlock, byte* outBlock) { word32 s0, s1, s2, s3; @@ -1411,6 +1371,8 @@ static void wc_AesEncrypt(Aes* aes, const byte* inBlock, byte* outBlock) s2 ^= rk[2]; s3 ^= rk[3]; + s0 |= PreFetchTe(); + /* * Nr - 1 full rounds: */ @@ -1478,28 +1440,28 @@ static void wc_AesEncrypt(Aes* aes, const byte* inBlock, byte* outBlock) */ s0 = - (Te[4][GETBYTE(t0, 3)] & 0xff000000) ^ - (Te[4][GETBYTE(t1, 2)] & 0x00ff0000) ^ - (Te[4][GETBYTE(t2, 1)] & 0x0000ff00) ^ - (Te[4][GETBYTE(t3, 0)] & 0x000000ff) ^ + (Te[2][GETBYTE(t0, 3)] & 0xff000000) ^ + (Te[3][GETBYTE(t1, 2)] & 0x00ff0000) ^ + (Te[0][GETBYTE(t2, 1)] & 0x0000ff00) ^ + (Te[1][GETBYTE(t3, 0)] & 0x000000ff) ^ rk[0]; s1 = - (Te[4][GETBYTE(t1, 3)] & 0xff000000) ^ - (Te[4][GETBYTE(t2, 2)] & 0x00ff0000) ^ - (Te[4][GETBYTE(t3, 1)] & 0x0000ff00) ^ - (Te[4][GETBYTE(t0, 0)] & 0x000000ff) ^ + (Te[2][GETBYTE(t1, 3)] & 0xff000000) ^ + (Te[3][GETBYTE(t2, 2)] & 0x00ff0000) ^ + (Te[0][GETBYTE(t3, 1)] & 0x0000ff00) ^ + (Te[1][GETBYTE(t0, 0)] & 0x000000ff) ^ rk[1]; s2 = - (Te[4][GETBYTE(t2, 3)] & 0xff000000) ^ - (Te[4][GETBYTE(t3, 2)] & 0x00ff0000) ^ - (Te[4][GETBYTE(t0, 1)] & 0x0000ff00) ^ - (Te[4][GETBYTE(t1, 0)] & 0x000000ff) ^ + (Te[2][GETBYTE(t2, 3)] & 0xff000000) ^ + (Te[3][GETBYTE(t3, 2)] & 0x00ff0000) ^ + (Te[0][GETBYTE(t0, 1)] & 0x0000ff00) ^ + (Te[1][GETBYTE(t1, 0)] & 0x000000ff) ^ rk[2]; s3 = - (Te[4][GETBYTE(t3, 3)] & 0xff000000) ^ - (Te[4][GETBYTE(t0, 2)] & 0x00ff0000) ^ - (Te[4][GETBYTE(t1, 1)] & 0x0000ff00) ^ - (Te[4][GETBYTE(t2, 0)] & 0x000000ff) ^ + (Te[2][GETBYTE(t3, 3)] & 0xff000000) ^ + (Te[3][GETBYTE(t0, 2)] & 0x00ff0000) ^ + (Te[0][GETBYTE(t1, 1)] & 0x0000ff00) ^ + (Te[1][GETBYTE(t2, 0)] & 0x000000ff) ^ rk[3]; /* write out */ @@ -1514,6 +1476,7 @@ static void wc_AesEncrypt(Aes* aes, const byte* inBlock, byte* outBlock) XMEMCPY(outBlock + sizeof(s0), &s1, sizeof(s1)); XMEMCPY(outBlock + 2 * sizeof(s0), &s2, sizeof(s2)); XMEMCPY(outBlock + 3 * sizeof(s0), &s3, sizeof(s3)); + } #endif /* HAVE_AES_CBC || WOLFSSL_AES_DIRECT || HAVE_AESGCM */ @@ -2055,10 +2018,10 @@ static void wc_AesDecrypt(Aes* aes, const byte* inBlock, byte* outBlock) { temp = rk[3]; rk[4] = rk[0] ^ - (Te[4][GETBYTE(temp, 2)] & 0xff000000) ^ - (Te[4][GETBYTE(temp, 1)] & 0x00ff0000) ^ - (Te[4][GETBYTE(temp, 0)] & 0x0000ff00) ^ - (Te[4][GETBYTE(temp, 3)] & 0x000000ff) ^ + (Te[2][GETBYTE(temp, 2)] & 0xff000000) ^ + (Te[3][GETBYTE(temp, 1)] & 0x00ff0000) ^ + (Te[0][GETBYTE(temp, 0)] & 0x0000ff00) ^ + (Te[1][GETBYTE(temp, 3)] & 0x000000ff) ^ rcon[i]; rk[5] = rk[1] ^ rk[4]; rk[6] = rk[2] ^ rk[5]; @@ -2077,10 +2040,10 @@ static void wc_AesDecrypt(Aes* aes, const byte* inBlock, byte* outBlock) { temp = rk[ 5]; rk[ 6] = rk[ 0] ^ - (Te[4][GETBYTE(temp, 2)] & 0xff000000) ^ - (Te[4][GETBYTE(temp, 1)] & 0x00ff0000) ^ - (Te[4][GETBYTE(temp, 0)] & 0x0000ff00) ^ - (Te[4][GETBYTE(temp, 3)] & 0x000000ff) ^ + (Te[2][GETBYTE(temp, 2)] & 0xff000000) ^ + (Te[3][GETBYTE(temp, 1)] & 0x00ff0000) ^ + (Te[0][GETBYTE(temp, 0)] & 0x0000ff00) ^ + (Te[1][GETBYTE(temp, 3)] & 0x000000ff) ^ rcon[i]; rk[ 7] = rk[ 1] ^ rk[ 6]; rk[ 8] = rk[ 2] ^ rk[ 7]; @@ -2100,10 +2063,10 @@ static void wc_AesDecrypt(Aes* aes, const byte* inBlock, byte* outBlock) { temp = rk[ 7]; rk[ 8] = rk[ 0] ^ - (Te[4][GETBYTE(temp, 2)] & 0xff000000) ^ - (Te[4][GETBYTE(temp, 1)] & 0x00ff0000) ^ - (Te[4][GETBYTE(temp, 0)] & 0x0000ff00) ^ - (Te[4][GETBYTE(temp, 3)] & 0x000000ff) ^ + (Te[2][GETBYTE(temp, 2)] & 0xff000000) ^ + (Te[3][GETBYTE(temp, 1)] & 0x00ff0000) ^ + (Te[0][GETBYTE(temp, 0)] & 0x0000ff00) ^ + (Te[1][GETBYTE(temp, 3)] & 0x000000ff) ^ rcon[i]; rk[ 9] = rk[ 1] ^ rk[ 8]; rk[10] = rk[ 2] ^ rk[ 9]; @@ -2112,10 +2075,10 @@ static void wc_AesDecrypt(Aes* aes, const byte* inBlock, byte* outBlock) break; temp = rk[11]; rk[12] = rk[ 4] ^ - (Te[4][GETBYTE(temp, 3)] & 0xff000000) ^ - (Te[4][GETBYTE(temp, 2)] & 0x00ff0000) ^ - (Te[4][GETBYTE(temp, 1)] & 0x0000ff00) ^ - (Te[4][GETBYTE(temp, 0)] & 0x000000ff); + (Te[2][GETBYTE(temp, 3)] & 0xff000000) ^ + (Te[3][GETBYTE(temp, 2)] & 0x00ff0000) ^ + (Te[0][GETBYTE(temp, 1)] & 0x0000ff00) ^ + (Te[1][GETBYTE(temp, 0)] & 0x000000ff); rk[13] = rk[ 5] ^ rk[12]; rk[14] = rk[ 6] ^ rk[13]; rk[15] = rk[ 7] ^ rk[14]; @@ -2147,25 +2110,25 @@ static void wc_AesDecrypt(Aes* aes, const byte* inBlock, byte* outBlock) for (i = 1; i < aes->rounds; i++) { rk += 4; rk[0] = - Td[0][Te[4][GETBYTE(rk[0], 3)] & 0xff] ^ - Td[1][Te[4][GETBYTE(rk[0], 2)] & 0xff] ^ - Td[2][Te[4][GETBYTE(rk[0], 1)] & 0xff] ^ - Td[3][Te[4][GETBYTE(rk[0], 0)] & 0xff]; + Td[0][Te[1][GETBYTE(rk[0], 3)] & 0xff] ^ + Td[1][Te[1][GETBYTE(rk[0], 2)] & 0xff] ^ + Td[2][Te[1][GETBYTE(rk[0], 1)] & 0xff] ^ + Td[3][Te[1][GETBYTE(rk[0], 0)] & 0xff]; rk[1] = - Td[0][Te[4][GETBYTE(rk[1], 3)] & 0xff] ^ - Td[1][Te[4][GETBYTE(rk[1], 2)] & 0xff] ^ - Td[2][Te[4][GETBYTE(rk[1], 1)] & 0xff] ^ - Td[3][Te[4][GETBYTE(rk[1], 0)] & 0xff]; + Td[0][Te[1][GETBYTE(rk[1], 3)] & 0xff] ^ + Td[1][Te[1][GETBYTE(rk[1], 2)] & 0xff] ^ + Td[2][Te[1][GETBYTE(rk[1], 1)] & 0xff] ^ + Td[3][Te[1][GETBYTE(rk[1], 0)] & 0xff]; rk[2] = - Td[0][Te[4][GETBYTE(rk[2], 3)] & 0xff] ^ - Td[1][Te[4][GETBYTE(rk[2], 2)] & 0xff] ^ - Td[2][Te[4][GETBYTE(rk[2], 1)] & 0xff] ^ - Td[3][Te[4][GETBYTE(rk[2], 0)] & 0xff]; + Td[0][Te[1][GETBYTE(rk[2], 3)] & 0xff] ^ + Td[1][Te[1][GETBYTE(rk[2], 2)] & 0xff] ^ + Td[2][Te[1][GETBYTE(rk[2], 1)] & 0xff] ^ + Td[3][Te[1][GETBYTE(rk[2], 0)] & 0xff]; rk[3] = - Td[0][Te[4][GETBYTE(rk[3], 3)] & 0xff] ^ - Td[1][Te[4][GETBYTE(rk[3], 2)] & 0xff] ^ - Td[2][Te[4][GETBYTE(rk[3], 1)] & 0xff] ^ - Td[3][Te[4][GETBYTE(rk[3], 0)] & 0xff]; + Td[0][Te[1][GETBYTE(rk[3], 3)] & 0xff] ^ + Td[1][Te[1][GETBYTE(rk[3], 2)] & 0xff] ^ + Td[2][Te[1][GETBYTE(rk[3], 1)] & 0xff] ^ + Td[3][Te[1][GETBYTE(rk[3], 0)] & 0xff]; } } #endif /* HAVE_AES_DECRYPT */ From c6256211d66ad7bac3357326cb08fdeaddbc3b54 Mon Sep 17 00:00:00 2001 From: toddouska Date: Mon, 12 Sep 2016 13:04:30 -0700 Subject: [PATCH 2/3] compress aes last round decrypt table, prefetch Td tables before aes decrypt rounds, prefecth compressed table before last round --- wolfcrypt/src/aes.c | 171 +++++++++++++++++++++++--------------------- 1 file changed, 88 insertions(+), 83 deletions(-) diff --git a/wolfcrypt/src/aes.c b/wolfcrypt/src/aes.c index 8705c38b5..65a5538ce 100644 --- a/wolfcrypt/src/aes.c +++ b/wolfcrypt/src/aes.c @@ -774,7 +774,7 @@ static const word32 Te[4][256] = { } }; -static const word32 Td[5][256] = { +static const word32 Td[4][256] = { { 0x51f4a750U, 0x7e416553U, 0x1a17a4c3U, 0x3a275e96U, 0x3bab6bcbU, 0x1f9d45f1U, 0xacfa58abU, 0x4be30393U, @@ -1039,75 +1039,46 @@ static const word32 Td[5][256] = { 0x1dc37216U, 0xe2250cbcU, 0x3c498b28U, 0x0d9541ffU, 0xa8017139U, 0x0cb3de08U, 0xb4e49cd8U, 0x56c19064U, 0xcb84617bU, 0x32b670d5U, 0x6c5c7448U, 0xb85742d0U, -}, -{ - 0x52525252U, 0x09090909U, 0x6a6a6a6aU, 0xd5d5d5d5U, - 0x30303030U, 0x36363636U, 0xa5a5a5a5U, 0x38383838U, - 0xbfbfbfbfU, 0x40404040U, 0xa3a3a3a3U, 0x9e9e9e9eU, - 0x81818181U, 0xf3f3f3f3U, 0xd7d7d7d7U, 0xfbfbfbfbU, - 0x7c7c7c7cU, 0xe3e3e3e3U, 0x39393939U, 0x82828282U, - 0x9b9b9b9bU, 0x2f2f2f2fU, 0xffffffffU, 0x87878787U, - 0x34343434U, 0x8e8e8e8eU, 0x43434343U, 0x44444444U, - 0xc4c4c4c4U, 0xdedededeU, 0xe9e9e9e9U, 0xcbcbcbcbU, - 0x54545454U, 0x7b7b7b7bU, 0x94949494U, 0x32323232U, - 0xa6a6a6a6U, 0xc2c2c2c2U, 0x23232323U, 0x3d3d3d3dU, - 0xeeeeeeeeU, 0x4c4c4c4cU, 0x95959595U, 0x0b0b0b0bU, - 0x42424242U, 0xfafafafaU, 0xc3c3c3c3U, 0x4e4e4e4eU, - 0x08080808U, 0x2e2e2e2eU, 0xa1a1a1a1U, 0x66666666U, - 0x28282828U, 0xd9d9d9d9U, 0x24242424U, 0xb2b2b2b2U, - 0x76767676U, 0x5b5b5b5bU, 0xa2a2a2a2U, 0x49494949U, - 0x6d6d6d6dU, 0x8b8b8b8bU, 0xd1d1d1d1U, 0x25252525U, - 0x72727272U, 0xf8f8f8f8U, 0xf6f6f6f6U, 0x64646464U, - 0x86868686U, 0x68686868U, 0x98989898U, 0x16161616U, - 0xd4d4d4d4U, 0xa4a4a4a4U, 0x5c5c5c5cU, 0xccccccccU, - 0x5d5d5d5dU, 0x65656565U, 0xb6b6b6b6U, 0x92929292U, - 0x6c6c6c6cU, 0x70707070U, 0x48484848U, 0x50505050U, - 0xfdfdfdfdU, 0xededededU, 0xb9b9b9b9U, 0xdadadadaU, - 0x5e5e5e5eU, 0x15151515U, 0x46464646U, 0x57575757U, - 0xa7a7a7a7U, 0x8d8d8d8dU, 0x9d9d9d9dU, 0x84848484U, - 0x90909090U, 0xd8d8d8d8U, 0xababababU, 0x00000000U, - 0x8c8c8c8cU, 0xbcbcbcbcU, 0xd3d3d3d3U, 0x0a0a0a0aU, - 0xf7f7f7f7U, 0xe4e4e4e4U, 0x58585858U, 0x05050505U, - 0xb8b8b8b8U, 0xb3b3b3b3U, 0x45454545U, 0x06060606U, - 0xd0d0d0d0U, 0x2c2c2c2cU, 0x1e1e1e1eU, 0x8f8f8f8fU, - 0xcacacacaU, 0x3f3f3f3fU, 0x0f0f0f0fU, 0x02020202U, - 0xc1c1c1c1U, 0xafafafafU, 0xbdbdbdbdU, 0x03030303U, - 0x01010101U, 0x13131313U, 0x8a8a8a8aU, 0x6b6b6b6bU, - 0x3a3a3a3aU, 0x91919191U, 0x11111111U, 0x41414141U, - 0x4f4f4f4fU, 0x67676767U, 0xdcdcdcdcU, 0xeaeaeaeaU, - 0x97979797U, 0xf2f2f2f2U, 0xcfcfcfcfU, 0xcecececeU, - 0xf0f0f0f0U, 0xb4b4b4b4U, 0xe6e6e6e6U, 0x73737373U, - 0x96969696U, 0xacacacacU, 0x74747474U, 0x22222222U, - 0xe7e7e7e7U, 0xadadadadU, 0x35353535U, 0x85858585U, - 0xe2e2e2e2U, 0xf9f9f9f9U, 0x37373737U, 0xe8e8e8e8U, - 0x1c1c1c1cU, 0x75757575U, 0xdfdfdfdfU, 0x6e6e6e6eU, - 0x47474747U, 0xf1f1f1f1U, 0x1a1a1a1aU, 0x71717171U, - 0x1d1d1d1dU, 0x29292929U, 0xc5c5c5c5U, 0x89898989U, - 0x6f6f6f6fU, 0xb7b7b7b7U, 0x62626262U, 0x0e0e0e0eU, - 0xaaaaaaaaU, 0x18181818U, 0xbebebebeU, 0x1b1b1b1bU, - 0xfcfcfcfcU, 0x56565656U, 0x3e3e3e3eU, 0x4b4b4b4bU, - 0xc6c6c6c6U, 0xd2d2d2d2U, 0x79797979U, 0x20202020U, - 0x9a9a9a9aU, 0xdbdbdbdbU, 0xc0c0c0c0U, 0xfefefefeU, - 0x78787878U, 0xcdcdcdcdU, 0x5a5a5a5aU, 0xf4f4f4f4U, - 0x1f1f1f1fU, 0xddddddddU, 0xa8a8a8a8U, 0x33333333U, - 0x88888888U, 0x07070707U, 0xc7c7c7c7U, 0x31313131U, - 0xb1b1b1b1U, 0x12121212U, 0x10101010U, 0x59595959U, - 0x27272727U, 0x80808080U, 0xececececU, 0x5f5f5f5fU, - 0x60606060U, 0x51515151U, 0x7f7f7f7fU, 0xa9a9a9a9U, - 0x19191919U, 0xb5b5b5b5U, 0x4a4a4a4aU, 0x0d0d0d0dU, - 0x2d2d2d2dU, 0xe5e5e5e5U, 0x7a7a7a7aU, 0x9f9f9f9fU, - 0x93939393U, 0xc9c9c9c9U, 0x9c9c9c9cU, 0xefefefefU, - 0xa0a0a0a0U, 0xe0e0e0e0U, 0x3b3b3b3bU, 0x4d4d4d4dU, - 0xaeaeaeaeU, 0x2a2a2a2aU, 0xf5f5f5f5U, 0xb0b0b0b0U, - 0xc8c8c8c8U, 0xebebebebU, 0xbbbbbbbbU, 0x3c3c3c3cU, - 0x83838383U, 0x53535353U, 0x99999999U, 0x61616161U, - 0x17171717U, 0x2b2b2b2bU, 0x04040404U, 0x7e7e7e7eU, - 0xbabababaU, 0x77777777U, 0xd6d6d6d6U, 0x26262626U, - 0xe1e1e1e1U, 0x69696969U, 0x14141414U, 0x63636363U, - 0x55555555U, 0x21212121U, 0x0c0c0c0cU, 0x7d7d7d7dU, } }; + +static const byte Td4[256] = +{ + 0x52U, 0x09U, 0x6aU, 0xd5U, 0x30U, 0x36U, 0xa5U, 0x38U, + 0xbfU, 0x40U, 0xa3U, 0x9eU, 0x81U, 0xf3U, 0xd7U, 0xfbU, + 0x7cU, 0xe3U, 0x39U, 0x82U, 0x9bU, 0x2fU, 0xffU, 0x87U, + 0x34U, 0x8eU, 0x43U, 0x44U, 0xc4U, 0xdeU, 0xe9U, 0xcbU, + 0x54U, 0x7bU, 0x94U, 0x32U, 0xa6U, 0xc2U, 0x23U, 0x3dU, + 0xeeU, 0x4cU, 0x95U, 0x0bU, 0x42U, 0xfaU, 0xc3U, 0x4eU, + 0x08U, 0x2eU, 0xa1U, 0x66U, 0x28U, 0xd9U, 0x24U, 0xb2U, + 0x76U, 0x5bU, 0xa2U, 0x49U, 0x6dU, 0x8bU, 0xd1U, 0x25U, + 0x72U, 0xf8U, 0xf6U, 0x64U, 0x86U, 0x68U, 0x98U, 0x16U, + 0xd4U, 0xa4U, 0x5cU, 0xccU, 0x5dU, 0x65U, 0xb6U, 0x92U, + 0x6cU, 0x70U, 0x48U, 0x50U, 0xfdU, 0xedU, 0xb9U, 0xdaU, + 0x5eU, 0x15U, 0x46U, 0x57U, 0xa7U, 0x8dU, 0x9dU, 0x84U, + 0x90U, 0xd8U, 0xabU, 0x00U, 0x8cU, 0xbcU, 0xd3U, 0x0aU, + 0xf7U, 0xe4U, 0x58U, 0x05U, 0xb8U, 0xb3U, 0x45U, 0x06U, + 0xd0U, 0x2cU, 0x1eU, 0x8fU, 0xcaU, 0x3fU, 0x0fU, 0x02U, + 0xc1U, 0xafU, 0xbdU, 0x03U, 0x01U, 0x13U, 0x8aU, 0x6bU, + 0x3aU, 0x91U, 0x11U, 0x41U, 0x4fU, 0x67U, 0xdcU, 0xeaU, + 0x97U, 0xf2U, 0xcfU, 0xceU, 0xf0U, 0xb4U, 0xe6U, 0x73U, + 0x96U, 0xacU, 0x74U, 0x22U, 0xe7U, 0xadU, 0x35U, 0x85U, + 0xe2U, 0xf9U, 0x37U, 0xe8U, 0x1cU, 0x75U, 0xdfU, 0x6eU, + 0x47U, 0xf1U, 0x1aU, 0x71U, 0x1dU, 0x29U, 0xc5U, 0x89U, + 0x6fU, 0xb7U, 0x62U, 0x0eU, 0xaaU, 0x18U, 0xbeU, 0x1bU, + 0xfcU, 0x56U, 0x3eU, 0x4bU, 0xc6U, 0xd2U, 0x79U, 0x20U, + 0x9aU, 0xdbU, 0xc0U, 0xfeU, 0x78U, 0xcdU, 0x5aU, 0xf4U, + 0x1fU, 0xddU, 0xa8U, 0x33U, 0x88U, 0x07U, 0xc7U, 0x31U, + 0xb1U, 0x12U, 0x10U, 0x59U, 0x27U, 0x80U, 0xecU, 0x5fU, + 0x60U, 0x51U, 0x7fU, 0xa9U, 0x19U, 0xb5U, 0x4aU, 0x0dU, + 0x2dU, 0xe5U, 0x7aU, 0x9fU, 0x93U, 0xc9U, 0x9cU, 0xefU, + 0xa0U, 0xe0U, 0x3bU, 0x4dU, 0xaeU, 0x2aU, 0xf5U, 0xb0U, + 0xc8U, 0xebU, 0xbbU, 0x3cU, 0x83U, 0x53U, 0x99U, 0x61U, + 0x17U, 0x2bU, 0x04U, 0x7eU, 0xbaU, 0x77U, 0xd6U, 0x26U, + 0xe1U, 0x69U, 0x14U, 0x63U, 0x55U, 0x21U, 0x0cU, 0x7dU, +}; + #define GETBYTE(x, y) (word32)((byte)((x) >> (8 * (y)))) @@ -1482,6 +1453,35 @@ static void wc_AesEncrypt(Aes* aes, const byte* inBlock, byte* outBlock) #ifdef HAVE_AES_DECRYPT #if defined(HAVE_AES_CBC) || defined(WOLFSSL_AES_DIRECT) + +/* load 4 Td Tables into cache by cache line stride */ +static INLINE word32 PreFetchTd(void) +{ + word32 x = 0; + int i,j; + + for (i = 0; i < 4; i++) { + for (j = 0; j < 256; j += WC_CACHE_LINE_SZ) { + x &= Td[i][j]; + } + } + return x; +} + + +/* load Td Table4 into cache by cache line stride */ +static INLINE word32 PreFetchTd4(void) +{ + word32 x = 0; + int i; + + for (i = 0; i < 256; i += WC_CACHE_LINE_SZ) { + x &= (word32)Td4[i]; + } + return x; +} + + static void wc_AesDecrypt(Aes* aes, const byte* inBlock, byte* outBlock) { word32 s0, s1, s2, s3; @@ -1538,6 +1538,8 @@ static void wc_AesDecrypt(Aes* aes, const byte* inBlock, byte* outBlock) s2 ^= rk[2]; s3 ^= rk[3]; + s0 |= PreFetchTd(); + /* * Nr - 1 full rounds: */ @@ -1602,29 +1604,32 @@ static void wc_AesDecrypt(Aes* aes, const byte* inBlock, byte* outBlock) * apply last round and * map cipher state to byte array block: */ + + s0 |= PreFetchTd4(); + s0 = - (Td[4][GETBYTE(t0, 3)] & 0xff000000) ^ - (Td[4][GETBYTE(t3, 2)] & 0x00ff0000) ^ - (Td[4][GETBYTE(t2, 1)] & 0x0000ff00) ^ - (Td[4][GETBYTE(t1, 0)] & 0x000000ff) ^ + ((word32)Td4[GETBYTE(t0, 3)] << 24) ^ + ((word32)Td4[GETBYTE(t3, 2)] << 16) ^ + ((word32)Td4[GETBYTE(t2, 1)] << 8) ^ + ((word32)Td4[GETBYTE(t1, 0)]) ^ rk[0]; s1 = - (Td[4][GETBYTE(t1, 3)] & 0xff000000) ^ - (Td[4][GETBYTE(t0, 2)] & 0x00ff0000) ^ - (Td[4][GETBYTE(t3, 1)] & 0x0000ff00) ^ - (Td[4][GETBYTE(t2, 0)] & 0x000000ff) ^ + ((word32)Td4[GETBYTE(t1, 3)] << 24) ^ + ((word32)Td4[GETBYTE(t0, 2)] << 16) ^ + ((word32)Td4[GETBYTE(t3, 1)] << 8) ^ + ((word32)Td4[GETBYTE(t2, 0)]) ^ rk[1]; s2 = - (Td[4][GETBYTE(t2, 3)] & 0xff000000) ^ - (Td[4][GETBYTE(t1, 2)] & 0x00ff0000) ^ - (Td[4][GETBYTE(t0, 1)] & 0x0000ff00) ^ - (Td[4][GETBYTE(t3, 0)] & 0x000000ff) ^ + ((word32)Td4[GETBYTE(t2, 3)] << 24) ^ + ((word32)Td4[GETBYTE(t1, 2)] << 16) ^ + ((word32)Td4[GETBYTE(t0, 1)] << 8) ^ + ((word32)Td4[GETBYTE(t3, 0)]) ^ rk[2]; s3 = - (Td[4][GETBYTE(t3, 3)] & 0xff000000) ^ - (Td[4][GETBYTE(t2, 2)] & 0x00ff0000) ^ - (Td[4][GETBYTE(t1, 1)] & 0x0000ff00) ^ - (Td[4][GETBYTE(t0, 0)] & 0x000000ff) ^ + ((word32)Td4[GETBYTE(t3, 3)] << 24) ^ + ((word32)Td4[GETBYTE(t2, 2)] << 16) ^ + ((word32)Td4[GETBYTE(t1, 1)] << 8) ^ + ((word32)Td4[GETBYTE(t0, 0)]) ^ rk[3]; /* write out */ From 6ae1a14c9f24754cccfb5c0cd84851f9b6650375 Mon Sep 17 00:00:00 2001 From: toddouska Date: Mon, 12 Sep 2016 21:09:08 -0700 Subject: [PATCH 3/3] do aes cache line stride by bytes, not word32s --- wolfcrypt/src/aes.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/wolfcrypt/src/aes.c b/wolfcrypt/src/aes.c index 65a5538ce..bfd46ffde 100644 --- a/wolfcrypt/src/aes.c +++ b/wolfcrypt/src/aes.c @@ -1260,7 +1260,8 @@ static INLINE word32 PreFetchTe(void) int i,j; for (i = 0; i < 4; i++) { - for (j = 0; j < 256; j += WC_CACHE_LINE_SZ) { + /* 256 elements, each one is 4 bytes */ + for (j = 0; j < 256; j += WC_CACHE_LINE_SZ/4) { x &= Te[i][j]; } } @@ -1461,7 +1462,8 @@ static INLINE word32 PreFetchTd(void) int i,j; for (i = 0; i < 4; i++) { - for (j = 0; j < 256; j += WC_CACHE_LINE_SZ) { + /* 256 elements, each one is 4 bytes */ + for (j = 0; j < 256; j += WC_CACHE_LINE_SZ/4) { x &= Td[i][j]; } }