From 83dd60307f011173f91cf5eb3536d748962ebfa7 Mon Sep 17 00:00:00 2001 From: "harshal.patil" Date: Thu, 12 Oct 2023 14:51:04 +0530 Subject: [PATCH] feat(mbedtls/esp_crypto_shared_gdma): support AXI-DMA in the crypto shared gdma layer - In case of AXI-DMA, the DMA descriptors need to be 8 bytes aligned lldesc_t do not satify this condition thus we need to replace it with dma_descriptor_t (align(4) and align(8)) in esp_crypto_shared_gdma. - Added new shared gdma start API that supports the dma_descriptor_t DMA descriptor. - Added some generic dma descriptor macros and helper functions - replace lldesc_t with dma_descriptor_t --- components/hal/include/hal/dma_types.h | 21 ++- components/mbedtls/port/aes/dma/esp_aes.c | 177 +++++++++++------- .../port/aes/dma/esp_aes_crypto_dma_impl.c | 26 +-- .../mbedtls/port/aes/dma/esp_aes_gdma_impl.c | 9 +- .../port/aes/dma/include/esp_aes_dma_priv.h | 7 +- components/mbedtls/port/aes/esp_aes_gcm.c | 24 +-- .../port/aes/include/esp_aes_internal.h | 8 +- .../esp_crypto_shared_gdma.c | 88 ++++++++- .../mbedtls/port/include/aes/esp_aes_gcm.h | 1 - .../mbedtls/port/include/esp_crypto_dma.h | 45 +++++ .../port/include/esp_crypto_shared_gdma.h | 23 ++- .../test_apps/main/test_aes_sha_parallel.c | 2 - tools/ci/check_copyright_ignore.txt | 1 - 13 files changed, 309 insertions(+), 123 deletions(-) create mode 100644 components/mbedtls/port/include/esp_crypto_dma.h diff --git a/components/hal/include/hal/dma_types.h b/components/hal/include/hal/dma_types.h index a0cae11ea3..52e6c542c7 100644 --- a/components/hal/include/hal/dma_types.h +++ b/components/hal/include/hal/dma_types.h @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: 2020-2023 Espressif Systems (Shanghai) CO LTD + * SPDX-FileCopyrightText: 2020-2024 Espressif Systems (Shanghai) CO LTD * * SPDX-License-Identifier: Apache-2.0 */ @@ -7,6 +7,7 @@ #pragma once #include +#include #include "esp_assert.h" #ifdef __cplusplus @@ -56,6 +57,24 @@ ESP_STATIC_ASSERT(sizeof(dma_descriptor_align8_t) == 16, "dma_descriptor_align8_ #define DMA_DESCRIPTOR_BUFFER_OWNER_DMA (1) /*!< DMA buffer is allowed to be accessed by DMA engine */ #define DMA_DESCRIPTOR_BUFFER_MAX_SIZE (4095) /*!< Maximum size of the buffer that can be attached to descriptor */ #define DMA_DESCRIPTOR_BUFFER_MAX_SIZE_4B_ALIGNED (4095-3) /*!< Maximum size of the buffer that can be attached to descriptor, and aligned to 4B */ +#define DMA_DESCRIPTOR_BUFFER_MAX_SIZE_16B_ALIGNED (4095-15) /*!< Maximum size of the buffer that can be attached to descriptor, and aligned to 16B */ + +// the size field has 12 bits, but 0 not for 4096. +// to avoid possible problem when the size is not word-aligned, we only use 4096-4 per desc. +/** Maximum size of data in the buffer that a DMA descriptor can hold. */ +#define DMA_DESCRIPTOR_BUFFER_MAX_SIZE_PER_DESC (4096-4) + +/** + * Get the number of DMA descriptors required for a given buffer size. + * + * @param data_size Size to check DMA descriptor number. + * @param max_desc_size Maximum length of each descriptor + * @return Number of DMA descriptors required. + */ +static inline size_t dma_desc_get_required_num(size_t data_size, size_t max_desc_size) +{ + return (data_size + max_desc_size - 1) / max_desc_size; +} #ifdef __cplusplus } diff --git a/components/mbedtls/port/aes/dma/esp_aes.c b/components/mbedtls/port/aes/dma/esp_aes.c index 8fe7b4d522..9aaa327ff9 100644 --- a/components/mbedtls/port/aes/dma/esp_aes.c +++ b/components/mbedtls/port/aes/dma/esp_aes.c @@ -33,7 +33,7 @@ #include "esp_private/periph_ctrl.h" #include "esp_log.h" #include "esp_attr.h" -#include "soc/lldesc.h" +#include "esp_crypto_dma.h" #include "esp_heap_caps.h" #include "esp_memory_utils.h" #include "sys/param.h" @@ -42,8 +42,8 @@ #endif #include "esp_crypto_lock.h" #include "hal/aes_hal.h" -#include "esp_aes_internal.h" #include "esp_aes_dma_priv.h" +#include "esp_aes_internal.h" #if CONFIG_IDF_TARGET_ESP32S2 #include "esp32s2/rom/cache.h" @@ -103,25 +103,19 @@ static bool s_check_dma_capable(const void *p); * * Must be in DMA capable memory, so stack is not a safe place to put them * * To avoid having to malloc/free them for every DMA operation */ -static DRAM_ATTR lldesc_t s_stream_in_desc; -static DRAM_ATTR lldesc_t s_stream_out_desc; +static DRAM_ATTR crypto_dma_desc_t s_stream_in_desc; +static DRAM_ATTR crypto_dma_desc_t s_stream_out_desc; static DRAM_ATTR uint8_t s_stream_in[AES_BLOCK_BYTES]; static DRAM_ATTR uint8_t s_stream_out[AES_BLOCK_BYTES]; -static inline void esp_aes_wait_dma_done(lldesc_t *output) +/** Append a descriptor to the chain, set head if chain empty + * + * @param[out] head Pointer to the first/head node of the DMA descriptor linked list + * @param item Pointer to the DMA descriptor node that has to be appended + */ +static inline void dma_desc_append(crypto_dma_desc_t **head, crypto_dma_desc_t *item) { - /* Wait for DMA write operation to complete */ - while (1) { - if ( esp_aes_dma_done(output) ) { - break; - } - } -} - -/* Append a descriptor to the chain, set head if chain empty */ -static inline void lldesc_append(lldesc_t **head, lldesc_t *item) -{ - lldesc_t *it; + crypto_dma_desc_t *it; if (*head == NULL) { *head = item; return; @@ -129,11 +123,61 @@ static inline void lldesc_append(lldesc_t **head, lldesc_t *item) it = *head; - while (it->empty != 0) { - it = (lldesc_t *)it->empty; + while (it->next != 0) { + it = (crypto_dma_desc_t *)it->next; + } + it->dw0.suc_eof = 0; + it->next = item; +} + +/** + * Generate a linked list pointing to a (huge) buffer in an descriptor array. + * + * The caller should ensure there is enough size to hold the array, by calling + * `dma_desc_get_required_num` with the same or less than the max_desc_size argument. + * + * @param[out] dmadesc Output of a descriptor array, the head should be fed to the DMA. + * @param data Buffer for the descriptors to point to. + * @param len Size (or length for TX) of the buffer + * @param max_desc_size Maximum length of each descriptor + * @param isrx The RX DMA may require the buffer to be word-aligned, set to true for a RX link, otherwise false. + */ +static inline void dma_desc_setup_link(crypto_dma_desc_t* dmadesc, const void *data, int len, int max_desc_size, bool isrx) +{ + int i = 0; + while (len) { + int dmachunklen = len; + if (dmachunklen > max_desc_size) { + dmachunklen = max_desc_size; + } + if (isrx) { + //Receive needs DMA length rounded to next 32-bit boundary + dmadesc[i].dw0.size = (dmachunklen + 3) & (~3); + dmadesc[i].dw0.length = (dmachunklen + 3) & (~3); + } else { + dmadesc[i].dw0.size = dmachunklen; + dmadesc[i].dw0.length = dmachunklen; + } + dmadesc[i].buffer = (void *)data; + dmadesc[i].dw0.suc_eof = 0; + dmadesc[i].dw0.owner = DMA_DESCRIPTOR_BUFFER_OWNER_DMA; + dmadesc[i].next = &dmadesc[i + 1]; + len -= dmachunklen; + data += dmachunklen; + i++; + } + dmadesc[i - 1].dw0.suc_eof = 1; //Mark last DMA desc as end of stream. + dmadesc[i - 1].next = NULL; +} + +static inline void esp_aes_wait_dma_done(crypto_dma_desc_t *output) +{ + /* Wait for DMA write operation to complete */ + while (1) { + if ( esp_aes_dma_done(output) ) { + break; + } } - it->eof = 0; - it->empty = (uint32_t)item; } void esp_aes_acquire_hardware( void ) @@ -221,7 +265,7 @@ static esp_err_t esp_aes_isr_initialise( void ) #endif // CONFIG_MBEDTLS_AES_USE_INTERRUPT /* Wait for AES hardware block operation to complete */ -static int esp_aes_dma_wait_complete(bool use_intr, lldesc_t *output_desc) +static int esp_aes_dma_wait_complete(bool use_intr, crypto_dma_desc_t *output_desc) { #if defined (CONFIG_MBEDTLS_AES_USE_INTERRUPT) if (use_intr) { @@ -335,10 +379,10 @@ cleanup: */ static int esp_aes_process_dma(esp_aes_context *ctx, const unsigned char *input, unsigned char *output, size_t len, uint8_t *stream_out) { - lldesc_t *in_desc_head = NULL, *out_desc_head = NULL; - lldesc_t *out_desc_tail = NULL; /* pointer to the final output descriptor */ - lldesc_t *block_desc = NULL, *block_in_desc = NULL, *block_out_desc = NULL; - size_t lldesc_num = 0; + crypto_dma_desc_t *in_desc_head = NULL, *out_desc_head = NULL; + crypto_dma_desc_t *out_desc_tail = NULL; /* pointer to the final output descriptor */ + crypto_dma_desc_t *block_desc = NULL, *block_in_desc = NULL, *block_out_desc = NULL; + size_t crypto_dma_desc_num = 0; unsigned stream_bytes = len % AES_BLOCK_BYTES; // bytes which aren't in a full block unsigned block_bytes = len - stream_bytes; // bytes which are in a full block unsigned blocks = (block_bytes / AES_BLOCK_BYTES) + ((stream_bytes > 0) ? 1 : 0); @@ -388,10 +432,10 @@ static int esp_aes_process_dma(esp_aes_context *ctx, const unsigned char *input, } /* Set up dma descriptors for input and output considering the 16 byte alignment requirement for EDMA */ - lldesc_num = lldesc_get_required_num_constrained(block_bytes, LLDESC_MAX_NUM_PER_DESC_16B_ALIGNED); + crypto_dma_desc_num = dma_desc_get_required_num(block_bytes, DMA_DESCRIPTOR_BUFFER_MAX_SIZE_16B_ALIGNED); /* Allocate both in and out descriptors to save a malloc/free per function call */ - block_desc = heap_caps_calloc(lldesc_num * 2, sizeof(lldesc_t), MALLOC_CAP_DMA); + block_desc = heap_caps_calloc(crypto_dma_desc_num * 2, sizeof(crypto_dma_desc_t), MALLOC_CAP_DMA); if (block_desc == NULL) { mbedtls_platform_zeroize(output, len); ESP_LOGE(TAG, "Failed to allocate memory"); @@ -399,36 +443,37 @@ static int esp_aes_process_dma(esp_aes_context *ctx, const unsigned char *input, } block_in_desc = block_desc; - block_out_desc = block_desc + lldesc_num; + block_out_desc = block_desc + crypto_dma_desc_num; + + dma_desc_setup_link(block_in_desc, input, block_bytes, DMA_DESCRIPTOR_BUFFER_MAX_SIZE_PER_DESC, 0); - lldesc_setup_link(block_in_desc, input, block_bytes, 0); //Limit max inlink descriptor length to be 16 byte aligned, require for EDMA - lldesc_setup_link_constrained(block_out_desc, output, block_bytes, LLDESC_MAX_NUM_PER_DESC_16B_ALIGNED, 0); + dma_desc_setup_link(block_out_desc, output, block_bytes, DMA_DESCRIPTOR_BUFFER_MAX_SIZE_16B_ALIGNED, 0); /* Setup in/out start descriptors */ - lldesc_append(&in_desc_head, block_in_desc); - lldesc_append(&out_desc_head, block_out_desc); + dma_desc_append(&in_desc_head, block_in_desc); + dma_desc_append(&out_desc_head, block_out_desc); - out_desc_tail = &block_out_desc[lldesc_num - 1]; + out_desc_tail = &block_out_desc[crypto_dma_desc_num - 1]; } /* Any leftover bytes which are appended as an additional DMA list */ if (stream_bytes > 0) { - memset(&s_stream_in_desc, 0, sizeof(lldesc_t)); - memset(&s_stream_out_desc, 0, sizeof(lldesc_t)); + memset(&s_stream_in_desc, 0, sizeof(crypto_dma_desc_t)); + memset(&s_stream_out_desc, 0, sizeof(crypto_dma_desc_t)); memset(s_stream_in, 0, AES_BLOCK_BYTES); memset(s_stream_out, 0, AES_BLOCK_BYTES); memcpy(s_stream_in, input + block_bytes, stream_bytes); - lldesc_setup_link(&s_stream_in_desc, s_stream_in, AES_BLOCK_BYTES, 0); - lldesc_setup_link(&s_stream_out_desc, s_stream_out, AES_BLOCK_BYTES, 0); + dma_desc_setup_link(&s_stream_in_desc, s_stream_in, AES_BLOCK_BYTES, DMA_DESCRIPTOR_BUFFER_MAX_SIZE_PER_DESC, 0); + dma_desc_setup_link(&s_stream_out_desc, s_stream_out, AES_BLOCK_BYTES, DMA_DESCRIPTOR_BUFFER_MAX_SIZE_PER_DESC, 0); /* Link with block descriptors */ - lldesc_append(&in_desc_head, &s_stream_in_desc); - lldesc_append(&out_desc_head, &s_stream_out_desc); + dma_desc_append(&in_desc_head, &s_stream_in_desc); + dma_desc_append(&out_desc_head, &s_stream_out_desc); out_desc_tail = &s_stream_out_desc; } @@ -494,13 +539,13 @@ cleanup: * 3. If AES interrupt is enabled and ISR initialisation fails * 4. Failure in any of the AES operations */ -int esp_aes_process_dma_gcm(esp_aes_context *ctx, const unsigned char *input, unsigned char *output, size_t len, lldesc_t *aad_desc, size_t aad_len) +int esp_aes_process_dma_gcm(esp_aes_context *ctx, const unsigned char *input, unsigned char *output, size_t len, crypto_dma_desc_t *aad_desc, size_t aad_len) { - lldesc_t *in_desc_head = NULL, *out_desc_head = NULL, *len_desc = NULL; - lldesc_t *out_desc_tail = NULL; /* pointer to the final output descriptor */ - lldesc_t stream_in_desc, stream_out_desc; - lldesc_t *block_desc = NULL, *block_in_desc = NULL, *block_out_desc = NULL; - size_t lldesc_num; + crypto_dma_desc_t *in_desc_head = NULL, *out_desc_head = NULL, *len_desc = NULL; + crypto_dma_desc_t *out_desc_tail = NULL; /* pointer to the final output descriptor */ + crypto_dma_desc_t stream_in_desc, stream_out_desc; + crypto_dma_desc_t *block_desc = NULL, *block_in_desc = NULL, *block_out_desc = NULL; + size_t crypto_dma_desc_num = 0; uint32_t len_buf[4] = {}; uint8_t stream_in[16] = {}; uint8_t stream_out[16] = {}; @@ -523,10 +568,10 @@ int esp_aes_process_dma_gcm(esp_aes_context *ctx, const unsigned char *input, un } /* Set up dma descriptors for input and output */ - lldesc_num = lldesc_get_required_num(block_bytes); + crypto_dma_desc_num = dma_desc_get_required_num(block_bytes, DMA_DESCRIPTOR_BUFFER_MAX_SIZE_PER_DESC); /* Allocate both in and out descriptors to save a malloc/free per function call, add 1 for length descriptor */ - block_desc = heap_caps_calloc( (lldesc_num * 2) + 1, sizeof(lldesc_t), MALLOC_CAP_DMA); + block_desc = heap_caps_calloc((crypto_dma_desc_num * 2) + 1, sizeof(crypto_dma_desc_t), MALLOC_CAP_DMA); if (block_desc == NULL) { mbedtls_platform_zeroize(output, len); ESP_LOGE(TAG, "Failed to allocate memory"); @@ -534,32 +579,32 @@ int esp_aes_process_dma_gcm(esp_aes_context *ctx, const unsigned char *input, un } block_in_desc = block_desc; - len_desc = block_desc + lldesc_num; - block_out_desc = block_desc + lldesc_num + 1; + len_desc = block_desc + crypto_dma_desc_num; + block_out_desc = block_desc + crypto_dma_desc_num + 1; if (aad_desc != NULL) { - lldesc_append(&in_desc_head, aad_desc); + dma_desc_append(&in_desc_head, aad_desc); } if (block_bytes > 0) { - lldesc_setup_link(block_in_desc, input, block_bytes, 0); - lldesc_setup_link(block_out_desc, output, block_bytes, 0); + dma_desc_setup_link(block_in_desc, input, block_bytes, DMA_DESCRIPTOR_BUFFER_MAX_SIZE_PER_DESC, 0); + dma_desc_setup_link(block_out_desc, output, block_bytes, DMA_DESCRIPTOR_BUFFER_MAX_SIZE_PER_DESC, 0); - lldesc_append(&in_desc_head, block_in_desc); - lldesc_append(&out_desc_head, block_out_desc); + dma_desc_append(&in_desc_head, block_in_desc); + dma_desc_append(&out_desc_head, block_out_desc); - out_desc_tail = &block_out_desc[lldesc_num - 1]; + out_desc_tail = &block_out_desc[crypto_dma_desc_num - 1]; } /* Any leftover bytes which are appended as an additional DMA list */ if (stream_bytes > 0) { memcpy(stream_in, input + block_bytes, stream_bytes); - lldesc_setup_link(&stream_in_desc, stream_in, AES_BLOCK_BYTES, 0); - lldesc_setup_link(&stream_out_desc, stream_out, AES_BLOCK_BYTES, 0); + dma_desc_setup_link(&stream_in_desc, stream_in, AES_BLOCK_BYTES, DMA_DESCRIPTOR_BUFFER_MAX_SIZE_PER_DESC, 0); + dma_desc_setup_link(&stream_out_desc, stream_out, AES_BLOCK_BYTES, DMA_DESCRIPTOR_BUFFER_MAX_SIZE_PER_DESC, 0); - lldesc_append(&in_desc_head, &stream_in_desc); - lldesc_append(&out_desc_head, &stream_out_desc); + dma_desc_append(&in_desc_head, &stream_in_desc); + dma_desc_append(&out_desc_head, &stream_out_desc); out_desc_tail = &stream_out_desc; } @@ -568,13 +613,13 @@ int esp_aes_process_dma_gcm(esp_aes_context *ctx, const unsigned char *input, un len_buf[1] = __builtin_bswap32(aad_len * 8); len_buf[3] = __builtin_bswap32(len * 8); - len_desc->length = sizeof(len_buf); - len_desc->size = sizeof(len_buf); - len_desc->owner = 1; - len_desc->eof = 1; - len_desc->buf = (uint8_t *)len_buf; + len_desc->dw0.length = sizeof(len_buf); + len_desc->dw0.size = sizeof(len_buf); + len_desc->dw0.owner = 1; + len_desc->dw0.suc_eof = 1; + len_desc->buffer = (uint8_t *)len_buf; - lldesc_append(&in_desc_head, len_desc); + dma_desc_append(&in_desc_head, len_desc); #if defined (CONFIG_MBEDTLS_AES_USE_INTERRUPT) /* Only use interrupt for long AES operations */ diff --git a/components/mbedtls/port/aes/dma/esp_aes_crypto_dma_impl.c b/components/mbedtls/port/aes/dma/esp_aes_crypto_dma_impl.c index df8fd4bf49..8878375231 100644 --- a/components/mbedtls/port/aes/dma/esp_aes_crypto_dma_impl.c +++ b/components/mbedtls/port/aes/dma/esp_aes_crypto_dma_impl.c @@ -1,18 +1,10 @@ -// Copyright 2020 Espressif Systems (Shanghai) PTE LTD -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - +/* + * SPDX-FileCopyrightText: 2020-2023 Espressif Systems (Shanghai) CO LTD + * + * SPDX-License-Identifier: Apache-2.0 + */ +#include "esp_crypto_dma.h" #include "esp_aes_dma_priv.h" #include "soc/soc_caps.h" @@ -20,7 +12,7 @@ #include "hal/crypto_dma_ll.h" -esp_err_t esp_aes_dma_start(const lldesc_t *input, const lldesc_t *output) +esp_err_t esp_aes_dma_start(const crypto_dma_desc_t *input, const crypto_dma_desc_t *output) { crypto_dma_ll_reset(); crypto_dma_ll_set_mode(CRYPTO_DMA_AES); @@ -36,7 +28,7 @@ esp_err_t esp_aes_dma_start(const lldesc_t *input, const lldesc_t *output) return ESP_OK; } -bool esp_aes_dma_done(const lldesc_t *output) +bool esp_aes_dma_done(const crypto_dma_desc_t *output) { - return (crypto_dma_ll_inlink_is_eof() && (output->owner == 0)); + return (crypto_dma_ll_inlink_is_eof() && (output->dw0.owner == 0)); } diff --git a/components/mbedtls/port/aes/dma/esp_aes_gdma_impl.c b/components/mbedtls/port/aes/dma/esp_aes_gdma_impl.c index 991a5600c5..c43bfd68ae 100644 --- a/components/mbedtls/port/aes/dma/esp_aes_gdma_impl.c +++ b/components/mbedtls/port/aes/dma/esp_aes_gdma_impl.c @@ -5,14 +5,15 @@ */ #include "esp_aes_dma_priv.h" +#include "esp_crypto_dma.h" #include "esp_crypto_shared_gdma.h" -esp_err_t esp_aes_dma_start(const lldesc_t *input, const lldesc_t *output) +esp_err_t esp_aes_dma_start(const crypto_dma_desc_t *input, const crypto_dma_desc_t *output) { - return esp_crypto_shared_gdma_start(input, output, GDMA_TRIG_PERIPH_AES); + return esp_crypto_shared_gdma_start_axi_ahb(input, output, GDMA_TRIG_PERIPH_AES); } -bool esp_aes_dma_done(const lldesc_t *output) +bool esp_aes_dma_done(const crypto_dma_desc_t *output) { - return (output->owner == 0); + return (output->dw0.owner == 0); } diff --git a/components/mbedtls/port/aes/dma/include/esp_aes_dma_priv.h b/components/mbedtls/port/aes/dma/include/esp_aes_dma_priv.h index c219a9a77e..72c56dc034 100644 --- a/components/mbedtls/port/aes/dma/include/esp_aes_dma_priv.h +++ b/components/mbedtls/port/aes/dma/include/esp_aes_dma_priv.h @@ -6,7 +6,8 @@ #pragma once -#include "soc/lldesc.h" +#include +#include "esp_crypto_dma.h" #include "soc/soc_caps.h" #include "esp_err.h" @@ -23,7 +24,7 @@ extern "C" { * - ESP_OK: Successfully started the DMA * - ESP_ERR_INVALID_STATE: No DMA channel available */ -esp_err_t esp_aes_dma_start(const lldesc_t *input, const lldesc_t *output); +esp_err_t esp_aes_dma_start(const crypto_dma_desc_t *input, const crypto_dma_desc_t *output); /** * @brief Check if the DMA engine is finished reading the result @@ -33,7 +34,7 @@ esp_err_t esp_aes_dma_start(const lldesc_t *input, const lldesc_t *output); * - true: DMA finished * - false: DMA not yet finished */ -bool esp_aes_dma_done(const lldesc_t *output); +bool esp_aes_dma_done(const crypto_dma_desc_t *output); /** * @brief Allocate AES peripheral interrupt handler diff --git a/components/mbedtls/port/aes/esp_aes_gcm.c b/components/mbedtls/port/aes/esp_aes_gcm.c index 76d68544d7..e7c34fc5b4 100644 --- a/components/mbedtls/port/aes/esp_aes_gcm.c +++ b/components/mbedtls/port/aes/esp_aes_gcm.c @@ -672,8 +672,8 @@ int esp_aes_gcm_crypt_and_tag( esp_gcm_context *ctx, #endif #if CONFIG_MBEDTLS_HARDWARE_GCM int ret; - lldesc_t aad_desc[2] = {}; - lldesc_t *aad_head_desc = NULL; + crypto_dma_desc_t aad_desc[2] = {}; + crypto_dma_desc_t *aad_head_desc = NULL; size_t remainder_bit; uint8_t stream_in[AES_BLOCK_BYTES] = {}; unsigned stream_bytes = aad_len % AES_BLOCK_BYTES; // bytes which aren't in a full block @@ -687,7 +687,7 @@ int esp_aes_gcm_crypt_and_tag( esp_gcm_context *ctx, /* Limit aad len to a single DMA descriptor to simplify DMA handling In practice, e.g. with mbedtls the length of aad will always be short */ - if (aad_len > LLDESC_MAX_NUM_PER_DESC) { + if (aad_len > DMA_DESCRIPTOR_BUFFER_MAX_SIZE_PER_DESC) { return MBEDTLS_ERR_GCM_BAD_INPUT; } /* IV and AD are limited to 2^32 bits, so 2^29 bytes */ @@ -723,20 +723,20 @@ int esp_aes_gcm_crypt_and_tag( esp_gcm_context *ctx, ctx->aes_ctx.key_in_hardware = aes_hal_setkey(ctx->aes_ctx.key, ctx->aes_ctx.key_bytes, mode); if (block_bytes > 0) { - aad_desc[0].length = block_bytes; - aad_desc[0].size = block_bytes; - aad_desc[0].owner = 1; - aad_desc[0].buf = aad; + aad_desc[0].dw0.length = block_bytes; + aad_desc[0].dw0.size = block_bytes; + aad_desc[0].dw0.owner = 1; + aad_desc[0].buffer = (void*)aad; } if (stream_bytes > 0) { memcpy(stream_in, aad + block_bytes, stream_bytes); - aad_desc[0].empty = (uint32_t)&aad_desc[1]; - aad_desc[1].length = AES_BLOCK_BYTES; - aad_desc[1].size = AES_BLOCK_BYTES; - aad_desc[1].owner = 1; - aad_desc[1].buf = stream_in; + aad_desc[0].next = &aad_desc[1]; + aad_desc[1].dw0.length = AES_BLOCK_BYTES; + aad_desc[1].dw0.size = AES_BLOCK_BYTES; + aad_desc[1].dw0.owner = 1; + aad_desc[1].buffer = (void*)stream_in; } if (block_bytes > 0) { diff --git a/components/mbedtls/port/aes/include/esp_aes_internal.h b/components/mbedtls/port/aes/include/esp_aes_internal.h index 889888e83f..e400e63f8a 100644 --- a/components/mbedtls/port/aes/include/esp_aes_internal.h +++ b/components/mbedtls/port/aes/include/esp_aes_internal.h @@ -6,10 +6,10 @@ #pragma once -#include "aes/esp_aes.h" -#include "aes/esp_aes_gcm.h" -#include "soc/soc_caps.h" #include +#include "aes/esp_aes.h" +#include "soc/soc_caps.h" +#include "esp_crypto_dma.h" #ifdef __cplusplus extern "C" { @@ -29,7 +29,7 @@ bool valid_key_length(const esp_aes_context *ctx); * @param aad_len GCM additional data length * @return int -1 on error */ -int esp_aes_process_dma_gcm(esp_aes_context *ctx, const unsigned char *input, unsigned char *output, size_t len, lldesc_t *aad_desc, size_t aad_len); +int esp_aes_process_dma_gcm(esp_aes_context *ctx, const unsigned char *input, unsigned char *output, size_t len, crypto_dma_desc_t *aad_desc, size_t aad_len); #endif #ifdef __cplusplus diff --git a/components/mbedtls/port/crypto_shared_gdma/esp_crypto_shared_gdma.c b/components/mbedtls/port/crypto_shared_gdma/esp_crypto_shared_gdma.c index 83026a6274..8df3fd2c27 100644 --- a/components/mbedtls/port/crypto_shared_gdma/esp_crypto_shared_gdma.c +++ b/components/mbedtls/port/crypto_shared_gdma/esp_crypto_shared_gdma.c @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD + * SPDX-FileCopyrightText: 2021-2023 Espressif Systems (Shanghai) CO LTD * * SPDX-License-Identifier: Apache-2.0 */ @@ -8,12 +8,21 @@ #include "freertos/FreeRTOS.h" #include "freertos/task.h" - -#include "hal/gdma_ll.h" -#include "soc/soc_caps.h" #include "esp_log.h" #include "esp_err.h" +#include "esp_crypto_dma.h" #include "esp_crypto_lock.h" +#include "soc/soc_caps.h" + +#if CONFIG_IDF_TARGET_ESP32P4 +#include "esp32p4/rom/cache.h" +#endif + +#if SOC_AHB_GDMA_VERSION == 1 +#include "hal/gdma_ll.h" +#elif SOC_AXI_GDMA_SUPPORTED +#include "hal/axi_dma_ll.h" +#endif /* SOC_AHB_GDMA_VERSION */ #define NEW_CHANNEL_TIMEOUT_MS 1000 #define NEW_CHANNEL_DELAY_MS 100 @@ -26,12 +35,15 @@ static gdma_channel_handle_t tx_channel; /* Allocate a new GDMA channel, will keep trying until NEW_CHANNEL_TIMEOUT_MS */ static inline esp_err_t crypto_shared_gdma_new_channel(gdma_channel_alloc_config_t *channel_config, gdma_channel_handle_t *channel) { - esp_err_t ret; + esp_err_t ret = ESP_FAIL; int time_waited_ms = 0; while (1) { - ret = gdma_new_channel(channel_config, channel); - +#if SOC_AXI_GDMA_SUPPORTED + ret = gdma_new_axi_channel(channel_config, channel); +#else /* !SOC_AXI_GDMA_SUPPORTED */ + ret = gdma_new_ahb_channel(channel_config, channel); +#endif /* SOC_AXI_GDMA_SUPPORTED */ if (ret == ESP_OK) { break; } else if (time_waited_ms >= NEW_CHANNEL_TIMEOUT_MS) { @@ -92,7 +104,6 @@ err: return ret; } - esp_err_t esp_crypto_shared_gdma_start(const lldesc_t *input, const lldesc_t *output, gdma_trigger_peripheral_t peripheral) { int rx_ch_id = 0; @@ -121,7 +132,68 @@ esp_err_t esp_crypto_shared_gdma_start(const lldesc_t *input, const lldesc_t *ou /* tx channel is reset by gdma_connect(), also reset rx to ensure a known state */ gdma_get_channel_id(rx_channel, &rx_ch_id); + +#if SOC_AHB_GDMA_VERSION == 1 gdma_ll_rx_reset_channel(&GDMA, rx_ch_id); +#endif /* SOC_AHB_GDMA_VERSION */ + + gdma_start(tx_channel, (intptr_t)input); + gdma_start(rx_channel, (intptr_t)output); + + return ESP_OK; +} + +esp_err_t esp_crypto_shared_gdma_start_axi_ahb(const crypto_dma_desc_t *input, const crypto_dma_desc_t *output, gdma_trigger_peripheral_t peripheral) +{ + int rx_ch_id = 0; + esp_err_t ret = ESP_OK; + +#if SOC_CACHE_INTERNAL_MEM_VIA_L1CACHE + // TODO: replace with `esp_cache_msync` + const crypto_dma_desc_t *it = input; + while(it != NULL) { + Cache_WriteBack_Addr(CACHE_MAP_L1_DCACHE | CACHE_MAP_L2_CACHE, (uint32_t)it->buffer, it->dw0.length); // try using esp_cache_msync() + Cache_WriteBack_Addr(CACHE_MAP_L1_DCACHE | CACHE_MAP_L2_CACHE, (uint32_t)it, sizeof(crypto_dma_desc_t)); + it = (const crypto_dma_desc_t*) it->next; + } + + it = output; + while(it != NULL) { + Cache_WriteBack_Addr(CACHE_MAP_L1_DCACHE | CACHE_MAP_L2_CACHE, (uint32_t)it->buffer, it->dw0.length); + Cache_WriteBack_Addr(CACHE_MAP_L1_DCACHE | CACHE_MAP_L2_CACHE, (uint32_t)it, sizeof(crypto_dma_desc_t)); + it = (const crypto_dma_desc_t*) it->next; + }; +#endif /* SOC_CACHE_INTERNAL_MEM_VIA_L1CACHE */ + + if (tx_channel == NULL) { + /* Allocate a pair of RX and TX for crypto, should only happen the first time we use the GMDA + or if user called esp_crypto_shared_gdma_release */ + ret = crypto_shared_gdma_init(); + } + + if (ret != ESP_OK) { + return ret; + } + + /* Tx channel is shared between AES and SHA, need to connect to peripheral every time */ + gdma_disconnect(tx_channel); + + if (peripheral == GDMA_TRIG_PERIPH_SHA) { + gdma_connect(tx_channel, GDMA_MAKE_TRIGGER(GDMA_TRIG_PERIPH_SHA, 0)); + } else if (peripheral == GDMA_TRIG_PERIPH_AES) { + gdma_connect(tx_channel, GDMA_MAKE_TRIGGER(GDMA_TRIG_PERIPH_AES, 0)); + } else { + return ESP_ERR_INVALID_ARG; + } + + /* tx channel is reset by gdma_connect(), also reset rx to ensure a known state */ + gdma_get_channel_id(rx_channel, &rx_ch_id); + +#if SOC_AHB_GDMA_VERSION == 1 + gdma_ll_rx_reset_channel(&GDMA, rx_ch_id); +#elif SOC_AXI_GDMA_SUPPORTED + axi_dma_ll_rx_reset_channel(&AXI_DMA, rx_ch_id); +#endif /* SOC_AHB_GDMA_VERSION */ gdma_start(tx_channel, (intptr_t)input); gdma_start(rx_channel, (intptr_t)output); diff --git a/components/mbedtls/port/include/aes/esp_aes_gcm.h b/components/mbedtls/port/include/aes/esp_aes_gcm.h index fb9cc1260c..c270c9f97b 100644 --- a/components/mbedtls/port/include/aes/esp_aes_gcm.h +++ b/components/mbedtls/port/include/aes/esp_aes_gcm.h @@ -12,7 +12,6 @@ #include "aes/esp_aes.h" #include "mbedtls/cipher.h" -#include "soc/lldesc.h" #ifdef __cplusplus extern "C" { diff --git a/components/mbedtls/port/include/esp_crypto_dma.h b/components/mbedtls/port/include/esp_crypto_dma.h new file mode 100644 index 0000000000..8eadc1aa26 --- /dev/null +++ b/components/mbedtls/port/include/esp_crypto_dma.h @@ -0,0 +1,45 @@ +/* + * SPDX-FileCopyrightText: 2023-2024 Espressif Systems (Shanghai) CO LTD + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include "hal/dma_types.h" +#include "soc/gdma_channel.h" +#include "soc/soc_caps.h" + + +#ifdef __cplusplus +extern "C" +{ +#endif + + +#if (SOC_AES_SUPPORT_DMA) && (SOC_SHA_SUPPORT_DMA) + +#if (SOC_AES_GDMA) && (SOC_SHA_GDMA) + +#if (SOC_GDMA_TRIG_PERIPH_AES0_BUS == SOC_GDMA_BUS_AHB) && (SOC_GDMA_TRIG_PERIPH_SHA0_BUS == SOC_GDMA_BUS_AHB) +#define DMA_DESC_MEM_ALIGN_SIZE 4 +typedef dma_descriptor_align4_t crypto_dma_desc_t; +#elif (SOC_GDMA_TRIG_PERIPH_AES0_BUS == SOC_GDMA_BUS_AXI) && (SOC_GDMA_TRIG_PERIPH_SHA0_BUS == SOC_GDMA_BUS_AXI) +#define DMA_DESC_MEM_ALIGN_SIZE 8 +typedef dma_descriptor_align8_t crypto_dma_desc_t; +#else +#error "As we support a shared crypto GDMA layer for the AES and the SHA peripheral, both the peripherals must use the same GDMA bus" +#endif /* (SOC_GDMA_TRIG_PERIPH_AES0_BUS == SOC_GDMA_BUS_AHB) && (SOC_GDMA_TRIG_PERIPH_AES0_BUS == SOC_GDMA_BUS_AHB) */ + +#elif (SOC_AES_CRYPTO_DMA) && (SOC_SHA_CRYPTO_DMA) +#define DMA_DESC_MEM_ALIGN_SIZE 4 +typedef dma_descriptor_align4_t crypto_dma_desc_t; + +#endif /* (SOC_AES_GDMA) && (SOC_SHA_GDMA) */ + +#endif /* (SOC_AES_SUPPORT_DMA) && (SOC_SHA_SUPPORT_DMA) */ + + +#ifdef __cplusplus +} +#endif diff --git a/components/mbedtls/port/include/esp_crypto_shared_gdma.h b/components/mbedtls/port/include/esp_crypto_shared_gdma.h index 4e69a5f8ea..fd49558912 100644 --- a/components/mbedtls/port/include/esp_crypto_shared_gdma.h +++ b/components/mbedtls/port/include/esp_crypto_shared_gdma.h @@ -1,14 +1,15 @@ /* - * SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD + * SPDX-FileCopyrightText: 2021-2023 Espressif Systems (Shanghai) CO LTD * * SPDX-License-Identifier: Apache-2.0 */ #pragma once -#include "soc/lldesc.h" +#include "esp_crypto_dma.h" #include "esp_private/gdma.h" #include "esp_err.h" +#include "soc/lldesc.h" #ifdef __cplusplus extern "C" { @@ -16,17 +17,31 @@ extern "C" { /** * @brief Start a GDMA transfer on the shared crypto DMA channel + * Only supports AHB-DMA. * * @note Will allocate a GDMA channel for AES & SHA if no such channel is already allocated * - * @param input Input linked list descriptor - * @param output Output linked list descriptor + * @param input Input linked list descriptor (lldesc_t *) + * @param output Output linked list descriptor (lldesc_t *) * @param peripheral Crypto peripheral to connect the DMA to, either GDMA_TRIG_PERIPH_AES or * GDMA_TRIG_PERIPH_SHA * @return esp_err_t ESP_FAIL if no GDMA channel available */ esp_err_t esp_crypto_shared_gdma_start(const lldesc_t *input, const lldesc_t *output, gdma_trigger_peripheral_t peripheral); +/** + * @brief Start a GDMA transfer on the shared crypto DMA channel + * Supports AXI-DMA and AHB-DMA. + * + * @note Will allocate a GDMA channel for AES & SHA if no such channel is already allocated + * + * @param input Input linked list descriptor (crypto_dma_desc_t *) + * @param output Output linked list descriptor (crypto_dma_desc_t *) + * @param peripheral Crypto peripheral to connect the DMA to, either GDMA_TRIG_PERIPH_AES or + * GDMA_TRIG_PERIPH_SHA + * @return esp_err_t ESP_FAIL if no GDMA channel available + */ +esp_err_t esp_crypto_shared_gdma_start_axi_ahb(const crypto_dma_desc_t *input, const crypto_dma_desc_t *output, gdma_trigger_peripheral_t peripheral); /** * @brief Frees any shared crypto DMA channel, if esp_crypto_shared_gdma_start is called after diff --git a/components/mbedtls/test_apps/main/test_aes_sha_parallel.c b/components/mbedtls/test_apps/main/test_aes_sha_parallel.c index ddbb5baffa..a018eca9ab 100644 --- a/components/mbedtls/test_apps/main/test_aes_sha_parallel.c +++ b/components/mbedtls/test_apps/main/test_aes_sha_parallel.c @@ -112,8 +112,6 @@ static void tskRunAES256Test(void *pvParameters) } -#include "esp_crypto_shared_gdma.h" - #define TASK_STACK_SIZE (20*1024) TEST_CASE("mbedtls AES/SHA multithreading", "[mbedtls]") diff --git a/tools/ci/check_copyright_ignore.txt b/tools/ci/check_copyright_ignore.txt index c1fbfb935a..4092aef14a 100644 --- a/tools/ci/check_copyright_ignore.txt +++ b/tools/ci/check_copyright_ignore.txt @@ -504,7 +504,6 @@ components/lwip/include/apps/ping/ping.h components/mbedtls/esp_crt_bundle/test_gen_crt_bundle/test_gen_crt_bundle.py components/mbedtls/port/aes/block/esp_aes.c components/mbedtls/port/aes/dma/esp_aes.c -components/mbedtls/port/aes/dma/esp_aes_crypto_dma_impl.c components/mbedtls/port/aes/esp_aes_xts.c components/mbedtls/port/include/aes/esp_aes.h components/mbedtls/port/include/aes_alt.h