feedc0de/esp-idf
1
0
Fork 0
forked from espressif/esp-idf
Code Pull Requests Activity

feat(mbedtls/esp_crypto_shared_gdma): support AXI-DMA in the crypto shared gdma layer

Browse Source 
- In case of AXI-DMA, the DMA descriptors need to be 8 bytes aligned
lldesc_t do not satify this condition thus we need to replace it with
dma_descriptor_t (align(4) and align(8)) in esp_crypto_shared_gdma.

- Added new shared gdma start API that supports the dma_descriptor_t
DMA descriptor.

- Added some generic dma descriptor macros and helper functions

- replace lldesc_t with dma_descriptor_t
This commit is contained in:
harshal.patil
2023-10-12 14:51:04 +05:30
parent 2c570ed53b
commit 83dd60307f
13 changed files with 309 additions and 123 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
components/hal/include/hal/dma_types.h
View File

@@ -1,5 +1,5 @@
/* /*
* SPDX-FileCopyrightText: 2020-2023 Espressif Systems (Shanghai) CO LTD * SPDX-FileCopyrightText: 2020-2024 Espressif Systems (Shanghai) CO LTD
* *
* SPDX-License-Identifier: Apache-2.0 * SPDX-License-Identifier: Apache-2.0
*/ */
@@ -7,6 +7,7 @@
#pragma once #pragma once
#include <stdint.h> #include <stdint.h>
#include <stddef.h>
#include "esp_assert.h" #include "esp_assert.h"
#ifdef __cplusplus #ifdef __cplusplus
@@ -56,6 +57,24 @@ ESP_STATIC_ASSERT(sizeof(dma_descriptor_align8_t) == 16, "dma_descriptor_align8_
#define DMA_DESCRIPTOR_BUFFER_OWNER_DMA (1) /*!< DMA buffer is allowed to be accessed by DMA engine */ #define DMA_DESCRIPTOR_BUFFER_OWNER_DMA (1) /*!< DMA buffer is allowed to be accessed by DMA engine */
#define DMA_DESCRIPTOR_BUFFER_MAX_SIZE (4095) /*!< Maximum size of the buffer that can be attached to descriptor */ #define DMA_DESCRIPTOR_BUFFER_MAX_SIZE (4095) /*!< Maximum size of the buffer that can be attached to descriptor */
#define DMA_DESCRIPTOR_BUFFER_MAX_SIZE_4B_ALIGNED (4095-3) /*!< Maximum size of the buffer that can be attached to descriptor, and aligned to 4B */ #define DMA_DESCRIPTOR_BUFFER_MAX_SIZE_4B_ALIGNED (4095-3) /*!< Maximum size of the buffer that can be attached to descriptor, and aligned to 4B */
#define DMA_DESCRIPTOR_BUFFER_MAX_SIZE_16B_ALIGNED (4095-15) /*!< Maximum size of the buffer that can be attached to descriptor, and aligned to 16B */
// the size field has 12 bits, but 0 not for 4096.
// to avoid possible problem when the size is not word-aligned, we only use 4096-4 per desc.
/** Maximum size of data in the buffer that a DMA descriptor can hold. */
#define DMA_DESCRIPTOR_BUFFER_MAX_SIZE_PER_DESC (4096-4)
/**
* Get the number of DMA descriptors required for a given buffer size.
*
* @param data_size Size to check DMA descriptor number.
* @param max_desc_size Maximum length of each descriptor
* @return Number of DMA descriptors required.
*/
static inline size_t dma_desc_get_required_num(size_t data_size, size_t max_desc_size)
{
return (data_size + max_desc_size - 1) / max_desc_size;
}
#ifdef __cplusplus #ifdef __cplusplus
} }

177
components/mbedtls/port/aes/dma/esp_aes.c
View File

@@ -33,7 +33,7 @@
#include "esp_private/periph_ctrl.h" #include "esp_private/periph_ctrl.h"
#include "esp_log.h" #include "esp_log.h"
#include "esp_attr.h" #include "esp_attr.h"
#include "soc/lldesc.h" #include "esp_crypto_dma.h"
#include "esp_heap_caps.h" #include "esp_heap_caps.h"
#include "esp_memory_utils.h" #include "esp_memory_utils.h"
#include "sys/param.h" #include "sys/param.h"
@@ -42,8 +42,8 @@
#endif #endif
#include "esp_crypto_lock.h" #include "esp_crypto_lock.h"
#include "hal/aes_hal.h" #include "hal/aes_hal.h"
#include "esp_aes_internal.h"
#include "esp_aes_dma_priv.h" #include "esp_aes_dma_priv.h"
#include "esp_aes_internal.h"
#if CONFIG_IDF_TARGET_ESP32S2 #if CONFIG_IDF_TARGET_ESP32S2
#include "esp32s2/rom/cache.h" #include "esp32s2/rom/cache.h"
@@ -103,25 +103,19 @@ static bool s_check_dma_capable(const void *p);
* * Must be in DMA capable memory, so stack is not a safe place to put them * * Must be in DMA capable memory, so stack is not a safe place to put them
* * To avoid having to malloc/free them for every DMA operation * * To avoid having to malloc/free them for every DMA operation
*/ */
static DRAM_ATTR lldesc_t s_stream_in_desc; static DRAM_ATTR crypto_dma_desc_t s_stream_in_desc;
static DRAM_ATTR lldesc_t s_stream_out_desc; static DRAM_ATTR crypto_dma_desc_t s_stream_out_desc;
static DRAM_ATTR uint8_t s_stream_in[AES_BLOCK_BYTES]; static DRAM_ATTR uint8_t s_stream_in[AES_BLOCK_BYTES];
static DRAM_ATTR uint8_t s_stream_out[AES_BLOCK_BYTES]; static DRAM_ATTR uint8_t s_stream_out[AES_BLOCK_BYTES];
static inline void esp_aes_wait_dma_done(lldesc_t *output) /** Append a descriptor to the chain, set head if chain empty
*
* @param[out] head Pointer to the first/head node of the DMA descriptor linked list
* @param item Pointer to the DMA descriptor node that has to be appended
*/
static inline void dma_desc_append(crypto_dma_desc_t **head, crypto_dma_desc_t *item)
{ {
/* Wait for DMA write operation to complete */ crypto_dma_desc_t *it;
while (1) {
if ( esp_aes_dma_done(output) ) {
break;
}
}
}
/* Append a descriptor to the chain, set head if chain empty */
static inline void lldesc_append(lldesc_t **head, lldesc_t *item)
{
lldesc_t *it;
if (*head == NULL) { if (*head == NULL) {
*head = item; *head = item;
return; return;
@@ -129,11 +123,61 @@ static inline void lldesc_append(lldesc_t **head, lldesc_t *item)
it = *head; it = *head;
while (it->empty != 0) { while (it->next != 0) {
it = (lldesc_t *)it->empty; it = (crypto_dma_desc_t *)it->next;
}
it->dw0.suc_eof = 0;
it->next = item;
}
/**
* Generate a linked list pointing to a (huge) buffer in an descriptor array.
*
* The caller should ensure there is enough size to hold the array, by calling
* `dma_desc_get_required_num` with the same or less than the max_desc_size argument.
*
* @param[out] dmadesc Output of a descriptor array, the head should be fed to the DMA.
* @param data Buffer for the descriptors to point to.
* @param len Size (or length for TX) of the buffer
* @param max_desc_size Maximum length of each descriptor
* @param isrx The RX DMA may require the buffer to be word-aligned, set to true for a RX link, otherwise false.
*/
static inline void dma_desc_setup_link(crypto_dma_desc_t* dmadesc, const void *data, int len, int max_desc_size, bool isrx)
{
int i = 0;
while (len) {
int dmachunklen = len;
if (dmachunklen > max_desc_size) {
dmachunklen = max_desc_size;
}
if (isrx) {
//Receive needs DMA length rounded to next 32-bit boundary
dmadesc[i].dw0.size = (dmachunklen + 3) & (~3);
dmadesc[i].dw0.length = (dmachunklen + 3) & (~3);
} else {
dmadesc[i].dw0.size = dmachunklen;
dmadesc[i].dw0.length = dmachunklen;
}
dmadesc[i].buffer = (void *)data;
dmadesc[i].dw0.suc_eof = 0;
dmadesc[i].dw0.owner = DMA_DESCRIPTOR_BUFFER_OWNER_DMA;
dmadesc[i].next = &dmadesc[i + 1];
len -= dmachunklen;
data += dmachunklen;
i++;
}
dmadesc[i - 1].dw0.suc_eof = 1; //Mark last DMA desc as end of stream.
dmadesc[i - 1].next = NULL;
}
static inline void esp_aes_wait_dma_done(crypto_dma_desc_t *output)
{
/* Wait for DMA write operation to complete */
while (1) {
if ( esp_aes_dma_done(output) ) {
break;
}
} }
it->eof = 0;
it->empty = (uint32_t)item;
} }
void esp_aes_acquire_hardware( void ) void esp_aes_acquire_hardware( void )
@@ -221,7 +265,7 @@ static esp_err_t esp_aes_isr_initialise( void )
#endif // CONFIG_MBEDTLS_AES_USE_INTERRUPT #endif // CONFIG_MBEDTLS_AES_USE_INTERRUPT
/* Wait for AES hardware block operation to complete */ /* Wait for AES hardware block operation to complete */
static int esp_aes_dma_wait_complete(bool use_intr, lldesc_t *output_desc) static int esp_aes_dma_wait_complete(bool use_intr, crypto_dma_desc_t *output_desc)
{ {
#if defined (CONFIG_MBEDTLS_AES_USE_INTERRUPT) #if defined (CONFIG_MBEDTLS_AES_USE_INTERRUPT)
if (use_intr) { if (use_intr) {
@@ -335,10 +379,10 @@ cleanup:
*/ */
static int esp_aes_process_dma(esp_aes_context *ctx, const unsigned char *input, unsigned char *output, size_t len, uint8_t *stream_out) static int esp_aes_process_dma(esp_aes_context *ctx, const unsigned char *input, unsigned char *output, size_t len, uint8_t *stream_out)
{ {
lldesc_t *in_desc_head = NULL, *out_desc_head = NULL; crypto_dma_desc_t *in_desc_head = NULL, *out_desc_head = NULL;
lldesc_t *out_desc_tail = NULL; /* pointer to the final output descriptor */ crypto_dma_desc_t *out_desc_tail = NULL; /* pointer to the final output descriptor */
lldesc_t *block_desc = NULL, *block_in_desc = NULL, *block_out_desc = NULL; crypto_dma_desc_t *block_desc = NULL, *block_in_desc = NULL, *block_out_desc = NULL;
size_t lldesc_num = 0; size_t crypto_dma_desc_num = 0;
unsigned stream_bytes = len % AES_BLOCK_BYTES; // bytes which aren't in a full block unsigned stream_bytes = len % AES_BLOCK_BYTES; // bytes which aren't in a full block
unsigned block_bytes = len - stream_bytes; // bytes which are in a full block unsigned block_bytes = len - stream_bytes; // bytes which are in a full block
unsigned blocks = (block_bytes / AES_BLOCK_BYTES) + ((stream_bytes > 0) ? 1 : 0); unsigned blocks = (block_bytes / AES_BLOCK_BYTES) + ((stream_bytes > 0) ? 1 : 0);
@@ -388,10 +432,10 @@ static int esp_aes_process_dma(esp_aes_context *ctx, const unsigned char *input,
} }
/* Set up dma descriptors for input and output considering the 16 byte alignment requirement for EDMA */ /* Set up dma descriptors for input and output considering the 16 byte alignment requirement for EDMA */
lldesc_num = lldesc_get_required_num_constrained(block_bytes, LLDESC_MAX_NUM_PER_DESC_16B_ALIGNED); crypto_dma_desc_num = dma_desc_get_required_num(block_bytes, DMA_DESCRIPTOR_BUFFER_MAX_SIZE_16B_ALIGNED);
/* Allocate both in and out descriptors to save a malloc/free per function call */ /* Allocate both in and out descriptors to save a malloc/free per function call */
block_desc = heap_caps_calloc(lldesc_num * 2, sizeof(lldesc_t), MALLOC_CAP_DMA); block_desc = heap_caps_calloc(crypto_dma_desc_num * 2, sizeof(crypto_dma_desc_t), MALLOC_CAP_DMA);
if (block_desc == NULL) { if (block_desc == NULL) {
mbedtls_platform_zeroize(output, len); mbedtls_platform_zeroize(output, len);
ESP_LOGE(TAG, "Failed to allocate memory"); ESP_LOGE(TAG, "Failed to allocate memory");
@@ -399,36 +443,37 @@ static int esp_aes_process_dma(esp_aes_context *ctx, const unsigned char *input,
} }
block_in_desc = block_desc; block_in_desc = block_desc;
block_out_desc = block_desc + lldesc_num; block_out_desc = block_desc + crypto_dma_desc_num;
dma_desc_setup_link(block_in_desc, input, block_bytes, DMA_DESCRIPTOR_BUFFER_MAX_SIZE_PER_DESC, 0);
lldesc_setup_link(block_in_desc, input, block_bytes, 0);
//Limit max inlink descriptor length to be 16 byte aligned, require for EDMA //Limit max inlink descriptor length to be 16 byte aligned, require for EDMA
lldesc_setup_link_constrained(block_out_desc, output, block_bytes, LLDESC_MAX_NUM_PER_DESC_16B_ALIGNED, 0); dma_desc_setup_link(block_out_desc, output, block_bytes, DMA_DESCRIPTOR_BUFFER_MAX_SIZE_16B_ALIGNED, 0);
/* Setup in/out start descriptors */ /* Setup in/out start descriptors */
lldesc_append(&in_desc_head, block_in_desc); dma_desc_append(&in_desc_head, block_in_desc);
lldesc_append(&out_desc_head, block_out_desc); dma_desc_append(&out_desc_head, block_out_desc);
out_desc_tail = &block_out_desc[lldesc_num - 1]; out_desc_tail = &block_out_desc[crypto_dma_desc_num - 1];
} }
/* Any leftover bytes which are appended as an additional DMA list */ /* Any leftover bytes which are appended as an additional DMA list */
if (stream_bytes > 0) { if (stream_bytes > 0) {
memset(&s_stream_in_desc, 0, sizeof(lldesc_t)); memset(&s_stream_in_desc, 0, sizeof(crypto_dma_desc_t));
memset(&s_stream_out_desc, 0, sizeof(lldesc_t)); memset(&s_stream_out_desc, 0, sizeof(crypto_dma_desc_t));
memset(s_stream_in, 0, AES_BLOCK_BYTES); memset(s_stream_in, 0, AES_BLOCK_BYTES);
memset(s_stream_out, 0, AES_BLOCK_BYTES); memset(s_stream_out, 0, AES_BLOCK_BYTES);
memcpy(s_stream_in, input + block_bytes, stream_bytes); memcpy(s_stream_in, input + block_bytes, stream_bytes);
lldesc_setup_link(&s_stream_in_desc, s_stream_in, AES_BLOCK_BYTES, 0); dma_desc_setup_link(&s_stream_in_desc, s_stream_in, AES_BLOCK_BYTES, DMA_DESCRIPTOR_BUFFER_MAX_SIZE_PER_DESC, 0);
lldesc_setup_link(&s_stream_out_desc, s_stream_out, AES_BLOCK_BYTES, 0); dma_desc_setup_link(&s_stream_out_desc, s_stream_out, AES_BLOCK_BYTES, DMA_DESCRIPTOR_BUFFER_MAX_SIZE_PER_DESC, 0);
/* Link with block descriptors */ /* Link with block descriptors */
lldesc_append(&in_desc_head, &s_stream_in_desc); dma_desc_append(&in_desc_head, &s_stream_in_desc);
lldesc_append(&out_desc_head, &s_stream_out_desc); dma_desc_append(&out_desc_head, &s_stream_out_desc);
out_desc_tail = &s_stream_out_desc; out_desc_tail = &s_stream_out_desc;
} }
@@ -494,13 +539,13 @@ cleanup:
* 3. If AES interrupt is enabled and ISR initialisation fails * 3. If AES interrupt is enabled and ISR initialisation fails
* 4. Failure in any of the AES operations * 4. Failure in any of the AES operations
*/ */
int esp_aes_process_dma_gcm(esp_aes_context *ctx, const unsigned char *input, unsigned char *output, size_t len, lldesc_t *aad_desc, size_t aad_len) int esp_aes_process_dma_gcm(esp_aes_context *ctx, const unsigned char *input, unsigned char *output, size_t len, crypto_dma_desc_t *aad_desc, size_t aad_len)
{ {
lldesc_t *in_desc_head = NULL, *out_desc_head = NULL, *len_desc = NULL; crypto_dma_desc_t *in_desc_head = NULL, *out_desc_head = NULL, *len_desc = NULL;
lldesc_t *out_desc_tail = NULL; /* pointer to the final output descriptor */ crypto_dma_desc_t *out_desc_tail = NULL; /* pointer to the final output descriptor */
lldesc_t stream_in_desc, stream_out_desc; crypto_dma_desc_t stream_in_desc, stream_out_desc;
lldesc_t *block_desc = NULL, *block_in_desc = NULL, *block_out_desc = NULL; crypto_dma_desc_t *block_desc = NULL, *block_in_desc = NULL, *block_out_desc = NULL;
size_t lldesc_num; size_t crypto_dma_desc_num = 0;
uint32_t len_buf[4] = {}; uint32_t len_buf[4] = {};
uint8_t stream_in[16] = {}; uint8_t stream_in[16] = {};
uint8_t stream_out[16] = {}; uint8_t stream_out[16] = {};
@@ -523,10 +568,10 @@ int esp_aes_process_dma_gcm(esp_aes_context *ctx, const unsigned char *input, un
} }
/* Set up dma descriptors for input and output */ /* Set up dma descriptors for input and output */
lldesc_num = lldesc_get_required_num(block_bytes); crypto_dma_desc_num = dma_desc_get_required_num(block_bytes, DMA_DESCRIPTOR_BUFFER_MAX_SIZE_PER_DESC);
/* Allocate both in and out descriptors to save a malloc/free per function call, add 1 for length descriptor */ /* Allocate both in and out descriptors to save a malloc/free per function call, add 1 for length descriptor */
block_desc = heap_caps_calloc( (lldesc_num * 2) + 1, sizeof(lldesc_t), MALLOC_CAP_DMA); block_desc = heap_caps_calloc((crypto_dma_desc_num * 2) + 1, sizeof(crypto_dma_desc_t), MALLOC_CAP_DMA);
if (block_desc == NULL) { if (block_desc == NULL) {
mbedtls_platform_zeroize(output, len); mbedtls_platform_zeroize(output, len);
ESP_LOGE(TAG, "Failed to allocate memory"); ESP_LOGE(TAG, "Failed to allocate memory");
@@ -534,32 +579,32 @@ int esp_aes_process_dma_gcm(esp_aes_context *ctx, const unsigned char *input, un
} }
block_in_desc = block_desc; block_in_desc = block_desc;
len_desc = block_desc + lldesc_num; len_desc = block_desc + crypto_dma_desc_num;
block_out_desc = block_desc + lldesc_num + 1; block_out_desc = block_desc + crypto_dma_desc_num + 1;
if (aad_desc != NULL) { if (aad_desc != NULL) {
lldesc_append(&in_desc_head, aad_desc); dma_desc_append(&in_desc_head, aad_desc);
} }
if (block_bytes > 0) { if (block_bytes > 0) {
lldesc_setup_link(block_in_desc, input, block_bytes, 0); dma_desc_setup_link(block_in_desc, input, block_bytes, DMA_DESCRIPTOR_BUFFER_MAX_SIZE_PER_DESC, 0);
lldesc_setup_link(block_out_desc, output, block_bytes, 0); dma_desc_setup_link(block_out_desc, output, block_bytes, DMA_DESCRIPTOR_BUFFER_MAX_SIZE_PER_DESC, 0);
lldesc_append(&in_desc_head, block_in_desc); dma_desc_append(&in_desc_head, block_in_desc);
lldesc_append(&out_desc_head, block_out_desc); dma_desc_append(&out_desc_head, block_out_desc);
out_desc_tail = &block_out_desc[lldesc_num - 1]; out_desc_tail = &block_out_desc[crypto_dma_desc_num - 1];
} }
/* Any leftover bytes which are appended as an additional DMA list */ /* Any leftover bytes which are appended as an additional DMA list */
if (stream_bytes > 0) { if (stream_bytes > 0) {
memcpy(stream_in, input + block_bytes, stream_bytes); memcpy(stream_in, input + block_bytes, stream_bytes);
lldesc_setup_link(&stream_in_desc, stream_in, AES_BLOCK_BYTES, 0); dma_desc_setup_link(&stream_in_desc, stream_in, AES_BLOCK_BYTES, DMA_DESCRIPTOR_BUFFER_MAX_SIZE_PER_DESC, 0);
lldesc_setup_link(&stream_out_desc, stream_out, AES_BLOCK_BYTES, 0); dma_desc_setup_link(&stream_out_desc, stream_out, AES_BLOCK_BYTES, DMA_DESCRIPTOR_BUFFER_MAX_SIZE_PER_DESC, 0);
lldesc_append(&in_desc_head, &stream_in_desc); dma_desc_append(&in_desc_head, &stream_in_desc);
lldesc_append(&out_desc_head, &stream_out_desc); dma_desc_append(&out_desc_head, &stream_out_desc);
out_desc_tail = &stream_out_desc; out_desc_tail = &stream_out_desc;
} }
@@ -568,13 +613,13 @@ int esp_aes_process_dma_gcm(esp_aes_context *ctx, const unsigned char *input, un
len_buf[1] = __builtin_bswap32(aad_len * 8); len_buf[1] = __builtin_bswap32(aad_len * 8);
len_buf[3] = __builtin_bswap32(len * 8); len_buf[3] = __builtin_bswap32(len * 8);
len_desc->length = sizeof(len_buf); len_desc->dw0.length = sizeof(len_buf);
len_desc->size = sizeof(len_buf); len_desc->dw0.size = sizeof(len_buf);
len_desc->owner = 1; len_desc->dw0.owner = 1;
len_desc->eof = 1; len_desc->dw0.suc_eof = 1;
len_desc->buf = (uint8_t *)len_buf; len_desc->buffer = (uint8_t *)len_buf;
lldesc_append(&in_desc_head, len_desc); dma_desc_append(&in_desc_head, len_desc);
#if defined (CONFIG_MBEDTLS_AES_USE_INTERRUPT) #if defined (CONFIG_MBEDTLS_AES_USE_INTERRUPT)
/* Only use interrupt for long AES operations */ /* Only use interrupt for long AES operations */

26
components/mbedtls/port/aes/dma/esp_aes_crypto_dma_impl.c
View File

@@ -1,18 +1,10 @@
// Copyright 2020 Espressif Systems (Shanghai) PTE LTD /*
// * SPDX-FileCopyrightText: 2020-2023 Espressif Systems (Shanghai) CO LTD
// Licensed under the Apache License, Version 2.0 (the "License"); *
// you may not use this file except in compliance with the License. * SPDX-License-Identifier: Apache-2.0
// You may obtain a copy of the License at */
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "esp_crypto_dma.h"
#include "esp_aes_dma_priv.h" #include "esp_aes_dma_priv.h"
#include "soc/soc_caps.h" #include "soc/soc_caps.h"
@@ -20,7 +12,7 @@
#include "hal/crypto_dma_ll.h" #include "hal/crypto_dma_ll.h"
esp_err_t esp_aes_dma_start(const lldesc_t *input, const lldesc_t *output) esp_err_t esp_aes_dma_start(const crypto_dma_desc_t *input, const crypto_dma_desc_t *output)
{ {
crypto_dma_ll_reset(); crypto_dma_ll_reset();
crypto_dma_ll_set_mode(CRYPTO_DMA_AES); crypto_dma_ll_set_mode(CRYPTO_DMA_AES);
@@ -36,7 +28,7 @@ esp_err_t esp_aes_dma_start(const lldesc_t *input, const lldesc_t *output)
return ESP_OK; return ESP_OK;
} }
bool esp_aes_dma_done(const lldesc_t *output) bool esp_aes_dma_done(const crypto_dma_desc_t *output)
{ {
return (crypto_dma_ll_inlink_is_eof() && (output->owner == 0)); return (crypto_dma_ll_inlink_is_eof() && (output->dw0.owner == 0));
} }

9
components/mbedtls/port/aes/dma/esp_aes_gdma_impl.c
View File

@@ -5,14 +5,15 @@
*/ */
#include "esp_aes_dma_priv.h" #include "esp_aes_dma_priv.h"
#include "esp_crypto_dma.h"
#include "esp_crypto_shared_gdma.h" #include "esp_crypto_shared_gdma.h"
esp_err_t esp_aes_dma_start(const lldesc_t *input, const lldesc_t *output) esp_err_t esp_aes_dma_start(const crypto_dma_desc_t *input, const crypto_dma_desc_t *output)
{ {
return esp_crypto_shared_gdma_start(input, output, GDMA_TRIG_PERIPH_AES); return esp_crypto_shared_gdma_start_axi_ahb(input, output, GDMA_TRIG_PERIPH_AES);
} }
bool esp_aes_dma_done(const lldesc_t *output) bool esp_aes_dma_done(const crypto_dma_desc_t *output)
{ {
return (output->owner == 0); return (output->dw0.owner == 0);
} }

7
components/mbedtls/port/aes/dma/include/esp_aes_dma_priv.h
View File

@@ -6,7 +6,8 @@
#pragma once #pragma once
#include "soc/lldesc.h" #include <stdbool.h>
#include "esp_crypto_dma.h"
#include "soc/soc_caps.h" #include "soc/soc_caps.h"
#include "esp_err.h" #include "esp_err.h"
@@ -23,7 +24,7 @@ extern "C" {
* - ESP_OK: Successfully started the DMA * - ESP_OK: Successfully started the DMA
* - ESP_ERR_INVALID_STATE: No DMA channel available * - ESP_ERR_INVALID_STATE: No DMA channel available
*/ */
esp_err_t esp_aes_dma_start(const lldesc_t *input, const lldesc_t *output); esp_err_t esp_aes_dma_start(const crypto_dma_desc_t *input, const crypto_dma_desc_t *output);
/** /**
* @brief Check if the DMA engine is finished reading the result * @brief Check if the DMA engine is finished reading the result
@@ -33,7 +34,7 @@ esp_err_t esp_aes_dma_start(const lldesc_t *input, const lldesc_t *output);
* - true: DMA finished * - true: DMA finished
* - false: DMA not yet finished * - false: DMA not yet finished
*/ */
bool esp_aes_dma_done(const lldesc_t *output); bool esp_aes_dma_done(const crypto_dma_desc_t *output);
/** /**
* @brief Allocate AES peripheral interrupt handler * @brief Allocate AES peripheral interrupt handler

24
components/mbedtls/port/aes/esp_aes_gcm.c
View File

@@ -672,8 +672,8 @@ int esp_aes_gcm_crypt_and_tag( esp_gcm_context *ctx,
#endif #endif
#if CONFIG_MBEDTLS_HARDWARE_GCM #if CONFIG_MBEDTLS_HARDWARE_GCM
int ret; int ret;
lldesc_t aad_desc[2] = {}; crypto_dma_desc_t aad_desc[2] = {};
lldesc_t *aad_head_desc = NULL; crypto_dma_desc_t *aad_head_desc = NULL;
size_t remainder_bit; size_t remainder_bit;
uint8_t stream_in[AES_BLOCK_BYTES] = {}; uint8_t stream_in[AES_BLOCK_BYTES] = {};
unsigned stream_bytes = aad_len % AES_BLOCK_BYTES; // bytes which aren't in a full block unsigned stream_bytes = aad_len % AES_BLOCK_BYTES; // bytes which aren't in a full block
@@ -687,7 +687,7 @@ int esp_aes_gcm_crypt_and_tag( esp_gcm_context *ctx,
/* Limit aad len to a single DMA descriptor to simplify DMA handling /* Limit aad len to a single DMA descriptor to simplify DMA handling
In practice, e.g. with mbedtls the length of aad will always be short In practice, e.g. with mbedtls the length of aad will always be short
*/ */
if (aad_len > LLDESC_MAX_NUM_PER_DESC) { if (aad_len > DMA_DESCRIPTOR_BUFFER_MAX_SIZE_PER_DESC) {
return MBEDTLS_ERR_GCM_BAD_INPUT; return MBEDTLS_ERR_GCM_BAD_INPUT;
} }
/* IV and AD are limited to 2^32 bits, so 2^29 bytes */ /* IV and AD are limited to 2^32 bits, so 2^29 bytes */
@@ -723,20 +723,20 @@ int esp_aes_gcm_crypt_and_tag( esp_gcm_context *ctx,
ctx->aes_ctx.key_in_hardware = aes_hal_setkey(ctx->aes_ctx.key, ctx->aes_ctx.key_bytes, mode); ctx->aes_ctx.key_in_hardware = aes_hal_setkey(ctx->aes_ctx.key, ctx->aes_ctx.key_bytes, mode);
if (block_bytes > 0) { if (block_bytes > 0) {
aad_desc[0].length = block_bytes; aad_desc[0].dw0.length = block_bytes;
aad_desc[0].size = block_bytes; aad_desc[0].dw0.size = block_bytes;
aad_desc[0].owner = 1; aad_desc[0].dw0.owner = 1;
aad_desc[0].buf = aad; aad_desc[0].buffer = (void*)aad;
} }
if (stream_bytes > 0) { if (stream_bytes > 0) {
memcpy(stream_in, aad + block_bytes, stream_bytes); memcpy(stream_in, aad + block_bytes, stream_bytes);
aad_desc[0].empty = (uint32_t)&aad_desc[1]; aad_desc[0].next = &aad_desc[1];
aad_desc[1].length = AES_BLOCK_BYTES; aad_desc[1].dw0.length = AES_BLOCK_BYTES;
aad_desc[1].size = AES_BLOCK_BYTES; aad_desc[1].dw0.size = AES_BLOCK_BYTES;
aad_desc[1].owner = 1; aad_desc[1].dw0.owner = 1;
aad_desc[1].buf = stream_in; aad_desc[1].buffer = (void*)stream_in;
} }
if (block_bytes > 0) { if (block_bytes > 0) {

8
components/mbedtls/port/aes/include/esp_aes_internal.h
View File

@@ -6,10 +6,10 @@
#pragma once #pragma once
#include "aes/esp_aes.h"
#include "aes/esp_aes_gcm.h"
#include "soc/soc_caps.h"
#include <stdbool.h> #include <stdbool.h>
#include "aes/esp_aes.h"
#include "soc/soc_caps.h"
#include "esp_crypto_dma.h"
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
@@ -29,7 +29,7 @@ bool valid_key_length(const esp_aes_context *ctx);
* @param aad_len GCM additional data length * @param aad_len GCM additional data length
* @return int -1 on error * @return int -1 on error
*/ */
int esp_aes_process_dma_gcm(esp_aes_context *ctx, const unsigned char *input, unsigned char *output, size_t len, lldesc_t *aad_desc, size_t aad_len); int esp_aes_process_dma_gcm(esp_aes_context *ctx, const unsigned char *input, unsigned char *output, size_t len, crypto_dma_desc_t *aad_desc, size_t aad_len);
#endif #endif
#ifdef __cplusplus #ifdef __cplusplus

88
components/mbedtls/port/crypto_shared_gdma/esp_crypto_shared_gdma.c
View File

@@ -1,5 +1,5 @@
/* /*
* SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD * SPDX-FileCopyrightText: 2021-2023 Espressif Systems (Shanghai) CO LTD
* *
* SPDX-License-Identifier: Apache-2.0 * SPDX-License-Identifier: Apache-2.0
*/ */
@@ -8,12 +8,21 @@
#include "freertos/FreeRTOS.h" #include "freertos/FreeRTOS.h"
#include "freertos/task.h" #include "freertos/task.h"
#include "hal/gdma_ll.h"
#include "soc/soc_caps.h"
#include "esp_log.h" #include "esp_log.h"
#include "esp_err.h" #include "esp_err.h"
#include "esp_crypto_dma.h"
#include "esp_crypto_lock.h" #include "esp_crypto_lock.h"
#include "soc/soc_caps.h"
#if CONFIG_IDF_TARGET_ESP32P4
#include "esp32p4/rom/cache.h"
#endif
#if SOC_AHB_GDMA_VERSION == 1
#include "hal/gdma_ll.h"
#elif SOC_AXI_GDMA_SUPPORTED
#include "hal/axi_dma_ll.h"
#endif /* SOC_AHB_GDMA_VERSION */
#define NEW_CHANNEL_TIMEOUT_MS 1000 #define NEW_CHANNEL_TIMEOUT_MS 1000
#define NEW_CHANNEL_DELAY_MS 100 #define NEW_CHANNEL_DELAY_MS 100
@@ -26,12 +35,15 @@ static gdma_channel_handle_t tx_channel;
/* Allocate a new GDMA channel, will keep trying until NEW_CHANNEL_TIMEOUT_MS */ /* Allocate a new GDMA channel, will keep trying until NEW_CHANNEL_TIMEOUT_MS */
static inline esp_err_t crypto_shared_gdma_new_channel(gdma_channel_alloc_config_t *channel_config, gdma_channel_handle_t *channel) static inline esp_err_t crypto_shared_gdma_new_channel(gdma_channel_alloc_config_t *channel_config, gdma_channel_handle_t *channel)
{ {
esp_err_t ret; esp_err_t ret = ESP_FAIL;
int time_waited_ms = 0; int time_waited_ms = 0;
while (1) { while (1) {
ret = gdma_new_channel(channel_config, channel); #if SOC_AXI_GDMA_SUPPORTED
ret = gdma_new_axi_channel(channel_config, channel);
#else /* !SOC_AXI_GDMA_SUPPORTED */
ret = gdma_new_ahb_channel(channel_config, channel);
#endif /* SOC_AXI_GDMA_SUPPORTED */
if (ret == ESP_OK) { if (ret == ESP_OK) {
break; break;
} else if (time_waited_ms >= NEW_CHANNEL_TIMEOUT_MS) { } else if (time_waited_ms >= NEW_CHANNEL_TIMEOUT_MS) {
@@ -92,7 +104,6 @@ err:
return ret; return ret;
} }
esp_err_t esp_crypto_shared_gdma_start(const lldesc_t *input, const lldesc_t *output, gdma_trigger_peripheral_t peripheral) esp_err_t esp_crypto_shared_gdma_start(const lldesc_t *input, const lldesc_t *output, gdma_trigger_peripheral_t peripheral)
{ {
int rx_ch_id = 0; int rx_ch_id = 0;
@@ -121,7 +132,68 @@ esp_err_t esp_crypto_shared_gdma_start(const lldesc_t *input, const lldesc_t *ou
/* tx channel is reset by gdma_connect(), also reset rx to ensure a known state */ /* tx channel is reset by gdma_connect(), also reset rx to ensure a known state */
gdma_get_channel_id(rx_channel, &rx_ch_id); gdma_get_channel_id(rx_channel, &rx_ch_id);
#if SOC_AHB_GDMA_VERSION == 1
gdma_ll_rx_reset_channel(&GDMA, rx_ch_id); gdma_ll_rx_reset_channel(&GDMA, rx_ch_id);
#endif /* SOC_AHB_GDMA_VERSION */
gdma_start(tx_channel, (intptr_t)input);
gdma_start(rx_channel, (intptr_t)output);
return ESP_OK;
}
esp_err_t esp_crypto_shared_gdma_start_axi_ahb(const crypto_dma_desc_t *input, const crypto_dma_desc_t *output, gdma_trigger_peripheral_t peripheral)
{
int rx_ch_id = 0;
esp_err_t ret = ESP_OK;
#if SOC_CACHE_INTERNAL_MEM_VIA_L1CACHE
// TODO: replace with `esp_cache_msync`
const crypto_dma_desc_t *it = input;
while(it != NULL) {
Cache_WriteBack_Addr(CACHE_MAP_L1_DCACHE | CACHE_MAP_L2_CACHE, (uint32_t)it->buffer, it->dw0.length); // try using esp_cache_msync()
Cache_WriteBack_Addr(CACHE_MAP_L1_DCACHE | CACHE_MAP_L2_CACHE, (uint32_t)it, sizeof(crypto_dma_desc_t));
it = (const crypto_dma_desc_t*) it->next;
}
it = output;
while(it != NULL) {
Cache_WriteBack_Addr(CACHE_MAP_L1_DCACHE | CACHE_MAP_L2_CACHE, (uint32_t)it->buffer, it->dw0.length);
Cache_WriteBack_Addr(CACHE_MAP_L1_DCACHE | CACHE_MAP_L2_CACHE, (uint32_t)it, sizeof(crypto_dma_desc_t));
it = (const crypto_dma_desc_t*) it->next;
};
#endif /* SOC_CACHE_INTERNAL_MEM_VIA_L1CACHE */
if (tx_channel == NULL) {
/* Allocate a pair of RX and TX for crypto, should only happen the first time we use the GMDA
or if user called esp_crypto_shared_gdma_release */
ret = crypto_shared_gdma_init();
}
if (ret != ESP_OK) {
return ret;
}
/* Tx channel is shared between AES and SHA, need to connect to peripheral every time */
gdma_disconnect(tx_channel);
if (peripheral == GDMA_TRIG_PERIPH_SHA) {
gdma_connect(tx_channel, GDMA_MAKE_TRIGGER(GDMA_TRIG_PERIPH_SHA, 0));
} else if (peripheral == GDMA_TRIG_PERIPH_AES) {
gdma_connect(tx_channel, GDMA_MAKE_TRIGGER(GDMA_TRIG_PERIPH_AES, 0));
} else {
return ESP_ERR_INVALID_ARG;
}
/* tx channel is reset by gdma_connect(), also reset rx to ensure a known state */
gdma_get_channel_id(rx_channel, &rx_ch_id);
#if SOC_AHB_GDMA_VERSION == 1
gdma_ll_rx_reset_channel(&GDMA, rx_ch_id);
#elif SOC_AXI_GDMA_SUPPORTED
axi_dma_ll_rx_reset_channel(&AXI_DMA, rx_ch_id);
#endif /* SOC_AHB_GDMA_VERSION */
gdma_start(tx_channel, (intptr_t)input); gdma_start(tx_channel, (intptr_t)input);
gdma_start(rx_channel, (intptr_t)output); gdma_start(rx_channel, (intptr_t)output);

1
components/mbedtls/port/include/aes/esp_aes_gcm.h
View File

@@ -12,7 +12,6 @@
#include "aes/esp_aes.h" #include "aes/esp_aes.h"
#include "mbedtls/cipher.h" #include "mbedtls/cipher.h"
#include "soc/lldesc.h"
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {

45
components/mbedtls/port/include/esp_crypto_dma.h Normal file
View File

@@ -0,0 +1,45 @@
/*
* SPDX-FileCopyrightText: 2023-2024 Espressif Systems (Shanghai) CO LTD
*
* SPDX-License-Identifier: Apache-2.0
*/
#pragma once
#include "hal/dma_types.h"
#include "soc/gdma_channel.h"
#include "soc/soc_caps.h"
#ifdef __cplusplus
extern "C"
{
#endif
#if (SOC_AES_SUPPORT_DMA) && (SOC_SHA_SUPPORT_DMA)
#if (SOC_AES_GDMA) && (SOC_SHA_GDMA)
#if (SOC_GDMA_TRIG_PERIPH_AES0_BUS == SOC_GDMA_BUS_AHB) && (SOC_GDMA_TRIG_PERIPH_SHA0_BUS == SOC_GDMA_BUS_AHB)
#define DMA_DESC_MEM_ALIGN_SIZE 4
typedef dma_descriptor_align4_t crypto_dma_desc_t;
#elif (SOC_GDMA_TRIG_PERIPH_AES0_BUS == SOC_GDMA_BUS_AXI) && (SOC_GDMA_TRIG_PERIPH_SHA0_BUS == SOC_GDMA_BUS_AXI)
#define DMA_DESC_MEM_ALIGN_SIZE 8
typedef dma_descriptor_align8_t crypto_dma_desc_t;
#else
#error "As we support a shared crypto GDMA layer for the AES and the SHA peripheral, both the peripherals must use the same GDMA bus"
#endif /* (SOC_GDMA_TRIG_PERIPH_AES0_BUS == SOC_GDMA_BUS_AHB) && (SOC_GDMA_TRIG_PERIPH_AES0_BUS == SOC_GDMA_BUS_AHB) */
#elif (SOC_AES_CRYPTO_DMA) && (SOC_SHA_CRYPTO_DMA)
#define DMA_DESC_MEM_ALIGN_SIZE 4
typedef dma_descriptor_align4_t crypto_dma_desc_t;
#endif /* (SOC_AES_GDMA) && (SOC_SHA_GDMA) */
#endif /* (SOC_AES_SUPPORT_DMA) && (SOC_SHA_SUPPORT_DMA) */
#ifdef __cplusplus
}
#endif

23
components/mbedtls/port/include/esp_crypto_shared_gdma.h
View File

@@ -1,14 +1,15 @@
/* /*
* SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD * SPDX-FileCopyrightText: 2021-2023 Espressif Systems (Shanghai) CO LTD
* *
* SPDX-License-Identifier: Apache-2.0 * SPDX-License-Identifier: Apache-2.0
*/ */
#pragma once #pragma once
#include "soc/lldesc.h" #include "esp_crypto_dma.h"
#include "esp_private/gdma.h" #include "esp_private/gdma.h"
#include "esp_err.h" #include "esp_err.h"
#include "soc/lldesc.h"
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
@@ -16,17 +17,31 @@ extern "C" {
/** /**
* @brief Start a GDMA transfer on the shared crypto DMA channel * @brief Start a GDMA transfer on the shared crypto DMA channel
* Only supports AHB-DMA.
* *
* @note Will allocate a GDMA channel for AES & SHA if no such channel is already allocated * @note Will allocate a GDMA channel for AES & SHA if no such channel is already allocated
* *
* @param input Input linked list descriptor * @param input Input linked list descriptor (lldesc_t *)
* @param output Output linked list descriptor * @param output Output linked list descriptor (lldesc_t *)
* @param peripheral Crypto peripheral to connect the DMA to, either GDMA_TRIG_PERIPH_AES or * @param peripheral Crypto peripheral to connect the DMA to, either GDMA_TRIG_PERIPH_AES or
* GDMA_TRIG_PERIPH_SHA * GDMA_TRIG_PERIPH_SHA
* @return esp_err_t ESP_FAIL if no GDMA channel available * @return esp_err_t ESP_FAIL if no GDMA channel available
*/ */
esp_err_t esp_crypto_shared_gdma_start(const lldesc_t *input, const lldesc_t *output, gdma_trigger_peripheral_t peripheral); esp_err_t esp_crypto_shared_gdma_start(const lldesc_t *input, const lldesc_t *output, gdma_trigger_peripheral_t peripheral);
/**
* @brief Start a GDMA transfer on the shared crypto DMA channel
* Supports AXI-DMA and AHB-DMA.
*
* @note Will allocate a GDMA channel for AES & SHA if no such channel is already allocated
*
* @param input Input linked list descriptor (crypto_dma_desc_t *)
* @param output Output linked list descriptor (crypto_dma_desc_t *)
* @param peripheral Crypto peripheral to connect the DMA to, either GDMA_TRIG_PERIPH_AES or
* GDMA_TRIG_PERIPH_SHA
* @return esp_err_t ESP_FAIL if no GDMA channel available
*/
esp_err_t esp_crypto_shared_gdma_start_axi_ahb(const crypto_dma_desc_t *input, const crypto_dma_desc_t *output, gdma_trigger_peripheral_t peripheral);
/** /**
* @brief Frees any shared crypto DMA channel, if esp_crypto_shared_gdma_start is called after * @brief Frees any shared crypto DMA channel, if esp_crypto_shared_gdma_start is called after

2
components/mbedtls/test_apps/main/test_aes_sha_parallel.c
View File

@@ -112,8 +112,6 @@ static void tskRunAES256Test(void *pvParameters)
} }
#include "esp_crypto_shared_gdma.h"
#define TASK_STACK_SIZE (20*1024) #define TASK_STACK_SIZE (20*1024)
TEST_CASE("mbedtls AES/SHA multithreading", "[mbedtls]") TEST_CASE("mbedtls AES/SHA multithreading", "[mbedtls]")

1
tools/ci/check_copyright_ignore.txt
View File

@@ -504,7 +504,6 @@ components/lwip/include/apps/ping/ping.h
components/mbedtls/esp_crt_bundle/test_gen_crt_bundle/test_gen_crt_bundle.py components/mbedtls/esp_crt_bundle/test_gen_crt_bundle/test_gen_crt_bundle.py
components/mbedtls/port/aes/block/esp_aes.c components/mbedtls/port/aes/block/esp_aes.c
components/mbedtls/port/aes/dma/esp_aes.c components/mbedtls/port/aes/dma/esp_aes.c
components/mbedtls/port/aes/dma/esp_aes_crypto_dma_impl.c
components/mbedtls/port/aes/esp_aes_xts.c components/mbedtls/port/aes/esp_aes_xts.c
components/mbedtls/port/include/aes/esp_aes.h components/mbedtls/port/include/aes/esp_aes.h
components/mbedtls/port/include/aes_alt.h components/mbedtls/port/include/aes_alt.h