Merge branch 'feat/adding_different_strategy_to_perform_tls_using_dynamic_feature' into 'master'

Add configuration to control dynamic buffer strategy in mbedtls

Closes IDF-12591

See merge request espressif/esp-idf!39469

components/esp-tls/esp_tls.h

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: 2017-2024 Espressif Systems (Shanghai) CO LTD
+ * SPDX-FileCopyrightText: 2017-2025 Espressif Systems (Shanghai) CO LTD
  *
  * SPDX-License-Identifier: Apache-2.0
  */
@@ -91,6 +91,12 @@ typedef enum {
    ESP_TLS_VER_TLS_MAX,         /* to indicate max */
 } esp_tls_proto_ver_t;
 
+typedef enum {
+    ESP_TLS_DYN_BUF_RX_STATIC = 1,    /*!< Strategy to disable dynamic RX buffer allocations and convert to static allocation post-handshake, reducing memory fragmentation */
+    ESP_TLS_DYN_BUF_STRATEGY_MAX,     /*!< to indicate max */
+} esp_tls_dyn_buf_strategy_t;
+
+
 /**
  * @brief      ESP-TLS configuration parameters
  *
@@ -213,6 +219,11 @@ typedef struct esp_tls_cfg {
     const int *ciphersuites_list;           /*!< Pointer to a zero-terminated array of IANA identifiers of TLS ciphersuites.
                                                 Please check the list validity by esp_tls_get_ciphersuites_list() API */
     esp_tls_proto_ver_t tls_version;        /*!< TLS protocol version of the connection, e.g., TLS 1.2, TLS 1.3 (default - no preference) */
+
+#if CONFIG_MBEDTLS_DYNAMIC_BUFFER
+    esp_tls_dyn_buf_strategy_t esp_tls_dyn_buf_strategy; /*!< ESP-TLS dynamic buffer strategy */
+#endif
+
 } esp_tls_cfg_t;
 
 #if defined(CONFIG_ESP_TLS_SERVER_SESSION_TICKETS)

components/esp-tls/esp_tls_mbedtls.c

@@ -20,7 +20,7 @@
 #include <errno.h>
 #include "esp_log.h"
 #include "esp_check.h"
-
+#include "mbedtls/esp_mbedtls_dynamic.h"
 #ifdef CONFIG_MBEDTLS_HARDWARE_ECDSA_SIGN
 #include "ecdsa/ecdsa_alt.h"
 #endif
@@ -115,6 +115,10 @@ esp_err_t esp_create_mbedtls_handle(const char *hostname, size_t hostlen, const
 
     mbedtls_ssl_conf_rng(&tls->conf, mbedtls_ctr_drbg_random, &tls->ctr_drbg);
 
+#if CONFIG_MBEDTLS_DYNAMIC_BUFFER
+    tls->esp_tls_dyn_buf_strategy = ((esp_tls_cfg_t *)cfg)->esp_tls_dyn_buf_strategy;
+#endif
+
     if (tls->role == ESP_TLS_CLIENT) {
         esp_ret = set_client_config(hostname, hostlen, (esp_tls_cfg_t *)cfg, tls);
         if (esp_ret != ESP_OK) {
@@ -256,6 +260,15 @@ int esp_mbedtls_handshake(esp_tls_t *tls, const esp_tls_cfg_t *cfg)
 #endif
     ret = mbedtls_ssl_handshake(&tls->ssl);
     if (ret == 0) {
+#if CONFIG_MBEDTLS_DYNAMIC_BUFFER
+        if (tls->esp_tls_dyn_buf_strategy != 0) {
+            ret = esp_mbedtls_dynamic_set_rx_buf_static(&tls->ssl);
+            if (ret != 0) {
+                ESP_LOGE(TAG, "esp_mbedtls_dynamic_set_rx_buf_static returned -0x%04X", -ret);
+                return ret;
+            }
+        }
+#endif
         tls->conn_state = ESP_TLS_DONE;
 
 #ifdef CONFIG_ESP_TLS_USE_DS_PERIPHERAL

components/esp-tls/private_include/esp_tls_private.h

@@ -98,6 +98,10 @@ struct esp_tls {
 
     esp_tls_error_handle_t error_handle;                                        /*!< handle to error descriptor */
 
+#if CONFIG_MBEDTLS_DYNAMIC_BUFFER
+    esp_tls_dyn_buf_strategy_t esp_tls_dyn_buf_strategy;                        /*!< ESP-TLS dynamic buffer strategy */
+#endif
+
 };
 
 // Function pointer for the server configuration API

components/esp_http_client/esp_http_client.c

@@ -34,6 +34,8 @@ static const char *TAG = "HTTP_CLIENT";
 
 ESP_STATIC_ASSERT((int)ESP_HTTP_CLIENT_TLS_VER_ANY == (int)ESP_TLS_VER_ANY, "Enum mismatch in esp_http_client and esp-tls");
 ESP_STATIC_ASSERT((int)ESP_HTTP_CLIENT_TLS_VER_MAX <= (int)ESP_TLS_VER_TLS_MAX, "HTTP client supported TLS is not supported in esp-tls");
+ESP_STATIC_ASSERT((int)HTTP_TLS_DYN_BUF_RX_STATIC == (int)ESP_TLS_DYN_BUF_RX_STATIC, "Enum mismatch in esp_http_client and esp-tls");
+ESP_STATIC_ASSERT((int)HTTP_TLS_DYN_BUF_STRATEGY_MAX <= (int)ESP_TLS_DYN_BUF_STRATEGY_MAX, "HTTP client supported TLS is not supported in esp-tls");
 
 #if CONFIG_ESP_HTTP_CLIENT_EVENT_POST_TIMEOUT == -1
 #define ESP_HTTP_CLIENT_EVENT_POST_TIMEOUT portMAX_DELAY
@@ -844,6 +846,14 @@ esp_http_client_handle_t esp_http_client_init(const esp_http_client_config_t *co
     }
     esp_transport_ssl_set_tls_version(ssl, config->tls_version);
 
+#if CONFIG_MBEDTLS_DYNAMIC_BUFFER
+    /* When tls_dyn_buf_strategy is 0, mbedTLS dynamic buffer allocation uses default behavior.
+     * No need to call esp_transport_ssl_set_esp_tls_dyn_buf_strategy() in this case */
+    if (config->tls_dyn_buf_strategy != 0 && config->tls_dyn_buf_strategy < HTTP_TLS_DYN_BUF_STRATEGY_MAX) {
+        esp_transport_ssl_set_esp_tls_dyn_buf_strategy(ssl, config->tls_dyn_buf_strategy);
+    }
+#endif
+
 #if CONFIG_ESP_TLS_USE_SECURE_ELEMENT
     if (config->use_secure_element) {
         esp_transport_ssl_use_secure_element(ssl);

components/esp_http_client/include/esp_http_client.h

@@ -139,6 +139,11 @@ typedef enum {
     HTTP_ADDR_TYPE_INET6 = AF_INET6,        /**< IPv6 address family. */
 } esp_http_client_addr_type_t;
 
+typedef enum {
+    HTTP_TLS_DYN_BUF_RX_STATIC = 1,     /*!< Strategy to disable dynamic RX buffer allocations and convert to static allocation post-handshake, reducing memory fragmentation */
+    HTTP_TLS_DYN_BUF_STRATEGY_MAX,      /*!< to indicate max */
+} esp_http_client_tls_dyn_buf_strategy_t;
+
 /**
  * @brief HTTP configuration
  */
@@ -215,6 +220,10 @@ typedef struct {
     struct esp_transport_item_t *transport;
 #endif
     esp_http_client_addr_type_t addr_type;  /*!< Address type used in http client configurations */
+
+#if CONFIG_MBEDTLS_DYNAMIC_BUFFER
+    esp_http_client_tls_dyn_buf_strategy_t tls_dyn_buf_strategy; /*!< TLS dynamic buffer strategy */
+#endif
 } esp_http_client_config_t;
 
 /**

components/mbedtls/port/dynamic/esp_mbedtls_dynamic_impl.c

@@ -18,6 +18,17 @@
 
 #define TX_IDLE_BUFFER_SIZE (MBEDTLS_SSL_HEADER_LEN + CACHE_BUFFER_SIZE)
 
+#define ESP_MBEDTLS_RETURN_IF_RX_BUF_STATIC(ssl) \
+    do { \
+        if (ssl->MBEDTLS_PRIVATE(in_buf)) { \
+            esp_mbedtls_ssl_buf_states state = esp_mbedtls_get_buf_state(ssl->MBEDTLS_PRIVATE(in_buf)); \
+            if (state == ESP_MBEDTLS_SSL_BUF_STATIC) { \
+                return 0; \
+            } \
+        } \
+    } while(0)
+
+
 static const char *TAG = "Dynamic Impl";
 
 static void esp_mbedtls_set_buf_state(unsigned char *buf, esp_mbedtls_ssl_buf_states state)
@@ -140,6 +151,29 @@ static void init_rx_buffer(mbedtls_ssl_context *ssl, unsigned char *buf)
     ssl->MBEDTLS_PRIVATE(in_left) = 0;
 }
 
+esp_err_t esp_mbedtls_dynamic_set_rx_buf_static(mbedtls_ssl_context *ssl)
+{
+    unsigned char cache_buf[16];
+    memcpy(cache_buf, ssl->MBEDTLS_PRIVATE(in_buf), 16);
+    esp_mbedtls_reset_free_rx_buffer(ssl);
+
+    struct esp_mbedtls_ssl_buf *esp_buf;
+    int buffer_len = tx_buffer_len(ssl, MBEDTLS_SSL_IN_BUFFER_LEN);
+    esp_buf = mbedtls_calloc(1, SSL_BUF_HEAD_OFFSET_SIZE + buffer_len);
+    if (!esp_buf) {
+        ESP_LOGE(TAG, "rx buf alloc(%d bytes) failed", SSL_BUF_HEAD_OFFSET_SIZE + buffer_len);
+        return ESP_ERR_NO_MEM;
+    }
+    esp_mbedtls_init_ssl_buf(esp_buf, buffer_len);
+    init_rx_buffer(ssl, esp_buf->buf);
+
+    memcpy(ssl->MBEDTLS_PRIVATE(in_ctr), cache_buf, 8);
+    memcpy(ssl->MBEDTLS_PRIVATE(in_iv), cache_buf + 8, 8);
+    esp_mbedtls_set_buf_state(ssl->MBEDTLS_PRIVATE(in_buf), ESP_MBEDTLS_SSL_BUF_STATIC);
+    return ESP_OK;
+
+}
+
 static int esp_mbedtls_alloc_tx_buf(mbedtls_ssl_context *ssl, int len)
 {
     struct esp_mbedtls_ssl_buf *esp_buf;
@@ -324,6 +358,12 @@ exit:
 
 int esp_mbedtls_add_rx_buffer(mbedtls_ssl_context *ssl)
 {
+    /*
+     * If RX buffer is set to static mode, this macro will return early
+     * and skip dynamic buffer allocation logic below
+     */
+    ESP_MBEDTLS_RETURN_IF_RX_BUF_STATIC(ssl);
+
     int cached = 0;
     int ret = 0;
     int buffer_len;
@@ -405,6 +445,12 @@ exit:
 
 int esp_mbedtls_free_rx_buffer(mbedtls_ssl_context *ssl)
 {
+    /*
+     * If RX buffer is set to static mode, this macro will return early
+     * and skip dynamic buffer free logic below
+     */
+    ESP_MBEDTLS_RETURN_IF_RX_BUF_STATIC(ssl);
+
     int ret = 0;
     unsigned char buf[16];
     struct esp_mbedtls_ssl_buf *esp_buf;

components/mbedtls/port/dynamic/esp_mbedtls_dynamic_impl.h

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: 2020-2022 Espressif Systems (Shanghai) CO LTD
+ * SPDX-FileCopyrightText: 2020-2025 Espressif Systems (Shanghai) CO LTD
  *
  * SPDX-License-Identifier: Apache-2.0
  */
@@ -26,6 +26,8 @@
 #include "esp_log.h"
 #include "sdkconfig.h"
 
+#include "mbedtls/esp_mbedtls_dynamic.h"
+
 #define TRACE_CHECK(_fn, _state) \
 ({ \
     ESP_LOGV(TAG, "%d " _state " to do \"%s\"", __LINE__, # _fn); \
@@ -48,8 +50,9 @@
 })
 
 typedef enum {
-    ESP_MBEDTLS_SSL_BUF_CACHED,
+    ESP_MBEDTLS_SSL_BUF_CACHED = 0,
     ESP_MBEDTLS_SSL_BUF_NO_CACHED,
+    ESP_MBEDTLS_SSL_BUF_STATIC,
 } esp_mbedtls_ssl_buf_states;
 
 struct esp_mbedtls_ssl_buf {

components/mbedtls/port/include/mbedtls/esp_mbedtls_dynamic.h

@@ -0,0 +1,28 @@
+/*
+ * SPDX-FileCopyrightText: 2025 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#pragma once
+
+#include "mbedtls/ssl.h"
+#include "esp_err.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * @brief Set the dynamic buffer rx statically after the handshake. This is to avoid frequent allocation and deallocation of dynamic buffer.
+ *
+ * @param ssl mbedtls ssl context
+ * @return esp_err_t
+ *         - ESP_OK: Successfully set the rx buffer to static
+ *         - ESP_ERR_NO_MEM: Failed to allocate memory for the rx buffer
+ */
+esp_err_t esp_mbedtls_dynamic_set_rx_buf_static(mbedtls_ssl_context *ssl);
+
+#ifdef __cplusplus
+}
+#endif

components/tcp_transport/include/esp_transport_ssl.h

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: 2015-2024 Espressif Systems (Shanghai) CO LTD
+ * SPDX-FileCopyrightText: 2015-2025 Espressif Systems (Shanghai) CO LTD
  *
  * SPDX-License-Identifier: Apache-2.0
  */
@@ -61,6 +61,15 @@ void esp_transport_ssl_crt_bundle_attach(esp_transport_handle_t t, esp_err_t ((*
  */
 void esp_transport_ssl_enable_global_ca_store(esp_transport_handle_t t);
 
+#if CONFIG_MBEDTLS_DYNAMIC_BUFFER
+/**
+ * @brief      Set ESP-TLS dynamic buffer strategy for ESP-TLS connection
+ *
+ * @param      t    ssl transport
+ * @param[in]  strategy      ESP-TLS dynamic buffer strategy
+ */
+void esp_transport_ssl_set_esp_tls_dyn_buf_strategy(esp_transport_handle_t t, esp_tls_dyn_buf_strategy_t strategy);
+#endif
 /**
  * @brief      Set TLS protocol version for ESP-TLS connection
  *

components/tcp_transport/transport_ssl.c

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: 2015-2024 Espressif Systems (Shanghai) CO LTD
+ * SPDX-FileCopyrightText: 2015-2025 Espressif Systems (Shanghai) CO LTD
  *
  * SPDX-License-Identifier: Apache-2.0
  */
@@ -365,6 +365,14 @@ void esp_transport_ssl_enable_global_ca_store(esp_transport_handle_t t)
     ssl->cfg.use_global_ca_store = true;
 }
 
+#if CONFIG_MBEDTLS_DYNAMIC_BUFFER
+void esp_transport_ssl_set_esp_tls_dyn_buf_strategy(esp_transport_handle_t t, esp_tls_dyn_buf_strategy_t strategy)
+{
+    GET_SSL_FROM_TRANSPORT_OR_RETURN(ssl, t);
+    ssl->cfg.esp_tls_dyn_buf_strategy = strategy;
+}
+#endif
+
 void esp_transport_ssl_set_tls_version(esp_transport_handle_t t, esp_tls_proto_ver_t tls_version)
 {
     GET_SSL_FROM_TRANSPORT_OR_RETURN(ssl, t);

examples/system/ota/simple_ota_example/main/Kconfig.projbuild

@@ -51,4 +51,13 @@ menu "Example Configuration"
             help
                 Select ethernet interface to pass the OTA data.
     endchoice
+
+    config EXAMPLE_TLS_DYN_BUF_RX_STATIC
+        bool "Use static rx buffer for dynamic buffer after TLS handshake"
+        depends on MBEDTLS_DYNAMIC_BUFFER
+        default n
+        help
+            This converts the dynamic RX buffer to static allocation after the TLS handshake
+            is complete. This reduces memory fragmentation by avoiding repeated dynamic
+            allocations during data transfer.
 endmenu

examples/system/ota/simple_ota_example/main/simple_ota_example.c

@@ -104,6 +104,12 @@ void simple_ota_example_task(void *pvParameter)
 #ifdef CONFIG_EXAMPLE_FIRMWARE_UPGRADE_BIND_IF
         .if_name = &ifr,
 #endif
+#if CONFIG_EXAMPLE_TLS_DYN_BUF_RX_STATIC
+        /* This part applies static buffer strategy for rx dynamic buffer.
+         * This is to avoid frequent allocation and deallocation of dynamic buffer.
+         */
+        .tls_dyn_buf_strategy = HTTP_TLS_DYN_BUF_RX_STATIC,
+#endif /* CONFIG_EXAMPLE_TLS_DYN_BUF_RX_STATIC */
     };
 
 #ifdef CONFIG_EXAMPLE_FIRMWARE_UPGRADE_URL_FROM_STDIN

examples/system/ota/simple_ota_example/pytest_simple_ota.py

@@ -426,6 +426,46 @@ def test_examples_protocol_simple_ota_example_tls1_3(dut: Dut) -> None:
         tls1_3_server.kill()
 
 
+@pytest.mark.ethernet_ota
+@pytest.mark.parametrize(
+    'config',
+    [
+        'tls1_2_dynamic',
+    ],
+    indirect=True,
+)
+@idf_parametrize('target', ['esp32'], indirect=['target'])
+def test_examples_protocol_simple_ota_example_tls1_2_dynamic(dut: Dut) -> None:
+    """
+    steps: |
+      1. join AP/Ethernet
+      2. Fetch OTA image over HTTPS
+      3. Reboot with the new OTA image
+    """
+    sha256_bootloader, sha256_app = calc_all_sha256(dut)
+    # Start server
+    thread1 = multiprocessing.Process(target=start_https_server, args=(dut.app.binary_path, '0.0.0.0', 8000))
+    thread1.daemon = True
+    thread1.start()
+    try:
+        # start test
+        dut.expect(f'Loaded app from partition at offset {OTA_0_ADDRESS}', timeout=30)
+        check_sha256(sha256_bootloader, str(dut.expect(r'SHA-256 for bootloader:\s+([a-f0-9]){64}')[0]))
+        check_sha256(sha256_app, str(dut.expect(r'SHA-256 for current firmware:\s+([a-f0-9]){64}')[0]))
+
+        host_ip = setting_connection(dut)
+
+        dut.expect('Starting OTA example task', timeout=30)
+        print(f'writing to device: https://{host_ip}:8000/simple_ota.bin')
+        dut.write(f'https://{host_ip}:8000/simple_ota.bin')
+        dut.expect('OTA Succeed, Rebooting...', timeout=120)
+        # after reboot
+        dut.expect(f'Loaded app from partition at offset {OTA_1_ADDRESS}', timeout=30)
+        dut.expect('OTA example app_main start', timeout=10)
+    finally:
+        thread1.terminate()
+
+
 if __name__ == '__main__':
     if sys.argv[2:]:  # if two or more arguments provided:
         # Usage: pytest_simple_ota.py <image_dir> <server_port> [cert_dir]

examples/system/ota/simple_ota_example/sdkconfig.ci.tls1_2_dynamic

@@ -0,0 +1,15 @@
+CONFIG_EXAMPLE_FIRMWARE_UPGRADE_URL="FROM_STDIN"
+CONFIG_EXAMPLE_SKIP_COMMON_NAME_CHECK=y
+CONFIG_EXAMPLE_CONNECT_ETHERNET=y
+CONFIG_EXAMPLE_CONNECT_WIFI=n
+CONFIG_EXAMPLE_USE_INTERNAL_ETHERNET=y
+CONFIG_EXAMPLE_ETH_PHY_IP101=y
+CONFIG_EXAMPLE_ETH_MDC_GPIO=23
+CONFIG_EXAMPLE_ETH_MDIO_GPIO=18
+CONFIG_EXAMPLE_ETH_PHY_RST_GPIO=5
+CONFIG_EXAMPLE_ETH_PHY_ADDR=1
+CONFIG_EXAMPLE_CONNECT_IPV6=y
+CONFIG_MBEDTLS_SSL_PROTO_TLS1_2=y
+CONFIG_MBEDTLS_SSL_PROTO_TLS1_3=y
+CONFIG_MBEDTLS_DYNAMIC_BUFFER=y
+CONFIG_EXAMPLE_TLS_DYN_BUF_RX_STATIC=y