feat(modem): Added test that performs OTA to exercise modem layers

components/esp_modem/include/vfs_resource/vfs_create.hpp

@@ -31,7 +31,7 @@
  */
 struct esp_modem_vfs_uart_creator {
     const char *dev_name;                       /*!< VFS device name, e.g. /dev/uart/n */
-    const struct esp_modem_uart_term_config uart;     /*!< UART driver init struct */
+    struct esp_modem_uart_term_config uart;     /*!< UART driver init struct */
 };
 
 /**

components/esp_modem/test/target_ota/CMakeLists.txt

@@ -0,0 +1,8 @@
+# The following lines of boilerplate have to be in your project's CMakeLists
+# in this exact order for cmake to work correctly
+cmake_minimum_required(VERSION 3.8)
+
+set(EXTRA_COMPONENT_DIRS "../.." "../../examples/modem_tcp_client/components")
+
+include($ENV{IDF_PATH}/tools/cmake/project.cmake)
+project(ota_test)

components/esp_modem/test/target_ota/README.md

@@ -0,0 +1,21 @@
+# Target test running OTA update
+
+## Overview
+
+The aim of this test is to exercise the most commonly failing scenario, running OTA over PPPoS with https.
+
+This project opens a data session, runs basic mqtt operations and initiates OTA update.
+It supports the following test configurations:
+* Using a real modem device (default config)
+* Using VFS device (only to exercise VFS DTE)
+* Using network-only DCE (connecting directly to PPP server) -- needs some configuration
+
+### Configuring the PPP server
+
+You need to run these applications on your host machine:
+* PPP server
+```bash
+sudo pppd /dev/ttyUSB1 115200 192.168.11.1:192.168.11.2 ms-dns 8.8.8.8 modem local noauth debug nocrtscts nodetach +ipv6
+```
+* MQTT broker: Running mosquitto in the default config is enough, configuring the broker's URL to the local PPP address: `config.broker.address.uri = "mqtt://192.168.11.1";`
+* HTTP server: Need to support HTTP/1.1 (to support ranges). You can use the script `http_server.py` and configure the OTA endpoint as `"https://192.168.11.1:1234/esp32.bin"`

components/esp_modem/test/target_ota/components/manual_ota/CMakeLists.txt

@@ -0,0 +1,3 @@
+idf_component_register(SRCS manual_ota.cpp transport_batch_tls.cpp
+                       INCLUDE_DIRS "."
+                       PRIV_REQUIRES extra_tcp_transports esp_http_client app_update)

components/esp_modem/test/target_ota/components/manual_ota/manual_ota.cpp

@@ -0,0 +1,259 @@
+/*
+ * SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Unlicense OR CC0-1.0
+ */
+#include "manual_ota.hpp"
+#include "esp_log.h"
+#include "esp_ota_ops.h"
+#include "esp_app_format.h"
+#include "esp_http_client.h"
+#include "esp_partition.h"
+#include "esp_transport_tcp.h"
+#include "transport_batch_tls.hpp"
+
+static const char *TAG = "manual_ota";
+
+bool manual_ota::begin()
+{
+    if (status != state::UNDEF) {
+        ESP_LOGE(TAG, "Invalid state");
+        return false;
+    }
+    status = state::INIT;
+    esp_transport_handle_t tcp = esp_transport_tcp_init();
+    ssl_ = esp_transport_batch_tls_init(tcp, max_buffer_size_);
+
+    esp_http_client_config_t config = { };
+    config.skip_cert_common_name_check = true;
+    config.url = uri_;
+    config.transport = ssl_;
+    const esp_partition_t *configured = esp_ota_get_boot_partition();
+    const esp_partition_t *running = esp_ota_get_running_partition();
+
+    if (configured != running) {
+        ESP_LOGE(TAG, "Configured OTA boot partition at offset 0x%08" PRIx32 ", but running from offset 0x%08" PRIx32, configured->address, running->address);
+        return false;
+    }
+
+    http_ = esp_http_client_init(&config);
+    if (http_ == nullptr) {
+        ESP_LOGE(TAG, "Failed to initialise HTTP connection");
+        return false;
+    }
+    esp_http_client_set_method(http_, HTTP_METHOD_HEAD);
+    esp_err_t err = esp_http_client_perform(http_);
+    if (err == ESP_OK) {
+        int http_status = esp_http_client_get_status_code(http_);
+        if (http_status != HttpStatus_Ok) {
+            ESP_LOGE(TAG, "Received incorrect http status %d", http_status);
+            return false;
+        }
+    } else {
+        ESP_LOGE(TAG, "ESP HTTP client perform failed: %d", err);
+        return false;
+    }
+    image_length_ = esp_http_client_get_content_length(http_);
+    ESP_LOGI(TAG, "image_length = %lld", image_length_);
+    esp_http_client_close(http_);
+
+    if (image_length_ > size_) {
+        char *header_val = nullptr;
+        asprintf(&header_val, "bytes=0-%d", max_buffer_size_ - 1);
+        if (header_val == nullptr) {
+            ESP_LOGE(TAG, "Failed to allocate memory for HTTP header");
+            return false;
+        }
+        esp_http_client_set_header(http_, "Range", header_val);
+        free(header_val);
+    }
+    esp_http_client_set_method(http_, HTTP_METHOD_GET);
+
+    partition_ = esp_ota_get_next_update_partition(nullptr);
+    if (partition_ == nullptr) {
+        ESP_LOGE(TAG, "Invalid update partition");
+        return false;
+    }
+    ESP_LOGI(TAG, "Writing to partition subtype %d at offset 0x%" PRIx32, partition_->subtype, partition_->address);
+
+    file_length_ = 0;
+    reconnect_attempts_ = 0;
+    buffer_.resize(max_buffer_size_);
+    status = state::IMAGE_CHECK;
+    return true;
+}
+
+bool manual_ota::perform()
+{
+    if (status != state::IMAGE_CHECK && status != state::START) {
+        ESP_LOGE(TAG, "Invalid state");
+        return false;
+    }
+    esp_err_t err = esp_http_client_open(http_, 0);
+    if (err != ESP_OK) {
+        if (image_length_ == file_length_) {
+            status = state::END;
+            return false;
+        }
+
+        esp_http_client_close(http_);
+        ESP_LOGI(TAG, "Failed to open HTTP connection: %s", esp_err_to_name(err));
+        if (reconnect_attempts_++ < max_reconnect_attempts_) {
+            if (prepare_reconnect()) {
+                return true;    // will retry in the next iteration
+            }
+        }
+        return fail_cleanup();
+    }
+    esp_http_client_fetch_headers(http_);
+
+    int batch_len = esp_transport_batch_tls_pre_read(ssl_, max_buffer_size_, timeout_ * 1000);
+    if (batch_len < 0) {
+        ESP_LOGE(TAG, "Error: Failed to pre-read plain text data!");
+        return fail_cleanup();
+    }
+
+    int data_read = esp_http_client_read(http_, buffer_.data(), batch_len);
+
+    if (data_read < 0) {
+        ESP_LOGE(TAG, "Error: SSL data read error");
+        return fail_cleanup();
+    } else if (data_read > 0) {
+        esp_http_client_close(http_);
+
+        if (status == state::IMAGE_CHECK) {
+            esp_app_desc_t new_app_info;
+            if (data_read > sizeof(esp_image_header_t) + sizeof(esp_image_segment_header_t) + sizeof(esp_app_desc_t)) {
+                // check current version with downloading
+                memcpy(&new_app_info, &buffer_[sizeof(esp_image_header_t) + sizeof(esp_image_segment_header_t)], sizeof(esp_app_desc_t));
+                ESP_LOGI(TAG, "New firmware version: %s", new_app_info.version);
+
+                esp_app_desc_t running_app_info;
+                const esp_partition_t *running = esp_ota_get_running_partition();
+                if (esp_ota_get_partition_description(running, &running_app_info) == ESP_OK) {
+                    ESP_LOGI(TAG, "Running firmware version: %s", running_app_info.version);
+                }
+
+                const esp_partition_t *last_invalid_app = esp_ota_get_last_invalid_partition();
+                esp_app_desc_t invalid_app_info;
+                if (esp_ota_get_partition_description(last_invalid_app, &invalid_app_info) == ESP_OK) {
+                    ESP_LOGI(TAG, "Last invalid firmware version: %s", invalid_app_info.version);
+                }
+
+                // check current version with last invalid partition
+                if (last_invalid_app != NULL) {
+                    if (memcmp(invalid_app_info.version, new_app_info.version, sizeof(new_app_info.version)) == 0) {
+                        ESP_LOGW(TAG, "New version is the same as invalid version.");
+                        ESP_LOGW(TAG, "Previously, there was an attempt to launch the firmware with %s version, but it failed.", invalid_app_info.version);
+                        ESP_LOGW(TAG, "The firmware has been rolled back to the previous version.");
+                        return fail_cleanup();
+                    }
+                }
+
+                status = state::START;
+                err = esp_ota_begin(partition_, OTA_WITH_SEQUENTIAL_WRITES, &update_handle_);
+                if (err != ESP_OK) {
+                    ESP_LOGE(TAG, "esp_ota_begin failed (%s)", esp_err_to_name(err));
+                    esp_ota_abort(update_handle_);
+                    return fail_cleanup();
+                }
+                ESP_LOGI(TAG, "esp_ota_begin succeeded");
+            } else {
+                ESP_LOGE(TAG, "Received chunk doesn't contain app descriptor");
+                esp_ota_abort(update_handle_);
+                return fail_cleanup();
+            }
+        }
+        err = esp_ota_write(update_handle_, (const void *)buffer_.data(), data_read);
+        if (err != ESP_OK) {
+            esp_ota_abort(update_handle_);
+            return fail_cleanup();
+        }
+        file_length_ += data_read;
+        ESP_LOGI(TAG, "Written image length %d", file_length_);
+
+        if (image_length_ == file_length_) {
+            status = state::END;
+            return false;
+        }
+
+        if (!prepare_reconnect()) {
+            esp_ota_abort(update_handle_);
+            return fail_cleanup();
+        }
+
+    } else if (data_read == 0) {
+        if (file_length_ == 0) {
+            // Try to handle possible HTTP redirections
+            int status_code = esp_http_client_get_status_code(http_);
+            ESP_LOGW(TAG, "Status code: %d", status_code);
+            err = esp_http_client_set_redirection(http_);
+            if (err != ESP_OK) {
+                ESP_LOGE(TAG, "URL redirection Failed");
+                esp_ota_abort(update_handle_);
+                return fail_cleanup();
+            }
+
+            err = esp_http_client_open(http_, 0);
+            if (err != ESP_OK) {
+                ESP_LOGE(TAG, "Failed to open HTTP connection: %s", esp_err_to_name(err));
+                return fail_cleanup();
+            }
+            esp_http_client_fetch_headers(http_);
+        }
+    }
+
+    return true;
+}
+
+bool manual_ota::prepare_reconnect()
+{
+    esp_http_client_set_method(http_, HTTP_METHOD_GET);
+    char *header_val = nullptr;
+    if ((image_length_ - file_length_) > max_buffer_size_) {
+        asprintf(&header_val, "bytes=%d-%d", file_length_, (file_length_ + max_buffer_size_ - 1));
+    } else {
+        asprintf(&header_val, "bytes=%d-", file_length_);
+    }
+    if (header_val == nullptr) {
+        ESP_LOGE(TAG, "Failed to allocate memory for HTTP header");
+        return false;
+    }
+    esp_http_client_set_header(http_, "Range", header_val);
+    free(header_val);
+    return true;
+}
+
+bool manual_ota::fail_cleanup()
+{
+    esp_http_client_close(http_);
+    esp_http_client_cleanup(http_);
+    status = state::FAIL;
+    return false;
+}
+
+bool manual_ota::end()
+{
+    if (status == state::END) {
+        if (!esp_http_client_is_complete_data_received(http_)) {
+            ESP_LOGE(TAG, "Error in receiving complete file");
+            return fail_cleanup();
+        }
+        esp_err_t err = esp_ota_end(update_handle_);
+        if (err != ESP_OK) {
+            if (err == ESP_ERR_OTA_VALIDATE_FAILED) {
+                ESP_LOGE(TAG, "Image validation failed, image is corrupted");
+            } else {
+                ESP_LOGE(TAG, "esp_ota_end failed (%s)!", esp_err_to_name(err));
+            }
+            return fail_cleanup();
+        }
+        err = esp_ota_set_boot_partition(partition_);
+        if (err != ESP_OK) {
+            ESP_LOGE(TAG, "esp_ota_set_boot_partition failed (%s)!", esp_err_to_name(err));
+            return fail_cleanup();
+        }
+        return true;
+    }
+    return false;
+}

components/esp_modem/test/target_ota/components/manual_ota/manual_ota.hpp

@@ -0,0 +1,71 @@
+/*
+ * SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Unlicense OR CC0-1.0
+ */
+#pragma once
+#include <vector>
+#include "esp_http_client.h"
+#include "esp_partition.h"
+#include "esp_transport_tcp.h"
+#include "esp_ota_ops.h"
+
+class manual_ota {
+public:
+    enum class state {
+        UNDEF,
+        INIT,
+        IMAGE_CHECK,
+        START,
+        END,
+        FAIL,
+    };
+    size_t size_{32};
+    int timeout_{2};
+
+    /**
+     * @brief Construct a new manual ota object
+     *
+     * @param uri URI of the binary image
+     */
+    explicit manual_ota(const char *uri): uri_(uri) {}
+
+    /**
+     * @brief Start the manual OTA process
+     *
+     * @return true if started successfully
+     */
+    bool begin();
+
+    /**
+     * @brief Performs one read-write OTA iteration
+     *
+     * @return true if the process is in progress
+     * @return false if the process finished, call end() to get OTA result
+     */
+    bool perform();
+
+    /**
+     * @brief Finishes an OTA update
+     *
+     * @return true if the OTA update completed successfully
+     */
+    bool end();
+
+private:
+    const char *uri_{};
+    esp_http_client_handle_t http_;
+    int64_t image_length_;
+    size_t file_length_;
+    const size_t max_buffer_size_{size_ * 1024};
+    const esp_partition_t *partition_{nullptr};
+    state status{state::UNDEF};
+    std::vector<char> buffer_{};
+    int reconnect_attempts_;
+    const int max_reconnect_attempts_{3};
+    esp_transport_handle_t ssl_;
+    esp_ota_handle_t update_handle_{0};
+
+    bool prepare_reconnect();
+    bool fail_cleanup();
+};

components/esp_modem/test/target_ota/components/manual_ota/transport_batch_tls.cpp

@@ -0,0 +1,232 @@
+/*
+ * SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Unlicense OR CC0-1.0
+ */
+#include <vector>
+#include "esp_log.h"
+#include "mbedtls_wrap.hpp"
+#include "esp_transport_tcp.h"
+
+#define TAG "batch-tls"
+
+class TlsTransport: public Tls {
+public:
+    explicit TlsTransport(esp_transport_handle_t parent) : Tls(), transport_(parent), read_len(0), offset(0) {}
+    int send(const unsigned char *buf, size_t len) override;
+    int recv(unsigned char *buf, size_t len) override;
+    static bool set_func(esp_transport_handle_t tls_transport);
+    int preread(size_t len, int timeout_ms);
+    bool prepare_buffer(size_t max_size);
+private:
+    esp_transport_handle_t transport_{};
+    int connect(const char *host, int port, int timeout_ms);
+    void delay() override;
+
+    struct transport {
+        static int connect(esp_transport_handle_t t, const char *host, int port, int timeout_ms);
+        static int read(esp_transport_handle_t t, char *buffer, int len, int timeout_ms);
+        static int write(esp_transport_handle_t t, const char *buffer, int len, int timeout_ms);
+        static int close(esp_transport_handle_t t);
+        static int poll_read(esp_transport_handle_t t, int timeout_ms);
+        static int poll_write(esp_transport_handle_t t, int timeout_ms);
+        static int destroy(esp_transport_handle_t t);
+    };
+    int last_timeout;
+    std::vector<char> buf;
+    size_t read_len;
+    size_t offset;
+};
+
+esp_transport_handle_t esp_transport_tls_init(esp_transport_handle_t parent)
+{
+    esp_transport_handle_t ssl = esp_transport_init();
+    auto *tls = new TlsTransport(parent);
+    esp_transport_set_context_data(ssl, tls);
+    TlsTransport::set_func(ssl);
+    return ssl;
+}
+
+int TlsTransport::send(const unsigned char *buf, size_t len)
+{
+    int ret = esp_transport_write(transport_, reinterpret_cast<const char *>(buf), len, 0);
+    ESP_LOGD(TAG, "writing(len=%d) ret=%d", len, ret);
+    return ret;
+}
+
+int TlsTransport::recv(unsigned char *buffer, size_t len)
+{
+    ESP_LOGD(TAG, "recv(len=%d)", len);
+    if (read_len != 0) {
+
+        if (read_len > len) {
+            memcpy((char *)buffer, buf.data() + offset, len);
+            read_len -= len;
+            offset += len;
+            ESP_LOGD(TAG, "read %d from batch read_len = %d", len, read_len);
+            return len;
+        } else {
+            int remaining = len = read_len;
+            if (remaining > 0) {
+                memcpy((char *)buffer, buf.data() + offset, remaining);
+                read_len = 0;
+                offset = 0;
+                return remaining;
+
+            }
+            read_len = 0;
+            offset = 0;
+            return ERR_TCP_TRANSPORT_CONNECTION_CLOSED_BY_FIN;
+        }
+    }
+    int ret = esp_transport_read(transport_, reinterpret_cast<char *>(buffer), len, last_timeout);
+
+    if (ret == ERR_TCP_TRANSPORT_CONNECTION_TIMEOUT) {
+        return MBEDTLS_ERR_SSL_WANT_READ;
+    }
+    return ret == ERR_TCP_TRANSPORT_CONNECTION_CLOSED_BY_FIN ? 0 : ret;
+}
+
+bool TlsTransport::set_func(esp_transport_handle_t tls_transport)
+{
+    return esp_transport_set_func(tls_transport, TlsTransport::transport::connect, TlsTransport::transport::read, TlsTransport::transport::write, TlsTransport::transport::close, TlsTransport::transport::poll_read, TlsTransport::transport::poll_write, TlsTransport::transport::destroy) == ESP_OK;
+}
+
+int TlsTransport::connect(const char *host, int port, int timeout_ms)
+{
+    return esp_transport_connect(transport_, host, port, timeout_ms);
+}
+
+void TlsTransport::delay()
+{
+    vTaskDelay(pdMS_TO_TICKS(500));
+}
+
+int TlsTransport::transport::connect(esp_transport_handle_t t, const char *host, int port, int timeout_ms)
+{
+    auto tls = static_cast<TlsTransport *>(esp_transport_get_context_data(t));
+    tls->init(is_server{false}, do_verify{false});
+
+    ESP_LOGD(TAG, "TLS-connect");
+    auto ret = tls->connect(host, port, timeout_ms);
+    if (ret < 0) {
+        ESP_LOGI(TAG, "Failed to connect to transport");
+        return ret;
+    }
+    if (tls->is_session_loaded()) {
+        tls->set_session();
+    }
+    ESP_LOGI(TAG, "Before handshake");
+    ret = tls->handshake();
+    if (ret < 0) {
+        ESP_LOGI(TAG, "Failed to handshake");
+        return ret;
+    }
+    tls->get_session();
+    ESP_LOGI(TAG, "After handshake");
+    return 0;
+}
+
+int TlsTransport::transport::read(esp_transport_handle_t t, char *buffer, int len, int timeout_ms)
+{
+    auto tls = static_cast<TlsTransport *>(esp_transport_get_context_data(t));
+    ESP_LOGD(TAG, "available=%d tls->read_len=%d", tls->get_available_bytes(), tls->read_len);
+    if (tls->get_available_bytes() <= 0 && tls->read_len == 0) {
+        ESP_LOGD(TAG, "red(len=%d, timeout=%d) tls->read_len=%d", len, timeout_ms, tls->read_len);
+        tls->last_timeout = timeout_ms;
+        int poll = esp_transport_poll_read(t, timeout_ms);
+        if (poll == -1) {
+            return ERR_TCP_TRANSPORT_CONNECTION_FAILED;
+        }
+        if (poll == 0) {
+            return ERR_TCP_TRANSPORT_CONNECTION_TIMEOUT;
+        }
+    }
+
+    auto ret =  tls->read(reinterpret_cast<unsigned char *>(buffer), len);
+    if (ret == MBEDTLS_ERR_SSL_WANT_READ) {
+        ret = ERR_TCP_TRANSPORT_CONNECTION_TIMEOUT;
+    }
+    ESP_LOGD(TAG, "red(len=%d, timeout=%d) ret=%d", len, timeout_ms, ret);
+    return ret;
+}
+
+int TlsTransport::transport::write(esp_transport_handle_t t, const char *buffer, int len, int timeout_ms)
+{
+    int poll;
+    if ((poll = esp_transport_poll_write(t, timeout_ms)) <= 0) {
+        ESP_LOGW(TAG, "Poll timeout or error timeout_ms=%d", timeout_ms);
+        return poll;
+    }
+
+    auto tls = static_cast<TlsTransport *>(esp_transport_get_context_data(t));
+    int ret = tls->write(reinterpret_cast<const unsigned char *>(buffer), len);
+    ESP_LOGD(TAG, "write ret=%d", ret);
+    return ret;
+}
+
+int TlsTransport::transport::close(esp_transport_handle_t t)
+{
+    auto tls = static_cast<TlsTransport *>(esp_transport_get_context_data(t));
+    int ret = esp_transport_close(tls->transport_);
+    tls->deinit();
+    return ret;
+}
+
+int TlsTransport::transport::poll_read(esp_transport_handle_t t, int timeout_ms)
+{
+    auto tls = static_cast<TlsTransport *>(esp_transport_get_context_data(t));
+    return esp_transport_poll_read(tls->transport_, timeout_ms);
+}
+
+int TlsTransport::transport::poll_write(esp_transport_handle_t t, int timeout_ms)
+{
+    auto tls = static_cast<TlsTransport *>(esp_transport_get_context_data(t));
+    return esp_transport_poll_write(tls->transport_, timeout_ms);
+}
+
+int TlsTransport::transport::destroy(esp_transport_handle_t t)
+{
+    auto tls = static_cast<TlsTransport *>(esp_transport_get_context_data(t));
+    return esp_transport_destroy(tls->transport_);
+}
+
+
+esp_transport_handle_t esp_transport_batch_tls_init(esp_transport_handle_t parent, const size_t max_buffer_size)
+{
+    esp_transport_handle_t ssl = esp_transport_init();
+    auto *tls = new TlsTransport(parent);
+    esp_transport_set_context_data(ssl, tls);
+    TlsTransport::set_func(ssl);
+    tls->prepare_buffer(max_buffer_size);
+    return ssl;
+}
+
+int TlsTransport::preread(size_t len, int timeout_ms)
+{
+    while (len != read_len) {
+        int l = esp_transport_read(transport_, buf.data() + read_len, len - read_len, timeout_ms);
+        ESP_LOGD(TAG, "need %d read %d already %d", len, l, read_len);
+        if ((l == ERR_TCP_TRANSPORT_CONNECTION_CLOSED_BY_FIN || l == ERR_TCP_TRANSPORT_CONNECTION_TIMEOUT ) && read_len > 0) {
+            return read_len;
+        }
+        if (l <= 0) {
+            read_len = 0;
+            return read_len;
+        }
+        read_len += l;
+    }
+    return read_len;
+}
+
+bool TlsTransport::prepare_buffer(size_t max_size)
+{
+    buf.resize(max_size);
+    return true;
+}
+
+int esp_transport_batch_tls_pre_read(esp_transport_handle_t t, int len, int timeout_ms)
+{
+    auto tls = static_cast<TlsTransport *>(esp_transport_get_context_data(t));
+    return tls->preread(len, timeout_ms);
+}

components/esp_modem/test/target_ota/components/manual_ota/transport_batch_tls.hpp

@@ -0,0 +1,25 @@
+/*
+ * SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Unlicense OR CC0-1.0
+ */
+#pragma once
+
+/**
+ * @brief Creates batch transport
+ *
+ * @param parent tcp-transport handle to the parent transport
+ * @param max_buffer_size maximum size of one batch
+ * @return created transport handle
+ */
+esp_transport_handle_t esp_transport_batch_tls_init(esp_transport_handle_t parent, const size_t max_buffer_size);
+
+/**
+ * @brief Performs batch read operation from the underlying transport
+ *
+ * @param t Transport handle
+ * @param len Batch size
+ * @param timeout_ms Timeout in ms
+ * @return true If read from the parent transport completed successfully
+ */
+bool esp_transport_batch_tls_pre_read(esp_transport_handle_t t, int len, int timeout_ms);

components/esp_modem/test/target_ota/http_server.py

@@ -0,0 +1,15 @@
+# SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Unlicense OR CC0-1.0
+import ssl
+from http.server import HTTPServer
+
+from RangeHTTPServer import RangeRequestHandler
+
+server_address = ('0.0.0.0', 1234)
+httpd = HTTPServer(server_address, RangeRequestHandler)
+httpd.socket = ssl.wrap_socket(httpd.socket,
+                               server_side=True,
+                               certfile='srv.crt',
+                               keyfile='srv.key',
+                               ssl_version=ssl.PROTOCOL_TLS)
+httpd.serve_forever()

components/esp_modem/test/target_ota/main/CMakeLists.txt

@@ -0,0 +1,2 @@
+idf_component_register(SRCS ota_test.cpp
+                       INCLUDE_DIRS ".")

components/esp_modem/test/target_ota/main/Kconfig.projbuild

@@ -0,0 +1,47 @@
+menu "Test Configuration"
+
+    choice TEST_DEVICE
+        prompt "Choose supported modem device (DCE)"
+        default TEST_DEVICE_MODEM_GENERIC
+        help
+            Select modem device connected to the ESP DTE.
+
+        config TEST_DEVICE_MODEM_GENERIC
+            bool "Common modem device"
+            help
+                Generic device that could be used with most common modems (BG96, SIM76xx, A76xx).
+
+        config TEST_DEVICE_PPPD_SERVER
+            bool "PPPD Server"
+            help
+                Test device is a pppd service in server mode, running on linux.
+    endchoice
+
+    config TEST_MODEM_APN
+        string "Modem APN"
+        depends on TEST_DEVICE_MODEM_GENERIC
+        default "lpwa.vodafone.com"
+        help
+            Set APN (Access Point Name), a logical name to choose data network
+
+    config TEST_USE_VFS_TERM
+        bool "Use VFS terminal"
+        default n
+        help
+            Demonstrate use of VFS as a communication terminal of the DTE.
+            VFS driver implements non-block reads, writes and selects to communicate with esp-modem,
+            but this implementation uses UART resource only.
+
+    config TEST_OTA_URI
+        string "URI of the binary"
+        default "https://192.168.11.1/esp32.bin"
+        help
+            HTTPS address of the update binary.
+
+    config BROKER_URI
+        string "Broker URL"
+        default "mqtt://test.mosquitto.org"
+        help
+            URL of an mqtt broker which this example connects to.
+
+endmenu

components/esp_modem/test/target_ota/main/network_dce.hpp

@@ -0,0 +1,55 @@
+/*
+ * SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Unlicense OR CC0-1.0
+ */
+
+#include "cxx_include/esp_modem_dte.hpp"
+#include "esp_modem_config.h"
+#include "cxx_include/esp_modem_api.hpp"
+#include "cxx_include/esp_modem_dce_factory.hpp"
+#include <memory>
+#include <utility>
+
+class NetModule;
+
+/**
+ * @brief Custom factory which can build and create a DCE using a custom module
+ */
+class NetDCE_Factory: public esp_modem::dce_factory::Factory {
+public:
+    template <typename T, typename ...Args>
+    static auto create(const esp_modem::dce_factory::config *cfg, Args &&... args) -> std::shared_ptr<esp_modem::DCE_T<T>>
+    {
+        return build_generic_DCE<T, esp_modem::DCE_T<T>, std::shared_ptr<esp_modem::DCE_T<T>>>(cfg, std::forward<Args>(args)...);
+    }
+};
+
+/**
+ * @brief This is a null-module, doesn't define any AT commands, just passes everything to pppd
+ */
+class NetModule: public esp_modem::ModuleIf {
+public:
+    explicit NetModule(std::shared_ptr<esp_modem::DTE> dte, const esp_modem_dce_config *cfg):
+        dte(std::move(dte)) {}
+
+    bool setup_data_mode() override
+    {
+        return true;
+    }
+
+    bool set_mode(esp_modem::modem_mode mode) override
+    {
+        return true;
+    }
+
+private:
+    std::shared_ptr<esp_modem::DTE> dte;
+};
+
+
+std::shared_ptr<esp_modem::DCE_T<NetModule>> create(std::shared_ptr<esp_modem::DTE> dte, esp_netif_t *netif)
+{
+    const esp_modem::dce_config config = {};
+    return NetDCE_Factory::create<NetModule>(&config, dte, netif);
+}

components/esp_modem/test/target_ota/main/ota_test.cpp

@@ -0,0 +1,295 @@
+/*
+ * SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Unlicense OR CC0-1.0
+ */
+
+#include <cstring>
+#include "freertos/FreeRTOS.h"
+#include "freertos/event_groups.h"
+#include "esp_netif.h"
+#include "esp_netif_ppp.h"
+#include "esp_log.h"
+#include "esp_event.h"
+#include "cxx_include/esp_modem_dte.hpp"
+#include "esp_modem_config.h"
+#include "cxx_include/esp_modem_api.hpp"
+#include "esp_vfs_dev.h"        // For optional VFS support
+#include "vfs_resource/vfs_create.hpp"
+#include "network_dce.hpp"
+#include "manual_ota.hpp"
+#include "mqtt_client.h"
+
+using namespace esp_modem;
+
+static const char *TAG = "ota_test";
+
+
+// Wrap event handlers to destruct correctly on returning from main
+class StatusHandler {
+public:
+    static constexpr auto IP_Event      = SignalGroup::bit0;
+    static constexpr auto MQTT_Connect  = SignalGroup::bit1;
+    static constexpr auto MQTT_Data     = SignalGroup::bit2;
+
+    StatusHandler()
+    {
+        ESP_ERROR_CHECK(esp_event_handler_register(IP_EVENT, ESP_EVENT_ANY_ID, on_event, this));
+    }
+
+    ~StatusHandler()
+    {
+        esp_event_handler_unregister(IP_EVENT, ESP_EVENT_ANY_ID, on_event);
+    }
+
+    void handle_mqtt(esp_mqtt_client_handle_t client)
+    {
+        mqtt = client;
+        esp_mqtt_client_register_event(client, MQTT_EVENT_ANY, on_event, this);
+    }
+
+    void remove_mqtt()
+    {
+        esp_mqtt_client_unregister_event(mqtt, MQTT_EVENT_ANY, on_event);
+        mqtt = nullptr;
+    }
+
+    esp_err_t wait_for(decltype(IP_Event) event, int milliseconds)
+    {
+        return signal.wait_any(event, milliseconds);
+    }
+
+    ip_event_t get_ip_event_type()
+    {
+        return ip_event_type;
+    }
+
+private:
+    static void on_event(void *arg, esp_event_base_t base, int32_t event, void *data)
+    {
+        auto *handler = static_cast<StatusHandler *>(arg);
+        if (base == IP_EVENT) {
+            handler->ip_event(event, data);
+        } else {
+            handler->mqtt_event(event, data);
+        }
+    }
+
+    void ip_event(int32_t id, void *data)
+    {
+        if (id == IP_EVENT_PPP_GOT_IP) {
+            auto *event = (ip_event_got_ip_t *)data;
+            ESP_LOGI(TAG, "IP          : " IPSTR, IP2STR(&event->ip_info.ip));
+            ESP_LOGI(TAG, "Netmask     : " IPSTR, IP2STR(&event->ip_info.netmask));
+            ESP_LOGI(TAG, "Gateway     : " IPSTR, IP2STR(&event->ip_info.gw));
+            signal.set(IP_Event);
+        } else if (id == IP_EVENT_PPP_LOST_IP) {
+            signal.set(IP_Event);
+        }
+        ip_event_type = static_cast<ip_event_t>(id);
+    }
+
+    void mqtt_event(int32_t event, void *data)
+    {
+        if (mqtt && event == MQTT_EVENT_CONNECTED) {
+            signal.set(MQTT_Connect);
+        } else if (mqtt && event == MQTT_EVENT_DATA) {
+            auto event_data = static_cast<esp_mqtt_event_handle_t>(data);
+            ESP_LOGI(TAG, " TOPIC: %.*s", event_data->topic_len, event_data->topic);
+            ESP_LOGI(TAG, " DATA: %.*s", event_data->data_len, event_data->data);
+            signal.set(MQTT_Data);
+        }
+    }
+
+    esp_modem::SignalGroup signal{};
+    esp_mqtt_client_handle_t mqtt{};
+    ip_event_t ip_event_type{};
+};
+
+// Wrap MQTT operations to destroy everything on returning from main
+struct PublishOnce {
+    esp_mqtt_client_handle_t mqtt_;
+    StatusHandler *events_;
+
+    PublishOnce(StatusHandler *events)
+    {
+        esp_mqtt_client_config_t config = { };
+        config.broker.address.uri = CONFIG_BROKER_URI;
+        mqtt_ = esp_mqtt_client_init(&config);
+        events_ = events;
+        events->handle_mqtt(mqtt_);
+    }
+
+    bool Connect()
+    {
+        return esp_mqtt_client_start(mqtt_) == ESP_OK;
+    }
+
+    bool SubscribePublish()
+    {
+        return  esp_mqtt_client_subscribe(mqtt_, "/topic/esp-modem", 0) >= 0 &&
+                esp_mqtt_client_publish(mqtt_, "/topic/esp-modem", "Hello modem", 0, 0, 0) >= 0;
+    }
+
+    ~PublishOnce()
+    {
+        events_->remove_mqtt();
+        esp_mqtt_client_destroy(mqtt_);
+    }
+};
+
+
+// OTA related
+static constexpr auto OTA_OK      = SignalGroup::bit0;
+static constexpr auto OTA_FAILED  = SignalGroup::bit1;
+
+void ota_task(void *ctx)
+{
+    auto ota_done = static_cast<esp_modem::SignalGroup *>(ctx);
+    manual_ota ota(CONFIG_TEST_OTA_URI);
+    ota.size_ = 64;
+
+    ota.begin();
+    while (true) {
+        if (!ota.perform()) {
+            break;
+        }
+    }
+    auto ret = ota.end();
+    ota_done->set(ret ? OTA_OK : OTA_FAILED);
+    vTaskDelete(nullptr);
+}
+
+
+// App related
+extern "C" void app_main(void)
+{
+    esp_log_level_set("*", ESP_LOG_INFO);
+    esp_log_level_set("ota_test", ESP_LOG_DEBUG);
+
+    // Initialize system functions
+    ESP_ERROR_CHECK(esp_event_loop_create_default());
+    ESP_ERROR_CHECK(esp_netif_init());
+
+    // Initialize DTE
+    esp_modem_dte_config_t dte_config = ESP_MODEM_DTE_DEFAULT_CONFIG();
+#ifdef CONFIG_TEST_USE_VFS_TERM
+    // To code-cover the vfs layers
+    struct esp_modem_vfs_uart_creator uart_config = ESP_MODEM_VFS_DEFAULT_UART_CONFIG("/dev/uart/1");
+    assert(vfs_create_uart(&uart_config, &dte_config.vfs_config) == true);
+
+    auto dte = create_vfs_dte(&dte_config);
+    esp_vfs_dev_uart_use_driver(uart_config.uart.port_num);
+#else
+    auto dte = create_uart_dte(&dte_config);
+#endif // CONFIG_TEST_USE_VFS_TERM
+    assert(dte);
+    dte->set_error_cb([](terminal_error err) {
+        ESP_LOGE(TAG, "DTE reported terminal error: %d", static_cast<int>(err));
+    });
+
+    // Initialize PPP netif
+    esp_netif_config_t netif_ppp_config = ESP_NETIF_DEFAULT_PPP();
+    esp_netif_t *esp_netif = esp_netif_new(&netif_ppp_config);
+    assert(esp_netif);
+
+    // Initialize DCE
+#ifdef CONFIG_TEST_DEVICE_PPPD_SERVER
+    auto dce = create(dte, esp_netif);
+#else
+    esp_modem_dce_config_t dce_config = ESP_MODEM_DCE_DEFAULT_CONFIG(CONFIG_TEST_MODEM_APN);
+    auto dce = create_generic_dce(&dce_config, dte, esp_netif);
+    assert(dce);
+#endif
+
+    StatusHandler handler;
+
+#ifndef CONFIG_TEST_DEVICE_PPPD_SERVER
+    if (dce->set_mode(esp_modem::modem_mode::CMUX_MANUAL_MODE) &&
+            dce->set_mode(esp_modem::modem_mode::CMUX_MANUAL_SWAP) &&
+            dce->set_mode(esp_modem::modem_mode::CMUX_MANUAL_DATA)) {
+#else
+    if (dce->set_mode(esp_modem::modem_mode::DATA_MODE)) {
+#endif
+        ESP_LOGI(TAG, "Modem has correctly entered the desired mode (CMUX/DATA/Manual CMUX)");
+    } else {
+        ESP_LOGE(TAG, "Failed to configure multiplexed command mode... exiting");
+        return;
+    }
+
+    if (!handler.wait_for(StatusHandler::IP_Event, 60000)) {
+        ESP_LOGE(TAG, "Cannot get IP within specified timeout... exiting");
+        return;
+    } else if (handler.get_ip_event_type() == IP_EVENT_PPP_GOT_IP) {
+        ESP_LOGI(TAG, "Got IP address");
+
+        /* When connected to network, subscribe and publish some MQTT data */
+        PublishOnce publish(&handler);
+        if (!publish.Connect()) {
+            ESP_LOGE(TAG, "Failed to connect to mqtt server");
+            return;
+        }
+        if (!handler.wait_for(StatusHandler::MQTT_Connect, 60000)) {
+            ESP_LOGE(TAG, "Cannot connect to %s within specified timeout... exiting", CONFIG_BROKER_URI);
+            return;
+        }
+        ESP_LOGI(TAG, "Connected");
+
+        if (!publish.SubscribePublish()) {
+            ESP_LOGE(TAG, "Failed to subscribe and publish to mqtt server");
+            return;
+        }
+
+        if (!handler.wait_for(StatusHandler::MQTT_Data, 60000)) {
+            ESP_LOGE(TAG, "Didn't receive published data within specified timeout... exiting");
+            return;
+        }
+        ESP_LOGI(TAG, "Received MQTT data");
+
+    } else if (handler.get_ip_event_type() == IP_EVENT_PPP_LOST_IP) {
+        ESP_LOGE(TAG, "PPP client has lost connection... exiting");
+        return;
+    }
+
+
+    esp_modem::SignalGroup ota_done{};
+    // now stop the LCP keepalive before performing OTA
+    esp_netif_ppp_config_t cfg;
+    ESP_ERROR_CHECK(esp_netif_ppp_get_params(esp_netif, &cfg));
+    cfg.ppp_lcp_echo_disabled = true;
+    ESP_ERROR_CHECK(esp_netif_ppp_set_params(esp_netif, &cfg));
+
+    // Run the OTA in a separate task to keep sending commands to the modem at the same time
+    xTaskCreate(ota_task, "ota_task", 8192, &ota_done, 5, nullptr);
+
+#ifndef CONFIG_TEST_DEVICE_PPPD_SERVER
+    while (true) {
+        std::string str;
+        if (dce->get_imsi(str) == esp_modem::command_result::OK) {
+            ESP_LOGI(TAG, "Modem IMSI number: %s", str.c_str());
+        }
+        if (ota_done.wait_any(OTA_OK | OTA_FAILED, 100)) {
+            break;
+        }
+    }
+#else
+    ota_done.wait_any(OTA_OK | OTA_FAILED, portMAX_DELAY);
+#endif // CONFIG_TEST_DEVICE_PPPD_SERVER
+
+#ifndef CONFIG_TEST_DEVICE_PPPD_SERVER
+    if (dce->set_mode(esp_modem::modem_mode::CMUX_MANUAL_EXIT)) {
+#else
+    if (dce->set_mode(esp_modem::modem_mode::COMMAND_MODE)) {
+#endif
+        ESP_LOGI(TAG, "Modem CMUX/DATA mode exit");
+    } else {
+        ESP_LOGE(TAG, "Failed to configure desired mode... exiting");
+        return;
+    }
+
+    if (ota_done.is_any(OTA_OK)) {
+        ESP_LOGI(TAG, "Prepare to restart system!");
+        esp_restart();
+    }
+
+}