From f2223dd719de3d28b6dc2cf3877092c6580829cb Mon Sep 17 00:00:00 2001 From: David Cermak Date: Thu, 21 Sep 2023 09:03:20 +0200 Subject: [PATCH] feat(modem): Added test that performs OTA to exercise modem layers --- .../include/vfs_resource/vfs_create.hpp | 2 +- .../esp_modem/test/target_ota/CMakeLists.txt | 8 + .../esp_modem/test/target_ota/README.md | 21 ++ .../components/manual_ota/CMakeLists.txt | 3 + .../components/manual_ota/manual_ota.cpp | 259 +++++++++++++++ .../components/manual_ota/manual_ota.hpp | 71 +++++ .../manual_ota/transport_batch_tls.cpp | 232 ++++++++++++++ .../manual_ota/transport_batch_tls.hpp | 25 ++ .../esp_modem/test/target_ota/http_server.py | 15 + .../test/target_ota/main/CMakeLists.txt | 2 + .../test/target_ota/main/Kconfig.projbuild | 47 +++ .../test/target_ota/main/network_dce.hpp | 55 ++++ .../test/target_ota/main/ota_test.cpp | 295 ++++++++++++++++++ 13 files changed, 1034 insertions(+), 1 deletion(-) create mode 100644 components/esp_modem/test/target_ota/CMakeLists.txt create mode 100644 components/esp_modem/test/target_ota/README.md create mode 100644 components/esp_modem/test/target_ota/components/manual_ota/CMakeLists.txt create mode 100644 components/esp_modem/test/target_ota/components/manual_ota/manual_ota.cpp create mode 100644 components/esp_modem/test/target_ota/components/manual_ota/manual_ota.hpp create mode 100644 components/esp_modem/test/target_ota/components/manual_ota/transport_batch_tls.cpp create mode 100644 components/esp_modem/test/target_ota/components/manual_ota/transport_batch_tls.hpp create mode 100644 components/esp_modem/test/target_ota/http_server.py create mode 100644 components/esp_modem/test/target_ota/main/CMakeLists.txt create mode 100644 components/esp_modem/test/target_ota/main/Kconfig.projbuild create mode 100644 components/esp_modem/test/target_ota/main/network_dce.hpp create mode 100644 components/esp_modem/test/target_ota/main/ota_test.cpp diff --git a/components/esp_modem/include/vfs_resource/vfs_create.hpp b/components/esp_modem/include/vfs_resource/vfs_create.hpp index 7b075143b..5d8485ddc 100644 --- a/components/esp_modem/include/vfs_resource/vfs_create.hpp +++ b/components/esp_modem/include/vfs_resource/vfs_create.hpp @@ -31,7 +31,7 @@ */ struct esp_modem_vfs_uart_creator { const char *dev_name; /*!< VFS device name, e.g. /dev/uart/n */ - const struct esp_modem_uart_term_config uart; /*!< UART driver init struct */ + struct esp_modem_uart_term_config uart; /*!< UART driver init struct */ }; /** diff --git a/components/esp_modem/test/target_ota/CMakeLists.txt b/components/esp_modem/test/target_ota/CMakeLists.txt new file mode 100644 index 000000000..ca5c2eaaa --- /dev/null +++ b/components/esp_modem/test/target_ota/CMakeLists.txt @@ -0,0 +1,8 @@ +# The following lines of boilerplate have to be in your project's CMakeLists +# in this exact order for cmake to work correctly +cmake_minimum_required(VERSION 3.8) + +set(EXTRA_COMPONENT_DIRS "../.." "../../examples/modem_tcp_client/components") + +include($ENV{IDF_PATH}/tools/cmake/project.cmake) +project(ota_test) diff --git a/components/esp_modem/test/target_ota/README.md b/components/esp_modem/test/target_ota/README.md new file mode 100644 index 000000000..74b98ef4a --- /dev/null +++ b/components/esp_modem/test/target_ota/README.md @@ -0,0 +1,21 @@ +# Target test running OTA update + +## Overview + +The aim of this test is to exercise the most commonly failing scenario, running OTA over PPPoS with https. + +This project opens a data session, runs basic mqtt operations and initiates OTA update. +It supports the following test configurations: +* Using a real modem device (default config) +* Using VFS device (only to exercise VFS DTE) +* Using network-only DCE (connecting directly to PPP server) -- needs some configuration + +### Configuring the PPP server + +You need to run these applications on your host machine: +* PPP server +```bash +sudo pppd /dev/ttyUSB1 115200 192.168.11.1:192.168.11.2 ms-dns 8.8.8.8 modem local noauth debug nocrtscts nodetach +ipv6 +``` +* MQTT broker: Running mosquitto in the default config is enough, configuring the broker's URL to the local PPP address: `config.broker.address.uri = "mqtt://192.168.11.1";` +* HTTP server: Need to support HTTP/1.1 (to support ranges). You can use the script `http_server.py` and configure the OTA endpoint as `"https://192.168.11.1:1234/esp32.bin"` diff --git a/components/esp_modem/test/target_ota/components/manual_ota/CMakeLists.txt b/components/esp_modem/test/target_ota/components/manual_ota/CMakeLists.txt new file mode 100644 index 000000000..5eb262e10 --- /dev/null +++ b/components/esp_modem/test/target_ota/components/manual_ota/CMakeLists.txt @@ -0,0 +1,3 @@ +idf_component_register(SRCS manual_ota.cpp transport_batch_tls.cpp + INCLUDE_DIRS "." + PRIV_REQUIRES extra_tcp_transports esp_http_client app_update) diff --git a/components/esp_modem/test/target_ota/components/manual_ota/manual_ota.cpp b/components/esp_modem/test/target_ota/components/manual_ota/manual_ota.cpp new file mode 100644 index 000000000..c1aaa3d36 --- /dev/null +++ b/components/esp_modem/test/target_ota/components/manual_ota/manual_ota.cpp @@ -0,0 +1,259 @@ +/* + * SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD + * + * SPDX-License-Identifier: Unlicense OR CC0-1.0 + */ +#include "manual_ota.hpp" +#include "esp_log.h" +#include "esp_ota_ops.h" +#include "esp_app_format.h" +#include "esp_http_client.h" +#include "esp_partition.h" +#include "esp_transport_tcp.h" +#include "transport_batch_tls.hpp" + +static const char *TAG = "manual_ota"; + +bool manual_ota::begin() +{ + if (status != state::UNDEF) { + ESP_LOGE(TAG, "Invalid state"); + return false; + } + status = state::INIT; + esp_transport_handle_t tcp = esp_transport_tcp_init(); + ssl_ = esp_transport_batch_tls_init(tcp, max_buffer_size_); + + esp_http_client_config_t config = { }; + config.skip_cert_common_name_check = true; + config.url = uri_; + config.transport = ssl_; + const esp_partition_t *configured = esp_ota_get_boot_partition(); + const esp_partition_t *running = esp_ota_get_running_partition(); + + if (configured != running) { + ESP_LOGE(TAG, "Configured OTA boot partition at offset 0x%08" PRIx32 ", but running from offset 0x%08" PRIx32, configured->address, running->address); + return false; + } + + http_ = esp_http_client_init(&config); + if (http_ == nullptr) { + ESP_LOGE(TAG, "Failed to initialise HTTP connection"); + return false; + } + esp_http_client_set_method(http_, HTTP_METHOD_HEAD); + esp_err_t err = esp_http_client_perform(http_); + if (err == ESP_OK) { + int http_status = esp_http_client_get_status_code(http_); + if (http_status != HttpStatus_Ok) { + ESP_LOGE(TAG, "Received incorrect http status %d", http_status); + return false; + } + } else { + ESP_LOGE(TAG, "ESP HTTP client perform failed: %d", err); + return false; + } + image_length_ = esp_http_client_get_content_length(http_); + ESP_LOGI(TAG, "image_length = %lld", image_length_); + esp_http_client_close(http_); + + if (image_length_ > size_) { + char *header_val = nullptr; + asprintf(&header_val, "bytes=0-%d", max_buffer_size_ - 1); + if (header_val == nullptr) { + ESP_LOGE(TAG, "Failed to allocate memory for HTTP header"); + return false; + } + esp_http_client_set_header(http_, "Range", header_val); + free(header_val); + } + esp_http_client_set_method(http_, HTTP_METHOD_GET); + + partition_ = esp_ota_get_next_update_partition(nullptr); + if (partition_ == nullptr) { + ESP_LOGE(TAG, "Invalid update partition"); + return false; + } + ESP_LOGI(TAG, "Writing to partition subtype %d at offset 0x%" PRIx32, partition_->subtype, partition_->address); + + file_length_ = 0; + reconnect_attempts_ = 0; + buffer_.resize(max_buffer_size_); + status = state::IMAGE_CHECK; + return true; +} + +bool manual_ota::perform() +{ + if (status != state::IMAGE_CHECK && status != state::START) { + ESP_LOGE(TAG, "Invalid state"); + return false; + } + esp_err_t err = esp_http_client_open(http_, 0); + if (err != ESP_OK) { + if (image_length_ == file_length_) { + status = state::END; + return false; + } + + esp_http_client_close(http_); + ESP_LOGI(TAG, "Failed to open HTTP connection: %s", esp_err_to_name(err)); + if (reconnect_attempts_++ < max_reconnect_attempts_) { + if (prepare_reconnect()) { + return true; // will retry in the next iteration + } + } + return fail_cleanup(); + } + esp_http_client_fetch_headers(http_); + + int batch_len = esp_transport_batch_tls_pre_read(ssl_, max_buffer_size_, timeout_ * 1000); + if (batch_len < 0) { + ESP_LOGE(TAG, "Error: Failed to pre-read plain text data!"); + return fail_cleanup(); + } + + int data_read = esp_http_client_read(http_, buffer_.data(), batch_len); + + if (data_read < 0) { + ESP_LOGE(TAG, "Error: SSL data read error"); + return fail_cleanup(); + } else if (data_read > 0) { + esp_http_client_close(http_); + + if (status == state::IMAGE_CHECK) { + esp_app_desc_t new_app_info; + if (data_read > sizeof(esp_image_header_t) + sizeof(esp_image_segment_header_t) + sizeof(esp_app_desc_t)) { + // check current version with downloading + memcpy(&new_app_info, &buffer_[sizeof(esp_image_header_t) + sizeof(esp_image_segment_header_t)], sizeof(esp_app_desc_t)); + ESP_LOGI(TAG, "New firmware version: %s", new_app_info.version); + + esp_app_desc_t running_app_info; + const esp_partition_t *running = esp_ota_get_running_partition(); + if (esp_ota_get_partition_description(running, &running_app_info) == ESP_OK) { + ESP_LOGI(TAG, "Running firmware version: %s", running_app_info.version); + } + + const esp_partition_t *last_invalid_app = esp_ota_get_last_invalid_partition(); + esp_app_desc_t invalid_app_info; + if (esp_ota_get_partition_description(last_invalid_app, &invalid_app_info) == ESP_OK) { + ESP_LOGI(TAG, "Last invalid firmware version: %s", invalid_app_info.version); + } + + // check current version with last invalid partition + if (last_invalid_app != NULL) { + if (memcmp(invalid_app_info.version, new_app_info.version, sizeof(new_app_info.version)) == 0) { + ESP_LOGW(TAG, "New version is the same as invalid version."); + ESP_LOGW(TAG, "Previously, there was an attempt to launch the firmware with %s version, but it failed.", invalid_app_info.version); + ESP_LOGW(TAG, "The firmware has been rolled back to the previous version."); + return fail_cleanup(); + } + } + + status = state::START; + err = esp_ota_begin(partition_, OTA_WITH_SEQUENTIAL_WRITES, &update_handle_); + if (err != ESP_OK) { + ESP_LOGE(TAG, "esp_ota_begin failed (%s)", esp_err_to_name(err)); + esp_ota_abort(update_handle_); + return fail_cleanup(); + } + ESP_LOGI(TAG, "esp_ota_begin succeeded"); + } else { + ESP_LOGE(TAG, "Received chunk doesn't contain app descriptor"); + esp_ota_abort(update_handle_); + return fail_cleanup(); + } + } + err = esp_ota_write(update_handle_, (const void *)buffer_.data(), data_read); + if (err != ESP_OK) { + esp_ota_abort(update_handle_); + return fail_cleanup(); + } + file_length_ += data_read; + ESP_LOGI(TAG, "Written image length %d", file_length_); + + if (image_length_ == file_length_) { + status = state::END; + return false; + } + + if (!prepare_reconnect()) { + esp_ota_abort(update_handle_); + return fail_cleanup(); + } + + } else if (data_read == 0) { + if (file_length_ == 0) { + // Try to handle possible HTTP redirections + int status_code = esp_http_client_get_status_code(http_); + ESP_LOGW(TAG, "Status code: %d", status_code); + err = esp_http_client_set_redirection(http_); + if (err != ESP_OK) { + ESP_LOGE(TAG, "URL redirection Failed"); + esp_ota_abort(update_handle_); + return fail_cleanup(); + } + + err = esp_http_client_open(http_, 0); + if (err != ESP_OK) { + ESP_LOGE(TAG, "Failed to open HTTP connection: %s", esp_err_to_name(err)); + return fail_cleanup(); + } + esp_http_client_fetch_headers(http_); + } + } + + return true; +} + +bool manual_ota::prepare_reconnect() +{ + esp_http_client_set_method(http_, HTTP_METHOD_GET); + char *header_val = nullptr; + if ((image_length_ - file_length_) > max_buffer_size_) { + asprintf(&header_val, "bytes=%d-%d", file_length_, (file_length_ + max_buffer_size_ - 1)); + } else { + asprintf(&header_val, "bytes=%d-", file_length_); + } + if (header_val == nullptr) { + ESP_LOGE(TAG, "Failed to allocate memory for HTTP header"); + return false; + } + esp_http_client_set_header(http_, "Range", header_val); + free(header_val); + return true; +} + +bool manual_ota::fail_cleanup() +{ + esp_http_client_close(http_); + esp_http_client_cleanup(http_); + status = state::FAIL; + return false; +} + +bool manual_ota::end() +{ + if (status == state::END) { + if (!esp_http_client_is_complete_data_received(http_)) { + ESP_LOGE(TAG, "Error in receiving complete file"); + return fail_cleanup(); + } + esp_err_t err = esp_ota_end(update_handle_); + if (err != ESP_OK) { + if (err == ESP_ERR_OTA_VALIDATE_FAILED) { + ESP_LOGE(TAG, "Image validation failed, image is corrupted"); + } else { + ESP_LOGE(TAG, "esp_ota_end failed (%s)!", esp_err_to_name(err)); + } + return fail_cleanup(); + } + err = esp_ota_set_boot_partition(partition_); + if (err != ESP_OK) { + ESP_LOGE(TAG, "esp_ota_set_boot_partition failed (%s)!", esp_err_to_name(err)); + return fail_cleanup(); + } + return true; + } + return false; +} diff --git a/components/esp_modem/test/target_ota/components/manual_ota/manual_ota.hpp b/components/esp_modem/test/target_ota/components/manual_ota/manual_ota.hpp new file mode 100644 index 000000000..ad09e748d --- /dev/null +++ b/components/esp_modem/test/target_ota/components/manual_ota/manual_ota.hpp @@ -0,0 +1,71 @@ +/* + * SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD + * + * SPDX-License-Identifier: Unlicense OR CC0-1.0 + */ +#pragma once +#include +#include "esp_http_client.h" +#include "esp_partition.h" +#include "esp_transport_tcp.h" +#include "esp_ota_ops.h" + +class manual_ota { +public: + enum class state { + UNDEF, + INIT, + IMAGE_CHECK, + START, + END, + FAIL, + }; + size_t size_{32}; + int timeout_{2}; + + /** + * @brief Construct a new manual ota object + * + * @param uri URI of the binary image + */ + explicit manual_ota(const char *uri): uri_(uri) {} + + /** + * @brief Start the manual OTA process + * + * @return true if started successfully + */ + bool begin(); + + /** + * @brief Performs one read-write OTA iteration + * + * @return true if the process is in progress + * @return false if the process finished, call end() to get OTA result + */ + bool perform(); + + /** + * @brief Finishes an OTA update + * + * @return true if the OTA update completed successfully + */ + bool end(); + +private: + const char *uri_{}; + esp_http_client_handle_t http_; + int64_t image_length_; + size_t file_length_; + const size_t max_buffer_size_{size_ * 1024}; + const esp_partition_t *partition_{nullptr}; + state status{state::UNDEF}; + std::vector buffer_{}; + int reconnect_attempts_; + const int max_reconnect_attempts_{3}; + esp_transport_handle_t ssl_; + esp_ota_handle_t update_handle_{0}; + + bool prepare_reconnect(); + bool fail_cleanup(); +}; diff --git a/components/esp_modem/test/target_ota/components/manual_ota/transport_batch_tls.cpp b/components/esp_modem/test/target_ota/components/manual_ota/transport_batch_tls.cpp new file mode 100644 index 000000000..013b4ef6b --- /dev/null +++ b/components/esp_modem/test/target_ota/components/manual_ota/transport_batch_tls.cpp @@ -0,0 +1,232 @@ +/* + * SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD + * + * SPDX-License-Identifier: Unlicense OR CC0-1.0 + */ +#include +#include "esp_log.h" +#include "mbedtls_wrap.hpp" +#include "esp_transport_tcp.h" + +#define TAG "batch-tls" + +class TlsTransport: public Tls { +public: + explicit TlsTransport(esp_transport_handle_t parent) : Tls(), transport_(parent), read_len(0), offset(0) {} + int send(const unsigned char *buf, size_t len) override; + int recv(unsigned char *buf, size_t len) override; + static bool set_func(esp_transport_handle_t tls_transport); + int preread(size_t len, int timeout_ms); + bool prepare_buffer(size_t max_size); +private: + esp_transport_handle_t transport_{}; + int connect(const char *host, int port, int timeout_ms); + void delay() override; + + struct transport { + static int connect(esp_transport_handle_t t, const char *host, int port, int timeout_ms); + static int read(esp_transport_handle_t t, char *buffer, int len, int timeout_ms); + static int write(esp_transport_handle_t t, const char *buffer, int len, int timeout_ms); + static int close(esp_transport_handle_t t); + static int poll_read(esp_transport_handle_t t, int timeout_ms); + static int poll_write(esp_transport_handle_t t, int timeout_ms); + static int destroy(esp_transport_handle_t t); + }; + int last_timeout; + std::vector buf; + size_t read_len; + size_t offset; +}; + +esp_transport_handle_t esp_transport_tls_init(esp_transport_handle_t parent) +{ + esp_transport_handle_t ssl = esp_transport_init(); + auto *tls = new TlsTransport(parent); + esp_transport_set_context_data(ssl, tls); + TlsTransport::set_func(ssl); + return ssl; +} + +int TlsTransport::send(const unsigned char *buf, size_t len) +{ + int ret = esp_transport_write(transport_, reinterpret_cast(buf), len, 0); + ESP_LOGD(TAG, "writing(len=%d) ret=%d", len, ret); + return ret; +} + +int TlsTransport::recv(unsigned char *buffer, size_t len) +{ + ESP_LOGD(TAG, "recv(len=%d)", len); + if (read_len != 0) { + + if (read_len > len) { + memcpy((char *)buffer, buf.data() + offset, len); + read_len -= len; + offset += len; + ESP_LOGD(TAG, "read %d from batch read_len = %d", len, read_len); + return len; + } else { + int remaining = len = read_len; + if (remaining > 0) { + memcpy((char *)buffer, buf.data() + offset, remaining); + read_len = 0; + offset = 0; + return remaining; + + } + read_len = 0; + offset = 0; + return ERR_TCP_TRANSPORT_CONNECTION_CLOSED_BY_FIN; + } + } + int ret = esp_transport_read(transport_, reinterpret_cast(buffer), len, last_timeout); + + if (ret == ERR_TCP_TRANSPORT_CONNECTION_TIMEOUT) { + return MBEDTLS_ERR_SSL_WANT_READ; + } + return ret == ERR_TCP_TRANSPORT_CONNECTION_CLOSED_BY_FIN ? 0 : ret; +} + +bool TlsTransport::set_func(esp_transport_handle_t tls_transport) +{ + return esp_transport_set_func(tls_transport, TlsTransport::transport::connect, TlsTransport::transport::read, TlsTransport::transport::write, TlsTransport::transport::close, TlsTransport::transport::poll_read, TlsTransport::transport::poll_write, TlsTransport::transport::destroy) == ESP_OK; +} + +int TlsTransport::connect(const char *host, int port, int timeout_ms) +{ + return esp_transport_connect(transport_, host, port, timeout_ms); +} + +void TlsTransport::delay() +{ + vTaskDelay(pdMS_TO_TICKS(500)); +} + +int TlsTransport::transport::connect(esp_transport_handle_t t, const char *host, int port, int timeout_ms) +{ + auto tls = static_cast(esp_transport_get_context_data(t)); + tls->init(is_server{false}, do_verify{false}); + + ESP_LOGD(TAG, "TLS-connect"); + auto ret = tls->connect(host, port, timeout_ms); + if (ret < 0) { + ESP_LOGI(TAG, "Failed to connect to transport"); + return ret; + } + if (tls->is_session_loaded()) { + tls->set_session(); + } + ESP_LOGI(TAG, "Before handshake"); + ret = tls->handshake(); + if (ret < 0) { + ESP_LOGI(TAG, "Failed to handshake"); + return ret; + } + tls->get_session(); + ESP_LOGI(TAG, "After handshake"); + return 0; +} + +int TlsTransport::transport::read(esp_transport_handle_t t, char *buffer, int len, int timeout_ms) +{ + auto tls = static_cast(esp_transport_get_context_data(t)); + ESP_LOGD(TAG, "available=%d tls->read_len=%d", tls->get_available_bytes(), tls->read_len); + if (tls->get_available_bytes() <= 0 && tls->read_len == 0) { + ESP_LOGD(TAG, "red(len=%d, timeout=%d) tls->read_len=%d", len, timeout_ms, tls->read_len); + tls->last_timeout = timeout_ms; + int poll = esp_transport_poll_read(t, timeout_ms); + if (poll == -1) { + return ERR_TCP_TRANSPORT_CONNECTION_FAILED; + } + if (poll == 0) { + return ERR_TCP_TRANSPORT_CONNECTION_TIMEOUT; + } + } + + auto ret = tls->read(reinterpret_cast(buffer), len); + if (ret == MBEDTLS_ERR_SSL_WANT_READ) { + ret = ERR_TCP_TRANSPORT_CONNECTION_TIMEOUT; + } + ESP_LOGD(TAG, "red(len=%d, timeout=%d) ret=%d", len, timeout_ms, ret); + return ret; +} + +int TlsTransport::transport::write(esp_transport_handle_t t, const char *buffer, int len, int timeout_ms) +{ + int poll; + if ((poll = esp_transport_poll_write(t, timeout_ms)) <= 0) { + ESP_LOGW(TAG, "Poll timeout or error timeout_ms=%d", timeout_ms); + return poll; + } + + auto tls = static_cast(esp_transport_get_context_data(t)); + int ret = tls->write(reinterpret_cast(buffer), len); + ESP_LOGD(TAG, "write ret=%d", ret); + return ret; +} + +int TlsTransport::transport::close(esp_transport_handle_t t) +{ + auto tls = static_cast(esp_transport_get_context_data(t)); + int ret = esp_transport_close(tls->transport_); + tls->deinit(); + return ret; +} + +int TlsTransport::transport::poll_read(esp_transport_handle_t t, int timeout_ms) +{ + auto tls = static_cast(esp_transport_get_context_data(t)); + return esp_transport_poll_read(tls->transport_, timeout_ms); +} + +int TlsTransport::transport::poll_write(esp_transport_handle_t t, int timeout_ms) +{ + auto tls = static_cast(esp_transport_get_context_data(t)); + return esp_transport_poll_write(tls->transport_, timeout_ms); +} + +int TlsTransport::transport::destroy(esp_transport_handle_t t) +{ + auto tls = static_cast(esp_transport_get_context_data(t)); + return esp_transport_destroy(tls->transport_); +} + + +esp_transport_handle_t esp_transport_batch_tls_init(esp_transport_handle_t parent, const size_t max_buffer_size) +{ + esp_transport_handle_t ssl = esp_transport_init(); + auto *tls = new TlsTransport(parent); + esp_transport_set_context_data(ssl, tls); + TlsTransport::set_func(ssl); + tls->prepare_buffer(max_buffer_size); + return ssl; +} + +int TlsTransport::preread(size_t len, int timeout_ms) +{ + while (len != read_len) { + int l = esp_transport_read(transport_, buf.data() + read_len, len - read_len, timeout_ms); + ESP_LOGD(TAG, "need %d read %d already %d", len, l, read_len); + if ((l == ERR_TCP_TRANSPORT_CONNECTION_CLOSED_BY_FIN || l == ERR_TCP_TRANSPORT_CONNECTION_TIMEOUT ) && read_len > 0) { + return read_len; + } + if (l <= 0) { + read_len = 0; + return read_len; + } + read_len += l; + } + return read_len; +} + +bool TlsTransport::prepare_buffer(size_t max_size) +{ + buf.resize(max_size); + return true; +} + +int esp_transport_batch_tls_pre_read(esp_transport_handle_t t, int len, int timeout_ms) +{ + auto tls = static_cast(esp_transport_get_context_data(t)); + return tls->preread(len, timeout_ms); +} diff --git a/components/esp_modem/test/target_ota/components/manual_ota/transport_batch_tls.hpp b/components/esp_modem/test/target_ota/components/manual_ota/transport_batch_tls.hpp new file mode 100644 index 000000000..e91b8b62e --- /dev/null +++ b/components/esp_modem/test/target_ota/components/manual_ota/transport_batch_tls.hpp @@ -0,0 +1,25 @@ +/* + * SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD + * + * SPDX-License-Identifier: Unlicense OR CC0-1.0 + */ +#pragma once + +/** + * @brief Creates batch transport + * + * @param parent tcp-transport handle to the parent transport + * @param max_buffer_size maximum size of one batch + * @return created transport handle + */ +esp_transport_handle_t esp_transport_batch_tls_init(esp_transport_handle_t parent, const size_t max_buffer_size); + +/** + * @brief Performs batch read operation from the underlying transport + * + * @param t Transport handle + * @param len Batch size + * @param timeout_ms Timeout in ms + * @return true If read from the parent transport completed successfully + */ +bool esp_transport_batch_tls_pre_read(esp_transport_handle_t t, int len, int timeout_ms); diff --git a/components/esp_modem/test/target_ota/http_server.py b/components/esp_modem/test/target_ota/http_server.py new file mode 100644 index 000000000..cdea1daa5 --- /dev/null +++ b/components/esp_modem/test/target_ota/http_server.py @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Unlicense OR CC0-1.0 +import ssl +from http.server import HTTPServer + +from RangeHTTPServer import RangeRequestHandler + +server_address = ('0.0.0.0', 1234) +httpd = HTTPServer(server_address, RangeRequestHandler) +httpd.socket = ssl.wrap_socket(httpd.socket, + server_side=True, + certfile='srv.crt', + keyfile='srv.key', + ssl_version=ssl.PROTOCOL_TLS) +httpd.serve_forever() diff --git a/components/esp_modem/test/target_ota/main/CMakeLists.txt b/components/esp_modem/test/target_ota/main/CMakeLists.txt new file mode 100644 index 000000000..4ee863eb9 --- /dev/null +++ b/components/esp_modem/test/target_ota/main/CMakeLists.txt @@ -0,0 +1,2 @@ +idf_component_register(SRCS ota_test.cpp + INCLUDE_DIRS ".") diff --git a/components/esp_modem/test/target_ota/main/Kconfig.projbuild b/components/esp_modem/test/target_ota/main/Kconfig.projbuild new file mode 100644 index 000000000..3d973fb00 --- /dev/null +++ b/components/esp_modem/test/target_ota/main/Kconfig.projbuild @@ -0,0 +1,47 @@ +menu "Test Configuration" + + choice TEST_DEVICE + prompt "Choose supported modem device (DCE)" + default TEST_DEVICE_MODEM_GENERIC + help + Select modem device connected to the ESP DTE. + + config TEST_DEVICE_MODEM_GENERIC + bool "Common modem device" + help + Generic device that could be used with most common modems (BG96, SIM76xx, A76xx). + + config TEST_DEVICE_PPPD_SERVER + bool "PPPD Server" + help + Test device is a pppd service in server mode, running on linux. + endchoice + + config TEST_MODEM_APN + string "Modem APN" + depends on TEST_DEVICE_MODEM_GENERIC + default "lpwa.vodafone.com" + help + Set APN (Access Point Name), a logical name to choose data network + + config TEST_USE_VFS_TERM + bool "Use VFS terminal" + default n + help + Demonstrate use of VFS as a communication terminal of the DTE. + VFS driver implements non-block reads, writes and selects to communicate with esp-modem, + but this implementation uses UART resource only. + + config TEST_OTA_URI + string "URI of the binary" + default "https://192.168.11.1/esp32.bin" + help + HTTPS address of the update binary. + + config BROKER_URI + string "Broker URL" + default "mqtt://test.mosquitto.org" + help + URL of an mqtt broker which this example connects to. + +endmenu diff --git a/components/esp_modem/test/target_ota/main/network_dce.hpp b/components/esp_modem/test/target_ota/main/network_dce.hpp new file mode 100644 index 000000000..db6f0058d --- /dev/null +++ b/components/esp_modem/test/target_ota/main/network_dce.hpp @@ -0,0 +1,55 @@ +/* + * SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD + * + * SPDX-License-Identifier: Unlicense OR CC0-1.0 + */ + +#include "cxx_include/esp_modem_dte.hpp" +#include "esp_modem_config.h" +#include "cxx_include/esp_modem_api.hpp" +#include "cxx_include/esp_modem_dce_factory.hpp" +#include +#include + +class NetModule; + +/** + * @brief Custom factory which can build and create a DCE using a custom module + */ +class NetDCE_Factory: public esp_modem::dce_factory::Factory { +public: + template + static auto create(const esp_modem::dce_factory::config *cfg, Args &&... args) -> std::shared_ptr> + { + return build_generic_DCE, std::shared_ptr>>(cfg, std::forward(args)...); + } +}; + +/** + * @brief This is a null-module, doesn't define any AT commands, just passes everything to pppd + */ +class NetModule: public esp_modem::ModuleIf { +public: + explicit NetModule(std::shared_ptr dte, const esp_modem_dce_config *cfg): + dte(std::move(dte)) {} + + bool setup_data_mode() override + { + return true; + } + + bool set_mode(esp_modem::modem_mode mode) override + { + return true; + } + +private: + std::shared_ptr dte; +}; + + +std::shared_ptr> create(std::shared_ptr dte, esp_netif_t *netif) +{ + const esp_modem::dce_config config = {}; + return NetDCE_Factory::create(&config, dte, netif); +} diff --git a/components/esp_modem/test/target_ota/main/ota_test.cpp b/components/esp_modem/test/target_ota/main/ota_test.cpp new file mode 100644 index 000000000..ec0edfbb0 --- /dev/null +++ b/components/esp_modem/test/target_ota/main/ota_test.cpp @@ -0,0 +1,295 @@ +/* + * SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD + * + * SPDX-License-Identifier: Unlicense OR CC0-1.0 + */ + +#include +#include "freertos/FreeRTOS.h" +#include "freertos/event_groups.h" +#include "esp_netif.h" +#include "esp_netif_ppp.h" +#include "esp_log.h" +#include "esp_event.h" +#include "cxx_include/esp_modem_dte.hpp" +#include "esp_modem_config.h" +#include "cxx_include/esp_modem_api.hpp" +#include "esp_vfs_dev.h" // For optional VFS support +#include "vfs_resource/vfs_create.hpp" +#include "network_dce.hpp" +#include "manual_ota.hpp" +#include "mqtt_client.h" + +using namespace esp_modem; + +static const char *TAG = "ota_test"; + + +// Wrap event handlers to destruct correctly on returning from main +class StatusHandler { +public: + static constexpr auto IP_Event = SignalGroup::bit0; + static constexpr auto MQTT_Connect = SignalGroup::bit1; + static constexpr auto MQTT_Data = SignalGroup::bit2; + + StatusHandler() + { + ESP_ERROR_CHECK(esp_event_handler_register(IP_EVENT, ESP_EVENT_ANY_ID, on_event, this)); + } + + ~StatusHandler() + { + esp_event_handler_unregister(IP_EVENT, ESP_EVENT_ANY_ID, on_event); + } + + void handle_mqtt(esp_mqtt_client_handle_t client) + { + mqtt = client; + esp_mqtt_client_register_event(client, MQTT_EVENT_ANY, on_event, this); + } + + void remove_mqtt() + { + esp_mqtt_client_unregister_event(mqtt, MQTT_EVENT_ANY, on_event); + mqtt = nullptr; + } + + esp_err_t wait_for(decltype(IP_Event) event, int milliseconds) + { + return signal.wait_any(event, milliseconds); + } + + ip_event_t get_ip_event_type() + { + return ip_event_type; + } + +private: + static void on_event(void *arg, esp_event_base_t base, int32_t event, void *data) + { + auto *handler = static_cast(arg); + if (base == IP_EVENT) { + handler->ip_event(event, data); + } else { + handler->mqtt_event(event, data); + } + } + + void ip_event(int32_t id, void *data) + { + if (id == IP_EVENT_PPP_GOT_IP) { + auto *event = (ip_event_got_ip_t *)data; + ESP_LOGI(TAG, "IP : " IPSTR, IP2STR(&event->ip_info.ip)); + ESP_LOGI(TAG, "Netmask : " IPSTR, IP2STR(&event->ip_info.netmask)); + ESP_LOGI(TAG, "Gateway : " IPSTR, IP2STR(&event->ip_info.gw)); + signal.set(IP_Event); + } else if (id == IP_EVENT_PPP_LOST_IP) { + signal.set(IP_Event); + } + ip_event_type = static_cast(id); + } + + void mqtt_event(int32_t event, void *data) + { + if (mqtt && event == MQTT_EVENT_CONNECTED) { + signal.set(MQTT_Connect); + } else if (mqtt && event == MQTT_EVENT_DATA) { + auto event_data = static_cast(data); + ESP_LOGI(TAG, " TOPIC: %.*s", event_data->topic_len, event_data->topic); + ESP_LOGI(TAG, " DATA: %.*s", event_data->data_len, event_data->data); + signal.set(MQTT_Data); + } + } + + esp_modem::SignalGroup signal{}; + esp_mqtt_client_handle_t mqtt{}; + ip_event_t ip_event_type{}; +}; + +// Wrap MQTT operations to destroy everything on returning from main +struct PublishOnce { + esp_mqtt_client_handle_t mqtt_; + StatusHandler *events_; + + PublishOnce(StatusHandler *events) + { + esp_mqtt_client_config_t config = { }; + config.broker.address.uri = CONFIG_BROKER_URI; + mqtt_ = esp_mqtt_client_init(&config); + events_ = events; + events->handle_mqtt(mqtt_); + } + + bool Connect() + { + return esp_mqtt_client_start(mqtt_) == ESP_OK; + } + + bool SubscribePublish() + { + return esp_mqtt_client_subscribe(mqtt_, "/topic/esp-modem", 0) >= 0 && + esp_mqtt_client_publish(mqtt_, "/topic/esp-modem", "Hello modem", 0, 0, 0) >= 0; + } + + ~PublishOnce() + { + events_->remove_mqtt(); + esp_mqtt_client_destroy(mqtt_); + } +}; + + +// OTA related +static constexpr auto OTA_OK = SignalGroup::bit0; +static constexpr auto OTA_FAILED = SignalGroup::bit1; + +void ota_task(void *ctx) +{ + auto ota_done = static_cast(ctx); + manual_ota ota(CONFIG_TEST_OTA_URI); + ota.size_ = 64; + + ota.begin(); + while (true) { + if (!ota.perform()) { + break; + } + } + auto ret = ota.end(); + ota_done->set(ret ? OTA_OK : OTA_FAILED); + vTaskDelete(nullptr); +} + + +// App related +extern "C" void app_main(void) +{ + esp_log_level_set("*", ESP_LOG_INFO); + esp_log_level_set("ota_test", ESP_LOG_DEBUG); + + // Initialize system functions + ESP_ERROR_CHECK(esp_event_loop_create_default()); + ESP_ERROR_CHECK(esp_netif_init()); + + // Initialize DTE + esp_modem_dte_config_t dte_config = ESP_MODEM_DTE_DEFAULT_CONFIG(); +#ifdef CONFIG_TEST_USE_VFS_TERM + // To code-cover the vfs layers + struct esp_modem_vfs_uart_creator uart_config = ESP_MODEM_VFS_DEFAULT_UART_CONFIG("/dev/uart/1"); + assert(vfs_create_uart(&uart_config, &dte_config.vfs_config) == true); + + auto dte = create_vfs_dte(&dte_config); + esp_vfs_dev_uart_use_driver(uart_config.uart.port_num); +#else + auto dte = create_uart_dte(&dte_config); +#endif // CONFIG_TEST_USE_VFS_TERM + assert(dte); + dte->set_error_cb([](terminal_error err) { + ESP_LOGE(TAG, "DTE reported terminal error: %d", static_cast(err)); + }); + + // Initialize PPP netif + esp_netif_config_t netif_ppp_config = ESP_NETIF_DEFAULT_PPP(); + esp_netif_t *esp_netif = esp_netif_new(&netif_ppp_config); + assert(esp_netif); + + // Initialize DCE +#ifdef CONFIG_TEST_DEVICE_PPPD_SERVER + auto dce = create(dte, esp_netif); +#else + esp_modem_dce_config_t dce_config = ESP_MODEM_DCE_DEFAULT_CONFIG(CONFIG_TEST_MODEM_APN); + auto dce = create_generic_dce(&dce_config, dte, esp_netif); + assert(dce); +#endif + + StatusHandler handler; + +#ifndef CONFIG_TEST_DEVICE_PPPD_SERVER + if (dce->set_mode(esp_modem::modem_mode::CMUX_MANUAL_MODE) && + dce->set_mode(esp_modem::modem_mode::CMUX_MANUAL_SWAP) && + dce->set_mode(esp_modem::modem_mode::CMUX_MANUAL_DATA)) { +#else + if (dce->set_mode(esp_modem::modem_mode::DATA_MODE)) { +#endif + ESP_LOGI(TAG, "Modem has correctly entered the desired mode (CMUX/DATA/Manual CMUX)"); + } else { + ESP_LOGE(TAG, "Failed to configure multiplexed command mode... exiting"); + return; + } + + if (!handler.wait_for(StatusHandler::IP_Event, 60000)) { + ESP_LOGE(TAG, "Cannot get IP within specified timeout... exiting"); + return; + } else if (handler.get_ip_event_type() == IP_EVENT_PPP_GOT_IP) { + ESP_LOGI(TAG, "Got IP address"); + + /* When connected to network, subscribe and publish some MQTT data */ + PublishOnce publish(&handler); + if (!publish.Connect()) { + ESP_LOGE(TAG, "Failed to connect to mqtt server"); + return; + } + if (!handler.wait_for(StatusHandler::MQTT_Connect, 60000)) { + ESP_LOGE(TAG, "Cannot connect to %s within specified timeout... exiting", CONFIG_BROKER_URI); + return; + } + ESP_LOGI(TAG, "Connected"); + + if (!publish.SubscribePublish()) { + ESP_LOGE(TAG, "Failed to subscribe and publish to mqtt server"); + return; + } + + if (!handler.wait_for(StatusHandler::MQTT_Data, 60000)) { + ESP_LOGE(TAG, "Didn't receive published data within specified timeout... exiting"); + return; + } + ESP_LOGI(TAG, "Received MQTT data"); + + } else if (handler.get_ip_event_type() == IP_EVENT_PPP_LOST_IP) { + ESP_LOGE(TAG, "PPP client has lost connection... exiting"); + return; + } + + + esp_modem::SignalGroup ota_done{}; + // now stop the LCP keepalive before performing OTA + esp_netif_ppp_config_t cfg; + ESP_ERROR_CHECK(esp_netif_ppp_get_params(esp_netif, &cfg)); + cfg.ppp_lcp_echo_disabled = true; + ESP_ERROR_CHECK(esp_netif_ppp_set_params(esp_netif, &cfg)); + + // Run the OTA in a separate task to keep sending commands to the modem at the same time + xTaskCreate(ota_task, "ota_task", 8192, &ota_done, 5, nullptr); + +#ifndef CONFIG_TEST_DEVICE_PPPD_SERVER + while (true) { + std::string str; + if (dce->get_imsi(str) == esp_modem::command_result::OK) { + ESP_LOGI(TAG, "Modem IMSI number: %s", str.c_str()); + } + if (ota_done.wait_any(OTA_OK | OTA_FAILED, 100)) { + break; + } + } +#else + ota_done.wait_any(OTA_OK | OTA_FAILED, portMAX_DELAY); +#endif // CONFIG_TEST_DEVICE_PPPD_SERVER + +#ifndef CONFIG_TEST_DEVICE_PPPD_SERVER + if (dce->set_mode(esp_modem::modem_mode::CMUX_MANUAL_EXIT)) { +#else + if (dce->set_mode(esp_modem::modem_mode::COMMAND_MODE)) { +#endif + ESP_LOGI(TAG, "Modem CMUX/DATA mode exit"); + } else { + ESP_LOGE(TAG, "Failed to configure desired mode... exiting"); + return; + } + + if (ota_done.is_any(OTA_OK)) { + ESP_LOGI(TAG, "Prepare to restart system!"); + esp_restart(); + } + +}