From 7f8541ff47a6dfed43425353cecf2e6c31f782a0 Mon Sep 17 00:00:00 2001 From: Shubham Kulkarni Date: Tue, 2 Feb 2021 13:25:40 +0530 Subject: [PATCH 1/3] esp_https_ota: Add support for partial image download Fix issue where binary_file_length field is greater than original image length --- .../esp_https_ota/include/esp_https_ota.h | 2 + components/esp_https_ota/src/esp_https_ota.c | 80 +++++++++++++++++-- .../en/api-reference/system/esp_https_ota.rst | 13 +++ 3 files changed, 89 insertions(+), 6 deletions(-) diff --git a/components/esp_https_ota/include/esp_https_ota.h b/components/esp_https_ota/include/esp_https_ota.h index 2966c5d0a3..2a987ca692 100644 --- a/components/esp_https_ota/include/esp_https_ota.h +++ b/components/esp_https_ota/include/esp_https_ota.h @@ -31,6 +31,8 @@ typedef struct { const esp_http_client_config_t *http_config; /*!< ESP HTTP client configuration */ http_client_init_cb_t http_client_init_cb; /*!< Callback after ESP HTTP client is initialised */ bool bulk_flash_erase; /*!< Erase entire flash partition during initialization. By default flash partition is erased during write operation and in chunk of 4K sector size */ + bool partial_http_download; /*!< Enable Firmware image to be downloaded over multiple HTTP requests */ + int max_http_request_size; /*!< Maximum request size for partial HTTP download */ } esp_https_ota_config_t; #define ESP_ERR_HTTPS_OTA_BASE (0x9000) diff --git a/components/esp_https_ota/src/esp_https_ota.c b/components/esp_https_ota/src/esp_https_ota.c index 6e06e7ccbe..5681c60d56 100644 --- a/components/esp_https_ota/src/esp_https_ota.c +++ b/components/esp_https_ota/src/esp_https_ota.c @@ -23,6 +23,7 @@ #define IMAGE_HEADER_SIZE sizeof(esp_image_header_t) + sizeof(esp_image_segment_header_t) + sizeof(esp_app_desc_t) + 1 #define DEFAULT_OTA_BUF_SIZE IMAGE_HEADER_SIZE +#define DEFAULT_REQUEST_SIZE (64 * 1024) static const char *TAG = "esp_https_ota"; typedef enum { @@ -39,8 +40,11 @@ struct esp_https_ota_handle { char *ota_upgrade_buf; size_t ota_upgrade_buf_size; int binary_file_len; + int image_length; + int max_http_request_size; esp_https_ota_state state; bool bulk_flash_erase; + bool partial_http_download; }; typedef struct esp_https_ota_handle esp_https_ota_t; @@ -167,6 +171,9 @@ esp_err_t esp_https_ota_begin(esp_https_ota_config_t *ota_config, esp_https_ota_ return ESP_ERR_NO_MEM; } + https_ota_handle->partial_http_download = ota_config->partial_http_download; + https_ota_handle->max_http_request_size = (ota_config->max_http_request_size == 0) ? DEFAULT_REQUEST_SIZE : ota_config->max_http_request_size; + /* Initiate HTTP Connection */ https_ota_handle->http_client = esp_http_client_init(ota_config->http_config); if (https_ota_handle->http_client == NULL) { @@ -175,6 +182,30 @@ esp_err_t esp_https_ota_begin(esp_https_ota_config_t *ota_config, esp_https_ota_ goto failure; } + if (https_ota_handle->partial_http_download) { + esp_http_client_set_method(https_ota_handle->http_client, HTTP_METHOD_HEAD); + err = esp_http_client_perform(https_ota_handle->http_client); + if (err != ESP_OK || esp_http_client_get_status_code(https_ota_handle->http_client) != HttpStatus_Ok) { + ESP_LOGE(TAG, "Failed to get image length"); + goto http_cleanup; + } + https_ota_handle->image_length = esp_http_client_get_content_length(https_ota_handle->http_client); + esp_http_client_close(https_ota_handle->http_client); + + if (https_ota_handle->image_length > https_ota_handle->max_http_request_size) { + char *header_val = NULL; + asprintf(&header_val, "bytes=0-%d", https_ota_handle->max_http_request_size - 1); + if (header_val == NULL) { + ESP_LOGE(TAG, "Failed to allocate memory for HTTP header"); + err = ESP_ERR_NO_MEM; + goto http_cleanup; + } + esp_http_client_set_header(https_ota_handle->http_client, "Range", header_val); + free(header_val); + } + esp_http_client_set_method(https_ota_handle->http_client, HTTP_METHOD_GET); + } + if (ota_config->http_client_init_cb) { err = ota_config->http_client_init_cb(https_ota_handle->http_client); if (err != ESP_OK) { @@ -243,7 +274,7 @@ esp_err_t esp_https_ota_get_img_desc(esp_https_ota_handle_t https_ota_handle, es * while loop is added to download complete image headers, even if the headers * are not sent in a single packet. */ - while (data_read_size > 0 && !esp_https_ota_is_complete_data_received(https_ota_handle)) { + while (data_read_size > 0 && !esp_http_client_is_complete_data_received(handle->http_client)) { data_read = esp_http_client_read(handle->http_client, (handle->ota_upgrade_buf + bytes_read), data_read_size); @@ -294,7 +325,13 @@ esp_err_t esp_https_ota_perform(esp_https_ota_handle_t https_ota_handle) then the image data read there should be written to OTA partition */ if (handle->binary_file_len) { - return _ota_write(handle, (const void *)handle->ota_upgrade_buf, handle->binary_file_len); + /* + * Header length gets added to handle->binary_file_len in _ota_write + * Clear handle->binary_file_len to avoid additional 289 bytes in binary_file_len + */ + int binary_file_len = handle->binary_file_len; + handle->binary_file_len = 0; + return _ota_write(handle, (const void *)handle->ota_upgrade_buf, binary_file_len); } /* falls through */ case ESP_HTTPS_OTA_IN_PROGRESS: @@ -303,10 +340,10 @@ esp_err_t esp_https_ota_perform(esp_https_ota_handle_t https_ota_handle) handle->ota_upgrade_buf_size); if (data_read == 0) { /* - * esp_https_ota_is_complete_data_received is added to check whether + * esp_http_client_is_complete_data_received is added to check whether * complete image is received. */ - bool is_recv_complete = esp_https_ota_is_complete_data_received(https_ota_handle); + bool is_recv_complete = esp_http_client_is_complete_data_received(handle->http_client); /* * As esp_http_client_read doesn't return negative error code if select fails, we rely on * `errno` to check for underlying transport connectivity closure if any. @@ -326,20 +363,51 @@ esp_err_t esp_https_ota_perform(esp_https_ota_handle_t https_ota_handle) } else { return ESP_FAIL; } - handle->state = ESP_HTTPS_OTA_SUCCESS; + if (!handle->partial_http_download || (handle->partial_http_download && handle->image_length == handle->binary_file_len)) { + handle->state = ESP_HTTPS_OTA_SUCCESS; + } break; default: ESP_LOGE(TAG, "Invalid ESP HTTPS OTA State"); return ESP_FAIL; break; } + if (handle->partial_http_download) { + if (handle->state == ESP_HTTPS_OTA_IN_PROGRESS && handle->image_length > handle->binary_file_len) { + esp_http_client_close(handle->http_client); + char *header_val = NULL; + if ((handle->image_length - handle->binary_file_len) > handle->max_http_request_size) { + asprintf(&header_val, "bytes=%d-%d", handle->binary_file_len, (handle->binary_file_len + handle->max_http_request_size - 1)); + } else { + asprintf(&header_val, "bytes=%d-", handle->binary_file_len); + } + if (header_val == NULL) { + ESP_LOGE(TAG, "Failed to allocate memory for HTTP header"); + return ESP_ERR_NO_MEM; + } + esp_http_client_set_header(handle->http_client, "Range", header_val); + free(header_val); + err = _http_connect(handle->http_client); + if (err != ESP_OK) { + ESP_LOGE(TAG, "Failed to establish HTTP connection"); + return ESP_FAIL; + } + return ESP_ERR_HTTPS_OTA_IN_PROGRESS; + } + } return ESP_OK; } bool esp_https_ota_is_complete_data_received(esp_https_ota_handle_t https_ota_handle) { + bool ret = false; esp_https_ota_t *handle = (esp_https_ota_t *)https_ota_handle; - return esp_http_client_is_complete_data_received(handle->http_client); + if (handle->partial_http_download) { + ret = (handle->image_length == handle->binary_file_len); + } else { + ret = esp_http_client_is_complete_data_received(handle->http_client); + } + return ret; } esp_err_t esp_https_ota_finish(esp_https_ota_handle_t https_ota_handle) diff --git a/docs/en/api-reference/system/esp_https_ota.rst b/docs/en/api-reference/system/esp_https_ota.rst index ccf18e37e8..b4a4c57970 100644 --- a/docs/en/api-reference/system/esp_https_ota.rst +++ b/docs/en/api-reference/system/esp_https_ota.rst @@ -29,6 +29,19 @@ Application Example return ESP_OK; } +Partial Image Download over HTTPS +--------------------------------- + +To use partial image download feature, enable ``partial_http_download`` configuration in ``esp_https_ota_config_t``. +When this configuration is enabled, firmware image will be downloaded in multiple HTTP requests of specified size. +Maximum content length of each request can be specified by setting ``max_http_request_size`` to required value. + +This option is useful while fetching image from a service like AWS S3, where mbedTLS Rx buffer size (:ref:`CONFIG_MBEDTLS_SSL_IN_CONTENT_LEN`) +can be set to lower value which is not possible without enabling this configuration. + +Default value of mbedTLS Rx buffer size is set to 16K. By using partial_http_download with max_http_request_size of 4K, +size of mbedTLS Rx buffer can be reduced to 4K. With this confiuration, memory saving of around 12K is expected. + .. only:: esp32 Signature Verification From 807bf9688c87098bc4b578adbb002392a2f7a618 Mon Sep 17 00:00:00 2001 From: Itay Perl Date: Wed, 10 Feb 2021 11:02:13 +0530 Subject: [PATCH 2/3] ESP HTTPS OTA: send POST request body if set Signed-off-by: Shubham Kulkarni Closes: https://github.com/espressif/esp-idf/issues/6390 Merges: https://github.com/espressif/esp-idf/pull/6391 --- components/esp_https_ota/src/esp_https_ota.c | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/components/esp_https_ota/src/esp_https_ota.c b/components/esp_https_ota/src/esp_https_ota.c index 5681c60d56..386fffd6b3 100644 --- a/components/esp_https_ota/src/esp_https_ota.c +++ b/components/esp_https_ota/src/esp_https_ota.c @@ -104,11 +104,29 @@ static esp_err_t _http_connect(esp_http_client_handle_t http_client) esp_err_t err = ESP_FAIL; int status_code, header_ret; do { - err = esp_http_client_open(http_client, 0); + char *post_data = NULL; + /* Send POST request if body is set. + * Note: Sending POST request is not supported if partial_http_download + * is enabled + */ + int post_len = esp_http_client_get_post_field(http_client, &post_data); + err = esp_http_client_open(http_client, post_len); if (err != ESP_OK) { ESP_LOGE(TAG, "Failed to open HTTP connection: %s", esp_err_to_name(err)); return err; } + if (post_len) { + int write_len = 0; + while (post_len > 0) { + write_len = esp_http_client_write(http_client, post_data, post_len); + if (write_len < 0) { + ESP_LOGE(TAG, "Write failed"); + return ESP_FAIL; + } + } + post_len -= write_len; + post_data += write_len; + } header_ret = esp_http_client_fetch_headers(http_client); if (header_ret < 0) { return header_ret; From b2eb199502a1dcdadf9d4558c4792117529e4a1a Mon Sep 17 00:00:00 2001 From: Hassan DRAGA Date: Fri, 26 Feb 2021 17:40:06 -0400 Subject: [PATCH 3/3] Update native_ota_example.c * assert(update_partition) should be before trying to use update_partition in ESP_LOGI(). * if we show detailed description about the error ESP_ERR_OTA_VALIDATE_FAILED, why show it again using esp_err_to_name() ? Merges https://github.com/espressif/esp-idf/pull/6623 Signed-off-by: Shubham Kulkarni --- .../system/ota/native_ota_example/main/native_ota_example.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/examples/system/ota/native_ota_example/main/native_ota_example.c b/examples/system/ota/native_ota_example/main/native_ota_example.c index 0df3f5f105..5e1d0f51d2 100644 --- a/examples/system/ota/native_ota_example/main/native_ota_example.c +++ b/examples/system/ota/native_ota_example/main/native_ota_example.c @@ -132,9 +132,9 @@ static void ota_example_task(void *pvParameter) esp_http_client_fetch_headers(client); update_partition = esp_ota_get_next_update_partition(NULL); + assert(update_partition != NULL); ESP_LOGI(TAG, "Writing to partition subtype %d at offset 0x%x", update_partition->subtype, update_partition->address); - assert(update_partition != NULL); int binary_file_length = 0; /*deal with all receive packet*/ @@ -234,8 +234,9 @@ static void ota_example_task(void *pvParameter) if (err != ESP_OK) { if (err == ESP_ERR_OTA_VALIDATE_FAILED) { ESP_LOGE(TAG, "Image validation failed, image is corrupted"); + } else { + ESP_LOGE(TAG, "esp_ota_end failed (%s)!", esp_err_to_name(err)); } - ESP_LOGE(TAG, "esp_ota_end failed (%s)!", esp_err_to_name(err)); http_cleanup(client); task_fatal_error(); }