From 3a768f51c3f932663af99384210d7d618ae885ea Mon Sep 17 00:00:00 2001 From: Peter Hoddie Date: Tue, 5 May 2020 18:03:25 -0700 Subject: [PATCH 1/2] app_update: add support for incremental flash erase This patch modifies the OTA implementation to incremental erase flash sectors as they are written, rather than bulk erasing them all at once when OTA begins. This solves several problems: - The bulk flash erase can take so long that it triggers the watchdog timer. Disabling the watchdog is an undesirable risk. - The bulk erase interferes with the responsiveness of the system to user input. - Incremental erasing eliminates the need to pass the firmware image size to esp_ota_begin to avoid erasing more flash sectors than required. - When installing an OTA received via HTTP, the size of the firmware image is known when the content-length header is received. Calling esp_ota_begin at that time causes a long delay to do the bulk erase which causes the TCP transfer speed to be measurably slowed when the transfer resumes after the erase. Incremental erasing eliminates this TCP slowdown. Update: Rework so erase optimization is only applied when image_size is set to OTA_WITH_SEQUENTIAL_WRITES #5246 Merges https://github.com/espressif/esp-idf/pull/5246 --- components/app_update/esp_ota_ops.c | 52 ++++++++++++--------- components/app_update/include/esp_ota_ops.h | 1 + 2 files changed, 32 insertions(+), 21 deletions(-) diff --git a/components/app_update/esp_ota_ops.c b/components/app_update/esp_ota_ops.c index 8ef9e0ba70..cabfe87cff 100644 --- a/components/app_update/esp_ota_ops.c +++ b/components/app_update/esp_ota_ops.c @@ -52,7 +52,7 @@ typedef struct ota_ops_entry_ { uint32_t handle; const esp_partition_t *part; - uint32_t erased_size; + bool need_erase; uint32_t wrote_size; uint8_t partial_bytes; uint8_t partial_data[16]; @@ -158,16 +158,17 @@ esp_err_t esp_ota_begin(const esp_partition_t *partition, size_t image_size, esp } #endif - // If input image size is 0 or OTA_SIZE_UNKNOWN, erase entire partition - if ((image_size == 0) || (image_size == OTA_SIZE_UNKNOWN)) { - ret = esp_partition_erase_range(partition, 0, partition->size); - } else { - const int aligned_erase_size = (image_size + SPI_FLASH_SEC_SIZE - 1) & ~(SPI_FLASH_SEC_SIZE - 1); - ret = esp_partition_erase_range(partition, 0, aligned_erase_size); - } - - if (ret != ESP_OK) { - return ret; + if (image_size != OTA_WITH_SEQUENTIAL_WRITES) { + // If input image size is 0 or OTA_SIZE_UNKNOWN, erase entire partition + if ((image_size == 0) || (image_size == OTA_SIZE_UNKNOWN)) { + ret = esp_partition_erase_range(partition, 0, partition->size); + } else { + const int aligned_erase_size = (image_size + SPI_FLASH_SEC_SIZE - 1) & ~(SPI_FLASH_SEC_SIZE - 1); + ret = esp_partition_erase_range(partition, 0, aligned_erase_size); + } + if (ret != ESP_OK) { + return ret; + } } new_entry = (ota_ops_entry_t *) calloc(sizeof(ota_ops_entry_t), 1); @@ -177,14 +178,9 @@ esp_err_t esp_ota_begin(const esp_partition_t *partition, size_t image_size, esp LIST_INSERT_HEAD(&s_ota_ops_entries_head, new_entry, entries); - if ((image_size == 0) || (image_size == OTA_SIZE_UNKNOWN)) { - new_entry->erased_size = partition->size; - } else { - new_entry->erased_size = image_size; - } - new_entry->part = partition; new_entry->handle = ++s_ota_ops_last_handle; + new_entry->need_erase = (image_size == OTA_WITH_SEQUENTIAL_WRITES); *out_handle = new_entry->handle; return ESP_OK; } @@ -203,8 +199,22 @@ esp_err_t esp_ota_write(esp_ota_handle_t handle, const void *data, size_t size) // find ota handle in linked list for (it = LIST_FIRST(&s_ota_ops_entries_head); it != NULL; it = LIST_NEXT(it, entries)) { if (it->handle == handle) { - // must erase the partition before writing to it - assert(it->erased_size > 0 && "must erase the partition before writing to it"); + if (it->need_erase) { + // must erase the partition before writing to it + uint32_t first_sector = it->wrote_size / SPI_FLASH_SEC_SIZE; + uint32_t last_sector = (it->wrote_size + size) / SPI_FLASH_SEC_SIZE; + + ret = ESP_OK; + if ((it->wrote_size % SPI_FLASH_SEC_SIZE) == 0) { + ret = esp_partition_erase_range(it->part, it->wrote_size, ((last_sector - first_sector) + 1) * SPI_FLASH_SEC_SIZE); + } else if (first_sector != last_sector) { + ret = esp_partition_erase_range(it->part, (first_sector + 1) * SPI_FLASH_SEC_SIZE, (last_sector - first_sector) * SPI_FLASH_SEC_SIZE); + } + if (ret != ESP_OK) { + return ret; + } + } + if (it->wrote_size == 0 && it->partial_bytes == 0 && size > 0 && data_bytes[0] != ESP_IMAGE_HEADER_MAGIC) { ESP_LOGE(TAG, "OTA image has invalid magic byte (expected 0xE9, saw 0x%02x)", data_bytes[0]); return ESP_ERR_OTA_VALIDATE_FAILED; @@ -270,7 +280,7 @@ esp_err_t esp_ota_write_with_offset(esp_ota_handle_t handle, const void *data, s for (it = LIST_FIRST(&s_ota_ops_entries_head); it != NULL; it = LIST_NEXT(it, entries)) { if (it->handle == handle) { // must erase the partition before writing to it - assert(it->erased_size > 0 && "must erase the partition before writing to it"); + assert(it->need_erase == 0 && "must erase the partition before writing to it"); /* esp_ota_write_with_offset is used to write data in non contiguous manner. * Hence, unaligned data(less than 16 bytes) cannot be cached if flash encryption is enabled. @@ -310,7 +320,7 @@ esp_err_t esp_ota_end(esp_ota_handle_t handle) /* 'it' holds the ota_ops_entry_t for 'handle' */ // esp_ota_end() is only valid if some data was written to this handle - if ((it->erased_size == 0) || (it->wrote_size == 0)) { + if (it->wrote_size == 0) { ret = ESP_ERR_INVALID_ARG; goto cleanup; } diff --git a/components/app_update/include/esp_ota_ops.h b/components/app_update/include/esp_ota_ops.h index ac0ae656e0..cf24e75fa7 100644 --- a/components/app_update/include/esp_ota_ops.h +++ b/components/app_update/include/esp_ota_ops.h @@ -29,6 +29,7 @@ extern "C" #endif #define OTA_SIZE_UNKNOWN 0xffffffff /*!< Used for esp_ota_begin() if new image size is unknown */ +#define OTA_WITH_SEQUENTIAL_WRITES 0xfffffffe /*!< Used for esp_ota_begin() if new image size is unknown and erase can be done in incremental manner (assuming write operation is in continuous sequence) */ #define ESP_ERR_OTA_BASE 0x1500 /*!< Base error code for ota_ops api */ #define ESP_ERR_OTA_PARTITION_CONFLICT (ESP_ERR_OTA_BASE + 0x01) /*!< Error if request was to write or erase the current running partition */ From 33d0024034bb553ed4908f6b8862e69b0982baa8 Mon Sep 17 00:00:00 2001 From: Mahavir Jain Date: Tue, 21 Jul 2020 18:23:28 +0530 Subject: [PATCH 2/2] esp_https_ota: change erase to incremental during flash write --- components/esp_https_ota/include/esp_https_ota.h | 1 + components/esp_https_ota/src/esp_https_ota.c | 6 ++++-- .../system/ota/native_ota_example/main/native_ota_example.c | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/components/esp_https_ota/include/esp_https_ota.h b/components/esp_https_ota/include/esp_https_ota.h index 1afa44a73a..b88fdfaa5c 100644 --- a/components/esp_https_ota/include/esp_https_ota.h +++ b/components/esp_https_ota/include/esp_https_ota.h @@ -30,6 +30,7 @@ typedef esp_err_t(*http_client_init_cb_t)(esp_http_client_handle_t); typedef struct { const esp_http_client_config_t *http_config; /*!< ESP HTTP client configuration */ http_client_init_cb_t http_client_init_cb; /*!< Callback after ESP HTTP client is initialised */ + bool bulk_flash_erase; /*!< Erase entire flash partition during initialization. By default flash partition is erased during write operation and in chunk of 4K sector size */ } esp_https_ota_config_t; #define ESP_ERR_HTTPS_OTA_BASE (0x9000) diff --git a/components/esp_https_ota/src/esp_https_ota.c b/components/esp_https_ota/src/esp_https_ota.c index 2559de7673..8980986362 100644 --- a/components/esp_https_ota/src/esp_https_ota.c +++ b/components/esp_https_ota/src/esp_https_ota.c @@ -39,6 +39,7 @@ struct esp_https_ota_handle { size_t ota_upgrade_buf_size; int binary_file_len; esp_https_ota_state state; + bool bulk_flash_erase; }; typedef struct esp_https_ota_handle esp_https_ota_t; @@ -207,7 +208,7 @@ esp_err_t esp_https_ota_begin(esp_https_ota_config_t *ota_config, esp_https_ota_ goto http_cleanup; } https_ota_handle->ota_upgrade_buf_size = alloc_size; - + https_ota_handle->bulk_flash_erase = ota_config->bulk_flash_erase; https_ota_handle->binary_file_len = 0; *handle = (esp_https_ota_handle_t)https_ota_handle; https_ota_handle->state = ESP_HTTPS_OTA_BEGIN; @@ -280,9 +281,10 @@ esp_err_t esp_https_ota_perform(esp_https_ota_handle_t https_ota_handle) esp_err_t err; int data_read; + const int erase_size = handle->bulk_flash_erase ? OTA_SIZE_UNKNOWN : OTA_WITH_SEQUENTIAL_WRITES; switch (handle->state) { case ESP_HTTPS_OTA_BEGIN: - err = esp_ota_begin(handle->update_partition, OTA_SIZE_UNKNOWN, &handle->update_handle); + err = esp_ota_begin(handle->update_partition, erase_size, &handle->update_handle); if (err != ESP_OK) { ESP_LOGE(TAG, "esp_ota_begin failed (%s)", esp_err_to_name(err)); return err; diff --git a/examples/system/ota/native_ota_example/main/native_ota_example.c b/examples/system/ota/native_ota_example/main/native_ota_example.c index ea29d916ef..a81cf9d7a2 100644 --- a/examples/system/ota/native_ota_example/main/native_ota_example.c +++ b/examples/system/ota/native_ota_example/main/native_ota_example.c @@ -183,7 +183,7 @@ static void ota_example_task(void *pvParameter) image_header_was_checked = true; - err = esp_ota_begin(update_partition, OTA_SIZE_UNKNOWN, &update_handle); + err = esp_ota_begin(update_partition, OTA_WITH_SEQUENTIAL_WRITES, &update_handle); if (err != ESP_OK) { ESP_LOGE(TAG, "esp_ota_begin failed (%s)", esp_err_to_name(err)); http_cleanup(client);