refactor(usb/host_msc_example): Increase transfer speeds

- Upgrade to MSC driver 1.1.1 for zero copy transfers
- Use setvbuf() to increase size of VFS file buffer
- Add example test

examples/peripherals/usb/host/msc/README.md

@@ -5,15 +5,24 @@
 
 ## Overview
 
-This example demonstrates usage of Mass Storage Class to get access to storage on USB memory stick.
+This example demonstrates usage of the MSC (Mass Storage Class) to access storage on a USB flash drive. Upon connection of the flash drive, it is mounted to the Virtual filesystem. The following example operations are then performed:
-Upon connection of USB stick, storage is mounted to Virtual filesystem. Example then creates `ESP` subdirectory(if not present already), as well as `text.txt` file. Its content is then repetitively printed to monitor until USB stick is manually ejected. User can decide whether or not to deinitialize the whole 
+
-USB stack or not by shorting GPIO10 to ground. When GPIO10 is left unconnected USB stack is not deinitialized, USB stick can be plugged-in again. 
+1. Print device info (capacity, sectors size, and count...)
+2. List all folders and files in the root directory of the USB flash drive
+3. Create `ESP` subdirectory (if not present already), as well as a `text.txt` file
+4. Run read/write benchmarks by transferring 1 MB of data to a `dummy` file
+
+
+### USB Reconnections
+
+The example is run in a loop so that it can demonstrate USB connection and reconnection handling. If you want to deinitialize the entire  USB Host Stack, you can short GPIO0 to GND. GPIO0 is usually mapped to a BOOT button, thus pressing the button will deinitialize the stack.
+
 
 ### Hardware Required
 
 * Development board with USB capable ESP SoC (ESP32-S2/ESP32-S3)
 * A USB cable for Power supply and programming
-* A USB memory stick
+* A USB flash drive
 
 ### Common Pin Assignments
 
@@ -30,7 +39,7 @@ ESP BOARD    USB CONNECTOR (type A)
                    --
 ```
 
-Additionally, GPIO10 can be shorted to ground in order to deinitialize USB stack after ejecting USB stick. 
+Additionally, GPIO0 can be shorted to ground in order to deinitialize USB stack. 
 
 ### Build and Flash
 
@@ -48,16 +57,27 @@ See the Getting Started Guide for full steps to configure and use ESP-IDF to bui
 
 ```
 ...
-I (274) cpu_start: Starting scheduler on PRO CPU.
+I (380) example: Waiting for USB flash drive to be connected
-I (339) APP: Waiting for USB stick to be connected
+I (790) example: MSC device connected
+...
 Device info:
-         PID: 0x5678 
+         Capacity: 29339 MB
-         VID: 0xFFFF 
+         Sector size: 512
-         iProduct: Disk 2.0 
+         Sector count: 60088319
+         PID: 0x5595
+         VID: 0x0781
+         iProduct:  SanDisk 3.2Gen1
          iManufacturer:  USB
-         iSerialNumber: 92072836B2589224378 
+         iSerialNumber: 0401545df64623a907abf299bae54c9
-I (719) APP: Writing file
+I (990) example: ls command output:
-I (749) APP: Reading file
+SYSTEM~1
-I (749) APP: Read from file: 'Hello World!'
+ESP
-I (759) APP: Done
+DUMMY
+I (1000) example: Reading file
+I (1010) example: Read from file '/usb/esp/test.txt': 'Hello World!'
+I (1030) example: Writing to file /usb/esp/dummy
+I (2160) example: Write speed 0.93 MiB/s
+I (2160) example: Reading from file /usb/esp/dummy
+I (3110) example: Read speed 1.10 MiB/s
+I (3140) example: Example finished, you can disconnect the USB flash drive
 ```

examples/peripherals/usb/host/msc/main/idf_component.yml

@@ -1,4 +1,4 @@
 ## IDF Component Manager Manifest File
 dependencies:
   idf: ">=4.4"
-  usb_host_msc: "^1.0.4"
+  usb_host_msc: "^1.1.1"

examples/peripherals/usb/host/msc/main/msc_example_main.c

@@ -8,43 +8,87 @@
 #include <string.h>
 #include <assert.h>
 #include <sys/stat.h>
+#include <dirent.h>
+#include <inttypes.h>
 #include "freertos/FreeRTOS.h"
 #include "freertos/task.h"
-#include "freertos/event_groups.h"
+#include "freertos/queue.h"
+#include "esp_timer.h"
 #include "esp_err.h"
 #include "esp_log.h"
 #include "usb/usb_host.h"
-#include "msc_host.h"
+#include "usb/msc_host.h"
-#include "msc_host_vfs.h"
+#include "usb/msc_host_vfs.h"
 #include "ffconf.h"
 #include "errno.h"
 #include "driver/gpio.h"
-#include "esp_vfs_fat.h"
-
-#define USB_DISCONNECT_PIN  GPIO_NUM_10
-
-#define READY_TO_UNINSTALL (HOST_NO_CLIENT | HOST_ALL_FREE)
-
-typedef enum {
-    HOST_NO_CLIENT = 0x1,
-    HOST_ALL_FREE = 0x2,
-    DEVICE_CONNECTED = 0x4,
-    DEVICE_DISCONNECTED = 0x8,
-    DEVICE_ADDRESS_MASK = 0xFF0,
-} app_event_t;
 
 static const char *TAG = "example";
-static EventGroupHandle_t usb_flags;
+#define MNT_PATH         "/usb"     // Path in the Virtual File System, where the USB flash drive is going to be mounted
+#define APP_QUIT_PIN     GPIO_NUM_0 // BOOT button on most boards
+#define BUFFER_SIZE      4096       // The read/write performance can be improved with larger buffer for the cost of RAM, 4kB is enough for most usecases
 
+/**
+ * @brief Application Queue and its messages ID
+ */
+static QueueHandle_t app_queue;
+typedef struct {
+    enum {
+        APP_QUIT,                // Signals request to exit the application
+        APP_DEVICE_CONNECTED,    // USB device connect event
+        APP_DEVICE_DISCONNECTED, // USB device disconnect event
+    } id;
+    union {
+        uint8_t new_dev_address; // Address of new USB device for APP_DEVICE_CONNECTED event if
+    } data;
+} app_message_t;
+
+/**
+ * @brief BOOT button pressed callback
+ *
+ * Signal application to exit the main task
+ *
+ * @param[in] arg Unused
+ */
+static void gpio_cb(void *arg)
+{
+    BaseType_t xTaskWoken = pdFALSE;
+    app_message_t message = {
+        .id = APP_QUIT,
+    };
+
+    if (app_queue) {
+        xQueueSendFromISR(app_queue, &message, &xTaskWoken);
+    }
+
+    if (xTaskWoken == pdTRUE) {
+        portYIELD_FROM_ISR();
+    }
+}
+
+/**
+ * @brief MSC driver callback
+ *
+ * Signal device connection/disconnection to the main task
+ *
+ * @param[in] event MSC event
+ * @param[in] arg   MSC event data
+ */
 static void msc_event_cb(const msc_host_event_t *event, void *arg)
 {
     if (event->event == MSC_DEVICE_CONNECTED) {
         ESP_LOGI(TAG, "MSC device connected");
-        // Obtained USB device address is placed after application events
+        app_message_t message = {
-        xEventGroupSetBits(usb_flags, DEVICE_CONNECTED | (event->device.address << 4));
+            .id = APP_DEVICE_CONNECTED,
+            .data.new_dev_address = event->device.address,
+        };
+        xQueueSend(app_queue, &message, portMAX_DELAY);
     } else if (event->event == MSC_DEVICE_DISCONNECTED) {
-        xEventGroupSetBits(usb_flags, DEVICE_DISCONNECTED);
         ESP_LOGI(TAG, "MSC device disconnected");
+        app_message_t message = {
+            .id = APP_DEVICE_DISCONNECTED,
+        };
+        xQueueSend(app_queue, &message, portMAX_DELAY);
     }
 }
 
@@ -57,24 +101,19 @@ static void print_device_info(msc_host_device_info_t *info)
     printf("\t Capacity: %llu MB\n", capacity);
     printf("\t Sector size: %"PRIu32"\n", info->sector_size);
     printf("\t Sector count: %"PRIu32"\n", info->sector_count);
-    printf("\t PID: 0x%4X \n", info->idProduct);
+    printf("\t PID: 0x%04X \n", info->idProduct);
-    printf("\t VID: 0x%4X \n", info->idVendor);
+    printf("\t VID: 0x%04X \n", info->idVendor);
     wprintf(L"\t iProduct: %S \n", info->iProduct);
     wprintf(L"\t iManufacturer: %S \n", info->iManufacturer);
     wprintf(L"\t iSerialNumber: %S \n", info->iSerialNumber);
 }
 
-static bool file_exists(const char *file_path)
-{
-    struct stat buffer;
-    return stat(file_path, &buffer) == 0;
-}
-
 static void file_operations(void)
 {
     const char *directory = "/usb/esp";
     const char *file_path = "/usb/esp/test.txt";
 
+    // Create /usb/esp directory
     struct stat s = {0};
     bool directory_exists = stat(directory, &s) == 0;
     if (!directory_exists) {
@@ -83,7 +122,8 @@ static void file_operations(void)
         }
     }
 
-    if (!file_exists(file_path)) {
+    // Create /usb/esp/test.txt file, if it doesn't exist
+    if (stat(file_path, &s) != 0) {
         ESP_LOGI(TAG, "Creating file");
         FILE *f = fopen(file_path, "w");
         if (f == NULL) {
@@ -94,6 +134,7 @@ static void file_operations(void)
         fclose(f);
     }
 
+    // Read back the file
     FILE *f;
     ESP_LOGI(TAG, "Reading file");
     f = fopen(file_path, "r");
@@ -109,69 +150,64 @@ static void file_operations(void)
     if (pos) {
         *pos = '\0';
     }
-    ESP_LOGI(TAG, "Read from file: '%s'", line);
+    ESP_LOGI(TAG, "Read from file '%s': '%s'", file_path, line);
 }
 
-// Handles common USB host library events
+void speed_test(void)
-static void handle_usb_events(void *args)
 {
-    while (1) {
+#define TEST_FILE "/usb/esp/dummy"
-        uint32_t event_flags;
+#define ITERATIONS  256  // 256 * 4kb = 1MB
-        usb_host_lib_handle_events(portMAX_DELAY, &event_flags);
+    int64_t test_start, test_end;
 
-        // Release devices once all clients has deregistered
+    FILE *f = fopen(TEST_FILE, "wb+");
-        if (event_flags & USB_HOST_LIB_EVENT_FLAGS_NO_CLIENTS) {
+    if (f == NULL) {
-            usb_host_device_free_all();
+        ESP_LOGE(TAG, "Failed to open file for writing");
-            xEventGroupSetBits(usb_flags, HOST_NO_CLIENT);
+        return;
     }
-        // Give ready_to_uninstall_usb semaphore to indicate that USB Host library
+    // Set larger buffer for this file. It results in larger and more effective USB transfers
-        // can be deinitialized, and terminate this task.
+    setvbuf(f, NULL, _IOFBF, BUFFER_SIZE);
-        if (event_flags & USB_HOST_LIB_EVENT_FLAGS_ALL_FREE) {
+
-            xEventGroupSetBits(usb_flags, HOST_ALL_FREE);
+    // Allocate application buffer used for read/write
+    uint8_t *data = malloc(BUFFER_SIZE);
+    assert(data);
+
+    ESP_LOGI(TAG, "Writing to file %s", TEST_FILE);
+    test_start = esp_timer_get_time();
+    for (int i = 0; i < ITERATIONS; i++) {
+        if (fwrite(data, BUFFER_SIZE, 1, f) == 0) {
+            return;
         }
     }
+    test_end = esp_timer_get_time();
+    ESP_LOGI(TAG, "Write speed %1.2f MiB/s", (BUFFER_SIZE * ITERATIONS) / (float)(test_end - test_start));
+    rewind(f);
 
-    vTaskDelete(NULL);
+    ESP_LOGI(TAG, "Reading from file %s", TEST_FILE);
+    test_start = esp_timer_get_time();
+    for (int i = 0; i < ITERATIONS; i++) {
+        if (0 == fread(data, BUFFER_SIZE, 1, f)) {
+            return;
+        }
+    }
+    test_end = esp_timer_get_time();
+    ESP_LOGI(TAG, "Read speed %1.2f MiB/s", (BUFFER_SIZE * ITERATIONS) / (float)(test_end - test_start));
+
+    fclose(f);
+    free(data);
 }
 
-static uint8_t wait_for_msc_device(void)
+/**
+ * @brief USB task
+ *
+ * Install USB Host Library and MSC driver.
+ * Handle USB Host Library events
+ *
+ * @param[in] args Unused
+ */
+static void usb_task(void *args)
 {
-    EventBits_t event;
-
-    ESP_LOGI(TAG, "Waiting for USB stick to be connected");
-    event = xEventGroupWaitBits(usb_flags, DEVICE_CONNECTED | DEVICE_ADDRESS_MASK,
-                                pdTRUE, pdFALSE, portMAX_DELAY);
-    ESP_LOGI(TAG, "connection...");
-    // Extract USB device address from event group bits
-    return (event & DEVICE_ADDRESS_MASK) >> 4;
-}
-
-static bool wait_for_event(EventBits_t event, TickType_t timeout)
-{
-    return xEventGroupWaitBits(usb_flags, event, pdTRUE, pdTRUE, timeout) & event;
-}
-
-void app_main(void)
-{
-    msc_host_device_handle_t msc_device;
-    msc_host_vfs_handle_t vfs_handle;
-    msc_host_device_info_t info;
-    BaseType_t task_created;
-
-    const gpio_config_t input_pin = {
-        .pin_bit_mask = BIT64(USB_DISCONNECT_PIN),
-        .mode = GPIO_MODE_INPUT,
-        .pull_up_en = GPIO_PULLUP_ENABLE,
-    };
-    ESP_ERROR_CHECK(gpio_config(&input_pin));
-
-    usb_flags = xEventGroupCreate();
-    assert(usb_flags);
-
     const usb_host_config_t host_config = { .intr_flags = ESP_INTR_FLAG_LEVEL1 };
     ESP_ERROR_CHECK(usb_host_install(&host_config));
-    task_created = xTaskCreate(handle_usb_events, "usb_events", 2048, NULL, 2, NULL);
-    assert(task_created);
 
     const msc_host_driver_config_t msc_config = {
         .create_backround_task = true,
@@ -181,37 +217,109 @@ void app_main(void)
     };
     ESP_ERROR_CHECK(msc_host_install(&msc_config));
 
+    while (true) {
+        uint32_t event_flags;
+        usb_host_lib_handle_events(portMAX_DELAY, &event_flags);
+
+        // Release devices once all clients has deregistered
+        if (event_flags & USB_HOST_LIB_EVENT_FLAGS_NO_CLIENTS) {
+            if (usb_host_device_free_all() == ESP_OK) {
+                break;
+            };
+        }
+        if (event_flags & USB_HOST_LIB_EVENT_FLAGS_ALL_FREE) {
+            break;
+        }
+    }
+
+    vTaskDelay(10); // Give clients some time to uninstall
+    ESP_LOGI(TAG, "Deinitializing USB");
+    ESP_ERROR_CHECK(usb_host_uninstall());
+    vTaskDelete(NULL);
+}
+
+void app_main(void)
+{
+    // Create FreeRTOS primitives
+    app_queue = xQueueCreate(5, sizeof(app_message_t));
+    assert(app_queue);
+
+    BaseType_t task_created = xTaskCreate(usb_task, "usb_task", 4096, NULL, 2, NULL);
+    assert(task_created);
+
+    // Init BOOT button: Pressing the button simulates app request to exit
+    // It will disconnect the USB device and uninstall the MSC driver and USB Host Lib
+    const gpio_config_t input_pin = {
+        .pin_bit_mask = BIT64(APP_QUIT_PIN),
+        .mode = GPIO_MODE_INPUT,
+        .pull_up_en = GPIO_PULLUP_ENABLE,
+        .intr_type = GPIO_INTR_NEGEDGE,
+    };
+    ESP_ERROR_CHECK(gpio_config(&input_pin));
+    ESP_ERROR_CHECK(gpio_install_isr_service(ESP_INTR_FLAG_LEVEL1));
+    ESP_ERROR_CHECK(gpio_isr_handler_add(APP_QUIT_PIN, gpio_cb, NULL));
+
+    ESP_LOGI(TAG, "Waiting for USB flash drive to be connected");
+    msc_host_device_handle_t msc_device = NULL;
+    msc_host_vfs_handle_t vfs_handle = NULL;
+
+    // Perform all example operations in a loop to allow USB reconnections
+    while (1) {
+        app_message_t msg;
+        xQueueReceive(app_queue, &msg, portMAX_DELAY);
+
+        if (msg.id == APP_DEVICE_CONNECTED) {
+            // 1. MSC flash drive connected. Open it and map it to Virtual File System
+            ESP_ERROR_CHECK(msc_host_install_device(msg.data.new_dev_address, &msc_device));
             const esp_vfs_fat_mount_config_t mount_config = {
                 .format_if_mount_failed = false,
                 .max_files = 3,
-        .allocation_unit_size = 1024,
+                .allocation_unit_size = 8192,
             };
+            ESP_ERROR_CHECK(msc_host_vfs_register(msc_device, MNT_PATH, &mount_config, &vfs_handle));
 
-    do {
+            // 2. Print information about the connected disk
-        uint8_t device_address = wait_for_msc_device();
+            msc_host_device_info_t info;
-
-        ESP_ERROR_CHECK(msc_host_install_device(device_address, &msc_device));
-
-        msc_host_print_descriptors(msc_device);
-
             ESP_ERROR_CHECK(msc_host_get_device_info(msc_device, &info));
+            msc_host_print_descriptors(msc_device);
             print_device_info(&info);
 
-        ESP_ERROR_CHECK(msc_host_vfs_register(msc_device, "/usb", &mount_config, &vfs_handle));
+            // 3. List all the files in root directory
+            ESP_LOGI(TAG, "ls command output:");
+            struct dirent *d;
+            DIR *dh = opendir(MNT_PATH);
+            assert(dh);
+            while ((d = readdir(dh)) != NULL) {
+                printf("%s\n", d->d_name);
+            }
+            closedir(dh);
 
-        while (!wait_for_event(DEVICE_DISCONNECTED, 200)) {
+            // 4. The disk is mounted to Virtual File System, perform some basic demo file operation
             file_operations();
+
+            // 5. Perform speed test
+            speed_test();
+
+            ESP_LOGI(TAG, "Example finished, you can disconnect the USB flash drive");
+        }
+        if ((msg.id == APP_DEVICE_DISCONNECTED) || (msg.id == APP_QUIT)) {
+            if (vfs_handle) {
+                ESP_ERROR_CHECK(msc_host_vfs_unregister(vfs_handle));
+                vfs_handle = NULL;
+            }
+            if (msc_device) {
+                ESP_ERROR_CHECK(msc_host_uninstall_device(msc_device));
+                msc_device = NULL;
+            }
+            if (msg.id == APP_QUIT) {
+                // This will cause the usb_task to exit
+                ESP_ERROR_CHECK(msc_host_uninstall());
+                break;
+            }
+        }
     }
 
-        xEventGroupClearBits(usb_flags, READY_TO_UNINSTALL);
-        ESP_ERROR_CHECK(msc_host_vfs_unregister(vfs_handle));
-        ESP_ERROR_CHECK(msc_host_uninstall_device(msc_device));
-
-    } while (gpio_get_level(USB_DISCONNECT_PIN) != 0);
-
-    ESP_LOGI(TAG, "Uninitializing USB ...");
-    ESP_ERROR_CHECK(msc_host_uninstall());
-    wait_for_event(READY_TO_UNINSTALL, portMAX_DELAY);
-    ESP_ERROR_CHECK(usb_host_uninstall());
     ESP_LOGI(TAG, "Done");
+    gpio_isr_handler_remove(APP_QUIT_PIN);
+    vQueueDelete(app_queue);
 }

examples/peripherals/usb/host/msc/pytest_usb_host_msc.py

@@ -0,0 +1,27 @@
+
+# SPDX-FileCopyrightText: 2022-2023 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: CC0-1.0
+import pytest
+from pytest_embedded import Dut
+
+
+@pytest.mark.esp32s2
+@pytest.mark.esp32s3
+@pytest.mark.usb_host_flash_disk
+def test_usb_host_msc_example(dut: Dut) -> None:
+    # Check result of file_operations()
+    dut.expect_exact("example: Read from file '/usb/esp/test.txt': 'Hello World!'")
+
+    # Check result of speed_test()
+    write_throughput = float(dut.expect(r'example: Write speed ([0-9]*[.]?[0-9]+) MiB')[1].decode())
+    read_throughput = float(dut.expect(r'example: Read speed ([0-9]*[.]?[0-9]+) MiB')[1].decode())
+
+    # These values should be updated for HS targets
+    if write_throughput > 0.9:
+        print('Write throughput put OK')
+    else:
+        print('write throughput too slow!')
+    if read_throughput > 1.0:
+        print('Read throughput put OK')
+    else:
+        print('Read throughput too slow!')