Merge branch 'feature/support_dynamic_payload_len_for_ws_server' into 'master'

esp_http_server: support dynamic payload len for ws server

Closes IDFGH-4619

See merge request espressif/esp-idf!12235

components/esp_http_server/include/esp_http_server.h

@@ -1591,6 +1591,11 @@ typedef struct httpd_ws_frame {
 
 /**
  * @brief Receive and parse a WebSocket frame
+ *
+ * @note    Calling httpd_ws_recv_frame() with max_len as 0 will give actual frame size in pkt->len.
+ *          The user can dynamically allocate space for pkt->payload as per this length and call httpd_ws_recv_frame() again to get the actual data.
+ *          Please refer to the corresponding example for usage.
+ *
  * @param[in]   req         Current request
  * @param[out]  pkt         WebSocket packet
  * @param[in]   max_len     Maximum length for receive

components/esp_http_server/src/esp_httpd_priv.h

@@ -102,6 +102,7 @@ struct httpd_req_aux {
     bool ws_handshake_detect;                       /*!< WebSocket handshake detection flag */
     httpd_ws_type_t ws_type;                        /*!< WebSocket frame type */
     bool ws_final;                                  /*!< WebSocket FIN bit (final frame or not) */
+    uint8_t mask_key[4];                            /*!< WebSocket mask key for this payload */
 #endif
 };
 

components/esp_http_server/src/httpd_ws.c

@@ -253,45 +253,46 @@ esp_err_t httpd_ws_recv_frame(httpd_req_t *req, httpd_ws_frame_t *frame, size_t
         ESP_LOGW(TAG, LOG_FMT("Frame pointer is invalid"));
         return ESP_ERR_INVALID_ARG;
     }
+    /* If frame len is 0, will get frame len from req. Otherwise regard frame len already achieved by calling httpd_ws_recv_frame before */
+    if (frame->len == 0) {
+        /* Assign the frame info from the previous reading */
+        frame->type = aux->ws_type;
+        frame->final = aux->ws_final;
 
-    /* Assign the frame info from the previous reading */
+        /* Grab the second byte */
-    frame->type = aux->ws_type;
+        uint8_t second_byte = 0;
-    frame->final = aux->ws_final;
+        if (httpd_recv_with_opt(req, (char *)&second_byte, sizeof(second_byte), false) <= 0) {
-
+            ESP_LOGW(TAG, LOG_FMT("Failed to receive the second byte"));
-    /* Grab the second byte */
-    uint8_t second_byte = 0;
-    if (httpd_recv_with_opt(req, (char *)&second_byte, sizeof(second_byte), false) <= 0) {
-        ESP_LOGW(TAG, LOG_FMT("Failed to receive the second byte"));
-        return ESP_FAIL;
-    }
-
-    /* Parse the second byte */
-    /* Please refer to RFC6455 Section 5.2 for more details */
-    bool masked = (second_byte & HTTPD_WS_MASK_BIT) != 0;
-
-    /* Interpret length */
-    uint8_t init_len = second_byte & HTTPD_WS_LENGTH_BITS;
-    if (init_len < 126) {
-        /* Case 1: If length is 0-125, then this length bit is 7 bits */
-        frame->len = init_len;
-    } else if (init_len == 126) {
-        /* Case 2: If length byte is 126, then this frame's length bit is 16 bits */
-        uint8_t length_bytes[2] = { 0 };
-        if (httpd_recv_with_opt(req, (char *)length_bytes, sizeof(length_bytes), false) <= 0) {
-            ESP_LOGW(TAG, LOG_FMT("Failed to receive 2 bytes length"));
             return ESP_FAIL;
         }
 
-        frame->len = ((uint32_t)(length_bytes[0] << 8U) | (length_bytes[1]));
+        /* Parse the second byte */
-    } else if (init_len == 127) {
+        /* Please refer to RFC6455 Section 5.2 for more details */
-        /* Case 3: If length is byte 127, then this frame's length bit is 64 bits */
+        bool masked = (second_byte & HTTPD_WS_MASK_BIT) != 0;
-        uint8_t length_bytes[8] = { 0 };
-        if (httpd_recv_with_opt(req, (char *)length_bytes, sizeof(length_bytes), false) <= 0) {
-            ESP_LOGW(TAG, LOG_FMT("Failed to receive 2 bytes length"));
-            return ESP_FAIL;
-        }
 
-        frame->len = (((uint64_t)length_bytes[0] << 56U) |
+        /* Interpret length */
+        uint8_t init_len = second_byte & HTTPD_WS_LENGTH_BITS;
+        if (init_len < 126) {
+            /* Case 1: If length is 0-125, then this length bit is 7 bits */
+            frame->len = init_len;
+        } else if (init_len == 126) {
+            /* Case 2: If length byte is 126, then this frame's length bit is 16 bits */
+            uint8_t length_bytes[2] = { 0 };
+            if (httpd_recv_with_opt(req, (char *)length_bytes, sizeof(length_bytes), false) <= 0) {
+                ESP_LOGW(TAG, LOG_FMT("Failed to receive 2 bytes length"));
+                return ESP_FAIL;
+            }
+
+            frame->len = ((uint32_t)(length_bytes[0] << 8U) | (length_bytes[1]));
+        } else if (init_len == 127) {
+            /* Case 3: If length is byte 127, then this frame's length bit is 64 bits */
+            uint8_t length_bytes[8] = { 0 };
+            if (httpd_recv_with_opt(req, (char *)length_bytes, sizeof(length_bytes), false) <= 0) {
+                ESP_LOGW(TAG, LOG_FMT("Failed to receive 2 bytes length"));
+                return ESP_FAIL;
+            }
+
+            frame->len = (((uint64_t)length_bytes[0] << 56U) |
                     ((uint64_t)length_bytes[1] << 48U) |
                     ((uint64_t)length_bytes[2] << 40U) |
                     ((uint64_t)length_bytes[3] << 32U) |
@@ -299,28 +300,31 @@ esp_err_t httpd_ws_recv_frame(httpd_req_t *req, httpd_ws_frame_t *frame, size_t
                     ((uint64_t)length_bytes[5] << 16U) |
                     ((uint64_t)length_bytes[6] <<  8U) |
                     ((uint64_t)length_bytes[7]));
+        }
+        /* If this frame is masked, dump the mask as well */
+        if (masked) {
+            if (httpd_recv_with_opt(req, (char *)aux->mask_key, sizeof(aux->mask_key), false) <= 0) {
+                ESP_LOGW(TAG, LOG_FMT("Failed to receive mask key"));
+                return ESP_FAIL;
+            }
+        } else {
+            /* If the WS frame from client to server is not masked, it should be rejected.
+             * Please refer to RFC6455 Section 5.2 for more details. */
+            ESP_LOGW(TAG, LOG_FMT("WS frame is not properly masked."));
+            return ESP_ERR_INVALID_STATE;
+        }
     }
-
     /* We only accept the incoming packet length that is smaller than the max_len (or it will overflow the buffer!) */
+    /* If max_len is 0, regard it OK for userspace to get frame len */
     if (frame->len > max_len) {
+        if (max_len == 0) {
+            ESP_LOGD(TAG, "regard max_len == 0 is OK for user to get frame len");
+            return ESP_OK;
+        }
         ESP_LOGW(TAG, LOG_FMT("WS Message too long"));
         return ESP_ERR_INVALID_SIZE;
     }
 
-    /* If this frame is masked, dump the mask as well */
-    uint8_t mask_key[4] = { 0 };
-    if (masked) {
-        if (httpd_recv_with_opt(req, (char *)mask_key, sizeof(mask_key), false) <= 0) {
-            ESP_LOGW(TAG, LOG_FMT("Failed to receive mask key"));
-            return ESP_FAIL;
-        }
-    } else {
-        /* If the WS frame from client to server is not masked, it should be rejected.
-         * Please refer to RFC6455 Section 5.2 for more details. */
-        ESP_LOGW(TAG, LOG_FMT("WS frame is not properly masked."));
-        return ESP_ERR_INVALID_STATE;
-    }
-
     /* Receive buffer */
     /* If there's nothing to receive, return and stop here. */
     if (frame->len == 0) {
@@ -338,7 +342,7 @@ esp_err_t httpd_ws_recv_frame(httpd_req_t *req, httpd_ws_frame_t *frame, size_t
     }
 
     /* Unmask payload */
-    httpd_ws_unmask_payload(frame->payload, frame->len, mask_key);
+    httpd_ws_unmask_payload(frame->payload, frame->len, aux->mask_key);
 
     return ESP_OK;
 }

examples/protocols/http_server/ws_echo_server/README.md

@@ -16,6 +16,20 @@ Each outgoing frame has the FIN flag set by default.
 In case an application wants to send fragmented data, it must be done manually by setting the
 `fragmented` option and using the `final` flag as described in [RFC6455, section 5.4](https://tools.ietf.org/html/rfc6455#section-5.4).
 
+`httpd_ws_recv_frame` support two ways to get frame payload.
+* Static buffer -- Allocate maximum expected packet length (either statically or dynamically) and call `httpd_ws_recv_frame()` referencing this buffer and it's size. (Unnecessarily large buffers might cause memory waste)
+
+```
+#define MAX_PAYLOAD_LEN 128
+uint8_t buf[MAX_PAYLOAD_LEN] = { 0 };
+httpd_ws_frame_t ws_pkt;
+ws_pkt.payload = buf;
+httpd_ws_recv_frame(req, &ws_pkt, MAX_PAYLOAD_LEN);
+```
+* Dynamic buffer -- Refer to the examples, which receive websocket data in these three steps:
+   1) Call `httpd_ws_recv_frame()` with zero buffer size
+   2) Allocate the size based on the received packet length
+   3) Call `httpd_ws_recv_frame()` with the allocated buffer
 
 ### Hardware Required
 

examples/protocols/http_server/ws_echo_server/main/ws_echo_server.c

@@ -67,12 +67,25 @@ static esp_err_t trigger_async_send(httpd_handle_t handle, httpd_req_t *req)
  */
 static esp_err_t echo_handler(httpd_req_t *req)
 {
-    uint8_t buf[128] = { 0 };
     httpd_ws_frame_t ws_pkt;
     memset(&ws_pkt, 0, sizeof(httpd_ws_frame_t));
-    ws_pkt.payload = buf;
     ws_pkt.type = HTTPD_WS_TYPE_TEXT;
-    esp_err_t ret = httpd_ws_recv_frame(req, &ws_pkt, 128);
+    /* Set max_len = 0 to get the frame len */
+    esp_err_t ret = httpd_ws_recv_frame(req, &ws_pkt, 0);
+    if (ret != ESP_OK) {
+        ESP_LOGE(TAG, "httpd_ws_recv_frame failed to get frame len with %d", ret);
+        return ret;
+    }
+    ESP_LOGI(TAG, "frame len is %d", ws_pkt.len);
+    /* ws_pkt.len + 1 is for NULL termination as we are expecting a string */
+    uint8_t *buf = calloc(1, ws_pkt.len + 1);
+    if (buf == NULL) {
+        ESP_LOGE(TAG, "Failed to calloc memory for buf");
+        return ESP_ERR_NO_MEM;
+    }
+    ws_pkt.payload = buf;
+    /* Set max_len = ws_pkt.len to get the frame payload */
+    ret = httpd_ws_recv_frame(req, &ws_pkt, ws_pkt.len);
     if (ret != ESP_OK) {
         ESP_LOGE(TAG, "httpd_ws_recv_frame failed with %d", ret);
         return ret;
@@ -81,6 +94,8 @@ static esp_err_t echo_handler(httpd_req_t *req)
     ESP_LOGI(TAG, "Packet type: %d", ws_pkt.type);
     if (ws_pkt.type == HTTPD_WS_TYPE_TEXT &&
         strcmp((char*)ws_pkt.payload,"Trigger async") == 0) {
+        free(buf);
+        buf = NULL;
         return trigger_async_send(req->handle, req);
     }
 
@@ -88,6 +103,8 @@ static esp_err_t echo_handler(httpd_req_t *req)
     if (ret != ESP_OK) {
         ESP_LOGE(TAG, "httpd_ws_send_frame failed with %d", ret);
     }
+    free(buf);
+    buf = NULL;
     return ret;
 }
 

examples/protocols/https_server/wss_server/README.md

@@ -8,6 +8,21 @@ See the `esp_https_server` component documentation for details.
 
 Before using the example, open the project configuration menu (`idf.py menuconfig`) to configure Wi-Fi or Ethernet. See "Establishing Wi-Fi or Ethernet Connection" section in [examples/protocols/README.md](../../README.md) for more details.
 
+`httpd_ws_recv_frame` support two ways to get frame payload.
+* Static buffer -- Allocate maximum expected packet length (either statically or dynamically) and call `httpd_ws_recv_frame()` referencing this buffer and it's size. (Unnecessarily large buffers might cause memory waste)
+
+```
+#define MAX_PAYLOAD_LEN 128
+uint8_t buf[MAX_PAYLOAD_LEN] = { 0 };
+httpd_ws_frame_t ws_pkt;
+ws_pkt.payload = buf;
+httpd_ws_recv_frame(req, &ws_pkt, MAX_PAYLOAD_LEN);
+```
+* Dynamic buffer -- Refer to the examples, which receive websocket data in these three steps:
+   1) Call `httpd_ws_recv_frame()` with zero buffer size
+   2) Allocate the size based on the received packet length
+   3) Call `httpd_ws_recv_frame()` with the allocated buffer
+
 ## Certificates
 
 You will need to approve a security exception in your browser. This is because of a self signed

examples/protocols/https_server/wss_server/main/wss_server_example.c

@@ -33,13 +33,26 @@ static const size_t max_clients = 4;
 
 static esp_err_t ws_handler(httpd_req_t *req)
 {
-    uint8_t buf[128] = { 0 };
     httpd_ws_frame_t ws_pkt;
     memset(&ws_pkt, 0, sizeof(httpd_ws_frame_t));
-    ws_pkt.payload = buf;
 
     // First receive the full ws message
-    esp_err_t ret = httpd_ws_recv_frame(req, &ws_pkt, 128);
+    /* Set max_len = 0 to get the frame len */
+    esp_err_t ret = httpd_ws_recv_frame(req, &ws_pkt, 0);
+    if (ret != ESP_OK) {
+        ESP_LOGE(TAG, "httpd_ws_recv_frame failed to get frame len with %d", ret);
+        return ret;
+    }
+    ESP_LOGI(TAG, "frame len is %d", ws_pkt.len);
+    /* ws_pkt.len + 1 is for NULL termination as we are expecting a string */
+    uint8_t *buf = calloc(1, ws_pkt.len + 1);
+    if (buf == NULL) {
+        ESP_LOGE(TAG, "Failed to calloc memory for buf");
+        return ESP_ERR_NO_MEM;
+    }
+    ws_pkt.payload = buf;
+    /* Set max_len = ws_pkt.len to get the frame payload */
+    ret = httpd_ws_recv_frame(req, &ws_pkt, ws_pkt.len);
     if (ret != ESP_OK) {
         ESP_LOGE(TAG, "httpd_ws_recv_frame failed with %d", ret);
         return ret;
@@ -48,6 +61,8 @@ static esp_err_t ws_handler(httpd_req_t *req)
     // If it was a PONG, update the keep-alive
     if (ws_pkt.type == HTTPD_WS_TYPE_PONG) {
         ESP_LOGD(TAG, "Received PONG message");
+        free(buf);
+        buf = NULL;
         return wss_keep_alive_client_is_active(httpd_get_global_user_ctx(req->handle),
                 httpd_req_to_sockfd(req));
 
@@ -60,8 +75,12 @@ static esp_err_t ws_handler(httpd_req_t *req)
         }
         ESP_LOGI(TAG, "ws_handler: httpd_handle_t=%p, sockfd=%d, client_info:%d", req->handle,
                  httpd_req_to_sockfd(req), httpd_ws_get_fd_info(req->handle, httpd_req_to_sockfd(req)));
+        free(buf);
+        buf = NULL;
         return ret;
     }
+    free(buf);
+    buf = NULL;
     return ESP_OK;
 }