Merge branch 'bugfix/mqtt_improved_receive' into 'master'

MQTT improved receive and minor updates to weekend tests

See merge request idf/esp-idf!4947

components/lwip/test_afl_host/Makefile

@@ -1,7 +1,7 @@
 COMPONENTS_DIR=../..
 CFLAGS=-std=gnu99 -Og -ggdb -ffunction-sections -fdata-sections -nostdlib -Wall  -Werror=all -Wno-int-to-pointer-cast -Wno-error=unused-function -Wno-error=unused-variable -Wno-error=deprecated-declarations -Wextra \
 -Wno-unused-parameter -Wno-sign-compare -Wno-address   -Wno-unused-variable -DESP_PLATFORM -D IDF_VER=\"v3.1\" -MMD -MP -DWITH_POSIX
-INC_DIRS=-I . -I ./build/config -I $(COMPONENTS_DIR)/newlib/platform_include -I $(COMPONENTS_DIR)/newlib/include -I $(COMPONENTS_DIR)/driver/include -I $(COMPONENTS_DIR)/esp32/include -I $(COMPONENTS_DIR)/ethernet/include -I $(COMPONENTS_DIR)/freertos/include -I $(COMPONENTS_DIR)/heap/include -I $(COMPONENTS_DIR)/lwip/lwip/src/include  -I $(COMPONENTS_DIR)/lwip/include/apps -I $(COMPONENTS_DIR)/lwip/lwip/src/include/netif -I $(COMPONENTS_DIR)/lwip/lwip/src/include/posix -I $(COMPONENTS_DIR)/lwip/port/esp32/include -I $(COMPONENTS_DIR)/lwip/lwip/src/include/posix -I $(COMPONENTS_DIR)/lwip/include/apps/ping -I $(COMPONENTS_DIR)/soc/esp32/include -I $(COMPONENTS_DIR)/soc/include -I $(COMPONENTS_DIR)/tcpip_adapter/include -I $(COMPONENTS_DIR)/esp_rom/include  -I $(COMPONENTS_DIR)/esp_common/include -I $(COMPONENTS_DIR)/xtensa/include
+INC_DIRS=-I . -I ./build/config -I $(COMPONENTS_DIR)/newlib/platform_include -I $(COMPONENTS_DIR)/newlib/include -I $(COMPONENTS_DIR)/driver/include -I $(COMPONENTS_DIR)/esp32/include -I $(COMPONENTS_DIR)/ethernet/include -I $(COMPONENTS_DIR)/freertos/include -I $(COMPONENTS_DIR)/heap/include -I $(COMPONENTS_DIR)/lwip/lwip/src/include  -I $(COMPONENTS_DIR)/lwip/include/apps -I $(COMPONENTS_DIR)/lwip/lwip/src/include/netif -I $(COMPONENTS_DIR)/lwip/lwip/src/include/posix -I $(COMPONENTS_DIR)/lwip/port/esp32/include -I $(COMPONENTS_DIR)/lwip/lwip/src/include/posix -I $(COMPONENTS_DIR)/lwip/include/apps/ping -I $(COMPONENTS_DIR)/lwip/include/apps/sntp  -I $(COMPONENTS_DIR)/soc/esp32/include -I $(COMPONENTS_DIR)/soc/include -I $(COMPONENTS_DIR)/tcpip_adapter/include -I $(COMPONENTS_DIR)/esp_rom/include  -I $(COMPONENTS_DIR)/esp_common/include -I $(COMPONENTS_DIR)/xtensa/include -I $(COMPONENTS_DIR)/xtensa/esp32/include -I $(COMPONENTS_DIR)/esp_wifi/include -I $(COMPONENTS_DIR)/esp_event/include
 TEST_NAME=test
 FUZZ=afl-fuzz
 GEN_CFG=generate_config

components/mdns/test_afl_fuzz_host/esp32_compat.h

@@ -20,7 +20,6 @@
 // Not to include
 #define ESP_MDNS_NETWORKING_H_
 #define _TCPIP_ADAPTER_H_
-#define __ESP_EVENT_H__
 
 
 #ifdef USE_BSD_STRING
@@ -116,6 +115,47 @@ typedef enum {
     WIFI_MODE_MAX
 } wifi_mode_t;
 
+extern const char * WIFI_EVENT;
+extern const char * IP_EVENT;
+extern const char * ETH_EVENT;
+
+typedef enum {
+    WIFI_EVENT_WIFI_READY = 0,           /**< ESP32 WiFi ready */
+    WIFI_EVENT_SCAN_DONE,                /**< ESP32 finish scanning AP */
+    WIFI_EVENT_STA_START,                /**< ESP32 station start */
+    WIFI_EVENT_STA_STOP,                 /**< ESP32 station stop */
+    WIFI_EVENT_STA_CONNECTED,            /**< ESP32 station connected to AP */
+    WIFI_EVENT_STA_DISCONNECTED,         /**< ESP32 station disconnected from AP */
+    WIFI_EVENT_STA_AUTHMODE_CHANGE,      /**< the auth mode of AP connected by ESP32 station changed */
+
+    WIFI_EVENT_STA_WPS_ER_SUCCESS,       /**< ESP32 station wps succeeds in enrollee mode */
+    WIFI_EVENT_STA_WPS_ER_FAILED,        /**< ESP32 station wps fails in enrollee mode */
+    WIFI_EVENT_STA_WPS_ER_TIMEOUT,       /**< ESP32 station wps timeout in enrollee mode */
+    WIFI_EVENT_STA_WPS_ER_PIN,           /**< ESP32 station wps pin code in enrollee mode */
+
+    WIFI_EVENT_AP_START,                 /**< ESP32 soft-AP start */
+    WIFI_EVENT_AP_STOP,                  /**< ESP32 soft-AP stop */
+    WIFI_EVENT_AP_STACONNECTED,          /**< a station connected to ESP32 soft-AP */
+    WIFI_EVENT_AP_STADISCONNECTED,       /**< a station disconnected from ESP32 soft-AP */
+
+    WIFI_EVENT_AP_PROBEREQRECVED,        /**< Receive probe request packet in soft-AP interface */
+} wifi_event_t;
+
+typedef enum {
+    ETHERNET_EVENT_START,                /**< ESP32 ethernet start */
+    ETHERNET_EVENT_STOP,                 /**< ESP32 ethernet stop */
+    ETHERNET_EVENT_CONNECTED,            /**< ESP32 ethernet phy link up */
+    ETHERNET_EVENT_DISCONNECTED,         /**< ESP32 ethernet phy link down */
+} eth_event_t;
+
+typedef enum {
+    IP_EVENT_STA_GOT_IP,               /*!< ESP32 station got IP from connected AP */
+    IP_EVENT_STA_LOST_IP,              /*!< ESP32 station lost IP and the IP is reset to 0 */
+    IP_EVENT_AP_STAIPASSIGNED,         /*!< ESP32 soft-AP assign an IP to a connected station */
+    IP_EVENT_GOT_IP6,                  /*!< ESP32 station or ap or ethernet interface v6IP addr is preferred */
+    IP_EVENT_ETH_GOT_IP,               /*!< ESP32 ethernet got IP from connected AP */
+} ip_event_t;
+
 /* status of DHCP client or DHCP server */
 typedef enum {
     TCPIP_ADAPTER_DHCP_INIT = 0,    /**< DHCP client/server in initial state */
@@ -124,34 +164,6 @@ typedef enum {
     TCPIP_ADAPTER_DHCP_STATUS_MAX
 } tcpip_adapter_dhcp_status_t;
 
-typedef enum {
-    SYSTEM_EVENT_WIFI_READY = 0,           /**< ESP32 WiFi ready */
-    SYSTEM_EVENT_SCAN_DONE,                /**< ESP32 finish scanning AP */
-    SYSTEM_EVENT_STA_START,                /**< ESP32 station start */
-    SYSTEM_EVENT_STA_STOP,                 /**< ESP32 station stop */
-    SYSTEM_EVENT_STA_CONNECTED,            /**< ESP32 station connected to AP */
-    SYSTEM_EVENT_STA_DISCONNECTED,         /**< ESP32 station disconnected from AP */
-    SYSTEM_EVENT_STA_AUTHMODE_CHANGE,      /**< the auth mode of AP connected by ESP32 station changed */
-    SYSTEM_EVENT_STA_GOT_IP,               /**< ESP32 station got IP from connected AP */
-    SYSTEM_EVENT_STA_LOST_IP,              /**< ESP32 station lost IP and the IP is reset to 0 */
-    SYSTEM_EVENT_STA_WPS_ER_SUCCESS,       /**< ESP32 station wps succeeds in enrollee mode */
-    SYSTEM_EVENT_STA_WPS_ER_FAILED,        /**< ESP32 station wps fails in enrollee mode */
-    SYSTEM_EVENT_STA_WPS_ER_TIMEOUT,       /**< ESP32 station wps timeout in enrollee mode */
-    SYSTEM_EVENT_STA_WPS_ER_PIN,           /**< ESP32 station wps pin code in enrollee mode */
-    SYSTEM_EVENT_AP_START,                 /**< ESP32 soft-AP start */
-    SYSTEM_EVENT_AP_STOP,                  /**< ESP32 soft-AP stop */
-    SYSTEM_EVENT_AP_STACONNECTED,          /**< a station connected to ESP32 soft-AP */
-    SYSTEM_EVENT_AP_STADISCONNECTED,       /**< a station disconnected from ESP32 soft-AP */
-    SYSTEM_EVENT_AP_PROBEREQRECVED,        /**< Receive probe request packet in soft-AP interface */
-    SYSTEM_EVENT_GOT_IP6,                  /**< ESP32 station or ap or ethernet interface v6IP addr is preferred */
-    SYSTEM_EVENT_ETH_START,                /**< ESP32 ethernet start */
-    SYSTEM_EVENT_ETH_STOP,                 /**< ESP32 ethernet stop */
-    SYSTEM_EVENT_ETH_CONNECTED,            /**< ESP32 ethernet phy link up */
-    SYSTEM_EVENT_ETH_DISCONNECTED,         /**< ESP32 ethernet phy link down */
-    SYSTEM_EVENT_ETH_GOT_IP,               /**< ESP32 ethernet got IP from connected AP */
-    SYSTEM_EVENT_MAX
-} system_event_id_t;
-
 struct udp_pcb {
     uint8_t dummy;
 };
@@ -180,7 +192,6 @@ typedef struct {
     ip4_addr_t gw;
 } tcpip_adapter_ip_info_t;
 
-// typedef int32_t system_event_id_t;
 typedef enum {
     TCPIP_ADAPTER_IF_STA = 0,     /**< ESP32 station interface */
     TCPIP_ADAPTER_IF_AP,          /**< ESP32 soft-AP interface */
@@ -191,18 +202,13 @@ typedef enum {
 typedef struct {
     ip6_addr_t ip;
 } tcpip_adapter_ip6_info_t;
-typedef struct {
-    tcpip_adapter_if_t if_index;
-    tcpip_adapter_ip6_info_t ip6_info;
-} system_event_got_ip6_t;
-typedef union {
-    system_event_got_ip6_t                     got_ip6;            /**< ESP32 station　or ap or ethernet ipv6 addr state change to preferred */
-} system_event_info_t;
 
 typedef struct {
-    system_event_id_t     event_id;      /**< event ID */
+    tcpip_adapter_if_t if_index;        /*!< Interface for which the event is received */
-    system_event_info_t   event_info;    /**< event information */
+    tcpip_adapter_ip6_info_t ip6_info;  /*!< IPv6 address of the interface */
-} system_event_t;
+} ip_event_got_ip6_t;
+
+typedef void* system_event_t;
 
 inline esp_err_t esp_wifi_get_mode(wifi_mode_t * mode)
 {

components/mdns/test_afl_fuzz_host/esp32_mock.c

@@ -9,6 +9,23 @@ void*     g_queue;
 int       g_queue_send_shall_fail = 0;
 int       g_size = 0;
 
+const char * WIFI_EVENT = "wifi_event";
+const char * IP_EVENT = "ip_event";
+const char * ETH_EVENT = "eth_event";
+
+esp_err_t esp_event_handler_register(const char * event_base,
+                                        int32_t event_id,
+                                        void* event_handler,
+                                        void* event_handler_arg)
+{
+    return ESP_OK;
+}
+
+esp_err_t esp_event_handler_unregister(const char * event_base, int32_t event_id, void* event_handler)
+{
+    return ESP_OK;
+}
+
 esp_err_t esp_timer_delete(esp_timer_handle_t timer)
 {
     return ESP_OK;

components/mdns/test_afl_fuzz_host/esp32_mock.h

@@ -40,6 +40,10 @@ void GetLastItem(void *pvBuffer);
 
 void ForceTaskDelete();
 
+esp_err_t esp_event_handler_register(const char * event_base, int32_t event_id, void* event_handler, void* event_handler_arg);
+
+esp_err_t esp_event_handler_unregister(const char * event_base, int32_t event_id, void* event_handler);
+
 #define _mdns_udp_pcb_write(tcpip_if, ip_protocol, ip, port, data, len) len
 
 #endif /* ESP32_MOCK_H_ */

components/mqtt/weekend_test/mqtt_publish_test.py

@@ -71,11 +71,11 @@ def test_single_config(dut, transport, qos, repeat, published):
     global expected_data
     global message_log
     sample_string = ''.join(random.choice(string.ascii_uppercase + string.ascii_lowercase + string.digits) for _ in range(16))
-    print("PUBLISH TEST: transport:{}, qos:{}, sequence:{}, sample msg:{}".format(transport, qos, published, sample_string))
     event_client_connected.clear()
     expected_count = 0
     message_log = ""
     expected_data = sample_string * repeat
+    print("PUBLISH TEST: transport:{}, qos:{}, sequence:{}, sample msg:'{}'".format(transport, qos, published, expected_data))
     client = None
     try:
         if transport in ["ws", "wss"]:
@@ -148,12 +148,12 @@ def test_weekend_mqtt_publish(env, extra_data):
     # Look for host:port in sdkconfig
     try:
         # python client subscribes to the topic to which esp client publishes and vice versa
-        publish_topic = dut1.app.get_sdkconfig()["CONFIG_SUBSCIBE_TOPIC"].replace('"','')
+        publish_topic = dut1.app.get_sdkconfig()["CONFIG_EXAMPLE_SUBSCIBE_TOPIC"].replace('"','')
-        subscribe_topic = dut1.app.get_sdkconfig()["CONFIG_PUBLISH_TOPIC"].replace('"','')
+        subscribe_topic = dut1.app.get_sdkconfig()["CONFIG_EXAMPLE_PUBLISH_TOPIC"].replace('"','')
-        broker_host["ssl"], broker_port["ssl"] = get_host_port_from_dut(dut1, "CONFIG_BROKER_SSL_URI")
+        broker_host["ssl"], broker_port["ssl"] = get_host_port_from_dut(dut1, "CONFIG_EXAMPLE_BROKER_SSL_URI")
-        broker_host["tcp"], broker_port["tcp"] = get_host_port_from_dut(dut1, "CONFIG_BROKER_TCP_URI")
+        broker_host["tcp"], broker_port["tcp"] = get_host_port_from_dut(dut1, "CONFIG_EXAMPLE_BROKER_TCP_URI")
-        broker_host["ws"], broker_port["ws"] = get_host_port_from_dut(dut1, "CONFIG_BROKER_WS_URI")
+        broker_host["ws"], broker_port["ws"] = get_host_port_from_dut(dut1, "CONFIG_EXAMPLE_BROKER_WS_URI")
-        broker_host["wss"], broker_port["wss"] = get_host_port_from_dut(dut1, "CONFIG_BROKER_WSS_URI")
+        broker_host["wss"], broker_port["wss"] = get_host_port_from_dut(dut1, "CONFIG_EXAMPLE_BROKER_WSS_URI")
     except Exception:
         print('ENV_TEST_FAILURE: Cannot find broker url in sdkconfig')
         raise
@@ -166,8 +166,10 @@ def test_weekend_mqtt_publish(env, extra_data):
         raise
     for qos in [0, 1, 2]:
         for transport in ["tcp", "ssl", "ws", "wss"]:
+            # simple test with empty message
+            test_single_config(dut1, transport, qos, 0, 5)
             # decide on broker what level of test will pass (local broker works the best)
-            if broker_host[transport].startswith("192.168"):
+            if broker_host[transport].startswith("192.168") and qos < 1:
                 # medium size, medium repeated
                 test_single_config(dut1, transport, qos, 5, 50)
                 # long data

components/tcp_transport/transport_ws.c

@@ -192,8 +192,8 @@ static int ws_read(esp_transport_handle_t t, char *buffer, int len, int timeout_
 {
     transport_ws_t *ws = esp_transport_get_context_data(t);
     int payload_len;
-    int payload_len_buff = len;
+    char ws_header[MAX_WEBSOCKET_HEADER_SIZE];
-    char *data_ptr = buffer, opcode, mask, *mask_key = NULL;
+    char *data_ptr = ws_header, opcode, mask, *mask_key = NULL;
     int rlen;
     int poll_read;
     if ((poll_read = esp_transport_poll_read(ws->parent, timeout_ms)) <= 0) {
@@ -202,7 +202,7 @@ static int ws_read(esp_transport_handle_t t, char *buffer, int len, int timeout_
 
     // Receive and process header first (based on header size)
     int header = 2;
-    if ((rlen = esp_transport_read(ws->parent, buffer, header, timeout_ms)) <= 0) {
+    if ((rlen = esp_transport_read(ws->parent, data_ptr, header, timeout_ms)) <= 0) {
         ESP_LOGE(TAG, "Error read data");
         return rlen;
     }
@@ -219,8 +219,6 @@ static int ws_read(esp_transport_handle_t t, char *buffer, int len, int timeout_
             return rlen;
         }
         payload_len = data_ptr[0] << 8 | data_ptr[1];
-        payload_len_buff = len - 4;
-        data_ptr += 2;
     } else if (payload_len == 127) {
         // headerLen += 8;
         header = 8;
@@ -235,29 +233,25 @@ static int ws_read(esp_transport_handle_t t, char *buffer, int len, int timeout_
         } else {
             payload_len = data_ptr[4] << 24 | data_ptr[5] << 16 | data_ptr[6] << 8 | data_ptr[7];
         }
-        data_ptr += 8;
-        payload_len_buff = len - 10;
     }
 
-    if (payload_len > payload_len_buff) {
+    if (payload_len > len) {
-        ESP_LOGD(TAG, "Actual data to receive (%d) are longer than ws buffer (%d)", payload_len, payload_len_buff);
+        ESP_LOGD(TAG, "Actual data to receive (%d) are longer than ws buffer (%d)", payload_len, len);
-        payload_len = payload_len_buff;
+        payload_len = len;
     }
 
     // Then receive and process payload
-    if ((rlen = esp_transport_read(ws->parent, data_ptr, payload_len, timeout_ms)) <= 0) {
+    if ((rlen = esp_transport_read(ws->parent, buffer, payload_len, timeout_ms)) <= 0) {
         ESP_LOGE(TAG, "Error read data");
         return rlen;
     }
 
     if (mask) {
-        mask_key = data_ptr;
+        mask_key = buffer;
-        data_ptr += 4;
+        data_ptr = buffer + 4;
         for (int i = 0; i < payload_len; i++) {
             buffer[i] = (data_ptr[i] ^ mask_key[i % 4]);
         }
-    } else {
-        memmove(buffer, data_ptr, payload_len);
     }
     return payload_len;
 }