From db121a56e13b4b93b6bcc9217f7890bc4783cabd Mon Sep 17 00:00:00 2001 From: David Cermak Date: Tue, 30 Apr 2019 11:41:39 +0200 Subject: [PATCH 1/4] tcp_transport: modified ws_read to read payload directly to the read buffer and separately from header bytes Previous version read all data to the buffer including header which reduced maximum payload read. This version uses a local array to receive header and reads payload bytes to the buffer --- components/mqtt/esp-mqtt | 2 +- components/tcp_transport/transport_ws.c | 24 +++++++++--------------- 2 files changed, 10 insertions(+), 16 deletions(-) diff --git a/components/mqtt/esp-mqtt b/components/mqtt/esp-mqtt index 39118d5182..18b6f2c582 160000 --- a/components/mqtt/esp-mqtt +++ b/components/mqtt/esp-mqtt @@ -1 +1 @@ -Subproject commit 39118d5182f804edc01ed030c8ba137a9c7f388e +Subproject commit 18b6f2c58231728805ff813d46021d07ed023dcb diff --git a/components/tcp_transport/transport_ws.c b/components/tcp_transport/transport_ws.c index a760ecfec0..e31149e178 100644 --- a/components/tcp_transport/transport_ws.c +++ b/components/tcp_transport/transport_ws.c @@ -192,8 +192,8 @@ static int ws_read(esp_transport_handle_t t, char *buffer, int len, int timeout_ { transport_ws_t *ws = esp_transport_get_context_data(t); int payload_len; - int payload_len_buff = len; - char *data_ptr = buffer, opcode, mask, *mask_key = NULL; + char ws_header[MAX_WEBSOCKET_HEADER_SIZE]; + char *data_ptr = ws_header, opcode, mask, *mask_key = NULL; int rlen; int poll_read; if ((poll_read = esp_transport_poll_read(ws->parent, timeout_ms)) <= 0) { @@ -202,7 +202,7 @@ static int ws_read(esp_transport_handle_t t, char *buffer, int len, int timeout_ // Receive and process header first (based on header size) int header = 2; - if ((rlen = esp_transport_read(ws->parent, buffer, header, timeout_ms)) <= 0) { + if ((rlen = esp_transport_read(ws->parent, data_ptr, header, timeout_ms)) <= 0) { ESP_LOGE(TAG, "Error read data"); return rlen; } @@ -219,8 +219,6 @@ static int ws_read(esp_transport_handle_t t, char *buffer, int len, int timeout_ return rlen; } payload_len = data_ptr[0] << 8 | data_ptr[1]; - payload_len_buff = len - 4; - data_ptr += 2; } else if (payload_len == 127) { // headerLen += 8; header = 8; @@ -235,29 +233,25 @@ static int ws_read(esp_transport_handle_t t, char *buffer, int len, int timeout_ } else { payload_len = data_ptr[4] << 24 | data_ptr[5] << 16 | data_ptr[6] << 8 | data_ptr[7]; } - data_ptr += 8; - payload_len_buff = len - 10; } - if (payload_len > payload_len_buff) { - ESP_LOGD(TAG, "Actual data to receive (%d) are longer than ws buffer (%d)", payload_len, payload_len_buff); - payload_len = payload_len_buff; + if (payload_len > len) { + ESP_LOGD(TAG, "Actual data to receive (%d) are longer than ws buffer (%d)", payload_len, len); + payload_len = len; } // Then receive and process payload - if ((rlen = esp_transport_read(ws->parent, data_ptr, payload_len, timeout_ms)) <= 0) { + if ((rlen = esp_transport_read(ws->parent, buffer, payload_len, timeout_ms)) <= 0) { ESP_LOGE(TAG, "Error read data"); return rlen; } if (mask) { - mask_key = data_ptr; - data_ptr += 4; + mask_key = buffer; + data_ptr = buffer + 4; for (int i = 0; i < payload_len; i++) { buffer[i] = (data_ptr[i] ^ mask_key[i % 4]); } - } else { - memmove(buffer, data_ptr, payload_len); } return payload_len; } From d19b20388fbcecc5b5faa76b677f3bb467108a91 Mon Sep 17 00:00:00 2001 From: David Cermak Date: Tue, 30 Apr 2019 13:52:01 +0200 Subject: [PATCH 2/4] lwip tests: adapt fuzzer test mocks to comply with 4.0 refactoring --- components/lwip/test_afl_host/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/lwip/test_afl_host/Makefile b/components/lwip/test_afl_host/Makefile index d31858417d..7c5cf49f3a 100644 --- a/components/lwip/test_afl_host/Makefile +++ b/components/lwip/test_afl_host/Makefile @@ -1,7 +1,7 @@ COMPONENTS_DIR=../.. CFLAGS=-std=gnu99 -Og -ggdb -ffunction-sections -fdata-sections -nostdlib -Wall -Werror=all -Wno-int-to-pointer-cast -Wno-error=unused-function -Wno-error=unused-variable -Wno-error=deprecated-declarations -Wextra \ -Wno-unused-parameter -Wno-sign-compare -Wno-address -Wno-unused-variable -DESP_PLATFORM -D IDF_VER=\"v3.1\" -MMD -MP -DWITH_POSIX -INC_DIRS=-I . -I ./build/config -I $(COMPONENTS_DIR)/newlib/platform_include -I $(COMPONENTS_DIR)/newlib/include -I $(COMPONENTS_DIR)/driver/include -I $(COMPONENTS_DIR)/esp32/include -I $(COMPONENTS_DIR)/ethernet/include -I $(COMPONENTS_DIR)/freertos/include -I $(COMPONENTS_DIR)/heap/include -I $(COMPONENTS_DIR)/lwip/lwip/src/include -I $(COMPONENTS_DIR)/lwip/include/apps -I $(COMPONENTS_DIR)/lwip/lwip/src/include/netif -I $(COMPONENTS_DIR)/lwip/lwip/src/include/posix -I $(COMPONENTS_DIR)/lwip/port/esp32/include -I $(COMPONENTS_DIR)/lwip/lwip/src/include/posix -I $(COMPONENTS_DIR)/lwip/include/apps/ping -I $(COMPONENTS_DIR)/soc/esp32/include -I $(COMPONENTS_DIR)/soc/include -I $(COMPONENTS_DIR)/tcpip_adapter/include -I $(COMPONENTS_DIR)/esp_rom/include -I $(COMPONENTS_DIR)/esp_common/include -I $(COMPONENTS_DIR)/xtensa/include +INC_DIRS=-I . -I ./build/config -I $(COMPONENTS_DIR)/newlib/platform_include -I $(COMPONENTS_DIR)/newlib/include -I $(COMPONENTS_DIR)/driver/include -I $(COMPONENTS_DIR)/esp32/include -I $(COMPONENTS_DIR)/ethernet/include -I $(COMPONENTS_DIR)/freertos/include -I $(COMPONENTS_DIR)/heap/include -I $(COMPONENTS_DIR)/lwip/lwip/src/include -I $(COMPONENTS_DIR)/lwip/include/apps -I $(COMPONENTS_DIR)/lwip/lwip/src/include/netif -I $(COMPONENTS_DIR)/lwip/lwip/src/include/posix -I $(COMPONENTS_DIR)/lwip/port/esp32/include -I $(COMPONENTS_DIR)/lwip/lwip/src/include/posix -I $(COMPONENTS_DIR)/lwip/include/apps/ping -I $(COMPONENTS_DIR)/lwip/include/apps/sntp -I $(COMPONENTS_DIR)/soc/esp32/include -I $(COMPONENTS_DIR)/soc/include -I $(COMPONENTS_DIR)/tcpip_adapter/include -I $(COMPONENTS_DIR)/esp_rom/include -I $(COMPONENTS_DIR)/esp_common/include -I $(COMPONENTS_DIR)/xtensa/include -I $(COMPONENTS_DIR)/xtensa/esp32/include -I $(COMPONENTS_DIR)/esp_wifi/include -I $(COMPONENTS_DIR)/esp_event/include TEST_NAME=test FUZZ=afl-fuzz GEN_CFG=generate_config From 38d15cbd637e8b8baacda9fc43e8e99d224530f5 Mon Sep 17 00:00:00 2001 From: David Cermak Date: Tue, 30 Apr 2019 13:54:08 +0200 Subject: [PATCH 3/4] mdns tests: adapt mdns fuzzer test to compile with event loop library --- .../mdns/test_afl_fuzz_host/esp32_compat.h | 86 ++++++++++--------- .../mdns/test_afl_fuzz_host/esp32_mock.c | 17 ++++ .../mdns/test_afl_fuzz_host/esp32_mock.h | 4 + 3 files changed, 67 insertions(+), 40 deletions(-) diff --git a/components/mdns/test_afl_fuzz_host/esp32_compat.h b/components/mdns/test_afl_fuzz_host/esp32_compat.h index e054c18bbd..acdafa31a8 100644 --- a/components/mdns/test_afl_fuzz_host/esp32_compat.h +++ b/components/mdns/test_afl_fuzz_host/esp32_compat.h @@ -20,7 +20,6 @@ // Not to include #define ESP_MDNS_NETWORKING_H_ #define _TCPIP_ADAPTER_H_ -#define __ESP_EVENT_H__ #ifdef USE_BSD_STRING @@ -116,6 +115,47 @@ typedef enum { WIFI_MODE_MAX } wifi_mode_t; +extern const char * WIFI_EVENT; +extern const char * IP_EVENT; +extern const char * ETH_EVENT; + +typedef enum { + WIFI_EVENT_WIFI_READY = 0, /**< ESP32 WiFi ready */ + WIFI_EVENT_SCAN_DONE, /**< ESP32 finish scanning AP */ + WIFI_EVENT_STA_START, /**< ESP32 station start */ + WIFI_EVENT_STA_STOP, /**< ESP32 station stop */ + WIFI_EVENT_STA_CONNECTED, /**< ESP32 station connected to AP */ + WIFI_EVENT_STA_DISCONNECTED, /**< ESP32 station disconnected from AP */ + WIFI_EVENT_STA_AUTHMODE_CHANGE, /**< the auth mode of AP connected by ESP32 station changed */ + + WIFI_EVENT_STA_WPS_ER_SUCCESS, /**< ESP32 station wps succeeds in enrollee mode */ + WIFI_EVENT_STA_WPS_ER_FAILED, /**< ESP32 station wps fails in enrollee mode */ + WIFI_EVENT_STA_WPS_ER_TIMEOUT, /**< ESP32 station wps timeout in enrollee mode */ + WIFI_EVENT_STA_WPS_ER_PIN, /**< ESP32 station wps pin code in enrollee mode */ + + WIFI_EVENT_AP_START, /**< ESP32 soft-AP start */ + WIFI_EVENT_AP_STOP, /**< ESP32 soft-AP stop */ + WIFI_EVENT_AP_STACONNECTED, /**< a station connected to ESP32 soft-AP */ + WIFI_EVENT_AP_STADISCONNECTED, /**< a station disconnected from ESP32 soft-AP */ + + WIFI_EVENT_AP_PROBEREQRECVED, /**< Receive probe request packet in soft-AP interface */ +} wifi_event_t; + +typedef enum { + ETHERNET_EVENT_START, /**< ESP32 ethernet start */ + ETHERNET_EVENT_STOP, /**< ESP32 ethernet stop */ + ETHERNET_EVENT_CONNECTED, /**< ESP32 ethernet phy link up */ + ETHERNET_EVENT_DISCONNECTED, /**< ESP32 ethernet phy link down */ +} eth_event_t; + +typedef enum { + IP_EVENT_STA_GOT_IP, /*!< ESP32 station got IP from connected AP */ + IP_EVENT_STA_LOST_IP, /*!< ESP32 station lost IP and the IP is reset to 0 */ + IP_EVENT_AP_STAIPASSIGNED, /*!< ESP32 soft-AP assign an IP to a connected station */ + IP_EVENT_GOT_IP6, /*!< ESP32 station or ap or ethernet interface v6IP addr is preferred */ + IP_EVENT_ETH_GOT_IP, /*!< ESP32 ethernet got IP from connected AP */ +} ip_event_t; + /* status of DHCP client or DHCP server */ typedef enum { TCPIP_ADAPTER_DHCP_INIT = 0, /**< DHCP client/server in initial state */ @@ -124,34 +164,6 @@ typedef enum { TCPIP_ADAPTER_DHCP_STATUS_MAX } tcpip_adapter_dhcp_status_t; -typedef enum { - SYSTEM_EVENT_WIFI_READY = 0, /**< ESP32 WiFi ready */ - SYSTEM_EVENT_SCAN_DONE, /**< ESP32 finish scanning AP */ - SYSTEM_EVENT_STA_START, /**< ESP32 station start */ - SYSTEM_EVENT_STA_STOP, /**< ESP32 station stop */ - SYSTEM_EVENT_STA_CONNECTED, /**< ESP32 station connected to AP */ - SYSTEM_EVENT_STA_DISCONNECTED, /**< ESP32 station disconnected from AP */ - SYSTEM_EVENT_STA_AUTHMODE_CHANGE, /**< the auth mode of AP connected by ESP32 station changed */ - SYSTEM_EVENT_STA_GOT_IP, /**< ESP32 station got IP from connected AP */ - SYSTEM_EVENT_STA_LOST_IP, /**< ESP32 station lost IP and the IP is reset to 0 */ - SYSTEM_EVENT_STA_WPS_ER_SUCCESS, /**< ESP32 station wps succeeds in enrollee mode */ - SYSTEM_EVENT_STA_WPS_ER_FAILED, /**< ESP32 station wps fails in enrollee mode */ - SYSTEM_EVENT_STA_WPS_ER_TIMEOUT, /**< ESP32 station wps timeout in enrollee mode */ - SYSTEM_EVENT_STA_WPS_ER_PIN, /**< ESP32 station wps pin code in enrollee mode */ - SYSTEM_EVENT_AP_START, /**< ESP32 soft-AP start */ - SYSTEM_EVENT_AP_STOP, /**< ESP32 soft-AP stop */ - SYSTEM_EVENT_AP_STACONNECTED, /**< a station connected to ESP32 soft-AP */ - SYSTEM_EVENT_AP_STADISCONNECTED, /**< a station disconnected from ESP32 soft-AP */ - SYSTEM_EVENT_AP_PROBEREQRECVED, /**< Receive probe request packet in soft-AP interface */ - SYSTEM_EVENT_GOT_IP6, /**< ESP32 station or ap or ethernet interface v6IP addr is preferred */ - SYSTEM_EVENT_ETH_START, /**< ESP32 ethernet start */ - SYSTEM_EVENT_ETH_STOP, /**< ESP32 ethernet stop */ - SYSTEM_EVENT_ETH_CONNECTED, /**< ESP32 ethernet phy link up */ - SYSTEM_EVENT_ETH_DISCONNECTED, /**< ESP32 ethernet phy link down */ - SYSTEM_EVENT_ETH_GOT_IP, /**< ESP32 ethernet got IP from connected AP */ - SYSTEM_EVENT_MAX -} system_event_id_t; - struct udp_pcb { uint8_t dummy; }; @@ -180,7 +192,6 @@ typedef struct { ip4_addr_t gw; } tcpip_adapter_ip_info_t; -// typedef int32_t system_event_id_t; typedef enum { TCPIP_ADAPTER_IF_STA = 0, /**< ESP32 station interface */ TCPIP_ADAPTER_IF_AP, /**< ESP32 soft-AP interface */ @@ -191,18 +202,13 @@ typedef enum { typedef struct { ip6_addr_t ip; } tcpip_adapter_ip6_info_t; -typedef struct { - tcpip_adapter_if_t if_index; - tcpip_adapter_ip6_info_t ip6_info; -} system_event_got_ip6_t; -typedef union { - system_event_got_ip6_t got_ip6; /**< ESP32 station　or ap or ethernet ipv6 addr state change to preferred */ -} system_event_info_t; typedef struct { - system_event_id_t event_id; /**< event ID */ - system_event_info_t event_info; /**< event information */ -} system_event_t; + tcpip_adapter_if_t if_index; /*!< Interface for which the event is received */ + tcpip_adapter_ip6_info_t ip6_info; /*!< IPv6 address of the interface */ +} ip_event_got_ip6_t; + +typedef void* system_event_t; inline esp_err_t esp_wifi_get_mode(wifi_mode_t * mode) { diff --git a/components/mdns/test_afl_fuzz_host/esp32_mock.c b/components/mdns/test_afl_fuzz_host/esp32_mock.c index 8cc986dfd6..2607b40cc3 100644 --- a/components/mdns/test_afl_fuzz_host/esp32_mock.c +++ b/components/mdns/test_afl_fuzz_host/esp32_mock.c @@ -9,6 +9,23 @@ void* g_queue; int g_queue_send_shall_fail = 0; int g_size = 0; +const char * WIFI_EVENT = "wifi_event"; +const char * IP_EVENT = "ip_event"; +const char * ETH_EVENT = "eth_event"; + +esp_err_t esp_event_handler_register(const char * event_base, + int32_t event_id, + void* event_handler, + void* event_handler_arg) +{ + return ESP_OK; +} + +esp_err_t esp_event_handler_unregister(const char * event_base, int32_t event_id, void* event_handler) +{ + return ESP_OK; +} + esp_err_t esp_timer_delete(esp_timer_handle_t timer) { return ESP_OK; diff --git a/components/mdns/test_afl_fuzz_host/esp32_mock.h b/components/mdns/test_afl_fuzz_host/esp32_mock.h index 5ff5ad6330..fb4a21a43f 100644 --- a/components/mdns/test_afl_fuzz_host/esp32_mock.h +++ b/components/mdns/test_afl_fuzz_host/esp32_mock.h @@ -40,6 +40,10 @@ void GetLastItem(void *pvBuffer); void ForceTaskDelete(); +esp_err_t esp_event_handler_register(const char * event_base, int32_t event_id, void* event_handler, void* event_handler_arg); + +esp_err_t esp_event_handler_unregister(const char * event_base, int32_t event_id, void* event_handler); + #define _mdns_udp_pcb_write(tcpip_if, ip_protocol, ip, port, data, len) len #endif /* ESP32_MOCK_H_ */ \ No newline at end of file From 7963ac65af2003aa790bca8667b8a49ca3f3c5a4 Mon Sep 17 00:00:00 2001 From: David Cermak Date: Mon, 6 May 2019 16:14:48 +0200 Subject: [PATCH 4/4] mqtt_tests: add weekend test for sending and receiving empty payload messages, update config options per new naming convetions --- .../mqtt/weekend_test/mqtt_publish_test.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/components/mqtt/weekend_test/mqtt_publish_test.py b/components/mqtt/weekend_test/mqtt_publish_test.py index 474bcd8c11..9263f3ada7 100644 --- a/components/mqtt/weekend_test/mqtt_publish_test.py +++ b/components/mqtt/weekend_test/mqtt_publish_test.py @@ -71,11 +71,11 @@ def test_single_config(dut, transport, qos, repeat, published): global expected_data global message_log sample_string = ''.join(random.choice(string.ascii_uppercase + string.ascii_lowercase + string.digits) for _ in range(16)) - print("PUBLISH TEST: transport:{}, qos:{}, sequence:{}, sample msg:{}".format(transport, qos, published, sample_string)) event_client_connected.clear() expected_count = 0 message_log = "" expected_data = sample_string * repeat + print("PUBLISH TEST: transport:{}, qos:{}, sequence:{}, sample msg:'{}'".format(transport, qos, published, expected_data)) client = None try: if transport in ["ws", "wss"]: @@ -148,12 +148,12 @@ def test_weekend_mqtt_publish(env, extra_data): # Look for host:port in sdkconfig try: # python client subscribes to the topic to which esp client publishes and vice versa - publish_topic = dut1.app.get_sdkconfig()["CONFIG_SUBSCIBE_TOPIC"].replace('"','') - subscribe_topic = dut1.app.get_sdkconfig()["CONFIG_PUBLISH_TOPIC"].replace('"','') - broker_host["ssl"], broker_port["ssl"] = get_host_port_from_dut(dut1, "CONFIG_BROKER_SSL_URI") - broker_host["tcp"], broker_port["tcp"] = get_host_port_from_dut(dut1, "CONFIG_BROKER_TCP_URI") - broker_host["ws"], broker_port["ws"] = get_host_port_from_dut(dut1, "CONFIG_BROKER_WS_URI") - broker_host["wss"], broker_port["wss"] = get_host_port_from_dut(dut1, "CONFIG_BROKER_WSS_URI") + publish_topic = dut1.app.get_sdkconfig()["CONFIG_EXAMPLE_SUBSCIBE_TOPIC"].replace('"','') + subscribe_topic = dut1.app.get_sdkconfig()["CONFIG_EXAMPLE_PUBLISH_TOPIC"].replace('"','') + broker_host["ssl"], broker_port["ssl"] = get_host_port_from_dut(dut1, "CONFIG_EXAMPLE_BROKER_SSL_URI") + broker_host["tcp"], broker_port["tcp"] = get_host_port_from_dut(dut1, "CONFIG_EXAMPLE_BROKER_TCP_URI") + broker_host["ws"], broker_port["ws"] = get_host_port_from_dut(dut1, "CONFIG_EXAMPLE_BROKER_WS_URI") + broker_host["wss"], broker_port["wss"] = get_host_port_from_dut(dut1, "CONFIG_EXAMPLE_BROKER_WSS_URI") except Exception: print('ENV_TEST_FAILURE: Cannot find broker url in sdkconfig') raise @@ -166,8 +166,10 @@ def test_weekend_mqtt_publish(env, extra_data): raise for qos in [0, 1, 2]: for transport in ["tcp", "ssl", "ws", "wss"]: + # simple test with empty message + test_single_config(dut1, transport, qos, 0, 5) # decide on broker what level of test will pass (local broker works the best) - if broker_host[transport].startswith("192.168"): + if broker_host[transport].startswith("192.168") and qos < 1: # medium size, medium repeated test_single_config(dut1, transport, qos, 5, 50) # long data