MQTT: Integrate esp-mqtt library into idf

added docs and tests for mqtt library, small fixes (removed warnings, option for custom outbox, websocket bug fixed for longer transports). refactored to use common tcp_transport component, support for CMake build system.
Closes #2108

examples/protocols/mqtt/tcp/CMakeLists.txt

@@ -0,0 +1,7 @@
+# The following four lines of boilerplate have to be in your project's CMakeLists
+# in this exact order for cmake to work correctly
+cmake_minimum_required(VERSION 3.5)
+
+include($ENV{IDF_PATH}/tools/cmake/project.cmake)
+
+project(mqtt_tcp)

examples/protocols/mqtt/tcp/Makefile

@@ -0,0 +1,7 @@
+#
+# This is a project Makefile. It is assumed the directory this Makefile resides in is a
+# project subdirectory.
+#
+PROJECT_NAME := mqtt_tcp
+
+include $(IDF_PATH)/make/project.mk

examples/protocols/mqtt/tcp/README.md

@@ -0,0 +1,61 @@
+# ESP-MQTT sample application
+(See the README.md file in the upper level 'examples' directory for more information about examples.)
+
+This example connects to the broker URI selected using `make menuconfig` (using mqtt tcp transport) and as a demonstration subscribes/unsubscribes and send a message on certain topic.
+Note: If the URI equals `FROM_STDIN` then the broker address is read from stdin upon application startup (used for testing)
+
+It uses ESP-MQTT library which implements mqtt client to connect to mqtt broker.
+
+## How to use example
+
+### Hardware Required
+
+This example can be executed on any ESP32 board, the only required interface is WiFi and connection to internet.
+
+### Configure the project
+
+```
+make menuconfig
+```
+
+* Set serial port under Serial Flasher Options.
+
+* Set ssid and password for the board to connect to AP.
+
+### Build and Flash
+
+Build the project and flash it to the board, then run monitor tool to view serial output:
+
+```
+make -j4 flash monitor
+```
+
+(To exit the serial monitor, type ``Ctrl-]``.)
+
+See the Getting Started Guide for full steps to configure and use ESP-IDF to build projects.
+
+## Example Output
+
+```
+I (3714) event: sta ip: 192.168.0.139, mask: 255.255.255.0, gw: 192.168.0.2
+I (3714) system_api: Base MAC address is not set, read default base MAC address from BLK0 of EFUSE
+I (3964) MQTT_CLIENT: Sending MQTT CONNECT message, type: 1, id: 0000
+I (4164) MQTT_EXAMPLE: MQTT_EVENT_CONNECTED
+I (4174) MQTT_EXAMPLE: sent publish successful, msg_id=41464
+I (4174) MQTT_EXAMPLE: sent subscribe successful, msg_id=17886
+I (4174) MQTT_EXAMPLE: sent subscribe successful, msg_id=42970
+I (4184) MQTT_EXAMPLE: sent unsubscribe successful, msg_id=50241
+I (4314) MQTT_EXAMPLE: MQTT_EVENT_PUBLISHED, msg_id=41464
+I (4484) MQTT_EXAMPLE: MQTT_EVENT_SUBSCRIBED, msg_id=17886
+I (4484) MQTT_EXAMPLE: sent publish successful, msg_id=0
+I (4684) MQTT_EXAMPLE: MQTT_EVENT_SUBSCRIBED, msg_id=42970
+I (4684) MQTT_EXAMPLE: sent publish successful, msg_id=0
+I (4884) MQTT_CLIENT: deliver_publish, message_length_read=19, message_length=19
+I (4884) MQTT_EXAMPLE: MQTT_EVENT_DATA
+TOPIC=/topic/qos0
+DATA=data
+I (5194) MQTT_CLIENT: deliver_publish, message_length_read=19, message_length=19
+I (5194) MQTT_EXAMPLE: MQTT_EVENT_DATA
+TOPIC=/topic/qos0
+DATA=data
+```

examples/protocols/mqtt/tcp/main/CMakeLists.txt

@@ -0,0 +1,4 @@
+set(COMPONENT_SRCS "app_main.c")
+set(COMPONENT_ADD_INCLUDEDIRS ".")
+
+register_component()

examples/protocols/mqtt/tcp/main/Kconfig.projbuild

@@ -0,0 +1,25 @@
+menu "Example Configuration"
+
+config WIFI_SSID
+    string "WiFi SSID"
+    default "myssid"
+    help
+        SSID (network name) for the example to connect to.
+
+config WIFI_PASSWORD
+    string "WiFi Password"
+    default "mypassword"
+    help
+        WiFi password (WPA or WPA2) for the example to use.
+
+config BROKER_URL
+    string "Broker URL"
+    default "mqtt://iot.eclipse.org"
+    help
+        URL of the broker to connect to
+
+config BROKER_URL_FROM_STDIN
+	bool
+	default y if BROKER_URL = "FROM_STDIN"
+
+endmenu

examples/protocols/mqtt/tcp/main/app_main.c

@@ -0,0 +1,171 @@
+#include <stdio.h>
+#include <stdint.h>
+#include <stddef.h>
+#include <string.h>
+#include "esp_wifi.h"
+#include "esp_system.h"
+#include "nvs_flash.h"
+#include "esp_event_loop.h"
+
+#include "freertos/FreeRTOS.h"
+#include "freertos/task.h"
+#include "freertos/semphr.h"
+#include "freertos/queue.h"
+#include "freertos/event_groups.h"
+
+#include "lwip/sockets.h"
+#include "lwip/dns.h"
+#include "lwip/netdb.h"
+
+#include "esp_log.h"
+#include "mqtt_client.h"
+
+static const char *TAG = "MQTT_EXAMPLE";
+
+static EventGroupHandle_t wifi_event_group;
+const static int CONNECTED_BIT = BIT0;
+
+
+static esp_err_t mqtt_event_handler(esp_mqtt_event_handle_t event)
+{
+    esp_mqtt_client_handle_t client = event->client;
+    int msg_id;
+    // your_context_t *context = event->context;
+    switch (event->event_id) {
+        case MQTT_EVENT_CONNECTED:
+            ESP_LOGI(TAG, "MQTT_EVENT_CONNECTED");
+            msg_id = esp_mqtt_client_publish(client, "/topic/qos1", "data_3", 0, 1, 0);
+            ESP_LOGI(TAG, "sent publish successful, msg_id=%d", msg_id);
+
+            msg_id = esp_mqtt_client_subscribe(client, "/topic/qos0", 0);
+            ESP_LOGI(TAG, "sent subscribe successful, msg_id=%d", msg_id);
+
+            msg_id = esp_mqtt_client_subscribe(client, "/topic/qos1", 1);
+            ESP_LOGI(TAG, "sent subscribe successful, msg_id=%d", msg_id);
+
+            msg_id = esp_mqtt_client_unsubscribe(client, "/topic/qos1");
+            ESP_LOGI(TAG, "sent unsubscribe successful, msg_id=%d", msg_id);
+            break;
+        case MQTT_EVENT_DISCONNECTED:
+            ESP_LOGI(TAG, "MQTT_EVENT_DISCONNECTED");
+            break;
+
+        case MQTT_EVENT_SUBSCRIBED:
+            ESP_LOGI(TAG, "MQTT_EVENT_SUBSCRIBED, msg_id=%d", event->msg_id);
+            msg_id = esp_mqtt_client_publish(client, "/topic/qos0", "data", 0, 0, 0);
+            ESP_LOGI(TAG, "sent publish successful, msg_id=%d", msg_id);
+            break;
+        case MQTT_EVENT_UNSUBSCRIBED:
+            ESP_LOGI(TAG, "MQTT_EVENT_UNSUBSCRIBED, msg_id=%d", event->msg_id);
+            break;
+        case MQTT_EVENT_PUBLISHED:
+            ESP_LOGI(TAG, "MQTT_EVENT_PUBLISHED, msg_id=%d", event->msg_id);
+            break;
+        case MQTT_EVENT_DATA:
+            ESP_LOGI(TAG, "MQTT_EVENT_DATA");
+            printf("TOPIC=%.*s\r\n", event->topic_len, event->topic);
+            printf("DATA=%.*s\r\n", event->data_len, event->data);
+            break;
+        case MQTT_EVENT_ERROR:
+            ESP_LOGI(TAG, "MQTT_EVENT_ERROR");
+            break;
+    }
+    return ESP_OK;
+}
+
+static esp_err_t wifi_event_handler(void *ctx, system_event_t *event)
+{
+    switch (event->event_id) {
+        case SYSTEM_EVENT_STA_START:
+            esp_wifi_connect();
+            break;
+        case SYSTEM_EVENT_STA_GOT_IP:
+            xEventGroupSetBits(wifi_event_group, CONNECTED_BIT);
+
+            break;
+        case SYSTEM_EVENT_STA_DISCONNECTED:
+            esp_wifi_connect();
+            xEventGroupClearBits(wifi_event_group, CONNECTED_BIT);
+            break;
+        default:
+            break;
+    }
+    return ESP_OK;
+}
+
+static void wifi_init(void)
+{
+    tcpip_adapter_init();
+    wifi_event_group = xEventGroupCreate();
+    ESP_ERROR_CHECK(esp_event_loop_init(wifi_event_handler, NULL));
+    wifi_init_config_t cfg = WIFI_INIT_CONFIG_DEFAULT();
+    ESP_ERROR_CHECK(esp_wifi_init(&cfg));
+    ESP_ERROR_CHECK(esp_wifi_set_storage(WIFI_STORAGE_RAM));
+    wifi_config_t wifi_config = {
+        .sta = {
+            .ssid = CONFIG_WIFI_SSID,
+            .password = CONFIG_WIFI_PASSWORD,
+        },
+    };
+    ESP_ERROR_CHECK(esp_wifi_set_mode(WIFI_MODE_STA));
+    ESP_ERROR_CHECK(esp_wifi_set_config(ESP_IF_WIFI_STA, &wifi_config));
+    ESP_LOGI(TAG, "start the WIFI SSID:[%s]", CONFIG_WIFI_SSID);
+    ESP_ERROR_CHECK(esp_wifi_start());
+    ESP_LOGI(TAG, "Waiting for wifi");
+    xEventGroupWaitBits(wifi_event_group, CONNECTED_BIT, false, true, portMAX_DELAY);
+}
+
+static void mqtt_app_start(void)
+{
+    esp_mqtt_client_config_t mqtt_cfg = {
+        .uri = CONFIG_BROKER_URL,
+        .event_handle = mqtt_event_handler,
+        // .user_context = (void *)your_context
+    };
+
+#if CONFIG_BROKER_URL_FROM_STDIN
+    char line[128];
+
+    if (strcmp(mqtt_cfg.uri, "FROM_STDIN") == 0) {
+        int count = 0;
+        printf("Please enter url of mqtt broker\n");
+        while (count < 128) {
+            int c = fgetc(stdin);
+            if (c == '\n') {
+                line[count] = '\0';
+                break;
+            } else if (c > 0 && c < 127) {
+                line[count] = c;
+                ++count;
+            }
+            vTaskDelay(10 / portTICK_PERIOD_MS);
+        }
+        mqtt_cfg.uri = line;
+        printf("Broker url: %s\n", line);
+    } else {
+        ESP_LOGE(TAG, "Configuration mismatch: wrong broker url");
+        abort();
+    }
+#endif /* CONFIG_BROKER_URL_FROM_STDIN */
+
+    esp_mqtt_client_handle_t client = esp_mqtt_client_init(&mqtt_cfg);
+    esp_mqtt_client_start(client);
+}
+
+void app_main()
+{
+    ESP_LOGI(TAG, "[APP] Startup..");
+    ESP_LOGI(TAG, "[APP] Free memory: %d bytes", esp_get_free_heap_size());
+    ESP_LOGI(TAG, "[APP] IDF version: %s", esp_get_idf_version());
+
+    esp_log_level_set("*", ESP_LOG_INFO);
+    esp_log_level_set("MQTT_CLIENT", ESP_LOG_VERBOSE);
+    esp_log_level_set("TRANSPORT_TCP", ESP_LOG_VERBOSE);
+    esp_log_level_set("TRANSPORT_SSL", ESP_LOG_VERBOSE);
+    esp_log_level_set("TRANSPORT", ESP_LOG_VERBOSE);
+    esp_log_level_set("OUTBOX", ESP_LOG_VERBOSE);
+
+    nvs_flash_init();
+    wifi_init();
+    mqtt_app_start();
+}

examples/protocols/mqtt/tcp/mqtt_tcp_example_test.py

@@ -0,0 +1,101 @@
+import re
+import os
+import sys
+from socket import *
+from threading import Thread
+import time
+
+global msgid
+
+def get_my_ip():
+    s1 = socket(AF_INET, SOCK_DGRAM)
+    s1.connect(("8.8.8.8", 80))
+    my_ip = s1.getsockname()[0]
+    s1.close()
+    return my_ip
+
+def mqqt_server_sketch(my_ip, port):
+    global msgid
+    print("Starting the server on {}".format(my_ip))
+    s=socket(AF_INET, SOCK_STREAM)
+    s.settimeout(60)
+    s.bind((my_ip, port))
+    s.listen(1)
+    q,addr=s.accept()
+    q.settimeout(30)
+    print("connection accepted")
+    # q.send(g_msg_to_client)
+    data = q.recv(1024)
+    # check if received initial empty message
+    print("received from client {}".format(data))
+    data = bytearray([0x20, 0x02, 0x00, 0x00])
+    q.send(data)
+    # try to receive qos1
+    data = q.recv(1024)
+    msgid = ord(data[15])*256+ord(data[16])
+    print("received from client {}, msgid: {}".format(data, msgid))
+    data = bytearray([0x40, 0x02, data[15], data[16]])
+    q.send(data)
+    time.sleep(5)
+    s.close()
+    print("server closed")
+
+# this is a test case write with tiny-test-fw.
+# to run test cases outside tiny-test-fw,
+# we need to set environment variable `TEST_FW_PATH`,
+# then get and insert `TEST_FW_PATH` to sys path before import FW module
+test_fw_path = os.getenv("TEST_FW_PATH")
+if test_fw_path and test_fw_path not in sys.path:
+    sys.path.insert(0, test_fw_path)
+
+import TinyFW
+import IDF
+
+
+
+
+@IDF.idf_example_test(env_tag="Example_WIFI")
+def test_examples_protocol_mqtt_qos1(env, extra_data):
+    global msgid
+    """
+    steps: (QoS1: Happy flow)
+      1. start the broker broker (with correctly sending ACK)
+      2. DUT client connects to a broker and publishes qos1 message
+      3. Test evaluates that qos1 message is queued and removed from queued after ACK received
+      4. Test the broker received the same message id evaluated in step 3
+    """
+    dut1 = env.get_dut("mqtt_tcp", "examples/protocols/mqtt/tcp")
+    # check and log bin size
+    binary_file = os.path.join(dut1.app.binary_path, "mqtt_tcp.bin")
+    bin_size = os.path.getsize(binary_file)
+    IDF.log_performance("mqtt_tcp_bin_size", "{}KB".format(bin_size//1024))
+    IDF.check_performance("mqtt_tcp_size", bin_size//1024)
+    # 1. start mqtt broker sketch
+    host_ip = get_my_ip()
+    thread1 = Thread(target = mqqt_server_sketch, args = (host_ip,1883))
+    thread1.start()
+    # 2. start the dut test and wait till client gets IP address
+    dut1.start_app()
+    # waiting for getting the IP address
+    data = dut1.expect(re.compile(r" sta ip: ([^,]+),"), timeout=30)
+    # time.sleep(15)
+    print ("writing to device: {}".format("mqtt://" + host_ip + "\n"))
+    dut1.write("mqtt://" + host_ip + "\n")
+    thread1.join()
+    print ("Message id received from server: {}".format(msgid))
+    # 3. check the message id was enqueued and then deleted
+    msgid_enqueued = dut1.expect(re.compile(r"OUTBOX: ENQUEUE msgid=([0-9]+)"), timeout=30)
+    # expect_txt="OUTBOX: ENQUEUE msgid=" + str(msgid)
+    # dut1.expect(re.compile(expect_txt), timeout=30)
+    msgid_deleted = dut1.expect(re.compile(r"OUTBOX: DELETED msgid=([0-9]+)"), timeout=30)
+    # expect_txt="OUTBOX: DELETED msgid=" + str(msgid)
+    # dut1.expect(re.compile(expect_txt), timeout=30)
+    # 4. check the msgid of received data are the same as that of enqueued and deleted from outbox
+    if (msgid_enqueued[0] == str(msgid) and msgid_deleted[0] == str(msgid)):
+        print("PASS: Received correct msg id")
+    else:
+        print("Failure!")
+        raise ValueError('Mismatch of msgid: received: {}, enqueued {}, deleted {}'.format(msgid, msgid_enqueued, msgid_deleted))
+
+if __name__ == '__main__':
+    test_examples_protocol_mqtt_qos1()

examples/protocols/mqtt/tcp/sdkconfig

@@ -0,0 +1 @@
+

examples/protocols/mqtt/tcp/sdkconfig.defaults

@@ -0,0 +1,7 @@
+CONFIG_BROKER_URL="FROM_STDIN"
+CONFIG_LOG_DEFAULT_LEVEL_NONE=
+CONFIG_LOG_DEFAULT_LEVEL_ERROR=
+CONFIG_LOG_DEFAULT_LEVEL_WARN=
+CONFIG_LOG_DEFAULT_LEVEL_INFO=
+CONFIG_LOG_DEFAULT_LEVEL_DEBUG=y
+CONFIG_LOG_DEFAULT_LEVEL_VERBOSE=