Merge pull request #265 from david-cermak/feat/linux_port_component

examples: Add simple mqtt client demo running on linux

.pre-commit-config.yaml

@@ -62,7 +62,7 @@ repos:
     hooks:
       - id: commit message scopes
         name: "commit message must be scoped with: mdns, modem, websocket, asio, mqtt_cxx, common"
-        entry: '\A(?!(feat|fix|ci|bump|test|docs)\((mdns|modem|common|websocket|asio|mqtt_cxx)\)\:)'
+        entry: '\A(?!(feat|fix|ci|bump|test|docs)\((mdns|modem|common|websocket|asio|mqtt_cxx|examples)\)\:)'
         language: pygrep
         args: [--multiline]
         stages: [commit-msg]

ci/check_copyright_config.yaml

@@ -29,6 +29,7 @@ DEFAULT:
 # this section sets the default license for examples and unit tests of components
 examples_and_unit_tests:
   include:
+   - 'examples/**'
    - 'components/**/examples/**'
    - 'components/**/test/**'
    - 'components/**/tests/**'

common_components/linux_compat/esp_timer/esp_timer_linux.c

@@ -7,6 +7,7 @@
 #include "esp_timer.h"
 #include "freertos/FreeRTOS.h"
 #include "freertos/task.h"
+#include <pthread.h>
 
 void *create_tt(esp_timer_cb_t cb);
 
@@ -37,3 +38,10 @@ esp_err_t esp_timer_delete(esp_timer_handle_t timer)
     destroy_tt(timer);
     return ESP_OK;
 }
+
+int64_t esp_timer_get_time(void)
+{
+    struct timespec spec;
+    clock_gettime(CLOCK_REALTIME, &spec);
+    return spec.tv_nsec / 1000 + spec.tv_sec * 1000000;
+}

common_components/linux_compat/esp_timer/include/esp_timer.h

@@ -9,6 +9,11 @@
 #include <stdint.h>
 #include "bsd/string.h"
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
 typedef struct esp_timer *esp_timer_handle_t;
 
 typedef void (*esp_timer_cb_t)(void *arg);
@@ -32,3 +37,9 @@ esp_err_t esp_timer_start_periodic(esp_timer_handle_t timer, uint64_t period);
 esp_err_t esp_timer_stop(esp_timer_handle_t timer);
 
 esp_err_t esp_timer_delete(esp_timer_handle_t timer);
+
+int64_t esp_timer_get_time(void);
+
+#ifdef __cplusplus
+}
+#endif

common_components/linux_compat/freertos/CMakeLists.txt

@@ -1,4 +1,5 @@
-idf_component_register(SRCS freertos_linux.c queue_unique_ptr.cpp
+idf_component_register(SRCS freertos_linux.c
+                            osal/queue.cpp osal/event_group.cpp osal/mutex.cpp
                        INCLUDE_DIRS include)
 
 set(THREADS_PREFER_PTHREAD_FLAG ON)

common_components/linux_compat/freertos/freertos_linux.c

@@ -0,0 +1,272 @@
+/*
+ * SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#include <unistd.h>
+#include "freertos/FreeRTOS.h"
+#include "freertos/task.h"
+#include <pthread.h>
+#include <assert.h>
+#include <stdlib.h>
+#include <string.h>
+#include "osal/osal_api.h"
+
+static uint64_t s_semaphore_data = 0;
+
+typedef enum queue_type_tag {
+    MUTEX_REC,
+    MUTEX,
+    SEMA,
+    QUEUE,
+} queue_type_t;
+
+struct generic_queue_handle {
+    queue_type_t type;
+    size_t item_size;
+    void *q;
+};
+
+static struct generic_queue_handle *create_generic_queue(queue_type_t type, uint32_t len, uint32_t item_size)
+{
+    struct generic_queue_handle *h = calloc(1, sizeof(struct generic_queue_handle));
+    h->item_size = len;
+    h->type = type;
+    switch (type) {
+    default:
+    case QUEUE:
+    case SEMA:
+        h->q = osal_queue_create();
+        break;
+
+    case MUTEX:
+    case MUTEX_REC:
+        h->q = osal_mutex_create();
+        break;
+    }
+    return h;
+}
+
+QueueHandle_t xQueueCreate(uint32_t uxQueueLength, uint32_t uxItemSize )
+{
+    return (QueueHandle_t)create_generic_queue(QUEUE, uxQueueLength, uxItemSize);
+}
+
+uint32_t xQueueSend(QueueHandle_t xQueue, const void *pvItemToQueue, TickType_t xTicksToWait)
+{
+    struct generic_queue_handle *h = xQueue;
+    return osal_queue_send(h->q, (uint8_t *)pvItemToQueue, h->item_size) ? pdTRUE : pdFAIL;
+}
+
+uint32_t xQueueSendToBack(QueueHandle_t xQueue, const void *pvItemToQueue, TickType_t xTicksToWait )
+{
+    return xQueueSend(xQueue, pvItemToQueue, xTicksToWait);
+}
+
+uint32_t xQueueReceive(QueueHandle_t xQueue, void *pvBuffer, TickType_t xTicksToWait)
+{
+    struct generic_queue_handle *h = xQueue;
+    return osal_queue_recv(h->q, (uint8_t *)pvBuffer, h->item_size, xTicksToWait) ? pdTRUE : pdFAIL;
+}
+
+BaseType_t xSemaphoreGive( QueueHandle_t xQueue)
+{
+    struct generic_queue_handle *h = xQueue;
+    if (h->type == MUTEX) {
+        osal_mutex_give(h->q);
+        return pdTRUE;
+    }
+    return xQueueSend(xQueue, &s_semaphore_data, portMAX_DELAY);
+}
+
+BaseType_t xSemaphoreGiveRecursive( QueueHandle_t xQueue)
+{
+    struct generic_queue_handle *h = xQueue;
+    if (h->type == MUTEX_REC) {
+        osal_mutex_give(h->q);
+        return pdTRUE;
+    }
+    return pdFALSE;
+}
+BaseType_t xSemaphoreTake( QueueHandle_t xQueue, TickType_t pvTask )
+{
+    struct generic_queue_handle *h = xQueue;
+    if (h->type == MUTEX) {
+        osal_mutex_take(h->q);
+        return pdTRUE;
+    }
+    return xQueueReceive(xQueue, &s_semaphore_data, portMAX_DELAY);
+}
+
+
+BaseType_t xSemaphoreTakeRecursive( QueueHandle_t xQueue, TickType_t pvTask )
+{
+    struct generic_queue_handle *h = xQueue;
+    if (h->type == MUTEX_REC) {
+        osal_mutex_take(h->q);
+        return pdTRUE;
+    }
+    return pdFALSE;
+}
+
+
+
+
+void vQueueDelete( QueueHandle_t xQueue )
+{
+    struct generic_queue_handle *h = xQueue;
+    if (h->q) {
+        if (h->type == MUTEX || h->type == MUTEX_REC) {
+            osal_mutex_delete(h->q);
+        } else {
+            osal_queue_delete(h->q);
+        }
+    }
+    free(xQueue);
+}
+
+QueueHandle_t xSemaphoreCreateBinary(void)
+{
+    QueueHandle_t sempaphore =  xQueueCreate(1, 1);
+    return sempaphore;
+}
+
+QueueHandle_t xSemaphoreCreateMutex(void)
+{
+    return (QueueHandle_t)create_generic_queue(MUTEX, 1, 1);
+}
+
+QueueHandle_t xSemaphoreCreateRecursiveMutex(void)
+{
+    return (QueueHandle_t)create_generic_queue(MUTEX_REC, 1, 1);
+}
+
+
+void vTaskDelete(TaskHandle_t *task)
+{
+    if (task == NULL) {
+        pthread_exit(0);
+    }
+    void *thread_rval = NULL;
+    pthread_join((pthread_t)task, &thread_rval);
+}
+
+void vTaskSuspend(void *task)
+{
+    vTaskDelete(task);
+}
+
+TickType_t xTaskGetTickCount( void )
+{
+    struct timespec spec;
+    clock_gettime(CLOCK_REALTIME, &spec);
+    return spec.tv_nsec / 1000000 + spec.tv_sec * 1000;
+}
+
+void vTaskDelay( const TickType_t xTicksToDelay )
+{
+    usleep(xTicksToDelay * 1000);
+}
+
+void *pthread_task(void *params)
+{
+    struct {
+        void *const param;
+        TaskFunction_t task;
+        bool started;
+    } *pthread_params = params;
+
+    void *const param = pthread_params->param;
+    TaskFunction_t task = pthread_params->task;
+    pthread_params->started = true;
+
+    task(param);
+
+    return NULL;
+}
+
+BaseType_t xTaskCreatePinnedToCore( TaskFunction_t pvTaskCode,
+                                    const char *const pcName,
+                                    const uint32_t usStackDepth,
+                                    void *const pvParameters,
+                                    UBaseType_t uxPriority,
+                                    TaskHandle_t *const pvCreatedTask,
+                                    const BaseType_t xCoreID)
+{
+    xTaskCreate(pvTaskCode, pcName, usStackDepth, pvParameters, uxPriority, pvCreatedTask);
+    return pdTRUE;
+}
+
+
+BaseType_t xTaskCreate(TaskFunction_t pvTaskCode, const char *const pcName, const uint32_t usStackDepth, void *const pvParameters, UBaseType_t uxPriority, TaskHandle_t *const pvCreatedTask)
+{
+    pthread_t new_thread = (pthread_t)NULL;
+    pthread_attr_t attr;
+    struct {
+        void *const param;
+        TaskFunction_t task;
+        bool started;
+    } pthread_params = { .param = pvParameters, .task = pvTaskCode};
+    int res = pthread_attr_init(&attr);
+    assert(res == 0);
+    res = pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE);
+    assert(res == 0);
+    res = pthread_create(&new_thread, &attr, pthread_task, &pthread_params);
+    assert(res == 0);
+
+    if (pvCreatedTask) {
+        *pvCreatedTask = (void *)new_thread;
+    }
+
+    // just wait till the task started so we can unwind params from the stack
+    while (pthread_params.started == false) {
+        usleep(1000);
+    }
+    return pdTRUE;
+}
+
+void xTaskNotifyGive(TaskHandle_t task)
+{
+
+}
+
+BaseType_t xTaskNotifyWait(uint32_t bits_entry_clear, uint32_t bits_exit_clear, uint32_t *value, TickType_t wait_time )
+{
+    return true;
+}
+
+TaskHandle_t xTaskGetCurrentTaskHandle(void)
+{
+    return NULL;
+}
+
+EventGroupHandle_t xEventGroupCreate( void )
+{
+    return osal_signal_create();
+}
+
+void vEventGroupDelete( EventGroupHandle_t xEventGroup )
+{
+    osal_signal_delete(xEventGroup);
+}
+
+EventBits_t xEventGroupClearBits( EventGroupHandle_t xEventGroup, const EventBits_t uxBitsToClear )
+{
+    return osal_signal_clear(xEventGroup, uxBitsToClear);
+}
+
+EventBits_t xEventGroupGetBits( EventGroupHandle_t xEventGroup)
+{
+    return osal_signal_get(xEventGroup);
+}
+
+EventBits_t xEventGroupSetBits( EventGroupHandle_t xEventGroup, const EventBits_t uxBitsToSet )
+{
+    return osal_signal_set(xEventGroup, uxBitsToSet);
+}
+
+EventBits_t xEventGroupWaitBits( EventGroupHandle_t xEventGroup, const EventBits_t uxBitsToWaitFor, const BaseType_t xClearOnExit, const BaseType_t xWaitForAllBits, TickType_t xTicksToWait )
+{
+    return osal_signal_wait(xEventGroup, uxBitsToWaitFor, xWaitForAllBits, xTicksToWait);
+}

common_components/linux_compat/freertos/include/esp_task.h

@@ -10,3 +10,4 @@
 
 #define ESP_TASK_PRIO_MAX 25
 #define ESP_TASKD_EVENT_PRIO 5
+#define ESP_TASKD_EVENT_STACK 1024

common_components/linux_compat/freertos/include/freertos/FreeRTOS.h

@@ -16,7 +16,9 @@
 typedef void *SemaphoreHandle_t;
 typedef void *QueueHandle_t;
 typedef void *TaskHandle_t;
+typedef void *EventGroupHandle_t;
 typedef uint32_t TickType_t;
+typedef TickType_t               EventBits_t;
 
 typedef void (*TaskFunction_t)( void * );
 typedef unsigned int    UBaseType_t;
@@ -30,6 +32,5 @@ typedef int             BaseType_t;
 
 #define pdMS_TO_TICKS(tick)    (tick)
 
-uint32_t esp_get_free_heap_size(void);
 uint32_t esp_random(void);
 void vTaskSuspendAll(void);

common_components/linux_compat/freertos/include/freertos/event_groups.h

@@ -0,0 +1,6 @@
+/*
+ * SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+#pragma once

common_components/linux_compat/freertos/include/freertos/queue.h

@@ -3,11 +3,4 @@
  *
  * SPDX-License-Identifier: Unlicense OR CC0-1.0
  */
-//
-// Created by david on 1/13/23.
-//
-
-#ifndef _QUEUE_H_
-#define _QUEUE_H_
-
-#endif //_QUEUE_H_
+#pragma once

common_components/linux_compat/freertos/include/freertos/semphr.h

@@ -3,11 +3,4 @@
  *
  * SPDX-License-Identifier: Unlicense OR CC0-1.0
  */
-//
-// Created by david on 1/13/23.
-//
-
-#ifndef _SEMAPHR_H_
-#define _SEMAPHR_H_
-
-#endif //_SEMAPHR_H_
+#pragma once

common_components/linux_compat/freertos/include/freertos/task.h

@@ -7,6 +7,10 @@
 
 #include "freertos/FreeRTOS.h"
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 #define TaskHandle_t TaskHandle_t
 #define vSemaphoreDelete( xSemaphore ) vQueueDelete( ( QueueHandle_t ) ( xSemaphore ) )
 
@@ -26,7 +30,7 @@ BaseType_t xTaskCreatePinnedToCore( TaskFunction_t pvTaskCode,
                                     TaskHandle_t *const pvCreatedTask,
                                     const BaseType_t xCoreID);
 
-void xTaskCreate(TaskFunction_t pvTaskCode, const char *const pcName, const uint32_t usStackDepth, void *const pvParameters, UBaseType_t uxPriority, TaskHandle_t *const pvCreatedTask);
+BaseType_t xTaskCreate(TaskFunction_t pvTaskCode, const char *const pcName, const uint32_t usStackDepth, void *const pvParameters, UBaseType_t uxPriority, TaskHandle_t *const pvCreatedTask);
 
 TickType_t xTaskGetTickCount( void );
 
@@ -35,11 +39,16 @@ void vQueueDelete( QueueHandle_t xQueue );
 QueueHandle_t xSemaphoreCreateBinary(void);
 
 QueueHandle_t xSemaphoreCreateMutex(void);
+QueueHandle_t xSemaphoreCreateRecursiveMutex(void);
 
 BaseType_t xSemaphoreGive( QueueHandle_t xQueue);
 
 BaseType_t xSemaphoreTake( QueueHandle_t xQueue, TickType_t pvTask );
 
+BaseType_t xSemaphoreGiveRecursive( QueueHandle_t xQueue);
+
+BaseType_t xSemaphoreTakeRecursive( QueueHandle_t xQueue, TickType_t pvTask );
+
 void vTaskDelete(TaskHandle_t *task);
 
 QueueHandle_t xQueueCreate( uint32_t uxQueueLength,
@@ -48,3 +57,27 @@ QueueHandle_t xQueueCreate( uint32_t uxQueueLength,
 uint32_t xQueueSend(QueueHandle_t xQueue, const void *pvItemToQueue, TickType_t xTicksToWait);
 
 uint32_t xQueueReceive(QueueHandle_t xQueue, void *pvBuffer, TickType_t xTicksToWait);
+
+void vTaskSuspend(void *task);
+
+EventGroupHandle_t xEventGroupCreate( void );
+void vEventGroupDelete( EventGroupHandle_t xEventGroup );
+EventBits_t xEventGroupClearBits( EventGroupHandle_t xEventGroup,
+                                  const EventBits_t uxBitsToClear );
+
+EventBits_t xEventGroupWaitBits( EventGroupHandle_t xEventGroup,
+                                 const EventBits_t uxBitsToWaitFor,
+                                 const BaseType_t xClearOnExit,
+                                 const BaseType_t xWaitForAllBits,
+                                 TickType_t xTicksToWait );
+
+EventBits_t xEventGroupGetBits( EventGroupHandle_t xEventGroup);
+
+EventBits_t xEventGroupSetBits( EventGroupHandle_t xEventGroup,
+                                const EventBits_t uxBitsToSet );
+
+uint32_t xQueueSendToBack(QueueHandle_t xQueue, const void *pvItemToQueue, TickType_t xTicksToWait );
+
+#ifdef __cplusplus
+}
+#endif //__cplusplus

common_components/linux_compat/freertos/osal/event_group.cpp

@@ -0,0 +1,109 @@
+/*
+ * SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+#include <mutex>
+#include <condition_variable>
+#include "osal_api.h"
+
+
+class SignalGroup {
+
+    struct SignalGroupInternal {
+        std::condition_variable notify;
+        std::mutex m;
+        uint32_t flags{ 0 };
+    };
+
+    using SignalT = std::unique_ptr<SignalGroupInternal>;
+
+public:
+
+    void set(uint32_t bits)
+    {
+        std::unique_lock<std::mutex> lock(event_group->m);
+        event_group->flags |= bits;
+        event_group->notify.notify_all();
+    }
+
+    uint32_t get()
+    {
+        return event_group->flags;
+    }
+
+    void clear(uint32_t bits)
+    {
+        std::unique_lock<std::mutex> lock(event_group->m);
+        event_group->flags &= ~bits;
+        event_group->notify.notify_all();
+    }
+
+    // waiting for all and clearing if set
+    bool wait(uint32_t flags, uint32_t time_ms)
+    {
+        std::unique_lock<std::mutex> lock(event_group->m);
+        return event_group->notify.wait_for(lock, std::chrono::milliseconds(time_ms), [&] {
+            if ((flags & event_group->flags) == flags)
+            {
+                event_group->flags &= ~flags;
+                return true;
+            }
+            return false;
+        });
+    }
+
+
+    // waiting for any bit, not clearing them
+    bool wait_any(uint32_t flags, uint32_t time_ms)
+    {
+        std::unique_lock<std::mutex> lock(event_group->m);
+        return event_group->notify.wait_for(lock, std::chrono::milliseconds(time_ms), [&] { return flags & event_group->flags; });
+    }
+
+private:
+    SignalT event_group{std::make_unique<SignalGroupInternal>()};
+};
+
+
+void *osal_signal_create()
+{
+    auto signal = new SignalGroup;
+    return signal;
+}
+
+void osal_signal_delete(void *s)
+{
+    delete static_cast<SignalGroup *>(s);
+}
+
+uint32_t osal_signal_clear(void *s, uint32_t bits)
+{
+    auto signal = static_cast<SignalGroup *>(s);
+    signal->clear(bits);
+    return signal->get();
+}
+
+uint32_t osal_signal_set(void *s, uint32_t bits)
+{
+    auto signal = static_cast<SignalGroup *>(s);
+    signal->set(bits);
+    return signal->get();
+}
+
+uint32_t osal_signal_get(void *s)
+{
+    auto signal = static_cast<SignalGroup *>(s);
+    return signal->get();
+}
+
+uint32_t osal_signal_wait(void *s, uint32_t flags, bool all, uint32_t timeout)
+{
+    auto signal = static_cast<SignalGroup *>(s);
+    if (all) {
+        signal->wait(flags, timeout);
+    } else {
+        signal->wait_any(flags, timeout);
+    }
+    return signal->get();
+}

common_components/linux_compat/freertos/osal/mutex.cpp

@@ -0,0 +1,31 @@
+/*
+ * SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+#include <queue>
+#include <mutex>
+#include "osal_api.h"
+
+void *osal_mutex_create()
+{
+    auto mut = new std::recursive_mutex();
+    return mut;
+}
+
+void osal_mutex_delete(void *mut)
+{
+    delete static_cast<std::recursive_mutex *>(mut);
+}
+
+void osal_mutex_take(void *m)
+{
+    auto mut = static_cast<std::recursive_mutex *>(m);
+    mut->lock();
+}
+
+void osal_mutex_give(void *m)
+{
+    auto mut = static_cast<std::recursive_mutex *>(m);
+    mut->unlock();
+}

common_components/linux_compat/freertos/osal/osal_api.h

@@ -0,0 +1,34 @@
+/*
+ * SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+#pragma once
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// queue api
+void *osal_queue_create(void);
+void osal_queue_delete(void *q);
+bool osal_queue_send(void *q, uint8_t *data, size_t len);
+bool osal_queue_recv(void *q, uint8_t *data, size_t len, uint32_t ms);
+
+// mutex api
+void *osal_mutex_create(void);
+void osal_mutex_delete(void *m);
+void osal_mutex_take(void *m);
+void osal_mutex_give(void *m);
+
+// event groups
+void *osal_signal_create(void);
+void osal_signal_delete(void *s);
+uint32_t osal_signal_clear(void *s, uint32_t bits);
+uint32_t osal_signal_set(void *s, uint32_t bits);
+uint32_t osal_signal_get(void *s);
+uint32_t osal_signal_wait(void *s, uint32_t flags, bool all, uint32_t timeout);
+
+#ifdef __cplusplus
+}
+#endif

common_components/linux_compat/freertos/osal/queue.cpp

@@ -0,0 +1,78 @@
+/*
+ * SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#include <cstdint>
+#include <vector>
+#include <cstring>
+#include <queue>
+#include <mutex>
+#include <condition_variable>
+#include <atomic>
+#include "osal_api.h"
+
+template <class T>
+class Queue {
+public:
+    void send(std::unique_ptr<T> t)
+    {
+        std::lock_guard<std::mutex> lock(m);
+        q.push(std::move(t));
+        c.notify_one();
+    }
+
+    std::unique_ptr<T> receive(std::chrono::milliseconds ms)
+    {
+        std::unique_lock<std::mutex> lock(m);
+        while (q.empty()) {
+            if (c.wait_for(lock, ms) == std::cv_status::timeout) {
+                return nullptr;
+            }
+        }
+        std::unique_ptr<T> val = std::move(q.front());
+        q.pop();
+        return val;
+    }
+
+private:
+    std::queue<std::unique_ptr<T>> q{};
+    mutable std::mutex m{};
+    std::condition_variable c{};
+};
+
+using item_t = std::vector<uint8_t>;
+
+void *osal_queue_create(void)
+{
+    auto *q = new Queue<item_t>();
+    return q;
+}
+
+void osal_queue_delete(void *q)
+{
+    auto *queue = static_cast<Queue<item_t> *>(q);
+    delete (queue);
+}
+
+bool osal_queue_send(void *q, uint8_t *data, size_t len)
+{
+    auto v = std::make_unique<item_t>(len);
+    v->assign(data, data + len);
+    auto queue = static_cast<Queue<item_t> *>(q);
+    queue->send(std::move(v));
+    return true;
+}
+
+
+bool osal_queue_recv(void *q, uint8_t *data, size_t len, uint32_t ms)
+{
+    auto queue = static_cast<Queue<item_t> *>(q);
+    auto v = queue->receive(std::chrono::milliseconds(ms));
+    if (v != nullptr) {
+        memcpy(data, (void *)v->data(), len);
+        return true;
+    }
+    return false;
+}

components/esp_modem/src/esp_modem_term_uart.cpp

@@ -8,6 +8,11 @@
 #include "esp_modem_config.h"
 #include "uart_resource.hpp"
 
+#ifndef UART_HW_FIFO_LEN
+// to build with IDF <= v5.1
+#define UART_HW_FIFO_LEN(uart_nr) UART_FIFO_LEN
+#endif
+
 namespace esp_modem {
 
 uart_resource::~uart_resource()
@@ -18,7 +23,7 @@ uart_resource::~uart_resource()
 }
 
 uart_resource::uart_resource(const esp_modem_uart_term_config *config, QueueHandle_t *event_queue, int fd)
-    : port(-1)
+    : port(UART_NUM_MAX)
 {
     esp_err_t res;
 
@@ -44,9 +49,9 @@ uart_resource::uart_resource(const esp_modem_uart_term_config *config, QueueHand
     ESP_MODEM_THROW_IF_ERROR(res, "config uart gpio failed");
     /* Set flow control threshold */
     if (config->flow_control == ESP_MODEM_FLOW_CONTROL_HW) {
-        res = uart_set_hw_flow_ctrl(config->port_num, UART_HW_FLOWCTRL_CTS_RTS, UART_FIFO_LEN - 8);
+        res = uart_set_hw_flow_ctrl(config->port_num, UART_HW_FLOWCTRL_CTS_RTS, UART_HW_FIFO_LEN(config->port_num) - 8);
     } else if (config->flow_control == ESP_MODEM_FLOW_CONTROL_SW) {
-        res = uart_set_sw_flow_ctrl(config->port_num, true, 8, UART_FIFO_LEN - 8);
+        res = uart_set_sw_flow_ctrl(config->port_num, true, 8, UART_HW_FIFO_LEN(config->port_num) - 8);
     }
     ESP_MODEM_THROW_IF_ERROR(res, "config uart flow control failed");
 

components/mdns/CMakeLists.txt

@@ -6,7 +6,7 @@ endif()
 
 idf_build_get_property(target IDF_TARGET)
 if(${target} STREQUAL "linux")
-    set(dependencies esp_netif_linux esp_timer_linux esp_system)
+    set(dependencies esp_netif_linux esp_timer esp_system)
     set(srcs "mdns.c" ${MDNS_NETWORKING})
 else()
     set(dependencies lwip console esp_netif)

components/mdns/tests/host_test/CMakeLists.txt

@@ -1,6 +1,6 @@
 cmake_minimum_required(VERSION 3.5)
 
-set(EXTRA_COMPONENT_DIRS "../..")
+set(EXTRA_COMPONENT_DIRS "../.." "../../../../common_components/linux_compat")
 
 include($ENV{IDF_PATH}/tools/cmake/project.cmake)
 set(COMPONENTS main esp_netif_linux)

components/mdns/tests/host_test/components/freertos/freertos_linux.c

@@ -1,174 +0,0 @@
-/*
- * SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
- *
- * SPDX-License-Identifier: Apache-2.0
- */
-
-#include <unistd.h>
-#include "freertos/FreeRTOS.h"
-#include "freertos/task.h"
-#include <pthread.h>
-#include <assert.h>
-#include <stdlib.h>
-#include <string.h>
-
-void *create_q(void);
-
-void destroy_q(void *q);
-
-bool send_q(void *q, uint8_t *data, size_t len);
-
-bool recv_q(void *q, uint8_t *data, size_t len, uint32_t ms);
-
-static uint64_t s_semaphore_data = 0;
-
-struct queue_handle {
-    size_t item_size;
-    void *q;
-};
-
-QueueHandle_t xQueueCreate( uint32_t uxQueueLength, uint32_t uxItemSize )
-{
-    struct queue_handle *h = calloc(1, sizeof(struct queue_handle));
-    h->item_size = uxItemSize;
-    h->q = create_q();
-    return (QueueHandle_t)h;
-}
-
-uint32_t xQueueSend(QueueHandle_t xQueue, const void *pvItemToQueue, TickType_t xTicksToWait)
-{
-    struct queue_handle *h = xQueue;
-    return send_q(h->q, (uint8_t *)pvItemToQueue, h->item_size) ? pdTRUE : pdFAIL;
-}
-
-uint32_t xQueueReceive(QueueHandle_t xQueue, void *pvBuffer, TickType_t xTicksToWait)
-{
-    struct queue_handle *h = xQueue;
-    return recv_q(h->q, (uint8_t *)pvBuffer, h->item_size, xTicksToWait) ? pdTRUE : pdFAIL;
-}
-
-BaseType_t xSemaphoreGive( QueueHandle_t xQueue)
-{
-    return xQueueSend(xQueue, &s_semaphore_data, portMAX_DELAY);
-}
-
-BaseType_t xSemaphoreTake( QueueHandle_t xQueue, TickType_t pvTask )
-{
-    return xQueueReceive(xQueue, &s_semaphore_data, portMAX_DELAY);
-}
-
-void vQueueDelete( QueueHandle_t xQueue )
-{
-    struct queue_handle *h = xQueue;
-    if (h->q) {
-        destroy_q(h->q);
-    }
-    free(xQueue);
-}
-
-QueueHandle_t xSemaphoreCreateBinary(void)
-{
-    QueueHandle_t sempaphore =  xQueueCreate(1, 1);
-    return sempaphore;
-}
-
-QueueHandle_t xSemaphoreCreateMutex(void)
-{
-    QueueHandle_t sempaphore =  xQueueCreate(1, 1);
-    if (sempaphore) {
-        xSemaphoreGive(sempaphore);
-    }
-    return sempaphore;
-}
-
-void vTaskDelete(TaskHandle_t *task)
-{
-    if (task == NULL) {
-        pthread_exit(0);
-    }
-    void *thread_rval = NULL;
-    pthread_join((pthread_t)task, &thread_rval);
-}
-
-TickType_t xTaskGetTickCount( void )
-{
-    struct timespec spec;
-    clock_gettime(CLOCK_REALTIME, &spec);
-    return spec.tv_nsec / 1000000 + spec.tv_sec * 1000;
-}
-
-void vTaskDelay( const TickType_t xTicksToDelay )
-{
-    usleep(xTicksToDelay * 1000);
-}
-
-void *pthread_task(void *params)
-{
-    struct {
-        void *const param;
-        TaskFunction_t task;
-        bool started;
-    } *pthread_params = params;
-
-    void *const param = pthread_params->param;
-    TaskFunction_t task = pthread_params->task;
-    pthread_params->started = true;
-
-    task(param);
-
-    return NULL;
-}
-
-BaseType_t xTaskCreatePinnedToCore( TaskFunction_t pvTaskCode,
-                                    const char *const pcName,
-                                    const uint32_t usStackDepth,
-                                    void *const pvParameters,
-                                    UBaseType_t uxPriority,
-                                    TaskHandle_t *const pvCreatedTask,
-                                    const BaseType_t xCoreID)
-{
-    xTaskCreate(pvTaskCode, pcName, usStackDepth, pvParameters, uxPriority, pvCreatedTask);
-    return pdTRUE;
-}
-
-
-void xTaskCreate(TaskFunction_t pvTaskCode, const char *const pcName, const uint32_t usStackDepth, void *const pvParameters, UBaseType_t uxPriority, TaskHandle_t *const pvCreatedTask)
-{
-    pthread_t new_thread = (pthread_t)NULL;
-    pthread_attr_t attr;
-    struct {
-        void *const param;
-        TaskFunction_t task;
-        bool started;
-    } pthread_params = { .param = pvParameters, .task = pvTaskCode};
-    int res = pthread_attr_init(&attr);
-    assert(res == 0);
-    res = pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE);
-    assert(res == 0);
-    res = pthread_create(&new_thread, &attr, pthread_task, &pthread_params);
-    assert(res == 0);
-
-    if (pvCreatedTask) {
-        *pvCreatedTask = (void *)new_thread;
-    }
-
-    // just wait till the task started so we can unwind params from the stack
-    while (pthread_params.started == false) {
-        usleep(1000);
-    }
-}
-
-void xTaskNotifyGive(TaskHandle_t task)
-{
-
-}
-
-BaseType_t xTaskNotifyWait(uint32_t bits_entry_clear, uint32_t bits_exit_clear, uint32_t *value, TickType_t wait_time )
-{
-    return true;
-}
-
-TaskHandle_t xTaskGetCurrentTaskHandle(void)
-{
-    return NULL;
-}

components/mdns/tests/host_test/components/freertos/queue_unique_ptr.cpp

@@ -1,43 +0,0 @@
-/*
- * SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
- *
- * SPDX-License-Identifier: Apache-2.0
- */
-
-#include "queue_unique_ptr.hpp"
-#include <cstdint>
-#include <vector>
-#include <memory>
-#include <cstring>
-
-extern "C" void *create_q(void)
-{
-    auto *q = new QueueMock<std::vector<uint8_t>>();
-    return q;
-}
-
-extern "C" void destroy_q(void *q)
-{
-    auto *queue = static_cast<QueueMock<std::vector<uint8_t>> *>(q);
-    delete (queue);
-}
-
-extern "C" bool send_q(void *q, uint8_t *data, size_t len)
-{
-    auto v = std::make_unique<std::vector<uint8_t>>(len);
-    v->assign(data, data + len);
-    auto queue = static_cast<QueueMock<std::vector<uint8_t>> *>(q);
-    queue->send(std::move(v));
-    return true;
-}
-
-extern "C" bool recv_q(void *q, uint8_t *data, size_t len, uint32_t ms)
-{
-    auto queue = static_cast<QueueMock<std::vector<uint8_t>> *>(q);
-    auto v = queue->receive(ms);
-    if (v == nullptr) {
-        return false;
-    }
-    memcpy(data, (void *)v->data(), len);
-    return true;
-}

components/mdns/tests/host_test/components/freertos/queue_unique_ptr.hpp

@@ -1,46 +0,0 @@
-/*
- * SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
- *
- * SPDX-License-Identifier: Apache-2.0
- */
-
-#pragma once
-
-#include <queue>
-#include <mutex>
-#include <condition_variable>
-#include <memory>
-#include <thread>
-#include <atomic>
-
-template <class T>
-class QueueMock {
-public:
-    QueueMock(void): q(), m(), c() {}
-    ~QueueMock(void) {}
-
-    void send(std::unique_ptr<T> t)
-    {
-        std::lock_guard<std::mutex> lock(m);
-        q.push(std::move(t));
-        c.notify_one();
-    }
-
-    std::unique_ptr<T> receive(uint32_t ms)
-    {
-        std::unique_lock<std::mutex> lock(m);
-        while (q.empty()) {
-            if (c.wait_for(lock, std::chrono::milliseconds(ms)) == std::cv_status::timeout) {
-                return nullptr;
-            }
-        }
-        std::unique_ptr<T> val = std::move(q.front());
-        q.pop();
-        return val;
-    }
-
-private:
-    std::queue<std::unique_ptr<T>> q;
-    mutable std::mutex m;
-    std::condition_variable c;
-};

examples/mqtt/CMakeLists.txt

@@ -0,0 +1,26 @@
+# This project serves as a demo to enable using esp-mqtt on ESP platform targets as well as on linux
+cmake_minimum_required(VERSION 3.16)
+
+# For ESP32 platform target
+set(EXTRA_COMPONENT_DIRS $ENV{IDF_PATH}/examples/common_components/protocol_examples_common)
+
+if(${IDF_TARGET} STREQUAL "linux")
+# For linux-target we have two options:
+# - With lwIP (must be defined on command line, e.g. idf.py -DWITH_LWIP=1)
+#     access networking from linux `tap` interface (TAP networking mode)
+# - Without lwIP (must be defined on command line, e.g. idf.py -DWITH_LWIP=0)
+#     no designated interface, accesses user network via linux/socket sys calls
+    if(WITH_LWIP STREQUAL 1)
+        set(EXTRA_COMPONENT_DIRS $ENV{IDF_PATH}/examples/common_components/protocol_examples_tapif_io
+                                 "../../common_components/linux_compat/esp_timer")
+        set(COMPONENTS main esp_netif lwip protocol_examples_tapif_io startup esp_hw_support esp_system nvs_flash mqtt esp_timer)
+    else()
+        list(APPEND EXTRA_COMPONENT_DIRS
+                "../../common_components/linux_compat/esp_timer"
+                "$ENV{IDF_PATH}/examples/protocols/linux_stubs/esp_stubs")
+        set(COMPONENTS main nvs_flash esp-tls esp_stubs mqtt protocol_examples_common esp_timer)
+    endif()
+endif()
+
+include($ENV{IDF_PATH}/tools/cmake/project.cmake)
+project(esp_mqtt_demo)

examples/mqtt/README.md

@@ -0,0 +1,41 @@
+# MQTT demo application that runs on linux
+
+## Overview
+
+This is a simple example demonstrating connecting to an MQTT broker, subscribing and publishing some data.
+This example uses IDF build system and could be configured to be build and executed:
+* for any ESP32 family chip
+* for linux target
+
+## How to use example
+
+### Hardware Required
+
+To run this example, you need any ESP32 development board or just PC/virtual machine/container running linux operating system.
+
+### Host build modes
+
+Linux build is supported in these two modes:
+* `WITH_LWIP=0`: Without lwIP component. The project uses linux BSD socket interface to interact with TCP/IP stack. There's no connection phase, we use the host network as users. This mode is often referred to as user-side networking.
+* `WITH_LWIP=1`: Including lwIP component, which is added to the list of required components and compiled on host. In this mode, we have to map the host network (linux TCP/IP stack) to the target network (lwip). We use IDF's [`tapif_io`](https://github.com/espressif/esp-idf/tree/master/examples/common_components/protocol_examples_tapif_io) component to create a network interface, which will be used to pass packets to and from the simulated target. Please refer to the [README](https://github.com/espressif/esp-idf/tree/master/examples/common_components/protocol_examples_tapif_io#readme) for more details about the host side networking.
+
+### Building on linux
+
+1) Configure linux target
+```bash
+idf.py --preview set-target linux
+```
+
+2) Build the project with preferred components (with or without lwip)
+```bash
+idf.py -DWITH_LWIP=0 build  # Building without lwip (user networking)
+idf.py -DWITH_LWIP=1 build  # Building with lwip (TAP networking)
+```
+
+3) Run the project
+
+It is possible to run the project elf file directly, or using `idf.py` monitor target (no need to flash):
+```bash
+idf.py monitor
+```
+idf.py -DWITH_LWIP=0 build  # Building without lwip (user networking)

examples/mqtt/main/CMakeLists.txt

@@ -0,0 +1,4 @@
+idf_component_register(SRCS "app_main.cpp"
+                    INCLUDE_DIRS ".")
+
+target_compile_options(${COMPONENT_LIB} PRIVATE "-Wno-format")

examples/mqtt/main/Kconfig.projbuild

@@ -0,0 +1,9 @@
+menu "Example Configuration"
+
+    config BROKER_URL
+        string "Broker URL"
+        default "mqtt://mqtt.eclipseprojects.io"
+        help
+            URL of the broker to connect to
+
+endmenu

examples/mqtt/main/app_main.cpp

@@ -0,0 +1,136 @@
+/*
+ * SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Unlicense OR CC0-1.0
+ */
+#include <stdio.h>
+#include <stdint.h>
+#include <stddef.h>
+#include <string.h>
+#include "esp_system.h"
+#include "nvs_flash.h"
+#include "esp_event.h"
+#include "esp_netif.h"
+#include "protocol_examples_common.h"
+#include "esp_netif.h"
+#include "esp_system.h"
+
+#include "esp_log.h"
+#include "mqtt_client.h"
+
+static const char *TAG = "esp_mqtt_demo";
+
+
+static void log_error_if_nonzero(const char *message, int error_code)
+{
+    if (error_code != 0) {
+        ESP_LOGE(TAG, "Last error %s: 0x%x", message, error_code);
+    }
+}
+
+/*
+ * @brief Event handler registered to receive MQTT events
+ *
+ *  This function is called by the MQTT client event loop.
+ *
+ * @param handler_args user data registered to the event.
+ * @param base Event base for the handler(always MQTT Base in this example).
+ * @param event_id The id for the received event.
+ * @param event_data The data for the event, esp_mqtt_event_handle_t.
+ */
+static void mqtt_event_handler(void *handler_args, esp_event_base_t base, int32_t event_id, void *event_data)
+{
+    ESP_LOGD(TAG, "Event dispatched from event loop base=%s, event_id=%d", base, event_id);
+    esp_mqtt_event_handle_t event = (esp_mqtt_event_handle_t)event_data;
+    esp_mqtt_client_handle_t client = event->client;
+    int msg_id;
+    switch ((esp_mqtt_event_id_t)event_id) {
+    case MQTT_EVENT_CONNECTED:
+        ESP_LOGI(TAG, "MQTT_EVENT_CONNECTED");
+        msg_id = esp_mqtt_client_publish(client, "/topic/qos1", "data_3", 0, 1, 0);
+        ESP_LOGI(TAG, "sent publish successful, msg_id=%d", msg_id);
+
+        msg_id = esp_mqtt_client_subscribe(client, "/topic/qos0", 0);
+        ESP_LOGI(TAG, "sent subscribe successful, msg_id=%d", msg_id);
+
+        msg_id = esp_mqtt_client_subscribe(client, "/topic/qos1", 1);
+        ESP_LOGI(TAG, "sent subscribe successful, msg_id=%d", msg_id);
+
+        msg_id = esp_mqtt_client_unsubscribe(client, "/topic/qos1");
+        ESP_LOGI(TAG, "sent unsubscribe successful, msg_id=%d", msg_id);
+        break;
+    case MQTT_EVENT_DISCONNECTED:
+        ESP_LOGI(TAG, "MQTT_EVENT_DISCONNECTED");
+        break;
+
+    case MQTT_EVENT_SUBSCRIBED:
+        ESP_LOGI(TAG, "MQTT_EVENT_SUBSCRIBED, msg_id=%d", event->msg_id);
+        msg_id = esp_mqtt_client_publish(client, "/topic/qos0", "data", 0, 0, 0);
+        ESP_LOGI(TAG, "sent publish successful, msg_id=%d", msg_id);
+        break;
+    case MQTT_EVENT_UNSUBSCRIBED:
+        ESP_LOGI(TAG, "MQTT_EVENT_UNSUBSCRIBED, msg_id=%d", event->msg_id);
+        break;
+    case MQTT_EVENT_PUBLISHED:
+        ESP_LOGI(TAG, "MQTT_EVENT_PUBLISHED, msg_id=%d", event->msg_id);
+        break;
+    case MQTT_EVENT_DATA:
+        ESP_LOGI(TAG, "MQTT_EVENT_DATA");
+        printf("TOPIC=%.*s\r\n", event->topic_len, event->topic);
+        printf("DATA=%.*s\r\n", event->data_len, event->data);
+        break;
+    case MQTT_EVENT_ERROR:
+        ESP_LOGI(TAG, "MQTT_EVENT_ERROR");
+        if (event->error_handle->error_type == MQTT_ERROR_TYPE_TCP_TRANSPORT) {
+            log_error_if_nonzero("reported from esp-tls", event->error_handle->esp_tls_last_esp_err);
+            log_error_if_nonzero("reported from tls stack", event->error_handle->esp_tls_stack_err);
+            log_error_if_nonzero("captured as transport's socket errno",  event->error_handle->esp_transport_sock_errno);
+            ESP_LOGI(TAG, "Last errno string (%s)", strerror(event->error_handle->esp_transport_sock_errno));
+
+        }
+        break;
+    default:
+        ESP_LOGI(TAG, "Other event id:%d", event->event_id);
+        break;
+    }
+}
+
+static void mqtt_app_start(void)
+{
+    esp_mqtt_client_config_t mqtt_cfg = {};
+    mqtt_cfg.broker.address.uri = CONFIG_BROKER_URL;
+    mqtt_cfg.credentials.client_id = "idf_on_linux_client";
+
+    esp_mqtt_client_handle_t client = esp_mqtt_client_init(&mqtt_cfg);
+    /* The last argument may be used to pass data to the event handler, in this example mqtt_event_handler */
+    esp_mqtt_client_register_event(client, (esp_mqtt_event_id_t)ESP_EVENT_ANY_ID, mqtt_event_handler, NULL);
+    esp_mqtt_client_start(client);
+}
+
+
+extern "C" void app_main(void)
+{
+    ESP_LOGI(TAG, "[APP] Startup..");
+    ESP_LOGI(TAG, "[APP] Free memory: %d bytes", esp_get_free_heap_size());
+    ESP_LOGI(TAG, "[APP] IDF version: %s", esp_get_idf_version());
+
+    esp_log_level_set("*", ESP_LOG_INFO);
+    esp_log_level_set("mqtt_client", ESP_LOG_VERBOSE);
+    esp_log_level_set("esp_mqtt_demo", ESP_LOG_VERBOSE);
+    esp_log_level_set("transport_base", ESP_LOG_VERBOSE);
+    esp_log_level_set("esp-tls", ESP_LOG_VERBOSE);
+    esp_log_level_set("transport", ESP_LOG_VERBOSE);
+    esp_log_level_set("outbox", ESP_LOG_VERBOSE);
+
+    ESP_ERROR_CHECK(nvs_flash_init());
+    ESP_ERROR_CHECK(esp_netif_init());
+    ESP_ERROR_CHECK(esp_event_loop_create_default());
+
+    /* This helper function configures Wi-Fi or Ethernet, as selected in menuconfig.
+     * Read "Establishing Wi-Fi or Ethernet Connection" section in
+     * examples/protocols/README.md for more information about this function.
+     */
+    ESP_ERROR_CHECK(example_connect());
+
+    mqtt_app_start();
+}

examples/mqtt/sdkconfig.defaults

@@ -0,0 +1,2 @@
+CONFIG_IDF_TARGET="linux"
+# CONFIG_ESP_EVENT_POST_FROM_ISR is not set