fix(mqtt5): Fix mqtt5 max inflight message counting

Split message validation path for mqtt5
Introduces tests on the conformance test suite

Fixes #312

test/host/main/CMakeLists.txt

@@ -1,4 +1,4 @@
-idf_component_register(SRCS  "test_mqtt_client.cpp" "test_log_intercept.cpp" "test_log_matchers.cpp" "test_log_parser.cpp"
+idf_component_register(SRCS  "test_mqtt_client.cpp" "test_mqtt5_client.cpp" "mqtt5_client_test_adapter.c" "test_log_intercept.cpp" "test_log_matchers.cpp" "test_log_parser.cpp"
                        REQUIRES cmock mqtt esp_timer esp_hw_support http_parser log
                        WHOLE_ARCHIVE)
 

test/host/main/mqtt5_client_test_adapter.c

@@ -0,0 +1,27 @@
+/*
+ * SPDX-FileCopyrightText: 2026 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+#include <stdint.h>
+
+#include "mqtt_client_priv.h"
+
+esp_err_t test_mqtt5_check_inflight_maximum(uint16_t send_count, uint16_t receive_maximum)
+{
+    struct esp_mqtt_client client = {0};
+    mqtt5_config_storage_t mqtt5_config = {0};
+    client.mqtt5_config = &mqtt5_config;
+    client.mqtt5_config->server_resp_property_info.receive_maximum = receive_maximum;
+    client.send_publish_packet_count = send_count;
+    return esp_mqtt5_client_check_inflight_maximum(&client);
+}
+
+int test_mqtt5_increment_packet_counter_with_dup(void)
+{
+    struct esp_mqtt_client client = {0};
+    uint8_t publish_header[] = {0x3a}; // PUBLISH, DUP=1, QoS=1
+    client.mqtt_state.connection.outbound_message.data = publish_header;
+    esp_mqtt5_increment_packet_counter(&client);
+    return client.send_publish_packet_count;
+}

test/host/main/test_mqtt5_client.cpp

@@ -0,0 +1,25 @@
+/*
+ * SPDX-FileCopyrightText: 2026 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+#include <catch2/catch_test_macros.hpp>
+#include <cstdint>
+
+#include "esp_err.h"
+
+extern "C" {
+    esp_err_t test_mqtt5_check_inflight_maximum(uint16_t send_count, uint16_t receive_maximum);
+    int test_mqtt5_increment_packet_counter_with_dup(void);
+}
+
+TEST_CASE("MQTT5 inflight quota uses an exact upper bound")
+{
+    REQUIRE(test_mqtt5_check_inflight_maximum(1, 2) == ESP_OK);
+    REQUIRE(test_mqtt5_check_inflight_maximum(2, 2) == ESP_FAIL);
+}
+
+TEST_CASE("MQTT5 first send on a connection counts even when PUBLISH has DUP set")
+{
+    REQUIRE(test_mqtt5_increment_packet_counter_with_dup() == 1);
+}

test/host/sdkconfig.defaults

@@ -1,4 +1,5 @@
 CONFIG_IDF_TARGET="linux"
+CONFIG_MQTT_PROTOCOL_5=y
 CONFIG_LOG_DEFAULT_LEVEL_DEBUG=y
 CONFIG_COMPILER_CXX_EXCEPTIONS=y
 CONFIG_COMPILER_CXX_RTTI=y