Merge branch 'bugfix/remove_unstable_network_tests_v4.3' into 'release/v4.3'

CI: Move mqtt publish tests from regular pipeline to weekend tests (v4.3)

See merge request espressif/esp-idf!14425

components/mdns/mdns.c

@@ -2456,7 +2456,7 @@ static bool _mdns_question_matches(mdns_parsed_question_t * question, uint16_t t
             && !strcasecmp(MDNS_DEFAULT_DOMAIN, question->domain)) {
             return true;
         }
-    } else if (type == MDNS_TYPE_SRV || type == MDNS_TYPE_TXT) {
+    } else if (service && (type == MDNS_TYPE_SRV || type == MDNS_TYPE_TXT)) {
         const char * name = _mdns_get_service_instance_name(service->service);
         if (name && question->host && !strcasecmp(name, question->host)
             && !strcasecmp(service->service->service, question->service)
@@ -2995,7 +2995,7 @@ void mdns_parse_packet(mdns_rx_packet_t * packet)
                     } else if (service) { // only detect txt collision if service existed
                         col = _mdns_check_txt_collision(service->service, data_ptr, data_len);
                     }
-                    if (col && !_mdns_server->interfaces[packet->tcpip_if].pcbs[packet->ip_protocol].probe_running) {
+                    if (col && !_mdns_server->interfaces[packet->tcpip_if].pcbs[packet->ip_protocol].probe_running && service) {
                         do_not_reply = true;
                         _mdns_init_pcb_probe(packet->tcpip_if, packet->ip_protocol, &service, 1, true);
                     } else if (ttl > 2250 && !col && !parsed_packet->authoritative && !parsed_packet->probe && !parsed_packet->questions && !_mdns_server->interfaces[packet->tcpip_if].pcbs[packet->ip_protocol].probe_running) {

components/mdns/test_afl_fuzz_host/esp32_mock.c

@@ -48,12 +48,8 @@ esp_err_t esp_timer_create(const esp_timer_create_args_t* create_args,
 
 uint32_t xTaskGetTickCount(void)
 {
-    struct timeval tv;
+    static uint32_t tick = 0;
-    struct timezone tz;
+    return tick++;
-    if (gettimeofday(&tv, &tz) == 0) {
-        return (tv.tv_sec * 1000) + (tv.tv_usec / 1000);
-    }
-    return 0;
 }
 
 /// Queue mock

components/mdns/test_afl_fuzz_host/sdkconfig.h

@@ -294,7 +294,7 @@
 #define CONFIG_MDNS_TASK_STACK_SIZE 4096
 #define CONFIG_MDNS_TASK_AFFINITY_CPU0 1
 #define CONFIG_MDNS_TASK_AFFINITY 0x0
-#define CONFIG_MDNS_SERVICE_ADD_TIMEOUT_MS 100
+#define CONFIG_MDNS_SERVICE_ADD_TIMEOUT_MS 1
 #define CONFIG_MDNS_TIMER_PERIOD_MS 100
 #define CONFIG_MQTT_PROTOCOL_311 1
 #define CONFIG_MQTT_TRANSPORT_SSL 1

components/mdns/test_afl_fuzz_host/test.c

@@ -204,9 +204,7 @@ int main(int argc, char** argv)
         //
         // Note: parameter1 is a file (mangled packet) which caused the crash
         file = fopen(argv[1], "r");
-        if (file) {
+        assert(file >= 0 );
-            len = fread(buf, 1, 1460, file);
-        }
         fclose(file);
     }
 

components/mqtt/weekend_test/mqtt_publish_test.py

@@ -1,178 +0,0 @@
-from __future__ import print_function, unicode_literals
-
-import random
-import re
-import ssl
-import string
-import sys
-import time
-from builtins import str
-from threading import Event, Thread
-
-import paho.mqtt.client as mqtt
-import ttfw_idf
-from tiny_test_fw import DUT
-
-event_client_connected = Event()
-event_stop_client = Event()
-event_client_received_correct = Event()
-message_log = ''
-broker_host = {}
-broker_port = {}
-expected_data = ''
-subscribe_topic = ''
-publish_topic = ''
-expected_count = 0
-
-
-# The callback for when the client receives a CONNACK response from the server.
-def on_connect(client, userdata, flags, rc):
-    print('Connected with result code ' + str(rc))
-    event_client_connected.set()
-    client.subscribe('/topic/qos0')
-
-
-def mqtt_client_task(client):
-    while not event_stop_client.is_set():
-        client.loop()
-
-
-def get_host_port_from_dut(dut1, config_option):
-    value = re.search(r'\:\/\/([^:]+)\:([0-9]+)', dut1.app.get_sdkconfig()[config_option])
-    if value is None:
-        return None, None
-    return value.group(1), int(value.group(2))
-
-
-# The callback for when a PUBLISH message is received from the server.
-def on_message(client, userdata, msg):
-    global message_log
-    global expected_count
-    payload = msg.payload.decode()
-    if payload == expected_data:
-        expected_count += 1
-        print('[{}] Received...'.format(msg.mid))
-    message_log += 'Received data:' + msg.topic + ' ' + payload + '\n'
-
-
-def test_single_config(dut, transport, qos, repeat, published, queue=0):
-    global expected_count
-    global expected_data
-    global message_log
-    sample_string = ''.join(random.choice(string.ascii_uppercase + string.ascii_lowercase + string.digits) for _ in range(16))
-    event_client_connected.clear()
-    expected_count = 0
-    message_log = ''
-    expected_data = sample_string * repeat
-    print("PUBLISH TEST: transport:{}, qos:{}, sequence:{}, enqueue:{}, sample msg:'{}'".format(transport, qos, published, queue, expected_data))
-    client = None
-    try:
-        if transport in ['ws', 'wss']:
-            client = mqtt.Client(transport='websockets')
-        else:
-            client = mqtt.Client()
-        client.on_connect = on_connect
-        client.on_message = on_message
-        if transport in ['ssl', 'wss']:
-            client.tls_set(None, None, None, cert_reqs=ssl.CERT_NONE, tls_version=ssl.PROTOCOL_TLSv1_2, ciphers=None)
-            client.tls_insecure_set(True)
-        print('Connecting...')
-        client.connect(broker_host[transport], broker_port[transport], 60)
-    except Exception:
-        print('ENV_TEST_FAILURE: Unexpected error while connecting to broker {}: {}:'.format(broker_host[transport], sys.exc_info()[0]))
-        raise
-    # Starting a py-client in a separate thread
-    thread1 = Thread(target=mqtt_client_task, args=(client,))
-    thread1.start()
-    print('Connecting py-client to broker {}:{}...'.format(broker_host[transport], broker_port[transport]))
-    if not event_client_connected.wait(timeout=30):
-        raise ValueError('ENV_TEST_FAILURE: Test script cannot connect to broker: {}'.format(broker_host[transport]))
-    client.subscribe(subscribe_topic, qos)
-    dut.write(' '.join(str(x) for x in (transport, sample_string, repeat, published, qos, queue)), eol='\n')
-    try:
-        # waiting till subscribed to defined topic
-        dut.expect(re.compile(r'MQTT_EVENT_SUBSCRIBED'), timeout=30)
-        for i in range(published):
-            client.publish(publish_topic, sample_string * repeat, qos)
-            print('Publishing...')
-        print('Checking esp-client received msg published from py-client...')
-        dut.expect(re.compile(r'Correct pattern received exactly x times'), timeout=60)
-        start = time.time()
-        while expected_count < published and time.time() - start <= 60:
-            time.sleep(1)
-        # Note: tolerate that messages qos=1 to be received more than once
-        if expected_count == published or (expected_count > published and qos == 1):
-            print('All data received from ESP32...')
-        else:
-            raise ValueError('Not all data received from ESP32: Expected:{}x{}, Received:{}x{}'.format(expected_count, published, expected_data, message_log))
-    finally:
-        event_stop_client.set()
-        thread1.join()
-    client.disconnect()
-    event_stop_client.clear()
-
-
-@ttfw_idf.idf_custom_test(env_tag='Example_WIFI')
-def test_weekend_mqtt_publish(env, extra_data):
-    # Using broker url dictionary for different transport
-    global broker_host
-    global broker_port
-    global publish_topic
-    global subscribe_topic
-    """
-    steps: |
-      1. join AP and connects to ssl broker
-      2. Test connects a client to the same broker
-      3. Test evaluates python client received correct qos0 message
-      4. Test ESP32 client received correct qos0 message
-    """
-    dut1 = env.get_dut('mqtt_publish_connect_test', 'tools/test_apps/protocols/mqtt/publish_connect_test')
-    # Look for host:port in sdkconfig
-    try:
-        # python client subscribes to the topic to which esp client publishes and vice versa
-        publish_topic = dut1.app.get_sdkconfig()['CONFIG_EXAMPLE_SUBSCIBE_TOPIC'].replace('"','')
-        subscribe_topic = dut1.app.get_sdkconfig()['CONFIG_EXAMPLE_PUBLISH_TOPIC'].replace('"','')
-        broker_host['ssl'], broker_port['ssl'] = get_host_port_from_dut(dut1, 'CONFIG_EXAMPLE_BROKER_SSL_URI')
-        broker_host['tcp'], broker_port['tcp'] = get_host_port_from_dut(dut1, 'CONFIG_EXAMPLE_BROKER_TCP_URI')
-        broker_host['ws'], broker_port['ws'] = get_host_port_from_dut(dut1, 'CONFIG_EXAMPLE_BROKER_WS_URI')
-        broker_host['wss'], broker_port['wss'] = get_host_port_from_dut(dut1, 'CONFIG_EXAMPLE_BROKER_WSS_URI')
-    except Exception:
-        print('ENV_TEST_FAILURE: Cannot find broker url in sdkconfig')
-        raise
-    dut1.start_app()
-    try:
-        ip_address = dut1.expect(re.compile(r' IPv4 address: ([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+)'), timeout=30)
-        print('Connected to AP with IP: {}'.format(ip_address))
-    except DUT.ExpectTimeout:
-        print('ENV_TEST_FAILURE: Cannot connect to AP')
-        raise
-    for qos in [0, 1, 2]:
-        for transport in ['tcp', 'ssl', 'ws', 'wss']:
-            for q in [0, 1]:
-                if broker_host[transport] is None:
-                    print('Skipping transport: {}...'.format(transport))
-                    continue
-                # simple test with empty message
-                test_single_config(dut1, transport, qos, 0, 5, q)
-                # decide on broker what level of test will pass (local broker works the best)
-                if broker_host[transport].startswith('192.168') and qos > 0 and q == 0:
-                    # medium size, medium repeated
-                    test_single_config(dut1, transport, qos, 5, 50, q)
-                    # long data
-                    test_single_config(dut1, transport, qos, 1000, 10, q)
-                    # short data, many repeats
-                    test_single_config(dut1, transport, qos, 2, 200, q)
-                elif transport in ['ws', 'wss']:
-                    # more relaxed criteria for websockets!
-                    test_single_config(dut1, transport, qos, 2, 5, q)
-                    test_single_config(dut1, transport, qos, 50, 1, q)
-                    test_single_config(dut1, transport, qos, 10, 20, q)
-                else:
-                    # common configuration should be good for most public mosquittos
-                    test_single_config(dut1, transport, qos, 5, 10, q)
-                    test_single_config(dut1, transport, qos, 500, 3, q)
-                    test_single_config(dut1, transport, qos, 1, 50, q)
-
-
-if __name__ == '__main__':
-    test_weekend_mqtt_publish(dut=ttfw_idf.ESP32QEMUDUT if sys.argv[1:] == ['qemu'] else ttfw_idf.ESP32DUT)

components/mqtt/weekend_test/test_weekend_mqtt_.yml

@@ -1,2 +0,0 @@
-CaseConfig:
-- name: test_weekend_mqtt_publish

tools/ci/config/target-test.yml

@@ -146,28 +146,17 @@
 
 test_weekend_mqtt:
   extends:
-    - .example_test_template
+    - .test_app_template
     - .rules:labels:weekend_test-only
   tags:
     - ESP32
     - Example_WIFI
-  variables:
+  script:
-    ENV_FILE: "$CI_PROJECT_DIR/components/mqtt/weekend_test/env.yml"
+    - export MQTT_PUBLISH_TEST=1
-    TEST_CASE_PATH: "$CI_PROJECT_DIR/components/mqtt/weekend_test"
+    - export TEST_PATH=$CI_PROJECT_DIR/tools/test_apps/protocols/mqtt/publish_connect_test
-    CONFIG_FILE_PATH: "$CI_PROJECT_DIR/components/mqtt/weekend_test"
+    - cd $IDF_PATH/tools/ci/python_packages/tiny_test_fw/bin
+    - run_cmd python Runner.py $TEST_PATH -c $TEST_PATH/publish_connect_mqtt_.yml -e $TEST_PATH/env.yml
 
-test_weekend_network:
-  extends:
-    - .example_test_template
-    - .rules:labels:weekend_test-only
-  image: $CI_DOCKER_REGISTRY/rpi-net-suite$BOT_DOCKER_IMAGE_TAG
-  tags:
-    - ESP32
-    - Example_WIFI
-  variables:
-    ENV_FILE: "$CI_PROJECT_DIR/components/lwip/weekend_test/env.yml"
-    TEST_CASE_PATH: "$CI_PROJECT_DIR/components/lwip/weekend_test"
-    CONFIG_FILE_PATH: "$CI_PROJECT_DIR/components/lwip/weekend_test"
 
 example_test_001A:
   extends: .example_test_template

tools/test_apps/protocols/mqtt/publish_connect_test/app_test.py

@@ -247,7 +247,7 @@ def test_app_protocol_mqtt_publish_connect(env, extra_data):
       2. connect to uri specified in the config
       3. send and receive data
     """
-    dut1 = env.get_dut('mqtt_publish_connect_test', 'tools/test_apps/protocols/mqtt/publish_connect_test', dut_class=ttfw_idf.ESP32DUT)
+    dut1 = env.get_dut('mqtt_publish_connect_test', 'tools/test_apps/protocols/mqtt/publish_connect_test')
     # check and log bin size
     binary_file = os.path.join(dut1.app.binary_path, 'mqtt_publish_connect_test.bin')
     bin_size = os.path.getsize(binary_file)
@@ -352,7 +352,10 @@ def test_app_protocol_mqtt_publish_connect(env, extra_data):
                 raise Exception('Unexpected negotiated protocol {}'.format(s.get_negotiated_protocol()))
 
     #
-    # start publish tests
+    # start publish tests only if enabled in the environment (for weekend tests only)
+    if not os.getenv('MQTT_PUBLISH_TEST'):
+        return
+
     def start_publish_case(transport, qos, repeat, published, queue):
         print('Starting Publish test: transport:{}, qos:{}, nr_of_msgs:{}, msg_size:{}, enqueue:{}'
               .format(transport, qos, published, repeat * DEFAULT_MSG_SIZE, queue))

tools/test_apps/protocols/mqtt/publish_connect_test/publish_connect_mqtt_.yml

@@ -0,0 +1,2 @@
+CaseConfig:
+- name: test_app_protocol_mqtt_publish_connect

tools/test_apps/protocols/mqtt/publish_connect_test/publish_connect_mqtt_qemu.yml

@@ -1,5 +1,5 @@
 CaseConfig:
-- name: test_weekend_mqtt_publish
+- name: test_app_protocol_mqtt_publish_connect
   overwrite:
     dut:
       class: ESP32QEMUDUT

tools/test_apps/protocols/mqtt/publish_connect_test/sdkconfig.qemu

@@ -17,3 +17,7 @@ CONFIG_EXAMPLE_BROKER_CERTIFICATE_OVERRIDE="${EXAMPLE_MQTT_BROKER_CERTIFICATE}"
 CONFIG_MBEDTLS_HARDWARE_AES=n
 CONFIG_MBEDTLS_HARDWARE_MPI=n
 CONFIG_MBEDTLS_HARDWARE_SHA=n
+CONFIG_ETH_USE_SPI_ETHERNET=n
+CONFIG_EXAMPLE_CONNECT_WIFI=n
+CONFIG_ESP_TLS_INSECURE=y
+CONFIG_ESP_TLS_SKIP_SERVER_CERT_VERIFY=y