From f7169ed06312302cadf45260cf58e2584317fc2e Mon Sep 17 00:00:00 2001 From: David Cermak Date: Thu, 10 Jun 2021 09:09:14 +0200 Subject: [PATCH 1/8] mqtt: Moved weekend tests to test apps --- .../mqtt/weekend_test/mqtt_publish_test.py | 178 ------------------ .../mqtt/weekend_test/test_weekend_mqtt_.yml | 2 - .../mqtt/publish_connect_test}/env.yml | 0 .../publish_connect_mqtt_.yml | 2 + .../publish_connect_mqtt_qemu.yml | 2 +- 5 files changed, 3 insertions(+), 181 deletions(-) delete mode 100644 components/mqtt/weekend_test/mqtt_publish_test.py delete mode 100644 components/mqtt/weekend_test/test_weekend_mqtt_.yml rename {components/mqtt/weekend_test => tools/test_apps/protocols/mqtt/publish_connect_test}/env.yml (100%) create mode 100644 tools/test_apps/protocols/mqtt/publish_connect_test/publish_connect_mqtt_.yml rename components/mqtt/weekend_test/test_weekend_mqtt_qemu.yml => tools/test_apps/protocols/mqtt/publish_connect_test/publish_connect_mqtt_qemu.yml (64%) diff --git a/components/mqtt/weekend_test/mqtt_publish_test.py b/components/mqtt/weekend_test/mqtt_publish_test.py deleted file mode 100644 index c9ea51c1a3..0000000000 --- a/components/mqtt/weekend_test/mqtt_publish_test.py +++ /dev/null @@ -1,178 +0,0 @@ -from __future__ import print_function, unicode_literals - -import random -import re -import ssl -import string -import sys -import time -from builtins import str -from threading import Event, Thread - -import paho.mqtt.client as mqtt -import ttfw_idf -from tiny_test_fw import DUT - -event_client_connected = Event() -event_stop_client = Event() -event_client_received_correct = Event() -message_log = '' -broker_host = {} -broker_port = {} -expected_data = '' -subscribe_topic = '' -publish_topic = '' -expected_count = 0 - - -# The callback for when the client receives a CONNACK response from the server. -def on_connect(client, userdata, flags, rc): - print('Connected with result code ' + str(rc)) - event_client_connected.set() - client.subscribe('/topic/qos0') - - -def mqtt_client_task(client): - while not event_stop_client.is_set(): - client.loop() - - -def get_host_port_from_dut(dut1, config_option): - value = re.search(r'\:\/\/([^:]+)\:([0-9]+)', dut1.app.get_sdkconfig()[config_option]) - if value is None: - return None, None - return value.group(1), int(value.group(2)) - - -# The callback for when a PUBLISH message is received from the server. -def on_message(client, userdata, msg): - global message_log - global expected_count - payload = msg.payload.decode() - if payload == expected_data: - expected_count += 1 - print('[{}] Received...'.format(msg.mid)) - message_log += 'Received data:' + msg.topic + ' ' + payload + '\n' - - -def test_single_config(dut, transport, qos, repeat, published, queue=0): - global expected_count - global expected_data - global message_log - sample_string = ''.join(random.choice(string.ascii_uppercase + string.ascii_lowercase + string.digits) for _ in range(16)) - event_client_connected.clear() - expected_count = 0 - message_log = '' - expected_data = sample_string * repeat - print("PUBLISH TEST: transport:{}, qos:{}, sequence:{}, enqueue:{}, sample msg:'{}'".format(transport, qos, published, queue, expected_data)) - client = None - try: - if transport in ['ws', 'wss']: - client = mqtt.Client(transport='websockets') - else: - client = mqtt.Client() - client.on_connect = on_connect - client.on_message = on_message - if transport in ['ssl', 'wss']: - client.tls_set(None, None, None, cert_reqs=ssl.CERT_NONE, tls_version=ssl.PROTOCOL_TLSv1_2, ciphers=None) - client.tls_insecure_set(True) - print('Connecting...') - client.connect(broker_host[transport], broker_port[transport], 60) - except Exception: - print('ENV_TEST_FAILURE: Unexpected error while connecting to broker {}: {}:'.format(broker_host[transport], sys.exc_info()[0])) - raise - # Starting a py-client in a separate thread - thread1 = Thread(target=mqtt_client_task, args=(client,)) - thread1.start() - print('Connecting py-client to broker {}:{}...'.format(broker_host[transport], broker_port[transport])) - if not event_client_connected.wait(timeout=30): - raise ValueError('ENV_TEST_FAILURE: Test script cannot connect to broker: {}'.format(broker_host[transport])) - client.subscribe(subscribe_topic, qos) - dut.write(' '.join(str(x) for x in (transport, sample_string, repeat, published, qos, queue)), eol='\n') - try: - # waiting till subscribed to defined topic - dut.expect(re.compile(r'MQTT_EVENT_SUBSCRIBED'), timeout=30) - for i in range(published): - client.publish(publish_topic, sample_string * repeat, qos) - print('Publishing...') - print('Checking esp-client received msg published from py-client...') - dut.expect(re.compile(r'Correct pattern received exactly x times'), timeout=60) - start = time.time() - while expected_count < published and time.time() - start <= 60: - time.sleep(1) - # Note: tolerate that messages qos=1 to be received more than once - if expected_count == published or (expected_count > published and qos == 1): - print('All data received from ESP32...') - else: - raise ValueError('Not all data received from ESP32: Expected:{}x{}, Received:{}x{}'.format(expected_count, published, expected_data, message_log)) - finally: - event_stop_client.set() - thread1.join() - client.disconnect() - event_stop_client.clear() - - -@ttfw_idf.idf_custom_test(env_tag='Example_WIFI') -def test_weekend_mqtt_publish(env, extra_data): - # Using broker url dictionary for different transport - global broker_host - global broker_port - global publish_topic - global subscribe_topic - """ - steps: | - 1. join AP and connects to ssl broker - 2. Test connects a client to the same broker - 3. Test evaluates python client received correct qos0 message - 4. Test ESP32 client received correct qos0 message - """ - dut1 = env.get_dut('mqtt_publish_connect_test', 'tools/test_apps/protocols/mqtt/publish_connect_test') - # Look for host:port in sdkconfig - try: - # python client subscribes to the topic to which esp client publishes and vice versa - publish_topic = dut1.app.get_sdkconfig()['CONFIG_EXAMPLE_SUBSCIBE_TOPIC'].replace('"','') - subscribe_topic = dut1.app.get_sdkconfig()['CONFIG_EXAMPLE_PUBLISH_TOPIC'].replace('"','') - broker_host['ssl'], broker_port['ssl'] = get_host_port_from_dut(dut1, 'CONFIG_EXAMPLE_BROKER_SSL_URI') - broker_host['tcp'], broker_port['tcp'] = get_host_port_from_dut(dut1, 'CONFIG_EXAMPLE_BROKER_TCP_URI') - broker_host['ws'], broker_port['ws'] = get_host_port_from_dut(dut1, 'CONFIG_EXAMPLE_BROKER_WS_URI') - broker_host['wss'], broker_port['wss'] = get_host_port_from_dut(dut1, 'CONFIG_EXAMPLE_BROKER_WSS_URI') - except Exception: - print('ENV_TEST_FAILURE: Cannot find broker url in sdkconfig') - raise - dut1.start_app() - try: - ip_address = dut1.expect(re.compile(r' IPv4 address: ([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+)'), timeout=30) - print('Connected to AP with IP: {}'.format(ip_address)) - except DUT.ExpectTimeout: - print('ENV_TEST_FAILURE: Cannot connect to AP') - raise - for qos in [0, 1, 2]: - for transport in ['tcp', 'ssl', 'ws', 'wss']: - for q in [0, 1]: - if broker_host[transport] is None: - print('Skipping transport: {}...'.format(transport)) - continue - # simple test with empty message - test_single_config(dut1, transport, qos, 0, 5, q) - # decide on broker what level of test will pass (local broker works the best) - if broker_host[transport].startswith('192.168') and qos > 0 and q == 0: - # medium size, medium repeated - test_single_config(dut1, transport, qos, 5, 50, q) - # long data - test_single_config(dut1, transport, qos, 1000, 10, q) - # short data, many repeats - test_single_config(dut1, transport, qos, 2, 200, q) - elif transport in ['ws', 'wss']: - # more relaxed criteria for websockets! - test_single_config(dut1, transport, qos, 2, 5, q) - test_single_config(dut1, transport, qos, 50, 1, q) - test_single_config(dut1, transport, qos, 10, 20, q) - else: - # common configuration should be good for most public mosquittos - test_single_config(dut1, transport, qos, 5, 10, q) - test_single_config(dut1, transport, qos, 500, 3, q) - test_single_config(dut1, transport, qos, 1, 50, q) - - -if __name__ == '__main__': - test_weekend_mqtt_publish(dut=ttfw_idf.ESP32QEMUDUT if sys.argv[1:] == ['qemu'] else ttfw_idf.ESP32DUT) diff --git a/components/mqtt/weekend_test/test_weekend_mqtt_.yml b/components/mqtt/weekend_test/test_weekend_mqtt_.yml deleted file mode 100644 index ae00e3524d..0000000000 --- a/components/mqtt/weekend_test/test_weekend_mqtt_.yml +++ /dev/null @@ -1,2 +0,0 @@ -CaseConfig: -- name: test_weekend_mqtt_publish diff --git a/components/mqtt/weekend_test/env.yml b/tools/test_apps/protocols/mqtt/publish_connect_test/env.yml similarity index 100% rename from components/mqtt/weekend_test/env.yml rename to tools/test_apps/protocols/mqtt/publish_connect_test/env.yml diff --git a/tools/test_apps/protocols/mqtt/publish_connect_test/publish_connect_mqtt_.yml b/tools/test_apps/protocols/mqtt/publish_connect_test/publish_connect_mqtt_.yml new file mode 100644 index 0000000000..70e13ef44e --- /dev/null +++ b/tools/test_apps/protocols/mqtt/publish_connect_test/publish_connect_mqtt_.yml @@ -0,0 +1,2 @@ +CaseConfig: +- name: test_app_protocol_mqtt_publish_connect diff --git a/components/mqtt/weekend_test/test_weekend_mqtt_qemu.yml b/tools/test_apps/protocols/mqtt/publish_connect_test/publish_connect_mqtt_qemu.yml similarity index 64% rename from components/mqtt/weekend_test/test_weekend_mqtt_qemu.yml rename to tools/test_apps/protocols/mqtt/publish_connect_test/publish_connect_mqtt_qemu.yml index bd6f0d66af..06f1235238 100644 --- a/components/mqtt/weekend_test/test_weekend_mqtt_qemu.yml +++ b/tools/test_apps/protocols/mqtt/publish_connect_test/publish_connect_mqtt_qemu.yml @@ -1,5 +1,5 @@ CaseConfig: -- name: test_weekend_mqtt_publish +- name: test_app_protocol_mqtt_publish_connect overwrite: dut: class: ESP32QEMUDUT From 6adaf783aa634cce8c3059cdefcbe6875fa0cfb2 Mon Sep 17 00:00:00 2001 From: David Cermak Date: Thu, 10 Jun 2021 09:10:16 +0200 Subject: [PATCH 2/8] CI/mqtt: Execute mqtt weekend tests from test apps --- tools/ci/config/target-test.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/ci/config/target-test.yml b/tools/ci/config/target-test.yml index 0404ef24b9..ca5999dea7 100644 --- a/tools/ci/config/target-test.yml +++ b/tools/ci/config/target-test.yml @@ -146,15 +146,15 @@ test_weekend_mqtt: extends: - - .example_test_template + - .test_app_template - .rules:labels:weekend_test-only tags: - ESP32 - Example_WIFI variables: - ENV_FILE: "$CI_PROJECT_DIR/components/mqtt/weekend_test/env.yml" - TEST_CASE_PATH: "$CI_PROJECT_DIR/components/mqtt/weekend_test" - CONFIG_FILE_PATH: "$CI_PROJECT_DIR/components/mqtt/weekend_test" + ENV_FILE: "$CI_PROJECT_DIR/tools/test_apps/protocols/mqtt/publish_connect_test/env.yml" + TEST_CASE_PATH: "$CI_PROJECT_DIR/tools/test_apps/protocols/mqtt/publish_connect_test" + CONFIG_FILE_PATH: "$CI_PROJECT_DIR/tools/test_apps/protocols/mqtt/publish_connect_test" test_weekend_network: extends: From 4373a892370f08d016da0ce4da4db39bfc438566 Mon Sep 17 00:00:00 2001 From: David Cermak Date: Thu, 10 Jun 2021 09:11:40 +0200 Subject: [PATCH 3/8] CI: Removed weekend network tests Temporarily, before getting reworked and stable for running in the CI --- tools/ci/config/target-test.yml | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/tools/ci/config/target-test.yml b/tools/ci/config/target-test.yml index ca5999dea7..325af0b144 100644 --- a/tools/ci/config/target-test.yml +++ b/tools/ci/config/target-test.yml @@ -156,18 +156,6 @@ test_weekend_mqtt: TEST_CASE_PATH: "$CI_PROJECT_DIR/tools/test_apps/protocols/mqtt/publish_connect_test" CONFIG_FILE_PATH: "$CI_PROJECT_DIR/tools/test_apps/protocols/mqtt/publish_connect_test" -test_weekend_network: - extends: - - .example_test_template - - .rules:labels:weekend_test-only - image: $CI_DOCKER_REGISTRY/rpi-net-suite$BOT_DOCKER_IMAGE_TAG - tags: - - ESP32 - - Example_WIFI - variables: - ENV_FILE: "$CI_PROJECT_DIR/components/lwip/weekend_test/env.yml" - TEST_CASE_PATH: "$CI_PROJECT_DIR/components/lwip/weekend_test" - CONFIG_FILE_PATH: "$CI_PROJECT_DIR/components/lwip/weekend_test" example_test_001A: extends: .example_test_template From 91c93611e1faa40fc6722462eb18379804e3add4 Mon Sep 17 00:00:00 2001 From: David Cermak Date: Thu, 10 Jun 2021 16:46:12 +0200 Subject: [PATCH 4/8] CI: Enable publish tests only when started from weekend pipeline --- tools/ci/config/target-test.yml | 9 +++++---- .../protocols/mqtt/publish_connect_test/app_test.py | 5 ++++- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/tools/ci/config/target-test.yml b/tools/ci/config/target-test.yml index 325af0b144..f299339163 100644 --- a/tools/ci/config/target-test.yml +++ b/tools/ci/config/target-test.yml @@ -151,10 +151,11 @@ test_weekend_mqtt: tags: - ESP32 - Example_WIFI - variables: - ENV_FILE: "$CI_PROJECT_DIR/tools/test_apps/protocols/mqtt/publish_connect_test/env.yml" - TEST_CASE_PATH: "$CI_PROJECT_DIR/tools/test_apps/protocols/mqtt/publish_connect_test" - CONFIG_FILE_PATH: "$CI_PROJECT_DIR/tools/test_apps/protocols/mqtt/publish_connect_test" + script: + - export MQTT_PUBLISH_TEST=1 + - export TEST_PATH=$CI_PROJECT_DIR/tools/test_apps/protocols/mqtt/publish_connect_test + - cd $IDF_PATH/tools/ci/python_packages/tiny_test_fw/bin + - run_cmd python Runner.py $TEST_PATH -c $TEST_PATH/publish_connect_mqtt_.yml -e $TEST_PATH/env.yml example_test_001A: diff --git a/tools/test_apps/protocols/mqtt/publish_connect_test/app_test.py b/tools/test_apps/protocols/mqtt/publish_connect_test/app_test.py index 2cb19f58f0..fb5374762c 100644 --- a/tools/test_apps/protocols/mqtt/publish_connect_test/app_test.py +++ b/tools/test_apps/protocols/mqtt/publish_connect_test/app_test.py @@ -352,7 +352,10 @@ def test_app_protocol_mqtt_publish_connect(env, extra_data): raise Exception('Unexpected negotiated protocol {}'.format(s.get_negotiated_protocol())) # - # start publish tests + # start publish tests only if enabled in the environment (for weekend tests only) + if not os.getenv('MQTT_PUBLISH_TEST'): + return + def start_publish_case(transport, qos, repeat, published, queue): print('Starting Publish test: transport:{}, qos:{}, nr_of_msgs:{}, msg_size:{}, enqueue:{}' .format(transport, qos, published, repeat * DEFAULT_MSG_SIZE, queue)) From 0d3c3a9173194657601c48b5d477d25215e96a9e Mon Sep 17 00:00:00 2001 From: David Cermak Date: Tue, 8 Jun 2021 13:27:15 +0200 Subject: [PATCH 5/8] mdns: Fix fuzzer from miss-interpreting adding services as timeouts --- components/mdns/test_afl_fuzz_host/esp32_mock.c | 8 ++------ components/mdns/test_afl_fuzz_host/sdkconfig.h | 2 +- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/components/mdns/test_afl_fuzz_host/esp32_mock.c b/components/mdns/test_afl_fuzz_host/esp32_mock.c index 89efa55fb6..5c26fe8d8e 100644 --- a/components/mdns/test_afl_fuzz_host/esp32_mock.c +++ b/components/mdns/test_afl_fuzz_host/esp32_mock.c @@ -48,12 +48,8 @@ esp_err_t esp_timer_create(const esp_timer_create_args_t* create_args, uint32_t xTaskGetTickCount(void) { - struct timeval tv; - struct timezone tz; - if (gettimeofday(&tv, &tz) == 0) { - return (tv.tv_sec * 1000) + (tv.tv_usec / 1000); - } - return 0; + static uint32_t tick = 0; + return tick++; } /// Queue mock diff --git a/components/mdns/test_afl_fuzz_host/sdkconfig.h b/components/mdns/test_afl_fuzz_host/sdkconfig.h index 43d4a278a5..1b5997b648 100644 --- a/components/mdns/test_afl_fuzz_host/sdkconfig.h +++ b/components/mdns/test_afl_fuzz_host/sdkconfig.h @@ -294,7 +294,7 @@ #define CONFIG_MDNS_TASK_STACK_SIZE 4096 #define CONFIG_MDNS_TASK_AFFINITY_CPU0 1 #define CONFIG_MDNS_TASK_AFFINITY 0x0 -#define CONFIG_MDNS_SERVICE_ADD_TIMEOUT_MS 100 +#define CONFIG_MDNS_SERVICE_ADD_TIMEOUT_MS 1 #define CONFIG_MDNS_TIMER_PERIOD_MS 100 #define CONFIG_MQTT_PROTOCOL_311 1 #define CONFIG_MQTT_TRANSPORT_SSL 1 From 355e152082e34d44e246eae94ce7ca3f5e13d0b1 Mon Sep 17 00:00:00 2001 From: David Cermak Date: Wed, 9 Jun 2021 09:58:58 +0200 Subject: [PATCH 6/8] mdns: Minor correction of the test code --- components/mdns/test_afl_fuzz_host/test.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/components/mdns/test_afl_fuzz_host/test.c b/components/mdns/test_afl_fuzz_host/test.c index 0c123f6fae..776a089cd9 100644 --- a/components/mdns/test_afl_fuzz_host/test.c +++ b/components/mdns/test_afl_fuzz_host/test.c @@ -204,9 +204,7 @@ int main(int argc, char** argv) // // Note: parameter1 is a file (mangled packet) which caused the crash file = fopen(argv[1], "r"); - if (file) { - len = fread(buf, 1, 1460, file); - } + assert(file >= 0 ); fclose(file); } From de30298223b7d2dd4360160e57e831b1440ff52d Mon Sep 17 00:00:00 2001 From: David Cermak Date: Wed, 9 Jun 2021 09:59:25 +0200 Subject: [PATCH 7/8] mdns: Fix crashes reported by the fuzzer --- components/mdns/mdns.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/mdns/mdns.c b/components/mdns/mdns.c index 2dcbfa3c77..e4004fd780 100644 --- a/components/mdns/mdns.c +++ b/components/mdns/mdns.c @@ -2456,7 +2456,7 @@ static bool _mdns_question_matches(mdns_parsed_question_t * question, uint16_t t && !strcasecmp(MDNS_DEFAULT_DOMAIN, question->domain)) { return true; } - } else if (type == MDNS_TYPE_SRV || type == MDNS_TYPE_TXT) { + } else if (service && (type == MDNS_TYPE_SRV || type == MDNS_TYPE_TXT)) { const char * name = _mdns_get_service_instance_name(service->service); if (name && question->host && !strcasecmp(name, question->host) && !strcasecmp(service->service->service, question->service) @@ -2995,7 +2995,7 @@ void mdns_parse_packet(mdns_rx_packet_t * packet) } else if (service) { // only detect txt collision if service existed col = _mdns_check_txt_collision(service->service, data_ptr, data_len); } - if (col && !_mdns_server->interfaces[packet->tcpip_if].pcbs[packet->ip_protocol].probe_running) { + if (col && !_mdns_server->interfaces[packet->tcpip_if].pcbs[packet->ip_protocol].probe_running && service) { do_not_reply = true; _mdns_init_pcb_probe(packet->tcpip_if, packet->ip_protocol, &service, 1, true); } else if (ttl > 2250 && !col && !parsed_packet->authoritative && !parsed_packet->probe && !parsed_packet->questions && !_mdns_server->interfaces[packet->tcpip_if].pcbs[packet->ip_protocol].probe_running) { From 965423d532f099b1ec290484c66b93f42830d95c Mon Sep 17 00:00:00 2001 From: David Cermak Date: Wed, 16 Jun 2021 09:58:20 +0200 Subject: [PATCH 8/8] CI: Prepare mqtt app test for QEMU tests Adds qemu configuration Generalize the get_dut() to enable choosing DUT class per configuration --- .../test_apps/protocols/mqtt/publish_connect_test/app_test.py | 2 +- .../protocols/mqtt/publish_connect_test/sdkconfig.qemu | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/tools/test_apps/protocols/mqtt/publish_connect_test/app_test.py b/tools/test_apps/protocols/mqtt/publish_connect_test/app_test.py index fb5374762c..8f0bd95c26 100644 --- a/tools/test_apps/protocols/mqtt/publish_connect_test/app_test.py +++ b/tools/test_apps/protocols/mqtt/publish_connect_test/app_test.py @@ -247,7 +247,7 @@ def test_app_protocol_mqtt_publish_connect(env, extra_data): 2. connect to uri specified in the config 3. send and receive data """ - dut1 = env.get_dut('mqtt_publish_connect_test', 'tools/test_apps/protocols/mqtt/publish_connect_test', dut_class=ttfw_idf.ESP32DUT) + dut1 = env.get_dut('mqtt_publish_connect_test', 'tools/test_apps/protocols/mqtt/publish_connect_test') # check and log bin size binary_file = os.path.join(dut1.app.binary_path, 'mqtt_publish_connect_test.bin') bin_size = os.path.getsize(binary_file) diff --git a/tools/test_apps/protocols/mqtt/publish_connect_test/sdkconfig.qemu b/tools/test_apps/protocols/mqtt/publish_connect_test/sdkconfig.qemu index e0d289d74e..62d7f5b7e3 100644 --- a/tools/test_apps/protocols/mqtt/publish_connect_test/sdkconfig.qemu +++ b/tools/test_apps/protocols/mqtt/publish_connect_test/sdkconfig.qemu @@ -17,3 +17,7 @@ CONFIG_EXAMPLE_BROKER_CERTIFICATE_OVERRIDE="${EXAMPLE_MQTT_BROKER_CERTIFICATE}" CONFIG_MBEDTLS_HARDWARE_AES=n CONFIG_MBEDTLS_HARDWARE_MPI=n CONFIG_MBEDTLS_HARDWARE_SHA=n +CONFIG_ETH_USE_SPI_ETHERNET=n +CONFIG_EXAMPLE_CONNECT_WIFI=n +CONFIG_ESP_TLS_INSECURE=y +CONFIG_ESP_TLS_SKIP_SERVER_CERT_VERIFY=y