Merge branch 'test/add_case_to_detect_ds2ds_issue_v3.1' into 'release/v3.1'

test: add test case to receive ds2ds packet (backport v3.1)

See merge request idf/esp-idf!3850

.gitlab-ci.yml

@@ -1224,6 +1224,38 @@ UT_010_04:
     - UT_T1_RMT
     - psram
 
+UT_017_01:
+  <<: *unit_test_template
+  tags:
+    - ESP32_IDF
+    - UT_T2_1
+
+UT_017_02:
+  <<: *unit_test_template
+  tags:
+    - ESP32_IDF
+    - UT_T2_1
+
+UT_017_03:
+  <<: *unit_test_template
+  tags:
+    - ESP32_IDF
+    - UT_T2_1
+
+UT_017_04:
+  <<: *unit_test_template
+  tags:
+    - ESP32_IDF
+    - UT_T2_1
+    - psram
+
+UT_017_05:
+  <<: *unit_test_template
+  tags:
+    - ESP32_IDF
+    - UT_T2_1
+    - 8Mpsram
+
 UT_601_01:
   <<: *unit_test_template
   tags:

components/driver/test/test_adc2.c

@@ -10,6 +10,7 @@
 #include "esp_wifi.h"
 #include "esp_log.h"
 #include "nvs_flash.h"
+#include "test_utils.h"
 
 static const char* TAG = "test_adc2";
 
@@ -44,7 +45,9 @@ TEST_CASE("adc2 work with wifi","[adc]")
 {
     int     read_raw;
     int     target_value;
-    
+
+    test_case_uses_tcpip();
+
     //adc and dac init
     TEST_ESP_OK( dac_output_enable( DAC_CHANNEL_1 ));
     TEST_ESP_OK( dac_output_enable( DAC_CHANNEL_2 ));

components/esp32/test/test_wifi.c

@@ -9,6 +9,8 @@
 #include "esp_wifi.h"
 #include "esp_log.h"
 #include "nvs_flash.h"
+#include "test_utils.h"
+#include "freertos/task.h"
 
 static const char* TAG = "test_wifi";
 
@@ -79,7 +81,9 @@ static void test_wifi_start_stop(wifi_init_config_t *cfg, wifi_config_t* wifi_co
 }
 
 TEST_CASE("wifi stop and deinit","[wifi]")
-{    
+{
+    test_case_uses_tcpip();
+
     wifi_init_config_t cfg = WIFI_INIT_CONFIG_DEFAULT();
     wifi_config_t wifi_config = {
         .sta = {
@@ -118,3 +122,68 @@ TEST_CASE("wifi stop and deinit","[wifi]")
 
     TEST_IGNORE_MESSAGE("this test case is ignored due to the critical memory leak of tcpip_adapter and event_loop.");
 }
+
+static void start_wifi_as_softap(void)
+{
+    wifi_init_config_t cfg = WIFI_INIT_CONFIG_DEFAULT();
+    cfg.nvs_enable = false;
+
+    wifi_config_t w_config = {
+        .ap.ssid = "default_ssid",
+        .ap.password = "default_password",
+        .ap.ssid_len = 0,
+        .ap.channel = 1,
+        .ap.authmode = WIFI_AUTH_WPA2_PSK,
+        .ap.ssid_hidden = false,
+        .ap.max_connection = 4,
+        .ap.beacon_interval = 100,
+    };
+
+    TEST_ESP_OK(esp_wifi_init(&cfg));
+    TEST_ESP_OK(esp_wifi_set_mode(WIFI_MODE_AP));
+    TEST_ESP_OK(esp_wifi_set_config(WIFI_IF_AP, &w_config));
+    TEST_ESP_OK(esp_wifi_start());
+
+}
+
+static void stop_wifi(void)
+{
+    TEST_ESP_OK(esp_wifi_stop());
+    TEST_ESP_OK(esp_wifi_deinit());
+}
+
+static void receive_ds2ds_packet(void)
+{
+    start_wifi_as_softap();
+    unity_wait_for_signal("sender ready");
+    unity_send_signal("receiver ready");
+
+    // wait for sender to send packets
+    vTaskDelay(1000/portTICK_PERIOD_MS);
+    stop_wifi();
+
+}
+
+static const char ds2ds_pdu[] = {
+    0x48, 0x03, 0x00, 0x00, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
+    0xE8, 0x65, 0xD4, 0xCB, 0x74, 0x19, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
+    0x60, 0x94, 0xE8, 0x65, 0xD4, 0xCB, 0x74, 0x1C, 0x26, 0xB9,
+    0x0D, 0x02, 0x7D, 0x13, 0x00, 0x00, 0x01, 0xE8, 0x65, 0xD4, 0xCB, 0x74,
+    0x1C, 0x00, 0x00, 0x26, 0xB9, 0x00, 0x00, 0x00, 0x00
+};
+
+static void send_ds2ds_packet(void)
+{
+    start_wifi_as_softap();
+    unity_send_signal("sender ready");
+    unity_wait_for_signal("receiver ready");
+
+    // send packet 20 times to make sure receiver will get this packet
+    for (uint16_t i = 0; i < 20; i++) {
+        esp_wifi_80211_tx(ESP_IF_WIFI_AP, ds2ds_pdu, sizeof(ds2ds_pdu), true);
+        vTaskDelay(50 / portTICK_PERIOD_MS);
+    }
+    stop_wifi();
+}
+
+TEST_CASE_MULTIPLE_DEVICES("receive ds2ds packet without exception", "[wifi][test_env=UT_T2_1]", receive_ds2ds_packet, send_ds2ds_packet);

components/newlib/test/test_time.c

@@ -133,12 +133,10 @@ TEST_CASE("test adjtime function", "[newlib]")
 static volatile bool exit_flag;
 static bool adjtime_test_result;
 static bool gettimeofday_test_result;
-static uint64_t count_adjtime;
-static uint64_t count_settimeofday;
-static uint64_t count_gettimeofday;
 
 static void adjtimeTask2(void *pvParameters)
 {
+    xSemaphoreHandle *sema = (xSemaphoreHandle *) pvParameters;
     struct timeval delta = {.tv_sec = 0, .tv_usec = 0};
     struct timeval outdelta;
 
@@ -148,34 +146,23 @@ static void adjtimeTask2(void *pvParameters)
         delta.tv_usec = 900000;
         if (delta.tv_sec >= 2146) delta.tv_sec = 1;
         adjtime(&delta, &outdelta);
-        count_adjtime++;
     }
+    xSemaphoreGive(*sema);
     vTaskDelete(NULL);
 }
 
-static void settimeofdayTask2(void *pvParameters)
+static void timeTask(void *pvParameters)
 {
+    xSemaphoreHandle *sema = (xSemaphoreHandle *) pvParameters;
     struct timeval tv_time = { .tv_sec = 1520000000, .tv_usec = 900000 };
 
     // although exit flag is set in another task, checking (exit_flag == false) is safe
     while (exit_flag == false) {
         tv_time.tv_sec += 1;
         settimeofday(&tv_time, NULL);
-        count_settimeofday++;
-        vTaskDelay(1);
-    }
-    vTaskDelete(NULL);
-}
-
-static void gettimeofdayTask2(void *pvParameters)
-{
-    struct timeval tv_time;
-    // although exit flag is set in another task, checking (exit_flag == false) is safe
-    while (exit_flag == false) {
         gettimeofday(&tv_time, NULL);
-        count_gettimeofday++;
-        vTaskDelay(1);
     }
+    xSemaphoreGive(*sema);
     vTaskDelete(NULL);
 }
 
@@ -183,32 +170,37 @@ TEST_CASE("test for no interlocking adjtime, gettimeofday and settimeofday funct
 {
     TaskHandle_t th[4];
     exit_flag = false;
-    count_adjtime = 0;
-    count_settimeofday = 0;
-    count_gettimeofday = 0;
     struct timeval tv_time = { .tv_sec = 1520000000, .tv_usec = 900000 };
     TEST_ASSERT_EQUAL(settimeofday(&tv_time, NULL), 0);
 
+    const int max_tasks = 2;
+    xSemaphoreHandle exit_sema[max_tasks];
+
+    for (int i = 0; i < max_tasks; ++i) {
+        exit_sema[i] = xSemaphoreCreateBinary();
+    }
+
 #ifndef CONFIG_FREERTOS_UNICORE
     printf("CPU0 and CPU1. Tasks run: 1 - adjtimeTask, 2 - gettimeofdayTask, 3 - settimeofdayTask \n");
-    xTaskCreatePinnedToCore(adjtimeTask2, "adjtimeTask1", 2048, NULL, UNITY_FREERTOS_PRIORITY - 1, &th[0], 0);
+    xTaskCreatePinnedToCore(adjtimeTask2, "adjtimeTask2", 2048, &exit_sema[0], UNITY_FREERTOS_PRIORITY - 1, &th[0], 0);
-    xTaskCreatePinnedToCore(gettimeofdayTask2, "gettimeofdayTask1", 2048, NULL, UNITY_FREERTOS_PRIORITY - 1, &th[1], 1);
+    xTaskCreatePinnedToCore(timeTask, "timeTask", 2048, &exit_sema[1], UNITY_FREERTOS_PRIORITY - 1, &th[1], 1);
-    xTaskCreatePinnedToCore(settimeofdayTask2, "settimeofdayTask1", 2048, NULL, UNITY_FREERTOS_PRIORITY - 1, &th[2], 0);
 #else
     printf("Only one CPU. Tasks run: 1 - adjtimeTask, 2 - gettimeofdayTask, 3 - settimeofdayTask\n");
-    xTaskCreate(adjtimeTask2, "adjtimeTask1", 2048, NULL, UNITY_FREERTOS_PRIORITY - 1, &th[0]);
+    xTaskCreate(adjtimeTask2, "adjtimeTask2", 2048, &exit_sema[0], UNITY_FREERTOS_PRIORITY - 1, &th[0]);
-    xTaskCreate(gettimeofdayTask2, "gettimeofdayTask1", 2048, NULL, UNITY_FREERTOS_PRIORITY - 1, &th[1]);
+    xTaskCreate(timeTask, "timeTask", 2048, &exit_sema[1], UNITY_FREERTOS_PRIORITY - 1, &th[1]);
-    xTaskCreate(settimeofdayTask2, "settimeofdayTask1", 2048, NULL, UNITY_FREERTOS_PRIORITY - 1, &th[2]);
 #endif
 
-    printf("start wait for 10 seconds\n");
+    printf("start wait for 5 seconds\n");
-    vTaskDelay(10000 / portTICK_PERIOD_MS);
+    vTaskDelay(5000 / portTICK_PERIOD_MS);
 
     // set exit flag to let thread exit
     exit_flag = true;
-    vTaskDelay(20 / portTICK_PERIOD_MS);
+    for (int i = 0; i < max_tasks; ++i) {
-    printf("count_adjtime %lld, count_settimeofday %lld, count_gettimeofday %lld\n", count_adjtime, count_settimeofday, count_gettimeofday);
+        if (!xSemaphoreTake(exit_sema[i], 2000/portTICK_PERIOD_MS)) {
-    TEST_ASSERT(count_adjtime > 1000LL && count_settimeofday > 1000LL && count_gettimeofday > 1000LL);
+            TEST_FAIL_MESSAGE("exit_sema not released by test task");
+        }
+        vSemaphoreDelete(exit_sema[i]);
+    }
 }
 
 static void adjtimeTask(void *pvParameters)
@@ -268,20 +260,15 @@ TEST_CASE("test for thread safety adjtime and gettimeofday functions", "[newlib]
     printf("CPU0 and CPU1. Tasks run: 1 - adjtimeTask, 2 - gettimeofdayTask\n");
     xTaskCreatePinnedToCore(adjtimeTask, "adjtimeTask1", 2048, NULL, UNITY_FREERTOS_PRIORITY - 1, &th[0], 0);
     xTaskCreatePinnedToCore(gettimeofdayTask, "gettimeofdayTask1", 2048, NULL, UNITY_FREERTOS_PRIORITY - 1, &th[1], 1);
-
+    xTaskCreatePinnedToCore(gettimeofdayTask, "gettimeofdayTask2", 2048, NULL, UNITY_FREERTOS_PRIORITY - 1, &th[2], 0);
-    xTaskCreatePinnedToCore(adjtimeTask, "adjtimeTask2", 2048, NULL, UNITY_FREERTOS_PRIORITY - 1, &th[2], 0);
-    xTaskCreatePinnedToCore(gettimeofdayTask, "gettimeofdayTask2", 2048, NULL, UNITY_FREERTOS_PRIORITY - 1, &th[3], 1);
 #else
     printf("Only one CPU. Tasks run: 1 - adjtimeTask, 2 - gettimeofdayTask\n");
     xTaskCreate(adjtimeTask, "adjtimeTask1", 2048, NULL, UNITY_FREERTOS_PRIORITY - 1, &th[0]);
     xTaskCreate(gettimeofdayTask, "gettimeofdayTask1", 2048, NULL, UNITY_FREERTOS_PRIORITY - 1, &th[1]);
-
-    xTaskCreate(adjtimeTask, "adjtimeTask2", 2048, NULL, UNITY_FREERTOS_PRIORITY - 1, &th[2]);
-    xTaskCreate(gettimeofdayTask, "gettimeofdayTask2", 2048, NULL, UNITY_FREERTOS_PRIORITY - 1, &th[3]);
 #endif
 
-    printf("start wait for 10 seconds\n");
+    printf("start wait for 5 seconds\n");
-    vTaskDelay(10000 / portTICK_PERIOD_MS);
+    vTaskDelay(5000 / portTICK_PERIOD_MS);
 
     // set exit flag to let thread exit
     exit_flag = true;

components/spiffs/test/test_spiffs.c

@@ -594,7 +594,7 @@ TEST_CASE("opendir, readdir, rewinddir, seekdir work as expected", "[spiffs]")
     test_teardown();
 }
 
-TEST_CASE("readdir with large number of files", "[spiffs][timeout=15]")
+TEST_CASE("readdir with large number of files", "[spiffs][timeout=30]")
 {
     test_setup();
     test_spiffs_readdir_many_files("/spiffs/dir2");

components/vfs/test/test_vfs_select.c

@@ -24,6 +24,7 @@
 #include "esp_vfs_dev.h"
 #include "lwip/sockets.h"
 #include "lwip/netdb.h"
+#include "test_utils.h"
 
 typedef struct {
     int fd;
@@ -114,6 +115,8 @@ static inline void start_task(const test_task_param_t *test_task_param)
 
 static void init(int *uart_fd, int *socket_fd)
 {
+    test_case_uses_tcpip();
+
     uart1_init();
     UART1.conf0.loopback = 1;
 
@@ -296,9 +299,10 @@ TEST_CASE("concurent selects work", "[vfs]")
     };
 
     int uart_fd, socket_fd;
-    const int dummy_socket_fd = open_dummy_socket();
     init(&uart_fd, &socket_fd);
 
+    const int dummy_socket_fd = open_dummy_socket();
+
     fd_set rfds;
     FD_ZERO(&rfds);
     FD_SET(uart_fd, &rfds);

tools/tiny-test-fw/DUT.py

@@ -623,7 +623,10 @@ class BaseDUT(object):
                 data = self.data_cache.get_data(time_remaining)
 
         if match_succeed:
-            # do callback and flush matched data cache
+            # sort matched items according to order of appearance in the input data,
+            # so that the callbacks are invoked in correct order
+            matched_expect_items = sorted(matched_expect_items, key=lambda it: it["index"])
+            # invoke callbacks and flush matched data cache
             slice_index = -1
             for expect_item in matched_expect_items:
                 # trigger callback

tools/unit-test-app/components/unity/include/test_utils.h

@@ -43,6 +43,32 @@ void ref_clock_deinit();
  */
 uint64_t ref_clock_get();
 
+
+/**
+ * @brief Reset automatic leak checking which happens in unit tests.
+ *
+ * Updates recorded "before" free memory values to the free memory values
+ * at time of calling. Resets leak checker if tracing is enabled in
+ * config.
+ *
+ * This can be called if a test case does something which allocates
+ * memory on first use, for example.
+ *
+ * @note Use with care as this can mask real memory leak problems.
+ */
+void unity_reset_leak_checks(void);
+
+
+/**
+ * @brief Call this function from a test case which requires TCP/IP or
+ * LWIP functionality.
+ *
+ * @note This should be the first function the test case calls, as it will
+ * allocate memory on first use (and also reset the test case leak checker).
+ */
+void test_case_uses_tcpip(void);
+
+
 /**
  * @brief wait for signals.
  *

tools/unit-test-app/components/unity/test_utils.c

@@ -17,6 +17,10 @@
 #include "test_utils.h"
 #include "rom/ets_sys.h"
 #include "rom/uart.h"
+#include "freertos/FreeRTOS.h"
+#include "freertos/task.h"
+#include "tcpip_adapter.h"
+#include "lwip/sockets.h"
 
 const esp_partition_t *get_test_data_partition()
 {
@@ -41,6 +45,32 @@ static void wait_user_control()
     }
 }
 
+void test_case_uses_tcpip()
+{
+    // Can be called more than once, does nothing on subsequent calls
+    tcpip_adapter_init();
+
+    // Allocate all sockets then free them
+    // (First time each socket is allocated some one-time allocations happen.)
+    int sockets[CONFIG_LWIP_MAX_SOCKETS];
+    for (int i = 0; i < CONFIG_LWIP_MAX_SOCKETS; i++) {
+        int type = (i % 2 == 0) ? SOCK_DGRAM : SOCK_STREAM;
+        int family = (i % 3 == 0) ? PF_INET6 : PF_INET;
+        sockets[i] = socket(family, type, IPPROTO_IP);
+    }
+    for (int i = 0; i < CONFIG_LWIP_MAX_SOCKETS; i++) {
+        close(sockets[i]);
+    }
+
+    // Allow LWIP tasks to finish initialising themselves
+    vTaskDelay(25 / portTICK_RATE_MS);
+
+    printf("Note: tcpip_adapter_init() has been called. Until next reset, TCP/IP task will periodicially allocate memory and consume CPU time.\n");
+
+    // Reset the leak checker as LWIP allocates a lot of memory on first run
+    unity_reset_leak_checks();
+}
+
 // signal functions, used for sync between unity DUTs for multiple devices cases
 void unity_wait_for_signal(const char* signal_name)
 {

tools/unit-test-app/components/unity/unity_platform.c

@@ -37,6 +37,16 @@ static size_t before_free_32bit;
 const size_t WARN_LEAK_THRESHOLD = 256;
 const size_t CRITICAL_LEAK_THRESHOLD = 4096;
 
+void unity_reset_leak_checks(void)
+{
+    before_free_8bit = heap_caps_get_free_size(MALLOC_CAP_8BIT);
+    before_free_32bit = heap_caps_get_free_size(MALLOC_CAP_32BIT);
+
+#ifdef CONFIG_HEAP_TRACING
+    heap_trace_start(HEAP_TRACE_LEAKS);
+#endif
+}
+
 /* setUp runs before every test */
 void setUp(void)
 {
@@ -54,12 +64,7 @@ void setUp(void)
     printf("%s", ""); /* sneakily lazy-allocate the reent structure for this test task */
     get_test_data_partition();  /* allocate persistent partition table structures */
 
-    before_free_8bit = heap_caps_get_free_size(MALLOC_CAP_8BIT);
+    unity_reset_leak_checks();
-    before_free_32bit = heap_caps_get_free_size(MALLOC_CAP_32BIT);
-
-#ifdef CONFIG_HEAP_TRACING
-    heap_trace_start(HEAP_TRACE_LEAKS);
-#endif
 }
 
 static void check_leak(size_t before_free, size_t after_free, const char *type)

tools/unit-test-app/main/app_main.c

@@ -3,7 +3,6 @@
 #include "freertos/task.h"
 #include "unity.h"
 #include "unity_config.h"
-#include "tcpip_adapter.h"
 
 void unityTask(void *pvParameters)
 {
@@ -13,10 +12,6 @@ void unityTask(void *pvParameters)
 
 void app_main()
 {
-    // TCP/IP adapter is initialized here because it leaks memory so the
-    // initialization in test cases would make the test fail because of leak.
-    tcpip_adapter_init();
-
     // Note: if unpinning this task, change the way run times are calculated in
     // unity_platform
     xTaskCreatePinnedToCore(unityTask, "unityTask", UNITY_FREERTOS_STACK_SIZE, NULL,

tools/unit-test-app/unit_test.py

@@ -27,7 +27,9 @@ EXCEPTION_PATTERN = re.compile(r"(Guru Meditation Error: Core\s+\d panic'ed \([\
 ABORT_PATTERN = re.compile(r"(abort\(\) was called at PC 0x[a-eA-E\d]{8} on core \d)")
 FINISH_PATTERN = re.compile(r"1 Tests (\d) Failures (\d) Ignored")
 
-STARTUP_TIMEOUT=10
+STARTUP_TIMEOUT = 10
+DUT_STARTUP_CHECK_RETRY_COUNT = 5
+TEST_HISTORY_CHECK_TIMEOUT = 1
 
 
 def format_test_case_config(test_case_data):
@@ -125,12 +127,12 @@ def reset_dut(dut):
     for _ in range(DUT_STARTUP_CHECK_RETRY_COUNT):
         dut.write("-")
         try:
-            dut.expect("0 Tests 0 Failures 0 Ignored", timeout=TEST_HISTROY_CHECK_TIMEOUT)
+            dut.expect("0 Tests 0 Failures 0 Ignored", timeout=TEST_HISTORY_CHECK_TIMEOUT)
             break
         except ExpectTimeout:
             pass
     else:
-        raise AssertationError("Reset {} ({}) failed!".format(dut.name, dut.port))
+        raise AssertionError("Reset {} ({}) failed!".format(dut.name, dut.port))
 
 
 def run_one_normal_case(dut, one_case, junit_test_case, failed_cases):
@@ -275,6 +277,10 @@ class Handler(threading.Thread):
         self.output = ""
         self.fail_name = None
         self.timeout = timeout
+        self.force_stop = threading.Event()  # it show the running status
+
+        reset_dut(self.dut)  # reset the board to make it start from begining
+
         threading.Thread.__init__(self, name="{} Handler".format(dut))
 
     def run(self):
@@ -321,14 +327,13 @@ class Handler(threading.Thread):
                 Utility.console_log("Ignored: " + self.child_case_name, color="orange")
             one_device_case_finish(not int(data[0]))
 
-        self.dut.reset()
+        try:
-        self.dut.write("-", flush=False)
+            time.sleep(1)
-        self.dut.expect_any(UT_APP_BOOT_UP_DONE, "0 Tests 0 Failures 0 Ignored")
+            self.dut.write("\"{}\"".format(self.parent_case_name))
-        time.sleep(1)
+            self.dut.expect("Running " + self.parent_case_name + "...")
-        self.dut.write("\"{}\"".format(self.parent_case_name))
+        except ExpectTimeout:
-        self.dut.expect("Running " + self.parent_case_name + "...")
+            Utility.console_log("No case detected!", color="orange")
-
+        while not self.finish and not self.force_stop.isSet():
-        while not self.finish:
             try:
                 self.dut.expect_any((re.compile('\(' + str(self.child_case_index) + '\)\s"(\w+)"'), get_child_case_name),
                                     (self.WAIT_SIGNAL_PATTERN, device_wait_action),  # wait signal pattern
@@ -340,6 +345,9 @@ class Handler(threading.Thread):
                 one_device_case_finish(False)
                 break
 
+    def stop(self):
+        self.force_stop.set()
+
 
 def get_case_info(one_case):
     parent_case = one_case["name"]
@@ -361,9 +369,9 @@ def run_one_multiple_devices_case(duts, ut_config, env, one_case, failed_cases,
     lock = threading.RLock()
     threads = []
     send_signal_list = []
-    failed_device = []
     result = True
     parent_case, case_num = get_case_info(one_case)
+
     for i in range(case_num):
         dut = get_dut(duts, env, "dut%d" % i, ut_config)
         threads.append(Handler(dut, send_signal_list, lock,
@@ -377,7 +385,8 @@ def run_one_multiple_devices_case(duts, ut_config, env, one_case, failed_cases,
         result = result and thread.result
         output += thread.output
         if not thread.result:
-            failed_device.append(thread.fail_name)
+            [thd.stop() for thd in threads]
+
     if result:
         Utility.console_log("Success: " + one_case["name"], color="green")
     else: