diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9576f6f827..2eccbaf843 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1224,6 +1224,38 @@ UT_010_04: - UT_T1_RMT - psram +UT_017_01: + <<: *unit_test_template + tags: + - ESP32_IDF + - UT_T2_1 + +UT_017_02: + <<: *unit_test_template + tags: + - ESP32_IDF + - UT_T2_1 + +UT_017_03: + <<: *unit_test_template + tags: + - ESP32_IDF + - UT_T2_1 + +UT_017_04: + <<: *unit_test_template + tags: + - ESP32_IDF + - UT_T2_1 + - psram + +UT_017_05: + <<: *unit_test_template + tags: + - ESP32_IDF + - UT_T2_1 + - 8Mpsram + UT_601_01: <<: *unit_test_template tags: diff --git a/components/driver/test/test_adc2.c b/components/driver/test/test_adc2.c index ec6e2ff4f6..40549428ef 100644 --- a/components/driver/test/test_adc2.c +++ b/components/driver/test/test_adc2.c @@ -10,6 +10,7 @@ #include "esp_wifi.h" #include "esp_log.h" #include "nvs_flash.h" +#include "test_utils.h" static const char* TAG = "test_adc2"; @@ -44,7 +45,9 @@ TEST_CASE("adc2 work with wifi","[adc]") { int read_raw; int target_value; - + + test_case_uses_tcpip(); + //adc and dac init TEST_ESP_OK( dac_output_enable( DAC_CHANNEL_1 )); TEST_ESP_OK( dac_output_enable( DAC_CHANNEL_2 )); diff --git a/components/esp32/test/test_esp32.c b/components/esp32/test/test_wifi.c similarity index 66% rename from components/esp32/test/test_esp32.c rename to components/esp32/test/test_wifi.c index 08700fa56f..8687650f3d 100644 --- a/components/esp32/test/test_esp32.c +++ b/components/esp32/test/test_wifi.c @@ -9,6 +9,8 @@ #include "esp_wifi.h" #include "esp_log.h" #include "nvs_flash.h" +#include "test_utils.h" +#include "freertos/task.h" static const char* TAG = "test_wifi"; @@ -79,7 +81,9 @@ static void test_wifi_start_stop(wifi_init_config_t *cfg, wifi_config_t* wifi_co } TEST_CASE("wifi stop and deinit","[wifi]") -{ +{ + test_case_uses_tcpip(); + wifi_init_config_t cfg = WIFI_INIT_CONFIG_DEFAULT(); wifi_config_t wifi_config = { .sta = { @@ -118,3 +122,68 @@ TEST_CASE("wifi stop and deinit","[wifi]") TEST_IGNORE_MESSAGE("this test case is ignored due to the critical memory leak of tcpip_adapter and event_loop."); } + +static void start_wifi_as_softap(void) +{ + wifi_init_config_t cfg = WIFI_INIT_CONFIG_DEFAULT(); + cfg.nvs_enable = false; + + wifi_config_t w_config = { + .ap.ssid = "default_ssid", + .ap.password = "default_password", + .ap.ssid_len = 0, + .ap.channel = 1, + .ap.authmode = WIFI_AUTH_WPA2_PSK, + .ap.ssid_hidden = false, + .ap.max_connection = 4, + .ap.beacon_interval = 100, + }; + + TEST_ESP_OK(esp_wifi_init(&cfg)); + TEST_ESP_OK(esp_wifi_set_mode(WIFI_MODE_AP)); + TEST_ESP_OK(esp_wifi_set_config(WIFI_IF_AP, &w_config)); + TEST_ESP_OK(esp_wifi_start()); + +} + +static void stop_wifi(void) +{ + TEST_ESP_OK(esp_wifi_stop()); + TEST_ESP_OK(esp_wifi_deinit()); +} + +static void receive_ds2ds_packet(void) +{ + start_wifi_as_softap(); + unity_wait_for_signal("sender ready"); + unity_send_signal("receiver ready"); + + // wait for sender to send packets + vTaskDelay(1000/portTICK_PERIOD_MS); + stop_wifi(); + +} + +static const char ds2ds_pdu[] = { + 0x48, 0x03, 0x00, 0x00, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xE8, 0x65, 0xD4, 0xCB, 0x74, 0x19, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x60, 0x94, 0xE8, 0x65, 0xD4, 0xCB, 0x74, 0x1C, 0x26, 0xB9, + 0x0D, 0x02, 0x7D, 0x13, 0x00, 0x00, 0x01, 0xE8, 0x65, 0xD4, 0xCB, 0x74, + 0x1C, 0x00, 0x00, 0x26, 0xB9, 0x00, 0x00, 0x00, 0x00 +}; + +static void send_ds2ds_packet(void) +{ + start_wifi_as_softap(); + unity_send_signal("sender ready"); + unity_wait_for_signal("receiver ready"); + + // send packet 20 times to make sure receiver will get this packet + for (uint16_t i = 0; i < 20; i++) { + esp_wifi_80211_tx(ESP_IF_WIFI_AP, ds2ds_pdu, sizeof(ds2ds_pdu), true); + vTaskDelay(50 / portTICK_PERIOD_MS); + } + stop_wifi(); +} + +TEST_CASE_MULTIPLE_DEVICES("receive ds2ds packet without exception", "[wifi][test_env=UT_T2_1]", receive_ds2ds_packet, send_ds2ds_packet); diff --git a/components/newlib/test/test_time.c b/components/newlib/test/test_time.c index 47df5b9de2..ccb717608d 100644 --- a/components/newlib/test/test_time.c +++ b/components/newlib/test/test_time.c @@ -133,12 +133,10 @@ TEST_CASE("test adjtime function", "[newlib]") static volatile bool exit_flag; static bool adjtime_test_result; static bool gettimeofday_test_result; -static uint64_t count_adjtime; -static uint64_t count_settimeofday; -static uint64_t count_gettimeofday; static void adjtimeTask2(void *pvParameters) { + xSemaphoreHandle *sema = (xSemaphoreHandle *) pvParameters; struct timeval delta = {.tv_sec = 0, .tv_usec = 0}; struct timeval outdelta; @@ -148,34 +146,23 @@ static void adjtimeTask2(void *pvParameters) delta.tv_usec = 900000; if (delta.tv_sec >= 2146) delta.tv_sec = 1; adjtime(&delta, &outdelta); - count_adjtime++; } + xSemaphoreGive(*sema); vTaskDelete(NULL); } -static void settimeofdayTask2(void *pvParameters) +static void timeTask(void *pvParameters) { + xSemaphoreHandle *sema = (xSemaphoreHandle *) pvParameters; struct timeval tv_time = { .tv_sec = 1520000000, .tv_usec = 900000 }; // although exit flag is set in another task, checking (exit_flag == false) is safe while (exit_flag == false) { tv_time.tv_sec += 1; settimeofday(&tv_time, NULL); - count_settimeofday++; - vTaskDelay(1); - } - vTaskDelete(NULL); -} - -static void gettimeofdayTask2(void *pvParameters) -{ - struct timeval tv_time; - // although exit flag is set in another task, checking (exit_flag == false) is safe - while (exit_flag == false) { gettimeofday(&tv_time, NULL); - count_gettimeofday++; - vTaskDelay(1); } + xSemaphoreGive(*sema); vTaskDelete(NULL); } @@ -183,32 +170,37 @@ TEST_CASE("test for no interlocking adjtime, gettimeofday and settimeofday funct { TaskHandle_t th[4]; exit_flag = false; - count_adjtime = 0; - count_settimeofday = 0; - count_gettimeofday = 0; struct timeval tv_time = { .tv_sec = 1520000000, .tv_usec = 900000 }; TEST_ASSERT_EQUAL(settimeofday(&tv_time, NULL), 0); + const int max_tasks = 2; + xSemaphoreHandle exit_sema[max_tasks]; + + for (int i = 0; i < max_tasks; ++i) { + exit_sema[i] = xSemaphoreCreateBinary(); + } + #ifndef CONFIG_FREERTOS_UNICORE printf("CPU0 and CPU1. Tasks run: 1 - adjtimeTask, 2 - gettimeofdayTask, 3 - settimeofdayTask \n"); - xTaskCreatePinnedToCore(adjtimeTask2, "adjtimeTask1", 2048, NULL, UNITY_FREERTOS_PRIORITY - 1, &th[0], 0); - xTaskCreatePinnedToCore(gettimeofdayTask2, "gettimeofdayTask1", 2048, NULL, UNITY_FREERTOS_PRIORITY - 1, &th[1], 1); - xTaskCreatePinnedToCore(settimeofdayTask2, "settimeofdayTask1", 2048, NULL, UNITY_FREERTOS_PRIORITY - 1, &th[2], 0); + xTaskCreatePinnedToCore(adjtimeTask2, "adjtimeTask2", 2048, &exit_sema[0], UNITY_FREERTOS_PRIORITY - 1, &th[0], 0); + xTaskCreatePinnedToCore(timeTask, "timeTask", 2048, &exit_sema[1], UNITY_FREERTOS_PRIORITY - 1, &th[1], 1); #else printf("Only one CPU. Tasks run: 1 - adjtimeTask, 2 - gettimeofdayTask, 3 - settimeofdayTask\n"); - xTaskCreate(adjtimeTask2, "adjtimeTask1", 2048, NULL, UNITY_FREERTOS_PRIORITY - 1, &th[0]); - xTaskCreate(gettimeofdayTask2, "gettimeofdayTask1", 2048, NULL, UNITY_FREERTOS_PRIORITY - 1, &th[1]); - xTaskCreate(settimeofdayTask2, "settimeofdayTask1", 2048, NULL, UNITY_FREERTOS_PRIORITY - 1, &th[2]); + xTaskCreate(adjtimeTask2, "adjtimeTask2", 2048, &exit_sema[0], UNITY_FREERTOS_PRIORITY - 1, &th[0]); + xTaskCreate(timeTask, "timeTask", 2048, &exit_sema[1], UNITY_FREERTOS_PRIORITY - 1, &th[1]); #endif - printf("start wait for 10 seconds\n"); - vTaskDelay(10000 / portTICK_PERIOD_MS); + printf("start wait for 5 seconds\n"); + vTaskDelay(5000 / portTICK_PERIOD_MS); // set exit flag to let thread exit exit_flag = true; - vTaskDelay(20 / portTICK_PERIOD_MS); - printf("count_adjtime %lld, count_settimeofday %lld, count_gettimeofday %lld\n", count_adjtime, count_settimeofday, count_gettimeofday); - TEST_ASSERT(count_adjtime > 1000LL && count_settimeofday > 1000LL && count_gettimeofday > 1000LL); + for (int i = 0; i < max_tasks; ++i) { + if (!xSemaphoreTake(exit_sema[i], 2000/portTICK_PERIOD_MS)) { + TEST_FAIL_MESSAGE("exit_sema not released by test task"); + } + vSemaphoreDelete(exit_sema[i]); + } } static void adjtimeTask(void *pvParameters) @@ -268,20 +260,15 @@ TEST_CASE("test for thread safety adjtime and gettimeofday functions", "[newlib] printf("CPU0 and CPU1. Tasks run: 1 - adjtimeTask, 2 - gettimeofdayTask\n"); xTaskCreatePinnedToCore(adjtimeTask, "adjtimeTask1", 2048, NULL, UNITY_FREERTOS_PRIORITY - 1, &th[0], 0); xTaskCreatePinnedToCore(gettimeofdayTask, "gettimeofdayTask1", 2048, NULL, UNITY_FREERTOS_PRIORITY - 1, &th[1], 1); - - xTaskCreatePinnedToCore(adjtimeTask, "adjtimeTask2", 2048, NULL, UNITY_FREERTOS_PRIORITY - 1, &th[2], 0); - xTaskCreatePinnedToCore(gettimeofdayTask, "gettimeofdayTask2", 2048, NULL, UNITY_FREERTOS_PRIORITY - 1, &th[3], 1); + xTaskCreatePinnedToCore(gettimeofdayTask, "gettimeofdayTask2", 2048, NULL, UNITY_FREERTOS_PRIORITY - 1, &th[2], 0); #else printf("Only one CPU. Tasks run: 1 - adjtimeTask, 2 - gettimeofdayTask\n"); xTaskCreate(adjtimeTask, "adjtimeTask1", 2048, NULL, UNITY_FREERTOS_PRIORITY - 1, &th[0]); xTaskCreate(gettimeofdayTask, "gettimeofdayTask1", 2048, NULL, UNITY_FREERTOS_PRIORITY - 1, &th[1]); - - xTaskCreate(adjtimeTask, "adjtimeTask2", 2048, NULL, UNITY_FREERTOS_PRIORITY - 1, &th[2]); - xTaskCreate(gettimeofdayTask, "gettimeofdayTask2", 2048, NULL, UNITY_FREERTOS_PRIORITY - 1, &th[3]); #endif - printf("start wait for 10 seconds\n"); - vTaskDelay(10000 / portTICK_PERIOD_MS); + printf("start wait for 5 seconds\n"); + vTaskDelay(5000 / portTICK_PERIOD_MS); // set exit flag to let thread exit exit_flag = true; diff --git a/components/spiffs/test/test_spiffs.c b/components/spiffs/test/test_spiffs.c index 60bca410bd..d5565659cf 100644 --- a/components/spiffs/test/test_spiffs.c +++ b/components/spiffs/test/test_spiffs.c @@ -594,7 +594,7 @@ TEST_CASE("opendir, readdir, rewinddir, seekdir work as expected", "[spiffs]") test_teardown(); } -TEST_CASE("readdir with large number of files", "[spiffs][timeout=15]") +TEST_CASE("readdir with large number of files", "[spiffs][timeout=30]") { test_setup(); test_spiffs_readdir_many_files("/spiffs/dir2"); diff --git a/components/vfs/test/test_vfs_select.c b/components/vfs/test/test_vfs_select.c index f9927a54f6..8ec6794f44 100644 --- a/components/vfs/test/test_vfs_select.c +++ b/components/vfs/test/test_vfs_select.c @@ -24,6 +24,7 @@ #include "esp_vfs_dev.h" #include "lwip/sockets.h" #include "lwip/netdb.h" +#include "test_utils.h" typedef struct { int fd; @@ -114,6 +115,8 @@ static inline void start_task(const test_task_param_t *test_task_param) static void init(int *uart_fd, int *socket_fd) { + test_case_uses_tcpip(); + uart1_init(); UART1.conf0.loopback = 1; @@ -296,9 +299,10 @@ TEST_CASE("concurent selects work", "[vfs]") }; int uart_fd, socket_fd; - const int dummy_socket_fd = open_dummy_socket(); init(&uart_fd, &socket_fd); + const int dummy_socket_fd = open_dummy_socket(); + fd_set rfds; FD_ZERO(&rfds); FD_SET(uart_fd, &rfds); diff --git a/tools/tiny-test-fw/DUT.py b/tools/tiny-test-fw/DUT.py index b782b5e1e1..d52f39684e 100644 --- a/tools/tiny-test-fw/DUT.py +++ b/tools/tiny-test-fw/DUT.py @@ -623,7 +623,10 @@ class BaseDUT(object): data = self.data_cache.get_data(time_remaining) if match_succeed: - # do callback and flush matched data cache + # sort matched items according to order of appearance in the input data, + # so that the callbacks are invoked in correct order + matched_expect_items = sorted(matched_expect_items, key=lambda it: it["index"]) + # invoke callbacks and flush matched data cache slice_index = -1 for expect_item in matched_expect_items: # trigger callback diff --git a/tools/unit-test-app/components/unity/include/test_utils.h b/tools/unit-test-app/components/unity/include/test_utils.h index 746d94a083..68e8e81d7c 100644 --- a/tools/unit-test-app/components/unity/include/test_utils.h +++ b/tools/unit-test-app/components/unity/include/test_utils.h @@ -43,6 +43,32 @@ void ref_clock_deinit(); */ uint64_t ref_clock_get(); + +/** + * @brief Reset automatic leak checking which happens in unit tests. + * + * Updates recorded "before" free memory values to the free memory values + * at time of calling. Resets leak checker if tracing is enabled in + * config. + * + * This can be called if a test case does something which allocates + * memory on first use, for example. + * + * @note Use with care as this can mask real memory leak problems. + */ +void unity_reset_leak_checks(void); + + +/** + * @brief Call this function from a test case which requires TCP/IP or + * LWIP functionality. + * + * @note This should be the first function the test case calls, as it will + * allocate memory on first use (and also reset the test case leak checker). + */ +void test_case_uses_tcpip(void); + + /** * @brief wait for signals. * diff --git a/tools/unit-test-app/components/unity/test_utils.c b/tools/unit-test-app/components/unity/test_utils.c index 01176a8163..36aae4c291 100644 --- a/tools/unit-test-app/components/unity/test_utils.c +++ b/tools/unit-test-app/components/unity/test_utils.c @@ -17,6 +17,10 @@ #include "test_utils.h" #include "rom/ets_sys.h" #include "rom/uart.h" +#include "freertos/FreeRTOS.h" +#include "freertos/task.h" +#include "tcpip_adapter.h" +#include "lwip/sockets.h" const esp_partition_t *get_test_data_partition() { @@ -41,6 +45,32 @@ static void wait_user_control() } } +void test_case_uses_tcpip() +{ + // Can be called more than once, does nothing on subsequent calls + tcpip_adapter_init(); + + // Allocate all sockets then free them + // (First time each socket is allocated some one-time allocations happen.) + int sockets[CONFIG_LWIP_MAX_SOCKETS]; + for (int i = 0; i < CONFIG_LWIP_MAX_SOCKETS; i++) { + int type = (i % 2 == 0) ? SOCK_DGRAM : SOCK_STREAM; + int family = (i % 3 == 0) ? PF_INET6 : PF_INET; + sockets[i] = socket(family, type, IPPROTO_IP); + } + for (int i = 0; i < CONFIG_LWIP_MAX_SOCKETS; i++) { + close(sockets[i]); + } + + // Allow LWIP tasks to finish initialising themselves + vTaskDelay(25 / portTICK_RATE_MS); + + printf("Note: tcpip_adapter_init() has been called. Until next reset, TCP/IP task will periodicially allocate memory and consume CPU time.\n"); + + // Reset the leak checker as LWIP allocates a lot of memory on first run + unity_reset_leak_checks(); +} + // signal functions, used for sync between unity DUTs for multiple devices cases void unity_wait_for_signal(const char* signal_name) { diff --git a/tools/unit-test-app/components/unity/unity_platform.c b/tools/unit-test-app/components/unity/unity_platform.c index fa1adf6333..c4ebb356e3 100644 --- a/tools/unit-test-app/components/unity/unity_platform.c +++ b/tools/unit-test-app/components/unity/unity_platform.c @@ -37,6 +37,16 @@ static size_t before_free_32bit; const size_t WARN_LEAK_THRESHOLD = 256; const size_t CRITICAL_LEAK_THRESHOLD = 4096; +void unity_reset_leak_checks(void) +{ + before_free_8bit = heap_caps_get_free_size(MALLOC_CAP_8BIT); + before_free_32bit = heap_caps_get_free_size(MALLOC_CAP_32BIT); + +#ifdef CONFIG_HEAP_TRACING + heap_trace_start(HEAP_TRACE_LEAKS); +#endif +} + /* setUp runs before every test */ void setUp(void) { @@ -54,12 +64,7 @@ void setUp(void) printf("%s", ""); /* sneakily lazy-allocate the reent structure for this test task */ get_test_data_partition(); /* allocate persistent partition table structures */ - before_free_8bit = heap_caps_get_free_size(MALLOC_CAP_8BIT); - before_free_32bit = heap_caps_get_free_size(MALLOC_CAP_32BIT); - -#ifdef CONFIG_HEAP_TRACING - heap_trace_start(HEAP_TRACE_LEAKS); -#endif + unity_reset_leak_checks(); } static void check_leak(size_t before_free, size_t after_free, const char *type) diff --git a/tools/unit-test-app/main/app_main.c b/tools/unit-test-app/main/app_main.c index 1dbcdd3b0c..73a8201ff8 100644 --- a/tools/unit-test-app/main/app_main.c +++ b/tools/unit-test-app/main/app_main.c @@ -3,7 +3,6 @@ #include "freertos/task.h" #include "unity.h" #include "unity_config.h" -#include "tcpip_adapter.h" void unityTask(void *pvParameters) { @@ -13,10 +12,6 @@ void unityTask(void *pvParameters) void app_main() { - // TCP/IP adapter is initialized here because it leaks memory so the - // initialization in test cases would make the test fail because of leak. - tcpip_adapter_init(); - // Note: if unpinning this task, change the way run times are calculated in // unity_platform xTaskCreatePinnedToCore(unityTask, "unityTask", UNITY_FREERTOS_STACK_SIZE, NULL, diff --git a/tools/unit-test-app/unit_test.py b/tools/unit-test-app/unit_test.py index f7ea874e70..ac4c779dd1 100644 --- a/tools/unit-test-app/unit_test.py +++ b/tools/unit-test-app/unit_test.py @@ -27,7 +27,9 @@ EXCEPTION_PATTERN = re.compile(r"(Guru Meditation Error: Core\s+\d panic'ed \([\ ABORT_PATTERN = re.compile(r"(abort\(\) was called at PC 0x[a-eA-E\d]{8} on core \d)") FINISH_PATTERN = re.compile(r"1 Tests (\d) Failures (\d) Ignored") -STARTUP_TIMEOUT=10 +STARTUP_TIMEOUT = 10 +DUT_STARTUP_CHECK_RETRY_COUNT = 5 +TEST_HISTORY_CHECK_TIMEOUT = 1 def format_test_case_config(test_case_data): @@ -125,12 +127,12 @@ def reset_dut(dut): for _ in range(DUT_STARTUP_CHECK_RETRY_COUNT): dut.write("-") try: - dut.expect("0 Tests 0 Failures 0 Ignored", timeout=TEST_HISTROY_CHECK_TIMEOUT) + dut.expect("0 Tests 0 Failures 0 Ignored", timeout=TEST_HISTORY_CHECK_TIMEOUT) break except ExpectTimeout: pass else: - raise AssertationError("Reset {} ({}) failed!".format(dut.name, dut.port)) + raise AssertionError("Reset {} ({}) failed!".format(dut.name, dut.port)) def run_one_normal_case(dut, one_case, junit_test_case, failed_cases): @@ -275,6 +277,10 @@ class Handler(threading.Thread): self.output = "" self.fail_name = None self.timeout = timeout + self.force_stop = threading.Event() # it show the running status + + reset_dut(self.dut) # reset the board to make it start from begining + threading.Thread.__init__(self, name="{} Handler".format(dut)) def run(self): @@ -321,14 +327,13 @@ class Handler(threading.Thread): Utility.console_log("Ignored: " + self.child_case_name, color="orange") one_device_case_finish(not int(data[0])) - self.dut.reset() - self.dut.write("-", flush=False) - self.dut.expect_any(UT_APP_BOOT_UP_DONE, "0 Tests 0 Failures 0 Ignored") - time.sleep(1) - self.dut.write("\"{}\"".format(self.parent_case_name)) - self.dut.expect("Running " + self.parent_case_name + "...") - - while not self.finish: + try: + time.sleep(1) + self.dut.write("\"{}\"".format(self.parent_case_name)) + self.dut.expect("Running " + self.parent_case_name + "...") + except ExpectTimeout: + Utility.console_log("No case detected!", color="orange") + while not self.finish and not self.force_stop.isSet(): try: self.dut.expect_any((re.compile('\(' + str(self.child_case_index) + '\)\s"(\w+)"'), get_child_case_name), (self.WAIT_SIGNAL_PATTERN, device_wait_action), # wait signal pattern @@ -340,6 +345,9 @@ class Handler(threading.Thread): one_device_case_finish(False) break + def stop(self): + self.force_stop.set() + def get_case_info(one_case): parent_case = one_case["name"] @@ -361,9 +369,9 @@ def run_one_multiple_devices_case(duts, ut_config, env, one_case, failed_cases, lock = threading.RLock() threads = [] send_signal_list = [] - failed_device = [] result = True parent_case, case_num = get_case_info(one_case) + for i in range(case_num): dut = get_dut(duts, env, "dut%d" % i, ut_config) threads.append(Handler(dut, send_signal_list, lock, @@ -377,7 +385,8 @@ def run_one_multiple_devices_case(duts, ut_config, env, one_case, failed_cases, result = result and thread.result output += thread.output if not thread.result: - failed_device.append(thread.fail_name) + [thd.stop() for thd in threads] + if result: Utility.console_log("Success: " + one_case["name"], color="green") else: