Merge branch 'feature/ut_filter_test_groups' into 'master'

unit test: allow filtering tests using TEST_GROUPS

See merge request idf/esp-idf!3695

components/esp32/test/CMakeLists.txt

@@ -1,38 +1,30 @@
-if (CONFIG_TEST_ESP32_SUBTEST_ONLY)
+set(COMPONENT_SRCDIRS ".")
-    set(COMPONENT_SRCDIRS "psram_4m")
+set(COMPONENT_ADD_INCLUDEDIRS ". ${CMAKE_CURRENT_BINARY_DIR}")
-    set(COMPONENT_ADD_INCLUDEDIRS ".")
 
-    register_component()
+set(COMPONENT_REQUIRES unity nvs_flash ulp)
 
-else()
+register_component()
-    set(COMPONENT_SRCDIRS ". psram_4m")
-    set(COMPONENT_ADD_INCLUDEDIRS ". ${CMAKE_CURRENT_BINARY_DIR}")
 
-    set(COMPONENT_REQUIRES unity nvs_flash ulp)
+add_custom_command(OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/test_tjpgd_logo.h"
+                COMMAND xxd -i "logo.jpg" "${CMAKE_CURRENT_BINARY_DIR}/test_tjpgd_logo.h"
+                WORKING_DIRECTORY ${COMPONENT_PATH}
+                DEPENDS "${CMAKE_CURRENT_LIST_DIR}/logo.jpg")
 
-    register_component()
+# Calculate MD5 value of header file esp_wifi_os_adapter.h
+execute_process(COMMAND md5sum ${IDF_PATH}/components/esp32/include/esp_wifi_os_adapter.h
+                COMMAND cut -c 1-7
+                OUTPUT_VARIABLE WIFI_OS_ADAPTER_MD5
+                OUTPUT_STRIP_TRAILING_WHITESPACE)
 
-    add_custom_command(OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/test_tjpgd_logo.h"
+# Calculate MD5 value of header file esp_wifi_crypto_types.h
-                    COMMAND xxd -i "logo.jpg" "${CMAKE_CURRENT_BINARY_DIR}/test_tjpgd_logo.h"
+execute_process(COMMAND md5sum ${IDF_PATH}/components/esp32/include/esp_wifi_crypto_types.h
-                    WORKING_DIRECTORY ${COMPONENT_PATH}
+                COMMAND cut -c 1-7
-                    DEPENDS "${CMAKE_CURRENT_LIST_DIR}/logo.jpg")
+                OUTPUT_VARIABLE WIFI_CRYPTO_MD5
+                OUTPUT_STRIP_TRAILING_WHITESPACE)
 
-    # Calculate MD5 value of header file esp_wifi_os_adapter.h
+add_definitions(-DWIFI_OS_ADAPTER_MD5=\"${WIFI_OS_ADAPTER_MD5}\")
-    execute_process(COMMAND md5sum ${IDF_PATH}/components/esp32/include/esp_wifi_os_adapter.h
+add_definitions(-DWIFI_CRYPTO_MD5=\"${WIFI_CRYPTO_MD5}\")
-                    COMMAND cut -c 1-7
-                    OUTPUT_VARIABLE WIFI_OS_ADAPTER_MD5
-                    OUTPUT_STRIP_TRAILING_WHITESPACE)
 
-    # Calculate MD5 value of header file esp_wifi_crypto_types.h
+add_custom_target(esp32_test_logo DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/test_tjpgd_logo.h")
-    execute_process(COMMAND md5sum ${IDF_PATH}/components/esp32/include/esp_wifi_crypto_types.h
-                    COMMAND cut -c 1-7
-                    OUTPUT_VARIABLE WIFI_CRYPTO_MD5
-                    OUTPUT_STRIP_TRAILING_WHITESPACE)
 
-    add_definitions(-DWIFI_OS_ADAPTER_MD5=\"${WIFI_OS_ADAPTER_MD5}\")
+add_dependencies(${COMPONENT_NAME} esp32_test_logo)
-    add_definitions(-DWIFI_CRYPTO_MD5=\"${WIFI_CRYPTO_MD5}\")
-
-    add_custom_target(esp32_test_logo DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/test_tjpgd_logo.h")
-
-    add_dependencies(${COMPONENT_NAME} esp32_test_logo)
-endif()

components/esp32/test/component.mk

@@ -6,11 +6,7 @@ COMPONENT_EXTRA_CLEAN := test_tjpgd_logo.h
 
 COMPONENT_ADD_LDFLAGS = -Wl,--whole-archive -l$(COMPONENT_NAME) -Wl,--no-whole-archive
 
-ifdef CONFIG_TEST_ESP32_SUBTEST_ONLY
+COMPONENT_SRCDIRS := .
-COMPONENT_SRCDIRS := psram_4m
-else
-COMPONENT_SRCDIRS := . test_vectors psram_4m
-endif
 
 # Calculate MD5 value of header file esp_wifi_os_adapter.h
 WIFI_OS_ADAPTER_MD5_VAL=\"$(shell md5sum $(IDF_PATH)/components/esp32/include/esp_wifi_os_adapter.h | cut -c 1-7)\"

components/esp32/test/test_4mpsram.c

@@ -34,7 +34,7 @@ static void test_psram_content()
 }
 
 // NOTE: this unit test rely on the config that PSRAM of 8MB is used only when CONFIG_SPIRAM_BNKSWITCH_ENABLE is set
-TEST_CASE("can use spi when not being used by psram", "[esp32]")
+TEST_CASE("can use spi when not being used by psram", "[psram_4m]")
 {
     spi_host_device_t host;
 #if !CONFIG_SPIRAM_SUPPORT || !CONFIG_SPIRAM_SPEED_80M || CONFIG_SPIRAM_BANKSWITCH_ENABLE

tools/unit-test-app/components/unity/Kconfig

@@ -13,14 +13,3 @@ config UNITY_FREERTOS_STACK_SIZE
 	default 8192
 
 endmenu
-
-menu "Test options"
-
-config TEST_ESP32_SUBTEST_ONLY
-    bool "Test only 4M PSRAM cases for esp32 component"
-    depends on SPIRAM_SUPPORT
-    default n
-    help
-        If this option is enabled, only 4M PSRAM cases are compiled. Otherwise all cases are included.
-
-endmenu

tools/unit-test-app/configs/psram_hspi

@@ -1,5 +1,5 @@
 TEST_COMPONENTS=esp32
-CONFIG_TEST_ESP32_SUBTEST_ONLY=y
+TEST_GROUPS=psram_4m
 CONFIG_ESPTOOLPY_FLASHFREQ_80M=y
 CONFIG_SPIRAM_SUPPORT=y
 CONFIG_SPIRAM_SPEED_80M=y

tools/unit-test-app/configs/psram_vspi

@@ -1,5 +1,5 @@
 TEST_COMPONENTS=esp32
-CONFIG_TEST_ESP32_SUBTEST_ONLY=y
+TEST_GROUPS=psram_4m
 CONFIG_ESPTOOLPY_FLASHFREQ_80M=y
 CONFIG_SPIRAM_SUPPORT=y
 CONFIG_SPIRAM_SPEED_80M=y

tools/unit-test-app/tools/UnitTestParser.py

@@ -31,6 +31,7 @@ class Parser(object):
     TAG_PATTERN = re.compile("([^=]+)(=)?(.+)?")
     DESCRIPTION_PATTERN = re.compile("\[([^]\[]+)\]")
     CONFIG_PATTERN = re.compile(r"{([^}]+)}")
+    TEST_GROUPS_PATTERN = re.compile(r"TEST_GROUPS=(.*)$")
 
     # file path (relative to idf path)
     TAG_DEF_FILE = os.path.join("tools", "unit-test-app", "tools", "TagDefinition.yml")
@@ -39,6 +40,7 @@ class Parser(object):
     MODULE_ARTIFACT_FILE = os.path.join("components", "idf_test", "ModuleDefinition.yml")
     TEST_CASE_FILE = os.path.join("components", "idf_test", "unit_test", "TestCaseAll.yml")
     UT_BIN_FOLDER = os.path.join("tools", "unit-test-app", "output")
+    UT_CONFIG_FOLDER = os.path.join("tools", "unit-test-app", "configs")
     ELF_FILE = "unit-test-app.elf"
     SDKCONFIG_FILE = "sdkconfig"
 
@@ -54,12 +56,15 @@ class Parser(object):
         self.test_case_names = set()
         self.parsing_errors = []
 
-    def parse_test_cases_for_one_config(self, config_output_folder, config_name):
+    def parse_test_cases_for_one_config(self, configs_folder, config_output_folder, config_name):
         """
         parse test cases from elf and save test cases need to be executed to unit test folder
+        :param configs_folder: folder where per-config sdkconfig framents are located (i.e. tools/unit-test-app/configs)
         :param config_output_folder: build folder of this config
         :param config_name: built unit test config name
         """
+        test_groups = self.get_test_groups(os.path.join(configs_folder, config_name))
+
         elf_file = os.path.join(config_output_folder, self.ELF_FILE)
         subprocess.check_output('xtensa-esp32-elf-objdump -t {} | grep test_desc > case_address.tmp'.format(elf_file),
                                 shell=True)
@@ -94,7 +99,11 @@ class Parser(object):
                 else:
                     self.test_case_names.add(tc["summary"] + config_name)
 
-                if tc["CI ready"] == "Yes":
+                test_group_included = True
+                if test_groups is not None and tc["group"] not in test_groups:
+                    test_group_included = False
+
+                if tc["CI ready"] == "Yes" and test_group_included:
                     # update test env list and the cases of same env list
                     if tc["test environment"] in self.test_env_tags:
                         self.test_env_tags[tc["test environment"]].append(tc["ID"])
@@ -128,6 +137,9 @@ class Parser(object):
         p = dict([(k, self.tag_def[k]["omitted"]) for k in self.tag_def])
         p["module"] = tags[0]
 
+        # Use the original value of the first tag as test group name
+        p["group"] = p["module"]
+
         if p["module"] not in self.module_map:
             p["module"] = "misc"
 
@@ -201,6 +213,20 @@ class Parser(object):
 
         return self.parse_tags_internal(configs, self.config_dependencies, self.CONFIG_PATTERN)
 
+
+    def get_test_groups(self, config_file):
+        """
+        If the config file includes TEST_GROUPS variable, return its value as a list of strings.
+        :param config_file file under configs/ directory for given configuration
+        :return: list of test groups, or None if TEST_GROUPS wasn't set
+        """
+        with open(config_file, "r") as f:
+            for line in f:
+                match = self.TEST_GROUPS_PATTERN.match(line)
+                if match is not None:
+                    return match.group(1).split(' ')
+        return None
+
     def parse_one_test_case(self, name, description, file_name, config_name, tags):
         """
         parse one test case
@@ -216,6 +242,7 @@ class Parser(object):
         test_case = deepcopy(TEST_CASE_PATTERN)
         test_case.update({"config": config_name,
                           "module": self.module_map[prop["module"]]['module'],
+                          "group": prop["group"],
                           "CI ready": "No" if prop["ignore"] == "Yes" else "Yes",
                           "ID": name,
                           "test point 2": prop["module"],
@@ -249,11 +276,12 @@ class Parser(object):
         test_cases = []
 
         output_folder = os.path.join(self.idf_path, self.UT_BIN_FOLDER)
+        configs_folder = os.path.join(self.idf_path, self.UT_CONFIG_FOLDER)
         test_configs = os.listdir(output_folder)
         for config in test_configs:
             config_output_folder = os.path.join(output_folder, config)
             if os.path.exists(config_output_folder):
-                test_cases.extend(self.parse_test_cases_for_one_config(config_output_folder, config))
+                test_cases.extend(self.parse_test_cases_for_one_config(configs_folder, config_output_folder, config))
 
         self.dump_test_cases(test_cases)