Merge branch 'feature/ut_parallel_build' into 'master'

unit_test: ut parallel build

Closes IDF-1570

See merge request espressif/esp-idf!8338

.gitignore

@@ -40,6 +40,10 @@ tools/unit-test-app/sdkconfig.old
 tools/unit-test-app/build
 tools/unit-test-app/builds
 tools/unit-test-app/output
+tools/unit-test-app/test_configs
+
+# Unit Test CMake compile log folder
+log_ut_cmake
 
 # IDF monitor test
 tools/test_idf_monitor/outputs

.gitlab-ci.yml

@@ -33,7 +33,6 @@ variables:
   # tell build system do not check submodule update as we download archive instead of clone
   IDF_SKIP_CHECK_SUBMODULES: 1
 
-  UNIT_TEST_BUILD_SYSTEM: cmake
   EXAMPLE_TEST_BUILD_SYSTEM: cmake
   IDF_PATH: "$CI_PROJECT_DIR"
   BATCH_BUILD: "1"

tools/build_apps.py

@@ -112,7 +112,7 @@ def main():
         try:
             build_system_class.build(build_info)
         except BuildError as e:
-            logging.error(e.message)
+            logging.error(str(e))
             if args.keep_going:
                 failed_builds.append(build_info)
             else:

tools/ci/build_unit_test.sh

@@ -0,0 +1,107 @@
+#!/bin/bash
+#
+# Build unit test app
+#
+# Runs as part of CI process.
+#
+
+# -----------------------------------------------------------------------------
+# Safety settings (see https://gist.github.com/ilg-ul/383869cbb01f61a51c4d).
+
+if [[ ! -z ${DEBUG_SHELL} ]]
+then
+  set -x # Activate the expand mode if DEBUG is anything but empty.
+fi
+
+set -o errexit # Exit if command failed.
+set -o pipefail # Exit if pipe failed.
+
+export PATH="$IDF_PATH/tools/ci:$IDF_PATH/tools:$PATH"
+
+# -----------------------------------------------------------------------------
+
+die() {
+    echo "${1:-"Unknown Error"}" 1>&2
+    exit 1
+}
+
+[ -z ${IDF_PATH} ] && die "IDF_PATH is not set"
+[ -z ${LOG_PATH} ] && die "LOG_PATH is not set"
+[ -z ${IDF_TARGET} ] && die "IDF_TARGET is not set"
+[ -d ${LOG_PATH} ] || mkdir -p ${LOG_PATH}
+
+# Relative to IDF_PATH
+# If changing the BUILD_PATH, remember to update the "artifacts" in gitlab-ci configs, and IDFApp.py.
+BUILD_PATH=${IDF_PATH}/tools/unit-test-app/builds
+OUTPUT_PATH=${IDF_PATH}/tools/unit-test-app/output
+mkdir -p ${BUILD_PATH}/${IDF_TARGET}
+mkdir -p ${OUTPUT_PATH}/${IDF_TARGET}
+
+if [ -z ${CI_NODE_TOTAL} ]; then
+    CI_NODE_TOTAL=1
+    echo "Assuming CI_NODE_TOTAL=${CI_NODE_TOTAL}"
+fi
+if [ -z ${CI_NODE_INDEX} ]; then
+    # Gitlab uses a 1-based index
+    CI_NODE_INDEX=1
+    echo "Assuming CI_NODE_INDEX=${CI_NODE_INDEX}"
+fi
+
+
+set -o nounset # Exit if variable not set.
+
+# Convert LOG_PATH to relative, to make the json file less verbose.
+LOG_PATH=$(realpath --relative-to ${IDF_PATH} ${LOG_PATH})
+
+ALL_BUILD_LIST_JSON="${BUILD_PATH}/${IDF_TARGET}/list.json"
+JOB_BUILD_LIST_JSON="${BUILD_PATH}/${IDF_TARGET}/list_job_${CI_NODE_INDEX}.json"
+
+echo "build_unit_test running for target $IDF_TARGET"
+
+cd ${IDF_PATH}
+
+# This part of the script produces the same result for all the unit test app build jobs. It may be moved to a separate stage
+# (pre-build) later, then the build jobs will receive ${BUILD_LIST_JSON} file as an artifact.
+
+${IDF_PATH}/tools/find_apps.py tools/unit-test-app \
+    -vv \
+    --format json \
+    --build-system cmake \
+    --target ${IDF_TARGET} \
+    --recursive \
+    --build-dir "builds/@t/@w" \
+    --build-log "${LOG_PATH}/@w.txt" \
+    --output ${ALL_BUILD_LIST_JSON} \
+    --config 'configs/*='
+
+# The part below is where the actual builds happen
+
+${IDF_PATH}/tools/build_apps.py \
+    -vv \
+    --format json \
+    --keep-going \
+    --parallel-count ${CI_NODE_TOTAL} \
+    --parallel-index ${CI_NODE_INDEX} \
+    --output-build-list ${JOB_BUILD_LIST_JSON} \
+    ${ALL_BUILD_LIST_JSON}\
+
+# Copy build artifacts to output directory
+build_names=$(cd ${BUILD_PATH}/${IDF_TARGET}; find . -maxdepth 1 \! -name . -prune -type d | cut -c 3-)
+for build_name in $build_names; do
+    src=${BUILD_PATH}/${IDF_TARGET}/${build_name}
+    dst=${OUTPUT_PATH}/${IDF_TARGET}/${build_name}
+    echo "Copying artifacts from ${src} to ${dst}"
+
+    rm -rf ${dst}
+    mkdir -p ${dst}
+    cp ${src}/{*.bin,*.elf,*.map,sdkconfig,flasher_args.json} ${dst}/
+
+    mkdir -p ${dst}/bootloader
+    cp ${src}/bootloader/*.bin ${dst}/bootloader/
+
+    mkdir -p ${dst}/partition_table
+    cp ${src}/partition_table/*.bin ${dst}/partition_table/
+done
+
+# Check for build warnings
+${IDF_PATH}/tools/ci/check_build_warnings.py -vv ${JOB_BUILD_LIST_JSON}

tools/ci/config/assign-test.yml

@@ -1,4 +1,3 @@
-
 assign_test:
   tags:
     - assign_test
@@ -8,12 +7,13 @@ assign_test:
   # we have a lot build example jobs. now we don't use dependencies, just download all artifacts of build stage.
   dependencies:
     - build_ssc_esp32
-    - build_esp_idf_tests_cmake
+    - build_esp_idf_tests_cmake_esp32
+    - build_esp_idf_tests_cmake_esp32s2
   variables:
     SUBMODULES_TO_FETCH: "components/esptool_py/esptool"
     EXAMPLE_CONFIG_OUTPUT_PATH: "$CI_PROJECT_DIR/examples/test_configs"
     TEST_APP_CONFIG_OUTPUT_PATH: "$CI_PROJECT_DIR/tools/test_apps/test_configs"
-    UNIT_TEST_CASE_FILE: "${CI_PROJECT_DIR}/components/idf_test/unit_test/TestCaseAll.yml"
+    UNIT_TEST_CASE_FILE: "${CI_PROJECT_DIR}/components/idf_test/unit_test"
   artifacts:
     paths:
       - components/idf_test/*/CIConfigs
@@ -55,8 +55,8 @@ update_test_cases:
       - master
       - schedules
   dependencies:
-    - build_esp_idf_tests_make
+    - build_esp_idf_tests_cmake_esp32
-    - build_esp_idf_tests_cmake
+    - build_esp_idf_tests_cmake_esp32s2
   artifacts:
     when: always
     paths:
@@ -64,7 +64,7 @@ update_test_cases:
     expire_in: 1 week
   variables:
     SUBMODULES_TO_FETCH: "components/esptool_py/esptool"
-    UNIT_TEST_CASE_FILE: "${CI_PROJECT_DIR}/components/idf_test/unit_test/TestCaseAll.yml"
+    UNIT_TEST_CASE_DIR: "${CI_PROJECT_DIR}/components/idf_test/unit_test"
     BOT_ACCOUNT_CONFIG_FILE: "${CI_PROJECT_DIR}/test-management/Config/Account.local.yml"
     AUTO_TEST_SCRIPT_PATH: "${CI_PROJECT_DIR}/auto_test_script"
     PYTHON_VER: 3
@@ -75,7 +75,8 @@ update_test_cases:
     - cd test-management
     - echo $BOT_JIRA_ACCOUNT > ${BOT_ACCOUNT_CONFIG_FILE}
     # update unit test cases
-    - python ImportTestCase.py $JIRA_TEST_MANAGEMENT_PROJECT unity -d $UNIT_TEST_CASE_FILE -r $GIT_SHA
+    - export UNIT_TEST_CASE_FILES=$(find $UNIT_TEST_CASE_DIR -maxdepth 1 -name "*.yml" | xargs)
+    - python ImportTestCase.py $JIRA_TEST_MANAGEMENT_PROJECT unity -d $UNIT_TEST_CASE_FILES -r $GIT_SHA
     # update example test cases
     - python ImportTestCase.py $JIRA_TEST_MANAGEMENT_PROJECT tiny_test_fw -d ${CI_PROJECT_DIR}/examples -r $GIT_SHA
     # organize test cases

tools/ci/config/build.yml

@@ -8,27 +8,6 @@
     BATCH_BUILD: "1"
     V: "0"
 
-.build_esp_idf_unit_test_template:
-  extends: .build_template
-  artifacts:
-    paths:
-      - components/idf_test/unit_test/TestCaseAll.yml
-      # Keep only significant files in the output folder (mainly to get rid of .map files)
-      - tools/unit-test-app/output/*/*.bin
-      - tools/unit-test-app/output/*/sdkconfig
-      - tools/unit-test-app/output/*/*.elf
-      - tools/unit-test-app/output/*/flasher_args.json
-      - tools/unit-test-app/output/*/bootloader/*.bin
-      - tools/unit-test-app/output/*/partition_table/*.bin
-    expire_in: 4 days
-  only:
-    variables:
-      - $BOT_TRIGGER_WITH_LABEL == null
-      - $BOT_LABEL_BUILD
-      - $BOT_LABEL_UNIT_TEST
-      - $BOT_LABEL_UNIT_TEST_S2
-      - $BOT_LABEL_REGULAR_TEST
-
 .build_ssc_template:
   extends: .build_template
   parallel: 3
@@ -61,36 +40,42 @@ build_ssc_esp32s2:
   variables:
     TARGET_NAME: "ESP32S2"
 
-build_esp_idf_tests_make:
+.build_esp_idf_tests_cmake:
-  extends: .build_esp_idf_unit_test_template
+  extends: .build_template
+  artifacts:
+    paths:
+      - tools/unit-test-app/output/${IDF_TARGET}
+      - tools/unit-test-app/builds/${IDF_TARGET}/*.json
+      - components/idf_test/unit_test/*.yml
+      - ${LOG_PATH}
+    when: always
+    expire_in: 4 days
+  only:
+    variables:
+      - $BOT_TRIGGER_WITH_LABEL == null
+      - $BOT_LABEL_BUILD
+      - $BOT_LABEL_UNIT_TEST
+      - $BOT_LABEL_UNIT_TEST_S2
+      - $BOT_LABEL_REGULAR_TEST
+  variables:
+    LOG_PATH: "$CI_PROJECT_DIR/log_ut_cmake"
   script:
     - export EXTRA_CFLAGS=${PEDANTIC_CFLAGS}
     - export EXTRA_CXXFLAGS=${PEDANTIC_CXXFLAGS}
-    - export IDF_TARGET=esp32
+    - mkdir -p ${LOG_PATH}
+    - ${CI_PROJECT_DIR}/tools/ci/build_unit_test.sh
     - cd $CI_PROJECT_DIR/tools/unit-test-app
-    - MAKEFLAGS= make help # make sure kconfig tools are built in single process
-    - make ut-clean-all-configs
-    - make ut-build-all-configs
     - python tools/UnitTestParser.py
-    # Check if the tests demand Make built binaries. If not, delete them
-    - if [ "$UNIT_TEST_BUILD_SYSTEM" == "make" ]; then exit 0; fi
-    - rm -rf builds output sdkconfig
-    - rm $CI_PROJECT_DIR/components/idf_test/unit_test/TestCaseAll.yml
 
-build_esp_idf_tests_cmake:
+build_esp_idf_tests_cmake_esp32:
-  extends: .build_esp_idf_unit_test_template
+  extends: .build_esp_idf_tests_cmake
-  script:
+  variables:
-    - export PATH="$IDF_PATH/tools:$PATH"
+    IDF_TARGET: esp32
-    - export EXTRA_CFLAGS=${PEDANTIC_CFLAGS}
+
-    - export EXTRA_CXXFLAGS=${PEDANTIC_CXXFLAGS}
+build_esp_idf_tests_cmake_esp32s2:
-    - cd $CI_PROJECT_DIR/tools/unit-test-app
+  extends: .build_esp_idf_tests_cmake
-    - idf.py ut-clean-all-configs
+  variables:
-    - idf.py ut-build-all-configs
+    IDF_TARGET: esp32s2
-    - python tools/UnitTestParser.py
-    # Check if the tests demand CMake built binaries. If not, delete them
-    - if [ "$UNIT_TEST_BUILD_SYSTEM" == "cmake" ]; then exit 0; fi
-    - rm -rf builds output sdkconfig
-    - rm $CI_PROJECT_DIR/components/idf_test/unit_test/TestCaseAll.yml
 
 build_examples_make:
   extends: .build_template

tools/ci/config/target-test.yml

@@ -110,8 +110,7 @@
   stage: target_test
   dependencies:
     - assign_test
-    - build_esp_idf_tests_make
+    - build_esp_idf_tests_cmake_esp32
-    - build_esp_idf_tests_cmake
   only:
     refs:
       - master
@@ -499,6 +498,9 @@ UT_034:
 
 .unit_test_s2_template:
   extends: .unit_test_template
+  dependencies:
+    - assign_test
+    - build_esp_idf_tests_cmake_esp32s2
   only:
     refs:
       # Due to lack of runners, the tests are only done by manual trigger

tools/ci/executable-list.txt

@@ -36,6 +36,7 @@ tools/ci/apply_bot_filter.py
 tools/ci/build_examples.sh
 tools/ci/build_examples_cmake.sh
 tools/ci/build_test_apps.sh
+tools/ci/build_unit_test.sh
 tools/ci/check-executable.sh
 tools/ci/check-line-endings.sh
 tools/ci/check_build_warnings.py
@@ -43,7 +44,6 @@ tools/ci/check_deprecated_kconfigs.py
 tools/ci/check_examples_cmake_make.sh
 tools/ci/check_examples_rom_header.sh
 tools/ci/check_idf_version.sh
-tools/ci/check_public_headers.sh
 tools/ci/check_ut_cmake_make.sh
 tools/ci/checkout_project_ref.py
 tools/ci/deploy_docs.py

tools/ci/python_packages/tiny_test_fw/Utility/CIAssignTest.py

@@ -53,7 +53,6 @@ from . import (CaseConfig, SearchCases, GitlabCIJob, console_log)
 
 
 class Group(object):
-
     MAX_EXECUTION_TIME = 30
     MAX_CASE = 15
     SORT_KEYS = ["env_tag"]

tools/ci/python_packages/ttfw_idf/CIAssignUnitTest.py

@@ -1,7 +1,7 @@
 """
 Command line tool to assign unit tests to CI test jobs.
 """
-
+import os
 import re
 import argparse
 
@@ -46,7 +46,7 @@ class Group(CIAssignTest.Group):
             for key in self.filters:
                 if self._get_case_attr(case, key) != self.filters[key]:
                     if key == "tags":
-                        if self._get_case_attr(case, key).issubset(self.filters[key]):
+                        if set(self._get_case_attr(case, key)).issubset(set(self.filters[key])):
                             continue
                     break
             else:
@@ -145,15 +145,33 @@ class UnitTestAssignTest(CIAssignTest.AssignTest):
         The unit test cases is stored in a yaml file which is created in job build-idf-test.
         """
 
+        def find_by_suffix(suffix, path):
+            res = []
+            for root, _, files in os.walk(path):
+                for file in files:
+                    if file.endswith(suffix):
+                        res.append(os.path.join(root, file))
+            return res
+
+        def get_test_cases_from_yml(yml_file):
             try:
-            with open(test_case_path, "r") as f:
+                with open(yml_file) as fr:
-                raw_data = yaml.load(f, Loader=Loader)
+                    raw_data = yaml.load(fr, Loader=Loader)
-            test_cases = raw_data["test cases"]
+                test_cases = raw_data['test cases']
-            for case in test_cases:
+            except (IOError, KeyError):
-                case["tags"] = set(case["tags"])
+                return []
-        except IOError:
+            else:
-            print("Test case path is invalid. Should only happen when use @bot to skip unit test.")
+                return test_cases
+
         test_cases = []
+        if os.path.isdir(test_case_path):
+            for yml_file in find_by_suffix('.yml', test_case_path):
+                test_cases.extend(get_test_cases_from_yml(yml_file))
+        elif os.path.isfile(test_case_path):
+            test_cases.extend(get_test_cases_from_yml(test_case_path))
+        else:
+            print("Test case path is invalid. Should only happen when use @bot to skip unit test.")
+
         # filter keys are lower case. Do map lower case keys with original keys.
         try:
             key_mapping = {x.lower(): x for x in test_cases[0].keys()}

tools/ci/python_packages/ttfw_idf/IDFApp.py

@@ -385,7 +385,7 @@ class UT(IDFApp):
             return path
 
         # ``make ut-build-all-configs`` or ``make ut-build-CONFIG`` will copy binary to output folder
-        path = os.path.join(self.idf_path, "tools", "unit-test-app", "output", config_name)
+        path = os.path.join(self.idf_path, "tools", "unit-test-app", "output", target, config_name)
         if os.path.exists(path):
             return path
 

tools/find_apps.py

@@ -35,7 +35,10 @@ def dict_from_sdkconfig(path):
         for line in f:
             m = regex.match(line)
             if m:
-                result[m.group(1)] = m.group(2)
+                val = m.group(2)
+                if val.startswith('"') and val.endswith('"'):
+                    val = val[1:-1]
+                result[m.group(1)] = val
     return result
 
 

tools/find_build_apps/cmake.py

@@ -15,12 +15,21 @@ IDF_PY = "idf.py"
 CMAKE_PROJECT_LINE = r"include($ENV{IDF_PATH}/tools/cmake/project.cmake)"
 
 SUPPORTED_TARGETS_REGEX = re.compile(r'Supported [Tt]argets((?:[\s|]+(?:ESP[0-9A-Z\-]+))+)')
+SDKCONFIG_LINE_REGEX = re.compile(r"^([^=]+)=\"?([^\"\n]*)\"?\n*$")
 
 FORMAL_TO_USUAL = {
     'ESP32': 'esp32',
     'ESP32-S2': 'esp32s2',
 }
 
+# If these keys are present in sdkconfig.defaults, they will be extracted and passed to CMake
+SDKCONFIG_TEST_OPTS = [
+    "EXCLUDE_COMPONENTS",
+    "TEST_EXCLUDE_COMPONENTS",
+    "TEST_COMPONENTS",
+    "TEST_GROUPS"
+]
+
 
 class CMakeBuildSystem(BuildSystem):
     NAME = BUILD_SYSTEM_CMAKE
@@ -69,6 +78,7 @@ class CMakeBuildSystem(BuildSystem):
             if not build_item.dry_run:
                 os.unlink(sdkconfig_file)
 
+        extra_cmakecache_items = {}
         logging.debug("Creating sdkconfig file: {}".format(sdkconfig_file))
         if not build_item.dry_run:
             with open(sdkconfig_file, "w") as f_out:
@@ -81,13 +91,11 @@ class CMakeBuildSystem(BuildSystem):
                         for line in f_in:
                             if not line.endswith("\n"):
                                 line += "\n"
+                            m = SDKCONFIG_LINE_REGEX.match(line)
+                            if m and m.group(1) in SDKCONFIG_TEST_OPTS:
+                                extra_cmakecache_items[m.group(1)] = m.group(2)
+                                continue
                             f_out.write(os.path.expandvars(line))
-            # Also save the sdkconfig file in the build directory
-            shutil.copyfile(
-                os.path.join(work_path, "sdkconfig"),
-                os.path.join(build_path, "sdkconfig"),
-            )
-
         else:
             for sdkconfig_name in sdkconfig_defaults_list:
                 sdkconfig_path = os.path.join(app_path, sdkconfig_name)
@@ -109,6 +117,11 @@ class CMakeBuildSystem(BuildSystem):
             work_path,
             "-DIDF_TARGET=" + build_item.target,
         ]
+        for key, val in extra_cmakecache_items.items():
+            args.append("-D{}={}".format(key, val))
+        if "TEST_EXCLUDE_COMPONENTS" in extra_cmakecache_items \
+                and "TEST_COMPONENTS" not in extra_cmakecache_items:
+            args.append("-DTESTS_ALL=1")
         if build_item.verbose:
             args.append("-v")
         args.append("build")
@@ -131,6 +144,12 @@ class CMakeBuildSystem(BuildSystem):
             subprocess.check_call(args, stdout=build_stdout, stderr=build_stderr)
         except subprocess.CalledProcessError as e:
             raise BuildError("Build failed with exit code {}".format(e.returncode))
+        else:
+            # Also save the sdkconfig file in the build directory
+            shutil.copyfile(
+                os.path.join(work_path, "sdkconfig"),
+                os.path.join(build_path, "sdkconfig"),
+            )
         finally:
             if log_file:
                 log_file.close()

tools/unit-test-app/configs/aes_no_hw

@@ -1,3 +1,4 @@
+# This config is for all targets
 TEST_EXCLUDE_COMPONENTS=libsodium bt app_update test_utils
 TEST_COMPONENTS=mbedtls
 CONFIG_MBEDTLS_HARDWARE_AES=n

tools/unit-test-app/configs/aes_no_hw_s2

@@ -1,4 +0,0 @@
-TEST_EXCLUDE_COMPONENTS=libsodium bt app_update test_utils
-TEST_COMPONENTS=mbedtls
-CONFIG_MBEDTLS_HARDWARE_AES=n
-CONFIG_IDF_TARGET="esp32s2"

tools/unit-test-app/configs/app_update

@@ -1,3 +1,5 @@
+# This config is split between targets since different component needs to be excluded (esp32, esp32s2)
+CONFIG_IDF_TARGET="esp32"
 TEST_COMPONENTS=app_update
 TEST_EXCLUDE_COMPONENTS=libsodium bt
 CONFIG_UNITY_FREERTOS_STACK_SIZE=12288

tools/unit-test-app/configs/app_update_s2

@@ -1,3 +1,5 @@
+# This config is split between targets since different component needs to be excluded (esp32, esp32s2)
+CONFIG_IDF_TARGET="esp32s2"
 TEST_COMPONENTS=app_update
 TEST_EXCLUDE_COMPONENTS=libsodium bt
 CONFIG_UNITY_FREERTOS_STACK_SIZE=12288
@@ -11,4 +13,3 @@ CONFIG_BOOTLOADER_HOLD_TIME_GPIO=2
 CONFIG_BOOTLOADER_OTA_DATA_ERASE=y
 CONFIG_BOOTLOADER_NUM_PIN_FACTORY_RESET=4
 CONFIG_BOOTLOADER_NUM_PIN_APP_TEST=18
-CONFIG_IDF_TARGET="esp32s2"

tools/unit-test-app/configs/bt

@@ -1,3 +1,4 @@
+CONFIG_IDF_TARGET="esp32"
 TEST_COMPONENTS=bt
 TEST_EXCLUDE_COMPONENTS=app_update
 CONFIG_BT_ENABLED=y

tools/unit-test-app/configs/cxx_exceptions

@@ -1,2 +1,4 @@
+# Only need to test this for one target (e.g. ESP32)
+CONFIG_IDF_TARGET="esp32"
 TEST_COMPONENTS=cxx
 CONFIG_COMPILER_CXX_EXCEPTIONS=y

tools/unit-test-app/configs/cxx_rtti

@@ -1,3 +1,5 @@
+# Only need to test this for one target (e.g. ESP32)
+CONFIG_IDF_TARGET="esp32"
 TEST_COMPONENTS=cxx
 CONFIG_COMPILER_CXX_EXCEPTIONS=y
 CONFIG_COMPILER_CXX_RTTI=y

tools/unit-test-app/configs/default

@@ -1 +1,3 @@
+# This config is split between targets since different component needs to be included (esp32, esp32s2)
+CONFIG_IDF_TARGET="esp32"
 TEST_COMPONENTS=freertos esp32 esp_timer driver heap pthread soc spi_flash vfs

tools/unit-test-app/configs/default_2

@@ -1 +1,3 @@
+# This config is split between targets since different component needs to be excluded (esp32, esp32s2)
+CONFIG_IDF_TARGET="esp32"
 TEST_EXCLUDE_COMPONENTS=libsodium bt app_update freertos esp32 esp_timer driver heap pthread soc spi_flash vfs test_utils

tools/unit-test-app/configs/default_2_s2

@@ -1,2 +1,3 @@
-TEST_EXCLUDE_COMPONENTS=libsodium bt app_update freertos esp32s2 esp_timer driver heap pthread soc spi_flash vfs
+# This config is split between targets since different component needs to be excluded (esp32, esp32s2)
 CONFIG_IDF_TARGET="esp32s2"
+TEST_EXCLUDE_COMPONENTS=libsodium bt app_update freertos esp32s2 esp_timer driver heap pthread soc spi_flash vfs

tools/unit-test-app/configs/default_s2

@@ -1,2 +1,3 @@
-TEST_COMPONENTS=freertos esp32s2 esp_timer driver heap pthread soc spi_flash vfs
+# This config is split between targets since different component needs to be included (esp32, esp32s2)
 CONFIG_IDF_TARGET="esp32s2"
+TEST_COMPONENTS=freertos esp32s2 esp_timer driver heap pthread soc spi_flash vfs

tools/unit-test-app/configs/flash_encryption

@@ -1,3 +1,5 @@
+# This config is for ESP32 only (no ESP32-S2 flash encryption support yet)
+CONFIG_IDF_TARGET="esp32"
 TEST_COMPONENTS=spi_flash
 TEST_GROUPS=flash_encryption
 CONFIG_SECURE_FLASH_ENC_ENABLED=y

tools/unit-test-app/configs/freertos_compliance

@@ -1,2 +1,4 @@
+# This config is split between targets since different component needs to be included (esp32, esp32s2)
+CONFIG_IDF_TARGET="esp32"
 TEST_COMPONENTS=driver esp32 esp_timer spi_flash
 CONFIG_FREERTOS_CHECK_PORT_CRITICAL_COMPLIANCE=y

tools/unit-test-app/configs/freertos_compliance_s2

@@ -1,3 +1,4 @@
+# This config is split between targets since different component needs to be included (esp32, esp32s2)
+CONFIG_IDF_TARGET="esp32s2"
 TEST_COMPONENTS=driver esp32s2 esp_timer spi_flash
 CONFIG_FREERTOS_CHECK_PORT_CRITICAL_COMPLIANCE=y
-CONFIG_IDF_TARGET="esp32s2"

tools/unit-test-app/configs/libsodium

@@ -1,3 +1,4 @@
+CONFIG_IDF_TARGET="esp32"
 TEST_COMPONENTS=libsodium
 TEST_EXCLUDE_COMPONENTS=bt app_update
 CONFIG_UNITY_FREERTOS_STACK_SIZE=12288

tools/unit-test-app/configs/psram

@@ -1,3 +1,4 @@
+CONFIG_IDF_TARGET="esp32"
 TEST_EXCLUDE_COMPONENTS=libsodium bt app_update driver esp32 esp_timer mbedtls spi_flash test_utils heap pthread soc
 CONFIG_ESP32_SPIRAM_SUPPORT=y
 CONFIG_ESP_INT_WDT_TIMEOUT_MS=800

tools/unit-test-app/configs/psram_2

@@ -1,3 +1,4 @@
+CONFIG_IDF_TARGET="esp32"
 TEST_COMPONENTS=esp32 esp_timer mbedtls spi_flash heap pthread soc
 CONFIG_ESP32_SPIRAM_SUPPORT=y
 CONFIG_ESP_INT_WDT_TIMEOUT_MS=800

tools/unit-test-app/configs/psram_3

@@ -1,3 +1,4 @@
+CONFIG_IDF_TARGET="esp32"
 TEST_COMPONENTS=driver
 CONFIG_ESP32_SPIRAM_SUPPORT=y
 CONFIG_ESP_INT_WDT_TIMEOUT_MS=800

tools/unit-test-app/configs/psram_8m

@@ -1,3 +1,4 @@
+CONFIG_IDF_TARGET="esp32"
 TEST_COMPONENTS=esp32 esp_timer
 CONFIG_ESP32_SPIRAM_SUPPORT=y
 CONFIG_SPIRAM_BANKSWITCH_ENABLE=y

tools/unit-test-app/configs/psram_hspi

@@ -1,3 +1,4 @@
+CONFIG_IDF_TARGET="esp32"
 TEST_COMPONENTS=esp32
 TEST_GROUPS=psram_4m
 CONFIG_ESPTOOLPY_FLASHFREQ_80M=y

tools/unit-test-app/configs/psram_vspi

@@ -1,3 +1,4 @@
+CONFIG_IDF_TARGET="esp32"
 TEST_COMPONENTS=esp32
 TEST_GROUPS=psram_4m
 CONFIG_ESPTOOLPY_FLASHFREQ_80M=y

tools/unit-test-app/configs/release

@@ -1,3 +1,4 @@
+CONFIG_IDF_TARGET="esp32"
 TEST_COMPONENTS=freertos esp32 esp_timer driver heap pthread soc spi_flash vfs
 CONFIG_COMPILER_OPTIMIZATION_SIZE=y
 CONFIG_BOOTLOADER_COMPILER_OPTIMIZATION_SIZE=y

tools/unit-test-app/configs/release_2

@@ -1,3 +1,5 @@
+# This config is split between targets since different component needs to be included (esp32, esp32s2)
+CONFIG_IDF_TARGET="esp32"
 TEST_EXCLUDE_COMPONENTS=libsodium bt app_update freertos esp32 esp_timer driver heap pthread soc spi_flash vfs test_utils
 CONFIG_COMPILER_OPTIMIZATION_SIZE=y
 CONFIG_BOOTLOADER_COMPILER_OPTIMIZATION_SIZE=y

tools/unit-test-app/configs/release_2_s2

@@ -1,5 +1,6 @@
+# This config is split between targets since different component needs to be excluded (esp32, esp32s2)
+CONFIG_IDF_TARGET="esp32s2"
 TEST_EXCLUDE_COMPONENTS=libsodium bt app_update freertos esp32s2 esp_timer driver heap pthread soc spi_flash vfs test_utils
 CONFIG_COMPILER_OPTIMIZATION_SIZE=y
 CONFIG_BOOTLOADER_COMPILER_OPTIMIZATION_SIZE=y
 CONFIG_COMPILER_OPTIMIZATION_ASSERTIONS_SILENT=y
-CONFIG_IDF_TARGET="esp32s2"

tools/unit-test-app/configs/release_s2

@@ -1,5 +1,6 @@
+# This config is split between targets since different component needs to be included (esp32, esp32s2)
+CONFIG_IDF_TARGET="esp32s2"
 TEST_COMPONENTS=freertos esp32s2 esp_timer driver heap pthread soc spi_flash vfs
 CONFIG_COMPILER_OPTIMIZATION_SIZE=y
 CONFIG_BOOTLOADER_COMPILER_OPTIMIZATION_SIZE=y
 CONFIG_COMPILER_OPTIMIZATION_ASSERTIONS_SILENT=y
-CONFIG_IDF_TARGET="esp32s2"

tools/unit-test-app/configs/single_core

@@ -1,3 +1,5 @@
+# This config is split between targets since different component needs to be included (esp32, esp32s2)
+CONFIG_IDF_TARGET="esp32"
 TEST_COMPONENTS=freertos esp32 esp_timer driver heap pthread soc spi_flash vfs
 CONFIG_MEMMAP_SMP=n
 CONFIG_FREERTOS_UNICORE=y

tools/unit-test-app/configs/single_core_2

@@ -1,3 +1,5 @@
+# This config is split between targets since different component needs to be excluded (esp32, esp32s2)
+CONFIG_IDF_TARGET="esp32"
 TEST_EXCLUDE_COMPONENTS=libsodium bt app_update freertos esp32 esp_timer driver heap pthread soc spi_flash vfs test_utils
 CONFIG_MEMMAP_SMP=n
 CONFIG_FREERTOS_UNICORE=y

tools/unit-test-app/configs/single_core_2_s2

@@ -1,5 +1,6 @@
+# This config is split between targets since different component needs to be excluded (esp32, esp32s2)
+CONFIG_IDF_TARGET="esp32s2"
 TEST_EXCLUDE_COMPONENTS=libsodium bt app_update freertos esp32s2 esp_timer driver heap pthread soc spi_flash vfs
 CONFIG_MEMMAP_SMP=n
 CONFIG_FREERTOS_UNICORE=y
 CONFIG_ESP32_RTCDATA_IN_FAST_MEM=y
-CONFIG_IDF_TARGET="esp32s2"

tools/unit-test-app/configs/single_core_s2

@@ -1,5 +1,6 @@
+# This config is split between targets since different component needs to be included (esp32, esp32s2)
+CONFIG_IDF_TARGET="esp32s2"
 TEST_COMPONENTS=freertos esp32s2 esp_timer driver heap pthread soc spi_flash vfs test_utils
 CONFIG_MEMMAP_SMP=n
 CONFIG_FREERTOS_UNICORE=y
 CONFIG_ESP32_RTCDATA_IN_FAST_MEM=y
-CONFIG_IDF_TARGET="esp32s2"

tools/unit-test-app/configs/spi_flash_legacy

@@ -1,3 +1,4 @@
+# This config is for all targets
 TEST_COMPONENTS=spi_flash
 CONFIG_ESP32_SPIRAM_SUPPORT=y
 CONFIG_SPI_FLASH_USE_LEGACY_IMPL=y

tools/unit-test-app/configs/spi_flash_legacy_s2

@@ -1,4 +0,0 @@
-TEST_COMPONENTS=spi_flash
-CONFIG_ESP32_SPIRAM_SUPPORT=y
-CONFIG_SPI_FLASH_USE_LEGACY_IMPL=y
-CONFIG_IDF_TARGET="esp32s2"

tools/unit-test-app/configs/test_utils

@@ -1,3 +1,4 @@
+# This config is for all targets
 # The test is isolated as it requires particular memory layout
 TEST_COMPONENTS=test_utils
 CONFIG_ESP_IPC_TASK_STACK_SIZE=2048

tools/unit-test-app/configs/test_utils_psram

@@ -1,3 +1,5 @@
+# This config is for esp32 only
+CONFIG_IDF_TARGET="esp32"
 # The test is isolated as it requires particular memory layout
 TEST_COMPONENTS=test_utils
 CONFIG_ESP_IPC_TASK_STACK_SIZE=2048

tools/unit-test-app/configs/test_utils_s2

@@ -1,5 +0,0 @@
-# The test is isolated as it requires particular memory layout
-TEST_COMPONENTS=test_utils
-CONFIG_ESP_IPC_TASK_STACK_SIZE=2048
-CONFIG_IDF_TARGET="esp32s2"
-

tools/unit-test-app/tools/UnitTestParser.py

@@ -4,6 +4,7 @@ import os
 import re
 import shutil
 import subprocess
+import sys
 
 from copy import deepcopy
 import CreateSectionTable
@@ -43,18 +44,24 @@ class Parser(object):
     MODULE_DEF_FILE = os.path.join("tools", "unit-test-app", "tools", "ModuleDefinition.yml")
     CONFIG_DEPENDENCY_FILE = os.path.join("tools", "unit-test-app", "tools", "ConfigDependency.yml")
     MODULE_ARTIFACT_FILE = os.path.join("components", "idf_test", "ModuleDefinition.yml")
-    TEST_CASE_FILE = os.path.join("components", "idf_test", "unit_test", "TestCaseAll.yml")
+    TEST_CASE_FILE_DIR = os.path.join("components", "idf_test", "unit_test")
     UT_BIN_FOLDER = os.path.join("tools", "unit-test-app", "output")
     UT_CONFIG_FOLDER = os.path.join("tools", "unit-test-app", "configs")
     ELF_FILE = "unit-test-app.elf"
     SDKCONFIG_FILE = "sdkconfig"
     STRIP_CONFIG_PATTERN = re.compile(r"(.+?)(_\d+)?$")
+    TOOLCHAIN_FOR_TARGET = {
+        "esp32": "xtensa-esp32-elf-",
+        "esp32s2": "xtensa-esp32s2-elf-",
+    }
 
-    def __init__(self, idf_path=os.getenv("IDF_PATH")):
+    def __init__(self, idf_path=os.getenv("IDF_PATH"), idf_target=os.getenv("IDF_TARGET")):
         self.test_env_tags = {}
         self.unit_jobs = {}
         self.file_name_cache = {}
         self.idf_path = idf_path
+        self.idf_target = idf_target
+        self.objdump = Parser.TOOLCHAIN_FOR_TARGET.get(idf_target, "") + "objdump"
         self.tag_def = yaml.load(open(os.path.join(idf_path, self.TAG_DEF_FILE), "r"), Loader=Loader)
         self.module_map = yaml.load(open(os.path.join(idf_path, self.MODULE_DEF_FILE), "r"), Loader=Loader)
         self.config_dependencies = yaml.load(open(os.path.join(idf_path, self.CONFIG_DEPENDENCY_FILE), "r"),
@@ -66,27 +73,19 @@ class Parser(object):
     def parse_test_cases_for_one_config(self, configs_folder, config_output_folder, config_name):
         """
         parse test cases from elf and save test cases need to be executed to unit test folder
-        :param configs_folder: folder where per-config sdkconfig framents are located (i.e. tools/unit-test-app/configs)
+        :param configs_folder: folder where per-config sdkconfig fragments are located (i.e. tools/unit-test-app/configs)
         :param config_output_folder: build folder of this config
         :param config_name: built unit test config name
         """
         tags = self.parse_tags(os.path.join(config_output_folder, self.SDKCONFIG_FILE))
         print("Tags of config %s: %s" % (config_name, tags))
-        # Search in tags to set the target
-        target_tag_dict = {"ESP32_IDF": "esp32", "ESP32S2_IDF": "esp32s2"}
-        for tag in target_tag_dict:
-            if tag in tags:
-                target = target_tag_dict[tag]
-                break
-        else:
-            target = "esp32"
 
         test_groups = self.get_test_groups(os.path.join(configs_folder, config_name))
 
         elf_file = os.path.join(config_output_folder, self.ELF_FILE)
-        subprocess.check_output('xtensa-esp32-elf-objdump -t {} | grep test_desc > case_address.tmp'.format(elf_file),
+        subprocess.check_output('{} -t {} | grep test_desc > case_address.tmp'.format(self.objdump, elf_file),
                                 shell=True)
-        subprocess.check_output('xtensa-esp32-elf-objdump -s {} > section_table.tmp'.format(elf_file), shell=True)
+        subprocess.check_output('{} -s {} > section_table.tmp'.format(self.objdump, elf_file), shell=True)
 
         table = CreateSectionTable.SectionTable("section_table.tmp")
         test_cases = []
@@ -105,13 +104,11 @@ class Parser(object):
 
                 name_addr = table.get_unsigned_int(section, test_addr, 4)
                 desc_addr = table.get_unsigned_int(section, test_addr + 4, 4)
-                file_name_addr = table.get_unsigned_int(section, test_addr + 12, 4)
                 function_count = table.get_unsigned_int(section, test_addr + 20, 4)
                 name = table.get_string("any", name_addr)
                 desc = table.get_string("any", desc_addr)
-                file_name = table.get_string("any", file_name_addr)
 
-                tc = self.parse_one_test_case(name, desc, file_name, config_name, stripped_config_name, tags, target)
+                tc = self.parse_one_test_case(name, desc, config_name, stripped_config_name, tags)
 
                 # check if duplicated case names
                 # we need to use it to select case,
@@ -229,7 +226,6 @@ class Parser(object):
         :param sdkconfig_file: sdk config file of the unit test config
         :return: required tags for runners
         """
-
         with open(sdkconfig_file, "r") as f:
             configs_raw_data = f.read()
 
@@ -250,12 +246,11 @@ class Parser(object):
                     return match.group(1).split(' ')
         return None
 
-    def parse_one_test_case(self, name, description, file_name, config_name, stripped_config_name, tags, target):
+    def parse_one_test_case(self, name, description, config_name, stripped_config_name, tags):
         """
         parse one test case
         :param name: test case name (summary)
         :param description: test case description (tag string)
-        :param file_name: the file defines this test case
         :param config_name: built unit test app name
         :param stripped_config_name: strip suffix from config name because they're the same except test components
         :param tags: tags to select runners
@@ -279,7 +274,7 @@ class Parser(object):
                           "multi_stage": prop["multi_stage"],
                           "timeout": int(prop["timeout"]),
                           "tags": tags,
-                          "chip_target": target})
+                          "chip_target": self.idf_target})
         return test_case
 
     def dump_test_cases(self, test_cases):
@@ -287,7 +282,7 @@ class Parser(object):
         dump parsed test cases to YAML file for test bench input
         :param test_cases: parsed test cases
         """
-        filename = os.path.join(self.idf_path, self.TEST_CASE_FILE)
+        filename = os.path.join(self.idf_path, self.TEST_CASE_FILE_DIR, self.idf_target + ".yml")
         try:
             os.mkdir(os.path.dirname(filename))
         except OSError:
@@ -305,7 +300,7 @@ class Parser(object):
         """ parse test cases from multiple built unit test apps """
         test_cases = []
 
-        output_folder = os.path.join(self.idf_path, self.UT_BIN_FOLDER)
+        output_folder = os.path.join(self.idf_path, self.UT_BIN_FOLDER, self.idf_target)
         configs_folder = os.path.join(self.idf_path, self.UT_CONFIG_FOLDER)
         test_configs = os.listdir(output_folder)
         for config in test_configs:
@@ -362,14 +357,22 @@ def main():
     test_parser()
 
     idf_path = os.getenv("IDF_PATH")
+    if not idf_path:
+        print("IDF_PATH must be set to use this script", file=sys.stderr)
+        raise SystemExit(1)
 
-    parser = Parser(idf_path)
+    idf_target = os.getenv("IDF_TARGET")
+    if not idf_target:
+        print("IDF_TARGET must be set to use this script", file=sys.stderr)
+        raise SystemExit(1)
+
+    parser = Parser(idf_path, idf_target)
     parser.parse_test_cases()
     parser.copy_module_def_file()
     if len(parser.parsing_errors) > 0:
         for error in parser.parsing_errors:
             print(error)
-        exit(-1)
+        exit(1)
 
 
 if __name__ == '__main__':