CI: test-apps introduction

Introducing feature of adding arbitrary projects which could be build or
executed in the CI for the only purpose as testing

Closes IDF-641

tools/ci/build_test_apps.sh

@@ -0,0 +1,96 @@
+#!/bin/bash
+#
+# Build test apps
+#
+# Runs as part of CI process.
+#
+
+# -----------------------------------------------------------------------------
+# Safety settings (see https://gist.github.com/ilg-ul/383869cbb01f61a51c4d).
+
+if [[ ! -z ${DEBUG_SHELL} ]]
+then
+  set -x # Activate the expand mode if DEBUG is anything but empty.
+fi
+
+set -o errexit # Exit if command failed.
+set -o pipefail # Exit if pipe failed.
+
+export PATH="$IDF_PATH/tools/ci:$IDF_PATH/tools:$PATH"
+
+# -----------------------------------------------------------------------------
+
+die() {
+    echo "${1:-"Unknown Error"}" 1>&2
+    exit 1
+}
+
+[ -z ${IDF_PATH} ] && die "IDF_PATH is not set"
+[ -z ${LOG_PATH} ] && die "LOG_PATH is not set"
+[ -z ${BUILD_PATH} ] && die "BUILD_PATH is not set"
+[ -z ${IDF_TARGET} ] && die "IDF_TARGET is not set"
+[ -d ${LOG_PATH} ] || mkdir -p ${LOG_PATH}
+[ -d ${BUILD_PATH} ] || mkdir -p ${BUILD_PATH}
+
+if [ -z ${CI_NODE_TOTAL} ]; then
+    CI_NODE_TOTAL=1
+    echo "Assuming CI_NODE_TOTAL=${CI_NODE_TOTAL}"
+fi
+if [ -z ${CI_NODE_INDEX} ]; then
+    # Gitlab uses a 1-based index
+    CI_NODE_INDEX=1
+    echo "Assuming CI_NODE_INDEX=${CI_NODE_INDEX}"
+fi
+
+
+set -o nounset # Exit if variable not set.
+
+# Convert LOG_PATH to relative, to make the json file less verbose.
+LOG_PATH=$(realpath --relative-to ${IDF_PATH} ${LOG_PATH})
+BUILD_PATH=$(realpath --relative-to ${IDF_PATH} ${BUILD_PATH})
+
+ALL_BUILD_LIST_JSON="${BUILD_PATH}/list.json"
+JOB_BUILD_LIST_JSON="${BUILD_PATH}/list_job_${CI_NODE_INDEX}.json"
+mkdir -p "${BUILD_PATH}/example_builds"
+
+echo "build_examples running for target $IDF_TARGET"
+
+cd ${IDF_PATH}
+
+# This part of the script produces the same result for all the example build jobs. It may be moved to a separate stage
+# (pre-build) later, then the build jobs will receive ${BUILD_LIST_JSON} file as an artifact.
+
+# If changing the work-dir or build-dir, remember to update the "artifacts" in gitlab-ci configs, and IDFApp.py.
+
+${IDF_PATH}/tools/find_apps.py tools/test_apps \
+    -vv \
+    --format json \
+    --build-system cmake \
+    --target ${IDF_TARGET} \
+    --recursive \
+    --build-dir "\${IDF_PATH}/${BUILD_PATH}/@f/@w/@t/build" \
+    --build-log "${LOG_PATH}/@f.txt" \
+    --output ${ALL_BUILD_LIST_JSON} \
+    --config 'sdkconfig.ci=default' \
+    --config 'sdkconfig.ci.*=' \
+    --config '=default' \
+
+# --config rules above explained:
+# 1. If sdkconfig.ci exists, use it build the example with configuration name "default"
+# 2. If sdkconfig.ci.* exists, use it to build the "*" configuration
+# 3. If none of the above exist, build the default configuration under the name "default"
+
+# The part below is where the actual builds happen
+
+${IDF_PATH}/tools/build_apps.py \
+    -vv \
+    --format json \
+    --keep-going \
+    --parallel-count ${CI_NODE_TOTAL} \
+    --parallel-index ${CI_NODE_INDEX} \
+    --output-build-list ${JOB_BUILD_LIST_JSON} \
+    ${ALL_BUILD_LIST_JSON}\
+
+
+# Check for build warnings
+${IDF_PATH}/tools/ci/check_build_warnings.py -vv ${JOB_BUILD_LIST_JSON}

tools/ci/config/assign-test.yml

@@ -5,19 +5,22 @@ assign_test:
   image: $CI_DOCKER_REGISTRY/ubuntu-test-env$BOT_DOCKER_IMAGE_TAG
   stage: assign_test
   # gitlab ci do not support match job with RegEx or wildcard now in dependencies.
-  # we have a lot build example jobs. now we don't use dependencies, just download all artificats of build stage.
+  # we have a lot build example jobs. now we don't use dependencies, just download all artifacts of build stage.
   dependencies:
     - build_ssc_esp32
     - build_esp_idf_tests_cmake
+    - build_test_apps_esp32
   variables:
     SUBMODULES_TO_FETCH: "components/esptool_py/esptool"
     EXAMPLE_CONFIG_OUTPUT_PATH: "$CI_PROJECT_DIR/examples/test_configs"
+    TEST_APP_CONFIG_OUTPUT_PATH: "$CI_PROJECT_DIR/tools/test_apps/test_configs"
     UNIT_TEST_CASE_FILE: "${CI_PROJECT_DIR}/components/idf_test/unit_test/TestCaseAll.yml"
   artifacts:
     paths:
       - components/idf_test/*/CIConfigs
       - components/idf_test/*/TC.sqlite
       - $EXAMPLE_CONFIG_OUTPUT_PATH
+      - $TEST_APP_CONFIG_OUTPUT_PATH
       - build_examples/artifact_index.json
     expire_in: 1 week
   only:
@@ -29,6 +32,8 @@ assign_test:
   script:
     # assign example tests
     - python tools/ci/python_packages/ttfw_idf/CIAssignExampleTest.py $IDF_PATH/examples $CI_TARGET_TEST_CONFIG_FILE $EXAMPLE_CONFIG_OUTPUT_PATH
+    # assign test apps
+    - python tools/ci/python_packages/ttfw_idf/CIAssignExampleTest.py --job-prefix test_app_test_ $IDF_PATH/tools/test_apps $CI_TARGET_TEST_CONFIG_FILE $TEST_APP_CONFIG_OUTPUT_PATH
     # assign unit test cases
     - python tools/ci/python_packages/ttfw_idf/CIAssignUnitTest.py $UNIT_TEST_CASE_FILE $CI_TARGET_TEST_CONFIG_FILE $IDF_PATH/components/idf_test/unit_test/CIConfigs
     # clone test script to assign tests

tools/ci/config/build.yml

@@ -184,6 +184,50 @@ build_examples_cmake_esp32s2:
   variables:
     IDF_TARGET: esp32s2
 
+.build_test_apps: &build_test_apps
+  extends: .build_template
+  parallel: 2
+  stage: pre_build
+  artifacts:
+    when: always
+    paths:
+      - build_test_apps/list.json
+      - build_test_apps/list_job_*.json
+      - build_test_apps/*/*/*/build/*.bin
+      - build_test_apps/*/*/*/sdkconfig
+      - build_test_apps/*/*/*/build/*.elf
+      - build_test_apps/*/*/*/build/*.map
+      - build_test_apps/*/*/*/build/flasher_args.json
+      - build_test_apps/*/*/*/build/bootloader/*.bin
+      - build_test_apps/*/*/*/build/partition_table/*.bin
+      - $LOG_PATH
+    expire_in: 3 days
+  variables:
+    LOG_PATH: "$CI_PROJECT_DIR/log_test_apps"
+    BUILD_PATH: "$CI_PROJECT_DIR/build_test_apps"
+  only:
+    variables:
+      - $BOT_TRIGGER_WITH_LABEL == null
+      - $BOT_LABEL_BUILD
+      - $BOT_LABEL_INTEGRATION_TEST
+      - $BOT_LABEL_REGULAR_TEST
+      - $BOT_LABEL_WEEKEND_TEST
+  script:
+    - mkdir -p ${BUILD_PATH}
+    - mkdir -p ${LOG_PATH}
+    - ${IDF_PATH}/tools/ci/build_test_apps.sh
+
+build_test_apps_esp32:
+  extends: .build_test_apps
+  variables:
+    IDF_TARGET: esp32
+
+build_test_apps_esp32s2:
+  extends: .build_test_apps
+  variables:
+    IDF_TARGET: esp32s2beta
+
+
 # If you want to add new build example jobs, please add it into dependencies of `.example_test_template`
 
 build_docs:

tools/ci/config/target-test.yml

@@ -83,6 +83,30 @@
     # run test
     - python Runner.py $TEST_CASE_PATH -c $CONFIG_FILE -e $ENV_FILE
 
+.test_app_template:
+  extends: .example_test_template
+  stage: pre_target_test
+  dependencies:
+    - assign_test
+    - build_test_apps_esp32
+  variables:
+    TEST_FW_PATH: "$CI_PROJECT_DIR/tools/tiny-test-fw"
+    TEST_CASE_PATH: "$CI_PROJECT_DIR/tools/test_apps"
+    CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/tools/test_apps/test_configs"
+    LOG_PATH: "$CI_PROJECT_DIR/TEST_LOGS"
+    ENV_FILE: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/EnvConfig.yml"
+  script:
+    - *define_config_file_name
+    # first test if config file exists, if not exist, exit 0
+    - test -e $CONFIG_FILE || exit 0
+    # clone test env configs
+    - git clone $TEST_ENV_CONFIG_REPOSITORY
+    - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs
+    - cd $TEST_FW_PATH
+    # run test
+    - python Runner.py $TEST_CASE_PATH -c $CONFIG_FILE -e $ENV_FILE
+
+
 .unit_test_template:
   extends: .example_test_template
   stage: target_test
@@ -279,6 +303,12 @@ example_test_010:
     - ESP32
     - Example_ExtFlash
 
+test_app_test_001:
+  extends: .test_app_template
+  tags:
+    - ESP32
+    - test_jtag_arm
+
 example_test_011:
   extends: .example_debug_template
   tags:

tools/ci/executable-list.txt

@@ -33,6 +33,7 @@ tools/check_python_dependencies.py
 tools/ci/apply_bot_filter.py
 tools/ci/build_examples.sh
 tools/ci/build_examples_cmake.sh
+tools/ci/build_test_apps.sh
 tools/ci/check-executable.sh
 tools/ci/check-line-endings.sh
 tools/ci/check_build_warnings.py

tools/ci/python_packages/tiny_test_fw/Utility/CIAssignTest.py

@@ -148,6 +148,7 @@ class AssignTest(object):
 
     def __init__(self, test_case_path, ci_config_file, case_group=Group):
         self.test_case_path = test_case_path
+        self.test_case_file_pattern = None
         self.test_cases = []
         self.jobs = self._parse_gitlab_ci_config(ci_config_file)
         self.case_group = case_group
@@ -177,7 +178,7 @@ class AssignTest(object):
         job_list.sort(key=lambda x: x["name"])
         return job_list
 
-    def _search_cases(self, test_case_path, case_filter=None):
+    def _search_cases(self, test_case_path, case_filter=None, test_case_file_pattern=None):
         """
         :param test_case_path: path contains test case folder
         :param case_filter: filter for test cases. the filter to use is default filter updated with case_filter param.
@@ -186,7 +187,7 @@ class AssignTest(object):
         _case_filter = self.DEFAULT_FILTER.copy()
         if case_filter:
             _case_filter.update(case_filter)
-        test_methods = SearchCases.Search.search_test_cases(test_case_path)
+        test_methods = SearchCases.Search.search_test_cases(test_case_path, test_case_file_pattern)
         return CaseConfig.filter_test_cases(test_methods, _case_filter)
 
     def _group_cases(self):
@@ -276,7 +277,7 @@ class AssignTest(object):
         failed_to_assign = []
         assigned_groups = []
         case_filter = self._apply_bot_filter()
-        self.test_cases = self._search_cases(self.test_case_path, case_filter)
+        self.test_cases = self._search_cases(self.test_case_path, case_filter, self.test_case_file_pattern)
         self._apply_bot_test_count()
         test_groups = self._group_cases()
 

tools/ci/python_packages/tiny_test_fw/Utility/SearchCases.py

@@ -93,14 +93,14 @@ class Search(object):
         return replicated_cases
 
     @classmethod
-    def search_test_cases(cls, test_case):
+    def search_test_cases(cls, test_case, test_case_file_pattern=None):
         """
         search all test cases from a folder or file, and then do case replicate.
 
         :param test_case: test case file(s) path
         :return: a list of replicated test methods
         """
-        test_case_files = cls._search_test_case_files(test_case, cls.TEST_CASE_FILE_PATTERN)
+        test_case_files = cls._search_test_case_files(test_case, test_case_file_pattern or cls.TEST_CASE_FILE_PATTERN)
         test_cases = []
         for test_case_file in test_case_files:
             test_cases += cls._search_cases_from_file(test_case_file)

tools/ci/python_packages/ttfw_idf/CIAssignExampleTest.py

@@ -82,8 +82,19 @@ if __name__ == '__main__':
                         help="output path of config files")
     parser.add_argument("--pipeline_id", "-p", type=int, default=None,
                         help="pipeline_id")
+    parser.add_argument("--job-prefix",
+                        help="prefix of the test job name in CI yml file")
+    parser.add_argument("--test-case-file-pattern",
+                        help="file name pattern used to find Python test case files")
     args = parser.parse_args()
 
+    if args.job_prefix:
+        CIExampleAssignTest.CI_TEST_JOB_PATTERN = re.compile(r"^{}.+".format(args.job_prefix))
+
+    assign_test = CIExampleAssignTest(args.test_case, args.ci_config_file, case_group=ExampleGroup)
+    if args.test_case_file_pattern:
+        assign_test.test_case_file_pattern = args.test_case_file_pattern
+
     assign_test = CIExampleAssignTest(args.test_case, args.ci_config_file, case_group=ExampleGroup)
     assign_test.assign_cases()
     assign_test.output_configs(args.output_path)

tools/ci/python_packages/ttfw_idf/IDFApp.py

@@ -402,6 +402,36 @@ class UT(IDFApp):
         raise OSError("Failed to get unit-test-app binary path")
 
 
+class TestApp(IDFApp):
+    def _get_sdkconfig_paths(self):
+        """
+        overrides the parent method to provide exact path of sdkconfig for example tests
+        """
+        return [os.path.join(self.binary_path, "..", "sdkconfig")]
+
+    def get_binary_path(self, app_path, config_name=None):
+        # local build folder
+        path = os.path.join(self.idf_path, app_path, "build")
+        if os.path.exists(path):
+            return path
+
+        if not config_name:
+            config_name = "default"
+
+        # Search for CI build folders.
+        # Path format: $IDF_PATH/build_test_apps/app_path_with_underscores/config/target
+        # (see tools/ci/build_test_apps.sh)
+        # For example: $IDF_PATH/build_test_apps/startup/default/esp32
+        app_path_underscored = app_path.replace(os.path.sep, "_")
+        build_root = os.path.join(self.idf_path, "build_test_apps")
+        for dirpath in os.listdir(build_root):
+            if os.path.basename(dirpath) == app_path_underscored:
+                path = os.path.join(build_root, dirpath, config_name, self.target, "build")
+                return path
+
+        raise OSError("Failed to find test app binary")
+
+
 class SSC(IDFApp):
     def get_binary_path(self, app_path, config_name=None, target=None):
         # TODO: to implement SSC get binary path

tools/ci/python_packages/ttfw_idf/__init__.py

@@ -15,7 +15,7 @@ import os
 import re
 
 from tiny_test_fw import TinyFW, Utility
-from .IDFApp import IDFApp, Example, LoadableElfExample, UT  # noqa: export all Apps for users
+from .IDFApp import IDFApp, Example, LoadableElfExample, UT, TestApp  # noqa: export all Apps for users
 from .IDFDUT import IDFDUT, ESP32DUT, ESP32S2DUT, ESP8266DUT, ESP32QEMUDUT  # noqa: export DUTs for users
 
 
@@ -88,6 +88,38 @@ def idf_unit_test(app=UT, dut=IDFDUT, chip="ESP32", module="unit-test", executio
     return test
 
 
+def idf_test_app_test(app=TestApp, dut=IDFDUT, chip="ESP32", module="misc", execution_time=1,
+                      level="integration", erase_nvs=True, **kwargs):
+    """
+    decorator for testing idf unit tests (with default values for some keyword args).
+
+    :param app: test application class
+    :param dut: dut class
+    :param chip: chip supported, string or tuple
+    :param module: module, string
+    :param execution_time: execution time in minutes, int
+    :param level: test level, could be used to filter test cases, string
+    :param erase_nvs: if need to erase_nvs in DUT.start_app()
+    :param kwargs: other keyword args
+    :return: test method
+    """
+    try:
+        # try to config the default behavior of erase nvs
+        dut.ERASE_NVS = erase_nvs
+    except AttributeError:
+        pass
+
+    original_method = TinyFW.test_method(app=app, dut=dut, chip=chip, module=module,
+                                         execution_time=execution_time, level=level, **kwargs)
+
+    def test(func):
+        test_func = original_method(func)
+        test_func.case_info["ID"] = format_case_id(chip, test_func.case_info["name"])
+        return test_func
+
+    return test
+
+
 def log_performance(item, value):
     """
     do print performance with pre-defined format to console

tools/test_apps/startup/CMakeLists.txt

@@ -0,0 +1,6 @@
+# The following lines of boilerplate have to be in your project's
+# CMakeLists in this exact order for cmake to work correctly
+cmake_minimum_required(VERSION 3.5)
+
+include($ENV{IDF_PATH}/tools/cmake/project.cmake)
+project(test_startup)

tools/test_apps/startup/README.txt

@@ -0,0 +1,4 @@
+This project tests if the app can start up in a certain configuration.
+To add new configuration, create one more sdkconfig.ci.NAME file in this directory.
+
+If you need to test for anything other than app starting up, create another test project.

tools/test_apps/startup/main/CMakeLists.txt

@@ -0,0 +1,2 @@
+idf_component_register(SRCS "test_startup_main.c"
+                       INCLUDE_DIRS ".")

tools/test_apps/startup/main/test_startup_main.c

@@ -0,0 +1,6 @@
+#include <stdio.h>
+
+void app_main(void)
+{
+    printf("app_main running\n");
+}

tools/test_apps/startup/sdkconfig.ci.flash_80m_qio

@@ -0,0 +1,2 @@
+CONFIG_ESPTOOLPY_FLASHFREQ_80M=y
+CONFIG_ESPTOOLPY_FLASHMODE_QIO=y

tools/test_apps/startup/test.py

@@ -0,0 +1,35 @@
+#!/usr/bin/env python
+
+import re
+import os
+import sys
+import glob
+
+try:
+    import IDF
+except ImportError:
+    # This environment variable is expected on the host machine
+    test_fw_path = os.getenv("TEST_FW_PATH")
+    if test_fw_path and test_fw_path not in sys.path:
+        sys.path.insert(0, test_fw_path)
+
+    import IDF
+
+import Utility
+
+
+@IDF.idf_test_app_test(env_tag="test_jtag_arm")
+def test_startup(env, extra_data):
+    config_files = glob.glob(os.path.join(os.path.dirname(__file__), "sdkconfig.ci.*"))
+    config_names = [s.replace("sdkconfig.ci.", "") for s in config_files]
+    for name in config_names:
+        Utility.console_log("Checking config \"{}\"... ".format(name), end="")
+        dut = env.get_dut("startup", "tools/test_apps/startup", app_config_name=name)
+        dut.start_app()
+        dut.expect("app_main running")
+        env.close_dut(dut.name)
+        Utility.console_log("done")
+
+
+if __name__ == '__main__':
+    test_startup()