CI: download only required bin for unit-tests. Refactor AssignTest related code

tools/ci/config/assign-test.yml

@@ -24,6 +24,7 @@ assign_test:
       - $TEST_APP_CONFIG_OUTPUT_PATH
       - build_examples/artifact_index.json
       - build_test_apps/artifact_index.json
+      - tools/unit-test-app/builds/artifact_index.json
     expire_in: 1 week
   only:
     variables:
@@ -35,11 +36,11 @@ assign_test:
       - $BOT_LABEL_CUSTOM_TEST
   script:
     # assign example tests
-    - python tools/ci/python_packages/ttfw_idf/CIAssignExampleTest.py $IDF_PATH/examples $CI_TARGET_TEST_CONFIG_FILE $EXAMPLE_CONFIG_OUTPUT_PATH
+    - python tools/ci/python_packages/ttfw_idf/IDFAssignTest.py example_test $IDF_PATH/examples $CI_TARGET_TEST_CONFIG_FILE $EXAMPLE_CONFIG_OUTPUT_PATH
     # assign test apps
-    - python tools/ci/python_packages/ttfw_idf/CIAssignExampleTest.py --custom-group test-apps --job-prefix test_app_test_ $IDF_PATH/tools/test_apps $CI_TARGET_TEST_CONFIG_FILE $TEST_APP_CONFIG_OUTPUT_PATH
+    - python tools/ci/python_packages/ttfw_idf/IDFAssignTest.py custom_test  $IDF_PATH/tools/test_apps $CI_TARGET_TEST_CONFIG_FILE $TEST_APP_CONFIG_OUTPUT_PATH
     # assign unit test cases
-    - python tools/ci/python_packages/ttfw_idf/CIAssignUnitTest.py $UNIT_TEST_CASE_FILE $CI_TARGET_TEST_CONFIG_FILE $IDF_PATH/components/idf_test/unit_test/CIConfigs
+    - python tools/ci/python_packages/ttfw_idf/IDFAssignTest.py unit_test $UNIT_TEST_CASE_FILE $CI_TARGET_TEST_CONFIG_FILE $IDF_PATH/components/idf_test/unit_test/CIConfigs
     # clone test script to assign tests
     - ./tools/ci/retry_failed.sh git clone $TEST_SCRIPT_REPOSITORY
     - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script

tools/ci/config/build.yml

@@ -45,7 +45,7 @@ build_ssc_esp32s2:
   artifacts:
     paths:
       - tools/unit-test-app/output/${IDF_TARGET}
-      - tools/unit-test-app/builds/${IDF_TARGET}/*.json
+      - tools/unit-test-app/builds/*.json
       - tools/unit-test-app/builds/${IDF_TARGET}/*/size.json
       - components/idf_test/unit_test/*.yml
       - $LOG_PATH

tools/ci/config/target-test.yml

@@ -109,7 +109,6 @@
   stage: target_test
   dependencies:
     - assign_test
-    - build_esp_idf_tests_cmake_esp32
   only:
     refs:
       - master
@@ -546,7 +545,6 @@ UT_034:
   extends: .unit_test_template
   dependencies:
     - assign_test
-    - build_esp_idf_tests_cmake_esp32s2
   only:
     refs:
       # Due to lack of runners, the tests are only done by manual trigger

tools/ci/find_apps_build_apps.sh

@@ -169,4 +169,7 @@ if [ "${TEST_TYPE}" = "unit_test" ]; then
     mkdir -p ${dst}/partition_table
     cp ${src}/partition_table/*.bin ${dst}/partition_table/
   done
+
+  # Copy app list json files to build path
+  mv ${BUILD_PATH}/${IDF_TARGET}/*.json ${BUILD_PATH}
 fi

tools/ci/python_packages/gitlab_api.py

@@ -109,7 +109,7 @@ class Gitlab(object):
             try:
                 data = job.artifact(a_path)
             except gitlab.GitlabGetError as e:
-                print("Failed to download '{}' form job {}".format(a_path, job_id))
+                print("Failed to download '{}' from job {}".format(a_path, job_id))
                 raise e
             raw_data_list.append(data)
             if destination:

tools/ci/python_packages/ttfw_idf/CIAssignExampleTest.py

@@ -1,114 +0,0 @@
-# Copyright 2015-2017 Espressif Systems (Shanghai) PTE LTD
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http:#www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""
-Command line tool to assign example tests to CI test jobs.
-"""
-
-# TODO: Need to handle running examples on different chips
-import os
-import re
-import argparse
-import json
-
-import gitlab_api
-from tiny_test_fw.Utility import CIAssignTest
-
-IDF_PATH_FROM_ENV = os.getenv("IDF_PATH")
-
-
-class ExampleGroup(CIAssignTest.Group):
-    SORT_KEYS = CI_JOB_MATCH_KEYS = ["env_tag", "target"]
-    BUILD_LOCAL_DIR = "build_examples"
-    BUILD_JOB_NAMES = ["build_examples_cmake_esp32", "build_examples_cmake_esp32s2"]
-
-
-class TestAppsGroup(ExampleGroup):
-    BUILD_LOCAL_DIR = "build_test_apps"
-    BUILD_JOB_NAMES = ["build_test_apps_esp32", "build_test_apps_esp32s2"]
-
-
-class CIExampleAssignTest(CIAssignTest.AssignTest):
-    CI_TEST_JOB_PATTERN = re.compile(r"^example_test_.+")
-
-
-def get_artifact_index_file(case_group=ExampleGroup):
-    if IDF_PATH_FROM_ENV:
-        artifact_index_file = os.path.join(IDF_PATH_FROM_ENV,
-                                           case_group.BUILD_LOCAL_DIR, "artifact_index.json")
-    else:
-        artifact_index_file = "artifact_index.json"
-    return artifact_index_file
-
-
-def create_artifact_index_file(project_id=None, pipeline_id=None, case_group=ExampleGroup):
-    if project_id is None:
-        project_id = os.getenv("CI_PROJECT_ID")
-    if pipeline_id is None:
-        pipeline_id = os.getenv("CI_PIPELINE_ID")
-    gitlab_inst = gitlab_api.Gitlab(project_id)
-    artifact_index_list = []
-
-    def format_build_log_path():
-        parallel = job_info["parallel_num"]  # Could be None if "parallel_num" not defined for the job
-        return "{}/list_job_{}.json".format(case_group.BUILD_LOCAL_DIR, parallel or 1)
-
-    for build_job_name in case_group.BUILD_JOB_NAMES:
-        job_info_list = gitlab_inst.find_job_id(build_job_name, pipeline_id=pipeline_id)
-        for job_info in job_info_list:
-            raw_data = gitlab_inst.download_artifact(job_info["id"], [format_build_log_path()])[0]
-            build_info_list = [json.loads(line) for line in raw_data.decode().splitlines()]
-            for build_info in build_info_list:
-                build_info["ci_job_id"] = job_info["id"]
-                artifact_index_list.append(build_info)
-    artifact_index_file = get_artifact_index_file(case_group=case_group)
-    try:
-        os.makedirs(os.path.dirname(artifact_index_file))
-    except OSError:
-        # already created
-        pass
-
-    with open(artifact_index_file, "w") as f:
-        json.dump(artifact_index_list, f)
-
-
-if __name__ == '__main__':
-    parser = argparse.ArgumentParser()
-    parser.add_argument("test_case",
-                        help="test case folder or file")
-    parser.add_argument("ci_config_file",
-                        help="gitlab ci config file")
-    parser.add_argument("output_path",
-                        help="output path of config files")
-    parser.add_argument("--pipeline_id", "-p", type=int, default=None,
-                        help="pipeline_id")
-    parser.add_argument("--job-prefix",
-                        help="prefix of the test job name in CI yml file")
-    parser.add_argument("--test-case-file-pattern",
-                        help="file name pattern used to find Python test case files")
-    parser.add_argument('--custom-group',
-                        help='select custom-group for the test cases, if other than ExampleTest',
-                        choices=['example', 'test-apps'], default='example')
-
-    args = parser.parse_args()
-
-    if args.job_prefix:
-        CIExampleAssignTest.CI_TEST_JOB_PATTERN = re.compile(r"^{}.+".format(args.job_prefix))
-
-    case_group = ExampleGroup if args.custom_group == 'example' else TestAppsGroup
-
-    assign_test = CIExampleAssignTest(args.test_case, args.ci_config_file, case_group=case_group)
-    assign_test.assign_cases()
-    assign_test.output_configs(args.output_path)
-    create_artifact_index_file(case_group=case_group)

tools/ci/python_packages/ttfw_idf/CIScanTests.py

@@ -3,12 +3,11 @@ import errno
 import json
 import logging
 import os
-import re
 from collections import defaultdict
 
 from find_apps import find_apps
 from find_build_apps import BUILD_SYSTEMS, BUILD_SYSTEM_CMAKE
-from ttfw_idf.CIAssignExampleTest import CIExampleAssignTest, TestAppsGroup, ExampleGroup
+from ttfw_idf.IDFAssignTest import ExampleAssignTest, TestAppsAssignTest
 
 VALID_TARGETS = [
     'esp32',
@@ -98,10 +97,9 @@ def main():
     test_cases = []
     for path in set(args.paths):
         if args.test_type == 'example_test':
-            assign = CIExampleAssignTest(path, args.ci_config_file, ExampleGroup)
+            assign = ExampleAssignTest(path, args.ci_config_file)
         elif args.test_type == 'test_apps':
-            CIExampleAssignTest.CI_TEST_JOB_PATTERN = re.compile(r'^test_app_test_.+')
+            assign = TestAppsAssignTest(path, args.ci_config_file)
-            assign = CIExampleAssignTest(path, args.ci_config_file, TestAppsGroup)
         else:
             raise SystemExit(1)  # which is impossible
 

tools/ci/python_packages/ttfw_idf/IDFApp.py

@@ -13,14 +13,16 @@
 # limitations under the License.
 
 """ IDF Test Applications """
-import subprocess
 import hashlib
 import json
 import os
+import re
+import subprocess
 import sys
+from abc import abstractmethod
 
 from tiny_test_fw import App
-from . import CIAssignExampleTest
+from .IDFAssignTest import ExampleGroup, TestAppsGroup, UnitTestGroup, IDFCaseGroup
 
 try:
     import gitlab_api
@@ -90,32 +92,46 @@ class Artifacts(object):
             ret = None
         return ret
 
-    def download_artifacts(self):
+    def _get_app_base_path(self):
         if self.artifact_info:
-            base_path = os.path.join(self.artifact_info["work_dir"], self.artifact_info["build_dir"])
+            return os.path.join(self.artifact_info["work_dir"], self.artifact_info["build_dir"])
-            job_id = self.artifact_info["ci_job_id"]
-
-            # 1. download flash args file
-            if self.artifact_info["build_system"] == "cmake":
-                flash_arg_file = os.path.join(base_path, "flasher_args.json")
-            else:
-                flash_arg_file = os.path.join(base_path, "download.config")
-
-            self.gitlab_inst.download_artifact(job_id, [flash_arg_file], self.dest_root_path)
-
-            # 2. download all binary files
-            flash_files, flash_settings, app_name = parse_flash_settings(os.path.join(self.dest_root_path,
-                                                                                      flash_arg_file))
-            artifact_files = [os.path.join(base_path, p[1]) for p in flash_files]
-            artifact_files.append(os.path.join(base_path, app_name + ".elf"))
-
-            self.gitlab_inst.download_artifact(job_id, artifact_files, self.dest_root_path)
-
-            # 3. download sdkconfig file
-            self.gitlab_inst.download_artifact(job_id, [os.path.join(os.path.dirname(base_path), "sdkconfig")],
-                                               self.dest_root_path)
         else:
-            base_path = None
+            return None
+
+    def _get_flash_arg_file(self, base_path, job_id):
+        if self.artifact_info["build_system"] == "cmake":
+            flash_arg_file = os.path.join(base_path, "flasher_args.json")
+        else:
+            flash_arg_file = os.path.join(base_path, "download.config")
+
+        self.gitlab_inst.download_artifact(job_id, [flash_arg_file], self.dest_root_path)
+        return flash_arg_file
+
+    def _download_binary_files(self, base_path, job_id, flash_arg_file):
+        flash_files, flash_settings, app_name = parse_flash_settings(os.path.join(self.dest_root_path,
+                                                                                  flash_arg_file))
+        artifact_files = [os.path.join(base_path, p[1]) for p in flash_files]
+        artifact_files.append(os.path.join(base_path, app_name + ".elf"))
+
+        self.gitlab_inst.download_artifact(job_id, artifact_files, self.dest_root_path)
+
+    def _download_sdkconfig_file(self, base_path, job_id):
+        self.gitlab_inst.download_artifact(job_id, [os.path.join(os.path.dirname(base_path), "sdkconfig")],
+                                           self.dest_root_path)
+
+    def download_artifacts(self):
+        if not self.artifact_info:
+            return None
+        base_path = self._get_app_base_path()
+        job_id = self.artifact_info["ci_job_id"]
+        # 1. download flash args file
+        flash_arg_file = self._get_flash_arg_file(base_path, job_id)
+
+        # 2. download all binary files
+        self._download_binary_files(base_path, job_id, flash_arg_file)
+
+        # 3. download sdkconfig file
+        self._download_sdkconfig_file(base_path, job_id)
         return base_path
 
     def download_artifact_files(self, file_names):
@@ -135,6 +151,20 @@ class Artifacts(object):
         return base_path
 
 
+class UnitTestArtifacts(Artifacts):
+    BUILDS_DIR_RE = re.compile(r'^builds/')
+
+    def _get_app_base_path(self):
+        if self.artifact_info:
+            output_dir = self.BUILDS_DIR_RE.sub('output/', self.artifact_info["build_dir"])
+            return os.path.join(self.artifact_info["app_dir"], output_dir)
+        else:
+            return None
+
+    def _download_sdkconfig_file(self, base_path, job_id):
+        self.gitlab_inst.download_artifact(job_id, [os.path.join(base_path, "sdkconfig")], self.dest_root_path)
+
+
 class IDFApp(App.BaseApp):
     """
     Implements common esp-idf application behavior.
@@ -144,13 +174,16 @@ class IDFApp(App.BaseApp):
     IDF_DOWNLOAD_CONFIG_FILE = "download.config"
     IDF_FLASH_ARGS_FILE = "flasher_args.json"
 
-    def __init__(self, app_path, config_name=None, target=None):
+    def __init__(self, app_path, config_name=None, target=None, case_group=IDFCaseGroup, artifact_cls=Artifacts):
         super(IDFApp, self).__init__(app_path)
+        self.app_path = app_path
         self.config_name = config_name
         self.target = target
         self.idf_path = self.get_sdk_path()
-        self.binary_path = self.get_binary_path(app_path, config_name, target)
+        self.case_group = case_group
-        self.elf_file = self._get_elf_file_path(self.binary_path)
+        self.artifact_cls = artifact_cls
+        self.binary_path = self.get_binary_path()
+        self.elf_file = self._get_elf_file_path()
         self._elf_file_sha256 = None
         assert os.path.exists(self.binary_path)
         if self.IDF_DOWNLOAD_CONFIG_FILE not in os.listdir(self.binary_path):
@@ -166,9 +199,16 @@ class IDFApp(App.BaseApp):
         self.flash_files, self.flash_settings = self._parse_flash_download_config()
         self.partition_table = self._parse_partition_table()
 
+    def __str__(self):
+        parts = ['app<{}>'.format(self.app_path)]
+        if self.config_name:
+            parts.extend('config<{}>'.format(self.config_name))
+        if self.target:
+            parts.extend('target<{}>'.format(self.target))
+        return ' '.join(parts)
+
     @classmethod
-    def get_sdk_path(cls):
+    def get_sdk_path(cls):  # type: () -> str
-        # type: () -> str
         idf_path = os.getenv("IDF_PATH")
         assert idf_path
         assert os.path.exists(idf_path)
@@ -184,7 +224,7 @@ class IDFApp(App.BaseApp):
 
     def get_sdkconfig(self):
         """
-        reads sdkconfig and returns a dictionary with all configuredvariables
+        reads sdkconfig and returns a dictionary with all configured variables
 
         :raise: AssertionError: if sdkconfig file does not exist in defined paths
         """
@@ -202,27 +242,35 @@ class IDFApp(App.BaseApp):
                     d[configs[0]] = configs[1].rstrip()
         return d
 
-    def get_binary_path(self, app_path, config_name=None, target=None):
+    @abstractmethod
-        # type: (str, str, str) -> str
+    def _try_get_binary_from_local_fs(self):
-        """
-        get binary path according to input app_path.
-
-        subclass must overwrite this method.
-
-        :param app_path: path of application
-        :param config_name: name of the application build config. Will match any config if None
-        :param target: target name. Will match for target if None
-        :return: abs app binary path
-        """
         pass
 
-    @staticmethod
+    def get_binary_path(self):
-    def _get_elf_file_path(binary_path):
+        path = self._try_get_binary_from_local_fs()
+        if path:
+            return path
+
+        artifacts = self.artifact_cls(self.idf_path,
+                                      self.case_group.get_artifact_index_file(),
+                                      self.app_path, self.config_name, self.target)
+        if isinstance(self, LoadableElfTestApp):
+            assert self.app_files
+            path = artifacts.download_artifact_files(self.app_files)
+        else:
+            path = artifacts.download_artifacts()
+
+        if path:
+            return os.path.join(self.idf_path, path)
+        else:
+            raise OSError("Failed to get binary for {}".format(self))
+
+    def _get_elf_file_path(self):
         ret = ""
-        file_names = os.listdir(binary_path)
+        file_names = os.listdir(self.binary_path)
         for fn in file_names:
             if os.path.splitext(fn)[1] == ".elf":
-                ret = os.path.join(binary_path, fn)
+                ret = os.path.join(self.binary_path, fn)
         return ret
 
     def _parse_flash_download_config(self):
@@ -281,7 +329,7 @@ class IDFApp(App.BaseApp):
                 if isinstance(raw_error, bytes):
                     raw_error = raw_error.decode()
                 if 'Traceback' in raw_error:
-                    # Some exception occured. It is possible that we've tried the wrong binary file.
+                    # Some exception occurred. It is possible that we've tried the wrong binary file.
                     errors.append((path, raw_error))
                     continue
 
@@ -333,62 +381,52 @@ class IDFApp(App.BaseApp):
 
 
 class Example(IDFApp):
+    def __init__(self, app_path, config_name='default', target='esp32', case_group=ExampleGroup, artifacts_cls=Artifacts):
+        if not config_name:
+            config_name = 'default'
+        if not target:
+            target = 'esp32'
+        super(Example, self).__init__(app_path, config_name, target, case_group, artifacts_cls)
+
     def _get_sdkconfig_paths(self):
         """
         overrides the parent method to provide exact path of sdkconfig for example tests
         """
         return [os.path.join(self.binary_path, "..", "sdkconfig")]
 
-    def _try_get_binary_from_local_fs(self, app_path, config_name=None, target=None, local_build_dir="build_examples"):
+    def _try_get_binary_from_local_fs(self):
         # build folder of example path
-        path = os.path.join(self.idf_path, app_path, "build")
+        path = os.path.join(self.idf_path, self.app_path, "build")
         if os.path.exists(path):
             return path
 
-        if not config_name:
-            config_name = "default"
-
-        if not target:
-            target = "esp32"
-
         # Search for CI build folders.
         # Path format: $IDF_PATH/build_examples/app_path_with_underscores/config/target
-        # (see tools/ci/build_examples_cmake.sh)
+        # (see tools/ci/build_examples.sh)
         # For example: $IDF_PATH/build_examples/examples_get-started_blink/default/esp32
-        app_path_underscored = app_path.replace(os.path.sep, "_")
+        app_path_underscored = self.app_path.replace(os.path.sep, "_")
-        example_path = os.path.join(self.idf_path, local_build_dir)
+        example_path = os.path.join(self.idf_path, self.case_group.LOCAL_BUILD_DIR)
         for dirpath in os.listdir(example_path):
             if os.path.basename(dirpath) == app_path_underscored:
-                path = os.path.join(example_path, dirpath, config_name, target, "build")
+                path = os.path.join(example_path, dirpath, self.config_name, self.target, "build")
                 if os.path.exists(path):
                     return path
                 else:
                     return None
 
-    def get_binary_path(self, app_path, config_name=None, target=None):
-        path = self._try_get_binary_from_local_fs(app_path, config_name, target)
-        if path:
-            return path
-        else:
-            artifacts = Artifacts(self.idf_path,
-                                  CIAssignExampleTest.get_artifact_index_file(case_group=CIAssignExampleTest.ExampleGroup),
-                                  app_path, config_name, target)
-            path = artifacts.download_artifacts()
-            if path:
-                return os.path.join(self.idf_path, path)
-            else:
-                raise OSError("Failed to find example binary")
-
 
 class UT(IDFApp):
-    def get_binary_path(self, app_path, config_name=None, target=None):
+    def __init__(self, app_path, config_name='default', target='esp32', case_group=UnitTestGroup, artifacts_cls=UnitTestArtifacts):
         if not config_name:
-            config_name = "default"
+            config_name = 'default'
+        if not target:
+            target = 'esp32'
+        super(UT, self).__init__(app_path, config_name, target, case_group, artifacts_cls)
 
-        path = os.path.join(self.idf_path, app_path)
+    def _try_get_binary_from_local_fs(self):
-        default_build_path = os.path.join(path, "build")
+        path = os.path.join(self.idf_path, self.app_path, "build")
-        if os.path.exists(default_build_path):
+        if os.path.exists(path):
-            return default_build_path
+            return path
 
         # first try to get from build folder of unit-test-app
         path = os.path.join(self.idf_path, "tools", "unit-test-app", "build")
@@ -396,66 +434,44 @@ class UT(IDFApp):
             # found, use bin in build path
             return path
 
-        # ``make ut-build-all-configs`` or ``make ut-build-CONFIG`` will copy binary to output folder
+        # ``build_unit_test.sh`` will copy binary to output folder
-        path = os.path.join(self.idf_path, "tools", "unit-test-app", "output", target, config_name)
+        path = os.path.join(self.idf_path, "tools", "unit-test-app", "output", self.target, self.config_name)
         if os.path.exists(path):
             return path
 
-        raise OSError("Failed to get unit-test-app binary path")
+        return None
 
 
 class TestApp(Example):
-    def get_binary_path(self, app_path, config_name=None, target=None):
+    def __init__(self, app_path, config_name='default', target='esp32', case_group=TestAppsGroup, artifacts_cls=Artifacts):
-        path = self._try_get_binary_from_local_fs(app_path, config_name, target, local_build_dir="build_test_apps")
+        super(TestApp, self).__init__(app_path, config_name, target, case_group, artifacts_cls)
-        if path:
-            return path
-        else:
-            artifacts = Artifacts(self.idf_path,
-                                  CIAssignExampleTest.get_artifact_index_file(case_group=CIAssignExampleTest.TestAppsGroup),
-                                  app_path, config_name, target)
-            path = artifacts.download_artifacts()
-            if path:
-                return os.path.join(self.idf_path, path)
-            else:
-                raise OSError("Failed to find example binary")
 
 
 class LoadableElfTestApp(TestApp):
-    def __init__(self, app_path, app_files, config_name=None, target=None):
+    def __init__(self, app_path, app_files, config_name='default', target='esp32', case_group=TestAppsGroup, artifacts_cls=Artifacts):
         # add arg `app_files` for loadable elf test_app.
         # Such examples only build elf files, so it doesn't generate flasher_args.json.
         # So we can't get app files from config file. Test case should pass it to application.
         super(IDFApp, self).__init__(app_path)
+        self.app_path = app_path
         self.app_files = app_files
-        self.config_name = config_name
+        self.config_name = config_name or 'default'
-        self.target = target
+        self.target = target or 'esp32'
         self.idf_path = self.get_sdk_path()
-        self.binary_path = self.get_binary_path(app_path, config_name, target)
+        self.case_group = case_group
-        self.elf_file = self._get_elf_file_path(self.binary_path)
+        self.artifact_cls = artifacts_cls
+        self.binary_path = self.get_binary_path()
+        self.elf_file = self._get_elf_file_path()
         assert os.path.exists(self.binary_path)
 
-    def get_binary_path(self, app_path, config_name=None, target=None):
-        path = self._try_get_binary_from_local_fs(app_path, config_name, target, local_build_dir="build_test_apps")
-        if path:
-            return path
-        else:
-            artifacts = Artifacts(self.idf_path,
-                                  CIAssignExampleTest.get_artifact_index_file(case_group=CIAssignExampleTest.TestAppsGroup),
-                                  app_path, config_name, target)
-            path = artifacts.download_artifact_files(self.app_files)
-            if path:
-                return os.path.join(self.idf_path, path)
-            else:
-                raise OSError("Failed to find the loadable ELF file")
-
 
 class SSC(IDFApp):
-    def get_binary_path(self, app_path, config_name=None, target=None):
+    def get_binary_path(self):
         # TODO: to implement SSC get binary path
-        return app_path
+        return self.app_path
 
 
 class AT(IDFApp):
-    def get_binary_path(self, app_path, config_name=None, target=None):
+    def get_binary_path(self):
         # TODO: to implement AT get binary path
-        return app_path
+        return self.app_path

tools/ci/python_packages/ttfw_idf/IDFAssignTest.py

@@ -1,9 +1,11 @@
 """
-Command line tool to assign unit tests to CI test jobs.
+Command line tool to assign tests to CI test jobs.
 """
+import argparse
+import errno
+import json
 import os
 import re
-import argparse
 
 import yaml
 
@@ -12,16 +14,88 @@ try:
 except ImportError:
     from yaml import Loader as Loader
 
+import gitlab_api
 from tiny_test_fw.Utility import CIAssignTest
 
+IDF_PATH_FROM_ENV = os.getenv("IDF_PATH")
 
-class Group(CIAssignTest.Group):
+
+class IDFCaseGroup(CIAssignTest.Group):
+    LOCAL_BUILD_DIR = None
+    BUILD_JOB_NAMES = None
+
+    @classmethod
+    def get_artifact_index_file(cls):
+        assert cls.LOCAL_BUILD_DIR
+        if IDF_PATH_FROM_ENV:
+            artifact_index_file = os.path.join(IDF_PATH_FROM_ENV, cls.LOCAL_BUILD_DIR, "artifact_index.json")
+        else:
+            artifact_index_file = "artifact_index.json"
+        return artifact_index_file
+
+
+class IDFAssignTest(CIAssignTest.AssignTest):
+    def format_build_log_path(self, parallel_num):
+        return "{}/list_job_{}.json".format(self.case_group.LOCAL_BUILD_DIR, parallel_num)
+
+    def create_artifact_index_file(self, project_id=None, pipeline_id=None):
+        if project_id is None:
+            project_id = os.getenv("CI_PROJECT_ID")
+        if pipeline_id is None:
+            pipeline_id = os.getenv("CI_PIPELINE_ID")
+        gitlab_inst = gitlab_api.Gitlab(project_id)
+
+        artifact_index_list = []
+        for build_job_name in self.case_group.BUILD_JOB_NAMES:
+            job_info_list = gitlab_inst.find_job_id(build_job_name, pipeline_id=pipeline_id)
+            for job_info in job_info_list:
+                parallel_num = job_info["parallel_num"] or 1  # Could be None if "parallel_num" not defined for the job
+                raw_data = gitlab_inst.download_artifact(job_info["id"],
+                                                         [self.format_build_log_path(parallel_num)])[0]
+                build_info_list = [json.loads(line) for line in raw_data.decode().splitlines()]
+                for build_info in build_info_list:
+                    build_info["ci_job_id"] = job_info["id"]
+                    artifact_index_list.append(build_info)
+        artifact_index_file = self.case_group.get_artifact_index_file()
+        try:
+            os.makedirs(os.path.dirname(artifact_index_file))
+        except OSError as e:
+            if e.errno != errno.EEXIST:
+                raise e
+
+        with open(artifact_index_file, "w") as f:
+            json.dump(artifact_index_list, f)
+
+
+SUPPORTED_TARGETS = [
+    'esp32',
+    'esp32s2',
+]
+
+
+class ExampleGroup(IDFCaseGroup):
+    SORT_KEYS = CI_JOB_MATCH_KEYS = ["env_tag", "target"]
+
+    LOCAL_BUILD_DIR = "build_examples"
+    BUILD_JOB_NAMES = ["build_examples_cmake_{}".format(target) for target in SUPPORTED_TARGETS]
+
+
+class TestAppsGroup(ExampleGroup):
+    LOCAL_BUILD_DIR = "build_test_apps"
+    BUILD_JOB_NAMES = ["build_test_apps_{}".format(target) for target in SUPPORTED_TARGETS]
+
+
+class UnitTestGroup(IDFCaseGroup):
     SORT_KEYS = ["test environment", "tags", "chip_target"]
+    CI_JOB_MATCH_KEYS = ["test environment"]
+
+    LOCAL_BUILD_DIR = "tools/unit-test-app/builds"
+    BUILD_JOB_NAMES = ["build_esp_idf_tests_cmake_{}".format(target) for target in SUPPORTED_TARGETS]
+
     MAX_CASE = 50
     ATTR_CONVERT_TABLE = {
         "execution_time": "execution time"
     }
-    CI_JOB_MATCH_KEYS = ["test environment"]
     DUT_CLS_NAME = {
         "esp32": "ESP32DUT",
         "esp32s2": "ESP32S2DUT",
@@ -29,14 +103,14 @@ class Group(CIAssignTest.Group):
     }
 
     def __init__(self, case):
-        super(Group, self).__init__(case)
+        super(UnitTestGroup, self).__init__(case)
         for tag in self._get_case_attr(case, "tags"):
             self.ci_job_match_keys.add(tag)
 
     @staticmethod
     def _get_case_attr(case, attr):
-        if attr in Group.ATTR_CONVERT_TABLE:
+        if attr in UnitTestGroup.ATTR_CONVERT_TABLE:
-            attr = Group.ATTR_CONVERT_TABLE[attr]
+            attr = UnitTestGroup.ATTR_CONVERT_TABLE[attr]
         return case[attr]
 
     def add_extra_case(self, case):
@@ -133,11 +207,25 @@ class Group(CIAssignTest.Group):
         return output_data
 
 
-class UnitTestAssignTest(CIAssignTest.AssignTest):
+class ExampleAssignTest(IDFAssignTest):
-    CI_TEST_JOB_PATTERN = re.compile(r"^UT_.+")
+    CI_TEST_JOB_PATTERN = re.compile(r'^example_test_.+')
 
-    def __init__(self, test_case_path, ci_config_file):
+    def __init__(self, est_case_path, ci_config_file):
-        CIAssignTest.AssignTest.__init__(self, test_case_path, ci_config_file, case_group=Group)
+        super(ExampleAssignTest, self).__init__(est_case_path, ci_config_file, case_group=ExampleGroup)
+
+
+class TestAppsAssignTest(IDFAssignTest):
+    CI_TEST_JOB_PATTERN = re.compile(r'^test_app_test_.+')
+
+    def __init__(self, est_case_path, ci_config_file):
+        super(TestAppsAssignTest, self).__init__(est_case_path, ci_config_file, case_group=TestAppsGroup)
+
+
+class UnitTestAssignTest(IDFAssignTest):
+    CI_TEST_JOB_PATTERN = re.compile(r'^UT_.+')
+
+    def __init__(self, est_case_path, ci_config_file):
+        super(UnitTestAssignTest, self).__init__(est_case_path, ci_config_file, case_group=UnitTestGroup)
 
     def search_cases(self, case_filter=None):
         """
@@ -203,14 +291,27 @@ class UnitTestAssignTest(CIAssignTest.AssignTest):
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser()
-    parser.add_argument("test_case",
+    parser.add_argument("case_group", choices=["example_test", "custom_test", "unit_test"])
-                        help="test case folder or file")
+    parser.add_argument("test_case", help="test case folder or file")
-    parser.add_argument("ci_config_file",
+    parser.add_argument("ci_config_file", help="gitlab ci config file")
-                        help="gitlab ci config file")
+    parser.add_argument("output_path", help="output path of config files")
-    parser.add_argument("output_path",
+    parser.add_argument("--pipeline_id", "-p", type=int, default=None, help="pipeline_id")
-                        help="output path of config files")
+    parser.add_argument("--test-case-file-pattern", help="file name pattern used to find Python test case files")
     args = parser.parse_args()
 
-    assign_test = UnitTestAssignTest(args.test_case, args.ci_config_file)
+    args_list = [args.test_case, args.ci_config_file]
-    assign_test.assign_cases()
+    if args.case_group == 'example_test':
-    assign_test.output_configs(args.output_path)
+        assigner = ExampleAssignTest(*args_list)
+    elif args.case_group == 'custom_test':
+        assigner = TestAppsAssignTest(*args_list)
+    elif args.case_group == 'unit_test':
+        assigner = UnitTestAssignTest(*args_list)
+    else:
+        raise SystemExit(1)  # which is impossible
+
+    if args.test_case_file_pattern:
+        assigner.CI_TEST_JOB_PATTERN = re.compile(r'{}'.format(args.test_case_file_pattern))
+
+    assigner.assign_cases()
+    assigner.output_configs(args.output_path)
+    assigner.create_artifact_index_file()