From 0478edc6ba16c93d763df2a912122ae198bca45f Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Tue, 21 Jul 2020 16:59:31 +0800 Subject: [PATCH] CI: download only required bin for unit-tests. Refactor AssignTest related code --- tools/ci/config/assign-test.yml | 7 +- tools/ci/config/build.yml | 2 +- tools/ci/config/target-test.yml | 2 - tools/ci/find_apps_build_apps.sh | 3 + tools/ci/python_packages/gitlab_api.py | 2 +- .../ttfw_idf/CIAssignExampleTest.py | 114 -------- .../python_packages/ttfw_idf/CIScanTests.py | 8 +- tools/ci/python_packages/ttfw_idf/IDFApp.py | 256 ++++++++++-------- .../{CIAssignUnitTest.py => IDFAssignTest.py} | 141 ++++++++-- 9 files changed, 269 insertions(+), 266 deletions(-) delete mode 100644 tools/ci/python_packages/ttfw_idf/CIAssignExampleTest.py rename tools/ci/python_packages/ttfw_idf/{CIAssignUnitTest.py => IDFAssignTest.py} (58%) diff --git a/tools/ci/config/assign-test.yml b/tools/ci/config/assign-test.yml index f355685659..795f8af4ce 100644 --- a/tools/ci/config/assign-test.yml +++ b/tools/ci/config/assign-test.yml @@ -24,6 +24,7 @@ assign_test: - $TEST_APP_CONFIG_OUTPUT_PATH - build_examples/artifact_index.json - build_test_apps/artifact_index.json + - tools/unit-test-app/builds/artifact_index.json expire_in: 1 week only: variables: @@ -35,11 +36,11 @@ assign_test: - $BOT_LABEL_CUSTOM_TEST script: # assign example tests - - python tools/ci/python_packages/ttfw_idf/CIAssignExampleTest.py $IDF_PATH/examples $CI_TARGET_TEST_CONFIG_FILE $EXAMPLE_CONFIG_OUTPUT_PATH + - python tools/ci/python_packages/ttfw_idf/IDFAssignTest.py example_test $IDF_PATH/examples $CI_TARGET_TEST_CONFIG_FILE $EXAMPLE_CONFIG_OUTPUT_PATH # assign test apps - - python tools/ci/python_packages/ttfw_idf/CIAssignExampleTest.py --custom-group test-apps --job-prefix test_app_test_ $IDF_PATH/tools/test_apps $CI_TARGET_TEST_CONFIG_FILE $TEST_APP_CONFIG_OUTPUT_PATH + - python tools/ci/python_packages/ttfw_idf/IDFAssignTest.py custom_test $IDF_PATH/tools/test_apps $CI_TARGET_TEST_CONFIG_FILE $TEST_APP_CONFIG_OUTPUT_PATH # assign unit test cases - - python tools/ci/python_packages/ttfw_idf/CIAssignUnitTest.py $UNIT_TEST_CASE_FILE $CI_TARGET_TEST_CONFIG_FILE $IDF_PATH/components/idf_test/unit_test/CIConfigs + - python tools/ci/python_packages/ttfw_idf/IDFAssignTest.py unit_test $UNIT_TEST_CASE_FILE $CI_TARGET_TEST_CONFIG_FILE $IDF_PATH/components/idf_test/unit_test/CIConfigs # clone test script to assign tests - ./tools/ci/retry_failed.sh git clone $TEST_SCRIPT_REPOSITORY - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script diff --git a/tools/ci/config/build.yml b/tools/ci/config/build.yml index 6938628d52..77d99cc15d 100644 --- a/tools/ci/config/build.yml +++ b/tools/ci/config/build.yml @@ -45,7 +45,7 @@ build_ssc_esp32s2: artifacts: paths: - tools/unit-test-app/output/${IDF_TARGET} - - tools/unit-test-app/builds/${IDF_TARGET}/*.json + - tools/unit-test-app/builds/*.json - tools/unit-test-app/builds/${IDF_TARGET}/*/size.json - components/idf_test/unit_test/*.yml - $LOG_PATH diff --git a/tools/ci/config/target-test.yml b/tools/ci/config/target-test.yml index 7e20dc10ed..98096ed223 100644 --- a/tools/ci/config/target-test.yml +++ b/tools/ci/config/target-test.yml @@ -109,7 +109,6 @@ stage: target_test dependencies: - assign_test - - build_esp_idf_tests_cmake_esp32 only: refs: - master @@ -546,7 +545,6 @@ UT_034: extends: .unit_test_template dependencies: - assign_test - - build_esp_idf_tests_cmake_esp32s2 only: refs: # Due to lack of runners, the tests are only done by manual trigger diff --git a/tools/ci/find_apps_build_apps.sh b/tools/ci/find_apps_build_apps.sh index 8f4b205cec..77c68d6859 100755 --- a/tools/ci/find_apps_build_apps.sh +++ b/tools/ci/find_apps_build_apps.sh @@ -169,4 +169,7 @@ if [ "${TEST_TYPE}" = "unit_test" ]; then mkdir -p ${dst}/partition_table cp ${src}/partition_table/*.bin ${dst}/partition_table/ done + + # Copy app list json files to build path + mv ${BUILD_PATH}/${IDF_TARGET}/*.json ${BUILD_PATH} fi diff --git a/tools/ci/python_packages/gitlab_api.py b/tools/ci/python_packages/gitlab_api.py index 561ffb863a..8f5ae4b67c 100644 --- a/tools/ci/python_packages/gitlab_api.py +++ b/tools/ci/python_packages/gitlab_api.py @@ -109,7 +109,7 @@ class Gitlab(object): try: data = job.artifact(a_path) except gitlab.GitlabGetError as e: - print("Failed to download '{}' form job {}".format(a_path, job_id)) + print("Failed to download '{}' from job {}".format(a_path, job_id)) raise e raw_data_list.append(data) if destination: diff --git a/tools/ci/python_packages/ttfw_idf/CIAssignExampleTest.py b/tools/ci/python_packages/ttfw_idf/CIAssignExampleTest.py deleted file mode 100644 index eb54e47092..0000000000 --- a/tools/ci/python_packages/ttfw_idf/CIAssignExampleTest.py +++ /dev/null @@ -1,114 +0,0 @@ -# Copyright 2015-2017 Espressif Systems (Shanghai) PTE LTD -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http:#www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -Command line tool to assign example tests to CI test jobs. -""" - -# TODO: Need to handle running examples on different chips -import os -import re -import argparse -import json - -import gitlab_api -from tiny_test_fw.Utility import CIAssignTest - -IDF_PATH_FROM_ENV = os.getenv("IDF_PATH") - - -class ExampleGroup(CIAssignTest.Group): - SORT_KEYS = CI_JOB_MATCH_KEYS = ["env_tag", "target"] - BUILD_LOCAL_DIR = "build_examples" - BUILD_JOB_NAMES = ["build_examples_cmake_esp32", "build_examples_cmake_esp32s2"] - - -class TestAppsGroup(ExampleGroup): - BUILD_LOCAL_DIR = "build_test_apps" - BUILD_JOB_NAMES = ["build_test_apps_esp32", "build_test_apps_esp32s2"] - - -class CIExampleAssignTest(CIAssignTest.AssignTest): - CI_TEST_JOB_PATTERN = re.compile(r"^example_test_.+") - - -def get_artifact_index_file(case_group=ExampleGroup): - if IDF_PATH_FROM_ENV: - artifact_index_file = os.path.join(IDF_PATH_FROM_ENV, - case_group.BUILD_LOCAL_DIR, "artifact_index.json") - else: - artifact_index_file = "artifact_index.json" - return artifact_index_file - - -def create_artifact_index_file(project_id=None, pipeline_id=None, case_group=ExampleGroup): - if project_id is None: - project_id = os.getenv("CI_PROJECT_ID") - if pipeline_id is None: - pipeline_id = os.getenv("CI_PIPELINE_ID") - gitlab_inst = gitlab_api.Gitlab(project_id) - artifact_index_list = [] - - def format_build_log_path(): - parallel = job_info["parallel_num"] # Could be None if "parallel_num" not defined for the job - return "{}/list_job_{}.json".format(case_group.BUILD_LOCAL_DIR, parallel or 1) - - for build_job_name in case_group.BUILD_JOB_NAMES: - job_info_list = gitlab_inst.find_job_id(build_job_name, pipeline_id=pipeline_id) - for job_info in job_info_list: - raw_data = gitlab_inst.download_artifact(job_info["id"], [format_build_log_path()])[0] - build_info_list = [json.loads(line) for line in raw_data.decode().splitlines()] - for build_info in build_info_list: - build_info["ci_job_id"] = job_info["id"] - artifact_index_list.append(build_info) - artifact_index_file = get_artifact_index_file(case_group=case_group) - try: - os.makedirs(os.path.dirname(artifact_index_file)) - except OSError: - # already created - pass - - with open(artifact_index_file, "w") as f: - json.dump(artifact_index_list, f) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument("test_case", - help="test case folder or file") - parser.add_argument("ci_config_file", - help="gitlab ci config file") - parser.add_argument("output_path", - help="output path of config files") - parser.add_argument("--pipeline_id", "-p", type=int, default=None, - help="pipeline_id") - parser.add_argument("--job-prefix", - help="prefix of the test job name in CI yml file") - parser.add_argument("--test-case-file-pattern", - help="file name pattern used to find Python test case files") - parser.add_argument('--custom-group', - help='select custom-group for the test cases, if other than ExampleTest', - choices=['example', 'test-apps'], default='example') - - args = parser.parse_args() - - if args.job_prefix: - CIExampleAssignTest.CI_TEST_JOB_PATTERN = re.compile(r"^{}.+".format(args.job_prefix)) - - case_group = ExampleGroup if args.custom_group == 'example' else TestAppsGroup - - assign_test = CIExampleAssignTest(args.test_case, args.ci_config_file, case_group=case_group) - assign_test.assign_cases() - assign_test.output_configs(args.output_path) - create_artifact_index_file(case_group=case_group) diff --git a/tools/ci/python_packages/ttfw_idf/CIScanTests.py b/tools/ci/python_packages/ttfw_idf/CIScanTests.py index f78f3ede59..5c2e9004ef 100644 --- a/tools/ci/python_packages/ttfw_idf/CIScanTests.py +++ b/tools/ci/python_packages/ttfw_idf/CIScanTests.py @@ -3,12 +3,11 @@ import errno import json import logging import os -import re from collections import defaultdict from find_apps import find_apps from find_build_apps import BUILD_SYSTEMS, BUILD_SYSTEM_CMAKE -from ttfw_idf.CIAssignExampleTest import CIExampleAssignTest, TestAppsGroup, ExampleGroup +from ttfw_idf.IDFAssignTest import ExampleAssignTest, TestAppsAssignTest VALID_TARGETS = [ 'esp32', @@ -98,10 +97,9 @@ def main(): test_cases = [] for path in set(args.paths): if args.test_type == 'example_test': - assign = CIExampleAssignTest(path, args.ci_config_file, ExampleGroup) + assign = ExampleAssignTest(path, args.ci_config_file) elif args.test_type == 'test_apps': - CIExampleAssignTest.CI_TEST_JOB_PATTERN = re.compile(r'^test_app_test_.+') - assign = CIExampleAssignTest(path, args.ci_config_file, TestAppsGroup) + assign = TestAppsAssignTest(path, args.ci_config_file) else: raise SystemExit(1) # which is impossible diff --git a/tools/ci/python_packages/ttfw_idf/IDFApp.py b/tools/ci/python_packages/ttfw_idf/IDFApp.py index 70677437aa..64c434f2c9 100644 --- a/tools/ci/python_packages/ttfw_idf/IDFApp.py +++ b/tools/ci/python_packages/ttfw_idf/IDFApp.py @@ -13,14 +13,16 @@ # limitations under the License. """ IDF Test Applications """ -import subprocess import hashlib import json import os +import re +import subprocess import sys +from abc import abstractmethod from tiny_test_fw import App -from . import CIAssignExampleTest +from .IDFAssignTest import ExampleGroup, TestAppsGroup, UnitTestGroup, IDFCaseGroup try: import gitlab_api @@ -90,32 +92,46 @@ class Artifacts(object): ret = None return ret - def download_artifacts(self): + def _get_app_base_path(self): if self.artifact_info: - base_path = os.path.join(self.artifact_info["work_dir"], self.artifact_info["build_dir"]) - job_id = self.artifact_info["ci_job_id"] - - # 1. download flash args file - if self.artifact_info["build_system"] == "cmake": - flash_arg_file = os.path.join(base_path, "flasher_args.json") - else: - flash_arg_file = os.path.join(base_path, "download.config") - - self.gitlab_inst.download_artifact(job_id, [flash_arg_file], self.dest_root_path) - - # 2. download all binary files - flash_files, flash_settings, app_name = parse_flash_settings(os.path.join(self.dest_root_path, - flash_arg_file)) - artifact_files = [os.path.join(base_path, p[1]) for p in flash_files] - artifact_files.append(os.path.join(base_path, app_name + ".elf")) - - self.gitlab_inst.download_artifact(job_id, artifact_files, self.dest_root_path) - - # 3. download sdkconfig file - self.gitlab_inst.download_artifact(job_id, [os.path.join(os.path.dirname(base_path), "sdkconfig")], - self.dest_root_path) + return os.path.join(self.artifact_info["work_dir"], self.artifact_info["build_dir"]) else: - base_path = None + return None + + def _get_flash_arg_file(self, base_path, job_id): + if self.artifact_info["build_system"] == "cmake": + flash_arg_file = os.path.join(base_path, "flasher_args.json") + else: + flash_arg_file = os.path.join(base_path, "download.config") + + self.gitlab_inst.download_artifact(job_id, [flash_arg_file], self.dest_root_path) + return flash_arg_file + + def _download_binary_files(self, base_path, job_id, flash_arg_file): + flash_files, flash_settings, app_name = parse_flash_settings(os.path.join(self.dest_root_path, + flash_arg_file)) + artifact_files = [os.path.join(base_path, p[1]) for p in flash_files] + artifact_files.append(os.path.join(base_path, app_name + ".elf")) + + self.gitlab_inst.download_artifact(job_id, artifact_files, self.dest_root_path) + + def _download_sdkconfig_file(self, base_path, job_id): + self.gitlab_inst.download_artifact(job_id, [os.path.join(os.path.dirname(base_path), "sdkconfig")], + self.dest_root_path) + + def download_artifacts(self): + if not self.artifact_info: + return None + base_path = self._get_app_base_path() + job_id = self.artifact_info["ci_job_id"] + # 1. download flash args file + flash_arg_file = self._get_flash_arg_file(base_path, job_id) + + # 2. download all binary files + self._download_binary_files(base_path, job_id, flash_arg_file) + + # 3. download sdkconfig file + self._download_sdkconfig_file(base_path, job_id) return base_path def download_artifact_files(self, file_names): @@ -135,6 +151,20 @@ class Artifacts(object): return base_path +class UnitTestArtifacts(Artifacts): + BUILDS_DIR_RE = re.compile(r'^builds/') + + def _get_app_base_path(self): + if self.artifact_info: + output_dir = self.BUILDS_DIR_RE.sub('output/', self.artifact_info["build_dir"]) + return os.path.join(self.artifact_info["app_dir"], output_dir) + else: + return None + + def _download_sdkconfig_file(self, base_path, job_id): + self.gitlab_inst.download_artifact(job_id, [os.path.join(base_path, "sdkconfig")], self.dest_root_path) + + class IDFApp(App.BaseApp): """ Implements common esp-idf application behavior. @@ -144,13 +174,16 @@ class IDFApp(App.BaseApp): IDF_DOWNLOAD_CONFIG_FILE = "download.config" IDF_FLASH_ARGS_FILE = "flasher_args.json" - def __init__(self, app_path, config_name=None, target=None): + def __init__(self, app_path, config_name=None, target=None, case_group=IDFCaseGroup, artifact_cls=Artifacts): super(IDFApp, self).__init__(app_path) + self.app_path = app_path self.config_name = config_name self.target = target self.idf_path = self.get_sdk_path() - self.binary_path = self.get_binary_path(app_path, config_name, target) - self.elf_file = self._get_elf_file_path(self.binary_path) + self.case_group = case_group + self.artifact_cls = artifact_cls + self.binary_path = self.get_binary_path() + self.elf_file = self._get_elf_file_path() self._elf_file_sha256 = None assert os.path.exists(self.binary_path) if self.IDF_DOWNLOAD_CONFIG_FILE not in os.listdir(self.binary_path): @@ -166,9 +199,16 @@ class IDFApp(App.BaseApp): self.flash_files, self.flash_settings = self._parse_flash_download_config() self.partition_table = self._parse_partition_table() + def __str__(self): + parts = ['app<{}>'.format(self.app_path)] + if self.config_name: + parts.extend('config<{}>'.format(self.config_name)) + if self.target: + parts.extend('target<{}>'.format(self.target)) + return ' '.join(parts) + @classmethod - def get_sdk_path(cls): - # type: () -> str + def get_sdk_path(cls): # type: () -> str idf_path = os.getenv("IDF_PATH") assert idf_path assert os.path.exists(idf_path) @@ -184,7 +224,7 @@ class IDFApp(App.BaseApp): def get_sdkconfig(self): """ - reads sdkconfig and returns a dictionary with all configuredvariables + reads sdkconfig and returns a dictionary with all configured variables :raise: AssertionError: if sdkconfig file does not exist in defined paths """ @@ -202,27 +242,35 @@ class IDFApp(App.BaseApp): d[configs[0]] = configs[1].rstrip() return d - def get_binary_path(self, app_path, config_name=None, target=None): - # type: (str, str, str) -> str - """ - get binary path according to input app_path. - - subclass must overwrite this method. - - :param app_path: path of application - :param config_name: name of the application build config. Will match any config if None - :param target: target name. Will match for target if None - :return: abs app binary path - """ + @abstractmethod + def _try_get_binary_from_local_fs(self): pass - @staticmethod - def _get_elf_file_path(binary_path): + def get_binary_path(self): + path = self._try_get_binary_from_local_fs() + if path: + return path + + artifacts = self.artifact_cls(self.idf_path, + self.case_group.get_artifact_index_file(), + self.app_path, self.config_name, self.target) + if isinstance(self, LoadableElfTestApp): + assert self.app_files + path = artifacts.download_artifact_files(self.app_files) + else: + path = artifacts.download_artifacts() + + if path: + return os.path.join(self.idf_path, path) + else: + raise OSError("Failed to get binary for {}".format(self)) + + def _get_elf_file_path(self): ret = "" - file_names = os.listdir(binary_path) + file_names = os.listdir(self.binary_path) for fn in file_names: if os.path.splitext(fn)[1] == ".elf": - ret = os.path.join(binary_path, fn) + ret = os.path.join(self.binary_path, fn) return ret def _parse_flash_download_config(self): @@ -281,7 +329,7 @@ class IDFApp(App.BaseApp): if isinstance(raw_error, bytes): raw_error = raw_error.decode() if 'Traceback' in raw_error: - # Some exception occured. It is possible that we've tried the wrong binary file. + # Some exception occurred. It is possible that we've tried the wrong binary file. errors.append((path, raw_error)) continue @@ -333,62 +381,52 @@ class IDFApp(App.BaseApp): class Example(IDFApp): + def __init__(self, app_path, config_name='default', target='esp32', case_group=ExampleGroup, artifacts_cls=Artifacts): + if not config_name: + config_name = 'default' + if not target: + target = 'esp32' + super(Example, self).__init__(app_path, config_name, target, case_group, artifacts_cls) + def _get_sdkconfig_paths(self): """ overrides the parent method to provide exact path of sdkconfig for example tests """ return [os.path.join(self.binary_path, "..", "sdkconfig")] - def _try_get_binary_from_local_fs(self, app_path, config_name=None, target=None, local_build_dir="build_examples"): + def _try_get_binary_from_local_fs(self): # build folder of example path - path = os.path.join(self.idf_path, app_path, "build") + path = os.path.join(self.idf_path, self.app_path, "build") if os.path.exists(path): return path - if not config_name: - config_name = "default" - - if not target: - target = "esp32" - # Search for CI build folders. # Path format: $IDF_PATH/build_examples/app_path_with_underscores/config/target - # (see tools/ci/build_examples_cmake.sh) + # (see tools/ci/build_examples.sh) # For example: $IDF_PATH/build_examples/examples_get-started_blink/default/esp32 - app_path_underscored = app_path.replace(os.path.sep, "_") - example_path = os.path.join(self.idf_path, local_build_dir) + app_path_underscored = self.app_path.replace(os.path.sep, "_") + example_path = os.path.join(self.idf_path, self.case_group.LOCAL_BUILD_DIR) for dirpath in os.listdir(example_path): if os.path.basename(dirpath) == app_path_underscored: - path = os.path.join(example_path, dirpath, config_name, target, "build") + path = os.path.join(example_path, dirpath, self.config_name, self.target, "build") if os.path.exists(path): return path else: return None - def get_binary_path(self, app_path, config_name=None, target=None): - path = self._try_get_binary_from_local_fs(app_path, config_name, target) - if path: - return path - else: - artifacts = Artifacts(self.idf_path, - CIAssignExampleTest.get_artifact_index_file(case_group=CIAssignExampleTest.ExampleGroup), - app_path, config_name, target) - path = artifacts.download_artifacts() - if path: - return os.path.join(self.idf_path, path) - else: - raise OSError("Failed to find example binary") - class UT(IDFApp): - def get_binary_path(self, app_path, config_name=None, target=None): + def __init__(self, app_path, config_name='default', target='esp32', case_group=UnitTestGroup, artifacts_cls=UnitTestArtifacts): if not config_name: - config_name = "default" + config_name = 'default' + if not target: + target = 'esp32' + super(UT, self).__init__(app_path, config_name, target, case_group, artifacts_cls) - path = os.path.join(self.idf_path, app_path) - default_build_path = os.path.join(path, "build") - if os.path.exists(default_build_path): - return default_build_path + def _try_get_binary_from_local_fs(self): + path = os.path.join(self.idf_path, self.app_path, "build") + if os.path.exists(path): + return path # first try to get from build folder of unit-test-app path = os.path.join(self.idf_path, "tools", "unit-test-app", "build") @@ -396,66 +434,44 @@ class UT(IDFApp): # found, use bin in build path return path - # ``make ut-build-all-configs`` or ``make ut-build-CONFIG`` will copy binary to output folder - path = os.path.join(self.idf_path, "tools", "unit-test-app", "output", target, config_name) + # ``build_unit_test.sh`` will copy binary to output folder + path = os.path.join(self.idf_path, "tools", "unit-test-app", "output", self.target, self.config_name) if os.path.exists(path): return path - raise OSError("Failed to get unit-test-app binary path") + return None class TestApp(Example): - def get_binary_path(self, app_path, config_name=None, target=None): - path = self._try_get_binary_from_local_fs(app_path, config_name, target, local_build_dir="build_test_apps") - if path: - return path - else: - artifacts = Artifacts(self.idf_path, - CIAssignExampleTest.get_artifact_index_file(case_group=CIAssignExampleTest.TestAppsGroup), - app_path, config_name, target) - path = artifacts.download_artifacts() - if path: - return os.path.join(self.idf_path, path) - else: - raise OSError("Failed to find example binary") + def __init__(self, app_path, config_name='default', target='esp32', case_group=TestAppsGroup, artifacts_cls=Artifacts): + super(TestApp, self).__init__(app_path, config_name, target, case_group, artifacts_cls) class LoadableElfTestApp(TestApp): - def __init__(self, app_path, app_files, config_name=None, target=None): + def __init__(self, app_path, app_files, config_name='default', target='esp32', case_group=TestAppsGroup, artifacts_cls=Artifacts): # add arg `app_files` for loadable elf test_app. # Such examples only build elf files, so it doesn't generate flasher_args.json. # So we can't get app files from config file. Test case should pass it to application. super(IDFApp, self).__init__(app_path) + self.app_path = app_path self.app_files = app_files - self.config_name = config_name - self.target = target + self.config_name = config_name or 'default' + self.target = target or 'esp32' self.idf_path = self.get_sdk_path() - self.binary_path = self.get_binary_path(app_path, config_name, target) - self.elf_file = self._get_elf_file_path(self.binary_path) + self.case_group = case_group + self.artifact_cls = artifacts_cls + self.binary_path = self.get_binary_path() + self.elf_file = self._get_elf_file_path() assert os.path.exists(self.binary_path) - def get_binary_path(self, app_path, config_name=None, target=None): - path = self._try_get_binary_from_local_fs(app_path, config_name, target, local_build_dir="build_test_apps") - if path: - return path - else: - artifacts = Artifacts(self.idf_path, - CIAssignExampleTest.get_artifact_index_file(case_group=CIAssignExampleTest.TestAppsGroup), - app_path, config_name, target) - path = artifacts.download_artifact_files(self.app_files) - if path: - return os.path.join(self.idf_path, path) - else: - raise OSError("Failed to find the loadable ELF file") - class SSC(IDFApp): - def get_binary_path(self, app_path, config_name=None, target=None): + def get_binary_path(self): # TODO: to implement SSC get binary path - return app_path + return self.app_path class AT(IDFApp): - def get_binary_path(self, app_path, config_name=None, target=None): + def get_binary_path(self): # TODO: to implement AT get binary path - return app_path + return self.app_path diff --git a/tools/ci/python_packages/ttfw_idf/CIAssignUnitTest.py b/tools/ci/python_packages/ttfw_idf/IDFAssignTest.py similarity index 58% rename from tools/ci/python_packages/ttfw_idf/CIAssignUnitTest.py rename to tools/ci/python_packages/ttfw_idf/IDFAssignTest.py index cec26ee62c..28e568ebb4 100644 --- a/tools/ci/python_packages/ttfw_idf/CIAssignUnitTest.py +++ b/tools/ci/python_packages/ttfw_idf/IDFAssignTest.py @@ -1,9 +1,11 @@ """ -Command line tool to assign unit tests to CI test jobs. +Command line tool to assign tests to CI test jobs. """ +import argparse +import errno +import json import os import re -import argparse import yaml @@ -12,16 +14,88 @@ try: except ImportError: from yaml import Loader as Loader +import gitlab_api from tiny_test_fw.Utility import CIAssignTest +IDF_PATH_FROM_ENV = os.getenv("IDF_PATH") -class Group(CIAssignTest.Group): + +class IDFCaseGroup(CIAssignTest.Group): + LOCAL_BUILD_DIR = None + BUILD_JOB_NAMES = None + + @classmethod + def get_artifact_index_file(cls): + assert cls.LOCAL_BUILD_DIR + if IDF_PATH_FROM_ENV: + artifact_index_file = os.path.join(IDF_PATH_FROM_ENV, cls.LOCAL_BUILD_DIR, "artifact_index.json") + else: + artifact_index_file = "artifact_index.json" + return artifact_index_file + + +class IDFAssignTest(CIAssignTest.AssignTest): + def format_build_log_path(self, parallel_num): + return "{}/list_job_{}.json".format(self.case_group.LOCAL_BUILD_DIR, parallel_num) + + def create_artifact_index_file(self, project_id=None, pipeline_id=None): + if project_id is None: + project_id = os.getenv("CI_PROJECT_ID") + if pipeline_id is None: + pipeline_id = os.getenv("CI_PIPELINE_ID") + gitlab_inst = gitlab_api.Gitlab(project_id) + + artifact_index_list = [] + for build_job_name in self.case_group.BUILD_JOB_NAMES: + job_info_list = gitlab_inst.find_job_id(build_job_name, pipeline_id=pipeline_id) + for job_info in job_info_list: + parallel_num = job_info["parallel_num"] or 1 # Could be None if "parallel_num" not defined for the job + raw_data = gitlab_inst.download_artifact(job_info["id"], + [self.format_build_log_path(parallel_num)])[0] + build_info_list = [json.loads(line) for line in raw_data.decode().splitlines()] + for build_info in build_info_list: + build_info["ci_job_id"] = job_info["id"] + artifact_index_list.append(build_info) + artifact_index_file = self.case_group.get_artifact_index_file() + try: + os.makedirs(os.path.dirname(artifact_index_file)) + except OSError as e: + if e.errno != errno.EEXIST: + raise e + + with open(artifact_index_file, "w") as f: + json.dump(artifact_index_list, f) + + +SUPPORTED_TARGETS = [ + 'esp32', + 'esp32s2', +] + + +class ExampleGroup(IDFCaseGroup): + SORT_KEYS = CI_JOB_MATCH_KEYS = ["env_tag", "target"] + + LOCAL_BUILD_DIR = "build_examples" + BUILD_JOB_NAMES = ["build_examples_cmake_{}".format(target) for target in SUPPORTED_TARGETS] + + +class TestAppsGroup(ExampleGroup): + LOCAL_BUILD_DIR = "build_test_apps" + BUILD_JOB_NAMES = ["build_test_apps_{}".format(target) for target in SUPPORTED_TARGETS] + + +class UnitTestGroup(IDFCaseGroup): SORT_KEYS = ["test environment", "tags", "chip_target"] + CI_JOB_MATCH_KEYS = ["test environment"] + + LOCAL_BUILD_DIR = "tools/unit-test-app/builds" + BUILD_JOB_NAMES = ["build_esp_idf_tests_cmake_{}".format(target) for target in SUPPORTED_TARGETS] + MAX_CASE = 50 ATTR_CONVERT_TABLE = { "execution_time": "execution time" } - CI_JOB_MATCH_KEYS = ["test environment"] DUT_CLS_NAME = { "esp32": "ESP32DUT", "esp32s2": "ESP32S2DUT", @@ -29,14 +103,14 @@ class Group(CIAssignTest.Group): } def __init__(self, case): - super(Group, self).__init__(case) + super(UnitTestGroup, self).__init__(case) for tag in self._get_case_attr(case, "tags"): self.ci_job_match_keys.add(tag) @staticmethod def _get_case_attr(case, attr): - if attr in Group.ATTR_CONVERT_TABLE: - attr = Group.ATTR_CONVERT_TABLE[attr] + if attr in UnitTestGroup.ATTR_CONVERT_TABLE: + attr = UnitTestGroup.ATTR_CONVERT_TABLE[attr] return case[attr] def add_extra_case(self, case): @@ -133,11 +207,25 @@ class Group(CIAssignTest.Group): return output_data -class UnitTestAssignTest(CIAssignTest.AssignTest): - CI_TEST_JOB_PATTERN = re.compile(r"^UT_.+") +class ExampleAssignTest(IDFAssignTest): + CI_TEST_JOB_PATTERN = re.compile(r'^example_test_.+') - def __init__(self, test_case_path, ci_config_file): - CIAssignTest.AssignTest.__init__(self, test_case_path, ci_config_file, case_group=Group) + def __init__(self, est_case_path, ci_config_file): + super(ExampleAssignTest, self).__init__(est_case_path, ci_config_file, case_group=ExampleGroup) + + +class TestAppsAssignTest(IDFAssignTest): + CI_TEST_JOB_PATTERN = re.compile(r'^test_app_test_.+') + + def __init__(self, est_case_path, ci_config_file): + super(TestAppsAssignTest, self).__init__(est_case_path, ci_config_file, case_group=TestAppsGroup) + + +class UnitTestAssignTest(IDFAssignTest): + CI_TEST_JOB_PATTERN = re.compile(r'^UT_.+') + + def __init__(self, est_case_path, ci_config_file): + super(UnitTestAssignTest, self).__init__(est_case_path, ci_config_file, case_group=UnitTestGroup) def search_cases(self, case_filter=None): """ @@ -203,14 +291,27 @@ class UnitTestAssignTest(CIAssignTest.AssignTest): if __name__ == '__main__': parser = argparse.ArgumentParser() - parser.add_argument("test_case", - help="test case folder or file") - parser.add_argument("ci_config_file", - help="gitlab ci config file") - parser.add_argument("output_path", - help="output path of config files") + parser.add_argument("case_group", choices=["example_test", "custom_test", "unit_test"]) + parser.add_argument("test_case", help="test case folder or file") + parser.add_argument("ci_config_file", help="gitlab ci config file") + parser.add_argument("output_path", help="output path of config files") + parser.add_argument("--pipeline_id", "-p", type=int, default=None, help="pipeline_id") + parser.add_argument("--test-case-file-pattern", help="file name pattern used to find Python test case files") args = parser.parse_args() - assign_test = UnitTestAssignTest(args.test_case, args.ci_config_file) - assign_test.assign_cases() - assign_test.output_configs(args.output_path) + args_list = [args.test_case, args.ci_config_file] + if args.case_group == 'example_test': + assigner = ExampleAssignTest(*args_list) + elif args.case_group == 'custom_test': + assigner = TestAppsAssignTest(*args_list) + elif args.case_group == 'unit_test': + assigner = UnitTestAssignTest(*args_list) + else: + raise SystemExit(1) # which is impossible + + if args.test_case_file_pattern: + assigner.CI_TEST_JOB_PATTERN = re.compile(r'{}'.format(args.test_case_file_pattern)) + + assigner.assign_cases() + assigner.output_configs(args.output_path) + assigner.create_artifact_index_file()