From 96aeead6f5cb36a91466062da4637f3e0aad8e02 Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Tue, 21 Nov 2023 09:02:59 +0100 Subject: [PATCH 01/18] ci: fix ttfw type hint --- tools/ci/python_packages/gitlab_api.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tools/ci/python_packages/gitlab_api.py b/tools/ci/python_packages/gitlab_api.py index 1482ddd52f..32652ec3b4 100644 --- a/tools/ci/python_packages/gitlab_api.py +++ b/tools/ci/python_packages/gitlab_api.py @@ -1,5 +1,6 @@ # SPDX-FileCopyrightText: 2022-2023 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 + import argparse import logging import os @@ -10,7 +11,7 @@ import tempfile import time import zipfile from functools import wraps -from typing import Any, Callable, Dict, List, Optional +from typing import Any, Callable, Dict, List, Optional, Union import gitlab @@ -63,7 +64,7 @@ class Gitlab(object): DOWNLOAD_ERROR_MAX_RETRIES = 3 - def __init__(self, project_id: Optional[int] = None): + def __init__(self, project_id: Union[int, str, None] = None): config_data_from_env = os.getenv('PYTHON_GITLAB_CONFIG') if config_data_from_env: # prefer to load config from env variable @@ -129,7 +130,7 @@ class Gitlab(object): archive_file.extractall(destination) @retry - def download_artifact(self, job_id: int, artifact_path: str, destination: Optional[str] = None) -> List[bytes]: + def download_artifact(self, job_id: int, artifact_path: List[str], destination: Optional[str] = None) -> List[bytes]: """ download specific path of job artifacts and extract to destination. From b709c880dde0f9e502396c77c605876ba90d5d46 Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Mon, 18 Dec 2023 15:42:56 +0100 Subject: [PATCH 02/18] ci: add linter for gitlab yaml files - remove duplicated artifacts default values - migrate check_artifacts_expire_time.py - migrate check_rules_yml.py --- .gitlab/ci/build.yml | 7 -- .gitlab/ci/common.yml | 3 + .gitlab/ci/docs.yml | 7 -- .gitlab/ci/host-test.yml | 12 -- .gitlab/ci/pre_check.yml | 9 -- .gitlab/ci/rules.yml | 10 -- .gitlab/ci/static-code-analysis.yml | 6 - .gitlab/ci/target-test.yml | 2 - .pre-commit-config.yaml | 17 +-- tools/ci/check_artifacts_expire_time.py | 54 -------- tools/ci/check_rules_yml.py | 157 ------------------------ tools/ci/exclude_check_tools_files.txt | 1 + tools/ci/executable-list.txt | 2 +- tools/ci/generate_rules.py | 8 +- tools/ci/gitlab_yaml_linter.py | 100 +++++++++++++++ tools/ci/idf_ci_utils.py | 76 +++++++++++- 16 files changed, 190 insertions(+), 281 deletions(-) delete mode 100644 tools/ci/check_artifacts_expire_time.py delete mode 100755 tools/ci/check_rules_yml.py create mode 100755 tools/ci/gitlab_yaml_linter.py diff --git a/.gitlab/ci/build.yml b/.gitlab/ci/build.yml index 64f4020e56..074c744ecf 100644 --- a/.gitlab/ci/build.yml +++ b/.gitlab/ci/build.yml @@ -32,8 +32,6 @@ # keep the size info to help track the binary size - size_info.txt - "**/build*/size.json" - when: always - expire_in: 4 days script: # CI specific options start from "--parallel-count xxx". could ignore when running locally - run_cmd python tools/ci/ci_build_apps.py $TEST_DIR -v @@ -376,12 +374,10 @@ build_only_tools_test_apps: BUILD_LOG_CMAKE: "${LOG_PATH}/cmake_@t_@w.txt" BUILD_COMMAND_ARGS: "" artifacts: - when: always paths: - log_template_app/* - size_info.txt - build_template_app/**/size.json - expire_in: 1 week script: # Set the variable for 'esp-idf-template' testing - ESP_IDF_TEMPLATE_GIT=${ESP_IDF_TEMPLATE_GIT:-"https://github.com/espressif/esp-idf-template.git"} @@ -554,7 +550,6 @@ pytest_build_system: paths: - XUNIT_RESULT.xml - test_build_system - when: always expire_in: 2 days reports: junit: XUNIT_RESULT.xml @@ -571,7 +566,6 @@ pytest_build_system_macos: paths: - XUNIT_RESULT.xml - test_build_system - when: always expire_in: 2 days reports: junit: XUNIT_RESULT.xml @@ -603,7 +597,6 @@ pytest_build_system_win: paths: - XUNIT_RESULT.xml - test_build_system - when: always expire_in: 2 days reports: junit: XUNIT_RESULT.xml diff --git a/.gitlab/ci/common.yml b/.gitlab/ci/common.yml index 7323505d8d..d2c48531be 100644 --- a/.gitlab/ci/common.yml +++ b/.gitlab/ci/common.yml @@ -345,6 +345,9 @@ default: - *setup_tools_and_idf_python_venv - add_gitlab_ssh_keys - fetch_submodules + artifacts: + expire_in: 1 week + when: always retry: max: 2 when: diff --git a/.gitlab/ci/docs.yml b/.gitlab/ci/docs.yml index 6e77737a36..809b1612d7 100644 --- a/.gitlab/ci/docs.yml +++ b/.gitlab/ci/docs.yml @@ -121,7 +121,6 @@ build_docs_html_full: artifacts: false optional: true artifacts: - when: always paths: - docs/_build/*/*/*.txt - docs/_build/*/*/html/* @@ -135,7 +134,6 @@ build_docs_html_full_prod: - .doc-rules:build:docs-full-prod dependencies: [] # Stop build_docs jobs from downloading all previous job's artifacts artifacts: - when: always paths: - docs/_build/*/*/*.txt - docs/_build/*/*/html/* @@ -152,7 +150,6 @@ build_docs_html_partial: artifacts: false optional: true artifacts: - when: always paths: - docs/_build/*/*/*.txt - docs/_build/*/*/html/* @@ -175,7 +172,6 @@ build_docs_pdf: artifacts: false optional: true artifacts: - when: always paths: - docs/_build/*/*/latex/* expire_in: 4 days @@ -188,7 +184,6 @@ build_docs_pdf_prod: - .doc-rules:build:docs-full-prod dependencies: [] # Stop build_docs jobs from downloading all previous job's artifacts artifacts: - when: always paths: - docs/_build/*/*/latex/* expire_in: 4 days @@ -266,11 +261,9 @@ check_doc_links: artifacts: false tags: ["build", "amd64", "internet"] artifacts: - when: always paths: - docs/_build/*/*/*.txt - docs/_build/*/*/linkcheck/*.txt - expire_in: 1 week allow_failure: true script: - cd docs diff --git a/.gitlab/ci/host-test.yml b/.gitlab/ci/host-test.yml index ab1d62e588..f025824206 100644 --- a/.gitlab/ci/host-test.yml +++ b/.gitlab/ci/host-test.yml @@ -28,7 +28,6 @@ test_nvs_coverage: artifacts: paths: - components/nvs_flash/test_nvs_host/coverage_report - expire_in: 1 week script: - cd components/nvs_flash/test_nvs_host - make coverage_report @@ -65,7 +64,6 @@ test_reproducible_build: - "**/build*/*.bin" - "**/build*/bootloader/*.bin" - "**/build*/partition_table/*.bin" - expire_in: 1 week test_spiffs_on_host: extends: .host_test_template @@ -110,7 +108,6 @@ test_cli_installer: paths: - tools/tools.new.json - tools/test_idf_tools/test_python_env_logs.txt - expire_in: 1 week image: name: $ESP_ENV_IMAGE entrypoint: [""] # use system python3. no extra pip package installed @@ -130,7 +127,6 @@ test_cli_installer: when: on_failure paths: - components/efuse/${IDF_TARGET}/esp_efuse_table.c - expire_in: 1 week script: - cd ${IDF_PATH}/components/efuse/ - ./efuse_table_gen.py -t "${IDF_TARGET}" ${IDF_PATH}/components/efuse/${IDF_TARGET}/esp_efuse_table.csv @@ -173,7 +169,6 @@ test_logtrace_proc: paths: - tools/esp_app_trace/test/logtrace/output - tools/esp_app_trace/test/logtrace/.coverage - expire_in: 1 week script: - cd ${IDF_PATH}/tools/esp_app_trace/test/logtrace - ./test.sh @@ -185,7 +180,6 @@ test_sysviewtrace_proc: paths: - tools/esp_app_trace/test/sysview/output - tools/esp_app_trace/test/sysview/.coverage - expire_in: 1 week script: - cd ${IDF_PATH}/tools/esp_app_trace/test/sysview - ./test.sh @@ -194,13 +188,11 @@ test_tools: extends: - .host_test_template artifacts: - when: always paths: - ${IDF_PATH}/*.out - ${IDF_PATH}/XUNIT_*.xml reports: junit: ${IDF_PATH}/XUNIT_*.xml - expire_in: 1 week variables: LC_ALL: C.UTF-8 INSTALL_QEMU: 1 # for test_idf_qemu.py @@ -280,13 +272,11 @@ test_pytest_qemu: - .host_test_template - .before_script:build artifacts: - when: always paths: - XUNIT_RESULT.xml - pytest_embedded_log/ reports: junit: XUNIT_RESULT.xml - expire_in: 1 week allow_failure: true # IDFCI-1752 parallel: matrix: @@ -316,14 +306,12 @@ test_pytest_linux: - .host_test_template - .before_script:build artifacts: - when: always paths: - XUNIT_RESULT.xml - pytest_embedded_log/ - "**/build*/build_log.txt" reports: junit: XUNIT_RESULT.xml - expire_in: 1 week script: - run_cmd python tools/ci/ci_build_apps.py components examples tools/test_apps -vv --target linux diff --git a/.gitlab/ci/pre_check.yml b/.gitlab/ci/pre_check.yml index 6ff57bfc86..b95c86d8d0 100644 --- a/.gitlab/ci/pre_check.yml +++ b/.gitlab/ci/pre_check.yml @@ -84,7 +84,6 @@ check_chip_support_components: paths: - esp_hw_support_part.h - bootloader_support_part.h - expire_in: 1 week script: - python tools/ci/check_soc_headers_leak.py - find ${IDF_PATH}/components/soc/*/include/soc/ -name "*_struct.h" -print0 | xargs -0 -n1 ./tools/ci/check_soc_struct_headers.py @@ -98,7 +97,6 @@ check_esp_err_to_name: when: on_failure paths: - components/esp_common/esp_err_to_name.c - expire_in: 1 week script: - cd ${IDF_PATH}/tools/ - ./gen_esp_err_to_name.py @@ -122,12 +120,6 @@ check_version_tag: script: - (git cat-file -t $CI_COMMIT_REF_NAME | grep tag) || (echo "ESP-IDF versions must be annotated tags." && exit 1) -check_artifacts_expire_time: - extends: .pre_check_template - script: - # check if we have set expire time for all artifacts - - python tools/ci/check_artifacts_expire_time.py - check_test_scripts_build_test_rules: extends: - .pre_check_template @@ -165,4 +157,3 @@ pipeline_variables: artifacts: reports: dotenv: pipeline.env - expire_in: 4 days diff --git a/.gitlab/ci/rules.yml b/.gitlab/ci/rules.yml index f6d5b68d69..dea9e18d24 100644 --- a/.gitlab/ci/rules.yml +++ b/.gitlab/ci/rules.yml @@ -344,9 +344,6 @@ .if-dev-push: &if-dev-push if: '$CI_COMMIT_REF_NAME != "master" && $CI_COMMIT_BRANCH !~ /^release\/v/ && $CI_COMMIT_TAG !~ /^v\d+\.\d+(\.\d+)?($|-)/ && $CI_COMMIT_TAG !~ /^qa-test/ && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event")' -.if-merge_request: &if-merge_request - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' - .if-schedule: &if-schedule if: '$CI_PIPELINE_SOURCE == "schedule"' @@ -356,9 +353,6 @@ .if-schedule-test-build-system-windows: &if-schedule-test-build-system-windows if: '$CI_PIPELINE_SOURCE == "schedule" && $SCHEDULED_BUILD_SYSTEM_TEST_WIN == "true"' -.if-trigger: &if-trigger - if: '$CI_PIPELINE_SOURCE == "trigger"' - .if-label-build-only: &if-label-build-only if: '$CI_JOB_STAGE == "target_test" && $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*For Maintainers: Only Build Tests(?:,[^,\n\r]+)*$/i' @@ -398,10 +392,6 @@ - <<: *if-protected-no_label when: always -.rules:mr: - rules: - - <<: *if-merge_request - .rules:tag:release: rules: - <<: *if-tag-release diff --git a/.gitlab/ci/static-code-analysis.yml b/.gitlab/ci/static-code-analysis.yml index ef99c8bf45..e1ef1d0d54 100644 --- a/.gitlab/ci/static-code-analysis.yml +++ b/.gitlab/ci/static-code-analysis.yml @@ -6,8 +6,6 @@ clang_tidy_check: artifacts: paths: - clang_tidy_reports/ - when: always - expire_in: 1 day variables: IDF_TOOLCHAIN: clang script: @@ -23,10 +21,8 @@ check_pylint: needs: - pipeline_variables artifacts: - when: always reports: codequality: pylint.json - expire_in: 1 week script: - | if [ -n "$CI_MERGE_REQUEST_IID" ]; then @@ -72,10 +68,8 @@ check_pylint: GIT_DEPTH: 0 REPORT_PATTERN: clang_tidy_reports/**/*.txt artifacts: - when: always paths: - $REPORT_PATTERN - expire_in: 1 week dependencies: # Here is not a hard dependency relationship, could be skipped when only python files changed. so we do not use "needs" here. - clang_tidy_check diff --git a/.gitlab/ci/target-test.yml b/.gitlab/ci/target-test.yml index 350b5d017f..28d46e23ce 100644 --- a/.gitlab/ci/target-test.yml +++ b/.gitlab/ci/target-test.yml @@ -18,13 +18,11 @@ extends: - .target_test_template artifacts: - when: always paths: - XUNIT_RESULT.xml - pytest_embedded_log/ reports: junit: XUNIT_RESULT.xml - expire_in: 1 week script: - retry_failed git clone $KNOWN_FAILURE_CASES_REPO known_failure_cases # get runner env config file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8735049c26..8ae2d5b14d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -91,20 +91,13 @@ repos: always_run: true files: '\.gitlab/CODEOWNERS' pass_filenames: false - - id: check-rules-yml - name: Check rules.yml all rules have at lease one job applied, all rules needed exist - entry: tools/ci/check_rules_yml.py - language: python - files: '\.gitlab/ci/.+\.yml|\.gitlab-ci.yml|\.gitmodules' - pass_filenames: false - additional_dependencies: - - PyYAML == 5.3.1 - id: check-generated-rules name: Check rules are generated (based on .gitlab/ci/dependencies/dependencies.yml) entry: tools/ci/generate_rules.py language: python files: '\.gitlab/ci/dependencies/.+|\.gitlab/ci/.*\.yml' pass_filenames: false + require_serial: true additional_dependencies: - PyYAML == 5.3.1 - id: mypy-check @@ -185,6 +178,14 @@ repos: language: python always_run: true require_serial: true + - id: gitlab-yaml-linter + name: Check gitlab yaml files + entry: tools/ci/gitlab_yaml_linter.py + language: python + files: '\.gitlab-ci\.yml|\.gitlab/ci/.+\.yml' + pass_filenames: false + additional_dependencies: + - PyYAML == 5.3.1 - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.0.1 hooks: diff --git a/tools/ci/check_artifacts_expire_time.py b/tools/ci/check_artifacts_expire_time.py deleted file mode 100644 index ff298cac38..0000000000 --- a/tools/ci/check_artifacts_expire_time.py +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env python -# SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD -# SPDX-License-Identifier: Apache-2.0 - -# internal use only -# check if expire time is set for all artifacts - -import os - -import yaml - -IDF_PATH = os.getenv('IDF_PATH') -if not IDF_PATH: - print('Please set IDF_PATH before running this script') - raise SystemExit(-1) - -GITLAB_CONFIG_FILE = os.path.join(IDF_PATH, '.gitlab-ci.yml') - - -def check_artifacts_expire_time() -> None: - with open(GITLAB_CONFIG_FILE, 'r') as f: - config = yaml.load(f, Loader=yaml.FullLoader) - - # load files listed in `include` - if 'include' in config: - for _file in config['include']: - with open(os.path.join(IDF_PATH or '', _file)) as f: - config.update(yaml.load(f, Loader=yaml.FullLoader)) - - print('expire time for jobs:') - errors = [] - - job_names = list(config.keys()) - job_names.sort() - - for job_name in job_names: - try: - if 'expire_in' not in config[job_name]['artifacts']: - errors.append(job_name) - else: - print('{}: {}'.format(job_name, config[job_name]['artifacts']['expire_in'])) - except (KeyError, TypeError): - # this is not job, or the job does not have artifacts - pass - - if errors: - print('\n\nThe following jobs did not set expire time for its artifacts') - for error in errors: - print(error) - raise SystemExit(-2) - - -if __name__ == '__main__': - check_artifacts_expire_time() diff --git a/tools/ci/check_rules_yml.py b/tools/ci/check_rules_yml.py deleted file mode 100755 index 5a55132456..0000000000 --- a/tools/ci/check_rules_yml.py +++ /dev/null @@ -1,157 +0,0 @@ -#!/usr/bin/env python -# -# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD -# SPDX-License-Identifier: Apache-2.0 - -""" -Check if all rules in rules.yml used or not in CI yaml files. -""" - -import argparse -import os -import re -import sys -from copy import deepcopy -from typing import Any, Dict, List, Optional, Set, Union - -import yaml -from idf_ci_utils import IDF_PATH - -ROOT_YML_FP = os.path.join(IDF_PATH, '.gitlab-ci.yml') - - -def load_yaml(file_path: str) -> Any: - return yaml.load(open(file_path), Loader=yaml.FullLoader) - - -class YMLConfig: - def __init__(self, root_yml_file_path: str) -> None: - self._config: Optional[Dict] = None - self._all_extends: Optional[Set] = None - - self.root_yml = load_yaml(root_yml_file_path) - assert self.root_yml - - @staticmethod - def _list(str_or_list: Union[str, List]) -> List: - if isinstance(str_or_list, str): - return [str_or_list] - if isinstance(str_or_list, list): - return str_or_list - raise ValueError( - 'Wrong type: {}. Only supports str or list.'.format(type(str_or_list)) - ) - - @property - def config(self) -> Dict: - if self._config: - return self._config - - all_config = dict() - for item in self.root_yml['include']: - all_config.update(load_yaml(os.path.join(IDF_PATH, item))) - self._config = all_config - return self._config - - @property - def all_extends(self) -> Set: - if self._all_extends: - return self._all_extends - - res = set([]) - for v in self.config.values(): - if 'extends' in v: - for item in self._list(v['extends']): - if item.startswith('.rules:'): - res.add(item) - self._all_extends = res - return self._all_extends - - def exists(self, key: str) -> bool: - if key in self.all_extends: - return True - return False - - -YML_CONFIG = YMLConfig(ROOT_YML_FP) - - -def get_needed_rules() -> Set[str]: - return deepcopy(YML_CONFIG.all_extends) - - -def validate_needed_rules(rules_yml: 'os.PathLike[str]') -> int: - res = 0 - needed_rules = deepcopy(YML_CONFIG.all_extends) - with open(rules_yml) as fr: - for index, line in enumerate(fr): - if line.startswith('.rules:'): - key = line.strip().rsplit(':', 1)[0] - if not YML_CONFIG.exists(key): - print( - '{}:{}:WARNING:rule "{}" unused'.format(rules_yml, index, key) - ) - else: - needed_rules.remove(key) - - if needed_rules: - for item in needed_rules: - print('ERROR: missing rule: "{}"'.format(item)) - res = 1 - - if res == 0: - print('Pass') - return res - - -def parse_submodule_paths( - gitsubmodules: str = os.path.join(IDF_PATH, '.gitmodules') -) -> List[str]: - path_regex = re.compile(r'^\s+path = (.+)$', re.MULTILINE) - with open(gitsubmodules, 'r') as f: - data = f.read() - - res = [] - for item in path_regex.finditer(data): - res.append(item.group(1)) - - return res - - -def validate_submodule_patterns() -> int: - submodule_paths = sorted(['.gitmodules'] + parse_submodule_paths()) - submodule_paths_in_patterns = sorted( - YML_CONFIG.config.get('.patterns-submodule', []) - ) - - res = 0 - if submodule_paths != submodule_paths_in_patterns: - res = 1 - print('please update the pattern ".patterns-submodule"') - should_remove = set(submodule_paths_in_patterns) - set(submodule_paths) - if should_remove: - print(f'- should remove: {should_remove}') - should_add = set(submodule_paths) - set(submodule_paths_in_patterns) - if should_add: - print(f'- should add: {should_add}') - - return res - - -if __name__ == '__main__': - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - 'rules_yml', - nargs='?', - default=os.path.join(IDF_PATH, '.gitlab', 'ci', 'rules.yml'), - help='rules.yml file path', - ) - args = parser.parse_args() - - exit_code = 0 - if validate_needed_rules(args.rules_yml): - exit_code = 1 - if validate_submodule_patterns(): - exit_code = 1 - - sys.exit(exit_code) diff --git a/tools/ci/exclude_check_tools_files.txt b/tools/ci/exclude_check_tools_files.txt index c801f3f43c..873ecd3e95 100644 --- a/tools/ci/exclude_check_tools_files.txt +++ b/tools/ci/exclude_check_tools_files.txt @@ -40,3 +40,4 @@ tools/templates/sample_component/main.c tools/ci/cleanup_ignore_lists.py tools/ci/artifacts_handler.py tools/unit-test-app/**/* +tools/ci/gitlab_yaml_linter.py diff --git a/tools/ci/executable-list.txt b/tools/ci/executable-list.txt index 04d372832e..2fce0de3c0 100644 --- a/tools/ci/executable-list.txt +++ b/tools/ci/executable-list.txt @@ -62,7 +62,6 @@ tools/ci/check_kconfigs.py tools/ci/check_readme_links.py tools/ci/check_requirement_files.py tools/ci/check_rules_components_patterns.py -tools/ci/check_rules_yml.py tools/ci/check_soc_struct_headers.py tools/ci/check_tools_files_patterns.py tools/ci/check_type_comments.py @@ -74,6 +73,7 @@ tools/ci/fix_empty_prototypes.sh tools/ci/generate_rules.py tools/ci/get-full-sources.sh tools/ci/get_supported_examples.sh +tools/ci/gitlab_yaml_linter.py tools/ci/mirror-submodule-update.sh tools/ci/multirun_with_pyenv.sh tools/ci/push_to_github.sh diff --git a/tools/ci/generate_rules.py b/tools/ci/generate_rules.py index 1d37ac69c6..072c22f974 100755 --- a/tools/ci/generate_rules.py +++ b/tools/ci/generate_rules.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2021-2023 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 import argparse @@ -11,8 +11,7 @@ from collections import defaultdict from itertools import product import yaml -from check_rules_yml import get_needed_rules -from idf_ci_utils import IDF_PATH +from idf_ci_utils import IDF_PATH, GitlabYmlConfig try: import pygraphviz as pgv @@ -100,6 +99,7 @@ class RulesWriter: self.cfg = self.expand_matrices() self.rules = self.expand_rules() + self.yml_config = GitlabYmlConfig() self.graph = None def expand_matrices(self): # type: () -> dict @@ -201,7 +201,7 @@ class RulesWriter: def new_rules_str(self): # type: () -> str res = [] for k, v in sorted(self.rules.items()): - if '.rules:' + k not in get_needed_rules(): + if '.rules:' + k not in self.yml_config.used_rules: print(f'WARNING: unused rule: {k}, skipping...') continue res.append(self.RULES_TEMPLATE.format(k, self._format_rule(k, v))) diff --git a/tools/ci/gitlab_yaml_linter.py b/tools/ci/gitlab_yaml_linter.py new file mode 100755 index 0000000000..0f9750f0d5 --- /dev/null +++ b/tools/ci/gitlab_yaml_linter.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python + +# SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 + +""" +Check gitlab ci yaml files +""" + +import argparse +import os +import typing as t +from functools import cached_property + +from idf_ci_utils import IDF_PATH, GitlabYmlConfig, get_submodule_dirs + + +class YmlLinter: + def __init__(self, yml_config: GitlabYmlConfig) -> None: + self.yml_config = yml_config + + self._errors: t.List[str] = [] + + @cached_property + def lint_functions(self) -> t.List[str]: + funcs = [] + for func in dir(self): + if func.startswith('_lint_'): + funcs.append(func) + + return funcs + + def lint(self) -> None: + exit_code = 0 + + for func in self.lint_functions: + getattr(self, func)() + + if self._errors: + print(f'Errors found while running {func}:') + exit_code = 1 + print('\t- ' + '\n\t- '.join(self._errors)) + self._errors = [] # reset + + exit(exit_code) + + # name it like _1_ to make it run first + def _lint_1_yml_parser(self) -> None: + for k, v in self.yml_config.config.items(): + if ( + k not in self.yml_config.global_keys + and k not in self.yml_config.anchors + and k not in self.yml_config.jobs + ): + raise SystemExit(f'Parser incorrect. Key {k} not in global keys, rules or jobs') + + def _lint_default_values_artifacts(self) -> None: + defaults_artifacts = self.yml_config.default.get('artifacts', {}) + + for job_name, d in self.yml_config.jobs.items(): + for k, v in d.get('artifacts', {}).items(): + if k not in defaults_artifacts: + continue + + if v == defaults_artifacts[k]: + self._errors.append(f'job {job_name} key {k} has same value as default value {v}') + + def _lint_submodule_patterns(self) -> None: + submodule_paths = sorted(['.gitmodules'] + get_submodule_dirs()) + submodule_paths_in_patterns = sorted(self.yml_config.config.get('.patterns-submodule', [])) + + if submodule_paths != submodule_paths_in_patterns: + unused_patterns = set(submodule_paths_in_patterns) - set(submodule_paths) + if unused_patterns: + for item in unused_patterns: + self._errors.append(f'non-exist pattern {item}. Please remove {item} from .patterns-submodule') + undefined_patterns = set(submodule_paths) - set(submodule_paths_in_patterns) + if undefined_patterns: + for item in undefined_patterns: + self._errors.append(f'undefined pattern {item}. Please add {item} to .patterns-submodule') + + def _lint_gitlab_yml_rules(self) -> None: + unused_rules = self.yml_config.rules - self.yml_config.used_rules + for item in unused_rules: + self._errors.append(f'Unused rule: {item}, please remove it') + undefined_rules = self.yml_config.used_rules - self.yml_config.rules + for item in undefined_rules: + self._errors.append(f'Undefined rule: {item}') + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument( + '--root-yml-filepath', help='root yml file path', default=os.path.join(IDF_PATH, '.gitlab-ci.yml') + ) + args = parser.parse_args() + + config = GitlabYmlConfig(args.root_yml_filepath) + linter = YmlLinter(config) + linter.lint() diff --git a/tools/ci/idf_ci_utils.py b/tools/ci/idf_ci_utils.py index 1776766eaf..db38d8a988 100644 --- a/tools/ci/idf_ci_utils.py +++ b/tools/ci/idf_ci_utils.py @@ -8,12 +8,13 @@ import logging import os import subprocess import sys -from typing import Any, List +import typing as t +from functools import cached_property IDF_PATH = os.path.abspath(os.getenv('IDF_PATH', os.path.join(os.path.dirname(__file__), '..', '..'))) -def get_submodule_dirs(full_path: bool = False) -> List[str]: +def get_submodule_dirs(full_path: bool = False) -> t.List[str]: """ To avoid issue could be introduced by multi-os or additional dependency, we use python and git to get this output @@ -71,7 +72,7 @@ def is_executable(full_path: str) -> bool: return os.access(full_path, os.X_OK) -def get_git_files(path: str = IDF_PATH, full_path: bool = False) -> List[str]: +def get_git_files(path: str = IDF_PATH, full_path: bool = False) -> t.List[str]: """ Get the result of git ls-files :param path: path to run git ls-files @@ -102,7 +103,10 @@ def is_in_directory(file_path: str, folder: str) -> bool: return os.path.realpath(file_path).startswith(os.path.realpath(folder) + os.sep) -def to_list(s: Any) -> List[Any]: +def to_list(s: t.Any) -> t.List[t.Any]: + if not s: + return [] + if isinstance(s, (set, tuple)): return list(s) @@ -110,3 +114,67 @@ def to_list(s: Any) -> List[Any]: return s return [s] + + +class GitlabYmlConfig: + def __init__(self, root_yml_filepath: str = os.path.join(IDF_PATH, '.gitlab-ci.yml')) -> None: + self._config: t.Dict[str, t.Any] = {} + self._defaults: t.Dict[str, t.Any] = {} + + self._load(root_yml_filepath) + + def _load(self, root_yml_filepath: str) -> None: + # avoid unused import in other pre-commit hooks + import yaml + + all_config = dict() + root_yml = yaml.load(open(root_yml_filepath), Loader=yaml.FullLoader) + for item in root_yml['include']: + all_config.update(yaml.load(open(os.path.join(IDF_PATH, item)), Loader=yaml.FullLoader)) + + if 'default' in all_config: + self._defaults = all_config.pop('default') + + self._config = all_config + + @property + def default(self) -> t.Dict[str, t.Any]: + return self._defaults + + @property + def config(self) -> t.Dict[str, t.Any]: + return self._config + + @cached_property + def global_keys(self) -> t.List[str]: + return ['default', 'include', 'workflow', 'variables', 'stages'] + + @cached_property + def anchors(self) -> t.Dict[str, t.Any]: + return {k: v for k, v in self.config.items() if k.startswith('.')} + + @cached_property + def jobs(self) -> t.Dict[str, t.Any]: + return {k: v for k, v in self.config.items() if not k.startswith('.') and k not in self.global_keys} + + @cached_property + def rules(self) -> t.Set[str]: + return {k for k, _ in self.anchors.items() if self._is_rule_key(k)} + + @cached_property + def used_rules(self) -> t.Set[str]: + res = set() + + for v in self.config.values(): + if not isinstance(v, dict): + continue + + for item in to_list(v.get('extends')): + if self._is_rule_key(item): + res.add(item) + + return res + + @staticmethod + def _is_rule_key(key: str) -> bool: + return key.startswith('.rules:') or key.endswith('template') From 06af021c9f0bacd025f012fee6097dd594732084 Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Tue, 28 Nov 2023 14:38:47 +0100 Subject: [PATCH 03/18] ci(pytest): stop ambiguous markers for multi-dut test case also add `tools/ci/idf_pytest/tests` for testing --- conftest.py | 68 +++-- pytest.ini | 1 + tools/ci/idf_pytest/constants.py | 105 +++++-- tools/ci/idf_pytest/plugin.py | 257 ++++++++++-------- tools/ci/idf_pytest/pytest.ini | 2 + tools/ci/idf_pytest/script.py | 49 +++- .../idf_pytest/tests/test_get_pytest_cases.py | 94 +++++++ tools/ci/idf_pytest/utils.py | 27 +- 8 files changed, 409 insertions(+), 194 deletions(-) create mode 100644 tools/ci/idf_pytest/pytest.ini create mode 100644 tools/ci/idf_pytest/tests/test_get_pytest_cases.py diff --git a/conftest.py b/conftest.py index 0fdc3ad45a..ed5bcaec0c 100644 --- a/conftest.py +++ b/conftest.py @@ -19,9 +19,9 @@ import logging import os import re import sys +import typing as t from copy import deepcopy from datetime import datetime -from typing import Callable, Optional import pytest from _pytest.config import Config @@ -34,13 +34,13 @@ try: from idf_ci_utils import IDF_PATH from idf_pytest.constants import DEFAULT_SDKCONFIG, ENV_MARKERS, SPECIAL_MARKERS, TARGET_MARKERS from idf_pytest.plugin import IDF_PYTEST_EMBEDDED_KEY, IdfPytestEmbedded - from idf_pytest.utils import format_case_id, get_target_marker_from_expr + from idf_pytest.utils import format_case_id except ImportError: sys.path.append(os.path.join(os.path.dirname(__file__), 'tools', 'ci')) from idf_ci_utils import IDF_PATH from idf_pytest.constants import DEFAULT_SDKCONFIG, ENV_MARKERS, SPECIAL_MARKERS, TARGET_MARKERS from idf_pytest.plugin import IDF_PYTEST_EMBEDDED_KEY, IdfPytestEmbedded - from idf_pytest.utils import format_case_id, get_target_marker_from_expr + from idf_pytest.utils import format_case_id try: import common_test_methods # noqa: F401 @@ -102,7 +102,7 @@ def test_case_name(request: FixtureRequest, target: str, config: str) -> str: @pytest.fixture @multi_dut_fixture -def build_dir(app_path: str, target: Optional[str], config: Optional[str]) -> str: +def build_dir(app_path: str, target: t.Optional[str], config: t.Optional[str]) -> str: """ Check local build dir with the following priority: @@ -138,7 +138,7 @@ def build_dir(app_path: str, target: Optional[str], config: Optional[str]) -> st @pytest.fixture(autouse=True) @multi_dut_fixture -def junit_properties(test_case_name: str, record_xml_attribute: Callable[[str, object], None]) -> None: +def junit_properties(test_case_name: str, record_xml_attribute: t.Callable[[str, object], None]) -> None: """ This fixture is autoused and will modify the junit report test case name to .. """ @@ -154,7 +154,7 @@ def set_test_case_name(request: FixtureRequest, test_case_name: str) -> None: # Log Util Functions # ###################### @pytest.fixture -def log_performance(record_property: Callable[[str, object], None]) -> Callable[[str, str], None]: +def log_performance(record_property: t.Callable[[str, object], None]) -> t.Callable[[str, str], None]: """ log performance item with pre-defined format to the console and record it under the ``properties`` tag in the junit report if available. @@ -172,7 +172,7 @@ def log_performance(record_property: Callable[[str, object], None]) -> Callable[ @pytest.fixture -def check_performance(idf_path: str) -> Callable[[str, float, str], None]: +def check_performance(idf_path: str) -> t.Callable[[str, float, str], None]: """ check if the given performance item meets the passing standard or not """ @@ -186,9 +186,9 @@ def check_performance(idf_path: str) -> Callable[[str, float, str], None]: """ def _find_perf_item(operator: str, path: str) -> float: - with open(path, 'r') as f: + with open(path) as f: data = f.read() - match = re.search(r'#define\s+IDF_PERFORMANCE_{}_{}\s+([\d.]+)'.format(operator, item.upper()), data) + match = re.search(fr'#define\s+IDF_PERFORMANCE_{operator}_{item.upper()}\s+([\d.]+)', data) return float(match.group(1)) # type: ignore def _check_perf(operator: str, standard_value: float) -> None: @@ -198,7 +198,7 @@ def check_performance(idf_path: str) -> Callable[[str, float, str], None]: ret = value >= standard_value if not ret: raise AssertionError( - "[Performance] {} value is {}, doesn't meet pass standard {}".format(item, value, standard_value) + f"[Performance] {item} value is {value}, doesn't meet pass standard {standard_value}" ) path_prefix = os.path.join(idf_path, 'components', 'idf_test', 'include') @@ -212,7 +212,7 @@ def check_performance(idf_path: str) -> Callable[[str, float, str], None]: for performance_file in performance_files: try: standard = _find_perf_item(op, performance_file) - except (IOError, AttributeError): + except (OSError, AttributeError): # performance file doesn't exist or match is not found in it continue @@ -221,13 +221,13 @@ def check_performance(idf_path: str) -> Callable[[str, float, str], None]: break if not found_item: - raise AssertionError('Failed to get performance standard for {}'.format(item)) + raise AssertionError(f'Failed to get performance standard for {item}') return real_func @pytest.fixture -def log_minimum_free_heap_size(dut: IdfDut, config: str) -> Callable[..., None]: +def log_minimum_free_heap_size(dut: IdfDut, config: str) -> t.Callable[..., None]: def real_func() -> None: res = dut.expect(r'Minimum free heap size: (\d+) bytes') logging.info( @@ -278,28 +278,52 @@ def pytest_addoption(parser: pytest.Parser) -> None: '--app-info-basedir', default=IDF_PATH, help='app info base directory. specify this value when you\'re building under a ' - 'different IDF_PATH. (Default: $IDF_PATH)', + 'different IDF_PATH. (Default: $IDF_PATH)', ) idf_group.addoption( '--app-info-filepattern', help='glob pattern to specify the files that include built app info generated by ' - '`idf-build-apps --collect-app-info ...`. will not raise ValueError when binary ' - 'paths not exist in local file system if not listed recorded in the app info.', + '`idf-build-apps --collect-app-info ...`. will not raise ValueError when binary ' + 'paths not exist in local file system if not listed recorded in the app info.', ) def pytest_configure(config: Config) -> None: # cli option "--target" - target = config.getoption('target') or '' + target = [_t.strip().lower() for _t in (config.getoption('target', '') or '').split(',') if _t.strip()] + + # add markers based on idf_pytest/constants.py + for name, description in { + **TARGET_MARKERS, + **ENV_MARKERS, + **SPECIAL_MARKERS, + }.items(): + config.addinivalue_line('markers', f'{name}: {description}') help_commands = ['--help', '--fixtures', '--markers', '--version'] for cmd in help_commands: if cmd in config.invocation_params.args: - target = 'unneeded' + target = ['unneeded'] break - if not target: # also could specify through markexpr via "-m" - target = get_target_marker_from_expr(config.getoption('markexpr') or '') + markexpr = config.getoption('markexpr') or '' + # check marker expr set via "pytest -m" + if not target and markexpr: + # we use `-m "esp32 and generic"` in our CI to filter the test cases + # this doesn't cover all use cases, but fit what we do in CI. + for marker in markexpr.split('and'): + marker = marker.strip() + if marker in TARGET_MARKERS: + target.append(marker) + + # "--target" must be set + if not target: + raise SystemExit( + """Pass `--target TARGET[,TARGET...]` to specify all targets the test cases are using. + - for single DUT, we run with `pytest --target esp32` + - for multi DUT, we run with `pytest --target esp32,esp32,esp32s2` to indicate all DUTs +""" + ) apps_list = None app_info_basedir = config.getoption('app_info_basedir') @@ -326,14 +350,10 @@ def pytest_configure(config: Config) -> None: config.stash[IDF_PYTEST_EMBEDDED_KEY] = IdfPytestEmbedded( target=target, - sdkconfig=config.getoption('sdkconfig'), apps_list=apps_list, ) config.pluginmanager.register(config.stash[IDF_PYTEST_EMBEDDED_KEY]) - for name, description in {**TARGET_MARKERS, **ENV_MARKERS, **SPECIAL_MARKERS}.items(): - config.addinivalue_line('markers', f'{name}: {description}') - def pytest_unconfigure(config: Config) -> None: _pytest_embedded = config.stash.get(IDF_PYTEST_EMBEDDED_KEY, None) diff --git a/pytest.ini b/pytest.ini index c9c0c43560..08ef25dfe7 100644 --- a/pytest.ini +++ b/pytest.ini @@ -12,6 +12,7 @@ addopts = --skip-check-coredump y --logfile-extension ".txt" --check-duplicates y + --ignore-glob */managed_components/* # ignore DeprecationWarning filterwarnings = diff --git a/tools/ci/idf_pytest/constants.py b/tools/ci/idf_pytest/constants.py index f5bb8743ae..ccbe00fb0d 100644 --- a/tools/ci/idf_pytest/constants.py +++ b/tools/ci/idf_pytest/constants.py @@ -6,7 +6,10 @@ Pytest Related Constants. Don't import third-party packages here. """ import os import typing as t +from collections import Counter from dataclasses import dataclass +from enum import Enum +from functools import cached_property from _pytest.python import Function from pytest_embedded.utils import to_list @@ -35,10 +38,11 @@ SPECIAL_MARKERS = { 'temp_skip': 'temp skip tests for specified targets both in ci and locally', 'nightly_run': 'tests should be executed as part of the nightly trigger pipeline', 'host_test': 'tests which should not be built at the build stage, and instead built in host_test stage', - 'qemu': 'build and test using qemu-system-xtensa, not real target', } ENV_MARKERS = { + # special markers + 'qemu': 'build and test using qemu, not real target', # single-dut markers 'generic': 'tests should be run on generic runners', 'flash_suspend': 'support flash suspend feature', @@ -89,7 +93,6 @@ ENV_MARKERS = { 'adc': 'ADC related tests should run on adc runners', 'xtal32k': 'Runner with external 32k crystal connected', 'no32kXtal': 'Runner with no external 32k crystal connected', - 'multi_dut_modbus_rs485': 'a pair of runners connected by RS485 bus', 'psramv0': 'Runner with PSRAM version 0', 'esp32eco3': 'Runner with esp32 eco3 connected', 'ecdsa_efuse': 'Runner with test ECDSA private keys programmed in efuse', @@ -98,6 +101,7 @@ ENV_MARKERS = { 'i2c_oled': 'Runner with ssd1306 I2C oled connected', 'httpbin': 'runner for tests that need to access the httpbin service', # multi-dut markers + 'multi_dut_modbus_rs485': 'a pair of runners connected by RS485 bus', 'ieee802154': 'ieee802154 related tests should run on ieee802154 runners.', 'openthread_br': 'tests should be used for openthread border router.', 'openthread_bbr': 'tests should be used for openthread border router linked to Internet.', @@ -113,6 +117,13 @@ ENV_MARKERS = { } +class CollectMode(str, Enum): + SINGLE_SPECIFIC = 'single_specific' + MULTI_SPECIFIC = 'multi_specific' + MULTI_ALL_WITH_PARAM = 'multi_all_with_param' + ALL = 'all' + + @dataclass class PytestApp: path: str @@ -122,38 +133,43 @@ class PytestApp: def __hash__(self) -> int: return hash((self.path, self.target, self.config)) + @cached_property + def build_dir(self) -> str: + return os.path.join(self.path, f'build_{self.target}_{self.config}') + @dataclass class PytestCase: - path: str - name: str - - apps: t.Set[PytestApp] - target: str + apps: t.List[PytestApp] item: Function def __hash__(self) -> int: return hash((self.path, self.name, self.apps, self.all_markers)) + @cached_property + def path(self) -> str: + return str(self.item.path) + + @cached_property + def name(self) -> str: + return self.item.originalname # type: ignore + + @cached_property + def targets(self) -> t.List[str]: + return [app.target for app in self.apps] + + @cached_property + def is_single_dut_test_case(self) -> bool: + return True if len(self.apps) == 1 else False + + # the following markers could be changed dynamically, don't use cached_property @property def all_markers(self) -> t.Set[str]: return {marker.name for marker in self.item.iter_markers()} - @property - def is_nightly_run(self) -> bool: - return 'nightly_run' in self.all_markers - @property def target_markers(self) -> t.Set[str]: - return {marker for marker in self.all_markers if marker in TARGET_MARKERS} - - @property - def env_markers(self) -> t.Set[str]: - return {marker for marker in self.all_markers if marker in ENV_MARKERS} - - @property - def skipped_targets(self) -> t.Set[str]: def _get_temp_markers_disabled_targets(marker_name: str) -> t.Set[str]: temp_marker = self.item.get_closest_marker(marker_name) @@ -179,4 +195,53 @@ class PytestCase: else: # we use `temp_skip` locally skip_targets = temp_skip_targets - return skip_targets + return {marker for marker in self.all_markers if marker in TARGET_MARKERS} - skip_targets + + @property + def env_markers(self) -> t.Set[str]: + return {marker for marker in self.all_markers if marker in ENV_MARKERS} + + @property + def target_with_amount_markers(self) -> t.Set[str]: + c: Counter = Counter() + for app in self.apps: + c[app.target] += 1 + + res = set() + for target, amount in c.items(): + if amount > 1: + res.add(f'{target}_{amount}') + else: + res.add(target) + + return res + + def all_built_in_app_lists(self, app_lists: t.Optional[t.List[str]] = None) -> bool: + if app_lists is None: + # ignore this feature + return True + + bin_found = [0] * len(self.apps) + for i, app in enumerate(self.apps): + if app.build_dir in app_lists: + bin_found[i] = 1 + + if sum(bin_found) == 0: + msg = f'Skip test case {self.name} because all following binaries are not listed in the app lists: ' + for app in self.apps: + msg += f'\n - {app.build_dir}' + + print(msg) + return False + + if sum(bin_found) == len(self.apps): + return True + + # some found, some not, looks suspicious + msg = f'Found some binaries of test case {self.name} are not listed in the app lists.' + for i, app in enumerate(self.apps): + if bin_found[i] == 0: + msg += f'\n - {app.build_dir}' + + msg += '\nMight be a issue of .build-test-rules.yml files' + return False diff --git a/tools/ci/idf_pytest/plugin.py b/tools/ci/idf_pytest/plugin.py index c5c5576a05..efae926208 100644 --- a/tools/ci/idf_pytest/plugin.py +++ b/tools/ci/idf_pytest/plugin.py @@ -1,9 +1,9 @@ # SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 -import logging import os import typing as t +from functools import cached_property from xml.etree import ElementTree as ET import pytest @@ -16,12 +16,13 @@ from pytest_embedded.plugin import parse_multi_dut_args from pytest_embedded.utils import find_by_suffix, to_list from pytest_ignore_test_results.ignore_results import ChildCase, ChildCasesStashKey -from .constants import DEFAULT_SDKCONFIG, PREVIEW_TARGETS, SUPPORTED_TARGETS, PytestApp, PytestCase -from .utils import format_case_id, merge_junit_files +from .constants import DEFAULT_SDKCONFIG, PREVIEW_TARGETS, SUPPORTED_TARGETS, CollectMode, PytestApp, PytestCase +from .utils import comma_sep_str_to_list, format_case_id, merge_junit_files IDF_PYTEST_EMBEDDED_KEY = pytest.StashKey['IdfPytestEmbedded']() ITEM_FAILED_CASES_KEY = pytest.StashKey[list]() ITEM_FAILED_KEY = pytest.StashKey[bool]() +ITEM_PYTEST_CASE_KEY = pytest.StashKey[PytestCase]() class IdfPytestEmbedded: @@ -33,80 +34,119 @@ class IdfPytestEmbedded: def __init__( self, - target: str, - sdkconfig: t.Optional[str] = None, + target: t.Union[t.List[str], str], + *, + single_target_duplicate_mode: bool = False, apps_list: t.Optional[t.List[str]] = None, ): - # CLI options to filter the test cases - self.target = target.lower() - self.sdkconfig = sdkconfig + if isinstance(target, str): + self.target = sorted(comma_sep_str_to_list(target)) + else: + self.target = sorted(target) + + if not self.target: + raise ValueError('`target` should not be empty') + + # these are useful while gathering all the multi-dut test cases + # when this mode is activated, + # + # pytest.mark.esp32 + # pytest.mark.parametrize('count', [2], indirect=True) + # def test_foo(dut): + # pass + # + # should be collected when running `pytest --target esp32` + # + # otherwise, it should be collected when running `pytest --target esp32,esp32` + self._single_target_duplicate_mode = single_target_duplicate_mode + self.apps_list = apps_list self.cases: t.List[PytestCase] = [] + @cached_property + def collect_mode(self) -> CollectMode: + if len(self.target) == 1: + if self.target[0] == CollectMode.MULTI_ALL_WITH_PARAM: + return CollectMode.MULTI_ALL_WITH_PARAM + else: + return CollectMode.SINGLE_SPECIFIC + else: + return CollectMode.MULTI_SPECIFIC + @staticmethod def get_param(item: Function, key: str, default: t.Any = None) -> t.Any: - # implement like this since this is a limitation of pytest, couldn't get fixture values while collecting - # https://github.com/pytest-dev/pytest/discussions/9689 + # funcargs is not calculated while collection + # callspec is something defined in parametrize if not hasattr(item, 'callspec'): return default return item.callspec.params.get(key, default) or default def item_to_pytest_case(self, item: Function) -> PytestCase: - count = 1 - case_path = str(item.path) - case_name = item.originalname - target = self.target + """ + Turn pytest item to PytestCase + """ + count = self.get_param(item, 'count', 1) - # funcargs is not calculated while collection - if hasattr(item, 'callspec'): - count = item.callspec.params.get('count', 1) - app_paths = to_list( - parse_multi_dut_args( - count, - self.get_param(item, 'app_path', os.path.dirname(case_path)), - ) - ) - configs = to_list(parse_multi_dut_args(count, self.get_param(item, 'config', 'default'))) - targets = to_list(parse_multi_dut_args(count, self.get_param(item, 'target', target))) - else: - app_paths = [os.path.dirname(case_path)] - configs = ['default'] - targets = [target] - - case_apps = set() - for i in range(count): - case_apps.add(PytestApp(app_paths[i], targets[i], configs[i])) - - return PytestCase( - case_path, - case_name, - case_apps, - self.target, - item, + # default app_path is where the test script locates + app_paths = to_list( + parse_multi_dut_args(count, os.path.relpath(self.get_param(item, 'app_path', os.path.dirname(item.path)))) ) + configs = to_list(parse_multi_dut_args(count, self.get_param(item, 'config', DEFAULT_SDKCONFIG))) + targets = to_list(parse_multi_dut_args(count, self.get_param(item, 'target', self.target[0]))) - @pytest.hookimpl(tryfirst=True) - def pytest_sessionstart(self, session: Session) -> None: - # same behavior for vanilla pytest-embedded '--target' - session.config.option.target = self.target + return PytestCase([PytestApp(app_paths[i], targets[i], configs[i]) for i in range(count)], item) @pytest.hookimpl(tryfirst=True) def pytest_collection_modifyitems(self, items: t.List[Function]) -> None: - item_to_case: t.Dict[Function, PytestCase] = {} + """ + Background info: - # Add Markers to the test cases + We're using `pytest.mark.[TARGET]` as a syntactic sugar to indicate that they are actually supported by all + the listed targets. For example, + + >>> @pytest.mark.esp32 + >>> @pytest.mark.esp32s2 + + should be treated as + + >>> @pytest.mark.parametrize('target', [ + >>> 'esp32', + >>> 'esp32s2', + >>> ], indirect=True) + + All single-dut test cases, and some of the multi-dut test cases with the same targets, are using this + way to indicate the supported targets. + + To avoid ambiguity, + + - when we're collecting single-dut test cases with esp32, we call + + `pytest --collect-only --target esp32` + + - when we're collecting multi-dut test cases, we list all the targets, even when they're the same + + `pytest --collect-only --target esp32,esp32` for two esp32 connected + `pytest --collect-only --target esp32,esp32s2` for esp32 and esp32s2 connected + + therefore, we have two different logic for searching test cases, explained in 2.1 and 2.2 + """ + # 1. Filter according to nighty_run related markers + if os.getenv('INCLUDE_NIGHTLY_RUN') == '1': + # nightly_run and non-nightly_run cases are both included + pass + elif os.getenv('NIGHTLY_RUN') == '1': + # only nightly_run cases are included + items[:] = [_item for _item in items if _item.get_closest_marker('nightly_run') is not None] + else: + # only non-nightly_run cases are included + items[:] = [_item for _item in items if _item.get_closest_marker('nightly_run') is None] + + # 2. Add markers according to special markers + item_to_case_dict: t.Dict[Function, PytestCase] = {} for item in items: - # generate PytestCase for each item - case = self.item_to_pytest_case(item) - item_to_case[item] = case - - # set default timeout 10 minutes for each case - if 'timeout' not in item.keywords: - item.add_marker(pytest.mark.timeout(10 * 60)) - - # add markers for special markers + item.stash[ITEM_PYTEST_CASE_KEY] = item_to_case_dict[item] = self.item_to_pytest_case(item) if 'supported_targets' in item.keywords: for _target in SUPPORTED_TARGETS: item.add_marker(_target) @@ -117,72 +157,55 @@ class IdfPytestEmbedded: for _target in [*SUPPORTED_TARGETS, *PREVIEW_TARGETS]: item.add_marker(_target) + # 3.1. CollectMode.SINGLE_SPECIFIC, like `pytest --target esp32` + if self.collect_mode == CollectMode.SINGLE_SPECIFIC: + filtered_items = [] + for item in items: + case = item_to_case_dict[item] + + # single-dut one + if case.is_single_dut_test_case and self.target[0] in case.target_markers: + filtered_items.append(item) + + # multi-dut ones and in single_target_duplicate_mode + elif self._single_target_duplicate_mode and not case.is_single_dut_test_case: + # ignore those test cases with `target` defined in parametrize, since these will be covered in 3.3 + if self.get_param(item, 'target', None) is None and self.target[0] in case.target_markers: + filtered_items.append(item) + + items[:] = filtered_items + # 3.2. CollectMode.MULTI_SPECIFIC, like `pytest --target esp32,esp32` + elif self.collect_mode == CollectMode.MULTI_SPECIFIC: + items[:] = [_item for _item in items if item_to_case_dict[_item].targets == self.target] + + # 3.3. CollectMode.MULTI_ALL_WITH_PARAM, intended to be used by `get_pytest_cases` + else: + items[:] = [ + _item + for _item in items + if not item_to_case_dict[_item].is_single_dut_test_case + and self.get_param(_item, 'target', None) is not None + ] + + # 4. filter by `self.apps_list`, skip the test case if not listed + # should only be used in CI + items[:] = [_item for _item in items if item_to_case_dict[_item].all_built_in_app_lists(self.apps_list)] + + # OKAY!!! All left ones will be executed, sort it and add more markers + items[:] = sorted( + items, key=lambda x: (os.path.dirname(x.path), self.get_param(x, 'config', DEFAULT_SDKCONFIG)) + ) + for item in items: + case = item_to_case_dict[item] + # set default timeout 10 minutes for each case + if 'timeout' not in item.keywords: + item.add_marker(pytest.mark.timeout(10 * 60)) + # add 'xtal_40mhz' tag as a default tag for esp32c2 target # only add this marker for esp32c2 cases - if self.target == 'esp32c2' and 'esp32c2' in case.target_markers and 'xtal_26mhz' not in case.all_markers: + if 'esp32c2' in self.target and 'esp32c2' in case.targets and 'xtal_26mhz' not in case.all_markers: item.add_marker('xtal_40mhz') - # Filter the test cases - filtered_items = [] - for item in items: - case = item_to_case[item] - # filter by "nightly_run" marker - if os.getenv('INCLUDE_NIGHTLY_RUN') == '1': - # Do not filter nightly_run cases - pass - elif os.getenv('NIGHTLY_RUN') == '1': - if not case.is_nightly_run: - logging.debug( - 'Skipping test case %s because of this test case is not a nightly run test case', item.name - ) - continue - else: - if case.is_nightly_run: - logging.debug( - 'Skipping test case %s because of this test case is a nightly run test case', item.name - ) - continue - - # filter by target - if self.target not in case.target_markers: - continue - - if self.target in case.skipped_targets: - continue - - # filter by sdkconfig - if self.sdkconfig: - if self.get_param(item, 'config', DEFAULT_SDKCONFIG) != self.sdkconfig: - continue - - # filter by apps_list, skip the test case if not listed - # should only be used in CI - if self.apps_list is not None: - bin_not_found = False - for case_app in case.apps: - # in ci, always use build__ as build dir - binary_path = os.path.join(case_app.path, f'build_{case_app.target}_{case_app.config}') - if binary_path not in self.apps_list: - logging.info( - 'Skipping test case %s because binary path %s is not listed in app info list files', - item.name, - binary_path, - ) - bin_not_found = True - break - - if bin_not_found: - continue - - # finally! - filtered_items.append(item) - - # sort the test cases with (app folder, config) - items[:] = sorted( - filtered_items, - key=lambda x: (os.path.dirname(x.path), self.get_param(x, 'config', DEFAULT_SDKCONFIG)) - ) - def pytest_report_collectionfinish(self, items: t.List[Function]) -> None: for item in items: self.cases.append(self.item_to_pytest_case(item)) diff --git a/tools/ci/idf_pytest/pytest.ini b/tools/ci/idf_pytest/pytest.ini new file mode 100644 index 0000000000..0ee949b898 --- /dev/null +++ b/tools/ci/idf_pytest/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +python_files = test_*.py diff --git a/tools/ci/idf_pytest/script.py b/tools/ci/idf_pytest/script.py index a970864478..7f16bc91e6 100644 --- a/tools/ci/idf_pytest/script.py +++ b/tools/ci/idf_pytest/script.py @@ -12,7 +12,7 @@ from idf_py_actions.constants import PREVIEW_TARGETS as TOOLS_PREVIEW_TARGETS from idf_py_actions.constants import SUPPORTED_TARGETS as TOOLS_SUPPORTED_TARGETS from pytest_embedded.utils import to_list -from .constants import PytestCase +from .constants import CollectMode, PytestCase from .plugin import IdfPytestEmbedded @@ -35,15 +35,25 @@ def get_pytest_files(paths: t.List[str]) -> t.List[str]: def get_pytest_cases( paths: t.Union[str, t.List[str]], - target: str = 'all', + target: str = CollectMode.ALL, marker_expr: t.Optional[str] = None, filter_expr: t.Optional[str] = None, ) -> t.List[PytestCase]: - if target == 'all': - targets = TOOLS_SUPPORTED_TARGETS + TOOLS_PREVIEW_TARGETS - else: - targets = [target] + """ + For single-dut test cases, `target` could be + - [TARGET], e.g. `esp32`, to get the test cases for the given target + - or `single_all`, to get all single-dut test cases + For multi-dut test cases, `target` could be + - [TARGET,[TARGET...]], e.g. `esp32,esp32s2`, to get the test cases for the given targets + - or `multi_all`, to get all multi-dut test cases + + :param paths: paths to search for pytest scripts + :param target: target to get test cases for, detailed above + :param marker_expr: pytest marker expression, `-m` + :param filter_expr: pytest filter expression, `-k` + :return: list of test cases + """ paths = to_list(paths) cases: t.List[PytestCase] = [] @@ -52,12 +62,12 @@ def get_pytest_cases( print(f'WARNING: no pytest scripts found for target {target} under paths {", ".join(paths)}') return cases - for target in targets: - collector = IdfPytestEmbedded(target) + def _get_pytest_cases(_target: str, _single_target_duplicate_mode: bool = False) -> t.List[PytestCase]: + collector = IdfPytestEmbedded(_target, single_target_duplicate_mode=_single_target_duplicate_mode) with io.StringIO() as buf: with redirect_stdout(buf): - cmd = ['--collect-only', *pytest_scripts, '--target', target, '-q'] + cmd = ['--collect-only', *pytest_scripts, '--target', _target, '-q'] if marker_expr: cmd.extend(['-m', marker_expr]) if filter_expr: @@ -66,11 +76,24 @@ def get_pytest_cases( if res.value != ExitCode.OK: if res.value == ExitCode.NO_TESTS_COLLECTED: - print(f'WARNING: no pytest app found for target {target} under paths {", ".join(paths)}') + print(f'WARNING: no pytest app found for target {_target} under paths {", ".join(paths)}') else: print(buf.getvalue()) - raise RuntimeError(f'pytest collection failed at {", ".join(paths)} with command \"{" ".join(cmd)}\"') + raise RuntimeError( + f'pytest collection failed at {", ".join(paths)} with command \"{" ".join(cmd)}\"' + ) - cases.extend(collector.cases) + return collector.cases # type: ignore - return cases + if target == CollectMode.ALL: + targets = TOOLS_SUPPORTED_TARGETS + TOOLS_PREVIEW_TARGETS + [CollectMode.MULTI_ALL_WITH_PARAM] + else: + targets = [target] + + for _target in targets: + if target == CollectMode.ALL: + cases.extend(_get_pytest_cases(_target, _single_target_duplicate_mode=True)) + else: + cases.extend(_get_pytest_cases(_target)) + + return sorted(cases, key=lambda x: (x.path, x.name, str(x.targets))) diff --git a/tools/ci/idf_pytest/tests/test_get_pytest_cases.py b/tools/ci/idf_pytest/tests/test_get_pytest_cases.py new file mode 100644 index 0000000000..64a54bcead --- /dev/null +++ b/tools/ci/idf_pytest/tests/test_get_pytest_cases.py @@ -0,0 +1,94 @@ +# SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 + +import os +import sys +from pathlib import Path + +from idf_pytest.constants import CollectMode + +try: + from idf_pytest.script import get_pytest_cases +except ImportError: + sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..')) + + from idf_pytest.script import get_pytest_cases + +TEMPLATE_SCRIPT = ''' +import pytest + +@pytest.mark.esp32 +@pytest.mark.esp32s2 +def test_foo_single(dut): + pass + +@pytest.mark.parametrize( + 'count, target', [ + (2, 'esp32|esp32s2'), + (3, 'esp32s2|esp32s2|esp32s3'), + ], indirect=True +) +def test_foo_multi(dut): + pass + +@pytest.mark.esp32 +@pytest.mark.esp32s2 +@pytest.mark.parametrize( + 'count', [2], indirect=True +) +def test_foo_multi_with_marker(dut): + pass +''' + + +def test_get_pytest_cases_single_specific(tmp_path: Path) -> None: + script = tmp_path / 'pytest_get_pytest_cases_single_specific.py' + script.write_text(TEMPLATE_SCRIPT) + cases = get_pytest_cases([str(tmp_path)], 'esp32') + + assert len(cases) == 1 + assert cases[0].targets == ['esp32'] + + +def test_get_pytest_cases_multi_specific(tmp_path: Path) -> None: + script = tmp_path / 'pytest_get_pytest_cases_multi_specific.py' + script.write_text(TEMPLATE_SCRIPT) + cases = get_pytest_cases([str(tmp_path)], 'esp32s3,esp32s2, esp32s2') + + assert len(cases) == 1 + assert cases[0].targets == ['esp32s2', 'esp32s2', 'esp32s3'] + + +def test_get_pytest_cases_multi_all(tmp_path: Path) -> None: + script = tmp_path / 'pytest_get_pytest_cases_multi_all.py' + script.write_text(TEMPLATE_SCRIPT) + cases = get_pytest_cases([str(tmp_path)], CollectMode.MULTI_ALL_WITH_PARAM) + + assert len(cases) == 2 + assert cases[0].targets == ['esp32', 'esp32s2'] + assert cases[1].targets == ['esp32s2', 'esp32s2', 'esp32s3'] + + +def test_get_pytest_cases_all(tmp_path: Path) -> None: + script = tmp_path / 'pytest_get_pytest_cases_all.py' + script.write_text(TEMPLATE_SCRIPT) + cases = get_pytest_cases([str(tmp_path)], CollectMode.ALL) + + assert len(cases) == 6 + assert cases[0].targets == ['esp32', 'esp32s2'] + assert cases[0].name == 'test_foo_multi' + + assert cases[1].targets == ['esp32s2', 'esp32s2', 'esp32s3'] + assert cases[1].name == 'test_foo_multi' + + assert cases[2].targets == ['esp32', 'esp32'] + assert cases[2].name == 'test_foo_multi_with_marker' + + assert cases[3].targets == ['esp32s2', 'esp32s2'] + assert cases[3].name == 'test_foo_multi_with_marker' + + assert cases[4].targets == ['esp32'] + assert cases[4].name == 'test_foo_single' + + assert cases[5].targets == ['esp32s2'] + assert cases[5].name == 'test_foo_single' diff --git a/tools/ci/idf_pytest/utils.py b/tools/ci/idf_pytest/utils.py index aef1a776c5..1e06354668 100644 --- a/tools/ci/idf_pytest/utils.py +++ b/tools/ci/idf_pytest/utils.py @@ -6,10 +6,10 @@ import os import typing as t from xml.etree import ElementTree as ET -from .constants import TARGET_MARKERS - -def format_case_id(target: t.Optional[str], config: t.Optional[str], case: str, is_qemu: bool = False, params: t.Optional[dict] = None) -> str: +def format_case_id( + target: t.Optional[str], config: t.Optional[str], case: str, is_qemu: bool = False, params: t.Optional[dict] = None +) -> str: parts = [] if target: parts.append((str(target) + '_qemu') if is_qemu else str(target)) @@ -23,23 +23,6 @@ def format_case_id(target: t.Optional[str], config: t.Optional[str], case: str, return '.'.join(parts) -def get_target_marker_from_expr(markexpr: str) -> str: - candidates = set() - # we use `-m "esp32 and generic"` in our CI to filter the test cases - # this doesn't cover all use cases, but fit what we do in CI. - for marker in markexpr.split('and'): - marker = marker.strip() - if marker in TARGET_MARKERS: - candidates.add(marker) - - if len(candidates) > 1: - raise ValueError(f'Specified more than one target markers: {candidates}. Please specify no more than one.') - elif len(candidates) == 1: - return candidates.pop() - else: - raise ValueError('Please specify one target marker via "--target [TARGET]" or via "-m [TARGET]"') - - def merge_junit_files(junit_files: t.List[str], target_path: str) -> None: if len(junit_files) <= 1: return @@ -78,3 +61,7 @@ def merge_junit_files(junit_files: t.List[str], target_path: str) -> None: with open(target_path, 'wb') as fw: fw.write(ET.tostring(merged_testsuite)) + + +def comma_sep_str_to_list(s: str) -> t.List[str]: + return [s.strip() for s in s.split(',') if s.strip()] From b6a93bf81e09b2dc83c90d65ff33ee3397b663a1 Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Thu, 30 Nov 2023 15:38:49 +0100 Subject: [PATCH 04/18] ci(i154): migrate from .gitlab/ci/rules.yml to .build-test-rules.yml --- .gitlab/ci/dependencies/dependencies.yml | 16 ----- .gitlab/ci/rules.yml | 32 ---------- .gitlab/ci/target-test.yml | 77 ------------------------ examples/zigbee/.build-test-rules.yml | 3 +- 4 files changed, 2 insertions(+), 126 deletions(-) diff --git a/.gitlab/ci/dependencies/dependencies.yml b/.gitlab/ci/dependencies/dependencies.yml index 7ab9e4a0ef..3f16b63d6c 100644 --- a/.gitlab/ci/dependencies/dependencies.yml +++ b/.gitlab/ci/dependencies/dependencies.yml @@ -119,7 +119,6 @@ - sdio # pytest*sdio* - usb # USB Device & Host tests - adc # pytest*adc* - - i154 - flash_multi - ecdsa - nvs_encr_hmac @@ -149,21 +148,6 @@ - "build:example_test" - build:target_test -# For i154 runners -"test:example_test-i154": - patterns: - - "example_test-i154" - - "target_test-i154" - labels: - - target_test - - example_test - included_in: - - "build:example_test-esp32s3" - - "build:example_test-esp32c6" - - "build:example_test-esp32h2" - - "build:example_test" - - build:target_test - "test:host_test": labels: - host_test diff --git a/.gitlab/ci/rules.yml b/.gitlab/ci/rules.yml index dea9e18d24..98e18be4f1 100644 --- a/.gitlab/ci/rules.yml +++ b/.gitlab/ci/rules.yml @@ -83,22 +83,6 @@ - "components/**/*" -.patterns-component_ut-i154: &patterns-component_ut-i154 - - "components/{esp_phy,esp_coex}/???[!t]*/**/*" - - "components/{esp_phy,esp_coex}/??[!s]?*/**/*" - - "components/{esp_phy,esp_coex}/???/**/*" - - "components/{esp_phy,esp_coex}/*" - - "components/ieee802154/**/*" - -.patterns-example_test-i154: &patterns-example_test-i154 - - "components/{esp_netif,ieee802154,lwip,openthread}/???[!t]*/**/*" - - "components/{esp_netif,ieee802154,lwip,openthread}/??[!s]?*/**/*" - - "components/{esp_netif,ieee802154,lwip,openthread}/???/**/*" - - "components/{esp_netif,ieee802154,lwip,openthread}/??/**/*" - - "components/{esp_netif,ieee802154,lwip,openthread}/*" - - "examples/common_components/iperf/**/*" - - "examples/openthread/**/*" - .patterns-target_test-wifi: &patterns-target_test-wifi - "components/{esp_wifi,esp_netif,lwip,esp_phy,wpa_supplicant,esp_coex}/???[!t]*/**/*" - "components/{esp_wifi,esp_netif,lwip,esp_phy,wpa_supplicant,esp_coex}/??[!s]?*/**/*" @@ -1640,8 +1624,6 @@ changes: *patterns-component_ut-adc - <<: *if-dev-push changes: *patterns-component_ut-flash_multi - - <<: *if-dev-push - changes: *patterns-component_ut-i154 - <<: *if-dev-push changes: *patterns-component_ut-nvs_encr_hmac - <<: *if-dev-push @@ -1662,8 +1644,6 @@ changes: *patterns-example_test-adc - <<: *if-dev-push changes: *patterns-example_test-ethernet - - <<: *if-dev-push - changes: *patterns-example_test-i154 - <<: *if-dev-push changes: *patterns-example_test-nvs_encr_hmac - <<: *if-dev-push @@ -2651,18 +2631,6 @@ - <<: *if-dev-push changes: *patterns-target_test-wifi -.rules:test:example_test-i154: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-example_test - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-example_test-i154 - .rules:test:host_test: rules: - <<: *if-revert-branch diff --git a/.gitlab/ci/target-test.yml b/.gitlab/ci/target-test.yml index 28d46e23ce..ddac7aa93a 100644 --- a/.gitlab/ci/target-test.yml +++ b/.gitlab/ci/target-test.yml @@ -1305,83 +1305,6 @@ pytest_components_esp32c6_adc: artifacts: false tags: [ esp32c6, adc ] -pytest_examples_esp32c6_i154: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32c6 - needs: - - job: build_pytest_examples_esp32c6 - artifacts: false - tags: [ esp32c6, ieee802154 ] - -pytest_examples_openthread_br: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-i154 - needs: - - job: build_pytest_examples_esp32s3 - artifacts: false - - job: build_pytest_examples_esp32c6 - artifacts: false - - job: build_pytest_examples_esp32h2 - artifacts: false - tags: [ esp32c6, openthread_br ] - script: - - retry_failed git clone $KNOWN_FAILURE_CASES_REPO known_failure_cases - # get runner env config file - - retry_failed git clone $TEST_ENV_CONFIG_REPO - - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs - # using runner tags as markers to filter the test cases - # Runner tags are comma separated, replace the comma with " and " for markers - - job_tags=$(python tools/ci/python_packages/gitlab_api.py get_job_tags $CI_PROJECT_ID --job_id $CI_JOB_ID) - - markers=$(echo $job_tags | sed -e "s/,/ and /g") - # download the artifacts, requires s3, c6, h2 chips - - run_cmd python tools/ci/artifacts_handler.py download --job-name "build_pytest_examples_esp32s3" - - run_cmd python tools/ci/artifacts_handler.py download --job-name "build_pytest_examples_esp32c6" - - run_cmd python tools/ci/artifacts_handler.py download --job-name "build_pytest_examples_esp32h2" - - run_cmd pytest $TEST_DIR - -m \"${markers}\" - --junitxml=XUNIT_RESULT.xml - --ignore-result-files known_failure_cases/known_failure_cases.txt - --parallel-count ${CI_NODE_TOTAL:-1} - --parallel-index ${CI_NODE_INDEX:-1} - ${PYTEST_EXTRA_FLAGS} - --app-info-filepattern \"list_job_*.txt\" - -pytest_examples_openthread_bbr: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-i154 - needs: - - job: build_pytest_examples_esp32s3 - artifacts: false - - job: build_pytest_examples_esp32c6 - artifacts: false - - job: build_pytest_examples_esp32h2 - artifacts: false - tags: [ esp32c6, openthread_bbr ] - script: - - retry_failed git clone $KNOWN_FAILURE_CASES_REPO known_failure_cases - # get runner env config file - - retry_failed git clone $TEST_ENV_CONFIG_REPO - - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs - # using runner tags as markers to filter the test cases - # Runner tags are comma separated, replace the comma with " and " for markers - - job_tags=$(python tools/ci/python_packages/gitlab_api.py get_job_tags $CI_PROJECT_ID --job_id $CI_JOB_ID) - - markers=$(echo $job_tags | sed -e "s/,/ and /g") - # download the artifacts, requires s3, c6, h2 chips - - run_cmd python tools/ci/artifacts_handler.py download --job-name "build_pytest_examples_esp32s3" - - run_cmd python tools/ci/artifacts_handler.py download --job-name "build_pytest_examples_esp32c6" - - run_cmd python tools/ci/artifacts_handler.py download --job-name "build_pytest_examples_esp32h2" - - run_cmd pytest $TEST_DIR - -m \"${markers}\" - --junitxml=XUNIT_RESULT.xml - --ignore-result-files known_failure_cases/known_failure_cases.txt - --parallel-count ${CI_NODE_TOTAL:-1} - --parallel-index ${CI_NODE_INDEX:-1} - ${PYTEST_EXTRA_FLAGS} - --app-info-filepattern \"list_job_*.txt\" - pytest_examples_openthread_sleep: extends: - .pytest_examples_dir_template diff --git a/examples/zigbee/.build-test-rules.yml b/examples/zigbee/.build-test-rules.yml index d733dc017f..ecffa999bd 100644 --- a/examples/zigbee/.build-test-rules.yml +++ b/examples/zigbee/.build-test-rules.yml @@ -1,8 +1,9 @@ # Documentation: .gitlab/ci/README.md#manifest-file-to-control-the-buildtest-apps .zigbee_dependencies: &zigbee_dependencies + depends_components: + - ieee802154 depends_filepatterns: - - components/ieee802154/**/* - examples/zigbee/light_sample/**/* examples/zigbee/esp_zigbee_gateway: From 3645c041e2b043dfd95b6899bbbe1c920bccf2c0 Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Thu, 30 Nov 2023 15:46:16 +0100 Subject: [PATCH 05/18] ci(flash_multi): migrate from .gitlab/ci/rules.yml to .build-test-rules.yml --- .gitlab/ci/dependencies/dependencies.yml | 1 - .gitlab/ci/rules.yml | 60 ------------------------ .gitlab/ci/target-test.yml | 36 -------------- 3 files changed, 97 deletions(-) diff --git a/.gitlab/ci/dependencies/dependencies.yml b/.gitlab/ci/dependencies/dependencies.yml index 3f16b63d6c..806654928f 100644 --- a/.gitlab/ci/dependencies/dependencies.yml +++ b/.gitlab/ci/dependencies/dependencies.yml @@ -119,7 +119,6 @@ - sdio # pytest*sdio* - usb # USB Device & Host tests - adc # pytest*adc* - - flash_multi - ecdsa - nvs_encr_hmac patterns: diff --git a/.gitlab/ci/rules.yml b/.gitlab/ci/rules.yml index 98e18be4f1..d0faa3bea6 100644 --- a/.gitlab/ci/rules.yml +++ b/.gitlab/ci/rules.yml @@ -248,12 +248,6 @@ - "components/esp_driver_sdmmc/include/driver/sdmmc*.h" - "components/sdmmc/**/*" -# for jobs: component_ut_pytest_flash_multi -.patterns-component_ut-flash_multi: &patterns-component_ut-flash_multi - - "components/spi_flash/**/*" - - "components/hal/spi_flash*.c" - - "components/hal/include/hal/spi_flash*.h" - # for jobs: USB host and device examples .patterns-example_test-usb: &patterns-example_test-usb - "components/hal/usb*.c" @@ -1622,8 +1616,6 @@ changes: *patterns-component_ut - <<: *if-dev-push changes: *patterns-component_ut-adc - - <<: *if-dev-push - changes: *patterns-component_ut-flash_multi - <<: *if-dev-push changes: *patterns-component_ut-nvs_encr_hmac - <<: *if-dev-push @@ -1700,19 +1692,6 @@ - <<: *if-dev-push changes: *patterns-target_test-adc -.rules:test:component_ut-esp32-flash_multi: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-component_ut - - <<: *if-label-component_ut_esp32 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-component_ut-flash_multi - .rules:test:component_ut-esp32-sdio: rules: - <<: *if-revert-branch @@ -1797,19 +1776,6 @@ - <<: *if-dev-push changes: *patterns-target_test-adc -.rules:test:component_ut-esp32c3-flash_multi: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-component_ut - - <<: *if-label-component_ut_esp32c3 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-component_ut-flash_multi - .rules:test:component_ut-esp32c3-nvs_encr_hmac: rules: - <<: *if-revert-branch @@ -1961,19 +1927,6 @@ - <<: *if-dev-push changes: *patterns-target_test-adc -.rules:test:component_ut-esp32s2-flash_multi: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-component_ut - - <<: *if-label-component_ut_esp32s2 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-component_ut-flash_multi - .rules:test:component_ut-esp32s2-sdio: rules: - <<: *if-revert-branch @@ -2015,19 +1968,6 @@ - <<: *if-dev-push changes: *patterns-target_test-adc -.rules:test:component_ut-esp32s3-flash_multi: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-component_ut - - <<: *if-label-component_ut_esp32s3 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-component_ut-flash_multi - .rules:test:component_ut-esp32s3-usb: rules: - <<: *if-revert-branch diff --git a/.gitlab/ci/target-test.yml b/.gitlab/ci/target-test.yml index ddac7aa93a..640d97c998 100644 --- a/.gitlab/ci/target-test.yml +++ b/.gitlab/ci/target-test.yml @@ -861,15 +861,6 @@ pytest_components_esp32_flash_encryption: artifacts: false tags: [ esp32, flash_encryption ] -pytest_components_esp32_flash_multi: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32-flash_multi - needs: - - job: build_pytest_components_esp32 - artifacts: false - tags: [ esp32, flash_multi ] - pytest_components_esp32_xtal32k: extends: - .pytest_components_dir_template @@ -934,15 +925,6 @@ pytest_components_esp32s2_adc: artifacts: false tags: [ esp32s2, adc ] -pytest_components_esp32s2_flash_multi: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32s2-flash_multi - needs: - - job: build_pytest_components_esp32s2 - artifacts: false - tags: [ esp32s2, flash_multi ] - pytest_components_esp32s3_generic: extends: - .pytest_components_dir_template @@ -1007,15 +989,6 @@ pytest_components_esp32s3_flash_encryption_f8r8: artifacts: false tags: [ esp32s3, flash_encryption_f8r8 ] -pytest_components_esp32s3_flash_multi: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32s3-flash_multi - needs: - - job: build_pytest_components_esp32s3 - artifacts: false - tags: [ esp32s3, flash_multi ] - pytest_components_esp32s3_mspi_f4r4: extends: - .pytest_components_dir_template @@ -1178,15 +1151,6 @@ pytest_components_esp32c3_nvs_encr_hmac: artifacts: false tags: [ esp32c3, nvs_encr_hmac ] -pytest_components_esp32c3_flash_multi: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32c3-flash_multi - needs: - - job: build_pytest_components_esp32c3 - artifacts: false - tags: [ esp32c3, flash_multi ] - pytest_components_esp32_sdspi: extends: - .pytest_components_dir_template From 7138b977f8ed0df27db21a51022e80394b6314f6 Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Thu, 30 Nov 2023 17:07:00 +0100 Subject: [PATCH 06/18] ci(nvs_encr_hmac): migrate from .gitlab/ci/rules.yml to .build-test-rules.yml --- .gitlab/ci/dependencies/dependencies.yml | 1 - .gitlab/ci/rules.yml | 42 ------------------------ .gitlab/ci/target-test.yml | 18 ---------- examples/security/.build-test-rules.yml | 5 +++ 4 files changed, 5 insertions(+), 61 deletions(-) diff --git a/.gitlab/ci/dependencies/dependencies.yml b/.gitlab/ci/dependencies/dependencies.yml index 806654928f..20adde4dfe 100644 --- a/.gitlab/ci/dependencies/dependencies.yml +++ b/.gitlab/ci/dependencies/dependencies.yml @@ -120,7 +120,6 @@ - usb # USB Device & Host tests - adc # pytest*adc* - ecdsa - - nvs_encr_hmac patterns: - "{0}-{1}-{2}" - "{0}-{2}" diff --git a/.gitlab/ci/rules.yml b/.gitlab/ci/rules.yml index d0faa3bea6..3700ecf93d 100644 --- a/.gitlab/ci/rules.yml +++ b/.gitlab/ci/rules.yml @@ -285,18 +285,6 @@ - "components/{hal,efuse}/*" - "components/mbedtls/port/ecdsa/*" -.patterns-component_ut-nvs_encr_hmac: &patterns-component_ut-nvs_encr_hmac - - "components/nvs_flash/**/*" - - "components/nvs_sec_provider/**/*" - -.patterns-example_test-nvs_encr_hmac: &patterns-example_test-nvs_encr_hmac - - "components/{nvs_flash,nvs_sec_provider}/???[!t]*/**/*" - - "components/{nvs_flash,nvs_sec_provider}/??[!s]?*/**/*" - - "components/{nvs_flash,nvs_sec_provider}/???/**/*" - - "components/{nvs_flash,nvs_sec_provider}/??/**/*" - - "components/{nvs_flash,nvs_sec_provider}/*" - - "examples/security/nvs_encryption_hmac/**/*" - ############## # if anchors # ############## @@ -1616,8 +1604,6 @@ changes: *patterns-component_ut - <<: *if-dev-push changes: *patterns-component_ut-adc - - <<: *if-dev-push - changes: *patterns-component_ut-nvs_encr_hmac - <<: *if-dev-push changes: *patterns-component_ut-sdio - <<: *if-dev-push @@ -1636,8 +1622,6 @@ changes: *patterns-example_test-adc - <<: *if-dev-push changes: *patterns-example_test-ethernet - - <<: *if-dev-push - changes: *patterns-example_test-nvs_encr_hmac - <<: *if-dev-push changes: *patterns-example_test-sdio - <<: *if-dev-push @@ -1776,19 +1760,6 @@ - <<: *if-dev-push changes: *patterns-target_test-adc -.rules:test:component_ut-esp32c3-nvs_encr_hmac: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-component_ut - - <<: *if-label-component_ut_esp32c3 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-component_ut-nvs_encr_hmac - .rules:test:component_ut-esp32c3-sdio: rules: - <<: *if-revert-branch @@ -2324,19 +2295,6 @@ when: never - <<: *if-example_test-ota-include_nightly_run-rule -.rules:test:example_test-esp32c3-nvs_encr_hmac: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-example_test - - <<: *if-label-example_test_esp32c3 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-example_test-nvs_encr_hmac - .rules:test:example_test-esp32c3-sdio: rules: - <<: *if-revert-branch diff --git a/.gitlab/ci/target-test.yml b/.gitlab/ci/target-test.yml index 640d97c998..a65167ab7a 100644 --- a/.gitlab/ci/target-test.yml +++ b/.gitlab/ci/target-test.yml @@ -544,15 +544,6 @@ pytest_examples_esp32c3_flash_encryption: artifacts: false tags: [ esp32c3, flash_encryption ] -pytest_examples_esp32c3_nvs_encr_hmac: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32c3-nvs_encr_hmac - needs: - - job: build_pytest_examples_esp32c3 - artifacts: false - tags: [ esp32c3, nvs_encr_hmac ] - pytest_examples_esp32s2_usb_device: extends: - .pytest_examples_dir_template @@ -1142,15 +1133,6 @@ pytest_components_esp32c3_flash_encryption: artifacts: false tags: [ esp32c3, flash_encryption ] -pytest_components_esp32c3_nvs_encr_hmac: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32c3-nvs_encr_hmac - needs: - - job: build_pytest_components_esp32c3 - artifacts: false - tags: [ esp32c3, nvs_encr_hmac ] - pytest_components_esp32_sdspi: extends: - .pytest_components_dir_template diff --git a/examples/security/.build-test-rules.yml b/examples/security/.build-test-rules.yml index e24e449691..81f7d70edd 100644 --- a/examples/security/.build-test-rules.yml +++ b/examples/security/.build-test-rules.yml @@ -13,3 +13,8 @@ examples/security/nvs_encryption_hmac: - if: IDF_TARGET not in ["esp32c3"] temporary: true reason: lack of runners + depends_components: + - nvs_flash + - nvs_sec_provider + depends_filepatterns: + - examples/security/nvs_encryption_hmac/**/* From 9a9b1cd11f86a88e9086162d7f3dcb05034057b2 Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Thu, 30 Nov 2023 15:54:55 +0100 Subject: [PATCH 07/18] ci(ecdsa): migrate from .gitlab/ci/rules.yml to .build-test-rules.yml --- .gitlab/ci/dependencies/dependencies.yml | 1 - .gitlab/ci/rules.yml | 35 ------------------------ .gitlab/ci/target-test.yml | 9 ------ components/mbedtls/.build-test-rules.yml | 4 +++ 4 files changed, 4 insertions(+), 45 deletions(-) diff --git a/.gitlab/ci/dependencies/dependencies.yml b/.gitlab/ci/dependencies/dependencies.yml index 20adde4dfe..295d44962c 100644 --- a/.gitlab/ci/dependencies/dependencies.yml +++ b/.gitlab/ci/dependencies/dependencies.yml @@ -119,7 +119,6 @@ - sdio # pytest*sdio* - usb # USB Device & Host tests - adc # pytest*adc* - - ecdsa patterns: - "{0}-{1}-{2}" - "{0}-{2}" diff --git a/.gitlab/ci/rules.yml b/.gitlab/ci/rules.yml index 3700ecf93d..bd96a26084 100644 --- a/.gitlab/ci/rules.yml +++ b/.gitlab/ci/rules.yml @@ -277,14 +277,6 @@ .patterns-example_test-adc: &patterns-example_test-adc - "examples/peripherals/adc/**/*" -.patterns-target_test-ecdsa: &patterns-target_test-ecdsa - - "components/{hal,efuse}/???[!t]*/**/*" - - "components/{hal,efuse}/??[!s]?*/**/*" - - "components/{hal,efuse}/???/**/*" - - "components/{hal,efuse}/??/**/*" - - "components/{hal,efuse}/*" - - "components/mbedtls/port/ecdsa/*" - ############## # if anchors # ############## @@ -601,8 +593,6 @@ changes: *patterns-downloadable-tools - <<: *if-dev-push changes: *patterns-target_test-adc - - <<: *if-dev-push - changes: *patterns-target_test-ecdsa - <<: *if-dev-push changes: *patterns-target_test-wifi @@ -639,8 +629,6 @@ changes: *patterns-downloadable-tools - <<: *if-dev-push changes: *patterns-target_test-adc - - <<: *if-dev-push - changes: *patterns-target_test-ecdsa - <<: *if-dev-push changes: *patterns-target_test-wifi @@ -677,8 +665,6 @@ changes: *patterns-downloadable-tools - <<: *if-dev-push changes: *patterns-target_test-adc - - <<: *if-dev-push - changes: *patterns-target_test-ecdsa - <<: *if-dev-push changes: *patterns-target_test-wifi @@ -715,8 +701,6 @@ changes: *patterns-downloadable-tools - <<: *if-dev-push changes: *patterns-target_test-adc - - <<: *if-dev-push - changes: *patterns-target_test-ecdsa - <<: *if-dev-push changes: *patterns-target_test-wifi @@ -939,8 +923,6 @@ changes: *patterns-downloadable-tools - <<: *if-dev-push changes: *patterns-target_test-adc - - <<: *if-dev-push - changes: *patterns-target_test-ecdsa - <<: *if-dev-push changes: *patterns-target_test-wifi @@ -965,8 +947,6 @@ changes: *patterns-downloadable-tools - <<: *if-dev-push changes: *patterns-target_test-adc - - <<: *if-dev-push - changes: *patterns-target_test-ecdsa - <<: *if-dev-push changes: *patterns-target_test-wifi @@ -1630,8 +1610,6 @@ changes: *patterns-example_test-wifi - <<: *if-dev-push changes: *patterns-target_test-adc - - <<: *if-dev-push - changes: *patterns-target_test-ecdsa - <<: *if-dev-push changes: *patterns-target_test-wifi @@ -1844,19 +1822,6 @@ - <<: *if-dev-push changes: *patterns-target_test-adc -.rules:test:component_ut-esp32h2-ecdsa: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-component_ut - - <<: *if-label-component_ut_esp32h2 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-target_test-ecdsa - .rules:test:component_ut-esp32p4: rules: - <<: *if-revert-branch diff --git a/.gitlab/ci/target-test.yml b/.gitlab/ci/target-test.yml index a65167ab7a..58fe684685 100644 --- a/.gitlab/ci/target-test.yml +++ b/.gitlab/ci/target-test.yml @@ -1216,15 +1216,6 @@ pytest_components_esp32h2_adc: artifacts: false tags: [ esp32h2, adc ] -pytest_components_esp32h2_ecdsa: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32h2-ecdsa - needs: - - job: build_pytest_components_esp32h2 - artifacts: false - tags: [ esp32h2, ecdsa_efuse ] - pytest_components_esp32h2_usb_serial_jtag: extends: - .pytest_components_dir_template diff --git a/components/mbedtls/.build-test-rules.yml b/components/mbedtls/.build-test-rules.yml index 705dd4b66b..f33fabc474 100644 --- a/components/mbedtls/.build-test-rules.yml +++ b/components/mbedtls/.build-test-rules.yml @@ -5,3 +5,7 @@ components/mbedtls/test_apps: - if: CONFIG_NAME == "psram" and SOC_SPIRAM_SUPPORTED != 1 - if: CONFIG_NAME == "psram_all_ext" and SOC_SPIRAM_SUPPORTED != 1 - if: CONFIG_NAME == "ecdsa_sign" and SOC_ECDSA_SUPPORTED != 1 + depends_components: + - efuse + depends_filepatterns: + - components/mbedtls/port/ecdsa/* From ed1c892a8abde00fd2cd877ed2adeaf7a60be3f1 Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Mon, 4 Dec 2023 09:08:41 +0100 Subject: [PATCH 08/18] ci(adc): migrate from .gitlab/ci/rules.yml to .build-test-rules.yml --- .gitlab/ci/dependencies/dependencies.yml | 1 - .gitlab/ci/rules.yml | 242 ------------------ .gitlab/ci/target-test.yml | 126 --------- .../driver/test_apps/.build-test-rules.yml | 6 + examples/peripherals/.build-test-rules.yml | 19 ++ 5 files changed, 25 insertions(+), 369 deletions(-) diff --git a/.gitlab/ci/dependencies/dependencies.yml b/.gitlab/ci/dependencies/dependencies.yml index 295d44962c..d1ecaeec2e 100644 --- a/.gitlab/ci/dependencies/dependencies.yml +++ b/.gitlab/ci/dependencies/dependencies.yml @@ -118,7 +118,6 @@ - ethernet # pytest*ethernet* - sdio # pytest*sdio* - usb # USB Device & Host tests - - adc # pytest*adc* patterns: - "{0}-{1}-{2}" - "{0}-{2}" diff --git a/.gitlab/ci/rules.yml b/.gitlab/ci/rules.yml index bd96a26084..9d14675c64 100644 --- a/.gitlab/ci/rules.yml +++ b/.gitlab/ci/rules.yml @@ -263,20 +263,6 @@ - "components/hal/esp32s*/include/hal/usb*.h" - "components/usb/**/*" -# for jobs: *_pytest_esp32x_adc: -.patterns-target_test-adc: &patterns-target_test-adc - - "components/{esp_adc,driver,hal,esp_hw_support,efuse}/???[!t]*/**/*" - - "components/{esp_adc,driver,hal,esp_hw_support,efuse}/??[!s]?*/**/*" - - "components/{esp_adc,driver,hal,esp_hw_support,efuse}/???/**/*" - - "components/{esp_adc,driver,hal,esp_hw_support,efuse}/??/**/*" - - "components/{esp_adc,driver,hal,esp_hw_support,efuse}/*" - -.patterns-component_ut-adc: &patterns-component_ut-adc - - "components/esp_adc/**/*" - -.patterns-example_test-adc: &patterns-example_test-adc - - "examples/peripherals/adc/**/*" - ############## # if anchors # ############## @@ -591,8 +577,6 @@ changes: *patterns-component_ut-wifi - <<: *if-dev-push changes: *patterns-downloadable-tools - - <<: *if-dev-push - changes: *patterns-target_test-adc - <<: *if-dev-push changes: *patterns-target_test-wifi @@ -627,8 +611,6 @@ changes: *patterns-component_ut-wifi - <<: *if-dev-push changes: *patterns-downloadable-tools - - <<: *if-dev-push - changes: *patterns-target_test-adc - <<: *if-dev-push changes: *patterns-target_test-wifi @@ -663,8 +645,6 @@ changes: *patterns-component_ut-wifi - <<: *if-dev-push changes: *patterns-downloadable-tools - - <<: *if-dev-push - changes: *patterns-target_test-adc - <<: *if-dev-push changes: *patterns-target_test-wifi @@ -699,8 +679,6 @@ changes: *patterns-component_ut-wifi - <<: *if-dev-push changes: *patterns-downloadable-tools - - <<: *if-dev-push - changes: *patterns-target_test-adc - <<: *if-dev-push changes: *patterns-target_test-wifi @@ -921,8 +899,6 @@ changes: *patterns-custom_test-wifi - <<: *if-dev-push changes: *patterns-downloadable-tools - - <<: *if-dev-push - changes: *patterns-target_test-adc - <<: *if-dev-push changes: *patterns-target_test-wifi @@ -945,8 +921,6 @@ changes: *patterns-custom_test-wifi - <<: *if-dev-push changes: *patterns-downloadable-tools - - <<: *if-dev-push - changes: *patterns-target_test-adc - <<: *if-dev-push changes: *patterns-target_test-wifi @@ -1582,8 +1556,6 @@ changes: *patterns-build_template-app - <<: *if-dev-push changes: *patterns-component_ut - - <<: *if-dev-push - changes: *patterns-component_ut-adc - <<: *if-dev-push changes: *patterns-component_ut-sdio - <<: *if-dev-push @@ -1598,8 +1570,6 @@ changes: *patterns-downloadable-tools - <<: *if-dev-push changes: *patterns-example_test - - <<: *if-dev-push - changes: *patterns-example_test-adc - <<: *if-dev-push changes: *patterns-example_test-ethernet - <<: *if-dev-push @@ -1608,8 +1578,6 @@ changes: *patterns-example_test-usb - <<: *if-dev-push changes: *patterns-example_test-wifi - - <<: *if-dev-push - changes: *patterns-target_test-adc - <<: *if-dev-push changes: *patterns-target_test-wifi @@ -1639,21 +1607,6 @@ - <<: *if-dev-push changes: *patterns-component_ut -.rules:test:component_ut-esp32-adc: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-component_ut - - <<: *if-label-component_ut_esp32 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-component_ut-adc - - <<: *if-dev-push - changes: *patterns-target_test-adc - .rules:test:component_ut-esp32-sdio: rules: - <<: *if-revert-branch @@ -1680,21 +1633,6 @@ - <<: *if-dev-push changes: *patterns-component_ut -.rules:test:component_ut-esp32c2-adc: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-component_ut - - <<: *if-label-component_ut_esp32c2 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-component_ut-adc - - <<: *if-dev-push - changes: *patterns-target_test-adc - .rules:test:component_ut-esp32c2-wifi: rules: - <<: *if-revert-branch @@ -1723,21 +1661,6 @@ - <<: *if-dev-push changes: *patterns-component_ut -.rules:test:component_ut-esp32c3-adc: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-component_ut - - <<: *if-label-component_ut_esp32c3 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-component_ut-adc - - <<: *if-dev-push - changes: *patterns-target_test-adc - .rules:test:component_ut-esp32c3-sdio: rules: - <<: *if-revert-branch @@ -1779,21 +1702,6 @@ - <<: *if-dev-push changes: *patterns-component_ut -.rules:test:component_ut-esp32c6-adc: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-component_ut - - <<: *if-label-component_ut_esp32c6 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-component_ut-adc - - <<: *if-dev-push - changes: *patterns-target_test-adc - .rules:test:component_ut-esp32h2: rules: - <<: *if-revert-branch @@ -1807,21 +1715,6 @@ - <<: *if-dev-push changes: *patterns-component_ut -.rules:test:component_ut-esp32h2-adc: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-component_ut - - <<: *if-label-component_ut_esp32h2 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-component_ut-adc - - <<: *if-dev-push - changes: *patterns-target_test-adc - .rules:test:component_ut-esp32p4: rules: - <<: *if-revert-branch @@ -1848,21 +1741,6 @@ - <<: *if-dev-push changes: *patterns-component_ut -.rules:test:component_ut-esp32s2-adc: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-component_ut - - <<: *if-label-component_ut_esp32s2 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-component_ut-adc - - <<: *if-dev-push - changes: *patterns-target_test-adc - .rules:test:component_ut-esp32s2-sdio: rules: - <<: *if-revert-branch @@ -1889,21 +1767,6 @@ - <<: *if-dev-push changes: *patterns-component_ut -.rules:test:component_ut-esp32s3-adc: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-component_ut - - <<: *if-label-component_ut_esp32s3 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-component_ut-adc - - <<: *if-dev-push - changes: *patterns-target_test-adc - .rules:test:component_ut-esp32s3-usb: rules: - <<: *if-revert-branch @@ -2111,21 +1974,6 @@ - <<: *if-dev-push changes: *patterns-example_test -.rules:test:example_test-esp32-adc: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-example_test - - <<: *if-label-example_test_esp32 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-example_test-adc - - <<: *if-dev-push - changes: *patterns-target_test-adc - .rules:test:example_test-esp32-ethernet: rules: - <<: *if-revert-branch @@ -2191,21 +2039,6 @@ - <<: *if-dev-push changes: *patterns-example_test -.rules:test:example_test-esp32c2-adc: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-example_test - - <<: *if-label-example_test_esp32c2 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-example_test-adc - - <<: *if-dev-push - changes: *patterns-target_test-adc - .rules:test:example_test-esp32c2-wifi: rules: - <<: *if-revert-branch @@ -2236,21 +2069,6 @@ - <<: *if-dev-push changes: *patterns-example_test -.rules:test:example_test-esp32c3-adc: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-example_test - - <<: *if-label-example_test_esp32c3 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-example_test-adc - - <<: *if-dev-push - changes: *patterns-target_test-adc - .rules:test:example_test-esp32c3-include_nightly_run-rule: rules: - <<: *if-revert-branch @@ -2303,21 +2121,6 @@ - <<: *if-dev-push changes: *patterns-example_test -.rules:test:example_test-esp32c6-adc: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-example_test - - <<: *if-label-example_test_esp32c6 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-example_test-adc - - <<: *if-dev-push - changes: *patterns-target_test-adc - .rules:test:example_test-esp32c6-wifi: rules: - <<: *if-revert-branch @@ -2348,21 +2151,6 @@ - <<: *if-dev-push changes: *patterns-example_test -.rules:test:example_test-esp32h2-adc: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-example_test - - <<: *if-label-example_test_esp32h2 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-example_test-adc - - <<: *if-dev-push - changes: *patterns-target_test-adc - .rules:test:example_test-esp32p4: rules: - <<: *if-revert-branch @@ -2393,21 +2181,6 @@ - <<: *if-dev-push changes: *patterns-example_test -.rules:test:example_test-esp32s2-adc: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-example_test - - <<: *if-label-example_test_esp32s2 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-example_test-adc - - <<: *if-dev-push - changes: *patterns-target_test-adc - .rules:test:example_test-esp32s2-sdio: rules: - <<: *if-revert-branch @@ -2464,21 +2237,6 @@ - <<: *if-dev-push changes: *patterns-example_test -.rules:test:example_test-esp32s3-adc: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-example_test - - <<: *if-label-example_test_esp32s3 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-example_test-adc - - <<: *if-dev-push - changes: *patterns-target_test-adc - .rules:test:example_test-esp32s3-wifi: rules: - <<: *if-revert-branch diff --git a/.gitlab/ci/target-test.yml b/.gitlab/ci/target-test.yml index 58fe684685..7f2062e482 100644 --- a/.gitlab/ci/target-test.yml +++ b/.gitlab/ci/target-test.yml @@ -598,69 +598,6 @@ pytest_examples_esp32_extflash: artifacts: false tags: [ esp32, external_flash ] -pytest_examples_esp32_adc: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32-adc - needs: - - job: build_pytest_examples_esp32 - artifacts: false - tags: [ esp32, adc ] - -pytest_examples_esp32s2_adc: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32s2-adc - needs: - - job: build_pytest_examples_esp32s2 - artifacts: false - tags: [ esp32s2, adc ] - -pytest_examples_esp32s3_adc: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32s3-adc - needs: - - job: build_pytest_examples_esp32s3 - artifacts: false - tags: [ esp32s3, adc ] - -pytest_examples_esp32c3_adc: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32c3-adc - needs: - - job: build_pytest_examples_esp32c3 - artifacts: false - tags: [ esp32c3, adc ] - -pytest_examples_esp32c2_adc: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32c2-adc - needs: - - job: build_pytest_examples_esp32c2 - artifacts: false - tags: [ esp32c2, adc, xtal_26mhz] - -pytest_examples_esp32c6_adc: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32c6-adc - needs: - - job: build_pytest_examples_esp32c6 - artifacts: false - tags: [ esp32c6, adc ] - -pytest_examples_esp32h2_adc: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32h2-adc - needs: - - job: build_pytest_examples_esp32h2 - artifacts: false - tags: [ esp32h2, adc ] - pytest_examples_esp32s3_emmc: extends: - .pytest_examples_dir_template @@ -704,15 +641,6 @@ pytest_components_esp32_wifi_two_dut: artifacts: false tags: [ esp32, wifi_two_dut ] -pytest_components_esp32_adc: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32-adc - needs: - - job: build_pytest_components_esp32 - artifacts: false - tags: [ esp32, adc ] - pytest_components_esp32_sdmmc: extends: - .pytest_components_dir_template @@ -907,15 +835,6 @@ pytest_components_esp32s2_generic_multi_device: artifacts: false tags: [ esp32s2, generic_multi_device ] -pytest_components_esp32s2_adc: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32s2-adc - needs: - - job: build_pytest_components_esp32s2 - artifacts: false - tags: [ esp32s2, adc ] - pytest_components_esp32s3_generic: extends: - .pytest_components_dir_template @@ -935,15 +854,6 @@ pytest_components_esp32s3_generic_multi_device: artifacts: false tags: [ esp32s3, generic_multi_device ] -pytest_components_esp32s3_adc: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32s3-adc - needs: - - job: build_pytest_components_esp32s3 - artifacts: false - tags: [ esp32s3, adc ] - pytest_components_esp32s3_octal_psram: extends: - .pytest_components_dir_template @@ -1025,15 +935,6 @@ pytest_components_esp32c2_generic: tags: [ esp32c2, generic, xtal_40mhz ] parallel: 3 -pytest_components_esp32c2_adc: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32c2-adc - needs: - - job: build_pytest_components_esp32c2 - artifacts: false - tags: [ esp32c2, adc, xtal_26mhz ] - pytest_components_esp32c2_generic_multi_device: extends: - .pytest_components_dir_template @@ -1115,15 +1016,6 @@ pytest_components_esp32c2_wifi_two_dut: artifacts: false tags: [ esp32c2, wifi_two_dut, xtal_26mhz ] -pytest_components_esp32c3_adc: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32c3-adc - needs: - - job: build_pytest_components_esp32c3 - artifacts: false - tags: [ esp32c3, adc ] - pytest_components_esp32c3_flash_encryption: extends: - .pytest_components_dir_template @@ -1207,15 +1099,6 @@ pytest_components_esp32h2_generic_multi_device: artifacts: false tags: [ esp32h2, generic_multi_device ] -pytest_components_esp32h2_adc: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32h2-adc - needs: - - job: build_pytest_components_esp32h2 - artifacts: false - tags: [ esp32h2, adc ] - pytest_components_esp32h2_usb_serial_jtag: extends: - .pytest_components_dir_template @@ -1233,15 +1116,6 @@ pytest_components_esp32c6_generic_multi_device: artifacts: false tags: [ esp32c6, generic_multi_device ] -pytest_components_esp32c6_adc: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32c6-adc - needs: - - job: build_pytest_components_esp32c6 - artifacts: false - tags: [ esp32c6, adc ] - pytest_examples_openthread_sleep: extends: - .pytest_examples_dir_template diff --git a/components/driver/test_apps/.build-test-rules.yml b/components/driver/test_apps/.build-test-rules.yml index 7696651dbc..fd259747f7 100644 --- a/components/driver/test_apps/.build-test-rules.yml +++ b/components/driver/test_apps/.build-test-rules.yml @@ -21,6 +21,12 @@ components/driver/test_apps/i2s_test_apps/legacy_i2s_driver: components/driver/test_apps/legacy_adc_driver: disable: - if: SOC_ADC_SUPPORTED != 1 + depends_components: + - efuse + - esp_driver_i2s + - esp_driver_spi + depends_filepatterns: + - components/driver/deprecated/**/*adc* components/driver/test_apps/legacy_i2c_driver: disable_test: diff --git a/examples/peripherals/.build-test-rules.yml b/examples/peripherals/.build-test-rules.yml index 1941ae6f5a..8680827930 100644 --- a/examples/peripherals/.build-test-rules.yml +++ b/examples/peripherals/.build-test-rules.yml @@ -1,5 +1,12 @@ # Documentation: .gitlab/ci/README.md#manifest-file-to-control-the-buildtest-apps +.adc_dependencies: &adc_dependencies + depends_components: + - esp_adc + - efuse + - esp_driver_i2s + - esp_driver_spi + .i2c_dependencies: &i2c_dependencies depends_filepatterns: # components @@ -10,10 +17,12 @@ examples/peripherals/adc/continuous_read: disable: - if: SOC_ADC_DMA_SUPPORTED != 1 + <<: *adc_dependencies examples/peripherals/adc/oneshot_read: disable: - if: SOC_ADC_SUPPORTED != 1 + <<: *adc_dependencies examples/peripherals/analog_comparator: disable: @@ -30,6 +39,16 @@ examples/peripherals/dac: disable: - if: SOC_DAC_SUPPORTED != 1 +examples/peripherals/dac/dac_cosine_wave: + disable: + - if: SOC_DAC_SUPPORTED != 1 + depends_components: + - esp_adc + - efuse + - esp_driver_i2s + - esp_driver_spi + - esp_driver_dac + examples/peripherals/gpio: depends_components: - esp_driver_gpio From d44f236f7b3248a0a50c6f445651dee81da89425 Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Thu, 30 Nov 2023 16:08:48 +0100 Subject: [PATCH 09/18] ci(usb): migrate from .gitlab/ci/rules.yml to .build-test-rules.yml --- .gitlab/ci/dependencies/dependencies.yml | 1 - .gitlab/ci/rules.yml | 45 ------------------- .gitlab/ci/target-test.yml | 18 -------- .../usb/test_apps/.build-test-rules.yml | 5 +++ examples/peripherals/.build-test-rules.yml | 15 +++++++ .../usb/device/.build-test-rules.yml | 9 ---- 6 files changed, 20 insertions(+), 73 deletions(-) delete mode 100644 examples/peripherals/usb/device/.build-test-rules.yml diff --git a/.gitlab/ci/dependencies/dependencies.yml b/.gitlab/ci/dependencies/dependencies.yml index d1ecaeec2e..331280081e 100644 --- a/.gitlab/ci/dependencies/dependencies.yml +++ b/.gitlab/ci/dependencies/dependencies.yml @@ -117,7 +117,6 @@ - - wifi # pytest*wifi* - ethernet # pytest*ethernet* - sdio # pytest*sdio* - - usb # USB Device & Host tests patterns: - "{0}-{1}-{2}" - "{0}-{2}" diff --git a/.gitlab/ci/rules.yml b/.gitlab/ci/rules.yml index 9d14675c64..7bc226eb5d 100644 --- a/.gitlab/ci/rules.yml +++ b/.gitlab/ci/rules.yml @@ -248,21 +248,6 @@ - "components/esp_driver_sdmmc/include/driver/sdmmc*.h" - "components/sdmmc/**/*" -# for jobs: USB host and device examples -.patterns-example_test-usb: &patterns-example_test-usb - - "components/hal/usb*.c" - - "components/hal/esp32s*/include/hal/usb*.h" - - "components/usb/*include/**/*" - - "components/usb/*" - - "examples/peripherals/usb/host/**/**/**/*" - - "examples/peripherals/usb/device/**/**/*" - -# for jobs: USB component (Host) pytest test_app -.patterns-component_ut-usb: &patterns-component_ut-usb - - "components/hal/usb*.c" - - "components/hal/esp32s*/include/hal/usb*.h" - - "components/usb/**/*" - ############## # if anchors # ############## @@ -1558,8 +1543,6 @@ changes: *patterns-component_ut - <<: *if-dev-push changes: *patterns-component_ut-sdio - - <<: *if-dev-push - changes: *patterns-component_ut-usb - <<: *if-dev-push changes: *patterns-component_ut-wifi - <<: *if-dev-push @@ -1574,8 +1557,6 @@ changes: *patterns-example_test-ethernet - <<: *if-dev-push changes: *patterns-example_test-sdio - - <<: *if-dev-push - changes: *patterns-example_test-usb - <<: *if-dev-push changes: *patterns-example_test-wifi - <<: *if-dev-push @@ -1767,19 +1748,6 @@ - <<: *if-dev-push changes: *patterns-component_ut -.rules:test:component_ut-esp32s3-usb: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-component_ut - - <<: *if-label-component_ut_esp32s3 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-component_ut-usb - .rules:test:component_ut-esp32s3-wifi: rules: - <<: *if-revert-branch @@ -2194,19 +2162,6 @@ - <<: *if-dev-push changes: *patterns-example_test-sdio -.rules:test:example_test-esp32s2-usb: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-example_test - - <<: *if-label-example_test_esp32s2 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-example_test-usb - .rules:test:example_test-esp32s2-wifi: rules: - <<: *if-revert-branch diff --git a/.gitlab/ci/target-test.yml b/.gitlab/ci/target-test.yml index 7f2062e482..95ec6630e1 100644 --- a/.gitlab/ci/target-test.yml +++ b/.gitlab/ci/target-test.yml @@ -544,15 +544,6 @@ pytest_examples_esp32c3_flash_encryption: artifacts: false tags: [ esp32c3, flash_encryption ] -pytest_examples_esp32s2_usb_device: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32s2-usb - needs: - - job: build_pytest_examples_esp32s2 - artifacts: false - tags: [ esp32s2, usb_device ] - pytest_examples_esp32_sdmmc: extends: - .pytest_examples_dir_template @@ -1156,15 +1147,6 @@ pytest_examples_esp32h2_zigbee: artifacts: false tags: [ esp32h2, zigbee_multi_dut ] -pytest_components_esp32s3_usb_host: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32s3-usb - needs: - - job: build_pytest_components_esp32s3 - artifacts: false - tags: [ esp32s3, usb_host_flash_disk ] - .pytest_test_apps_dir_template: extends: .pytest_template variables: diff --git a/components/usb/test_apps/.build-test-rules.yml b/components/usb/test_apps/.build-test-rules.yml index d11df13540..77a723b155 100644 --- a/components/usb/test_apps/.build-test-rules.yml +++ b/components/usb/test_apps/.build-test-rules.yml @@ -3,3 +3,8 @@ components/usb/test_apps: enable: - if: SOC_USB_OTG_SUPPORTED == 1 + depends_components: + - usb + depends_filepatterns: + - components/hal/usb*.c + - components/hal/esp32*/include/hal/usb*.h diff --git a/examples/peripherals/.build-test-rules.yml b/examples/peripherals/.build-test-rules.yml index 8680827930..057af470e4 100644 --- a/examples/peripherals/.build-test-rules.yml +++ b/examples/peripherals/.build-test-rules.yml @@ -465,6 +465,21 @@ examples/peripherals/usb: disable: - if: SOC_USB_OTG_SUPPORTED != 1 +examples/peripherals/usb/device: + enable: + - if: SOC_USB_OTG_SUPPORTED == 1 + disable_test: + - if: IDF_TARGET == "esp32s3" + temporary: true + reason: lack of runners + depends_components: + - usb + depends_filepatterns: + - components/hal/usb*.c + - components/hal/esp32*/include/hal/usb*.h + - examples/peripherals/usb/host/**/* + - examples/peripherals/usb/device/**/* + examples/peripherals/usb_serial_jtag/usb_serial_jtag_echo: disable: - if: SOC_USB_SERIAL_JTAG_SUPPORTED != 1 diff --git a/examples/peripherals/usb/device/.build-test-rules.yml b/examples/peripherals/usb/device/.build-test-rules.yml deleted file mode 100644 index 2e88c76571..0000000000 --- a/examples/peripherals/usb/device/.build-test-rules.yml +++ /dev/null @@ -1,9 +0,0 @@ -# Documentation: .gitlab/ci/README.md#manifest-file-to-control-the-buildtest-apps - -examples/peripherals/usb/device: - enable: - - if: SOC_USB_OTG_SUPPORTED == 1 - disable_test: - - if: IDF_TARGET == "esp32s3" - temporary: true - reason: lack of runners From 6f6092ed8d941df2ed8de08cbec3b6d060e1e3d7 Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Thu, 30 Nov 2023 16:27:53 +0100 Subject: [PATCH 10/18] ci(sdio): migrate from .gitlab/ci/rules.yml to .build-test-rules.yml --- .gitlab/ci/dependencies/dependencies.yml | 1 - .gitlab/ci/rules.yml | 109 ------------------ .gitlab/ci/target-test.yml | 101 ---------------- .../fatfs/test_apps/.build-test-rules.yml | 2 - 4 files changed, 213 deletions(-) diff --git a/.gitlab/ci/dependencies/dependencies.yml b/.gitlab/ci/dependencies/dependencies.yml index 331280081e..0cf72a4f15 100644 --- a/.gitlab/ci/dependencies/dependencies.yml +++ b/.gitlab/ci/dependencies/dependencies.yml @@ -116,7 +116,6 @@ - *all_targets - - wifi # pytest*wifi* - ethernet # pytest*ethernet* - - sdio # pytest*sdio* patterns: - "{0}-{1}-{2}" - "{0}-{2}" diff --git a/.gitlab/ci/rules.yml b/.gitlab/ci/rules.yml index 7bc226eb5d..e7159bf91c 100644 --- a/.gitlab/ci/rules.yml +++ b/.gitlab/ci/rules.yml @@ -221,33 +221,6 @@ .patterns-custom_test-wifi: &patterns-custom_test-wifi - "tools/test_apps/phy/**/*" -# for jobs: SDIO related example_test -.patterns-example_test-sdio: &patterns-example_test-sdio - - "components/hal/sdio*.c" - - "components/hal/include/hal/sdio*.h" - - "components/esp_driver_sdspi/src/sdspi*.c" - - "components/esp_driver_sdio/src/sdio*.c" - - "components/esp_driver_sdmmc/src/sdmmc*.c" - - "components/esp_driver_sdspi/include/driver/sdspi*.h" - - "components/esp_driver_sdio/include/driver/sdio*.h" - - "components/esp_driver_sdmmc/include/driver/sdmmc*.h" - - "components/sdmmc/??[!s][!t]*/**/*" - - "components/sdmmc/???/**/*" - - "components/sdmmc/*" - - "examples/peripherals/sdio/**/*" - -# for jobs: SDIO related component_test -.patterns-component_ut-sdio: &patterns-component_ut-sdio - - "components/hal/sdio*.c" - - "components/hal/include/hal/sdio*.h" - - "components/esp_driver_sdspi/src/sdspi*.c" - - "components/esp_driver_sdio/src/sdio*.c" - - "components/esp_driver_sdmmc/src/sdmmc*.c" - - "components/esp_driver_sdspi/include/driver/sdspi*.h" - - "components/esp_driver_sdio/include/driver/sdio*.h" - - "components/esp_driver_sdmmc/include/driver/sdmmc*.h" - - "components/sdmmc/**/*" - ############## # if anchors # ############## @@ -1541,8 +1514,6 @@ changes: *patterns-build_template-app - <<: *if-dev-push changes: *patterns-component_ut - - <<: *if-dev-push - changes: *patterns-component_ut-sdio - <<: *if-dev-push changes: *patterns-component_ut-wifi - <<: *if-dev-push @@ -1555,8 +1526,6 @@ changes: *patterns-example_test - <<: *if-dev-push changes: *patterns-example_test-ethernet - - <<: *if-dev-push - changes: *patterns-example_test-sdio - <<: *if-dev-push changes: *patterns-example_test-wifi - <<: *if-dev-push @@ -1588,19 +1557,6 @@ - <<: *if-dev-push changes: *patterns-component_ut -.rules:test:component_ut-esp32-sdio: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-component_ut - - <<: *if-label-component_ut_esp32 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-component_ut-sdio - .rules:test:component_ut-esp32c2: rules: - <<: *if-revert-branch @@ -1642,19 +1598,6 @@ - <<: *if-dev-push changes: *patterns-component_ut -.rules:test:component_ut-esp32c3-sdio: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-component_ut - - <<: *if-label-component_ut_esp32c3 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-component_ut-sdio - .rules:test:component_ut-esp32c3-wifi: rules: - <<: *if-revert-branch @@ -1722,19 +1665,6 @@ - <<: *if-dev-push changes: *patterns-component_ut -.rules:test:component_ut-esp32s2-sdio: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-component_ut - - <<: *if-label-component_ut_esp32s2 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-component_ut-sdio - .rules:test:component_ut-esp32s3: rules: - <<: *if-revert-branch @@ -1964,19 +1894,6 @@ when: never - <<: *if-example_test-ota-include_nightly_run-rule -.rules:test:example_test-esp32-sdio: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-example_test - - <<: *if-label-example_test_esp32 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-example_test-sdio - .rules:test:example_test-esp32-wifi: rules: - <<: *if-revert-branch @@ -2046,19 +1963,6 @@ when: never - <<: *if-example_test-ota-include_nightly_run-rule -.rules:test:example_test-esp32c3-sdio: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-example_test - - <<: *if-label-example_test_esp32c3 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-example_test-sdio - .rules:test:example_test-esp32c3-wifi: rules: - <<: *if-revert-branch @@ -2149,19 +2053,6 @@ - <<: *if-dev-push changes: *patterns-example_test -.rules:test:example_test-esp32s2-sdio: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-example_test - - <<: *if-label-example_test_esp32s2 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-example_test-sdio - .rules:test:example_test-esp32s2-wifi: rules: - <<: *if-revert-branch diff --git a/.gitlab/ci/target-test.yml b/.gitlab/ci/target-test.yml index 95ec6630e1..11d0b8173e 100644 --- a/.gitlab/ci/target-test.yml +++ b/.gitlab/ci/target-test.yml @@ -127,14 +127,6 @@ pytest_examples_esp32_ccs811: artifacts: false tags: [ esp32, ccs811 ] -pytest_examples_esp32_sdio: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32-sdio - needs: - - job: build_pytest_examples_esp32 - artifacts: false - tags: [ esp32, sdio_master_slave ] pytest_examples_esp32s2_generic: extends: @@ -553,33 +545,6 @@ pytest_examples_esp32_sdmmc: artifacts: false tags: [ esp32, sdcard_sdmode ] -pytest_examples_esp32_sdspi: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32-sdio - needs: - - job: build_pytest_examples_esp32 - artifacts: false - tags: [ esp32, sdcard_spimode ] - -pytest_examples_esp32s2_sdspi: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32s2-sdio - needs: - - job: build_pytest_examples_esp32s2 - artifacts: false - tags: [ esp32s2, sdcard_spimode ] - -pytest_examples_esp32c3_sdspi: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32c3-sdio - needs: - - job: build_pytest_examples_esp32c3 - artifacts: false - tags: [ esp32c3, sdcard_spimode ] - pytest_examples_esp32_extflash: extends: - .pytest_examples_dir_template @@ -650,46 +615,6 @@ pytest_components_esp32s3_sdmmc: artifacts: false tags: [ esp32s3, sdcard ] -pytest_components_esp32_sdio: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32-sdio - needs: - - job: build_pytest_components_esp32 - artifacts: false - tags: [ esp32, sdio_master_slave ] - -pytest_components_esp32_esp32c6_sdio: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32-sdio - needs: - - job: build_pytest_components_esp32 - artifacts: false - - job: build_pytest_components_esp32c6 - artifacts: false - tags: [ esp32c6, sdio_multidev_32_c6 ] - script: - - retry_failed git clone $KNOWN_FAILURE_CASES_REPO known_failure_cases - # get runner env config file - - retry_failed git clone $TEST_ENV_CONFIG_REPO - - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs - # using runner tags as markers to filter the test cases - # Runner tags are comma separated, replace the comma with " and " for markers - - job_tags=$(python tools/ci/python_packages/gitlab_api.py get_job_tags $CI_PROJECT_ID --job_id $CI_JOB_ID) - - markers=$(echo $job_tags | sed -e "s/,/ and /g") - # download the artifacts, requires esp32 and esp32c6 chips - - run_cmd python tools/ci/artifacts_handler.py download --job-name "build_pytest_components_esp32" - - run_cmd python tools/ci/artifacts_handler.py download --job-name "build_pytest_components_esp32c6" - - run_cmd pytest $TEST_DIR - -m \"${markers}\" - --junitxml=XUNIT_RESULT.xml - --ignore-result-files known_failure_cases/known_failure_cases.txt - --parallel-count ${CI_NODE_TOTAL:-1} - --parallel-index ${CI_NODE_INDEX:-1} - ${PYTEST_EXTRA_FLAGS} - --app-info-filepattern \"list_job_*.txt\" - pytest_components_esp32_lan8720: extends: - .pytest_components_dir_template @@ -1016,32 +941,6 @@ pytest_components_esp32c3_flash_encryption: artifacts: false tags: [ esp32c3, flash_encryption ] -pytest_components_esp32_sdspi: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32-sdio - needs: - - job: build_pytest_components_esp32 - artifacts: false - tags: [ esp32, sdcard_spimode ] - -pytest_components_esp32s2_sdspi: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32s2-sdio - needs: - - job: build_pytest_components_esp32s2 - artifacts: false - tags: [ esp32s2, sdcard_spimode ] - -pytest_components_esp32c3_sdspi: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32c3-sdio - needs: - - job: build_pytest_components_esp32c3 - artifacts: false - tags: [ esp32c3, sdcard_spimode ] pytest_components_esp32c6_generic: extends: diff --git a/components/fatfs/test_apps/.build-test-rules.yml b/components/fatfs/test_apps/.build-test-rules.yml index 0a1970ea43..552ed42304 100644 --- a/components/fatfs/test_apps/.build-test-rules.yml +++ b/components/fatfs/test_apps/.build-test-rules.yml @@ -28,12 +28,10 @@ components/fatfs/test_apps/sdcard: - if: IDF_TARGET == "esp32p4" temporary: true reason: target esp32p4 is not supported yet # TODO: IDF-7501 - disable_test: - if: IDF_TARGET not in ["esp32", "esp32c3"] temporary: true reason: lack of runners - depends_components: - esp_driver_sdmmc - esp_driver_spi From 83d4e8c6ab3d61d2649531a891a77e3ab0410184 Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Tue, 2 Jan 2024 14:03:29 +0100 Subject: [PATCH 11/18] ci(flash_encryption_wifi_high_traffic): migrate from .gitlab/ci/rules.yml to .build-test-rules.yml --- .gitlab/ci/dependencies/dependencies.yml | 13 --------- .gitlab/ci/rules.yml | 36 ------------------------ .gitlab/ci/target-test.yml | 18 ------------ examples/system/.build-test-rules.yml | 3 ++ 4 files changed, 3 insertions(+), 67 deletions(-) diff --git a/.gitlab/ci/dependencies/dependencies.yml b/.gitlab/ci/dependencies/dependencies.yml index 0cf72a4f15..f7ceff278a 100644 --- a/.gitlab/ci/dependencies/dependencies.yml +++ b/.gitlab/ci/dependencies/dependencies.yml @@ -129,19 +129,6 @@ - "build:{0}" - build:target_test -# For example_test*flash_encryption_wifi_high_traffic jobs -# set `INCLUDE_NIGHTLY_RUN` variable when triggered on development branches -"test:example_test-{0}-include_nightly_run-rule": - matrix: - - - esp32 - - esp32c3 - specific_rules: - - "if-example_test-ota-include_nightly_run-rule" - included_in: - - "build:example_test-{0}" - - "build:example_test" - - build:target_test - "test:host_test": labels: - host_test diff --git a/.gitlab/ci/rules.yml b/.gitlab/ci/rules.yml index e7159bf91c..eb4e53624f 100644 --- a/.gitlab/ci/rules.yml +++ b/.gitlab/ci/rules.yml @@ -181,14 +181,6 @@ - "components/bt/esp_ble_mesh/lib/lib" - ".gitmodules" -# for jobs: flash_encryption_wifi_high_traffic -.patterns-example_test-ota-nightly_run: &patterns-example_test-ota-nightly_run - - "examples/system/ota/**/*" - - "examples/common_components/protocol_examples_common/**/*" - - "components/app_update/include/*" - - "components/app_update/*" - - "components/esp_https_ota/**/*" - # for jobs: example_test*ethernet* .patterns-example_test-ethernet: &patterns-example_test-ethernet - "tools/ci/python_packages/common_test_methods.py" @@ -264,15 +256,6 @@ .if-revert-branch: &if-revert-branch if: '$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME =~ /^revert-/' -##################### -# Specific if rules # -##################### -.if-example_test-ota-include_nightly_run-rule: &if-example_test-ota-include_nightly_run-rule - <<: *if-dev-push - changes: *patterns-example_test-ota-nightly_run - variables: - INCLUDE_NIGHTLY_RUN: "1" - ######### # Rules # ######### @@ -1471,7 +1454,6 @@ - <<: *if-revert-branch when: never - <<: *if-protected - - <<: *if-example_test-ota-include_nightly_run-rule - <<: *if-label-build - <<: *if-label-component_ut - <<: *if-label-component_ut_esp32 @@ -1885,15 +1867,6 @@ - <<: *if-dev-push changes: *patterns-example_test-ethernet -.rules:test:example_test-esp32-include_nightly_run-rule: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-example_test-ota-include_nightly_run-rule - .rules:test:example_test-esp32-wifi: rules: - <<: *if-revert-branch @@ -1954,15 +1927,6 @@ - <<: *if-dev-push changes: *patterns-example_test -.rules:test:example_test-esp32c3-include_nightly_run-rule: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-example_test-ota-include_nightly_run-rule - .rules:test:example_test-esp32c3-wifi: rules: - <<: *if-revert-branch diff --git a/.gitlab/ci/target-test.yml b/.gitlab/ci/target-test.yml index 11d0b8173e..91ffca345a 100644 --- a/.gitlab/ci/target-test.yml +++ b/.gitlab/ci/target-test.yml @@ -264,24 +264,6 @@ pytest_examples_esp32_wifi_high_traffic: artifacts: false tags: [ esp32, wifi_high_traffic ] -pytest_examples_esp32_flash_encryption_wifi_high_traffic: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32-include_nightly_run-rule - needs: - - job: build_pytest_examples_esp32 - artifacts: false - tags: [ esp32, flash_encryption_wifi_high_traffic ] - -pytest_examples_esp32c3_flash_encryption_wifi_high_traffic: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32c3-include_nightly_run-rule - needs: - - job: build_pytest_examples_esp32c3 - artifacts: false - tags: [ esp32c3, flash_encryption_wifi_high_traffic ] - pytest_examples_esp32_ethernet: extends: - .pytest_examples_dir_template diff --git a/examples/system/.build-test-rules.yml b/examples/system/.build-test-rules.yml index 9bc604cda2..c622e50dac 100644 --- a/examples/system/.build-test-rules.yml +++ b/examples/system/.build-test-rules.yml @@ -188,6 +188,9 @@ examples/system/ota/simple_ota_example: - if: IDF_TARGET == "esp32c2" or IDF_TARGET == "esp32c6" temporary: true reason: lack of runners + depends_components: + - app_update + - esp_https_ota examples/system/perfmon: enable: From bb375ee79a708308e90c3293ab92697145592622 Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Tue, 2 Jan 2024 14:06:00 +0100 Subject: [PATCH 12/18] ci(ethernet): migrate from .gitlab/ci/rules.yml to .build-test-rules.yml --- .gitlab/ci/dependencies/dependencies.yml | 1 - .gitlab/ci/rules.yml | 28 --------------- .gitlab/ci/target-test.yml | 45 ------------------------ examples/network/.build-test-rules.yml | 11 ++++++ examples/protocols/.build-test-rules.yml | 16 +++++++++ examples/system/.build-test-rules.yml | 9 +++++ tools/test_apps/.build-test-rules.yml | 10 ++++++ 7 files changed, 46 insertions(+), 74 deletions(-) diff --git a/.gitlab/ci/dependencies/dependencies.yml b/.gitlab/ci/dependencies/dependencies.yml index f7ceff278a..8b2ce40e30 100644 --- a/.gitlab/ci/dependencies/dependencies.yml +++ b/.gitlab/ci/dependencies/dependencies.yml @@ -115,7 +115,6 @@ - *target_test - *all_targets - - wifi # pytest*wifi* - - ethernet # pytest*ethernet* patterns: - "{0}-{1}-{2}" - "{0}-{2}" diff --git a/.gitlab/ci/rules.yml b/.gitlab/ci/rules.yml index eb4e53624f..79c5a5554a 100644 --- a/.gitlab/ci/rules.yml +++ b/.gitlab/ci/rules.yml @@ -181,19 +181,6 @@ - "components/bt/esp_ble_mesh/lib/lib" - ".gitmodules" -# for jobs: example_test*ethernet* -.patterns-example_test-ethernet: &patterns-example_test-ethernet - - "tools/ci/python_packages/common_test_methods.py" - - "examples/common_components/**/*" - - "examples/protocols/**/*" - - "examples/system/ota/**/*" - - "examples/ethernet/iperf/**/*" - - "examples/network/vlan_support/**/*" - - "components/esp_eth/??[!s][!t]*/**/*" - - "components/esp_eth/???/**/*" - - "components/esp_eth/*" - - "components/esp_netif/esp_netif_handlers.c" - # for jobs: example_test*wifi* .patterns-example_test-wifi: &patterns-example_test-wifi - "tools/ci/python_packages/common_test_methods.py" @@ -1506,8 +1493,6 @@ changes: *patterns-downloadable-tools - <<: *if-dev-push changes: *patterns-example_test - - <<: *if-dev-push - changes: *patterns-example_test-ethernet - <<: *if-dev-push changes: *patterns-example_test-wifi - <<: *if-dev-push @@ -1854,19 +1839,6 @@ - <<: *if-dev-push changes: *patterns-example_test -.rules:test:example_test-esp32-ethernet: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-example_test - - <<: *if-label-example_test_esp32 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-example_test-ethernet - .rules:test:example_test-esp32-wifi: rules: - <<: *if-revert-branch diff --git a/.gitlab/ci/target-test.yml b/.gitlab/ci/target-test.yml index 91ffca345a..91a639b910 100644 --- a/.gitlab/ci/target-test.yml +++ b/.gitlab/ci/target-test.yml @@ -246,15 +246,6 @@ pytest_examples_esp32p4_generic: artifacts: false tags: [ esp32p4, generic ] -pytest_examples_esp32_ethernet_ota: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32-ethernet - needs: - - job: build_pytest_examples_esp32 - artifacts: false - tags: [ esp32, ethernet_ota ] - pytest_examples_esp32_wifi_high_traffic: extends: - .pytest_examples_dir_template @@ -264,24 +255,6 @@ pytest_examples_esp32_wifi_high_traffic: artifacts: false tags: [ esp32, wifi_high_traffic ] -pytest_examples_esp32_ethernet: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32-ethernet - needs: - - job: build_pytest_examples_esp32 - artifacts: false - tags: [ esp32, ethernet] - -pytest_examples_esp32_ethernet_httpbin: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32-ethernet - needs: - - job: build_pytest_examples_esp32 - artifacts: false - tags: [ esp32, httpbin] - pytest_examples_esp32_8mb_flash: extends: - .pytest_examples_dir_template @@ -435,24 +408,6 @@ pytest_examples_esp32_wifi_wlan: artifacts: false tags: [ esp32, wifi_wlan ] -pytest_examples_esp32_ethernet_router: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32-ethernet - needs: - - job: build_pytest_examples_esp32 - artifacts: false - tags: [ esp32, ethernet_router ] - -pytest_examples_esp32_ethernet_vlan: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32-ethernet - needs: - - job: build_pytest_examples_esp32 - artifacts: false - tags: [ esp32, ethernet_vlan ] - pytest_examples_esp32_ethernet_bridge: extends: - .pytest_examples_dir_template diff --git a/examples/network/.build-test-rules.yml b/examples/network/.build-test-rules.yml index 93f5b739c1..8a717aff7d 100644 --- a/examples/network/.build-test-rules.yml +++ b/examples/network/.build-test-rules.yml @@ -22,7 +22,18 @@ examples/network/simple_sniffer: examples/network/sta2eth: disable: - if: SOC_WIFI_SUPPORTED != 1 + examples/network/vlan_support: disable_test: - if: IDF_TARGET not in ["esp32"] reason: Runner uses esp32 ethernet kit + depends_components: + - esp_eth + depends_filepatterns: + - tools/ci/python_packages/common_test_methods.py + - examples/common_components/**/* + - examples/protocols/**/* + - examples/system/ota/**/* + - examples/ethernet/iperf/**/* + - examples/network/vlan_support/**/* + - components/esp_netif/esp_netif_handlers.c diff --git a/examples/protocols/.build-test-rules.yml b/examples/protocols/.build-test-rules.yml index 2b0b52480c..535f8796f7 100644 --- a/examples/protocols/.build-test-rules.yml +++ b/examples/protocols/.build-test-rules.yml @@ -1,5 +1,11 @@ # Documentation: .gitlab/ci/README.md#manifest-file-to-control-the-buildtest-apps +.ethernet_dependencies: ðernet_dependencies + # TODO: IDFCI-1821 + depends_filepatterns: + - tools/ci/python_packages/common_test_methods.py + - components/esp_netif/esp_netif_handlers.c + examples/protocols/esp_http_client: enable: - if: INCLUDE_DEFAULT == 1 or IDF_TARGET == "linux" @@ -11,6 +17,7 @@ examples/protocols/esp_http_client: - if: IDF_TARGET == "esp32p4" temporary: true reason: not supported on p4 + <<: *ethernet_dependencies examples/protocols/esp_local_ctrl: disable: @@ -32,6 +39,7 @@ examples/protocols/http_request: - if: IDF_TARGET != "esp32" temporary: true reason: only test on esp32 + <<: *ethernet_dependencies examples/protocols/http_server: disable: @@ -78,6 +86,7 @@ examples/protocols/https_mbedtls: - if: IDF_TARGET != "esp32" temporary: true reason: lack of runners + <<: *ethernet_dependencies examples/protocols/https_request: disable: @@ -88,6 +97,7 @@ examples/protocols/https_request: - if: IDF_TARGET != "esp32" temporary: true reason: lack of runners + <<: *ethernet_dependencies examples/protocols/https_server/simple: disable: @@ -118,6 +128,7 @@ examples/protocols/https_x509_bundle: - if: IDF_TARGET != "esp32" temporary: true reason: lack of runners + <<: *ethernet_dependencies examples/protocols/icmp_echo: disable: @@ -158,6 +169,7 @@ examples/protocols/mqtt/ssl: - if: IDF_TARGET != "esp32" temporary: true reason: lack of runners + <<: *ethernet_dependencies examples/protocols/mqtt/ssl_ds: disable: @@ -186,6 +198,7 @@ examples/protocols/mqtt/tcp: - if: IDF_TARGET != "esp32" temporary: true reason: lack of runners + <<: *ethernet_dependencies examples/protocols/mqtt/ws: disable: @@ -196,6 +209,7 @@ examples/protocols/mqtt/ws: - if: IDF_TARGET != "esp32" temporary: true reason: lack of runners + <<: *ethernet_dependencies examples/protocols/mqtt/wss: disable: @@ -206,6 +220,7 @@ examples/protocols/mqtt/wss: - if: IDF_TARGET != "esp32" temporary: true reason: lack of runners + <<: *ethernet_dependencies examples/protocols/mqtt5: disable: @@ -216,6 +231,7 @@ examples/protocols/mqtt5: - if: IDF_TARGET != "esp32" temporary: true reason: lack of runners + <<: *ethernet_dependencies examples/protocols/smtp_client: disable: diff --git a/examples/system/.build-test-rules.yml b/examples/system/.build-test-rules.yml index c622e50dac..d011ce4725 100644 --- a/examples/system/.build-test-rules.yml +++ b/examples/system/.build-test-rules.yml @@ -1,5 +1,11 @@ # Documentation: .gitlab/ci/README.md#manifest-file-to-control-the-buildtest-apps +.ethernet_dependencies: ðernet_dependencies + # TODO: IDFCI-1821 + depends_filepatterns: + - tools/ci/python_packages/common_test_methods.py + - components/esp_netif/esp_netif_handlers.c + examples/system/app_trace_basic: disable: - if: IDF_TARGET in ["esp32c6", "esp32h2", "esp32p4"] @@ -151,6 +157,7 @@ examples/system/ota/advanced_https_ota: - if: IDF_TARGET == "esp32c2" or IDF_TARGET == "esp32c6" temporary: true reason: lack of runners + <<: *ethernet_dependencies examples/system/ota/native_ota_example: disable: @@ -161,6 +168,7 @@ examples/system/ota/native_ota_example: - if: IDF_TARGET == "esp32c6" temporary: true reason: lack of runners + <<: *ethernet_dependencies examples/system/ota/otatool: disable: @@ -177,6 +185,7 @@ examples/system/ota/pre_encrypted_ota: - if: IDF_TARGET == "esp32c2" or IDF_TARGET == "esp32c6" temporary: true reason: lack of runners + <<: *ethernet_dependencies examples/system/ota/simple_ota_example: disable: diff --git a/tools/test_apps/.build-test-rules.yml b/tools/test_apps/.build-test-rules.yml index 47177fff01..22f992ed66 100644 --- a/tools/test_apps/.build-test-rules.yml +++ b/tools/test_apps/.build-test-rules.yml @@ -52,6 +52,16 @@ tools/test_apps/protocols/mqtt/publish_connect_test: - if: IDF_TARGET == "esp32s2" or IDF_TARGET == "esp32c3" temporary: true reason: lack of runners + depends_components: + - esp_eth + depends_filepatterns: + - tools/ci/python_packages/common_test_methods.py + - examples/common_components/**/* + - examples/protocols/**/* + - examples/system/ota/**/* + - examples/ethernet/iperf/**/* + - examples/network/vlan_support/**/* + - components/esp_netif/esp_netif_handlers.c tools/test_apps/protocols/netif_components: enable: From 114af706bbfdc5433593ad17b770cfa6905c538f Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Thu, 30 Nov 2023 17:01:23 +0100 Subject: [PATCH 13/18] ci(wifi): migrate from .gitlab/ci/rules.yml to .build-test-rules.yml --- .gitlab/ci/dependencies/dependencies.yml | 25 -- .gitlab/ci/rules.yml | 288 ----------------------- .gitlab/ci/target-test.yml | 252 -------------------- examples/network/.build-test-rules.yml | 11 + examples/protocols/.build-test-rules.yml | 26 ++ examples/system/.build-test-rules.yml | 16 +- 6 files changed, 52 insertions(+), 566 deletions(-) diff --git a/.gitlab/ci/dependencies/dependencies.yml b/.gitlab/ci/dependencies/dependencies.yml index 8b2ce40e30..95a8342292 100644 --- a/.gitlab/ci/dependencies/dependencies.yml +++ b/.gitlab/ci/dependencies/dependencies.yml @@ -103,31 +103,6 @@ - "build:{0}-{1}" - build:target_test -# ------------- -# Special Cases -# ------------- - -# To reduce the specific runners' usage. -# Do not create these jobs by default patterns on development branches -# Can be triggered by labels or related changes -"test:{0}-{1}-{2}": - matrix: - - *target_test - - *all_targets - - - wifi # pytest*wifi* - patterns: - - "{0}-{1}-{2}" - - "{0}-{2}" - - "target_test-{2}" - labels: - - "{0}_{1}" - - "{0}" - - target_test - included_in: - - "build:{0}-{1}" - - "build:{0}" - - build:target_test - "test:host_test": labels: - host_test diff --git a/.gitlab/ci/rules.yml b/.gitlab/ci/rules.yml index 79c5a5554a..6afb0f43b4 100644 --- a/.gitlab/ci/rules.yml +++ b/.gitlab/ci/rules.yml @@ -83,16 +83,6 @@ - "components/**/*" -.patterns-target_test-wifi: &patterns-target_test-wifi - - "components/{esp_wifi,esp_netif,lwip,esp_phy,wpa_supplicant,esp_coex}/???[!t]*/**/*" - - "components/{esp_wifi,esp_netif,lwip,esp_phy,wpa_supplicant,esp_coex}/??[!s]?*/**/*" - - "components/{esp_wifi,esp_netif,lwip,esp_phy,wpa_supplicant,esp_coex}/???/**/*" - - "components/{esp_wifi,esp_netif,lwip,esp_phy,wpa_supplicant,esp_coex}/??/**/*" - - "components/{esp_wifi,esp_netif,lwip,esp_phy,wpa_supplicant,esp_coex}/*" - -.patterns-component_ut-wifi: &patterns-component_ut-wifi - - "components/esp_wifi/**/*" - .patterns-build_macos: &patterns-build_macos - "tools/ci/test_configure_ci_environment.sh" @@ -181,25 +171,6 @@ - "components/bt/esp_ble_mesh/lib/lib" - ".gitmodules" -# for jobs: example_test*wifi* -.patterns-example_test-wifi: &patterns-example_test-wifi - - "tools/ci/python_packages/common_test_methods.py" - - "examples/common_components/protocol_examples_common/**/*" - - "examples/protocols/**/*" - - "examples/wifi/**/*" - - "examples/network/simple_sniffer/**/*" - - - "components/mbedtls/port/dynamic/*" - # ota - - "examples/system/ota/**/*" - - "components/app_update/include/*" - - "components/app_update/*" - - "components/esp_https_ota/**/*" - -# for jobs: custom_test*wifi* -.patterns-custom_test-wifi: &patterns-custom_test-wifi - - "tools/test_apps/phy/**/*" - ############## # if anchors # ############## @@ -489,24 +460,8 @@ changes: *patterns-build_system - <<: *if-dev-push changes: *patterns-component_ut - - <<: *if-dev-push - changes: *patterns-component_ut-adc - - <<: *if-dev-push - changes: *patterns-component_ut-flash_multi - - <<: *if-dev-push - changes: *patterns-component_ut-i154 - - <<: *if-dev-push - changes: *patterns-component_ut-nvs_encr_hmac - - <<: *if-dev-push - changes: *patterns-component_ut-sdio - - <<: *if-dev-push - changes: *patterns-component_ut-usb - - <<: *if-dev-push - changes: *patterns-component_ut-wifi - <<: *if-dev-push changes: *patterns-downloadable-tools - - <<: *if-dev-push - changes: *patterns-target_test-wifi .rules:build:component_ut-esp32: rules: @@ -523,24 +478,8 @@ changes: *patterns-build_system - <<: *if-dev-push changes: *patterns-component_ut - - <<: *if-dev-push - changes: *patterns-component_ut-adc - - <<: *if-dev-push - changes: *patterns-component_ut-flash_multi - - <<: *if-dev-push - changes: *patterns-component_ut-i154 - - <<: *if-dev-push - changes: *patterns-component_ut-nvs_encr_hmac - - <<: *if-dev-push - changes: *patterns-component_ut-sdio - - <<: *if-dev-push - changes: *patterns-component_ut-usb - - <<: *if-dev-push - changes: *patterns-component_ut-wifi - <<: *if-dev-push changes: *patterns-downloadable-tools - - <<: *if-dev-push - changes: *patterns-target_test-wifi .rules:build:component_ut-esp32c2: rules: @@ -823,12 +762,8 @@ changes: *patterns-build_system - <<: *if-dev-push changes: *patterns-custom_test - - <<: *if-dev-push - changes: *patterns-custom_test-wifi - <<: *if-dev-push changes: *patterns-downloadable-tools - - <<: *if-dev-push - changes: *patterns-target_test-wifi .rules:build:custom_test-esp32: rules: @@ -845,12 +780,8 @@ changes: *patterns-build_system - <<: *if-dev-push changes: *patterns-custom_test - - <<: *if-dev-push - changes: *patterns-custom_test-wifi - <<: *if-dev-push changes: *patterns-downloadable-tools - - <<: *if-dev-push - changes: *patterns-target_test-wifi .rules:build:custom_test-esp32c2: rules: @@ -867,16 +798,8 @@ changes: *patterns-build_system - <<: *if-dev-push changes: *patterns-custom_test - - <<: *if-dev-push - changes: *patterns-custom_test-wifi - <<: *if-dev-push changes: *patterns-downloadable-tools - - <<: *if-dev-push - changes: *patterns-target_test-adc - - <<: *if-dev-push - changes: *patterns-target_test-ecdsa - - <<: *if-dev-push - changes: *patterns-target_test-wifi .rules:build:custom_test-esp32c3: rules: @@ -893,16 +816,8 @@ changes: *patterns-build_system - <<: *if-dev-push changes: *patterns-custom_test - - <<: *if-dev-push - changes: *patterns-custom_test-wifi - <<: *if-dev-push changes: *patterns-downloadable-tools - - <<: *if-dev-push - changes: *patterns-target_test-adc - - <<: *if-dev-push - changes: *patterns-target_test-ecdsa - - <<: *if-dev-push - changes: *patterns-target_test-wifi .rules:build:custom_test-esp32c6: rules: @@ -1483,20 +1398,12 @@ changes: *patterns-build_template-app - <<: *if-dev-push changes: *patterns-component_ut - - <<: *if-dev-push - changes: *patterns-component_ut-wifi - <<: *if-dev-push changes: *patterns-custom_test - - <<: *if-dev-push - changes: *patterns-custom_test-wifi - <<: *if-dev-push changes: *patterns-downloadable-tools - <<: *if-dev-push changes: *patterns-example_test - - <<: *if-dev-push - changes: *patterns-example_test-wifi - - <<: *if-dev-push - changes: *patterns-target_test-wifi .rules:labels:nvs_coverage: rules: @@ -1537,21 +1444,6 @@ - <<: *if-dev-push changes: *patterns-component_ut -.rules:test:component_ut-esp32c2-wifi: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-component_ut - - <<: *if-label-component_ut_esp32c2 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-component_ut-wifi - - <<: *if-dev-push - changes: *patterns-target_test-wifi - .rules:test:component_ut-esp32c3: rules: - <<: *if-revert-branch @@ -1565,21 +1457,6 @@ - <<: *if-dev-push changes: *patterns-component_ut -.rules:test:component_ut-esp32c3-wifi: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-component_ut - - <<: *if-label-component_ut_esp32c3 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-component_ut-wifi - - <<: *if-dev-push - changes: *patterns-target_test-wifi - .rules:test:component_ut-esp32c6: rules: - <<: *if-revert-branch @@ -1645,21 +1522,6 @@ - <<: *if-dev-push changes: *patterns-component_ut -.rules:test:component_ut-esp32s3-wifi: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-component_ut - - <<: *if-label-component_ut_esp32s3 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-component_ut-wifi - - <<: *if-dev-push - changes: *patterns-target_test-wifi - .rules:test:custom_test-esp32: rules: - <<: *if-revert-branch @@ -1686,21 +1548,6 @@ - <<: *if-dev-push changes: *patterns-custom_test -.rules:test:custom_test-esp32c2-wifi: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-custom_test - - <<: *if-label-custom_test_esp32c2 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-custom_test-wifi - - <<: *if-dev-push - changes: *patterns-target_test-wifi - .rules:test:custom_test-esp32c3: rules: - <<: *if-revert-branch @@ -1714,21 +1561,6 @@ - <<: *if-dev-push changes: *patterns-custom_test -.rules:test:custom_test-esp32c3-wifi: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-custom_test - - <<: *if-label-custom_test_esp32c3 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-custom_test-wifi - - <<: *if-dev-push - changes: *patterns-target_test-wifi - .rules:test:custom_test-esp32c6: rules: - <<: *if-revert-branch @@ -1781,21 +1613,6 @@ - <<: *if-dev-push changes: *patterns-custom_test -.rules:test:custom_test-esp32s2-wifi: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-custom_test - - <<: *if-label-custom_test_esp32s2 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-custom_test-wifi - - <<: *if-dev-push - changes: *patterns-target_test-wifi - .rules:test:custom_test-esp32s3: rules: - <<: *if-revert-branch @@ -1809,21 +1626,6 @@ - <<: *if-dev-push changes: *patterns-custom_test -.rules:test:custom_test-esp32s3-wifi: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-custom_test - - <<: *if-label-custom_test_esp32s3 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-custom_test-wifi - - <<: *if-dev-push - changes: *patterns-target_test-wifi - .rules:test:example_test-esp32: rules: - <<: *if-revert-branch @@ -1839,21 +1641,6 @@ - <<: *if-dev-push changes: *patterns-example_test -.rules:test:example_test-esp32-wifi: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-example_test - - <<: *if-label-example_test_esp32 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-example_test-wifi - - <<: *if-dev-push - changes: *patterns-target_test-wifi - .rules:test:example_test-esp32c2: rules: - <<: *if-revert-branch @@ -1869,21 +1656,6 @@ - <<: *if-dev-push changes: *patterns-example_test -.rules:test:example_test-esp32c2-wifi: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-example_test - - <<: *if-label-example_test_esp32c2 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-example_test-wifi - - <<: *if-dev-push - changes: *patterns-target_test-wifi - .rules:test:example_test-esp32c3: rules: - <<: *if-revert-branch @@ -1899,21 +1671,6 @@ - <<: *if-dev-push changes: *patterns-example_test -.rules:test:example_test-esp32c3-wifi: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-example_test - - <<: *if-label-example_test_esp32c3 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-example_test-wifi - - <<: *if-dev-push - changes: *patterns-target_test-wifi - .rules:test:example_test-esp32c6: rules: - <<: *if-revert-branch @@ -1929,21 +1686,6 @@ - <<: *if-dev-push changes: *patterns-example_test -.rules:test:example_test-esp32c6-wifi: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-example_test - - <<: *if-label-example_test_esp32c6 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-example_test-wifi - - <<: *if-dev-push - changes: *patterns-target_test-wifi - .rules:test:example_test-esp32h2: rules: - <<: *if-revert-branch @@ -1989,21 +1731,6 @@ - <<: *if-dev-push changes: *patterns-example_test -.rules:test:example_test-esp32s2-wifi: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-example_test - - <<: *if-label-example_test_esp32s2 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-example_test-wifi - - <<: *if-dev-push - changes: *patterns-target_test-wifi - .rules:test:example_test-esp32s3: rules: - <<: *if-revert-branch @@ -2019,21 +1746,6 @@ - <<: *if-dev-push changes: *patterns-example_test -.rules:test:example_test-esp32s3-wifi: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-example_test - - <<: *if-label-example_test_esp32s3 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-example_test-wifi - - <<: *if-dev-push - changes: *patterns-target_test-wifi - .rules:test:host_test: rules: - <<: *if-revert-branch diff --git a/.gitlab/ci/target-test.yml b/.gitlab/ci/target-test.yml index 91a639b910..b945baffb5 100644 --- a/.gitlab/ci/target-test.yml +++ b/.gitlab/ci/target-test.yml @@ -246,15 +246,6 @@ pytest_examples_esp32p4_generic: artifacts: false tags: [ esp32p4, generic ] -pytest_examples_esp32_wifi_high_traffic: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32-wifi - needs: - - job: build_pytest_examples_esp32 - artifacts: false - tags: [ esp32, wifi_high_traffic ] - pytest_examples_esp32_8mb_flash: extends: - .pytest_examples_dir_template @@ -264,150 +255,6 @@ pytest_examples_esp32_8mb_flash: artifacts: false tags: [ esp32, ethernet_flash_8m ] -pytest_examples_esp32_wifi_ap: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32-wifi - needs: - - job: build_pytest_examples_esp32 - artifacts: false - tags: [ esp32, wifi_ap ] - -pytest_examples_esp32c3_wifi_ap: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32c3-wifi - needs: - - job: build_pytest_examples_esp32c3 - artifacts: false - tags: [ esp32c3, wifi_ap ] - -pytest_examples_esp32s3_wifi_ap: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32s3-wifi - needs: - - job: build_pytest_examples_esp32s3 - artifacts: false - tags: [ esp32s3, wifi_ap ] - -pytest_examples_esp32s2_wifi_ap: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32s2-wifi - needs: - - job: build_pytest_examples_esp32s2 - artifacts: false - tags: [ esp32s2, wifi_ap ] - -pytest_examples_esp32c2_wifi_ap: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32c2-wifi - needs: - - job: build_pytest_examples_esp32c2 - artifacts: false - tags: [ esp32c2, wifi_ap, xtal_40mhz ] - -pytest_examples_esp32c2_26m_wifi_ap: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32c2-wifi - needs: - - job: build_pytest_examples_esp32c2 - artifacts: false - tags: [ esp32c2, wifi_ap, xtal_26mhz ] - -pytest_examples_esp32c6_wifi_ap: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32c6-wifi - needs: - - job: build_pytest_examples_esp32c6 - artifacts: false - tags: [ esp32c6, wifi_ap ] - -pytest_examples_esp32_wifi_router: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32-wifi - needs: - - job: build_pytest_examples_esp32 - artifacts: false - tags: [ esp32, wifi_router ] - -pytest_examples_esp32c3_wifi_router: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32c3-wifi - needs: - - job: build_pytest_examples_esp32c3 - artifacts: false - tags: [ esp32c3, wifi_router ] - -pytest_examples_esp32s3_wifi_router: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32s3-wifi - needs: - - job: build_pytest_examples_esp32s3 - artifacts: false - tags: [ esp32s3, wifi_router ] - -pytest_examples_esp32s2_wifi_router: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32s2-wifi - needs: - - job: build_pytest_examples_esp32s2 - artifacts: false - tags: [ esp32s2, wifi_router ] - -pytest_examples_esp32c2_wifi_router: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32c2-wifi - needs: - - job: build_pytest_examples_esp32c2 - artifacts: false - tags: [ esp32c2, wifi_router, xtal_40mhz ] - -pytest_examples_esp32c2_26m_wifi_router: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32c2-wifi - needs: - - job: build_pytest_examples_esp32c2 - artifacts: false - tags: [ esp32c2, wifi_router, xtal_26mhz ] - -pytest_examples_esp32c6_wifi_router: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32c6-wifi - needs: - - job: build_pytest_examples_esp32c6 - artifacts: false - tags: [ esp32c6, wifi_router ] - -pytest_examples_esp32_wifi_iperf: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32-wifi - needs: - - job: build_pytest_examples_esp32 - artifacts: false - tags: [ esp32, Example_ShieldBox_Basic ] - -pytest_examples_esp32_wifi_wlan: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32-wifi - needs: - - job: build_pytest_examples_esp32 - artifacts: false - tags: [ esp32, wifi_wlan ] - pytest_examples_esp32_ethernet_bridge: extends: - .pytest_examples_dir_template @@ -428,42 +275,6 @@ pytest_examples_esp32_flash_encryption: artifacts: false tags: [ esp32, flash_encryption ] -pytest_examples_esp32_wifi_two_dut: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32-wifi - needs: - - job: build_pytest_examples_esp32 - artifacts: false - tags: [ esp32, wifi_two_dut ] - -pytest_examples_esp32c3_wifi_two_dut: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32c3-wifi - needs: - - job: build_pytest_examples_esp32c3 - artifacts: false - tags: [ esp32c3, wifi_two_dut ] - -pytest_examples_esp32s3_wifi_two_dut: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32s3-wifi - needs: - - job: build_pytest_examples_esp32s3 - artifacts: false - tags: [ esp32s3, wifi_two_dut ] - -pytest_examples_esp32c2_wifi_two_dut: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32c2-wifi - needs: - - job: build_pytest_examples_esp32c2 - artifacts: false - tags: [ esp32c2, wifi_two_dut, xtal_26mhz ] - pytest_examples_esp32c3_flash_encryption: extends: - .pytest_examples_dir_template @@ -834,15 +645,6 @@ pytest_components_esp32c3_generic_multi_device: artifacts: false tags: [ esp32c3, generic_multi_device ] -pytest_components_esp32c3_wifi_two_dut: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32c3-wifi - needs: - - job: build_pytest_components_esp32c3 - artifacts: false - tags: [ esp32c3, wifi_two_dut ] - pytest_components_esp32c3_usb_serial_jtag: extends: - .pytest_components_dir_template @@ -851,24 +653,6 @@ pytest_components_esp32c3_usb_serial_jtag: - build_pytest_components_esp32c3 tags: [ esp32c3, usb_serial_jtag ] -pytest_components_esp32s3_wifi_two_dut: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32s3-wifi - needs: - - job: build_pytest_components_esp32s3 - artifacts: false - tags: [ esp32s3, wifi_two_dut ] - -pytest_components_esp32c2_wifi_two_dut: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32c2-wifi - needs: - - job: build_pytest_components_esp32c2 - artifacts: false - tags: [ esp32c2, wifi_two_dut, xtal_26mhz ] - pytest_components_esp32c3_flash_encryption: extends: - .pytest_components_dir_template @@ -1108,39 +892,3 @@ pytest_test_apps_esp32s3_mspi_f4r4: - job: build_pytest_test_apps_esp32s3 artifacts: false tags: [ esp32s3, MSPI_F4R4 ] - -pytest_test_apps_esp32s2_wifi_two_dut: - extends: - - .pytest_test_apps_dir_template - - .rules:test:custom_test-esp32s2-wifi - needs: - - job: build_pytest_test_apps_esp32s2 - artifacts: false - tags: [ esp32s2, wifi_two_dut ] - -pytest_test_apps_esp32s3_wifi_two_dut: - extends: - - .pytest_test_apps_dir_template - - .rules:test:custom_test-esp32s3-wifi - needs: - - job: build_pytest_test_apps_esp32s3 - artifacts: false - tags: [ esp32s3, wifi_two_dut ] - -pytest_test_apps_esp32c2_wifi_two_dut: - extends: - - .pytest_test_apps_dir_template - - .rules:test:custom_test-esp32c2-wifi - needs: - - job: build_pytest_test_apps_esp32c2 - artifacts: false - tags: [ esp32c2, wifi_two_dut, xtal_26mhz ] - -pytest_test_apps_esp32c3_wifi_two_dut: - extends: - - .pytest_test_apps_dir_template - - .rules:test:custom_test-esp32c3-wifi - needs: - - job: build_pytest_test_apps_esp32c3 - artifacts: false - tags: [ esp32c3, wifi_two_dut] diff --git a/examples/network/.build-test-rules.yml b/examples/network/.build-test-rules.yml index 8a717aff7d..1285265a53 100644 --- a/examples/network/.build-test-rules.yml +++ b/examples/network/.build-test-rules.yml @@ -18,6 +18,17 @@ examples/network/simple_sniffer: - if: IDF_TARGET not in ["esp32", "esp32c3", "esp32s3"] temporary: true reason: lack of runners + depends_filepatterns: + - tools/ci/python_packages/common_test_methods.py + - examples/common_components/protocol_examples_common/**/* + - examples/protocols/**/* + - examples/wifi/**/* + - examples/network/simple_sniffer/**/* + - components/mbedtls/port/dynamic/* + - examples/system/ota/**/* + depends_components: + - app_update + - esp_https_ota examples/network/sta2eth: disable: diff --git a/examples/protocols/.build-test-rules.yml b/examples/protocols/.build-test-rules.yml index 535f8796f7..e22a8681fc 100644 --- a/examples/protocols/.build-test-rules.yml +++ b/examples/protocols/.build-test-rules.yml @@ -6,6 +6,19 @@ - tools/ci/python_packages/common_test_methods.py - components/esp_netif/esp_netif_handlers.c +.wifi_dependencies: &wifi_dependencies + depends_filepatterns: + - tools/ci/python_packages/common_test_methods.py + - examples/common_components/protocol_examples_common/**/* + - examples/protocols/**/* + - examples/wifi/**/* + - examples/network/simple_sniffer/**/* + - components/mbedtls/port/dynamic/* + - examples/system/ota/**/* + depends_components: + - app_update + - esp_https_ota + examples/protocols/esp_http_client: enable: - if: INCLUDE_DEFAULT == 1 or IDF_TARGET == "linux" @@ -29,6 +42,7 @@ examples/protocols/esp_local_ctrl: - if: IDF_TARGET not in ["esp32", "esp32c3", "esp32s3"] temporary: true reason: lack of runners + <<: *wifi_dependencies examples/protocols/http_request: disable: @@ -50,6 +64,7 @@ examples/protocols/http_server: - if: IDF_TARGET not in ["esp32", "esp32c3", "esp32s3"] temporary: true reason: lack of runners + <<: *wifi_dependencies examples/protocols/http_server/captive_portal: disable: @@ -60,12 +75,14 @@ examples/protocols/http_server/captive_portal: - if: IDF_TARGET != "esp32" temporary: true reason: only test on esp32 + <<: *wifi_dependencies examples/protocols/http_server/restful_server: disable: - if: IDF_TARGET in ["esp32h2", "esp32p4"] temporary: true reason: not supported on p4 # TODO: IDF-8076 + <<: *wifi_dependencies examples/protocols/http_server/ws_echo_server: disable: @@ -76,6 +93,7 @@ examples/protocols/http_server/ws_echo_server: - if: IDF_TARGET != "esp32" temporary: true reason: only test on esp32 + <<: *wifi_dependencies examples/protocols/https_mbedtls: disable: @@ -108,6 +126,7 @@ examples/protocols/https_server/simple: - if: IDF_TARGET not in ["esp32", "esp32c3", "esp32s3"] temporary: true reason: lack of runners + <<: *wifi_dependencies examples/protocols/https_server/wss_server: disable: @@ -118,6 +137,7 @@ examples/protocols/https_server/wss_server: - if: IDF_TARGET != "esp32" temporary: true reason: only test on esp32 + <<: *wifi_dependencies examples/protocols/https_x509_bundle: disable: @@ -137,6 +157,7 @@ examples/protocols/icmp_echo: reason: not supported on p4 disable_test: - if: SOC_WIFI_SUPPORTED != 1 + <<: *wifi_dependencies examples/protocols/l2tap: disable: @@ -244,6 +265,7 @@ examples/protocols/sntp: - if: IDF_TARGET == "esp32" temporary: true reason: the other targets are not tested yet + <<: *wifi_dependencies examples/protocols/sockets: disable: @@ -266,6 +288,7 @@ examples/protocols/sockets/tcp_client: - if: IDF_TARGET == "esp32p4" temporary: true reason: not supported on p4 + <<: *wifi_dependencies examples/protocols/sockets/tcp_server: disable: @@ -274,6 +297,7 @@ examples/protocols/sockets/tcp_server: reason: not supported on p4 disable_test: - if: SOC_WIFI_SUPPORTED != 1 + <<: *wifi_dependencies examples/protocols/sockets/udp_client: disable: @@ -282,6 +306,7 @@ examples/protocols/sockets/udp_client: reason: not supported on p4 disable_test: - if: SOC_WIFI_SUPPORTED != 1 + <<: *wifi_dependencies examples/protocols/sockets/udp_server: disable: @@ -290,6 +315,7 @@ examples/protocols/sockets/udp_server: reason: not supported on p4 disable_test: - if: SOC_WIFI_SUPPORTED != 1 + <<: *wifi_dependencies examples/protocols/static_ip: disable: diff --git a/examples/system/.build-test-rules.yml b/examples/system/.build-test-rules.yml index d011ce4725..a90a4aee62 100644 --- a/examples/system/.build-test-rules.yml +++ b/examples/system/.build-test-rules.yml @@ -157,7 +157,21 @@ examples/system/ota/advanced_https_ota: - if: IDF_TARGET == "esp32c2" or IDF_TARGET == "esp32c6" temporary: true reason: lack of runners - <<: *ethernet_dependencies + depends_filepatterns: + - components/esp_netif/esp_netif_handlers.c + - components/mbedtls/port/dynamic/* + - examples/common_components/**/* + - examples/ethernet/iperf/**/* + - examples/network/simple_sniffer/**/* + - examples/network/vlan_support/**/* + - examples/protocols/**/* + - examples/system/ota/**/* + - examples/wifi/**/* + - tools/ci/python_packages/common_test_methods.py + depends_components: + - app_update + - esp_https_ota + - esp_eth examples/system/ota/native_ota_example: disable: From fa34008c614ff0f90dd401c325fe3eb69416a9a2 Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Wed, 20 Dec 2023 13:27:40 +0100 Subject: [PATCH 14/18] docs(pytest): improve docs, add new features and changes --- .../contribute/esp-idf-tests-with-pytest.rst | 961 +++++++++--------- 1 file changed, 496 insertions(+), 465 deletions(-) diff --git a/docs/en/contribute/esp-idf-tests-with-pytest.rst b/docs/en/contribute/esp-idf-tests-with-pytest.rst index 90509dd5c8..b07ef455da 100644 --- a/docs/en/contribute/esp-idf-tests-with-pytest.rst +++ b/docs/en/contribute/esp-idf-tests-with-pytest.rst @@ -1,30 +1,35 @@ -================= -pytest in ESP-IDF -================= +=============================== +ESP-IDF Tests with Pytest Guide +=============================== :link_to_translation:`zh_CN:[中文]` -ESP-IDF has numerous types of tests that are meant to be executed on an ESP chip (known as **on target testing**). Target tests are usually compiled as part of an IDF project used for testing (known as a **test app**), where test apps follows the same build, flash, and monitor process of any other standard IDF project. +ESP-IDF provides a variety of testing mechanisms that runs directly on target ESP chips (referred to as **target test**). These target tests are typically integrated into an ESP-IDF project specifically designed for testing purposes (known as a **test app**). Similar to standard ESP-IDF projects, test apps follow the same build, flash, and monitoring procedures. -Typically, on target testing will require a connected host (e.g., a PC) that is responsible for triggering a particular test case, providing test data, and inspecting test results. +In target testing, a connected host (for instance, a PC) is typically required to trigger specific test cases, provide test data, and evaluate test results. -ESP-IDF uses the pytest framework (and some pytest plugins) on the host side to automate on target testing. This guide introduces pytest in ESP-IDF and covers the following concepts: +On the host side, ESP-IDF employs the pytest framework (alongside certain pytest plugins) to automate target testing. This guide delves into pytest in ESP-IDF, covering the following aspects: -1. The different types of test apps in ESP-IDF. -2. Using the pytest framework in Python scripts to automate target testing. -3. ESP-IDF Continuous Integration (CI) target testing process. -4. How to run target tests locally with pytest. +1. Common concepts in ESP-IDF target testing. +2. Using the pytest framework in Python scripts for target testing automation. +3. ESP-IDF Continuous Integration (CI) target testing workflow. +4. Running target tests locally using pytest. 5. pytest tips and tricks. .. note:: In ESP-IDF, we use the following pytest plugins by default: - - `pytest-embedded `__ with default services ``esp,idf`` - - `pytest-rerunfailures `__ + - `pytest-embedded `__ with default services ``esp,idf`` + - `pytest-rerunfailures `__ + - `pytest-ignore-test-results `__ All the concepts and usages introduced in this guide are based on the default behavior of these plugins, thus may not be available in vanilla pytest. +.. important:: + + This guide specifically targets ESP-IDF contributors. Some of the concepts, like the custom markers, may not be directly applicable to personal projects using the ESP-IDF SDK. For running pytest-embedded in personal projects, please refer to `pytest-embedded documentation `__, and explore the `provided examples `__. + Installation ============ @@ -34,253 +39,179 @@ All dependencies could be installed by running the install script with the ``--e $ install.sh --enable-pytest +We have implemented several mechanisms to ensure the successful execution of all installation processes. If you encounter any issues during the installation, please submit an issue report to our `GitHub issue tracker `__. -Common Issues During Installation ---------------------------------- +Common Concepts +=============== -No Package 'dbus-1' Found -^^^^^^^^^^^^^^^^^^^^^^^^^ +A **test app** is a set of binaries which is being built from an IDF project that is used to test a particular feature of your project. Test apps are usually located under ``${IDF_PATH}/examples``, ``${IDF_PATH}/tools/test_apps``, and ``${IDF_PATH}/components//test_apps``. -.. code:: text +A **Device under test (DUT)** is a set of ESP chip(s) which connect to a host (e.g., a PC). The host is responsible for flashing the apps to the DUT, triggering the test cases, and inspecting the test results. - configure: error: Package requirements (dbus-1 >= 1.8) were not met: +A typical ESP-IDF project that contains a pytest script will have the following structure: - No package 'dbus-1' found +.. code-block:: text - Consider adjusting the PKG_CONFIG_PATH environment variable if you - installed software in a non-standard prefix. + . + └── my_app/ + ├── main/ + │ └── ... + ├── CMakeLists.txt + └── pytest_foo.py -If you encounter the error message above, you may need to install some missing packages. +Sometimes, for some multi-dut tests, one test case requires multiple test apps. In this case, the test app folder structure would be like this: -If you are using Ubuntu, you may need to run: +.. code-block:: text -.. code:: shell - - sudo apt-get install libdbus-glib-1-dev - -or - -.. code:: shell - - sudo apt-get install libdbus-1-dev - -For other Linux distributions, please Google the error message above and find which missing packages need to be installed for your particular distribution. - -Invalid Command 'bdist_wheel' -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code:: text - - error: invalid command 'bdist_wheel' - -If you encounter the error message above, you may need to install some missing Python packages such as: - -.. code:: shell - - python -m pip install -U pip - -or - -.. code:: shell - - python -m pip install wheel - -.. note:: - - Before running the pip commands, please make sure you are using the IDF Python virtual environment. - - -Test Apps -========= - -ESP-IDF contains different types of test apps that can be automated using pytest. - -Component Tests ---------------- - -ESP-IDF components typically contain component specific test apps that execute component specific unit tests. Component test apps are the recommended way to test components. All the test apps should be located under ``${IDF_PATH}/components//test_apps``, for example: - -.. code:: text - - components/ - └── my_component/ - ├── include/ - │ └── ... - ├── test_apps/ - │ ├── test_app_1 - │ │ ├── main/ - │ │ │ └── ... - │ │ ├── CMakeLists.txt - │ │ └── pytest_my_component_app_1.py - │ ├── test_app_2 - │ │ ├── ... - │ │ └── pytest_my_component_app_2.py - │ └── parent_folder - │ ├── test_app_3 - │ │ ├── ... - │ │ └── pytest_my_component_app_3.py - │ └── ... - ├── my_component.c - └── CMakeLists.txt - -Example Tests -------------- - -The purpose of ESP-IDF examples is to demonstrate parts of ESP-IDF functionality to users (refer to :idf_file:`Examples Readme ` for more information). - -However, to ensure that these examples operate correctly, examples can be treated as test apps and executed automatically by using pytest. All examples should be located under ``${IDF_PATH}/examples``, with tested example including a Python test script, for example: - -.. code:: text - - examples/ - └── parent_folder/ - └── example_1/ - ├── main/ - │ └── ... - ├── CMakeLists.txt - └── pytest_example_1.py - -Custom Tests ------------- - -Custom Tests are tests that aim to test some arbitrary functionality of ESP-IDF, thus are not intended to demonstrate IDF functionality to users in any way. - -All custom test apps are located under ``${IDF_PATH}/tools/test_apps``. For more information please refer to the :idf_file:`Custom Test Readme `. + . + ├── my_app_foo/ + │ ├── main/ + │ │ └── ... + │ └── CMakeLists.txt + ├── my_app_bar/ + │ ├── main/ + │ │ └── ... + │ └── CMakeLists.txt + └── pytest_foo_bar.py pytest in ESP-IDF ================= -.. _pytest-execution-process: +Single DUT Test Cases +--------------------- -pytest Execution Process ------------------------- +Getting Started +^^^^^^^^^^^^^^^ -1. Bootstrapping Phase - - Create session-scoped caches: - - - port-target cache - - port-app cache - -2. Collection Phase - - A. Gather all Python files with the prefix ``pytest_``. - B. Gather all test functions with the prefix ``test_``. - C. Apply the `params `__, and duplicate the test functions. - D. Filter the test cases with CLI options. For the detailed usages, see :ref:`filter-the-test-cases`. - -3. Execution Phase - - A. Construct the `fixtures `__. In ESP-IDF, the common fixtures are initialized in this order: - - a. ``pexpect_proc``: `pexpect `__ instance - - b. ``app``: `IdfApp `__ instance - - The test app's information (e.g., sdkconfig, flash_files, partition_table, etc) would be parsed at this phase. - - c. ``serial``: `IdfSerial `__ instance - - The port of the host to which the target is connected is auto-detected. In the case of multiple targets connected to the host, the test target's type is parsed from the app. The test app binary files are flashed to the test target automatically. - - d. ``dut``: `IdfDut `__ instance - - B. Run the real test function. - - C. Deconstruct the fixtures in this order: - - a. ``dut`` - - i. close the ``serial`` port. - ii. (Only for apps with `Unity test framework `__) generate JUnit report of the Unity test cases. - - b. ``serial`` - c. ``app`` - d. ``pexpect_proc``: Close the file descriptor - - D. (Only for apps with `Unity test framework `__) - - If ``dut.expect_from_unity_output()`` is called, an ``AssertionError`` is raised upon detection of a Unity test failure. - -4. Reporting Phase - - A. Generate JUnit report of the test functions. - B. Modify the JUnit report test case name into ESP-IDF test case ID format: ``..``. - -5. Finalizatoin Phase (Only for apps with `Unity test framework `__) - - Combine the JUnit reports if the JUnit reports of the Unity test cases are generated. - -Basic Example -------------- - -This following Python test script example is taken from :idf_file:`pytest_console_basic.py `. - -.. code:: python +.. code-block:: python @pytest.mark.esp32 - @pytest.mark.esp32c3 + @pytest.mark.esp32s2 @pytest.mark.generic - @pytest.mark.parametrize('config', [ - 'history', - 'nohistory', - ], indirect=True) - def test_console_advanced(config: str, dut: IdfDut) -> None: - if config == 'history': - dut.expect('Command history enabled') - elif config == 'nohistory': - dut.expect('Command history disabled') + def test_hello_world(dut) -> None: + dut.expect('Hello world!') -To demonstrate how pytest is typically used in an ESP-IDF test script, let us go through this simple test script line by line in the following subsections. +This is a simple test script that could run with our getting-started example :example:`get-started/hello_world`. -Target Markers -^^^^^^^^^^^^^^ +First two lines are the target markers: -Pytest markers can be used to indicate which targets (i.e., which ESP chip) a particular test case should should run on. For example: - -.. code:: python - - @pytest.mark.esp32 # <-- support esp32 - @pytest.mark.esp32c3 # <-- support esp32c3 - @pytest.mark.generic # <-- test env "generic" - -The example above indicates that a particular test case is supported on the ESP32 and ESP32-C3. Furthermore, the target's board type should be ``generic``. For more details regarding the ``generic`` type, you may run ``pytest --markers`` to get detailed information regarding all markers. +* The ``@pytest.mark.esp32`` is a marker that indicates that this test case should be run on the ESP32. +* The ``@pytest.mark.esp32s2`` is a marker that indicates that this test case should be run on the ESP32-S2. .. note:: If the test case can be run on all targets officially supported by ESP-IDF (call ``idf.py --list-targets`` for more details), you can use a special marker ``supported_targets`` to apply all of them in one line. -Parameterized Markers -^^^^^^^^^^^^^^^^^^^^^ + We also supports ``preview_targets`` and ``all_targets`` as special target markers (call ``idf.py --list-targets --preview`` for a full targets list including preview targets). -You can use ``pytest.mark.parametrize`` with ``config`` to apply the same test to different apps with different sdkconfig files. For more information about ``sdkconfig.ci.xxx`` files, please refer to the Configuration Files section under :idf_file:`this readme `. +Next, we have the environment marker: -.. code:: python +* The ``@pytest.mark.generic`` is a marker that indicates that this test case should be run on the ``generic`` board type. + +.. note:: + + For the detailed explanation of the environment markers, please refer to :idf_file:`ENV_MARKERS definition ` + +Finally, we have the test function. With a ``dut`` fixture. In single-dut test cases, the ``dut`` fixture is an instance of ``IdfDut`` class, for multi-dut test cases, it is a tuple of ``IdfDut`` instances. For more details regarding the ``IdfDut`` class, please refer to `pytest-embedded IdfDut API reference `__. + +Same App with Different sdkconfig Files +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +For some test cases, you may need to run the same app with different sdkconfig files. For detailed documentation regarding sdkconfig related concepts, please refer to `idf-build-apps Documentation `__. + +Here's a simple example that demonstrates how to run the same app with different sdkconfig files. Assume we have the following folder structure: + +.. code-block:: text + + . + └── my_app/ + ├── main/ + │ └── ... + ├── CMakeLists.txt + ├── sdkconfig.ci.foo + ├── sdkconfig.ci.bar + └── pytest_foo.py + +If the test case needs to run all supported targets with these two sdkconfig files, you can use the following code: + +.. code-block:: python + + @pytest.mark.esp32 + @pytest.mark.esp32s2 + @pytest.mark.parametrize('config', [ # <-- parameterize the sdkconfig file + 'foo', # <-- run with sdkconfig.ci.foo + 'bar', # <-- run with sdkconfig.ci.bar + ], indirect=True) # <-- `indirect=True` is required, indicates this param is pre-calculated before other fixtures + def test_foo_bar(dut, config) -> None: + if config == 'foo': + dut.expect('This is from sdkconfig.ci.foo') + elif config == 'bar': + dut.expect('This is from sdkconfig.ci.bar') + +All markers will impact the test case simultaneously. Overall, this test function would be replicated to 4 test cases: + +- ``test_foo_bar``, with esp32 target, and sdkconfig.ci.foo as the sdkconfig file +- ``test_foo_bar``, with esp32 target, and sdkconfig.ci.bar as the sdkconfig file +- ``test_foo_bar``, with esp32s2 target, and sdkconfig.ci.foo as the sdkconfig file +- ``test_foo_bar``, with esp32s2 target, and sdkconfig.ci.bar as the sdkconfig file + +Sometimes in the test script or the log file, you may see the following format: + +- ``esp32.foo.test_foo_bar`` +- ``esp32.bar.test_foo_bar`` +- ``esp32s2.foo.test_foo_bar`` +- ``esp32s2.bar.test_foo_bar`` + +We call this format the **test case ID**. The test case ID should be considered as the unique identifier of a test case. It is composed of the following parts: + +- ``esp32``: the target name +- ``foo``: the config name +- ``test_foo_bar``: the test function name + +The test case ID is used to identify the test case in the JUnit report. + +.. note:: + + Nearly all the CLI options of pytest-embedded supports parameterization. To see all supported CLI options, you may run ``pytest --help`` and check the ``embedded-...`` sections for vanilla pytest-embedded ones, and the ``idf`` sections for ESP-IDF specific ones. + +.. note:: + + The target markers, like ``@pytest.mark.esp32`` and ``@pytest.mark.esp32s2``, are actually syntactic sugar for parameterization. In fact they are defined as: + + .. code-block:: python + + @pytest.mark.parametrize('target', [ + 'esp32', + 'esp32s2', + ], indirect=True) + +Same App with Different sdkconfig Files, Different Targets +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +For some test cases, you may need to run the same app with different sdkconfig files. These sdkconfig files supports different targets. We may use ``pytest.param`` to achieve this. Let's use the same folder structure as above. + +.. code-block:: python @pytest.mark.parametrize('config', [ - 'history', # <-- run with app built by sdkconfig.ci.history - 'nohistory', # <-- run with app built by sdkconfig.ci.nohistory - ], indirect=True) # <-- `indirect=True` is required + pytest.param('foo', marks=[pytest.mark.esp32]), + pytest.param('bar', marks=[pytest.mark.esp32s2]), + ], indirect=True) -Overall, this test function would be replicated to 4 test cases: +Now this test function would be replicated to 2 test cases (represented as test case IDs): -- ``esp32.history.test_console_advanced`` -- ``esp32.nohistory.test_console_advanced`` -- ``esp32c3.history.test_console_advanced`` -- ``esp32c3.nohistory.test_console_advanced`` +* ``esp32.foo.test_foo_bar`` +* ``esp32s2.bar.test_foo_bar`` -Testing Serial Output -^^^^^^^^^^^^^^^^^^^^^ +Testing Serial Output (Expecting) +--------------------------------- To ensure that test has executed successfully on target, the test script can test that serial output of the target using the ``dut.expect()`` function, for example: -.. code:: python +.. code-block:: python - def test_console_advanced(config: str, dut: IdfDut) -> None: # The value of argument ``config`` is assigned by the parameterization. - if config == 'history': - dut.expect('Command history enabled') - elif config == 'nohistory': - dut.expect('Command history disabled') + def test_hello_world(dut) -> None: + dut.expect('\d+') # <-- `expect`ing from a regex + dut.expect_exact('Hello world!') # <-- `expect_exact`ly the string The ``dut.expect(...)`` will first compile the expected string into regex, which in turn is then used to seek through the serial output until the compiled regex is matched, or until a timeout occurs. @@ -288,115 +219,377 @@ Please pay extra attention to the expected string when it contains regex keyword For more information regarding the different types of ``expect`` functions, please refer to the `pytest-embedded Expecting documentation `__. -Advanced Examples ------------------ +Multi-DUT Test Cases +-------------------- Multi-Target Tests with the Same App ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -In some cases a test may involve multiple targets running the same test app. In this case, multiple DUTs can be instantiated using ``parameterize``, for example: +In some cases a test may involve multiple targets running the same test app. Parametrize ``count`` to the number of DUTs you want to test with. -.. code:: python +.. code-block:: python + @pytest.mark.parametrize('count', [ + 2, + ], indirect=True) + @pytest.mark.parametrize('target', [ + 'esp32|esp32s2', + 'esp32s3', + ], indirect=True) + def test_hello_world(dut) -> None: + dut[0].expect('Hello world!') + dut[1].expect('Hello world!') + +The ``|`` symbol in all parametrized items is used for separating the settings for each DUT. In this example, the test case would be tested with: + +* esp32, esp32s2 +* esp32s3, esp32s3 + +After setting the param ``count`` to 2, all the fixtures are changed into tuples. + +.. important:: + + ``count`` is mandatory for multi-DUT tests. + +.. note:: + + For detailed multi-dut parametrization documentation, please refer to `pytest-embedded Multi-DUT documentation `__. + +.. warning:: + + In some test scripts, you may see target markers like ``@pytest.mark.esp32`` and ``@pytest.mark.esp32s2`` used together with multi-DUT test cases. This is deprecated and should be replaced with the ``target`` parametrization. + + For example, + + .. code-block:: python + + @pytest.mark.esp32 @pytest.mark.esp32s2 - @pytest.mark.esp32s3 - @pytest.mark.usb_host @pytest.mark.parametrize('count', [ 2, ], indirect=True) - def test_usb_host(dut: Tuple[IdfDut, IdfDut]) -> None: - device = dut[0] # <-- assume the first dut is the device - host = dut[1] # <-- and the second dut is the host - ... + def test_hello_world(dut) -> None: + dut[0].expect('Hello world!') + dut[1].expect('Hello world!') + + should be replaced with: -After setting the param ``count`` to 2, all these fixtures are changed into tuples. + .. code-block:: python + + @pytest.mark.parametrize('count', [ + 2, + ], indirect=True) + @pytest.mark.parametrize('target', [ + 'esp32', + 'esp32s2', + ], indirect=True) + def test_hello_world(dut) -> None: + dut[0].expect('Hello world!') + dut[1].expect('Hello world!') + + This could help avoid the ambiguity of the target markers when multi-DUT test cases are using different type of targets. Multi-Target Tests with Different Apps ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -In some cases (in particular protocol tests), a test may involve multiple targets running different test apps (e.g., separate targets to act as master and slave). In this case, multiple DUTs with different test apps can be instantiated using ``parameterize``. +In some cases, a test may involve multiple targets running different test apps (e.g., separate targets to act as master and slave). Usually in ESP-IDF, the folder structure would be like this: -This code example is taken from :idf_file:`pytest_wifi_getting_started.py `. +.. code-block:: text -.. code:: python + . + ├── master/ + │ ├── main/ + │ │ └── ... + │ └── CMakeLists.txt + ├── slave/ + │ ├── main/ + │ │ └── ... + │ └── CMakeLists.txt + └── pytest_master_slave.py + +In this case, we can parametrize the ``app_path`` to the path of the test apps you want to test with. + +.. code-block:: python - @pytest.mark.esp32 @pytest.mark.multi_dut_generic - @pytest.mark.parametrize( - 'count, app_path', [ - (2, - f'{os.path.join(os.path.dirname(__file__), "softAP")}|{os.path.join(os.path.dirname(__file__), "station")}'), - ], indirect=True - ) - def test_wifi_getting_started(dut: Tuple[IdfDut, IdfDut]) -> None: - softap = dut[0] - station = dut[1] - ... - -Here the first DUT was flashed with the app :idf_file:`softAP `, and the second DUT was flashed with the app :idf_file:`station `. + @pytest.mark.parametrize('count', [ + 2, + ], indirect=True) + @pytest.mark.parametrize('app_path, target', [ + (f'{os.path.join(os.path.dirname(__file__), "master")}|{os.path.join(os.path.dirname(__file__), "slave")}', 'esp32|esp32s2'), + (f'{os.path.join(os.path.dirname(__file__), "master")}|{os.path.join(os.path.dirname(__file__), "slave")}', 'esp32s2|esp32'), + ], indirect=True) + def test_master_slave(dut) -> None: + master = dut[0] + slave = dut[1] + + master.write('Hello world!') + slave.expect_exact('Hello world!') .. note:: - Here the ``app_path`` should be set with absolute path. The ``__file__`` macro in Python would return the absolute path of the test script itself. + When parametrizing two items, like ``app_path, target`` here, make sure you're passing a list of tuples to the ``parametrize`` decorator. Each tuple should contain the values for each item. -Multi-Target Tests with Different Apps and Targets -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +The test case here will be replicated to 2 test cases: -This code example is taken from :idf_file:`pytest_wifi_getting_started.py `. As the comment says, for now it is not running in the ESP-IDF CI. +* dut-0, an ESP32, running app ``master``, and dut-1, an ESP32-S2, running app ``slave`` +* dut-0, an ESP32-S2, running app ``master``, and dut-1, an ESP32, running app ``slave`` -.. code:: python +Test Cases with Unity Test Framework +------------------------------------ + +We use `Unity test framework `__ in our unit tests. Overall, we have three types of test cases (`Unity test framework `__): + +* Normal test cases (single DUT) +* Multi-stage test cases (single DUT) +* Multi-device test cases (multi-DUT) + +All single-DUT test cases (including normal test cases and multi-stage test cases) can be run using the following command: + +.. code-block:: python + + def test_unity_single_dut(dut: IdfDut): + dut.run_all_single_board_cases() + +Using this command will skip all the test cases containing the ``[ignore]`` tag. + +If you need to run a group of test cases, you may run: + +.. code-block:: python + + def test_unity_single_dut(dut: IdfDut): + dut.run_all_single_board_cases(group='psram') + +It would trigger all test cases with the ``[psram]`` tag. + +.. warning:: + + You may also see that there are some test scripts with the following statements, which are deprecated. Please use the suggested one as above. + + .. code-block:: python + + def test_unity_single_dut(dut: IdfDut): + dut.expect_exact('Press ENTER to see the list of tests') + dut.write('*') + dut.expect_unity_test_output() + +We also provide a fixture ``case_tester`` to trigger all kinds of test cases easier. For example: + +.. code-block:: python + + def test_unity_single_dut(case_tester): + case_tester.run_all_normal_cases() # to run all normal test cases + case_tester.run_all_multi_dev_cases() # to run all multi-device test cases + case_tester.run_all_multi_stage_cases() # to run all multi-stage test cases + +For a full list of the available functions, please refer to `pytest-embedded case_tester API reference `__. + +Running Target Tests in CI +========================== + +The workflow in CI is as follows: + +.. blockdiag:: + :caption: Target Test Child Pipeline Workflow + :align: center + + blockdiag child-pipeline-workflow { + default_group_color = lightgray; + + group { + label = "build" + + build_test_related_apps; build_non_test_related_apps; + } + + group { + label = "assign_test" + + build_job_report; generate_pytest_child_pipeline; + } + + group { + label = "target_test" + + "Specific Target Test Jobs"; + } + + group { + label = ".post" + + target_test_report; + } + + build_test_related_apps, build_non_test_related_apps -> generate_pytest_child_pipeline, build_job_report -> "Specific Target Test Jobs" -> target_test_report; + } + +All build jobs and target test jobs are generated automatically by our CI script :project:`tools/ci/dynamic_pipelines`. + +Build Jobs +---------- + +In CI, all ESP-IDF projects under ``components``, ``examples``, and ``tools/test_apps``, are built with all supported targets and sdkconfig files. The binaries are built under ``build__``. For example + +.. code-block:: text + + . + ├── build_esp32_history/ + │ └── ... + ├── build_esp32_nohistory/ + │ └── ... + ├── build_esp32s2_history/ + │ └── ... + ├── ... + ├── main/ + ├── CMakeLists.txt + ├── sdkconfig.ci.history + ├── sdkconfig.ci.nohistory + └── ... + +There are two types of build jobs, ``build_test_related_apps`` and ``build_non_test_related_apps``. + +For ``build_test_related_apps``, all the built binaries will be uploaded to our internal MinIO server. You may find the download link in the build report posted in the internal MR. + +For ``build_non_test_related_apps``, all the built binaries will be removed after the build job is finished. Only the build log files will be uploaded to our internal MinIO server. You may also find the download link in the build report posted in the internal MR. + +Target Test Jobs +---------------- + +In CI, all generated target test jobs are named according to the pattern " - ". For example, single-dut test job ``esp32 - generic``, or multi-dut test job ``esp32,esp32 - multi_dut_generic``. + +The binaries in the target test jobs are downloaded from our internal MinIO servers. For most of the test cases, only the files that are required by flash (like .bin files, flash_args files, etc) would be downloaded. For some test cases, like jtag test cases, .elf files are also downloaded. + +Running Tests Locally +===================== + +Installation +------------ + +First you need to install ESP-IDF with additional Python requirements: + +.. code-block:: shell + + $ cd $IDF_PATH + $ bash install.sh --enable-ci --enable-pytest + $ . ./export.sh + +Build Directories +----------------- + +By default, each test case looks for the required binary files in the following directories (in order): + +- ``build__`` +- ``build_`` +- ``build_`` +- ``build`` + +As long as one of the above directories exists, the test case uses that directory to flash the binaries. If non of the above directories exists, the test case fails with an error. + +Test Your Test Script +--------------------- + +Single-DUT Test Cases with ``sdkconfig.defaults`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +This is the simplest use case. Let's take :project:`examples/get-started/hello_world` as an example. Assume we're testing with a ESP32 board. + +.. code-block:: shell + + $ cd $IDF_PATH/examples/get-started/hello_world + $ idf.py set-target esp32 build + $ pytest --target esp32 + +Single-DUT Test Cases with ``sdkconfig.ci.xxx`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Some test cases may need to run with different sdkconfig files. Let's take :project:`examples/system/console/basic` as an example. Assume we're testing with a ESP32 board, and test with ``sdkconfig.ci.history``. + +.. code-block:: shell + + $ cd $IDF_PATH/examples/system/console/basic + $ idf.py -DSDKCONFIG_DEFAULTS='sdkconfig.defaults;sdkconfig.ci.history' -B build_esp32_history set-target esp32 build + $ pytest --target esp32 -k "not nohistory" + +.. note:: + + Here if we use ``pytest --target esp32 -k history``, both test cases will be selected, since ``pytest -k`` will use string matching to filter the test cases. + +If you want to build and test with all sdkconfig files at the same time, you should use our CI script as an helper script: + +.. code-block:: shell + + $ cd $IDF_PATH/examples/system/console/basic + $ python $IDF_PATH/tools/ci/ci_build_apps.py . --target esp32 -v --pytest-apps + $ pytest --target esp32 + +The app with ``sdkconfig.ci.history`` will be built in ``build_esp32_history``, and the app with ``sdkconfig.ci.nohistory`` will be built in ``build_esp32_nohistory``. ``pytest --target esp32`` will run tests on both apps. + +Multi-DUT Test Cases +^^^^^^^^^^^^^^^^^^^^ + +Some test cases may need to run with multiple DUTs. Let's take :project:`examples/openthread` as an example. The test case function looks like this: + +.. code-block:: python @pytest.mark.parametrize( - 'count, app_path, target', [ - (2, - f'{os.path.join(os.path.dirname(__file__), "softAP")}|{os.path.join(os.path.dirname(__file__), "station")}', - 'esp32|esp32s2'), - (2, - f'{os.path.join(os.path.dirname(__file__), "softAP")}|{os.path.join(os.path.dirname(__file__), "station")}', - 'esp32s2|esp32'), + 'config, count, app_path, target', [ + ('rcp|cli_h2|br', 3, + f'{os.path.join(os.path.dirname(__file__), "ot_rcp")}' + f'|{os.path.join(os.path.dirname(__file__), "ot_cli")}' + f'|{os.path.join(os.path.dirname(__file__), "ot_br")}', + 'esp32c6|esp32h2|esp32s3'), ], indirect=True, ) - def test_wifi_getting_started(dut: Tuple[IdfDut, IdfDut]) -> None: - softap = dut[0] - station = dut[1] + def test_thread_connect(dut:Tuple[IdfDut, IdfDut, IdfDut]) -> None: ... -Overall, this test function would be replicated to 2 test cases: +The test cases will run with -- softAP with ESP32 target, and station with ESP32-S2 target -- softAP with ESP32-S2 target, and station with ESP32 target +- ESP32-C6, flashed with ``ot_rcp`` +- ESP32-H2, flashed with ``ot_cli`` +- ESP32-S3, flashed with ``ot_br`` -Support Different Targets with Different sdkconfig Files -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Of course we can build the required binaries manually, but we can also use our CI script as an helper script: -This code example is taken from :idf_file:`pytest_panic.py ` as an advanced example. +.. code-block:: shell -.. code:: python + $ cd $IDF_PATH/examples/openthread + $ python $IDF_PATH/tools/ci/ci_build_apps.py . --target all -v --pytest-apps -k test_thread_connect + $ pytest --target esp32c6,esp32h2,esp32s3 -k test_thread_connect - CONFIGS = [ - pytest.param('coredump_flash_bin_crc', marks=[pytest.mark.esp32, pytest.mark.esp32s2]), - pytest.param('coredump_flash_elf_sha', marks=[pytest.mark.esp32]), # sha256 only supported on esp32 - pytest.param('coredump_uart_bin_crc', marks=[pytest.mark.esp32, pytest.mark.esp32s2]), - pytest.param('coredump_uart_elf_crc', marks=[pytest.mark.esp32, pytest.mark.esp32s2]), - pytest.param('gdbstub', marks=[pytest.mark.esp32, pytest.mark.esp32s2]), - pytest.param('panic', marks=[pytest.mark.esp32, pytest.mark.esp32s2]), - ] +.. important:: - @pytest.mark.parametrize('config', CONFIGS, indirect=True) - ... + It is mandatory to list all the targets for multi-DUT test cases. Otherwise, the test case would fail with an error. + +Debug CI Test Cases +------------------- + +Sometimes you can't reprocude the CI test case failure locally. In this case, you may need to debug the test case with the binaries built in CI. + +Run pytest with ``--pipeline-id `` to force pytest to download the binaries from CI. For example: + +.. code-block:: shell + + $ cd $IDF_PATH/examples/get-started/hello_world + $ pytest --target esp32 --pipeline-id 123456 + +Even if you have ``build_esp32_default``, or ``build`` directory locally, pytest would still download the binaries from pipeline 123456 and place the binaries in ``build_esp32_default``. Then run the test case with this binary. + +Pytest Tips and Tricks +====================== Custom Classes -^^^^^^^^^^^^^^ +-------------- Usually, you may want to write a custom class under these conditions: 1. Add more reusable functions for a certain number of DUTs. -2. Add custom setup and teardown functions in different phases described in Section :ref:`pytest-execution-process`. +2. Add custom setup and teardown functions This code example is taken from :idf_file:`panic/conftest.py `. -.. code:: python +.. code-block:: python class PanicTestDut(IdfDut): ... @@ -417,13 +610,13 @@ This code example is taken from :idf_file:`panic/conftest.py `__ `autouse `__ fixture. This function replaces the ``IdfDut`` class with your custom class. Mark Flaky Tests -^^^^^^^^^^^^^^^^ +---------------- Certain test cases are based on Ethernet or Wi-Fi. However, the test may be flaky due to networking issues. Thus, it is possible to mark a particular test case as flaky. This code example is taken from :idf_file:`pytest_esp_eth.py `. -.. code:: python +.. code-block:: python @pytest.mark.flaky(reruns=3, reruns_delay=5) def test_esp_eth_ip101(dut: IdfDut) -> None: @@ -432,7 +625,7 @@ This code example is taken from :idf_file:`pytest_esp_eth.py ` -.. code:: python +.. code-block:: python @pytest.mark.xfail('config.getvalue("target") == "esp32s2"', reason='raised IllegalInstruction instead') def test_cache_error(dut: PanicTestDut, config: str, test_func_name: str) -> None: @@ -451,195 +644,33 @@ This code example is taken from :idf_file:`pytest_panic.py `. - -Running Tests in CI -=================== - -The workflow in CI is simple, build jobs > target test jobs. - -Build Jobs ----------- - -Build Job Names -^^^^^^^^^^^^^^^ - -- Component-based Unit Tests: ``build_pytest_components_`` -- Example Tests: ``build_pytest_examples_`` -- Custom Tests: ``build_pytest_test_apps_`` - -Build Job Commands -^^^^^^^^^^^^^^^^^^ - -The command used by CI to build all the relevant tests is: ``python $IDF_PATH/tools/ci/ci_build_apps.py --target -vv --pytest-apps`` - -All apps which supported the specified target would be built with all supported sdkconfig files under ``build__``. - -For example, If you run ``python $IDF_PATH/tools/ci/ci_build_apps.py $IDF_PATH/examples/system/console/basic --target esp32 --pytest-apps``, the folder structure would be like this: - -.. code:: text - - basic - ├── build_esp32_history/ - │ └── ... - ├── build_esp32_nohistory/ - │ └── ... - ├── main/ - ├── CMakeLists.txt - ├── pytest_console_basic.py - └── ... - -All the build folders would be uploaded as artifacts under the same directories. - -Target Test Jobs ----------------- - -Target Test Job Names -^^^^^^^^^^^^^^^^^^^^^ - -- Component-based Unit Tests: ``component_ut_pytest__`` -- Example Tests: ``example_test_pytest__`` -- Custom Tests: ``test_app_test_pytest__`` - -Target Test Job Commands -^^^^^^^^^^^^^^^^^^^^^^^^ - -The command used by CI to run all the relevant tests is: ``pytest --target -m `` - -All test cases with the specified target marker and the test env marker under the parent folder would be executed. - -The binaries in the target test jobs are downloaded from build jobs. the artifacts would be placed under the same directories. - -Running Tests Locally -===================== - -First you need to install ESP-IDF with additional Python requirements: - -.. code-block:: shell - - $ cd $IDF_PATH - $ bash install.sh --enable-pytest - $ . ./export.sh - -By default, the pytest script will look for the build directory in this order: - -- ``build__`` -- ``build_`` -- ``build_`` -- ``build`` - -Which means, the simplest way to run pytest is calling ``idf.py build``. - -For example, if you want to run all the esp32 tests under the ``$IDF_PATH/examples/get-started/hello_world`` folder, you should run: - -.. code-block:: shell - - $ cd examples/get-started/hello_world - $ idf.py build - $ pytest --target esp32 - -If you have multiple sdkconfig files in your test app, like those ``sdkconfig.ci.*`` files, the simple ``idf.py build`` won't apply the extra sdkconfig files. Let us take ``$IDF_PATH/examples/system/console/basic`` as an example. - -If you want to test this app with config ``history``, and build with ``idf.py build``, you should run - -.. code-block:: shell - - $ cd examples/system/console/basic - $ idf.py -DSDKCONFIG_DEFAULTS="sdkconfig.defaults;sdkconfig.ci.history" build - $ pytest --target esp32 --sdkconfig history - -If you want to build and test with all sdkconfig files at the same time, you should use our CI script as an helper script: - -.. code-block:: shell - - $ cd examples/system/console/basic - $ python $IDF_PATH/tools/ci/ci_build_apps.py . --target esp32 -vv --pytest-apps - $ pytest --target esp32 - -The app with ``sdkconfig.ci.history`` will be built in ``build_esp32_history``, and the app with ``sdkconfig.ci.nohistory`` will be built in ``build_esp32_nohistory``. ``pytest --target esp32`` will run tests on both apps. - -Tips and Tricks -=============== - -.. _filter-the-test-cases: - -Filter the Test Cases ---------------------- - -- Filter by target with ``pytest --target `` - - pytest would run all the test cases that support specified target. - -- Filter by sdkconfig file with ``pytest --sdkconfig `` - - If ```` is ``default``, pytest would run all the test cases with the sdkconfig file ``sdkconfig.defaults``. - - In other cases, pytest would run all the test cases with sdkconfig file ``sdkconfig.ci.``. - -- Filter by test-case name with ``pytest -k `` to run a single test-case, e.g. ``pytest -k test_int_wdt_cache_disabled``. - Add New Markers --------------- We are using two types of custom markers, target markers which indicate that the test cases should support this target, and env markers which indicate that the test cases should be assigned to runners with these tags in CI. -You can add new markers by adding one line under the ``${IDF_PATH}/conftest.py``. If it is a target marker, it should be added into ``TARGET_MARKERS``. If it is a marker that specifies a type of test environment, it should be added into ``ENV_MARKERS``. The syntax should be: ``: ``. - -Generate JUnit Report ---------------------- - -You can call pytest with ``--junitxml `` to generate the JUnit report. In ESP-IDF, the test case name would be unified as ``..: ``. Skip Auto Flash Binary ---------------------- @@ -662,13 +693,12 @@ Sometimes you may need to add some extra logging lines while running the test ca You can use `Python logging module `__ to achieve this. -Useful Logging Functions (as Fixture) -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Here are some logging functions provided as fixtures, ``log_performance`` -""""""""""""""""""" +^^^^^^^^^^^^^^^^^^^ -.. code:: python +.. code-block:: python def test_hello_world( dut: IdfDut, @@ -676,9 +706,10 @@ Useful Logging Functions (as Fixture) ) -> None: log_performance('test', 1) + The above example would log the performance item with pre-defined format: ``[performance][test]: 1`` and record it under the ``properties`` tag in the JUnit report if ``--junitxml `` is specified. The JUnit test case node would look like: -.. code:: html +.. code-block:: html @@ -687,11 +718,11 @@ The above example would log the performance item with pre-defined format: ``[per ``check_performance`` -""""""""""""""""""""" +^^^^^^^^^^^^^^^^^^^^^ We provide C macros ``TEST_PERFORMANCE_LESS_THAN`` and ``TEST_PERFORMANCE_GREATER_THAN`` to log the performance item and check if the value is in the valid range. Sometimes the performance item value could not be measured in C code, so we also provide a Python function for the same purpose. Please note that using C macros is the preferred approach, since the Python function could not recognize the threshold values of the same performance item under different ifdef blocks well. -.. code:: python +.. code-block:: python def test_hello_world( dut: IdfDut, @@ -708,4 +739,4 @@ Further Readings ================ - pytest documentation: https://docs.pytest.org/en/latest/contents.html -- pytest-embedded documentation: https://docs.espressif.com/projects/pytest-embedded/en/latest/ \ No newline at end of file +- pytest-embedded documentation: https://docs.espressif.com/projects/pytest-embedded/en/latest/ From 3f03e56bb834f44206ecd2a8d0dc90a72c21a183 Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Tue, 9 Jan 2024 12:08:58 +0100 Subject: [PATCH 15/18] ci: add --keep-going to build all test apps in `build_template_app` job --- tools/ci/build_template_app.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/ci/build_template_app.sh b/tools/ci/build_template_app.sh index 3d3994fb51..392c6e8833 100755 --- a/tools/ci/build_template_app.sh +++ b/tools/ci/build_template_app.sh @@ -63,6 +63,7 @@ build_stage2() { --build-dir ${BUILD_DIR} \ --build-log ${BUILD_LOG_CMAKE} \ --size-file size.json \ + --keep-going \ --collect-size-info size_info.txt \ --default-build-targets esp32 esp32s2 esp32s3 esp32c2 esp32c3 esp32c6 esp32h2 esp32p4 } From c1301c61451ee48f30cf695643826bd96d41d4d1 Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Mon, 27 Nov 2023 11:54:56 +0100 Subject: [PATCH 16/18] build: bypass build issues --- components/fatfs/test_apps/.build-test-rules.yml | 3 +-- .../cmake/import_prebuilt/main/project_include.cmake | 8 ++++++++ examples/protocols/.build-test-rules.yml | 3 +-- examples/protocols/sockets/tcp_client/README.md | 4 ++-- examples/storage/littlefs/main/idf_component.yml | 5 +---- examples/system/.build-test-rules.yml | 3 +++ 6 files changed, 16 insertions(+), 10 deletions(-) diff --git a/components/fatfs/test_apps/.build-test-rules.yml b/components/fatfs/test_apps/.build-test-rules.yml index 552ed42304..15288644e4 100644 --- a/components/fatfs/test_apps/.build-test-rules.yml +++ b/components/fatfs/test_apps/.build-test-rules.yml @@ -15,14 +15,13 @@ components/fatfs/test_apps/flash_wl: disable_test: - if: IDF_TARGET not in ["esp32", "esp32c3"] reason: only one target per arch needed - depends_components: - esp_partition - spi_flash - fatfs - vfs - wear_leveling - + components/fatfs/test_apps/sdcard: disable: - if: IDF_TARGET == "esp32p4" diff --git a/examples/build_system/cmake/import_prebuilt/main/project_include.cmake b/examples/build_system/cmake/import_prebuilt/main/project_include.cmake index 9bf6e7cb6e..4f262f8e42 100644 --- a/examples/build_system/cmake/import_prebuilt/main/project_include.cmake +++ b/examples/build_system/cmake/import_prebuilt/main/project_include.cmake @@ -1,6 +1,14 @@ # For users checking this example, ignore the following code. This is so that # the prebuilt project is built automatically in ESP-IDF CI. if("$ENV{CI}") + # otherwise these file won't be rebuilt when switching the built target within the same job + file(REMOVE + ${CMAKE_SOURCE_DIR}/prebuilt/sdkconfig + ${CMAKE_SOURCE_DIR}/main/libprebuilt.a + ${CMAKE_SOURCE_DIR}/main/prebuilt.h + ) + file(REMOVE_RECURSE ${CMAKE_SOURCE_DIR}/prebuilt/build) + execute_process(COMMAND ${IDFTOOL} build WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/prebuilt) endif() diff --git a/examples/protocols/.build-test-rules.yml b/examples/protocols/.build-test-rules.yml index e22a8681fc..f1ba770c6f 100644 --- a/examples/protocols/.build-test-rules.yml +++ b/examples/protocols/.build-test-rules.yml @@ -282,8 +282,7 @@ examples/protocols/sockets/non_blocking: examples/protocols/sockets/tcp_client: disable_test: - if: SOC_WIFI_SUPPORTED != 1 - enable: - - if: INCLUDE_DEFAULT == 1 or IDF_TARGET == "linux" + # linux target won't work with CONFIG_EXAMPLE_SOCKET_IP_INPUT_STDIN=y disable: - if: IDF_TARGET == "esp32p4" temporary: true diff --git a/examples/protocols/sockets/tcp_client/README.md b/examples/protocols/sockets/tcp_client/README.md index 89b32f3b0a..59218a487e 100644 --- a/examples/protocols/sockets/tcp_client/README.md +++ b/examples/protocols/sockets/tcp_client/README.md @@ -1,5 +1,5 @@ -| Supported Targets | ESP32 | ESP32-C2 | ESP32-C3 | ESP32-C6 | ESP32-H2 | ESP32-S2 | ESP32-S3 | Linux | -| ----------------- | ----- | -------- | -------- | -------- | -------- | -------- | -------- | ----- | +| Supported Targets | ESP32 | ESP32-C2 | ESP32-C3 | ESP32-C6 | ESP32-H2 | ESP32-S2 | ESP32-S3 | +| ----------------- | ----- | -------- | -------- | -------- | -------- | -------- | -------- | # TCP Client example diff --git a/examples/storage/littlefs/main/idf_component.yml b/examples/storage/littlefs/main/idf_component.yml index 95c7329ae1..6f83c9dfd6 100644 --- a/examples/storage/littlefs/main/idf_component.yml +++ b/examples/storage/littlefs/main/idf_component.yml @@ -1,6 +1,3 @@ ## IDF Component Manager Manifest File dependencies: - joltwallet/littlefs: "==1.5.5" - ## Required IDF version - idf: - version: ">=5.2.0" + joltwallet/littlefs: "~=1.10.0" diff --git a/examples/system/.build-test-rules.yml b/examples/system/.build-test-rules.yml index a90a4aee62..bfac822ab9 100644 --- a/examples/system/.build-test-rules.yml +++ b/examples/system/.build-test-rules.yml @@ -195,6 +195,9 @@ examples/system/ota/pre_encrypted_ota: - if: IDF_TARGET in ["esp32h2", "esp32p4"] temporary: true reason: target esp32h2, esp32p4 is not supported yet + - if: CONFIG_NAME == "partial_download" and IDF_TARGET == "esp32c3" + temporary: true + reason: build failed disable_test: - if: IDF_TARGET == "esp32c2" or IDF_TARGET == "esp32c6" temporary: true From a98923bc85045920cb7cfa0b86e4ce2874d070bb Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Wed, 10 Jan 2024 15:37:20 +0100 Subject: [PATCH 17/18] test: temp disable ip101 test cases --- components/esp_eth/test_apps/pytest_esp_eth.py | 1 + examples/protocols/l2tap/pytest_example_l2tap_echo.py | 1 + 2 files changed, 2 insertions(+) diff --git a/components/esp_eth/test_apps/pytest_esp_eth.py b/components/esp_eth/test_apps/pytest_esp_eth.py index 921636f441..2b49b9c6ad 100644 --- a/components/esp_eth/test_apps/pytest_esp_eth.py +++ b/components/esp_eth/test_apps/pytest_esp_eth.py @@ -204,6 +204,7 @@ def test_esp_emac_hal(dut: IdfDut) -> None: @pytest.mark.esp32 @pytest.mark.ip101 +@pytest.mark.temp_skip_ci(targets=['esp32'], reason='runner under maintenance') @pytest.mark.parametrize('config', [ 'default_ip101', ], indirect=True) diff --git a/examples/protocols/l2tap/pytest_example_l2tap_echo.py b/examples/protocols/l2tap/pytest_example_l2tap_echo.py index 7bbf205931..074d9ee383 100644 --- a/examples/protocols/l2tap/pytest_example_l2tap_echo.py +++ b/examples/protocols/l2tap/pytest_example_l2tap_echo.py @@ -101,6 +101,7 @@ def actual_test(dut: Dut) -> None: @pytest.mark.esp32 # internally tested using ESP32 with IP101 but may support all targets with SPI Ethernet @pytest.mark.ip101 +@pytest.mark.temp_skip_ci(targets=['esp32'], reason='runner under maintenance') @pytest.mark.flaky(reruns=3, reruns_delay=5) def test_esp_netif_l2tap_example(dut: Dut) -> None: actual_test(dut) From fba96d58c26520df8af41073d7253468d03677ac Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Mon, 18 Dec 2023 15:29:58 +0100 Subject: [PATCH 18/18] ci: dynamic pipeline build: - upgrade idf-build-apps to 2.x - unify get_pytest_apps and get_cmake_apps to get_all_apps - returns (test_apps, non_test_apps) tuple - add tests for the new get_all_apps assign: - generate build report - generate target test pipeline based on the build report target test: - download artifacts from minio server - users can use `pytest --pipeline-id xxxxx` to download and flash the binaries from the artifacts .post: - generate target test reports --- .gitignore | 41 +- .gitlab-ci.yml | 1 - .gitlab/ci/build.yml | 440 +----- .gitlab/ci/common.yml | 5 + .gitlab/ci/dependencies/dependencies.yml | 37 - .gitlab/ci/host-test.yml | 21 +- .gitlab/ci/pre_check.yml | 17 +- .gitlab/ci/rules.yml | 1381 +---------------- .gitlab/ci/target-test.yml | 894 ----------- .pre-commit-config.yaml | 5 +- .../test_apps/esp_flash_stress/README.md | 4 +- conftest.py | 220 ++- tools/ci/artifacts_handler.py | 42 +- tools/ci/check_build_test_rules.py | 12 +- tools/ci/check_tools_files_patterns.py | 2 +- tools/ci/ci_build_apps.py | 265 +--- tools/ci/dynamic_pipelines/__init__.py | 9 + tools/ci/dynamic_pipelines/constants.py | 31 + tools/ci/dynamic_pipelines/models.py | 169 ++ tools/ci/dynamic_pipelines/report.py | 276 ++++ .../ci/dynamic_pipelines/scripts/__init__.py | 25 + .../scripts/child_pipeline_build_apps.py | 73 + .../scripts/generate_build_child_pipeline.py | 193 +++ .../scripts/generate_build_report.py | 59 + .../generate_target_test_child_pipeline.py | 131 ++ .../scripts/generate_target_test_report.py | 62 + .../templates/.dynamic_jobs.yml | 85 + .../templates/generate_target_test_report.yml | 10 + .../templates/report.template.html | 23 + .../templates/test_child_pipeline.yml | 41 + tools/ci/dynamic_pipelines/utils.py | 37 + tools/ci/exclude_check_tools_files.txt | 8 + tools/ci/idf_ci/__init__.py | 0 tools/ci/idf_ci/app.py | 40 + tools/ci/idf_ci/uploader.py | 150 ++ tools/ci/idf_ci_utils.py | 21 +- tools/ci/idf_pytest/constants.py | 80 +- tools/ci/idf_pytest/plugin.py | 47 +- tools/ci/idf_pytest/script.py | 122 +- tools/ci/idf_pytest/tests/conftest.py | 48 + .../ci/idf_pytest/tests/test_get_all_apps.py | 100 ++ .../idf_pytest/tests/test_get_pytest_cases.py | 10 +- tools/requirements/requirements.ci.txt | 2 +- tools/requirements/requirements.pytest.txt | 2 +- 44 files changed, 2184 insertions(+), 3057 deletions(-) delete mode 100644 .gitlab/ci/target-test.yml create mode 100644 tools/ci/dynamic_pipelines/__init__.py create mode 100644 tools/ci/dynamic_pipelines/constants.py create mode 100644 tools/ci/dynamic_pipelines/models.py create mode 100644 tools/ci/dynamic_pipelines/report.py create mode 100644 tools/ci/dynamic_pipelines/scripts/__init__.py create mode 100644 tools/ci/dynamic_pipelines/scripts/child_pipeline_build_apps.py create mode 100644 tools/ci/dynamic_pipelines/scripts/generate_build_child_pipeline.py create mode 100644 tools/ci/dynamic_pipelines/scripts/generate_build_report.py create mode 100644 tools/ci/dynamic_pipelines/scripts/generate_target_test_child_pipeline.py create mode 100644 tools/ci/dynamic_pipelines/scripts/generate_target_test_report.py create mode 100644 tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml create mode 100644 tools/ci/dynamic_pipelines/templates/generate_target_test_report.yml create mode 100644 tools/ci/dynamic_pipelines/templates/report.template.html create mode 100644 tools/ci/dynamic_pipelines/templates/test_child_pipeline.yml create mode 100644 tools/ci/dynamic_pipelines/utils.py create mode 100644 tools/ci/idf_ci/__init__.py create mode 100644 tools/ci/idf_ci/app.py create mode 100644 tools/ci/idf_ci/uploader.py create mode 100644 tools/ci/idf_pytest/tests/conftest.py create mode 100644 tools/ci/idf_pytest/tests/test_get_all_apps.py diff --git a/.gitignore b/.gitignore index 7aa73b086b..b8a8909c37 100644 --- a/.gitignore +++ b/.gitignore @@ -24,18 +24,6 @@ GPATH # cache dir .cache/ -# Components Unit Test Apps files -components/**/build/ -components/**/build_*_*/ -components/**/sdkconfig -components/**/sdkconfig.old - -# Example project files -examples/**/build/ -examples/**/build_esp*_*/ -examples/**/sdkconfig -examples/**/sdkconfig.old - # Doc build artifacts docs/_build/ docs/doxygen_sqlite3.db @@ -44,16 +32,23 @@ docs/doxygen_sqlite3.db docs/_static/DejaVuSans.ttf docs/_static/NotoSansSC-Regular.otf +# Components Unit Test Apps files +components/**/build/ +components/**/build_*_*/ +components/**/sdkconfig +components/**/sdkconfig.old + +# Example project files +examples/**/build/ +examples/**/build_*_*/ +examples/**/sdkconfig +examples/**/sdkconfig.old + # Unit test app files -tools/unit-test-app/sdkconfig -tools/unit-test-app/sdkconfig.old tools/unit-test-app/build tools/unit-test-app/build_*_*/ -tools/unit-test-app/output -tools/unit-test-app/test_configs - -# Unit Test CMake compile log folder -log_ut_cmake +tools/unit-test-app/sdkconfig +tools/unit-test-app/sdkconfig.old # test application build files tools/test_apps/**/build/ @@ -61,7 +56,8 @@ tools/test_apps/**/build_*_*/ tools/test_apps/**/sdkconfig tools/test_apps/**/sdkconfig.old -TEST_LOGS +TEST_LOGS/ +build_summary_*.xml # gcov coverage reports *.gcda @@ -101,8 +97,9 @@ managed_components # pytest log pytest_embedded_log/ -list_job_*.txt -size_info.txt +list_job*.txt +size_info*.txt +XUNIT_RESULT*.xml # clang config (for LSP) .clangd diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d8f4c3eadf..87487c7620 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -28,5 +28,4 @@ include: - '.gitlab/ci/build.yml' - '.gitlab/ci/integration_test.yml' - '.gitlab/ci/host-test.yml' - - '.gitlab/ci/target-test.yml' - '.gitlab/ci/deploy.yml' diff --git a/.gitlab/ci/build.yml b/.gitlab/ci/build.yml index 074c744ecf..798271d94f 100644 --- a/.gitlab/ci/build.yml +++ b/.gitlab/ci/build.yml @@ -1,7 +1,7 @@ .build_template: stage: build extends: - - .after_script:build:ccache + - .after_script:build:ccache:upload-when-fail image: $ESP_ENV_IMAGE tags: - build @@ -43,8 +43,8 @@ examples/bluetooth/esp_ble_mesh/ble_mesh_console examples/bluetooth/hci/controller_hci_uart_esp32 examples/wifi/iperf - --modified-components ${MODIFIED_COMPONENTS} - --modified-files ${MODIFIED_FILES} + --modified-components ${MR_MODIFIED_COMPONENTS} + --modified-files ${MR_MODIFIED_FILES} # for detailed documents, please refer to .gitlab/ci/README.md#uploaddownload-artifacts-to-internal-minio-server - python tools/ci/artifacts_handler.py upload @@ -62,307 +62,14 @@ --copy-sdkconfig --parallel-count ${CI_NODE_TOTAL:-1} --parallel-index ${CI_NODE_INDEX:-1} - --modified-components ${MODIFIED_COMPONENTS} - --modified-files ${MODIFIED_FILES} + --modified-components ${MR_MODIFIED_COMPONENTS} + --modified-files ${MR_MODIFIED_FILES} $TEST_BUILD_OPTS_EXTRA - python tools/ci/artifacts_handler.py upload -.build_pytest_template: - extends: - - .build_cmake_template - script: - # CI specific options start from "--parallel-count xxx". could ignore when running locally - - run_cmd python tools/ci/ci_build_apps.py $TEST_DIR -v - -t $IDF_TARGET - -m \"not host_test\" - --pytest-apps - --parallel-count ${CI_NODE_TOTAL:-1} - --parallel-index ${CI_NODE_INDEX:-1} - --collect-app-info "list_job_${CI_JOB_NAME_SLUG}.txt" - --modified-components ${MODIFIED_COMPONENTS} - --modified-files ${MODIFIED_FILES} - - python tools/ci/artifacts_handler.py upload - -.build_pytest_no_jtag_template: - extends: - - .build_cmake_template - script: - # CI specific options start from "--parallel-count xxx". could ignore when running locally - - run_cmd python tools/ci/ci_build_apps.py $TEST_DIR -v - -t $IDF_TARGET - -m \"not host_test and not jtag\" - --pytest-apps - --parallel-count ${CI_NODE_TOTAL:-1} - --parallel-index ${CI_NODE_INDEX:-1} - --collect-app-info "list_job_${CI_JOB_NAME_SLUG}.txt" - --modified-components ${MODIFIED_COMPONENTS} - --modified-files ${MODIFIED_FILES} - - python tools/ci/artifacts_handler.py upload - -.build_pytest_jtag_template: - extends: - - .build_cmake_template - script: - # CI specific options start from "--parallel-count xxx". could ignore when running locally - - run_cmd python tools/ci/ci_build_apps.py $TEST_DIR -v - -t $IDF_TARGET - -m \"not host_test and jtag\" - --pytest-apps - --parallel-count ${CI_NODE_TOTAL:-1} - --parallel-index ${CI_NODE_INDEX:-1} - --collect-app-info "list_job_${CI_JOB_NAME_SLUG}.txt" - --modified-components ${MODIFIED_COMPONENTS} - --modified-files ${MODIFIED_FILES} - - python tools/ci/artifacts_handler.py upload - -build_pytest_examples_esp32: - extends: - - .build_pytest_no_jtag_template - - .rules:build:example_test-esp32 - parallel: 6 - variables: - IDF_TARGET: esp32 - TEST_DIR: examples - -build_pytest_examples_esp32s2: - extends: - - .build_pytest_no_jtag_template - - .rules:build:example_test-esp32s2 - parallel: 3 - variables: - IDF_TARGET: esp32s2 - TEST_DIR: examples - -build_pytest_examples_esp32s3: - extends: - - .build_pytest_no_jtag_template - - .rules:build:example_test-esp32s3 - parallel: 4 - variables: - IDF_TARGET: esp32s3 - TEST_DIR: examples - -build_pytest_examples_esp32c3: - extends: - - .build_pytest_no_jtag_template - - .rules:build:example_test-esp32c3 - parallel: 4 - variables: - IDF_TARGET: esp32c3 - TEST_DIR: examples - -build_pytest_examples_esp32c2: - extends: - - .build_pytest_no_jtag_template - - .rules:build:example_test-esp32c2 - parallel: 2 - variables: - IDF_TARGET: esp32c2 - TEST_DIR: examples - -build_pytest_examples_esp32c6: - extends: - - .build_pytest_no_jtag_template - - .rules:build:example_test-esp32c6 - parallel: 2 - variables: - IDF_TARGET: esp32c6 - TEST_DIR: examples - -build_pytest_examples_esp32h2: - extends: - - .build_pytest_no_jtag_template - - .rules:build:example_test-esp32h2 - parallel: 2 - variables: - IDF_TARGET: esp32h2 - TEST_DIR: examples - -build_pytest_examples_esp32p4: - extends: - - .build_pytest_no_jtag_template - - .rules:build:example_test-esp32p4 - parallel: 2 - variables: - IDF_TARGET: esp32p4 - TEST_DIR: examples - -build_pytest_examples_jtag: # for all targets - extends: - - .build_pytest_jtag_template - - .rules:build:example_test - variables: - IDF_TARGET: all - TEST_DIR: examples - -build_pytest_components_esp32: - extends: - - .build_pytest_template - - .rules:build:component_ut-esp32 - parallel: 5 - variables: - IDF_TARGET: esp32 - TEST_DIR: components - -build_pytest_components_esp32s2: - extends: - - .build_pytest_template - - .rules:build:component_ut-esp32s2 - parallel: 4 - variables: - IDF_TARGET: esp32s2 - TEST_DIR: components - -build_pytest_components_esp32s3: - extends: - - .build_pytest_template - - .rules:build:component_ut-esp32s3 - parallel: 4 - variables: - IDF_TARGET: esp32s3 - TEST_DIR: components - -build_pytest_components_esp32c3: - extends: - - .build_pytest_template - - .rules:build:component_ut-esp32c3 - parallel: 4 - variables: - IDF_TARGET: esp32c3 - TEST_DIR: components - -build_pytest_components_esp32c2: - extends: - - .build_pytest_template - - .rules:build:component_ut-esp32c2 - parallel: 3 - variables: - IDF_TARGET: esp32c2 - TEST_DIR: components - -build_pytest_components_esp32c6: - extends: - - .build_pytest_template - - .rules:build:component_ut-esp32c6 - parallel: 3 - variables: - IDF_TARGET: esp32c6 - TEST_DIR: components - -build_pytest_components_esp32h2: - extends: - - .build_pytest_template - - .rules:build:component_ut-esp32h2 - parallel: 4 - variables: - IDF_TARGET: esp32h2 - TEST_DIR: components - -build_pytest_components_esp32p4: - extends: - - .build_pytest_template - - .rules:build:component_ut-esp32p4 - parallel: 4 - variables: - IDF_TARGET: esp32p4 - TEST_DIR: components - -build_only_components_apps: - extends: - - .build_cmake_template - - .rules:build:component_ut - parallel: 5 - script: - - set_component_ut_vars - # CI specific options start from "--parallel-count xxx". could ignore when running locally - - run_cmd python tools/ci/ci_build_apps.py $COMPONENT_UT_DIRS -v - -t all - --parallel-count ${CI_NODE_TOTAL:-1} - --parallel-index ${CI_NODE_INDEX:-1} - --modified-components ${MODIFIED_COMPONENTS} - --modified-files ${MODIFIED_FILES} - - python tools/ci/artifacts_handler.py upload - -build_pytest_test_apps_esp32: - extends: - - .build_pytest_template - - .rules:build:custom_test-esp32 - variables: - IDF_TARGET: esp32 - TEST_DIR: tools/test_apps - -build_pytest_test_apps_esp32s2: - extends: - - .build_pytest_template - - .rules:build:custom_test-esp32s2 - variables: - IDF_TARGET: esp32s2 - TEST_DIR: tools/test_apps - -build_pytest_test_apps_esp32s3: - extends: - - .build_pytest_template - - .rules:build:custom_test-esp32s3 - parallel: 2 - variables: - IDF_TARGET: esp32s3 - TEST_DIR: tools/test_apps - -build_pytest_test_apps_esp32c3: - extends: - - .build_pytest_template - - .rules:build:custom_test-esp32c3 - variables: - IDF_TARGET: esp32c3 - TEST_DIR: tools/test_apps - -build_pytest_test_apps_esp32c2: - extends: - - .build_pytest_template - - .rules:build:custom_test-esp32c2 - variables: - IDF_TARGET: esp32c2 - TEST_DIR: tools/test_apps - -build_pytest_test_apps_esp32c6: - extends: - - .build_pytest_template - - .rules:build:custom_test-esp32c6 - variables: - IDF_TARGET: esp32c6 - TEST_DIR: tools/test_apps - -build_pytest_test_apps_esp32h2: - extends: - - .build_pytest_template - - .rules:build:custom_test-esp32h2 - variables: - IDF_TARGET: esp32h2 - TEST_DIR: tools/test_apps - -build_pytest_test_apps_esp32p4: - extends: - - .build_pytest_template - - .rules:build:custom_test-esp32p4 - variables: - IDF_TARGET: esp32p4 - TEST_DIR: tools/test_apps - -build_only_tools_test_apps: - extends: - - .build_cmake_template - - .rules:build:custom_test - parallel: 9 - script: - # CI specific options start from "--parallel-count xxx". could ignore when running locally - - run_cmd python tools/ci/ci_build_apps.py tools/test_apps -v - -t all - --parallel-count ${CI_NODE_TOTAL:-1} - --parallel-index ${CI_NODE_INDEX:-1} - --modified-components ${MODIFIED_COMPONENTS} - --modified-files ${MODIFIED_FILES} - - python tools/ci/artifacts_handler.py upload - +###################### +# build_template_app # +###################### .build_template_app_template: extends: - .build_template @@ -400,96 +107,27 @@ fast_template_app: BUILD_COMMAND_ARGS: "-p" #------------------------------------------------------------------------------ -build_examples_cmake_esp32: - extends: - - .build_cmake_template - - .rules:build:example_test-esp32 - parallel: 8 - variables: - IDF_TARGET: esp32 - TEST_DIR: examples - -build_examples_cmake_esp32s2: - extends: - - .build_cmake_template - - .rules:build:example_test-esp32s2 - parallel: 7 - variables: - IDF_TARGET: esp32s2 - TEST_DIR: examples - -build_examples_cmake_esp32s3: - extends: - - .build_cmake_template - - .rules:build:example_test-esp32s3 - parallel: 11 - variables: - IDF_TARGET: esp32s3 - TEST_DIR: examples - -build_examples_cmake_esp32c2: - extends: - - .build_cmake_template - - .rules:build:example_test-esp32c2 - parallel: 7 - variables: - IDF_TARGET: esp32c2 - TEST_DIR: examples - -build_examples_cmake_esp32c3: - extends: - - .build_cmake_template - - .rules:build:example_test-esp32c3 - parallel: 9 - variables: - IDF_TARGET: esp32c3 - TEST_DIR: examples - -build_examples_cmake_esp32c6: - extends: - - .build_cmake_template - - .rules:build:example_test-esp32c6 - parallel: 11 - variables: - IDF_TARGET: esp32c6 - TEST_DIR: examples - -build_examples_cmake_esp32h2: - extends: - - .build_cmake_template - - .rules:build:example_test-esp32h2 - parallel: 9 - variables: - IDF_TARGET: esp32h2 - TEST_DIR: examples - -build_examples_cmake_esp32p4: - extends: - - .build_cmake_template - - .rules:build:example_test-esp32p4 - parallel: 4 - variables: - IDF_TARGET: esp32p4 - TEST_DIR: examples - +######################################## +# Clang Build Apps Without Tests Cases # +######################################## build_clang_test_apps_esp32: extends: - .build_cmake_clang_template - - .rules:build:custom_test-esp32 + - .rules:build variables: IDF_TARGET: esp32 build_clang_test_apps_esp32s2: extends: - .build_cmake_clang_template - - .rules:build:custom_test-esp32s2 + - .rules:build variables: IDF_TARGET: esp32s2 build_clang_test_apps_esp32s3: extends: - .build_cmake_clang_template - - .rules:build:custom_test-esp32s3 + - .rules:build variables: IDF_TARGET: esp32s3 @@ -506,26 +144,29 @@ build_clang_test_apps_esp32s3: build_clang_test_apps_esp32c3: extends: - .build_clang_test_apps_riscv - - .rules:build:custom_test-esp32c3 + - .rules:build variables: IDF_TARGET: esp32c3 build_clang_test_apps_esp32c2: extends: - .build_clang_test_apps_riscv - - .rules:build:custom_test-esp32c2 + - .rules:build variables: IDF_TARGET: esp32c2 build_clang_test_apps_esp32c6: extends: - .build_clang_test_apps_riscv - - .rules:build:custom_test-esp32c6 + - .rules:build # TODO: c6 builds fail in master due to missing headers allow_failure: true variables: IDF_TARGET: esp32c6 +###################### +# Build System Tests # +###################### .test_build_system_template: stage: host_test extends: @@ -634,3 +275,44 @@ build_template_app: needs: - job: fast_template_app artifacts: false + +#################### +# Dynamic Pipeline # +#################### +generate_build_child_pipeline: + extends: + - .build_template + dependencies: # set dependencies to null to avoid missing artifacts issue + needs: + - pipeline_variables + artifacts: + paths: + - build_child_pipeline.yml + - test_related_apps.txt + - non_test_related_apps.txt + script: + - run_cmd python tools/ci/dynamic_pipelines/scripts/generate_build_child_pipeline.py + --modified-components ${MR_MODIFIED_COMPONENTS} + --modified-files ${MR_MODIFIED_FILES} + +build_child_pipeline: + stage: build + needs: + - job: fast_template_app + artifacts: false + - pipeline_variables + - generate_build_child_pipeline + variables: + IS_MR_PIPELINE: $IS_MR_PIPELINE + MR_MODIFIED_COMPONENTS: $MR_MODIFIED_COMPONENTS + MR_MODIFIED_FILES: $MR_MODIFIED_FILES + PARENT_PIPELINE_ID: $CI_PIPELINE_ID + BUILD_AND_TEST_ALL_APPS: $BUILD_AND_TEST_ALL_APPS + # https://gitlab.com/gitlab-org/gitlab/-/issues/214340 + inherit: + variables: false + trigger: + include: + - artifact: build_child_pipeline.yml + job: generate_build_child_pipeline + strategy: depend diff --git a/.gitlab/ci/common.yml b/.gitlab/ci/common.yml index d2c48531be..32beb8b84e 100644 --- a/.gitlab/ci/common.yml +++ b/.gitlab/ci/common.yml @@ -85,6 +85,7 @@ variables: ################################################ .common_before_scripts: &common-before_scripts | source tools/ci/utils.sh + is_based_on_commits $REQUIRED_ANCESTOR_COMMITS if [[ -n "$IDF_DONT_USE_MIRRORS" ]]; then @@ -208,6 +209,10 @@ variables: - export EXTRA_CXXFLAGS=${PEDANTIC_CXXFLAGS} .after_script:build:ccache: + after_script: + - *show_ccache_statistics + +.after_script:build:ccache:upload-when-fail: after_script: - *show_ccache_statistics - *upload_failed_job_log_artifacts diff --git a/.gitlab/ci/dependencies/dependencies.yml b/.gitlab/ci/dependencies/dependencies.yml index 95a8342292..4f87a9ab3b 100644 --- a/.gitlab/ci/dependencies/dependencies.yml +++ b/.gitlab/ci/dependencies/dependencies.yml @@ -66,43 +66,6 @@ included_in: - build:check -# --------------- -# Build Test Jobs -# --------------- -"build:{0}-{1}": - matrix: - - *target_test - - *all_targets - labels: - - build - patterns: - - build_components - - build_system - - build_target_test - - downloadable-tools - included_in: - - "build:{0}" - - build:target_test - -#################### -# Target Test Jobs # -#################### -"test:{0}-{1}": - matrix: - - *target_test - - *all_targets - labels: # For each rule, use labels and - - - "{0}" - - "{0}_{1}" - - target_test - patterns: # For each rule, use patterns and build- - - "{0}" - - "build-{0}" - included_in: # Parent rules - - "build:{0}" - - "build:{0}-{1}" - - build:target_test - "test:host_test": labels: - host_test diff --git a/.gitlab/ci/host-test.yml b/.gitlab/ci/host-test.yml index f025824206..ed71ead85c 100644 --- a/.gitlab/ci/host-test.yml +++ b/.gitlab/ci/host-test.yml @@ -289,8 +289,8 @@ test_pytest_qemu: --pytest-apps -m qemu --collect-app-info "list_job_${CI_JOB_NAME_SLUG}.txt" - --modified-components ${MODIFIED_COMPONENTS} - --modified-files ${MODIFIED_FILES} + --modified-components ${MR_MODIFIED_COMPONENTS} + --modified-files ${MR_MODIFIED_FILES} - retry_failed git clone $KNOWN_FAILURE_CASES_REPO known_failure_cases - run_cmd pytest --target $IDF_TARGET @@ -318,8 +318,8 @@ test_pytest_linux: --pytest-apps -m host_test --collect-app-info "list_job_${CI_JOB_NAME_SLUG}.txt" - --modified-components ${MODIFIED_COMPONENTS} - --modified-files ${MODIFIED_FILES} + --modified-components ${MR_MODIFIED_COMPONENTS} + --modified-files ${MR_MODIFIED_FILES} - retry_failed git clone $KNOWN_FAILURE_CASES_REPO known_failure_cases - run_cmd pytest --target linux @@ -327,3 +327,16 @@ test_pytest_linux: --junitxml=XUNIT_RESULT.xml --ignore-result-files known_failure_cases/known_failure_cases.txt --app-info-filepattern \"list_job_*.txt\" + +test_idf_pytest_plugin: + extends: + - .host_test_template + - .rules:patterns:idf-pytest-plugin + variables: + SUBMODULES_TO_FETCH: "none" + artifacts: + reports: + junit: XUNIT_RESULT.xml + script: + - cd tools/ci/idf_pytest + - pytest --junitxml=${CI_PROJECT_DIR}/XUNIT_RESULT.xml diff --git a/.gitlab/ci/pre_check.yml b/.gitlab/ci/pre_check.yml index b95c86d8d0..4282bf7997 100644 --- a/.gitlab/ci/pre_check.yml +++ b/.gitlab/ci/pre_check.yml @@ -145,9 +145,22 @@ pipeline_variables: tags: - build script: - - MODIFIED_FILES=$(echo $GIT_DIFF_OUTPUT | xargs) + # MODIFIED_FILES is a list of files that changed, could be used everywhere + - MODIFIED_FILES=$(echo "$GIT_DIFF_OUTPUT" | xargs) - echo "MODIFIED_FILES=$MODIFIED_FILES" >> pipeline.env - - echo "MODIFIED_COMPONENTS=$(run_cmd python tools/ci/ci_get_mr_info.py components --modified-files $MODIFIED_FILES | xargs)" >> pipeline.env + # MR_MODIFIED_FILES and MR_MODIFIED_COMPONENTS are semicolon separated lists that is used in MR only + # for non MR pipeline, these are empty lists + - | + if [ $IS_MR_PIPELINE == "0" ]; then + echo "MR_MODIFIED_FILES=" >> pipeline.env + echo "MR_MODIFIED_COMPONENTS=" >> pipeline.env + else + MR_MODIFIED_FILES=$(echo "$GIT_DIFF_OUTPUT" | tr '\n' ';') + echo "MR_MODIFIED_FILES=\"$MR_MODIFIED_FILES\"" >> pipeline.env + + MR_MODIFIED_COMPONENTS=$(run_cmd python tools/ci/ci_get_mr_info.py components --modified-files $MODIFIED_FILES | tr '\n' ';') + echo "MR_MODIFIED_COMPONENTS=\"$MR_MODIFIED_COMPONENTS\"" >> pipeline.env + fi - | if echo "$CI_MERGE_REQUEST_LABELS" | egrep "(^|,)BUILD_AND_TEST_ALL_APPS(,|$)"; then echo "BUILD_AND_TEST_ALL_APPS=1" >> pipeline.env diff --git a/.gitlab/ci/rules.yml b/.gitlab/ci/rules.yml index 6afb0f43b4..6132556695 100644 --- a/.gitlab/ci/rules.yml +++ b/.gitlab/ci/rules.yml @@ -23,19 +23,6 @@ .patterns-sonarqube-files: &patterns-sonarqube-files - "tools/ci/sonar_exclude_list.txt" -.patterns-example_test: &patterns-example_test - - "tools/ci/idf_pytest/**/*" - - "tools/ci/python_packages/gitlab_api.py" - - "tools/ci/python_packages/idf_http_server_test/**/*" - - "tools/ci/python_packages/idf_iperf_test_util/**/*" - - "tools/ci/python_packages/common_test_methods.py" - - - "tools/esp_prov/**/*" - - "examples/**/*" - -.patterns-build-example_test: &patterns-build-example_test - - "tools/ci/get_supported_examples.sh" - .patterns-build_components: &patterns-build_components # components files except "test*/" "host*/" folders # ?? to include folders less than 4 characters @@ -67,22 +54,6 @@ - "tools/ci/ci_build_apps.py" - "tools/test_build_system/**/*" -.patterns-custom_test: &patterns-custom_test - - "tools/ci/idf_pytest/**/*" - - "tools/ci/python_packages/gitlab_api.py" - - "tools/ci/python_packages/common_test_methods.py" - - - "tools/test_apps/**/*" - - "tools/ldgen/**/*" - -.patterns-component_ut: &patterns-component_ut - - "tools/ci/idf_pytest/**/*" - - "tools/ci/python_packages/gitlab_api.py" - - "tools/ci/python_packages/common_test_methods.py" - - "tools/test_apps/configs/sdkconfig.debug_helpers" - - - "components/**/*" - .patterns-build_macos: &patterns-build_macos - "tools/ci/test_configure_ci_environment.sh" @@ -171,6 +142,9 @@ - "components/bt/esp_ble_mesh/lib/lib" - ".gitmodules" +.patterns-idf-pytest-plugin: &patterns-idf-pytest-plugin + - "tools/ci/idf_pytest/**/*" + ############## # if anchors # ############## @@ -280,6 +254,12 @@ - <<: *if-dev-push changes: *patterns-sonarqube-files +.rules:patterns:idf-pytest-plugin: + rules: + - <<: *if-protected + - <<: *if-dev-push + changes: *patterns-idf-pytest-plugin + # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! # DO NOT place comments or maintain any code from this line # @@ -295,99 +275,9 @@ .if-label-build: &if-label-build if: '$BOT_LABEL_BUILD || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*build(?:,[^,\n\r]+)*$/i' -.if-label-component_ut: &if-label-component_ut - if: '$BOT_LABEL_COMPONENT_UT || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*component_ut(?:,[^,\n\r]+)*$/i' - -.if-label-component_ut_esp32: &if-label-component_ut_esp32 - if: '$BOT_LABEL_COMPONENT_UT_ESP32 || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*component_ut_esp32(?:,[^,\n\r]+)*$/i' - -.if-label-component_ut_esp32c2: &if-label-component_ut_esp32c2 - if: '$BOT_LABEL_COMPONENT_UT_ESP32C2 || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*component_ut_esp32c2(?:,[^,\n\r]+)*$/i' - -.if-label-component_ut_esp32c3: &if-label-component_ut_esp32c3 - if: '$BOT_LABEL_COMPONENT_UT_ESP32C3 || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*component_ut_esp32c3(?:,[^,\n\r]+)*$/i' - -.if-label-component_ut_esp32c5: &if-label-component_ut_esp32c5 - if: '$BOT_LABEL_COMPONENT_UT_ESP32C5 || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*component_ut_esp32c5(?:,[^,\n\r]+)*$/i' - -.if-label-component_ut_esp32c6: &if-label-component_ut_esp32c6 - if: '$BOT_LABEL_COMPONENT_UT_ESP32C6 || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*component_ut_esp32c6(?:,[^,\n\r]+)*$/i' - -.if-label-component_ut_esp32h2: &if-label-component_ut_esp32h2 - if: '$BOT_LABEL_COMPONENT_UT_ESP32H2 || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*component_ut_esp32h2(?:,[^,\n\r]+)*$/i' - -.if-label-component_ut_esp32p4: &if-label-component_ut_esp32p4 - if: '$BOT_LABEL_COMPONENT_UT_ESP32P4 || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*component_ut_esp32p4(?:,[^,\n\r]+)*$/i' - -.if-label-component_ut_esp32s2: &if-label-component_ut_esp32s2 - if: '$BOT_LABEL_COMPONENT_UT_ESP32S2 || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*component_ut_esp32s2(?:,[^,\n\r]+)*$/i' - -.if-label-component_ut_esp32s3: &if-label-component_ut_esp32s3 - if: '$BOT_LABEL_COMPONENT_UT_ESP32S3 || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*component_ut_esp32s3(?:,[^,\n\r]+)*$/i' - -.if-label-custom_test: &if-label-custom_test - if: '$BOT_LABEL_CUSTOM_TEST || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*custom_test(?:,[^,\n\r]+)*$/i' - -.if-label-custom_test_esp32: &if-label-custom_test_esp32 - if: '$BOT_LABEL_CUSTOM_TEST_ESP32 || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*custom_test_esp32(?:,[^,\n\r]+)*$/i' - -.if-label-custom_test_esp32c2: &if-label-custom_test_esp32c2 - if: '$BOT_LABEL_CUSTOM_TEST_ESP32C2 || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*custom_test_esp32c2(?:,[^,\n\r]+)*$/i' - -.if-label-custom_test_esp32c3: &if-label-custom_test_esp32c3 - if: '$BOT_LABEL_CUSTOM_TEST_ESP32C3 || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*custom_test_esp32c3(?:,[^,\n\r]+)*$/i' - -.if-label-custom_test_esp32c5: &if-label-custom_test_esp32c5 - if: '$BOT_LABEL_CUSTOM_TEST_ESP32C5 || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*custom_test_esp32c5(?:,[^,\n\r]+)*$/i' - -.if-label-custom_test_esp32c6: &if-label-custom_test_esp32c6 - if: '$BOT_LABEL_CUSTOM_TEST_ESP32C6 || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*custom_test_esp32c6(?:,[^,\n\r]+)*$/i' - -.if-label-custom_test_esp32h2: &if-label-custom_test_esp32h2 - if: '$BOT_LABEL_CUSTOM_TEST_ESP32H2 || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*custom_test_esp32h2(?:,[^,\n\r]+)*$/i' - -.if-label-custom_test_esp32p4: &if-label-custom_test_esp32p4 - if: '$BOT_LABEL_CUSTOM_TEST_ESP32P4 || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*custom_test_esp32p4(?:,[^,\n\r]+)*$/i' - -.if-label-custom_test_esp32s2: &if-label-custom_test_esp32s2 - if: '$BOT_LABEL_CUSTOM_TEST_ESP32S2 || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*custom_test_esp32s2(?:,[^,\n\r]+)*$/i' - -.if-label-custom_test_esp32s3: &if-label-custom_test_esp32s3 - if: '$BOT_LABEL_CUSTOM_TEST_ESP32S3 || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*custom_test_esp32s3(?:,[^,\n\r]+)*$/i' - .if-label-docker: &if-label-docker if: '$BOT_LABEL_DOCKER || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*docker(?:,[^,\n\r]+)*$/i' -.if-label-example_test: &if-label-example_test - if: '$BOT_LABEL_EXAMPLE_TEST || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*example_test(?:,[^,\n\r]+)*$/i' - -.if-label-example_test_esp32: &if-label-example_test_esp32 - if: '$BOT_LABEL_EXAMPLE_TEST_ESP32 || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*example_test_esp32(?:,[^,\n\r]+)*$/i' - -.if-label-example_test_esp32c2: &if-label-example_test_esp32c2 - if: '$BOT_LABEL_EXAMPLE_TEST_ESP32C2 || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*example_test_esp32c2(?:,[^,\n\r]+)*$/i' - -.if-label-example_test_esp32c3: &if-label-example_test_esp32c3 - if: '$BOT_LABEL_EXAMPLE_TEST_ESP32C3 || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*example_test_esp32c3(?:,[^,\n\r]+)*$/i' - -.if-label-example_test_esp32c5: &if-label-example_test_esp32c5 - if: '$BOT_LABEL_EXAMPLE_TEST_ESP32C5 || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*example_test_esp32c5(?:,[^,\n\r]+)*$/i' - -.if-label-example_test_esp32c6: &if-label-example_test_esp32c6 - if: '$BOT_LABEL_EXAMPLE_TEST_ESP32C6 || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*example_test_esp32c6(?:,[^,\n\r]+)*$/i' - -.if-label-example_test_esp32h2: &if-label-example_test_esp32h2 - if: '$BOT_LABEL_EXAMPLE_TEST_ESP32H2 || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*example_test_esp32h2(?:,[^,\n\r]+)*$/i' - -.if-label-example_test_esp32p4: &if-label-example_test_esp32p4 - if: '$BOT_LABEL_EXAMPLE_TEST_ESP32P4 || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*example_test_esp32p4(?:,[^,\n\r]+)*$/i' - -.if-label-example_test_esp32s2: &if-label-example_test_esp32s2 - if: '$BOT_LABEL_EXAMPLE_TEST_ESP32S2 || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*example_test_esp32s2(?:,[^,\n\r]+)*$/i' - -.if-label-example_test_esp32s3: &if-label-example_test_esp32s3 - if: '$BOT_LABEL_EXAMPLE_TEST_ESP32S3 || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*example_test_esp32s3(?:,[^,\n\r]+)*$/i' - .if-label-host_test: &if-label-host_test if: '$BOT_LABEL_HOST_TEST || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*host_test(?:,[^,\n\r]+)*$/i' @@ -403,9 +293,6 @@ .if-label-submodule: &if-label-submodule if: '$BOT_LABEL_SUBMODULE || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*submodule(?:,[^,\n\r]+)*$/i' -.if-label-target_test: &if-label-target_test - if: '$BOT_LABEL_TARGET_TEST || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*target_test(?:,[^,\n\r]+)*$/i' - .if-label-windows: &if-label-windows if: '$BOT_LABEL_WINDOWS || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*windows(?:,[^,\n\r]+)*$/i' @@ -437,518 +324,6 @@ - <<: *if-dev-push changes: *patterns-downloadable-tools -.rules:build:component_ut: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build - - <<: *if-label-component_ut - - <<: *if-label-component_ut_esp32 - - <<: *if-label-component_ut_esp32c2 - - <<: *if-label-component_ut_esp32c3 - - <<: *if-label-component_ut_esp32c5 - - <<: *if-label-component_ut_esp32c6 - - <<: *if-label-component_ut_esp32h2 - - <<: *if-label-component_ut_esp32p4 - - <<: *if-label-component_ut_esp32s2 - - <<: *if-label-component_ut_esp32s3 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-build_components - - <<: *if-dev-push - changes: *patterns-build_system - - <<: *if-dev-push - changes: *patterns-component_ut - - <<: *if-dev-push - changes: *patterns-downloadable-tools - -.rules:build:component_ut-esp32: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build - - <<: *if-label-component_ut - - <<: *if-label-component_ut_esp32 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-build_components - - <<: *if-dev-push - changes: *patterns-build_system - - <<: *if-dev-push - changes: *patterns-component_ut - - <<: *if-dev-push - changes: *patterns-downloadable-tools - -.rules:build:component_ut-esp32c2: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build - - <<: *if-label-component_ut - - <<: *if-label-component_ut_esp32c2 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-build_components - - <<: *if-dev-push - changes: *patterns-build_system - - <<: *if-dev-push - changes: *patterns-component_ut - - <<: *if-dev-push - changes: *patterns-component_ut-adc - - <<: *if-dev-push - changes: *patterns-component_ut-flash_multi - - <<: *if-dev-push - changes: *patterns-component_ut-i154 - - <<: *if-dev-push - changes: *patterns-component_ut-nvs_encr_hmac - - <<: *if-dev-push - changes: *patterns-component_ut-sdio - - <<: *if-dev-push - changes: *patterns-component_ut-usb - - <<: *if-dev-push - changes: *patterns-component_ut-wifi - - <<: *if-dev-push - changes: *patterns-downloadable-tools - - <<: *if-dev-push - changes: *patterns-target_test-wifi - -.rules:build:component_ut-esp32c3: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build - - <<: *if-label-component_ut - - <<: *if-label-component_ut_esp32c3 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-build_components - - <<: *if-dev-push - changes: *patterns-build_system - - <<: *if-dev-push - changes: *patterns-component_ut - - <<: *if-dev-push - changes: *patterns-component_ut-adc - - <<: *if-dev-push - changes: *patterns-component_ut-flash_multi - - <<: *if-dev-push - changes: *patterns-component_ut-i154 - - <<: *if-dev-push - changes: *patterns-component_ut-nvs_encr_hmac - - <<: *if-dev-push - changes: *patterns-component_ut-sdio - - <<: *if-dev-push - changes: *patterns-component_ut-usb - - <<: *if-dev-push - changes: *patterns-component_ut-wifi - - <<: *if-dev-push - changes: *patterns-downloadable-tools - - <<: *if-dev-push - changes: *patterns-target_test-wifi - -.rules:build:component_ut-esp32c6: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build - - <<: *if-label-component_ut - - <<: *if-label-component_ut_esp32c6 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-build_components - - <<: *if-dev-push - changes: *patterns-build_system - - <<: *if-dev-push - changes: *patterns-component_ut - - <<: *if-dev-push - changes: *patterns-component_ut-adc - - <<: *if-dev-push - changes: *patterns-component_ut-flash_multi - - <<: *if-dev-push - changes: *patterns-component_ut-i154 - - <<: *if-dev-push - changes: *patterns-component_ut-nvs_encr_hmac - - <<: *if-dev-push - changes: *patterns-component_ut-sdio - - <<: *if-dev-push - changes: *patterns-component_ut-usb - - <<: *if-dev-push - changes: *patterns-component_ut-wifi - - <<: *if-dev-push - changes: *patterns-downloadable-tools - - <<: *if-dev-push - changes: *patterns-target_test-adc - - <<: *if-dev-push - changes: *patterns-target_test-ecdsa - - <<: *if-dev-push - changes: *patterns-target_test-wifi - -.rules:build:component_ut-esp32h2: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build - - <<: *if-label-component_ut - - <<: *if-label-component_ut_esp32h2 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-build_components - - <<: *if-dev-push - changes: *patterns-build_system - - <<: *if-dev-push - changes: *patterns-component_ut - - <<: *if-dev-push - changes: *patterns-component_ut-adc - - <<: *if-dev-push - changes: *patterns-component_ut-flash_multi - - <<: *if-dev-push - changes: *patterns-component_ut-i154 - - <<: *if-dev-push - changes: *patterns-component_ut-nvs_encr_hmac - - <<: *if-dev-push - changes: *patterns-component_ut-sdio - - <<: *if-dev-push - changes: *patterns-component_ut-usb - - <<: *if-dev-push - changes: *patterns-component_ut-wifi - - <<: *if-dev-push - changes: *patterns-downloadable-tools - - <<: *if-dev-push - changes: *patterns-target_test-adc - - <<: *if-dev-push - changes: *patterns-target_test-ecdsa - - <<: *if-dev-push - changes: *patterns-target_test-wifi - -.rules:build:component_ut-esp32p4: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build - - <<: *if-label-component_ut - - <<: *if-label-component_ut_esp32p4 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-build_components - - <<: *if-dev-push - changes: *patterns-build_system - - <<: *if-dev-push - changes: *patterns-component_ut - - <<: *if-dev-push - changes: *patterns-component_ut-adc - - <<: *if-dev-push - changes: *patterns-component_ut-flash_multi - - <<: *if-dev-push - changes: *patterns-component_ut-i154 - - <<: *if-dev-push - changes: *patterns-component_ut-nvs_encr_hmac - - <<: *if-dev-push - changes: *patterns-component_ut-sdio - - <<: *if-dev-push - changes: *patterns-component_ut-usb - - <<: *if-dev-push - changes: *patterns-component_ut-wifi - - <<: *if-dev-push - changes: *patterns-downloadable-tools - - <<: *if-dev-push - changes: *patterns-target_test-adc - - <<: *if-dev-push - changes: *patterns-target_test-ecdsa - - <<: *if-dev-push - changes: *patterns-target_test-wifi - -.rules:build:component_ut-esp32s2: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build - - <<: *if-label-component_ut - - <<: *if-label-component_ut_esp32s2 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-build_components - - <<: *if-dev-push - changes: *patterns-build_system - - <<: *if-dev-push - changes: *patterns-component_ut - - <<: *if-dev-push - changes: *patterns-component_ut-adc - - <<: *if-dev-push - changes: *patterns-component_ut-flash_multi - - <<: *if-dev-push - changes: *patterns-component_ut-i154 - - <<: *if-dev-push - changes: *patterns-component_ut-nvs_encr_hmac - - <<: *if-dev-push - changes: *patterns-component_ut-sdio - - <<: *if-dev-push - changes: *patterns-component_ut-usb - - <<: *if-dev-push - changes: *patterns-component_ut-wifi - - <<: *if-dev-push - changes: *patterns-downloadable-tools - - <<: *if-dev-push - changes: *patterns-target_test-adc - - <<: *if-dev-push - changes: *patterns-target_test-ecdsa - - <<: *if-dev-push - changes: *patterns-target_test-wifi - -.rules:build:component_ut-esp32s3: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build - - <<: *if-label-component_ut - - <<: *if-label-component_ut_esp32s3 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-build_components - - <<: *if-dev-push - changes: *patterns-build_system - - <<: *if-dev-push - changes: *patterns-component_ut - - <<: *if-dev-push - changes: *patterns-component_ut-adc - - <<: *if-dev-push - changes: *patterns-component_ut-flash_multi - - <<: *if-dev-push - changes: *patterns-component_ut-i154 - - <<: *if-dev-push - changes: *patterns-component_ut-nvs_encr_hmac - - <<: *if-dev-push - changes: *patterns-component_ut-sdio - - <<: *if-dev-push - changes: *patterns-component_ut-usb - - <<: *if-dev-push - changes: *patterns-component_ut-wifi - - <<: *if-dev-push - changes: *patterns-downloadable-tools - - <<: *if-dev-push - changes: *patterns-target_test-adc - - <<: *if-dev-push - changes: *patterns-target_test-ecdsa - - <<: *if-dev-push - changes: *patterns-target_test-wifi - -.rules:build:custom_test: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build - - <<: *if-label-custom_test - - <<: *if-label-custom_test_esp32 - - <<: *if-label-custom_test_esp32c2 - - <<: *if-label-custom_test_esp32c3 - - <<: *if-label-custom_test_esp32c5 - - <<: *if-label-custom_test_esp32c6 - - <<: *if-label-custom_test_esp32h2 - - <<: *if-label-custom_test_esp32p4 - - <<: *if-label-custom_test_esp32s2 - - <<: *if-label-custom_test_esp32s3 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-build_components - - <<: *if-dev-push - changes: *patterns-build_system - - <<: *if-dev-push - changes: *patterns-custom_test - - <<: *if-dev-push - changes: *patterns-downloadable-tools - -.rules:build:custom_test-esp32: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build - - <<: *if-label-custom_test - - <<: *if-label-custom_test_esp32 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-build_components - - <<: *if-dev-push - changes: *patterns-build_system - - <<: *if-dev-push - changes: *patterns-custom_test - - <<: *if-dev-push - changes: *patterns-downloadable-tools - -.rules:build:custom_test-esp32c2: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build - - <<: *if-label-custom_test - - <<: *if-label-custom_test_esp32c2 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-build_components - - <<: *if-dev-push - changes: *patterns-build_system - - <<: *if-dev-push - changes: *patterns-custom_test - - <<: *if-dev-push - changes: *patterns-downloadable-tools - -.rules:build:custom_test-esp32c3: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build - - <<: *if-label-custom_test - - <<: *if-label-custom_test_esp32c3 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-build_components - - <<: *if-dev-push - changes: *patterns-build_system - - <<: *if-dev-push - changes: *patterns-custom_test - - <<: *if-dev-push - changes: *patterns-downloadable-tools - -.rules:build:custom_test-esp32c6: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build - - <<: *if-label-custom_test - - <<: *if-label-custom_test_esp32c6 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-build_components - - <<: *if-dev-push - changes: *patterns-build_system - - <<: *if-dev-push - changes: *patterns-custom_test - - <<: *if-dev-push - changes: *patterns-custom_test-wifi - - <<: *if-dev-push - changes: *patterns-downloadable-tools - - <<: *if-dev-push - changes: *patterns-target_test-adc - - <<: *if-dev-push - changes: *patterns-target_test-ecdsa - - <<: *if-dev-push - changes: *patterns-target_test-wifi - -.rules:build:custom_test-esp32h2: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build - - <<: *if-label-custom_test - - <<: *if-label-custom_test_esp32h2 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-build_components - - <<: *if-dev-push - changes: *patterns-build_system - - <<: *if-dev-push - changes: *patterns-custom_test - - <<: *if-dev-push - changes: *patterns-custom_test-wifi - - <<: *if-dev-push - changes: *patterns-downloadable-tools - - <<: *if-dev-push - changes: *patterns-target_test-adc - - <<: *if-dev-push - changes: *patterns-target_test-ecdsa - - <<: *if-dev-push - changes: *patterns-target_test-wifi - -.rules:build:custom_test-esp32p4: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build - - <<: *if-label-custom_test - - <<: *if-label-custom_test_esp32p4 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-build_components - - <<: *if-dev-push - changes: *patterns-build_system - - <<: *if-dev-push - changes: *patterns-custom_test - - <<: *if-dev-push - changes: *patterns-custom_test-wifi - - <<: *if-dev-push - changes: *patterns-downloadable-tools - - <<: *if-dev-push - changes: *patterns-target_test-adc - - <<: *if-dev-push - changes: *patterns-target_test-ecdsa - - <<: *if-dev-push - changes: *patterns-target_test-wifi - -.rules:build:custom_test-esp32s2: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build - - <<: *if-label-custom_test - - <<: *if-label-custom_test_esp32s2 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-build_components - - <<: *if-dev-push - changes: *patterns-build_system - - <<: *if-dev-push - changes: *patterns-custom_test - - <<: *if-dev-push - changes: *patterns-custom_test-wifi - - <<: *if-dev-push - changes: *patterns-downloadable-tools - - <<: *if-dev-push - changes: *patterns-target_test-adc - - <<: *if-dev-push - changes: *patterns-target_test-ecdsa - - <<: *if-dev-push - changes: *patterns-target_test-wifi - -.rules:build:custom_test-esp32s3: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build - - <<: *if-label-custom_test - - <<: *if-label-custom_test_esp32s3 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-build_components - - <<: *if-dev-push - changes: *patterns-build_system - - <<: *if-dev-push - changes: *patterns-custom_test - - <<: *if-dev-push - changes: *patterns-custom_test-wifi - - <<: *if-dev-push - changes: *patterns-downloadable-tools - - <<: *if-dev-push - changes: *patterns-target_test-adc - - <<: *if-dev-push - changes: *patterns-target_test-ecdsa - - <<: *if-dev-push - changes: *patterns-target_test-wifi - .rules:build:docker: rules: - <<: *if-revert-branch @@ -965,377 +340,6 @@ - <<: *if-dev-push changes: *patterns-submodule -.rules:build:example_test: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-example_test-ota-include_nightly_run-rule - - <<: *if-label-build - - <<: *if-label-example_test - - <<: *if-label-example_test_esp32 - - <<: *if-label-example_test_esp32c2 - - <<: *if-label-example_test_esp32c3 - - <<: *if-label-example_test_esp32c5 - - <<: *if-label-example_test_esp32c6 - - <<: *if-label-example_test_esp32h2 - - <<: *if-label-example_test_esp32p4 - - <<: *if-label-example_test_esp32s2 - - <<: *if-label-example_test_esp32s3 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-build-example_test - - <<: *if-dev-push - changes: *patterns-build_components - - <<: *if-dev-push - changes: *patterns-build_system - - <<: *if-dev-push - changes: *patterns-downloadable-tools - - <<: *if-dev-push - changes: *patterns-example_test - - <<: *if-dev-push - changes: *patterns-example_test-adc - - <<: *if-dev-push - changes: *patterns-example_test-ethernet - - <<: *if-dev-push - changes: *patterns-example_test-i154 - - <<: *if-dev-push - changes: *patterns-example_test-nvs_encr_hmac - - <<: *if-dev-push - changes: *patterns-example_test-sdio - - <<: *if-dev-push - changes: *patterns-example_test-usb - - <<: *if-dev-push - changes: *patterns-example_test-wifi - - <<: *if-dev-push - changes: *patterns-target_test-adc - - <<: *if-dev-push - changes: *patterns-target_test-ecdsa - - <<: *if-dev-push - changes: *patterns-target_test-wifi - -.rules:build:example_test-esp32: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-example_test-ota-include_nightly_run-rule - - <<: *if-label-build - - <<: *if-label-example_test - - <<: *if-label-example_test_esp32 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-build-example_test - - <<: *if-dev-push - changes: *patterns-build_components - - <<: *if-dev-push - changes: *patterns-build_system - - <<: *if-dev-push - changes: *patterns-downloadable-tools - - <<: *if-dev-push - changes: *patterns-example_test - - <<: *if-dev-push - changes: *patterns-example_test-adc - - <<: *if-dev-push - changes: *patterns-example_test-ethernet - - <<: *if-dev-push - changes: *patterns-example_test-i154 - - <<: *if-dev-push - changes: *patterns-example_test-nvs_encr_hmac - - <<: *if-dev-push - changes: *patterns-example_test-sdio - - <<: *if-dev-push - changes: *patterns-example_test-usb - - <<: *if-dev-push - changes: *patterns-example_test-wifi - - <<: *if-dev-push - changes: *patterns-target_test-adc - - <<: *if-dev-push - changes: *patterns-target_test-ecdsa - - <<: *if-dev-push - changes: *patterns-target_test-wifi - -.rules:build:example_test-esp32c2: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build - - <<: *if-label-example_test - - <<: *if-label-example_test_esp32c2 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-build-example_test - - <<: *if-dev-push - changes: *patterns-build_components - - <<: *if-dev-push - changes: *patterns-build_system - - <<: *if-dev-push - changes: *patterns-downloadable-tools - - <<: *if-dev-push - changes: *patterns-example_test - - <<: *if-dev-push - changes: *patterns-example_test-adc - - <<: *if-dev-push - changes: *patterns-example_test-ethernet - - <<: *if-dev-push - changes: *patterns-example_test-i154 - - <<: *if-dev-push - changes: *patterns-example_test-nvs_encr_hmac - - <<: *if-dev-push - changes: *patterns-example_test-sdio - - <<: *if-dev-push - changes: *patterns-example_test-usb - - <<: *if-dev-push - changes: *patterns-example_test-wifi - - <<: *if-dev-push - changes: *patterns-target_test-adc - - <<: *if-dev-push - changes: *patterns-target_test-ecdsa - - <<: *if-dev-push - changes: *patterns-target_test-wifi - -.rules:build:example_test-esp32c3: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-example_test-ota-include_nightly_run-rule - - <<: *if-label-build - - <<: *if-label-example_test - - <<: *if-label-example_test_esp32c3 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-build-example_test - - <<: *if-dev-push - changes: *patterns-build_components - - <<: *if-dev-push - changes: *patterns-build_system - - <<: *if-dev-push - changes: *patterns-downloadable-tools - - <<: *if-dev-push - changes: *patterns-example_test - - <<: *if-dev-push - changes: *patterns-example_test-adc - - <<: *if-dev-push - changes: *patterns-example_test-ethernet - - <<: *if-dev-push - changes: *patterns-example_test-i154 - - <<: *if-dev-push - changes: *patterns-example_test-nvs_encr_hmac - - <<: *if-dev-push - changes: *patterns-example_test-sdio - - <<: *if-dev-push - changes: *patterns-example_test-usb - - <<: *if-dev-push - changes: *patterns-example_test-wifi - - <<: *if-dev-push - changes: *patterns-target_test-adc - - <<: *if-dev-push - changes: *patterns-target_test-ecdsa - - <<: *if-dev-push - changes: *patterns-target_test-wifi - -.rules:build:example_test-esp32c6: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build - - <<: *if-label-example_test - - <<: *if-label-example_test_esp32c6 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-build-example_test - - <<: *if-dev-push - changes: *patterns-build_components - - <<: *if-dev-push - changes: *patterns-build_system - - <<: *if-dev-push - changes: *patterns-downloadable-tools - - <<: *if-dev-push - changes: *patterns-example_test - - <<: *if-dev-push - changes: *patterns-example_test-adc - - <<: *if-dev-push - changes: *patterns-example_test-ethernet - - <<: *if-dev-push - changes: *patterns-example_test-i154 - - <<: *if-dev-push - changes: *patterns-example_test-nvs_encr_hmac - - <<: *if-dev-push - changes: *patterns-example_test-sdio - - <<: *if-dev-push - changes: *patterns-example_test-usb - - <<: *if-dev-push - changes: *patterns-example_test-wifi - - <<: *if-dev-push - changes: *patterns-target_test-adc - - <<: *if-dev-push - changes: *patterns-target_test-ecdsa - - <<: *if-dev-push - changes: *patterns-target_test-wifi - -.rules:build:example_test-esp32h2: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build - - <<: *if-label-example_test - - <<: *if-label-example_test_esp32h2 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-build-example_test - - <<: *if-dev-push - changes: *patterns-build_components - - <<: *if-dev-push - changes: *patterns-build_system - - <<: *if-dev-push - changes: *patterns-downloadable-tools - - <<: *if-dev-push - changes: *patterns-example_test - - <<: *if-dev-push - changes: *patterns-example_test-adc - - <<: *if-dev-push - changes: *patterns-example_test-ethernet - - <<: *if-dev-push - changes: *patterns-example_test-i154 - - <<: *if-dev-push - changes: *patterns-example_test-nvs_encr_hmac - - <<: *if-dev-push - changes: *patterns-example_test-sdio - - <<: *if-dev-push - changes: *patterns-example_test-usb - - <<: *if-dev-push - changes: *patterns-example_test-wifi - - <<: *if-dev-push - changes: *patterns-target_test-adc - - <<: *if-dev-push - changes: *patterns-target_test-ecdsa - - <<: *if-dev-push - changes: *patterns-target_test-wifi - -.rules:build:example_test-esp32p4: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build - - <<: *if-label-example_test - - <<: *if-label-example_test_esp32p4 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-build-example_test - - <<: *if-dev-push - changes: *patterns-build_components - - <<: *if-dev-push - changes: *patterns-build_system - - <<: *if-dev-push - changes: *patterns-downloadable-tools - - <<: *if-dev-push - changes: *patterns-example_test - - <<: *if-dev-push - changes: *patterns-example_test-adc - - <<: *if-dev-push - changes: *patterns-example_test-ethernet - - <<: *if-dev-push - changes: *patterns-example_test-i154 - - <<: *if-dev-push - changes: *patterns-example_test-nvs_encr_hmac - - <<: *if-dev-push - changes: *patterns-example_test-sdio - - <<: *if-dev-push - changes: *patterns-example_test-usb - - <<: *if-dev-push - changes: *patterns-example_test-wifi - - <<: *if-dev-push - changes: *patterns-target_test-adc - - <<: *if-dev-push - changes: *patterns-target_test-ecdsa - - <<: *if-dev-push - changes: *patterns-target_test-wifi - -.rules:build:example_test-esp32s2: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build - - <<: *if-label-example_test - - <<: *if-label-example_test_esp32s2 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-build-example_test - - <<: *if-dev-push - changes: *patterns-build_components - - <<: *if-dev-push - changes: *patterns-build_system - - <<: *if-dev-push - changes: *patterns-downloadable-tools - - <<: *if-dev-push - changes: *patterns-example_test - - <<: *if-dev-push - changes: *patterns-example_test-adc - - <<: *if-dev-push - changes: *patterns-example_test-ethernet - - <<: *if-dev-push - changes: *patterns-example_test-i154 - - <<: *if-dev-push - changes: *patterns-example_test-nvs_encr_hmac - - <<: *if-dev-push - changes: *patterns-example_test-sdio - - <<: *if-dev-push - changes: *patterns-example_test-usb - - <<: *if-dev-push - changes: *patterns-example_test-wifi - - <<: *if-dev-push - changes: *patterns-target_test-adc - - <<: *if-dev-push - changes: *patterns-target_test-ecdsa - - <<: *if-dev-push - changes: *patterns-target_test-wifi - -.rules:build:example_test-esp32s3: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build - - <<: *if-label-example_test - - <<: *if-label-example_test_esp32s3 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-build-example_test - - <<: *if-dev-push - changes: *patterns-build_components - - <<: *if-dev-push - changes: *patterns-build_system - - <<: *if-dev-push - changes: *patterns-downloadable-tools - - <<: *if-dev-push - changes: *patterns-example_test - - <<: *if-dev-push - changes: *patterns-example_test-adc - - <<: *if-dev-push - changes: *patterns-example_test-ethernet - - <<: *if-dev-push - changes: *patterns-example_test-i154 - - <<: *if-dev-push - changes: *patterns-example_test-nvs_encr_hmac - - <<: *if-dev-push - changes: *patterns-example_test-sdio - - <<: *if-dev-push - changes: *patterns-example_test-usb - - <<: *if-dev-push - changes: *patterns-example_test-wifi - - <<: *if-dev-push - changes: *patterns-target_test-adc - - <<: *if-dev-push - changes: *patterns-target_test-ecdsa - - <<: *if-dev-push - changes: *patterns-target_test-wifi - .rules:build:macos: rules: - <<: *if-revert-branch @@ -1357,53 +361,14 @@ when: never - <<: *if-protected - <<: *if-label-build - - <<: *if-label-component_ut - - <<: *if-label-component_ut_esp32 - - <<: *if-label-component_ut_esp32c2 - - <<: *if-label-component_ut_esp32c3 - - <<: *if-label-component_ut_esp32c5 - - <<: *if-label-component_ut_esp32c6 - - <<: *if-label-component_ut_esp32h2 - - <<: *if-label-component_ut_esp32p4 - - <<: *if-label-component_ut_esp32s2 - - <<: *if-label-component_ut_esp32s3 - - <<: *if-label-custom_test - - <<: *if-label-custom_test_esp32 - - <<: *if-label-custom_test_esp32c2 - - <<: *if-label-custom_test_esp32c3 - - <<: *if-label-custom_test_esp32c5 - - <<: *if-label-custom_test_esp32c6 - - <<: *if-label-custom_test_esp32h2 - - <<: *if-label-custom_test_esp32p4 - - <<: *if-label-custom_test_esp32s2 - - <<: *if-label-custom_test_esp32s3 - - <<: *if-label-example_test - - <<: *if-label-example_test_esp32 - - <<: *if-label-example_test_esp32c2 - - <<: *if-label-example_test_esp32c3 - - <<: *if-label-example_test_esp32c5 - - <<: *if-label-example_test_esp32c6 - - <<: *if-label-example_test_esp32h2 - - <<: *if-label-example_test_esp32p4 - - <<: *if-label-example_test_esp32s2 - - <<: *if-label-example_test_esp32s3 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-build-example_test - <<: *if-dev-push changes: *patterns-build_components - <<: *if-dev-push changes: *patterns-build_system - <<: *if-dev-push changes: *patterns-build_template-app - - <<: *if-dev-push - changes: *patterns-component_ut - - <<: *if-dev-push - changes: *patterns-custom_test - <<: *if-dev-push changes: *patterns-downloadable-tools - - <<: *if-dev-push - changes: *patterns-example_test .rules:labels:nvs_coverage: rules: @@ -1418,334 +383,6 @@ - <<: *if-schedule-test-build-system-windows - <<: *if-label-windows -.rules:test:component_ut-esp32: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-component_ut - - <<: *if-label-component_ut_esp32 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-component_ut - -.rules:test:component_ut-esp32c2: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-component_ut - - <<: *if-label-component_ut_esp32c2 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-component_ut - -.rules:test:component_ut-esp32c3: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-component_ut - - <<: *if-label-component_ut_esp32c3 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-component_ut - -.rules:test:component_ut-esp32c6: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-component_ut - - <<: *if-label-component_ut_esp32c6 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-component_ut - -.rules:test:component_ut-esp32h2: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-component_ut - - <<: *if-label-component_ut_esp32h2 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-component_ut - -.rules:test:component_ut-esp32p4: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-component_ut - - <<: *if-label-component_ut_esp32p4 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-component_ut - -.rules:test:component_ut-esp32s2: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-component_ut - - <<: *if-label-component_ut_esp32s2 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-component_ut - -.rules:test:component_ut-esp32s3: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-component_ut - - <<: *if-label-component_ut_esp32s3 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-component_ut - -.rules:test:custom_test-esp32: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-custom_test - - <<: *if-label-custom_test_esp32 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-custom_test - -.rules:test:custom_test-esp32c2: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-custom_test - - <<: *if-label-custom_test_esp32c2 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-custom_test - -.rules:test:custom_test-esp32c3: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-custom_test - - <<: *if-label-custom_test_esp32c3 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-custom_test - -.rules:test:custom_test-esp32c6: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-custom_test - - <<: *if-label-custom_test_esp32c6 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-custom_test - -.rules:test:custom_test-esp32h2: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-custom_test - - <<: *if-label-custom_test_esp32h2 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-custom_test - -.rules:test:custom_test-esp32p4: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-custom_test - - <<: *if-label-custom_test_esp32p4 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-custom_test - -.rules:test:custom_test-esp32s2: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-custom_test - - <<: *if-label-custom_test_esp32s2 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-custom_test - -.rules:test:custom_test-esp32s3: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-custom_test - - <<: *if-label-custom_test_esp32s3 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-custom_test - -.rules:test:example_test-esp32: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-example_test - - <<: *if-label-example_test_esp32 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-build-example_test - - <<: *if-dev-push - changes: *patterns-example_test - -.rules:test:example_test-esp32c2: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-example_test - - <<: *if-label-example_test_esp32c2 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-build-example_test - - <<: *if-dev-push - changes: *patterns-example_test - -.rules:test:example_test-esp32c3: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-example_test - - <<: *if-label-example_test_esp32c3 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-build-example_test - - <<: *if-dev-push - changes: *patterns-example_test - -.rules:test:example_test-esp32c6: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-example_test - - <<: *if-label-example_test_esp32c6 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-build-example_test - - <<: *if-dev-push - changes: *patterns-example_test - -.rules:test:example_test-esp32h2: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-example_test - - <<: *if-label-example_test_esp32h2 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-build-example_test - - <<: *if-dev-push - changes: *patterns-example_test - -.rules:test:example_test-esp32p4: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-example_test - - <<: *if-label-example_test_esp32p4 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-build-example_test - - <<: *if-dev-push - changes: *patterns-example_test - -.rules:test:example_test-esp32s2: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-example_test - - <<: *if-label-example_test_esp32s2 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-build-example_test - - <<: *if-dev-push - changes: *patterns-example_test - -.rules:test:example_test-esp32s3: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build-only - when: never - - <<: *if-label-example_test - - <<: *if-label-example_test_esp32s3 - - <<: *if-label-target_test - - <<: *if-dev-push - changes: *patterns-build-example_test - - <<: *if-dev-push - changes: *patterns-example_test - .rules:test:host_test: rules: - <<: *if-revert-branch diff --git a/.gitlab/ci/target-test.yml b/.gitlab/ci/target-test.yml deleted file mode 100644 index b945baffb5..0000000000 --- a/.gitlab/ci/target-test.yml +++ /dev/null @@ -1,894 +0,0 @@ -.target_test_template: - image: $TARGET_TEST_ENV_IMAGE - extends: - - .before_script:fetch:target_test - stage: target_test - timeout: 1 hour - dependencies: [] - cache: - # Usually do not need submodule-cache in target_test - - key: pip-cache-${LATEST_GIT_TAG} - paths: - - .cache/pip - policy: pull - after_script: - - python tools/ci/artifacts_handler.py upload --type logs junit_reports - -.pytest_template: - extends: - - .target_test_template - artifacts: - paths: - - XUNIT_RESULT.xml - - pytest_embedded_log/ - reports: - junit: XUNIT_RESULT.xml - script: - - retry_failed git clone $KNOWN_FAILURE_CASES_REPO known_failure_cases - # get runner env config file - - retry_failed git clone $TEST_ENV_CONFIG_REPO - - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs - # using runner tags as markers to filter the test cases - # Runner tags are comma separated, replace the comma with " and " for markers - - job_tags=$(python tools/ci/python_packages/gitlab_api.py get_job_tags $CI_PROJECT_ID --job_id $CI_JOB_ID) - - markers=$(echo $job_tags | sed -e "s/,/ and /g") - - if [ -n "$BUILD_JOB_NAME" ]; then - job_name=$BUILD_JOB_NAME; - else - job_name=${BUILD_JOB_PREFIX}$(python tools/ci/ci_get_mr_info.py target_in_tags $job_tags); - fi - - run_cmd python tools/ci/artifacts_handler.py download --job-name "$job_name" --type build_dir_without_map_and_elf_files - - if [ -n "$REQUIRES_ELF_FILES" ]; then - run_cmd python tools/ci/artifacts_handler.py download --job-name "$job_name" --type map_and_elf_files; - fi - - run_cmd pytest $TEST_DIR - -m \"${markers}\" - --junitxml=XUNIT_RESULT.xml - --ignore-result-files known_failure_cases/known_failure_cases.txt - --parallel-count ${CI_NODE_TOTAL:-1} - --parallel-index ${CI_NODE_INDEX:-1} - ${PYTEST_EXTRA_FLAGS} - --app-info-filepattern \"list_job_*.txt\" - -.pytest_examples_dir_template: - extends: .pytest_template - variables: - TEST_DIR: examples - BUILD_JOB_PREFIX: build_pytest_examples_ - -.pytest_examples_dir_jtag_template: - extends: .pytest_examples_dir_template - needs: - - job: build_pytest_examples_jtag - artifacts: false - variables: - BUILD_JOB_NAME: build_pytest_examples_jtag - REQUIRES_ELF_FILES: "1" - PYTEST_EXTRA_FLAGS: "--log-cli-level DEBUG" - -pytest_examples_esp32_generic: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32 - needs: - - job: build_pytest_examples_esp32 - artifacts: false - tags: [ esp32, generic ] - parallel: 3 - -pytest_examples_esp32_esp32eco3: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32 - needs: - - job: build_pytest_examples_esp32 - artifacts: false - tags: [ esp32, esp32eco3 ] - -pytest_examples_esp32_ir_transceiver: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32 - needs: - - job: build_pytest_examples_esp32 - artifacts: false - tags: [ esp32, ir_transceiver ] - -pytest_examples_esp32_twai_transceiver: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32 - needs: - - job: build_pytest_examples_esp32 - artifacts: false - tags: [ esp32, twai_transceiver ] - -pytest_examples_esp32_twai_network: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32 - needs: - - job: build_pytest_examples_esp32 - artifacts: false - tags: [ esp32, twai_network ] - -pytest_examples_esp32_jtag: - extends: - - .pytest_examples_dir_jtag_template - - .rules:test:example_test-esp32 - tags: [ esp32, jtag ] - -pytest_examples_esp32_ccs811: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32 - needs: - - job: build_pytest_examples_esp32 - artifacts: false - tags: [ esp32, ccs811 ] - - -pytest_examples_esp32s2_generic: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32s2 - needs: - - job: build_pytest_examples_esp32s2 - artifacts: false - tags: [ esp32s2, generic ] - parallel: 3 - -pytest_examples_esp32s2_jtag: - extends: - - .pytest_examples_dir_jtag_template - - .rules:test:example_test-esp32s2 - tags: [ esp32s2, jtag ] - -pytest_examples_esp32s3_generic: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32s3 - needs: - - job: build_pytest_examples_esp32s3 - artifacts: false - tags: [ esp32s3, generic ] - parallel: 3 - -pytest_examples_esp32s3_usb_serial_jtag: - extends: - - .pytest_examples_dir_jtag_template - - .rules:test:example_test-esp32s3 - tags: [ esp32s3, usb_serial_jtag ] - -pytest_examples_esp32s3_f4r8: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32s3 - needs: - - job: build_pytest_examples_esp32s3 - artifacts: false - tags: [ esp32s3, MSPI_F4R8 ] - -pytest_examples_esp32c2_generic: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32c2 - needs: - - job: build_pytest_examples_esp32c2 - artifacts: false - tags: [ esp32c2, generic, xtal_40mhz ] - parallel: 3 - -pytest_examples_esp32c2_jtag: - extends: - - .pytest_examples_dir_jtag_template - - .rules:test:example_test-esp32c2 - tags: [ esp32c2, jtag ] - -pytest_examples_esp32c3_generic: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32c3 - needs: - - job: build_pytest_examples_esp32c3 - artifacts: false - tags: [ esp32c3, generic ] - parallel: 3 - -pytest_examples_esp32c3_usb_serial_jtag: - extends: - - .pytest_examples_dir_jtag_template - - .rules:test:example_test-esp32c3 - tags: [ esp32c3, usb_serial_jtag ] - -pytest_examples_esp32c3_flash_suspend: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32c3 - needs: - - job: build_pytest_examples_esp32c3 - artifacts: false - tags: [ esp32c3, flash_suspend ] - -pytest_examples_esp32c6_generic: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32c6 - needs: - - job: build_pytest_examples_esp32c6 - artifacts: false - tags: [ esp32c6, generic ] - -pytest_examples_esp32c6_usj_device: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32c6 - needs: - - job: build_pytest_examples_esp32c6 - artifacts: false - tags: [ esp32c6, usj_device ] - -pytest_examples_esp32h2_generic: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32h2 - needs: - - job: build_pytest_examples_esp32h2 - artifacts: false - tags: [ esp32h2, generic ] - -pytest_examples_esp32p4_generic: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32p4 - needs: - - job: build_pytest_examples_esp32p4 - artifacts: false - tags: [ esp32p4, generic ] - -pytest_examples_esp32_8mb_flash: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32 - needs: - - job: build_pytest_examples_esp32 - artifacts: false - tags: [ esp32, ethernet_flash_8m ] - -pytest_examples_esp32_ethernet_bridge: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32 - needs: - - job: build_pytest_examples_esp32 - artifacts: false - tags: [ esp32, eth_w5500 ] - variables: - PYTEST_EXTRA_FLAGS: "--dev-passwd ${ETHERNET_TEST_PASSWORD} --dev-user ${ETHERNET_TEST_USER}" - -pytest_examples_esp32_flash_encryption: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32 - needs: - - job: build_pytest_examples_esp32 - artifacts: false - tags: [ esp32, flash_encryption ] - -pytest_examples_esp32c3_flash_encryption: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32c3 - needs: - - job: build_pytest_examples_esp32c3 - artifacts: false - tags: [ esp32c3, flash_encryption ] - -pytest_examples_esp32_sdmmc: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32 - needs: - - job: build_pytest_examples_esp32 - artifacts: false - tags: [ esp32, sdcard_sdmode ] - -pytest_examples_esp32_extflash: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32 - needs: - - job: build_pytest_examples_esp32 - artifacts: false - tags: [ esp32, external_flash ] - -pytest_examples_esp32s3_emmc: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32s3 - needs: - - job: build_pytest_examples_esp32s3 - artifacts: false - tags: [ esp32s3, emmc ] - -.pytest_components_dir_template: - extends: .pytest_template - variables: - TEST_DIR: components - BUILD_JOB_PREFIX: build_pytest_components_ - -pytest_components_esp32_generic: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32 - needs: - - job: build_pytest_components_esp32 - artifacts: false - tags: [ esp32, generic ] - parallel: 7 - -pytest_components_esp32_generic_multi_device: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32 - needs: - - job: build_pytest_components_esp32 - artifacts: false - tags: [ esp32, generic_multi_device ] - -pytest_components_esp32_wifi_two_dut: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32 - needs: - - job: build_pytest_components_esp32 - artifacts: false - tags: [ esp32, wifi_two_dut ] - -pytest_components_esp32_sdmmc: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32 - needs: - - job: build_pytest_components_esp32 - artifacts: false - tags: [ esp32, sdcard ] - -pytest_components_esp32s3_sdmmc: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32s3 - needs: - - job: build_pytest_components_esp32s3 - artifacts: false - tags: [ esp32s3, sdcard ] - -pytest_components_esp32_lan8720: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32 - needs: - - job: build_pytest_components_esp32 - artifacts: false - tags: [ esp32, eth_lan8720 ] - -pytest_components_esp32_rtl8201: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32 - needs: - - job: build_pytest_components_esp32 - artifacts: false - tags: [ esp32, eth_rtl8201 ] - -pytest_components_esp32_w5500: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32 - needs: - - job: build_pytest_components_esp32 - artifacts: false - tags: [ esp32, eth_w5500 ] - -pytest_components_esp32_ksz8851snl: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32 - needs: - - job: build_pytest_components_esp32 - artifacts: false - tags: [ esp32, eth_ksz8851snl ] - -pytest_components_esp32_dm9051: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32 - needs: - - job: build_pytest_components_esp32 - artifacts: false - tags: [ esp32, eth_dm9051 ] - -pytest_components_esp32_ksz8041: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32 - needs: - - job: build_pytest_components_esp32 - artifacts: false - tags: [ esp32, eth_ksz8041 ] - -pytest_components_esp32_dp83848: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32 - needs: - - job: build_pytest_components_esp32 - artifacts: false - tags: [ esp32, eth_dp83848 ] - -pytest_components_esp32_ethernet: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32 - needs: - - job: build_pytest_components_esp32 - artifacts: false - tags: [ esp32, ethernet ] - -pytest_components_esp32_flash_encryption: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32 - needs: - - job: build_pytest_components_esp32 - artifacts: false - tags: [ esp32, flash_encryption ] - -pytest_components_esp32_xtal32k: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32 - needs: - - job: build_pytest_components_esp32 - artifacts: false - tags: [ esp32, xtal32k ] - -pytest_components_esp32_no32kXtal: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32 - needs: - - job: build_pytest_components_esp32 - artifacts: false - tags: [ esp32, no32kXtal ] - -pytest_components_esp32_rs485_multi: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32 - needs: - - job: build_pytest_components_esp32 - artifacts: false - tags: [ esp32, multi_dut_modbus_rs485 ] - -pytest_components_esp32_psramv0: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32 - needs: - - job: build_pytest_components_esp32 - artifacts: false - tags: [ esp32, psramv0 ] - -pytest_components_esp32s2_generic: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32s2 - needs: - - job: build_pytest_components_esp32s2 - artifacts: false - tags: [ esp32s2, generic ] - parallel: 5 - -pytest_components_esp32s2_generic_multi_device: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32s2 - needs: - - job: build_pytest_components_esp32s2 - artifacts: false - tags: [ esp32s2, generic_multi_device ] - -pytest_components_esp32s3_generic: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32s3 - needs: - - job: build_pytest_components_esp32s3 - artifacts: false - tags: [ esp32s3, generic ] - parallel: 5 - -pytest_components_esp32s3_generic_multi_device: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32s3 - needs: - - job: build_pytest_components_esp32s3 - artifacts: false - tags: [ esp32s3, generic_multi_device ] - -pytest_components_esp32s3_octal_psram: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32s3 - needs: - - job: build_pytest_components_esp32s3 - artifacts: false - tags: [ esp32s3, octal_psram ] - -pytest_components_esp32s3_quad_psram: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32s3 - needs: - - job: build_pytest_components_esp32s3 - artifacts: false - tags: [ esp32s3, quad_psram ] - -pytest_components_esp32s3_flash_encryption_f4r8: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32s3 - needs: - - job: build_pytest_components_esp32s3 - artifacts: false - tags: [ esp32s3, flash_encryption_f4r8 ] - -pytest_components_esp32s3_flash_encryption_f8r8: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32s3 - needs: - - job: build_pytest_components_esp32s3 - artifacts: false - tags: [ esp32s3, flash_encryption_f8r8 ] - -pytest_components_esp32s3_mspi_f4r4: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32s3 - needs: - - job: build_pytest_components_esp32s3 - artifacts: false - tags: [ esp32s3, MSPI_F4R4 ] - -pytest_components_esp32s3_mspi_f4r8: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32s3 - needs: - - job: build_pytest_components_esp32s3 - artifacts: false - tags: [ esp32s3, MSPI_F4R8 ] - -pytest_components_esp32s3_mspi_f8r8: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32s3 - needs: - - job: build_pytest_components_esp32s3 - artifacts: false - tags: [ esp32s3, MSPI_F8R8 ] - -pytest_components_esp32s3_usb_serial_jtag: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32s3 - needs: - - build_pytest_components_esp32s3 - tags: [ esp32s3, usb_serial_jtag ] - -pytest_components_esp32c2_generic: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32c2 - needs: - - job: build_pytest_components_esp32c2 - artifacts: false - tags: [ esp32c2, generic, xtal_40mhz ] - parallel: 3 - -pytest_components_esp32c2_generic_multi_device: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32c2 - needs: - - job: build_pytest_components_esp32c2 - artifacts: false - tags: [ esp32c2, generic_multi_device, xtal_40mhz ] - -pytest_components_esp32c2_xtal_26mhz: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32c2 - needs: - - job: build_pytest_components_esp32c2 - artifacts: false - tags: [ esp32c2, generic, xtal_26mhz ] - -pytest_components_esp32c3_generic: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32c3 - needs: - - job: build_pytest_components_esp32c3 - artifacts: false - tags: [ esp32c3, generic ] - parallel: 3 - -pytest_components_esp32c3_i2c_oled: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32c3 - needs: - - job: build_pytest_components_esp32c3 - artifacts: false - tags: [ esp32c3, i2c_oled ] - -pytest_components_esp32c3_generic_multi_device: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32c3 - needs: - - job: build_pytest_components_esp32c3 - artifacts: false - tags: [ esp32c3, generic_multi_device ] - -pytest_components_esp32c3_usb_serial_jtag: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32c3 - needs: - - build_pytest_components_esp32c3 - tags: [ esp32c3, usb_serial_jtag ] - -pytest_components_esp32c3_flash_encryption: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32c3 - needs: - - job: build_pytest_components_esp32c3 - artifacts: false - tags: [ esp32c3, flash_encryption ] - - -pytest_components_esp32c6_generic: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32c6 - needs: - - job: build_pytest_components_esp32c6 - artifacts: false - tags: [ esp32c6, generic ] - parallel: 2 - -pytest_components_esp32c6_usb_serial_jtag: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32c6 - needs: - - build_pytest_components_esp32c6 - tags: [ esp32c6, usb_serial_jtag ] - -pytest_components_esp32h2_generic: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32h2 - needs: - - job: build_pytest_components_esp32h2 - artifacts: false - tags: [ esp32h2, generic ] - parallel: 2 - -pytest_components_esp32p4_generic: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32p4 - needs: - - job: build_pytest_components_esp32p4 - artifacts: false - tags: [ esp32p4, generic ] - parallel: 2 - -pytest_components_esp32h2_generic_multi_device: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32h2 - needs: - - job: build_pytest_components_esp32h2 - artifacts: false - tags: [ esp32h2, generic_multi_device ] - -pytest_components_esp32h2_usb_serial_jtag: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32h2 - needs: - - build_pytest_components_esp32h2 - tags: [ esp32h2, usb_serial_jtag ] - -pytest_components_esp32c6_generic_multi_device: - extends: - - .pytest_components_dir_template - - .rules:test:component_ut-esp32c6 - needs: - - job: build_pytest_components_esp32c6 - artifacts: false - tags: [ esp32c6, generic_multi_device ] - -pytest_examples_openthread_sleep: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32h2 - needs: - - job: build_pytest_examples_esp32c6 - artifacts: false - - job: build_pytest_examples_esp32h2 - artifacts: false - tags: [ esp32c6, openthread_sleep ] - script: - - retry_failed git clone $KNOWN_FAILURE_CASES_REPO known_failure_cases - # get runner env config file - - retry_failed git clone $TEST_ENV_CONFIG_REPO - - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs - # using runner tags as markers to filter the test cases - # Runner tags are comma separated, replace the comma with " and " for markers - - job_tags=$(python tools/ci/python_packages/gitlab_api.py get_job_tags $CI_PROJECT_ID --job_id $CI_JOB_ID) - - markers=$(echo $job_tags | sed -e "s/,/ and /g") - # download the artifacts, requires c6, h2 chips - - run_cmd python tools/ci/artifacts_handler.py download --job-name "build_pytest_examples_esp32c6" - - run_cmd python tools/ci/artifacts_handler.py download --job-name "build_pytest_examples_esp32h2" - - run_cmd pytest $TEST_DIR - -m \"${markers}\" - --junitxml=XUNIT_RESULT.xml - --ignore-result-files known_failure_cases/known_failure_cases.txt - --parallel-count ${CI_NODE_TOTAL:-1} - --parallel-index ${CI_NODE_INDEX:-1} - ${PYTEST_EXTRA_FLAGS} - --app-info-filepattern \"list_job_*.txt\" - -pytest_examples_esp32h2_zigbee: - extends: - - .pytest_examples_dir_template - - .rules:test:example_test-esp32h2 - needs: - - job: build_pytest_examples_esp32h2 - artifacts: false - tags: [ esp32h2, zigbee_multi_dut ] - -.pytest_test_apps_dir_template: - extends: .pytest_template - variables: - TEST_DIR: tools/test_apps - BUILD_JOB_PREFIX: build_pytest_test_apps_ - REQUIRES_ELF_FILES: "1" - -pytest_test_apps_esp32_generic: - extends: - - .pytest_test_apps_dir_template - - .rules:test:custom_test-esp32 - needs: - - job: build_pytest_test_apps_esp32 - artifacts: false - tags: [ esp32, generic ] - -pytest_test_apps_esp32_jtag: - extends: - - .pytest_test_apps_dir_template - - .rules:test:custom_test-esp32 - needs: - - job: build_pytest_test_apps_esp32 - artifacts: false - tags: [ esp32, jtag ] - variables: - PYTEST_EXTRA_FLAGS: "--log-cli-level DEBUG" - -pytest_test_apps_esp32_ethernet: - extends: - - .pytest_test_apps_dir_template - - .rules:test:custom_test-esp32 - needs: - - job: build_pytest_test_apps_esp32 - artifacts: false - tags: [ esp32, ethernet ] - -pytest_test_apps_esp32s2_generic: - extends: - - .pytest_test_apps_dir_template - - .rules:test:custom_test-esp32s2 - needs: - - job: build_pytest_test_apps_esp32s2 - artifacts: false - tags: [ esp32s2, generic ] - -pytest_test_apps_esp32s3_generic: - extends: - - .pytest_test_apps_dir_template - - .rules:test:custom_test-esp32s3 - needs: - - job: build_pytest_test_apps_esp32s3 - artifacts: false - tags: [ esp32s3, generic ] - -pytest_test_apps_esp32c2_generic: - extends: - - .pytest_test_apps_dir_template - - .rules:test:custom_test-esp32c2 - needs: - - job: build_pytest_test_apps_esp32c2 - artifacts: false - tags: [ esp32c2, generic, xtal_40mhz ] - -pytest_test_apps_esp32c3_generic: - extends: - - .pytest_test_apps_dir_template - - .rules:test:custom_test-esp32c3 - needs: - - job: build_pytest_test_apps_esp32c3 - artifacts: false - tags: [ esp32c3, generic ] - -pytest_test_apps_esp32c6_generic: - extends: - - .pytest_test_apps_dir_template - - .rules:test:custom_test-esp32c6 - needs: - - job: build_pytest_test_apps_esp32c6 - artifacts: false - tags: [ esp32c6, generic ] - -pytest_test_apps_esp32h2_generic: - extends: - - .pytest_test_apps_dir_template - - .rules:test:custom_test-esp32h2 - needs: - - job: build_pytest_test_apps_esp32h2 - artifacts: false - tags: [ esp32h2, generic ] - -pytest_test_apps_esp32p4_generic: - extends: - - .pytest_test_apps_dir_template - - .rules:test:custom_test-esp32p4 - needs: - - job: build_pytest_test_apps_esp32p4 - artifacts: false - tags: [ esp32p4, generic ] - -pytest_test_apps_esp32s3_mspi_f8r8: - extends: - - .pytest_test_apps_dir_template - - .rules:test:custom_test-esp32s3 - needs: - - job: build_pytest_test_apps_esp32s3 - artifacts: false - tags: [ esp32s3, MSPI_F8R8 ] - -pytest_test_apps_esp32s3_mspi_f4r8: - extends: - - .pytest_test_apps_dir_template - - .rules:test:custom_test-esp32s3 - needs: - - job: build_pytest_test_apps_esp32s3 - artifacts: false - tags: [ esp32s3, MSPI_F4R8 ] - -pytest_test_apps_esp32s3_mspi_f4r4: - extends: - - .pytest_test_apps_dir_template - - .rules:test:custom_test-esp32s3 - needs: - - job: build_pytest_test_apps_esp32s3 - artifacts: false - tags: [ esp32s3, MSPI_F4R4 ] diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8ae2d5b14d..6b19ed496f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -95,7 +95,7 @@ repos: name: Check rules are generated (based on .gitlab/ci/dependencies/dependencies.yml) entry: tools/ci/generate_rules.py language: python - files: '\.gitlab/ci/dependencies/.+|\.gitlab/ci/.*\.yml' + files: '\.gitlab/ci/dependencies/.+|\.gitlab/ci/.*\.yml|.gitlab-ci.yml' pass_filenames: false require_serial: true additional_dependencies: @@ -108,6 +108,7 @@ repos: - 'mypy-extensions==0.4.3' - 'types-setuptools==57.4.14' - 'types-PyYAML==0.1.9' + - 'types-requests' exclude: > (?x)^( .*_pb2.py @@ -154,7 +155,7 @@ repos: require_serial: true additional_dependencies: - PyYAML == 5.3.1 - - idf_build_apps~=1.0 + - idf-build-apps~=2.0.0rc1 - id: sort-build-test-rules-ymls name: sort .build-test-rules.yml files entry: tools/ci/check_build_test_rules.py sort-yaml diff --git a/components/spi_flash/test_apps/esp_flash_stress/README.md b/components/spi_flash/test_apps/esp_flash_stress/README.md index bf47d80ec6..a8b7833fa3 100644 --- a/components/spi_flash/test_apps/esp_flash_stress/README.md +++ b/components/spi_flash/test_apps/esp_flash_stress/README.md @@ -1,2 +1,2 @@ -| Supported Targets | ESP32 | ESP32-C2 | ESP32-C3 | ESP32-C6 | ESP32-H2 | ESP32-P4 | ESP32-S2 | ESP32-S3 | -| ----------------- | ----- | -------- | -------- | -------- | -------- | -------- | -------- | -------- | +| Supported Targets | ESP32 | ESP32-C2 | ESP32-C3 | ESP32-C6 | ESP32-H2 | ESP32-S2 | ESP32-S3 | +| ----------------- | ----- | -------- | -------- | -------- | -------- | -------- | -------- | diff --git a/conftest.py b/conftest.py index ed5bcaec0c..e9048d461f 100644 --- a/conftest.py +++ b/conftest.py @@ -1,53 +1,54 @@ -# SPDX-FileCopyrightText: 2021-2023 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2021-2024 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 - # pylint: disable=W0621 # redefined-outer-name -# This file is a pytest root configuration file and provide the following functionalities: -# 1. Defines a few fixtures that could be used under the whole project. -# 2. Defines a few hook functions. -# # IDF is using [pytest](https://github.com/pytest-dev/pytest) and -# [pytest-embedded plugin](https://github.com/espressif/pytest-embedded) as its example test framework. -# -# This is an experimental feature, and if you found any bug or have any question, please report to -# https://github.com/espressif/pytest-embedded/issues +# [pytest-embedded plugin](https://github.com/espressif/pytest-embedded) as its test framework. + +# if you found any bug or have any question, +# please report to https://github.com/espressif/pytest-embedded/issues +# or discuss at https://github.com/espressif/pytest-embedded/discussions + +import os +import sys + +import gitlab + +if os.path.join(os.path.dirname(__file__), 'tools', 'ci') not in sys.path: + sys.path.append(os.path.join(os.path.dirname(__file__), 'tools', 'ci')) + +if os.path.join(os.path.dirname(__file__), 'tools', 'ci', 'python_packages') not in sys.path: + sys.path.append(os.path.join(os.path.dirname(__file__), 'tools', 'ci', 'python_packages')) import glob -import json +import io import logging import os import re -import sys import typing as t +import zipfile from copy import deepcopy from datetime import datetime +import common_test_methods # noqa: F401 +import gitlab_api import pytest +import requests +import yaml from _pytest.config import Config from _pytest.fixtures import FixtureRequest +from artifacts_handler import ArtifactType +from dynamic_pipelines.constants import TEST_RELATED_APPS_DOWNLOAD_URLS_FILENAME +from idf_ci.app import import_apps_from_txt +from idf_ci.uploader import AppUploader +from idf_ci_utils import IDF_PATH +from idf_pytest.constants import DEFAULT_SDKCONFIG, ENV_MARKERS, SPECIAL_MARKERS, TARGET_MARKERS, PytestCase +from idf_pytest.plugin import IDF_PYTEST_EMBEDDED_KEY, ITEM_PYTEST_CASE_KEY, IdfPytestEmbedded +from idf_pytest.utils import format_case_id from pytest_embedded.plugin import multi_dut_argument, multi_dut_fixture from pytest_embedded_idf.dut import IdfDut from pytest_embedded_idf.unity_tester import CaseTester -try: - from idf_ci_utils import IDF_PATH - from idf_pytest.constants import DEFAULT_SDKCONFIG, ENV_MARKERS, SPECIAL_MARKERS, TARGET_MARKERS - from idf_pytest.plugin import IDF_PYTEST_EMBEDDED_KEY, IdfPytestEmbedded - from idf_pytest.utils import format_case_id -except ImportError: - sys.path.append(os.path.join(os.path.dirname(__file__), 'tools', 'ci')) - from idf_ci_utils import IDF_PATH - from idf_pytest.constants import DEFAULT_SDKCONFIG, ENV_MARKERS, SPECIAL_MARKERS, TARGET_MARKERS - from idf_pytest.plugin import IDF_PYTEST_EMBEDDED_KEY, IdfPytestEmbedded - from idf_pytest.utils import format_case_id - -try: - import common_test_methods # noqa: F401 -except ImportError: - sys.path.append(os.path.join(os.path.dirname(__file__), 'tools', 'ci', 'python_packages')) - import common_test_methods # noqa: F401 - ############ # Fixtures # @@ -100,9 +101,91 @@ def test_case_name(request: FixtureRequest, target: str, config: str) -> str: return format_case_id(target, config, request.node.originalname, is_qemu=is_qemu, params=filtered_params) # type: ignore +@pytest.fixture(scope='session') +def pipeline_id(request: FixtureRequest) -> t.Optional[str]: + return request.config.getoption('pipeline_id', None) or os.getenv('PARENT_PIPELINE_ID', None) # type: ignore + + +class BuildReportDownloader: + def __init__(self, presigned_url_yaml: str) -> None: + self.app_presigned_urls_dict: t.Dict[str, t.Dict[str, str]] = yaml.safe_load(presigned_url_yaml) + + def download_app( + self, app_build_path: str, artifact_type: ArtifactType = ArtifactType.BUILD_DIR_WITHOUT_MAP_AND_ELF_FILES + ) -> None: + url = self.app_presigned_urls_dict[app_build_path][artifact_type.value] + + logging.debug('Downloading app from %s', url) + with io.BytesIO() as f: + for chunk in requests.get(url).iter_content(chunk_size=1024 * 1024): + if chunk: + f.write(chunk) + + f.seek(0) + + with zipfile.ZipFile(f) as zip_ref: + zip_ref.extractall() + + +@pytest.fixture(scope='session') +def app_downloader(pipeline_id: t.Optional[str]) -> t.Union[AppUploader, BuildReportDownloader, None]: + if not pipeline_id: + return None + + if ( + 'IDF_S3_BUCKET' in os.environ + and 'IDF_S3_ACCESS_KEY' in os.environ + and 'IDF_S3_SECRET_KEY' in os.environ + and 'IDF_S3_SERVER' in os.environ + and 'IDF_S3_BUCKET' in os.environ + ): + return AppUploader(pipeline_id) + + logging.info('Downloading build report from the build pipeline %s', pipeline_id) + test_app_presigned_urls_file = None + try: + gl = gitlab_api.Gitlab(os.getenv('CI_PROJECT_ID', 'espressif/esp-idf')) + except gitlab.exceptions.GitlabAuthenticationError: + msg = """To download artifacts from gitlab, please create ~/.python-gitlab.cfg with the following content: + +[global] +default = internal +ssl_verify = true +timeout = 5 + +[internal] +url = +private_token = +api_version = 4 +""" + raise SystemExit(msg) + + for child_pipeline in gl.project.pipelines.get(pipeline_id, lazy=True).bridges.list(iterator=True): + if child_pipeline.name == 'build_child_pipeline': + for job in gl.project.pipelines.get(child_pipeline.downstream_pipeline['id'], lazy=True).jobs.list( + iterator=True + ): + if job.name == 'generate_pytest_build_report': + test_app_presigned_urls_file = gl.download_artifact( + job.id, [TEST_RELATED_APPS_DOWNLOAD_URLS_FILENAME] + )[0] + break + + if test_app_presigned_urls_file: + return BuildReportDownloader(test_app_presigned_urls_file) + + return None + + @pytest.fixture @multi_dut_fixture -def build_dir(app_path: str, target: t.Optional[str], config: t.Optional[str]) -> str: +def build_dir( + request: FixtureRequest, + app_path: str, + target: t.Optional[str], + config: t.Optional[str], + app_downloader: t.Optional[AppUploader], +) -> str: """ Check local build dir with the following priority: @@ -114,14 +197,25 @@ def build_dir(app_path: str, target: t.Optional[str], config: t.Optional[str]) - Returns: valid build directory """ - check_dirs = [] - if target is not None and config is not None: - check_dirs.append(f'build_{target}_{config}') - if target is not None: - check_dirs.append(f'build_{target}') - if config is not None: - check_dirs.append(f'build_{config}') - check_dirs.append('build') + # download from minio on CI + case: PytestCase = request._pyfuncitem.stash[ITEM_PYTEST_CASE_KEY] + if app_downloader: + # somehow hardcoded... + app_build_path = os.path.join(os.path.relpath(app_path, IDF_PATH), f'build_{target}_{config}') + if case.requires_elf_or_map: + app_downloader.download_app(app_build_path) + else: + app_downloader.download_app(app_build_path, ArtifactType.BUILD_DIR_WITHOUT_MAP_AND_ELF_FILES) + check_dirs = [f'build_{target}_{config}'] + else: + check_dirs = [] + if target is not None and config is not None: + check_dirs.append(f'build_{target}_{config}') + if target is not None: + check_dirs.append(f'build_{target}') + if config is not None: + check_dirs.append(f'build_{config}') + check_dirs.append('build') for check_dir in check_dirs: binary_path = os.path.join(app_path, check_dir) @@ -145,6 +239,13 @@ def junit_properties(test_case_name: str, record_xml_attribute: t.Callable[[str, record_xml_attribute('name', test_case_name) +@pytest.fixture(autouse=True) +@multi_dut_fixture +def ci_job_url(record_xml_attribute: t.Callable[[str, object], None]) -> None: + if ci_job_url := os.getenv('CI_JOB_URL'): + record_xml_attribute('ci_job_url', ci_job_url) + + @pytest.fixture(autouse=True) def set_test_case_name(request: FixtureRequest, test_case_name: str) -> None: request.node.funcargs['test_case_name'] = test_case_name @@ -247,12 +348,12 @@ def log_minimum_free_heap_size(dut: IdfDut, config: str) -> t.Callable[..., None return real_func -@pytest.fixture +@pytest.fixture(scope='session') def dev_password(request: FixtureRequest) -> str: return request.config.getoption('dev_passwd') or '' -@pytest.fixture +@pytest.fixture(scope='session') def dev_user(request: FixtureRequest) -> str: return request.config.getoption('dev_user') or '' @@ -274,18 +375,17 @@ def pytest_addoption(parser: pytest.Parser) -> None: '--dev-passwd', help='password associated with some specific device/service used during the test execution', ) - idf_group.addoption( - '--app-info-basedir', - default=IDF_PATH, - help='app info base directory. specify this value when you\'re building under a ' - 'different IDF_PATH. (Default: $IDF_PATH)', - ) idf_group.addoption( '--app-info-filepattern', help='glob pattern to specify the files that include built app info generated by ' '`idf-build-apps --collect-app-info ...`. will not raise ValueError when binary ' 'paths not exist in local file system if not listed recorded in the app info.', ) + idf_group.addoption( + '--pipeline-id', + help='main pipeline id, not the child pipeline id. Specify this option to download the artifacts ' + 'from the minio server for debugging purpose.', + ) def pytest_configure(config: Config) -> None: @@ -325,32 +425,16 @@ def pytest_configure(config: Config) -> None: """ ) - apps_list = None - app_info_basedir = config.getoption('app_info_basedir') + apps = None app_info_filepattern = config.getoption('app_info_filepattern') if app_info_filepattern: - apps_list = [] - for file in glob.glob(os.path.join(IDF_PATH, app_info_filepattern)): - with open(file) as fr: - for line in fr.readlines(): - if not line.strip(): - continue - - # each line is a valid json - app_info = json.loads(line.strip()) - if app_info_basedir and app_info['app_dir'].startswith(app_info_basedir): - relative_app_dir = os.path.relpath(app_info['app_dir'], app_info_basedir) - apps_list.append(os.path.join(IDF_PATH, os.path.join(relative_app_dir, app_info['build_dir']))) - print('Detected app: ', apps_list[-1]) - else: - print( - f'WARNING: app_info base dir {app_info_basedir} not recognizable in {app_info["app_dir"]}, skipping...' - ) - continue + apps = [] + for f in glob.glob(os.path.join(IDF_PATH, app_info_filepattern)): + apps.extend(import_apps_from_txt(f)) config.stash[IDF_PYTEST_EMBEDDED_KEY] = IdfPytestEmbedded( target=target, - apps_list=apps_list, + apps=apps, ) config.pluginmanager.register(config.stash[IDF_PYTEST_EMBEDDED_KEY]) diff --git a/tools/ci/artifacts_handler.py b/tools/ci/artifacts_handler.py index 73930a3a05..c51a69f300 100644 --- a/tools/ci/artifacts_handler.py +++ b/tools/ci/artifacts_handler.py @@ -12,6 +12,7 @@ from pathlib import Path from zipfile import ZipFile import urllib3 +from idf_pytest.constants import DEFAULT_BUILD_LOG_FILENAME from minio import Minio @@ -33,7 +34,7 @@ TYPE_PATTERNS_DICT = { '**/build*/*.elf', ], ArtifactType.BUILD_DIR_WITHOUT_MAP_AND_ELF_FILES: [ - '**/build*/build_log.txt', + f'**/build*/{DEFAULT_BUILD_LOG_FILENAME}', '**/build*/*.bin', '**/build*/bootloader/*.bin', '**/build*/partition_table/*.bin', @@ -41,17 +42,17 @@ TYPE_PATTERNS_DICT = { '**/build*/flash_project_args', '**/build*/config/sdkconfig.json', '**/build*/project_description.json', - 'list_job_*.txt', + 'list_job*.txt', ], ArtifactType.LOGS: [ - '**/build*/build_log.txt', + f'**/build*/{DEFAULT_BUILD_LOG_FILENAME}', ], ArtifactType.SIZE_REPORTS: [ '**/build*/size.json', 'size_info.txt', ], ArtifactType.JUNIT_REPORTS: [ - 'XUNIT_RESULT.xml', + 'XUNIT_RESULT*.xml', ], ArtifactType.MODIFIED_FILES_AND_COMPONENTS_REPORT: [ 'pipeline.env', @@ -66,6 +67,23 @@ def getenv(env_var: str) -> str: raise Exception(f'Environment variable {env_var} not set') from e +def get_minio_client() -> Minio: + return Minio( + getenv('IDF_S3_SERVER').replace('https://', ''), + access_key=getenv('IDF_S3_ACCESS_KEY'), + secret_key=getenv('IDF_S3_SECRET_KEY'), + http_client=urllib3.PoolManager( + num_pools=10, + timeout=urllib3.Timeout.DEFAULT_TIMEOUT, + retries=urllib3.Retry( + total=5, + backoff_factor=0.2, + status_forcelist=[500, 502, 503, 504], + ), + ), + ) + + def _download_files( pipeline_id: int, *, @@ -131,7 +149,7 @@ def _upload_files( try: if has_file: - obj_name = f'{pipeline_id}/{artifact_type.value}/{job_name.split(" ")[0]}/{job_id}.zip' + obj_name = f'{pipeline_id}/{artifact_type.value}/{job_name.rsplit(" ", maxsplit=1)[0]}/{job_id}.zip' print(f'Created archive file: {job_id}.zip, uploading as {obj_name}') client.fput_object(getenv('IDF_S3_BUCKET'), obj_name, f'{job_id}.zip') @@ -168,19 +186,7 @@ if __name__ == '__main__': args = parser.parse_args() - client = Minio( - getenv('IDF_S3_SERVER').replace('https://', ''), - access_key=getenv('IDF_S3_ACCESS_KEY'), - secret_key=getenv('IDF_S3_SECRET_KEY'), - http_client=urllib3.PoolManager( - timeout=urllib3.Timeout.DEFAULT_TIMEOUT, - retries=urllib3.Retry( - total=5, - backoff_factor=0.2, - status_forcelist=[500, 502, 503, 504], - ), - ), - ) + client = get_minio_client() ci_pipeline_id = args.pipeline_id or getenv('CI_PIPELINE_ID') # required if args.action == 'download': diff --git a/tools/ci/check_build_test_rules.py b/tools/ci/check_build_test_rules.py index f204f0702a..e70fb0e324 100755 --- a/tools/ci/check_build_test_rules.py +++ b/tools/ci/check_build_test_rules.py @@ -13,7 +13,7 @@ from pathlib import Path from typing import Dict, List, Optional, Tuple import yaml -from idf_ci_utils import IDF_PATH +from idf_ci_utils import IDF_PATH, get_all_manifest_files YES = u'\u2713' NO = u'\u2717' @@ -148,9 +148,7 @@ def check_readme( 'all', recursive=True, exclude_list=exclude_dirs or [], - manifest_files=[ - str(p) for p in Path(IDF_PATH).glob('**/.build-test-rules.yml') - ], + manifest_files=get_all_manifest_files(), default_build_targets=SUPPORTED_TARGETS + extra_default_build_targets, ) ) @@ -304,9 +302,7 @@ def check_test_scripts( 'all', recursive=True, exclude_list=exclude_dirs or [], - manifest_files=[ - str(p) for p in Path(IDF_PATH).glob('**/.build-test-rules.yml') - ], + manifest_files=get_all_manifest_files(), default_build_targets=SUPPORTED_TARGETS + extra_default_build_targets, ) ) @@ -382,7 +378,7 @@ def sort_yaml(files: List[str]) -> None: def check_exist() -> None: exit_code = 0 - config_files = [str(p) for p in Path(IDF_PATH).glob('**/.build-test-rules.yml')] + config_files = get_all_manifest_files() for file in config_files: if 'managed_components' in Path(file).parts: continue diff --git a/tools/ci/check_tools_files_patterns.py b/tools/ci/check_tools_files_patterns.py index 9b6407f451..89e593fa73 100755 --- a/tools/ci/check_tools_files_patterns.py +++ b/tools/ci/check_tools_files_patterns.py @@ -39,7 +39,7 @@ def check(pattern_yml: str, exclude_list: str) -> Tuple[Set, Set]: git_files = get_git_files(os.path.join(IDF_PATH, 'tools'), full_path=True) for f in git_files: f = Path(f) - if f in rules_files_set or f in exclude_files_set: + if f in rules_files_set or f in exclude_files_set or str(f).startswith(os.path.join(IDF_PATH, 'tools', 'test_apps')): continue missing_files.add(os.path.relpath(f, IDF_PATH)) diff --git a/tools/ci/ci_build_apps.py b/tools/ci/ci_build_apps.py index a37de3c34d..d1ef4cec20 100644 --- a/tools/ci/ci_build_apps.py +++ b/tools/ci/ci_build_apps.py @@ -10,13 +10,15 @@ import os import sys import typing as t import unittest -from collections import defaultdict from pathlib import Path import yaml -from idf_build_apps import LOGGER, App, build_apps, find_apps, setup_logging -from idf_build_apps.constants import SUPPORTED_TARGETS -from idf_ci_utils import IDF_PATH +from dynamic_pipelines.constants import DEFAULT_TEST_PATHS +from idf_build_apps import build_apps, setup_logging +from idf_build_apps.utils import semicolon_separated_str_to_list +from idf_pytest.constants import (DEFAULT_BUILD_TEST_RULES_FILEPATH, DEFAULT_CONFIG_RULES_STR, + DEFAULT_FULL_BUILD_TEST_FILEPATTERNS, DEFAULT_IGNORE_WARNING_FILEPATH) +from idf_pytest.script import get_all_apps CI_ENV_VARS = { 'EXTRA_CFLAGS': '-Werror -Werror=deprecated-declarations -Werror=unused-variable ' @@ -27,118 +29,6 @@ CI_ENV_VARS = { } -def get_pytest_apps( - paths: t.List[str], - target: str, - config_rules_str: t.List[str], - marker_expr: str, - filter_expr: str, - preserve_all: bool = False, - extra_default_build_targets: t.Optional[t.List[str]] = None, - modified_components: t.Optional[t.List[str]] = None, - modified_files: t.Optional[t.List[str]] = None, - ignore_app_dependencies_filepatterns: t.Optional[t.List[str]] = None, -) -> t.List[App]: - from idf_pytest.script import get_pytest_cases - - pytest_cases = get_pytest_cases(paths, target, marker_expr, filter_expr) - - _paths: t.Set[str] = set() - test_related_app_configs = defaultdict(set) - for case in pytest_cases: - for app in case.apps: - _paths.add(app.path) - test_related_app_configs[app.path].add(app.config) - - if not extra_default_build_targets: - extra_default_build_targets = [] - - app_dirs = list(_paths) - if not app_dirs: - raise RuntimeError('No apps found') - - LOGGER.info(f'Found {len(app_dirs)} apps') - app_dirs.sort() - - apps = find_apps( - app_dirs, - target=target, - build_dir='build_@t_@w', - config_rules_str=config_rules_str, - build_log_path='build_log.txt', - size_json_path='size.json', - check_warnings=True, - manifest_rootpath=IDF_PATH, - manifest_files=[str(p) for p in Path(IDF_PATH).glob('**/.build-test-rules.yml')], - default_build_targets=SUPPORTED_TARGETS + extra_default_build_targets, - modified_components=modified_components, - modified_files=modified_files, - ignore_app_dependencies_filepatterns=ignore_app_dependencies_filepatterns, - ) - - for app in apps: - is_test_related = app.config_name in test_related_app_configs[app.app_dir] - if not preserve_all and not is_test_related: - app.preserve = False - - if app.target == 'linux': - app._size_json_path = None # no esp_idf_size for linux target - - return apps # type: ignore - - -def get_cmake_apps( - paths: t.List[str], - target: str, - config_rules_str: t.List[str], - preserve_all: bool = False, - extra_default_build_targets: t.Optional[t.List[str]] = None, - modified_components: t.Optional[t.List[str]] = None, - modified_files: t.Optional[t.List[str]] = None, - ignore_app_dependencies_filepatterns: t.Optional[t.List[str]] = None, -) -> t.List[App]: - from idf_pytest.constants import PytestApp - from idf_pytest.script import get_pytest_cases - - apps = find_apps( - paths, - recursive=True, - target=target, - build_dir='build_@t_@w', - config_rules_str=config_rules_str, - build_log_path='build_log.txt', - size_json_path='size.json', - check_warnings=True, - preserve=False, - manifest_rootpath=IDF_PATH, - manifest_files=[str(p) for p in Path(IDF_PATH).glob('**/.build-test-rules.yml')], - default_build_targets=SUPPORTED_TARGETS + extra_default_build_targets, - modified_components=modified_components, - modified_files=modified_files, - ignore_app_dependencies_filepatterns=ignore_app_dependencies_filepatterns, - ) - - apps_for_build = [] - pytest_cases_apps = [app for case in get_pytest_cases(paths, target) for app in case.apps] - for app in apps: - if preserve_all: # relpath - app.preserve = True - - if PytestApp(os.path.realpath(app.app_dir), app.target, app.config_name) in pytest_cases_apps: - LOGGER.debug('Skipping build app with pytest scripts: %s', app) - continue - - if app.target == 'linux': - app._size_json_path = None # no esp_idf_size for linux target - - apps_for_build.append(app) - - return apps_for_build - - -APPS_BUILD_PER_JOB = 30 - - def main(args: argparse.Namespace) -> None: extra_default_build_targets: t.List[str] = [] if args.default_build_test_rules: @@ -148,39 +38,24 @@ def main(args: argparse.Namespace) -> None: if configs: extra_default_build_targets = configs.get('extra_default_build_targets') or [] - if args.pytest_apps: - LOGGER.info('Only build apps with pytest scripts') - apps = get_pytest_apps( - args.paths, - args.target, - args.config, - args.marker_expr, - args.filter_expr, - args.preserve_all, - extra_default_build_targets, - args.modified_components, - args.modified_files, - args.ignore_app_dependencies_filepatterns, - ) - else: - LOGGER.info('build apps. will skip pytest apps with pytest scripts') - apps = get_cmake_apps( - args.paths, - args.target, - args.config, - args.preserve_all, - extra_default_build_targets, - args.modified_components, - args.modified_files, - args.ignore_app_dependencies_filepatterns, - ) - - LOGGER.info('Found %d apps after filtering', len(apps)) - LOGGER.info( - 'Suggest setting the parallel count to %d for this build job', - len(apps) // APPS_BUILD_PER_JOB + 1, + test_related_apps, non_test_related_apps = get_all_apps( + args.paths, + args.target, + config_rules_str=args.config, + marker_expr=args.marker_expr, + filter_expr=args.filter_expr, + preserve_all=args.preserve_all, + extra_default_build_targets=extra_default_build_targets, + modified_files=args.modified_components, + modified_components=args.modified_files, + ignore_app_dependencies_filepatterns=args.ignore_app_dependencies_filepatterns, ) + if args.pytest_apps: + apps = test_related_apps + else: + apps = non_test_related_apps + if args.extra_preserve_dirs: for app in apps: if app.preserve: @@ -192,7 +67,7 @@ def main(args: argparse.Namespace) -> None: app.preserve = True res = build_apps( - apps, + sorted(apps), parallel_count=args.parallel_count, parallel_index=args.parallel_index, dry_run=False, @@ -206,12 +81,10 @@ def main(args: argparse.Namespace) -> None: modified_components=args.modified_components, modified_files=args.modified_files, ignore_app_dependencies_filepatterns=args.ignore_app_dependencies_filepatterns, + junitxml=args.junitxml, ) - if isinstance(res, tuple): - sys.exit(res[0]) - else: - sys.exit(res) + sys.exit(res) if __name__ == '__main__': @@ -219,7 +92,7 @@ if __name__ == '__main__': description='Build all the apps for different test types. Will auto remove those non-test apps binaries', formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) - parser.add_argument('paths', nargs='+', help='Paths to the apps to build.') + parser.add_argument('paths', nargs='*', help='Paths to the apps to build.') parser.add_argument( '-t', '--target', @@ -228,7 +101,7 @@ if __name__ == '__main__': ) parser.add_argument( '--config', - default=['sdkconfig.ci=default', 'sdkconfig.ci.*=', '=default'], + default=DEFAULT_CONFIG_RULES_STR, nargs='+', help='Adds configurations (sdkconfig file names) to build. This can either be ' 'FILENAME[=NAME] or FILEPATTERN. FILENAME is the name of the sdkconfig file, ' @@ -272,7 +145,7 @@ if __name__ == '__main__': ) parser.add_argument( '--ignore-warning-file', - default=os.path.join(IDF_PATH, 'tools', 'ci', 'ignore_build_warnings.txt'), + default=DEFAULT_IGNORE_WARNING_FILEPATH, type=argparse.FileType('r'), help='Ignore the warning strings in the specified file. Each line should be a regex string.', ) @@ -290,7 +163,8 @@ if __name__ == '__main__': parser.add_argument( '--pytest-apps', action='store_true', - help='Only build apps with pytest scripts. Will build apps without pytest scripts if this flag is unspecified.', + help='Only build apps required by pytest scripts. ' + 'Will build non-test-related apps if this flag is unspecified.', ) parser.add_argument( '-m', @@ -307,7 +181,7 @@ if __name__ == '__main__': ) parser.add_argument( '--default-build-test-rules', - default=os.path.join(IDF_PATH, '.gitlab', 'ci', 'default-build-test-rules.yml'), + default=DEFAULT_BUILD_TEST_RULES_FILEPATH, help='default build test rules config file', ) parser.add_argument( @@ -318,69 +192,64 @@ if __name__ == '__main__': ) parser.add_argument( '--modified-components', - nargs='*', - default=None, - help='space-separated list which specifies the modified components. app with `depends_components` set in the ' - 'corresponding manifest files would only be built if depends on any of the specified components.', + type=semicolon_separated_str_to_list, + help='semicolon-separated string which specifies the modified components. ' + 'app with `depends_components` set in the corresponding manifest files would only be built ' + 'if depends on any of the specified components. ' + 'If set to "", the value would be considered as None. ' + 'If set to ";", the value would be considered as an empty list', ) parser.add_argument( '--modified-files', - nargs='*', - default=None, - help='space-separated list which specifies the modified files. app with `depends_filepatterns` set in the ' - 'corresponding manifest files would only be built if any of the specified file pattern matches any of the ' - 'specified modified files.', + type=semicolon_separated_str_to_list, + help='semicolon-separated string which specifies the modified files. ' + 'app with `depends_filepatterns` set in the corresponding manifest files would only be built ' + 'if any of the specified file pattern matches any of the specified modified files. ' + 'If set to "", the value would be considered as None. ' + 'If set to ";", the value would be considered as an empty list', ) parser.add_argument( '-if', '--ignore-app-dependencies-filepatterns', - nargs='*', - default=None, - help='space-separated list which specifies the file patterns used for ignoring checking the app dependencies. ' - 'The `depends_components` and `depends_filepatterns` set in the manifest files will be ignored when any of the ' - 'specified file patterns matches any of the modified files. Must be used together with --modified-files', + type=semicolon_separated_str_to_list, + help='semicolon-separated string which specifies the file patterns used for ' + 'ignoring checking the app dependencies. ' + 'The `depends_components` and `depends_filepatterns` set in the manifest files ' + 'will be ignored when any of the specified file patterns matches any of the modified files. ' + 'Must be used together with --modified-files. ' + 'If set to "", the value would be considered as None. ' + 'If set to ";", the value would be considered as an empty list', + ) + parser.add_argument( + '--junitxml', + default='build_summary_@p.xml', + help='Path to the junitxml file. If specified, the junitxml file will be generated', ) arguments = parser.parse_args() setup_logging(arguments.verbose) + # set default paths + if not arguments.paths: + arguments.paths = DEFAULT_TEST_PATHS + # skip setting flags in CI if not arguments.skip_setting_flags and not os.getenv('CI_JOB_ID'): for _k, _v in CI_ENV_VARS.items(): os.environ[_k] = _v - LOGGER.info(f'env var {_k} set to "{_v}"') + print(f'env var {_k} set to "{_v}"') if os.getenv('IS_MR_PIPELINE') == '0' or os.getenv('BUILD_AND_TEST_ALL_APPS') == '1': # if it's not MR pipeline or env var BUILD_AND_TEST_ALL_APPS=1, # remove component dependency related arguments - if 'modified_components' in arguments: - arguments.modified_components = None - if 'modified_files' in arguments: - arguments.modified_files = None + arguments.modified_components = None + arguments.modified_files = None + arguments.ignore_app_dependencies_filepatterns = None - # file patterns to tigger full build - if 'modified_components' in arguments and not arguments.ignore_app_dependencies_filepatterns: - arguments.ignore_app_dependencies_filepatterns = [ - # tools - 'tools/cmake/**/*', - 'tools/tools.json', - # components - 'components/cxx/**/*', - 'components/esp_common/**/*', - 'components/esp_hw_support/**/*', - 'components/esp_rom/**/*', - 'components/esp_system/**/*', - 'components/esp_timer/**/*', - 'components/freertos/**/*', - 'components/hal/**/*', - 'components/heap/**/*', - 'components/log/**/*', - 'components/newlib/**/*', - 'components/riscv/**/*', - 'components/soc/**/*', - 'components/xtensa/**/*', - ] + # default file patterns to tigger full build + if arguments.modified_files is not None and arguments.ignore_app_dependencies_filepatterns is None: + arguments.ignore_app_dependencies_filepatterns = DEFAULT_FULL_BUILD_TEST_FILEPATTERNS main(arguments) diff --git a/tools/ci/dynamic_pipelines/__init__.py b/tools/ci/dynamic_pipelines/__init__.py new file mode 100644 index 0000000000..28b2a3c5e3 --- /dev/null +++ b/tools/ci/dynamic_pipelines/__init__.py @@ -0,0 +1,9 @@ +# SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 + +import os +import sys + +tools_dir = os.path.realpath(os.path.join(os.path.dirname(__file__), '..', '..')) +if tools_dir not in sys.path: + sys.path.append(tools_dir) diff --git a/tools/ci/dynamic_pipelines/constants.py b/tools/ci/dynamic_pipelines/constants.py new file mode 100644 index 0000000000..a90e6769f2 --- /dev/null +++ b/tools/ci/dynamic_pipelines/constants.py @@ -0,0 +1,31 @@ +# SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 + +import os + +from idf_ci_utils import IDF_PATH + +# use relative path to avoid absolute path in pipeline +DEFAULT_TEST_PATHS = [ + 'examples', + os.path.join('tools', 'test_apps'), + 'components', +] + +DEFAULT_APPS_BUILD_PER_JOB = 60 +DEFAULT_CASES_TEST_PER_JOB = 60 + +DEFAULT_BUILD_CHILD_PIPELINE_FILEPATH = os.path.join(IDF_PATH, 'build_child_pipeline.yml') +DEFAULT_TARGET_TEST_CHILD_PIPELINE_FILEPATH = os.path.join(IDF_PATH, 'target_test_child_pipeline.yml') + +TEST_RELATED_BUILD_JOB_NAME = 'build_test_related_apps' +NON_TEST_RELATED_BUILD_JOB_NAME = 'build_non_test_related_apps' + +COMMENT_START_MARKER = '### Dynamic Pipeline Report' +TEST_RELATED_APPS_FILENAME = 'test_related_apps.txt' +NON_TEST_RELATED_APPS_FILENAME = 'non_test_related_apps.txt' + +TEST_RELATED_APPS_DOWNLOAD_URLS_FILENAME = 'test_related_apps_download_urls.yml' +REPORT_TEMPLATE_FILEPATH = os.path.join( + IDF_PATH, 'tools', 'ci', 'dynamic_pipelines', 'templates', 'report.template.html' +) diff --git a/tools/ci/dynamic_pipelines/models.py b/tools/ci/dynamic_pipelines/models.py new file mode 100644 index 0000000000..a71fa2830c --- /dev/null +++ b/tools/ci/dynamic_pipelines/models.py @@ -0,0 +1,169 @@ +# SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 + +import inspect +import typing as t +from dataclasses import dataclass +from xml.etree.ElementTree import Element + +import yaml + + +class Job: + def __init__( + self, + *, + name: str, + extends: t.Optional[t.List[str]] = None, + tags: t.Optional[t.List[str]] = None, + stage: t.Optional[str] = None, + parallel: int = 1, + variables: t.Dict[str, str] = None, + script: t.Optional[t.List[str]] = None, + before_script: t.Optional[t.List[str]] = None, + after_script: t.Optional[t.List[str]] = None, + needs: t.Optional[t.List[str]] = None, + **kwargs: t.Any, + ) -> None: + self.name = name + self.extends = extends + self.tags = tags + self.stage = stage + self.parallel = parallel + self.variables = variables or {} + self.script = script + self.before_script = before_script + self.after_script = after_script + self.needs = needs + + for k, v in kwargs.items(): + setattr(self, k, v) + + def __str__(self) -> str: + return yaml.dump(self.to_dict()) # type: ignore + + def set_variable(self, key: str, value: str) -> None: + self.variables[key] = value + + def to_dict(self) -> t.Dict[str, t.Any]: + res = {} + for k, v in inspect.getmembers(self): + if k.startswith('_'): + continue + + # name is the dict key + if k == 'name': + continue + + # parallel 1 is not allowed + if k == 'parallel' and v == 1: + continue + + if v is None: + continue + + if inspect.ismethod(v) or inspect.isfunction(v): + continue + + res[k] = v + + return {self.name: res} + + +class EmptyJob(Job): + def __init__( + self, + *, + name: t.Optional[str] = None, + tags: t.Optional[t.List[str]] = None, + stage: t.Optional[str] = None, + before_script: t.Optional[t.List[str]] = None, + after_script: t.Optional[t.List[str]] = None, + **kwargs: t.Any, + ) -> None: + super().__init__( + name=name or 'fake_pass_job', + tags=tags or ['build', 'shiny'], + stage=stage or 'build', + script=['echo "This is a fake job to pass the pipeline"'], + before_script=before_script or [], + after_script=after_script or [], + **kwargs, + ) + + +class BuildJob(Job): + def __init__( + self, + *, + extends: t.Optional[t.List[str]] = None, + tags: t.Optional[t.List[str]] = None, + stage: t.Optional[str] = None, + **kwargs: t.Any, + ) -> None: + super().__init__( + extends=extends or ['.dynamic_build_template'], + tags=tags or ['build', 'shiny'], + stage=stage or 'build', + **kwargs, + ) + + +class TargetTestJob(Job): + def __init__( + self, + *, + extends: t.Optional[t.List[str]] = None, + stage: t.Optional[str] = None, + **kwargs: t.Any, + ) -> None: + super().__init__( + extends=extends or ['.dynamic_target_test_template'], + stage=stage or 'target_test', + **kwargs, + ) + + +@dataclass +class TestCase: + name: str + file: str + time: float + failure: t.Optional[str] = None + skipped: t.Optional[str] = None + ci_job_url: t.Optional[str] = None + + @property + def is_failure(self) -> bool: + return self.failure is not None + + @property + def is_skipped(self) -> bool: + return self.skipped is not None + + @property + def is_success(self) -> bool: + return not self.is_failure and not self.is_skipped + + @classmethod + def from_test_case_node(cls, node: Element) -> t.Optional['TestCase']: + if 'name' not in node.attrib: + print('WARNING: Node Invalid: ', node) + return None + + kwargs = { + 'name': node.attrib['name'], + 'file': node.attrib.get('file'), + 'time': float(node.attrib.get('time') or 0), + 'ci_job_url': node.attrib.get('ci_job_url') or '', + } + + failure_node = node.find('failure') + if failure_node is not None: + kwargs['failure'] = failure_node.attrib['message'] + + skipped_node = node.find('skipped') + if skipped_node is not None: + kwargs['skipped'] = skipped_node.attrib['message'] + + return cls(**kwargs) # type: ignore diff --git a/tools/ci/dynamic_pipelines/report.py b/tools/ci/dynamic_pipelines/report.py new file mode 100644 index 0000000000..5ed2ab2605 --- /dev/null +++ b/tools/ci/dynamic_pipelines/report.py @@ -0,0 +1,276 @@ +# SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 + +import abc +import html +import os +import re +import typing as t + +import yaml +from artifacts_handler import ArtifactType +from gitlab_api import Gitlab +from idf_build_apps import App +from idf_build_apps.constants import BuildStatus +from idf_ci.uploader import AppUploader +from prettytable import PrettyTable + +from .constants import COMMENT_START_MARKER, REPORT_TEMPLATE_FILEPATH, TEST_RELATED_APPS_DOWNLOAD_URLS_FILENAME +from .models import TestCase + + +class ReportGenerator: + REGEX_PATTERN = '#### {}[^####]+' + + def __init__(self, project_id: int, mr_iid: int, pipeline_id: int, *, title: str): + gl_project = Gitlab(project_id).project + if mr_iid is not None: + self.mr = gl_project.mergerequests.get(mr_iid) + else: + self.mr = None + self.pipeline_id = pipeline_id + + self.title = title + self.output_filepath = self.title.lower().replace(' ', '_') + '.html' + + @staticmethod + def get_download_link_for_url(url: str) -> str: + if url: + return f'Download' + + return '' + + def generate_html_report(self, table_str: str) -> str: + # we're using bootstrap table + table_str = table_str.replace('', '
') + + with open(REPORT_TEMPLATE_FILEPATH) as fr: + template = fr.read() + + return template.replace('{{title}}', self.title).replace('{{table}}', table_str) + + @staticmethod + def table_to_html_str(table: PrettyTable) -> str: + return html.unescape(table.get_html_string()) # type: ignore + + @abc.abstractmethod + def _get_report_str(self) -> str: + raise NotImplementedError + + def post_report(self, job_id: int, commit_id: str) -> None: + # report in html format, otherwise will exceed the limit + with open(self.output_filepath, 'w') as fw: + fw.write(self._get_report_str()) + + # for example, {URL}/-/esp-idf/-/jobs/{id}/artifacts/list_job_84.txt + # CI_PAGES_URL is {URL}/esp-idf, which missed one `-` + url = os.getenv('CI_PAGES_URL', '').replace('esp-idf', '-/esp-idf') + + comment = f'''#### {self.title} + +Full {self.title} here: {url}/-/jobs/{job_id}/artifacts/{self.output_filepath} (with commit {commit_id}) + +''' + + if self.mr is None: + print('No MR found, skip posting comment') + return + + for note in self.mr.notes.list(iterator=True): + if note.body.startswith(COMMENT_START_MARKER): + updated_str = re.sub(self.REGEX_PATTERN.format(self.title), comment, note.body) + if updated_str == note.body: # not updated + updated_str = f'{note.body.strip()}\n\n{comment}' + + note.body = updated_str + note.save() + break + else: + new_comment = f'''{COMMENT_START_MARKER} + +{comment}''' + self.mr.notes.create({'body': new_comment}) + + +class BuildReportGenerator(ReportGenerator): + def __init__( + self, + project_id: int, + mr_iid: int, + pipeline_id: int, + *, + title: str = 'Build Report', + apps: t.List[App], + ): + super().__init__(project_id, mr_iid, pipeline_id, title=title) + self.apps = apps + + self.apps_presigned_url_filepath = TEST_RELATED_APPS_DOWNLOAD_URLS_FILENAME + + def _get_report_str(self) -> str: + if not self.apps: + print('No apps found, skip generating build report') + return 'No Apps Built' + + uploader = AppUploader(self.pipeline_id) + + table_str = '' + + failed_apps = [app for app in self.apps if app.build_status == BuildStatus.FAILED] + if failed_apps: + table_str += '

Failed Apps

' + + failed_apps_table = PrettyTable() + failed_apps_table.field_names = [ + 'App Dir', + 'Build Dir', + 'Failed Reason', + 'Build Log', + ] + for app in failed_apps: + failed_apps_table.add_row( + [ + app.app_dir, + app.build_dir, + app.build_comment or '', + self.get_download_link_for_url(uploader.get_app_presigned_url(app, ArtifactType.LOGS)), + ] + ) + + table_str += self.table_to_html_str(failed_apps_table) + + built_test_related_apps = [app for app in self.apps if app.build_status == BuildStatus.SUCCESS and app.preserve] + if built_test_related_apps: + table_str += '

Built Apps (Test Related)

' + + built_apps_table = PrettyTable() + built_apps_table.field_names = [ + 'App Dir', + 'Build Dir', + 'Bin Files with Build Log (without map and elf)', + 'Map and Elf Files', + ] + app_presigned_urls_dict: t.Dict[str, t.Dict[str, str]] = {} + for app in built_test_related_apps: + _d = { + ArtifactType.BUILD_DIR_WITHOUT_MAP_AND_ELF_FILES.value: uploader.get_app_presigned_url( + app, ArtifactType.BUILD_DIR_WITHOUT_MAP_AND_ELF_FILES + ), + ArtifactType.MAP_AND_ELF_FILES.value: uploader.get_app_presigned_url( + app, ArtifactType.MAP_AND_ELF_FILES + ), + } + + built_apps_table.add_row( + [ + app.app_dir, + app.build_dir, + self.get_download_link_for_url(_d[ArtifactType.BUILD_DIR_WITHOUT_MAP_AND_ELF_FILES]), + self.get_download_link_for_url(_d[ArtifactType.MAP_AND_ELF_FILES]), + ] + ) + + app_presigned_urls_dict[app.build_path] = _d + + # also generate a yaml file that includes the apps and the presigned urls + # for helping debugging locally + with open(self.apps_presigned_url_filepath, 'w') as fw: + yaml.dump(app_presigned_urls_dict, fw) + + table_str += self.table_to_html_str(built_apps_table) + + built_non_test_related_apps = [ + app for app in self.apps if app.build_status == BuildStatus.SUCCESS and not app.preserve + ] + if built_non_test_related_apps: + table_str += '

Built Apps (Non Test Related)

' + + built_apps_table = PrettyTable() + built_apps_table.field_names = [ + 'App Dir', + 'Build Dir', + 'Build Log', + ] + for app in built_non_test_related_apps: + built_apps_table.add_row( + [ + app.app_dir, + app.build_dir, + self.get_download_link_for_url(uploader.get_app_presigned_url(app, ArtifactType.LOGS)), + ] + ) + + table_str += self.table_to_html_str(built_apps_table) + + skipped_apps = [app for app in self.apps if app.build_status == BuildStatus.SKIPPED] + if skipped_apps: + table_str += '

Skipped Apps

' + + skipped_apps_table = PrettyTable() + skipped_apps_table.field_names = ['App Dir', 'Build Dir', 'Skipped Reason', 'Build Log'] + for app in skipped_apps: + skipped_apps_table.add_row( + [ + app.app_dir, + app.build_dir, + app.build_comment or '', + self.get_download_link_for_url(uploader.get_app_presigned_url(app, ArtifactType.LOGS)), + ] + ) + + table_str += self.table_to_html_str(skipped_apps_table) + + return self.generate_html_report(table_str) + + +class TargetTestReportGenerator(ReportGenerator): + def __init__( + self, + project_id: int, + mr_iid: int, + pipeline_id: int, + *, + title: str = 'Target Test Report', + test_cases: t.List[TestCase], + ): + super().__init__(project_id, mr_iid, pipeline_id, title=title) + + self.test_cases = test_cases + + def _get_report_str(self) -> str: + table_str = '' + + failed_test_cases = [tc for tc in self.test_cases if tc.is_failure] + if failed_test_cases: + table_str += '

Failed Test Cases

' + + failed_test_cases_table = PrettyTable() + failed_test_cases_table.field_names = ['Test Case', 'Test Script File Path', 'Failure Reason', 'Job URL'] + for tc in failed_test_cases: + failed_test_cases_table.add_row([tc.name, tc.file, tc.failure, tc.ci_job_url]) + + table_str += self.table_to_html_str(failed_test_cases_table) + + skipped_test_cases = [tc for tc in self.test_cases if tc.is_skipped] + if skipped_test_cases: + table_str += '

Skipped Test Cases

' + + skipped_test_cases_table = PrettyTable() + skipped_test_cases_table.field_names = ['Test Case', 'Test Script File Path', 'Skipped Reason'] + for tc in skipped_test_cases: + skipped_test_cases_table.add_row([tc.name, tc.file, tc.skipped]) + + table_str += self.table_to_html_str(skipped_test_cases_table) + + successful_test_cases = [tc for tc in self.test_cases if tc.is_success] + if successful_test_cases: + table_str += '

Succeeded Test Cases

' + + successful_test_cases_table = PrettyTable() + successful_test_cases_table.field_names = ['Test Case', 'Test Script File Path', 'Job URL'] + for tc in successful_test_cases: + successful_test_cases_table.add_row([tc.name, tc.file, tc.ci_job_url]) + + table_str += self.table_to_html_str(successful_test_cases_table) + + return self.generate_html_report(table_str) diff --git a/tools/ci/dynamic_pipelines/scripts/__init__.py b/tools/ci/dynamic_pipelines/scripts/__init__.py new file mode 100644 index 0000000000..2ef7c132ff --- /dev/null +++ b/tools/ci/dynamic_pipelines/scripts/__init__.py @@ -0,0 +1,25 @@ +# SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 + +import os +import sys + +IDF_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', '..', '..')) + +# run this scripts only in idf path, to ensure the relative path is the same +os.chdir(IDF_PATH) + +if 'IDF_PATH' not in os.environ: + os.environ['IDF_PATH'] = IDF_PATH + +tools_path = os.path.join(os.path.dirname(__file__), '..', '..', '..') +if tools_path not in sys.path: + sys.path.append(tools_path) + +tools_ci_path = os.path.join(os.path.dirname(__file__), '..', '..') +if tools_ci_path not in sys.path: + sys.path.append(tools_ci_path) + +tools_ci_python_packages_path = os.path.join(os.path.dirname(__file__), '..', '..', 'python_packages') +if tools_ci_python_packages_path not in sys.path: + sys.path.append(tools_ci_python_packages_path) diff --git a/tools/ci/dynamic_pipelines/scripts/child_pipeline_build_apps.py b/tools/ci/dynamic_pipelines/scripts/child_pipeline_build_apps.py new file mode 100644 index 0000000000..cf4d3ebe9c --- /dev/null +++ b/tools/ci/dynamic_pipelines/scripts/child_pipeline_build_apps.py @@ -0,0 +1,73 @@ +# SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 + +import argparse +import sys + +import __init__ # noqa: F401 # inject the system path +from dynamic_pipelines.constants import TEST_RELATED_APPS_FILENAME +from idf_build_apps import build_apps, setup_logging +from idf_build_apps.utils import semicolon_separated_str_to_list +from idf_ci.app import import_apps_from_txt +from idf_pytest.constants import DEFAULT_IGNORE_WARNING_FILEPATH + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Build Apps for Dynamic Pipeline') + parser.add_argument('app_list_file', default=TEST_RELATED_APPS_FILENAME, help='List of apps to build') + parser.add_argument( + '--build-verbose', + action='store_true', + help='Enable verbose output from build system.', + ) + parser.add_argument('--parallel-count', default=1, type=int, help='Number of parallel build jobs.') + parser.add_argument( + '--parallel-index', + default=1, + type=int, + help='Index (1-based) of the job, out of the number specified by --parallel-count.', + ) + parser.add_argument( + '--ignore-warning-file', + default=DEFAULT_IGNORE_WARNING_FILEPATH, + type=argparse.FileType('r'), + help='Ignore the warning strings in the specified file. Each line should be a regex string.', + ) + parser.add_argument( + '--modified-components', + type=semicolon_separated_str_to_list, + help='semicolon-separated string which specifies the modified components. ' + 'app with `depends_components` set in the corresponding manifest files would only be built ' + 'if depends on any of the specified components. ' + 'If set to "", the value would be considered as None. ' + 'If set to ";", the value would be considered as an empty list', + ) + parser.add_argument( + '--collect-app-info', + default='list_job_@p.txt', + help='If specified, the test case name and app info json will be written to this file', + ) + parser.add_argument( + '--junitxml', + default='build_summary_@p.xml', + help='Path to the junitxml file. If specified, the junitxml file will be generated', + ) + + args = parser.parse_args() + + setup_logging(verbose=1) + + sys.exit( + build_apps( + import_apps_from_txt(args.app_list_file), + build_verbose=args.build_verbose, + keep_going=True, + ignore_warning_file=args.ignore_warning_file, + modified_components=args.modified_components, + check_app_dependencies=True, + parallel_count=args.parallel_count, + parallel_index=args.parallel_index, + collect_size_info='size_info_@p.txt', + collect_app_info=args.collect_app_info, + junitxml=args.junitxml, + ) + ) diff --git a/tools/ci/dynamic_pipelines/scripts/generate_build_child_pipeline.py b/tools/ci/dynamic_pipelines/scripts/generate_build_child_pipeline.py new file mode 100644 index 0000000000..5194d73671 --- /dev/null +++ b/tools/ci/dynamic_pipelines/scripts/generate_build_child_pipeline.py @@ -0,0 +1,193 @@ +# SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 + +"""This file is used for generating the child pipeline for build jobs.""" + +import argparse +import os +import typing as t + +import __init__ # noqa: F401 # inject the system path +import yaml +from dynamic_pipelines.constants import (DEFAULT_APPS_BUILD_PER_JOB, DEFAULT_BUILD_CHILD_PIPELINE_FILEPATH, + DEFAULT_TEST_PATHS, NON_TEST_RELATED_APPS_FILENAME, + NON_TEST_RELATED_BUILD_JOB_NAME, TEST_RELATED_APPS_FILENAME, + TEST_RELATED_BUILD_JOB_NAME) +from dynamic_pipelines.models import BuildJob, EmptyJob +from dynamic_pipelines.utils import dump_jobs_to_yaml +from idf_build_apps.utils import semicolon_separated_str_to_list +from idf_ci.app import dump_apps_to_txt +from idf_ci_utils import IDF_PATH +from idf_pytest.constants import DEFAULT_CONFIG_RULES_STR, DEFAULT_FULL_BUILD_TEST_FILEPATTERNS, CollectMode +from idf_pytest.script import get_all_apps + + +def main(arguments: argparse.Namespace) -> None: + # load from default build test rules config file + extra_default_build_targets: t.List[str] = [] + if arguments.default_build_test_rules: + with open(arguments.default_build_test_rules) as fr: + configs = yaml.safe_load(fr) + + if configs: + extra_default_build_targets = configs.get('extra_default_build_targets') or [] + + build_jobs = [] + ########################################### + # special case with -k, ignore other args # + ########################################### + if arguments.filter_expr: + # build only test related apps + test_related_apps, _ = get_all_apps( + arguments.paths, + target=CollectMode.ALL, + config_rules_str=DEFAULT_CONFIG_RULES_STR, + filter_expr=arguments.filter_expr, + marker_expr='not host_test', + extra_default_build_targets=extra_default_build_targets, + ) + dump_apps_to_txt(sorted(test_related_apps), TEST_RELATED_APPS_FILENAME) + print(f'Generate test related apps file {TEST_RELATED_APPS_FILENAME} with {len(test_related_apps)} apps') + + test_apps_build_job = BuildJob( + name=TEST_RELATED_BUILD_JOB_NAME, + parallel=len(test_related_apps) // DEFAULT_APPS_BUILD_PER_JOB + 1, + variables={ + 'APP_LIST_FILE': TEST_RELATED_APPS_FILENAME, + }, + ) + + build_jobs.append(test_apps_build_job) + else: + ############# + # all cases # + ############# + test_related_apps, non_test_related_apps = get_all_apps( + arguments.paths, + CollectMode.ALL, + marker_expr='not host_test', + config_rules_str=DEFAULT_CONFIG_RULES_STR, + extra_default_build_targets=extra_default_build_targets, + modified_components=arguments.modified_components, + modified_files=arguments.modified_files, + ignore_app_dependencies_filepatterns=arguments.ignore_app_dependencies_filepatterns, + ) + + dump_apps_to_txt(sorted(test_related_apps), TEST_RELATED_APPS_FILENAME) + print(f'Generate test related apps file {TEST_RELATED_APPS_FILENAME} with {len(test_related_apps)} apps') + dump_apps_to_txt(sorted(non_test_related_apps), NON_TEST_RELATED_APPS_FILENAME) + print( + f'Generate non-test related apps file {NON_TEST_RELATED_APPS_FILENAME} with {len(non_test_related_apps)} apps' + ) + + if test_related_apps: + test_apps_build_job = BuildJob( + name=TEST_RELATED_BUILD_JOB_NAME, + parallel=len(test_related_apps) // DEFAULT_APPS_BUILD_PER_JOB + 1, + variables={ + 'APP_LIST_FILE': TEST_RELATED_APPS_FILENAME, + }, + ) + build_jobs.append(test_apps_build_job) + + if non_test_related_apps: + non_test_apps_build_job = BuildJob( + name=NON_TEST_RELATED_BUILD_JOB_NAME, + parallel=len(non_test_related_apps) // DEFAULT_APPS_BUILD_PER_JOB + 1, + variables={ + 'APP_LIST_FILE': NON_TEST_RELATED_APPS_FILENAME, + }, + ) + build_jobs.append(non_test_apps_build_job) + + # check if there's no jobs + if not build_jobs: + print('No apps need to be built. Create one empty job instead') + build_jobs.append(EmptyJob()) + extra_include_yml = [] + else: + extra_include_yml = ['tools/ci/dynamic_pipelines/templates/test_child_pipeline.yml'] + + dump_jobs_to_yaml(build_jobs, arguments.yaml_output, extra_include_yml) + print(f'Generate child pipeline yaml file {arguments.yaml_output} with {sum(j.parallel for j in build_jobs)} jobs') + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description='Generate build child pipeline', + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + parser.add_argument( + '-o', + '--yaml-output', + default=DEFAULT_BUILD_CHILD_PIPELINE_FILEPATH, + help='Output YAML path', + ) + parser.add_argument( + '-p', + '--paths', + nargs='+', + default=DEFAULT_TEST_PATHS, + help='Paths to the apps to build.', + ) + parser.add_argument( + '-k', + '--filter-expr', + help='only build tests matching given filter expression. For example: -k "test_hello_world". Works only' + 'for pytest', + ) + parser.add_argument( + '--default-build-test-rules', + default=os.path.join(IDF_PATH, '.gitlab', 'ci', 'default-build-test-rules.yml'), + help='default build test rules config file', + ) + parser.add_argument( + '--modified-components', + type=semicolon_separated_str_to_list, + help='semicolon-separated string which specifies the modified components. ' + 'app with `depends_components` set in the corresponding manifest files would only be built ' + 'if depends on any of the specified components. ' + 'If set to "", the value would be considered as None. ' + 'If set to ";", the value would be considered as an empty list', + ) + parser.add_argument( + '--modified-files', + type=semicolon_separated_str_to_list, + help='semicolon-separated string which specifies the modified files. ' + 'app with `depends_filepatterns` set in the corresponding manifest files would only be built ' + 'if any of the specified file pattern matches any of the specified modified files. ' + 'If set to "", the value would be considered as None. ' + 'If set to ";", the value would be considered as an empty list', + ) + parser.add_argument( + '-if', + '--ignore-app-dependencies-filepatterns', + type=semicolon_separated_str_to_list, + help='semicolon-separated string which specifies the file patterns used for ' + 'ignoring checking the app dependencies. ' + 'The `depends_components` and `depends_filepatterns` set in the manifest files will be ignored ' + 'when any of the specified file patterns matches any of the modified files. ' + 'Must be used together with --modified-files. ' + 'If set to "", the value would be considered as None. ' + 'If set to ";", the value would be considered as an empty list', + ) + + args = parser.parse_args() + + if os.getenv('IS_MR_PIPELINE') == '0' or os.getenv('BUILD_AND_TEST_ALL_APPS') == '1': + print('Build and run all test cases, and compile all cmake apps') + args.modified_components = None + args.modified_files = None + args.ignore_app_dependencies_filepatterns = None + elif args.filter_expr is not None: + print('Build and run only test cases matching "%s"' % args.filter_expr) + args.modified_components = None + args.modified_files = None + args.ignore_app_dependencies_filepatterns = None + else: + print('Build and run only test cases matching the modified components and files') + if args.modified_files and not args.ignore_app_dependencies_filepatterns: + # setting default values + args.ignore_app_dependencies_filepatterns = DEFAULT_FULL_BUILD_TEST_FILEPATTERNS + + main(args) diff --git a/tools/ci/dynamic_pipelines/scripts/generate_build_report.py b/tools/ci/dynamic_pipelines/scripts/generate_build_report.py new file mode 100644 index 0000000000..cef16271fe --- /dev/null +++ b/tools/ci/dynamic_pipelines/scripts/generate_build_report.py @@ -0,0 +1,59 @@ +# SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 + +import argparse +import glob +import os + +import __init__ # noqa: F401 # inject the system path +from dynamic_pipelines.report import BuildReportGenerator +from idf_ci.app import import_apps_from_txt + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description='Update Build Report in MR pipelines', + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + parser.add_argument( + '--project-id', + type=int, + default=os.getenv('CI_PROJECT_ID'), + help='Project ID', + ) + parser.add_argument( + '--mr-iid', + type=int, + default=os.getenv('CI_MERGE_REQUEST_IID'), + help='Merge Request IID', + ) + parser.add_argument( + '--pipeline-id', + type=int, + default=os.getenv('PARENT_PIPELINE_ID'), + help='Pipeline ID', + ) + parser.add_argument( + '--job-id', + type=int, + default=os.getenv('CI_JOB_ID'), + help='Job ID', + ) + parser.add_argument( + '--commit-id', + default=os.getenv('CI_COMMIT_SHORT_SHA'), + help='MR commit ID', + ) + parser.add_argument( + '--app-list-filepattern', + default='list_job_*.txt', + help='App list file pattern', + ) + + args = parser.parse_args() + + apps = [] + for f in glob.glob(args.app_list_filepattern): + apps.extend(import_apps_from_txt(f)) + + report_generator = BuildReportGenerator(args.project_id, args.mr_iid, args.pipeline_id, apps=apps) + report_generator.post_report(args.job_id, args.commit_id) diff --git a/tools/ci/dynamic_pipelines/scripts/generate_target_test_child_pipeline.py b/tools/ci/dynamic_pipelines/scripts/generate_target_test_child_pipeline.py new file mode 100644 index 0000000000..28e8d335cd --- /dev/null +++ b/tools/ci/dynamic_pipelines/scripts/generate_target_test_child_pipeline.py @@ -0,0 +1,131 @@ +# SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 + +"""This file is used for generating the child pipeline for target test jobs. + +1. Check the build jobs' artifacts to get the built apps' information. +2. Post the Build Report if it's running in an MR pipeline. +3. Generate the child pipeline for target test jobs. +""" + +import argparse +import glob +import os +import typing as t +from collections import Counter, defaultdict + +import __init__ # noqa: F401 # inject the system path +from dynamic_pipelines.constants import (DEFAULT_CASES_TEST_PER_JOB, DEFAULT_TARGET_TEST_CHILD_PIPELINE_FILEPATH, + DEFAULT_TEST_PATHS) +from dynamic_pipelines.models import EmptyJob, Job, TargetTestJob +from dynamic_pipelines.utils import dump_jobs_to_yaml +from gitlab.v4.objects import Project +from gitlab_api import Gitlab +from idf_build_apps import App +from idf_ci.app import import_apps_from_txt +from idf_pytest.script import get_pytest_cases + + +def get_tags_with_amount(s: str) -> t.List[str]: + c: Counter = Counter() + for _t in s.split(','): + c[_t] += 1 + + res = set() + for target, amount in c.items(): + if amount > 1: + res.add(f'{target}_{amount}') + else: + res.add(target) + + return sorted(res) + + +def generate_target_test_child_pipeline(project: Project, paths: str, apps: t.List[App], output_filepath: str) -> None: + pytest_cases = get_pytest_cases( + paths, + apps=apps, + marker_expr='not host_test', # since it's generating target-test child pipeline + ) + + res = defaultdict(list) + for case in pytest_cases: + if not case.env_markers: + print(f'No env markers found for {case.item.originalname} in {case.path}. Ignoring...') + continue + + res[(case.target_selector, tuple(sorted(case.env_markers)))].append(case) + + target_test_jobs: t.List[Job] = [] + for (target_selector, env_markers), cases in res.items(): + runner_tags = get_tags_with_amount(target_selector) + list(env_markers) + # we don't need to get all runner, as long as we get one runner, it's fine + runner_list = project.runners.list(status='online', tag_list=','.join(runner_tags), get_all=False) + if not runner_list: + print(f'WARNING: No runner found with tag {",".join(runner_tags)}, ignoring the following test cases:') + for case in cases: + print(f' - {case.name}') + continue + + target_test_job = TargetTestJob( + name=f'{target_selector} - {",".join(env_markers)}', + tags=runner_tags, + parallel=len(cases) // DEFAULT_CASES_TEST_PER_JOB + 1, + ) + target_test_job.set_variable('TARGET_SELECTOR', f"'{target_selector}'") + target_test_job.set_variable('ENV_MARKERS', "'" + ' and '.join(env_markers) + "'") + target_test_job.set_variable('PYTEST_NODES', ' '.join([f"'{case.item.nodeid}'" for case in cases])) + + target_test_jobs.append(target_test_job) + + if not target_test_jobs: + print('No target test cases required, create one empty job instead') + target_test_jobs.append(EmptyJob()) + extra_include_yml = [] + else: + extra_include_yml = ['tools/ci/dynamic_pipelines/templates/generate_target_test_report.yml'] + + dump_jobs_to_yaml(target_test_jobs, output_filepath, extra_include_yml) + print(f'Generate child pipeline yaml file {output_filepath} with {sum(j.parallel for j in target_test_jobs)} jobs') + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description='Generate Target Test Child Pipeline. Update Build Report in MR pipelines', + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + parser.add_argument( + '-p', + '--paths', + nargs='+', + default=DEFAULT_TEST_PATHS, + help='Paths to the apps to build.', + ) + parser.add_argument( + '--project-id', + type=int, + default=os.getenv('CI_PROJECT_ID'), + help='Project ID', + ) + parser.add_argument( + '--pipeline-id', + type=int, + default=os.getenv('PARENT_PIPELINE_ID'), + help='Pipeline ID', + ) + parser.add_argument( + '-o', + '--output', + default=DEFAULT_TARGET_TEST_CHILD_PIPELINE_FILEPATH, + help='Output child pipeline file path', + ) + + args = parser.parse_args() + + app_list_filepattern = 'list_job_*.txt' + apps = [] + for f in glob.glob(app_list_filepattern): + apps.extend(import_apps_from_txt(f)) + + gl_project = Gitlab(args.project_id).project + generate_target_test_child_pipeline(gl_project, args.paths, apps, args.output) diff --git a/tools/ci/dynamic_pipelines/scripts/generate_target_test_report.py b/tools/ci/dynamic_pipelines/scripts/generate_target_test_report.py new file mode 100644 index 0000000000..cd736ad9f1 --- /dev/null +++ b/tools/ci/dynamic_pipelines/scripts/generate_target_test_report.py @@ -0,0 +1,62 @@ +# SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 + +import argparse +import glob +import os +import xml.etree.ElementTree as ET + +import __init__ # noqa: F401 # inject the system path +from dynamic_pipelines.models import TestCase +from dynamic_pipelines.report import TargetTestReportGenerator + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description='Update Build Report in MR pipelines', + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + parser.add_argument( + '--project-id', + type=int, + default=os.getenv('CI_PROJECT_ID'), + help='Project ID', + ) + parser.add_argument( + '--mr-iid', + type=int, + default=os.getenv('CI_MERGE_REQUEST_IID'), + help='Merge Request IID', + ) + parser.add_argument( + '--pipeline-id', + type=int, + default=os.getenv('PARENT_PIPELINE_ID'), + help='Pipeline ID', + ) + parser.add_argument( + '--job-id', + type=int, + default=os.getenv('CI_JOB_ID'), + help='Job ID', + ) + parser.add_argument( + '--commit-id', + default=os.getenv('CI_COMMIT_SHORT_SHA'), + help='MR commit ID', + ) + parser.add_argument( + '--junit-report-filepattern', + default='XUNIT_RESULT*.xml', + help='Junit Report file pattern', + ) + + args = parser.parse_args() + + test_cases = [] + for f in glob.glob(args.junit_report_filepattern): + root = ET.parse(f).getroot() + for tc in root.findall('.//testcase'): + test_cases.append(TestCase.from_test_case_node(tc)) + + report_generator = TargetTestReportGenerator(args.project_id, args.mr_iid, args.pipeline_id, test_cases=test_cases) + report_generator.post_report(args.job_id, args.commit_id) diff --git a/tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml b/tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml new file mode 100644 index 0000000000..781a97490d --- /dev/null +++ b/tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml @@ -0,0 +1,85 @@ +# This file is used to generate build jobs for pytest case dynamic pipeline +# don't add real jobs in this file + +######################## +# Build Jobs Templates # +######################## +.dynamic_build_template: + extends: + - .before_script:build + - .after_script:build:ccache:upload-when-fail + image: $ESP_ENV_IMAGE + stage: build + variables: + # Enable ccache for all build jobs. See configure_ci_environment.sh for more ccache related settings. + IDF_CCACHE_ENABLE: "1" + needs: + - pipeline: $PARENT_PIPELINE_ID + job: generate_build_child_pipeline + artifacts: + paths: + # The other artifacts patterns are defined under tools/ci/artifacts_handler.py + # Now we're uploading/downloading the binary files from our internal storage server + # + # keep the log file to help debug + - "**/build*/build_log.txt" + # build spec files + - build_summary_*.xml + # list of built apps + - list_job_*.txt + when: always + expire_in: 1 week + script: + # CI specific options start from "--parallel-count xxx". could ignore when running locally + - run_cmd python tools/ci/dynamic_pipelines/scripts/child_pipeline_build_apps.py $APP_LIST_FILE + --parallel-count ${CI_NODE_TOTAL:-1} + --parallel-index ${CI_NODE_INDEX:-1} + --collect-app-info "list_job_${CI_JOB_NAME_SLUG}.txt" + --modified-components ${MR_MODIFIED_COMPONENTS} + --junitxml "build_summary_${CI_JOB_NAME_SLUG}.xml" + +.dynamic_target_test_template: + extends: + - .before_script:fetch:target_test + image: $TARGET_TEST_ENV_IMAGE + stage: target_test + timeout: 1 hour + variables: + SUBMODULES_TO_FETCH: "none" + # set while generating the pipeline + PYTEST_NODES: "" + TARGET_SELECTOR: "" + ENV_MARKERS: "" + cache: + # Usually do not need submodule-cache in target_test + - key: pip-cache-${LATEST_GIT_TAG} + paths: + - .cache/pip + policy: pull + artifacts: + paths: + - XUNIT_RESULT*.xml + - pytest_embedded_log/ +# Child pipeline reports won't be collected in the main one +# https://gitlab.com/groups/gitlab-org/-/epics/8205 +# reports: +# junit: XUNIT_RESULT.xml + script: + # get known failure cases + - retry_failed git clone $KNOWN_FAILURE_CASES_REPO known_failure_cases + # get runner env config file + - retry_failed git clone $TEST_ENV_CONFIG_REPO + - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs + # CI specific options start from "--known-failure-cases-file xxx". could ignore when running locally + - run_cmd pytest ${PYTEST_NODES} + --target ${TARGET_SELECTOR} + -m ${ENV_MARKERS} + --pipeline-id $PARENT_PIPELINE_ID + --junitxml=XUNIT_RESULT_${CI_JOB_NAME_SLUG}.xml + --ignore-result-files known_failure_cases/known_failure_cases.txt + --parallel-count ${CI_NODE_TOTAL:-1} + --parallel-index ${CI_NODE_INDEX:-1} + ${PYTEST_EXTRA_FLAGS} + --app-info-filepattern "list_job_*.txt" + after_script: + - python tools/ci/artifacts_handler.py upload --type logs junit_reports diff --git a/tools/ci/dynamic_pipelines/templates/generate_target_test_report.yml b/tools/ci/dynamic_pipelines/templates/generate_target_test_report.yml new file mode 100644 index 0000000000..62cba4d77f --- /dev/null +++ b/tools/ci/dynamic_pipelines/templates/generate_target_test_report.yml @@ -0,0 +1,10 @@ +generate_pytest_report: + stage: .post + tags: [build, shiny] + image: $ESP_ENV_IMAGE + when: always + artifacts: + paths: + - target_test_report.html + script: + - python tools/ci/dynamic_pipelines/scripts/generate_target_test_report.py diff --git a/tools/ci/dynamic_pipelines/templates/report.template.html b/tools/ci/dynamic_pipelines/templates/report.template.html new file mode 100644 index 0000000000..f8da79ff7e --- /dev/null +++ b/tools/ci/dynamic_pipelines/templates/report.template.html @@ -0,0 +1,23 @@ + + + + + {{title}} + + + + + + +
{{table}}
+ + + + + diff --git a/tools/ci/dynamic_pipelines/templates/test_child_pipeline.yml b/tools/ci/dynamic_pipelines/templates/test_child_pipeline.yml new file mode 100644 index 0000000000..1b531790eb --- /dev/null +++ b/tools/ci/dynamic_pipelines/templates/test_child_pipeline.yml @@ -0,0 +1,41 @@ +generate_pytest_build_report: + stage: assign_test + image: $ESP_ENV_IMAGE + tags: + - build + - shiny + when: always + artifacts: + paths: + - build_report.html + - test_related_apps_download_urls.yml + script: + - python tools/ci/dynamic_pipelines/scripts/generate_build_report.py + +generate_pytest_child_pipeline: + # finally, we can get some use out of the default behavior that downloads all artifacts from the previous stage + stage: assign_test + image: $ESP_ENV_IMAGE + tags: + - build + - shiny + artifacts: + paths: + - target_test_child_pipeline.yml + script: + - python tools/ci/dynamic_pipelines/scripts/generate_target_test_child_pipeline.py + +Pytest Target Test Jobs: + stage: target_test + needs: + - generate_pytest_child_pipeline + variables: + PARENT_PIPELINE_ID: $PARENT_PIPELINE_ID + # https://gitlab.com/gitlab-org/gitlab/-/issues/214340 + inherit: + variables: false + trigger: + include: + - artifact: target_test_child_pipeline.yml + job: generate_pytest_child_pipeline + strategy: depend diff --git a/tools/ci/dynamic_pipelines/utils.py b/tools/ci/dynamic_pipelines/utils.py new file mode 100644 index 0000000000..25fee44caa --- /dev/null +++ b/tools/ci/dynamic_pipelines/utils.py @@ -0,0 +1,37 @@ +# SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 + +import typing as t + +import yaml + +from .models import Job + + +def dump_jobs_to_yaml( + jobs: t.List[Job], output_filepath: str, extra_include_yml: t.Optional[t.List[str]] = None +) -> None: + yaml_dict = {} + for job in jobs: + yaml_dict.update(job.to_dict()) + + # global stuffs + yaml_dict.update( + { + 'include': [ + 'tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml', + '.gitlab/ci/common.yml', + ], + # https://gitlab.com/gitlab-org/gitlab/-/issues/222370#note_662695503 + 'workflow': { + 'rules': [ + {'if': '$CI_MERGE_REQUEST_IID'}, + {'if': '$CI_COMMIT_BRANCH'}, + ], + }, + } + ) + yaml_dict['include'].extend(extra_include_yml or []) + + with open(output_filepath, 'w') as fw: + yaml.dump(yaml_dict, fw, indent=2) diff --git a/tools/ci/exclude_check_tools_files.txt b/tools/ci/exclude_check_tools_files.txt index 873ecd3e95..f8986d11aa 100644 --- a/tools/ci/exclude_check_tools_files.txt +++ b/tools/ci/exclude_check_tools_files.txt @@ -41,3 +41,11 @@ tools/ci/cleanup_ignore_lists.py tools/ci/artifacts_handler.py tools/unit-test-app/**/* tools/ci/gitlab_yaml_linter.py +tools/ci/dynamic_pipelines/**/* +tools/ci/idf_ci/**/* +tools/ci/get_supported_examples.sh +tools/ci/python_packages/common_test_methods.py +tools/ci/python_packages/gitlab_api.py +tools/ci/python_packages/idf_http_server_test/**/* +tools/ci/python_packages/idf_iperf_test_util/**/* +tools/esp_prov/**/* diff --git a/tools/ci/idf_ci/__init__.py b/tools/ci/idf_ci/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tools/ci/idf_ci/app.py b/tools/ci/idf_ci/app.py new file mode 100644 index 0000000000..7ade06a817 --- /dev/null +++ b/tools/ci/idf_ci/app.py @@ -0,0 +1,40 @@ +# SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 + +import sys +import typing as t +from typing import Literal + +from idf_build_apps import App, CMakeApp, json_to_app +from idf_ci.uploader import AppUploader, get_app_uploader + + +class IdfCMakeApp(CMakeApp): + uploader: t.ClassVar[t.Optional['AppUploader']] = get_app_uploader() + build_system: Literal['idf_cmake'] = 'idf_cmake' + + def _post_build(self) -> None: + super()._post_build() + + if self.uploader: + self.uploader.upload_app(self.build_path) + + +def dump_apps_to_txt(apps: t.List[App], output_filepath: str) -> None: + with open(output_filepath, 'w') as fw: + for app in apps: + fw.write(app.model_dump_json() + '\n') + + +def import_apps_from_txt(input_filepath: str) -> t.List[App]: + apps: t.List[App] = [] + with open(input_filepath) as fr: + for line in fr: + if line := line.strip(): + try: + apps.append(json_to_app(line, extra_classes=[IdfCMakeApp])) + except Exception: # noqa + print('Failed to deserialize app from line: %s' % line) + sys.exit(1) + + return apps diff --git a/tools/ci/idf_ci/uploader.py b/tools/ci/idf_ci/uploader.py new file mode 100644 index 0000000000..95b6819d57 --- /dev/null +++ b/tools/ci/idf_ci/uploader.py @@ -0,0 +1,150 @@ +# SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 + +import glob +import os +import typing as t +from datetime import timedelta +from zipfile import ZIP_DEFLATED, ZipFile + +import minio +from artifacts_handler import ArtifactType, get_minio_client, getenv +from idf_build_apps import App +from idf_build_apps.utils import rmdir +from idf_ci_utils import IDF_PATH +from idf_pytest.constants import DEFAULT_BUILD_LOG_FILENAME + + +class AppUploader: + TYPE_PATTERNS_DICT = { + ArtifactType.MAP_AND_ELF_FILES: [ + 'bootloader/*.map', + 'bootloader/*.elf', + '*.map', + '*.elf', + ], + ArtifactType.BUILD_DIR_WITHOUT_MAP_AND_ELF_FILES: [ + '*.bin', + 'bootloader/*.bin', + 'partition_table/*.bin', + 'flasher_args.json', + 'flash_project_args', + 'config/sdkconfig.json', + 'project_description.json', + ], + ArtifactType.LOGS: [ + DEFAULT_BUILD_LOG_FILENAME, + ], + } + + def __init__(self, pipeline_id: t.Union[str, int, None] = None) -> None: + self.pipeline_id = str(pipeline_id or '1') + + self._client = get_minio_client() + + def get_app_object_name(self, app_path: str, zip_name: str, artifact_type: ArtifactType) -> str: + return f'{self.pipeline_id}/{artifact_type.value}/{app_path}/{zip_name}' + + def _upload_app(self, app_build_path: str, artifact_type: ArtifactType) -> bool: + app_path, build_dir = os.path.split(app_build_path) + zip_filename = f'{build_dir}.zip' + + has_file = False + with ZipFile( + zip_filename, + 'w', + compression=ZIP_DEFLATED, + # 1 is the fastest compression level + # the size differs not much between 1 and 9 + compresslevel=1, + ) as zw: + for pattern in self.TYPE_PATTERNS_DICT[artifact_type]: + for file in glob.glob(os.path.join(app_build_path, pattern), recursive=True): + zw.write(file) + has_file = True + + uploaded = False + try: + if has_file: + obj_name = self.get_app_object_name(app_path, zip_filename, artifact_type) + print(f'Created archive file: {zip_filename}, uploading as {obj_name}') + self._client.fput_object(getenv('IDF_S3_BUCKET'), obj_name, zip_filename) + uploaded = True + finally: + os.remove(zip_filename) + return uploaded + + def upload_app(self, app_build_path: str, artifact_type: t.Optional[ArtifactType] = None) -> None: + uploaded = False + if not artifact_type: + for _artifact_type in [ + ArtifactType.MAP_AND_ELF_FILES, + ArtifactType.BUILD_DIR_WITHOUT_MAP_AND_ELF_FILES, + ArtifactType.LOGS, + ]: + uploaded |= self._upload_app(app_build_path, _artifact_type) + else: + uploaded = self._upload_app(app_build_path, artifact_type) + + if uploaded: + rmdir(app_build_path, exclude_file_patterns=DEFAULT_BUILD_LOG_FILENAME) + + def _download_app(self, app_build_path: str, artifact_type: ArtifactType) -> None: + app_path, build_dir = os.path.split(app_build_path) + zip_filename = f'{build_dir}.zip' + + # path are relative to IDF_PATH + current_dir = os.getcwd() + os.chdir(IDF_PATH) + try: + obj_name = self.get_app_object_name(app_path, zip_filename, artifact_type) + print(f'Downloading {obj_name}') + try: + try: + self._client.stat_object(getenv('IDF_S3_BUCKET'), obj_name) + except minio.error.S3Error as e: + raise SystemExit( + f'No such file on minio server: {obj_name}. ' + f'Probably the build failed or the artifacts got expired. ' + f'Full error message: {str(e)}' + ) + else: + self._client.fget_object(getenv('IDF_S3_BUCKET'), obj_name, zip_filename) + print(f'Downloaded to {zip_filename}') + except minio.error.S3Error as e: + raise SystemExit('Shouldn\'t happen, please report this bug in the CI channel' + str(e)) + + with ZipFile(zip_filename, 'r') as zr: + zr.extractall() + + os.remove(zip_filename) + finally: + os.chdir(current_dir) + + def download_app(self, app_build_path: str, artifact_type: t.Optional[ArtifactType] = None) -> None: + if not artifact_type: + for _artifact_type in [ArtifactType.MAP_AND_ELF_FILES, ArtifactType.BUILD_DIR_WITHOUT_MAP_AND_ELF_FILES]: + self._download_app(app_build_path, _artifact_type) + else: + self._download_app(app_build_path, artifact_type) + + def get_app_presigned_url(self, app: App, artifact_type: ArtifactType) -> str: + obj_name = self.get_app_object_name(app.app_dir, f'{app.build_dir}.zip', artifact_type) + try: + self._client.stat_object( + getenv('IDF_S3_BUCKET'), + obj_name, + ) + except minio.error.S3Error: + return '' + else: + return self._client.get_presigned_url( # type: ignore + 'GET', getenv('IDF_S3_BUCKET'), obj_name, expires=timedelta(days=4) + ) + + +def get_app_uploader() -> t.Optional['AppUploader']: + if parent_pipeline_id := os.getenv('PARENT_PIPELINE_ID'): + return AppUploader(parent_pipeline_id) + + return None diff --git a/tools/ci/idf_ci_utils.py b/tools/ci/idf_ci_utils.py index db38d8a988..9ddc40ded5 100644 --- a/tools/ci/idf_ci_utils.py +++ b/tools/ci/idf_ci_utils.py @@ -10,6 +10,7 @@ import subprocess import sys import typing as t from functools import cached_property +from pathlib import Path IDF_PATH = os.path.abspath(os.getenv('IDF_PATH', os.path.join(os.path.dirname(__file__), '..', '..'))) @@ -99,10 +100,6 @@ def get_git_files(path: str = IDF_PATH, full_path: bool = False) -> t.List[str]: return [os.path.join(path, f) for f in files] if full_path else files -def is_in_directory(file_path: str, folder: str) -> bool: - return os.path.realpath(file_path).startswith(os.path.realpath(folder) + os.sep) - - def to_list(s: t.Any) -> t.List[t.Any]: if not s: return [] @@ -178,3 +175,19 @@ class GitlabYmlConfig: @staticmethod def _is_rule_key(key: str) -> bool: return key.startswith('.rules:') or key.endswith('template') + + +def get_all_manifest_files() -> t.List[str]: + """ + + :rtype: object + """ + paths: t.List[str] = [] + + for p in Path(IDF_PATH).glob('**/.build-test-rules.yml'): + if 'managed_components' in p.parts: + continue + + paths.append(str(p)) + + return paths diff --git a/tools/ci/idf_pytest/constants.py b/tools/ci/idf_pytest/constants.py index ccbe00fb0d..2eec0efda1 100644 --- a/tools/ci/idf_pytest/constants.py +++ b/tools/ci/idf_pytest/constants.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2023-2024 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 """ @@ -6,12 +6,13 @@ Pytest Related Constants. Don't import third-party packages here. """ import os import typing as t -from collections import Counter from dataclasses import dataclass from enum import Enum from functools import cached_property +from pathlib import Path from _pytest.python import Function +from idf_ci_utils import IDF_PATH from pytest_embedded.utils import to_list SUPPORTED_TARGETS = ['esp32', 'esp32s2', 'esp32c3', 'esp32s3', 'esp32c2', 'esp32c6', 'esp32h2', 'esp32p4'] @@ -113,9 +114,34 @@ ENV_MARKERS = { 'sdio_master_slave': 'Test sdio multi board, esp32+esp32', 'sdio_multidev_32_c6': 'Test sdio multi board, esp32+esp32c6', 'usj_device': 'Test usb_serial_jtag and usb_serial_jtag is used as serial only (not console)', - 'twai_std': 'twai runner with all twai supported targets connect to usb-can adapter' + 'twai_std': 'twai runner with all twai supported targets connect to usb-can adapter', } +DEFAULT_CONFIG_RULES_STR = ['sdkconfig.ci=default', 'sdkconfig.ci.*=', '=default'] +DEFAULT_IGNORE_WARNING_FILEPATH = os.path.join(IDF_PATH, 'tools', 'ci', 'ignore_build_warnings.txt') +DEFAULT_BUILD_TEST_RULES_FILEPATH = os.path.join(IDF_PATH, '.gitlab', 'ci', 'default-build-test-rules.yml') +DEFAULT_FULL_BUILD_TEST_FILEPATTERNS = [ + # tools + 'tools/cmake/**/*', + 'tools/tools.json', + # components + 'components/cxx/**/*', + 'components/esp_common/**/*', + 'components/esp_hw_support/**/*', + 'components/esp_rom/**/*', + 'components/esp_system/**/*', + 'components/esp_timer/**/*', + 'components/freertos/**/*', + 'components/hal/**/*', + 'components/heap/**/*', + 'components/log/**/*', + 'components/newlib/**/*', + 'components/riscv/**/*', + 'components/soc/**/*', + 'components/xtensa/**/*', +] +DEFAULT_BUILD_LOG_FILENAME = 'build_log.txt' + class CollectMode(str, Enum): SINGLE_SPECIFIC = 'single_specific' @@ -163,6 +189,10 @@ class PytestCase: def is_single_dut_test_case(self) -> bool: return True if len(self.apps) == 1 else False + @cached_property + def is_host_test(self) -> bool: + return 'host_test' in self.all_markers or 'linux' in self.targets + # the following markers could be changed dynamically, don't use cached_property @property def all_markers(self) -> t.Set[str]: @@ -202,24 +232,35 @@ class PytestCase: return {marker for marker in self.all_markers if marker in ENV_MARKERS} @property - def target_with_amount_markers(self) -> t.Set[str]: - c: Counter = Counter() - for app in self.apps: - c[app.target] += 1 + def target_selector(self) -> str: + return ','.join(app.target for app in self.apps) - res = set() - for target, amount in c.items(): - if amount > 1: - res.add(f'{target}_{amount}') - else: - res.add(target) + @property + def requires_elf_or_map(self) -> bool: + """ + This property determines whether the test case requires elf or map file. By default, one app in the test case + only requires .bin files. - return res + :return: True if the test case requires elf or map file, False otherwise + """ + if 'jtag' in self.env_markers or 'usb_serial_jtag' in self.env_markers: + return True - def all_built_in_app_lists(self, app_lists: t.Optional[t.List[str]] = None) -> bool: + if any('panic' in Path(app.path).parts for app in self.apps): + return True + + return False + + def all_built_in_app_lists(self, app_lists: t.Optional[t.List[str]] = None) -> t.Optional[str]: + """ + Check if all binaries of the test case are built in the app lists. + + :param app_lists: app lists to check + :return: debug string if not all binaries are built in the app lists, None otherwise + """ if app_lists is None: # ignore this feature - return True + return None bin_found = [0] * len(self.apps) for i, app in enumerate(self.apps): @@ -232,10 +273,10 @@ class PytestCase: msg += f'\n - {app.build_dir}' print(msg) - return False + return msg if sum(bin_found) == len(self.apps): - return True + return None # some found, some not, looks suspicious msg = f'Found some binaries of test case {self.name} are not listed in the app lists.' @@ -244,4 +285,5 @@ class PytestCase: msg += f'\n - {app.build_dir}' msg += '\nMight be a issue of .build-test-rules.yml files' - return False + print(msg) + return msg diff --git a/tools/ci/idf_pytest/plugin.py b/tools/ci/idf_pytest/plugin.py index efae926208..af9e61e247 100644 --- a/tools/ci/idf_pytest/plugin.py +++ b/tools/ci/idf_pytest/plugin.py @@ -1,8 +1,9 @@ -# SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2023-2024 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 import os import typing as t +from collections import defaultdict from functools import cached_property from xml.etree import ElementTree as ET @@ -11,6 +12,8 @@ from _pytest.config import ExitCode from _pytest.main import Session from _pytest.python import Function from _pytest.runner import CallInfo +from idf_build_apps import App +from idf_build_apps.constants import BuildStatus from pytest_embedded import Dut from pytest_embedded.plugin import parse_multi_dut_args from pytest_embedded.utils import find_by_suffix, to_list @@ -37,7 +40,7 @@ class IdfPytestEmbedded: target: t.Union[t.List[str], str], *, single_target_duplicate_mode: bool = False, - apps_list: t.Optional[t.List[str]] = None, + apps: t.Optional[t.List[App]] = None, ): if isinstance(target, str): self.target = sorted(comma_sep_str_to_list(target)) @@ -60,10 +63,18 @@ class IdfPytestEmbedded: # otherwise, it should be collected when running `pytest --target esp32,esp32` self._single_target_duplicate_mode = single_target_duplicate_mode - self.apps_list = apps_list + self.apps_list = ( + [os.path.join(app.app_dir, app.build_dir) for app in apps if app.build_status == BuildStatus.SUCCESS] + if apps + else None + ) self.cases: t.List[PytestCase] = [] + # record the additional info + # test case id: {key: value} + self.additional_info: t.Dict[str, t.Dict[str, t.Any]] = defaultdict(dict) + @cached_property def collect_mode(self) -> CollectMode: if len(self.target) == 1: @@ -90,13 +101,19 @@ class IdfPytestEmbedded: count = self.get_param(item, 'count', 1) # default app_path is where the test script locates - app_paths = to_list( - parse_multi_dut_args(count, os.path.relpath(self.get_param(item, 'app_path', os.path.dirname(item.path)))) - ) + app_paths = to_list(parse_multi_dut_args(count, self.get_param(item, 'app_path', os.path.dirname(item.path)))) configs = to_list(parse_multi_dut_args(count, self.get_param(item, 'config', DEFAULT_SDKCONFIG))) targets = to_list(parse_multi_dut_args(count, self.get_param(item, 'target', self.target[0]))) - return PytestCase([PytestApp(app_paths[i], targets[i], configs[i]) for i in range(count)], item) + def abspath_or_relpath(s: str) -> str: + if os.path.abspath(s) and s.startswith(os.getcwd()): + return os.path.relpath(s) + + return s + + return PytestCase( + [PytestApp(abspath_or_relpath(app_paths[i]), targets[i], configs[i]) for i in range(count)], item + ) @pytest.hookimpl(tryfirst=True) def pytest_collection_modifyitems(self, items: t.List[Function]) -> None: @@ -189,11 +206,17 @@ class IdfPytestEmbedded: # 4. filter by `self.apps_list`, skip the test case if not listed # should only be used in CI - items[:] = [_item for _item in items if item_to_case_dict[_item].all_built_in_app_lists(self.apps_list)] + _items = [] + for item in items: + case = item_to_case_dict[item] + if msg := case.all_built_in_app_lists(self.apps_list): + self.additional_info[case.name]['skip_reason'] = msg + else: + _items.append(item) # OKAY!!! All left ones will be executed, sort it and add more markers items[:] = sorted( - items, key=lambda x: (os.path.dirname(x.path), self.get_param(x, 'config', DEFAULT_SDKCONFIG)) + _items, key=lambda x: (os.path.dirname(x.path), self.get_param(x, 'config', DEFAULT_SDKCONFIG)) ) for item in items: case = item_to_case_dict[item] @@ -207,8 +230,7 @@ class IdfPytestEmbedded: item.add_marker('xtal_40mhz') def pytest_report_collectionfinish(self, items: t.List[Function]) -> None: - for item in items: - self.cases.append(self.item_to_pytest_case(item)) + self.cases = [item.stash[ITEM_PYTEST_CASE_KEY] for item in items] def pytest_custom_test_case_name(self, item: Function) -> str: return item.funcargs.get('test_case_name', item.nodeid) # type: ignore @@ -275,6 +297,9 @@ class IdfPytestEmbedded: if 'file' in case.attrib: case.attrib['file'] = case.attrib['file'].replace('/IDF/', '') # our unity test framework + if ci_job_url := os.getenv('CI_JOB_URL'): + case.attrib['ci_job_url'] = ci_job_url + xml.write(junit) def pytest_sessionfinish(self, session: Session, exitstatus: int) -> None: diff --git a/tools/ci/idf_pytest/script.py b/tools/ci/idf_pytest/script.py index 7f16bc91e6..7e3232c0c7 100644 --- a/tools/ci/idf_pytest/script.py +++ b/tools/ci/idf_pytest/script.py @@ -1,18 +1,24 @@ # SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 +import fnmatch import io +import logging +import os.path import typing as t from contextlib import redirect_stdout from pathlib import Path import pytest from _pytest.config import ExitCode +from idf_build_apps import App, find_apps +from idf_build_apps.constants import SUPPORTED_TARGETS, BuildStatus +from idf_ci.app import IdfCMakeApp +from idf_ci_utils import IDF_PATH, get_all_manifest_files, to_list from idf_py_actions.constants import PREVIEW_TARGETS as TOOLS_PREVIEW_TARGETS from idf_py_actions.constants import SUPPORTED_TARGETS as TOOLS_SUPPORTED_TARGETS -from pytest_embedded.utils import to_list -from .constants import CollectMode, PytestCase +from .constants import DEFAULT_BUILD_LOG_FILENAME, DEFAULT_CONFIG_RULES_STR, CollectMode, PytestCase from .plugin import IdfPytestEmbedded @@ -36,8 +42,10 @@ def get_pytest_files(paths: t.List[str]) -> t.List[str]: def get_pytest_cases( paths: t.Union[str, t.List[str]], target: str = CollectMode.ALL, + *, marker_expr: t.Optional[str] = None, filter_expr: t.Optional[str] = None, + apps: t.Optional[t.List[App]] = None, ) -> t.List[PytestCase]: """ For single-dut test cases, `target` could be @@ -49,9 +57,10 @@ def get_pytest_cases( - or `multi_all`, to get all multi-dut test cases :param paths: paths to search for pytest scripts - :param target: target to get test cases for, detailed above + :param target: target or keywords to get test cases for, detailed above :param marker_expr: pytest marker expression, `-m` :param filter_expr: pytest filter expression, `-k` + :param apps: built app list, skip the tests required by apps not in the list :return: list of test cases """ paths = to_list(paths) @@ -63,7 +72,7 @@ def get_pytest_cases( return cases def _get_pytest_cases(_target: str, _single_target_duplicate_mode: bool = False) -> t.List[PytestCase]: - collector = IdfPytestEmbedded(_target, single_target_duplicate_mode=_single_target_duplicate_mode) + collector = IdfPytestEmbedded(_target, single_target_duplicate_mode=_single_target_duplicate_mode, apps=apps) with io.StringIO() as buf: with redirect_stdout(buf): @@ -97,3 +106,108 @@ def get_pytest_cases( cases.extend(_get_pytest_cases(_target)) return sorted(cases, key=lambda x: (x.path, x.name, str(x.targets))) + + +def get_all_apps( + paths: t.List[str], + target: str = CollectMode.ALL, + *, + marker_expr: t.Optional[str] = None, + filter_expr: t.Optional[str] = None, + config_rules_str: t.Optional[t.List[str]] = None, + preserve_all: bool = False, + extra_default_build_targets: t.Optional[t.List[str]] = None, + modified_components: t.Optional[t.List[str]] = None, + modified_files: t.Optional[t.List[str]] = None, + ignore_app_dependencies_filepatterns: t.Optional[t.List[str]] = None, +) -> t.Tuple[t.Set[App], t.Set[App]]: + """ + Return the tuple of test-required apps and non-test-related apps + + :param paths: paths to search for pytest scripts + :param target: target or keywords to get test cases for, explained in `get_pytest_cases` + :param marker_expr: pytest marker expression, `-m` + :param filter_expr: pytest filter expression, `-k` + :param config_rules_str: config rules string + :param preserve_all: preserve all apps + :param extra_default_build_targets: extra default build targets + :param modified_components: modified components + :param modified_files: modified files + :param ignore_app_dependencies_filepatterns: ignore app dependencies filepatterns + :return: tuple of test-required apps and non-test-related apps + """ + all_apps = find_apps( + paths, + target, + build_system=IdfCMakeApp, + recursive=True, + build_dir='build_@t_@w', + config_rules_str=config_rules_str or DEFAULT_CONFIG_RULES_STR, + build_log_filename=DEFAULT_BUILD_LOG_FILENAME, + size_json_filename='size.json', + check_warnings=True, + manifest_rootpath=IDF_PATH, + manifest_files=get_all_manifest_files(), + default_build_targets=SUPPORTED_TARGETS + (extra_default_build_targets or []), + modified_components=modified_components, + modified_files=modified_files, + ignore_app_dependencies_filepatterns=ignore_app_dependencies_filepatterns, + include_skipped_apps=True, + ) + + pytest_cases = get_pytest_cases( + paths, + target, + marker_expr=marker_expr, + filter_expr=filter_expr, + ) + + modified_pytest_cases = [] + if modified_files: + modified_pytest_scripts = [ + os.path.dirname(f) for f in modified_files if fnmatch.fnmatch(os.path.basename(f), 'pytest_*.py') + ] + if modified_pytest_scripts: + modified_pytest_cases = get_pytest_cases( + modified_pytest_scripts, + target, + marker_expr=marker_expr, + filter_expr=filter_expr, + ) + + # app_path, target, config + pytest_app_path_tuple_dict: t.Dict[t.Tuple[Path, str, str], PytestCase] = {} + for case in pytest_cases: + for app in case.apps: + pytest_app_path_tuple_dict[(Path(app.path), app.target, app.config)] = case + + modified_pytest_app_path_tuple_dict: t.Dict[t.Tuple[Path, str, str], PytestCase] = {} + for case in modified_pytest_cases: + for app in case.apps: + modified_pytest_app_path_tuple_dict[(Path(app.path), app.target, app.config)] = case + + test_related_apps: t.Set[App] = set() + non_test_related_apps: t.Set[App] = set() + for app in all_apps: + # override build_status if test script got modified + if case := modified_pytest_app_path_tuple_dict.get((Path(app.app_dir), app.target, app.config_name)): + test_related_apps.add(app) + app.build_status = BuildStatus.SHOULD_BE_BUILT + app.preserve = True + logging.debug('Found app: %s - required by modified test case %s', app, case.path) + elif app.build_status != BuildStatus.SKIPPED: + if case := pytest_app_path_tuple_dict.get((Path(app.app_dir), app.target, app.config_name)): + test_related_apps.add(app) + # should be built if + app.build_status = BuildStatus.SHOULD_BE_BUILT + app.preserve = True + logging.debug('Found test-related app: %s - required by %s', app, case.path) + else: + non_test_related_apps.add(app) + app.preserve = preserve_all + logging.debug('Found non-test-related app: %s', app) + + print(f'Found {len(test_related_apps)} test-related apps') + print(f'Found {len(non_test_related_apps)} non-test-related apps') + + return test_related_apps, non_test_related_apps diff --git a/tools/ci/idf_pytest/tests/conftest.py b/tools/ci/idf_pytest/tests/conftest.py new file mode 100644 index 0000000000..bfd05d8dfb --- /dev/null +++ b/tools/ci/idf_pytest/tests/conftest.py @@ -0,0 +1,48 @@ +# SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 + +import os +import sys +from pathlib import Path + +tools_ci_dir = os.path.join(os.path.dirname(__file__), '..', '..') +if tools_ci_dir not in sys.path: + sys.path.append(tools_ci_dir) + +tools_dir = os.path.join(os.path.dirname(__file__), '..', '..', '..') +if tools_dir not in sys.path: + sys.path.append(tools_dir) + + +def create_project(name: str, folder: Path) -> Path: + p = folder / name + p.mkdir(parents=True, exist_ok=True) + (p / 'main').mkdir(parents=True, exist_ok=True) + + with open(p / 'CMakeLists.txt', 'w') as fw: + fw.write( + """cmake_minimum_required(VERSION 3.16) +include($ENV{{IDF_PATH}}/tools/cmake/project.cmake) +project({}) +""".format( + name + ) + ) + + with open(p / 'main' / 'CMakeLists.txt', 'w') as fw: + fw.write( + """idf_component_register(SRCS "{}.c" +INCLUDE_DIRS ".") +""".format( + name + ) + ) + + with open(p / 'main' / f'{name}.c', 'w') as fw: + fw.write( + """#include +void app_main(void) {} +""" + ) + + return p diff --git a/tools/ci/idf_pytest/tests/test_get_all_apps.py b/tools/ci/idf_pytest/tests/test_get_all_apps.py new file mode 100644 index 0000000000..e2aa2c95b2 --- /dev/null +++ b/tools/ci/idf_pytest/tests/test_get_all_apps.py @@ -0,0 +1,100 @@ +# SPDX-FileCopyrightText: 2023-2024 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 + +from pathlib import Path + +from idf_pytest.script import SUPPORTED_TARGETS, get_all_apps + +from conftest import create_project + + +def test_get_all_apps_non(tmp_path: Path) -> None: + create_project('foo', tmp_path) + create_project('bar', tmp_path) + + test_related_apps, non_test_related_apps = get_all_apps([str(tmp_path)]) + + assert test_related_apps == set() + assert len(non_test_related_apps) == 2 * len(SUPPORTED_TARGETS) + + +def test_get_all_apps_single_dut_test_script(tmp_path: Path) -> None: + create_project('foo', tmp_path) + with open(tmp_path / 'foo' / 'pytest_get_all_apps_single_dut_test_script.py', 'w') as fw: + fw.write( + """import pytest + +@pytest.mark.esp32 +@pytest.mark.esp32s2 +def test_foo(dut): + pass +""" + ) + create_project('bar', tmp_path) + + test_related_apps, non_test_related_apps = get_all_apps([str(tmp_path)], target='all') + + assert len(test_related_apps) == 2 + assert len(non_test_related_apps) == 2 * len(SUPPORTED_TARGETS) - 2 + + +def test_get_all_apps_multi_dut_test_script(tmp_path: Path) -> None: + create_project('foo', tmp_path) + with open(tmp_path / 'foo' / 'pytest_get_all_apps_multi_dut_test_script.py', 'w') as fw: + fw.write( + """import pytest + +@pytest.mark.parametrize( + 'count, target', [ + (2, 'esp32s2|esp32s3'), + (3, 'esp32|esp32s3|esp32'), + ], indirect=True +) +def test_foo(dut): + pass +""" + ) + + test_related_apps, non_test_related_apps = get_all_apps([str(tmp_path)], target='all') + + assert len(test_related_apps) == 3 # 32, s2, s3 + assert len(non_test_related_apps) == len(SUPPORTED_TARGETS) - 3 + + +def test_get_all_apps_modified_pytest_script(tmp_path: Path) -> None: + create_project('foo', tmp_path) + create_project('bar', tmp_path) + + (tmp_path / 'pytest_get_all_apps_modified_pytest_script.py').write_text( + """import pytest +import os + +@pytest.mark.parametrize('count, target', [(2, 'esp32')], indirect=True) +@pytest.mark.parametrize('app_path', [ + '{}|{}'.format(os.path.join(os.path.dirname(__file__), 'foo'), os.path.join(os.path.dirname(__file__), 'bar')), + ], indirect=True +) +def test_multi_foo_bar(dut): + pass +""", + encoding='utf-8', + ) + + test_related_apps, non_test_related_apps = get_all_apps([str(tmp_path)], target='all') + assert len(test_related_apps) == 2 # foo-esp32, bar-esp32 + assert len(non_test_related_apps) == 2 * len(SUPPORTED_TARGETS) - 2 + + test_related_apps, non_test_related_apps = get_all_apps( + [str(tmp_path)], target='all', modified_files=[], modified_components=[] + ) + assert len(test_related_apps) == 0 + assert len(non_test_related_apps) == 0 + + test_related_apps, non_test_related_apps = get_all_apps( + [str(tmp_path)], + target='all', + modified_files=[str(tmp_path / 'pytest_get_all_apps_modified_pytest_script.py')], + modified_components=[], + ) + assert len(test_related_apps) == 2 + assert len(non_test_related_apps) == 0 diff --git a/tools/ci/idf_pytest/tests/test_get_pytest_cases.py b/tools/ci/idf_pytest/tests/test_get_pytest_cases.py index 64a54bcead..e078146ce4 100644 --- a/tools/ci/idf_pytest/tests/test_get_pytest_cases.py +++ b/tools/ci/idf_pytest/tests/test_get_pytest_cases.py @@ -1,18 +1,10 @@ # SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 -import os -import sys from pathlib import Path from idf_pytest.constants import CollectMode - -try: - from idf_pytest.script import get_pytest_cases -except ImportError: - sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..')) - - from idf_pytest.script import get_pytest_cases +from idf_pytest.script import get_pytest_cases TEMPLATE_SCRIPT = ''' import pytest diff --git a/tools/requirements/requirements.ci.txt b/tools/requirements/requirements.ci.txt index 0e5c231133..4ab65cf374 100644 --- a/tools/requirements/requirements.ci.txt +++ b/tools/requirements/requirements.ci.txt @@ -3,7 +3,7 @@ # ci coverage -idf-build-apps +idf-build-apps~=2.0.0rc1 jsonschema junit_xml python-gitlab diff --git a/tools/requirements/requirements.pytest.txt b/tools/requirements/requirements.pytest.txt index 8ca65461b5..3c6dcbf223 100644 --- a/tools/requirements/requirements.pytest.txt +++ b/tools/requirements/requirements.pytest.txt @@ -10,7 +10,7 @@ pytest-timeout pytest-ignore-test-results # build -idf-build-apps +idf-build-apps~=2.0.0rc1 # dependencies in pytest test scripts scapy