From 9a9438b79ab1aa6d93276472b4a41cee4d150b02 Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Tue, 13 May 2025 09:40:23 +0200 Subject: [PATCH 1/2] ci: improve rules for deploy jobs --- .gitlab/ci/deploy.yml | 6 +- .gitlab/ci/docs.yml | 15 ++--- .gitlab/ci/post_deploy.yml | 17 ++++-- .gitlab/ci/pre_check.yml | 2 +- .gitlab/ci/rules.yml | 71 +++++++++--------------- .gitlab/ci/static-code-analysis.yml | 2 +- tools/ci/generate_rules.py | 86 ++++++++++++++--------------- 7 files changed, 89 insertions(+), 110 deletions(-) diff --git a/.gitlab/ci/deploy.yml b/.gitlab/ci/deploy.yml index 1df1004ee8..eb4fb62130 100644 --- a/.gitlab/ci/deploy.yml +++ b/.gitlab/ci/deploy.yml @@ -3,7 +3,6 @@ image: $ESP_ENV_IMAGE tags: [ deploy ] -# Check this before push_to_github check_submodule_sync: extends: - .deploy_job_template @@ -31,8 +30,9 @@ push_to_github: extends: - .deploy_job_template - .before_script:minimal - - .rules:push_to_github + - .rules:protected:deploy needs: + # submodule must be synced before pushing to github - check_submodule_sync tags: [ brew, github_sync ] variables: @@ -50,7 +50,7 @@ deploy_update_SHA_in_esp-dockerfiles: extends: - .deploy_job_template - .before_script:minimal - - .rules:protected-no_label-always + - .rules:protected:deploy dependencies: [] variables: GIT_DEPTH: 2 diff --git a/.gitlab/ci/docs.yml b/.gitlab/ci/docs.yml index 89bf179e9d..f0c1434d88 100644 --- a/.gitlab/ci/docs.yml +++ b/.gitlab/ci/docs.yml @@ -19,12 +19,9 @@ .patterns-docs-preview: &patterns-docs-preview - "docs/**/*" -.if-protected: &if-protected +.if-protected-check: &if-protected-check if: '($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_BRANCH =~ /^release\/v/ || $CI_COMMIT_TAG =~ /^v\d+\.\d+(\.\d+)?($|-)/)' -.if-protected-no_label: &if-protected-no_label - if: '($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_BRANCH =~ /^release\/v/ || $CI_COMMIT_TAG =~ /^v\d+\.\d+(\.\d+)?($|-)/) && $BOT_TRIGGER_WITH_LABEL == null' - .if-qa-test-tag: &if-qa-test-tag if: '$CI_COMMIT_TAG =~ /^qa-test/' @@ -41,7 +38,7 @@ rules: - <<: *if-qa-test-tag when: never - - <<: *if-protected + - <<: *if-protected-check - <<: *if-label-build_docs - <<: *if-label-docs_full - <<: *if-dev-push @@ -64,7 +61,7 @@ check_readme_links: tags: ["build", "amd64", "internet"] allow_failure: true rules: - - <<: *if-protected + - <<: *if-protected-check - <<: *if-dev-push changes: *patterns-example-readme script: @@ -188,8 +185,7 @@ deploy_docs_production: # The DOCS_PROD_* variables used by this job are "Protected" so these branches must all be marked "Protected" in Gitlab settings extends: - .deploy_docs_template - rules: - - <<: *if-protected-no_label + - .rules:protected:deploy stage: post_deploy dependencies: # set dependencies to null to avoid missing artifacts issue needs: # ensure runs after push_to_github succeeded @@ -208,8 +204,7 @@ deploy_docs_production: check_doc_links: extends: - .build_docs_template - rules: - - <<: *if-protected-no_label + - .rules:protected:deploy stage: post_deploy needs: - job: deploy_docs_production diff --git a/.gitlab/ci/post_deploy.yml b/.gitlab/ci/post_deploy.yml index efa91f44eb..41050c2eaa 100644 --- a/.gitlab/ci/post_deploy.yml +++ b/.gitlab/ci/post_deploy.yml @@ -1,7 +1,11 @@ -generate_failed_jobs_report: +.post_deploy_template: stage: post_deploy - tags: [build, shiny] image: $ESP_ENV_IMAGE + +generate_failed_jobs_report: + extends: + - .post_deploy_template + tags: [build, shiny] when: always dependencies: [] # Do not download artifacts from the previous stages artifacts: @@ -13,12 +17,13 @@ generate_failed_jobs_report: - python tools/ci/dynamic_pipelines/scripts/generate_report.py --report-type job sync_support_status: - stage: post_deploy extends: - - .rules:sync_support_status + - .post_deploy_template + - .rules:master:push + tags: [ brew, github_sync ] needs: - push_to_github - image: $ESP_ENV_IMAGE - tags: [ brew, github_sync ] + cache: [] + before_script: [] script: - curl --fail --request POST --form token="$IDF_STATUS_TRIG_TOKEN" --form ref="$IDF_STATUS_BRANCH" --form "variables[UPLOAD_TO_S3]=true" "$IDF_STATUS_TRIG_URL" diff --git a/.gitlab/ci/pre_check.yml b/.gitlab/ci/pre_check.yml index 29e89745e2..5dc66bef8c 100644 --- a/.gitlab/ci/pre_check.yml +++ b/.gitlab/ci/pre_check.yml @@ -10,7 +10,7 @@ check_version: # esp_idf_version.h in a branch before tagging the next version. extends: - .pre_check_template - - .rules:protected + - .rules:protected:check tags: [ brew, github_sync ] variables: # need a full clone to get the latest tag diff --git a/.gitlab/ci/rules.yml b/.gitlab/ci/rules.yml index 6cb01c6558..64c1d1c859 100644 --- a/.gitlab/ci/rules.yml +++ b/.gitlab/ci/rules.yml @@ -167,22 +167,19 @@ ############## # if anchors # ############## -.if-ref-master: &if-ref-master - if: '$CI_COMMIT_REF_NAME == "master"' +.if-master-push: &if-master-push + if: '$CI_COMMIT_REF_NAME == "master" && $CI_PIPELINE_SOURCE == "push"' -.if-ref-master-no_label: &if-ref-master-no_label - if: '$CI_COMMIT_REF_NAME == "master" && $BOT_TRIGGER_WITH_LABEL == null' - -.if-tag-release: &if-tag-release +.if-release-tag: &if-release-tag if: '$CI_COMMIT_TAG =~ /^v\d+\.\d+(\.\d+)?($|-)/' -.if-protected: &if-protected +.if-protected-check: &if-protected-check if: '($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_BRANCH =~ /^release\/v/ || $CI_COMMIT_TAG =~ /^v\d+\.\d+(\.\d+)?($|-)/) || $CI_COMMIT_TAG =~ /^qa-test/' -.if-protected-no_label: &if-protected-no_label - if: '($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_BRANCH =~ /^release\/v/ || $CI_COMMIT_TAG =~ /^v\d+\.\d+(\.\d+)?($|-)/) && $BOT_TRIGGER_WITH_LABEL == null' +.if-protected-deploy: &if-protected-deploy + if: '($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_BRANCH =~ /^release\/v/ || $CI_COMMIT_TAG =~ /^v\d+\.\d+(\.\d+)?($|-)/) && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "api")' -.if-protected-ref-push: &if-protected-ref-push +.if-protected-branch-push: &if-protected-branch-push # rules:changes always evaluates to true for new branch pipelines or when there is no Git push event if: '($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_BRANCH =~ /^release\/v/) && $CI_PIPELINE_SOURCE == "push"' @@ -192,9 +189,6 @@ .if-dev-push: &if-dev-push if: '$CI_COMMIT_REF_NAME != "master" && $CI_COMMIT_BRANCH !~ /^release\/v/ && $CI_COMMIT_TAG !~ /^v\d+\.\d+(\.\d+)?($|-)/ && $CI_COMMIT_TAG !~ /^qa-test/ && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event")' -.if-schedule: &if-schedule - if: '$CI_PIPELINE_SOURCE == "schedule"' - .if-schedule-nightly: &if-schedule-nightly if: '$CI_PIPELINE_SOURCE == "schedule" && $INCLUDE_NIGHTLY_RUN == "1"' @@ -214,51 +208,41 @@ # Rules # ######### ### Branches ### -.rules:protected: +.rules:protected:check: rules: - - <<: *if-protected + - <<: *if-protected-check -.rules:push_to_github: +.rules:protected:deploy: rules: - <<: *if-qa-test-tag when: never - - <<: *if-protected-no_label + - <<: *if-protected-deploy -# Not uploading on release branches -.rules:sync_support_status: +.rules:master:push: rules: - - <<: *if-ref-master-no_label - -.rules:protected-no_label-always: - rules: - - <<: *if-qa-test-tag - when: never - - <<: *if-protected-no_label - when: always + - <<: *if-master-push .rules:tag:release: rules: - - <<: *if-tag-release + - <<: *if-release-tag .rules:dev-push: rules: - <<: *if-dev-push -# Do not upload caches on dev branches by default .rules:upload-python-cache: rules: - - <<: *if-tag-release + - <<: *if-release-tag - <<: *if-schedule-nightly - - <<: *if-protected-ref-push + - <<: *if-protected-branch-push changes: *patterns-python-cache - <<: *if-label-upload_cache when: manual .rules:upload-submodule-cache: rules: - # Needn't upload submodule cache in schedule pipeline - - <<: *if-tag-release - - <<: *if-protected-ref-push + - <<: *if-release-tag + - <<: *if-protected-branch-push changes: *patterns-submodule - <<: *if-label-upload_cache when: manual @@ -266,11 +250,10 @@ ### Patterns ### .rules:patterns:clang_tidy: rules: - - <<: *if-protected + - <<: *if-protected-check - <<: *if-dev-push changes: *patterns-c-files - #.rules:patterns:static-code-analysis-preview: # rules: # - <<: *if-dev-push @@ -282,7 +265,7 @@ .rules:patterns:idf-pytest-plugin: rules: - - <<: *if-protected + - <<: *if-protected-check - <<: *if-dev-push changes: *patterns-idf-pytest-plugin @@ -326,7 +309,7 @@ rules: - <<: *if-revert-branch when: never - - <<: *if-protected + - <<: *if-protected-check - <<: *if-label-build - <<: *if-dev-push changes: *patterns-build_components @@ -339,7 +322,7 @@ rules: - <<: *if-revert-branch when: never - - <<: *if-protected + - <<: *if-protected-check - <<: *if-label-build - <<: *if-dev-push changes: *patterns-build_check @@ -354,7 +337,7 @@ rules: - <<: *if-revert-branch when: never - - <<: *if-protected + - <<: *if-protected-check - <<: *if-label-build - <<: *if-label-docker - <<: *if-dev-push @@ -370,7 +353,7 @@ rules: - <<: *if-revert-branch when: never - - <<: *if-protected + - <<: *if-protected-check - <<: *if-label-build - <<: *if-label-macos - <<: *if-label-macos_test @@ -385,7 +368,7 @@ rules: - <<: *if-revert-branch when: never - - <<: *if-protected + - <<: *if-protected-check - <<: *if-label-build - <<: *if-dev-push changes: *patterns-build_components @@ -415,7 +398,7 @@ rules: - <<: *if-revert-branch when: never - - <<: *if-protected + - <<: *if-protected-check - <<: *if-label-build-only when: never - <<: *if-label-host_test @@ -426,7 +409,7 @@ rules: - <<: *if-revert-branch when: never - - <<: *if-protected + - <<: *if-protected-check - <<: *if-label-build-only when: never - <<: *if-label-submodule diff --git a/.gitlab/ci/static-code-analysis.yml b/.gitlab/ci/static-code-analysis.yml index eb40c9c3ae..15eabc1598 100644 --- a/.gitlab/ci/static-code-analysis.yml +++ b/.gitlab/ci/static-code-analysis.yml @@ -85,7 +85,7 @@ clang_tidy_check: #code_quality_report: # extends: # - .sonar_scan_template -# - .rules:protected +# - .rules:protected:check # allow_failure: true # it's using exit code to indicate the code analysis result, # # we don't want to block ci when critical issues founded # script: diff --git a/tools/ci/generate_rules.py b/tools/ci/generate_rules.py index 4dc9534039..f9257eaf09 100755 --- a/tools/ci/generate_rules.py +++ b/tools/ci/generate_rules.py @@ -1,30 +1,24 @@ #!/usr/bin/env python # -# SPDX-FileCopyrightText: 2021-2024 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2021-2025 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 import argparse import inspect import os import sys +import typing as t from collections import defaultdict from itertools import product import yaml -from idf_ci_utils import GitlabYmlConfig from idf_ci_utils import IDF_PATH +from idf_ci_utils import GitlabYmlConfig -try: +if t.TYPE_CHECKING: import pygraphviz as pgv -except ImportError: # used when pre-commit, skip generating image - pass - -try: - from typing import Union -except ImportError: # used for type hint - pass -def _list(str_or_list): # type: (Union[str, list]) -> list +def _list(str_or_list: t.Union[str, t.List]) -> t.List: if isinstance(str_or_list, str): return [str_or_list] elif isinstance(str_or_list, list): @@ -33,7 +27,7 @@ def _list(str_or_list): # type: (Union[str, list]) -> list raise ValueError('Wrong type: {}. Only supports str or list.'.format(type(str_or_list))) -def _format_nested_dict(_dict, f_tuple): # type: (dict[str, dict], tuple[str, ...]) -> dict[str, dict] +def _format_nested_dict(_dict: t.Dict[str, t.Dict], f_tuple: t.Tuple[str, ...]) -> t.Dict[str, t.Dict]: res = {} for k, v in _dict.items(): k = k.split('__')[0] @@ -47,7 +41,7 @@ def _format_nested_dict(_dict, f_tuple): # type: (dict[str, dict], tuple[str, . return res -def _format_nested_list(_list, f_tuple): # type: (list[str], tuple[str, ...]) -> list[str] +def _format_nested_list(_list: t.List[str], f_tuple: t.Tuple[str, ...]) -> t.List[str]: res = [] for item in _list: if isinstance(item, list): @@ -61,26 +55,23 @@ def _format_nested_list(_list, f_tuple): # type: (list[str], tuple[str, ...]) - class RulesWriter: - AUTO_GENERATE_MARKER = inspect.cleandoc(r''' + AUTO_GENERATE_MARKER = inspect.cleandoc(r""" ################## # Auto Generated # ################## - ''') + """) - LABEL_TEMPLATE = inspect.cleandoc(r''' + LABEL_TEMPLATE = inspect.cleandoc(r""" .if-label-{0}: &if-label-{0} if: '$BOT_LABEL_{1} || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*{0}(?:,[^,\n\r]+)*$/i' - ''') + """) - RULE_PROTECTED = ' - <<: *if-protected' - RULE_PROTECTED_NO_LABEL = ' - <<: *if-protected-no_label' - RULE_BUILD_ONLY = ' - <<: *if-label-build-only\n' \ - ' when: never' - RULE_REVERT_BRANCH = ' - <<: *if-revert-branch\n' \ - ' when: never' + RULE_PROTECTED_CHECK = ' - <<: *if-protected-check' + RULE_PROTECTED_PUSH = ' - <<: *if-protected-push' + RULE_BUILD_ONLY = ' - <<: *if-label-build-only\n when: never' + RULE_REVERT_BRANCH = ' - <<: *if-revert-branch\n when: never' RULE_LABEL_TEMPLATE = ' - <<: *if-label-{0}' - RULE_PATTERN_TEMPLATE = ' - <<: *if-dev-push\n' \ - ' changes: *patterns-{0}' + RULE_PATTERN_TEMPLATE = ' - <<: *if-dev-push\n changes: *patterns-{0}' SPECIFIC_RULE_TEMPLATE = ' - <<: *{0}' RULES_TEMPLATE = inspect.cleandoc(r""" .rules:{0}: @@ -120,7 +111,7 @@ class RulesWriter: return res @staticmethod - def _expand_matrix(name, cfg): # type: (str, dict) -> dict + def _expand_matrix(name: str, cfg: t.Dict[str, t.Any]) -> t.Dict[str, t.Any]: """ Expand matrix into multi keys :param cfg: single rule dict @@ -138,7 +129,7 @@ class RulesWriter: res.update(_format_nested_dict(default, comb)) return res - def expand_rules(self): # type: () -> dict[str, dict[str, list]] + def expand_rules(self) -> t.Dict[str, t.Dict[str, t.List[str]]]: res = defaultdict(lambda: defaultdict(set)) # type: dict[str, dict[str, set]] for k, v in self.cfg.items(): if not v: @@ -169,13 +160,13 @@ class RulesWriter: continue res[item]['patterns'].add(_pat) - sorted_res = defaultdict(lambda: defaultdict(list)) # type: dict[str, dict[str, list]] + sorted_res = defaultdict(lambda: defaultdict(list)) # type: t.Dict[str, t.Dict[str, t.List[str]]] for k, v in res.items(): for vk, vv in v.items(): sorted_res[k][vk] = sorted(vv) return sorted_res - def new_labels_str(self): # type: () -> str + def new_labels_str(self) -> str: _labels = set([]) for k, v in self.cfg.items(): if not v: @@ -191,14 +182,14 @@ class RulesWriter: return res @classmethod - def _format_label(cls, label): # type: (str) -> str + def _format_label(cls, label: str) -> str: return cls.LABEL_TEMPLATE.format(label, cls.bot_label_str(label)) @staticmethod - def bot_label_str(label): # type: (str) -> str + def bot_label_str(label: str) -> str: return label.upper().replace('-', '_') - def new_rules_str(self): # type: () -> str + def new_rules_str(self) -> str: res = [] for k, v in sorted(self.rules.items()): if k.startswith('pattern'): @@ -211,13 +202,13 @@ class RulesWriter: res.append(self.RULES_TEMPLATE.format(k, self._format_rule(k, v))) return '\n\n'.join(res) - def _format_rule(self, name, cfg): # type: (str, dict) -> str + def _format_rule(self, name: str, cfg: t.Dict[str, t.Any]) -> str: _rules = [self.RULE_REVERT_BRANCH] if name.endswith('-production'): - _rules.append(self.RULE_PROTECTED_NO_LABEL) + _rules.append(self.RULE_PROTECTED_PUSH) else: if not (name.endswith('-preview') or name.startswith('labels:')): - _rules.append(self.RULE_PROTECTED) + _rules.append(self.RULE_PROTECTED_CHECK) if name.startswith('test:'): _rules.append(self.RULE_BUILD_ONLY) @@ -235,7 +226,7 @@ class RulesWriter: print('WARNING: pattern {} not exists'.format(pattern)) return '\n'.join(_rules) - def update_rules_yml(self): # type: () -> bool + def update_rules_yml(self) -> bool: with open(self.rules_yml) as fr: file_str = fr.read() @@ -255,7 +246,9 @@ PATTERN_COLOR = 'cyan' RULE_COLOR = 'blue' -def build_graph(rules_dict): # type: (dict[str, dict[str, list]]) -> pgv.AGraph +def build_graph(rules_dict: t.Dict[str, t.Dict[str, t.List[str]]]) -> 'pgv.AGraph': + from pygraphviz import pgv + graph = pgv.AGraph(directed=True, rankdir='LR', concentrate=True) for k, v in rules_dict.items(): @@ -281,7 +274,7 @@ def build_graph(rules_dict): # type: (dict[str, dict[str, list]]) -> pgv.AGraph return graph -def output_graph(graph, output_path='output.png'): # type: (pgv.AGraph, str) -> None +def output_graph(graph: 'pgv.AGraph', output_path: str = 'output.png') -> None: graph.layout('dot') if output_path.endswith('.png'): img_path = output_path @@ -292,13 +285,16 @@ def output_graph(graph, output_path='output.png'): # type: (pgv.AGraph, str) -> if __name__ == '__main__': parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument('rules_yml', nargs='?', default=os.path.join(IDF_PATH, '.gitlab', 'ci', 'rules.yml'), - help='rules.yml file path') - parser.add_argument('dependencies_yml', nargs='?', default=os.path.join(IDF_PATH, '.gitlab', 'ci', 'dependencies', - 'dependencies.yml'), - help='dependencies.yml file path') - parser.add_argument('--graph', - help='Specify PNG image output path. Use this argument to generate dependency graph') + parser.add_argument( + 'rules_yml', nargs='?', default=os.path.join(IDF_PATH, '.gitlab', 'ci', 'rules.yml'), help='rules.yml file path' + ) + parser.add_argument( + 'dependencies_yml', + nargs='?', + default=os.path.join(IDF_PATH, '.gitlab', 'ci', 'dependencies', 'dependencies.yml'), + help='dependencies.yml file path', + ) + parser.add_argument('--graph', help='Specify PNG image output path. Use this argument to generate dependency graph') args = parser.parse_args() writer = RulesWriter(args.rules_yml, args.dependencies_yml) From dc05b46ab56887b491faca903deaa41887989064 Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Tue, 13 May 2025 13:26:15 +0200 Subject: [PATCH 2/2] ci: inherit more variables this is somehow a temporary workaround, the long-term solution shall be related to `inherit:variables` --- .gitlab/ci/build.yml | 1 + tools/ci/dynamic_pipelines/templates/test_child_pipeline.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.gitlab/ci/build.yml b/.gitlab/ci/build.yml index ae3045175e..68e0f52eb1 100644 --- a/.gitlab/ci/build.yml +++ b/.gitlab/ci/build.yml @@ -314,6 +314,7 @@ build_child_pipeline: PARENT_PIPELINE_ID: $CI_PIPELINE_ID BUILD_AND_TEST_ALL_APPS: $BUILD_AND_TEST_ALL_APPS REPORT_EXIT_CODE: $REPORT_EXIT_CODE + OOCD_DISTRO_URL: $OOCD_DISTRO_URL # https://gitlab.com/gitlab-org/gitlab/-/issues/214340 inherit: variables: false diff --git a/tools/ci/dynamic_pipelines/templates/test_child_pipeline.yml b/tools/ci/dynamic_pipelines/templates/test_child_pipeline.yml index 9b5c04318e..00d6e0352a 100644 --- a/tools/ci/dynamic_pipelines/templates/test_child_pipeline.yml +++ b/tools/ci/dynamic_pipelines/templates/test_child_pipeline.yml @@ -37,6 +37,7 @@ Pytest Target Test Jobs: needs: - generate_pytest_child_pipeline variables: + OOCD_DISTRO_URL: $OOCD_DISTRO_URL PARENT_PIPELINE_ID: $PARENT_PIPELINE_ID REPORT_EXIT_CODE: $REPORT_EXIT_CODE # https://gitlab.com/gitlab-org/gitlab/-/issues/214340