Merge branch 'ci/fix-deploy-jobs-rules' into 'master'

ci: improve deploy rules Closes IDFCI-2938 See merge request espressif/esp-idf!39068
2025-07-30 02:37:19 +02:00 · 2025-05-14 15:11:10 +08:00
parent cd10016953 dc05b46ab5
commit 0c1e5a13e2
9 changed files with 91 additions and 110 deletions
--- a/.gitlab/ci/build.yml
+++ b/.gitlab/ci/build.yml
@ -314,6 +314,7 @@ build_child_pipeline:
    PARENT_PIPELINE_ID: $CI_PIPELINE_ID
    BUILD_AND_TEST_ALL_APPS: $BUILD_AND_TEST_ALL_APPS
    REPORT_EXIT_CODE: $REPORT_EXIT_CODE
+    OOCD_DISTRO_URL: $OOCD_DISTRO_URL
  # https://gitlab.com/gitlab-org/gitlab/-/issues/214340
  inherit:
    variables: false
--- a/.gitlab/ci/deploy.yml
+++ b/.gitlab/ci/deploy.yml
@ -3,7 +3,6 @@
  image: $ESP_ENV_IMAGE
  tags: [ deploy ]

-# Check this before push_to_github
 check_submodule_sync:
  extends:
    - .deploy_job_template
@ -31,8 +30,9 @@ push_to_github:
  extends:
    - .deploy_job_template
    - .before_script:minimal
-    - .rules:push_to_github
+    - .rules:protected:deploy
  needs:
+    # submodule must be synced before pushing to github
    - check_submodule_sync
  tags: [ brew, github_sync ]
  variables:
@ -50,7 +50,7 @@ deploy_update_SHA_in_esp-dockerfiles:
  extends:
    - .deploy_job_template
    - .before_script:minimal
-    - .rules:protected-no_label-always
+    - .rules:protected:deploy
  dependencies: []
  variables:
    GIT_DEPTH: 2
--- a/.gitlab/ci/docs.yml
+++ b/.gitlab/ci/docs.yml
@ -19,12 +19,9 @@
 .patterns-docs-preview: &patterns-docs-preview
  - "docs/**/*"

-.if-protected: &if-protected
+.if-protected-check: &if-protected-check
  if: '($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_BRANCH =~ /^release\/v/ || $CI_COMMIT_TAG =~ /^v\d+\.\d+(\.\d+)?($|-)/)'

-.if-protected-no_label: &if-protected-no_label
-  if: '($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_BRANCH =~ /^release\/v/ || $CI_COMMIT_TAG =~ /^v\d+\.\d+(\.\d+)?($|-)/) && $BOT_TRIGGER_WITH_LABEL == null'
-
 .if-qa-test-tag: &if-qa-test-tag
  if: '$CI_COMMIT_TAG =~ /^qa-test/'

@ -41,7 +38,7 @@
  rules:
    - <<: *if-qa-test-tag
      when: never
-    - <<: *if-protected
+    - <<: *if-protected-check
    - <<: *if-label-build_docs
    - <<: *if-label-docs_full
    - <<: *if-dev-push
@ -64,7 +61,7 @@ check_readme_links:
  tags: ["build", "amd64", "internet"]
  allow_failure: true
  rules:
-    - <<: *if-protected
+    - <<: *if-protected-check
    - <<: *if-dev-push
      changes: *patterns-example-readme
  script:
@ -188,8 +185,7 @@ deploy_docs_production:
  # The DOCS_PROD_* variables used by this job are "Protected" so these branches must all be marked "Protected" in Gitlab settings
  extends:
    - .deploy_docs_template
-  rules:
-    - <<: *if-protected-no_label
+    - .rules:protected:deploy
  stage: post_deploy
  dependencies:  # set dependencies to null to avoid missing artifacts issue
  needs: # ensure runs after push_to_github succeeded
@ -208,8 +204,7 @@ deploy_docs_production:
 check_doc_links:
  extends:
    - .build_docs_template
-  rules:
-    - <<: *if-protected-no_label
+    - .rules:protected:deploy
  stage: post_deploy
  needs:
    - job: deploy_docs_production
--- a/.gitlab/ci/post_deploy.yml
+++ b/.gitlab/ci/post_deploy.yml
@ -1,7 +1,11 @@
-generate_failed_jobs_report:
+.post_deploy_template:
  stage: post_deploy
-  tags: [build, shiny]
  image: $ESP_ENV_IMAGE
+
+generate_failed_jobs_report:
+  extends:
+    - .post_deploy_template
+  tags: [build, shiny]
  when: always
  dependencies: []  # Do not download artifacts from the previous stages
  artifacts:
@ -13,12 +17,13 @@ generate_failed_jobs_report:
    - python tools/ci/dynamic_pipelines/scripts/generate_report.py --report-type job

 sync_support_status:
-  stage: post_deploy
  extends:
-    - .rules:sync_support_status
+    - .post_deploy_template
+    - .rules:master:push
+  tags: [ brew, github_sync ]
  needs:
    - push_to_github
-  image: $ESP_ENV_IMAGE
-  tags: [ brew, github_sync ]
+  cache: []
+  before_script: []
  script:
    - curl --fail --request POST --form token="$IDF_STATUS_TRIG_TOKEN" --form ref="$IDF_STATUS_BRANCH" --form "variables[UPLOAD_TO_S3]=true" "$IDF_STATUS_TRIG_URL"
--- a/.gitlab/ci/pre_check.yml
+++ b/.gitlab/ci/pre_check.yml
@ -10,7 +10,7 @@ check_version:
  # esp_idf_version.h in a branch before tagging the next version.
  extends:
    - .pre_check_template
-    - .rules:protected
+    - .rules:protected:check
  tags: [ brew, github_sync ]
  variables:
    # need a full clone to get the latest tag
--- a/.gitlab/ci/rules.yml
+++ b/.gitlab/ci/rules.yml
@ -167,22 +167,19 @@
 ##############
 # if anchors #
 ##############
-.if-ref-master: &if-ref-master
-  if: '$CI_COMMIT_REF_NAME == "master"'
+.if-master-push: &if-master-push
+  if: '$CI_COMMIT_REF_NAME == "master" && $CI_PIPELINE_SOURCE == "push"'

-.if-ref-master-no_label: &if-ref-master-no_label
-  if: '$CI_COMMIT_REF_NAME == "master" && $BOT_TRIGGER_WITH_LABEL == null'
-
-.if-tag-release: &if-tag-release
+.if-release-tag: &if-release-tag
  if: '$CI_COMMIT_TAG =~ /^v\d+\.\d+(\.\d+)?($|-)/'

-.if-protected: &if-protected
+.if-protected-check: &if-protected-check
  if: '($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_BRANCH =~ /^release\/v/ || $CI_COMMIT_TAG =~ /^v\d+\.\d+(\.\d+)?($|-)/) || $CI_COMMIT_TAG =~ /^qa-test/'

-.if-protected-no_label: &if-protected-no_label
-  if: '($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_BRANCH =~ /^release\/v/ || $CI_COMMIT_TAG =~ /^v\d+\.\d+(\.\d+)?($|-)/) && $BOT_TRIGGER_WITH_LABEL == null'
+.if-protected-deploy: &if-protected-deploy
+  if: '($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_BRANCH =~ /^release\/v/ || $CI_COMMIT_TAG =~ /^v\d+\.\d+(\.\d+)?($|-)/) && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "api")'

-.if-protected-ref-push: &if-protected-ref-push
+.if-protected-branch-push: &if-protected-branch-push
  # rules:changes always evaluates to true for new branch pipelines or when there is no Git push event
  if: '($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_BRANCH =~ /^release\/v/) && $CI_PIPELINE_SOURCE == "push"'

@ -192,9 +189,6 @@
 .if-dev-push: &if-dev-push
  if: '$CI_COMMIT_REF_NAME != "master" && $CI_COMMIT_BRANCH !~ /^release\/v/ && $CI_COMMIT_TAG !~ /^v\d+\.\d+(\.\d+)?($|-)/ && $CI_COMMIT_TAG !~ /^qa-test/ && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event")'

-.if-schedule: &if-schedule
-  if: '$CI_PIPELINE_SOURCE == "schedule"'
-
 .if-schedule-nightly: &if-schedule-nightly
  if: '$CI_PIPELINE_SOURCE == "schedule" && $INCLUDE_NIGHTLY_RUN == "1"'

@ -214,51 +208,41 @@
 # Rules #
 #########
 ### Branches ###
-.rules:protected:
+.rules:protected:check:
  rules:
-    - <<: *if-protected
+    - <<: *if-protected-check

-.rules:push_to_github:
+.rules:protected:deploy:
  rules:
    - <<: *if-qa-test-tag
      when: never
-    - <<: *if-protected-no_label
+    - <<: *if-protected-deploy

-# Not uploading on release branches
-.rules:sync_support_status:
+.rules:master:push:
  rules:
-    - <<: *if-ref-master-no_label
-
-.rules:protected-no_label-always:
-  rules:
-    - <<: *if-qa-test-tag
-      when: never
-    - <<: *if-protected-no_label
-      when: always
+    - <<: *if-master-push

 .rules:tag:release:
  rules:
-    - <<: *if-tag-release
+    - <<: *if-release-tag

 .rules:dev-push:
  rules:
    - <<: *if-dev-push

-# Do not upload caches on dev branches by default
 .rules:upload-python-cache:
  rules:
-    - <<: *if-tag-release
+    - <<: *if-release-tag
    - <<: *if-schedule-nightly
-    - <<: *if-protected-ref-push
+    - <<: *if-protected-branch-push
      changes: *patterns-python-cache
    - <<: *if-label-upload_cache
      when: manual

 .rules:upload-submodule-cache:
  rules:
-    # Needn't upload submodule cache in schedule pipeline
-    - <<: *if-tag-release
-    - <<: *if-protected-ref-push
+    - <<: *if-release-tag
+    - <<: *if-protected-branch-push
      changes: *patterns-submodule
    - <<: *if-label-upload_cache
      when: manual
@ -266,11 +250,10 @@
 ### Patterns ###
 .rules:patterns:clang_tidy:
  rules:
-    - <<: *if-protected
+    - <<: *if-protected-check
    - <<: *if-dev-push
      changes: *patterns-c-files

-
 #.rules:patterns:static-code-analysis-preview:
 #  rules:
 #    - <<: *if-dev-push
@ -282,7 +265,7 @@

 .rules:patterns:idf-pytest-plugin:
  rules:
-    - <<: *if-protected
+    - <<: *if-protected-check
    - <<: *if-dev-push
      changes: *patterns-idf-pytest-plugin

@ -326,7 +309,7 @@
  rules:
    - <<: *if-revert-branch
      when: never
-    - <<: *if-protected
+    - <<: *if-protected-check
    - <<: *if-label-build
    - <<: *if-dev-push
      changes: *patterns-build_components
@ -339,7 +322,7 @@
  rules:
    - <<: *if-revert-branch
      when: never
-    - <<: *if-protected
+    - <<: *if-protected-check
    - <<: *if-label-build
    - <<: *if-dev-push
      changes: *patterns-build_check
@ -354,7 +337,7 @@
  rules:
    - <<: *if-revert-branch
      when: never
-    - <<: *if-protected
+    - <<: *if-protected-check
    - <<: *if-label-build
    - <<: *if-label-docker
    - <<: *if-dev-push
@ -370,7 +353,7 @@
  rules:
    - <<: *if-revert-branch
      when: never
-    - <<: *if-protected
+    - <<: *if-protected-check
    - <<: *if-label-build
    - <<: *if-label-macos
    - <<: *if-label-macos_test
@ -385,7 +368,7 @@
  rules:
    - <<: *if-revert-branch
      when: never
-    - <<: *if-protected
+    - <<: *if-protected-check
    - <<: *if-label-build
    - <<: *if-dev-push
      changes: *patterns-build_components
@ -415,7 +398,7 @@
  rules:
    - <<: *if-revert-branch
      when: never
-    - <<: *if-protected
+    - <<: *if-protected-check
    - <<: *if-label-build-only
      when: never
    - <<: *if-label-host_test
@ -426,7 +409,7 @@
  rules:
    - <<: *if-revert-branch
      when: never
-    - <<: *if-protected
+    - <<: *if-protected-check
    - <<: *if-label-build-only
      when: never
    - <<: *if-label-submodule
--- a/.gitlab/ci/static-code-analysis.yml
+++ b/.gitlab/ci/static-code-analysis.yml
@ -85,7 +85,7 @@ clang_tidy_check:
 #code_quality_report:
 #  extends:
 #    - .sonar_scan_template
-#    - .rules:protected
+#    - .rules:protected:check
 #  allow_failure: true  # it's using exit code to indicate the code analysis result,
 #                       # we don't want to block ci when critical issues founded
 #  script:
--- a/tools/ci/dynamic_pipelines/templates/test_child_pipeline.yml
+++ b/tools/ci/dynamic_pipelines/templates/test_child_pipeline.yml
@ -37,6 +37,7 @@ Pytest Target Test Jobs:
  needs:
    - generate_pytest_child_pipeline
  variables:
+    OOCD_DISTRO_URL: $OOCD_DISTRO_URL
    PARENT_PIPELINE_ID: $PARENT_PIPELINE_ID
    REPORT_EXIT_CODE: $REPORT_EXIT_CODE
  # https://gitlab.com/gitlab-org/gitlab/-/issues/214340
--- a/tools/ci/generate_rules.py
+++ b/tools/ci/generate_rules.py
@ -1,30 +1,24 @@
 #!/usr/bin/env python
 #
-# SPDX-FileCopyrightText: 2021-2024 Espressif Systems (Shanghai) CO LTD
+# SPDX-FileCopyrightText: 2021-2025 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
 import argparse
 import inspect
 import os
 import sys
+import typing as t
 from collections import defaultdict
 from itertools import product

 import yaml
-from idf_ci_utils import GitlabYmlConfig
 from idf_ci_utils import IDF_PATH
+from idf_ci_utils import GitlabYmlConfig

-try:
+if t.TYPE_CHECKING:
    import pygraphviz as pgv
-except ImportError:  # used when pre-commit, skip generating image
-    pass
-
-try:
-    from typing import Union
-except ImportError:  # used for type hint
-    pass


-def _list(str_or_list):  # type: (Union[str, list]) -> list
+def _list(str_or_list: t.Union[str, t.List]) -> t.List:
    if isinstance(str_or_list, str):
        return [str_or_list]
    elif isinstance(str_or_list, list):
@ -33,7 +27,7 @@ def _list(str_or_list):  # type: (Union[str, list]) -> list
        raise ValueError('Wrong type: {}. Only supports str or list.'.format(type(str_or_list)))


-def _format_nested_dict(_dict, f_tuple):  # type: (dict[str, dict], tuple[str, ...]) -> dict[str, dict]
+def _format_nested_dict(_dict: t.Dict[str, t.Dict], f_tuple: t.Tuple[str, ...]) -> t.Dict[str, t.Dict]:
    res = {}
    for k, v in _dict.items():
        k = k.split('__')[0]
@ -47,7 +41,7 @@ def _format_nested_dict(_dict, f_tuple):  # type: (dict[str, dict], tuple[str, .
    return res


-def _format_nested_list(_list, f_tuple):  # type: (list[str], tuple[str, ...]) -> list[str]
+def _format_nested_list(_list: t.List[str], f_tuple: t.Tuple[str, ...]) -> t.List[str]:
    res = []
    for item in _list:
        if isinstance(item, list):
@ -61,26 +55,23 @@ def _format_nested_list(_list, f_tuple):  # type: (list[str], tuple[str, ...]) -


 class RulesWriter:
-    AUTO_GENERATE_MARKER = inspect.cleandoc(r'''
+    AUTO_GENERATE_MARKER = inspect.cleandoc(r"""
    ##################
    # Auto Generated #
    ##################
-    ''')
+    """)

-    LABEL_TEMPLATE = inspect.cleandoc(r'''
+    LABEL_TEMPLATE = inspect.cleandoc(r"""
    .if-label-{0}: &if-label-{0}
      if: '$BOT_LABEL_{1} || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*{0}(?:,[^,\n\r]+)*$/i'
-    ''')
+    """)

-    RULE_PROTECTED = '    - <<: *if-protected'
-    RULE_PROTECTED_NO_LABEL = '    - <<: *if-protected-no_label'
-    RULE_BUILD_ONLY = '    - <<: *if-label-build-only\n' \
-                      '      when: never'
-    RULE_REVERT_BRANCH = '    - <<: *if-revert-branch\n' \
-                         '      when: never'
+    RULE_PROTECTED_CHECK = '    - <<: *if-protected-check'
+    RULE_PROTECTED_PUSH = '    - <<: *if-protected-push'
+    RULE_BUILD_ONLY = '    - <<: *if-label-build-only\n      when: never'
+    RULE_REVERT_BRANCH = '    - <<: *if-revert-branch\n      when: never'
    RULE_LABEL_TEMPLATE = '    - <<: *if-label-{0}'
-    RULE_PATTERN_TEMPLATE = '    - <<: *if-dev-push\n' \
-                            '      changes: *patterns-{0}'
+    RULE_PATTERN_TEMPLATE = '    - <<: *if-dev-push\n      changes: *patterns-{0}'
    SPECIFIC_RULE_TEMPLATE = '    - <<: *{0}'
    RULES_TEMPLATE = inspect.cleandoc(r"""
    .rules:{0}:
@ -120,7 +111,7 @@ class RulesWriter:
        return res

    @staticmethod
-    def _expand_matrix(name, cfg):  # type: (str, dict) -> dict
+    def _expand_matrix(name: str, cfg: t.Dict[str, t.Any]) -> t.Dict[str, t.Any]:
        """
        Expand matrix into multi keys
        :param cfg: single rule dict
@ -138,7 +129,7 @@ class RulesWriter:
            res.update(_format_nested_dict(default, comb))
        return res

-    def expand_rules(self):  # type: () -> dict[str, dict[str, list]]
+    def expand_rules(self) -> t.Dict[str, t.Dict[str, t.List[str]]]:
        res = defaultdict(lambda: defaultdict(set))  # type: dict[str, dict[str, set]]
        for k, v in self.cfg.items():
            if not v:
@ -169,13 +160,13 @@ class RulesWriter:
                                continue
                            res[item]['patterns'].add(_pat)

-        sorted_res = defaultdict(lambda: defaultdict(list))  # type: dict[str, dict[str, list]]
+        sorted_res = defaultdict(lambda: defaultdict(list))  # type: t.Dict[str, t.Dict[str, t.List[str]]]
        for k, v in res.items():
            for vk, vv in v.items():
                sorted_res[k][vk] = sorted(vv)
        return sorted_res

-    def new_labels_str(self):  # type: () -> str
+    def new_labels_str(self) -> str:
        _labels = set([])
        for k, v in self.cfg.items():
            if not v:
@ -191,14 +182,14 @@ class RulesWriter:
        return res

    @classmethod
-    def _format_label(cls, label):  # type: (str) -> str
+    def _format_label(cls, label: str) -> str:
        return cls.LABEL_TEMPLATE.format(label, cls.bot_label_str(label))

    @staticmethod
-    def bot_label_str(label):  # type: (str) -> str
+    def bot_label_str(label: str) -> str:
        return label.upper().replace('-', '_')

-    def new_rules_str(self):  # type: () -> str
+    def new_rules_str(self) -> str:
        res = []
        for k, v in sorted(self.rules.items()):
            if k.startswith('pattern'):
@ -211,13 +202,13 @@ class RulesWriter:
            res.append(self.RULES_TEMPLATE.format(k, self._format_rule(k, v)))
        return '\n\n'.join(res)

-    def _format_rule(self, name, cfg):  # type: (str, dict) -> str
+    def _format_rule(self, name: str, cfg: t.Dict[str, t.Any]) -> str:
        _rules = [self.RULE_REVERT_BRANCH]
        if name.endswith('-production'):
-            _rules.append(self.RULE_PROTECTED_NO_LABEL)
+            _rules.append(self.RULE_PROTECTED_PUSH)
        else:
            if not (name.endswith('-preview') or name.startswith('labels:')):
-                _rules.append(self.RULE_PROTECTED)
+                _rules.append(self.RULE_PROTECTED_CHECK)
            if name.startswith('test:'):
                _rules.append(self.RULE_BUILD_ONLY)

@ -235,7 +226,7 @@ class RulesWriter:
                    print('WARNING: pattern {} not exists'.format(pattern))
        return '\n'.join(_rules)

-    def update_rules_yml(self):  # type: () -> bool
+    def update_rules_yml(self) -> bool:
        with open(self.rules_yml) as fr:
            file_str = fr.read()

@ -255,7 +246,9 @@ PATTERN_COLOR = 'cyan'
 RULE_COLOR = 'blue'


-def build_graph(rules_dict):  # type: (dict[str, dict[str, list]]) -> pgv.AGraph
+def build_graph(rules_dict: t.Dict[str, t.Dict[str, t.List[str]]]) -> 'pgv.AGraph':
+    from pygraphviz import pgv
+
    graph = pgv.AGraph(directed=True, rankdir='LR', concentrate=True)

    for k, v in rules_dict.items():
@ -281,7 +274,7 @@ def build_graph(rules_dict):  # type: (dict[str, dict[str, list]]) -> pgv.AGraph
    return graph


-def output_graph(graph, output_path='output.png'):  # type: (pgv.AGraph, str) -> None
+def output_graph(graph: 'pgv.AGraph', output_path: str = 'output.png') -> None:
    graph.layout('dot')
    if output_path.endswith('.png'):
        img_path = output_path
@ -292,13 +285,16 @@ def output_graph(graph, output_path='output.png'):  # type: (pgv.AGraph, str) ->

 if __name__ == '__main__':
    parser = argparse.ArgumentParser(description=__doc__)
-    parser.add_argument('rules_yml', nargs='?', default=os.path.join(IDF_PATH, '.gitlab', 'ci', 'rules.yml'),
-                        help='rules.yml file path')
-    parser.add_argument('dependencies_yml', nargs='?', default=os.path.join(IDF_PATH, '.gitlab', 'ci', 'dependencies',
-                                                                            'dependencies.yml'),
-                        help='dependencies.yml file path')
-    parser.add_argument('--graph',
-                        help='Specify PNG image output path. Use this argument to generate dependency graph')
+    parser.add_argument(
+        'rules_yml', nargs='?', default=os.path.join(IDF_PATH, '.gitlab', 'ci', 'rules.yml'), help='rules.yml file path'
+    )
+    parser.add_argument(
+        'dependencies_yml',
+        nargs='?',
+        default=os.path.join(IDF_PATH, '.gitlab', 'ci', 'dependencies', 'dependencies.yml'),
+        help='dependencies.yml file path',
+    )
+    parser.add_argument('--graph', help='Specify PNG image output path. Use this argument to generate dependency graph')
    args = parser.parse_args()

    writer = RulesWriter(args.rules_yml, args.dependencies_yml)