From 0af3e2b1b2dcc8703f896163a39ce470c685a362 Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Mon, 23 Jun 2025 09:07:58 +0200 Subject: [PATCH] ci: replace ci_fetch_submodules.py with submodule cache --- .gitlab/ci/common.yml | 21 +--- .gitlab/ci/deploy.yml | 10 +- .gitlab/ci/host-test.yml | 2 - .gitlab/ci/integration_test.yml | 1 - .gitlab/ci/pre_check.yml | 15 +-- .gitlab/ci/pre_commit.yml | 12 +- .gitlab/ci/rules.yml | 3 + .gitlab/ci/test-win.yml | 12 +- .gitlab/ci/upload_cache.yml | 6 +- tools/ci/ci_fetch_submodule.py | 116 +++--------------- .../templates/.dynamic_jobs.yml | 1 - tools/ci/utils.sh | 2 +- 12 files changed, 49 insertions(+), 152 deletions(-) diff --git a/.gitlab/ci/common.yml b/.gitlab/ci/common.yml index 9830fdf066..7a287f6545 100644 --- a/.gitlab/ci/common.yml +++ b/.gitlab/ci/common.yml @@ -30,26 +30,15 @@ variables: # - set GIT_STRATEGY: "clone" to shiny runners # - set GIT_STRATEGY: "fetch" to brew runners GIT_STRATEGY: clone - # we will download archive for each submodule instead of clone. - # we don't do "recursive" when fetch submodule as they're not used in CI now. - GIT_SUBMODULE_STRATEGY: none - # since we're using merged-result pipelines, the last commit should work for most cases GIT_DEPTH: 1 - # --no-recurse-submodules: we use cache for submodules + GIT_SUBMODULE_STRATEGY: none # here we use cache for submodules, so we don't need to fetch them every time + # since we're using merged-result pipelines, the last commit should work for most cases # --prune --prune-tags: in case remote branch or tag is force pushed GIT_FETCH_EXTRA_FLAGS: "--no-recurse-submodules --prune --prune-tags" # we're using .cache folder for caches GIT_CLEAN_FLAGS: -ffdx -e .cache/ LATEST_GIT_TAG: v6.0-dev - SUBMODULE_FETCH_TOOL: "tools/ci/ci_fetch_submodule.py" - # by default we will fetch all submodules - # jobs can overwrite this variable to only fetch submodules they required - # set to "none" if don't need to fetch submodules - SUBMODULES_TO_FETCH: "all" - # tell build system do not check submodule update as we download archive instead of clone - IDF_SKIP_CHECK_SUBMODULES: 1 - IDF_PATH: "$CI_PROJECT_DIR" V: "0" CHECKOUT_REF_SCRIPT: "$CI_PROJECT_DIR/tools/ci/checkout_project_ref.py" @@ -392,11 +381,9 @@ default: paths: - .cache/pip policy: pull - - key: submodule-cache-${LATEST_GIT_TAG} - fallback_keys: - - submodule-cache + - key: git-submodule-cache-${LATEST_GIT_TAG} paths: - - .cache/submodule_archives + - .git/modules policy: pull before_script: - *common-before_scripts diff --git a/.gitlab/ci/deploy.yml b/.gitlab/ci/deploy.yml index 8e20a89813..5062004835 100644 --- a/.gitlab/ci/deploy.yml +++ b/.gitlab/ci/deploy.yml @@ -11,9 +11,7 @@ check_submodule_sync: tags: [ brew, github_sync ] retry: 2 variables: - # for brew runners, we always set GIT_STRATEGY to fetch - GIT_STRATEGY: fetch - SUBMODULES_TO_FETCH: "none" + GIT_STRATEGY: fetch # use brew local mirror first PUBLIC_IDF_URL: "https://github.com/espressif/esp-idf.git" dependencies: [] script: @@ -36,10 +34,8 @@ push_to_github: - check_submodule_sync tags: [ brew, github_sync ] variables: - # for brew runners, we always set GIT_STRATEGY to fetch - GIT_STRATEGY: fetch - # github also need full record of commits - GIT_DEPTH: 0 + GIT_STRATEGY: fetch # use brew local mirror first + GIT_DEPTH: 0 # github needs full record of commits script: - add_github_ssh_keys - git remote remove github &>/dev/null || true diff --git a/.gitlab/ci/host-test.yml b/.gitlab/ci/host-test.yml index c3d364f7f3..80e9eb2728 100644 --- a/.gitlab/ci/host-test.yml +++ b/.gitlab/ci/host-test.yml @@ -382,8 +382,6 @@ test_idf_pytest_plugin: extends: - .host_test_template - .rules:patterns:idf-pytest-plugin - variables: - SUBMODULES_TO_FETCH: "none" artifacts: reports: junit: XUNIT_RESULT.xml diff --git a/.gitlab/ci/integration_test.yml b/.gitlab/ci/integration_test.yml index c14e515de6..9e1a3d3d3a 100644 --- a/.gitlab/ci/integration_test.yml +++ b/.gitlab/ci/integration_test.yml @@ -36,7 +36,6 @@ gen_integration_pipeline: cache: [] tags: [fast_run, shiny] variables: - SUBMODULES_TO_FETCH: "none" GIT_LFS_SKIP_SMUDGE: 1 needs: - job: fast_template_app diff --git a/.gitlab/ci/pre_check.yml b/.gitlab/ci/pre_check.yml index 515b674ad0..b13e6d89f7 100644 --- a/.gitlab/ci/pre_check.yml +++ b/.gitlab/ci/pre_check.yml @@ -15,7 +15,6 @@ check_version: # need a full clone to get the latest tag # the --shallow-since=$(git log -1 --format=%as $LATEST_GIT_TAG) option is not accurate GIT_STRATEGY: fetch - SUBMODULES_TO_FETCH: "none" GIT_DEPTH: 0 script: - export IDF_PATH=$PWD @@ -33,8 +32,6 @@ check_blobs: extends: - .pre_check_template - .rules:build:check - variables: - SUBMODULES_TO_FETCH: "components/esp_wifi/lib;components/esp_phy/lib;components/esp_coex/lib" script: # Check if Wi-Fi library header files match between IDF and the version used when compiling the libraries - IDF_TARGET=esp32 $IDF_PATH/components/esp_wifi/test_md5/test_md5.sh @@ -107,7 +104,6 @@ check_version_tag: # need a full clone to get the latest tag # the --shallow-since=$(git log -1 --format=%as $LATEST_GIT_TAG) option is not accurate GIT_STRATEGY: fetch - SUBMODULES_TO_FETCH: "none" GIT_DEPTH: 0 script: - (git cat-file -t $CI_COMMIT_REF_NAME | grep tag) || (echo "ESP-IDF versions must be annotated tags." && exit 1) @@ -210,13 +206,12 @@ baseline_manifest_sha: when: always redundant_pass_job: - stage: pre_check + extends: + - .pre_check_template tags: [shiny, fast_run] - image: $ESP_ENV_IMAGE - dependencies: null - before_script: [] cache: [] - extends: [] + variables: + GIT_STRATEGY: none + before_script: [] script: - echo "This job is redundant to ensure the 'retry_failed_jobs' job can exist and not be skipped" - when: always diff --git a/.gitlab/ci/pre_commit.yml b/.gitlab/ci/pre_commit.yml index 49449106e7..d5a371bde8 100644 --- a/.gitlab/ci/pre_commit.yml +++ b/.gitlab/ci/pre_commit.yml @@ -24,9 +24,9 @@ check_pre_commit_upload_cache: paths: - .cache/pre-commit policy: pull-push - - key: submodule-cache-${LATEST_GIT_TAG} + - key: git-submodule-cache-${LATEST_GIT_TAG} paths: - - .cache/submodule_archives + - .git/modules policy: pull check_pre_commit: @@ -41,9 +41,9 @@ check_pre_commit: paths: - .cache/pre-commit policy: pull - - key: submodule-cache-${LATEST_GIT_TAG} + - key: git-submodule-cache-${LATEST_GIT_TAG} paths: - - .cache/submodule_archives + - .git/modules policy: pull check_powershell: @@ -74,7 +74,7 @@ check_powershell: paths: - .cache/pre-commit policy: pull - - key: submodule-cache-${LATEST_GIT_TAG} + - key: git-submodule-cache-${LATEST_GIT_TAG} paths: - - .cache/submodule_archives + - .git/modules policy: pull diff --git a/.gitlab/ci/rules.yml b/.gitlab/ci/rules.yml index 4a4db3746b..42c75f7533 100644 --- a/.gitlab/ci/rules.yml +++ b/.gitlab/ci/rules.yml @@ -249,6 +249,9 @@ changes: *patterns-submodule - <<: *if-label-upload_cache when: manual + - <<: *if-dev-push + changes: + - .gitlab/ci/upload_cache.yml ### Patterns ### .rules:patterns:clang_tidy: diff --git a/.gitlab/ci/test-win.yml b/.gitlab/ci/test-win.yml index 02177c4405..b19e8a1d93 100644 --- a/.gitlab/ci/test-win.yml +++ b/.gitlab/ci/test-win.yml @@ -3,8 +3,7 @@ extends: .rules:test:host_test stage: host_test image: $ESP_ENV_IMAGE - tags: - - windows-build + tags: [windows-build, brew] dependencies: # set dependencies to null to avoid missing artifacts issue # run host_test jobs immediately, only after upload cache needs: @@ -14,6 +13,8 @@ - job: upload-submodules-cache optional: true artifacts: false + variables: + GIT_STRATEGY: fetch # use brew local mirror first before_script: [] after_script: [] @@ -60,7 +61,7 @@ test_tools_win: - python -m pip install jsonschema - .\install.ps1 --enable-ci --enable-pytest - .\export.ps1 - - python "${SUBMODULE_FETCH_TOOL}" -s "all" + - git submodule update --init - cd ${IDF_PATH}/tools/test_idf_py - pytest --noconftest test_idf_py.py --junitxml=${IDF_PATH}/XUNIT_IDF_PY.xml - pytest --noconftest test_hints.py --junitxml=${IDF_PATH}/XUNIT_HINTS.xml @@ -78,7 +79,7 @@ test_tools_win: script: - .\install.ps1 --enable-ci --enable-pytest - . .\export.ps1 - - python "${SUBMODULE_FETCH_TOOL}" -s "all" + - git submodule update --init - cd ${IDF_PATH}\tools\test_build_system - pytest --parallel-count ${CI_NODE_TOTAL} --parallel-index ${CI_NODE_INDEX} --junitxml=${CI_PROJECT_DIR}\XUNIT_RESULT.xml @@ -88,8 +89,7 @@ pytest_build_system_win: - .rules:labels:windows_pytest_build_system parallel: 2 needs: [] - tags: - - windows-build + tags: [windows-build, brew] artifacts: paths: - XUNIT_RESULT.xml diff --git a/.gitlab/ci/upload_cache.yml b/.gitlab/ci/upload_cache.yml index 173b63224b..c173ba42a6 100644 --- a/.gitlab/ci/upload_cache.yml +++ b/.gitlab/ci/upload_cache.yml @@ -35,14 +35,14 @@ upload-submodules-cache: - $GEO - cache cache: - key: submodule-cache-${LATEST_GIT_TAG} + key: git-submodule-cache-${LATEST_GIT_TAG} paths: - - .cache/submodule_archives + - .git/modules policy: push script: # use the default gitlab server - unset LOCAL_GITLAB_HTTPS_HOST - - rm -rf .cache/submodule_archives # clear old submodule archives + - rm -rf .git/modules # clear old submodules - add_gitlab_ssh_keys - fetch_submodules parallel: diff --git a/tools/ci/ci_fetch_submodule.py b/tools/ci/ci_fetch_submodule.py index 821ce93caf..5f3ab23d2d 100644 --- a/tools/ci/ci_fetch_submodule.py +++ b/tools/ci/ci_fetch_submodule.py @@ -1,113 +1,33 @@ #!/usr/bin/env python -# SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2022-2025 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 # internal use only for CI # download archive of one commit instead of cloning entire submodule repo import argparse -import os -import re -import shutil import subprocess +import sys import time -from typing import Any, List - -import gitlab_api - -SUBMODULE_PATTERN = re.compile(r"\[submodule \"([^\"]+)\"]") -PATH_PATTERN = re.compile(r'path\s+=\s+(\S+)') -URL_PATTERN = re.compile(r'url\s+=\s+(\S+)') - -SUBMODULE_ARCHIVE_TEMP_FOLDER = 'submodule_archive' -# need to match the one defined in CI yaml files for caching purpose -SUBMODULE_ARCHIVE_CACHE_DIR = '.cache/submodule_archives' - - -class SubModule(object): - # We don't need to support recursive submodule clone now - - GIT_LS_TREE_OUTPUT_PATTERN = re.compile(r'\d+\s+commit\s+([0-9a-f]+)\s+') - - def __init__(self, gitlab_inst: gitlab_api.Gitlab, path: str, url: str) -> None: - self.path = path - self.url = url - self.gitlab_inst = gitlab_inst - self.project_id = self._get_project_id(url) - self.commit_id = self._get_commit_id(path) - - def _get_commit_id(self, path: str) -> str: - output = subprocess.check_output(['git', 'ls-tree', 'HEAD', path]).decode() - # example output: 160000 commit d88a262fbdf35e5abb372280eb08008749c3faa0 components/esp_wifi/lib - match = self.GIT_LS_TREE_OUTPUT_PATTERN.search(output) - return match.group(1) if match is not None else '' - - def _get_project_id(self, url: str) -> Any: - base_name = os.path.basename(url) - project_id = self.gitlab_inst.get_project_id(os.path.splitext(base_name)[0], # remove .git - namespace='espressif') - return project_id - - def download_archive(self) -> None: - print('Update submodule: {}: {}'.format(self.path, self.commit_id)) - path_name = self.gitlab_inst.download_archive(self.commit_id, SUBMODULE_ARCHIVE_TEMP_FOLDER, - self.project_id, SUBMODULE_ARCHIVE_CACHE_DIR) - renamed_path = os.path.join(os.path.dirname(path_name), os.path.basename(self.path)) - os.rename(path_name, renamed_path) - shutil.rmtree(self.path, ignore_errors=True) - shutil.move(renamed_path, os.path.dirname(self.path)) - - -def update_submodule(git_module_file: str, submodules_to_update: List) -> None: - gitlab_inst = gitlab_api.Gitlab() - submodules = [] - with open(git_module_file, 'r') as f: - data = f.read() - match = SUBMODULE_PATTERN.search(data) - if match is not None: - while True: - next_match = SUBMODULE_PATTERN.search(data, pos=match.end()) - if next_match: - end_pos = next_match.start() - else: - end_pos = len(data) - path_match = PATH_PATTERN.search(data, pos=match.end(), endpos=end_pos) - url_match = URL_PATTERN.search(data, pos=match.end(), endpos=end_pos) - path = path_match.group(1) if path_match is not None else '' - url = url_match.group(1) if url_match is not None else '' - - filter_result = True - if submodules_to_update: - if path not in submodules_to_update: - filter_result = False - if filter_result: - submodules.append(SubModule(gitlab_inst, path, url)) - - match = next_match - if not match: - break - - shutil.rmtree(SUBMODULE_ARCHIVE_TEMP_FOLDER, ignore_errors=True) - - for submodule in submodules: - submodule.download_archive() - if __name__ == '__main__': - start_time = time.time() parser = argparse.ArgumentParser() parser.add_argument('--repo_path', '-p', default='.', help='repo path') - parser.add_argument('--submodule', '-s', default='all', - help='Submodules to update. By default update all submodules. ' - 'For multiple submodules, separate them with `;`. ' - '`all` and `none` are special values that indicates we fetch all / none submodules') + parser.add_argument( + '--submodule', + '-s', + default='all', + help='Submodules to update. By default update all submodules. ' + 'For multiple submodules, separate them with `;`. ' + '`all` and `none` are special values that indicates we fetch all / none submodules', + ) args = parser.parse_args() - if args.submodule == 'none': - print("don't need to update submodules") - exit(0) - if args.submodule == 'all': - _submodules = [] - else: - _submodules = args.submodule.split(';') - update_submodule(os.path.join(args.repo_path, '.gitmodules'), _submodules) + + print('This script is deprecated, please use the following git command with gitlab cache `.git/modules` instead.') + print('Calling `git submodule update --init --depth=1` ...') + + start_time = time.time() + subprocess.check_call( + ['git', 'submodule', 'update', '--init', '--depth=1'], stdout=sys.stdout, stderr=sys.stderr, cwd=args.repo_path + ) print('total time spent on update submodule: {:.02f}s'.format(time.time() - start_time)) diff --git a/tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml b/tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml index 2629a05aec..7e34f60b22 100644 --- a/tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml +++ b/tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml @@ -50,7 +50,6 @@ stage: target_test timeout: 1 hour variables: - SUBMODULES_TO_FETCH: "none" # set while generating the pipeline PYTEST_NODES: "" TARGET_SELECTOR: "" diff --git a/tools/ci/utils.sh b/tools/ci/utils.sh index 17951e2173..0f705cf6e8 100644 --- a/tools/ci/utils.sh +++ b/tools/ci/utils.sh @@ -36,7 +36,7 @@ function add_doc_server_ssh_keys() { function fetch_submodules() { section_start "fetch_submodules" "Fetching submodules..." - python "${SUBMODULE_FETCH_TOOL}" -s "${SUBMODULES_TO_FETCH}" + git submodule update --init --depth 1 --force section_end "fetch_submodules" }