From 4d7372beab782ce8c8a88a7a7e25571298e86921 Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Wed, 25 Jun 2025 07:56:37 +0200 Subject: [PATCH] Revert "ci: replace ci_fetch_submodules.py with submodule cache" This reverts commit 0af3e2b1b2dcc8703f896163a39ce470c685a362. --- .gitlab/ci/common.yml | 14 ++- .gitlab/ci/deploy.yml | 1 + .gitlab/ci/host-test.yml | 2 + .gitlab/ci/integration_test.yml | 1 + .gitlab/ci/pre_check.yml | 4 + .gitlab/ci/pre_commit.yml | 12 +- .gitlab/ci/rules.yml | 3 - .gitlab/ci/test-win.yml | 4 +- .gitlab/ci/upload_cache.yml | 6 +- tools/ci/ci_fetch_submodule.py | 106 ++++++++++++++++-- .../templates/.dynamic_jobs.yml | 1 + tools/ci/utils.sh | 2 +- 12 files changed, 130 insertions(+), 26 deletions(-) diff --git a/.gitlab/ci/common.yml b/.gitlab/ci/common.yml index 7a287f6545..953b4cbe2c 100644 --- a/.gitlab/ci/common.yml +++ b/.gitlab/ci/common.yml @@ -39,6 +39,14 @@ variables: GIT_CLEAN_FLAGS: -ffdx -e .cache/ LATEST_GIT_TAG: v6.0-dev + SUBMODULE_FETCH_TOOL: "tools/ci/ci_fetch_submodule.py" + # by default we will fetch all submodules + # jobs can overwrite this variable to only fetch submodules they required + # set to "none" if don't need to fetch submodules + SUBMODULES_TO_FETCH: "all" + # tell build system do not check submodule update as we download archive instead of clone + IDF_SKIP_CHECK_SUBMODULES: 1 + IDF_PATH: "$CI_PROJECT_DIR" V: "0" CHECKOUT_REF_SCRIPT: "$CI_PROJECT_DIR/tools/ci/checkout_project_ref.py" @@ -381,9 +389,11 @@ default: paths: - .cache/pip policy: pull - - key: git-submodule-cache-${LATEST_GIT_TAG} + - key: submodule-cache-${LATEST_GIT_TAG} + fallback_keys: + - submodule-cache paths: - - .git/modules + - .cache/submodule_archives policy: pull before_script: - *common-before_scripts diff --git a/.gitlab/ci/deploy.yml b/.gitlab/ci/deploy.yml index 7f5feae032..1a2b3c871a 100644 --- a/.gitlab/ci/deploy.yml +++ b/.gitlab/ci/deploy.yml @@ -12,6 +12,7 @@ check_submodule_sync: retry: 2 variables: GIT_STRATEGY: fetch # use brew local mirror first + SUBMODULES_TO_FETCH: "none" PUBLIC_IDF_URL: "https://github.com/espressif/esp-idf.git" dependencies: [] script: diff --git a/.gitlab/ci/host-test.yml b/.gitlab/ci/host-test.yml index de5c1a424d..e6ea1e16bc 100644 --- a/.gitlab/ci/host-test.yml +++ b/.gitlab/ci/host-test.yml @@ -383,6 +383,8 @@ test_idf_pytest_plugin: extends: - .host_test_template - .rules:patterns:idf-pytest-plugin + variables: + SUBMODULES_TO_FETCH: "none" artifacts: reports: junit: XUNIT_RESULT.xml diff --git a/.gitlab/ci/integration_test.yml b/.gitlab/ci/integration_test.yml index 9e1a3d3d3a..c14e515de6 100644 --- a/.gitlab/ci/integration_test.yml +++ b/.gitlab/ci/integration_test.yml @@ -36,6 +36,7 @@ gen_integration_pipeline: cache: [] tags: [fast_run, shiny] variables: + SUBMODULES_TO_FETCH: "none" GIT_LFS_SKIP_SMUDGE: 1 needs: - job: fast_template_app diff --git a/.gitlab/ci/pre_check.yml b/.gitlab/ci/pre_check.yml index b13e6d89f7..6effe5764f 100644 --- a/.gitlab/ci/pre_check.yml +++ b/.gitlab/ci/pre_check.yml @@ -15,6 +15,7 @@ check_version: # need a full clone to get the latest tag # the --shallow-since=$(git log -1 --format=%as $LATEST_GIT_TAG) option is not accurate GIT_STRATEGY: fetch + SUBMODULES_TO_FETCH: "none" GIT_DEPTH: 0 script: - export IDF_PATH=$PWD @@ -32,6 +33,8 @@ check_blobs: extends: - .pre_check_template - .rules:build:check + variables: + SUBMODULES_TO_FETCH: "components/esp_wifi/lib;components/esp_phy/lib;components/esp_coex/lib" script: # Check if Wi-Fi library header files match between IDF and the version used when compiling the libraries - IDF_TARGET=esp32 $IDF_PATH/components/esp_wifi/test_md5/test_md5.sh @@ -104,6 +107,7 @@ check_version_tag: # need a full clone to get the latest tag # the --shallow-since=$(git log -1 --format=%as $LATEST_GIT_TAG) option is not accurate GIT_STRATEGY: fetch + SUBMODULES_TO_FETCH: "none" GIT_DEPTH: 0 script: - (git cat-file -t $CI_COMMIT_REF_NAME | grep tag) || (echo "ESP-IDF versions must be annotated tags." && exit 1) diff --git a/.gitlab/ci/pre_commit.yml b/.gitlab/ci/pre_commit.yml index d5a371bde8..49449106e7 100644 --- a/.gitlab/ci/pre_commit.yml +++ b/.gitlab/ci/pre_commit.yml @@ -24,9 +24,9 @@ check_pre_commit_upload_cache: paths: - .cache/pre-commit policy: pull-push - - key: git-submodule-cache-${LATEST_GIT_TAG} + - key: submodule-cache-${LATEST_GIT_TAG} paths: - - .git/modules + - .cache/submodule_archives policy: pull check_pre_commit: @@ -41,9 +41,9 @@ check_pre_commit: paths: - .cache/pre-commit policy: pull - - key: git-submodule-cache-${LATEST_GIT_TAG} + - key: submodule-cache-${LATEST_GIT_TAG} paths: - - .git/modules + - .cache/submodule_archives policy: pull check_powershell: @@ -74,7 +74,7 @@ check_powershell: paths: - .cache/pre-commit policy: pull - - key: git-submodule-cache-${LATEST_GIT_TAG} + - key: submodule-cache-${LATEST_GIT_TAG} paths: - - .git/modules + - .cache/submodule_archives policy: pull diff --git a/.gitlab/ci/rules.yml b/.gitlab/ci/rules.yml index 5dbd84f034..96e19867e2 100644 --- a/.gitlab/ci/rules.yml +++ b/.gitlab/ci/rules.yml @@ -249,9 +249,6 @@ changes: *patterns-submodule - <<: *if-label-upload_cache when: manual - - <<: *if-dev-push - changes: - - .gitlab/ci/upload_cache.yml ### Patterns ### .rules:patterns:clang_tidy: diff --git a/.gitlab/ci/test-win.yml b/.gitlab/ci/test-win.yml index 1722b0b5d5..23879f20f4 100644 --- a/.gitlab/ci/test-win.yml +++ b/.gitlab/ci/test-win.yml @@ -62,7 +62,7 @@ test_tools_win: - python -m pip install jsonschema - .\install.ps1 --enable-ci --enable-pytest - .\export.ps1 - - git submodule update --init + - python "${SUBMODULE_FETCH_TOOL}" -s "all" - cd ${IDF_PATH}/tools/test_idf_py - pytest --noconftest test_idf_py.py --junitxml=${IDF_PATH}/XUNIT_IDF_PY.xml - pytest --noconftest test_hints.py --junitxml=${IDF_PATH}/XUNIT_HINTS.xml @@ -80,7 +80,7 @@ test_tools_win: script: - .\install.ps1 --enable-ci --enable-pytest - . .\export.ps1 - - git submodule update --init + - python "${SUBMODULE_FETCH_TOOL}" -s "all" - cd ${IDF_PATH}\tools\test_build_system - pytest --parallel-count ${CI_NODE_TOTAL} --parallel-index ${CI_NODE_INDEX} --junitxml=${CI_PROJECT_DIR}\XUNIT_RESULT.xml diff --git a/.gitlab/ci/upload_cache.yml b/.gitlab/ci/upload_cache.yml index c173ba42a6..173b63224b 100644 --- a/.gitlab/ci/upload_cache.yml +++ b/.gitlab/ci/upload_cache.yml @@ -35,14 +35,14 @@ upload-submodules-cache: - $GEO - cache cache: - key: git-submodule-cache-${LATEST_GIT_TAG} + key: submodule-cache-${LATEST_GIT_TAG} paths: - - .git/modules + - .cache/submodule_archives policy: push script: # use the default gitlab server - unset LOCAL_GITLAB_HTTPS_HOST - - rm -rf .git/modules # clear old submodules + - rm -rf .cache/submodule_archives # clear old submodule archives - add_gitlab_ssh_keys - fetch_submodules parallel: diff --git a/tools/ci/ci_fetch_submodule.py b/tools/ci/ci_fetch_submodule.py index 5f3ab23d2d..2137e1ef78 100644 --- a/tools/ci/ci_fetch_submodule.py +++ b/tools/ci/ci_fetch_submodule.py @@ -6,11 +6,99 @@ # download archive of one commit instead of cloning entire submodule repo import argparse +import os +import re +import shutil import subprocess -import sys import time +from typing import Any +from typing import List + +import gitlab_api + +SUBMODULE_PATTERN = re.compile(r'\[submodule \"([^\"]+)\"]') +PATH_PATTERN = re.compile(r'path\s+=\s+(\S+)') +URL_PATTERN = re.compile(r'url\s+=\s+(\S+)') + +SUBMODULE_ARCHIVE_TEMP_FOLDER = 'submodule_archive' +# need to match the one defined in CI yaml files for caching purpose +SUBMODULE_ARCHIVE_CACHE_DIR = '.cache/submodule_archives' + + +class SubModule(object): + # We don't need to support recursive submodule clone now + + GIT_LS_TREE_OUTPUT_PATTERN = re.compile(r'\d+\s+commit\s+([0-9a-f]+)\s+') + + def __init__(self, gitlab_inst: gitlab_api.Gitlab, path: str, url: str) -> None: + self.path = path + self.url = url + self.gitlab_inst = gitlab_inst + self.project_id = self._get_project_id(url) + self.commit_id = self._get_commit_id(path) + + def _get_commit_id(self, path: str) -> str: + output = subprocess.check_output(['git', 'ls-tree', 'HEAD', path]).decode() + # example output: 160000 commit d88a262fbdf35e5abb372280eb08008749c3faa0 components/esp_wifi/lib + match = self.GIT_LS_TREE_OUTPUT_PATTERN.search(output) + return match.group(1) if match is not None else '' + + def _get_project_id(self, url: str) -> Any: + base_name = os.path.basename(url) + project_id = self.gitlab_inst.get_project_id( + os.path.splitext(base_name)[0], # remove .git + namespace='espressif', + ) + return project_id + + def download_archive(self) -> None: + print('Update submodule: {}: {}'.format(self.path, self.commit_id)) + path_name = self.gitlab_inst.download_archive( + self.commit_id, SUBMODULE_ARCHIVE_TEMP_FOLDER, self.project_id, SUBMODULE_ARCHIVE_CACHE_DIR + ) + renamed_path = os.path.join(os.path.dirname(path_name), os.path.basename(self.path)) + os.rename(path_name, renamed_path) + shutil.rmtree(self.path, ignore_errors=True) + shutil.move(renamed_path, os.path.dirname(self.path)) + + +def update_submodule(git_module_file: str, submodules_to_update: List) -> None: + gitlab_inst = gitlab_api.Gitlab() + submodules = [] + with open(git_module_file, 'r') as f: + data = f.read() + match = SUBMODULE_PATTERN.search(data) + if match is not None: + while True: + next_match = SUBMODULE_PATTERN.search(data, pos=match.end()) + if next_match: + end_pos = next_match.start() + else: + end_pos = len(data) + path_match = PATH_PATTERN.search(data, pos=match.end(), endpos=end_pos) + url_match = URL_PATTERN.search(data, pos=match.end(), endpos=end_pos) + path = path_match.group(1) if path_match is not None else '' + url = url_match.group(1) if url_match is not None else '' + + filter_result = True + if submodules_to_update: + if path not in submodules_to_update: + filter_result = False + if filter_result: + submodules.append(SubModule(gitlab_inst, path, url)) + + match = next_match + if not match: + break + + shutil.rmtree(SUBMODULE_ARCHIVE_TEMP_FOLDER, ignore_errors=True) + + for submodule in submodules: + submodule.download_archive() + if __name__ == '__main__': + start_time = time.time() parser = argparse.ArgumentParser() parser.add_argument('--repo_path', '-p', default='.', help='repo path') parser.add_argument( @@ -22,12 +110,12 @@ if __name__ == '__main__': '`all` and `none` are special values that indicates we fetch all / none submodules', ) args = parser.parse_args() - - print('This script is deprecated, please use the following git command with gitlab cache `.git/modules` instead.') - print('Calling `git submodule update --init --depth=1` ...') - - start_time = time.time() - subprocess.check_call( - ['git', 'submodule', 'update', '--init', '--depth=1'], stdout=sys.stdout, stderr=sys.stderr, cwd=args.repo_path - ) + if args.submodule == 'none': + print("don't need to update submodules") + exit(0) + if args.submodule == 'all': + _submodules = [] + else: + _submodules = args.submodule.split(';') + update_submodule(os.path.join(args.repo_path, '.gitmodules'), _submodules) print('total time spent on update submodule: {:.02f}s'.format(time.time() - start_time)) diff --git a/tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml b/tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml index 7e34f60b22..2629a05aec 100644 --- a/tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml +++ b/tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml @@ -50,6 +50,7 @@ stage: target_test timeout: 1 hour variables: + SUBMODULES_TO_FETCH: "none" # set while generating the pipeline PYTEST_NODES: "" TARGET_SELECTOR: "" diff --git a/tools/ci/utils.sh b/tools/ci/utils.sh index 0f705cf6e8..17951e2173 100644 --- a/tools/ci/utils.sh +++ b/tools/ci/utils.sh @@ -36,7 +36,7 @@ function add_doc_server_ssh_keys() { function fetch_submodules() { section_start "fetch_submodules" "Fetching submodules..." - git submodule update --init --depth 1 --force + python "${SUBMODULE_FETCH_TOOL}" -s "${SUBMODULES_TO_FETCH}" section_end "fetch_submodules" }