Merge branch 'ci/remove-submodule-fetch-script' into 'master'

Ci/remove submodule fetch script

Closes IDFCI-2980

See merge request espressif/esp-idf!39992

.gitlab/ci/build.yml

@@ -3,10 +3,7 @@
   extends:
     - .after_script:build:ccache-show-stats:upload-failed-job-logs
   image: $ESP_ENV_IMAGE
-  tags:
-    - build
-    # build only on shiny servers since shiny storage server is at the same location
-    - shiny
+  tags: [build, shiny]
   variables:
     # Enable ccache for all build jobs. See configure_ci_environment.sh for more ccache related settings.
     IDF_CCACHE_ENABLE: "1"

.gitlab/ci/common.yml

@@ -30,26 +30,15 @@ variables:
   # - set GIT_STRATEGY: "clone" to shiny runners
   # - set GIT_STRATEGY: "fetch" to brew runners
   GIT_STRATEGY: clone
-  # we will download archive for each submodule instead of clone.
-  # we don't do "recursive" when fetch submodule as they're not used in CI now.
-  GIT_SUBMODULE_STRATEGY: none
-  # since we're using merged-result pipelines, the last commit should work for most cases
   GIT_DEPTH: 1
-  # --no-recurse-submodules: we use cache for submodules
+  GIT_SUBMODULE_STRATEGY: none  # here we use cache for submodules, so we don't need to fetch them every time
+  # since we're using merged-result pipelines, the last commit should work for most cases
   # --prune --prune-tags: in case remote branch or tag is force pushed
   GIT_FETCH_EXTRA_FLAGS: "--no-recurse-submodules --prune --prune-tags"
   # we're using .cache folder for caches
   GIT_CLEAN_FLAGS: -ffdx -e .cache/
   LATEST_GIT_TAG: v6.0-dev
 
-  SUBMODULE_FETCH_TOOL: "tools/ci/ci_fetch_submodule.py"
-  # by default we will fetch all submodules
-  # jobs can overwrite this variable to only fetch submodules they required
-  # set to "none" if don't need to fetch submodules
-  SUBMODULES_TO_FETCH: "all"
-  # tell build system do not check submodule update as we download archive instead of clone
-  IDF_SKIP_CHECK_SUBMODULES: 1
-
   IDF_PATH: "$CI_PROJECT_DIR"
   V: "0"
   CHECKOUT_REF_SCRIPT: "$CI_PROJECT_DIR/tools/ci/checkout_project_ref.py"
@@ -392,11 +381,9 @@ default:
       paths:
         - .cache/pip
       policy: pull
-    - key: submodule-cache-${LATEST_GIT_TAG}
-      fallback_keys:
-        - submodule-cache
+    - key: git-submodule-cache-${LATEST_GIT_TAG}
       paths:
-        - .cache/submodule_archives
+        - .git/modules
       policy: pull
   before_script:
     - *common-before_scripts

.gitlab/ci/deploy.yml

@@ -11,9 +11,7 @@ check_submodule_sync:
   tags: [ brew, github_sync ]
   retry: 2
   variables:
-    # for brew runners, we always set GIT_STRATEGY to fetch
-    GIT_STRATEGY: fetch
-    SUBMODULES_TO_FETCH: "none"
+    GIT_STRATEGY: fetch  # use brew local mirror first
     PUBLIC_IDF_URL: "https://github.com/espressif/esp-idf.git"
   dependencies: []
   script:
@@ -36,10 +34,8 @@ push_to_github:
     - check_submodule_sync
   tags: [ brew, github_sync ]
   variables:
-    # for brew runners, we always set GIT_STRATEGY to fetch
-    GIT_STRATEGY: fetch
-    # github also need full record of commits
-    GIT_DEPTH: 0
+    GIT_STRATEGY: fetch  # use brew local mirror first
+    GIT_DEPTH: 0  # github needs full record of commits
   script:
     - add_github_ssh_keys
     - git remote remove github &>/dev/null || true
@@ -58,11 +54,11 @@ deploy_update_SHA_in_esp-dockerfiles:
   dependencies: []
   variables:
     GIT_DEPTH: 2
-  tags: [ shiny, build ]
+  tags: [build, shiny]
   script:
     - 'curl --header "PRIVATE-TOKEN: ${ESPCI_SCRIPTS_TOKEN}" -o create_MR_in_esp_dockerfile.sh $GITLAB_HTTP_SERVER/api/v4/projects/1260/repository/files/create_MR_in_esp_dockerfile%2Fcreate_MR_in_esp_dockerfile.sh/raw\?ref\=master'
     - chmod +x create_MR_in_esp_dockerfile.sh
     - ./create_MR_in_esp_dockerfile.sh
   environment:
-    name: eploy_update_SHA_in_esp-dockerfiles_production
+    name: deploy_update_SHA_in_esp-dockerfiles_production
     deployment_tier: production

.gitlab/ci/docs.yml

@@ -58,7 +58,7 @@
 check_readme_links:
   extends:
     - .pre_check_template
-  tags: ["build", "amd64", "internet"]
+  tags: ["amd64", "brew"]
   allow_failure: true
   rules:
     - <<: *if-protected-check

.gitlab/ci/host-test.yml

@@ -2,8 +2,7 @@
   extends: .rules:test:host_test
   stage: host_test
   image: $ESP_ENV_IMAGE
-  tags:
-    - host_test
+  tags: [build, shiny]
   dependencies: # set dependencies to null to avoid missing artifacts issue
   # run host_test jobs immediately, only after upload cache
   needs:
@@ -384,8 +383,6 @@ test_idf_pytest_plugin:
   extends:
     - .host_test_template
     - .rules:patterns:idf-pytest-plugin
-  variables:
-    SUBMODULES_TO_FETCH: "none"
   artifacts:
     reports:
       junit: XUNIT_RESULT.xml

.gitlab/ci/integration_test.yml

@@ -36,7 +36,6 @@ gen_integration_pipeline:
   cache: []
   tags: [fast_run, shiny]
   variables:
-    SUBMODULES_TO_FETCH: "none"
     GIT_LFS_SKIP_SMUDGE: 1
   needs:
     - job: fast_template_app

.gitlab/ci/pre_check.yml

@@ -1,8 +1,7 @@
 .pre_check_template:
   stage: pre_check
   image: $ESP_ENV_IMAGE
-  tags:
-    - host_test
+  tags: [build, shiny]
   dependencies: # set dependencies to null to avoid missing artifacts issue
 
 check_version:
@@ -16,7 +15,6 @@ check_version:
     # need a full clone to get the latest tag
     # the --shallow-since=$(git log -1 --format=%as $LATEST_GIT_TAG) option is not accurate
     GIT_STRATEGY: fetch
-    SUBMODULES_TO_FETCH: "none"
     GIT_DEPTH: 0
   script:
     - export IDF_PATH=$PWD
@@ -34,8 +32,6 @@ check_blobs:
   extends:
     - .pre_check_template
     - .rules:build:check
-  variables:
-    SUBMODULES_TO_FETCH: "components/esp_wifi/lib;components/esp_phy/lib;components/esp_coex/lib"
   script:
     # Check if Wi-Fi library header files match between IDF and the version used when compiling the libraries
     - IDF_TARGET=esp32 $IDF_PATH/components/esp_wifi/test_md5/test_md5.sh
@@ -108,7 +104,6 @@ check_version_tag:
     # need a full clone to get the latest tag
     # the --shallow-since=$(git log -1 --format=%as $LATEST_GIT_TAG) option is not accurate
     GIT_STRATEGY: fetch
-    SUBMODULES_TO_FETCH: "none"
     GIT_DEPTH: 0
   script:
     - (git cat-file -t $CI_COMMIT_REF_NAME | grep tag) || (echo "ESP-IDF versions must be annotated tags." && exit 1)
@@ -211,13 +206,12 @@ baseline_manifest_sha:
     when: always
 
 redundant_pass_job:
-  stage: pre_check
+  extends:
+    - .pre_check_template
   tags: [shiny, fast_run]
-  image: $ESP_ENV_IMAGE
-  dependencies: null
-  before_script: []
   cache: []
-  extends: []
+  variables:
+    GIT_STRATEGY: none
+  before_script: []
   script:
     - echo "This job is redundant to ensure the 'retry_failed_jobs' job can exist and not be skipped"
-  when: always

.gitlab/ci/pre_commit.yml

@@ -24,9 +24,9 @@ check_pre_commit_upload_cache:
       paths:
         - .cache/pre-commit
       policy: pull-push
-    - key: submodule-cache-${LATEST_GIT_TAG}
+    - key: git-submodule-cache-${LATEST_GIT_TAG}
       paths:
-        - .cache/submodule_archives
+        - .git/modules
       policy: pull
 
 check_pre_commit:
@@ -41,9 +41,9 @@ check_pre_commit:
       paths:
         - .cache/pre-commit
       policy: pull
-    - key: submodule-cache-${LATEST_GIT_TAG}
+    - key: git-submodule-cache-${LATEST_GIT_TAG}
       paths:
-        - .cache/submodule_archives
+        - .git/modules
       policy: pull
 
 check_powershell:
@@ -74,7 +74,7 @@ check_powershell:
       paths:
         - .cache/pre-commit
       policy: pull
-    - key: submodule-cache-${LATEST_GIT_TAG}
+    - key: git-submodule-cache-${LATEST_GIT_TAG}
       paths:
-        - .cache/submodule_archives
+        - .git/modules
       policy: pull

.gitlab/ci/rules.yml

@@ -249,6 +249,9 @@
       changes: *patterns-submodule
     - <<: *if-label-upload_cache
       when: manual
+    - <<: *if-dev-push
+      changes:
+        - .gitlab/ci/upload_cache.yml
 
 ### Patterns ###
 .rules:patterns:clang_tidy:

.gitlab/ci/test-win.yml

@@ -3,8 +3,7 @@
   extends: .rules:test:host_test
   stage: host_test
   image: $ESP_ENV_IMAGE
-  tags:
-    - windows-build
+  tags: [windows-build, brew]
   dependencies: # set dependencies to null to avoid missing artifacts issue
   # run host_test jobs immediately, only after upload cache
   needs:
@@ -14,6 +13,8 @@
     - job: upload-submodules-cache
       optional: true
       artifacts: false
+  variables:
+    GIT_STRATEGY: fetch  # use brew local mirror first
   before_script: []
   after_script: []
 
@@ -61,7 +62,7 @@ test_tools_win:
     - python -m pip install jsonschema
     - .\install.ps1 --enable-ci --enable-pytest
     - .\export.ps1
-    - python "${SUBMODULE_FETCH_TOOL}" -s "all"
+    - git submodule update --init
     - cd ${IDF_PATH}/tools/test_idf_py
     - pytest --noconftest test_idf_py.py       --junitxml=${IDF_PATH}/XUNIT_IDF_PY.xml
     - pytest --noconftest test_hints.py        --junitxml=${IDF_PATH}/XUNIT_HINTS.xml
@@ -79,7 +80,7 @@ test_tools_win:
   script:
     - .\install.ps1 --enable-ci --enable-pytest
     - . .\export.ps1
-    - python "${SUBMODULE_FETCH_TOOL}" -s "all"
+    - git submodule update --init
     - cd ${IDF_PATH}\tools\test_build_system
     - pytest --parallel-count ${CI_NODE_TOTAL} --parallel-index ${CI_NODE_INDEX} --junitxml=${CI_PROJECT_DIR}\XUNIT_RESULT.xml
 
@@ -89,8 +90,7 @@ pytest_build_system_win:
     - .rules:labels:windows_pytest_build_system
   parallel: 2
   needs: []
-  tags:
-    - windows-build
+  tags: [windows-build, brew]
   artifacts:
     paths:
       - XUNIT_RESULT.xml

.gitlab/ci/upload_cache.yml

@@ -35,14 +35,14 @@ upload-submodules-cache:
     - $GEO
     - cache
   cache:
-    key: submodule-cache-${LATEST_GIT_TAG}
+    key: git-submodule-cache-${LATEST_GIT_TAG}
     paths:
-      - .cache/submodule_archives
+      - .git/modules
     policy: push
   script:
     # use the default gitlab server
     - unset LOCAL_GITLAB_HTTPS_HOST
-    - rm -rf .cache/submodule_archives  # clear old submodule archives
+    - rm -rf .git/modules  # clear old submodules
     - add_gitlab_ssh_keys
     - fetch_submodules
   parallel:

tools/ci/ci_fetch_submodule.py

@@ -1,113 +1,33 @@
 #!/usr/bin/env python
-# SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD
+# SPDX-FileCopyrightText: 2022-2025 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
 
 # internal use only for CI
 # download archive of one commit instead of cloning entire submodule repo
 
 import argparse
-import os
-import re
-import shutil
 import subprocess
+import sys
 import time
-from typing import Any, List
-
-import gitlab_api
-
-SUBMODULE_PATTERN = re.compile(r"\[submodule \"([^\"]+)\"]")
-PATH_PATTERN = re.compile(r'path\s+=\s+(\S+)')
-URL_PATTERN = re.compile(r'url\s+=\s+(\S+)')
-
-SUBMODULE_ARCHIVE_TEMP_FOLDER = 'submodule_archive'
-# need to match the one defined in CI yaml files for caching purpose
-SUBMODULE_ARCHIVE_CACHE_DIR = '.cache/submodule_archives'
-
-
-class SubModule(object):
-    # We don't need to support recursive submodule clone now
-
-    GIT_LS_TREE_OUTPUT_PATTERN = re.compile(r'\d+\s+commit\s+([0-9a-f]+)\s+')
-
-    def __init__(self, gitlab_inst: gitlab_api.Gitlab, path: str, url: str) -> None:
-        self.path = path
-        self.url = url
-        self.gitlab_inst = gitlab_inst
-        self.project_id = self._get_project_id(url)
-        self.commit_id = self._get_commit_id(path)
-
-    def _get_commit_id(self, path: str) -> str:
-        output = subprocess.check_output(['git', 'ls-tree', 'HEAD', path]).decode()
-        # example output: 160000 commit d88a262fbdf35e5abb372280eb08008749c3faa0	components/esp_wifi/lib
-        match = self.GIT_LS_TREE_OUTPUT_PATTERN.search(output)
-        return match.group(1) if match is not None else ''
-
-    def _get_project_id(self, url: str) -> Any:
-        base_name = os.path.basename(url)
-        project_id = self.gitlab_inst.get_project_id(os.path.splitext(base_name)[0],  # remove .git
-                                                     namespace='espressif')
-        return project_id
-
-    def download_archive(self) -> None:
-        print('Update submodule: {}: {}'.format(self.path, self.commit_id))
-        path_name = self.gitlab_inst.download_archive(self.commit_id, SUBMODULE_ARCHIVE_TEMP_FOLDER,
-                                                      self.project_id, SUBMODULE_ARCHIVE_CACHE_DIR)
-        renamed_path = os.path.join(os.path.dirname(path_name), os.path.basename(self.path))
-        os.rename(path_name, renamed_path)
-        shutil.rmtree(self.path, ignore_errors=True)
-        shutil.move(renamed_path, os.path.dirname(self.path))
-
-
-def update_submodule(git_module_file: str, submodules_to_update: List) -> None:
-    gitlab_inst = gitlab_api.Gitlab()
-    submodules = []
-    with open(git_module_file, 'r') as f:
-        data = f.read()
-    match = SUBMODULE_PATTERN.search(data)
-    if match is not None:
-        while True:
-            next_match = SUBMODULE_PATTERN.search(data, pos=match.end())
-            if next_match:
-                end_pos = next_match.start()
-            else:
-                end_pos = len(data)
-            path_match = PATH_PATTERN.search(data, pos=match.end(), endpos=end_pos)
-            url_match = URL_PATTERN.search(data, pos=match.end(), endpos=end_pos)
-            path = path_match.group(1) if path_match is not None else ''
-            url = url_match.group(1) if url_match is not None else ''
-
-            filter_result = True
-            if submodules_to_update:
-                if path not in submodules_to_update:
-                    filter_result = False
-            if filter_result:
-                submodules.append(SubModule(gitlab_inst, path, url))
-
-            match = next_match
-            if not match:
-                break
-
-    shutil.rmtree(SUBMODULE_ARCHIVE_TEMP_FOLDER, ignore_errors=True)
-
-    for submodule in submodules:
-        submodule.download_archive()
-
 
 if __name__ == '__main__':
-    start_time = time.time()
     parser = argparse.ArgumentParser()
     parser.add_argument('--repo_path', '-p', default='.', help='repo path')
-    parser.add_argument('--submodule', '-s', default='all',
+    parser.add_argument(
+        '--submodule',
+        '-s',
+        default='all',
         help='Submodules to update. By default update all submodules. '
         'For multiple submodules, separate them with `;`. '
-                             '`all` and `none` are special values that indicates we fetch all / none submodules')
+        '`all` and `none` are special values that indicates we fetch all / none submodules',
+    )
     args = parser.parse_args()
-    if args.submodule == 'none':
-        print("don't need to update submodules")
-        exit(0)
-    if args.submodule == 'all':
-        _submodules = []
-    else:
-        _submodules = args.submodule.split(';')
-    update_submodule(os.path.join(args.repo_path, '.gitmodules'), _submodules)
+
+    print('This script is deprecated, please use the following git command with gitlab cache `.git/modules` instead.')
+    print('Calling `git submodule update --init --depth=1` ...')
+
+    start_time = time.time()
+    subprocess.check_call(
+        ['git', 'submodule', 'update', '--init', '--depth=1'], stdout=sys.stdout, stderr=sys.stderr, cwd=args.repo_path
+    )
     print('total time spent on update submodule: {:.02f}s'.format(time.time() - start_time))

tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml

@@ -50,7 +50,6 @@
   stage: target_test
   timeout: 1 hour
   variables:
-    SUBMODULES_TO_FETCH: "none"
     # set while generating the pipeline
     PYTEST_NODES: ""
     TARGET_SELECTOR: ""

tools/ci/utils.sh

@@ -36,7 +36,7 @@ function add_doc_server_ssh_keys() {
 
 function fetch_submodules() {
   section_start "fetch_submodules" "Fetching submodules..."
-  python "${SUBMODULE_FETCH_TOOL}" -s "${SUBMODULES_TO_FETCH}"
+  git submodule update --init --depth 1 --force
   section_end "fetch_submodules"
 }