Revert "ci: replace ci_fetch_submodules.py with submodule cache"

This reverts commit 0af3e2b1b2.

.gitlab/ci/common.yml

@@ -39,6 +39,14 @@ variables:
   GIT_CLEAN_FLAGS: -ffdx -e .cache/
   LATEST_GIT_TAG: v6.0-dev
 
+  SUBMODULE_FETCH_TOOL: "tools/ci/ci_fetch_submodule.py"
+  # by default we will fetch all submodules
+  # jobs can overwrite this variable to only fetch submodules they required
+  # set to "none" if don't need to fetch submodules
+  SUBMODULES_TO_FETCH: "all"
+  # tell build system do not check submodule update as we download archive instead of clone
+  IDF_SKIP_CHECK_SUBMODULES: 1
+
   IDF_PATH: "$CI_PROJECT_DIR"
   V: "0"
   CHECKOUT_REF_SCRIPT: "$CI_PROJECT_DIR/tools/ci/checkout_project_ref.py"
@@ -381,9 +389,11 @@ default:
       paths:
         - .cache/pip
       policy: pull
-    - key: git-submodule-cache-${LATEST_GIT_TAG}
+    - key: submodule-cache-${LATEST_GIT_TAG}
+      fallback_keys:
+        - submodule-cache
       paths:
-        - .git/modules
+        - .cache/submodule_archives
       policy: pull
   before_script:
     - *common-before_scripts

.gitlab/ci/deploy.yml

@@ -12,6 +12,7 @@ check_submodule_sync:
   retry: 2
   variables:
     GIT_STRATEGY: fetch  # use brew local mirror first
+    SUBMODULES_TO_FETCH: "none"
     PUBLIC_IDF_URL: "https://github.com/espressif/esp-idf.git"
   dependencies: []
   script:

.gitlab/ci/host-test.yml

@@ -383,6 +383,8 @@ test_idf_pytest_plugin:
   extends:
     - .host_test_template
     - .rules:patterns:idf-pytest-plugin
+  variables:
+    SUBMODULES_TO_FETCH: "none"
   artifacts:
     reports:
       junit: XUNIT_RESULT.xml

.gitlab/ci/integration_test.yml

@@ -36,6 +36,7 @@ gen_integration_pipeline:
   cache: []
   tags: [fast_run, shiny]
   variables:
+    SUBMODULES_TO_FETCH: "none"
     GIT_LFS_SKIP_SMUDGE: 1
   needs:
     - job: fast_template_app

.gitlab/ci/pre_check.yml

@@ -15,6 +15,7 @@ check_version:
     # need a full clone to get the latest tag
     # the --shallow-since=$(git log -1 --format=%as $LATEST_GIT_TAG) option is not accurate
     GIT_STRATEGY: fetch
+    SUBMODULES_TO_FETCH: "none"
     GIT_DEPTH: 0
   script:
     - export IDF_PATH=$PWD
@@ -32,6 +33,8 @@ check_blobs:
   extends:
     - .pre_check_template
     - .rules:build:check
+  variables:
+    SUBMODULES_TO_FETCH: "components/esp_wifi/lib;components/esp_phy/lib;components/esp_coex/lib"
   script:
     # Check if Wi-Fi library header files match between IDF and the version used when compiling the libraries
     - IDF_TARGET=esp32 $IDF_PATH/components/esp_wifi/test_md5/test_md5.sh
@@ -104,6 +107,7 @@ check_version_tag:
     # need a full clone to get the latest tag
     # the --shallow-since=$(git log -1 --format=%as $LATEST_GIT_TAG) option is not accurate
     GIT_STRATEGY: fetch
+    SUBMODULES_TO_FETCH: "none"
     GIT_DEPTH: 0
   script:
     - (git cat-file -t $CI_COMMIT_REF_NAME | grep tag) || (echo "ESP-IDF versions must be annotated tags." && exit 1)

.gitlab/ci/pre_commit.yml

@@ -24,9 +24,9 @@ check_pre_commit_upload_cache:
       paths:
         - .cache/pre-commit
       policy: pull-push
-    - key: git-submodule-cache-${LATEST_GIT_TAG}
+    - key: submodule-cache-${LATEST_GIT_TAG}
       paths:
-        - .git/modules
+        - .cache/submodule_archives
       policy: pull
 
 check_pre_commit:
@@ -41,9 +41,9 @@ check_pre_commit:
       paths:
         - .cache/pre-commit
       policy: pull
-    - key: git-submodule-cache-${LATEST_GIT_TAG}
+    - key: submodule-cache-${LATEST_GIT_TAG}
       paths:
-        - .git/modules
+        - .cache/submodule_archives
       policy: pull
 
 check_powershell:
@@ -74,7 +74,7 @@ check_powershell:
       paths:
         - .cache/pre-commit
       policy: pull
-    - key: git-submodule-cache-${LATEST_GIT_TAG}
+    - key: submodule-cache-${LATEST_GIT_TAG}
       paths:
-        - .git/modules
+        - .cache/submodule_archives
       policy: pull

.gitlab/ci/rules.yml

@@ -249,9 +249,6 @@
       changes: *patterns-submodule
     - <<: *if-label-upload_cache
       when: manual
-    - <<: *if-dev-push
-      changes:
-        - .gitlab/ci/upload_cache.yml
 
 ### Patterns ###
 .rules:patterns:clang_tidy:

.gitlab/ci/test-win.yml

@@ -62,7 +62,7 @@ test_tools_win:
     - python -m pip install jsonschema
     - .\install.ps1 --enable-ci --enable-pytest
     - .\export.ps1
-    - git submodule update --init
+    - python "${SUBMODULE_FETCH_TOOL}" -s "all"
     - cd ${IDF_PATH}/tools/test_idf_py
     - pytest --noconftest test_idf_py.py       --junitxml=${IDF_PATH}/XUNIT_IDF_PY.xml
     - pytest --noconftest test_hints.py        --junitxml=${IDF_PATH}/XUNIT_HINTS.xml
@@ -80,7 +80,7 @@ test_tools_win:
   script:
     - .\install.ps1 --enable-ci --enable-pytest
     - . .\export.ps1
-    - git submodule update --init
+    - python "${SUBMODULE_FETCH_TOOL}" -s "all"
     - cd ${IDF_PATH}\tools\test_build_system
     - pytest --parallel-count ${CI_NODE_TOTAL} --parallel-index ${CI_NODE_INDEX} --junitxml=${CI_PROJECT_DIR}\XUNIT_RESULT.xml
 

.gitlab/ci/upload_cache.yml

@@ -35,14 +35,14 @@ upload-submodules-cache:
     - $GEO
     - cache
   cache:
-    key: git-submodule-cache-${LATEST_GIT_TAG}
+    key: submodule-cache-${LATEST_GIT_TAG}
     paths:
-      - .git/modules
+      - .cache/submodule_archives
     policy: push
   script:
     # use the default gitlab server
     - unset LOCAL_GITLAB_HTTPS_HOST
-    - rm -rf .git/modules  # clear old submodules
+    - rm -rf .cache/submodule_archives  # clear old submodule archives
     - add_gitlab_ssh_keys
     - fetch_submodules
   parallel:

tools/ci/ci_fetch_submodule.py

@@ -6,11 +6,99 @@
 # download archive of one commit instead of cloning entire submodule repo
 
 import argparse
+import os
+import re
+import shutil
 import subprocess
-import sys
 import time
+from typing import Any
+from typing import List
+
+import gitlab_api
+
+SUBMODULE_PATTERN = re.compile(r'\[submodule \"([^\"]+)\"]')
+PATH_PATTERN = re.compile(r'path\s+=\s+(\S+)')
+URL_PATTERN = re.compile(r'url\s+=\s+(\S+)')
+
+SUBMODULE_ARCHIVE_TEMP_FOLDER = 'submodule_archive'
+# need to match the one defined in CI yaml files for caching purpose
+SUBMODULE_ARCHIVE_CACHE_DIR = '.cache/submodule_archives'
+
+
+class SubModule(object):
+    # We don't need to support recursive submodule clone now
+
+    GIT_LS_TREE_OUTPUT_PATTERN = re.compile(r'\d+\s+commit\s+([0-9a-f]+)\s+')
+
+    def __init__(self, gitlab_inst: gitlab_api.Gitlab, path: str, url: str) -> None:
+        self.path = path
+        self.url = url
+        self.gitlab_inst = gitlab_inst
+        self.project_id = self._get_project_id(url)
+        self.commit_id = self._get_commit_id(path)
+
+    def _get_commit_id(self, path: str) -> str:
+        output = subprocess.check_output(['git', 'ls-tree', 'HEAD', path]).decode()
+        # example output: 160000 commit d88a262fbdf35e5abb372280eb08008749c3faa0	components/esp_wifi/lib
+        match = self.GIT_LS_TREE_OUTPUT_PATTERN.search(output)
+        return match.group(1) if match is not None else ''
+
+    def _get_project_id(self, url: str) -> Any:
+        base_name = os.path.basename(url)
+        project_id = self.gitlab_inst.get_project_id(
+            os.path.splitext(base_name)[0],  # remove .git
+            namespace='espressif',
+        )
+        return project_id
+
+    def download_archive(self) -> None:
+        print('Update submodule: {}: {}'.format(self.path, self.commit_id))
+        path_name = self.gitlab_inst.download_archive(
+            self.commit_id, SUBMODULE_ARCHIVE_TEMP_FOLDER, self.project_id, SUBMODULE_ARCHIVE_CACHE_DIR
+        )
+        renamed_path = os.path.join(os.path.dirname(path_name), os.path.basename(self.path))
+        os.rename(path_name, renamed_path)
+        shutil.rmtree(self.path, ignore_errors=True)
+        shutil.move(renamed_path, os.path.dirname(self.path))
+
+
+def update_submodule(git_module_file: str, submodules_to_update: List) -> None:
+    gitlab_inst = gitlab_api.Gitlab()
+    submodules = []
+    with open(git_module_file, 'r') as f:
+        data = f.read()
+    match = SUBMODULE_PATTERN.search(data)
+    if match is not None:
+        while True:
+            next_match = SUBMODULE_PATTERN.search(data, pos=match.end())
+            if next_match:
+                end_pos = next_match.start()
+            else:
+                end_pos = len(data)
+            path_match = PATH_PATTERN.search(data, pos=match.end(), endpos=end_pos)
+            url_match = URL_PATTERN.search(data, pos=match.end(), endpos=end_pos)
+            path = path_match.group(1) if path_match is not None else ''
+            url = url_match.group(1) if url_match is not None else ''
+
+            filter_result = True
+            if submodules_to_update:
+                if path not in submodules_to_update:
+                    filter_result = False
+            if filter_result:
+                submodules.append(SubModule(gitlab_inst, path, url))
+
+            match = next_match
+            if not match:
+                break
+
+    shutil.rmtree(SUBMODULE_ARCHIVE_TEMP_FOLDER, ignore_errors=True)
+
+    for submodule in submodules:
+        submodule.download_archive()
+
 
 if __name__ == '__main__':
+    start_time = time.time()
     parser = argparse.ArgumentParser()
     parser.add_argument('--repo_path', '-p', default='.', help='repo path')
     parser.add_argument(
@@ -22,12 +110,12 @@ if __name__ == '__main__':
         '`all` and `none` are special values that indicates we fetch all / none submodules',
     )
     args = parser.parse_args()
-
+    if args.submodule == 'none':
-    print('This script is deprecated, please use the following git command with gitlab cache `.git/modules` instead.')
+        print("don't need to update submodules")
-    print('Calling `git submodule update --init --depth=1` ...')
+        exit(0)
-
+    if args.submodule == 'all':
-    start_time = time.time()
+        _submodules = []
-    subprocess.check_call(
+    else:
-        ['git', 'submodule', 'update', '--init', '--depth=1'], stdout=sys.stdout, stderr=sys.stderr, cwd=args.repo_path
+        _submodules = args.submodule.split(';')
-    )
+    update_submodule(os.path.join(args.repo_path, '.gitmodules'), _submodules)
     print('total time spent on update submodule: {:.02f}s'.format(time.time() - start_time))

tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml

@@ -50,6 +50,7 @@
   stage: target_test
   timeout: 1 hour
   variables:
+    SUBMODULES_TO_FETCH: "none"
     # set while generating the pipeline
     PYTEST_NODES: ""
     TARGET_SELECTOR: ""

tools/ci/utils.sh

@@ -36,7 +36,7 @@ function add_doc_server_ssh_keys() {
 
 function fetch_submodules() {
   section_start "fetch_submodules" "Fetching submodules..."
-  git submodule update --init --depth 1 --force
+  python "${SUBMODULE_FETCH_TOOL}" -s "${SUBMODULES_TO_FETCH}"
   section_end "fetch_submodules"
 }