mirror of
https://github.com/espressif/esp-idf.git
synced 2025-08-01 19:54:32 +02:00
CI: modify fetch submodule method:
download archive for submodules instead of clone
This commit is contained in:
@@ -23,9 +23,14 @@ variables:
|
|||||||
# GIT_STRATEGY is not defined here.
|
# GIT_STRATEGY is not defined here.
|
||||||
# Use an option from "CI / CD Settings" - "General pipelines".
|
# Use an option from "CI / CD Settings" - "General pipelines".
|
||||||
|
|
||||||
# "normal" strategy for fetching only top-level submodules since nothing requires the sub-submodules code for building IDF.
|
# we will download archive for each submodule instead of clone.
|
||||||
# If the "recursive" strategy is used we have a problem with using relative URLs for sub-submodules.
|
# we don't do "recursive" when fetch submodule as they're not used in CI now.
|
||||||
GIT_SUBMODULE_STRATEGY: normal
|
GIT_SUBMODULE_STRATEGY: none
|
||||||
|
SUBMODULE_FETCH_TOOL: "tools/ci/ci_fetch_submodule.py"
|
||||||
|
# by default we will fetch all submodules
|
||||||
|
# jobs can overwrite this variable to only fetch submodules they required
|
||||||
|
# set to "none" if don't need to fetch submodules
|
||||||
|
SUBMODULES_TO_FETCH: "all"
|
||||||
|
|
||||||
UNIT_TEST_BUILD_SYSTEM: cmake
|
UNIT_TEST_BUILD_SYSTEM: cmake
|
||||||
EXAMPLE_TEST_BUILD_SYSTEM: cmake
|
EXAMPLE_TEST_BUILD_SYSTEM: cmake
|
||||||
@@ -44,6 +49,7 @@ variables:
|
|||||||
CI_TARGET_TEST_CONFIG_FILE: "$CI_PROJECT_DIR/tools/ci/config/target-test.yml"
|
CI_TARGET_TEST_CONFIG_FILE: "$CI_PROJECT_DIR/tools/ci/config/target-test.yml"
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
# before each job, we need to check if this job is filtered by bot stage/job filter
|
# before each job, we need to check if this job is filtered by bot stage/job filter
|
||||||
.apply_bot_filter: &apply_bot_filter
|
.apply_bot_filter: &apply_bot_filter
|
||||||
python $APPLY_BOT_FILTER_SCRIPT || exit 0
|
python $APPLY_BOT_FILTER_SCRIPT || exit 0
|
||||||
@@ -72,12 +78,10 @@ variables:
|
|||||||
tools/idf_tools.py --non-interactive install && eval "$(tools/idf_tools.py --non-interactive export)" || exit 1
|
tools/idf_tools.py --non-interactive install && eval "$(tools/idf_tools.py --non-interactive export)" || exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
.show_submodule_urls: &show_submodule_urls |
|
.fetch_submodules: &fetch_submodules |
|
||||||
git config --get-regexp '^submodule\..*\.url$' || true
|
python $SUBMODULE_FETCH_TOOL -s $SUBMODULES_TO_FETCH
|
||||||
|
|
||||||
before_script:
|
before_script:
|
||||||
- echo "Running common script"
|
|
||||||
- *show_submodule_urls
|
|
||||||
- source tools/ci/setup_python.sh
|
- source tools/ci/setup_python.sh
|
||||||
# apply bot filter in before script
|
# apply bot filter in before script
|
||||||
- *apply_bot_filter
|
- *apply_bot_filter
|
||||||
@@ -93,6 +97,8 @@ before_script:
|
|||||||
|
|
||||||
- *setup_tools_unless_target_test
|
- *setup_tools_unless_target_test
|
||||||
|
|
||||||
|
- *fetch_submodules
|
||||||
|
|
||||||
- *setup_custom_toolchain
|
- *setup_custom_toolchain
|
||||||
|
|
||||||
# used for check scripts which we want to run unconditionally
|
# used for check scripts which we want to run unconditionally
|
||||||
|
106
tools/ci/ci_fetch_submodule.py
Normal file
106
tools/ci/ci_fetch_submodule.py
Normal file
@@ -0,0 +1,106 @@
|
|||||||
|
#!/usr/bin/env python
|
||||||
|
|
||||||
|
# internal use only for CI
|
||||||
|
# download archive of one commit instead of cloning entire submodule repo
|
||||||
|
|
||||||
|
import re
|
||||||
|
import os
|
||||||
|
import subprocess
|
||||||
|
import argparse
|
||||||
|
import shutil
|
||||||
|
import time
|
||||||
|
|
||||||
|
import gitlab_api
|
||||||
|
|
||||||
|
SUBMODULE_PATTERN = re.compile(r"\[submodule \"([^\"]+)\"]")
|
||||||
|
PATH_PATTERN = re.compile(r"path\s+=\s+(\S+)")
|
||||||
|
URL_PATTERN = re.compile(r"url\s+=\s+(\S+)")
|
||||||
|
|
||||||
|
SUBMODULE_ARCHIVE_TEMP_FOLDER = "submodule_archive"
|
||||||
|
|
||||||
|
|
||||||
|
class SubModule(object):
|
||||||
|
# We don't need to support recursive submodule clone now
|
||||||
|
|
||||||
|
GIT_LS_TREE_OUTPUT_PATTERN = re.compile(r"\d+\s+commit\s+([0-9a-f]+)\s+")
|
||||||
|
|
||||||
|
def __init__(self, gitlab_inst, path, url):
|
||||||
|
self.path = path
|
||||||
|
self.gitlab_inst = gitlab_inst
|
||||||
|
self.project_id = self._get_project_id(url)
|
||||||
|
self.commit_id = self._get_commit_id(path)
|
||||||
|
|
||||||
|
def _get_commit_id(self, path):
|
||||||
|
output = subprocess.check_output(["git", "ls-tree", "HEAD", path])
|
||||||
|
# example output: 160000 commit d88a262fbdf35e5abb372280eb08008749c3faa0 components/esp_wifi/lib
|
||||||
|
match = self.GIT_LS_TREE_OUTPUT_PATTERN.search(output)
|
||||||
|
return match.group(1)
|
||||||
|
|
||||||
|
def _get_project_id(self, url):
|
||||||
|
base_name = os.path.basename(url)
|
||||||
|
project_id = self.gitlab_inst.get_project_id(os.path.splitext(base_name)[0], # remove .git
|
||||||
|
namespace="espressif")
|
||||||
|
return project_id
|
||||||
|
|
||||||
|
def download_archive(self):
|
||||||
|
print("Update submodule: {}: {}".format(self.path, self.commit_id))
|
||||||
|
path_name = self.gitlab_inst.download_archive(self.commit_id, SUBMODULE_ARCHIVE_TEMP_FOLDER,
|
||||||
|
self.project_id)
|
||||||
|
renamed_path = os.path.join(os.path.dirname(path_name), os.path.basename(self.path))
|
||||||
|
os.rename(path_name, renamed_path)
|
||||||
|
shutil.rmtree(self.path, ignore_errors=True)
|
||||||
|
shutil.move(renamed_path, os.path.dirname(self.path))
|
||||||
|
|
||||||
|
|
||||||
|
def update_submodule(git_module_file, submodules_to_update):
|
||||||
|
gitlab_inst = gitlab_api.Gitlab()
|
||||||
|
submodules = []
|
||||||
|
with open(git_module_file, "r") as f:
|
||||||
|
data = f.read()
|
||||||
|
match = SUBMODULE_PATTERN.search(data)
|
||||||
|
while True:
|
||||||
|
next_match = SUBMODULE_PATTERN.search(data, pos=match.end())
|
||||||
|
if next_match:
|
||||||
|
end_pos = next_match.start()
|
||||||
|
else:
|
||||||
|
end_pos = len(data)
|
||||||
|
path_match = PATH_PATTERN.search(data, pos=match.end(), endpos=end_pos)
|
||||||
|
url_match = URL_PATTERN.search(data, pos=match.end(), endpos=end_pos)
|
||||||
|
path = path_match.group(1)
|
||||||
|
url = url_match.group(1)
|
||||||
|
|
||||||
|
filter_result = True
|
||||||
|
if submodules_to_update:
|
||||||
|
if path not in submodules_to_update:
|
||||||
|
filter_result = False
|
||||||
|
if filter_result:
|
||||||
|
submodules.append(SubModule(gitlab_inst, path, url))
|
||||||
|
|
||||||
|
match = next_match
|
||||||
|
if not match:
|
||||||
|
break
|
||||||
|
|
||||||
|
shutil.rmtree(SUBMODULE_ARCHIVE_TEMP_FOLDER, ignore_errors=True)
|
||||||
|
|
||||||
|
for submodule in submodules:
|
||||||
|
submodule.download_archive()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
start_time = time.time()
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument("--repo_path", "-p", default=".", help="repo path")
|
||||||
|
parser.add_argument("--submodule", "-s", default="all",
|
||||||
|
help="Submodules to update. By default update all submodules. "
|
||||||
|
"For multiple submodules, separate them with `;`. "
|
||||||
|
"`all` and `none` are special values that indicates we fetch all / none submodules")
|
||||||
|
args = parser.parse_args()
|
||||||
|
if args.submodule == "none":
|
||||||
|
print("don't need to update submodules")
|
||||||
|
exit(0)
|
||||||
|
if args.submodule == "all":
|
||||||
|
_submodules = []
|
||||||
|
else:
|
||||||
|
_submodules = args.submodule.split(";")
|
||||||
|
update_submodule(os.path.join(args.repo_path, ".gitmodules"), _submodules)
|
||||||
|
print("total time spent on update submodule: {:.02f}s".format(time.time() - start_time))
|
@@ -10,6 +10,7 @@ assign_test:
|
|||||||
- build_ssc
|
- build_ssc
|
||||||
- build_esp_idf_tests_cmake
|
- build_esp_idf_tests_cmake
|
||||||
variables:
|
variables:
|
||||||
|
SUBMODULES_TO_FETCH: "components/esptool_py/esptool"
|
||||||
EXAMPLE_CONFIG_OUTPUT_PATH: "$CI_PROJECT_DIR/examples/test_configs"
|
EXAMPLE_CONFIG_OUTPUT_PATH: "$CI_PROJECT_DIR/examples/test_configs"
|
||||||
UNIT_TEST_CASE_FILE: "${CI_PROJECT_DIR}/components/idf_test/unit_test/TestCaseAll.yml"
|
UNIT_TEST_CASE_FILE: "${CI_PROJECT_DIR}/components/idf_test/unit_test/TestCaseAll.yml"
|
||||||
artifacts:
|
artifacts:
|
||||||
@@ -55,6 +56,7 @@ update_test_cases:
|
|||||||
- ${CI_PROJECT_DIR}/test-management/*.log
|
- ${CI_PROJECT_DIR}/test-management/*.log
|
||||||
expire_in: 1 week
|
expire_in: 1 week
|
||||||
variables:
|
variables:
|
||||||
|
SUBMODULES_TO_FETCH: "components/esptool_py/esptool"
|
||||||
UNIT_TEST_CASE_FILE: "${CI_PROJECT_DIR}/components/idf_test/unit_test/TestCaseAll.yml"
|
UNIT_TEST_CASE_FILE: "${CI_PROJECT_DIR}/components/idf_test/unit_test/TestCaseAll.yml"
|
||||||
BOT_ACCOUNT_CONFIG_FILE: "${CI_PROJECT_DIR}/test-management/Config/Account.local.yml"
|
BOT_ACCOUNT_CONFIG_FILE: "${CI_PROJECT_DIR}/test-management/Config/Account.local.yml"
|
||||||
AUTO_TEST_SCRIPT_PATH: "${CI_PROJECT_DIR}/auto_test_script"
|
AUTO_TEST_SCRIPT_PATH: "${CI_PROJECT_DIR}/auto_test_script"
|
||||||
|
@@ -10,7 +10,7 @@ check_submodule_sync:
|
|||||||
retry: 2
|
retry: 2
|
||||||
variables:
|
variables:
|
||||||
GIT_STRATEGY: clone
|
GIT_STRATEGY: clone
|
||||||
GIT_SUBMODULE_STRATEGY: none
|
SUBMODULES_TO_FETCH: "none"
|
||||||
PUBLIC_IDF_URL: "https://github.com/espressif/esp-idf.git"
|
PUBLIC_IDF_URL: "https://github.com/espressif/esp-idf.git"
|
||||||
before_script: []
|
before_script: []
|
||||||
after_script: []
|
after_script: []
|
||||||
|
@@ -33,7 +33,7 @@
|
|||||||
CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/examples/test_configs"
|
CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/examples/test_configs"
|
||||||
LOG_PATH: "$CI_PROJECT_DIR/TEST_LOGS"
|
LOG_PATH: "$CI_PROJECT_DIR/TEST_LOGS"
|
||||||
ENV_FILE: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/EnvConfig.yml"
|
ENV_FILE: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/EnvConfig.yml"
|
||||||
GIT_SUBMODULE_STRATEGY: none
|
SUBMODULES_TO_FETCH: "components/esptool_py/esptool"
|
||||||
script:
|
script:
|
||||||
- *define_config_file_name
|
- *define_config_file_name
|
||||||
# first test if config file exists, if not exist, exit 0
|
# first test if config file exists, if not exist, exit 0
|
||||||
@@ -41,9 +41,6 @@
|
|||||||
# clone test env configs
|
# clone test env configs
|
||||||
- git clone $TEST_ENV_CONFIG_REPOSITORY
|
- git clone $TEST_ENV_CONFIG_REPOSITORY
|
||||||
- python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs
|
- python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs
|
||||||
# Get esptool
|
|
||||||
- git submodule sync
|
|
||||||
- git submodule update --init components/esptool_py/esptool
|
|
||||||
- cd tools/ci/python_packages/tiny_test_fw/bin
|
- cd tools/ci/python_packages/tiny_test_fw/bin
|
||||||
# run test
|
# run test
|
||||||
- python Runner.py $TEST_CASE_PATH -c $CONFIG_FILE -e $ENV_FILE
|
- python Runner.py $TEST_CASE_PATH -c $CONFIG_FILE -e $ENV_FILE
|
||||||
@@ -134,7 +131,7 @@
|
|||||||
- $LOG_PATH
|
- $LOG_PATH
|
||||||
expire_in: 1 week
|
expire_in: 1 week
|
||||||
variables:
|
variables:
|
||||||
GIT_SUBMODULE_STRATEGY: none
|
SUBMODULES_TO_FETCH: "components/esptool_py/esptool"
|
||||||
LOCAL_ENV_CONFIG_PATH: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/ESP32_IDF"
|
LOCAL_ENV_CONFIG_PATH: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/ESP32_IDF"
|
||||||
LOG_PATH: "${CI_PROJECT_DIR}/${CI_COMMIT_SHA}"
|
LOG_PATH: "${CI_PROJECT_DIR}/${CI_COMMIT_SHA}"
|
||||||
TEST_CASE_FILE_PATH: "$CI_PROJECT_DIR/components/idf_test/integration_test"
|
TEST_CASE_FILE_PATH: "$CI_PROJECT_DIR/components/idf_test/integration_test"
|
||||||
@@ -195,8 +192,6 @@ example_test_001:
|
|||||||
example_test_002:
|
example_test_002:
|
||||||
extends: .example_test_template
|
extends: .example_test_template
|
||||||
image: $CI_DOCKER_REGISTRY/ubuntu-test-env$BOT_DOCKER_IMAGE_TAG
|
image: $CI_DOCKER_REGISTRY/ubuntu-test-env$BOT_DOCKER_IMAGE_TAG
|
||||||
variables:
|
|
||||||
GIT_SUBMODULE_STRATEGY: normal
|
|
||||||
tags:
|
tags:
|
||||||
- ESP32
|
- ESP32
|
||||||
- Example_ShieldBox_Basic
|
- Example_ShieldBox_Basic
|
||||||
|
Reference in New Issue
Block a user