Merge branch 'ci/optimize_ats_ci_flow_v4.3' into 'release/v4.3'

CI: optimize ATS ci flow (v4.3)

See merge request espressif/esp-idf!20177

.gitlab-ci.yml

@@ -59,8 +59,6 @@ variables:
 
   # target test repo parameters
   TEST_ENV_CONFIG_REPO: "https://gitlab-ci-token:${BOT_TOKEN}@${CI_SERVER_HOST}:${CI_SERVER_PORT}/qa/ci-test-runner-configs.git"
-  CI_AUTO_TEST_SCRIPT_REPO_URL: "https://gitlab-ci-token:${BOT_TOKEN}@${CI_SERVER_HOST}:${CI_SERVER_PORT}/qa/auto_test_script.git"
-  CI_AUTO_TEST_SCRIPT_REPO_BRANCH: "ci/v4.1"
 
   # Versioned esp-idf-doc env image to use for all document building jobs
   ESP_IDF_DOC_ENV_IMAGE: "$CI_DOCKER_REGISTRY/esp-idf-doc-env:v10"

tools/ci/checkout_project_ref.py

@@ -9,8 +9,21 @@ import os
 import re
 import subprocess
 
-IDF_GIT_DESCRIBE_PATTERN = re.compile(r'^v(\d)\.(\d)')
+
-RETRY_COUNT = 3
+def _idf_version_from_cmake():
+    version_path = os.path.join(os.environ['IDF_PATH'], 'tools/cmake/version.cmake')
+    regex = re.compile(r'^\s*set\s*\(\s*IDF_VERSION_([A-Z]{5})\s+(\d+)')
+    try:
+        ver = {}
+        with open(version_path) as f:
+            for line in f:
+                m = regex.match(line)
+                if m:
+                    ver[m.group(1)] = m.group(2)
+        return (int(ver['MAJOR']), int(ver['MINOR']))
+    except (KeyError, OSError):
+        print('WARNING: Cannot find ESP-IDF version in version.cmake')
+        return (0, 0)
 
 
 def get_customized_project_revision(proj_name):
@@ -44,20 +57,12 @@ def target_branch_candidates(proj_name):
         candidates.insert(0, customized_candidate)
 
     # branch name read from IDF
-    try:
+    major_revision, minor_revision = _idf_version_from_cmake()
-        git_describe = subprocess.check_output(['git', 'describe', 'HEAD'])
+    # release branch
-        match = IDF_GIT_DESCRIBE_PATTERN.search(git_describe.decode())
+    candidates.append('release/v{}.{}'.format(major_revision, minor_revision))
-        if match:
+    # branch to match all major branches, like v3.x or v3
-            major_revision = match.group(1)
+    candidates.append('release/v{}.x'.format(major_revision))
-            minor_revision = match.group(2)
+    candidates.append('release/v{}'.format(major_revision))
-            # release branch
-            candidates.append('release/v{}.{}'.format(major_revision, minor_revision))
-            # branch to match all major branches, like v3.x or v3
-            candidates.append('release/v{}.x'.format(major_revision))
-            candidates.append('release/v{}'.format(major_revision))
-    except subprocess.CalledProcessError:
-        # this should not happen as IDF should have describe message
-        pass
 
     return [c for c in candidates if c]  # filter out null value
 
@@ -84,23 +89,11 @@ if __name__ == '__main__':
 
     ref_to_use = ''
     for candidate in candidate_branches:
-        # check if candidate branch exists
+        try:
-        branch_match = subprocess.check_output(['git', 'branch', '-a', '--list', 'origin/' + candidate])
+            subprocess.check_call(['git', 'checkout', '-f', candidate], stdout=subprocess.PIPE)  # not print the stdout
-        if branch_match:
+            print('CI using ref {} for project {}'.format(candidate, args.project))
-            ref_to_use = candidate
             break
-
+        except subprocess.CalledProcessError:
-    if ref_to_use:
+            pass
-        for _ in range(RETRY_COUNT):
-            # Add retry for projects with git-lfs
-            try:
-                subprocess.check_call(['git', 'checkout', '-f', ref_to_use], stdout=subprocess.PIPE)  # not print the stdout
-                print('CI using ref {} for project {}'.format(ref_to_use, args.project))
-                break
-            except subprocess.CalledProcessError:
-                pass
-        else:
-            print('Failed to use ref {} for project {}'.format(ref_to_use, args.project))
-            exit(1)
     else:
         print('using default branch')

tools/ci/config/assign-test.yml

@@ -26,6 +26,7 @@ assign_test:
     PYTHONPATH: ${CI_PROJECT_DIR}/auto_test_script/packages
     # auto_test_script only supports python 3.7.x
     PYTHON_VER: 3.7.7
+    GIT_LFS_SKIP_SMUDGE: 1
   artifacts:
     paths:
       - components/idf_test/*/CIConfigs
@@ -45,11 +46,12 @@ assign_test:
     - python tools/ci/python_packages/ttfw_idf/IDFAssignTest.py component_ut $COMPONENT_UT_DIRS -c $CI_TARGET_TEST_CONFIG_FILE -o $COMPONENT_UT_OUTPUT_DIR/test_configs
     - python tools/ci/python_packages/ttfw_idf/IDFAssignTest.py unit_test $UNIT_TEST_DIR -c $CI_TARGET_TEST_CONFIG_FILE -o $UNIT_TEST_DIR/CIConfigs
     # clone test script to assign tests
-    # can not retry if downing git lfs files failed, so using empty_branch first.
+    - retry_failed git clone ${CI_AUTO_TEST_SCRIPT_REPO_URL} auto_test_script
-    - retry_failed git clone ${CI_AUTO_TEST_SCRIPT_REPO_URL} -b empty_branch
+    - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script
-    - retry_failed git -C auto_test_script checkout -f ${CI_AUTO_TEST_SCRIPT_REPO_BRANCH}
+    - cd auto_test_script
-    - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script --customized_only
+    - ./tools/ci/setup_idfci.sh
     # assign integration test cases
+    - cd ${CI_PROJECT_DIR}
     - python ${ASSIGN_TEST_CASE_SCRIPT} -t ${INTEGRATION_TEST_CASE_PATH} -c $CI_TARGET_TEST_CONFIG_FILE -b $IDF_PATH/SSC/ssc_bin -o $INTEGRATION_TEST_DIR/test_configs
 
 update_test_cases:

tools/ci/config/target-test.yml

@@ -114,6 +114,8 @@
   extends:
     - .target_test_job_template
     - .rules:tests:integration_test
+    - .before_script_lesser
+  image: ${CI_INTEGRATION_TEST_ENV_IMAGE}
   needs:
     - assign_test
   variables:
@@ -124,21 +126,23 @@
     CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/tools/ci/integration_test/test_configs"
     PREPARE_TEST_BIN_SCRIPT: "${CI_PROJECT_DIR}/tools/ci/integration_test/prepare_test_bins.py"
     CI_RUNNER_SCRIPT: "${CI_PROJECT_DIR}/auto_test_script/bin/CIRunner.py"
-    PYTHONPATH: ${CI_PROJECT_DIR}/auto_test_script/packages
+    PYTHONPATH: "${CI_PROJECT_DIR}/auto_test_script/packages:${CI_PROJECT_DIR}/tools/ci/python_packages:${PYTHONPATH}"
-    # auto_test_script only supports python 3.7.x
+    INITIAL_CONDITION_RETRY_COUNT: "1"
-    PYTHON_VER: 3.7.7
+    GIT_LFS_SKIP_SMUDGE: 1
   script:
     - *define_config_file_name
     # first test if config file exists, if not exist, exit 0
     - test -e $CONFIG_FILE || exit 0
+    - add_gitlab_ssh_keys
     # clone local test env configs
     - retry_failed git clone $TEST_ENV_CONFIG_REPO
     - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs
     # clone test bench
-    # can not retry if downing git lfs files failed, so using empty_branch first.
+    - retry_failed git clone ${CI_AUTO_TEST_SCRIPT_REPO_URL} auto_test_script
-    - retry_failed git clone ${CI_AUTO_TEST_SCRIPT_REPO_URL} -b empty_branch
+    - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script
-    - retry_failed git -C auto_test_script checkout -f ${CI_AUTO_TEST_SCRIPT_REPO_BRANCH}
+    - cd auto_test_script
-    - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script --customized_only
+    - ./tools/ci/setup_idfci.sh
+    # Merge known issues
     - cat ${INTEGRATION_TEST_DIR}/KnownIssues >> ${TEST_CASE_FILE_PATH}/KnownIssues
     # run test
     - python ${PREPARE_TEST_BIN_SCRIPT} $CONFIG_FILE
@@ -632,16 +636,16 @@ nvs_compatible_test:
     - *define_config_file_name
     # first test if config file exists, if not exist, exit 0
     - test -e $CONFIG_FILE || exit 0
+    - add_gitlab_ssh_keys
     # clone local test env configs
     - retry_failed git clone $TEST_ENV_CONFIG_REPO
     - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs
     # clone test bench
-    # can not retry if downing git lfs files failed, so using empty_branch first.
+    - retry_failed git clone ${CI_AUTO_TEST_SCRIPT_REPO_URL} auto_test_script
-    - retry_failed git clone ${CI_AUTO_TEST_SCRIPT_REPO_URL} -b empty_branch
+    - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script
-    - retry_failed git -C auto_test_script checkout -f ${CI_AUTO_TEST_SCRIPT_REPO_BRANCH}
-    - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script --customized_only
-    # prepare nvs bins
     - cd auto_test_script
+    - ./tools/ci/setup_idfci.sh
+    # prepare nvs bins
     - ./tools/prepare_nvs_bin.sh
     # run test
     - python ${PREPARE_TEST_BIN_SCRIPT} $CONFIG_FILE