diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2d6ec408f7..512e0bca68 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -67,8 +67,6 @@ variables: # target test repo parameters TEST_ENV_CONFIG_REPO: "https://gitlab-ci-token:${BOT_TOKEN}@${CI_SERVER_HOST}:${CI_SERVER_PORT}/qa/ci-test-runner-configs.git" - CI_AUTO_TEST_SCRIPT_REPO_URL: "https://gitlab-ci-token:${BOT_TOKEN}@${CI_SERVER_HOST}:${CI_SERVER_PORT}/qa/auto_test_script.git" - CI_AUTO_TEST_SCRIPT_REPO_BRANCH: "ci/v4.1" .setup_tools_unless_target_test: &setup_tools_unless_target_test | if [[ -n "$IDF_DONT_USE_MIRRORS" ]]; then diff --git a/.gitlab/ci/assign-test.yml b/.gitlab/ci/assign-test.yml index 6466d15bd8..41cc88c230 100644 --- a/.gitlab/ci/assign-test.yml +++ b/.gitlab/ci/assign-test.yml @@ -108,6 +108,8 @@ assign_integration_test: extends: - .assign_test_template - .rules:test:integration_test + - .before_script_minimal + image: ${CI_INTEGRATION_TEST_ENV_IMAGE} needs: - build_ssc_esp32 - build_ssc_esp32c3 @@ -120,13 +122,14 @@ assign_integration_test: INTEGRATION_TEST_CASE_PATH: "${CI_PROJECT_DIR}/auto_test_script/TestCaseFiles" ASSIGN_TEST_CASE_SCRIPT: "${CI_PROJECT_DIR}/auto_test_script/bin/CIAssignTestCases.py" PYTHONPATH: ${CI_PROJECT_DIR}/auto_test_script/packages - PYTHON_VER: 3.7.7 + GIT_LFS_SKIP_SMUDGE: 1 script: + - add_gitlab_ssh_keys # clone test script to assign tests - # can not retry if downing git lfs files failed, so using empty_branch first. - - retry_failed git clone ${CI_AUTO_TEST_SCRIPT_REPO_URL} -b empty_branch - - retry_failed git -C auto_test_script checkout -f ${CI_AUTO_TEST_SCRIPT_REPO_BRANCH} - - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script --customized_only + - retry_failed git clone ${CI_AUTO_TEST_SCRIPT_REPO_URL} auto_test_script + - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script + - cd auto_test_script + - ./tools/ci/setup_idfci.sh # assign integration test cases - python ${ASSIGN_TEST_CASE_SCRIPT} -t ${INTEGRATION_TEST_CASE_PATH} -c $CI_TARGET_TEST_CONFIG_FILE -b ${BUILD_DIR} -o $TEST_DIR/test_configs diff --git a/.gitlab/ci/target-test.yml b/.gitlab/ci/target-test.yml index 2bb6799ebd..07529d7b15 100644 --- a/.gitlab/ci/target-test.yml +++ b/.gitlab/ci/target-test.yml @@ -781,6 +781,8 @@ component_ut_test_lan8720: extends: - .target_test_job_template - .rules:test:integration_test + - .before_script_minimal + image: ${CI_INTEGRATION_TEST_ENV_IMAGE} needs: # the assign already needs all the build jobs - assign_integration_test variables: @@ -791,22 +793,23 @@ component_ut_test_lan8720: KNOWN_ISSUE_FILE: "${CI_PROJECT_DIR}/tools/ci/integration_test/KnownIssues" CI_RUNNER_SCRIPT: "${CI_PROJECT_DIR}/auto_test_script/bin/CIRunner.py" PREPARE_TEST_BIN_SCRIPT: "${CI_PROJECT_DIR}/tools/ci/integration_test/prepare_test_bins.py" - PYTHONPATH: "${CI_PROJECT_DIR}/auto_test_script/packages:${PYTHONPATH}" + PYTHONPATH: "${CI_PROJECT_DIR}/auto_test_script/packages:${CI_PROJECT_DIR}/tools/ci/python_packages:${PYTHONPATH}" INITIAL_CONDITION_RETRY_COUNT: "1" - # auto_test_script only supports python 3.7.x - PYTHON_VER: 3.7.7 + GIT_LFS_SKIP_SMUDGE: 1 script: - *define_config_file_name # first test if config file exists, if not exist, exit 0 - test -e $CONFIG_FILE || exit 0 + - add_gitlab_ssh_keys # clone local test env configs - retry_failed git clone $TEST_ENV_CONFIG_REPO - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs # clone test bench - # can not retry if downing git lfs files failed, so using empty_branch first. - - retry_failed git clone ${CI_AUTO_TEST_SCRIPT_REPO_URL} -b empty_branch - - retry_failed git -C auto_test_script checkout -f ${CI_AUTO_TEST_SCRIPT_REPO_BRANCH} - - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script --customized_only + - retry_failed git clone ${CI_AUTO_TEST_SCRIPT_REPO_URL} auto_test_script + - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script + - cd auto_test_script + - ./tools/ci/setup_idfci.sh + # Merge known issues - cat ${KNOWN_ISSUE_FILE} >> ${TEST_CASE_FILE_PATH}/KnownIssues # run test - python ${PREPARE_TEST_BIN_SCRIPT} $CONFIG_FILE @@ -833,16 +836,16 @@ nvs_compatible_test: - *define_config_file_name # first test if config file exists, if not exist, exit 0 - test -e $CONFIG_FILE || exit 0 + - add_gitlab_ssh_keys # clone local test env configs - retry_failed git clone $TEST_ENV_CONFIG_REPO - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs # clone test bench - # can not retry if downing git lfs files failed, so using empty_branch first. - - retry_failed git clone ${CI_AUTO_TEST_SCRIPT_REPO_URL} -b empty_branch - - retry_failed git -C auto_test_script checkout -f ${CI_AUTO_TEST_SCRIPT_REPO_BRANCH} - - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script --customized_only - # prepare nvs bins + - retry_failed git clone ${CI_AUTO_TEST_SCRIPT_REPO_URL} auto_test_script + - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script - cd auto_test_script + - ./tools/ci/setup_idfci.sh + # prepare nvs bins - ./tools/prepare_nvs_bin.sh # run test - python ${PREPARE_TEST_BIN_SCRIPT} $CONFIG_FILE diff --git a/tools/ci/checkout_project_ref.py b/tools/ci/checkout_project_ref.py index a64e5f488a..60cd2b6297 100755 --- a/tools/ci/checkout_project_ref.py +++ b/tools/ci/checkout_project_ref.py @@ -10,7 +10,22 @@ import re import subprocess IDF_GIT_DESCRIBE_PATTERN = re.compile(r'^v(\d)\.(\d)') -RETRY_COUNT = 3 + + +def _idf_version_from_cmake(): + version_path = os.path.join(os.environ['IDF_PATH'], 'tools/cmake/version.cmake') + regex = re.compile(r'^\s*set\s*\(\s*IDF_VERSION_([A-Z]{5})\s+(\d+)') + try: + ver = {} + with open(version_path) as f: + for line in f: + m = regex.match(line) + if m: + ver[m.group(1)] = m.group(2) + return (int(ver['MAJOR']), int(ver['MINOR'])) + except (KeyError, OSError): + print('WARNING: Cannot find ESP-IDF version in version.cmake') + return (0, 0) def get_customized_project_revision(proj_name): @@ -44,20 +59,12 @@ def target_branch_candidates(proj_name): candidates.insert(0, customized_candidate) # branch name read from IDF - try: - git_describe = subprocess.check_output(['git', 'describe', 'HEAD']) - match = IDF_GIT_DESCRIBE_PATTERN.search(git_describe.decode()) - if match: - major_revision = match.group(1) - minor_revision = match.group(2) - # release branch - candidates.append('release/v{}.{}'.format(major_revision, minor_revision)) - # branch to match all major branches, like v3.x or v3 - candidates.append('release/v{}.x'.format(major_revision)) - candidates.append('release/v{}'.format(major_revision)) - except subprocess.CalledProcessError: - # this should not happen as IDF should have describe message - pass + major_revision, minor_revision = _idf_version_from_cmake() + # release branch + candidates.append('release/v{}.{}'.format(major_revision, minor_revision)) + # branch to match all major branches, like v3.x or v3 + candidates.append('release/v{}.x'.format(major_revision)) + candidates.append('release/v{}'.format(major_revision)) return [c for c in candidates if c] # filter out null value @@ -100,16 +107,10 @@ if __name__ == '__main__': continue if ref_to_use: - for _ in range(RETRY_COUNT): - # Add retry for projects with git-lfs - try: - subprocess.check_call(['git', 'checkout', '-f', ref_to_use], stdout=subprocess.PIPE) # not print the stdout - print('CI using ref {} for project {}'.format(ref_to_use, args.project)) - break - except subprocess.CalledProcessError: - pass - else: - print('Failed to use ref {} for project {}'.format(ref_to_use, args.project)) - exit(1) + try: + subprocess.check_call(['git', 'checkout', '-f', ref_to_use], stdout=subprocess.PIPE) # not print the stdout + print('CI using ref {} for project {}'.format(ref_to_use, args.project)) + except subprocess.CalledProcessError: + pass else: print('using default branch')