diff --git a/.gitlab/ci/common.yml b/.gitlab/ci/common.yml index 18f78b1e05..0b076cecd5 100644 --- a/.gitlab/ci/common.yml +++ b/.gitlab/ci/common.yml @@ -172,6 +172,11 @@ variables: # Show ccache statistics if enabled globally test "$CI_CCACHE_STATS" == 1 && test -n "$(which ccache)" && ccache --show-stats || true +.upload_failed_job_log_artifacts: &upload_failed_job_log_artifacts | + if [ $CI_JOB_STATUS = "failed" ]; then + python tools/ci/artifacts_handler.py upload --type logs + fi + .before_script:minimal: before_script: - *common-before_scripts @@ -198,6 +203,7 @@ variables: .after_script:build:ccache: after_script: - *show_ccache_statistics + - *upload_failed_job_log_artifacts ############################## # Git Strategy Job Templates # @@ -222,7 +228,8 @@ variables: else git remote add origin "${MIRROR_REPO_URL}" fi - git fetch origin --no-recurse-submodules + # mirror url may fail with authentication issue + git fetch origin --no-recurse-submodules || true fi # set remote url to CI_REPOSITORY_URL @@ -263,10 +270,14 @@ variables: git fetch origin $CI_COMMIT_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS} export GIT_DIFF_OUTPUT=$(git diff --name-only $CI_MERGE_REQUEST_DIFF_BASE_SHA $CI_COMMIT_SHA) # other pipelines, like the protected branches pipelines - else + elif [[ "$CI_COMMIT_BEFORE_SHA" != "0000000000000000000000000000000000000000" ]]; then git fetch origin $CI_COMMIT_BEFORE_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS} git fetch origin $CI_COMMIT_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS} export GIT_DIFF_OUTPUT=$(git diff --name-only $CI_COMMIT_BEFORE_SHA $CI_COMMIT_SHA) + else + # pipeline source could be web, scheduler, etc. + git fetch origin $CI_COMMIT_SHA --depth=2 ${GIT_FETCH_EXTRA_FLAGS} + export GIT_DIFF_OUTPUT=$(git diff --name-only $CI_COMMIT_SHA~1 $CI_COMMIT_SHA) fi - *git_checkout_fetch_head - *common-before_scripts diff --git a/.gitlab/ci/docs.yml b/.gitlab/ci/docs.yml index eed9f87dc5..95141ffdb9 100644 --- a/.gitlab/ci/docs.yml +++ b/.gitlab/ci/docs.yml @@ -196,8 +196,6 @@ build_docs_pdf_prod: .deploy_docs_template: image: $ESP_IDF_DOC_ENV_IMAGE - extends: - - .before_script:fetch:git_describe variables: DOCS_BUILD_DIR: "${IDF_PATH}/docs/_build/" PYTHONUNBUFFERED: 1 diff --git a/.gitlab/ci/host-test.yml b/.gitlab/ci/host-test.yml index 7950c07125..fee0070f2e 100644 --- a/.gitlab/ci/host-test.yml +++ b/.gitlab/ci/host-test.yml @@ -325,6 +325,7 @@ test_pytest_linux: paths: - XUNIT_RESULT.xml - pytest_embedded_log/ + - "**/build*/build_log.txt" reports: junit: XUNIT_RESULT.xml expire_in: 1 week diff --git a/.gitlab/ci/pre_check.yml b/.gitlab/ci/pre_check.yml index 5962f84ca1..78892a5cdf 100644 --- a/.gitlab/ci/pre_check.yml +++ b/.gitlab/ci/pre_check.yml @@ -186,6 +186,7 @@ pipeline_variables: echo "BUILD_AND_TEST_ALL_APPS=1" >> pipeline.env fi - cat pipeline.env + - python tools/ci/artifacts_handler.py upload --type modified_files_and_components_report artifacts: reports: dotenv: pipeline.env diff --git a/.gitlab/ci/static-code-analysis.yml b/.gitlab/ci/static-code-analysis.yml index c583dfbc86..d12741fc3d 100644 --- a/.gitlab/ci/static-code-analysis.yml +++ b/.gitlab/ci/static-code-analysis.yml @@ -32,7 +32,7 @@ check_pylint: if [ -n "$CI_MERGE_REQUEST_IID" ]; then export files=$(echo "$GIT_DIFF_OUTPUT" | grep ".py$" | xargs); else - export files=$(find . -iname "*.py" -print); + export files=$(git ls-files "*.py" | xargs); fi - if [ -z "$files" ]; then echo "No python files found"; exit 0; fi - run_cmd pylint --exit-zero --load-plugins=pylint_gitlab --output-format=gitlab-codeclimate:pylint.json $files diff --git a/.gitlab/ci/target-test.yml b/.gitlab/ci/target-test.yml index 824d5c6273..cc8a237bee 100644 --- a/.gitlab/ci/target-test.yml +++ b/.gitlab/ci/target-test.yml @@ -11,6 +11,8 @@ paths: - .cache/pip policy: pull + after_script: + - python tools/ci/artifacts_handler.py upload --type logs junit_reports .pytest_template: extends: diff --git a/tools/ci/artifacts_handler.py b/tools/ci/artifacts_handler.py index 070a88e5ca..73930a3a05 100644 --- a/tools/ci/artifacts_handler.py +++ b/tools/ci/artifacts_handler.py @@ -21,6 +21,8 @@ class ArtifactType(str, Enum): LOGS = 'logs' SIZE_REPORTS = 'size_reports' + JUNIT_REPORTS = 'junit_reports' + MODIFIED_FILES_AND_COMPONENTS_REPORT = 'modified_files_and_components_report' TYPE_PATTERNS_DICT = { @@ -48,6 +50,12 @@ TYPE_PATTERNS_DICT = { '**/build*/size.json', 'size_info.txt', ], + ArtifactType.JUNIT_REPORTS: [ + 'XUNIT_RESULT.xml', + ], + ArtifactType.MODIFIED_FILES_AND_COMPONENTS_REPORT: [ + 'pipeline.env', + ], }