From 651f60e66359178d78a15a75601f0a2d883f113a Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Thu, 19 Oct 2023 11:53:18 +0200 Subject: [PATCH 1/9] ci: improve check pylint get files command --- .gitlab/ci/static-code-analysis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/ci/static-code-analysis.yml b/.gitlab/ci/static-code-analysis.yml index c583dfbc86..d12741fc3d 100644 --- a/.gitlab/ci/static-code-analysis.yml +++ b/.gitlab/ci/static-code-analysis.yml @@ -32,7 +32,7 @@ check_pylint: if [ -n "$CI_MERGE_REQUEST_IID" ]; then export files=$(echo "$GIT_DIFF_OUTPUT" | grep ".py$" | xargs); else - export files=$(find . -iname "*.py" -print); + export files=$(git ls-files "*.py" | xargs); fi - if [ -z "$files" ]; then echo "No python files found"; exit 0; fi - run_cmd pylint --exit-zero --load-plugins=pylint_gitlab --output-format=gitlab-codeclimate:pylint.json $files From 911ad39808258bc2dd783f901de12e7468a9ab9d Mon Sep 17 00:00:00 2001 From: Aleksei Apaseev Date: Fri, 27 Oct 2023 17:32:47 +0800 Subject: [PATCH 2/9] ci: Upload artifacts of target stage jobs to s3 bucket --- .gitlab/ci/target-test.yml | 4 ++++ tools/ci/artifacts_handler.py | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/.gitlab/ci/target-test.yml b/.gitlab/ci/target-test.yml index 824d5c6273..ab5b531afc 100644 --- a/.gitlab/ci/target-test.yml +++ b/.gitlab/ci/target-test.yml @@ -49,6 +49,7 @@ --parallel-index ${CI_NODE_INDEX:-1} ${PYTEST_EXTRA_FLAGS} --app-info-filepattern \"list_job_*.txt\" + - python tools/ci/artifacts_handler.py upload --type logs junit_reports .pytest_examples_dir_template: extends: .pytest_template @@ -1297,6 +1298,7 @@ pytest_examples_openthread_br: --parallel-index ${CI_NODE_INDEX:-1} ${PYTEST_EXTRA_FLAGS} --app-info-filepattern \"list_job_*.txt\" + - python tools/ci/artifacts_handler.py upload --type logs junit_reports pytest_examples_openthread_bbr: extends: @@ -1331,6 +1333,7 @@ pytest_examples_openthread_bbr: --parallel-index ${CI_NODE_INDEX:-1} ${PYTEST_EXTRA_FLAGS} --app-info-filepattern \"list_job_*.txt\" + - python tools/ci/artifacts_handler.py upload --type logs junit_reports pytest_examples_openthread_sleep: extends: @@ -1362,6 +1365,7 @@ pytest_examples_openthread_sleep: --parallel-index ${CI_NODE_INDEX:-1} ${PYTEST_EXTRA_FLAGS} --app-info-filepattern \"list_job_*.txt\" + - python tools/ci/artifacts_handler.py upload --type logs junit_reports pytest_examples_esp32h2_zigbee: extends: diff --git a/tools/ci/artifacts_handler.py b/tools/ci/artifacts_handler.py index 070a88e5ca..496e88e6b8 100644 --- a/tools/ci/artifacts_handler.py +++ b/tools/ci/artifacts_handler.py @@ -21,6 +21,7 @@ class ArtifactType(str, Enum): LOGS = 'logs' SIZE_REPORTS = 'size_reports' + JUNIT_REPORTS = 'junit_reports' TYPE_PATTERNS_DICT = { @@ -48,6 +49,9 @@ TYPE_PATTERNS_DICT = { '**/build*/size.json', 'size_info.txt', ], + ArtifactType.JUNIT_REPORTS: [ + 'XUNIT_RESULT.xml', + ], } From 21b06f43bf1d6551304d5872767f6881f99b9541 Mon Sep 17 00:00:00 2001 From: "igor udot (horw)" Date: Mon, 6 Nov 2023 14:34:33 +0800 Subject: [PATCH 3/9] ci: added build log artifacts for linux --- .gitlab/ci/host-test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab/ci/host-test.yml b/.gitlab/ci/host-test.yml index 7950c07125..fee0070f2e 100644 --- a/.gitlab/ci/host-test.yml +++ b/.gitlab/ci/host-test.yml @@ -325,6 +325,7 @@ test_pytest_linux: paths: - XUNIT_RESULT.xml - pytest_embedded_log/ + - "**/build*/build_log.txt" reports: junit: XUNIT_RESULT.xml expire_in: 1 week From e6a4ad3d556c771c2df70d370ccd24f721ca9cdb Mon Sep 17 00:00:00 2001 From: Aleksei Apaseev Date: Fri, 3 Nov 2023 18:09:34 +0800 Subject: [PATCH 4/9] ci: Move upload artifacts execution of build and target_test stages to the after script section --- .gitlab/ci/common.yml | 7 +++++++ .gitlab/ci/target-test.yml | 6 ++---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/.gitlab/ci/common.yml b/.gitlab/ci/common.yml index 18f78b1e05..c804b66611 100644 --- a/.gitlab/ci/common.yml +++ b/.gitlab/ci/common.yml @@ -172,6 +172,12 @@ variables: # Show ccache statistics if enabled globally test "$CI_CCACHE_STATS" == 1 && test -n "$(which ccache)" && ccache --show-stats || true +.upload_failed_job_log_artifacts: &upload_failed_job_log_artifacts | + - | + if [ $CI_JOB_STATUS = "failed" ]; then + python tools/ci/artifacts_handler.py upload --type logs + fi + .before_script:minimal: before_script: - *common-before_scripts @@ -198,6 +204,7 @@ variables: .after_script:build:ccache: after_script: - *show_ccache_statistics + - *upload_failed_job_log_artifacts ############################## # Git Strategy Job Templates # diff --git a/.gitlab/ci/target-test.yml b/.gitlab/ci/target-test.yml index ab5b531afc..cc8a237bee 100644 --- a/.gitlab/ci/target-test.yml +++ b/.gitlab/ci/target-test.yml @@ -11,6 +11,8 @@ paths: - .cache/pip policy: pull + after_script: + - python tools/ci/artifacts_handler.py upload --type logs junit_reports .pytest_template: extends: @@ -49,7 +51,6 @@ --parallel-index ${CI_NODE_INDEX:-1} ${PYTEST_EXTRA_FLAGS} --app-info-filepattern \"list_job_*.txt\" - - python tools/ci/artifacts_handler.py upload --type logs junit_reports .pytest_examples_dir_template: extends: .pytest_template @@ -1298,7 +1299,6 @@ pytest_examples_openthread_br: --parallel-index ${CI_NODE_INDEX:-1} ${PYTEST_EXTRA_FLAGS} --app-info-filepattern \"list_job_*.txt\" - - python tools/ci/artifacts_handler.py upload --type logs junit_reports pytest_examples_openthread_bbr: extends: @@ -1333,7 +1333,6 @@ pytest_examples_openthread_bbr: --parallel-index ${CI_NODE_INDEX:-1} ${PYTEST_EXTRA_FLAGS} --app-info-filepattern \"list_job_*.txt\" - - python tools/ci/artifacts_handler.py upload --type logs junit_reports pytest_examples_openthread_sleep: extends: @@ -1365,7 +1364,6 @@ pytest_examples_openthread_sleep: --parallel-index ${CI_NODE_INDEX:-1} ${PYTEST_EXTRA_FLAGS} --app-info-filepattern \"list_job_*.txt\" - - python tools/ci/artifacts_handler.py upload --type logs junit_reports pytest_examples_esp32h2_zigbee: extends: From 6947dd533afff043415db435a469b27629062eed Mon Sep 17 00:00:00 2001 From: Chen Yudong Date: Thu, 16 Nov 2023 10:54:37 +0800 Subject: [PATCH 5/9] ci: fix fetch_from_mirror_url_if_exists --- .gitlab/ci/common.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab/ci/common.yml b/.gitlab/ci/common.yml index c804b66611..27dad45334 100644 --- a/.gitlab/ci/common.yml +++ b/.gitlab/ci/common.yml @@ -229,7 +229,8 @@ variables: else git remote add origin "${MIRROR_REPO_URL}" fi - git fetch origin --no-recurse-submodules + # mirror url may fail with authentication issue + git fetch origin --no-recurse-submodules || true fi # set remote url to CI_REPOSITORY_URL From ceb9e4205800c48febe0e2e623d3735e4e8d43f8 Mon Sep 17 00:00:00 2001 From: Chen Yudong Date: Mon, 20 Nov 2023 16:19:47 +0800 Subject: [PATCH 6/9] ci: fix fetch git diff from pipeline schedules --- .gitlab/ci/common.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.gitlab/ci/common.yml b/.gitlab/ci/common.yml index 27dad45334..d5e2fed1de 100644 --- a/.gitlab/ci/common.yml +++ b/.gitlab/ci/common.yml @@ -271,10 +271,14 @@ variables: git fetch origin $CI_COMMIT_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS} export GIT_DIFF_OUTPUT=$(git diff --name-only $CI_MERGE_REQUEST_DIFF_BASE_SHA $CI_COMMIT_SHA) # other pipelines, like the protected branches pipelines - else + elif [[ "$CI_COMMIT_BEFORE_SHA" != "0000000000000000000000000000000000000000" ]]; then git fetch origin $CI_COMMIT_BEFORE_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS} git fetch origin $CI_COMMIT_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS} export GIT_DIFF_OUTPUT=$(git diff --name-only $CI_COMMIT_BEFORE_SHA $CI_COMMIT_SHA) + else + # pipeline source could be web, scheduler, etc. + git fetch origin $CI_COMMIT_SHA --depth=2 ${GIT_FETCH_EXTRA_FLAGS} + export GIT_DIFF_OUTPUT=$(git diff --name-only $CI_COMMIT_SHA~1 $CI_COMMIT_SHA) fi - *git_checkout_fetch_head - *common-before_scripts From 8ce0b8bf73ea26dd474cf8174e03620c49fa6fbe Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Thu, 23 Nov 2023 08:03:42 +0100 Subject: [PATCH 7/9] ci: fix upload failed job log script --- .gitlab/ci/common.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.gitlab/ci/common.yml b/.gitlab/ci/common.yml index d5e2fed1de..0b076cecd5 100644 --- a/.gitlab/ci/common.yml +++ b/.gitlab/ci/common.yml @@ -173,10 +173,9 @@ variables: test "$CI_CCACHE_STATS" == 1 && test -n "$(which ccache)" && ccache --show-stats || true .upload_failed_job_log_artifacts: &upload_failed_job_log_artifacts | - - | - if [ $CI_JOB_STATUS = "failed" ]; then - python tools/ci/artifacts_handler.py upload --type logs - fi + if [ $CI_JOB_STATUS = "failed" ]; then + python tools/ci/artifacts_handler.py upload --type logs + fi .before_script:minimal: before_script: From 8dcf3e5f8081d690d9be731c2f80bebd5115b5d6 Mon Sep 17 00:00:00 2001 From: Aleksei Apaseev Date: Fri, 8 Dec 2023 10:49:17 +0800 Subject: [PATCH 8/9] ci: Upload report about modified files and components to s3 bucket --- .gitlab/ci/pre_check.yml | 1 + tools/ci/artifacts_handler.py | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/.gitlab/ci/pre_check.yml b/.gitlab/ci/pre_check.yml index 5962f84ca1..78892a5cdf 100644 --- a/.gitlab/ci/pre_check.yml +++ b/.gitlab/ci/pre_check.yml @@ -186,6 +186,7 @@ pipeline_variables: echo "BUILD_AND_TEST_ALL_APPS=1" >> pipeline.env fi - cat pipeline.env + - python tools/ci/artifacts_handler.py upload --type modified_files_and_components_report artifacts: reports: dotenv: pipeline.env diff --git a/tools/ci/artifacts_handler.py b/tools/ci/artifacts_handler.py index 496e88e6b8..73930a3a05 100644 --- a/tools/ci/artifacts_handler.py +++ b/tools/ci/artifacts_handler.py @@ -22,6 +22,7 @@ class ArtifactType(str, Enum): LOGS = 'logs' SIZE_REPORTS = 'size_reports' JUNIT_REPORTS = 'junit_reports' + MODIFIED_FILES_AND_COMPONENTS_REPORT = 'modified_files_and_components_report' TYPE_PATTERNS_DICT = { @@ -52,6 +53,9 @@ TYPE_PATTERNS_DICT = { ArtifactType.JUNIT_REPORTS: [ 'XUNIT_RESULT.xml', ], + ArtifactType.MODIFIED_FILES_AND_COMPONENTS_REPORT: [ + 'pipeline.env', + ], } From 44354375016dbc0a5070f83bf1a75e371f85a1f1 Mon Sep 17 00:00:00 2001 From: Marius Vikhammer Date: Thu, 16 Nov 2023 10:23:25 +0800 Subject: [PATCH 9/9] ci(docs): revert git strategy change for deploy docs --- .gitlab/ci/docs.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.gitlab/ci/docs.yml b/.gitlab/ci/docs.yml index eed9f87dc5..95141ffdb9 100644 --- a/.gitlab/ci/docs.yml +++ b/.gitlab/ci/docs.yml @@ -196,8 +196,6 @@ build_docs_pdf_prod: .deploy_docs_template: image: $ESP_IDF_DOC_ENV_IMAGE - extends: - - .before_script:fetch:git_describe variables: DOCS_BUILD_DIR: "${IDF_PATH}/docs/_build/" PYTHONUNBUFFERED: 1