feedc0de/esp-idf
1
0
Fork 0
forked from espressif/esp-idf
Code Pull Requests Activity

Merge branch 'ci/build_jobs_use_cache' into 'master'

Browse Source 
ci: Upload Artifacts to Minio Server Instead of Gitlab Server

Closes IDFCI-1653

See merge request espressif/esp-idf!25664
This commit is contained in:
Fu Hanxi
2023-10-16 00:49:47 +08:00
parent 05b345d6bc 79bc66b323
commit 786a641851
19 changed files with 1026 additions and 1274 deletions
Download Patch File Download Diff File Expand all files Collapse all files

222
.gitlab-ci.yml
View File

@@ -1,15 +1,3 @@
stages:
- upload_cache
- pre_check
- build
- assign_test
- build_doc
- target_test
- host_test
- test_deploy
- deploy
- post_deploy
workflow: workflow:
rules: rules:
# Disable those non-protected push triggered pipelines # Disable those non-protected push triggered pipelines
@@ -27,216 +15,10 @@ workflow:
IS_MR_PIPELINE: 0 IS_MR_PIPELINE: 0
- when: always - when: always
variables: # Place the default settings in `.gitlab/ci/common.yml` instead
# System environment
# Common parameters for the 'make' during CI tests
MAKEFLAGS: "-j5 --no-keep-going"
# GitLab-CI environment
# XXX_ATTEMPTS variables (https://docs.gitlab.com/ce/ci/yaml/README.html#job-stages-attempts) are not defined here.
# Use values from "CI / CD Settings" - "Variables".
# GIT_STRATEGY is not defined here.
# Use an option from "CI / CD Settings" - "General pipelines".
# we will download archive for each submodule instead of clone.
# we don't do "recursive" when fetch submodule as they're not used in CI now.
GIT_SUBMODULE_STRATEGY: none
SUBMODULE_FETCH_TOOL: "tools/ci/ci_fetch_submodule.py"
# by default we will fetch all submodules
# jobs can overwrite this variable to only fetch submodules they required
# set to "none" if don't need to fetch submodules
SUBMODULES_TO_FETCH: "all"
# tell build system do not check submodule update as we download archive instead of clone
IDF_SKIP_CHECK_SUBMODULES: 1
IDF_PATH: "$CI_PROJECT_DIR"
BATCH_BUILD: "1"
V: "0"
CHECKOUT_REF_SCRIPT: "$CI_PROJECT_DIR/tools/ci/checkout_project_ref.py"
PYTHON_VER: 3.8.17
# Docker images
BOT_DOCKER_IMAGE_TAG: ":latest"
ESP_ENV_IMAGE: "$CI_DOCKER_REGISTRY/esp-env-v5.2:2"
ESP_IDF_DOC_ENV_IMAGE: "$CI_DOCKER_REGISTRY/esp-idf-doc-env-v5.2:2-1"
QEMU_IMAGE: "${CI_DOCKER_REGISTRY}/qemu-v5.2:2-20230522"
TARGET_TEST_ENV_IMAGE: "$CI_DOCKER_REGISTRY/target-test-env-v5.2:2"
SONARQUBE_SCANNER_IMAGE: "${CI_DOCKER_REGISTRY}/sonarqube-scanner:5"
PRE_COMMIT_IMAGE: "$CI_DOCKER_REGISTRY/esp-idf-pre-commit:1"
# target test config file, used by assign test job
CI_TARGET_TEST_CONFIG_FILE: "$CI_PROJECT_DIR/.gitlab/ci/target-test.yml"
# target test repo parameters
TEST_ENV_CONFIG_REPO: "https://gitlab-ci-token:${BOT_TOKEN}@${CI_SERVER_HOST}:${CI_SERVER_PORT}/qa/ci-test-runner-configs.git"
# cache python dependencies
PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
# Set this variable to the branch of idf-constraints repo in order to test a custom Python constraint file. The
# branch name must be without the remote part ("origin/"). Keep the variable empty in order to use the constraint
# file from https://dl.espressif.com/dl/esp-idf.
CI_PYTHON_CONSTRAINT_BRANCH: ""
# Update the filename for a specific ESP-IDF release. It is used only with CI_PYTHON_CONSTRAINT_BRANCH.
CI_PYTHON_CONSTRAINT_FILE: "espidf.constraints.v5.2.txt"
# Set this variable to repository name of a Python tool you wish to install and test in the context of ESP-IDF CI.
# Keep the variable empty when not used.
CI_PYTHON_TOOL_REPO: ""
# Set this variable to the branch of a Python tool repo specified in CI_PYTHON_TOOL_REPO. The
# branch name must be without the remote part ("origin/"). Keep the variable empty when not used.
# This is used only if CI_PYTHON_TOOL_REPO is not empty.
CI_PYTHON_TOOL_BRANCH: ""
IDF_CI_BUILD: 1
cache:
# pull only for most of the use cases since it's cache dir.
# Only set "push" policy for "upload_cache" stage jobs
- key: pip-cache
paths:
- .cache/pip
policy: pull
- key: submodule-cache
paths:
- .cache/submodule_archives
policy: pull
.common_before_scripts: &common-before_scripts |
source tools/ci/utils.sh
is_based_on_commits $REQUIRED_ANCESTOR_COMMITS
if [[ -n "$IDF_DONT_USE_MIRRORS" ]]; then
export IDF_MIRROR_PREFIX_MAP=
fi
if echo "$CI_MERGE_REQUEST_LABELS" | egrep "(^|,)include_nightly_run(,|$)"; then
export INCLUDE_NIGHTLY_RUN="1"
fi
# configure cmake related flags
source tools/ci/configure_ci_environment.sh
# add extra python packages
export PYTHONPATH="$IDF_PATH/tools:$IDF_PATH/tools/esp_app_trace:$IDF_PATH/components/partition_table:$IDF_PATH/tools/ci/python_packages:$PYTHONPATH"
.setup_tools_and_idf_python_venv: &setup_tools_and_idf_python_venv |
# must use after setup_tools_except_target_test
# otherwise the export.sh won't work properly
# download constraint file for dev
if [[ -n "$CI_PYTHON_CONSTRAINT_BRANCH" ]]; then
wget -O /tmp/constraint.txt --header="Authorization:Bearer ${ESPCI_TOKEN}" ${GITLAB_HTTP_SERVER}/api/v4/projects/2581/repository/files/${CI_PYTHON_CONSTRAINT_FILE}/raw?ref=${CI_PYTHON_CONSTRAINT_BRANCH}
mkdir -p ~/.espressif
mv /tmp/constraint.txt ~/.espressif/${CI_PYTHON_CONSTRAINT_FILE}
fi
# Mirror
if [[ -n "$IDF_DONT_USE_MIRRORS" ]]; then
export IDF_MIRROR_PREFIX_MAP=
fi
# install latest python packages
# target test jobs
if [[ "${CI_JOB_STAGE}" == "target_test" ]]; then
# ttfw jobs
if ! echo "${CI_JOB_NAME}" | egrep ".*pytest.*"; then
run_cmd bash install.sh --enable-ci --enable-ttfw
else
run_cmd bash install.sh --enable-ci --enable-pytest
fi
elif [[ "${CI_JOB_STAGE}" == "build_doc" ]]; then
run_cmd bash install.sh --enable-ci --enable-docs
elif [[ "${CI_JOB_STAGE}" == "build" ]]; then
run_cmd bash install.sh --enable-ci --enable-pytest
else
if ! echo "${CI_JOB_NAME}" | egrep ".*pytest.*"; then
run_cmd bash install.sh --enable-ci
else
run_cmd bash install.sh --enable-ci --enable-pytest
fi
fi
# Install esp-clang if necessary
if [[ "$IDF_TOOLCHAIN" == "clang" ]]; then
$IDF_PATH/tools/idf_tools.py --non-interactive install esp-clang
fi
source ./export.sh
# Custom clang
if [[ ! -z "$CI_CLANG_DISTRO_URL" ]]; then
echo "Using custom clang from ${CI_CLANG_DISTRO_URL}"
wget $CI_CLANG_DISTRO_URL
ARCH_NAME=$(basename $CI_CLANG_DISTRO_URL)
tar -x -f $ARCH_NAME
export PATH=$PWD/esp-clang/bin:$PATH
fi
# Custom OpenOCD
if [[ ! -z "$OOCD_DISTRO_URL" && "$CI_JOB_STAGE" == "target_test" ]]; then
echo "Using custom OpenOCD from ${OOCD_DISTRO_URL}"
wget $OOCD_DISTRO_URL
ARCH_NAME=$(basename $OOCD_DISTRO_URL)
tar -x -f $ARCH_NAME
export OPENOCD_SCRIPTS=$PWD/openocd-esp32/share/openocd/scripts
export PATH=$PWD/openocd-esp32/bin:$PATH
fi
if [[ -n "$CI_PYTHON_TOOL_REPO" ]]; then
git clone --quiet --depth=1 -b ${CI_PYTHON_TOOL_BRANCH} https://gitlab-ci-token:${ESPCI_TOKEN}@${GITLAB_HTTPS_HOST}/espressif/${CI_PYTHON_TOOL_REPO}.git
pip install ./${CI_PYTHON_TOOL_REPO}
rm -rf ${CI_PYTHON_TOOL_REPO}
fi
before_script:
- *common-before_scripts
- *setup_tools_and_idf_python_venv
- add_gitlab_ssh_keys
- fetch_submodules
.before_script_minimal:
before_script:
- *common-before_scripts
.before_script_macos:
before_script:
- *common-before_scripts
# On macOS, these tools need to be installed
- export IDF_TOOLS_PATH="${HOME}/.espressif_runner_${CI_RUNNER_ID}_${CI_CONCURRENT_ID}"
- $IDF_PATH/tools/idf_tools.py --non-interactive install cmake ninja
# This adds tools (compilers) and the version-specific Python environment to PATH
- *setup_tools_and_idf_python_venv
- fetch_submodules
.before_script_build_jobs:
before_script:
- *common-before_scripts
- *setup_tools_and_idf_python_venv
- add_gitlab_ssh_keys
- fetch_submodules
- export EXTRA_CFLAGS=${PEDANTIC_CFLAGS}
- export EXTRA_CXXFLAGS=${PEDANTIC_CXXFLAGS}
default:
retry:
max: 2
when:
# In case of a runner failure we could hop to another one, or a network error could go away.
- runner_system_failure
# Job execution timeout may be caused by a network issue.
- job_execution_timeout
include: include:
- '.gitlab/ci/common.yml'
- '.gitlab/ci/rules.yml' - '.gitlab/ci/rules.yml'
- '.gitlab/ci/upload_cache.yml' - '.gitlab/ci/upload_cache.yml'
- '.gitlab/ci/docs.yml' - '.gitlab/ci/docs.yml'

77
.gitlab/ci/README.md
View File

@@ -22,6 +22,11 @@
- [Manifest File to Control the Build/Test apps](#manifest-file-to-control-the-buildtest-apps) - [Manifest File to Control the Build/Test apps](#manifest-file-to-control-the-buildtest-apps)
- [Grammar](#grammar) - [Grammar](#grammar)
- [Special Rules](#special-rules) - [Special Rules](#special-rules)
- [Upload/Download Artifacts to Internal Minio Server](#uploaddownload-artifacts-to-internal-minio-server)
- [Env Vars](#env-vars)
- [Artifacts Types and File Patterns](#artifacts-types-and-file-patterns)
- [Upload](#upload)
- [Download](#download)
## General Workflow ## General Workflow
@@ -245,3 +250,75 @@ In ESP-IDF CI, there's a few more special rules are additionally supported to di
- Add MR labels `BUILD_AND_TEST_ALL_APPS` - Add MR labels `BUILD_AND_TEST_ALL_APPS`
- Run in protected branches - Run in protected branches
## Upload/Download Artifacts to Internal Minio Server
### Users Without Access to Minio
If you don't have access to the internal Minio server, you can still download the artifacts from the shared link in the job log.
The log will look like this:
```shell
Pipeline ID : 587355
Job name : build_clang_test_apps_esp32
Job ID : 40272275
Created archive file: 40272275.zip, uploading as 587355/build_dir_without_map_and_elf_files/build_clang_test_apps_esp32/40272275.zip
Please download the archive file includes build_dir_without_map_and_elf_files from [INTERNAL_URL]
```
### Users With Access to Minio
#### Env Vars for Minio
Minio takes these env vars to connect to the server:
- `IDF_S3_SERVER`
- `IDF_S3_ACCESS_KEY`
- `IDF_S3_SECRET_KEY`
- `IDF_S3_BUCKET`
#### Artifacts Types and File Patterns
The artifacts types and corresponding file patterns are defined in tools/ci/artifacts_handler.py, inside `ArtifactType` and `TYPE_PATTERNS_DICT`.
#### Upload
```shell
python tools/ci/artifacts_handler.py upload
```
will upload the files that match the file patterns to minio object storage with name:
`<pipeline_id>/<artifact_type>/<job_name>/<job_id>.zip`
For example, job 39043328 will upload these four files:
- `575500/map_and_elf_files/build_pytest_examples_esp32/39043328.zip`
- `575500/build_dir_without_map_and_elf_files/build_pytest_examples_esp32/39043328.zip`
- `575500/logs/build_pytest_examples_esp32/39043328.zip`
- `575500/size_reports/build_pytest_examples_esp32/39043328.zip`
#### Download
You may run
```shell
python tools/ci/artifacts_handler.py download --pipeline_id <pipeline_id>
```
to download all files of the pipeline, or
```shell
python tools/ci/artifacts_handler.py download --pipeline_id <pipeline_id> --job_name <job_name_or_pattern>
```
to download all files with the specified job name or pattern, or
```shell
python tools/ci/artifacts_handler.py download --pipeline_id <pipeline_id> --job_name <job_name_or_pattern> --type <artifact_type> <artifact_type> ...
```
to download all files with the specified job name or pattern and artifact type(s).
You may check all detailed documentation with `python tools/ci/artifacts_handler.py download -h`

34
.gitlab/ci/assign-test.yml
View File

@@ -3,45 +3,15 @@
stage: assign_test stage: assign_test
tags: tags:
- assign_test - assign_test
dependencies: []
variables: variables:
SUBMODULES_TO_FETCH: "none" SUBMODULES_TO_FETCH: "none"
artifacts:
paths:
- ${TEST_DIR}/test_configs/
- artifact_index.json
when: always
expire_in: 1 week
script:
- run_cmd python tools/ci/python_packages/ttfw_idf/IDFAssignTest.py $TEST_TYPE $TEST_DIR -c $CI_TARGET_TEST_CONFIG_FILE -o $TEST_DIR/test_configs
assign_unit_test:
extends:
- .assign_test_template
- .rules:build:unit_test
needs:
- job: build_esp_idf_tests_cmake_esp32
optional: true
- job: build_esp_idf_tests_cmake_esp32s2
optional: true
- job: build_esp_idf_tests_cmake_esp32c2
optional: true
- job: build_esp_idf_tests_cmake_esp32c3
optional: true
- job: build_esp_idf_tests_cmake_esp32c6
optional: true
- job: build_esp_idf_tests_cmake_esp32h2
optional: true
- job: build_esp_idf_tests_cmake_esp32s3
optional: true
variables:
TEST_TYPE: unit_test
TEST_DIR: components/idf_test/unit_test
assign_integration_test: assign_integration_test:
extends: extends:
- .assign_test_template - .assign_test_template
- .rules:test:integration_test - .rules:test:integration_test
- .before_script_minimal - .before_script:minimal
image: ${CI_INTEGRATION_TEST_ENV_IMAGE} image: ${CI_INTEGRATION_TEST_ENV_IMAGE}
needs: needs:
- build_ssc_esp32 - build_ssc_esp32

253
.gitlab/ci/build.yml
View File

@@ -1,5 +1,7 @@
.build_template: .build_template:
stage: build stage: build
extends:
- .after_script:build:ccache
image: $ESP_ENV_IMAGE image: $ESP_ENV_IMAGE
tags: tags:
- build - build
@@ -8,15 +10,13 @@
variables: variables:
# Enable ccache for all build jobs. See configure_ci_environment.sh for more ccache related settings. # Enable ccache for all build jobs. See configure_ci_environment.sh for more ccache related settings.
IDF_CCACHE_ENABLE: "1" IDF_CCACHE_ENABLE: "1"
after_script:
# Show ccache statistics if enabled globally
- test "$CI_CCACHE_STATS" == 1 && test -n "$(which ccache)" && ccache --show-stats || true
dependencies: [] dependencies: []
.build_cmake_template: .build_cmake_template:
extends: extends:
- .build_template - .build_template
- .before_script_build_jobs - .before_script:build
- .after_script:build:ccache
dependencies: # set dependencies to null to avoid missing artifacts issue dependencies: # set dependencies to null to avoid missing artifacts issue
needs: needs:
- job: fast_template_app - job: fast_template_app
@@ -25,48 +25,16 @@
optional: true # only MR pipelines would have this optional: true # only MR pipelines would have this
artifacts: artifacts:
paths: paths:
- "**/build*/size.json" # The other artifacts patterns are defined under tools/ci/artifacts_handler.py
# Now we're uploading/downloading the binary files from our internal storage server
#
# keep the log file to help debug
- "**/build*/build_log.txt" - "**/build*/build_log.txt"
- "**/build*/*.bin" # keep the size info to help track the binary size
# upload to s3 server to save the artifacts size
# - "**/build*/*.map"
# ttfw tests require elf files
- "**/build*/*.elf"
- "**/build*/flasher_args.json"
- "**/build*/flash_project_args"
- "**/build*/config/sdkconfig.json"
# ttfw tests require sdkconfig file
- "**/build*/sdkconfig"
- "**/build*/bootloader/*.bin"
- "**/build*/partition_table/*.bin"
- list_job_*.txt
- size_info.txt - size_info.txt
# unit test specific - "**/build*/size.json"
- components/idf_test/unit_test/*.yml
when: always when: always
expire_in: 4 days expire_in: 4 days
after_script:
# Show ccache statistics if enabled globally
- test "$CI_CCACHE_STATS" == 1 && test -n "$(which ccache)" && ccache --show-stats || true
# upload the binary files to s3 server
- echo -e "\e[0Ksection_start:`date +%s`:upload_binaries_to_s3_server[collapsed=true]\r\e[0KUploading binaries to s3 Server"
- shopt -s globstar
# use || true to bypass the no-file error
- zip ${CI_JOB_ID}.zip **/build*/*.bin || true
- zip ${CI_JOB_ID}.zip **/build*/*.elf || true
- zip ${CI_JOB_ID}.zip **/build*/*.map || true
- zip ${CI_JOB_ID}.zip **/build*/flasher_args.json || true
- zip ${CI_JOB_ID}.zip **/build*/flash_project_args || true
- zip ${CI_JOB_ID}.zip **/build*/config/sdkconfig.json || true
- zip ${CI_JOB_ID}.zip **/build*/sdkconfig || true
- zip ${CI_JOB_ID}.zip **/build*/bootloader/*.bin || true
- zip ${CI_JOB_ID}.zip **/build*/partition_table/*.bin || true
- mc alias set shiny-s3 ${SHINY_S3_SERVER} ${SHINY_S3_ACCESS_KEY} ${SHINY_S3_SECRET_KEY}
- mc cp ${CI_JOB_ID}.zip shiny-s3/idf-artifacts/${CI_PIPELINE_ID}/${CI_JOB_ID}.zip
- echo -e "\e[0Ksection_end:`date +%s`:upload_binaries_to_s3_server\r\e[0K"
- echo "Please download the full binary files (including *.elf and *.map files) from the following share link"
# would be clean up after 4 days
- mc share download shiny-s3/idf-artifacts/${CI_PIPELINE_ID}/${CI_JOB_ID}.zip --expire=96h
script: script:
# CI specific options start from "--parallel-count xxx". could ignore when running locally # CI specific options start from "--parallel-count xxx". could ignore when running locally
- run_cmd python tools/ci/ci_build_apps.py $TEST_DIR -v - run_cmd python tools/ci/ci_build_apps.py $TEST_DIR -v
@@ -80,6 +48,7 @@
examples/wifi/iperf examples/wifi/iperf
--modified-components ${MR_MODIFIED_COMPONENTS} --modified-components ${MR_MODIFIED_COMPONENTS}
--modified-files ${MR_MODIFIED_FILES} --modified-files ${MR_MODIFIED_FILES}
- upload_artifacts_to_s3
.build_cmake_clang_template: .build_cmake_clang_template:
extends: extends:
@@ -98,41 +67,27 @@
--modified-components ${MR_MODIFIED_COMPONENTS} --modified-components ${MR_MODIFIED_COMPONENTS}
--modified-files ${MR_MODIFIED_FILES} --modified-files ${MR_MODIFIED_FILES}
$TEST_BUILD_OPTS_EXTRA $TEST_BUILD_OPTS_EXTRA
- upload_artifacts_to_s3
.build_pytest_template: .build_pytest_template:
extends: extends:
- .build_cmake_template - .build_cmake_template
- .before_script_build_jobs
artifacts:
paths:
- "**/build*/size.json"
- "**/build*/build_log.txt"
- "**/build*/*.bin"
# upload to s3 server to save the artifacts size
# - "**/build*/*.map"
# - "**/build*/*.elf"
- "**/build*/flasher_args.json"
- "**/build*/flash_project_args"
- "**/build*/config/sdkconfig.json"
- "**/build*/bootloader/*.bin"
- "**/build*/partition_table/*.bin"
- list_job_*.txt
- size_info.txt
when: always
expire_in: 4 days
script: script:
# CI specific options start from "--parallel-count xxx". could ignore when running locally # CI specific options start from "--parallel-count xxx". could ignore when running locally
- run_cmd python tools/ci/ci_build_apps.py $TEST_DIR -v - run_cmd python tools/ci/ci_build_apps.py $TEST_DIR -v
-t $IDF_TARGET -t $IDF_TARGET
-m \"not host_test\"
--pytest-apps --pytest-apps
--parallel-count ${CI_NODE_TOTAL:-1} --parallel-count ${CI_NODE_TOTAL:-1}
--parallel-index ${CI_NODE_INDEX:-1} --parallel-index ${CI_NODE_INDEX:-1}
--collect-app-info "list_job_${CI_JOB_NAME_SLUG}.txt" --collect-app-info "list_job_${CI_JOB_NAME_SLUG}.txt"
--modified-components ${MR_MODIFIED_COMPONENTS} --modified-components ${MR_MODIFIED_COMPONENTS}
--modified-files ${MR_MODIFIED_FILES} --modified-files ${MR_MODIFIED_FILES}
- upload_artifacts_to_s3
.build_pytest_no_jtag_template: .build_pytest_no_jtag_template:
extends: .build_pytest_template extends:
- .build_cmake_template
script: script:
# CI specific options start from "--parallel-count xxx". could ignore when running locally # CI specific options start from "--parallel-count xxx". could ignore when running locally
- run_cmd python tools/ci/ci_build_apps.py $TEST_DIR -v - run_cmd python tools/ci/ci_build_apps.py $TEST_DIR -v
@@ -144,28 +99,11 @@
--collect-app-info "list_job_${CI_JOB_NAME_SLUG}.txt" --collect-app-info "list_job_${CI_JOB_NAME_SLUG}.txt"
--modified-components ${MR_MODIFIED_COMPONENTS} --modified-components ${MR_MODIFIED_COMPONENTS}
--modified-files ${MR_MODIFIED_FILES} --modified-files ${MR_MODIFIED_FILES}
- upload_artifacts_to_s3
.build_pytest_jtag_template: .build_pytest_jtag_template:
extends: extends:
- .build_cmake_template - .build_cmake_template
- .before_script_build_jobs
artifacts:
paths:
- "**/build*/size.json"
- "**/build*/build_log.txt"
- "**/build*/*.bin"
# upload to s3 server to save the artifacts size
# - "**/build*/*.map"
- "**/build*/*.elf" # need elf for gdb
- "**/build*/flasher_args.json"
- "**/build*/flash_project_args"
- "**/build*/config/sdkconfig.json"
- "**/build*/bootloader/*.bin"
- "**/build*/partition_table/*.bin"
- list_job_*.txt
- size_info.txt
when: always
expire_in: 4 days
script: script:
# CI specific options start from "--parallel-count xxx". could ignore when running locally # CI specific options start from "--parallel-count xxx". could ignore when running locally
- run_cmd python tools/ci/ci_build_apps.py $TEST_DIR -v - run_cmd python tools/ci/ci_build_apps.py $TEST_DIR -v
@@ -177,6 +115,7 @@
--collect-app-info "list_job_${CI_JOB_NAME_SLUG}.txt" --collect-app-info "list_job_${CI_JOB_NAME_SLUG}.txt"
--modified-components ${MR_MODIFIED_COMPONENTS} --modified-components ${MR_MODIFIED_COMPONENTS}
--modified-files ${MR_MODIFIED_FILES} --modified-files ${MR_MODIFIED_FILES}
- upload_artifacts_to_s3
build_pytest_examples_esp32: build_pytest_examples_esp32:
extends: extends:
@@ -223,17 +162,9 @@ build_pytest_examples_esp32c2:
IDF_TARGET: esp32c2 IDF_TARGET: esp32c2
TEST_DIR: examples TEST_DIR: examples
build_pytest_examples_jtag: # for all targets
extends:
- .build_pytest_jtag_template
- .rules:build:example_test-esp32
variables:
IDF_TARGET: all
TEST_DIR: examples
build_pytest_examples_esp32c6: build_pytest_examples_esp32c6:
extends: extends:
- .build_pytest_template - .build_pytest_no_jtag_template
- .rules:build:example_test-esp32c6 - .rules:build:example_test-esp32c6
parallel: 2 parallel: 2
variables: variables:
@@ -242,13 +173,21 @@ build_pytest_examples_esp32c6:
build_pytest_examples_esp32h2: build_pytest_examples_esp32h2:
extends: extends:
- .build_pytest_template - .build_pytest_no_jtag_template
- .rules:build:example_test-esp32h2 - .rules:build:example_test-esp32h2
parallel: 2 parallel: 2
variables: variables:
IDF_TARGET: esp32h2 IDF_TARGET: esp32h2
TEST_DIR: examples TEST_DIR: examples
build_pytest_examples_jtag: # for all targets
extends:
- .build_pytest_jtag_template
- .rules:build:example_test
variables:
IDF_TARGET: all
TEST_DIR: examples
build_pytest_components_esp32: build_pytest_components_esp32:
extends: extends:
- .build_pytest_template - .build_pytest_template
@@ -326,33 +265,11 @@ build_only_components_apps:
--parallel-index ${CI_NODE_INDEX:-1} --parallel-index ${CI_NODE_INDEX:-1}
--modified-components ${MR_MODIFIED_COMPONENTS} --modified-components ${MR_MODIFIED_COMPONENTS}
--modified-files ${MR_MODIFIED_FILES} --modified-files ${MR_MODIFIED_FILES}
- upload_artifacts_to_s3
.build_pytest_test_apps_template:
extends: .build_pytest_template
artifacts:
paths:
- "**/build*/size.json"
- "**/build*/build_log.txt"
- "**/build*/*.bin"
# upload to s3 server to save the artifacts size
# - "**/build*/*.map"
# pytest test apps requires elf files for coredump tests
- "**/build*/*.elf"
- "**/build*/flasher_args.json"
- "**/build*/flash_project_args"
- "**/build*/config/sdkconfig.json"
- "**/build*/bootloader/*.elf"
- "**/build*/bootloader/*.bin"
- "**/build*/partition_table/*.bin"
- "**/build*/project_description.json"
- list_job_*.txt
- size_info.txt
when: always
expire_in: 4 days
build_pytest_test_apps_esp32: build_pytest_test_apps_esp32:
extends: extends:
- .build_pytest_test_apps_template - .build_pytest_template
- .rules:build:custom_test-esp32 - .rules:build:custom_test-esp32
variables: variables:
IDF_TARGET: esp32 IDF_TARGET: esp32
@@ -360,7 +277,7 @@ build_pytest_test_apps_esp32:
build_pytest_test_apps_esp32s2: build_pytest_test_apps_esp32s2:
extends: extends:
- .build_pytest_test_apps_template - .build_pytest_template
- .rules:build:custom_test-esp32s2 - .rules:build:custom_test-esp32s2
variables: variables:
IDF_TARGET: esp32s2 IDF_TARGET: esp32s2
@@ -368,7 +285,7 @@ build_pytest_test_apps_esp32s2:
build_pytest_test_apps_esp32s3: build_pytest_test_apps_esp32s3:
extends: extends:
- .build_pytest_test_apps_template - .build_pytest_template
- .rules:build:custom_test-esp32s3 - .rules:build:custom_test-esp32s3
parallel: 2 parallel: 2
variables: variables:
@@ -377,7 +294,7 @@ build_pytest_test_apps_esp32s3:
build_pytest_test_apps_esp32c3: build_pytest_test_apps_esp32c3:
extends: extends:
- .build_pytest_test_apps_template - .build_pytest_template
- .rules:build:custom_test-esp32c3 - .rules:build:custom_test-esp32c3
variables: variables:
IDF_TARGET: esp32c3 IDF_TARGET: esp32c3
@@ -385,7 +302,7 @@ build_pytest_test_apps_esp32c3:
build_pytest_test_apps_esp32c2: build_pytest_test_apps_esp32c2:
extends: extends:
- .build_pytest_test_apps_template - .build_pytest_template
- .rules:build:custom_test-esp32c2 - .rules:build:custom_test-esp32c2
variables: variables:
IDF_TARGET: esp32c2 IDF_TARGET: esp32c2
@@ -393,7 +310,7 @@ build_pytest_test_apps_esp32c2:
build_pytest_test_apps_esp32c6: build_pytest_test_apps_esp32c6:
extends: extends:
- .build_pytest_test_apps_template - .build_pytest_template
- .rules:build:custom_test-esp32c6 - .rules:build:custom_test-esp32c6
variables: variables:
IDF_TARGET: esp32c6 IDF_TARGET: esp32c6
@@ -401,7 +318,7 @@ build_pytest_test_apps_esp32c6:
build_pytest_test_apps_esp32h2: build_pytest_test_apps_esp32h2:
extends: extends:
- .build_pytest_test_apps_template - .build_pytest_template
- .rules:build:custom_test-esp32h2 - .rules:build:custom_test-esp32h2
variables: variables:
IDF_TARGET: esp32h2 IDF_TARGET: esp32h2
@@ -420,11 +337,12 @@ build_only_tools_test_apps:
--parallel-index ${CI_NODE_INDEX:-1} --parallel-index ${CI_NODE_INDEX:-1}
--modified-components ${MR_MODIFIED_COMPONENTS} --modified-components ${MR_MODIFIED_COMPONENTS}
--modified-files ${MR_MODIFIED_FILES} --modified-files ${MR_MODIFIED_FILES}
- upload_artifacts_to_s3
.build_template_app_template: .build_template_app_template:
extends: extends:
- .build_template - .build_template
- .before_script_build_jobs - .before_script:build
variables: variables:
LOG_PATH: "${CI_PROJECT_DIR}/log_template_app" LOG_PATH: "${CI_PROJECT_DIR}/log_template_app"
BUILD_PATH: "${CI_PROJECT_DIR}/build_template_app" BUILD_PATH: "${CI_PROJECT_DIR}/build_template_app"
@@ -523,97 +441,6 @@ build_ssc_esp32h2:
variables: variables:
TARGET_NAME: "ESP32H2" TARGET_NAME: "ESP32H2"
.build_esp_idf_tests_cmake_template:
extends:
- .build_cmake_template
- .before_script_build_jobs
artifacts:
paths:
- "**/build*/size.json"
- "**/build*/build_log.txt"
- "**/build*/*.bin"
# upload to s3 server to save the artifacts size
# - "**/build*/*.map"
# ttfw tests require elf files
- "**/build*/*.elf"
- "**/build*/flasher_args.json"
- "**/build*/flash_project_args"
- "**/build*/config/sdkconfig.json"
- "**/build*/sdkconfig"
- "**/build*/bootloader/*.bin"
- "**/build*/partition_table/*.bin"
- list_job_*.txt
- size_info.txt
- components/idf_test/unit_test/*.yml
when: always
expire_in: 4 days
script:
# CI specific options start from "--parallel-count xxx". could ignore when running locally
- run_cmd python tools/ci/ci_build_apps.py tools/unit-test-app -v
-t $IDF_TARGET
--config "configs/*="
--copy-sdkconfig
--preserve-all
--parallel-count ${CI_NODE_TOTAL:-1}
--parallel-index ${CI_NODE_INDEX:-1}
- run_cmd python tools/unit-test-app/tools/UnitTestParser.py tools/unit-test-app ${CI_NODE_INDEX:-1}
build_esp_idf_tests_cmake_esp32:
extends:
- .build_esp_idf_tests_cmake_template
- .rules:build:unit_test-esp32
variables:
IDF_TARGET: esp32
build_esp_idf_tests_cmake_esp32s2:
extends:
- .build_esp_idf_tests_cmake_template
- .rules:build:unit_test-esp32s2
variables:
IDF_TARGET: esp32s2
build_esp_idf_tests_cmake_esp32s3:
extends:
- .build_esp_idf_tests_cmake_template
- .rules:build:unit_test-esp32s3
variables:
IDF_TARGET: esp32s3
build_esp_idf_tests_cmake_esp32c2:
extends:
- .build_esp_idf_tests_cmake_template
- .rules:build:unit_test-esp32c2
variables:
IDF_TARGET: esp32c2
build_esp_idf_tests_cmake_esp32c3:
extends:
- .build_esp_idf_tests_cmake_template
- .rules:build:unit_test-esp32c3
variables:
IDF_TARGET: esp32c3
build_esp_idf_tests_cmake_esp32c6:
extends:
- .build_esp_idf_tests_cmake_template
- .rules:build:unit_test-esp32c6
variables:
IDF_TARGET: esp32c6
build_esp_idf_tests_cmake_esp32h2:
extends:
- .build_esp_idf_tests_cmake_template
- .rules:build:unit_test-esp32h2
variables:
IDF_TARGET: esp32h2
build_esp_idf_tests_cmake_esp32p4:
extends:
- .build_esp_idf_tests_cmake_template
- .rules:build:unit_test-esp32p4
variables:
IDF_TARGET: esp32p4
build_examples_cmake_esp32: build_examples_cmake_esp32:
extends: extends:
- .build_cmake_template - .build_cmake_template
@@ -770,7 +597,7 @@ pytest_build_system:
pytest_build_system_macos: pytest_build_system_macos:
extends: extends:
- .test_build_system_template - .test_build_system_template
- .before_script_macos - .before_script:build:macos
- .rules:build:macos - .rules:build:macos
tags: tags:
- macos_shell - macos_shell
@@ -786,7 +613,7 @@ pytest_build_system_macos:
build_docker: build_docker:
extends: extends:
- .before_script_minimal - .before_script:minimal
- .rules:build:docker - .rules:build:docker
stage: host_test stage: host_test
needs: [] needs: []

234
.gitlab/ci/common.yml Normal file
View File

@@ -0,0 +1,234 @@
#####################
# Default Variables #
#####################
stages:
- upload_cache
- pre_check
- build
- assign_test
- build_doc
- target_test
- host_test
- test_deploy
- deploy
- post_deploy
variables:
# System environment
# Common parameters for the 'make' during CI tests
MAKEFLAGS: "-j5 --no-keep-going"
# GitLab-CI environment
# XXX_ATTEMPTS variables (https://docs.gitlab.com/ee/ci/runners/configure_runners.html#job-stages-attempts) are not defined here.
# Use values from "CI / CD Settings" - "Variables".
# GIT_STRATEGY is not defined here.
# Use an option from "CI / CD Settings" - "General pipelines".
# we will download archive for each submodule instead of clone.
# we don't do "recursive" when fetch submodule as they're not used in CI now.
GIT_SUBMODULE_STRATEGY: none
SUBMODULE_FETCH_TOOL: "tools/ci/ci_fetch_submodule.py"
# by default we will fetch all submodules
# jobs can overwrite this variable to only fetch submodules they required
# set to "none" if don't need to fetch submodules
SUBMODULES_TO_FETCH: "all"
# tell build system do not check submodule update as we download archive instead of clone
IDF_SKIP_CHECK_SUBMODULES: 1
IDF_PATH: "$CI_PROJECT_DIR"
BATCH_BUILD: "1"
V: "0"
CHECKOUT_REF_SCRIPT: "$CI_PROJECT_DIR/tools/ci/checkout_project_ref.py"
PYTHON_VER: 3.8.17
# Docker images
BOT_DOCKER_IMAGE_TAG: ":latest"
ESP_ENV_IMAGE: "$CI_DOCKER_REGISTRY/esp-env-v5.2:2"
ESP_IDF_DOC_ENV_IMAGE: "$CI_DOCKER_REGISTRY/esp-idf-doc-env-v5.2:2-1"
QEMU_IMAGE: "${CI_DOCKER_REGISTRY}/qemu-v5.2:2-20230522"
TARGET_TEST_ENV_IMAGE: "$CI_DOCKER_REGISTRY/target-test-env-v5.2:2"
SONARQUBE_SCANNER_IMAGE: "${CI_DOCKER_REGISTRY}/sonarqube-scanner:5"
PRE_COMMIT_IMAGE: "$CI_DOCKER_REGISTRY/esp-idf-pre-commit:1"
# target test config file, used by assign test job
CI_TARGET_TEST_CONFIG_FILE: "$CI_PROJECT_DIR/.gitlab/ci/target-test.yml"
# target test repo parameters
TEST_ENV_CONFIG_REPO: "https://gitlab-ci-token:${BOT_TOKEN}@${CI_SERVER_HOST}:${CI_SERVER_PORT}/qa/ci-test-runner-configs.git"
# cache python dependencies
PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
# Set this variable to the branch of idf-constraints repo in order to test a custom Python constraint file. The
# branch name must be without the remote part ("origin/"). Keep the variable empty in order to use the constraint
# file from https://dl.espressif.com/dl/esp-idf.
CI_PYTHON_CONSTRAINT_BRANCH: ""
# Update the filename for a specific ESP-IDF release. It is used only with CI_PYTHON_CONSTRAINT_BRANCH.
CI_PYTHON_CONSTRAINT_FILE: "espidf.constraints.v5.2.txt"
# Set this variable to repository name of a Python tool you wish to install and test in the context of ESP-IDF CI.
# Keep the variable empty when not used.
CI_PYTHON_TOOL_REPO: ""
# Set this variable to the branch of a Python tool repo specified in CI_PYTHON_TOOL_REPO. The
# branch name must be without the remote part ("origin/"). Keep the variable empty when not used.
# This is used only if CI_PYTHON_TOOL_REPO is not empty.
CI_PYTHON_TOOL_BRANCH: ""
IDF_CI_BUILD: 1
################################################
# `before_script` and `after_script` Templates #
################################################
.common_before_scripts: &common-before_scripts |
source tools/ci/utils.sh
is_based_on_commits $REQUIRED_ANCESTOR_COMMITS
if [[ -n "$IDF_DONT_USE_MIRRORS" ]]; then
export IDF_MIRROR_PREFIX_MAP=
fi
if echo "$CI_MERGE_REQUEST_LABELS" | egrep "(^|,)include_nightly_run(,|$)"; then
export INCLUDE_NIGHTLY_RUN="1"
fi
# configure cmake related flags
source tools/ci/configure_ci_environment.sh
# add extra python packages
export PYTHONPATH="$IDF_PATH/tools:$IDF_PATH/tools/esp_app_trace:$IDF_PATH/components/partition_table:$IDF_PATH/tools/ci/python_packages:$PYTHONPATH"
.setup_tools_and_idf_python_venv: &setup_tools_and_idf_python_venv |
# must use after setup_tools_except_target_test
# otherwise the export.sh won't work properly
# download constraint file for dev
if [[ -n "$CI_PYTHON_CONSTRAINT_BRANCH" ]]; then
wget -O /tmp/constraint.txt --header="Authorization:Bearer ${ESPCI_TOKEN}" ${GITLAB_HTTP_SERVER}/api/v4/projects/2581/repository/files/${CI_PYTHON_CONSTRAINT_FILE}/raw?ref=${CI_PYTHON_CONSTRAINT_BRANCH}
mkdir -p ~/.espressif
mv /tmp/constraint.txt ~/.espressif/${CI_PYTHON_CONSTRAINT_FILE}
fi
# Mirror
if [[ -n "$IDF_DONT_USE_MIRRORS" ]]; then
export IDF_MIRROR_PREFIX_MAP=
fi
# install latest python packages
# target test jobs
if [[ "${CI_JOB_STAGE}" == "target_test" ]]; then
# ttfw jobs
if ! echo "${CI_JOB_NAME}" | egrep ".*pytest.*"; then
run_cmd bash install.sh --enable-ci --enable-ttfw
else
run_cmd bash install.sh --enable-ci --enable-pytest
fi
elif [[ "${CI_JOB_STAGE}" == "build_doc" ]]; then
run_cmd bash install.sh --enable-ci --enable-docs
elif [[ "${CI_JOB_STAGE}" == "build" ]]; then
run_cmd bash install.sh --enable-ci --enable-pytest
else
if ! echo "${CI_JOB_NAME}" | egrep ".*pytest.*"; then
run_cmd bash install.sh --enable-ci
else
run_cmd bash install.sh --enable-ci --enable-pytest
fi
fi
# Install esp-clang if necessary
if [[ "$IDF_TOOLCHAIN" == "clang" ]]; then
$IDF_PATH/tools/idf_tools.py --non-interactive install esp-clang
fi
source ./export.sh
# Custom clang
if [[ ! -z "$CI_CLANG_DISTRO_URL" ]]; then
echo "Using custom clang from ${CI_CLANG_DISTRO_URL}"
wget $CI_CLANG_DISTRO_URL
ARCH_NAME=$(basename $CI_CLANG_DISTRO_URL)
tar -x -f $ARCH_NAME
export PATH=$PWD/esp-clang/bin:$PATH
fi
# Custom OpenOCD
if [[ ! -z "$OOCD_DISTRO_URL" && "$CI_JOB_STAGE" == "target_test" ]]; then
echo "Using custom OpenOCD from ${OOCD_DISTRO_URL}"
wget $OOCD_DISTRO_URL
ARCH_NAME=$(basename $OOCD_DISTRO_URL)
tar -x -f $ARCH_NAME
export OPENOCD_SCRIPTS=$PWD/openocd-esp32/share/openocd/scripts
export PATH=$PWD/openocd-esp32/bin:$PATH
fi
if [[ -n "$CI_PYTHON_TOOL_REPO" ]]; then
git clone --quiet --depth=1 -b ${CI_PYTHON_TOOL_BRANCH} https://gitlab-ci-token:${ESPCI_TOKEN}@${GITLAB_HTTPS_HOST}/espressif/${CI_PYTHON_TOOL_REPO}.git
pip install ./${CI_PYTHON_TOOL_REPO}
rm -rf ${CI_PYTHON_TOOL_REPO}
fi
.show_ccache_statistics: &show_ccache_statistics |
# Show ccache statistics if enabled globally
test "$CI_CCACHE_STATS" == 1 && test -n "$(which ccache)" && ccache --show-stats || true
.before_script:minimal:
before_script:
- *common-before_scripts
.before_script:build:macos:
before_script:
- *common-before_scripts
# On macOS, these tools need to be installed
- export IDF_TOOLS_PATH="${HOME}/.espressif_runner_${CI_RUNNER_ID}_${CI_CONCURRENT_ID}"
- $IDF_PATH/tools/idf_tools.py --non-interactive install cmake ninja
# This adds tools (compilers) and the version-specific Python environment to PATH
- *setup_tools_and_idf_python_venv
- fetch_submodules
.before_script:build:
before_script:
- *common-before_scripts
- *setup_tools_and_idf_python_venv
- add_gitlab_ssh_keys
- fetch_submodules
- export EXTRA_CFLAGS=${PEDANTIC_CFLAGS}
- export EXTRA_CXXFLAGS=${PEDANTIC_CXXFLAGS}
.after_script:build:ccache:
after_script:
- *show_ccache_statistics
#############
# `default` #
#############
default:
cache:
# pull only for most of the use cases since it's cache dir.
# Only set "push" policy for "upload_cache" stage jobs
- key: pip-cache
paths:
- .cache/pip
policy: pull
- key: submodule-cache
paths:
- .cache/submodule_archives
policy: pull
before_script:
- *common-before_scripts
- *setup_tools_and_idf_python_venv
- add_gitlab_ssh_keys
- fetch_submodules
retry:
max: 2
when:
# In case of a runner failure we could hop to another one, or a network error could go away.
- runner_system_failure
# Job execution timeout may be caused by a network issue.
- job_execution_timeout

5
.gitlab/ci/dependencies/dependencies.yml
View File

@@ -11,7 +11,6 @@
.target_test: &target_test .target_test: &target_test
- example_test - example_test
- custom_test - custom_test
- unit_test
- component_ut - component_ut
############## ##############
@@ -116,14 +115,12 @@ build:integration_test:
# ------------- # -------------
# Special Cases # Special Cases
# ------------- # -------------
"test:component_ut-{0}": # component_ut will trigger by unit_test as well, since now we have 2 kinds of UT "test:component_ut-{0}":
matrix: matrix:
- *all_targets - *all_targets
labels: labels:
- component_ut - component_ut
- "component_ut_{0}" - "component_ut_{0}"
- unit_test
- "unit_test_{0}"
- target_test - target_test
patterns: patterns:
- component_ut - component_ut

4
.gitlab/ci/deploy.yml
View File

@@ -31,7 +31,7 @@ check_submodule_sync:
push_to_github: push_to_github:
extends: extends:
- .deploy_job_template - .deploy_job_template
- .before_script_minimal - .before_script:minimal
- .rules:protected-no_label - .rules:protected-no_label
dependencies: [] dependencies: []
script: script:
@@ -52,7 +52,7 @@ deploy_update_SHA_in_esp-dockerfiles:
deploy_test_result: deploy_test_result:
extends: extends:
- .deploy_job_template - .deploy_job_template
- .before_script_minimal - .before_script:minimal
- .rules:ref:master-always - .rules:ref:master-always
image: $CI_DOCKER_REGISTRY/bot-env:1 image: $CI_DOCKER_REGISTRY/bot-env:1
dependencies: [] dependencies: []

6
.gitlab/ci/host-test.yml
View File

@@ -114,7 +114,7 @@ test_idf_py:
test_idf_tools: test_idf_tools:
extends: extends:
- .host_test_template - .host_test_template
- .before_script_minimal - .before_script:minimal
artifacts: artifacts:
when: on_failure when: on_failure
paths: paths:
@@ -294,7 +294,7 @@ test_gen_soc_caps_kconfig:
test_pytest_qemu: test_pytest_qemu:
extends: extends:
- .host_test_template - .host_test_template
- .before_script_build_jobs - .before_script:build
image: $QEMU_IMAGE image: $QEMU_IMAGE
artifacts: artifacts:
when: always when: always
@@ -328,7 +328,7 @@ test_pytest_qemu:
test_pytest_linux: test_pytest_linux:
extends: extends:
- .host_test_template - .host_test_template
- .before_script_build_jobs - .before_script:build
artifacts: artifacts:
when: always when: always
paths: paths:

8
.gitlab/ci/pre_check.yml
View File

@@ -8,7 +8,7 @@
.check_pre_commit_template: .check_pre_commit_template:
extends: extends:
- .pre_check_template - .pre_check_template
- .before_script_minimal - .before_script:minimal
image: $PRE_COMMIT_IMAGE image: $PRE_COMMIT_IMAGE
check_pre_commit_master_release: check_pre_commit_master_release:
@@ -164,7 +164,7 @@ check_artifacts_expire_time:
check_test_scripts_build_test_rules: check_test_scripts_build_test_rules:
extends: extends:
- .pre_check_template - .pre_check_template
- .before_script_build_jobs - .before_script:build
script: script:
# required pytest related packages # required pytest related packages
- run_cmd bash install.sh --enable-pytest - run_cmd bash install.sh --enable-pytest
@@ -173,7 +173,7 @@ check_test_scripts_build_test_rules:
check_configure_ci_environment_parsing: check_configure_ci_environment_parsing:
extends: extends:
- .pre_check_template - .pre_check_template
- .before_script_build_jobs - .before_script:build
- .rules:build - .rules:build
script: script:
- cd tools/ci - cd tools/ci
@@ -183,7 +183,7 @@ mr_variables:
extends: extends:
- .pre_check_template - .pre_check_template
- .rules:mr - .rules:mr
- .before_script_minimal - .before_script:minimal
tags: tags:
- build - build
script: script:

483
.gitlab/ci/rules.yml
View File

@@ -620,33 +620,6 @@
.if-label-target_test: &if-label-target_test .if-label-target_test: &if-label-target_test
if: '$BOT_LABEL_TARGET_TEST || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*target_test(?:,[^,\n\r]+)*$/i' if: '$BOT_LABEL_TARGET_TEST || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*target_test(?:,[^,\n\r]+)*$/i'
.if-label-unit_test: &if-label-unit_test
if: '$BOT_LABEL_UNIT_TEST || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*unit_test(?:,[^,\n\r]+)*$/i'
.if-label-unit_test_esp32: &if-label-unit_test_esp32
if: '$BOT_LABEL_UNIT_TEST_ESP32 || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*unit_test_esp32(?:,[^,\n\r]+)*$/i'
.if-label-unit_test_esp32c2: &if-label-unit_test_esp32c2
if: '$BOT_LABEL_UNIT_TEST_ESP32C2 || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*unit_test_esp32c2(?:,[^,\n\r]+)*$/i'
.if-label-unit_test_esp32c3: &if-label-unit_test_esp32c3
if: '$BOT_LABEL_UNIT_TEST_ESP32C3 || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*unit_test_esp32c3(?:,[^,\n\r]+)*$/i'
.if-label-unit_test_esp32c6: &if-label-unit_test_esp32c6
if: '$BOT_LABEL_UNIT_TEST_ESP32C6 || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*unit_test_esp32c6(?:,[^,\n\r]+)*$/i'
.if-label-unit_test_esp32h2: &if-label-unit_test_esp32h2
if: '$BOT_LABEL_UNIT_TEST_ESP32H2 || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*unit_test_esp32h2(?:,[^,\n\r]+)*$/i'
.if-label-unit_test_esp32p4: &if-label-unit_test_esp32p4
if: '$BOT_LABEL_UNIT_TEST_ESP32P4 || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*unit_test_esp32p4(?:,[^,\n\r]+)*$/i'
.if-label-unit_test_esp32s2: &if-label-unit_test_esp32s2
if: '$BOT_LABEL_UNIT_TEST_ESP32S2 || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*unit_test_esp32s2(?:,[^,\n\r]+)*$/i'
.if-label-unit_test_esp32s3: &if-label-unit_test_esp32s3
if: '$BOT_LABEL_UNIT_TEST_ESP32S3 || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*unit_test_esp32s3(?:,[^,\n\r]+)*$/i'
.rules:build: .rules:build:
rules: rules:
- <<: *if-revert-branch - <<: *if-revert-branch
@@ -691,15 +664,6 @@
- <<: *if-label-component_ut_esp32s2 - <<: *if-label-component_ut_esp32s2
- <<: *if-label-component_ut_esp32s3 - <<: *if-label-component_ut_esp32s3
- <<: *if-label-target_test - <<: *if-label-target_test
- <<: *if-label-unit_test
- <<: *if-label-unit_test_esp32
- <<: *if-label-unit_test_esp32c2
- <<: *if-label-unit_test_esp32c3
- <<: *if-label-unit_test_esp32c6
- <<: *if-label-unit_test_esp32h2
- <<: *if-label-unit_test_esp32p4
- <<: *if-label-unit_test_esp32s2
- <<: *if-label-unit_test_esp32s3
- <<: *if-dev-push - <<: *if-dev-push
changes: *patterns-build_components changes: *patterns-build_components
- <<: *if-dev-push - <<: *if-dev-push
@@ -738,8 +702,6 @@
- <<: *if-label-component_ut - <<: *if-label-component_ut
- <<: *if-label-component_ut_esp32 - <<: *if-label-component_ut_esp32
- <<: *if-label-target_test - <<: *if-label-target_test
- <<: *if-label-unit_test
- <<: *if-label-unit_test_esp32
- <<: *if-dev-push - <<: *if-dev-push
changes: *patterns-build_components changes: *patterns-build_components
- <<: *if-dev-push - <<: *if-dev-push
@@ -778,8 +740,6 @@
- <<: *if-label-component_ut - <<: *if-label-component_ut
- <<: *if-label-component_ut_esp32c2 - <<: *if-label-component_ut_esp32c2
- <<: *if-label-target_test - <<: *if-label-target_test
- <<: *if-label-unit_test
- <<: *if-label-unit_test_esp32c2
- <<: *if-dev-push - <<: *if-dev-push
changes: *patterns-build_components changes: *patterns-build_components
- <<: *if-dev-push - <<: *if-dev-push
@@ -818,8 +778,6 @@
- <<: *if-label-component_ut - <<: *if-label-component_ut
- <<: *if-label-component_ut_esp32c3 - <<: *if-label-component_ut_esp32c3
- <<: *if-label-target_test - <<: *if-label-target_test
- <<: *if-label-unit_test
- <<: *if-label-unit_test_esp32c3
- <<: *if-dev-push - <<: *if-dev-push
changes: *patterns-build_components changes: *patterns-build_components
- <<: *if-dev-push - <<: *if-dev-push
@@ -858,8 +816,6 @@
- <<: *if-label-component_ut - <<: *if-label-component_ut
- <<: *if-label-component_ut_esp32c6 - <<: *if-label-component_ut_esp32c6
- <<: *if-label-target_test - <<: *if-label-target_test
- <<: *if-label-unit_test
- <<: *if-label-unit_test_esp32c6
- <<: *if-dev-push - <<: *if-dev-push
changes: *patterns-build_components changes: *patterns-build_components
- <<: *if-dev-push - <<: *if-dev-push
@@ -898,8 +854,6 @@
- <<: *if-label-component_ut - <<: *if-label-component_ut
- <<: *if-label-component_ut_esp32h2 - <<: *if-label-component_ut_esp32h2
- <<: *if-label-target_test - <<: *if-label-target_test
- <<: *if-label-unit_test
- <<: *if-label-unit_test_esp32h2
- <<: *if-dev-push - <<: *if-dev-push
changes: *patterns-build_components changes: *patterns-build_components
- <<: *if-dev-push - <<: *if-dev-push
@@ -938,8 +892,6 @@
- <<: *if-label-component_ut - <<: *if-label-component_ut
- <<: *if-label-component_ut_esp32s2 - <<: *if-label-component_ut_esp32s2
- <<: *if-label-target_test - <<: *if-label-target_test
- <<: *if-label-unit_test
- <<: *if-label-unit_test_esp32s2
- <<: *if-dev-push - <<: *if-dev-push
changes: *patterns-build_components changes: *patterns-build_components
- <<: *if-dev-push - <<: *if-dev-push
@@ -978,8 +930,6 @@
- <<: *if-label-component_ut - <<: *if-label-component_ut
- <<: *if-label-component_ut_esp32s3 - <<: *if-label-component_ut_esp32s3
- <<: *if-label-target_test - <<: *if-label-target_test
- <<: *if-label-unit_test
- <<: *if-label-unit_test_esp32s3
- <<: *if-dev-push - <<: *if-dev-push
changes: *patterns-build_components changes: *patterns-build_components
- <<: *if-dev-push - <<: *if-dev-push
@@ -1240,6 +1190,54 @@
- <<: *if-dev-push - <<: *if-dev-push
changes: *patterns-submodule changes: *patterns-submodule
.rules:build:example_test:
rules:
- <<: *if-revert-branch
when: never
- <<: *if-protected
- <<: *if-example_test-ota-include_nightly_run-rule
- <<: *if-label-build
- <<: *if-label-example_test
- <<: *if-label-example_test_esp32
- <<: *if-label-example_test_esp32c2
- <<: *if-label-example_test_esp32c3
- <<: *if-label-example_test_esp32c6
- <<: *if-label-example_test_esp32h2
- <<: *if-label-example_test_esp32p4
- <<: *if-label-example_test_esp32s2
- <<: *if-label-example_test_esp32s3
- <<: *if-label-target_test
- <<: *if-dev-push
changes: *patterns-build-example_test
- <<: *if-dev-push
changes: *patterns-build_components
- <<: *if-dev-push
changes: *patterns-build_system
- <<: *if-dev-push
changes: *patterns-downloadable-tools
- <<: *if-dev-push
changes: *patterns-example_test
- <<: *if-dev-push
changes: *patterns-example_test-adc
- <<: *if-dev-push
changes: *patterns-example_test-ethernet
- <<: *if-dev-push
changes: *patterns-example_test-i154
- <<: *if-dev-push
changes: *patterns-example_test-nvs_encr_hmac
- <<: *if-dev-push
changes: *patterns-example_test-sdio
- <<: *if-dev-push
changes: *patterns-example_test-usb
- <<: *if-dev-push
changes: *patterns-example_test-wifi
- <<: *if-dev-push
changes: *patterns-target_test-adc
- <<: *if-dev-push
changes: *patterns-target_test-ecdsa
- <<: *if-dev-push
changes: *patterns-target_test-wifi
.rules:build:example_test-esp32: .rules:build:example_test-esp32:
rules: rules:
- <<: *if-revert-branch - <<: *if-revert-branch
@@ -1636,15 +1634,6 @@
- <<: *if-label-integration_test_ble - <<: *if-label-integration_test_ble
- <<: *if-label-integration_test_wifi - <<: *if-label-integration_test_wifi
- <<: *if-label-target_test - <<: *if-label-target_test
- <<: *if-label-unit_test
- <<: *if-label-unit_test_esp32
- <<: *if-label-unit_test_esp32c2
- <<: *if-label-unit_test_esp32c3
- <<: *if-label-unit_test_esp32c6
- <<: *if-label-unit_test_esp32h2
- <<: *if-label-unit_test_esp32p4
- <<: *if-label-unit_test_esp32s2
- <<: *if-label-unit_test_esp32s3
- <<: *if-dev-push - <<: *if-dev-push
changes: *patterns-build-example_test changes: *patterns-build-example_test
- <<: *if-dev-push - <<: *if-dev-push
@@ -1701,251 +1690,6 @@
changes: *patterns-target_test-ecdsa changes: *patterns-target_test-ecdsa
- <<: *if-dev-push - <<: *if-dev-push
changes: *patterns-target_test-wifi changes: *patterns-target_test-wifi
- <<: *if-dev-push
changes: *patterns-unit_test
- <<: *if-dev-push
changes: *patterns-unit_test-sdio
.rules:build:unit_test:
rules:
- <<: *if-revert-branch
when: never
- <<: *if-protected
- <<: *if-label-build
- <<: *if-label-target_test
- <<: *if-label-unit_test
- <<: *if-label-unit_test_esp32
- <<: *if-label-unit_test_esp32c2
- <<: *if-label-unit_test_esp32c3
- <<: *if-label-unit_test_esp32c6
- <<: *if-label-unit_test_esp32h2
- <<: *if-label-unit_test_esp32p4
- <<: *if-label-unit_test_esp32s2
- <<: *if-label-unit_test_esp32s3
- <<: *if-dev-push
changes: *patterns-build_components
- <<: *if-dev-push
changes: *patterns-build_system
- <<: *if-dev-push
changes: *patterns-downloadable-tools
- <<: *if-dev-push
changes: *patterns-target_test-adc
- <<: *if-dev-push
changes: *patterns-target_test-ecdsa
- <<: *if-dev-push
changes: *patterns-target_test-wifi
- <<: *if-dev-push
changes: *patterns-unit_test
- <<: *if-dev-push
changes: *patterns-unit_test-sdio
.rules:build:unit_test-esp32:
rules:
- <<: *if-revert-branch
when: never
- <<: *if-protected
- <<: *if-label-build
- <<: *if-label-target_test
- <<: *if-label-unit_test
- <<: *if-label-unit_test_esp32
- <<: *if-dev-push
changes: *patterns-build_components
- <<: *if-dev-push
changes: *patterns-build_system
- <<: *if-dev-push
changes: *patterns-downloadable-tools
- <<: *if-dev-push
changes: *patterns-target_test-adc
- <<: *if-dev-push
changes: *patterns-target_test-ecdsa
- <<: *if-dev-push
changes: *patterns-target_test-wifi
- <<: *if-dev-push
changes: *patterns-unit_test
- <<: *if-dev-push
changes: *patterns-unit_test-sdio
.rules:build:unit_test-esp32c2:
rules:
- <<: *if-revert-branch
when: never
- <<: *if-protected
- <<: *if-label-build
- <<: *if-label-target_test
- <<: *if-label-unit_test
- <<: *if-label-unit_test_esp32c2
- <<: *if-dev-push
changes: *patterns-build_components
- <<: *if-dev-push
changes: *patterns-build_system
- <<: *if-dev-push
changes: *patterns-downloadable-tools
- <<: *if-dev-push
changes: *patterns-target_test-adc
- <<: *if-dev-push
changes: *patterns-target_test-ecdsa
- <<: *if-dev-push
changes: *patterns-target_test-wifi
- <<: *if-dev-push
changes: *patterns-unit_test
- <<: *if-dev-push
changes: *patterns-unit_test-sdio
.rules:build:unit_test-esp32c3:
rules:
- <<: *if-revert-branch
when: never
- <<: *if-protected
- <<: *if-label-build
- <<: *if-label-target_test
- <<: *if-label-unit_test
- <<: *if-label-unit_test_esp32c3
- <<: *if-dev-push
changes: *patterns-build_components
- <<: *if-dev-push
changes: *patterns-build_system
- <<: *if-dev-push
changes: *patterns-downloadable-tools
- <<: *if-dev-push
changes: *patterns-target_test-adc
- <<: *if-dev-push
changes: *patterns-target_test-ecdsa
- <<: *if-dev-push
changes: *patterns-target_test-wifi
- <<: *if-dev-push
changes: *patterns-unit_test
- <<: *if-dev-push
changes: *patterns-unit_test-sdio
.rules:build:unit_test-esp32c6:
rules:
- <<: *if-revert-branch
when: never
- <<: *if-protected
- <<: *if-label-build
- <<: *if-label-target_test
- <<: *if-label-unit_test
- <<: *if-label-unit_test_esp32c6
- <<: *if-dev-push
changes: *patterns-build_components
- <<: *if-dev-push
changes: *patterns-build_system
- <<: *if-dev-push
changes: *patterns-downloadable-tools
- <<: *if-dev-push
changes: *patterns-target_test-adc
- <<: *if-dev-push
changes: *patterns-target_test-ecdsa
- <<: *if-dev-push
changes: *patterns-target_test-wifi
- <<: *if-dev-push
changes: *patterns-unit_test
- <<: *if-dev-push
changes: *patterns-unit_test-sdio
.rules:build:unit_test-esp32h2:
rules:
- <<: *if-revert-branch
when: never
- <<: *if-protected
- <<: *if-label-build
- <<: *if-label-target_test
- <<: *if-label-unit_test
- <<: *if-label-unit_test_esp32h2
- <<: *if-dev-push
changes: *patterns-build_components
- <<: *if-dev-push
changes: *patterns-build_system
- <<: *if-dev-push
changes: *patterns-downloadable-tools
- <<: *if-dev-push
changes: *patterns-target_test-adc
- <<: *if-dev-push
changes: *patterns-target_test-ecdsa
- <<: *if-dev-push
changes: *patterns-target_test-wifi
- <<: *if-dev-push
changes: *patterns-unit_test
- <<: *if-dev-push
changes: *patterns-unit_test-sdio
.rules:build:unit_test-esp32p4:
rules:
- <<: *if-revert-branch
when: never
- <<: *if-protected
- <<: *if-label-build
- <<: *if-label-target_test
- <<: *if-label-unit_test
- <<: *if-label-unit_test_esp32p4
- <<: *if-dev-push
changes: *patterns-build_components
- <<: *if-dev-push
changes: *patterns-build_system
- <<: *if-dev-push
changes: *patterns-downloadable-tools
- <<: *if-dev-push
changes: *patterns-target_test-adc
- <<: *if-dev-push
changes: *patterns-target_test-ecdsa
- <<: *if-dev-push
changes: *patterns-target_test-wifi
- <<: *if-dev-push
changes: *patterns-unit_test
- <<: *if-dev-push
changes: *patterns-unit_test-sdio
.rules:build:unit_test-esp32s2:
rules:
- <<: *if-revert-branch
when: never
- <<: *if-protected
- <<: *if-label-build
- <<: *if-label-target_test
- <<: *if-label-unit_test
- <<: *if-label-unit_test_esp32s2
- <<: *if-dev-push
changes: *patterns-build_components
- <<: *if-dev-push
changes: *patterns-build_system
- <<: *if-dev-push
changes: *patterns-downloadable-tools
- <<: *if-dev-push
changes: *patterns-target_test-adc
- <<: *if-dev-push
changes: *patterns-target_test-ecdsa
- <<: *if-dev-push
changes: *patterns-target_test-wifi
- <<: *if-dev-push
changes: *patterns-unit_test
- <<: *if-dev-push
changes: *patterns-unit_test-sdio
.rules:build:unit_test-esp32s3:
rules:
- <<: *if-revert-branch
when: never
- <<: *if-protected
- <<: *if-label-build
- <<: *if-label-target_test
- <<: *if-label-unit_test
- <<: *if-label-unit_test_esp32s3
- <<: *if-dev-push
changes: *patterns-build_components
- <<: *if-dev-push
changes: *patterns-build_system
- <<: *if-dev-push
changes: *patterns-downloadable-tools
- <<: *if-dev-push
changes: *patterns-target_test-adc
- <<: *if-dev-push
changes: *patterns-target_test-ecdsa
- <<: *if-dev-push
changes: *patterns-target_test-wifi
- <<: *if-dev-push
changes: *patterns-unit_test
- <<: *if-dev-push
changes: *patterns-unit_test-sdio
.rules:labels:nvs_coverage: .rules:labels:nvs_coverage:
rules: rules:
@@ -1963,8 +1707,6 @@
- <<: *if-label-component_ut - <<: *if-label-component_ut
- <<: *if-label-component_ut_esp32 - <<: *if-label-component_ut_esp32
- <<: *if-label-target_test - <<: *if-label-target_test
- <<: *if-label-unit_test
- <<: *if-label-unit_test_esp32
- <<: *if-dev-push - <<: *if-dev-push
changes: *patterns-component_ut changes: *patterns-component_ut
@@ -2019,8 +1761,6 @@
- <<: *if-label-component_ut - <<: *if-label-component_ut
- <<: *if-label-component_ut_esp32c2 - <<: *if-label-component_ut_esp32c2
- <<: *if-label-target_test - <<: *if-label-target_test
- <<: *if-label-unit_test
- <<: *if-label-unit_test_esp32c2
- <<: *if-dev-push - <<: *if-dev-push
changes: *patterns-component_ut changes: *patterns-component_ut
@@ -2064,8 +1804,6 @@
- <<: *if-label-component_ut - <<: *if-label-component_ut
- <<: *if-label-component_ut_esp32c3 - <<: *if-label-component_ut_esp32c3
- <<: *if-label-target_test - <<: *if-label-target_test
- <<: *if-label-unit_test
- <<: *if-label-unit_test_esp32c3
- <<: *if-dev-push - <<: *if-dev-push
changes: *patterns-component_ut changes: *patterns-component_ut
@@ -2148,8 +1886,6 @@
- <<: *if-label-component_ut - <<: *if-label-component_ut
- <<: *if-label-component_ut_esp32c6 - <<: *if-label-component_ut_esp32c6
- <<: *if-label-target_test - <<: *if-label-target_test
- <<: *if-label-unit_test
- <<: *if-label-unit_test_esp32c6
- <<: *if-dev-push - <<: *if-dev-push
changes: *patterns-component_ut changes: *patterns-component_ut
@@ -2191,8 +1927,6 @@
- <<: *if-label-component_ut - <<: *if-label-component_ut
- <<: *if-label-component_ut_esp32h2 - <<: *if-label-component_ut_esp32h2
- <<: *if-label-target_test - <<: *if-label-target_test
- <<: *if-label-unit_test
- <<: *if-label-unit_test_esp32h2
- <<: *if-dev-push - <<: *if-dev-push
changes: *patterns-component_ut changes: *patterns-component_ut
@@ -2234,8 +1968,6 @@
- <<: *if-label-component_ut - <<: *if-label-component_ut
- <<: *if-label-component_ut_esp32s2 - <<: *if-label-component_ut_esp32s2
- <<: *if-label-target_test - <<: *if-label-target_test
- <<: *if-label-unit_test
- <<: *if-label-unit_test_esp32s2
- <<: *if-dev-push - <<: *if-dev-push
changes: *patterns-component_ut changes: *patterns-component_ut
@@ -2290,8 +2022,6 @@
- <<: *if-label-component_ut - <<: *if-label-component_ut
- <<: *if-label-component_ut_esp32s3 - <<: *if-label-component_ut_esp32s3
- <<: *if-label-target_test - <<: *if-label-target_test
- <<: *if-label-unit_test
- <<: *if-label-unit_test_esp32s3
- <<: *if-dev-push - <<: *if-dev-push
changes: *patterns-component_ut changes: *patterns-component_ut
@@ -2977,120 +2707,3 @@
- <<: *if-label-submodule - <<: *if-label-submodule
- <<: *if-dev-push - <<: *if-dev-push
changes: *patterns-submodule changes: *patterns-submodule
.rules:test:unit_test-esp32:
rules:
- <<: *if-revert-branch
when: never
- <<: *if-protected
- <<: *if-label-build-only
when: never
- <<: *if-label-target_test
- <<: *if-label-unit_test
- <<: *if-label-unit_test_esp32
- <<: *if-dev-push
changes: *patterns-unit_test
.rules:test:unit_test-esp32c2:
rules:
- <<: *if-revert-branch
when: never
- <<: *if-protected
- <<: *if-label-build-only
when: never
- <<: *if-label-target_test
- <<: *if-label-unit_test
- <<: *if-label-unit_test_esp32c2
- <<: *if-dev-push
changes: *patterns-unit_test
.rules:test:unit_test-esp32c3:
rules:
- <<: *if-revert-branch
when: never
- <<: *if-protected
- <<: *if-label-build-only
when: never
- <<: *if-label-target_test
- <<: *if-label-unit_test
- <<: *if-label-unit_test_esp32c3
- <<: *if-dev-push
changes: *patterns-unit_test
.rules:test:unit_test-esp32c3-sdio:
rules:
- <<: *if-revert-branch
when: never
- <<: *if-protected
- <<: *if-label-build-only
when: never
- <<: *if-label-target_test
- <<: *if-label-unit_test
- <<: *if-label-unit_test_esp32c3
- <<: *if-dev-push
changes: *patterns-unit_test-sdio
.rules:test:unit_test-esp32c6:
rules:
- <<: *if-revert-branch
when: never
- <<: *if-protected
- <<: *if-label-build-only
when: never
- <<: *if-label-target_test
- <<: *if-label-unit_test
- <<: *if-label-unit_test_esp32c6
- <<: *if-dev-push
changes: *patterns-unit_test
.rules:test:unit_test-esp32h2:
rules:
- <<: *if-revert-branch
when: never
- <<: *if-protected
- <<: *if-label-build-only
when: never
- <<: *if-label-target_test
- <<: *if-label-unit_test
- <<: *if-label-unit_test_esp32h2
- <<: *if-dev-push
changes: *patterns-unit_test
.rules:test:unit_test-esp32s2:
rules:
- <<: *if-revert-branch
when: never
- <<: *if-protected
- <<: *if-label-build-only
when: never
- <<: *if-label-target_test
- <<: *if-label-unit_test
- <<: *if-label-unit_test_esp32s2
- <<: *if-dev-push
changes: *patterns-unit_test
.rules:test:unit_test-esp32s2-sdio:
rules:
- <<: *if-revert-branch
when: never
- <<: *if-protected
- <<: *if-label-build-only
when: never
- <<: *if-label-target_test
- <<: *if-label-unit_test
- <<: *if-label-unit_test_esp32s2
- <<: *if-dev-push
changes: *patterns-unit_test-sdio
.rules:test:unit_test-esp32s3:
rules:
- <<: *if-revert-branch
when: never
- <<: *if-protected
- <<: *if-label-build-only
when: never
- <<: *if-label-target_test
- <<: *if-label-unit_test
- <<: *if-label-unit_test_esp32s3
- <<: *if-dev-push
changes: *patterns-unit_test

757
.gitlab/ci/target-test.yml
View File

File diff suppressed because it is too large Load Diff

4
.gitlab/ci/upload_cache.yml
View File

@@ -9,7 +9,7 @@
upload-pip-cache: upload-pip-cache:
extends: extends:
- .upload_cache_template - .upload_cache_template
- .before_script_minimal - .before_script:minimal
- .rules:patterns:python-cache - .rules:patterns:python-cache
tags: tags:
- $GEO - $GEO
@@ -29,7 +29,7 @@ upload-pip-cache:
upload-submodules-cache: upload-submodules-cache:
extends: extends:
- .upload_cache_template - .upload_cache_template
- .before_script_minimal - .before_script:minimal
- .rules:patterns:submodule - .rules:patterns:submodule
tags: tags:
- $GEO - $GEO

187
tools/ci/artifacts_handler.py Normal file
View File

@@ -0,0 +1,187 @@
# SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0
import argparse
import fnmatch
import glob
import os
import typing as t
from enum import Enum
from pathlib import Path
from zipfile import ZipFile
import urllib3
from minio import Minio
class ArtifactType(str, Enum):
MAP_AND_ELF_FILES = 'map_and_elf_files'
BUILD_DIR_WITHOUT_MAP_AND_ELF_FILES = 'build_dir_without_map_and_elf_files'
LOGS = 'logs'
SIZE_REPORTS = 'size_reports'
TYPE_PATTERNS_DICT = {
ArtifactType.MAP_AND_ELF_FILES: [
'**/build*/**/*.map',
'**/build*/**/*.elf',
],
ArtifactType.BUILD_DIR_WITHOUT_MAP_AND_ELF_FILES: [
'**/build*/build_log.txt',
'**/build*/**/*.bin',
'**/build*/flasher_args.json',
'**/build*/flash_project_args',
'**/build*/config/sdkconfig.json',
'**/build*/project_description.json',
'list_job_*.txt',
],
ArtifactType.LOGS: [
'**/build*/build_log.txt',
],
ArtifactType.SIZE_REPORTS: [
'**/build*/size.json',
'size_info.txt',
],
}
def getenv(env_var: str) -> str:
try:
return os.environ[env_var]
except KeyError as e:
raise Exception(f'Environment variable {env_var} not set') from e
def _download_files(
pipeline_id: int,
*,
artifact_type: t.Optional[ArtifactType] = None,
job_name: t.Optional[str] = None,
job_id: t.Optional[int] = None,
) -> None:
if artifact_type:
prefix = f'{pipeline_id}/{artifact_type.value}/'
else:
prefix = f'{pipeline_id}/'
for obj in client.list_objects(getenv('IDF_S3_BUCKET'), prefix=prefix, recursive=True):
obj_name = obj.object_name
obj_p = Path(obj_name)
# <pipeline_id>/<action_type>/<job_name>/<job_id>.zip
if len(obj_p.parts) != 4:
print(f'Invalid object name: {obj_name}')
continue
if job_name:
# could be a pattern
if not fnmatch.fnmatch(obj_p.parts[2], job_name):
print(f'Job name {job_name} does not match {obj_p.parts[2]}')
continue
if job_id:
if obj_p.parts[3] != f'{job_id}.zip':
print(f'Job ID {job_id} does not match {obj_p.parts[3]}')
continue
client.fget_object(getenv('IDF_S3_BUCKET'), obj_name, obj_name)
print(f'Downloaded {obj_name}')
if obj_name.endswith('.zip'):
with ZipFile(obj_name, 'r') as zr:
zr.extractall()
print(f'Extracted {obj_name}')
os.remove(obj_name)
def _upload_files(
pipeline_id: int,
*,
artifact_type: ArtifactType,
job_name: str,
job_id: str,
) -> None:
has_file = False
with ZipFile(f'{job_id}.zip', 'w') as zw:
for pattern in TYPE_PATTERNS_DICT[artifact_type]:
for file in glob.glob(pattern, recursive=True):
zw.write(file)
has_file = True
try:
if has_file:
obj_name = f'{pipeline_id}/{artifact_type.value}/{job_name.split(" ")[0]}/{job_id}.zip'
print(f'Created archive file: {job_id}.zip, uploading as {obj_name}')
client.fput_object(getenv('IDF_S3_BUCKET'), obj_name, f'{job_id}.zip')
url = client.get_presigned_url('GET', getenv('IDF_S3_BUCKET'), obj_name)
print(f'Please download the archive file which includes {artifact_type.value} from {url}')
finally:
os.remove(f'{job_id}.zip')
if __name__ == '__main__':
parser = argparse.ArgumentParser(
description='Download or upload files from/to S3, the object name would be '
'[PIPELINE_ID]/[ACTION_TYPE]/[JOB_NAME]/[JOB_ID].zip.'
'\n'
'For example: 123456/binaries/build_pytest_examples_esp32/123456789.zip',
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
)
common_args = argparse.ArgumentParser(add_help=False, formatter_class=argparse.ArgumentDefaultsHelpFormatter)
common_args.add_argument('--pipeline-id', type=int, help='Pipeline ID')
common_args.add_argument(
'--type', type=str, nargs='+', choices=[a.value for a in ArtifactType], help='Types of files to download'
)
action = parser.add_subparsers(dest='action', help='Download or Upload')
download = action.add_parser('download', help='Download files from S3', parents=[common_args])
upload = action.add_parser('upload', help='Upload files to S3', parents=[common_args])
download.add_argument('--job-name', type=str, help='Job name pattern')
download.add_argument('--job-id', type=int, help='Job ID')
upload.add_argument('--job-name', type=str, help='Job name')
upload.add_argument('--job-id', type=int, help='Job ID')
args = parser.parse_args()
client = Minio(
getenv('IDF_S3_SERVER').replace('https://', ''),
access_key=getenv('IDF_S3_ACCESS_KEY'),
secret_key=getenv('IDF_S3_SECRET_KEY'),
http_client=urllib3.PoolManager(
timeout=urllib3.Timeout.DEFAULT_TIMEOUT,
retries=urllib3.Retry(
total=5,
backoff_factor=0.2,
status_forcelist=[500, 502, 503, 504],
),
),
)
ci_pipeline_id = args.pipeline_id or getenv('CI_PIPELINE_ID') # required
if args.action == 'download':
method = _download_files
ci_job_name = args.job_name # optional
ci_job_id = args.job_id # optional
else:
method = _upload_files # type: ignore
ci_job_name = args.job_name or getenv('CI_JOB_NAME') # required
ci_job_id = args.job_id or getenv('CI_JOB_ID') # required
if args.type:
types = [ArtifactType(t) for t in args.type]
else:
types = list(ArtifactType)
print(f'{"Pipeline ID":15}: {ci_pipeline_id}')
if ci_job_name:
print(f'{"Job name":15}: {ci_job_name}')
if ci_job_id:
print(f'{"Job ID":15}: {ci_job_id}')
for _t in types:
method(ci_pipeline_id, artifact_type=_t, job_name=ci_job_name, job_id=ci_job_id) # type: ignore

16
tools/ci/ci_get_mr_info.py
View File

@@ -3,7 +3,7 @@
# internal use only for CI # internal use only for CI
# get latest MR information by source branch # get latest MR information by source branch
# #
# SPDX-FileCopyrightText: 2020-2022 Espressif Systems (Shanghai) CO LTD # SPDX-FileCopyrightText: 2020-2023 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0 # SPDX-License-Identifier: Apache-2.0
# #
@@ -73,6 +73,16 @@ def get_mr_components(source_branch: str) -> t.List[str]:
return list(components) return list(components)
def get_target_in_tags(tags: str) -> str:
from idf_pytest.constants import TARGET_MARKERS
for x in tags.split(','):
if x in TARGET_MARKERS:
return x
raise RuntimeError(f'No target marker found in {tags}')
def _print_list(_list: t.List[str], separator: str = '\n') -> None: def _print_list(_list: t.List[str], separator: str = '\n') -> None:
print(separator.join(_list)) print(separator.join(_list))
@@ -88,6 +98,8 @@ if __name__ == '__main__':
actions.add_parser('files', parents=[common_args]) actions.add_parser('files', parents=[common_args])
actions.add_parser('commits', parents=[common_args]) actions.add_parser('commits', parents=[common_args])
actions.add_parser('components', parents=[common_args]) actions.add_parser('components', parents=[common_args])
target = actions.add_parser('target_in_tags')
target.add_argument('tags', help='comma separated tags, e.g., esp32,generic')
args = parser.parse_args() args = parser.parse_args()
@@ -99,5 +111,7 @@ if __name__ == '__main__':
_print_list([commit.id for commit in get_mr_commits(args.src_branch)]) _print_list([commit.id for commit in get_mr_commits(args.src_branch)])
elif args.action == 'components': elif args.action == 'components':
_print_list(get_mr_components(args.src_branch)) _print_list(get_mr_components(args.src_branch))
elif args.action == 'target_in_tags':
print(get_target_in_tags(args.tags))
else: else:
raise NotImplementedError('not possible to get here') raise NotImplementedError('not possible to get here')

1
tools/ci/exclude_check_tools_files.txt
View File

@@ -40,3 +40,4 @@ tools/templates/sample_component/include/main.h
tools/templates/sample_component/main.c tools/templates/sample_component/main.c
tools/ci/cleanup_ignore_lists.py tools/ci/cleanup_ignore_lists.py
tools/ci/idf_pytest/**/* tools/ci/idf_pytest/**/*
tools/ci/artifacts_handler.py

2
tools/ci/python_packages/gitlab_api.py
View File

@@ -80,7 +80,7 @@ class Gitlab(object):
self.gitlab_inst = gitlab.Gitlab.from_config(gitlab_id=gitlab_id, config_files=config_files) self.gitlab_inst = gitlab.Gitlab.from_config(gitlab_id=gitlab_id, config_files=config_files)
self.gitlab_inst.auth() self.gitlab_inst.auth()
if project_id: if project_id:
self.project = self.gitlab_inst.projects.get(project_id) self.project = self.gitlab_inst.projects.get(project_id, lazy=True)
else: else:
self.project = None self.project = None

5
tools/ci/utils.sh
View File

@@ -49,6 +49,11 @@ function set_component_ut_vars() {
echo "exported variables COMPONENT_UT_DIRS, COMPONENT_UT_EXCLUDES" echo "exported variables COMPONENT_UT_DIRS, COMPONENT_UT_EXCLUDES"
} }
function upload_artifacts_to_s3() {
# for detailed documents, please refer to .gitlab/ci/README.md#uploaddownload-artifacts-to-internal-minio-server
python tools/ci/artifacts_handler.py upload
}
function error() { function error() {
printf "\033[0;31m%s\n\033[0m" "${1}" >&2 printf "\033[0;31m%s\n\033[0m" "${1}" >&2
} }

1
tools/requirements/requirements.ci.txt
View File

@@ -10,3 +10,4 @@ python-gitlab
pyyaml pyyaml
SimpleWebSocketServer SimpleWebSocketServer
pylint-gitlab pylint-gitlab
minio

1
tools/unit-test-app/tools/ConfigDependency.yml
View File

@@ -7,6 +7,7 @@
"ESP32C3_IDF": "CONFIG_IDF_TARGET_ESP32C3=y" "ESP32C3_IDF": "CONFIG_IDF_TARGET_ESP32C3=y"
"ESP32C6_IDF": "CONFIG_IDF_TARGET_ESP32C6=y" "ESP32C6_IDF": "CONFIG_IDF_TARGET_ESP32C6=y"
"ESP32H2_IDF": "CONFIG_IDF_TARGET_ESP32H2=y" "ESP32H2_IDF": "CONFIG_IDF_TARGET_ESP32H2=y"
"ESP32P4_IDF": "CONFIG_IDF_TARGET_ESP32P4=y"
"quad_psram": '{CONFIG_SPIRAM_MODE_QUAD=y} and {CONFIG_IDF_TARGET_ESP32S3=y}' "quad_psram": '{CONFIG_SPIRAM_MODE_QUAD=y} and {CONFIG_IDF_TARGET_ESP32S3=y}'
"octal_psram": '{CONFIG_SPIRAM_MODE_OCT=y} and {CONFIG_IDF_TARGET_ESP32S3=y}' "octal_psram": '{CONFIG_SPIRAM_MODE_OCT=y} and {CONFIG_IDF_TARGET_ESP32S3=y}'
"xtal_26mhz": '{CONFIG_XTAL_FREQ_26=y} and {CONFIG_IDF_TARGET_ESP32C2=y}' "xtal_26mhz": '{CONFIG_XTAL_FREQ_26=y} and {CONFIG_IDF_TARGET_ESP32C2=y}'