From e0f374d92bb9dfb1e5976182a4eae2892ca86959 Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Fri, 18 Oct 2024 10:51:07 +0200 Subject: [PATCH 1/8] ci: stop print presigned url, since the credential is masked --- tools/ci/artifacts_handler.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tools/ci/artifacts_handler.py b/tools/ci/artifacts_handler.py index 6147f61bdd..06752b0283 100644 --- a/tools/ci/artifacts_handler.py +++ b/tools/ci/artifacts_handler.py @@ -152,11 +152,8 @@ def _upload_files( try: if has_file: obj_name = f'{pipeline_id}/{artifact_type.value}/{sanitize_job_name(job_name)}/{job_id}.zip' - print(f'Created archive file: {job_id}.zip, uploading as {obj_name}') - client.fput_object(getenv('IDF_S3_BUCKET'), obj_name, f'{job_id}.zip') - url = client.get_presigned_url('GET', getenv('IDF_S3_BUCKET'), obj_name) - print(f'Please download the archive file which includes {artifact_type.value} from {url}') + print(f'Created archive file: {job_id}.zip, uploaded as {obj_name}') finally: os.remove(f'{job_id}.zip') From ffbdd8a1454246a810b0bf1d1c68956bf29b07b6 Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Fri, 18 Oct 2024 13:05:28 +0200 Subject: [PATCH 2/8] ci: add timeout for build jobs --- tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml b/tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml index bf5fb01c57..89a0b0d7ff 100644 --- a/tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml +++ b/tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml @@ -10,6 +10,7 @@ - .after_script:build:ccache-show-stats:upload-failed-job-logs image: $ESP_ENV_IMAGE stage: build + timeout: 1 hour variables: # Enable ccache for all build jobs. See configure_ci_environment.sh for more ccache related settings. IDF_CCACHE_ENABLE: "1" From 257878ddb4dcb9afe7403a99cf54e5d45301e469 Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Wed, 16 Oct 2024 12:22:09 +0200 Subject: [PATCH 3/8] ci: change logging level from debug to info for build jobs --- .gitlab/ci/build.yml | 2 +- tools/ci/build_template_app.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab/ci/build.yml b/.gitlab/ci/build.yml index 376946d0bd..e4e820099b 100644 --- a/.gitlab/ci/build.yml +++ b/.gitlab/ci/build.yml @@ -126,7 +126,7 @@ gcc_static_analyzer: ANALYZING_APP: "examples/get-started/hello_world" script: - echo "CONFIG_COMPILER_STATIC_ANALYZER=y" >> ${ANALYZING_APP}/sdkconfig.defaults - - python -m idf_build_apps build -vv -p ${ANALYZING_APP} -t all + - python -m idf_build_apps build -v -p ${ANALYZING_APP} -t all ######################################## # Clang Build Apps Without Tests Cases # diff --git a/tools/ci/build_template_app.sh b/tools/ci/build_template_app.sh index c77c85026a..59457ad0cb 100755 --- a/tools/ci/build_template_app.sh +++ b/tools/ci/build_template_app.sh @@ -54,7 +54,7 @@ build_stage2() { # Override EXTRA_CFLAGS and EXTRA_CXXFLAGS in the environment export EXTRA_CFLAGS=${PEDANTIC_CFLAGS/-Werror=unused-variable -Werror=unused-but-set-variable -Werror=unused-function/} export EXTRA_CXXFLAGS=${PEDANTIC_CXXFLAGS/-Werror=unused-variable -Werror=unused-but-set-variable -Werror=unused-function/} - python -m idf_build_apps build -vv \ + python -m idf_build_apps build -v \ -p ${TEMPLATE_APP_PATH} \ -t all \ ${CONFIG_STR} \ @@ -69,7 +69,7 @@ build_stage2() { build_stage1() { CONFIG_STR=$(get_config_str sdkconfig.ci2.*=) - python -m idf_build_apps build -vv \ + python -m idf_build_apps build -v \ -p ${TEMPLATE_APP_PATH} \ -t all \ ${CONFIG_STR} \ From ffdb0f012e03eda26bad3aa35e3b3a0090c10d60 Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Mon, 21 Oct 2024 09:05:21 +0200 Subject: [PATCH 4/8] ci: remove gcc dependency in target test --- .../newlib/test_apps/newlib/pytest_newlib.py | 29 +------------------ 1 file changed, 1 insertion(+), 28 deletions(-) diff --git a/components/newlib/test_apps/newlib/pytest_newlib.py b/components/newlib/test_apps/newlib/pytest_newlib.py index 78c01517d5..8b2c5cc450 100644 --- a/components/newlib/test_apps/newlib/pytest_newlib.py +++ b/components/newlib/test_apps/newlib/pytest_newlib.py @@ -1,35 +1,9 @@ -# SPDX-FileCopyrightText: 2022-2023 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2022-2024 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: CC0-1.0 -import subprocess -from os import path - import pytest -import yaml from pytest_embedded import Dut -def validate_sbom(dut: Dut) -> None: - dirname = path.dirname(path.abspath(__file__)) - sbom_file = path.join(path.dirname(path.dirname(dirname)), 'sbom.yml') - gcc_input_file = path.join(dirname, 'test_sbom', 'newlib_version.c') - gcc = 'riscv32-esp-elf-gcc' - if dut.target in dut.XTENSA_TARGETS: - gcc = f'xtensa-{dut.target}-elf-gcc' - gcc_process = subprocess.run(f'{gcc} -E {gcc_input_file}', - shell=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - check=True) - output_lines = gcc_process.stdout.splitlines() - assert output_lines, 'Can not get newlib version' - toolchain_newlib_version = output_lines[-1].replace(' ', '.') - with open(sbom_file, 'r', encoding='utf-8') as yaml_file: - sbom_newlib_version = yaml.safe_load(yaml_file).get('version') - assert sbom_newlib_version, 'Can not get newlib version from sbom.yml' - assert toolchain_newlib_version == sbom_newlib_version, 'toolchain_newlib_version != sbom_newlib_version' - - @pytest.mark.generic @pytest.mark.parametrize( 'config', @@ -44,5 +18,4 @@ def validate_sbom(dut: Dut) -> None: indirect=True ) def test_newlib(dut: Dut) -> None: - validate_sbom(dut) dut.run_all_single_board_cases() From f326d5d7ce7b87b3cad81f4866d6298a077b2720 Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Fri, 25 Oct 2024 12:49:58 +0200 Subject: [PATCH 5/8] ci: print esp-coredump output when failed in panic tests --- .../system/panic/test_panic_util/panic_dut.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/tools/test_apps/system/panic/test_panic_util/panic_dut.py b/tools/test_apps/system/panic/test_panic_util/panic_dut.py index 2619008fe0..ff8004cb85 100644 --- a/tools/test_apps/system/panic/test_panic_util/panic_dut.py +++ b/tools/test_apps/system/panic/test_panic_util/panic_dut.py @@ -151,9 +151,15 @@ class PanicTestDut(IdfDut): logging.info('espcoredump output is written to %s', self.coredump_output.name) self.serial.close() - subprocess.check_call(espcoredump_args, stdout=self.coredump_output) - self.coredump_output.flush() - self.coredump_output.seek(0) + try: + subprocess.check_call(espcoredump_args, stdout=self.coredump_output, stderr=self.coredump_output) + except subprocess.CalledProcessError: + self.coredump_output.flush() + with open(output_file_name, 'r') as file: + logging.error('espcoredump failed with output: %s', file.read()) + raise + finally: + self.coredump_output.seek(0) def process_coredump_uart( self, expected: Optional[List[Union[str, re.Pattern]]] = None, wait_reboot: bool = True From e1781df75b9b618c98404bdfedfc285ccbaaba03 Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Mon, 4 Nov 2024 11:05:56 +0100 Subject: [PATCH 6/8] feat: idf_tools.py export support env var "IDF_SKIP_TOOLS_CHECK" --- tools/idf_tools.py | 8 ++++++-- tools/test_idf_tools/test_idf_tools.py | 13 +++++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/tools/idf_tools.py b/tools/idf_tools.py index 395eebefec..b5e4f024ec 100755 --- a/tools/idf_tools.py +++ b/tools/idf_tools.py @@ -2122,8 +2122,12 @@ def process_tool( if not tool.versions_installed: if tool.get_install_type() == IDFTool.INSTALL_ALWAYS: - handle_missing_versions(tool, tool_name, install_cmd, prefer_system_hint) - tool_found = False + if os.getenv('IDF_SKIP_TOOLS_CHECK', '0') == '1': + warn(f'Tool {tool_name} is not installed and IDF_SKIP_TOOLS_CHECK is set. ' + 'This may cause build failures.') + else: + handle_missing_versions(tool, tool_name, install_cmd, prefer_system_hint) + tool_found = False # If a tool found, but it is optional and does not have versions installed, use whatever is in PATH. return tool_export_paths, tool_export_vars, tool_found diff --git a/tools/test_idf_tools/test_idf_tools.py b/tools/test_idf_tools/test_idf_tools.py index 1275430ef0..87eab1dee0 100755 --- a/tools/test_idf_tools/test_idf_tools.py +++ b/tools/test_idf_tools/test_idf_tools.py @@ -312,6 +312,19 @@ class TestUsage(TestUsageBase): self.assertNotIn(tool_to_test, output) + def test_export_with_required_tools_check_skipped(self): + self.run_idf_tools_with_error(['export'], assertError=True) + + new_os_environ = os.environ.copy() + new_os_environ['IDF_SKIP_TOOLS_CHECK'] = '1' + with patch('os.environ', new_os_environ): + self.run_idf_tools_with_action(['export']) + + self.run_idf_tools_with_action(['install', OPENOCD]) + output = self.run_idf_tools_with_action(['export']) + self.assertIn('%s/tools/openocd-esp32/%s/openocd-esp32/bin' % + (self.temp_tools_dir, OPENOCD_VERSION), output) + # TestUsageUnix tests installed tools on UNIX platforms @unittest.skipIf(sys.platform == 'win32', reason='Tools for UNIX differ') From 7e465dd5268ed5e61527611d9679c44518fb11d3 Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Fri, 8 Nov 2024 11:15:04 +0100 Subject: [PATCH 7/8] ci: ignore test-specific 3rd-party libs while building clang projects --- .gitlab/ci/build.yml | 22 ++-------------------- 1 file changed, 2 insertions(+), 20 deletions(-) diff --git a/.gitlab/ci/build.yml b/.gitlab/ci/build.yml index e4e820099b..f3d0f7df4d 100644 --- a/.gitlab/ci/build.yml +++ b/.gitlab/ci/build.yml @@ -12,7 +12,7 @@ IDF_CCACHE_ENABLE: "1" dependencies: [] -.build_cmake_template: +.build_cmake_clang_template: extends: - .build_template - .before_script:build @@ -34,29 +34,11 @@ - "**/build*/size.json" expire_in: 1 week when: always - script: - # CI specific options start from "--parallel-count xxx". could ignore when running locally - - run_cmd python tools/ci/ci_build_apps.py $TEST_DIR -v - -t $IDF_TARGET - --copy-sdkconfig - --parallel-count ${CI_NODE_TOTAL:-1} - --parallel-index ${CI_NODE_INDEX:-1} - --extra-preserve-dirs - examples/bluetooth/esp_ble_mesh/ble_mesh_console - examples/bluetooth/hci/controller_hci_uart_esp32 - examples/wifi/iperf - --modified-components ${MR_MODIFIED_COMPONENTS} - --modified-files ${MR_MODIFIED_FILES} - # for detailed documents, please refer to .gitlab/ci/README.md#uploaddownload-artifacts-to-internal-minio-server - - python tools/ci/artifacts_handler.py upload - -.build_cmake_clang_template: - extends: - - .build_cmake_template variables: IDF_TOOLCHAIN: clang TEST_BUILD_OPTS_EXTRA: "" TEST_DIR: tools/test_apps/system/clang_build_test + PYTEST_IGNORE_COLLECT_IMPORT_ERROR: "1" script: # CI specific options start from "--parallel-count xxx". could ignore when running locally - run_cmd python tools/ci/ci_build_apps.py $TEST_DIR -v From be2ec1615c8f1bf7daa89cccdfba026f9068ff20 Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Wed, 16 Oct 2024 12:21:21 +0200 Subject: [PATCH 8/8] ci: target-test job skip installing toolchain, only install python env also run with collapsed time section to better track run time --- .gitlab/ci/common.yml | 57 +++++++++++-------- .gitlab/ci/host-test.yml | 11 ++-- .../templates/.dynamic_jobs.yml | 13 ++++- tools/ci/utils.sh | 15 +++++ 4 files changed, 65 insertions(+), 31 deletions(-) diff --git a/.gitlab/ci/common.yml b/.gitlab/ci/common.yml index 5328c835dd..c11e1574ee 100644 --- a/.gitlab/ci/common.yml +++ b/.gitlab/ci/common.yml @@ -57,7 +57,7 @@ variables: # Docker images ESP_ENV_IMAGE: "${CI_DOCKER_REGISTRY}/esp-env-v5.4:1" ESP_IDF_DOC_ENV_IMAGE: "${CI_DOCKER_REGISTRY}/esp-idf-doc-env-v5.4:1-1" - TARGET_TEST_ENV_IMAGE: "${CI_DOCKER_REGISTRY}/target-test-env-v5.4:1" + TARGET_TEST_ENV_IMAGE: "${CI_DOCKER_REGISTRY}/target-test-env-v5.4:2" SONARQUBE_SCANNER_IMAGE: "${CI_DOCKER_REGISTRY}/sonarqube-scanner:5" PRE_COMMIT_IMAGE: "${CI_DOCKER_REGISTRY}/esp-idf-pre-commit:1" @@ -141,40 +141,48 @@ variables: export IDF_MIRROR_PREFIX_MAP= fi - # install latest python packages - # target test jobs - if [[ "${CI_JOB_STAGE}" == "target_test" ]]; then - run_cmd bash install.sh --enable-ci --enable-pytest --enable-test-specific - elif [[ "${CI_JOB_STAGE}" == "build_doc" ]]; then - run_cmd bash install.sh --enable-ci --enable-docs - elif [[ "${CI_JOB_STAGE}" == "build" ]]; then - run_cmd bash install.sh --enable-ci - else - if ! echo "${CI_JOB_NAME}" | egrep ".*pytest.*"; then + if [[ "${CI_JOB_STAGE}" != "target_test" ]]; then + section_start "running_install_sh" "Running install.sh" + if [[ "${CI_JOB_STAGE}" == "build_doc" ]]; then + run_cmd bash install.sh --enable-ci --enable-docs + elif [[ "${CI_JOB_STAGE}" == "build" ]]; then run_cmd bash install.sh --enable-ci else - run_cmd bash install.sh --enable-ci --enable-pytest --enable-test-specific + if ! echo "${CI_JOB_NAME}" | egrep ".*pytest.*"; then + run_cmd bash install.sh --enable-ci + else + run_cmd bash install.sh --enable-ci --enable-pytest --enable-test-specific + fi fi + section_end "running_install_sh" + else + section_start "install_python_env" "Install Python environment" + run_cmd python tools/idf_tools.py install-python-env --features ci,pytest,test-specific + section_end "install_python_env" fi - # Install esp-clang if necessary + if [[ ! -z "$INSTALL_EXTRA_TOOLS" ]]; then + section_start "installing_optional_tools" "Install optional tools ${INSTALL_EXTRA_TOOLS}" + $IDF_PATH/tools/idf_tools.py --non-interactive install $INSTALL_EXTRA_TOOLS + section_end "installing_optional_tools" + fi + + # Install esp-clang if necessary (esp-clang is separately installed) if [[ "$IDF_TOOLCHAIN" == "clang" && -z "$CI_CLANG_DISTRO_URL" ]]; then $IDF_PATH/tools/idf_tools.py --non-interactive install esp-clang fi - # Install QEMU if necessary - if [[ ! -z "$INSTALL_QEMU" ]]; then - $IDF_PATH/tools/idf_tools.py --non-interactive install qemu-xtensa qemu-riscv32 + if [[ "${CI_JOB_STAGE}" == "target_test" ]]; then + section_start "IDF_SKIP_TOOLS_CHECK" "Skip required tools check" + export IDF_SKIP_TOOLS_CHECK=1 + section_end "IDF_SKIP_TOOLS_CHECK" fi - - # Since the version 3.21 CMake passes source files and include dirs to ninja using absolute paths. - # Needed for pytest junit reports. - $IDF_PATH/tools/idf_tools.py --non-interactive install cmake - + section_start "source_export" "Source export.sh" source ./export.sh + section_end "source_export" # Custom clang toolchain - if [[ ! -z "$CI_CLANG_DISTRO_URL" ]]; then + if [[ "$IDF_TOOLCHAIN" == "clang" && ! -z "$CI_CLANG_DISTRO_URL" ]]; then echo "Using custom clang from ${CI_CLANG_DISTRO_URL}" wget $CI_CLANG_DISTRO_URL ARCH_NAME=$(basename $CI_CLANG_DISTRO_URL) @@ -198,6 +206,8 @@ variables: rm -rf ${CI_PYTHON_TOOL_REPO} fi + info "setup tools and python venv done" + .show_ccache_statistics: &show_ccache_statistics | # Show ccache statistics if enabled globally test "$CI_CCACHE_STATS" == 1 && test -n "$(which ccache)" && ccache --show-stats -vv || true @@ -222,10 +232,11 @@ variables: - export IDF_TOOLS_PATH="${HOME}/.espressif_runner_${CI_RUNNER_ID}_${CI_CONCURRENT_ID}" # remove idf-env.json, since it may contains enabled "features" - rm -f $IDF_TOOLS_PATH/idf-env.json - - $IDF_PATH/tools/idf_tools.py --non-interactive install cmake ninja # This adds tools (compilers) and the version-specific Python environment to PATH - *setup_tools_and_idf_python_venv - fetch_submodules + variables: + INSTALL_EXTRA_TOOLS: cmake ninja .after_script:build:macos:upload-failed-job-logs:ccache-show-stats: after_script: diff --git a/.gitlab/ci/host-test.yml b/.gitlab/ci/host-test.yml index 5140b97729..42285b2022 100644 --- a/.gitlab/ci/host-test.yml +++ b/.gitlab/ci/host-test.yml @@ -195,7 +195,7 @@ test_tools: junit: ${IDF_PATH}/XUNIT_*.xml variables: LC_ALL: C.UTF-8 - INSTALL_QEMU: 1 # for test_idf_qemu.py + INSTALL_EXTRA_TOOLS: "qemu-xtensa qemu-riscv32" # for test_idf_qemu.py script: - stat=0 - cd ${IDF_PATH}/tools/ci/test_autocomplete @@ -282,9 +282,10 @@ test_pytest_qemu: junit: XUNIT_RESULT.xml parallel: matrix: - - IDF_TARGET: [esp32, esp32c3] - variables: - INSTALL_QEMU: 1 + - IDF_TARGET: "esp32" + INSTALL_EXTRA_TOOLS: "qemu-xtensa" + - IDF_TARGET: "esp32c3" + INSTALL_EXTRA_TOOLS: "qemu-riscv32" script: - run_cmd python tools/ci/ci_build_apps.py . -v --target $IDF_TARGET @@ -348,7 +349,7 @@ test_pytest_macos: variables: PYTEST_IGNORE_COLLECT_IMPORT_ERROR: "1" script: - - run_cmd python tools/ci/ci_build_apps.py components examples tools/test_apps -vv + - run_cmd python tools/ci/ci_build_apps.py components examples tools/test_apps -v --target linux --pytest-apps -m \"host_test and macos_shell\" diff --git a/tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml b/tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml index 89a0b0d7ff..4d78137e60 100644 --- a/tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml +++ b/tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml @@ -14,6 +14,9 @@ variables: # Enable ccache for all build jobs. See configure_ci_environment.sh for more ccache related settings. IDF_CCACHE_ENABLE: "1" + # Since the version 3.21 CMake passes source files and include dirs to ninja using absolute paths. + # Needed for pytest junit reports. + INSTALL_EXTRA_TOOLS: cmake needs: - pipeline: $PARENT_PIPELINE_ID job: generate_build_child_pipeline @@ -52,6 +55,7 @@ PYTEST_NODES: "" TARGET_SELECTOR: "" ENV_MARKERS: "" + INSTALL_EXTRA_TOOLS: "xtensa-esp-elf-gdb riscv32-esp-elf-gdb openocd-esp32 esp-rom-elfs" PYTEST_EXTRA_FLAGS: "--dev-passwd ${ETHERNET_TEST_PASSWORD} --dev-user ${ETHERNET_TEST_USER} --capture=fd --verbosity=0" cache: # Usually do not need submodule-cache in target_test @@ -71,10 +75,10 @@ expire_in: 1 week script: # get known failure cases - - python tools/ci/get_known_failure_cases_file.py + - run_cmd python tools/ci/get_known_failure_cases_file.py # get runner env config file - retry_failed git clone $TEST_ENV_CONFIG_REPO - - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs + - run_cmd python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs # CI specific options start from "--known-failure-cases-file xxx". could ignore when running locally - run_cmd pytest ${PYTEST_NODES} --target ${TARGET_SELECTOR} @@ -86,4 +90,7 @@ --parallel-index ${CI_NODE_INDEX:-1} ${PYTEST_EXTRA_FLAGS} after_script: - - python tools/ci/artifacts_handler.py upload --type logs junit_reports + - source tools/ci/utils.sh + - section_start "upload_junit_reports" + - run_cmd python tools/ci/artifacts_handler.py upload --type logs junit_reports + - section_end "upload_junit_reports" diff --git a/tools/ci/utils.sh b/tools/ci/utils.sh index f860fef110..17951e2173 100644 --- a/tools/ci/utils.sh +++ b/tools/ci/utils.sh @@ -35,7 +35,9 @@ function add_doc_server_ssh_keys() { } function fetch_submodules() { + section_start "fetch_submodules" "Fetching submodules..." python "${SUBMODULE_FETCH_TOOL}" -s "${SUBMODULES_TO_FETCH}" + section_end "fetch_submodules" } function get_all_submodules() { @@ -49,6 +51,19 @@ function set_component_ut_vars() { echo "exported variables COMPONENT_UT_DIRS, COMPONENT_UT_EXCLUDES" } +# https://docs.gitlab.com/ee/ci/yaml/script.html#use-a-script-to-improve-display-of-collapsible-sections +function section_start() { + local section_title="${1}" + local section_description="${2:-$section_title}" + + echo -e "section_start:`date +%s`:${section_title}[collapsed=true]\r\e[0K${section_description}" +} +function section_end() { + local section_title="${1}" + + echo -e "section_end:`date +%s`:${section_title}\r\e[0K" +} + function error() { printf "\033[0;31m%s\n\033[0m" "${1}" >&2 }