diff --git a/.gitlab/ci/build.yml b/.gitlab/ci/build.yml index db483fc73c..187dafcce2 100644 --- a/.gitlab/ci/build.yml +++ b/.gitlab/ci/build.yml @@ -4,7 +4,6 @@ tags: - build variables: - SIZE_INFO_LOCATION: "$CI_PROJECT_DIR/size_info.txt" # Enable ccache for all build jobs. See configure_ci_environment.sh for more ccache related settings. IDF_CCACHE_ENABLE: "1" after_script: @@ -12,36 +11,104 @@ - test "$CI_CCACHE_STATS" == 1 && test -n "$(which ccache)" && ccache --show-stats || true dependencies: [] -.build_pytest_template: +.build_cmake_template: extends: - .build_template - .before_script_build_jobs - dependencies: # set dependencies to null to avoid missing artifacts issue + dependencies: # set dependencies to null to avoid missing artifacts issue needs: - job: fast_template_app artifacts: false + tags: + - build + # build only on shiny servers since shiny storage server is at the same location + - shiny artifacts: paths: - "**/build*/size.json" - "**/build*/build_log.txt" - "**/build*/*.bin" + # upload to s3 server to save the artifacts size + # - "**/build*/*.map" + # ttfw tests require elf files - "**/build*/*.elf" - - "**/build*/*.map" + - "**/build*/flasher_args.json" + - "**/build*/flash_project_args" + - "**/build*/config/sdkconfig.json" + # ttfw tests require sdkconfig file + - "**/build*/sdkconfig" + - "**/build*/bootloader/*.bin" + - "**/build*/partition_table/*.bin" + - list_job_*.json + - size_info.txt + # unit test specific + - components/idf_test/unit_test/*.yml + when: always + expire_in: 4 days + after_script: + # Show ccache statistics if enabled globally + - test "$CI_CCACHE_STATS" == 1 && test -n "$(which ccache)" && ccache --show-stats || true + # upload the binary files to s3 server + - echo -e "\e[0Ksection_start:`date +%s`:upload_binaries_to_s3_server[collapsed=true]\r\e[0KUploading binaries to s3 Server" + - shopt -s globstar + # use || true to bypass the no-file error + - zip ${CI_JOB_ID}.zip **/build*/*.bin || true + - zip ${CI_JOB_ID}.zip **/build*/*.elf || true + - zip ${CI_JOB_ID}.zip **/build*/*.map || true + - zip ${CI_JOB_ID}.zip **/build*/flasher_args.json || true + - zip ${CI_JOB_ID}.zip **/build*/flash_project_args || true + - zip ${CI_JOB_ID}.zip **/build*/config/sdkconfig.json || true + - zip ${CI_JOB_ID}.zip **/build*/sdkconfig || true + - zip ${CI_JOB_ID}.zip **/build*/bootloader/*.bin || true + - zip ${CI_JOB_ID}.zip **/build*/partition_table/*.bin || true + - mc alias set shiny-s3 ${SHINY_S3_SERVER} ${SHINY_S3_ACCESS_KEY} ${SHINY_S3_SECRET_KEY} + - mc cp ${CI_JOB_ID}.zip shiny-s3/idf-artifacts/${CI_PIPELINE_ID}/${CI_JOB_ID}.zip + - echo -e "\e[0Ksection_end:`date +%s`:upload_binaries_to_s3_server\r\e[0K" + - echo "Please download the full binary files (including *.elf and *.map files) from the following share link" + # would be clean up after 4 days + - mc share download shiny-s3/idf-artifacts/${CI_PIPELINE_ID}/${CI_JOB_ID}.zip --expire=96h + script: + # CI specific options start from "--collect-size-info xxx". could ignore when running locally + - run_cmd python tools/ci/ci_build_apps.py $TEST_DIR -v + -t $IDF_TARGET + --copy-sdkconfig + --collect-size-info size_info.txt + --collect-app-info list_job_${CI_NODE_INDEX:-1}.json + --parallel-count ${CI_NODE_TOTAL:-1} + --parallel-index ${CI_NODE_INDEX:-1} + --extra-preserve-dirs + examples/bluetooth/esp_ble_mesh/ble_mesh_console + examples/bluetooth/hci/controller_hci_uart_esp32 + examples/wifi/iperf + +.build_pytest_template: + extends: + - .build_cmake_template + - .before_script_build_jobs + artifacts: + paths: + - "**/build*/size.json" + - "**/build*/build_log.txt" + - "**/build*/*.bin" + # upload to s3 server to save the artifacts size + # - "**/build*/*.map" + # - "**/build*/*.elf" - "**/build*/flasher_args.json" - "**/build*/flash_project_args" - "**/build*/config/sdkconfig.json" - "**/build*/bootloader/*.bin" - "**/build*/bootloader/*.elf" - "**/build*/partition_table/*.bin" - - $SIZE_INFO_LOCATION + - list_job_*.json + - size_info.txt when: always - expire_in: 3 days + expire_in: 4 days script: # CI specific options start from "--collect-size-info xxx". could ignore when running locally - run_cmd python tools/ci/ci_build_apps.py $TEST_DIR -v -t $IDF_TARGET --pytest-apps - --collect-size-info $SIZE_INFO_LOCATION + --collect-size-info size_info.txt --parallel-count ${CI_NODE_TOTAL:-1} --parallel-index ${CI_NODE_INDEX:-1} @@ -49,7 +116,7 @@ build_pytest_examples_esp32: extends: - .build_pytest_template - .rules:build:example_test-esp32 - parallel: 3 + parallel: 4 variables: IDF_TARGET: esp32 TEST_DIR: examples @@ -67,7 +134,7 @@ build_pytest_examples_esp32s3: extends: - .build_pytest_template - .rules:build:example_test-esp32s3 - parallel: 3 + parallel: 4 variables: IDF_TARGET: esp32s3 TEST_DIR: examples @@ -102,6 +169,7 @@ build_pytest_components_esp32s2: extends: - .build_pytest_template - .rules:build:component_ut-esp32s2 + parallel: 2 variables: IDF_TARGET: esp32s2 TEST_DIR: components @@ -110,6 +178,7 @@ build_pytest_components_esp32s3: extends: - .build_pytest_template - .rules:build:component_ut-esp32s3 + parallel: 2 variables: IDF_TARGET: esp32s3 TEST_DIR: components @@ -118,6 +187,7 @@ build_pytest_components_esp32c3: extends: - .build_pytest_template - .rules:build:component_ut-esp32c3 + parallel: 2 variables: IDF_TARGET: esp32c3 TEST_DIR: components @@ -139,14 +209,35 @@ build_non_test_components_apps: # CI specific options start from "--collect-size-info xxx". could ignore when running locally - run_cmd python tools/ci/ci_build_apps.py $COMPONENT_UT_DIRS -v -t all - --collect-size-info $SIZE_INFO_LOCATION + --collect-size-info size_info.txt --collect-app-info list_job_${CI_NODE_INDEX:-1}.json --parallel-count ${CI_NODE_TOTAL:-1} --parallel-index ${CI_NODE_INDEX:-1} +.build_pytest_test_apps_template: + extends: .build_pytest_template + artifacts: + paths: + - "**/build*/size.json" + - "**/build*/build_log.txt" + - "**/build*/*.bin" + # upload to s3 server to save the artifacts size + # - "**/build*/*.map" + # pytest test apps requires elf files for coredump tests + - "**/build*/*.elf" + - "**/build*/flasher_args.json" + - "**/build*/flash_project_args" + - "**/build*/config/sdkconfig.json" + - "**/build*/bootloader/*.bin" + - "**/build*/partition_table/*.bin" + - list_job_*.json + - size_info.txt + when: always + expire_in: 4 days + build_pytest_test_apps_esp32: extends: - - .build_pytest_template + - .build_pytest_test_apps_template - .rules:build:custom_test-esp32 variables: IDF_TARGET: esp32 @@ -154,7 +245,7 @@ build_pytest_test_apps_esp32: build_pytest_test_apps_esp32s2: extends: - - .build_pytest_template + - .build_pytest_test_apps_template - .rules:build:custom_test-esp32s2 variables: IDF_TARGET: esp32s2 @@ -162,7 +253,7 @@ build_pytest_test_apps_esp32s2: build_pytest_test_apps_esp32s3: extends: - - .build_pytest_template + - .build_pytest_test_apps_template - .rules:build:custom_test-esp32s3 variables: IDF_TARGET: esp32s3 @@ -170,7 +261,7 @@ build_pytest_test_apps_esp32s3: build_pytest_test_apps_esp32c3: extends: - - .build_pytest_template + - .build_pytest_test_apps_template - .rules:build:custom_test-esp32c3 variables: IDF_TARGET: esp32c3 @@ -178,7 +269,7 @@ build_pytest_test_apps_esp32c3: build_pytest_test_apps_esp32c2: extends: - - .build_pytest_template + - .build_pytest_test_apps_template - .rules:build:custom_test-esp32c2 variables: IDF_TARGET: esp32c2 @@ -269,19 +360,17 @@ build_ssc_esp32s3: .build_esp_idf_tests_cmake_template: extends: - - .build_template + - .build_cmake_template - .before_script_build_jobs - dependencies: # set dependencies to null to avoid missing artifacts issue - needs: - - job: fast_template_app - artifacts: false artifacts: paths: - "**/build*/size.json" - "**/build*/build_log.txt" - "**/build*/*.bin" + # upload to s3 server to save the artifacts size + # - "**/build*/*.map" + # ttfw tests require elf files - "**/build*/*.elf" - - "**/build*/*.map" - "**/build*/flasher_args.json" - "**/build*/flash_project_args" - "**/build*/config/sdkconfig.json" @@ -289,12 +378,10 @@ build_ssc_esp32s3: - "**/build*/bootloader/*.bin" - "**/build*/partition_table/*.bin" - list_job_*.json - - $SIZE_INFO_LOCATION + - size_info.txt - components/idf_test/unit_test/*.yml when: always expire_in: 4 days - variables: - LDGEN_CHECK_MAPPING: 1 script: # CI specific options start from "--collect-size-info xxx". could ignore when running locally - run_cmd python tools/ci/ci_build_apps.py tools/unit-test-app -v @@ -302,7 +389,7 @@ build_ssc_esp32s3: --config "configs/*=" --copy-sdkconfig --preserve-all - --collect-size-info $SIZE_INFO_LOCATION + --collect-size-info size_info.txt --collect-app-info list_job_${CI_NODE_INDEX:-1}.json --parallel-count ${CI_NODE_TOTAL:-1} --parallel-index ${CI_NODE_INDEX:-1} @@ -345,52 +432,11 @@ build_esp_idf_tests_cmake_esp32c3: variables: IDF_TARGET: esp32c3 -.build_cmake_template: - extends: - - .build_template - - .before_script_build_jobs - dependencies: # set dependencies to null to avoid missing artifacts issue - needs: - - job: fast_template_app - artifacts: false - variables: - LDGEN_CHECK_MAPPING: 1 - artifacts: - paths: - - "**/build*/size.json" - - "**/build*/build_log.txt" - - "**/build*/*.bin" - - "**/build*/*.elf" - - "**/build*/*.map" - - "**/build*/flasher_args.json" - - "**/build*/flash_project_args" - - "**/build*/config/sdkconfig.json" - - "**/build*/sdkconfig" - - "**/build*/bootloader/*.bin" - - "**/build*/partition_table/*.bin" - - list_job_*.json - - $SIZE_INFO_LOCATION - when: always - expire_in: 4 days - script: - # CI specific options start from "--collect-size-info xxx". could ignore when running locally - - run_cmd python tools/ci/ci_build_apps.py $TEST_DIR -v - -t $IDF_TARGET - --copy-sdkconfig - --collect-size-info $SIZE_INFO_LOCATION - --collect-app-info list_job_${CI_NODE_INDEX:-1}.json - --parallel-count ${CI_NODE_TOTAL:-1} - --parallel-index ${CI_NODE_INDEX:-1} - --extra-preserve-dirs - examples/bluetooth/esp_ble_mesh/ble_mesh_console - examples/bluetooth/hci/controller_hci_uart_esp32 - examples/wifi/iperf - build_examples_cmake_esp32: extends: - .build_cmake_template - .rules:build:example_test-esp32 - parallel: 12 + parallel: 9 variables: IDF_TARGET: esp32 TEST_DIR: examples @@ -399,7 +445,7 @@ build_examples_cmake_esp32s2: extends: - .build_cmake_template - .rules:build:example_test-esp32s2 - parallel: 8 + parallel: 6 variables: IDF_TARGET: esp32s2 TEST_DIR: examples @@ -408,7 +454,7 @@ build_examples_cmake_esp32s3: extends: - .build_cmake_template - .rules:build:example_test-esp32s3 - parallel: 8 + parallel: 7 variables: IDF_TARGET: esp32s3 TEST_DIR: examples @@ -417,7 +463,7 @@ build_examples_cmake_esp32c2: extends: - .build_cmake_template - .rules:build:example_test-esp32c2 - parallel: 8 + parallel: 4 variables: IDF_TARGET: esp32c2 TEST_DIR: examples @@ -426,7 +472,7 @@ build_examples_cmake_esp32c3: extends: - .build_cmake_template - .rules:build:example_test-esp32c3 - parallel: 8 + parallel: 6 variables: IDF_TARGET: esp32c3 TEST_DIR: examples diff --git a/.gitlab/ci/pre_check.yml b/.gitlab/ci/pre_check.yml index dd0180d60f..1c0cfa1c1d 100644 --- a/.gitlab/ci/pre_check.yml +++ b/.gitlab/ci/pre_check.yml @@ -163,4 +163,6 @@ check_test_scripts_build_test_rules: - .pre_check_template - .before_script_build_jobs script: + # required pytest related packages + - run_cmd bash install.sh --enable-pytest - python tools/ci/check_build_test_rules.py check-test-scripts examples/ tools/test_apps components diff --git a/components/esp_pm/linker.lf b/components/esp_pm/linker.lf index 839459abb6..6ef1e90e06 100644 --- a/components/esp_pm/linker.lf +++ b/components/esp_pm/linker.lf @@ -21,8 +21,9 @@ entries: esp_clk:esp_rtc_get_time_us (noflash) if GPIO_ESP32_SUPPORT_SWITCH_SLP_PULL = y: sleep_gpio:gpio_sleep_mode_config_apply (noflash) - if IDF_TARGET_ESP32 = n && IDF_TARGET_ESP32S2 = n: + if SOC_PM_SUPPORT_CPU_PD = y || SOC_PM_SUPPORT_TAGMEM_PD = y: sleep_retention:sleep_enable_memory_retention (noflash) + if SOC_PM_SUPPORT_CPU_PD = y: sleep_retention:cpu_domain_pd_allowed (noflash) [mapping:esp_system_pm] diff --git a/components/hal/linker.lf b/components/hal/linker.lf index 9b23d5a562..c8178e04eb 100644 --- a/components/hal/linker.lf +++ b/components/hal/linker.lf @@ -30,7 +30,8 @@ entries: lcd_hal: lcd_hal_cal_pclk_freq (noflash) if ADC_ONESHOT_CTRL_FUNC_IN_IRAM = y: adc_oneshot_hal (noflash) - adc_hal_common: get_controller (noflash) + if COMPILER_OPTIMIZATION_DEFAULT = y: + adc_hal_common: get_controller (noflash) adc_hal_common: adc_hal_set_controller (noflash) if SOC_ADC_ARBITER_SUPPORTED = y: adc_hal_common: adc_hal_arbiter_config (noflash) @@ -40,7 +41,8 @@ entries: if ADC_CONTINUOUS_ISR_IRAM_SAFE = y: adc_hal: adc_hal_get_reading_result (noflash) adc_hal: adc_hal_digi_start (noflash) - adc_hal: adc_hal_digi_dma_link_descriptors (noflash) + if COMPILER_OPTIMIZATION_DEFAULT = y: + adc_hal: adc_hal_digi_dma_link_descriptors (noflash) adc_hal: adc_hal_digi_stop (noflash) if IDF_TARGET_ESP32 = y || IDF_TARGET_ESP32S2 = y: adc_hal: adc_hal_check_event (noflash) diff --git a/examples/get-started/blink/pytest_blink.py b/examples/get-started/blink/pytest_blink.py index 47d3aaccbd..b7ea802c3b 100644 --- a/examples/get-started/blink/pytest_blink.py +++ b/examples/get-started/blink/pytest_blink.py @@ -1,7 +1,6 @@ # SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: CC0-1.0 -import hashlib import logging import os @@ -9,21 +8,6 @@ import pytest from pytest_embedded_idf.dut import IdfDut -def verify_elf_sha256_embedding(bin_path: str, sha256_reported: str) -> None: - elf_file = os.path.join(bin_path, 'blink.elf') - sha256 = hashlib.sha256() - with open(elf_file, 'rb') as f: - sha256.update(f.read()) - sha256_expected = sha256.hexdigest() - - logging.info(f'ELF file SHA256: {sha256_expected}') - logging.info(f'ELF file SHA256 (reported by the app): {sha256_reported}') - - # the app reports only the first several hex characters of the SHA256, check that they match - if not sha256_expected.startswith(sha256_reported): - raise ValueError('ELF file SHA256 mismatch') - - @pytest.mark.supported_targets @pytest.mark.generic def test_blink(dut: IdfDut) -> None: @@ -31,7 +15,3 @@ def test_blink(dut: IdfDut) -> None: binary_file = os.path.join(dut.app.binary_path, 'blink.bin') bin_size = os.path.getsize(binary_file) logging.info('blink_bin_size : {}KB'.format(bin_size // 1024)) - - sha256_reported = dut.expect(r'ELF file SHA256:\s+([a-f0-9]+)').group(1).decode('utf-8') - - verify_elf_sha256_embedding(dut.app.binary_path, sha256_reported) diff --git a/examples/get-started/hello_world/pytest_hello_world.py b/examples/get-started/hello_world/pytest_hello_world.py index 3cb161c774..7a55984ab1 100644 --- a/examples/get-started/hello_world/pytest_hello_world.py +++ b/examples/get-started/hello_world/pytest_hello_world.py @@ -1,22 +1,46 @@ # SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: CC0-1.0 +import hashlib +import logging from typing import Callable import pytest from pytest_embedded_idf.dut import IdfDut +from pytest_embedded_qemu.app import QemuApp from pytest_embedded_qemu.dut import QemuDut @pytest.mark.supported_targets @pytest.mark.generic -def test_hello_world(dut: IdfDut, log_minimum_free_heap_size: Callable[..., None]) -> None: +def test_hello_world( + dut: IdfDut, log_minimum_free_heap_size: Callable[..., None] +) -> None: dut.expect('Hello world!') log_minimum_free_heap_size() +def verify_elf_sha256_embedding(app: QemuApp, sha256_reported: str) -> None: + sha256 = hashlib.sha256() + with open(app.elf_file, 'rb') as f: + sha256.update(f.read()) + sha256_expected = sha256.hexdigest() + + logging.info(f'ELF file SHA256: {sha256_expected}') + logging.info(f'ELF file SHA256 (reported by the app): {sha256_reported}') + + # the app reports only the first several hex characters of the SHA256, check that they match + if not sha256_expected.startswith(sha256_reported): + raise ValueError('ELF file SHA256 mismatch') + + @pytest.mark.esp32 # we only support qemu on esp32 for now @pytest.mark.host_test @pytest.mark.qemu -def test_hello_world_host(dut: QemuDut) -> None: +def test_hello_world_host(app: QemuApp, dut: QemuDut) -> None: + sha256_reported = ( + dut.expect(r'ELF file SHA256:\s+([a-f0-9]+)').group(1).decode('utf-8') + ) + verify_elf_sha256_embedding(app, sha256_reported) + dut.expect('Hello world!') diff --git a/tools/ci/configure_ci_environment.sh b/tools/ci/configure_ci_environment.sh index 512f26e526..417663742f 100644 --- a/tools/ci/configure_ci_environment.sh +++ b/tools/ci/configure_ci_environment.sh @@ -44,3 +44,5 @@ if [ "${CI_CCACHE_DISABLE_SECONDARY}" != "1" ] && [ -n "${REDIS_CACHE}" ]; then export CCACHE_SECONDARY_STORAGE="redis://${REDIS_CACHE}" echo "INFO: Using CCACHE_SECONDARY_STORAGE=${CCACHE_SECONDARY_STORAGE}" fi + +export LDGEN_CHECK_MAPPING="1"