fix: make get_adjusted_boot_time() not static

fix(nimble): Support for reattempt connection in ext connect

See merge request espressif/esp-idf!28222

.github/ISSUE_TEMPLATE/02_runtime_bug.yml

@@ -22,6 +22,14 @@ body:
       placeholder: ex. v3.2-dev-1148-g96cd3b75c
     validations:
       required: true
+  - type: input
+    id: chip_revision
+    attributes:
+      label: Espressif SoC revision.
+      description: On which Espressif SoC revision does your application run on? Run `esptool chip_id` to find it.
+      placeholder: ex. ESP32-C3 (QFN32) (revision v0.3)
+    validations:
+      required: true
   - type: dropdown
     id: operating_system
     attributes:
@@ -63,7 +71,7 @@ body:
     attributes:
       label: Development Kit.
       description: On which Development Kit does this issue occur on?
-      placeholder: ex. ESP32-Wrover-Kit v2 | Custom Board
+      placeholder: ex. ESP32-Wrover-Kit v2 | Custom Board | QEMU
     validations:
       required: true
   - type: dropdown

.github/workflows/dangerjs.yml

@@ -0,0 +1,27 @@
+name: DangerJS Pull Request review
+
+on:
+  pull_request_target:
+    types: [opened, edited, reopened, synchronize]
+
+permissions:
+  pull-requests: write
+  contents: write
+
+jobs:
+  pull-request-style-linter:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Check out PR head
+      uses: actions/checkout@v4
+      with:
+        ref: ${{ github.event.pull_request.head.sha }}
+
+    - name: DangerJS pull request linter
+      uses: espressif/shared-github-dangerjs@v1
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        instructions-gitlab-mirror: 'true'
+        instructions-contributions-file: 'CONTRIBUTING.md'
+        instructions-cla-link: 'https://cla-assistant.io/espressif/esp-idf'

.github/workflows/issue_comment.yml

@@ -12,7 +12,7 @@ jobs:
     name: Sync Issue Comments to Jira
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Sync issue comments to JIRA
         uses: espressif/github-actions/sync_issues_to_jira@master
         env:

.github/workflows/new_issues.yml

@@ -12,7 +12,7 @@ jobs:
     name: Sync issues to Jira
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Sync GitHub issues to Jira project
         uses: espressif/github-actions/sync_issues_to_jira@master
         env:

.github/workflows/new_prs.yml

@@ -15,7 +15,7 @@ jobs:
     name: Sync PRs to Jira
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Sync PRs to Jira project
         uses: espressif/github-actions/sync_issues_to_jira@master
         with:

.github/workflows/pr_approved.yml

@@ -11,7 +11,7 @@ jobs:
         (github.event.label.name == 'PR-Sync-Rebase')  ||
         (github.event.label.name == 'PR-Sync-Update')
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Sync approved PRs to internal codebase
         uses: espressif/github-actions/github_pr_to_internal_pr@master
         env:

.github/workflows/pre_commit_check.yml

@@ -10,9 +10,11 @@ permissions:
 jobs:
   pre_commit_check:
     runs-on: ubuntu-latest
+    env:
+      SKIP: "cleanup-ignore-lists"  # Comma-separated string of ignored pre-commit check IDs
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
       - name: Fetch head and base refs
         # This is necessary for pre-commit to check the changes in the PR branch
         run: |
@@ -21,7 +23,7 @@ jobs:
       - name: Set up Python environment
         uses: actions/setup-python@master
         with:
-          python-version: v3.7
+          python-version: v3.8
       - name: Install python packages
         run: |
           pip install pre-commit

.github/workflows/vulnerability_scan.yml

@@ -0,0 +1,34 @@
+name: Vulnerability scan
+
+on:
+  schedule:
+    - cron: '0 0 * * *'
+  workflow_dispatch:
+
+jobs:
+  vulnerability-scan:
+    strategy:
+      # We don't want to run all jobs in parallel, because this would
+      # overload NVD and we would get 503
+      max-parallel: 1
+      matrix:
+        # References/branches which should be scanned for vulnerabilities are
+        # defined in the VULNERABILITY_SCAN_REFS variable as json list.
+        # For example: ['master', 'release/v5.2', 'release/v5.1', 'release/v5.0', 'release/v4.4']
+        ref: ${{ fromJSON(vars.VULNERABILITY_SCAN_REFS) }}
+    name: Vulnerability scan
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          submodules: recursive
+          ref: ${{ matrix.ref }}
+
+      - name: Vulnerability scan
+        env:
+          SBOM_MATTERMOST_WEBHOOK: ${{ secrets.SBOM_MATTERMOST_WEBHOOK }}
+          NVDAPIKEY: ${{ secrets.NVDAPIKEY }}
+        uses: espressif/esp-idf-sbom-action@master
+        with:
+          ref: ${{ matrix.ref }}

.gitignore

@@ -78,6 +78,10 @@ test_multi_heap_host
 *.swp
 *.swo
 
+# Sublime Text files
+*.sublime-project
+*.sublime-workspace
+
 # Clion IDE CMake build & config
 .idea/
 cmake-build-*/
@@ -97,3 +101,11 @@ managed_components
 
 # pytest log
 pytest_embedded_log/
+list_job_*.txt
+size_info.txt
+
+# clang config (for LSP)
+.clangd
+
+# Vale
+.vale/styles/*

.gitlab-ci.yml

@@ -1,241 +1,32 @@
-stages:
-  - upload_cache
-  - pre_check
-  - build
-  - assign_test
-  - build_doc
-  - target_test
-  - host_test
-  - test_deploy
-  - deploy
-  - post_deploy
-
 workflow:
   rules:
     # Disable those non-protected push triggered pipelines
-    - if: '$CI_COMMIT_REF_NAME != "master" && $CI_COMMIT_BRANCH !~ /^release\/v/ && $CI_COMMIT_TAG !~ /^v\d+\.\d+(\.\d+)?($|-)/ && $CI_PIPELINE_SOURCE == "push"'
+    - if: '$CI_COMMIT_REF_NAME != "master" && $CI_COMMIT_BRANCH !~ /^release\/v/ && $CI_COMMIT_TAG !~ /^v\d+\.\d+(\.\d+)?($|-)/ && $CI_COMMIT_TAG !~ /^qa-test/ && $CI_PIPELINE_SOURCE == "push"'
       when: never
-    # when running merged result pipelines, it would create a temp commit id. use $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA instead of $CI_COMMIT_SHA.
-    # Please use PIPELINE_COMMIT_SHA at all places that require a commit sha
+    # when running merged result pipelines, CI_COMMIT_SHA represents the temp commit it created.
+    # Please use PIPELINE_COMMIT_SHA at all places that require a commit sha of the original commit.
     - if: $CI_OPEN_MERGE_REQUESTS != null
       variables:
         PIPELINE_COMMIT_SHA: $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA
+        IS_MR_PIPELINE: 1
     - if: $CI_OPEN_MERGE_REQUESTS == null
       variables:
         PIPELINE_COMMIT_SHA: $CI_COMMIT_SHA
+        IS_MR_PIPELINE: 0
     - when: always
 
-variables:
-# System environment
-
-  # Common parameters for the 'make' during CI tests
-  MAKEFLAGS: "-j5 --no-keep-going"
-
-# GitLab-CI environment
-
-  # XXX_ATTEMPTS variables (https://docs.gitlab.com/ce/ci/yaml/README.html#job-stages-attempts) are not defined here.
-  # Use values from  "CI / CD Settings" - "Variables".
-
-  # GIT_STRATEGY is not defined here.
-  # Use an option from  "CI / CD Settings" - "General pipelines".
-
-  # we will download archive for each submodule instead of clone.
-  # we don't do "recursive" when fetch submodule as they're not used in CI now.
-  GIT_SUBMODULE_STRATEGY: none
-  SUBMODULE_FETCH_TOOL: "tools/ci/ci_fetch_submodule.py"
-  # by default we will fetch all submodules
-  # jobs can overwrite this variable to only fetch submodules they required
-  # set to "none" if don't need to fetch submodules
-  SUBMODULES_TO_FETCH: "all"
-  # tell build system do not check submodule update as we download archive instead of clone
-  IDF_SKIP_CHECK_SUBMODULES: 1
-
-  IDF_PATH: "$CI_PROJECT_DIR"
-  BATCH_BUILD: "1"
-  V: "0"
-  CHECKOUT_REF_SCRIPT: "$CI_PROJECT_DIR/tools/ci/checkout_project_ref.py"
-  PYTHON_VER: 3.7.10
-
-  CLANG_TIDY_RUNNER_PROJ: 2107  # idf/clang-tidy-runner
-  IDF_BUILD_APPS_PROJ: 2818  # espressif/idf-build-apps
-
-  # Docker images
-  BOT_DOCKER_IMAGE_TAG: ":latest"
-
-  ESP_ENV_IMAGE: "$CI_DOCKER_REGISTRY/esp-env-v5.1:1"
-  CLANG_STATIC_ANALYSIS_IMAGE: "${CI_DOCKER_REGISTRY}/clang-static-analysis-v5.1:1-1"
-  ESP_IDF_DOC_ENV_IMAGE: "$CI_DOCKER_REGISTRY/esp-idf-doc-env-v5.1:1-1"
-  QEMU_IMAGE: "${CI_DOCKER_REGISTRY}/qemu-v5.1:1-20220802"
-  TARGET_TEST_ENV_IMAGE: "$CI_DOCKER_REGISTRY/target-test-env-v5.1:1"
-
-  SONARQUBE_SCANNER_IMAGE: "${CI_DOCKER_REGISTRY}/sonarqube-scanner:3"
-
-  PRE_COMMIT_IMAGE: "$CI_DOCKER_REGISTRY/esp-idf-pre-commit:1"
-
-  # target test config file, used by assign test job
-  CI_TARGET_TEST_CONFIG_FILE: "$CI_PROJECT_DIR/.gitlab/ci/target-test.yml"
-
-  # target test repo parameters
-  TEST_ENV_CONFIG_REPO: "https://gitlab-ci-token:${BOT_TOKEN}@${CI_SERVER_HOST}:${CI_SERVER_PORT}/qa/ci-test-runner-configs.git"
-
-  # cache python dependencies
-  PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
-
-  # Set this variable to the branch of idf-constraints repo in order to test a custom Python constraint file. The
-  # branch name must be without the remote part ("origin/"). Keep the variable empty in order to use the constraint
-  # file from https://dl.espressif.com/dl/esp-idf.
-  CI_PYTHON_CONSTRAINT_BRANCH: ""
-
-  # Update the filename for a specific ESP-IDF release. It is used only with CI_PYTHON_CONSTRAINT_BRANCH.
-  CI_PYTHON_CONSTRAINT_FILE: "espidf.constraints.v5.1.txt"
-
-  # Set this variable to repository name of a Python tool you wish to install and test in the context of ESP-IDF CI.
-  # Keep the variable empty when not used.
-  CI_PYTHON_TOOL_REPO: ""
-
-  # Set this variable to the branch of a Python tool repo specified in CI_PYTHON_TOOL_REPO. The
-  # branch name must be without the remote part ("origin/"). Keep the variable empty when not used.
-  # This is used only if CI_PYTHON_TOOL_REPO is not empty.
-  CI_PYTHON_TOOL_BRANCH: ""
-
-  IDF_CI_BUILD: 1
-
-cache:
-  # pull only for most of the use cases since it's cache dir.
-  # Only set "push" policy for "upload_cache" stage jobs
-  - key: pip-cache
-    paths:
-      - .cache/pip
-    policy: pull
-  - key: submodule-cache
-    paths:
-      - .cache/submodule_archives
-    policy: pull
-
-.common_before_scripts: &common-before_scripts |
-  source tools/ci/utils.sh
-  is_based_on_commits $REQUIRED_ANCESTOR_COMMITS
-
-  if [[ -n "$IDF_DONT_USE_MIRRORS" ]]; then
-    export IDF_MIRROR_PREFIX_MAP=
-  fi
-
-  if echo "$CI_MERGE_REQUEST_LABELS" | egrep "^([^,\n\r]+,)*include_nightly_run(,[^,\n\r]+)*$"; then
-    export INCLUDE_NIGHTLY_RUN="1"
-  fi
-
-  # configure cmake related flags
-  source tools/ci/configure_ci_environment.sh
-
-  # add extra python packages
-  export PYTHONPATH="$IDF_PATH/tools:$IDF_PATH/tools/esp_app_trace:$IDF_PATH/components/partition_table:$IDF_PATH/tools/ci/python_packages:$PYTHONPATH"
-
-.setup_tools_and_idf_python_venv: &setup_tools_and_idf_python_venv |
-  # must use after setup_tools_except_target_test
-  # otherwise the export.sh won't work properly
-
-  # download constraint file for dev
-  if [[ -n "$CI_PYTHON_CONSTRAINT_BRANCH" ]]; then
-    wget -O /tmp/constraint.txt --header="Authorization:Bearer ${ESPCI_TOKEN}" ${GITLAB_HTTP_SERVER}/api/v4/projects/2581/repository/files/${CI_PYTHON_CONSTRAINT_FILE}/raw?ref=${CI_PYTHON_CONSTRAINT_BRANCH}
-    mkdir -p ~/.espressif
-    mv /tmp/constraint.txt ~/.espressif/${CI_PYTHON_CONSTRAINT_FILE}
-  fi
-
-  # Mirror
-  if [[ -n "$IDF_DONT_USE_MIRRORS" ]]; then
-    export IDF_MIRROR_PREFIX_MAP=
-  fi
-
-  # install latest python packages
-  # target test jobs
-  if [[ "${CI_JOB_STAGE}" == "target_test" ]]; then
-    # ttfw jobs
-    if ! echo "${CI_JOB_NAME}" | egrep ".*pytest.*"; then
-      run_cmd bash install.sh --enable-ci --enable-ttfw
-    else
-      run_cmd bash install.sh --enable-ci --enable-pytest
-    fi
-  elif [[ "${CI_JOB_STAGE}" == "build_doc" ]]; then
-    run_cmd bash install.sh --enable-ci --enable-docs
-  elif [[ "${CI_JOB_STAGE}" == "build" ]]; then
-    run_cmd bash install.sh --enable-ci --enable-pytest
-  else
-    if ! echo "${CI_JOB_NAME}" | egrep ".*pytest.*"; then
-      run_cmd bash install.sh --enable-ci
-    else
-      run_cmd bash install.sh --enable-ci --enable-pytest
-    fi
-  fi
-
-  # Install esp-clang if necessary
-  if [[ "$IDF_TOOLCHAIN" == "clang" ]]; then
-    $IDF_PATH/tools/idf_tools.py --non-interactive install esp-clang
-  fi
-
-  source ./export.sh
-
-  # Custom OpenOCD
-  if [[ ! -z "$OOCD_DISTRO_URL" && "$CI_JOB_STAGE" == "target_test" ]]; then
-    echo "Using custom OpenOCD from ${OOCD_DISTRO_URL}"
-    wget $OOCD_DISTRO_URL
-    ARCH_NAME=$(basename $OOCD_DISTRO_URL)
-    tar -x -f $ARCH_NAME
-    export OPENOCD_SCRIPTS=$PWD/openocd-esp32/share/openocd/scripts
-    export PATH=$PWD/openocd-esp32/bin:$PATH
-  fi
-
-  if [[ -n "$CI_PYTHON_TOOL_REPO" ]]; then
-    git clone --quiet --depth=1 -b ${CI_PYTHON_TOOL_BRANCH} https://gitlab-ci-token:${ESPCI_TOKEN}@${GITLAB_HTTPS_HOST}/espressif/${CI_PYTHON_TOOL_REPO}.git
-    pip install ./${CI_PYTHON_TOOL_REPO}
-    rm -rf ${CI_PYTHON_TOOL_REPO}
-  fi
-
-before_script:
-  - *common-before_scripts
-  - *setup_tools_and_idf_python_venv
-  - add_gitlab_ssh_keys
-  - fetch_submodules
-
-.before_script_minimal:
-  before_script:
-    - *common-before_scripts
-
-.before_script_macos:
-  before_script:
-    - *common-before_scripts
-    # On macOS, these tools need to be installed
-    - export IDF_TOOLS_PATH="${HOME}/.espressif_runner_${CI_RUNNER_ID}_${CI_CONCURRENT_ID}"
-    - $IDF_PATH/tools/idf_tools.py --non-interactive install cmake ninja
-    # This adds tools (compilers) and the version-specific Python environment to PATH
-    - *setup_tools_and_idf_python_venv
-    - fetch_submodules
-
-.before_script_build_jobs:
-  before_script:
-    - *common-before_scripts
-    - *setup_tools_and_idf_python_venv
-    - add_gitlab_ssh_keys
-    - fetch_submodules
-    - export EXTRA_CFLAGS=${PEDANTIC_CFLAGS}
-    - export EXTRA_CXXFLAGS=${PEDANTIC_CXXFLAGS}
-
-default:
-  retry:
-    max: 2
-    when:
-      # In case of a runner failure we could hop to another one, or a network error could go away.
-      - runner_system_failure
-      # Job execution timeout may be caused by a network issue.
-      - job_execution_timeout
+# Place the default settings in `.gitlab/ci/common.yml` instead
 
 include:
+  - '.gitlab/ci/danger.yml'
+  - '.gitlab/ci/common.yml'
   - '.gitlab/ci/rules.yml'
   - '.gitlab/ci/upload_cache.yml'
   - '.gitlab/ci/docs.yml'
   - '.gitlab/ci/static-code-analysis.yml'
   - '.gitlab/ci/pre_check.yml'
   - '.gitlab/ci/build.yml'
-  - '.gitlab/ci/assign-test.yml'
+  - '.gitlab/ci/integration_test.yml'
   - '.gitlab/ci/host-test.yml'
   - '.gitlab/ci/target-test.yml'
   - '.gitlab/ci/deploy.yml'

.gitlab/CODEOWNERS

@@ -50,10 +50,11 @@
 /.github/workflows/                   @esp-idf-codeowners/ci
 /.gitlab-ci.yml                       @esp-idf-codeowners/ci
 /.gitlab/ci/                          @esp-idf-codeowners/ci
-/.gitlab/dangerjs/                    @esp-idf-codeowners/ci @esp-idf-codeowners/tools
 /.pre-commit-config.yaml              @esp-idf-codeowners/ci
 /.readthedocs.yml                     @esp-idf-codeowners/docs
+/.vale.ini                            @esp-idf-codeowners/docs
 /CMakeLists.txt                       @esp-idf-codeowners/build-config
+/COMPATIBILITY*.md                    @esp-idf-codeowners/peripherals
 /CONTRIBUTING.md                      @esp-idf-codeowners/docs
 /Kconfig                              @esp-idf-codeowners/build-config
 /README*.md                           @esp-idf-codeowners/docs
@@ -68,7 +69,7 @@
 
 # sort-order-reset
 
-/components/app_trace/                @esp-idf-codeowners/tools
+/components/app_trace/                @esp-idf-codeowners/debugging
 /components/app_update/               @esp-idf-codeowners/system @esp-idf-codeowners/app-utilities
 /components/bootloader*/              @esp-idf-codeowners/system @esp-idf-codeowners/security
 /components/bootloader_support/bootloader_flash/    @esp-idf-codeowners/peripherals
@@ -80,11 +81,14 @@
 /components/efuse/                    @esp-idf-codeowners/system
 /components/esp_adc/                  @esp-idf-codeowners/peripherals
 /components/esp_app_format/           @esp-idf-codeowners/system @esp-idf-codeowners/app-utilities
+/components/esp_bootloader_format/    @esp-idf-codeowners/system @esp-idf-codeowners/app-utilities
 /components/esp_coex/                 @esp-idf-codeowners/wifi @esp-idf-codeowners/bluetooth @esp-idf-codeowners/ieee802154
 /components/esp_common/               @esp-idf-codeowners/system
+/components/esp_driver_*/             @esp-idf-codeowners/peripherals
+/components/esp_driver_sdmmc/         @esp-idf-codeowners/peripherals @esp-idf-codeowners/storage
 /components/esp_eth/                  @esp-idf-codeowners/network
 /components/esp_event/                @esp-idf-codeowners/system
-/components/esp_gdbstub/              @esp-idf-codeowners/tools
+/components/esp_gdbstub/              @esp-idf-codeowners/debugging
 /components/esp_hid/                  @esp-idf-codeowners/bluetooth
 /components/esp_http_client/          @esp-idf-codeowners/app-utilities
 /components/esp_http_server/          @esp-idf-codeowners/app-utilities
@@ -106,7 +110,7 @@
 /components/esp_timer/                @esp-idf-codeowners/system
 /components/esp-tls/                  @esp-idf-codeowners/app-utilities
 /components/esp_wifi/                 @esp-idf-codeowners/wifi
-/components/espcoredump/              @esp-idf-codeowners/tools
+/components/espcoredump/              @esp-idf-codeowners/debugging
 /components/esptool_py/               @esp-idf-codeowners/tools
 /components/fatfs/                    @esp-idf-codeowners/storage
 /components/fatfs/**/*.py             @esp-idf-codeowners/tools
@@ -122,11 +126,12 @@
 /components/lwip/                     @esp-idf-codeowners/lwip
 /components/mbedtls/                  @esp-idf-codeowners/app-utilities/mbedtls @esp-idf-codeowners/security
 /components/mqtt/                     @esp-idf-codeowners/network
-/components/newlib/                   @esp-idf-codeowners/system @esp-idf-codeowners/tools
+/components/newlib/                   @esp-idf-codeowners/system @esp-idf-codeowners/toolchain
 /components/nvs_flash/                @esp-idf-codeowners/storage
+/components/nvs_sec_provider/         @esp-idf-codeowners/storage @esp-idf-codeowners/security
 /components/openthread/               @esp-idf-codeowners/ieee802154
 /components/partition_table/          @esp-idf-codeowners/system
-/components/perfmon/                  @esp-idf-codeowners/tools
+/components/perfmon/                  @esp-idf-codeowners/debugging
 /components/protobuf-c/               @esp-idf-codeowners/app-utilities
 /components/protocomm/                @esp-idf-codeowners/app-utilities/provisioning
 /components/pthread/                  @esp-idf-codeowners/system
@@ -146,32 +151,35 @@
 /components/wpa_supplicant/           @esp-idf-codeowners/wifi @esp-idf-codeowners/app-utilities/mbedtls
 /components/xtensa/                   @esp-idf-codeowners/system
 
-/docs/                                  @esp-idf-codeowners/docs
-/docs/**/api-reference/bluetooth/       @esp-idf-codeowners/bluetooth
-/docs/**/api-reference/network/         @esp-idf-codeowners/network
-/docs/**/api-reference/peripherals/     @esp-idf-codeowners/peripherals
-/docs/**/api-reference/peripherals/usb* @esp-idf-codeowners/peripherals @esp-idf-codeowners/peripherals/usb
-/docs/**/api-reference/protocols/       @esp-idf-codeowners/network @esp-idf-codeowners/app-utilities
-/docs/**/api-reference/provisioning/    @esp-idf-codeowners/app-utilities/provisioning
-/docs/**/api-reference/storage/         @esp-idf-codeowners/storage
-/docs/**/api-reference/system/          @esp-idf-codeowners/system
-/docs/**/security/                      @esp-idf-codeowners/security
-/docs/**/migration-guides/              @esp-idf-codeowners/docs @esp-idf-codeowners/all-maintainers
+/docs/                                   @esp-idf-codeowners/docs
+/docs/en/api-guides/jtag-debugging/      @esp-idf-codeowners/debugging
+/docs/**/api-reference/bluetooth/        @esp-idf-codeowners/bluetooth
+/docs/**/api-reference/network/          @esp-idf-codeowners/network @esp-idf-codeowners/wifi
+/docs/**/api-reference/peripherals/      @esp-idf-codeowners/peripherals
+/docs/**/api-reference/peripherals/usb*  @esp-idf-codeowners/peripherals @esp-idf-codeowners/peripherals/usb
+/docs/**/api-reference/protocols/        @esp-idf-codeowners/network @esp-idf-codeowners/app-utilities
+/docs/**/api-reference/provisioning/     @esp-idf-codeowners/app-utilities/provisioning
+/docs/**/api-reference/storage/          @esp-idf-codeowners/storage
+/docs/**/api-reference/system/           @esp-idf-codeowners/system
+/docs/**/security/                       @esp-idf-codeowners/security
+/docs/**/migration-guides/               @esp-idf-codeowners/docs @esp-idf-codeowners/all-maintainers
 
 /examples/README.md           @esp-idf-codeowners/docs @esp-idf-codeowners/ci
 /examples/**/*.py             @esp-idf-codeowners/ci @esp-idf-codeowners/tools
 /examples/bluetooth/          @esp-idf-codeowners/bluetooth
 /examples/build_system/       @esp-idf-codeowners/build-config
-/examples/common_components/  @esp-idf-codeowners/system
+/examples/common_components/  @esp-idf-codeowners/system @esp-idf-codeowners/wifi @esp-idf-codeowners/lwip @esp-idf-codeowners/network
 /examples/custom_bootloader/  @esp-idf-codeowners/system
 /examples/cxx/                @esp-idf-codeowners/system
 /examples/ethernet/           @esp-idf-codeowners/network
 /examples/get-started/        @esp-idf-codeowners/system
+/examples/ieee802154/         @esp-idf-codeowners/ieee802154
 /examples/mesh/               @esp-idf-codeowners/wifi
 /examples/network/            @esp-idf-codeowners/network @esp-idf-codeowners/wifi
 /examples/openthread/         @esp-idf-codeowners/ieee802154
 /examples/peripherals/        @esp-idf-codeowners/peripherals
 /examples/peripherals/usb/    @esp-idf-codeowners/peripherals @esp-idf-codeowners/peripherals/usb
+/examples/phy/                @esp-idf-codeowners/bluetooth @esp-idf-codeowners/wifi @esp-idf-codeowners/ieee802154
 /examples/protocols/          @esp-idf-codeowners/network @esp-idf-codeowners/app-utilities
 /examples/provisioning/       @esp-idf-codeowners/app-utilities/provisioning
 /examples/security/           @esp-idf-codeowners/security
@@ -186,7 +194,10 @@
 /tools/catch/                 @esp-idf-codeowners/ci
 /tools/ci/                    @esp-idf-codeowners/ci
 /tools/cmake/                 @esp-idf-codeowners/build-config
+/tools/cmake/toolchain-*.cmake      @esp-idf-codeowners/toolchain
+/tools/esp_app_trace/         @esp-idf-codeowners/debugging
 /tools/esp_prov/              @esp-idf-codeowners/app-utilities
+/tools/gdb_panic_server.py    @esp-idf-codeowners/debugging
 /tools/kconfig*/              @esp-idf-codeowners/build-config
 /tools/ldgen/                 @esp-idf-codeowners/build-config
 /tools/mass_mfg/              @esp-idf-codeowners/app-utilities
@@ -195,6 +206,7 @@
 
 ## Note: owners here should be the same as the owners for the same example subdir, above
 /tools/test_apps/build_system/       @esp-idf-codeowners/build-config
+/tools/test_apps/linux_compatible/   @esp-idf-codeowners/system
 /tools/test_apps/protocols/          @esp-idf-codeowners/network @esp-idf-codeowners/app-utilities
 /tools/test_apps/security/           @esp-idf-codeowners/security
 /tools/test_apps/system/             @esp-idf-codeowners/system
@@ -203,6 +215,8 @@
 
 /tools/test_build_system/     @esp-idf-codeowners/tools @esp-idf-codeowners/build-config
 
+/tools/tools.json             @esp-idf-codeowners/tools @esp-idf-codeowners/toolchain @esp-idf-codeowners/debugging
+
 /tools/unit-test-app/         @esp-idf-codeowners/system @esp-idf-codeowners/tools
 
 # sort-order-reset

.gitlab/ci/README.md

@@ -21,11 +21,14 @@
       - [Shell Script Related](#shell-script-related)
   - [Manifest File to Control the Build/Test apps](#manifest-file-to-control-the-buildtest-apps)
     - [Grammar](#grammar)
-      - [Operands](#operands)
-      - [Operators](#operators)
-      - [Limitation:](#limitation)
-    - [How does it work?](#how-does-it-work)
-    - [Example](#example)
+    - [Special Rules](#special-rules)
+  - [Upload/Download Artifacts to Internal Minio Server](#uploaddownload-artifacts-to-internal-minio-server)
+    - [Users Without Access to Minio](#users-without-access-to-minio)
+    - [Users With Access to Minio](#users-with-access-to-minio)
+      - [Env Vars for Minio](#env-vars-for-minio)
+      - [Artifacts Types and File Patterns](#artifacts-types-and-file-patterns)
+      - [Upload](#upload)
+      - [Download](#download)
 
 ## General Workflow
 
@@ -56,19 +59,17 @@
 - `example_test[_esp32/esp32s2/...]`
 - `fuzzer_test`
 - `host_test`
-- `integration_test[_wifi/ble]`
+- `integration_test`
 - `iperf_stress_test`
 - `macos`
 - `macos_test`
 - `nvs_coverage`
 - `submodule`
-- `unit_test[_esp32/esp32s2/...]`
-- `weekend_test`
 - `windows`
 
 There are two general labels (not recommended since these two labels will trigger a lot of jobs)
 
-- `target_test`: includes all target for `example_test`, `custom_test`, `component_ut`, `unit_test`, `integration_test`
+- `target_test`: includes all target for `example_test`, `custom_test`, `component_ut`, `integration_test`
 - `all_test`: includes all test labels
 
 ### How to trigger a `detached` pipeline without pushing new commits?
@@ -144,10 +145,11 @@ check if there's a suitable `.if-<if-anchor-you-need>` anchor
 1. if there is, create a rule following [`rules` Template Naming Rules](#rules-template-naming-rules).For detail information, please refer to [GitLab Documentation `rules-if`](https://docs.gitlab.com/ee/ci/yaml/README.html#rulesif). Here's an example.
 
     ```yaml
-    .rules:dev:
+    .rules:patterns:python-files:
       rules:
-        - <<: *if-trigger
+        - <<: *if-protected
         - <<: *if-dev-push
+          changes: *patterns-python-files
     ```
 
 2. if there isn't
@@ -196,7 +198,7 @@ if a name has multi phrases, use `-` to concatenate them.
 
     - `target_test`
 
-      a combination of `example_test`, `custom_test`, `unit_test`, `component_ut`, `integration_test` and all targets
+      a combination of `example_test`, `custom_test`, `component_ut`, `integration_test` and all targets
 
 #### `rules` Template Naming Rules
 
@@ -242,3 +244,82 @@ We're using the latest version of [idf-build-apps][idf-build-apps]. Please refer
 
 [idf-build-apps]: https://github.com/espressif/idf-build-apps
 [manifest-doc]: https://docs.espressif.com/projects/idf-build-apps/en/latest/manifest.html
+
+### Special Rules
+
+In ESP-IDF CI, there's a few more special rules are additionally supported to disable the check app dependencies feature:
+
+- Add MR labels `BUILD_AND_TEST_ALL_APPS`
+- Run in protected branches
+
+## Upload/Download Artifacts to Internal Minio Server
+
+### Users Without Access to Minio
+
+If you don't have access to the internal Minio server, you can still download the artifacts from the shared link in the job log.
+
+The log will look like this:
+
+```shell
+Pipeline ID    : 587355
+Job name       : build_clang_test_apps_esp32
+Job ID         : 40272275
+Created archive file: 40272275.zip, uploading as 587355/build_dir_without_map_and_elf_files/build_clang_test_apps_esp32/40272275.zip
+Please download the archive file includes build_dir_without_map_and_elf_files from [INTERNAL_URL]
+```
+
+### Users With Access to Minio
+
+#### Env Vars for Minio
+
+Minio takes these env vars to connect to the server:
+
+- `IDF_S3_SERVER`
+- `IDF_S3_ACCESS_KEY`
+- `IDF_S3_SECRET_KEY`
+- `IDF_S3_BUCKET`
+
+#### Artifacts Types and File Patterns
+
+The artifacts types and corresponding file patterns are defined in tools/ci/artifacts_handler.py, inside `ArtifactType` and `TYPE_PATTERNS_DICT`.
+
+#### Upload
+
+```shell
+python tools/ci/artifacts_handler.py upload
+```
+
+ will upload the files that match the file patterns to minio object storage with name:
+
+`<pipeline_id>/<artifact_type>/<job_name>/<job_id>.zip`
+
+For example, job 39043328 will upload these four files:
+
+- `575500/map_and_elf_files/build_pytest_examples_esp32/39043328.zip`
+- `575500/build_dir_without_map_and_elf_files/build_pytest_examples_esp32/39043328.zip`
+- `575500/logs/build_pytest_examples_esp32/39043328.zip`
+- `575500/size_reports/build_pytest_examples_esp32/39043328.zip`
+
+#### Download
+
+You may run
+
+```shell
+python tools/ci/artifacts_handler.py download --pipeline_id <pipeline_id>
+```
+
+to download all files of the pipeline, or
+
+```shell
+python tools/ci/artifacts_handler.py download --pipeline_id <pipeline_id> --job_name <job_name_or_pattern>
+```
+
+to download all files with the specified job name or pattern, or
+
+```shell
+python tools/ci/artifacts_handler.py download --pipeline_id <pipeline_id> --job_name <job_name_or_pattern> --type <artifact_type> <artifact_type> ...
+```
+
+to download all files with the specified job name or pattern and artifact type(s).
+
+You may check all detailed documentation with `python tools/ci/artifacts_handler.py download -h`

.gitlab/ci/assign-test.yml

@@ -1,99 +0,0 @@
-.assign_test_template:
-  image: $TARGET_TEST_ENV_IMAGE
-  stage: assign_test
-  tags:
-    - assign_test
-  variables:
-    SUBMODULES_TO_FETCH: "none"
-  artifacts:
-    paths:
-      - ${TEST_DIR}/test_configs/
-      - artifact_index.json
-    when: always
-    expire_in: 1 week
-  script:
-    - run_cmd python tools/ci/python_packages/ttfw_idf/IDFAssignTest.py $TEST_TYPE $TEST_DIR -c $CI_TARGET_TEST_CONFIG_FILE -o $TEST_DIR/test_configs
-
-assign_example_test:
-  extends:
-    - .assign_test_template
-    - .rules:build:example_test
-  needs:
-    - job: build_examples_cmake_esp32
-      artifacts: false
-      optional: true
-    - job: build_examples_cmake_esp32s2
-      artifacts: false
-      optional: true
-    - job: build_examples_cmake_esp32c2
-      artifacts: false
-      optional: true
-    - job: build_examples_cmake_esp32c3
-      artifacts: false
-      optional: true
-    - job: build_examples_cmake_esp32c6
-      artifacts: false
-      optional: true
-    - job: build_examples_cmake_esp32h2
-      artifacts: false
-      optional: true
-    - job: build_examples_cmake_esp32s3
-      artifacts: false
-      optional: true
-  variables:
-    TEST_TYPE: example_test
-    TEST_DIR: examples
-
-assign_unit_test:
-  extends:
-    - .assign_test_template
-    - .rules:build:unit_test
-  needs:
-    - job: build_esp_idf_tests_cmake_esp32
-      optional: true
-    - job: build_esp_idf_tests_cmake_esp32s2
-      optional: true
-    - job: build_esp_idf_tests_cmake_esp32c2
-      optional: true
-    - job: build_esp_idf_tests_cmake_esp32c3
-      optional: true
-    - job: build_esp_idf_tests_cmake_esp32c6
-      optional: true
-    - job: build_esp_idf_tests_cmake_esp32h2
-      optional: true
-    - job: build_esp_idf_tests_cmake_esp32s3
-      optional: true
-  variables:
-    TEST_TYPE: unit_test
-    TEST_DIR: components/idf_test/unit_test
-
-assign_integration_test:
-  extends:
-    - .assign_test_template
-    - .rules:test:integration_test
-    - .before_script_minimal
-  image: ${CI_INTEGRATION_TEST_ENV_IMAGE}
-  needs:
-    - build_ssc_esp32
-    - build_ssc_esp32c3
-    - build_ssc_esp32c2
-  artifacts:
-    paths:
-      - $TEST_DIR/test_configs
-    expire_in: 1 week
-  variables:
-    TEST_DIR: ${CI_PROJECT_DIR}/tools/ci/integration_test
-    BUILD_DIR: ${CI_PROJECT_DIR}/SSC/ssc_bin
-    INTEGRATION_TEST_CASE_PATH: "${CI_PROJECT_DIR}/auto_test_script/TestCaseFiles"
-    ASSIGN_TEST_CASE_SCRIPT: "${CI_PROJECT_DIR}/auto_test_script/bin/CIAssignTestCases.py"
-    PYTHONPATH: ${CI_PROJECT_DIR}/auto_test_script/packages
-    GIT_LFS_SKIP_SMUDGE: 1
-  script:
-    - add_gitlab_ssh_keys
-    # clone test script to assign tests
-    - retry_failed git clone ${CI_AUTO_TEST_SCRIPT_REPO_URL} auto_test_script
-    - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script
-    - cd auto_test_script
-    - ./tools/ci/setup_idfci.sh
-    # assign integration test cases
-    - python ${ASSIGN_TEST_CASE_SCRIPT} -t ${INTEGRATION_TEST_CASE_PATH} -c $CI_TARGET_TEST_CONFIG_FILE -b ${BUILD_DIR} -o $TEST_DIR/test_configs

.gitlab/ci/build.yml

@@ -1,5 +1,7 @@
 .build_template:
   stage: build
+  extends:
+    - .after_script:build:ccache
   image: $ESP_ENV_IMAGE
   tags:
     - build
@@ -8,76 +10,45 @@
   variables:
     # Enable ccache for all build jobs. See configure_ci_environment.sh for more ccache related settings.
     IDF_CCACHE_ENABLE: "1"
-  after_script:
-    # Show ccache statistics if enabled globally
-    - test "$CI_CCACHE_STATS" == 1 && test -n "$(which ccache)" && ccache --show-stats || true
   dependencies: []
 
 .build_cmake_template:
   extends:
     - .build_template
-    - .before_script_build_jobs
+    - .before_script:build
+    - .after_script:build:ccache
   dependencies: # set dependencies to null to avoid missing artifacts issue
   needs:
     - job: fast_template_app
       artifacts: false
+    - pipeline_variables
   artifacts:
     paths:
-      - "**/build*/size.json"
+      # The other artifacts patterns are defined under tools/ci/artifacts_handler.py
+      # Now we're uploading/downloading the binary files from our internal storage server
+      #
+      # keep the log file to help debug
       - "**/build*/build_log.txt"
-      - "**/build*/*.bin"
-      # upload to s3 server to save the artifacts size
-      # - "**/build*/*.map"
-      # ttfw tests require elf files
-      - "**/build*/*.elf"
-      - "**/build*/flasher_args.json"
-      - "**/build*/flash_project_args"
-      - "**/build*/config/sdkconfig.json"
-      # ttfw tests require sdkconfig file
-      - "**/build*/sdkconfig"
-      - "**/build*/bootloader/*.bin"
-      - "**/build*/partition_table/*.bin"
-      - list_job_*.json
+      # keep the size info to help track the binary size
       - size_info.txt
-      # unit test specific
-      - components/idf_test/unit_test/*.yml
+      - "**/build*/size.json"
     when: always
     expire_in: 4 days
-  after_script:
-    # Show ccache statistics if enabled globally
-    - test "$CI_CCACHE_STATS" == 1 && test -n "$(which ccache)" && ccache --show-stats || true
-    # upload the binary files to s3 server
-    - echo -e "\e[0Ksection_start:`date +%s`:upload_binaries_to_s3_server[collapsed=true]\r\e[0KUploading binaries to s3 Server"
-    - shopt -s globstar
-    # use || true to bypass the no-file error
-    - zip ${CI_JOB_ID}.zip **/build*/*.bin || true
-    - zip ${CI_JOB_ID}.zip **/build*/*.elf || true
-    - zip ${CI_JOB_ID}.zip **/build*/*.map || true
-    - zip ${CI_JOB_ID}.zip **/build*/flasher_args.json || true
-    - zip ${CI_JOB_ID}.zip **/build*/flash_project_args || true
-    - zip ${CI_JOB_ID}.zip **/build*/config/sdkconfig.json || true
-    - zip ${CI_JOB_ID}.zip **/build*/sdkconfig || true
-    - zip ${CI_JOB_ID}.zip **/build*/bootloader/*.bin || true
-    - zip ${CI_JOB_ID}.zip **/build*/partition_table/*.bin || true
-    - mc alias set shiny-s3 ${SHINY_S3_SERVER} ${SHINY_S3_ACCESS_KEY} ${SHINY_S3_SECRET_KEY}
-    - mc cp ${CI_JOB_ID}.zip shiny-s3/idf-artifacts/${CI_PIPELINE_ID}/${CI_JOB_ID}.zip
-    - echo -e "\e[0Ksection_end:`date +%s`:upload_binaries_to_s3_server\r\e[0K"
-    - echo "Please download the full binary files (including *.elf and *.map files) from the following share link"
-    # would be clean up after 4 days
-    - mc share download shiny-s3/idf-artifacts/${CI_PIPELINE_ID}/${CI_JOB_ID}.zip --expire=96h
   script:
-    # CI specific options start from "--collect-size-info xxx". could ignore when running locally
+    # CI specific options start from "--parallel-count xxx". could ignore when running locally
     - run_cmd python tools/ci/ci_build_apps.py $TEST_DIR -v
       -t $IDF_TARGET
       --copy-sdkconfig
-      --collect-size-info size_info.txt
-      --collect-app-info list_job_${CI_NODE_INDEX:-1}.json
       --parallel-count ${CI_NODE_TOTAL:-1}
       --parallel-index ${CI_NODE_INDEX:-1}
       --extra-preserve-dirs
         examples/bluetooth/esp_ble_mesh/ble_mesh_console
         examples/bluetooth/hci/controller_hci_uart_esp32
         examples/wifi/iperf
+      --modified-components ${MODIFIED_COMPONENTS}
+      --modified-files ${MODIFIED_FILES}
+    # for detailed documents, please refer to .gitlab/ci/README.md#uploaddownload-artifacts-to-internal-minio-server
+    - python tools/ci/artifacts_handler.py upload
 
 .build_cmake_clang_template:
   extends:
@@ -87,88 +58,64 @@
     TEST_BUILD_OPTS_EXTRA: ""
     TEST_DIR: tools/test_apps/system/cxx_pthread_bluetooth
   script:
-    # CI specific options start from "--collect-size-info xxx". could ignore when running locally
+    # CI specific options start from "--parallel-count xxx". could ignore when running locally
     - run_cmd python tools/ci/ci_build_apps.py $TEST_DIR -v
       -t $IDF_TARGET
       --copy-sdkconfig
-      --collect-size-info size_info.txt
-      --collect-app-info list_job_${CI_NODE_INDEX:-1}.json
       --parallel-count ${CI_NODE_TOTAL:-1}
       --parallel-index ${CI_NODE_INDEX:-1}
+      --modified-components ${MODIFIED_COMPONENTS}
+      --modified-files ${MODIFIED_FILES}
       $TEST_BUILD_OPTS_EXTRA
+    - python tools/ci/artifacts_handler.py upload
 
 .build_pytest_template:
   extends:
     - .build_cmake_template
-    - .before_script_build_jobs
-  artifacts:
-    paths:
-      - "**/build*/size.json"
-      - "**/build*/build_log.txt"
-      - "**/build*/*.bin"
-      # upload to s3 server to save the artifacts size
-      # - "**/build*/*.map"
-      # - "**/build*/*.elf"
-      - "**/build*/flasher_args.json"
-      - "**/build*/flash_project_args"
-      - "**/build*/config/sdkconfig.json"
-      - "**/build*/bootloader/*.bin"
-      - "**/build*/partition_table/*.bin"
-      - list_job_*.json
-      - size_info.txt
-    when: always
-    expire_in: 4 days
   script:
-    # CI specific options start from "--collect-size-info xxx". could ignore when running locally
+    # CI specific options start from "--parallel-count xxx". could ignore when running locally
     - run_cmd python tools/ci/ci_build_apps.py $TEST_DIR -v
       -t $IDF_TARGET
+      -m \"not host_test\"
       --pytest-apps
-      --collect-size-info size_info.txt
       --parallel-count ${CI_NODE_TOTAL:-1}
       --parallel-index ${CI_NODE_INDEX:-1}
+      --collect-app-info "list_job_${CI_JOB_NAME_SLUG}.txt"
+      --modified-components ${MODIFIED_COMPONENTS}
+      --modified-files ${MODIFIED_FILES}
+    - python tools/ci/artifacts_handler.py upload
 
 .build_pytest_no_jtag_template:
-  extends: .build_pytest_template
+  extends:
+    - .build_cmake_template
   script:
-    # CI specific options start from "--collect-size-info xxx". could ignore when running locally
+    # CI specific options start from "--parallel-count xxx". could ignore when running locally
     - run_cmd python tools/ci/ci_build_apps.py $TEST_DIR -v
       -t $IDF_TARGET
       -m \"not host_test and not jtag\"
       --pytest-apps
-      --collect-size-info size_info.txt
       --parallel-count ${CI_NODE_TOTAL:-1}
       --parallel-index ${CI_NODE_INDEX:-1}
+      --collect-app-info "list_job_${CI_JOB_NAME_SLUG}.txt"
+      --modified-components ${MODIFIED_COMPONENTS}
+      --modified-files ${MODIFIED_FILES}
+    - python tools/ci/artifacts_handler.py upload
 
 .build_pytest_jtag_template:
   extends:
     - .build_cmake_template
-    - .before_script_build_jobs
-  artifacts:
-    paths:
-      - "**/build*/size.json"
-      - "**/build*/build_log.txt"
-      - "**/build*/*.bin"
-      # upload to s3 server to save the artifacts size
-      # - "**/build*/*.map"
-      - "**/build*/*.elf"  # need elf for gdb
-      - "**/build*/flasher_args.json"
-      - "**/build*/flash_project_args"
-      - "**/build*/config/sdkconfig.json"
-      - "**/build*/bootloader/*.bin"
-      - "**/build*/partition_table/*.bin"
-      - list_job_*.json
-      - size_info.txt
-    when: always
-    expire_in: 4 days
   script:
-    # CI specific options start from "--collect-size-info xxx". could ignore when running locally
+    # CI specific options start from "--parallel-count xxx". could ignore when running locally
     - run_cmd python tools/ci/ci_build_apps.py $TEST_DIR -v
       -t $IDF_TARGET
       -m \"not host_test and jtag\"
       --pytest-apps
-      --collect-size-info size_info.txt
       --parallel-count ${CI_NODE_TOTAL:-1}
       --parallel-index ${CI_NODE_INDEX:-1}
+      --collect-app-info "list_job_${CI_JOB_NAME_SLUG}.txt"
+      --modified-components ${MODIFIED_COMPONENTS}
+      --modified-files ${MODIFIED_FILES}
+    - python tools/ci/artifacts_handler.py upload
 
 build_pytest_examples_esp32:
   extends:
@@ -215,25 +162,9 @@ build_pytest_examples_esp32c2:
     IDF_TARGET: esp32c2
     TEST_DIR: examples
 
-build_pytest_examples_esp32h4:
-  extends:
-    - .build_pytest_no_jtag_template
-    - .rules:build:example_test-esp32h4
-  variables:
-    IDF_TARGET: esp32h4
-    TEST_DIR: examples
-
-build_pytest_examples_jtag:  # for all targets
-  extends:
-    - .build_pytest_jtag_template
-    - .rules:build:example_test-esp32
-  variables:
-    IDF_TARGET: all
-    TEST_DIR: examples
-
 build_pytest_examples_esp32c6:
   extends:
-    - .build_pytest_template
+    - .build_pytest_no_jtag_template
     - .rules:build:example_test-esp32c6
   parallel: 2
   variables:
@@ -242,12 +173,30 @@ build_pytest_examples_esp32c6:
 
 build_pytest_examples_esp32h2:
   extends:
-    - .build_pytest_template
+    - .build_pytest_no_jtag_template
     - .rules:build:example_test-esp32h2
+  parallel: 2
   variables:
     IDF_TARGET: esp32h2
     TEST_DIR: examples
 
+build_pytest_examples_esp32p4:
+  extends:
+    - .build_pytest_no_jtag_template
+    - .rules:build:example_test-esp32p4
+  parallel: 2
+  variables:
+    IDF_TARGET: esp32p4
+    TEST_DIR: examples
+
+build_pytest_examples_jtag:  # for all targets
+  extends:
+    - .build_pytest_jtag_template
+    - .rules:build:example_test
+  variables:
+    IDF_TARGET: all
+    TEST_DIR: examples
+
 build_pytest_components_esp32:
   extends:
     - .build_pytest_template
@@ -311,6 +260,15 @@ build_pytest_components_esp32h2:
     IDF_TARGET: esp32h2
     TEST_DIR: components
 
+build_pytest_components_esp32p4:
+  extends:
+    - .build_pytest_template
+    - .rules:build:component_ut-esp32p4
+  parallel: 4
+  variables:
+    IDF_TARGET: esp32p4
+    TEST_DIR: components
+
 build_only_components_apps:
   extends:
     - .build_cmake_template
@@ -318,40 +276,18 @@ build_only_components_apps:
   parallel: 5
   script:
     - set_component_ut_vars
-    # CI specific options start from "--collect-size-info xxx". could ignore when running locally
+    # CI specific options start from "--parallel-count xxx". could ignore when running locally
     - run_cmd python tools/ci/ci_build_apps.py $COMPONENT_UT_DIRS -v
       -t all
-      --collect-size-info size_info.txt
-      --collect-app-info list_job_${CI_NODE_INDEX:-1}.json
       --parallel-count ${CI_NODE_TOTAL:-1}
       --parallel-index ${CI_NODE_INDEX:-1}
-
-.build_pytest_test_apps_template:
-  extends: .build_pytest_template
-  artifacts:
-    paths:
-      - "**/build*/size.json"
-      - "**/build*/build_log.txt"
-      - "**/build*/*.bin"
-      # upload to s3 server to save the artifacts size
-      # - "**/build*/*.map"
-      # pytest test apps requires elf files for coredump tests
-      - "**/build*/*.elf"
-      - "**/build*/flasher_args.json"
-      - "**/build*/flash_project_args"
-      - "**/build*/config/sdkconfig.json"
-      - "**/build*/bootloader/*.elf"
-      - "**/build*/bootloader/*.bin"
-      - "**/build*/partition_table/*.bin"
-      - "**/build*/project_description.json"
-      - list_job_*.json
-      - size_info.txt
-    when: always
-    expire_in: 4 days
+      --modified-components ${MODIFIED_COMPONENTS}
+      --modified-files ${MODIFIED_FILES}
+    - python tools/ci/artifacts_handler.py upload
 
 build_pytest_test_apps_esp32:
   extends:
-    - .build_pytest_test_apps_template
+    - .build_pytest_template
     - .rules:build:custom_test-esp32
   variables:
     IDF_TARGET: esp32
@@ -359,7 +295,7 @@ build_pytest_test_apps_esp32:
 
 build_pytest_test_apps_esp32s2:
   extends:
-    - .build_pytest_test_apps_template
+    - .build_pytest_template
     - .rules:build:custom_test-esp32s2
   variables:
     IDF_TARGET: esp32s2
@@ -367,7 +303,7 @@ build_pytest_test_apps_esp32s2:
 
 build_pytest_test_apps_esp32s3:
   extends:
-    - .build_pytest_test_apps_template
+    - .build_pytest_template
     - .rules:build:custom_test-esp32s3
   parallel: 2
   variables:
@@ -376,7 +312,7 @@ build_pytest_test_apps_esp32s3:
 
 build_pytest_test_apps_esp32c3:
   extends:
-    - .build_pytest_test_apps_template
+    - .build_pytest_template
     - .rules:build:custom_test-esp32c3
   variables:
     IDF_TARGET: esp32c3
@@ -384,7 +320,7 @@ build_pytest_test_apps_esp32c3:
 
 build_pytest_test_apps_esp32c2:
   extends:
-    - .build_pytest_test_apps_template
+    - .build_pytest_template
     - .rules:build:custom_test-esp32c2
   variables:
     IDF_TARGET: esp32c2
@@ -392,7 +328,7 @@ build_pytest_test_apps_esp32c2:
 
 build_pytest_test_apps_esp32c6:
   extends:
-    - .build_pytest_test_apps_template
+    - .build_pytest_template
     - .rules:build:custom_test-esp32c6
   variables:
     IDF_TARGET: esp32c6
@@ -400,34 +336,43 @@ build_pytest_test_apps_esp32c6:
 
 build_pytest_test_apps_esp32h2:
   extends:
-    - .build_pytest_test_apps_template
+    - .build_pytest_template
     - .rules:build:custom_test-esp32h2
   variables:
     IDF_TARGET: esp32h2
     TEST_DIR: tools/test_apps
 
+build_pytest_test_apps_esp32p4:
+  extends:
+    - .build_pytest_template
+    - .rules:build:custom_test-esp32p4
+  variables:
+    IDF_TARGET: esp32p4
+    TEST_DIR: tools/test_apps
+
 build_only_tools_test_apps:
   extends:
     - .build_cmake_template
     - .rules:build:custom_test
   parallel: 9
   script:
-    # CI specific options start from "--collect-size-info xxx". could ignore when running locally
+    # CI specific options start from "--parallel-count xxx". could ignore when running locally
     - run_cmd python tools/ci/ci_build_apps.py tools/test_apps -v
       -t all
-      --collect-size-info size_info.txt
-      --collect-app-info list_job_${CI_NODE_INDEX:-1}.json
       --parallel-count ${CI_NODE_TOTAL:-1}
       --parallel-index ${CI_NODE_INDEX:-1}
+      --modified-components ${MODIFIED_COMPONENTS}
+      --modified-files ${MODIFIED_FILES}
+    - python tools/ci/artifacts_handler.py upload
 
 .build_template_app_template:
   extends:
     - .build_template
-    - .before_script_build_jobs
+    - .before_script:build
   variables:
     LOG_PATH: "${CI_PROJECT_DIR}/log_template_app"
     BUILD_PATH: "${CI_PROJECT_DIR}/build_template_app"
-    BUILD_DIR: "@t/@w"
+    BUILD_DIR: "${BUILD_PATH}/@t/@w"
     BUILD_LOG_CMAKE: "${LOG_PATH}/cmake_@t_@w.txt"
     BUILD_COMMAND_ARGS: ""
   artifacts:
@@ -459,151 +404,6 @@ fast_template_app:
     BUILD_COMMAND_ARGS: "-p"
 #------------------------------------------------------------------------------
 
-.build_ssc_template:
-  extends:
-    - .build_template
-    - .rules:build:integration_test
-  needs:
-    - job: fast_template_app
-      artifacts: false
-  artifacts:
-    paths:
-      - SSC/ssc_bin
-    expire_in: 1 week
-  script:
-    - retry_failed git clone $SSC_REPOSITORY
-    - python $CHECKOUT_REF_SCRIPT SSC SSC
-    - cd SSC
-    - MAKEFLAGS= ./ci_build_ssc.sh $TARGET_NAME
-
-build_ssc_esp32:
-  extends: .build_ssc_template
-  parallel: 3
-  variables:
-    TARGET_NAME: "ESP32"
-
-build_ssc_esp32s2:
-  extends: .build_ssc_template
-  parallel: 2
-  variables:
-    TARGET_NAME: "ESP32S2"
-
-build_ssc_esp32c2:
-  extends: .build_ssc_template
-  parallel: 2
-  variables:
-    TARGET_NAME: "ESP32C2"
-
-build_ssc_esp32c3:
-  extends: .build_ssc_template
-  parallel: 3
-  variables:
-    TARGET_NAME: "ESP32C3"
-
-build_ssc_esp32s3:
-  extends: .build_ssc_template
-  parallel: 3
-  variables:
-    TARGET_NAME: "ESP32S3"
-
-build_ssc_esp32c6:
-  extends: .build_ssc_template
-  parallel: 3
-  variables:
-    TARGET_NAME: "ESP32C6"
-
-build_ssc_esp32h2:
-  extends: .build_ssc_template
-  parallel: 2
-  variables:
-    TARGET_NAME: "ESP32H2"
-
-.build_esp_idf_tests_cmake_template:
-  extends:
-    - .build_cmake_template
-    - .before_script_build_jobs
-  artifacts:
-    paths:
-      - "**/build*/size.json"
-      - "**/build*/build_log.txt"
-      - "**/build*/*.bin"
-      # upload to s3 server to save the artifacts size
-      # - "**/build*/*.map"
-      # ttfw tests require elf files
-      - "**/build*/*.elf"
-      - "**/build*/flasher_args.json"
-      - "**/build*/flash_project_args"
-      - "**/build*/config/sdkconfig.json"
-      - "**/build*/sdkconfig"
-      - "**/build*/bootloader/*.bin"
-      - "**/build*/partition_table/*.bin"
-      - list_job_*.json
-      - size_info.txt
-      - components/idf_test/unit_test/*.yml
-    when: always
-    expire_in: 4 days
-  script:
-    # CI specific options start from "--collect-size-info xxx". could ignore when running locally
-    - run_cmd python tools/ci/ci_build_apps.py tools/unit-test-app -v
-      -t $IDF_TARGET
-      --config "configs/*="
-      --copy-sdkconfig
-      --preserve-all
-      --collect-size-info size_info.txt
-      --collect-app-info list_job_${CI_NODE_INDEX:-1}.json
-      --parallel-count ${CI_NODE_TOTAL:-1}
-      --parallel-index ${CI_NODE_INDEX:-1}
-    - run_cmd python tools/unit-test-app/tools/UnitTestParser.py tools/unit-test-app ${CI_NODE_INDEX:-1}
-
-build_esp_idf_tests_cmake_esp32:
-  extends:
-    - .build_esp_idf_tests_cmake_template
-    - .rules:build:unit_test-esp32
-  variables:
-    IDF_TARGET: esp32
-
-build_esp_idf_tests_cmake_esp32s2:
-  extends:
-    - .build_esp_idf_tests_cmake_template
-    - .rules:build:unit_test-esp32s2
-  variables:
-    IDF_TARGET: esp32s2
-
-build_esp_idf_tests_cmake_esp32s3:
-  extends:
-    - .build_esp_idf_tests_cmake_template
-    - .rules:build:unit_test-esp32s3
-  variables:
-    IDF_TARGET: esp32s3
-
-build_esp_idf_tests_cmake_esp32c2:
-  extends:
-    - .build_esp_idf_tests_cmake_template
-    - .rules:build:unit_test-esp32c2
-  variables:
-    IDF_TARGET: esp32c2
-
-build_esp_idf_tests_cmake_esp32c3:
-  extends:
-    - .build_esp_idf_tests_cmake_template
-    - .rules:build:unit_test-esp32c3
-  variables:
-    IDF_TARGET: esp32c3
-
-build_esp_idf_tests_cmake_esp32c6:
-  extends:
-    - .build_esp_idf_tests_cmake_template
-    - .rules:build:unit_test-esp32c6
-  variables:
-    IDF_TARGET: esp32c6
-
-build_esp_idf_tests_cmake_esp32h2:
-  extends:
-    - .build_esp_idf_tests_cmake_template
-    - .rules:build:unit_test-esp32h2
-  variables:
-    IDF_TARGET: esp32h2
-
 build_examples_cmake_esp32:
   extends:
     - .build_cmake_template
@@ -626,7 +426,7 @@ build_examples_cmake_esp32s3:
   extends:
     - .build_cmake_template
     - .rules:build:example_test-esp32s3
-  parallel: 7
+  parallel: 11
   variables:
     IDF_TARGET: esp32s3
     TEST_DIR: examples
@@ -635,7 +435,7 @@ build_examples_cmake_esp32c2:
   extends:
     - .build_cmake_template
     - .rules:build:example_test-esp32c2
-  parallel: 6
+  parallel: 7
   variables:
     IDF_TARGET: esp32c2
     TEST_DIR: examples
@@ -644,24 +444,16 @@ build_examples_cmake_esp32c3:
   extends:
     - .build_cmake_template
     - .rules:build:example_test-esp32c3
-  parallel: 6
+  parallel: 9
   variables:
     IDF_TARGET: esp32c3
     TEST_DIR: examples
 
-build_examples_cmake_esp32h4:
-  extends:
-    - .build_cmake_template
-    - .rules:build:example_test-esp32h4
-  variables:
-    IDF_TARGET: esp32h4
-    TEST_DIR: examples
-
 build_examples_cmake_esp32c6:
   extends:
     - .build_cmake_template
     - .rules:build:example_test-esp32c6
-  parallel: 6
+  parallel: 11
   variables:
     IDF_TARGET: esp32c6
     TEST_DIR: examples
@@ -670,11 +462,20 @@ build_examples_cmake_esp32h2:
   extends:
     - .build_cmake_template
     - .rules:build:example_test-esp32h2
-  parallel: 2
+  parallel: 9
   variables:
     IDF_TARGET: esp32h2
     TEST_DIR: examples
 
+build_examples_cmake_esp32p4:
+  extends:
+    - .build_cmake_template
+    - .rules:build:example_test-esp32p4
+  parallel: 4
+  variables:
+    IDF_TARGET: esp32p4
+    TEST_DIR: examples
+
 build_clang_test_apps_esp32:
   extends:
     - .build_cmake_clang_template
@@ -729,50 +530,26 @@ build_clang_test_apps_esp32c6:
   variables:
     IDF_TARGET: esp32c6
 
-build_clang_test_apps_esp32h4:
-  extends:
-    - .build_clang_test_apps_riscv
-    - .rules:build:custom_test-esp32h4
-  variables:
-    IDF_TARGET: esp32h4
-
 .test_build_system_template:
   stage: host_test
   extends:
     - .build_template
-    - .rules:build
+    - .rules:build:check
   needs:
     - job: fast_template_app
       artifacts: false
+      optional: true
   script:
     - ${IDF_PATH}/tools/ci/test_configure_ci_environment.sh
-    - rm -rf test_build_system
-    - mkdir test_build_system
-    - cd test_build_system
-    - ${IDF_PATH}/tools/ci/${SHELL_TEST_SCRIPT}
-
-test_build_system_cmake:
-  extends: .test_build_system_template
-  variables:
-    SHELL_TEST_SCRIPT: test_build_system_cmake.sh
-
-test_build_system_cmake_macos:
-  extends:
-    - .test_build_system_template
-    - .before_script_macos
-    - .rules:build:macos
-  tags:
-    - macos_shell
-  variables:
-    SHELL_TEST_SCRIPT: test_build_system_cmake.sh
-
-test_build_system_spaces:
-  extends: .test_build_system_template
-  variables:
-    SHELL_TEST_SCRIPT: test_build_system_spaces.py
+    - cd ${IDF_PATH}/tools/test_build_system
+    - retry_failed git clone $KNOWN_FAILURE_CASES_REPO known_failure_cases
+    - pytest --parallel-count ${CI_NODE_TOTAL:-1} --parallel-index ${CI_NODE_INDEX:-1}
+      --work-dir ${CI_PROJECT_DIR}/test_build_system --junitxml=${CI_PROJECT_DIR}/XUNIT_RESULT.xml
+      --ignore-result-files known_failure_cases/known_failure_cases.txt
 
 pytest_build_system:
   extends: .test_build_system_template
+  parallel: 3
   artifacts:
     paths:
       - XUNIT_RESULT.xml
@@ -781,14 +558,59 @@ pytest_build_system:
     expire_in: 2 days
     reports:
       junit: XUNIT_RESULT.xml
+
+pytest_build_system_macos:
+  extends:
+    - .test_build_system_template
+    - .before_script:build:macos
+    - .rules:build:macos
+  tags:
+    - macos_shell
+  parallel: 3
+  artifacts:
+    paths:
+      - XUNIT_RESULT.xml
+      - test_build_system
+    when: always
+    expire_in: 2 days
+    reports:
+      junit: XUNIT_RESULT.xml
+
+.test_build_system_template_win:
+  stage: host_test
+  variables:
+  # Enable ccache for all build jobs. See configure_ci_environment.sh for more ccache related settings.
+    IDF_CCACHE_ENABLE: "1"
+    PYTHONPATH: "$PYTHONPATH;$IDF_PATH\\tools;$IDF_PATH\\tools\\esp_app_trace;$IDF_PATH\\components\\partition_table;$IDF_PATH\\tools\\ci\\python_packages"
+  before_script: []
+  after_script: []
+  timeout: 4 hours
   script:
-    - ${IDF_PATH}/tools/ci/test_configure_ci_environment.sh
-    - cd ${IDF_PATH}/tools/test_build_system
-    - pytest --work-dir ${CI_PROJECT_DIR}/test_build_system --junitxml=${CI_PROJECT_DIR}/XUNIT_RESULT.xml
+    - .\install.ps1 --enable-ci --enable-pytest
+    - . .\export.ps1
+    - python "${SUBMODULE_FETCH_TOOL}" -s "all"
+    - cd ${IDF_PATH}\tools\test_build_system
+    - pytest --junitxml=${CI_PROJECT_DIR}\XUNIT_RESULT.xml
+
+pytest_build_system_win:
+  extends:
+    - .test_build_system_template_win
+    - .rules:labels:windows_pytest_build_system
+  needs: []
+  tags:
+    - windows-target
+  artifacts:
+    paths:
+      - XUNIT_RESULT.xml
+      - test_build_system
+    when: always
+    expire_in: 2 days
+    reports:
+      junit: XUNIT_RESULT.xml
 
 build_docker:
   extends:
-    - .before_script_minimal
+    - .before_script:minimal
     - .rules:build:docker
   stage: host_test
   needs: []

.gitlab/ci/common.yml

@@ -0,0 +1,355 @@
+#####################
+# Default Variables #
+#####################
+stages:
+  - upload_cache
+  - pre_check
+  - build
+  - assign_test
+  - target_test
+  - host_test
+  - build_doc
+  - test_deploy
+  - deploy
+  - post_deploy
+
+variables:
+# System environment
+
+  # Common parameters for the 'make' during CI tests
+  MAKEFLAGS: "-j5 --no-keep-going"
+
+# GitLab-CI environment
+
+  # now we have pack-objects cache, so clone strategy is faster than fetch
+  GIT_STRATEGY: clone
+  # we will download archive for each submodule instead of clone.
+  # we don't do "recursive" when fetch submodule as they're not used in CI now.
+  GIT_SUBMODULE_STRATEGY: none
+  # since we're using merged-result pipelines, the last commit should work for most cases
+  GIT_DEPTH: 1
+  # --no-recurse-submodules: we use cache for submodules
+  # --prune --prune-tags: in case remote branch or tag is force pushed
+  GIT_FETCH_EXTRA_FLAGS: "--no-recurse-submodules --prune --prune-tags"
+  # we're using .cache folder for caches
+  GIT_CLEAN_FLAGS: -ffdx -e .cache/
+  LATEST_GIT_TAG: v5.3-dev
+
+  SUBMODULE_FETCH_TOOL: "tools/ci/ci_fetch_submodule.py"
+  # by default we will fetch all submodules
+  # jobs can overwrite this variable to only fetch submodules they required
+  # set to "none" if don't need to fetch submodules
+  SUBMODULES_TO_FETCH: "all"
+  # tell build system do not check submodule update as we download archive instead of clone
+  IDF_SKIP_CHECK_SUBMODULES: 1
+
+  IDF_PATH: "$CI_PROJECT_DIR"
+  V: "0"
+  CHECKOUT_REF_SCRIPT: "$CI_PROJECT_DIR/tools/ci/checkout_project_ref.py"
+  PYTHON_VER: 3.8.17
+
+  # Docker images
+  ESP_ENV_IMAGE: "${CI_DOCKER_REGISTRY}/esp-env-v5.3:1"
+  ESP_IDF_DOC_ENV_IMAGE: "${CI_DOCKER_REGISTRY}/esp-idf-doc-env-v5.3:1-1"
+  TARGET_TEST_ENV_IMAGE: "${CI_DOCKER_REGISTRY}/target-test-env-v5.3:1"
+  SONARQUBE_SCANNER_IMAGE: "${CI_DOCKER_REGISTRY}/sonarqube-scanner:5"
+  PRE_COMMIT_IMAGE: "${CI_DOCKER_REGISTRY}/esp-idf-pre-commit:1"
+
+  # target test repo parameters
+  TEST_ENV_CONFIG_REPO: "https://gitlab-ci-token:${BOT_TOKEN}@${CI_SERVER_HOST}:${CI_SERVER_PORT}/qa/ci-test-runner-configs.git"
+
+  # cache python dependencies
+  PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
+
+  # Set this variable to the branch of idf-constraints repo in order to test a custom Python constraint file. The
+  # branch name must be without the remote part ("origin/"). Keep the variable empty in order to use the constraint
+  # file from https://dl.espressif.com/dl/esp-idf.
+  CI_PYTHON_CONSTRAINT_BRANCH: ""
+
+  # Update the filename for a specific ESP-IDF release. It is used only with CI_PYTHON_CONSTRAINT_BRANCH.
+  CI_PYTHON_CONSTRAINT_FILE: "espidf.constraints.v5.3.txt"
+
+  # Set this variable to repository name of a Python tool you wish to install and test in the context of ESP-IDF CI.
+  # Keep the variable empty when not used.
+  CI_PYTHON_TOOL_REPO: ""
+
+  # Set this variable to the branch of a Python tool repo specified in CI_PYTHON_TOOL_REPO. The
+  # branch name must be without the remote part ("origin/"). Keep the variable empty when not used.
+  # This is used only if CI_PYTHON_TOOL_REPO is not empty.
+  CI_PYTHON_TOOL_BRANCH: ""
+
+  IDF_CI_BUILD: 1
+
+################################################
+# `before_script` and `after_script` Templates #
+################################################
+.common_before_scripts: &common-before_scripts |
+  source tools/ci/utils.sh
+  is_based_on_commits $REQUIRED_ANCESTOR_COMMITS
+
+  if [[ -n "$IDF_DONT_USE_MIRRORS" ]]; then
+    export IDF_MIRROR_PREFIX_MAP=
+  fi
+
+  if echo "$CI_MERGE_REQUEST_LABELS" | egrep "(^|,)include_nightly_run(,|$)"; then
+    export INCLUDE_NIGHTLY_RUN="1"
+    export NIGHTLY_RUN="1"
+  fi
+
+  # configure cmake related flags
+  source tools/ci/configure_ci_environment.sh
+
+  # add extra python packages
+  export PYTHONPATH="$IDF_PATH/tools:$IDF_PATH/tools/esp_app_trace:$IDF_PATH/components/partition_table:$IDF_PATH/tools/ci/python_packages:$PYTHONPATH"
+
+.setup_tools_and_idf_python_venv: &setup_tools_and_idf_python_venv |
+  # must use after setup_tools_except_target_test
+  # otherwise the export.sh won't work properly
+
+  # download constraint file for dev
+  if [[ -n "$CI_PYTHON_CONSTRAINT_BRANCH" ]]; then
+    wget -O /tmp/constraint.txt --header="Authorization:Bearer ${ESPCI_TOKEN}" ${GITLAB_HTTP_SERVER}/api/v4/projects/2581/repository/files/${CI_PYTHON_CONSTRAINT_FILE}/raw?ref=${CI_PYTHON_CONSTRAINT_BRANCH}
+    mkdir -p ~/.espressif
+    mv /tmp/constraint.txt ~/.espressif/${CI_PYTHON_CONSTRAINT_FILE}
+  fi
+
+  # Mirror
+  if [[ -n "$IDF_DONT_USE_MIRRORS" ]]; then
+    export IDF_MIRROR_PREFIX_MAP=
+  fi
+
+  # install latest python packages
+  # target test jobs
+  if [[ "${CI_JOB_STAGE}" == "target_test" ]]; then
+    run_cmd bash install.sh --enable-ci --enable-pytest
+  elif [[ "${CI_JOB_STAGE}" == "build_doc" ]]; then
+    run_cmd bash install.sh --enable-ci --enable-docs
+  elif [[ "${CI_JOB_STAGE}" == "build" ]]; then
+    run_cmd bash install.sh --enable-ci --enable-pytest
+  else
+    if ! echo "${CI_JOB_NAME}" | egrep ".*pytest.*"; then
+      run_cmd bash install.sh --enable-ci
+    else
+      run_cmd bash install.sh --enable-ci --enable-pytest
+    fi
+  fi
+
+  # Install esp-clang if necessary
+  if [[ "$IDF_TOOLCHAIN" == "clang" ]]; then
+    $IDF_PATH/tools/idf_tools.py --non-interactive install esp-clang
+  fi
+
+  # Install QEMU if necessary
+  if [[ ! -z "$INSTALL_QEMU" ]]; then
+    $IDF_PATH/tools/idf_tools.py --non-interactive install qemu-xtensa qemu-riscv32
+  fi
+
+  # Since the version 3.21 CMake passes source files and include dirs to ninja using absolute paths.
+  # Needed for pytest junit reports.
+  $IDF_PATH/tools/idf_tools.py --non-interactive install cmake
+
+  source ./export.sh
+
+  # Custom clang
+  if [[ ! -z "$CI_CLANG_DISTRO_URL" ]]; then
+    echo "Using custom clang from ${CI_CLANG_DISTRO_URL}"
+    wget $CI_CLANG_DISTRO_URL
+    ARCH_NAME=$(basename $CI_CLANG_DISTRO_URL)
+    tar -x -f $ARCH_NAME
+    export PATH=$PWD/esp-clang/bin:$PATH
+  fi
+
+  # Custom OpenOCD
+  if [[ ! -z "$OOCD_DISTRO_URL" && "$CI_JOB_STAGE" == "target_test" ]]; then
+    echo "Using custom OpenOCD from ${OOCD_DISTRO_URL}"
+    wget $OOCD_DISTRO_URL
+    ARCH_NAME=$(basename $OOCD_DISTRO_URL)
+    tar -x -f $ARCH_NAME
+    export OPENOCD_SCRIPTS=$PWD/openocd-esp32/share/openocd/scripts
+    export PATH=$PWD/openocd-esp32/bin:$PATH
+  fi
+
+  if [[ -n "$CI_PYTHON_TOOL_REPO" ]]; then
+    git clone --quiet --depth=1 -b ${CI_PYTHON_TOOL_BRANCH} https://gitlab-ci-token:${ESPCI_TOKEN}@${GITLAB_HTTPS_HOST}/espressif/${CI_PYTHON_TOOL_REPO}.git
+    pip install ./${CI_PYTHON_TOOL_REPO}
+    rm -rf ${CI_PYTHON_TOOL_REPO}
+  fi
+
+.show_ccache_statistics: &show_ccache_statistics |
+  # Show ccache statistics if enabled globally
+  test "$CI_CCACHE_STATS" == 1 && test -n "$(which ccache)" && ccache --show-stats || true
+
+.upload_failed_job_log_artifacts: &upload_failed_job_log_artifacts |
+  if [ $CI_JOB_STATUS = "failed" ]; then
+    python tools/ci/artifacts_handler.py upload --type logs
+  fi
+
+.before_script:minimal:
+  before_script:
+    - *common-before_scripts
+
+.before_script:build:macos:
+  before_script:
+    - *common-before_scripts
+    # On macOS, these tools need to be installed
+    - export IDF_TOOLS_PATH="${HOME}/.espressif_runner_${CI_RUNNER_ID}_${CI_CONCURRENT_ID}"
+    - $IDF_PATH/tools/idf_tools.py --non-interactive install cmake ninja
+    # This adds tools (compilers) and the version-specific Python environment to PATH
+    - *setup_tools_and_idf_python_venv
+    - fetch_submodules
+
+.before_script:build:
+  before_script:
+    - *common-before_scripts
+    - *setup_tools_and_idf_python_venv
+    - add_gitlab_ssh_keys
+    - fetch_submodules
+    - export EXTRA_CFLAGS=${PEDANTIC_CFLAGS}
+    - export EXTRA_CXXFLAGS=${PEDANTIC_CXXFLAGS}
+
+.after_script:build:ccache:
+  after_script:
+    - *show_ccache_statistics
+    - *upload_failed_job_log_artifacts
+
+##############################
+# Git Strategy Job Templates #
+##############################
+.git_init: &git_init |
+  mkdir -p "${CI_PROJECT_DIR}"
+  cd "${CI_PROJECT_DIR}"
+  git init
+
+.git_fetch_from_mirror_url_if_exists: &git_fetch_from_mirror_url_if_exists |
+  # check if set mirror
+  if [ -n "${LOCAL_GITLAB_HTTPS_HOST:-}" ] && [ -n "${ESPCI_TOKEN:-}" ]; then
+    MIRROR_REPO_URL="https://bot:${ESPCI_TOKEN}@${LOCAL_GITLAB_HTTPS_HOST}/${CI_PROJECT_PATH}"
+  elif [ -n "${LOCAL_GIT_MIRROR:-}" ]; then
+    MIRROR_REPO_URL="${LOCAL_GIT_MIRROR}/${CI_PROJECT_PATH}"
+  fi
+
+  # fetch from mirror first if set
+  if [ -n "${MIRROR_REPO_URL:-}" ]; then
+    if git remote -v | grep origin; then
+      git remote set-url origin "${MIRROR_REPO_URL}"
+    else
+      git remote add origin "${MIRROR_REPO_URL}"
+    fi
+    # mirror url may fail with authentication issue
+    git fetch origin --no-recurse-submodules || true
+  fi
+
+  # set remote url to CI_REPOSITORY_URL
+  if git remote -v | grep origin; then
+    git remote set-url origin "${CI_REPOSITORY_URL}"
+  else
+    git remote add origin "${CI_REPOSITORY_URL}"
+  fi
+
+.git_checkout_fetch_head: &git_checkout_fetch_head |
+  git checkout FETCH_HEAD
+  git clean ${GIT_CLEAN_FLAGS}
+
+# git diff requires two commits, with different CI env var
+#
+# By default, we use git strategy "clone" with depth 1 to speed up the clone process.
+# But for jobs requires running `git diff`, we need to fetch more commits to get the correct diffs.
+#
+# Since there's no way to get the correct git_depth before the job starts,
+# we can't set `GIT_DEPTH` in the job definition.
+#
+# Set git strategy to "none" and fetch manually instead.
+.before_script:fetch:git_diff:
+  variables:
+    GIT_STRATEGY: none
+  before_script:
+    - *git_init
+    - *git_fetch_from_mirror_url_if_exists
+    - |
+      # merged results pipelines, by default
+      if [[ -n $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA ]]; then
+        git fetch origin $CI_MERGE_REQUEST_DIFF_BASE_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
+        git fetch origin $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
+        export GIT_DIFF_OUTPUT=$(git diff --name-only $CI_MERGE_REQUEST_DIFF_BASE_SHA $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA)
+      # merge request pipelines, when the mr got conflicts
+      elif [[ -n $CI_MERGE_REQUEST_DIFF_BASE_SHA ]]; then
+        git fetch origin $CI_MERGE_REQUEST_DIFF_BASE_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
+        git fetch origin $CI_COMMIT_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
+        export GIT_DIFF_OUTPUT=$(git diff --name-only $CI_MERGE_REQUEST_DIFF_BASE_SHA $CI_COMMIT_SHA)
+      # other pipelines, like the protected branches pipelines
+      elif [[ "$CI_COMMIT_BEFORE_SHA" != "0000000000000000000000000000000000000000" ]]; then
+        git fetch origin $CI_COMMIT_BEFORE_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
+        git fetch origin $CI_COMMIT_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
+        export GIT_DIFF_OUTPUT=$(git diff --name-only $CI_COMMIT_BEFORE_SHA $CI_COMMIT_SHA)
+      else
+        # pipeline source could be web, scheduler, etc.
+        git fetch origin $CI_COMMIT_SHA --depth=2 ${GIT_FETCH_EXTRA_FLAGS}
+        export GIT_DIFF_OUTPUT=$(git diff --name-only $CI_COMMIT_SHA~1 $CI_COMMIT_SHA)
+      fi
+    - *git_checkout_fetch_head
+    - *common-before_scripts
+    - *setup_tools_and_idf_python_venv
+    - add_gitlab_ssh_keys
+
+# git describe requires commit history until the latest tag
+.before_script:fetch:git_describe:
+  variables:
+    GIT_STRAEGY: none
+  before_script:
+    - *git_init
+    - *git_fetch_from_mirror_url_if_exists
+    - |
+      git fetch origin refs/tags/"${LATEST_GIT_TAG}":refs/tags/"${LATEST_GIT_TAG}" --depth=1
+      git repack -d
+      git fetch origin $CI_COMMIT_SHA --shallow-since=$(git log -1 --format=%as "${LATEST_GIT_TAG}")
+    - *git_checkout_fetch_head
+    - *common-before_scripts
+    - *setup_tools_and_idf_python_venv
+    - add_gitlab_ssh_keys
+
+# target test runners may locate in different places
+# for runners set git mirror, we fetch from the mirror first, then fetch the HEAD commit
+.before_script:fetch:target_test:
+  variables:
+    GIT_STRATEGY: none
+  before_script:
+    - *git_init
+    - *git_fetch_from_mirror_url_if_exists
+    - git fetch origin "${CI_COMMIT_SHA}" --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
+    - *git_checkout_fetch_head
+    - *common-before_scripts
+    - *setup_tools_and_idf_python_venv
+    - add_gitlab_ssh_keys
+    # no submodules
+
+#############
+# `default` #
+#############
+default:
+  cache:
+    # pull only for most of the use cases since it's cache dir.
+    # Only set "push" policy for "upload_cache" stage jobs
+    - key: pip-cache-${LATEST_GIT_TAG}
+      fallback_keys:
+        - pip-cache
+      paths:
+        - .cache/pip
+      policy: pull
+    - key: submodule-cache-${LATEST_GIT_TAG}
+      fallback_keys:
+        - submodule-cache
+      paths:
+        - .cache/submodule_archives
+      policy: pull
+  before_script:
+    - *common-before_scripts
+    - *setup_tools_and_idf_python_venv
+    - add_gitlab_ssh_keys
+    - fetch_submodules
+  retry:
+    max: 2
+    when:
+      # In case of a runner failure we could hop to another one, or a network error could go away.
+      - runner_system_failure
+      # Job execution timeout may be caused by a network issue.
+      - job_execution_timeout

.gitlab/ci/danger.yml

@@ -0,0 +1,18 @@
+# Extenal DangerJS
+include:
+  - project: espressif/shared-ci-dangerjs
+    ref: master
+    file: danger.yaml
+
+run-danger-mr-linter:
+  stage: pre_check
+  variables:
+    GIT_STRATEGY: none # no repo checkout
+    ENABLE_CHECK_AREA_LABELS: 'true'
+    ENABLE_CHECK_DOCS_TRANSLATION: 'true'
+    ENABLE_CHECK_RELEASE_NOTES_DESCRIPTION: 'true'
+    ENABLE_CHECK_UPDATED_CHANGELOG: 'false'
+  before_script: []
+  cache: []
+  tags:
+    - dangerjs

.gitlab/ci/default-build-test-rules.yml

@@ -7,12 +7,10 @@
 #
 # This file should ONLY be used during bringup. Should be reset to empty after the bringup process
 extra_default_build_targets:
- - esp32c6
- - esp32h2
+ - esp32p4
 
 # bypass_check_test_targets:
-#  - esp32h2
-#  - esp32c6
+#  - esp32p4
 #
 # These lines would
 # - enable the README.md check for esp32c6. Don't forget to add the build jobs in .gitlab/ci/build.yml

.gitlab/ci/dependencies/dependencies.yml

@@ -3,15 +3,14 @@
   - esp32s2
   - esp32s3
   - esp32c3
-  - esp32h4
   - esp32c2
   - esp32c6
   - esp32h2
+  - esp32p4
 
 .target_test: &target_test
   - example_test
   - custom_test
-  - unit_test
   - component_ut
 
 ##############
@@ -26,6 +25,7 @@
     - downloadable-tools
   included_in:
     - build:target_test
+    - build:check
 
 # -------------------
 # Specific Build Jobs
@@ -40,14 +40,6 @@
     - build_system
     - downloadable-tools
 
-"build:windows":
-  labels:
-    - build
-    - windows
-  patterns:
-    - build_system
-    - windows
-
 "build:macos":
   labels:
     - build
@@ -55,9 +47,24 @@
     - macos_test  # for backward compatibility
   patterns:
     - build_system
-    - macos
+    - build_macos
     - downloadable-tools
 
+# ---------------------------
+# Add patterns to build rules
+# ---------------------------
+"patterns:template-app":
+  patterns:
+    - build_template-app
+  included_in:
+    - build:target_test
+
+"patterns:build-check":
+  patterns:
+    - build_check
+  included_in:
+    - build:check
+
 # ---------------
 # Build Test Jobs
 # ---------------
@@ -76,16 +83,6 @@
     - "build:{0}"
     - build:target_test
 
-build:integration_test:
-  labels:
-    - build
-  patterns:
-    - build_components
-    - build_system
-  included_in:
-    - build:target_test
-
-
 ####################
 # Target Test Jobs #
 ####################
@@ -108,22 +105,6 @@ build:integration_test:
 # -------------
 # Special Cases
 # -------------
-"test:component_ut-{0}": # component_ut will trigger by unit_test as well, since now we have 2 kinds of UT
-  matrix:
-    - *all_targets
-  labels:
-    - component_ut
-    - "component_ut_{0}"
-    - unit_test
-    - "unit_test_{0}"
-    - target_test
-  patterns:
-    - component_ut
-    - "build-component_ut-{0}"
-  included_in:
-    - build:component_ut
-    - "build:component_ut-{0}"
-    - build:target_test
 
 # To reduce the specific runners' usage.
 # Do not create these jobs by default patterns on development branches
@@ -132,14 +113,15 @@ build:integration_test:
   matrix:
     - *target_test
     - *all_targets
-    - - bt        # example_test_005
-      - wifi      # example_test_002, example_test*wifi*
-      - ethernet  # example_test*ethernet*
-      - sdio      # component_ut_pytest_esp32_sdio
+    - - wifi      # pytest*wifi*
+      - ethernet  # pytest*ethernet*
+      - sdio      # pytest*sdio*
       - usb       # USB Device & Host tests
-      - adc       # component_ut_pytest_esp32x_adc
+      - adc       # pytest*adc*
       - i154
       - flash_multi
+      - ecdsa
+      - nvs_encr_hmac
   patterns:
     - "{0}-{1}-{2}"
     - "{0}-{2}"
@@ -181,23 +163,6 @@ build:integration_test:
     - "build:example_test"
     - build:target_test
 
-"test:integration_test_{0}":
-  matrix:
-    - - wifi
-      - ble
-  labels:
-    - integration_test_{0}
-    - integration_test
-    - target_test
-  patterns:
-    - integration_test-{0}
-    - target_test-{0}
-    # - maybe others
-  included_in:
-    - test:integration_test
-    - build:integration_test
-    - build:target_test
-
 "test:host_test":
   labels:
     - host_test
@@ -213,37 +178,12 @@ build:integration_test:
 #################################
 # Triggered Only By Labels Jobs #
 #################################
-"labels:iperf_stress_test": # example_test
-  labels:
-    - iperf_stress_test
-  included_in:
-    - build:example_test
-    - build:example_test-esp32
-    - build:target_test
-
-"labels:weekend_test": # custom test
-  labels:
-    - weekend_test
-  included_in:
-    - build:custom_test
-    - build:custom_test-esp32
-    - build:target_test
-
 "labels:nvs_coverage": # host_test
   labels:
     - nvs_coverage
 
-"labels:fuzzer_test-weekend_test": # host test
+"labels:windows_pytest_build_system":
   labels:
-    - fuzzer_test
-    - weekend_test
-
-"labels-protected:lan8720":  # UT # FIXME: IDFCI-1176 temporary run this on master/release or with label
-  labels:
-    - lan8720
-  included_in:
-    - build:unit_test
-    - build:unit_test-esp32
-    - build:target_test
-    - build:component_ut
-    - build:component_ut-esp32
+    - windows
+  specific_rules:
+    - if-schedule-test-build-system-windows

.gitlab/ci/deploy.yml

@@ -1,8 +1,7 @@
 .deploy_job_template:
   stage: deploy
   image: $ESP_ENV_IMAGE
-  tags:
-    - deploy
+  tags: [ deploy ]
 
 # Check this before push_to_github
 check_submodule_sync:
@@ -10,11 +9,11 @@ check_submodule_sync:
     - .deploy_job_template
     - .rules:test:submodule
   stage: test_deploy
-  tags:
-    - github_sync
+  tags: [ brew, github_sync ]
   retry: 2
   variables:
-    GIT_STRATEGY: clone
+    # for brew runners, we always set GIT_STRATEGY to fetch
+    GIT_STRATEGY: fetch
     SUBMODULES_TO_FETCH: "none"
     PUBLIC_IDF_URL: "https://github.com/espressif/esp-idf.git"
   dependencies: []
@@ -31,51 +30,32 @@ check_submodule_sync:
 push_to_github:
   extends:
     - .deploy_job_template
-    - .before_script_minimal
-    - .rules:protected-no_label
-  dependencies: []
+    - .before_script:minimal
+    - .rules:push_to_github
+  needs:
+    - check_submodule_sync
+  tags: [ brew, github_sync ]
+  variables:
+    # for brew runners, we always set GIT_STRATEGY to fetch
+    GIT_STRATEGY: fetch
+    # github also need full record of commits
+    GIT_DEPTH: 0
   script:
     - add_github_ssh_keys
     - git remote remove github &>/dev/null || true
     - git remote add github git@github.com:espressif/esp-idf.git
     - tools/ci/push_to_github.sh
 
-deploy_test_result:
+deploy_update_SHA_in_esp-dockerfiles:
   extends:
     - .deploy_job_template
-    - .before_script_minimal
-    - .rules:ref:master-always
-  image: $CI_DOCKER_REGISTRY/bot-env:1
+    - .before_script:minimal
+    - .rules:protected-no_label-always
   dependencies: []
-  tags:
-    - deploy_test
-  artifacts:
-    when: always
-    paths:
-      - ${CI_PROJECT_DIR}/test-management/*.log
-    expire_in: 1 week
   variables:
-    BOT_ACCOUNT_CONFIG_FILE: "${CI_PROJECT_DIR}/test-management/Config/Account.local.yml"
-    TEST_RESULTS_PATH: "${CI_PROJECT_DIR}/TEST_RESULTS"
+    GIT_DEPTH: 2
+  tags: [ shiny, build ]
   script:
-    - add_gitlab_ssh_keys
-    - export GIT_SHA=$(echo ${CI_COMMIT_SHA} | cut -c 1-8)
-    - export REV_COUNT=$(git rev-list --count ${GIT_SHA} --)
-    - export SUMMARY="IDF CI test result for $GIT_SHA (r${REV_COUNT})"
-    # Download test result
-    - export PYTHONPATH="$IDF_PATH/tools:$IDF_PATH/tools/ci/python_packages:$PYTHONPATH"
-    - python3 ${IDF_PATH}/tools/ci/get_all_test_results.py --path ${TEST_RESULTS_PATH} --include_retried
-    - if [[ -z $(find ${TEST_RESULTS_PATH} -name "*.xml") ]]; then exit 0; fi
-    # Clone test-management repo
-    - retry_failed git clone $TEST_MANAGEMENT_REPO
-    - python3 $CHECKOUT_REF_SCRIPT test-management test-management
-    - cd test-management
-    - echo $BOT_JIRA_ACCOUNT > ${BOT_ACCOUNT_CONFIG_FILE}
-    # Make sure all requirements are installed
-    - pip3 install -r requirements.txt
-    # Update test cases
-    - python3 ImportTestCase.py $JIRA_TEST_MANAGEMENT_PROJECT from_xml -d ${TEST_RESULTS_PATH} -r $GIT_SHA -l IDFCI
-    # update test results
-    - python3 ImportTestResult.py -r "$GIT_SHA (r${REV_COUNT})" -j $JIRA_TEST_MANAGEMENT_PROJECT -s "$SUMMARY" -l IDFCI -p ${TEST_RESULTS_PATH} --pipeline_url ${CI_PIPELINE_URL}
-  # May need a long time to upload all test results.
-  timeout: 4 hours
+    - 'curl --header "PRIVATE-TOKEN: ${ESPCI_SCRIPTS_TOKEN}" -o create_MR_in_esp_dockerfile.sh $GITLAB_HTTP_SERVER/api/v4/projects/1260/repository/files/create_MR_in_esp_dockerfile%2Fcreate_MR_in_esp_dockerfile.sh/raw\?ref\=master'
+    - chmod +x create_MR_in_esp_dockerfile.sh
+    - ./create_MR_in_esp_dockerfile.sh

.gitlab/ci/docs.yml

@@ -3,6 +3,7 @@
   - "docs/**/*"
   - "**/*.rst"
   - "CONTRIBUTING.rst"
+  - "**/soc_caps.h"
 
 .patterns-docs-partial: &patterns-docs-partial
   - "components/**/*.h"
@@ -24,6 +25,9 @@
 .if-protected-no_label: &if-protected-no_label
   if: '($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_BRANCH =~ /^release\/v/ || $CI_COMMIT_TAG =~ /^v\d+\.\d+(\.\d+)?($|-)/) && $BOT_TRIGGER_WITH_LABEL == null'
 
+.if-qa-test-tag: &if-qa-test-tag
+  if: '$CI_COMMIT_TAG =~ /^qa-test/'
+
 .if-label-build_docs: &if-label-build_docs
   if: '$BOT_LABEL_BUILD_DOCS || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*build_docs(?:,[^,\n\r]+)*$/i'
 
@@ -31,18 +35,34 @@
   if: '$BOT_LABEL_DOCS_FULL || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*docs_full(?:,[^,\n\r]+)*$/i'
 
 .if-dev-push: &if-dev-push
-  if: '$CI_COMMIT_REF_NAME != "master" && $CI_COMMIT_BRANCH !~ /^release\/v/ && $CI_COMMIT_TAG !~ /^v\d+\.\d+(\.\d+)?($|-)/ && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event")'
+  if: '$CI_COMMIT_REF_NAME != "master" && $CI_COMMIT_BRANCH !~ /^release\/v/ && $CI_COMMIT_TAG !~ /^v\d+\.\d+(\.\d+)?($|-)/ && $CI_COMMIT_TAG !~ /^qa-test/ && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event")'
+
+.if-schedule: &if-schedule
+  if: '$CI_PIPELINE_SOURCE == "schedule"'
 
 .doc-rules:build:docs-full:
   rules:
-    - <<: *if-protected
+    - <<: *if-qa-test-tag
+      when: never
+    - <<: *if-schedule
     - <<: *if-label-build_docs
     - <<: *if-label-docs_full
     - <<: *if-dev-push
       changes: *patterns-docs-full
 
+.doc-rules:build:docs-full-prod:
+  rules:
+    - <<: *if-qa-test-tag
+      when: never
+    - <<: *if-protected-no_label
+
 .doc-rules:build:docs-partial:
   rules:
+    - <<: *if-qa-test-tag
+      when: never
+    - <<: *if-dev-push
+      changes: *patterns-docs-full
+      when: never
     - <<: *if-dev-push
       changes: *patterns-docs-partial
 
@@ -72,14 +92,14 @@ check_docs_lang_sync:
   stage: build_doc
   tags:
     - build_docs
-  dependencies: []
   script:
+    - if [ -n "${BREATHE_ALT_INSTALL_URL}" ]; then pip uninstall -y breathe && pip install -U ${BREATHE_ALT_INSTALL_URL}; fi
     - cd docs
     - build-docs -t $DOCTGT -bs $DOC_BUILDERS -l $DOCLANG build
   parallel:
     matrix:
       - DOCLANG: ["en", "zh_CN"]
-        DOCTGT: ["esp32", "esp32s2", "esp32s3", "esp32c3", "esp32c2", "esp32c6", "esp32h2"]
+        DOCTGT: ["esp32", "esp32s2", "esp32s3", "esp32c3", "esp32c2", "esp32c6", "esp32h2", "esp32p4"]
 
 check_docs_gh_links:
   image: $ESP_IDF_DOC_ENV_IMAGE
@@ -96,6 +116,24 @@ build_docs_html_full:
   extends:
     - .build_docs_template
     - .doc-rules:build:docs-full
+  needs:
+    - job: fast_template_app
+      artifacts: false
+      optional: true
+  artifacts:
+    when: always
+    paths:
+      - docs/_build/*/*/*.txt
+      - docs/_build/*/*/html/*
+    expire_in: 4 days
+  variables:
+    DOC_BUILDERS: "html"
+
+build_docs_html_full_prod:
+  extends:
+    - .build_docs_template
+    - .doc-rules:build:docs-full-prod
+  dependencies: [] # Stop build_docs jobs from downloading all previous job's artifacts
   artifacts:
     when: always
     paths:
@@ -109,6 +147,10 @@ build_docs_html_partial:
   extends:
     - .build_docs_template
     - .doc-rules:build:docs-partial
+  needs:
+    - job: fast_template_app
+      artifacts: false
+      optional: true
   artifacts:
     when: always
     paths:
@@ -122,12 +164,29 @@ build_docs_html_partial:
       - DOCLANG: "en"
         DOCTGT: "esp32"
       - DOCLANG: "zh_CN"
-        DOCTGT: "esp32c6"
+        DOCTGT: "esp32p4"
 
 build_docs_pdf:
   extends:
     - .build_docs_template
     - .doc-rules:build:docs-full
+  needs:
+    - job: fast_template_app
+      artifacts: false
+      optional: true
+  artifacts:
+    when: always
+    paths:
+      - docs/_build/*/*/latex/*
+    expire_in: 4 days
+  variables:
+    DOC_BUILDERS: "latex"
+
+build_docs_pdf_prod:
+  extends:
+    - .build_docs_template
+    - .doc-rules:build:docs-full-prod
+  dependencies: [] # Stop build_docs jobs from downloading all previous job's artifacts
   artifacts:
     when: always
     paths:
@@ -180,13 +239,12 @@ deploy_docs_production:
   # The DOCS_PROD_* variables used by this job are "Protected" so these branches must all be marked "Protected" in Gitlab settings
   extends:
     - .deploy_docs_template
-  rules:
-    - <<: *if-protected-no_label
+    - .doc-rules:build:docs-full-prod
   stage: post_deploy
   dependencies:  # set dependencies to null to avoid missing artifacts issue
   needs: # ensure runs after push_to_github succeeded
-    - build_docs_html_full
-    - build_docs_pdf
+    - build_docs_html_full_prod
+    - build_docs_pdf_prod
     - job: push_to_github
       artifacts: false
   variables:
@@ -201,8 +259,7 @@ deploy_docs_production:
 check_doc_links:
   extends:
     - .build_docs_template
-  rules:
-    - <<: *if-protected-no_label
+    - .doc-rules:build:docs-full-prod
   stage: post_deploy
   needs:
     - job: deploy_docs_production

.gitlab/ci/host-test.yml

@@ -4,7 +4,7 @@
   image: $ESP_ENV_IMAGE
   tags:
     - host_test
-  dependencies: []
+  dependencies: # set dependencies to null to avoid missing artifacts issue
   # run host_test jobs immediately, only after upload cache
   needs:
     - job: upload-pip-cache
@@ -13,6 +13,7 @@
     - job: upload-submodules-cache
       optional: true
       artifacts: false
+    - pipeline_variables
 
 test_nvs_on_host:
   extends: .host_test_template
@@ -40,22 +41,6 @@ test_partition_table_on_host:
     - cd components/partition_table/test_gen_esp32part_host
     - ./gen_esp32part_tests.py
 
-test_wl_on_host:
-  extends: .host_test_template
-  artifacts:
-    paths:
-      - components/wear_levelling/test_wl_host/coverage_report.zip
-    expire_in: 1 week
-  script:
-    - cd components/wear_levelling/test_wl_host
-    - make test
-
-test_fatfs_on_host:
-  extends: .host_test_template
-  script:
-    - cd components/fatfs/test_fatfs_host/
-    - make test
-
 test_ldgen_on_host:
   extends: .host_test_template
   script:
@@ -108,22 +93,18 @@ test_certificate_bundle_on_host:
     - cd components/mbedtls/esp_crt_bundle/test_gen_crt_bundle/
     - ./test_gen_crt_bundle.py
 
-
-test_idf_py:
+test_gdbstub_on_host:
   extends: .host_test_template
-  variables:
-    LC_ALL: C.UTF-8
   script:
-    - cd ${IDF_PATH}/tools/test_idf_py
-    - ./test_idf_py.py
-    - ./test_hints.py
+    - cd components/esp_gdbstub/test_gdbstub_host
+    - make test
 
 # Test for create virtualenv. It must be invoked from Python, not from virtualenv.
 # Use docker image system python without any extra dependencies
-test_idf_tools:
+test_cli_installer:
   extends:
     - .host_test_template
-    - .before_script_minimal
+    - .before_script:minimal
   artifacts:
     when: on_failure
     paths:
@@ -135,10 +116,10 @@ test_idf_tools:
     entrypoint: [""]  # use system python3. no extra pip package installed
   script:
     # Tools must be downloaded for testing
-    - python3 ${IDF_PATH}/tools/idf_tools.py download
+    - python3 ${IDF_PATH}/tools/idf_tools.py download required qemu-riscv32 qemu-xtensa
     - cd ${IDF_PATH}/tools/test_idf_tools
     - python3 -m pip install jsonschema
-    - python3 ./test_idf_tools.py
+    - python3 ./test_idf_tools.py -v
     - python3 ./test_idf_tools_python_env.py
 
 .test_efuse_table_on_host_template:
@@ -185,30 +166,6 @@ test_efuse_table_on_host_esp32c6:
   variables:
     IDF_TARGET: esp32c6
 
-test_efuse_table_on_host_esp32h4:
-  extends: .test_efuse_table_on_host_template
-  variables:
-    IDF_TARGET: esp32h4
-
-test_espcoredump:
-  extends: .host_test_template
-  artifacts:
-    when: always
-    paths:
-      - components/espcoredump/test/**/.coverage
-      - components/espcoredump/test/**/output
-    expire_in: 1 week
-  variables:
-    IDF_COREDUMP_ELF_REPO: "https://gitlab-ci-token:${BOT_TOKEN}@${CI_SERVER_HOST}:${CI_SERVER_PORT}/idf/idf-coredump-elf.git"
-    IDF_COREDUMP_ELF_TAG: idf-20220928
-    # install CMake version specified in tools.json
-    SETUP_TOOLS_LIST: "all"
-  script:
-    - eval $($IDF_PATH/tools/idf_tools.py export)
-    - retry_failed git clone ${IDF_COREDUMP_ELF_REPO} -b $IDF_COREDUMP_ELF_TAG
-    - cd ${IDF_PATH}/components/espcoredump/test/
-    - ./test_espcoredump.sh ${CI_PROJECT_DIR}/idf-coredump-elf
-
 test_logtrace_proc:
   extends: .host_test_template
   artifacts:
@@ -233,43 +190,39 @@ test_sysviewtrace_proc:
     - cd ${IDF_PATH}/tools/esp_app_trace/test/sysview
     - ./test.sh
 
-test_mkdfu:
-  extends: .host_test_template
-  variables:
-    LC_ALL: C.UTF-8
-  script:
-    - cd ${IDF_PATH}/tools/test_mkdfu
-    - ./test_mkdfu.py
-
-test_mkuf2:
-  extends: .host_test_template
-  script:
-    - cd ${IDF_PATH}/tools/test_mkuf2
-    - ./test_mkuf2.py
-
-test_autocomplete:
+test_tools:
   extends:
     - .host_test_template
   artifacts:
-    when: on_failure
+    when: always
     paths:
       - ${IDF_PATH}/*.out
+      - ${IDF_PATH}/XUNIT_*.xml
+    reports:
+      junit: ${IDF_PATH}/XUNIT_*.xml
     expire_in: 1 week
+  variables:
+    LC_ALL: C.UTF-8
+    INSTALL_QEMU: 1  # for test_idf_qemu.py
   script:
-    - ${IDF_PATH}/tools/ci/test_autocomplete.py
-
-test_detect_python:
-  extends:
-    - .host_test_template
-  script:
+    - stat=0
+    - cd ${IDF_PATH}/tools/ci/test_autocomplete
+    - pytest --noconftest test_autocomplete.py --junitxml=${IDF_PATH}/XUNIT_AUTOCOMP.xml                  || stat=1
+    - cd ${IDF_PATH}/tools/test_idf_py
+    - pytest --noconftest test_idf_py.py       --junitxml=${IDF_PATH}/XUNIT_IDF_PY.xml                    || stat=1
+    - pytest --noconftest test_hints.py        --junitxml=${IDF_PATH}/XUNIT_HINTS.xml                     || stat=1
+    - pytest --noconftest test_idf_qemu.py     --junitxml=${IDF_PATH}/XUNIT_IDF_PY_QEMU.xml               || stat=1
+    - cd ${IDF_PATH}/tools/test_mkdfu
+    - pytest --noconftest test_mkdfu.py        --junitxml=${IDF_PATH}/XUNIT_MKDFU.xml                     || stat=1
     - cd ${IDF_PATH}
-    - shellcheck -s sh tools/detect_python.sh
-    - shellcheck -s bash tools/detect_python.sh
-    - shellcheck -s dash tools/detect_python.sh
+    - shellcheck -s sh tools/detect_python.sh                                                             || stat=1
+    - shellcheck -s bash tools/detect_python.sh                                                           || stat=1
+    - shellcheck -s dash tools/detect_python.sh                                                           || stat=1
     - "bash -c '. tools/detect_python.sh && echo Our Python: ${ESP_PYTHON?Python is not set}'"
     - "dash -c '. tools/detect_python.sh && echo Our Python: ${ESP_PYTHON?Python is not set}'"
     - "zsh -c '. tools/detect_python.sh && echo Our Python: ${ESP_PYTHON?Python is not set}'"
     - "fish -c 'source tools/detect_python.fish && echo Our Python: $ESP_PYTHON'"
+    - exit "$stat"
 
 test_split_path_by_spaces:
   extends: .host_test_template
@@ -303,8 +256,8 @@ test_sockets_on_host:
     - grep "Socket unable to connect" test.log
     # test the udp-client example with lwip sockets
     - cd ${IDF_PATH}/examples/protocols/sockets/udp_client
-    - echo 'CONFIG_EXAMPLE_IPV4_ADDR="127.0.0.1"' >> sdkconfig.defaults
     - idf.py --preview set-target linux
+    - cat sdkconfig.ci.linux > sdkconfig
     - idf.py build
     - timeout 5 ./build/udp_client.elf >test.log || true
     - grep "Message sent" test.log
@@ -325,8 +278,7 @@ test_gen_soc_caps_kconfig:
 test_pytest_qemu:
   extends:
     - .host_test_template
-    - .before_script_build_jobs
-  image: $QEMU_IMAGE
+    - .before_script:build
   artifacts:
     when: always
     paths:
@@ -335,28 +287,40 @@ test_pytest_qemu:
     reports:
       junit: XUNIT_RESULT.xml
     expire_in: 1 week
+  allow_failure: true # IDFCI-1752
+  parallel:
+    matrix:
+      - IDF_TARGET: [esp32, esp32c3]
+  variables:
+    INSTALL_QEMU: 1
   script:
     - run_cmd python tools/ci/ci_build_apps.py . -vv
-      --target esp32
+      --target $IDF_TARGET
       --pytest-apps
       -m qemu
+      --collect-app-info "list_job_${CI_JOB_NAME_SLUG}.txt"
+      --modified-components ${MODIFIED_COMPONENTS}
+      --modified-files ${MODIFIED_FILES}
     - retry_failed git clone $KNOWN_FAILURE_CASES_REPO known_failure_cases
-    - pytest
-      --target esp32
+    - run_cmd pytest
+      --target $IDF_TARGET
       -m qemu
       --embedded-services idf,qemu
       --junitxml=XUNIT_RESULT.xml
-      --known-failure-cases-file known_failure_cases/known_failure_cases.txt
+      --ignore-result-files known_failure_cases/known_failure_cases.txt
+      --app-info-filepattern \"list_job_*.txt\"
+      --qemu-extra-args \"-global driver=timer.$IDF_TARGET.timg,property=wdt_disable,value=true\"
 
 test_pytest_linux:
   extends:
     - .host_test_template
-    - .before_script_build_jobs
+    - .before_script:build
   artifacts:
     when: always
     paths:
       - XUNIT_RESULT.xml
       - pytest_embedded_log/
+      - "**/build*/build_log.txt"
     reports:
       junit: XUNIT_RESULT.xml
     expire_in: 1 week
@@ -365,6 +329,13 @@ test_pytest_linux:
       --target linux
       --pytest-apps
       -m host_test
+      --collect-app-info "list_job_${CI_JOB_NAME_SLUG}.txt"
+      --modified-components ${MODIFIED_COMPONENTS}
+      --modified-files ${MODIFIED_FILES}
     - retry_failed git clone $KNOWN_FAILURE_CASES_REPO known_failure_cases
-    - run_cmd pytest --target linux -m host_test --junitxml=XUNIT_RESULT.xml
-      --known-failure-cases-file known_failure_cases/known_failure_cases.txt
+    - run_cmd pytest
+      --target linux
+      -m host_test
+      --junitxml=XUNIT_RESULT.xml
+      --ignore-result-files known_failure_cases/known_failure_cases.txt
+      --app-info-filepattern \"list_job_*.txt\"

.gitlab/ci/integration_test.yml

@@ -0,0 +1,69 @@
+# generate dynamic integration pipeline by `idf-integration-ci` project
+
+.patterns-integration_test: &patterns-integration_test
+  # add all possible patterns to make sure `gen_integration_pipeline` can be triggered.
+  # fine-grained control will be done while generating the pipeline
+  # find `patterns` in `idf-integration-ci` project
+  - "components/**/*"
+  - "tools/**/*"
+  - ".gitlab-ci.yml"
+  - ".gitlab/ci/common.yml"
+  - ".gitlab/ci/integration_test.yml"
+  - ".gitmodules"
+  - "CMakeLists.txt"
+  - "install.sh"
+  - "export.sh"
+  - "Kconfig"
+  - "sdkconfig.rename"
+
+# Simplify the rules
+.integration_test_rules:
+  rules:
+    - if: '$CI_PIPELINE_SOURCE != "merge_request_event"'
+    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
+      changes: *patterns-integration_test
+    # support trigger by ci labels
+    - if: '$CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*target_test(?:,[^,\n\r]+)*$/i'
+    - if: '$CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*integration_test(?:,[^,\n\r]+)*$/i'
+    - if: '$CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*build(?:,[^,\n\r]+)*$/i'
+
+gen_integration_pipeline:
+  extends:
+    - .before_script:minimal
+    - .integration_test_rules
+  image: ${CI_INTEGRATION_ASSIGN_ENV}
+  stage: assign_test
+  cache: []
+  tags:
+    - assign_test
+  variables:
+    SUBMODULES_TO_FETCH: "none"
+    GIT_LFS_SKIP_SMUDGE: 1
+  needs:
+    - job: fast_template_app
+      artifacts: false
+      optional: true
+  artifacts:
+    paths:
+      - idf-integration-ci/child_pipeline/
+    expire_in: 2 weeks
+  script:
+    - add_gitlab_ssh_keys
+    - retry_failed git clone ${CI_GEN_INTEGRATION_PIPELINE_REPO} idf-integration-ci
+    - python $CHECKOUT_REF_SCRIPT idf-integration-ci idf-integration-ci
+    - cd idf-integration-ci
+    - python tools/generate_child_pipeline.py -o child_pipeline/
+
+child_integration_test_pipeline:
+  extends:
+    - .integration_test_rules
+  stage: assign_test
+  needs:
+    - gen_integration_pipeline
+  trigger:
+    include:
+      - artifact: idf-integration-ci/child_pipeline/pipeline.yml
+        job: gen_integration_pipeline
+    forward:
+      yaml_variables: false
+    strategy: depend

.gitlab/ci/pre_check.yml

@@ -5,43 +5,17 @@
     - host_test
   dependencies: []
 
-.check_pre_commit_template:
+check_pre_commit:
   extends:
     - .pre_check_template
-    - .before_script_minimal
+    - .before_script:minimal
   image: $PRE_COMMIT_IMAGE
-
-check_pre_commit_master_release:
-  extends:
-    - .check_pre_commit_template
-    - .rules:protected
+  needs:
+    - pipeline_variables
   script:
-    - git diff-tree --no-commit-id --name-only -r $PIPELINE_COMMIT_SHA | xargs pre-commit run --files
-
-check_pre_commit_MR:
-  extends:
-    - .check_pre_commit_template
-    - .rules:dev
-  script:
-    - python ${CI_PROJECT_DIR}/tools/ci/ci_get_mr_info.py files ${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME} | xargs pre-commit run --files
-
-check_MR_style_dangerjs:
-  extends:
-    - .pre_check_template
-  image: node:14.18.0-alpine3.14
-  variables:
-    DANGER_GITLAB_API_TOKEN: ${ESPCI_TOKEN}
-    DANGER_GITLAB_HOST: ${GITLAB_HTTP_SERVER}
-    DANGER_GITLAB_API_BASE_URL: ${GITLAB_HTTP_SERVER}/api/v4
-    DANGER_JIRA_USER: ${DANGER_JIRA_USER}
-    DANGER_JIRA_PASSWORD: ${DANGER_JIRA_PASSWORD}
-  before_script:
-    - npm install -g danger@11.2.3 --silent --no-progress > /dev/null
-    - npm install axios@1.3.3 --silent --no-progress > /dev/null
-  script:
-    - danger ci --dangerfile=".gitlab/dangerjs/dangerfile.js" --failOnErrors -v
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
+    - fetch_submodules
+    - pre-commit run --files $MODIFIED_FILES
+    - pre-commit run --hook-stage post-commit validate-sbom-manifest
 
 check_version:
   # Don't run this for feature/bugfix branches, so that it is possible to modify
@@ -49,33 +23,22 @@ check_version:
   extends:
     - .pre_check_template
     - .rules:protected
+    - .before_script:fetch:git_describe
   script:
     - export IDF_PATH=$PWD
     - tools/ci/check_idf_version.sh
 
-check_rom_api_header:
+check_api_usage:
   extends: .pre_check_template
   script:
     - tools/ci/check_examples_rom_header.sh
     - tools/ci/check_api_violation.sh
-
-test_check_kconfigs:
-  extends: .pre_check_template
-  artifacts:
-    when: on_failure
-    paths:
-      - components/*/Kconfig*.new
-      - examples/*/*/*/Kconfig*.new
-      - examples/*/*/*/*/Kconfig*.new
-      - tools/*/Kconfig*.new
-      - tools/*/*/Kconfig*.new
-      - tools/*/*/*/Kconfig*.new
-    expire_in: 1 week
-  script:
-    - python ${IDF_PATH}/tools/ci/test_check_kconfigs.py
+    - tools/ci/check_examples_extra_component_dirs.sh
 
 check_blobs:
-  extends: .pre_check_template
+  extends:
+    - .pre_check_template
+    - .rules:build:check
   variables:
     SUBMODULES_TO_FETCH: "components/esp_wifi/lib;components/esp_phy/lib;components/esp_coex/lib"
   script:
@@ -100,7 +63,7 @@ check_blobs:
 check_public_headers:
   extends:
     - .pre_check_template
-    - .rules:build
+    - .rules:build:check
   script:
     - IDF_TARGET=esp32 python tools/ci/check_public_headers.py --jobs 4 --prefix xtensa-esp32-elf-
     - IDF_TARGET=esp32s2 python tools/ci/check_public_headers.py --jobs 4 --prefix xtensa-esp32s2-elf-
@@ -109,11 +72,12 @@ check_public_headers:
     - IDF_TARGET=esp32c2 python tools/ci/check_public_headers.py --jobs 4 --prefix riscv32-esp-elf-
     - IDF_TARGET=esp32c6 python tools/ci/check_public_headers.py --jobs 4 --prefix riscv32-esp-elf-
     - IDF_TARGET=esp32h2 python tools/ci/check_public_headers.py --jobs 4 --prefix riscv32-esp-elf-
+    - IDF_TARGET=esp32p4 python tools/ci/check_public_headers.py --jobs 4 --prefix riscv32-esp-elf-
 
 check_chip_support_components:
   extends:
     - .pre_check_template
-    - .rules:build
+    - .rules:build:check
   artifacts:
     when: on_failure
     paths:
@@ -128,7 +92,7 @@ check_chip_support_components:
 check_esp_err_to_name:
   extends:
     - .pre_check_template
-    - .rules:build
+    - .rules:build:check
   artifacts:
     when: on_failure
     paths:
@@ -148,10 +112,12 @@ check_esp_system:
 
 # For release tag pipelines only, make sure the tag was created with 'git tag -a' so it will update
 # the version returned by 'git describe'
+# Don't forget to update the env var `LATEST_GIT_TAG` in .gitlab/ci/common.yml
 check_version_tag:
   extends:
     - .pre_check_template
     - .rules:tag:release
+    - .before_script:fetch:git_describe
   script:
     - (git cat-file -t $CI_COMMIT_REF_NAME | grep tag) || (echo "ESP-IDF versions must be annotated tags." && exit 1)
 
@@ -164,7 +130,7 @@ check_artifacts_expire_time:
 check_test_scripts_build_test_rules:
   extends:
     - .pre_check_template
-    - .before_script_build_jobs
+    - .before_script:build
   script:
     # required pytest related packages
     - run_cmd bash install.sh --enable-pytest
@@ -173,7 +139,29 @@ check_test_scripts_build_test_rules:
 check_configure_ci_environment_parsing:
   extends:
     - .pre_check_template
-    - .before_script_build_jobs
+    - .before_script:build
+    - .rules:build
   script:
     - cd tools/ci
     - python -m unittest ci_build_apps.py
+
+pipeline_variables:
+  extends:
+    - .pre_check_template
+    - .before_script:fetch:git_diff
+  tags:
+    - build
+  script:
+    - MODIFIED_FILES=$(echo $GIT_DIFF_OUTPUT | xargs)
+    - echo "MODIFIED_FILES=$MODIFIED_FILES" >> pipeline.env
+    - echo "MODIFIED_COMPONENTS=$(run_cmd python tools/ci/ci_get_mr_info.py components --modified-files $MODIFIED_FILES | xargs)" >> pipeline.env
+    - |
+      if echo "$CI_MERGE_REQUEST_LABELS" | egrep "(^|,)BUILD_AND_TEST_ALL_APPS(,|$)"; then
+        echo "BUILD_AND_TEST_ALL_APPS=1" >> pipeline.env
+      fi
+    - cat pipeline.env
+    - python tools/ci/artifacts_handler.py upload --type modified_files_and_components_report
+  artifacts:
+    reports:
+      dotenv: pipeline.env
+    expire_in: 4 days

.gitlab/ci/static-code-analysis.yml

@@ -5,44 +5,37 @@ clang_tidy_check:
     - .rules:patterns:clang_tidy
   artifacts:
     paths:
-      - $OUTPUT_DIR
+      - clang_tidy_reports/
     when: always
     expire_in: 1 day
   variables:
-    CLANG_TIDY_DIRS_TXT: ${CI_PROJECT_DIR}/tools/ci/clang_tidy_dirs.txt
-    RULES_FILE: ${CI_PROJECT_DIR}/tools/ci/static-analysis-rules.yml
-    OUTPUT_DIR: ${CI_PROJECT_DIR}/clang_tidy_reports
     IDF_TOOLCHAIN: clang
   script:
-    - internal_pip_install $CLANG_TIDY_RUNNER_PROJ pyclang
-    - export PATH=$PATH:$(python -c "import sys; print(sys.executable.rsplit('/', 1)[0])")
-    - dirs=$(cat ${CLANG_TIDY_DIRS_TXT} | while read line; do echo ${CI_PROJECT_DIR}/${line}; done | xargs)
-    - run_cmd idf_clang ${dirs}
-      --output-path ${OUTPUT_DIR}
-      --limit-file ${RULES_FILE}
+    - run_cmd idf_clang_tidy $(cat tools/ci/clang_tidy_dirs.txt | xargs)
+      --output-path clang_tidy_reports
+      --limit-file tools/ci/static-analysis-rules.yml
       --xtensa-include-dir
-      --run-clang-tidy-py run-clang-tidy
 
 check_pylint:
   extends:
     - .pre_check_template
     - .rules:patterns:python-files
-    - .before_script_minimal
-  image: $SONARQUBE_SCANNER_IMAGE
+  needs:
+    - pipeline_variables
   artifacts:
     when: always
-    paths:
-      - pylint-report.txt
+    reports:
+      codequality: pylint.json
     expire_in: 1 week
   script:
-    - export PYTHONPATH="$IDF_PATH/tools:$IDF_PATH/tools/ci/python_packages:$PYTHONPATH"
     - |
       if [ -n "$CI_MERGE_REQUEST_IID" ]; then
-        export files=$(python ${CI_PROJECT_DIR}/tools/ci/ci_get_mr_info.py files ${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME} | grep ".py");
+        export files=$(echo $GIT_DIFF_OUTPUT | grep ".py$" | xargs);
       else
-        export files=$(find . -iname "*.py" -print);
+        export files=$(git ls-files "*.py" | xargs);
       fi
-    - pylint --rcfile=.pylintrc $files -r n --output-format=parseable > pylint-report.txt || exit 0
+    - if [ -z "$files" ]; then echo "No python files found"; exit 0; fi
+    - run_cmd pylint --exit-zero --load-plugins=pylint_gitlab --output-format=gitlab-codeclimate:pylint.json $files
 
 # build stage
 # Sonarqube related jobs put here for this reason:
@@ -83,7 +76,6 @@ check_pylint:
     expire_in: 1 week
   dependencies:  # Here is not a hard dependency relationship, could be skipped when only python files changed. so we do not use "needs" here.
     - clang_tidy_check
-    - check_pylint
 
 code_quality_check:
   extends:
@@ -92,7 +84,7 @@ code_quality_check:
   allow_failure: true  # since now it's using exit code to indicate the code analysis result,
                        # we don't want to block ci when critical issues founded
   script:
-    - export CI_MERGE_REQUEST_COMMITS=$(python ${CI_PROJECT_DIR}/tools/ci/ci_get_mr_info.py commits ${CI_COMMIT_REF_NAME} | tr '\n' ',')
+    - export CI_MERGE_REQUEST_COMMITS=$(python ${CI_PROJECT_DIR}/tools/ci/ci_get_mr_info.py commits --src-branch ${CI_COMMIT_REF_NAME} | tr '\n' ',')
     # test if this branch have merge request, if not, exit 0
     - test -n "$CI_MERGE_REQUEST_IID" || exit 0
     - test -n "$CI_MERGE_REQUEST_COMMITS" || exit 0
@@ -107,7 +99,6 @@ code_quality_check:
       -Dsonar.gitlab.ref_name=$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME
       -Dsonar.host.url=$SONAR_HOST_URL
       -Dsonar.login=$SONAR_LOGIN
-      -Dsonar.python.pylint.reportPath=pylint-report.txt
 
 code_quality_report:
   extends:
@@ -124,4 +115,3 @@ code_quality_report:
       -Dsonar.gitlab.ref_name=$CI_COMMIT_REF_NAME
       -Dsonar.host.url=$SONAR_HOST_URL
       -Dsonar.login=$SONAR_LOGIN
-      -Dsonar.python.pylint.reportPath=pylint-report.txt

.gitlab/ci/upload_cache.yml

@@ -9,13 +9,13 @@
 upload-pip-cache:
   extends:
     - .upload_cache_template
-    - .before_script_minimal
-    - .rules:patterns:python-cache
+    - .before_script:minimal
+    - .rules:upload-python-cache
   tags:
     - $GEO
     - cache
   cache:
-    key: pip-cache
+    key: pip-cache-${LATEST_GIT_TAG}
     paths:
       - .cache/pip
     policy: push
@@ -29,13 +29,13 @@ upload-pip-cache:
 upload-submodules-cache:
   extends:
     - .upload_cache_template
-    - .before_script_minimal
-    - .rules:patterns:submodule
+    - .before_script:minimal
+    - .rules:upload-submodule-cache
   tags:
     - $GEO
     - cache
   cache:
-    key: submodule-cache
+    key: submodule-cache-${LATEST_GIT_TAG}
     paths:
       - .cache/submodule_archives
     policy: push

.gitlab/dangerjs/dangerfile.js

@@ -1,44 +0,0 @@
-/*
- * Modules with checks are stored in ".gitlab/dangerjs/<module_name>". To import them, use path relative to "dangerfile.js"
- */
-
-async function runChecks() {
-    // Checks for merge request title
-    require("./mrTitleNoDraftOrWip.js")();
-
-    // Checks for merge request description
-    require("./mrDescriptionLongEnough.js")();
-    require("./mrDescriptionHasReleaseNotes.js")();
-    await require("./mrDescriptionJiraLinks.js")();
-
-    // Checks for documentation
-    require("./mrDocsTranslation.js")();
-
-    // Checks for MR commits
-    require("./mrCommitsTooManyCommits.js")();
-    require("./mrCommitsCommitMessage.js")();
-    require("./mrCommitsEmail.js")();
-
-    // Checks for MR code
-    require("./mrSizeTooLarge.js")();
-
-    // Checks for MR area labels
-    await require("./mrAreaLabels.js")();
-
-    // Add success log if no issues
-    if (
-        results.fails.length === 0 &&
-        results.warnings.length === 0 &&
-        results.messages.length === 0
-    ) {
-        return message("Good Job! All checks are passing!");
-    }
-}
-
-runChecks();
-
-// Add retry link
-const retryLink = `${process.env.DANGER_GITLAB_HOST}/${process.env.CI_PROJECT_PATH}/-/jobs/${process.env.CI_JOB_ID}`;
-markdown(
-    `***\n#### :repeat: You can enforce automatic MR checks by retrying the [DangerJS job](${retryLink})\n***`
-);

.gitlab/dangerjs/mrAreaLabels.js

@@ -1,20 +0,0 @@
-/**
- * Check if MR has area labels (light blue labels)
- *
- * @dangerjs WARN
- */
-module.exports = async function () {
-    const projectId = 103; // ESP-IDF
-    const areaLabelColor = /^#d2ebfa$/i; // match color code (case-insensitive)
-    const projectLabels = await danger.gitlab.api.Labels.all(projectId); // Get all project labels
-    const areaLabels = projectLabels
-        .filter((label) => areaLabelColor.test(label.color))
-        .map((label) => label.name); // Filter only area labels
-    const mrLabels = danger.gitlab.mr.labels; // Get MR labels
-
-    if (!mrLabels.some((label) => areaLabels.includes(label))) {
-        warn(
-            `Please add some [area labels](${process.env.DANGER_GITLAB_HOST}/espressif/esp-idf/-/labels) to this MR.`
-        );
-    }
-};

.gitlab/dangerjs/mrCommitsCommitMessage.js

@@ -1,70 +0,0 @@
-/**
- * Check if commit messages are sufficiently descriptive (not too short).
- *
- * Search for commit messages that appear to be automatically generated by Gitlab or temporary messages and report them.
- *
- * @dangerjs WARN
- */
-
-module.exports = function () {
-    const mrCommits = danger.gitlab.commits;
-
-    const detectRegexes = [
-        /^Rebased.*onto.*/i, // Automatically generated message by Gitlab
-        /^Fast-forwarded.*to.*/i, // Automatically generated message by Gitlab
-        /^Applied suggestion to.*/i, // Automatically generated message by Gitlab
-        /^Suggestion applied for line.*/i, // Automatically generated message by Gitlab
-        /^Opened merge request/i, // Automatically generated message by Gitlab
-        /^Merged.*/i, // Automatically generated message by Gitlab
-        /^Opened issue #\d+:.*/i, // Automatically generated message by Gitlab
-        /^Closed issue #\d+:.*/i, // Automatically generated message by Gitlab
-        /^Tagged.*as.*/i, // Automatically generated message by Gitlab
-        /^Deleted tag.*/i, // Automatically generated message by Gitlab
-        /^WIP.*/i, // Message starts with prefix "WIP"
-        /^Cleaned.*/i, // Message starts "Cleaned"
-        /clean ?up/i, // Message contains "clean up"
-        /^[^A-Za-z0-9\s].*/, // Message starts with special characters
-    ];
-
-    // Search for the messages in each commit
-    let partMessages = [];
-    for (const commit of mrCommits) {
-        const commitMessage = commit.message;
-        const commitMessageTitle = commit.title;
-
-        // Check if the commit message contains any Jira ticket references
-        const jiraTicketRegex = /[A-Z0-9]+-[0-9]+/g;
-        const jiraTicketMatches = commitMessage.match(jiraTicketRegex);
-        if (jiraTicketMatches) {
-            const jiraTicketNames = jiraTicketMatches.join(", ");
-            partMessages.push(
-                `- the commit message \`${commitMessageTitle}\` probably contains Jira ticket reference (\`${jiraTicketNames}\`). Please remove Jira tickets from commit messages.`
-            );
-            continue;
-        }
-
-        // Check if the commit message matches any regex from "detectRegexes"
-        if (detectRegexes.some((regex) => commitMessage.match(regex))) {
-            partMessages.push(
-                `- the commit message \`${commitMessageTitle}\` appears to be a temporary or automatically generated message`
-            );
-            continue;
-        }
-
-        // Check if the commit message is not too short
-        const shortCommitMessageThreshold = 20; // commit message is considered too short below this number of characters
-        if (commitMessage.length < shortCommitMessageThreshold) {
-            partMessages.push(
-                `- the commit message \`${commitMessageTitle}\` may not be sufficiently descriptive`
-            );
-        }
-    }
-
-	// Create report
-	if (partMessages.length) {
-		partMessages.sort();
-		let dangerMessage = `\nSome issues found for the commit messages in this MR:\n${partMessages.join('\n')}
-		\nPlease consider updating these commit messages. It is recommended to follow this [commit messages guide](https://gitlab.espressif.cn:6688/espressif/esp-idf/-/wikis/dev-proc/Commit-messages)`;
-		warn(dangerMessage);
-	}
-};

.gitlab/dangerjs/mrCommitsEmail.js

@@ -1,16 +0,0 @@
-/**
- * Check if the author is accidentally making a commit using a personal email
- *
- * @dangerjs INFO
- */
-module.exports = function () {
-    const mrCommitAuthorEmails = danger.gitlab.commits.map(commit => commit.author_email);
-    const mrCommitCommitterEmails = danger.gitlab.commits.map(commit => commit.committer_email);
-	const emailPattern = /.*@espressif\.com/;
-	const filteredEmails = [...mrCommitAuthorEmails, ...mrCommitCommitterEmails].filter((email) => !emailPattern.test(email));
-	if (filteredEmails.length) {
-		return message(
-			`Some of the commits were authored or committed by developers outside Espressif: ${filteredEmails.join(', ')}. Please check if this is expected.`
-		);
-	}
-};

.gitlab/dangerjs/mrCommitsTooManyCommits.js

@@ -1,15 +0,0 @@
-/**
- * Check if MR has not an excessive numbers of commits (if squashed)
- *
- * @dangerjs INFO
- */
-module.exports = function () {
-    const tooManyCommitThreshold = 2; // above this number of commits, squash commits is suggested
-    const mrCommits = danger.gitlab.commits;
-
-    if (mrCommits.length > tooManyCommitThreshold) {
-        return message(
-            `You might consider squashing your ${mrCommits.length} commits (simplifying branch history).`
-        );
-    }
-};

.gitlab/dangerjs/mrDescriptionHasReleaseNotes.js

@@ -1,28 +0,0 @@
-/**
- * Check if MR Description contains mandatory section "Release notes"
- *
- * Extracts the content of the "Release notes" section from the GitLab merge request description.
- *
- * @dangerjs WARN (if section missing, is empty or wrong markdown format)
- */
-module.exports = function () {
-    const mrDescription = danger.gitlab.mr.description;
-    const regexSectionReleaseNotes = /## Release notes([\s\S]*?)(?=## |$)/;
-    const sectionReleaseNotes = mrDescription.match(regexSectionReleaseNotes);
-
-    if (!sectionReleaseNotes) {
-        warn(
-            'The `Release Notes` section seems to be missing. Please check if the section header in MR description is present and in the correct markdown format ("## Release Notes")\n'
-        );
-        return null;
-    }
-
-    let content = sectionReleaseNotes[1].replace(/(\r\n|\n|\r)/gm, "").trim(); // Remove empty lines and whitespace
-
-    if (!content.length) {
-        warn(
-            "The `Release Notes` section seems to be empty (no section content)\n"
-        );
-        return null;
-    }
-};

.gitlab/dangerjs/mrDescriptionJiraLinks.js

@@ -1,230 +0,0 @@
-/** Check that there are valid JIRA links in MR description.
- *
- * This check extracts the "Related" section from the MR description and
- * searches for JIRA ticket references in the format "Closes [JIRA ticket key]".
- *
- * It then extracts the closing GitHub links from the corresponding JIRA tickets and
- * checks if the linked GitHub issues are still in open state.
- *
- * Finally, it checks if the required GitHub closing links are present in the MR's commit messages.
- *
- */
-module.exports = async function () {
-    const axios = require("axios");
-    const mrDescription = danger.gitlab.mr.description;
-    const mrCommitMessages = danger.gitlab.commits.map(
-        (commit) => commit.message
-    );
-    const jiraTicketRegex = /[A-Z0-9]+-[0-9]+/;
-
-    let partMessages = []; // Create a blank field for future records of individual issues
-
-    // Parse section "Related" from MR Description
-    const sectionRelated = extractSectionRelated(mrDescription);
-
-    if (
-        !sectionRelated.header || // No section Related in MR description or ...
-        !jiraTicketRegex.test(sectionRelated.content) // no Jira links in section Related
-    ) {
-        return message(
-            "Please consider adding references to JIRA issues in the `Related` section of the MR description."
-        );
-    }
-
-    // Get closing (only) JIRA tickets
-    const jiraTickets = findClosingJiraTickets(sectionRelated.content);
-
-    for (const ticket of jiraTickets) {
-        ticket.jiraUIUrl = `https://jira.espressif.com:8443/browse/${ticket.ticketName}`;
-
-        if (!ticket.correctFormat) {
-            partMessages.push(
-                `- closing ticket \`${ticket.record}\` seems to be in the wrong format (or inaccessible to Jira DangerBot).. The correct format is for example \`- Closes JIRA-123\`.`
-            );
-        }
-
-        // Get closing GitHub issue links from JIRA tickets
-        const closingGithubLink = await getGitHubClosingLink(ticket.ticketName);
-        if (closingGithubLink) {
-            ticket.closingGithubLink = closingGithubLink;
-        } else if (closingGithubLink === null) {
-            partMessages.push(
-                `- the Jira issue number [\`${ticket.ticketName}\`](${ticket.jiraUIUrl}) seems to be invalid (please check if the ticket number is correct)`
-            );
-            continue; // Handle unreachable JIRA tickets; skip the following checks
-        } else {
-            continue; // Jira ticket have no GitHub closing link; skip the following checks
-        }
-
-        // Get still open GitHub issues
-        const githubIssueStatusOpen = await isGithubIssueOpen(
-            ticket.closingGithubLink
-        );
-        ticket.isOpen = githubIssueStatusOpen;
-        if (githubIssueStatusOpen === null) {
-            // Handle unreachable GitHub issues
-            partMessages.push(
-                `- the GitHub issue [\`${ticket.closingGithubLink}\`](${ticket.closingGithubLink}) does not seem to exist on GitHub (referenced from JIRA ticket [\`${ticket.ticketName}\`](${ticket.jiraUIUrl}) )`
-            );
-            continue; // skip the following checks
-        }
-
-        // Search in commit message if there are all GitHub closing links (from Related section) for still open GH issues
-        if (ticket.isOpen) {
-            if (
-                !mrCommitMessages.some((item) =>
-                    item.includes(`Closes ${ticket.closingGithubLink}`)
-                )
-            ) {
-                partMessages.push(
-                    `- please add \`Closes ${ticket.closingGithubLink}\` to the commit message`
-                );
-            }
-        }
-    }
-
-    // Create report / DangerJS check feedback if issues with Jira links found
-    if (partMessages.length) {
-        createReport();
-    }
-
-    // ---------------------------------------------------------------
-
-    /**
-     * This function takes in a string mrDescription which contains a Markdown-formatted text
-     * related to a Merge Request (MR) in a GitLab repository. It searches for a section titled "Related"
-     * and extracts the content of that section. If the section is not found, it returns an object
-     * indicating that the header and content are null. If the section is found but empty, it returns
-     * an object indicating that the header is present but the content is null. If the section is found
-     * with content, it returns an object indicating that the header is present and the content of the
-     * "Related" section.
-     *
-     * @param {string} mrDescription - The Markdown-formatted text related to the Merge Request.
-     * @returns {{
-     *  header: string | boolean | null,
-     * 	content: string | null
-     * 	}} - An object containing the header and content of the "Related" section, if present.
-     */
-
-    function extractSectionRelated(mrDescription) {
-        const regexSectionRelated = /## Related([\s\S]*?)(?=## |$)/;
-        const sectionRelated = mrDescription.match(regexSectionRelated);
-        if (!sectionRelated) {
-            return { header: null, content: null }; // Section "Related" is missing
-        }
-
-        const content = sectionRelated[1].replace(/(\r\n|\n|\r)/gm, ""); // Remove empty lines
-        if (!content.length) {
-            return { header: true, content: null }; // Section "Related" is present, but empty
-        }
-
-        return { header: true, content: sectionRelated[1] }; // Found section "Related" with content
-    }
-
-    /**
-     * Finds all JIRA tickets that are being closed in the given sectionRelatedcontent.
-     * The function searches for lines that start with - Closes and have the format Closes [uppercase letters]-[numbers].
-     * @param {string} sectionRelatedcontent - A string that contains lines with mentions of JIRA tickets
-     * @returns {Array} An array of objects with ticketName property that has the correct format
-     */
-
-    function findClosingJiraTickets(sectionRelatedcontent) {
-        let closingTickets = [];
-        const lines = sectionRelatedcontent.split("\n");
-        for (const line of lines) {
-            if (!line.startsWith("- Closes")) {
-                continue; // Not closing-type ticket, skip
-            }
-
-            const correctJiraClosingLinkFormat = new RegExp(
-                `^- Closes ${jiraTicketRegex.source}$`
-            );
-            const matchedJiraTicket = line.match(jiraTicketRegex);
-            if (matchedJiraTicket) {
-                if (!correctJiraClosingLinkFormat.test(line)) {
-                    closingTickets.push({
-                        record: line,
-                        ticketName: matchedJiraTicket[0],
-                        correctFormat: false,
-                    });
-                } else {
-                    closingTickets.push({
-                        record: line,
-                        ticketName: matchedJiraTicket[0],
-                        correctFormat: true,
-                    });
-                }
-            }
-        }
-        return closingTickets;
-    }
-
-    /**
-     * This function takes a JIRA issue key and retrieves the description from JIRA's API.
-     * It then searches the description for a GitHub closing link in the format "Closes https://github.com/owner/repo/issues/123".
-     * If a GitHub closing link is found, it is returned. If no GitHub closing link is found, it returns null.
-     * @param {string} jiraIssueKey - The key of the JIRA issue to search for the GitHub closing link.
-     * @returns {Promise<string|null>} - A promise that resolves to a string containing the GitHub closing link if found,
-     * or null if not found.
-     */
-    async function getGitHubClosingLink(jiraIssueKey) {
-        let jiraDescription = "";
-
-        // Get JIRA ticket description content
-        try {
-            const response = await axios({
-                url: `https://jira.espressif.com:8443/rest/api/latest/issue/${jiraIssueKey}`,
-                auth: {
-                    username: process.env.DANGER_JIRA_USER,
-                    password: process.env.DANGER_JIRA_PASSWORD,
-                },
-            });
-            jiraDescription = response.data.fields.description
-                ? response.data.fields.description
-                : ""; // if the Jira ticket has an unfilled Description, the ".description" property is missing in API response - in that case set "jiraDescription" to an empty string
-        } catch (error) {
-            return null;
-        }
-
-        // Find GitHub closing link in description
-        const regexClosingGhLink =
-            /Closes\s+(https:\/\/github.com\/\S+\/\S+\/issues\/\d+)/;
-        const closingGithubLink = jiraDescription.match(regexClosingGhLink);
-
-        if (closingGithubLink) {
-            return closingGithubLink[1];
-        } else {
-            return false; // Jira issue has no GitHub closing link in description
-        }
-    }
-
-    /**
-     * Check if a GitHub issue linked in a merge request is still open.
-     *
-     * @param {string} link - The link to the GitHub issue.
-     * @returns {Promise<boolean>} A promise that resolves to a boolean indicating if the issue is open.
-     * @throws {Error} If the link is invalid or if there was an error fetching the issue.
-     */
-    async function isGithubIssueOpen(link) {
-        const parsedUrl = new URL(link);
-        const [owner, repo] = parsedUrl.pathname.split("/").slice(1, 3);
-        const issueNumber = parsedUrl.pathname.split("/").slice(-1)[0];
-
-        try {
-            const response = await axios.get(
-                `https://api.github.com/repos/${owner}/${repo}/issues/${issueNumber}`
-            );
-            return response.data.state === "open"; // return True if GitHub issue is open
-        } catch (error) {
-            return null; // GET request to issue fails
-        }
-    }
-
-    function createReport() {
-        partMessages.sort();
-        let dangerMessage = `Some issues found for the related JIRA tickets in this MR:\n${partMessages.join(
-            "\n"
-        )}`;
-        warn(dangerMessage);
-    }
-};

.gitlab/dangerjs/mrDescriptionLongEnough.js

@@ -1,17 +0,0 @@
-/**
- * Check if MR Description has accurate description".
- *
- * @dangerjs WARN
- */
-module.exports = function () {
-    const mrDescription = danger.gitlab.mr.description;
-    const descriptionChunk = mrDescription.match(/^([^#]*)/)[1].trim(); // Extract all text before the first section header (i.e., the text before the "## Release notes")
-
-    const shortMrDescriptionThreshold = 50; // Description is considered too short below this number of characters
-
-    if (descriptionChunk.length < shortMrDescriptionThreshold) {
-        return warn(
-            "The MR description looks very brief, please check if more details can be added."
-        );
-    }
-};

.gitlab/dangerjs/mrDocsTranslation.js

@@ -1,33 +0,0 @@
-/**
- * Check if documentation needs translation labels
- *
- * #TODO: this simple logic will be improved in future MRs - Jira IDF-6855.
- *
- * @dangerjs WARN
- */
-module.exports = function () {
-    const mrLabels = danger.gitlab.mr.labels;
-    const changesInDocsEN = /docs\/en/.test(
-        danger.git.modified_files ? danger.git.modified_files[0] : ""
-    ); // Test if changes in directory "docs/EN"
-    const changesInDocsCH = /docs\/zh_CN/.test(
-        danger.git.modified_files ? danger.git.modified_files[0] : ""
-    ); // Test if changes in directory "docs/CH"
-
-    // Only English docs has been changed
-    if (changesInDocsEN && !changesInDocsCH) {
-        if (!mrLabels.includes("needs translation: CN")) {
-            return warn(
-                "The updated documentation will need to be translated into Chinese, please add the MR label `needs translation: CN`"
-            );
-        }
-    }
-    // Only Chineese docs has been changed
-    if (!changesInDocsEN && changesInDocsCH) {
-        if (!mrLabels.includes("needs translation: EN")) {
-            return warn(
-                "The updated documentation will need to be translated into English, please add the MR label `needs translation: EN`"
-            );
-        }
-    }
-};

.gitlab/dangerjs/mrSizeTooLarge.js

@@ -1,15 +0,0 @@
-/**
- * Check if MR is too large (more than 1000 lines of changes)
- *
- * @dangerjs INFO
- */
-module.exports = async function () {
-    const bigMrLinesOfCodeThreshold = 1000;
-    const totalLines = await danger.git.linesOfCode();
-
-    if (totalLines > bigMrLinesOfCodeThreshold) {
-        return message(
-            `This MR seems to be quite large (total lines of code: ${totalLines}), you might consider splitting it into smaller MRs`
-        );
-    }
-};

.gitlab/dangerjs/mrTitleNoDraftOrWip.js

@@ -1,22 +0,0 @@
-/**
- * Check if MR Title contains prefix "WIP: ...".
- *
- * @dangerjs WARN
- */
-module.exports = function () {
-	const mrTitle = danger.gitlab.mr.title;
-    const regexes = [
-        { prefix: 'WIP', regex: /^WIP:/i },
-        { prefix: 'W.I.P', regex: /^W\.I\.P/i },
-        { prefix: '[WIP]', regex: /^\[WIP/i },
-        { prefix: '[W.I.P]', regex: /^\[W\.I\.P/i },
-        { prefix: '(WIP)', regex: /^\(WIP/i },
-        { prefix: '(W.I.P)', regex: /^\(W\.I\.P/i },
-    ];
-
-	for (const item of regexes) {
-		if (item.regex.test(mrTitle)) {
-			return warn(`Please remove the \`${item.prefix}\` prefix from the MR name before merging this MR.`);
-		}
-	}
-};

.gitlab/merge_request_templates/Mixed Template.md

@@ -42,7 +42,7 @@ _For other small/non-public changes, which are not expected to be in the release
 
 _Don't touch the subsection titles below, they will be parsed by scripts._
 
-## Release notes (Mandatory)
+## Release notes <!-- Mandatory -->
 
 _Changes made in this MR that should go into the **Release Notes** should be listed here. Please use **past tense** and *specify the area (see maintainers page of IDF internal wiki)*. If there is a subscope, include it and separate with slash (`/`). Minor changes can go to the descriptions above without a release notes entry._
 

.gitmodules

@@ -2,6 +2,26 @@
 # All the relative URL paths are intended to be GitHub ones
 # For Espressif's public projects please use '../../espressif/proj', not a '../proj'
 #
+# Submodules SBOM information
+# ---------------------------
+# Submodules, which are used directly and not forked into espressif namespace should
+# contain SBOM information here. Other submodules should have the SBOM manifest file
+# included in the root of their project's repository.
+#
+# The sbom-hash entry records the submodule's checkout SHA as presented in git-tree
+# commit object. For example spiffs submodule
+#
+# $ git ls-tree HEAD components/spiffs/spiffs
+# 160000 commit 0dbb3f71c5f6fae3747a9d935372773762baf852	components/spiffs/spiffs
+#
+# The hash can be also obtained with git submodule command
+#
+# $ git submodule status components/spiffs/spiffs
+# 0dbb3f71c5f6fae3747a9d935372773762baf852 components/spiffs/spiffs (0.2-255-g0dbb3f71c5f6)
+#
+# The submodule SHA recorded here has to match with SHA, which is presented in git-tree.
+# This is checked by CI. Also please don't forget to update the submodule version
+# if you are changing the sbom-hash. This is important for SBOM generation.
 
 [submodule "components/bt/controller/lib_esp32"]
 	path = components/bt/controller/lib_esp32
@@ -10,14 +30,31 @@
 [submodule "components/bootloader/subproject/components/micro-ecc/micro-ecc"]
 	path = components/bootloader/subproject/components/micro-ecc/micro-ecc
 	url = ../../kmackay/micro-ecc.git
+	sbom-version = 1.1
+	sbom-cpe = cpe:2.3:a:micro-ecc_project:micro-ecc:{}:*:*:*:*:*:*:*
+	sbom-supplier = Person: Ken MacKay
+	sbom-url = https://github.com/kmackay/micro-ecc
+	sbom-description = A small and fast ECDH and ECDSA implementation for 8-bit, 32-bit, and 64-bit processors
+	sbom-hash = 24c60e243580c7868f4334a1ba3123481fe1aa48
 
 [submodule "components/spiffs/spiffs"]
 	path = components/spiffs/spiffs
 	url = ../../pellepl/spiffs.git
+	sbom-version = 0.2-255-g0dbb3f71c5f6
+	sbom-supplier = Person: Peter Andersson
+	sbom-url = https://github.com/pellepl/spiffs
+	sbom-description = Wear-leveled SPI flash file system for embedded devices
+	sbom-hash = 0dbb3f71c5f6fae3747a9d935372773762baf852
 
 [submodule "components/json/cJSON"]
 	path = components/json/cJSON
 	url = ../../DaveGamble/cJSON.git
+	sbom-version = 1.7.16
+	sbom-cpe = cpe:2.3:a:cjson_project:cjson:{}:*:*:*:*:*:*:*
+	sbom-supplier = Person: Dave Gamble
+	sbom-url = https://github.com/DaveGamble/cJSON
+	sbom-description = Ultralightweight JSON parser in ANSI C
+	sbom-hash = cb8693b058ba302f4829ec6d03f609ac6f848546
 
 [submodule "components/mbedtls/mbedtls"]
 	path = components/mbedtls/mbedtls
@@ -34,10 +71,21 @@
 [submodule "components/protobuf-c/protobuf-c"]
 	path = components/protobuf-c/protobuf-c
 	url = ../../protobuf-c/protobuf-c.git
+	sbom-version = 1.4.1
+	sbom-cpe = cpe:2.3:a:protobuf-c_project:protobuf-c:{}:*:*:*:*:*:*:*
+	sbom-supplier = Organization: protobuf-c community <https://groups.google.com/g/protobuf-c>
+	sbom-url = https://github.com/protobuf-c/protobuf-c
+	sbom-description = Protocol Buffers implementation in C
+	sbom-hash = abc67a11c6db271bedbb9f58be85d6f4e2ea8389
 
 [submodule "components/unity/unity"]
 	path = components/unity/unity
 	url = ../../ThrowTheSwitch/Unity.git
+	sbom-version = v2.6.0-RC1
+	sbom-supplier = Organization: ThrowTheSwitch community <http://www.throwtheswitch.org>
+	sbom-url = https://github.com/ThrowTheSwitch/Unity
+	sbom-description = Simple Unit Testing for C
+	sbom-hash = bf560290f6020737eafaa8b5cbd2177c3956c03f
 
 [submodule "components/bt/host/nimble/nimble"]
 	path = components/bt/host/nimble/nimble
@@ -50,6 +98,11 @@
 [submodule "components/cmock/CMock"]
 	path = components/cmock/CMock
 	url = ../../ThrowTheSwitch/CMock.git
+	sbom-version = v2.5.2-2-geeecc49ce8af
+	sbom-supplier = Organization: ThrowTheSwitch community <http://www.throwtheswitch.org>
+	sbom-url = https://github.com/ThrowTheSwitch/CMock
+	sbom-description = CMock - Mock/stub generator for C
+	sbom-hash = eeecc49ce8af123cf8ad40efdb9673e37b56230f
 
 [submodule "components/openthread/openthread"]
 	path = components/openthread/openthread
@@ -90,3 +143,7 @@
 [submodule "components/esp_coex/lib"]
 	path = components/esp_coex/lib
 	url = ../../espressif/esp-coex-lib.git
+
+[submodule "components/bt/esp_ble_mesh/lib/lib"]
+	path = components/bt/esp_ble_mesh/lib/lib
+	url = ../../espressif/esp-ble-mesh-lib.git

.pre-commit-config.yaml

@@ -1,6 +1,8 @@
 # See https://pre-commit.com for more information
 # See https://pre-commit.com/hooks.html for more hooks
 
+default_stages: [commit]
+
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.0.1
@@ -28,13 +30,19 @@ repos:
       - id: mixed-line-ending
         args: ['-f=lf']
       - id: double-quote-string-fixer
+      - id: no-commit-to-branch
+        name: Do not use more than one slash in the branch name
+        args: ['--pattern', '^[^/]*/[^/]*/']
+      - id: no-commit-to-branch
+        name: Do not use uppercase letters in the branch name
+        args: ['--pattern', '^[^A-Z]*[A-Z]']
   - repo: https://github.com/PyCQA/flake8
     rev: 5.0.4
     hooks:
       - id: flake8
         args: ['--config=.flake8', '--tee', '--benchmark']
   - repo: https://github.com/pycqa/isort
-    rev: 5.11.5  # python 3.7 compatible
+    rev: 5.12.0  # python 3.8 compatible
     hooks:
       - id: isort
         name: isort (python)
@@ -60,6 +68,8 @@ repos:
         name: Validate Kconfig files
         entry: tools/ci/check_kconfigs.py
         language: python
+        additional_dependencies:
+          - esp-idf-kconfig
         files: '^Kconfig$|Kconfig.*$'
       - id: check-deprecated-kconfigs-options
         name: Check if any Kconfig Options Deprecated
@@ -127,6 +137,14 @@ repos:
         additional_dependencies:
           - PyYAML == 5.3.1
         pass_filenames: false
+      - id: check-rules-components-patterns
+        name: check patterns-build_components in rules.yml
+        entry: tools/ci/check_rules_components_patterns.py
+        language: python
+        files: 'components/.+|.gitlab/ci/rules.yml'
+        additional_dependencies:
+          - PyYAML == 5.3.1
+        pass_filenames: false
       - id: check-generated-soc-caps-kconfig
         name: Check soc caps kconfig files are generated (based on components/soc/IDF_TARGET/include/soc/soc_caps.h)
         entry: tools/gen_soc_caps_kconfig/gen_soc_caps_kconfig.py -d 'components/soc/*/include/soc/' 'components/esp_rom/*/' 'components/spi_flash/*/'
@@ -139,10 +157,11 @@ repos:
         name: Check if all apps readme files match given .build-test-rules.yml files. Modify the supported target tables
         entry: tools/ci/check_build_test_rules.py check-readmes
         language: python
-        files: 'tools/test_apps/.+|examples/.+|components/.+'
+        files: 'tools/test_apps/.+|examples/.+|components/.+|tools/idf_py_actions/constants.py|tools/ci/check_build_test_rules.py'
+        require_serial: true
         additional_dependencies:
           - PyYAML == 5.3.1
-          - idf_build_apps
+          - idf_build_apps~=1.0
       - id: sort-build-test-rules-ymls
         name: sort .build-test-rules.yml files
         entry: tools/ci/check_build_test_rules.py sort-yaml
@@ -151,13 +170,55 @@ repos:
         additional_dependencies:
           - PyYAML == 5.3.1
           - ruamel.yaml
+      - id: check-build-test-rules-path-exists
+        name: check path in .build-test-rules.yml exists
+        entry: tools/ci/check_build_test_rules.py check-exist
+        language: python
+        additional_dependencies:
+          - PyYAML == 5.3.1
+        always_run: true
+        pass_filenames: false
+        require_serial: true
+      - id: cleanup-ignore-lists
+        name: Remove non-existing patterns from ignore lists
+        entry: tools/ci/cleanup_ignore_lists.py
+        language: python
+        always_run: true
+        require_serial: true
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.0.1
     hooks:
       - id: file-contents-sorter
         files: 'tools\/ci\/(executable-list\.txt|mypy_ignore_list\.txt|check_copyright_ignore\.txt)'
   - repo: https://github.com/espressif/check-copyright/
-    rev: v1.0.1
+    rev: v1.0.3
     hooks:
       - id: check-copyright
         args: ['--ignore', 'tools/ci/check_copyright_ignore.txt', '--config', 'tools/ci/check_copyright_config.yaml']
+  - repo: https://github.com/espressif/conventional-precommit-linter
+    rev: v1.6.0
+    hooks:
+      - id: conventional-precommit-linter
+        stages: [commit-msg]
+  - repo: https://github.com/espressif/astyle_py.git
+    rev: v1.0.5
+    hooks:
+      - id: astyle_py
+        # If you are modifying astyle version, update tools/format.sh as well
+        args: ['--astyle-version=3.4.7', '--rules=tools/ci/astyle-rules.yml']
+  - repo: https://github.com/shellcheck-py/shellcheck-py
+    rev: v0.9.0.5
+    hooks:
+    - id: shellcheck
+      name: shellcheck bash
+      args: ['--shell', 'bash', '-x']
+      files: 'install.sh|export.sh'
+    - id: shellcheck
+      name: shellcheck dash (export.sh)
+      args: ['--shell', 'dash', '-x']
+      files: 'export.sh'
+  - repo: https://github.com/espressif/esp-idf-sbom.git
+    rev: v0.13.0
+    hooks:
+      - id: validate-sbom-manifest
+        stages: [post-commit]

.pylintrc

@@ -1,27 +1,77 @@
-[MASTER]
+[MAIN]
+
+# Analyse import fallback blocks. This can be used to support both Python 2 and
+# 3 compatible code, which means that the block might have code that exists
+# only in one or another interpreter, leading to false positives when analysed.
+analyse-fallback-blocks=no
+
+# Clear in-memory caches upon conclusion of linting. Useful if running pylint
+# in a server-like mode.
+clear-cache-post-run=no
+
+# Load and enable all available extensions. Use --list-extensions to see a list
+# all available extensions.
+#enable-all-extensions=
+
+# In error mode, messages with a category besides ERROR or FATAL are
+# suppressed, and no reports are done by default. Error mode is compatible with
+# disabling specific errors.
+#errors-only=
+
+# Always return a 0 (non-error) status code, even if lint errors are found.
+# This is primarily useful in continuous integration scripts.
+#exit-zero=
 
 # A comma-separated list of package or module names from where C extensions may
 # be loaded. Extensions are loading into the active Python interpreter and may
 # run arbitrary code.
+extension-pkg-allow-list=
+
+# A comma-separated list of package or module names from where C extensions may
+# be loaded. Extensions are loading into the active Python interpreter and may
+# run arbitrary code. (This is an alternative name to extension-pkg-allow-list
+# for backward compatibility.)
 extension-pkg-whitelist=
 
-# Specify a score threshold to be exceeded before program exits with error.
+# Return non-zero exit code if any of these messages/categories are detected,
+# even if score is above --fail-under value. Syntax same as enable. Messages
+# specified are enabled, while categories only check already-enabled messages.
+fail-on=
+
+# Specify a score threshold under which the program will exit with error.
 fail-under=10
 
-# Add files or directories to the blacklist. They should be base names, not
-# paths.
+# Interpret the stdin as a python script, whose filename needs to be passed as
+# the module_or_package argument.
+#from-stdin=
+
+# Files or directories to be skipped. They should be base names, not paths.
 ignore=CVS
 
-# Add files or directories matching the regex patterns to the blacklist. The
-# regex matches against base names, not paths.
-ignore-patterns=
+# Add files or directories matching the regular expressions patterns to the
+# ignore-list. The regex matches against paths and can be in Posix or Windows
+# format. Because '\\' represents the directory delimiter on Windows systems,
+# it can't be used as an escape character.
+ignore-paths=
+
+# Files or directories matching the regular expression patterns are skipped.
+# The regex matches against base names, not paths. The default value ignores
+# Emacs file locks
+ignore-patterns=^\.#
+
+# List of module names for which member attributes should not be checked
+# (useful for modules/projects where namespaces are manipulated during runtime
+# and thus existing member attributes cannot be deduced by static analysis). It
+# supports qualified module names, as well as Unix pattern matching.
+ignored-modules=
 
 # Python code to execute, usually for sys.path manipulation such as
 # pygtk.require().
 #init-hook=
 
 # Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the
-# number of processors available to use.
+# number of processors available to use, and will cap the count on Windows to
+# avoid hangs.
 jobs=1
 
 # Control the amount of potential inferred values when inferring a single
@@ -36,6 +86,19 @@ load-plugins=
 # Pickle collected data for later comparisons.
 persistent=yes
 
+# Minimum Python version to use for version dependent checks. Will default to
+# the version used to run pylint.
+py-version=3.8
+
+# Discover python modules and packages in the file system subtree.
+recursive=no
+
+# Add paths to the list of the source roots. Supports globbing patterns. The
+# source root is an absolute path or a path relative to the current working
+# directory used to determine a package namespace for modules located under the
+# source root.
+source-roots=
+
 # When enabled, pylint would attempt to guess common misconfiguration and emit
 # user-friendly hints instead of false-positive error messages.
 suggestion-mode=yes
@@ -44,348 +107,8 @@ suggestion-mode=yes
 # active Python interpreter and may run arbitrary code.
 unsafe-load-any-extension=no
 
-
-[MESSAGES CONTROL]
-
-# Only show warnings with the listed confidence levels. Leave empty to show
-# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED.
-confidence=
-
-# Disable the message, report, category or checker with the given id(s). You
-# can either give multiple identifiers separated by comma (,) or put this
-# option multiple times (only on the command line, not in the configuration
-# file where it should appear only once). You can also use "--disable=all" to
-# disable everything first and then reenable specific checks. For example, if
-# you want to run only the similarities checker, you can use "--disable=all
-# --enable=similarities". If you want to run only the classes checker, but have
-# no Warning level messages displayed, use "--disable=all --enable=classes
-# --disable=W".
-disable=print-statement,
-        parameter-unpacking,
-        unpacking-in-except,
-        old-raise-syntax,
-        backtick,
-        long-suffix,
-        old-ne-operator,
-        old-octal-literal,
-        import-star-module-level,
-        non-ascii-bytes-literal,
-        raw-checker-failed,
-        bad-inline-option,
-        locally-disabled,
-        file-ignored,
-        suppressed-message,
-        useless-suppression,
-        deprecated-pragma,
-        use-symbolic-message-instead,
-        apply-builtin,
-        basestring-builtin,
-        buffer-builtin,
-        cmp-builtin,
-        coerce-builtin,
-        execfile-builtin,
-        file-builtin,
-        long-builtin,
-        raw_input-builtin,
-        reduce-builtin,
-        standarderror-builtin,
-        unicode-builtin,
-        xrange-builtin,
-        coerce-method,
-        delslice-method,
-        getslice-method,
-        setslice-method,
-        no-absolute-import,
-        old-division,
-        dict-iter-method,
-        dict-view-method,
-        next-method-called,
-        metaclass-assignment,
-        indexing-exception,
-        raising-string,
-        reload-builtin,
-        oct-method,
-        hex-method,
-        nonzero-method,
-        cmp-method,
-        input-builtin,
-        round-builtin,
-        intern-builtin,
-        unichr-builtin,
-        map-builtin-not-iterating,
-        zip-builtin-not-iterating,
-        range-builtin-not-iterating,
-        filter-builtin-not-iterating,
-        using-cmp-argument,
-        eq-without-hash,
-        div-method,
-        idiv-method,
-        rdiv-method,
-        exception-message-attribute,
-        invalid-str-codec,
-        sys-max-int,
-        bad-python3-import,
-        deprecated-string-function,
-        deprecated-str-translate-call,
-        deprecated-itertools-function,
-        deprecated-types-field,
-        next-method-defined,
-        dict-items-not-iterating,
-        dict-keys-not-iterating,
-        dict-values-not-iterating,
-        deprecated-operator-function,
-        deprecated-urllib-function,
-        xreadlines-attribute,
-        deprecated-sys-function,
-        exception-escape,
-        comprehension-escape,
-        missing-function-docstring, # Modified since here, include this line
-        missing-class-docstring,
-        missing-module-docstring,
-        wrong-import-order,
-        invalid-name,
-        too-few-public-methods,
-        too-many-locals,
-        bad-super-call,  # since we still haven't drop python2 support
-        too-many-nested-blocks,
-        too-many-branches,
-        too-many-statements,
-        ungrouped-imports,  # since we have isort in pre-commit
-        no-name-in-module,  # since we have flake8 to check this
-        too-many-instance-attributes,
-
-# Enable the message, report, category or checker with the given id(s). You can
-# either give multiple identifier separated by comma (,) or put this option
-# multiple time (only on the command line, not in the configuration file where
-# it should appear only once). See also the "--disable" option for examples.
-enable=c-extension-no-member
-
-
-[REPORTS]
-
-# Python expression which should return a score less than or equal to 10. You
-# have access to the variables 'error', 'warning', 'refactor', and 'convention'
-# which contain the number of messages in each category, as well as 'statement'
-# which is the total number of statements analyzed. This score is used by the
-# global evaluation report (RP0004).
-evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)
-
-# Template used to display messages. This is a python new-style format string
-# used to format the message information. See doc for all details.
-#msg-template=
-
-# Set the output format. Available formats are text, parseable, colorized, json
-# and msvs (visual studio). You can also give a reporter class, e.g.
-# mypackage.mymodule.MyReporterClass.
-output-format=text
-
-# Tells whether to display a full report or only the messages.
-reports=no
-
-# Activate the evaluation score.
-score=yes
-
-
-[REFACTORING]
-
-# Maximum number of nested blocks for function / method body
-max-nested-blocks=5
-
-# Complete name of functions that never returns. When checking for
-# inconsistent-return-statements if a never returning function is called then
-# it will be considered as an explicit return statement and no message will be
-# printed.
-never-returning-functions=sys.exit
-
-
-[TYPECHECK]
-
-# List of decorators that produce context managers, such as
-# contextlib.contextmanager. Add to this list to register other decorators that
-# produce valid context managers.
-contextmanager-decorators=contextlib.contextmanager
-
-# List of members which are set dynamically and missed by pylint inference
-# system, and so shouldn't trigger E1101 when accessed. Python regular
-# expressions are accepted.
-generated-members=
-
-# Tells whether missing members accessed in mixin class should be ignored. A
-# mixin class is detected if its name ends with "mixin" (case insensitive).
-ignore-mixin-members=yes
-
-# Tells whether to warn about missing members when the owner of the attribute
-# is inferred to be None.
-ignore-none=yes
-
-# This flag controls whether pylint should warn about no-member and similar
-# checks whenever an opaque object is returned when inferring. The inference
-# can return multiple potential results while evaluating a Python object, but
-# some branches might not be evaluated, which results in partial inference. In
-# that case, it might be useful to still emit no-member and other checks for
-# the rest of the inferred objects.
-ignore-on-opaque-inference=yes
-
-# List of class names for which member attributes should not be checked (useful
-# for classes with dynamically set attributes). This supports the use of
-# qualified names.
-ignored-classes=optparse.Values,thread._local,_thread._local
-
-# List of module names for which member attributes should not be checked
-# (useful for modules/projects where namespaces are manipulated during runtime
-# and thus existing member attributes cannot be deduced by static analysis). It
-# supports qualified module names, as well as Unix pattern matching.
-ignored-modules=
-
-# Show a hint with possible names when a member name was not found. The aspect
-# of finding the hint is based on edit distance.
-missing-member-hint=yes
-
-# The minimum edit distance a name should have in order to be considered a
-# similar match for a missing member name.
-missing-member-hint-distance=1
-
-# The total number of similar names that should be taken in consideration when
-# showing a hint for a missing member.
-missing-member-max-choices=1
-
-# List of decorators that change the signature of a decorated function.
-signature-mutators=ttfw_idf.idf_example_test,ttfw_idf.idf_unit_test,ttfw_idf.idf_custom_test,ttfw_idf.idf_component_unit_test
-
-
-[SPELLING]
-
-# Limits count of emitted suggestions for spelling mistakes.
-max-spelling-suggestions=4
-
-# Spelling dictionary name. Available dictionaries: none. To make it work,
-# install the python-enchant package.
-spelling-dict=
-
-# List of comma separated words that should not be checked.
-spelling-ignore-words=
-
-# A path to a file that contains the private dictionary; one word per line.
-spelling-private-dict-file=
-
-# Tells whether to store unknown words to the private dictionary (see the
-# --spelling-private-dict-file option) instead of raising a message.
-spelling-store-unknown-words=no
-
-
-[FORMAT]
-
-# Expected format of line ending, e.g. empty (any line ending), LF or CRLF.
-expected-line-ending-format=
-
-# Regexp for a line that is allowed to be longer than the limit.
-ignore-long-lines=^\s*(# )?<?https?://\S+>?$
-
-# Number of spaces of indent required inside a hanging or continued line.
-indent-after-paren=4
-
-# String used as indentation unit. This is usually "    " (4 spaces) or "\t" (1
-# tab).
-indent-string='    '
-
-# Maximum number of characters on a single line.
-max-line-length=160
-
-# Maximum number of lines in a module.
-max-module-lines=1000
-
-# List of optional constructs for which whitespace checking is disabled. `dict-
-# separator` is used to allow tabulation in dicts, etc.: {1  : 1,\n222: 2}.
-# `trailing-comma` allows a space between comma and closing bracket: (a, ).
-# `empty-line` allows space-only lines.
-no-space-check=trailing-comma,
-               dict-separator
-
-# Allow the body of a class to be on the same line as the declaration if body
-# contains single statement.
-single-line-class-stmt=no
-
-# Allow the body of an if to be on the same line as the test if there is no
-# else.
-single-line-if-stmt=no
-
-
-[STRING]
-
-# This flag controls whether inconsistent-quotes generates a warning when the
-# character used as a quote delimiter is used inconsistently within a module.
-check-quote-consistency=no
-
-# This flag controls whether the implicit-str-concat should generate a warning
-# on implicit string concatenation in sequences defined over several lines.
-check-str-concat-over-line-jumps=no
-
-
-[LOGGING]
-
-# The type of string formatting that logging methods do. `old` means using %
-# formatting, `new` is for `{}` formatting.
-logging-format-style=old
-
-# Logging modules to check that the string format arguments are in logging
-# function parameter format.
-logging-modules=logging
-
-
-[MISCELLANEOUS]
-
-# List of note tags to take in consideration, separated by a comma.
-notes=FIXME,
-      XXX,
-      TODO
-
-# Regular expression of note tags to take in consideration.
-#notes-rgx=
-
-
-[SIMILARITIES]
-
-# Ignore comments when computing similarities.
-ignore-comments=yes
-
-# Ignore docstrings when computing similarities.
-ignore-docstrings=yes
-
-# Ignore imports when computing similarities.
-ignore-imports=no
-
-# Minimum lines number of a similarity.
-min-similarity-lines=4
-
-
-[VARIABLES]
-
-# List of additional names supposed to be defined in builtins. Remember that
-# you should avoid defining new builtins when possible.
-additional-builtins=
-
-# Tells whether unused global variables should be treated as a violation.
-allow-global-unused-variables=yes
-
-# List of strings which can identify a callback function by name. A callback
-# name must start or end with one of those strings.
-callbacks=cb_,
-          _cb
-
-# A regular expression matching the name of dummy variables (i.e. expected to
-# not be used).
-dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_
-
-# Argument names that match this expression will be ignored. Default to name
-# with leading underscore.
-ignored-argument-names=_.*|^ignored_|^unused_
-
-# Tells whether we should check for unused import in __init__ files.
-init-import=no
-
-# List of qualified module names which can have objects that can redefine
-# builtins.
-redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io
+# In verbose mode, extra non-checker-related info will be displayed.
+#verbose=
 
 
 [BASIC]
@@ -394,13 +117,15 @@ redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io
 argument-naming-style=snake_case
 
 # Regular expression matching correct argument names. Overrides argument-
-# naming-style.
+# naming-style. If left empty, argument names will be checked with the set
+# naming style.
 #argument-rgx=
 
 # Naming style matching correct attribute names.
 attr-naming-style=snake_case
 
 # Regular expression matching correct attribute names. Overrides attr-naming-
+# style. If left empty, attribute names will be checked with the set naming
 # style.
 #attr-rgx=
 
@@ -420,20 +145,30 @@ bad-names-rgxs=
 class-attribute-naming-style=any
 
 # Regular expression matching correct class attribute names. Overrides class-
-# attribute-naming-style.
+# attribute-naming-style. If left empty, class attribute names will be checked
+# with the set naming style.
 #class-attribute-rgx=
 
+# Naming style matching correct class constant names.
+class-const-naming-style=UPPER_CASE
+
+# Regular expression matching correct class constant names. Overrides class-
+# const-naming-style. If left empty, class constant names will be checked with
+# the set naming style.
+#class-const-rgx=
+
 # Naming style matching correct class names.
 class-naming-style=PascalCase
 
 # Regular expression matching correct class names. Overrides class-naming-
-# style.
+# style. If left empty, class names will be checked with the set naming style.
 #class-rgx=
 
 # Naming style matching correct constant names.
 const-naming-style=UPPER_CASE
 
 # Regular expression matching correct constant names. Overrides const-naming-
+# style. If left empty, constant names will be checked with the set naming
 # style.
 #const-rgx=
 
@@ -445,7 +180,8 @@ docstring-min-length=-1
 function-naming-style=snake_case
 
 # Regular expression matching correct function names. Overrides function-
-# naming-style.
+# naming-style. If left empty, function names will be checked with the set
+# naming style.
 #function-rgx=
 
 # Good variable names which should always be accepted, separated by a comma.
@@ -467,21 +203,22 @@ include-naming-hint=no
 inlinevar-naming-style=any
 
 # Regular expression matching correct inline iteration names. Overrides
-# inlinevar-naming-style.
+# inlinevar-naming-style. If left empty, inline iteration names will be checked
+# with the set naming style.
 #inlinevar-rgx=
 
 # Naming style matching correct method names.
 method-naming-style=snake_case
 
 # Regular expression matching correct method names. Overrides method-naming-
-# style.
+# style. If left empty, method names will be checked with the set naming style.
 #method-rgx=
 
 # Naming style matching correct module names.
 module-naming-style=snake_case
 
 # Regular expression matching correct module names. Overrides module-naming-
-# style.
+# style. If left empty, module names will be checked with the set naming style.
 #module-rgx=
 
 # Colon-delimited sets of names that determine each other's naming style when
@@ -497,21 +234,61 @@ no-docstring-rgx=^_
 # These decorators are taken in consideration only for invalid-name.
 property-classes=abc.abstractproperty
 
+# Regular expression matching correct type alias names. If left empty, type
+# alias names will be checked with the set naming style.
+#typealias-rgx=
+
+# Regular expression matching correct type variable names. If left empty, type
+# variable names will be checked with the set naming style.
+#typevar-rgx=
+
 # Naming style matching correct variable names.
 variable-naming-style=snake_case
 
 # Regular expression matching correct variable names. Overrides variable-
-# naming-style.
+# naming-style. If left empty, variable names will be checked with the set
+# naming style.
 #variable-rgx=
 
 
+[CLASSES]
+
+# Warn about protected attribute access inside special methods
+check-protected-access-in-special-methods=no
+
+# List of method names used to declare (i.e. assign) instance attributes.
+defining-attr-methods=__init__,
+                      __new__,
+                      setUp,
+                      asyncSetUp,
+                      __post_init__
+
+# List of member names, which should be excluded from the protected access
+# warning.
+exclude-protected=_asdict,_fields,_replace,_source,_make,os._exit
+
+# List of valid names for the first argument in a class method.
+valid-classmethod-first-arg=cls
+
+# List of valid names for the first argument in a metaclass class method.
+valid-metaclass-classmethod-first-arg=mcs
+
+
 [DESIGN]
 
+# List of regular expressions of class ancestor names to ignore when counting
+# public methods (see R0903)
+exclude-too-few-public-methods=
+
+# List of qualified class names to ignore when counting class parents (see
+# R0901)
+ignored-parents=
+
 # Maximum number of arguments for function / method.
 max-args=5
 
 # Maximum number of attributes for a class (see R0902).
-max-attributes=12
+max-attributes=7
 
 # Maximum number of boolean expressions in an if statement (see R0916).
 max-bool-expr=5
@@ -538,27 +315,40 @@ max-statements=50
 min-public-methods=2
 
 
-[CLASSES]
+[EXCEPTIONS]
 
-# List of method names used to declare (i.e. assign) instance attributes.
-defining-attr-methods=__init__,
-                      __new__,
-                      setUp,
-                      __post_init__
+# Exceptions that will emit a warning when caught.
+overgeneral-exceptions=builtins.BaseException,builtins.Exception
 
-# List of member names, which should be excluded from the protected access
-# warning.
-exclude-protected=_asdict,
-                  _fields,
-                  _replace,
-                  _source,
-                  _make
 
-# List of valid names for the first argument in a class method.
-valid-classmethod-first-arg=cls
+[FORMAT]
 
-# List of valid names for the first argument in a metaclass class method.
-valid-metaclass-classmethod-first-arg=cls
+# Expected format of line ending, e.g. empty (any line ending), LF or CRLF.
+expected-line-ending-format=
+
+# Regexp for a line that is allowed to be longer than the limit.
+ignore-long-lines=^\s*(# )?<?https?://\S+>?$
+
+# Number of spaces of indent required inside a hanging or continued line.
+indent-after-paren=4
+
+# String used as indentation unit. This is usually "    " (4 spaces) or "\t" (1
+# tab).
+indent-string='    '
+
+# Maximum number of characters on a single line.
+max-line-length=160
+
+# Maximum number of lines in a module.
+max-module-lines=1000
+
+# Allow the body of a class to be on the same line as the declaration if body
+# contains single statement.
+single-line-class-stmt=no
+
+# Allow the body of an if to be on the same line as the test if there is no
+# else.
+single-line-if-stmt=no
 
 
 [IMPORTS]
@@ -567,27 +357,26 @@ valid-metaclass-classmethod-first-arg=cls
 # one.
 allow-any-import-level=
 
+# Allow explicit reexports by alias from a package __init__.
+allow-reexport-from-package=no
+
 # Allow wildcard imports from modules that define __all__.
 allow-wildcard-with-all=no
 
-# Analyse import fallback blocks. This can be used to support both Python 2 and
-# 3 compatible code, which means that the block might have code that exists
-# only in one or another interpreter, leading to false positives when analysed.
-analyse-fallback-blocks=no
-
 # Deprecated modules which should not be used, separated by a comma.
-deprecated-modules=optparse,tkinter.tix
+deprecated-modules=
 
-# Create a graph of external dependencies in the given file (report RP0402 must
-# not be disabled).
+# Output a graph (.gv or any supported image format) of external dependencies
+# to the given file (report RP0402 must not be disabled).
 ext-import-graph=
 
-# Create a graph of every (i.e. internal and external) dependencies in the
-# given file (report RP0402 must not be disabled).
+# Output a graph (.gv or any supported image format) of all (i.e. internal and
+# external) dependencies to the given file (report RP0402 must not be
+# disabled).
 import-graph=
 
-# Create a graph of internal dependencies in the given file (report RP0402 must
-# not be disabled).
+# Output a graph (.gv or any supported image format) of internal dependencies
+# to the given file (report RP0402 must not be disabled).
 int-import-graph=
 
 # Force import order to recognize a module as part of the standard
@@ -601,9 +390,252 @@ known-third-party=enchant
 preferred-modules=
 
 
-[EXCEPTIONS]
+[LOGGING]
 
-# Exceptions that will emit a warning when being caught. Defaults to
-# "BaseException, Exception".
-overgeneral-exceptions=BaseException,
-                       Exception
+# The type of string formatting that logging methods do. `old` means using %
+# formatting, `new` is for `{}` formatting.
+logging-format-style=old
+
+# Logging modules to check that the string format arguments are in logging
+# function parameter format.
+logging-modules=logging
+
+
+[MESSAGES CONTROL]
+
+# Only show warnings with the listed confidence levels. Leave empty to show
+# all. Valid levels: HIGH, CONTROL_FLOW, INFERENCE, INFERENCE_FAILURE,
+# UNDEFINED.
+confidence=HIGH,
+           CONTROL_FLOW,
+           INFERENCE,
+           INFERENCE_FAILURE,
+           UNDEFINED
+
+# Disable the message, report, category or checker with the given id(s). You
+# can either give multiple identifiers separated by comma (,) or put this
+# option multiple times (only on the command line, not in the configuration
+# file where it should appear only once). You can also use "--disable=all" to
+# disable everything first and then re-enable specific checks. For example, if
+# you want to run only the similarities checker, you can use "--disable=all
+# --enable=similarities". If you want to run only the classes checker, but have
+# no Warning level messages displayed, use "--disable=all --enable=classes
+# --disable=W".
+disable=raw-checker-failed,
+        bad-inline-option,
+        locally-disabled,
+        file-ignored,
+        suppressed-message,
+        useless-suppression,
+        deprecated-pragma,
+        use-symbolic-message-instead,
+        missing-function-docstring, # Modified since here, include this line
+        missing-class-docstring,
+        missing-module-docstring,
+        wrong-import-order,
+        invalid-name,
+        too-few-public-methods,
+        too-many-locals,
+        ungrouped-imports,  # since we have isort in pre-commit
+        no-name-in-module,  # since we have flake8 to check this
+        too-many-instance-attributes,
+
+# Enable the message, report, category or checker with the given id(s). You can
+# either give multiple identifier separated by comma (,) or put this option
+# multiple time (only on the command line, not in the configuration file where
+# it should appear only once). See also the "--disable" option for examples.
+enable=c-extension-no-member
+
+
+[METHOD_ARGS]
+
+# List of qualified names (i.e., library.method) which require a timeout
+# parameter e.g. 'requests.api.get,requests.api.post'
+timeout-methods=requests.api.delete,requests.api.get,requests.api.head,requests.api.options,requests.api.patch,requests.api.post,requests.api.put,requests.api.request
+
+
+[MISCELLANEOUS]
+
+# List of note tags to take in consideration, separated by a comma.
+notes=FIXME,
+      XXX,
+      TODO
+
+# Regular expression of note tags to take in consideration.
+notes-rgx=
+
+
+[REFACTORING]
+
+# Maximum number of nested blocks for function / method body
+max-nested-blocks=5
+
+# Complete name of functions that never returns. When checking for
+# inconsistent-return-statements if a never returning function is called then
+# it will be considered as an explicit return statement and no message will be
+# printed.
+never-returning-functions=sys.exit,argparse.parse_error
+
+
+[REPORTS]
+
+# Python expression which should return a score less than or equal to 10. You
+# have access to the variables 'fatal', 'error', 'warning', 'refactor',
+# 'convention', and 'info' which contain the number of messages in each
+# category, as well as 'statement' which is the total number of statements
+# analyzed. This score is used by the global evaluation report (RP0004).
+evaluation=max(0, 0 if fatal else 10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10))
+
+# Template used to display messages. This is a python new-style format string
+# used to format the message information. See doc for all details.
+msg-template=
+
+# Set the output format. Available formats are text, parseable, colorized, json
+# and msvs (visual studio). You can also give a reporter class, e.g.
+# mypackage.mymodule.MyReporterClass.
+#output-format=
+
+# Tells whether to display a full report or only the messages.
+reports=no
+
+# Activate the evaluation score.
+score=yes
+
+
+[SIMILARITIES]
+
+# Comments are removed from the similarity computation
+ignore-comments=yes
+
+# Docstrings are removed from the similarity computation
+ignore-docstrings=yes
+
+# Imports are removed from the similarity computation
+ignore-imports=yes
+
+# Signatures are removed from the similarity computation
+ignore-signatures=yes
+
+# Minimum lines number of a similarity.
+min-similarity-lines=4
+
+
+[SPELLING]
+
+# Limits count of emitted suggestions for spelling mistakes.
+max-spelling-suggestions=4
+
+# Spelling dictionary name. No available dictionaries : You need to install
+# both the python package and the system dependency for enchant to work..
+spelling-dict=
+
+# List of comma separated words that should be considered directives if they
+# appear at the beginning of a comment and should not be checked.
+spelling-ignore-comment-directives=fmt: on,fmt: off,noqa:,noqa,nosec,isort:skip,mypy:
+
+# List of comma separated words that should not be checked.
+spelling-ignore-words=
+
+# A path to a file that contains the private dictionary; one word per line.
+spelling-private-dict-file=
+
+# Tells whether to store unknown words to the private dictionary (see the
+# --spelling-private-dict-file option) instead of raising a message.
+spelling-store-unknown-words=no
+
+
+[STRING]
+
+# This flag controls whether inconsistent-quotes generates a warning when the
+# character used as a quote delimiter is used inconsistently within a module.
+check-quote-consistency=no
+
+# This flag controls whether the implicit-str-concat should generate a warning
+# on implicit string concatenation in sequences defined over several lines.
+check-str-concat-over-line-jumps=no
+
+
+[TYPECHECK]
+
+# List of decorators that produce context managers, such as
+# contextlib.contextmanager. Add to this list to register other decorators that
+# produce valid context managers.
+contextmanager-decorators=contextlib.contextmanager
+
+# List of members which are set dynamically and missed by pylint inference
+# system, and so shouldn't trigger E1101 when accessed. Python regular
+# expressions are accepted.
+generated-members=
+
+# Tells whether to warn about missing members when the owner of the attribute
+# is inferred to be None.
+ignore-none=yes
+
+# This flag controls whether pylint should warn about no-member and similar
+# checks whenever an opaque object is returned when inferring. The inference
+# can return multiple potential results while evaluating a Python object, but
+# some branches might not be evaluated, which results in partial inference. In
+# that case, it might be useful to still emit no-member and other checks for
+# the rest of the inferred objects.
+ignore-on-opaque-inference=yes
+
+# List of symbolic message names to ignore for Mixin members.
+ignored-checks-for-mixins=no-member,
+                          not-async-context-manager,
+                          not-context-manager,
+                          attribute-defined-outside-init
+
+# List of class names for which member attributes should not be checked (useful
+# for classes with dynamically set attributes). This supports the use of
+# qualified names.
+ignored-classes=optparse.Values,thread._local,_thread._local,argparse.Namespace
+
+# Show a hint with possible names when a member name was not found. The aspect
+# of finding the hint is based on edit distance.
+missing-member-hint=yes
+
+# The minimum edit distance a name should have in order to be considered a
+# similar match for a missing member name.
+missing-member-hint-distance=1
+
+# The total number of similar names that should be taken in consideration when
+# showing a hint for a missing member.
+missing-member-max-choices=1
+
+# Regex pattern to define which classes are considered mixins.
+mixin-class-rgx=.*[Mm]ixin
+
+# List of decorators that change the signature of a decorated function.
+signature-mutators=
+
+
+[VARIABLES]
+
+# List of additional names supposed to be defined in builtins. Remember that
+# you should avoid defining new builtins when possible.
+additional-builtins=
+
+# Tells whether unused global variables should be treated as a violation.
+allow-global-unused-variables=yes
+
+# List of names allowed to shadow builtins
+allowed-redefined-builtins=
+
+# List of strings which can identify a callback function by name. A callback
+# name must start or end with one of those strings.
+callbacks=cb_,
+          _cb
+
+# A regular expression matching the name of dummy variables (i.e. expected to
+# not be used).
+dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_
+
+# Argument names that match this expression will be ignored.
+ignored-argument-names=_.*|^ignored_|^unused_
+
+# Tells whether we should check for unused import in __init__ files.
+init-import=no
+
+# List of qualified module names which can have objects that can redefine
+# builtins.
+redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io

.shellcheckrc

@@ -0,0 +1,5 @@
+# This is shellcheck config file
+# Files that are checked: install.sh, export.sh
+
+# Do not complain about variables determined at runtime (IDF_PATH)
+disable=SC1090

.vale.ini

@@ -0,0 +1,119 @@
+###################
+### Vale Config ###
+###################
+
+# This is a Vale linter configuration file.
+# - Repo: esp-idf
+# - Based on Default config: v0-1-1
+# It lists all necessary parameters to configure Vale for your project.
+# For official documentation on all config settings, see
+# https://vale.sh/docs/topics/config
+
+
+##############
+### Global ###
+##############
+
+# This section lists core settings applying to Vale itself.
+
+
+# Specify path to external resources (e.g., styles and vocab files).
+# The path value may be absolute or relative to this configuration file.
+StylesPath = .vale/styles
+
+
+# Specify the minimum alert severity that Vale will report.
+MinAlertLevel = suggestion # "suggestion", "warning", or "error"
+
+
+# Specify vocabulary for special treatment.
+# Create a folder in <StylesPath>/Vocab/<name>/and add its name here
+# The folder should contain two files:
+# - accept.txt -- lists words with accepted case-sensitive spelling
+# - reject.txt -- lists words whose occurrences throw an error
+# Vocab = Espressif
+
+
+# Specify the packages to import into your project.
+# A package is a zip file containing a number of rules (style) written in YAML.
+# For a list of official packages, see Package Hub at https://vale.sh/hub/
+# For official documentation on packages, see
+# https://vale.sh/docs/topics/packages/
+# Before linting, navigate to your project and run `vale sync` to download
+# the official packages specified below.
+# Packages = Package1, Package2, \
+# https://example.com/path/to/package/Package.zip
+Packages = Google, Microsoft, RedHat, \
+https://dl.espressif.com/dl/esp-vale-config/Espressif-latest.zip
+
+
+###############
+### Formats ###
+###############
+
+# This section enables association of "unknown" formats with the ones
+# supported by Vale. For official documentation on supported formats, see
+# https://vale.sh/docs/topics/scoping/
+[formats]
+
+# For example, treat MDX files as Markdown files.
+# mdx = md
+
+
+################################
+### Format-specific settings ###
+################################
+
+# This section lists the settings that apply to specific file formats
+# based on their glob pattern.
+# Settings provided under a more specific glob pattern,
+# such as [*.{md,txt}] will override those in [*].
+[*.{md,rst}]
+
+
+# Enable styles to activate all rules included in them.
+# BasedOnStyles = Style1, Style2
+BasedOnStyles = Vale, Espressif-latest
+
+
+### Deactivate individual rules ###
+### in enabled styles.
+# Style1.Rule1 = NO
+Vale.Repetition = NO
+Vale.Spelling = NO
+Espressif-latest.Admonitions = NO
+Espressif-latest.Contractions = NO
+Espressif-latest.Monospace = NO
+Espressif-latest.PascalCamelCase = NO
+
+
+### Change default severity level ###
+### of an activated rule.
+# Choose between "suggestion", "warning", or "error".
+# Style1.Rule2 = error
+
+
+### Activate individual rules ###
+### in non-enabled styles stored in <StylesPath>.
+# Style1.Rule = YES
+Google.Gender = YES
+Google.GenderBias = YES
+Google.Slang = YES
+Google.Spacing = YES
+Microsoft.DateNumbers = YES
+Microsoft.Ellipses = YES
+Microsoft.FirstPerson = YES
+Microsoft.Hyphens = YES
+Microsoft.Ordinal = YES
+Microsoft.OxfordComma = YES
+Microsoft.Percentages = YES
+Microsoft.RangeTime = YES
+Microsoft.Semicolon = YES
+Microsoft.SentenceLength = YES
+Microsoft.Suspended = YES
+Microsoft.Units = YES
+Microsoft.URLFormat = YES
+Microsoft.We = YES
+Microsoft.Wordiness = YES
+RedHat.Contractions = YES
+RedHat.RepeatedWords = YES

CMakeLists.txt

@@ -20,8 +20,11 @@ if(NOT BOOTLOADER_BUILD)
         if(CMAKE_C_COMPILER_ID MATCHES "GNU")
             list(APPEND compile_options "-freorder-blocks")
         endif()
-    elseif(CONFIG_COMPILER_OPTIMIZATION_DEFAULT)
+    elseif(CONFIG_COMPILER_OPTIMIZATION_DEBUG)
         list(APPEND compile_options "-Og")
+        if(CMAKE_C_COMPILER_ID MATCHES "GNU" AND NOT CONFIG_IDF_TARGET_LINUX)
+            list(APPEND compile_options "-fno-shrink-wrap")  # Disable shrink-wrapping to reduce binary size
+        endif()
     elseif(CONFIG_COMPILER_OPTIMIZATION_NONE)
         list(APPEND compile_options "-O0")
     elseif(CONFIG_COMPILER_OPTIMIZATION_PERF)
@@ -37,6 +40,9 @@ else()  # BOOTLOADER_BUILD
         endif()
     elseif(CONFIG_BOOTLOADER_COMPILER_OPTIMIZATION_DEBUG)
         list(APPEND compile_options "-Og")
+        if(CMAKE_C_COMPILER_ID MATCHES "GNU" AND NOT CONFIG_IDF_TARGET_LINUX)
+            list(APPEND compile_options "-fno-shrink-wrap")  # Disable shrink-wrapping to reduce binary size
+        endif()
     elseif(CONFIG_BOOTLOADER_COMPILER_OPTIMIZATION_NONE)
         list(APPEND compile_options "-O0")
     elseif(CONFIG_BOOTLOADER_COMPILER_OPTIMIZATION_PERF)
@@ -115,6 +121,10 @@ if(CMAKE_C_COMPILER_ID MATCHES "Clang")
     list(APPEND compile_options "-Wno-c2x-extensions")
     # warning on xMPU_SETTINGS for esp32s2 has size 0 for C and 1 for C++
     list(APPEND compile_options "-Wno-extern-c-compat")
+    if(NOT (CONFIG_IDF_TARGET_LINUX AND CMAKE_HOST_SYSTEM_NAME STREQUAL "Darwin"))
+        # warning: implicit truncation from 'int' to a one-bit wide bit-field changes value from 1 to -1
+        list(APPEND compile_options "-Wno-single-bit-bitfield-constant-conversion")
+    endif()
 endif()
 # More warnings may exist in unit tests and example projects.
 
@@ -184,6 +194,13 @@ if(CONFIG_COMPILER_DISABLE_GCC12_WARNINGS)
                                 "-Wno-use-after-free")
 endif()
 
+if(CONFIG_COMPILER_DISABLE_GCC13_WARNINGS)
+    list(APPEND compile_options "-Wno-xor-used-as-pow")
+    list(APPEND c_compile_options "-Wno-enum-int-mismatch")
+    list(APPEND cxx_compile_options "-Wno-self-move"
+                                    "-Wno-dangling-reference")
+endif()
+
 # GCC-specific options
 if(CMAKE_C_COMPILER_ID STREQUAL "GNU")
     list(APPEND compile_options "-fstrict-volatile-bitfields"
@@ -224,6 +241,10 @@ if(CMAKE_C_COMPILER_ID MATCHES "Clang")
     list(APPEND compile_options "-fno-use-cxa-atexit")
 endif()
 
+if(COMPILER_RT_LIB_NAME)
+    list(APPEND link_options "-rtlib=${CONFIG_COMPILER_RT_LIB_NAME}")
+endif()
+
 # For the transition period from 32-bit time_t to 64-bit time_t,
 # auto-detect the size of this type and set corresponding variable.
 include(CheckTypeSize)

COMPATIBILITY.md

@@ -0,0 +1,101 @@
+# Compatibility Between ESP-IDF Releases and Revisions of Espressif SoCs
+
+* [中文版](./COMPATIBILITY_CN.md)
+
+Espressif keeps improving the performance of its SoCs by providing new chip revisions. However, some of the improvements require special software support. Some of the software supports are even mandatory for the chip revisions to run normally.
+
+This document describes the compatibility between ESP-IDF releases and Espressif SoC revisions.
+
+NOTE: This document on release branches may be out-of-date. Check the [Compatibility file on master](https://github.com/espressif/esp-idf/blob/master/COMPATIBILITY.md) for the most accurate information.
+
+See [Compatibility Advisory for Chip Revision Numbering Scheme](https://www.espressif.com.cn/sites/default/files/advisory_downloads/AR2022-005%20Compatibility%20Advisory%20for%20Chip%20Revision%20Numbering%20%20Scheme.pdf) on the versioning of Espressif SoC revisions.
+
+You can run `esptool chip_id` to detect the series and revision of an SoC. See [SoC Errata](https://www.espressif.com.cn/en/support/documents/technical-documents?keys=errata) for more on how to distinguish between chip revisions, and the improvements provided by chip revisions. And run `idf.py --version` to know the version of current ESP-IDF.
+
+## ESP-IDF Support for Different Chip Revisions
+
+The sections below show the requirements to ESP-IDF version of chip revisions. Each chip revision corresponds to specific `Recommended` and `Required` versions of ESP-IDF:
+
+- `Recommended`: shows from which version of ESP-IDF you can make use of all the improvements of the chip revision. Running binary compiled with ESP-IDF below the `Recommended` version of a chip revision, software may not benefit from the bugfix/features provided by the chip revision. The chip will have almost the same behavior as its previous revision.
+
+- `Required`: shows the minimum version required to run the chip revision normally. Running binary compiled below the `Required` version, the binary may have unpredictable behavior.
+
+Though the software can make use of all the features of a chip revision, if its version is higher than the `Recommended` version of the chip, it is still recommended to use the latest bugfix version of the release branch. The latest bugfix version fixes a number of issues and helps improve product stability.
+
+For example, if we have a chip, whose `Required`/`Recommended` version of `release/v5.1` branch is `v5.1.2`/`v5.1.4`, and the latest release on that branch is `v5.1.6`. Then the chip will not boot up with ESP-IDF `v5.1`-`v5.1.1` or will have unpredictable behavior, and application may not make use of all benefits of the chip, when running with ESP-IDF `v5.1.2` or `v5.1.3`. Though `v5.1.4` well supports the chip revision, it is still recommended to upgrade ESP-IDF to `v5.1.6`.
+
+### ESP32
+
+#### v0.0, v1.0, v3.0
+
+Supported since initial version of ESP-IDF.
+
+#### v1.1
+
+To be added.
+
+#### v2.0
+
+To be added.
+
+#### v3.1
+
+To be added.
+
+### ESP32-S2
+
+#### v0.0
+
+Supported since ESP-IDF v4.2.
+
+#### v1.0
+
+| Release branch         | Recommended | Required |
+|------------------------|-------------|----------|
+| release/v4.2           | v4.2.3      | v4.2     |
+| release/v4.3           | v4.3.3      | v4.3     |
+| release/v4.4           | v4.4.6      | v4.4     |
+| release/v5.0           | v5.0.4      | v5.0     |
+| release/v5.1           | v5.1.2      | v5.1     |
+| release/v5.2 and above | v5.2        | v5.2     |
+
+### ESP32-C3
+
+#### v0.2, v0.3
+
+Supported since ESP-IDF v4.3.
+
+#### v0.4
+
+To be added.
+
+### ESP32-S3
+
+#### v0.1
+
+Supported since ESP-IDF v4.4.
+
+#### v0.2
+
+To be added.
+
+### ESP32-C2 & ESP8684
+
+#### v1.0
+
+Supported since ESP-IDF v5.0.
+
+#### v1.1
+
+To be added.
+
+#### v1.2
+
+To be added.
+
+
+## What If the ESP-IDF Version Is Lower than the `Required` Version?
+
+Latest ESP-IDF versions can prevent from downloading to, or even execute binaries on unsupported chips. ESP-IDF of versions v4.4.5+, v5.0.1+, v5.1 and above have both esptool download check and bootloader loading check against the chip revision. While ESP-IDF v4.3.5 has only esptool downloading check.
+
+For earlier ESP-IDF versions without such checking, which is incompatible with the given chip revision, the chips running such versions will have unpredictable behavior.

COMPATIBILITY_CN.md

@@ -0,0 +1,101 @@
+# ESP-IDF 版本与乐鑫芯片版本兼容性
+
+* [English Version](./COMPATIBILITY.md)
+
+为不断提高芯片性能，乐鑫会为其芯片发布新版本。但新芯片版本中的某些性能提升依赖特殊的软件支持，有时候新芯片版本必须在一定的软件版本下才能正常运行。
+
+本文档介绍了具体 ESP-IDF 版本与乐鑫芯片版本之间的兼容性情况。
+
+注意：各分支上的兼容性文档可能不是最新版本，请参考 [master 分支上的兼容性文件](https://github.com/espressif/esp-idf/blob/master/COMPATIBILITY_CN.md) 以获取最新信息。
+
+有关乐鑫芯片版本的编码方式，请参考 [关于芯片版本 (Chip Revision) 编码方式的兼容性公告](https://www.espressif.com/sites/default/files/advisory_downloads/AR2022-005%20%E5%85%B3%E4%BA%8E%E8%8A%AF%E7%89%87%E7%89%88%E6%9C%AC%E7%BC%96%E7%A0%81%E6%96%B9%E5%BC%8F%20%28Chip%20Revision%29%20%E7%9A%84%E5%85%BC%E5%AE%B9%E6%80%A7%E5%85%AC%E5%91%8A.pdf)。
+
+运行 `esptool chip_id` 可查看芯片系列及其版本。有关区分芯片版本及版本改进内容的更多信息，请参考 [芯片勘误表](https://www.espressif.com.cn/zh-hans/support/documents/technical-documents?keys=%E5%8B%98%E8%AF%AF%E8%A1%A8)。运行 `idf.py --version` 可查看当前的 ESP-IDF 版本。
+
+## ESP-IDF 对各芯片版本的支持
+
+下文介绍了 ESP-IDF 对各芯片版本的支持情况，每个芯片版本都有对应的 ESP-IDF `推荐版本` 和 `需求版本`：
+
+- `推荐版本`：表示从该版本的 ESP-IDF 开始，软件可以利用芯片版本提升的性能。如果在该芯片版本上运行低于 `推荐版本` 的 ESP-IDF 来编译二进制文件，软件可能无法利用该芯片版本修复的错误或新增的功能，芯片行为将与其上一版本几乎相同。
+
+- `需求版本`：表示该芯片版本正常运行所需的最低 ESP-IDF 版本。如果在该芯片版本上运行低于 `需求版本` 的 ESP-IDF 来编译二进制文件，可能会出现不可预测的芯片行为。
+
+即便使用的软件版本已高于该芯片版本的对应 `推荐版本`，软件已经能够利用该芯片版本的所有功能，我们仍建议用户升级到该发布分支的最新 bugfix 版本。新的 bugfix 版本修复了一些问题，有助于提升产品稳定性。
+
+例如，对于某一芯片版本，其 `release/v5.1` 分支的 `需求版本` 和 `推荐版本` 分别是 `v5.1.2` 和 `v5.1.4`，而该分支的最新版本是 `v5.1.6`。那么，在使用 ESP-IDF `v5.1` - `v5.1.1` 时，芯片将无法启动，或会出现不可预测的行为，而在使用 ESP-IDF `v5.1.2` 或 `v5.1.3` 时，应用程序可能无法使用芯片的部分性能。此外，虽然 `v5.1.4` 已支持该芯片版本，但仍建议将 ESP-IDF 升级到 `v5.1.6`。
+
+### ESP32
+
+#### v0.0、v1.0 和 v3.0
+
+从最初版本的 ESP-IDF 开始支持。
+
+#### v1.1
+
+待更新。
+
+#### v2.0
+
+待更新。
+
+#### v3.1
+
+待更新。
+
+### ESP32-S2
+
+#### v0.0
+
+从 ESP-IDF v4.2 开始支持。
+
+#### v1.0
+
+| 发布分支               | 推荐版本    | 需求版本 |
+|------------------------|-------------|----------|
+| release/v4.2           | v4.2.3      | v4.2     |
+| release/v4.3           | v4.3.3      | v4.3     |
+| release/v4.4           | v4.4.6      | v4.4     |
+| release/v5.0           | v5.0.4      | v5.0     |
+| release/v5.1           | v5.1.2      | v5.1     |
+| release/v5.2 及以上     | v5.2        | v5.2     |
+
+### ESP32-C3
+
+#### v0.2 和 v0.3
+
+从 ESP-IDF v4.3 开始支持。
+
+#### v0.4
+
+待更新。
+
+### ESP32-S3
+
+#### v0.1
+
+从 ESP-IDF v4.4 开始支持。
+
+#### v0.2
+
+待更新。
+
+### ESP32-C2 & ESP8684
+
+#### v1.0
+
+从 ESP-IDF v5.0 开始支持。
+
+#### v1.1
+
+待更新。
+
+#### v1.2
+
+待更新。
+
+
+## 如果 ESP-IDF 版本低于 `需求版本` 会出现什么情况？
+
+使用最新的 ESP-IDF 版本时，软件会阻止下载二进制文件到不支持的芯片版本上，甚至可以防止二进制文件在不支持的芯片版本上被执行。v4.4.5+、v5.0.1+、v5.1 及以上版本的 ESP-IDF 都支持针对芯片版本的 esptool 下载检查和引导加载器加载检查，但 ESP-IDF v4.3.5 只支持 esptool 下载检查。
+
+更早的 ESP-IDF 版本没有此类检查，若与芯片版本不兼容，芯片运行软件时可能会出现不可预测的行为。

Kconfig

@@ -11,14 +11,43 @@ mainmenu "Espressif IoT Development Framework Configuration"
         default "y"
 
     config IDF_ENV_FPGA
-        # This option is for internal use only
         bool
         option env="IDF_ENV_FPGA"
+        help
+            - This option is for internal use only.
+            - Enabling this option will help enable all FPGA support so as to
+              run ESP-IDF on an FPGA. This can help reproduce some issues that
+              only happens on FPGA condition, or when you have to burn some
+              efuses multiple times.
+
+    config IDF_ENV_BRINGUP
+        bool
+        help
+            - This option is ONLY used when doing new chip bringup.
+            - This option will only enable necessary hw / sw settings for running
+              a hello_world application.
+
 
     config IDF_CI_BUILD
         bool
         default y if "$(IDF_CI_BUILD)" = "y" || "$(IDF_CI_BUILD)" = 1
 
+    config IDF_DOC_BUILD
+        bool
+        default y if "$(IDF_DOC_BUILD)" = "y" || "$(IDF_DOC_BUILD)" = 1
+
+    config IDF_TOOLCHAIN
+        # This option records the IDF target when sdkconfig is generated the first time.
+        # It is not updated if environment variable $IDF_TOOLCHAIN changes later, and
+        # the build system is responsible for detecting the mismatch between
+        # CONFIG_IDF_TOOLCHAIN and $IDF_TOOLCHAIN.
+        string
+        default "$IDF_TOOLCHAIN"
+
+    config IDF_TOOLCHAIN_CLANG
+        bool
+        default "y" if IDF_TOOLCHAIN="clang"
+
     config IDF_TARGET_ARCH_RISCV
         bool
         default "n"
@@ -40,6 +69,12 @@ mainmenu "Espressif IoT Development Framework Configuration"
         string
         default "$IDF_TARGET"
 
+    config IDF_INIT_VERSION
+        # This option records the IDF version when sdkconfig is generated the first time.
+        # It is not updated if environment variable $IDF_VERSION changes later
+        string
+        default "$IDF_INIT_VERSION"
+
     config IDF_TARGET_LINUX
         bool
         default "y" if IDF_TARGET="linux"
@@ -66,28 +101,6 @@ mainmenu "Espressif IoT Development Framework Configuration"
         select FREERTOS_UNICORE
         select IDF_TARGET_ARCH_RISCV
 
-    config IDF_TARGET_ESP32H4
-        bool
-        default "y" if IDF_TARGET="esp32h4"
-        select FREERTOS_UNICORE
-        select IDF_TARGET_ARCH_RISCV
-
-    choice IDF_TARGET_ESP32H4_BETA_VERSION
-        prompt "ESP32-H4 beta version"
-        depends on IDF_TARGET_ESP32H4
-        default IDF_TARGET_ESP32H4_BETA_VERSION_2
-        help
-            Currently ESP32-H4 has several beta versions for internal use only.
-            Select the one that matches your chip model.
-
-        config IDF_TARGET_ESP32H4_BETA_VERSION_1
-            bool
-            prompt "ESP32-H4 beta1"
-        config IDF_TARGET_ESP32H4_BETA_VERSION_2
-            bool
-            prompt "ESP32-H4 beta2"
-    endchoice
-
     config IDF_TARGET_ESP32C2
         bool
         default "y" if IDF_TARGET="esp32c2"
@@ -100,6 +113,18 @@ mainmenu "Espressif IoT Development Framework Configuration"
         select FREERTOS_UNICORE
         select IDF_TARGET_ARCH_RISCV
 
+    config IDF_TARGET_ESP32C5
+        bool
+        default "y" if IDF_TARGET="esp32c5"
+        select FREERTOS_UNICORE
+        select IDF_TARGET_ARCH_RISCV
+        select IDF_ENV_BRINGUP
+
+    config IDF_TARGET_ESP32P4
+        bool
+        default "y" if IDF_TARGET="esp32p4"
+        select IDF_TARGET_ARCH_RISCV
+
     config IDF_TARGET_ESP32H2
         bool
         default "y" if IDF_TARGET="esp32h2"
@@ -116,11 +141,11 @@ mainmenu "Espressif IoT Development Framework Configuration"
         default 0x0002 if IDF_TARGET_ESP32S2
         default 0x0005 if IDF_TARGET_ESP32C3
         default 0x0009 if IDF_TARGET_ESP32S3
-        default 0x000A if IDF_TARGET_ESP32H4_BETA_VERSION_1
         default 0x000C if IDF_TARGET_ESP32C2
         default 0x000D if IDF_TARGET_ESP32C6
-        default 0x000E if IDF_TARGET_ESP32H4_BETA_VERSION_2 # ESP32-TODO: IDF-3475
         default 0x0010 if IDF_TARGET_ESP32H2
+        default 0x0012 if IDF_TARGET_ESP32P4
+        default 0x0013 if IDF_TARGET_ESP32C5
         default 0xFFFF
 
 
@@ -288,11 +313,11 @@ mainmenu "Espressif IoT Development Framework Configuration"
 
         choice COMPILER_OPTIMIZATION
             prompt "Optimization Level"
-            default COMPILER_OPTIMIZATION_DEFAULT
+            default COMPILER_OPTIMIZATION_DEBUG
             help
                 This option sets compiler optimization level (gcc -O argument) for the app.
 
-                - The "Default" setting will add the -0g flag to CFLAGS.
+                - The "Debug" setting will add the -0g flag to CFLAGS.
                 - The "Size" setting will add the -0s flag to CFLAGS.
                 - The "Performance" setting will add the -O2 flag to CFLAGS.
                 - The "None" setting will add the -O0 flag to CFLAGS.
@@ -311,7 +336,7 @@ mainmenu "Espressif IoT Development Framework Configuration"
                 Compiler optimization for the IDF bootloader is set separately,
                 see the BOOTLOADER_COMPILER_OPTIMIZATION setting.
 
-            config COMPILER_OPTIMIZATION_DEFAULT
+            config COMPILER_OPTIMIZATION_DEBUG
                 bool "Debug (-Og)"
             config COMPILER_OPTIMIZATION_SIZE
                 bool "Optimize for size (-Os)"
@@ -511,12 +536,47 @@ mainmenu "Espressif IoT Development Framework Configuration"
                 Enable this option if use GCC 12 or newer, and want to disable warnings which don't appear with
                 GCC 11.
 
+        config COMPILER_DISABLE_GCC13_WARNINGS
+            bool "Disable new warnings introduced in GCC 13"
+            default "n"
+            help
+                Enable this option if use GCC 13 or newer, and want to disable warnings which don't appear with
+                GCC 12.
+
         config COMPILER_DUMP_RTL_FILES
             bool "Dump RTL files during compilation"
             help
                 If enabled, RTL files will be produced during compilation. These files
                 can be used by other tools, for example to calculate call graphs.
 
+        choice COMPILER_RT_LIB
+            prompt "Compiler runtime library"
+            default COMPILER_RT_LIB_CLANGRT if IDF_TOOLCHAIN_CLANG
+            default COMPILER_RT_LIB_HOST if IDF_TARGET_LINUX
+            default COMPILER_RT_LIB_GCCLIB
+            help
+                Select runtime library to be used by compiler.
+                - GCC toolchain supports libgcc only.
+                - Clang allows to choose between libgcc or libclang_rt.
+                - For host builds ("linux" target), uses the default library.
+
+            config COMPILER_RT_LIB_GCCLIB
+                depends on !IDF_TARGET_LINUX
+                bool "libgcc"
+            config COMPILER_RT_LIB_CLANGRT
+                depends on IDF_TOOLCHAIN_CLANG && !IDF_TARGET_LINUX
+                bool "libclang_rt"
+            config COMPILER_RT_LIB_HOST
+                depends on IDF_TARGET_LINUX
+                bool "Host"
+        endchoice
+
+        config COMPILER_RT_LIB_NAME
+            string
+            default "clang_rt.builtins" if COMPILER_RT_LIB_CLANGRT
+            default "gcc" if COMPILER_RT_LIB_GCCLIB
+            default "" if COMPILER_RT_LIB_HOST
+
     endmenu # Compiler Options
 
     menu "Component config"
@@ -531,3 +591,10 @@ mainmenu "Espressif IoT Development Framework Configuration"
 
             Note you should still enable a certain experimental feature option to use it, and you
             should read the corresponding risk warning and known issue list carefully.
+
+            Current experimental feature list:
+
+            - CONFIG_ESPTOOLPY_FLASHFREQ_120M && CONFIG_ESPTOOLPY_FLASH_SAMPLE_MODE_DTR
+            - CONFIG_SPIRAM_SPEED_120M && CONFIG_SPIRAM_MODE_OCT
+            - CONFIG_BOOTLOADER_CACHE_32BIT_ADDR_QUAD_FLASH
+            - CONFIG_MBEDTLS_USE_CRYPTO_ROM_IMPL

README.md

@@ -6,7 +6,7 @@ ESP-IDF is the development framework for Espressif SoCs supported on Windows, Li
 
 # ESP-IDF Release Support Schedule
 
-![Support Schedule](https://dl.espressif.com/dl/esp-idf/support-periods.svg)
+![Support Schedule](https://dl.espressif.com/dl/esp-idf/support-periods.svg?v=1)
 
 - Please read [the support policy](SUPPORT_POLICY.md) and [the documentation](https://docs.espressif.com/projects/esp-idf/en/latest/esp32/versions.html) for more information about ESP-IDF versions.
 - Please see the [End-of-Life Advisories](https://www.espressif.com/en/support/documents/advisories?keys=&field_type_of_advisory_tid%5B%5D=817) for information about ESP-IDF releases with discontinued support.
@@ -15,19 +15,22 @@ ESP-IDF is the development framework for Espressif SoCs supported on Windows, Li
 
 The following table shows ESP-IDF support of Espressif SoCs where ![alt text][preview] and ![alt text][supported] denote preview status and support, respectively. The preview support is usually limited in time and intended for beta versions of chips. Please use an ESP-IDF release where the desired SoC is already supported.
 
-|Chip         |          v4.2          |         v4.3           |          v4.4          |          v5.0          |         v5.1           |                                                            |
-|:----------- | :---------------------:| :---------------------:| :---------------------:| :---------------------:| :--------------------: | :----------------------------------------------------------|
+|Chip         |         v4.3           |          v4.4          |          v5.0          |         v5.1           |         v5.2           |                                                            |
+|:----------- | :---------------------:| :---------------------:| :---------------------:| :--------------------: | :--------------------: | :----------------------------------------------------------|
 |ESP32        | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] |                                                            |
 |ESP32-S2     | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] |                                                            |
-|ESP32-C3     |                        | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] |                                                            |
-|ESP32-S3     |                        |                        | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] | [Announcement](https://www.espressif.com/en/news/ESP32_S3) |
-|ESP32-C2     |                        |                        |                        | ![alt text][supported] | ![alt text][supported] | [Announcement](https://www.espressif.com/en/news/ESP32-C2) |
-|ESP32-C6     |                        |                        |                        |                        | ![alt text][supported] | [Announcement](https://www.espressif.com/en/news/ESP32_C6) |
-|ESP32-H2     |                        |                        |                        |                        | ![alt text][supported] | [Announcement](https://www.espressif.com/en/news/ESP32_H2) |
+|ESP32-C3     | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] |                                                            |
+|ESP32-S3     |                        | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] | [Announcement](https://www.espressif.com/en/news/ESP32_S3) |
+|ESP32-C2     |                        |                        | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] | [Announcement](https://www.espressif.com/en/news/ESP32-C2) |
+|ESP32-C6     |                        |                        |                        | ![alt text][supported] | ![alt text][supported] | [Announcement](https://www.espressif.com/en/news/ESP32_C6) |
+|ESP32-H2     |                        |                        |                        | ![alt text][supported] | ![alt text][supported] | [Announcement](https://www.espressif.com/en/news/ESP32_H2) |
+|ESP32-P4     |                        |                        |                        |                        | ![alt text][preview]   | [Announcement](https://www.espressif.com/en/news/ESP32-P4) |
 
 [supported]: https://img.shields.io/badge/-supported-green "supported"
 [preview]: https://img.shields.io/badge/-preview-orange "preview"
 
+There are variants of revisions for a series of chips. See [Compatibility Between ESP-IDF Releases and Revisions of Espressif SoCs](https://github.com/espressif/esp-idf/blob/master/COMPATIBILITY.md) for the details of the compatibility between ESP-IDF and chip revisions.
+
 Espressif SoCs released before 2016 (ESP8266 and ESP8285) are supported by [RTOS SDK](https://github.com/espressif/ESP8266_RTOS_SDK) instead.
 
 # Developing With ESP-IDF
@@ -118,6 +121,8 @@ This can be combined with other targets, ie `idf.py -p PORT erase-flash flash` w
 
 * Documentation for the latest version: https://docs.espressif.com/projects/esp-idf/. This documentation is built from the [docs directory](docs) of this repository.
 
+* [Beginner's Guide to Key Concepts and Resources of ESP-IDF](https://youtu.be/J8zc8mMNKtc?feature=shared)
+
 * The [esp32.com forum](https://esp32.com/) is a place to ask questions and find community resources.
 
 * [Check the Issues section on github](https://github.com/espressif/esp-idf/issues) if you find a bug or have a feature request. Please check existing Issues before opening a new one.

README_CN.md

@@ -6,7 +6,7 @@ ESP-IDF 是乐鑫官方推出的物联网开发框架，支持 Windows、Linux
 
 # ESP-IDF 版本支持期限
 
-![支持期限](https://dl.espressif.com/dl/esp-idf/support-periods.svg)
+![支持期限](https://dl.espressif.com/dl/esp-idf/support-periods.svg?v=1)
 
 - 请参考 [ESP-IDF 支持政策](SUPPORT_POLICY_CN.md) 以及 [相关文档](https://docs.espressif.com/projects/esp-idf/zh_CN/latest/esp32/versions.html) 了解更多关于 ESP-IDF 版本的信息。
 - 请参考 [ESP-IDF 版本停止维护 (EOL) 公告](https://www.espressif.com/zh-hans/support/documents/advisories?keys=&field_type_of_advisory_tid%5B%5D=817)。
@@ -15,19 +15,22 @@ ESP-IDF 是乐鑫官方推出的物联网开发框架，支持 Windows、Linux
 
 下表总结了乐鑫芯片在 ESP-IDF 各版本中的支持状态，其中 ![alt text][supported] 代表已支持，![alt text][preview] 代表目前处于预览支持状态。预览支持状态通常有时间限制，而且仅适用于测试版芯片。请确保使用与芯片相匹配的 ESP-IDF 版本。
 
-|芯片         |         v4.2          |         v4.3           |          v4.4          |          v5.0          |          v5.1          |                                                                 |
-|:----------- |:---------------------:| :---------------------:| :---------------------:| :---------------------:| :--------------------: | :-------------------------------------------------------------- |
-|ESP32        |![alt text][supported] | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] |                                                                 |
-|ESP32-S2     |![alt text][supported] | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] |                                                                 |
-|ESP32-C3     |                       | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] |                                                                 |
-|ESP32-S3     |                       |                        | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] | [芯片发布公告](https://www.espressif.com/zh-hans/news/ESP32_S3) |
-|ESP32-C2     |                       |                        |                        | ![alt text][supported] | ![alt text][supported] | [芯片发布公告](https://www.espressif.com/zh-hans/news/ESP32-C2) |
-|ESP32-C6     |                       |                        |                        |                        | ![alt text][supported] | [芯片发布公告](https://www.espressif.com/zh-hans/news/ESP32_C6) |
-|ESP32-H2     |                       |                        |                        |                        | ![alt text][supported] | [芯片发布公告](https://www.espressif.com/zh-hans/news/ESP32_H2) |
+|芯片         |         v4.3           |          v4.4          |          v5.0          |          v5.1          |          v5.2          |                                                                 |
+|:----------- | :---------------------:| :---------------------:| :---------------------:| :--------------------: | :--------------------: | :-------------------------------------------------------------- |
+|ESP32        | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] |                                                                 |
+|ESP32-S2     | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] |                                                                 |
+|ESP32-C3     | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] |                                                                 |
+|ESP32-S3     |                        | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] | [芯片发布公告](https://www.espressif.com/zh-hans/news/ESP32_S3) |
+|ESP32-C2     |                        |                        | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] | [芯片发布公告](https://www.espressif.com/zh-hans/news/ESP32-C2) |
+|ESP32-C6     |                        |                        |                        | ![alt text][supported] | ![alt text][supported] | [芯片发布公告](https://www.espressif.com/zh-hans/news/ESP32_C6) |
+|ESP32-H2     |                        |                        |                        | ![alt text][supported] | ![alt text][supported] | [芯片发布公告](https://www.espressif.com/zh-hans/news/ESP32_H2) |
+|ESP32-P4     |                        |                        |                        |                        | ![alt text][preview]   | [芯片发布公告](https://www.espressif.com/en/news/ESP32-P4)      |
 
 [supported]: https://img.shields.io/badge/-%E6%94%AF%E6%8C%81-green "supported"
 [preview]: https://img.shields.io/badge/-%E9%A2%84%E8%A7%88-orange "preview"
 
+每款乐鑫芯片都可能有不同版本。建议参考 [ESP-IDF 版本与乐鑫芯片版本兼容性](https://github.com/espressif/esp-idf/blob/master/COMPATIBILITY_CN.md)，了解 ESP-IDF 版本与各芯片版本之间的兼容性。
+
 对于 2016 年之前发布的乐鑫芯片（包括 ESP8266 和 ESP8285），请参考 [RTOS SDK](https://github.com/espressif/ESP8266_RTOS_SDK)。
 
 # 使用 ESP-IDF 进行开发
@@ -118,6 +121,8 @@ ESP-IDF 中的子模块采用相对路径（[详见 .gitmodules 文件](.gitmodu
 
 * 最新版的文档：https://docs.espressif.com/projects/esp-idf/ ，该文档是由本仓库 [docs 目录](docs) 构建得到。
 
+* [初学者指南：主要概念和资源](https://www.bilibili.com/video/BV1114y1r7du/) 
+
 * 可以前往 [esp32.com 论坛](https://esp32.com/) 提问，挖掘社区资源。
 
 * 如果你在使用中发现了错误或者需要新的功能，请先[查看 GitHub Issues](https://github.com/espressif/esp-idf/issues)，确保该问题没有重复提交。

SECURITY.md

@@ -6,4 +6,4 @@ Please refer to https://docs.espressif.com/projects/esp-idf/en/latest/esp32/vers
 
 ## Reporting a Vulnerability
 
-If you think you have found a security vulnerability in Espressif solutions (including ESP-IDF), then please send an email to our Bug Bounty team at bugbounty@espressif.com. Please do **NOT** create a public GitHub issue.
+Please refer to [Espressif Security Incident Response Process](https://www.espressif.com/sites/default/files/Espressif%20Security%20Incident%20Response%20Process%20v1.0_EN.pdf) on the guidelines to report a security vulnerability. Please do **NOT** create a public GitHub issue.

components/README.md

@@ -1,4 +1,3 @@
-
 # Core Components
 
 ## Overview
@@ -9,16 +8,9 @@ This document contains details about what the core components are, what they con
 
 The core components are organized into two groups.
 
-The first group (referred to as `G0` from now on) contains `hal`, `xtensa` and `riscv` (referred to as `arch` components from now on), `esp_rom`, `esp_common`, and `soc`. This
-group contain information about and low-level access to underlying hardware; or in the case of `esp_common`, hardware-agnostic code and utilities.
-These components can depend on each other, but as much as possible have no dependencies outside the group. The reason for this is that, due to the
-nature of what these components contain, the likelihood is high that a lot of other components will require these. Ideally, then, the dependency
-relationship only goes one way. This makes it easier for these components, as a group, to be usable in another project. One can conceivably implement
-a competing SDK to ESP-IDF on top of these components.
+The first group (referred to as `G0`) includes `hal`, `arch` (where `arch` is either `riscv` or `xtensa` depending on the chip), `esp_rom`, `esp_common`, and `soc`. This group contains information about and provides low-level access to the underlying hardware. In the case of `esp_common`, it contains hardware-agnostic code and utilities. These components may have dependencies on each other within the group, but outside dependencies should be minimized. The reason for this approach is that these components are fundamental, and many other components may require them. Ideally, the dependency relationship only goes one way, making it easier for this group to be usable in other projects.
 
-The second group (referred to as `G1` from now on) sits at a higher level than the first group. This group contains the components `esp_hw_support`, `esp_system`, `newlib`, `spi_flash`,
-`freertos`, `log`, and `heap`. Like the first group, circular dependencies within the group are allowed; and being at a higher level, dependency on the first group
-is allowed. These components represent software mechanisms essential to building other components.
+The second group (referred to as `G1`) operates at a higher level than the first group. `G1` includes the components `esp_hw_support`, `esp_system`, `newlib`, `spi_flash`, `freertos`, `log`, and `heap`. Like the first group, circular dependencies within this group are allowed, and these components can have dependencies on the first group. G1 components represent essential software mechanisms for building other components.
 
 ## Descriptions
 
@@ -33,7 +25,7 @@ into routines that achieve a meaningful action or state of the peripheral.
 
 Example:
 
-- `spi_flash_ll_set_address` is a low-level function part of the hardware abstraction `spi_flash_hal_read_block`
+-   `spi_flash_ll_set_address` is a low-level function part of the hardware abstraction `spi_flash_hal_read_block`
 
 #### `arch`
 
@@ -42,9 +34,9 @@ This can also contain files provided by the architecture vendor.
 
 Example:
 
-- `xt_set_exception_handler`
-- `rv_utils_intr_enable`
-- `ERI_PERFMON_MAX`
+-   `xt_set_exception_handler`
+-   `rv_utils_intr_enable`
+-   `ERI_PERFMON_MAX`
 
 #### `esp_common`
 
@@ -52,9 +44,9 @@ Contains hardware-agnostic definitions, constants, macros, utilities, 'pure' and
 
 Example:
 
-- `BIT(nr)` and other bit manipulation utilities in the future
-- `IDF_DEPRECATED(REASON)`
-- `ESP_IDF_VERSION_MAJOR`
+-   `BIT(nr)` and other bit manipulation utilities in the future
+-   `IDF_DEPRECATED(REASON)`
+-   `ESP_IDF_VERSION_MAJOR`
 
 #### `soc`
 
@@ -62,9 +54,9 @@ Contains description of the underlying hardware: register structure, addresses,
 
 Example:
 
-- `DR_REG_DPORT_BASE`
-- `SOC_MCPWM_SUPPORTED`
-- `uart_dev_s`
+-   `DR_REG_DPORT_BASE`
+-   `SOC_MCPWM_SUPPORTED`
+-   `uart_dev_s`
 
 #### `esp_rom`
 
@@ -72,8 +64,8 @@ Contains headers, linker scripts, abstraction layer, patches, and other related
 
 Example:
 
-- `esp32.rom.eco3.ld`
-- `rom/aes.h`
+-   `esp32.rom.eco3.ld`
+-   `rom/aes.h`
 
 ### `G1` Components
 
@@ -99,8 +91,20 @@ Some functions n the standard library are implemented here, especially those nee
 
 Example:
 
-- `malloc` is implemented in terms of the component `heap`'s functions
-- `gettimeofday` is implemented in terms of system time in `esp_system`
+-   `malloc` is implemented in terms of the component `heap`'s functions
+-   `gettimeofday` is implemented in terms of system time in `esp_system`
+
+#### `esp_mm`
+
+Memory management. Currently, this encompasses:
+
+-   Memory mapping for MMU supported memories
+-   Memory synchronisation via Cache
+-   Utils such as APIs to convert between virtual address and physical address
+
+#### `esp_psram`
+
+Contains implementation of PSRAM services
 
 #### `esp_system`
 
@@ -108,10 +112,10 @@ Contains implementation of system services and controls system behavior. The imp
 here may take hardware resources and/or decide on a hardware state needed for support of a system service/feature/mechanism.
 Currently, this encompasses the following, but not limited to:
 
-- Startup and initialization
-- Panic and debug
-- Reset and reset reason
-- Task and interrupt watchdogs
+-   Startup and initialization
+-   Panic and debug
+-   Reset and reset reason
+-   Task and interrupt watchdogs
 
 #### `esp_hw_support`
 
@@ -119,19 +123,19 @@ Contains implementations that provide hardware operations, arbitration, or resou
 is used in the system. Unlike `esp_system`, implementations here do not decide on a hardware state or takes hardware resource, acting
 merely as facilitator to hardware access. Currently, this encompasses the following, but not limited to:
 
-- Interrupt allocation
-- Sleep functions
-- Memory functions (external SPIRAM, async memory, etc.)
-- Clock and clock control
-- Random generation
-- CPU utilities
-- MAC settings
+-   Interrupt allocation
+-   Sleep functions
+-   Memory functions (external SPIRAM, async memory, etc.)
+-   Clock and clock control
+-   Random generation
+-   CPU utilities
+-   MAC settings
 
 ### `esp_hw_support` vs `esp_system`
 
 This section details list some implementations and the reason for placing it in either `esp_hw_support` or `esp_system`.
 
-#### `task_wdt.c` (`esp_system`) vs  `intr_alloc.c` (`esp_hw_support`)
+#### `task_wdt.c` (`esp_system`) vs `intr_alloc.c` (`esp_hw_support`)
 
 The task watchdog fits the definition of taking and configuring hardware resources (wdt, interrupt) for implementation of a system service/mechanism.
 

components/app_trace/CMakeLists.txt

@@ -1,3 +1,9 @@
+idf_build_get_property(target IDF_TARGET)
+
+if(${target} STREQUAL "linux")
+    return() # This component is not supported by the POSIX/Linux simulator
+endif()
+
 set(srcs
     "app_trace.c"
     "app_trace_util.c"
@@ -57,8 +63,7 @@ endif()
 idf_component_register(SRCS "${srcs}"
                        INCLUDE_DIRS "${include_dirs}"
                        PRIV_INCLUDE_DIRS "${priv_include_dirs}"
-                       # Requires "driver" for GPTimer in "SEGGER_SYSVIEW_Config_FreeRTOS.c"
-                       PRIV_REQUIRES soc driver
+                       PRIV_REQUIRES esp_driver_gptimer esp_driver_gpio esp_driver_uart
                        REQUIRES esp_timer
                        LDFRAGMENTS linker.lf)
 
@@ -66,6 +71,12 @@ idf_component_register(SRCS "${srcs}"
 idf_component_get_property(app_trace app_trace COMPONENT_LIB)
 
 if(CONFIG_APPTRACE_GCOV_ENABLE)
+    if(CMAKE_C_COMPILER_ID MATCHES "Clang")
+        # Coverage info is not supported when clang is used
+        # TODO: LLVM-214
+        message(FATAL_ERROR "Coverage info is not supported when building with Clang!")
+    endif()
+
     # The original Gcov library from toolchain will be objcopy with symbols redefinitions (see file gcov/io_sym.map).
     # This needs because ESP has no file-system onboard, and redefined functions solves this problem and transmits
     # output file to host PC.

components/app_trace/Kconfig

@@ -231,7 +231,7 @@ menu "Application Level Tracing"
 
         choice APPTRACE_SV_CPU
             prompt "CPU to trace"
-            depends on APPTRACE_SV_DEST_UART && !FREERTOS_UNICORE
+            depends on APPTRACE_SV_DEST_UART && !ESP_SYSTEM_SINGLE_CORE_MODE
             default APPTRACE_SV_DEST_CPU_0
             help
                 Define the CPU to trace by SystemView.
@@ -252,8 +252,8 @@ menu "Application Level Tracing"
         choice APPTRACE_SV_TS_SOURCE
             prompt "Timer to use as timestamp source"
             depends on APPTRACE_SV_ENABLE
-            default APPTRACE_SV_TS_SOURCE_CCOUNT if FREERTOS_UNICORE && !PM_ENABLE && !IDF_TARGET_ESP32C3
-            default APPTRACE_SV_TS_SOURCE_GPTIMER if !FREERTOS_UNICORE && !PM_ENABLE && !IDF_TARGET_ESP32C3
+            default APPTRACE_SV_TS_SOURCE_CCOUNT if ESP_SYSTEM_SINGLE_CORE_MODE && !PM_ENABLE && !IDF_TARGET_ESP32C3
+            default APPTRACE_SV_TS_SOURCE_GPTIMER if !ESP_SYSTEM_SINGLE_CORE_MODE && !PM_ENABLE && !IDF_TARGET_ESP32C3
             default APPTRACE_SV_TS_SOURCE_ESP_TIMER if PM_ENABLE || IDF_TARGET_ESP32C3
             help
                 SystemView needs to use a hardware timer as the source of timestamps
@@ -261,7 +261,7 @@ menu "Application Level Tracing"
 
             config APPTRACE_SV_TS_SOURCE_CCOUNT
                 bool "CPU cycle counter (CCOUNT)"
-                depends on FREERTOS_UNICORE && !PM_ENABLE && !IDF_TARGET_ESP32C3
+                depends on ESP_SYSTEM_SINGLE_CORE_MODE && !PM_ENABLE && !IDF_TARGET_ESP32C3
 
             config APPTRACE_SV_TS_SOURCE_GPTIMER
                 bool "General Purpose Timer (Timer Group)"
@@ -389,4 +389,11 @@ menu "Application Level Tracing"
         help
             Enables support for GCOV data transfer to host.
 
+    config APPTRACE_GCOV_DUMP_TASK_STACK_SIZE
+        int "Gcov dump task stack size"
+        depends on APPTRACE_GCOV_ENABLE
+        default 2048
+        help
+            Configures stack size of Gcov dump task
+
 endmenu

components/app_trace/app_trace.c

@@ -77,7 +77,7 @@ esp_err_t esp_apptrace_init(void)
     return ESP_OK;
 }
 
-ESP_SYSTEM_INIT_FN(esp_apptrace_init, ESP_SYSTEM_INIT_ALL_CORES, 115)
+ESP_SYSTEM_INIT_FN(esp_apptrace_init, SECONDARY, ESP_SYSTEM_INIT_ALL_CORES, 115)
 {
     return esp_apptrace_init();
 }

components/app_trace/gcov/gcov_rtio.c

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: 2017-2021 Espressif Systems (Shanghai) CO LTD
+ * SPDX-FileCopyrightText: 2017-2023 Espressif Systems (Shanghai) CO LTD
  *
  * SPDX-License-Identifier: Apache-2.0
  */
@@ -76,7 +76,8 @@ gcov_exit:
 void gcov_create_task(void *arg)
 {
     ESP_EARLY_LOGV(TAG, "%s", __FUNCTION__);
-    xTaskCreatePinnedToCore(&gcov_dump_task, "gcov_dump_task", 2048, (void *)&s_gcov_task_running, configMAX_PRIORITIES - 1, NULL, 0);
+    xTaskCreatePinnedToCore(&gcov_dump_task, "gcov_dump_task", CONFIG_APPTRACE_GCOV_DUMP_TASK_STACK_SIZE,
+		(void *)&s_gcov_task_running, configMAX_PRIORITIES - 1, NULL, 0);
 }
 
 void gcov_create_task_tick_hook(void)
@@ -172,6 +173,13 @@ long gcov_rtio_ftell(void *stream)
     return ret;
 }
 
+int gcov_rtio_feof(void *stream)
+{
+    int ret = esp_apptrace_feof(ESP_APPTRACE_DEST_TRAX, stream);
+    ESP_EARLY_LOGV(TAG, "%s(%p) = %d", __FUNCTION__, stream, ret);
+    return ret;
+}
+
 void gcov_rtio_setbuf(void *arg1 __attribute__ ((unused)), void *arg2 __attribute__ ((unused)))
 {
     return;

components/app_trace/gcov/io_sym.map

@@ -5,3 +5,4 @@ fread          gcov_rtio_fread
 fseek          gcov_rtio_fseek
 ftell          gcov_rtio_ftell
 setbuf         gcov_rtio_setbuf
+feof           gcov_rtio_feof

components/app_trace/heap_trace_tohost.c

@@ -8,7 +8,7 @@
 #define HEAP_TRACE_SRCFILE /* don't warn on inclusion here */
 #include "esp_heap_trace.h"
 #undef HEAP_TRACE_SRCFILE
-
+#include "esp_heap_caps.h"
 #if CONFIG_APPTRACE_SV_ENABLE
 #include "esp_app_trace.h"
 #include "esp_sysview_trace.h"
@@ -85,7 +85,7 @@ void heap_trace_dump_caps(__attribute__((unused)) const uint32_t caps)
 }
 
 /* Add a new allocation to the heap trace records */
-static IRAM_ATTR void record_allocation(const heap_trace_record_t *record)
+static HEAP_IRAM_ATTR void record_allocation(const heap_trace_record_t *record)
 {
     if (!s_tracing) {
         return;
@@ -100,7 +100,7 @@ static IRAM_ATTR void record_allocation(const heap_trace_record_t *record)
    For HEAP_TRACE_ALL, this means filling in the freed_by pointer.
    For HEAP_TRACE_LEAKS, this means removing the record from the log.
 */
-static IRAM_ATTR void record_free(void *p, void **callers)
+static HEAP_IRAM_ATTR void record_free(void *p, void **callers)
 {
     if (!s_tracing) {
         return;

components/app_trace/host_file_io.c

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: 2017-2021 Espressif Systems (Shanghai) CO LTD
+ * SPDX-FileCopyrightText: 2017-2023 Espressif Systems (Shanghai) CO LTD
  *
  * SPDX-License-Identifier: Apache-2.0
  */
@@ -29,6 +29,7 @@ const static char *TAG = "esp_host_file_io";
 #define ESP_APPTRACE_FILE_CMD_FSEEK     0x4
 #define ESP_APPTRACE_FILE_CMD_FTELL     0x5
 #define ESP_APPTRACE_FILE_CMD_STOP      0x6 // indicates that there is no files to transfer
+#define ESP_APPTRACE_FILE_CMD_FEOF      0x7
 
 /** File operation header */
 typedef struct {
@@ -68,6 +69,11 @@ typedef struct {
     void *  file;
 } esp_apptrace_fseek_args_t;
 
+/** Helper structure for feof */
+typedef struct {
+    void *file;
+} esp_apptrace_feof_args_t;
+
 /** Helper structure for ftell */
 typedef struct {
     void *file;
@@ -228,8 +234,11 @@ size_t esp_apptrace_fwrite(esp_apptrace_dest_t dest, const void *ptr, size_t siz
         ESP_EARLY_LOGE(TAG, "Failed to read response (%d)!", ret);
         return 0;
     }
-
-    return resp/size; // return the number of items written
+    /* OpenOCD writes it like that:
+     *      fwrite(buf, size, 1, file);
+     * So, if 1 was returned that means fwrite succeed
+     */
+    return resp == 1 ? nmemb : 0;
 }
 
 static void esp_apptrace_fread_args_prepare(uint8_t *buf, void *priv)
@@ -275,6 +284,10 @@ size_t esp_apptrace_fread(esp_apptrace_dest_t dest, void *ptr, size_t size, size
         ESP_EARLY_LOGE(TAG, "Failed to read file data (%d)!", ret);
         return 0;
     }
+    /* OpenOCD reads it like that:
+     *      fread(buf, 1 ,size, file);
+     * So, total read bytes count returns
+     */
     return resp/size; // return the number of items read
 }
 
@@ -354,4 +367,34 @@ int esp_apptrace_fstop(esp_apptrace_dest_t dest)
     return ret;
 }
 
+static void esp_apptrace_feof_args_prepare(uint8_t *buf, void *priv)
+{
+    esp_apptrace_feof_args_t *args = priv;
+
+    memcpy(buf, &args->file, sizeof(args->file));
+}
+
+int esp_apptrace_feof(esp_apptrace_dest_t dest, void *stream)
+{
+    esp_apptrace_feof_args_t cmd_args;
+
+    cmd_args.file = stream;
+    esp_err_t ret = esp_apptrace_file_cmd_send(dest, ESP_APPTRACE_FILE_CMD_FEOF, esp_apptrace_feof_args_prepare,
+                        &cmd_args, sizeof(cmd_args));
+    if (ret != ESP_OK) {
+        ESP_EARLY_LOGE(TAG, "Failed to send file cmd (%d)!", ret);
+        return EOF;
+    }
+
+    // now read the answer
+    int resp;
+    ret = esp_apptrace_file_rsp_recv(dest, (uint8_t *)&resp, sizeof(resp));
+    if (ret != ESP_OK) {
+        ESP_EARLY_LOGE(TAG, "Failed to read response (%d)!", ret);
+        return EOF;
+    }
+
+    return resp;
+}
+
 #endif

components/app_trace/include/esp_app_trace.h

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: 2017-2021 Espressif Systems (Shanghai) CO LTD
+ * SPDX-FileCopyrightText: 2017-2023 Espressif Systems (Shanghai) CO LTD
  *
  * SPDX-License-Identifier: Apache-2.0
  */
@@ -255,6 +255,17 @@ int esp_apptrace_ftell(esp_apptrace_dest_t dest, void *stream);
  */
 int esp_apptrace_fstop(esp_apptrace_dest_t dest);
 
+/**
+ * @brief Test end-of-file indicator on a stream.
+ *		  This function has the same semantic as 'feof' except for the first argument.
+ *
+ * @param dest   Indicates HW interface to use.
+ * @param stream File handle returned by esp_apptrace_fopen.
+ *
+ * @return Non-Zero if end-of-file indicator is set for stream. See feof for details.
+ */
+int esp_apptrace_feof(esp_apptrace_dest_t dest, void *stream);
+
 /**
  * @brief Triggers gcov info dump.
  *		  This function waits for the host to connect to target before dumping data.

components/app_trace/linker.lf

@@ -15,7 +15,7 @@ entries:
       SEGGER_SYSVIEW_FreeRTOS (noflash)
 
 [mapping:app_trace_driver]
-archive: libdriver.a
+archive: libesp_driver_gptimer.a
 entries:
     if APPTRACE_SV_TS_SOURCE_GPTIMER = y:
       gptimer (noflash)

components/app_trace/port/xtensa/port.c

@@ -298,7 +298,7 @@ static inline void esp_apptrace_trax_memory_enable(void)
 #if CONFIG_IDF_TARGET_ESP32
         /* Enable trace memory on PRO CPU */
         DPORT_WRITE_PERI_REG(DPORT_PRO_TRACEMEM_ENA_REG, DPORT_PRO_TRACEMEM_ENA_M);
-#if CONFIG_FREERTOS_UNICORE == 0
+#if CONFIG_ESP_SYSTEM_SINGLE_CORE_MODE == 0
         /* Enable trace memory on APP CPU */
         DPORT_WRITE_PERI_REG(DPORT_APP_TRACEMEM_ENA_REG, DPORT_APP_TRACEMEM_ENA_M);
 #endif

components/app_trace/sys_view/Sample/FreeRTOSV10.4/Config/esp/SEGGER_SYSVIEW_Config_FreeRTOS.c

@@ -137,7 +137,7 @@ gptimer_handle_t s_sv_gptimer;
 #endif
 
 #elif CONFIG_FREERTOS_SYSTICK_USES_SYSTIMER
-    #define SYSTICK_INTR_ID (ETS_SYSTIMER_TARGET0_EDGE_INTR_SOURCE)
+    #define SYSTICK_INTR_ID (ETS_SYSTIMER_TARGET0_INTR_SOURCE)
 #endif // CONFIG_FREERTOS_TICK_SUPPORT_CORETIMER
 
 // SystemView is single core specific: it implies that SEGGER_SYSVIEW_LOCK()

components/app_trace/sys_view/esp/SEGGER_RTT_esp.c

@@ -35,7 +35,7 @@ static uint8_t s_down_buf[SYSVIEW_DOWN_BUF_SIZE];
 #if CONFIG_APPTRACE_SV_DEST_UART
 
 #define ESP_APPTRACE_DEST_SYSVIEW ESP_APPTRACE_DEST_UART
-#if CONFIG_APPTRACE_SV_DEST_CPU_0 || CONFIG_FREERTOS_UNICORE
+#if CONFIG_APPTRACE_SV_DEST_CPU_0 || CONFIG_ESP_SYSTEM_SINGLE_CORE_MODE
 #define APPTRACE_SV_DEST_CPU 0
 #else
 #define APPTRACE_SV_DEST_CPU 1
@@ -72,13 +72,13 @@ void SEGGER_RTT_ESP_FlushNoLock(unsigned long min_sz, unsigned long tmo)
     if (s_events_buf_filled > 0) {
       res = esp_apptrace_write(ESP_APPTRACE_DEST_SYSVIEW, s_events_buf, s_events_buf_filled, tmo);
       if (res != ESP_OK) {
-        ESP_LOGE(TAG, "Failed to flush buffered events (%d)!\n", res);
+        ESP_LOGE(TAG, "Failed to flush buffered events (%d)!", res);
       }
     }
     // flush even if we failed to write buffered events, because no new events will be sent after STOP
     res = esp_apptrace_flush_nolock(ESP_APPTRACE_DEST_SYSVIEW, min_sz, tmo);
     if (res != ESP_OK) {
-      ESP_LOGE(TAG, "Failed to flush apptrace data (%d)!\n", res);
+      ESP_LOGE(TAG, "Failed to flush apptrace data (%d)!", res);
     }
     s_events_buf_filled = 0;
 }
@@ -167,8 +167,8 @@ unsigned SEGGER_RTT_WriteSkipNoLock(unsigned BufferIndex, const void* pBuffer, u
       (event_id == SYSVIEW_EVTID_TASK_STOP_EXEC) ||
       (event_id == SYSVIEW_EVTID_TASK_START_READY) ||
       (event_id == SYSVIEW_EVTID_TASK_STOP_READY) ||
-      (event_id == SYSVIEW_EVTID_USER_START) ||
-      (event_id == SYSVIEW_EVTID_USER_STOP) ||
+      (event_id == SYSVIEW_EVTID_MARK_START) ||
+      (event_id == SYSVIEW_EVTID_MARK_STOP) ||
       (event_id == SYSVIEW_EVTID_TIMER_ENTER) ||
       (event_id == SYSVIEW_EVTID_TIMER_EXIT) ||
       (event_id == SYSVIEW_EVTID_STACK_INFO) ||
@@ -296,7 +296,7 @@ int SEGGER_RTT_ConfigDownBuffer(unsigned BufferIndex, const char* sName, void* p
  * linked whenever SystemView is used.
  */
 
-ESP_SYSTEM_INIT_FN(sysview_init, BIT(0), 120)
+ESP_SYSTEM_INIT_FN(sysview_init, SECONDARY, BIT(0), 120)
 {
     SEGGER_SYSVIEW_Conf();
     return ESP_OK;

components/app_trace/sys_view/ext/logging.c

@@ -18,7 +18,7 @@ int esp_sysview_vprintf(const char * format, va_list args)
     portENTER_CRITICAL(&s_log_mutex);
     size_t len = vsnprintf(log_buffer, sizeof(log_buffer), format, args);
     if (len > sizeof(log_buffer) - 1) {
-        log_buffer[sizeof(log_buffer - 1)] = 0;
+        log_buffer[sizeof(log_buffer) - 1] = 0;
     }
     SEGGER_SYSVIEW_Print(log_buffer);
     portEXIT_CRITICAL(&s_log_mutex);

components/app_trace/test/CMakeLists.txt

@@ -1,4 +0,0 @@
-idf_component_register(SRC_DIRS "."
-                    PRIV_INCLUDE_DIRS "."
-                    PRIV_REQUIRES cmock driver)
-target_compile_options(${COMPONENT_LIB} PRIVATE "-Wno-format")

components/app_trace/test_apps/.build-test-rules.yml

@@ -0,0 +1,9 @@
+# Documentation: .gitlab/ci/README.md#manifest-file-to-control-the-buildtest-apps
+
+components/app_trace/test_apps:
+  depends_components:
+    - app_trace
+    - esp_timer
+    - soc
+    - driver
+    - esp_hw_support

components/app_trace/test_apps/CMakeLists.txt

@@ -0,0 +1,9 @@
+cmake_minimum_required(VERSION 3.16)
+
+include($ENV{IDF_PATH}/tools/cmake/project.cmake)
+set(COMPONENTS main)
+list(PREPEND SDKCONFIG_DEFAULTS
+    "$ENV{IDF_PATH}/tools/test_apps/configs/sdkconfig.debug_helpers"
+    "sdkconfig.defaults")
+
+project(app_trace_test)

components/app_trace/test_apps/README.md

@@ -0,0 +1,16 @@
+| Supported Targets | ESP32 | ESP32-C2 | ESP32-C3 | ESP32-C6 | ESP32-H2 | ESP32-P4 | ESP32-S2 | ESP32-S3 |
+| ----------------- | ----- | -------- | -------- | -------- | -------- | -------- | -------- | -------- |
+
+# app_trace test
+
+To build and run this test app for app_trace related tests:
+```bash
+IDF_TARGET=esp32 idf.py @app_trace build flash monitor
+```
+
+To build and run this test app for SystemView related tests:
+```bash
+IDF_TARGET=esp32 idf.py @sysview build flash monitor
+```
+
+`@app_trace` and `@sysview` arguments apply additional `idf.py` options, from [app_trace](app_trace) and [sysview](sysview) files.

components/app_trace/test_apps/app_trace

@@ -0,0 +1 @@
+-DSDKCONFIG_DEFAULTS="sdkconfig.defaults;sdkconfig.ci.app_trace" -B build/app_trace -DSDKCONFIG=build/app_trace/sdkconfig

components/app_trace/test_apps/main/CMakeLists.txt

@@ -0,0 +1,4 @@
+idf_component_register(SRCS "test_app_trace_main.c" "test_trace.c"
+                    INCLUDE_DIRS "."
+                    PRIV_REQUIRES app_trace unity esp_driver_gptimer
+                    WHOLE_ARCHIVE)

components/app_trace/test_apps/main/test_app_trace_main.c

@@ -0,0 +1,48 @@
+/*
+ * SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#include "unity.h"
+#include "unity_test_runner.h"
+#include "esp_heap_caps.h"
+
+#define TEST_MEMORY_LEAK_THRESHOLD_DEFAULT 0
+static int leak_threshold = TEST_MEMORY_LEAK_THRESHOLD_DEFAULT;
+void set_leak_threshold(int threshold)
+{
+    leak_threshold = threshold;
+}
+
+static size_t before_free_8bit;
+static size_t before_free_32bit;
+
+static void check_leak(size_t before_free, size_t after_free, const char *type)
+{
+    ssize_t delta = after_free - before_free;
+    printf("MALLOC_CAP_%s: Before %u bytes free, After %u bytes free (delta %d)\n", type, before_free, after_free, delta);
+    TEST_ASSERT_MESSAGE(delta >= leak_threshold, "memory leak");
+}
+
+void setUp(void)
+{
+    before_free_8bit = heap_caps_get_free_size(MALLOC_CAP_8BIT);
+    before_free_32bit = heap_caps_get_free_size(MALLOC_CAP_32BIT);
+}
+
+void tearDown(void)
+{
+    size_t after_free_8bit = heap_caps_get_free_size(MALLOC_CAP_8BIT);
+    size_t after_free_32bit = heap_caps_get_free_size(MALLOC_CAP_32BIT);
+    check_leak(before_free_8bit, after_free_8bit, "8BIT");
+    check_leak(before_free_32bit, after_free_32bit, "32BIT");
+
+    leak_threshold = TEST_MEMORY_LEAK_THRESHOLD_DEFAULT;
+}
+
+void app_main(void)
+{
+    printf("Running app_trace component tests\n");
+    unity_run_menu();
+}

components/app_trace/test_apps/main/test_trace.c

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
+ * SPDX-FileCopyrightText: 2021-2023 Espressif Systems (Shanghai) CO LTD
  *
  * SPDX-License-Identifier: Apache-2.0
  */
@@ -16,7 +16,7 @@
 #include "freertos/FreeRTOS.h"
 #include "freertos/semphr.h"
 #include "freertos/task.h"
-#if CONFIG_APPTRACE_ENABLE == 1
+
 #include "esp_app_trace.h"
 #include "esp_app_trace_util.h"
 
@@ -423,7 +423,6 @@ TEST_CASE("App trace test (1 task + 1 crashed timer ISR @ 1 core)", "[trace][ign
     esp_apptrace_test(&test_cfg);
 }
 
-
 TEST_CASE("App trace test (1 crashed task)", "[trace][ignore]")
 {
     esp_apptrace_test_task_arg_t s_test_tasks[1];
@@ -717,6 +716,7 @@ typedef struct {
 static bool esp_sysview_test_timer_isr(gptimer_handle_t timer, const gptimer_alarm_event_data_t *edata, void *user_ctx)
 {
     esp_sysviewtrace_timer_arg_t *tim_arg = (esp_sysviewtrace_timer_arg_t *)user_ctx;
+    (void) tim_arg;
     return false;
 }
 
@@ -724,7 +724,7 @@ static void esp_sysviewtrace_test_task(void *p)
 {
     esp_sysviewtrace_task_arg_t *arg = (esp_sysviewtrace_task_arg_t *) p;
     volatile uint32_t tmp = 0;
-    printf("%x: run sysview task\n", (uint32_t)xTaskGetCurrentTaskHandle());
+    printf("%p: run sysview task\n", xTaskGetCurrentTaskHandle());
 
     if (arg->timer) {
         gptimer_alarm_config_t alarm_config = {
@@ -744,7 +744,7 @@ static void esp_sysviewtrace_test_task(void *p)
     int i = 0;
     while (1) {
         static uint32_t count;
-        printf("%d", arg->id);
+        printf("%" PRIu32, arg->id);
         if ((++count % 80) == 0) {
             printf("\n");
         }
@@ -890,22 +890,22 @@ TEST_CASE("SysView trace test 2", "[trace][ignore]")
     TEST_ESP_OK(gptimer_new_timer(&timer_config, &tim_arg2.gptimer));
 
     xTaskCreatePinnedToCore(esp_sysviewtrace_test_task, "svtrace0", 2048, &arg1, 3, &thnd, 0);
-    printf("Created task %x\n", (uint32_t)thnd);
+    printf("Created task %p\n", thnd);
 #if CONFIG_FREERTOS_UNICORE == 0
     xTaskCreatePinnedToCore(esp_sysviewtrace_test_task, "svtrace1", 2048, &arg2, 4, &thnd, 1);
 #else
     xTaskCreatePinnedToCore(esp_sysviewtrace_test_task, "svtrace1", 2048, &arg2, 4, &thnd, 0);
 #endif
-    printf("Created task %x\n", (uint32_t)thnd);
+    printf("Created task %p\n", thnd);
 
     xTaskCreatePinnedToCore(esp_sysviewtrace_test_task, "svsync0", 2048, &arg3, 3, &thnd, 0);
-    printf("Created task %x\n", (uint32_t)thnd);
+    printf("Created task %p\n", thnd);
 #if CONFIG_FREERTOS_UNICORE == 0
     xTaskCreatePinnedToCore(esp_sysviewtrace_test_task, "svsync1", 2048, &arg4, 5, &thnd, 1);
 #else
     xTaskCreatePinnedToCore(esp_sysviewtrace_test_task, "svsync1", 2048, &arg4, 5, &thnd, 0);
 #endif
-    printf("Created task %x\n", (uint32_t)thnd);
+    printf("Created task %p\n", thnd);
 
     xSemaphoreTake(arg1.done, portMAX_DELAY);
     vSemaphoreDelete(arg1.done);
@@ -924,4 +924,3 @@ TEST_CASE("SysView trace test 2", "[trace][ignore]")
     TEST_ESP_OK(gptimer_del_timer(tim_arg2.gptimer));
 }
 #endif // #if CONFIG_APPTRACE_SV_ENABLE == 0
-#endif // #if CONFIG_APPTRACE_ENABLE == 1

components/app_trace/test_apps/sdkconfig.ci.app_trace

@@ -0,0 +1 @@
+# app_trace is already enabled by sdkconfig.defaults, so no options are needed here

components/app_trace/test_apps/sdkconfig.ci.sysview

@@ -0,0 +1 @@
+CONFIG_APPTRACE_SV_ENABLE=y

components/app_trace/test_apps/sdkconfig.defaults

@@ -0,0 +1,2 @@
+CONFIG_ESP_TASK_WDT_CHECK_IDLE_TASK_CPU0=n
+CONFIG_APPTRACE_DEST_JTAG=y

components/app_trace/test_apps/sysview

@@ -0,0 +1 @@
+-DSDKCONFIG_DEFAULTS="sdkconfig.defaults;sdkconfig.ci.sysview" -B build/sysview -DSDKCONFIG=build/sysview/sdkconfig

components/app_update/.build-test-rules.yml

@@ -1,7 +0,0 @@
-# Documentation: .gitlab/ci/README.md#manifest-file-to-control-the-buildtest-apps
-
-components/app_update/test_apps:
-  disable:
-    - if: IDF_TARGET == "esp32c6" or IDF_TARGET == "esp32h2"
-      temporary: true
-      reason: target esp32c6, esp32h2 is not supported yet

components/app_update/CMakeLists.txt

@@ -1,6 +1,12 @@
+idf_build_get_property(target IDF_TARGET)
+
+if(${target} STREQUAL "linux")
+    return() # This component is not supported by the POSIX/Linux simulator
+endif()
+
 idf_component_register(SRCS "esp_ota_ops.c" "esp_ota_app_desc.c"
                     INCLUDE_DIRS "include"
-                    REQUIRES partition_table bootloader_support esp_app_format esp_partition
+                    REQUIRES partition_table bootloader_support esp_app_format esp_bootloader_format esp_partition
                     PRIV_REQUIRES esptool_py efuse spi_flash)
 
 if(NOT BOOTLOADER_BUILD)

components/app_update/esp_ota_app_desc.c

@@ -15,7 +15,7 @@ const esp_app_desc_t *esp_ota_get_app_description(void)
     return esp_app_get_description();
 }
 
-int IRAM_ATTR esp_ota_get_app_elf_sha256(char* dst, size_t size)
+int esp_ota_get_app_elf_sha256(char* dst, size_t size)
 {
    return esp_app_get_elf_sha256(dst, size);
 }

components/app_update/esp_ota_ops.c

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: 2015-2021 Espressif Systems (Shanghai) CO LTD
+ * SPDX-FileCopyrightText: 2015-2023 Espressif Systems (Shanghai) CO LTD
  *
  * SPDX-License-Identifier: Apache-2.0
  */
@@ -28,24 +28,8 @@
 #include "esp_system.h"
 #include "esp_efuse.h"
 #include "esp_attr.h"
-
-#if CONFIG_IDF_TARGET_ESP32
-#include "esp32/rom/secure_boot.h"
-#elif CONFIG_IDF_TARGET_ESP32S2
-#include "esp32s2/rom/secure_boot.h"
-#elif CONFIG_IDF_TARGET_ESP32C3
-#include "esp32c3/rom/secure_boot.h"
-#elif CONFIG_IDF_TARGET_ESP32S3
-#include "esp32s3/rom/secure_boot.h"
-#elif CONFIG_IDF_TARGET_ESP32H4
-#include "esp32h4/rom/secure_boot.h"
-#elif CONFIG_IDF_TARGET_ESP32C2
-#include "esp32c2/rom/secure_boot.h"
-#elif CONFIG_IDF_TARGET_ESP32C6
-#include "esp32c6/rom/secure_boot.h"
-#elif CONFIG_IDF_TARGET_ESP32H2
-#include "esp32h2/rom/secure_boot.h"
-#endif
+#include "esp_bootloader_desc.h"
+#include "esp_flash.h"
 
 #define SUB_TYPE_ID(i) (i & 0x0F)
 
@@ -197,13 +181,18 @@ esp_err_t esp_ota_write(esp_ota_handle_t handle, const void *data, size_t size)
         return ESP_ERR_INVALID_ARG;
     }
 
+    if (size == 0) {
+        ESP_LOGD(TAG, "write data size is 0");
+        return ESP_OK;
+    }
+
     // find ota handle in linked list
     for (it = LIST_FIRST(&s_ota_ops_entries_head); it != NULL; it = LIST_NEXT(it, entries)) {
         if (it->handle == handle) {
             if (it->need_erase) {
                 // must erase the partition before writing to it
-                uint32_t first_sector = it->wrote_size / SPI_FLASH_SEC_SIZE;
-                uint32_t last_sector = (it->wrote_size + size) / SPI_FLASH_SEC_SIZE;
+                uint32_t first_sector = it->wrote_size / SPI_FLASH_SEC_SIZE; // first affected sector
+                uint32_t last_sector = (it->wrote_size + size - 1) / SPI_FLASH_SEC_SIZE; // last affected sector
 
                 ret = ESP_OK;
                 if ((it->wrote_size % SPI_FLASH_SEC_SIZE) == 0) {
@@ -630,6 +619,32 @@ const esp_partition_t* esp_ota_get_next_update_partition(const esp_partition_t *
 
 }
 
+esp_err_t esp_ota_get_bootloader_description(const esp_partition_t *bootloader_partition, esp_bootloader_desc_t *desc)
+{
+    if (desc == NULL) {
+        return ESP_ERR_INVALID_ARG;
+    }
+    esp_partition_t partition = { 0 };
+    if (bootloader_partition == NULL) {
+        partition.flash_chip = esp_flash_default_chip;
+        partition.encrypted = esp_flash_encryption_enabled();
+        partition.address = CONFIG_BOOTLOADER_OFFSET_IN_FLASH;
+        partition.size = CONFIG_PARTITION_TABLE_OFFSET - CONFIG_BOOTLOADER_OFFSET_IN_FLASH;
+    } else {
+        memcpy(&partition, bootloader_partition, sizeof(partition));
+    }
+    esp_err_t err = esp_partition_read(&partition, sizeof(esp_image_header_t) + sizeof(esp_image_segment_header_t), desc, sizeof(esp_bootloader_desc_t));
+    if (err != ESP_OK) {
+        return err;
+    }
+
+    if (desc->magic_byte != ESP_BOOTLOADER_DESC_MAGIC_BYTE) {
+        return ESP_ERR_NOT_FOUND;
+    }
+
+    return ESP_OK;
+}
+
 esp_err_t esp_ota_get_partition_description(const esp_partition_t *partition, esp_app_desc_t *app_desc)
 {
     if (partition == NULL || app_desc == NULL) {
@@ -898,9 +913,27 @@ esp_err_t esp_ota_erase_last_boot_app_partition(void)
     return ESP_OK;
 }
 
-#if SOC_EFUSE_SECURE_BOOT_KEY_DIGESTS > 1 && CONFIG_SECURE_BOOT_V2_ENABLED
-esp_err_t esp_ota_revoke_secure_boot_public_key(esp_ota_secure_boot_public_key_index_t index) {
+#if SOC_SUPPORT_SECURE_BOOT_REVOKE_KEY && CONFIG_SECURE_BOOT_V2_ENABLED
 
+// Validates the image at "app_pos" with the secure boot digests other than "revoked_key_index"
+static bool validate_img(esp_ota_secure_boot_public_key_index_t revoked_key_index, esp_partition_pos_t *app_pos)
+{
+    bool verified = false;
+    for (int i = 0; i < SOC_EFUSE_SECURE_BOOT_KEY_DIGESTS; i++) {
+        if (i == revoked_key_index) {
+            continue;
+        }
+        if (esp_secure_boot_verify_with_efuse_digest_index(i, app_pos) == ESP_OK) {
+            verified = true;
+            ESP_LOGI(TAG, "Application successfully verified with public key digest %d", i);
+            break;
+        }
+    }
+    return verified;
+}
+
+esp_err_t esp_ota_revoke_secure_boot_public_key(esp_ota_secure_boot_public_key_index_t index)
+{
     if (!esp_secure_boot_enabled()) {
         ESP_LOGE(TAG, "Secure boot v2 has not been enabled.");
         return ESP_FAIL;
@@ -913,14 +946,21 @@ esp_err_t esp_ota_revoke_secure_boot_public_key(esp_ota_secure_boot_public_key_i
         return ESP_ERR_INVALID_ARG;
     }
 
-    esp_image_sig_public_key_digests_t app_digests = { 0 };
-    esp_err_t err = esp_secure_boot_get_signature_blocks_for_running_app(true, &app_digests);
-    if (err != ESP_OK || app_digests.num_digests == 0) {
-        ESP_LOGE(TAG, "This app is not signed, but check signature on update is enabled in config. It won't be possible to verify any update.");
+    const esp_partition_t *running_app_part = esp_ota_get_running_partition();
+    esp_err_t ret = ESP_FAIL;
+#ifdef CONFIG_BOOTLOADER_APP_ROLLBACK_ENABLE
+    esp_ota_img_states_t running_app_state;
+    ret = esp_ota_get_state_partition(running_app_part, &running_app_state);
+    if (ret != ESP_OK) {
+        ESP_LOGE(TAG, "esp_ota_get_state_partition returned: %s", esp_err_to_name(ret));
         return ESP_FAIL;
     }
+    if (running_app_state != ESP_OTA_IMG_VALID) {
+        ESP_LOGE(TAG, "The current running application is not marked as a valid image. Aborting..");
+        return ESP_FAIL;
+    }
+#endif
 
-    esp_err_t ret;
     esp_secure_boot_key_digests_t trusted_keys;
     ret = esp_secure_boot_read_key_digests(&trusted_keys);
     if (ret != ESP_OK) {
@@ -929,55 +969,38 @@ esp_err_t esp_ota_revoke_secure_boot_public_key(esp_ota_secure_boot_public_key_i
     }
 
     if (trusted_keys.key_digests[index] == NULL) {
-        ESP_LOGI(TAG, "Trusted Key block(%d) already revoked.", index);
+        ESP_LOGI(TAG, "Given public key digest(%d) is already revoked.", index);
         return ESP_OK;
     }
 
-    esp_image_sig_public_key_digests_t trusted_digests = { 0 };
-    for (unsigned i = 0; i < SECURE_BOOT_NUM_BLOCKS; i++) {
-        if (i == index) {
-            continue; // omitting - to find if there is a valid key after revoking this digest
-        }
+    /* Check if the application can be verified with a key other than the one being revoked */
+    esp_partition_pos_t running_app_pos = {
+        .offset = running_app_part->address,
+        .size   = running_app_part->size,
+    };
 
-        if (trusted_keys.key_digests[i] != NULL) {
-            bool all_zeroes = true;
-            for (unsigned j = 0; j < ESP_SECURE_BOOT_DIGEST_LEN; j++) {
-                all_zeroes = all_zeroes && (*(uint8_t *)(trusted_keys.key_digests[i] + j) == 0);
-            }
-            if (!all_zeroes) {
-                memcpy(trusted_digests.key_digests[trusted_digests.num_digests++], (uint8_t *)trusted_keys.key_digests[i], ESP_SECURE_BOOT_DIGEST_LEN);
-            } else {
-                ESP_LOGD(TAG, "Empty trusted key block (%d).", i);
-            }
-        }
-    }
-
-    bool match = false;
-    for (unsigned i = 0; i < trusted_digests.num_digests; i++) {
-        if (match == true) {
-            break;
-        }
-
-        for (unsigned j = 0; j < app_digests.num_digests; j++) {
-            if (memcmp(trusted_digests.key_digests[i], app_digests.key_digests[j], ESP_SECURE_BOOT_DIGEST_LEN) == 0) {
-                ESP_LOGI(TAG, "App key block(%d) matches Trusted key block(%d)[%d -> Next active trusted key block].", j, i, i);
-                esp_err_t err = esp_efuse_set_digest_revoke(index);
-                if (err != ESP_OK) {
-                    ESP_LOGE(TAG, "Failed to revoke digest (0x%x).", err);
-                    return ESP_FAIL;
-                }
-                ESP_LOGI(TAG, "Revoked signature block %d.", index);
-                match = true;
-                break;
-            }
-        }
-    }
-
-    if (match == false) {
-        ESP_LOGE(TAG, "Running app doesn't have another valid secure boot key. Cannot revoke current key(%d).", index);
+    if (!validate_img(index, &running_app_pos)) {
+        ESP_LOGE(TAG, "Application cannot be verified with any key other than the one being revoked");
         return ESP_FAIL;
     }
 
+    /* Check if bootloder can be verified with any key other than the one being revoked */
+    esp_partition_pos_t bootloader_pos = {
+        .offset = ESP_BOOTLOADER_OFFSET,
+        .size   = (ESP_PARTITION_TABLE_OFFSET - ESP_BOOTLOADER_OFFSET),
+    };
+
+    if (!validate_img(index, &bootloader_pos)) {
+        ESP_LOGE(TAG, "Bootloader cannot be verified with any key other than the one being revoked");
+        return ESP_FAIL;
+    }
+
+    esp_err_t err = esp_efuse_set_digest_revoke(index);
+    if (err != ESP_OK) {
+        ESP_LOGE(TAG, "Failed to revoke digest (0x%x).", err);
+        return ESP_FAIL;
+    }
+    ESP_LOGI(TAG, "Revoked signature block %d.", index);
     return ESP_OK;
 }
 #endif

components/app_update/include/esp_ota_ops.h

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: 2015-2021 Espressif Systems (Shanghai) CO LTD
+ * SPDX-FileCopyrightText: 2015-2023 Espressif Systems (Shanghai) CO LTD
  *
  * SPDX-License-Identifier: Apache-2.0
  */
@@ -13,6 +13,7 @@
 #include "esp_err.h"
 #include "esp_partition.h"
 #include "esp_app_desc.h"
+#include "esp_bootloader_desc.h"
 #include "esp_flash_partitions.h"
 #include "soc/soc_caps.h"
 
@@ -110,7 +111,7 @@ esp_err_t esp_ota_begin(const esp_partition_t* partition, size_t image_size, esp
  * @param size    Size of data buffer in bytes.
  *
  * @return
- *    - ESP_OK: Data was written to flash successfully.
+ *    - ESP_OK: Data was written to flash successfully, or size = 0
  *    - ESP_ERR_INVALID_ARG: handle is invalid.
  *    - ESP_ERR_OTA_VALIDATE_FAILED: First byte of image contains invalid app image magic byte.
  *    - ESP_ERR_FLASH_OP_TIMEOUT or ESP_ERR_FLASH_OP_FAIL: Flash write failed.
@@ -251,6 +252,23 @@ const esp_partition_t* esp_ota_get_next_update_partition(const esp_partition_t *
  */
 esp_err_t esp_ota_get_partition_description(const esp_partition_t *partition, esp_app_desc_t *app_desc);
 
+/**
+ * @brief Returns the description structure of the bootloader.
+ *
+ * @param[in] bootloader_partition Pointer to bootloader partition.
+ *                                 If NULL, then the current bootloader is used (the default location).
+ *                                 offset = CONFIG_BOOTLOADER_OFFSET_IN_FLASH,
+ *                                 size = CONFIG_PARTITION_TABLE_OFFSET - CONFIG_BOOTLOADER_OFFSET_IN_FLASH,
+ * @param[out] desc     Structure of info about bootloader.
+ * @return
+ *  - ESP_OK                Successful.
+ *  - ESP_ERR_NOT_FOUND     Description structure is not found in the bootloader image. Magic byte is incorrect.
+ *  - ESP_ERR_INVALID_ARG   Arguments is NULL.
+ *  - ESP_ERR_INVALID_SIZE  Read would go out of bounds of the partition.
+ *  - or one of error codes from lower-level flash driver.
+ */
+esp_err_t esp_ota_get_bootloader_description(const esp_partition_t *bootloader_partition, esp_bootloader_desc_t *desc);
+
 /**
  * @brief Returns number of ota partitions provided in partition table.
  *
@@ -332,10 +350,10 @@ typedef enum {
 } esp_ota_secure_boot_public_key_index_t;
 
 /**
- * @brief Revokes the old signature digest. To be called in the application after the rollback logic.
+ * @brief Revokes the signature digest denoted by the given index. This should be called in the application only after the rollback logic otherwise the device may end up in unrecoverable state.
  *
- * Relevant for Secure boot v2 on ESP32-S2, ESP32-S3, ESP32-C3, ESP32-H4 where upto 3 key digests can be stored (Key \#N-1, Key \#N, Key \#N+1).
- * When key \#N-1 used to sign an app is invalidated, an OTA update is to be sent with an app signed with key \#N-1 & Key \#N.
+ * Relevant for Secure boot v2 on ESP32-S2, ESP32-S3, ESP32-C3, ESP32-C6, ESP32-H2 where up to 3 key digests can be stored (Key \#N-1, Key \#N, Key \#N+1).
+ * When a key used to sign an app is invalidated, an OTA update is to be sent with an app signed with at least one of the other two keys which has not been revoked already.
  * After successfully booting the OTA app should call this function to revoke Key \#N-1.
  *
  * @param index - The index of the signature block to be revoked

components/app_update/test_apps/.build-test-rules.yml

@@ -0,0 +1,7 @@
+# Documentation: .gitlab/ci/README.md#manifest-file-to-control-the-buildtest-apps
+
+components/app_update/test_apps:
+  disable:
+    - if: IDF_TARGET in ["esp32c6", "esp32h2", "esp32p4"]
+      temporary: true
+      reason: target esp32c6, esp32h2, esp32p4 is not supported yet # TODO: IDF-8068

components/app_update/test_apps/README.md

@@ -1,3 +1,2 @@
 | Supported Targets | ESP32 | ESP32-C2 | ESP32-C3 | ESP32-S2 | ESP32-S3 |
 | ----------------- | ----- | -------- | -------- | -------- | -------- |
-

components/app_update/test_apps/main/test_switch_ota.c

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
+ * SPDX-FileCopyrightText: 2021-2023 Espressif Systems (Shanghai) CO LTD
  *
  * SPDX-License-Identifier: Apache-2.0
  */
@@ -185,8 +185,8 @@ static void erase_ota_data(void)
 static void reboot_as_deep_sleep(void)
 {
     ESP_LOGI(TAG, "reboot as deep sleep");
-    esp_sleep_enable_timer_wakeup(2000);
-    esp_deep_sleep_start();
+    esp_deep_sleep(20000);
+    TEST_FAIL_MESSAGE("Should never be reachable except when sleep is rejected, abort");
 }
 
 /* @brief Copies a current app to next partition (OTA0-15), after that ESP is rebooting and run this (the next) OTAx.

components/app_update/test_apps/pytest_app_update_ut.py

@@ -19,7 +19,7 @@ def run_multiple_stages(dut: Dut, test_case_num: int, stages: int) -> None:
 
 
 @pytest.mark.supported_targets
-@pytest.mark.temp_skip_ci(targets=['esp32c6', 'esp32h2'], reason='c6/h2 support TBD')
+@pytest.mark.temp_skip_ci(targets=['esp32c6', 'esp32h2', 'esp32p4'], reason='c6/h2/p4 support TBD')  # TODO: IDF-8959
 @pytest.mark.generic
 def test_app_update(dut: Dut) -> None:
     extra_data = dut.parse_test_menu()

components/bootloader/CMakeLists.txt

@@ -1,3 +1,9 @@
+idf_build_get_property(target IDF_TARGET)
+
+if(${target} STREQUAL "linux")
+    return() # This component is not supported by the POSIX/Linux simulator
+endif()
+
 idf_component_register(PRIV_REQUIRES partition_table esptool_py)
 
 # Do not generate flash file when building bootloader or is in early expansion of the build
@@ -7,8 +13,9 @@ endif()
 
 add_dependencies(bootloader partition_table_bin)
 
-# When secure boot is enabled, do not flash bootloader along with invocation of `idf.py flash`
-if(NOT CONFIG_SECURE_BOOT)
+# When secure boot is enabled and CONFIG_SECURE_BOOT_FLASH_BOOTLOADER_DEFAULT is not enabled
+# do not flash the bootloader along with the other artifacts using the command `idf.py flash`
+if(NOT CONFIG_SECURE_BOOT OR CONFIG_SECURE_BOOT_FLASH_BOOTLOADER_DEFAULT)
     set(flash_bootloader FLASH_IN_PROJECT)
 endif()
 
@@ -18,7 +25,7 @@ esptool_py_flash_target_image(bootloader-flash bootloader
     "${BOOTLOADER_BUILD_DIR}/bootloader.bin")
 
 # Also attach an image to the project flash target
-if(NOT CONFIG_SECURE_BOOT)
+if(NOT CONFIG_SECURE_BOOT OR CONFIG_SECURE_BOOT_FLASH_BOOTLOADER_DEFAULT)
     esptool_py_flash_target_image(flash bootloader
         ${CONFIG_BOOTLOADER_OFFSET_IN_FLASH}
         "${BOOTLOADER_BUILD_DIR}/bootloader.bin")

components/bootloader/Kconfig.projbuild

@@ -1,8 +1,12 @@
 menu "Bootloader config"
 
+    orsource "../esp_bootloader_format/Kconfig.bootloader"
+
     config BOOTLOADER_OFFSET_IN_FLASH
         hex
         default 0x1000 if IDF_TARGET_ESP32 || IDF_TARGET_ESP32S2
+        # the first 2 sectors are reserved for the key manager with AES-XTS (flash encryption) purpose
+        default 0x2000 if IDF_TARGET_ESP32P4
         default 0x0
         help
             Offset address that 2nd bootloader will be flashed to.
@@ -62,34 +66,82 @@ menu "Bootloader config"
         default 4 if BOOTLOADER_LOG_LEVEL_DEBUG
         default 5 if BOOTLOADER_LOG_LEVEL_VERBOSE
 
-    config BOOTLOADER_SPI_CUSTOM_WP_PIN
-        bool "Use custom SPI Flash WP Pin when flash pins set in eFuse (read help)"
-        depends on IDF_TARGET_ESP32 && (ESPTOOLPY_FLASHMODE_QIO || ESPTOOLPY_FLASHMODE_QOUT)
-        default y if BOOTLOADER_SPI_WP_PIN != 7  # backwards compatibility, can remove in IDF 5
-        default n
-        help
-            This setting is only used if the SPI flash pins have been overridden by setting the eFuses
-            SPI_PAD_CONFIG_xxx, and the SPI flash mode is QIO or QOUT.
+    menu "Serial Flash Configurations"
+        config BOOTLOADER_SPI_CUSTOM_WP_PIN
+            bool "Use custom SPI Flash WP Pin when flash pins set in eFuse (read help)"
+            depends on IDF_TARGET_ESP32 && (ESPTOOLPY_FLASHMODE_QIO || ESPTOOLPY_FLASHMODE_QOUT)
+            default y if BOOTLOADER_SPI_WP_PIN != 7  # backwards compatibility, can remove in IDF 5
+            default n
+            help
+                This setting is only used if the SPI flash pins have been overridden by setting the eFuses
+                SPI_PAD_CONFIG_xxx, and the SPI flash mode is QIO or QOUT.
 
-            When this is the case, the eFuse config only defines 3 of the 4 Quad I/O data pins. The WP pin (aka
-            ESP32 pin "SD_DATA_3" or SPI flash pin "IO2") is not specified in eFuse. The same pin is also used
-            for external SPIRAM if it is enabled.
+                When this is the case, the eFuse config only defines 3 of the 4 Quad I/O data pins. The WP pin (aka
+                ESP32 pin "SD_DATA_3" or SPI flash pin "IO2") is not specified in eFuse. The same pin is also used
+                for external SPIRAM if it is enabled.
 
-            If this config item is set to N (default), the correct WP pin will be automatically used for any
-            Espressif chip or module with integrated flash. If a custom setting is needed, set this config item to
-            Y and specify the GPIO number connected to the WP.
+                If this config item is set to N (default), the correct WP pin will be automatically used for any
+                Espressif chip or module with integrated flash. If a custom setting is needed, set this config item to
+                Y and specify the GPIO number connected to the WP.
 
-    config BOOTLOADER_SPI_WP_PIN
-        int "Custom SPI Flash WP Pin"
-        range 0 33
-        default 7
-        depends on IDF_TARGET_ESP32 && (ESPTOOLPY_FLASHMODE_QIO || ESPTOOLPY_FLASHMODE_QOUT)
-        #depends on BOOTLOADER_SPI_CUSTOM_WP_PIN  # backwards compatibility, can uncomment in IDF 5
-        help
-            The option "Use custom SPI Flash WP Pin" must be set or this value is ignored
+        config BOOTLOADER_SPI_WP_PIN
+            int "Custom SPI Flash WP Pin"
+            range 0 33
+            default 7
+            depends on IDF_TARGET_ESP32 && (ESPTOOLPY_FLASHMODE_QIO || ESPTOOLPY_FLASHMODE_QOUT)
+            #depends on BOOTLOADER_SPI_CUSTOM_WP_PIN  # backwards compatibility, can uncomment in IDF 5
+            help
+                The option "Use custom SPI Flash WP Pin" must be set or this value is ignored
 
-            If burning a customized set of SPI flash pins in eFuse and using QIO or QOUT mode for flash, set this
-            value to the GPIO number of the SPI flash WP pin.
+                If burning a customized set of SPI flash pins in eFuse and using QIO or QOUT mode for flash, set this
+                value to the GPIO number of the SPI flash WP pin.
+
+        config BOOTLOADER_FLASH_DC_AWARE
+            bool "Allow app adjust Dummy Cycle bits in SPI Flash for higher frequency (READ HELP FIRST)"
+            help
+                This will force 2nd bootloader to be loaded by DOUT mode, and will restore Dummy Cycle setting by
+                resetting the Flash
+
+        config BOOTLOADER_FLASH_XMC_SUPPORT
+            bool "Enable the support for flash chips of XMC (READ DOCS FIRST)"
+            default y
+            help
+                Perform the startup flow recommended by XMC. Please consult XMC for the details of this flow.
+                XMC chips will be forbidden to be used, when this option is disabled.
+
+                DON'T DISABLE THIS UNLESS YOU KNOW WHAT YOU ARE DOING.
+
+                comment "Features below require specific hardware (READ DOCS FIRST!)"
+
+        config BOOTLOADER_FLASH_32BIT_ADDR
+            bool
+            default y if ESPTOOLPY_FLASHSIZE_32MB || ESPTOOLPY_FLASHSIZE_64MB || ESPTOOLPY_FLASHSIZE_128MB
+            default n
+            help
+                This is a helper config for 32bits address flash. Invisible for users.
+
+        config BOOTLOADER_FLASH_NEEDS_32BIT_FEAT
+            bool
+            default y if BOOTLOADER_FLASH_32BIT_ADDR && !ESPTOOLPY_OCT_FLASH
+            help
+                This is a helper config for 32bits address flash. Invisible for users.
+
+        config BOOTLOADER_CACHE_32BIT_ADDR_QUAD_FLASH
+            bool "Enable cache access to 32-bit-address (over 16MB) range of SPI Flash (READ DOCS FIRST)"
+            depends on BOOTLOADER_FLASH_NEEDS_32BIT_FEAT && IDF_TARGET_ESP32S3 && IDF_EXPERIMENTAL_FEATURES
+            default n
+            help
+                Enabling this option allows the CPU to access 32-bit-address flash beyond 16M range.
+                1. This option only valid for 4-line flash. Octal flash doesn't need this.
+                2. This option is experimental, which means it can’t use on all flash chips stable, for more
+                information, please contact Espressif Business support.
+
+        config BOOTLOADER_CACHE_32BIT_ADDR_OCTAL_FLASH
+            bool
+            default y if ESPTOOLPY_OCT_FLASH && BOOTLOADER_FLASH_32BIT_ADDR
+            default n
+
+    endmenu
 
     choice BOOTLOADER_VDDSDIO_BOOST
         bool "VDDSDIO LDO voltage"
@@ -424,15 +476,6 @@ menu "Bootloader config"
             - "Reserve RTC FAST memory for custom purposes"
             - "GPIO triggers factory reset"
 
-    config BOOTLOADER_FLASH_XMC_SUPPORT
-        bool "Enable the support for flash chips of XMC (READ HELP FIRST)"
-        default y
-        help
-            Perform the startup flow recommended by XMC. Please consult XMC for the details of this flow.
-            XMC chips will be forbidden to be used, when this option is disabled.
-
-            DON'T DISABLE THIS UNLESS YOU KNOW WHAT YOU ARE DOING.
-
 endmenu  # Bootloader
 
 
@@ -704,14 +747,12 @@ menu "Security features"
 
     config SECURE_BOOT_VERIFICATION_KEY
         string "Secure boot public signature verification key"
-        depends on SECURE_SIGNED_APPS && !SECURE_BOOT_BUILD_SIGNED_BINARIES && !SECURE_SIGNED_APPS_RSA_SCHEME
+        depends on SECURE_SIGNED_APPS && SECURE_SIGNED_APPS_ECDSA_SCHEME && !SECURE_BOOT_BUILD_SIGNED_BINARIES
         default "signature_verification_key.bin"
         help
             Path to a public key file used to verify signed images.
             Secure Boot V1: This ECDSA public key is compiled into the bootloader and/or
             app, to verify app images.
-            Secure Boot V2: This RSA public key is compiled into the signature block at
-            the end of the bootloader/app.
 
             Key file is in raw binary format, and can be extracted from a
             PEM formatted private key using the espsecure.py
@@ -735,6 +776,23 @@ menu "Security features"
             This can lead to permanent bricking of the device, in case all keys are revoked
             because of signature verification failure.
 
+    config SECURE_BOOT_FLASH_BOOTLOADER_DEFAULT
+        bool "Flash bootloader along with other artifacts when using the default flash command"
+        depends on SECURE_BOOT_V2_ENABLED && SECURE_BOOT_BUILD_SIGNED_BINARIES
+        default N
+        help
+            When Secure Boot V2 is enabled, by default the bootloader is not flashed along with other artifacts
+            like the application and the partition table images, i.e. bootloader has to be seperately flashed
+            using the command `idf.py bootloader flash`, whereas, the application and partition table can be flashed
+            using the command `idf.py flash` itself.
+            Enabling this option allows flashing the bootloader along with the other artifacts
+            by invocation of the command `idf.py flash`.
+
+            If this option is enabled make sure that even the bootloader is signed using the correct secure boot key,
+            otherwise the bootloader signature verification would fail, as hash of the public key which is present in
+            the bootloader signature would not match with the digest stored into the efuses
+            and thus the device will not be able to boot up.
+
     choice SECURE_BOOTLOADER_KEY_ENCODING
         bool "Hardware Key Encoding"
         depends on SECURE_BOOTLOADER_REFLASHABLE
@@ -784,11 +842,11 @@ menu "Security features"
             before enabling.
 
     choice SECURE_FLASH_ENCRYPTION_KEYSIZE
-        bool "Size of generated AES-XTS key"
+        bool "Size of generated XTS-AES key"
         default SECURE_FLASH_ENCRYPTION_AES128
         depends on SOC_FLASH_ENCRYPTION_XTS_AES_OPTIONS && SECURE_FLASH_ENC_ENABLED
         help
-            Size of generated AES-XTS key.
+            Size of generated XTS-AES key.
 
             - AES-128 uses a 256-bit key (32 bytes) derived from 128 bits (16 bytes) burned in half Efuse key block.
               Internally, it calculates SHA256(128 bits)
@@ -846,7 +904,9 @@ menu "Security features"
         default y if (SOC_EFUSE_DIS_ICACHE || IDF_TARGET_ESP32) && SECURE_FLASH_ENC_ENABLED
 
     menu "Potentially insecure options"
-        visible if SECURE_FLASH_ENCRYPTION_MODE_DEVELOPMENT || SECURE_BOOT_INSECURE || SECURE_SIGNED_ON_UPDATE_NO_SECURE_BOOT # NOERROR
+        visible if (SECURE_FLASH_ENCRYPTION_MODE_DEVELOPMENT || \
+            SECURE_BOOT_INSECURE || \
+            SECURE_SIGNED_ON_UPDATE_NO_SECURE_BOOT) # NOERROR
 
         # NOTE: Options in this menu NEED to have SECURE_BOOT_INSECURE
         # and/or SECURE_FLASH_ENCRYPTION_MODE_DEVELOPMENT in "depends on", as the menu
@@ -950,7 +1010,8 @@ menu "Security features"
 
         config SECURE_FLASH_UART_BOOTLOADER_ALLOW_CACHE
             bool "Leave UART bootloader flash cache enabled"
-            depends on SECURE_FLASH_ENCRYPTION_MODE_DEVELOPMENT && (IDF_TARGET_ESP32 || SOC_EFUSE_DIS_DOWNLOAD_ICACHE || SOC_EFUSE_DIS_DOWNLOAD_DCACHE) # NOERROR
+            depends on SECURE_FLASH_ENCRYPTION_MODE_DEVELOPMENT && \
+                (IDF_TARGET_ESP32 || SOC_EFUSE_DIS_DOWNLOAD_ICACHE || SOC_EFUSE_DIS_DOWNLOAD_DCACHE) # NOERROR
             default N
             select SECURE_FLASH_SKIP_WRITE_PROTECTION_CACHE if SECURE_FLASH_HAS_WRITE_PROTECTION_CACHE
             help
@@ -1009,6 +1070,22 @@ menu "Security features"
                 DIS_DOWNLOAD_MANUAL_ENCRYPT, DIS_USB_JTAG, DIS_USB_SERIAL_JTAG, STRAP_JTAG_SEL, USB_PHY_SEL.
     endmenu  # Potentially Insecure
 
+    config SECURE_FLASH_ENCRYPT_ONLY_IMAGE_LEN_IN_APP_PART
+        bool "Encrypt only the app image that is present in the partition of type app"
+        depends on SECURE_FLASH_ENC_ENABLED && !SECURE_FLASH_REQUIRE_ALREADY_ENABLED
+        default y
+        help
+            If set (default), optimise encryption time for the partition of type APP,
+            by only encrypting the app image that is present in the partition,
+            instead of the whole partition.
+            The image length used for encryption is derived from the image metadata, which
+            includes the size of the app image, checksum, hash and also the signature sector
+            when secure boot is enabled.
+
+            If not set, the whole partition of type APP would be encrypted,
+            which increases the encryption time but might be useful if there
+            is any custom data appended to the firmware image.
+
     config SECURE_FLASH_CHECK_ENC_EN_IN_APP
         bool "Check Flash Encryption enabled on app startup"
         depends on SECURE_FLASH_ENC_ENABLED

components/bootloader/project_include.cmake

@@ -124,13 +124,20 @@ idf_build_get_property(sdkconfig SDKCONFIG)
 idf_build_get_property(python PYTHON)
 idf_build_get_property(extra_cmake_args EXTRA_CMAKE_ARGS)
 
+# We cannot pass lists are a parameter to the external project without modifying the ';' spearator
+string(REPLACE ";" "|" BOOTLOADER_IGNORE_EXTRA_COMPONENT "${BOOTLOADER_IGNORE_EXTRA_COMPONENT}")
+
 externalproject_add(bootloader
     SOURCE_DIR "${CMAKE_CURRENT_LIST_DIR}/subproject"
     BINARY_DIR "${BOOTLOADER_BUILD_DIR}"
+    # Modiying the list separator for the arguments, as such, we won't need to manually
+    # replace the new separator by the default ';' in the subproject
+    LIST_SEPARATOR |
     CMAKE_ARGS  -DSDKCONFIG=${sdkconfig} -DIDF_PATH=${idf_path} -DIDF_TARGET=${idf_target}
                 -DPYTHON_DEPS_CHECKED=1 -DPYTHON=${python}
                 -DEXTRA_COMPONENT_DIRS=${CMAKE_CURRENT_LIST_DIR}
                 -DPROJECT_SOURCE_DIR=${PROJECT_SOURCE_DIR}
+                -DIGNORE_EXTRA_COMPONENT=${BOOTLOADER_IGNORE_EXTRA_COMPONENT}
                 ${sign_key_arg} ${ver_key_arg}
                 ${extra_cmake_args}
     INSTALL_COMMAND ""
@@ -148,5 +155,5 @@ endif()
 #
 # So for now we just have the top-level build remove the final build products...
 set_property(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" APPEND PROPERTY
-    ADDITIONAL_MAKE_CLEAN_FILES
+    ADDITIONAL_CLEAN_FILES
     ${bootloader_binary_files})

components/bootloader/sdkconfig.rename

@@ -23,3 +23,7 @@ CONFIG_FLASH_ENCRYPTION_UART_BOOTLOADER_ALLOW_CACHE         CONFIG_SECURE_FLASH_
 
 # Secure Boot Scheme
 CONFIG_SECURE_BOOT_ENABLED                                  CONFIG_SECURE_BOOT_V1_ENABLED
+
+CONFIG_SPI_FLASH_32BIT_ADDR_ENABLE                          CONFIG_BOOTLOADER_CACHE_32BIT_ADDR_QUAD_FLASH
+CONFIG_SPI_FLASH_QUAD_32BIT_ADDR_ENABLE                     CONFIG_BOOTLOADER_CACHE_32BIT_ADDR_QUAD_FLASH
+CONFIG_SPI_FLASH_OCTAL_32BIT_ADDR_ENABLE                    CONFIG_BOOTLOADER_CACHE_32BIT_ADDR_OCTAL_FLASH

components/bootloader/subproject/CMakeLists.txt

@@ -41,11 +41,18 @@ if(EXISTS ${PROJECT_EXTRA_COMPONENTS})
     list(APPEND EXTRA_COMPONENT_DIRS "${PROJECT_EXTRA_COMPONENTS}")
 endif()
 
-# Consider each directory in project's bootloader_components as a component to be compiled
+if(IGNORE_EXTRA_COMPONENT)
+    # Prefix all entries of the list with ${PROJECT_EXTRA_COMPONENTS} absolute path
+    list(TRANSFORM IGNORE_EXTRA_COMPONENT
+         PREPEND "${PROJECT_EXTRA_COMPONENTS}/"
+         OUTPUT_VARIABLE EXTRA_COMPONENT_EXCLUDE_DIRS)
+endif()
+
+# Consider each directory in the project's bootloader_components as a component to be compiled
 file(GLOB proj_components RELATIVE ${PROJECT_EXTRA_COMPONENTS} ${PROJECT_EXTRA_COMPONENTS}/*)
 foreach(component ${proj_components})
-  # Only directories are considered as components
-  if(IS_DIRECTORY ${curdir}/${child})
+  # Only directories are considered components
+  if(IS_DIRECTORY "${PROJECT_EXTRA_COMPONENTS}/${component}" AND NOT ${component} IN_LIST IGNORE_EXTRA_COMPONENT)
     list(APPEND COMPONENTS ${component})
   endif()
 endforeach()
@@ -53,6 +60,7 @@ endforeach()
 set(BOOTLOADER_BUILD 1)
 include("${IDF_PATH}/tools/cmake/project.cmake")
 set(common_req log esp_rom esp_common esp_hw_support newlib)
+idf_build_set_property(EXTRA_COMPONENT_EXCLUDE_DIRS "${EXTRA_COMPONENT_EXCLUDE_DIRS}")
 idf_build_set_property(__COMPONENT_REQUIRES_COMMON "${common_req}")
 idf_build_set_property(__OUTPUT_SDKCONFIG 0)
 project(bootloader)
@@ -202,7 +210,11 @@ elseif(CONFIG_SECURE_BOOTLOADER_REFLASHABLE)
             "* Not recommended to re-use the same secure boot keyfile on multiple production devices."
         DEPENDS gen_secure_bootloader_key gen_bootloader_digest_bin
         VERBATIM)
-elseif(CONFIG_SECURE_BOOT_V2_ENABLED AND (CONFIG_IDF_TARGET_ESP32S2 OR CONFIG_IDF_TARGET_ESP32C3))
+elseif(
+        CONFIG_SECURE_BOOT_V2_ENABLED AND
+        (CONFIG_SOC_EFUSE_SECURE_BOOT_KEY_DIGESTS GREATER 1) AND
+        NOT CONFIG_SECURE_BOOT_FLASH_BOOTLOADER_DEFAULT
+    )
     add_custom_command(TARGET bootloader.elf POST_BUILD
     COMMAND ${CMAKE_COMMAND} -E echo
         "=============================================================================="
@@ -221,7 +233,7 @@ elseif(CONFIG_SECURE_BOOT_V2_ENABLED AND (CONFIG_IDF_TARGET_ESP32S2 OR CONFIG_ID
         "=============================================================================="
     DEPENDS gen_signed_bootloader
     VERBATIM)
-elseif(CONFIG_SECURE_BOOT_V2_ENABLED)
+elseif(CONFIG_SECURE_BOOT_V2_ENABLED AND NOT CONFIG_SECURE_BOOT_FLASH_BOOTLOADER_DEFAULT)
     add_custom_command(TARGET bootloader.elf POST_BUILD
     COMMAND ${CMAKE_COMMAND} -E echo
         "=============================================================================="

components/bootloader/subproject/main/ld/esp32/bootloader.ld

@@ -25,7 +25,7 @@ MEMORY
   */
   iram_seg (RWX) :                  org = 0x40080400, len = 0xfc00
   /* 64k at the end of DRAM, after ROM bootloader stack */
-  dram_seg (RW) :                  	org = 0x3FFF0000, len = 0x10000
+  dram_seg (RW) :                  	org = 0x3FFF0000, len = 0x6000
 }
 
 /*  Default entry point:  */
@@ -42,7 +42,9 @@ SECTIONS
     *(.stub .gnu.warning .gnu.linkonce.literal.* .gnu.linkonce.t.*.literal .gnu.linkonce.t.*)
      *(.iram1 .iram1.*) /* catch stray IRAM_ATTR */
     *liblog.a:(.literal .text .literal.* .text.*)
+    /* we use either libgcc or compiler-rt, so put similar entries for them here */
     *libgcc.a:(.literal .text .literal.* .text.*)
+    *libclang_rt.builtins.a:(.literal .text .literal.* .text.*)
     *libbootloader_support.a:bootloader_clock_loader.*(.literal .text .literal.* .text.*)
     *libbootloader_support.a:bootloader_common_loader.*(.literal .text .literal.* .text.*)
     *libbootloader_support.a:bootloader_flash.*(.literal .text .literal.* .text.*)
@@ -109,9 +111,14 @@ SECTIONS
     _bss_end = ABSOLUTE(.);
   } >dram_seg
 
-  .dram0.data :
+  .dram0.bootdesc : ALIGN(0x10)
   {
     _data_start = ABSOLUTE(.);
+    *(.data_bootloader_desc .data_bootloader_desc.*)               /* Should be the first.  Bootloader version info.        DO NOT PUT ANYTHING BEFORE IT! */
+  } > dram_seg
+
+  .dram0.data : ALIGN(0x10)
+  {
     *(.data)
     *(.data.*)
     *(.gnu.linkonce.d.*)

components/bootloader/subproject/main/ld/esp32/bootloader.rom.ld

@@ -1,9 +0,0 @@
-PROVIDE ( ets_update_cpu_frequency = 0x40008550 ); /* Updates g_ticks_per_us on the current CPU only; not on the other core */
-PROVIDE ( MD5Final = 0x4005db1c );
-PROVIDE ( MD5Init = 0x4005da7c );
-PROVIDE ( MD5Update = 0x4005da9c );
-
-/* bootloader will use following functions from xtensa hal library */
-xthal_get_ccount = 0x4000c050;
-xthal_get_ccompare = 0x4000c078;
-xthal_set_ccompare = 0x4000c058;

components/bootloader/subproject/main/ld/esp32c2/bootloader.ld

@@ -63,13 +63,14 @@ SECTIONS
     *(.stub .gnu.warning .gnu.linkonce.literal.* .gnu.linkonce.t.*.literal .gnu.linkonce.t.*)
      *(.iram1 .iram1.*) /* catch stray IRAM_ATTR */
     *liblog.a:(.literal .text .literal.* .text.*)
+    /* we use either libgcc or compiler-rt, so put similar entries for them here */
     *libgcc.a:(.literal .text .literal.* .text.*)
+    *libclang_rt.builtins.a:(.literal .text .literal.* .text.*)
     *libbootloader_support.a:bootloader_clock_loader.*(.literal .text .literal.* .text.*)
     *libbootloader_support.a:bootloader_common_loader.*(.literal .text .literal.* .text.*)
     *libbootloader_support.a:bootloader_flash.*(.literal .text .literal.* .text.*)
     *libbootloader_support.a:bootloader_random.*(.literal .text .literal.* .text.*)
     *libbootloader_support.a:bootloader_random*.*(.literal.bootloader_random_disable .text.bootloader_random_disable)
-    *libbootloader_support.a:bootloader_random*.*(.literal.bootloader_random_enable .text.bootloader_random_enable)
     *libbootloader_support.a:bootloader_efuse.*(.literal .text .literal.* .text.*)
     *libbootloader_support.a:bootloader_utility.*(.literal .text .literal.* .text.*)
     *libbootloader_support.a:bootloader_sha.*(.literal .text .literal.* .text.*)
@@ -131,9 +132,14 @@ SECTIONS
     _bss_end = ABSOLUTE(.);
   } > dram_seg
 
-  .dram0.data :
+  .dram0.bootdesc : ALIGN(0x10)
   {
     _data_start = ABSOLUTE(.);
+    *(.data_bootloader_desc .data_bootloader_desc.*)               /* Should be the first.  Bootloader version info.        DO NOT PUT ANYTHING BEFORE IT! */
+  } > dram_seg
+
+  .dram0.data :
+  {
     *(.data)
     *(.data.*)
     *(.gnu.linkonce.d.*)

components/bootloader/subproject/main/ld/esp32c3/bootloader.ld

@@ -63,13 +63,14 @@ SECTIONS
     *(.stub .gnu.warning .gnu.linkonce.literal.* .gnu.linkonce.t.*.literal .gnu.linkonce.t.*)
     *(.iram1 .iram1.*) /* catch stray IRAM_ATTR */
     *liblog.a:(.literal .text .literal.* .text.*)
+    /* we use either libgcc or compiler-rt, so put similar entries for them here */
     *libgcc.a:(.literal .text .literal.* .text.*)
+    *libclang_rt.builtins.a:(.literal .text .literal.* .text.*)
     *libbootloader_support.a:bootloader_clock_loader.*(.literal .text .literal.* .text.*)
     *libbootloader_support.a:bootloader_common_loader.*(.literal .text .literal.* .text.*)
     *libbootloader_support.a:bootloader_flash.*(.literal .text .literal.* .text.*)
     *libbootloader_support.a:bootloader_random.*(.literal .text .literal.* .text.*)
     *libbootloader_support.a:bootloader_random*.*(.literal.bootloader_random_disable .text.bootloader_random_disable)
-    *libbootloader_support.a:bootloader_random*.*(.literal.bootloader_random_enable .text.bootloader_random_enable)
     *libbootloader_support.a:bootloader_efuse.*(.literal .text .literal.* .text.*)
     *libbootloader_support.a:bootloader_utility.*(.literal .text .literal.* .text.*)
     *libbootloader_support.a:bootloader_sha.*(.literal .text .literal.* .text.*)
@@ -131,9 +132,14 @@ SECTIONS
     _bss_end = ABSOLUTE(.);
   } > dram_seg
 
-  .dram0.data :
+  .dram0.bootdesc : ALIGN(0x10)
   {
     _data_start = ABSOLUTE(.);
+    *(.data_bootloader_desc .data_bootloader_desc.*)               /* Should be the first.  Bootloader version info.        DO NOT PUT ANYTHING BEFORE IT! */
+  } > dram_seg
+
+  .dram0.data :
+  {
     *(.data)
     *(.data.*)
     *(.gnu.linkonce.d.*)

components/bootloader/subproject/main/ld/esp32c6/bootloader.ld

@@ -60,7 +60,9 @@ SECTIONS
     *(.stub .gnu.warning .gnu.linkonce.literal.* .gnu.linkonce.t.*.literal .gnu.linkonce.t.*)
     *(.iram1 .iram1.*) /* catch stray IRAM_ATTR */
     *liblog.a:(.literal .text .literal.* .text.*)
+    /* we use either libgcc or compiler-rt, so put similar entries for them here */
     *libgcc.a:(.literal .text .literal.* .text.*)
+    *libclang_rt.builtins.a:(.literal .text .literal.* .text.*)
     *libbootloader_support.a:bootloader_clock_loader.*(.literal .text .literal.* .text.*)
     *libbootloader_support.a:bootloader_common_loader.*(.literal .text .literal.* .text.*)
     *libbootloader_support.a:bootloader_flash.*(.literal .text .literal.* .text.*)
@@ -128,9 +130,14 @@ SECTIONS
     _bss_end = ABSOLUTE(.);
   } > dram_seg
 
-  .dram0.data :
+  .dram0.bootdesc : ALIGN(0x10)
   {
     _data_start = ABSOLUTE(.);
+    *(.data_bootloader_desc .data_bootloader_desc.*)               /* Should be the first.  Bootloader version info.        DO NOT PUT ANYTHING BEFORE IT! */
+  } > dram_seg
+
+  .dram0.data :
+  {
     *(.data)
     *(.data.*)
     *(.gnu.linkonce.d.*)

components/bootloader/subproject/main/ld/esp32h2/bootloader.ld

@@ -61,13 +61,14 @@ SECTIONS
     *(.stub .gnu.warning .gnu.linkonce.literal.* .gnu.linkonce.t.*.literal .gnu.linkonce.t.*)
     *(.iram1 .iram1.*) /* catch stray IRAM_ATTR */
     *liblog.a:(.literal .text .literal.* .text.*)
+    /* we use either libgcc or compiler-rt, so put similar entries for them here */
     *libgcc.a:(.literal .text .literal.* .text.*)
+    *libclang_rt.builtins.a:(.literal .text .literal.* .text.*)
     *libbootloader_support.a:bootloader_clock_loader.*(.literal .text .literal.* .text.*)
     *libbootloader_support.a:bootloader_common_loader.*(.literal .text .literal.* .text.*)
     *libbootloader_support.a:bootloader_flash.*(.literal .text .literal.* .text.*)
     *libbootloader_support.a:bootloader_random.*(.literal .text .literal.* .text.*)
     *libbootloader_support.a:bootloader_random*.*(.literal.bootloader_random_disable .text.bootloader_random_disable)
-    *libbootloader_support.a:bootloader_random*.*(.literal.bootloader_random_enable .text.bootloader_random_enable)
     *libbootloader_support.a:bootloader_efuse.*(.literal .text .literal.* .text.*)
     *libbootloader_support.a:bootloader_utility.*(.literal .text .literal.* .text.*)
     *libbootloader_support.a:bootloader_sha.*(.literal .text .literal.* .text.*)
@@ -129,6 +130,12 @@ SECTIONS
     _bss_end = ABSOLUTE(.);
   } > dram_seg
 
+  .dram0.bootdesc : ALIGN(0x10)
+  {
+    _data_start = ABSOLUTE(.);
+    *(.data_bootloader_desc .data_bootloader_desc.*)               /* Should be the first.  Bootloader version info.        DO NOT PUT ANYTHING BEFORE IT! */
+  } > dram_seg
+
   .dram0.data :
   {
     _data_start = ABSOLUTE(.);

components/bootloader/subproject/main/ld/esp32h4/bootloader.rom.ld

@@ -1,6 +0,0 @@
-/*
- * SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
- *
- * SPDX-License-Identifier: Apache-2.0
- */
-/* No definition for ESP32-H4 target */

components/bootloader/subproject/main/ld/esp32p4/bootloader.ld

@@ -1,38 +1,37 @@
 /*
- * SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
+ * SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
  *
  * SPDX-License-Identifier: Apache-2.0
  */
-/** Simplified memory map for the bootloader.
+/**
+ * TODO: IDF-8002
+ * Simplified memory map for the bootloader.
  *  Make sure the bootloader can load into main memory without overwriting itself.
  *
- *  ESP32-H4 ROM static data usage is as follows:
- *  - 0x3fccb900 - 0x3fcdd210: Shared buffers, used in UART/USB/SPI download mode only
- *  - 0x3fcdd210 - 0x3fcdf210: PRO CPU stack, can be reclaimed as heap after RTOS startup
- *  - 0x3fcdf210 - 0x3fce0000: ROM .bss and .data (not easily reclaimable)
+ *  ESP32-P4 ROM static data usage is as follows:
+ *  - 0x4ff296b8 - 0x4ff3afc0: Shared buffers, used in UART/USB/SPI download mode only
+ *  - 0x4ff3afc0 - 0x4ff3fba4: CPU1 stack, can be reclaimed as heap after RTOS startup
+ *  - 0x4ff3fba4 - 0x4ff40000: ROM .bss and .data (not easily reclaimable)
  *
  *  The 2nd stage bootloader can take space up to the end of ROM shared
- *  buffers area (0x3fce9704). For alignment purpose we shall use value (0x3fce9700).
+ *  buffers area (0x4087c610).
  */
 
-/* The offset between Dbus and Ibus. Used to convert between 0x403xxxxx and 0x3fcxxxxx addresses. */
-iram_dram_offset = 0x700000;
-
-/* We consider 0x3fce9700 to be the last usable address for 2nd stage bootloader stack overhead, dram_seg,
+/* We consider 0x4087c610 to be the last usable address for 2nd stage bootloader stack overhead, dram_seg,
  * and work out iram_seg and iram_loader_seg addresses from there, backwards.
  */
 
 /* These lengths can be adjusted, if necessary: */
-bootloader_usable_dram_end = 0x3fcdd120;
+bootloader_usable_dram_end = 0x4ff3abd0;
 bootloader_stack_overhead = 0x2000; /* For safety margin between bootloader data section and startup stacks */
-bootloader_dram_seg_len = 0x5000;
+bootloader_dram_seg_len = 0x4000;
 bootloader_iram_loader_seg_len = 0x7000;
 bootloader_iram_seg_len = 0x2000;
 
 /* Start of the lower region is determined by region size and the end of the higher region */
 bootloader_dram_seg_end = bootloader_usable_dram_end - bootloader_stack_overhead;
 bootloader_dram_seg_start = bootloader_dram_seg_end - bootloader_dram_seg_len;
-bootloader_iram_loader_seg_start = bootloader_dram_seg_start - bootloader_iram_loader_seg_len + iram_dram_offset;
+bootloader_iram_loader_seg_start = bootloader_dram_seg_start - bootloader_iram_loader_seg_len;
 bootloader_iram_seg_start = bootloader_iram_loader_seg_start - bootloader_iram_seg_len;
 
 MEMORY
@@ -42,6 +41,14 @@ MEMORY
   dram_seg (RW) :                   org = bootloader_dram_seg_start, len = bootloader_dram_seg_len
 }
 
+/* The app may use RAM for static allocations up to the start of iram_loader_seg.
+ * If you have changed something above and this assert fails:
+ * 1. Check what the new value of bootloader_iram_loader_seg start is.
+ * 2. Update the value in this assert.
+ * 3. Update SRAM_DRAM_END in components/esp_system/ld/esp32p4/memory.ld.in to the same value.
+ */
+ASSERT(bootloader_iram_loader_seg_start == 0x4FF2DBD0, "bootloader_iram_loader_seg_start inconsistent with SRAM_DRAM_END");
+
 /* Default entry point: */
 ENTRY(call_start_cpu0);
 
@@ -53,7 +60,7 @@ SECTIONS
     . = ALIGN (16);
     _loader_text_start = ABSOLUTE(.);
     *(.stub .gnu.warning .gnu.linkonce.literal.* .gnu.linkonce.t.*.literal .gnu.linkonce.t.*)
-     *(.iram1 .iram1.*) /* catch stray IRAM_ATTR */
+    *(.iram1 .iram1.*) /* catch stray IRAM_ATTR */
     *liblog.a:(.literal .text .literal.* .text.*)
     *libgcc.a:(.literal .text .literal.* .text.*)
     *libbootloader_support.a:bootloader_clock_loader.*(.literal .text .literal.* .text.*)
@@ -70,8 +77,10 @@ SECTIONS
     *libbootloader_support.a:bootloader_soc.*(.literal .text .literal.* .text.*)
     *libbootloader_support.a:esp_image_format.*(.literal .text .literal.* .text.*)
     *libbootloader_support.a:flash_encrypt.*(.literal .text .literal.* .text.*)
+    *libbootloader_support.a:flash_encryption_secure_features.*(.literal .text .literal.* .text.*)
     *libbootloader_support.a:flash_partitions.*(.literal .text .literal.* .text.*)
     *libbootloader_support.a:secure_boot.*(.literal .text .literal.* .text.*)
+    *libbootloader_support.a:secure_boot_secure_features.*(.literal .text .literal.* .text.*)
     *libbootloader_support.a:secure_boot_signatures_bootloader.*(.literal .text .literal.* .text.*)
     *libmicro-ecc.a:*.*(.literal .text .literal.* .text.*)
     *libspi_flash.a:*.*(.literal .text .literal.* .text.*)
@@ -204,37 +213,42 @@ SECTIONS
 
 }
 
+
 /**
  *  Appendix: Memory Usage of ROM bootloader
  *
- * 0x3fccb81c ------------------> _dram0_0_start
+ * 0x4ff296b8 ------------------> _dram0_0_start
  *            |               |
  *            |               |
  *            |               |   1. Large buffers that are only used in certain boot modes, see shared_buffers.h
  *            |               |
  *            |               |
- * 0x3fcdd120 ------------------> __stack_sentry
+ * 0x4ff3afc0 ------------------> __stack_sentry
  *            |               |
  *            |               |   2. Startup pro cpu stack (freed when IDF app is running)
  *            |               |
- * 0x3fcdf120 ------------------> __stack (pro cpu)
+ * 0x4ff3cfc0 ------------------> __stack (pro cpu)
+ *            |               |
+ *            |               |      Startup app cpu stack
+ *            |               |
+ * 0x4ff3efc0 ------------------> __stack_app (app cpu)
  *            |               |
  *            |               |
  *            |               |   3. Shared memory only used in startup code or nonos/early boot*
  *            |               |      (can be freed when IDF runs)
  *            |               |
  *            |               |
- * 0x3fcdfa6c ------------------> _dram0_rtos_reserved_start
+ * 0x4ff3fba4 ------------------> _dram0_rtos_reserved_start
  *            |               |
  *            |               |
  *            |               |   4. Shared memory used in startup code and when IDF runs
  *            |               |
  *            |               |
- * 0x3fcdfe40 ------------------> _dram0_rtos_reserved_end
+ * 0x4ff3ff94 ------------------> _dram0_rtos_reserved_end
  *            |               |
- * 0x3fcdfe4c ------------------> _data_start_interface
+ * 0x4ff3ffc8 ------------------> _data_start_interface
  *            |               |
  *            |               |   5. End of DRAM is the 'interface' data with constant addresses (ECO compatible)
  *            |               |
- * 0x3fce0000 ------------------> _data_end_interface
+ * 0x4ff40000 ------------------> _data_end_interface
  */