versions: Update version to 5.1.0

ulp: fix -Wformat errors in ULP tests

See merge request espressif/esp-idf!19379

.editorconfig

@@ -39,5 +39,7 @@ indent_size = 4
 max_line_length = 120
 
 [{*.sh,*.yml,*.yaml}]
-indent_style = space
+indent_size = 2
+
+[*.ini]
 indent_size = 2

.flake8

@@ -143,27 +143,23 @@ exclude =
         components/bootloader/subproject/components/micro-ecc/micro-ecc,
         components/bt/host/nimble/nimble,
         components/cmock/CMock,
-        components/esptool_py/esptool,
-        components/expat/expat,
         components/json/cJSON,
-        components/libsodium/libsodium,
         components/mbedtls/mbedtls,
-        components/nghttp/nghttp2,
         components/openthread/openthread,
         components/tinyusb,
         components/unity/unity,
-        examples/build_system/cmake/import_lib/main/lib/tinyxml2,
+        components/spiffs/spiffs,
         examples/peripherals/secure_element/atecc608_ecdsa/components/esp-cryptoauthlib,
     # autogenerated scripts
         components/protocomm/python/constants_pb2.py,
         components/protocomm/python/sec0_pb2.py,
         components/protocomm/python/sec1_pb2.py,
+        components/protocomm/python/sec2_pb2.py,
         components/protocomm/python/session_pb2.py,
         components/wifi_provisioning/python/wifi_scan_pb2.py,
         components/wifi_provisioning/python/wifi_config_pb2.py,
         components/wifi_provisioning/python/wifi_constants_pb2.py,
         components/esp_local_ctrl/python/esp_local_ctrl_pb2.py,
-        examples/provisioning/legacy/custom_config/components/custom_provisioning/python/custom_config_pb2.py,
 
 per-file-ignores =
     # Sphinx conf.py files use star imports to setup config variables

.github/workflows/docker.yml

@@ -0,0 +1,87 @@
+name: docker
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+
+on:
+  push:
+    branches:
+      - 'master'
+      - 'release/*'
+    tags:
+      - 'v*.*'
+
+env:
+  # Platforms to build the image for
+  BUILD_PLATFORMS: linux/amd64,linux/arm64
+  DOCKERHUB_REPO: ${{ github.repository_owner }}/idf
+
+jobs:
+  docker:
+    # Disable the job in forks
+    if: ${{ github.repository_owner == 'espressif' }}
+
+    runs-on: ubuntu-latest
+    steps:
+      # Depending on the branch/tag, set CLONE_BRANCH_OR_TAG variable (used in the Dockerfile
+      # as a build arg) and TAG_NAME (used when tagging the image).
+      #
+      # The following 3 steps cover the alternatives (tag, release branch, master branch):
+      - name: Set variables (tags)
+        if: ${{ github.ref_type == 'tag' }}
+        run: |
+          echo "CLONE_BRANCH_OR_TAG=$GITHUB_REF_NAME" >> $GITHUB_ENV
+          echo "TAG_NAME=$GITHUB_REF_NAME" >> $GITHUB_ENV
+      - name: Set variables (release branches)
+        if: ${{ github.ref_type == 'branch' && startsWith(github.ref_name, 'release/') }}
+        run: |
+          echo "CLONE_BRANCH_OR_TAG=$GITHUB_REF_NAME" >> $GITHUB_ENV
+          echo "TAG_NAME=release-${GITHUB_REF_NAME##release/}" >> $GITHUB_ENV
+      - name: Set variables (main branch)
+        if: ${{ github.ref_type == 'branch' && github.ref_name == 'master' }}
+        run: |
+          echo "CLONE_BRANCH_OR_TAG=master" >> $GITHUB_ENV
+          echo "TAG_NAME=latest" >> $GITHUB_ENV
+
+      # Display the variables set above, just in case.
+      - name: Check variables
+        run: |
+          echo "CLONE_BRANCH_OR_TAG: $CLONE_BRANCH_OR_TAG"
+          echo "CHECKOUT_REF: $CHECKOUT_REF"
+          echo "TAG_NAME: $TAG_NAME"
+
+      # The following steps are the standard boilerplate from
+      # https://github.com/marketplace/actions/build-and-push-docker-images
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Set up QEMU for multiarch builds
+        uses: docker/setup-qemu-action@v2
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      - name: Build and push
+        uses: docker/build-push-action@v3
+        with:
+          context: tools/docker
+          push: true
+          tags: ${{ env.DOCKERHUB_REPO }}:${{ env.TAG_NAME }}
+          platforms: ${{ env.BUILD_PLATFORMS }}
+          build-args: |
+            IDF_CLONE_URL=${{ github.server_url }}/${{ github.repository }}.git
+            IDF_CLONE_BRANCH_OR_TAG=${{ env.CLONE_BRANCH_OR_TAG }}
+
+      - name: Update Docker Hub repository description (master branch)
+        if: ${{ github.ref_type == 'branch' && github.ref_name == 'master' }}
+        uses: peter-evans/dockerhub-description@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          # Token based authentication is not supported here:
+          # https://github.com/peter-evans/dockerhub-description/issues/10
+          # https://github.com/docker/roadmap/issues/115#issuecomment-891694974
+          password: ${{ secrets.DOCKERHUB_PASSWORD }}
+          repository: ${{ env.DOCKERHUB_REPO }}
+          readme-filepath: ./tools/docker/README.md

.github/workflows/issue_comment.yml

@@ -3,12 +3,16 @@ name: Sync issue comments to JIRA
 # This workflow will be triggered when new issue comment is created (including PR comments)
 on: issue_comment
 
+# Limit to single concurrent run for workflows which can create Jira issues.
+# Same concurrency group is used in new_issues.yml
+concurrency: jira_issues
+
 jobs:
   sync_issue_comments_to_jira:
     name: Sync Issue Comments to Jira
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@master
+      - uses: actions/checkout@v2
       - name: Sync issue comments to JIRA
         uses: espressif/github-actions/sync_issues_to_jira@master
         env:

.github/workflows/new_issues.yml

@@ -3,12 +3,16 @@ name: Sync issues to Jira
 # This workflow will be triggered when a new issue is opened
 on: issues
 
+# Limit to single concurrent run for workflows which can create Jira issues.
+# Same concurrency group is used in issue_comment.yml
+concurrency: jira_issues
+
 jobs:
   sync_issues_to_jira:
     name: Sync issues to Jira
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@master
+      - uses: actions/checkout@v2
       - name: Sync GitHub issues to Jira project
         uses: espressif/github-actions/sync_issues_to_jira@master
         env:

.github/workflows/new_prs.yml

@@ -6,12 +6,16 @@ on:
   schedule:
     - cron: "0 * * * *"
 
+# Limit to single concurrent run for workflows which can create Jira issues.
+# Same concurrency group is used in issue_comment.yml
+concurrency: jira_issues
+
 jobs:
   sync_prs_to_jira:
     name: Sync PRs to Jira
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@master
+      - uses: actions/checkout@v2
       - name: Sync PRs to Jira project
         uses: espressif/github-actions/sync_issues_to_jira@master
         with:

.github/workflows/pr_approved.yml

@@ -0,0 +1,23 @@
+name: Sync approved PRs to internal codebase
+on:
+  pull_request_target:
+    types: [labeled]
+
+jobs:
+  sync_prs_to_internal_codebase:
+    name: GitHub PR to Internal Codebase Sync
+    runs-on: ubuntu-latest
+    if: (github.event.label.name == 'PR-Sync-Merge')   ||
+        (github.event.label.name == 'PR-Sync-Rebase')  ||
+        (github.event.label.name == 'PR-Sync-Update')
+    steps:
+      - uses: actions/checkout@v2
+      - name: Sync approved PRs to internal codebase
+        uses: espressif/github-actions/github_pr_to_internal_pr@master
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GITLAB_URL: ${{ secrets.GITLAB_URL }}
+          GITLAB_TOKEN: ${{ secrets.GITLAB_TOKEN }}
+          GIT_CONFIG_NAME: ${{ secrets.GIT_CONFIG_NAME }}
+          GIT_CONFIG_EMAIL: ${{ secrets.GIT_CONFIG_EMAIL }}
+          JIRA_PROJECT: IDFGH

.github/workflows/pre_commit_check.yml

@@ -0,0 +1,37 @@
+name: Check pre-commit rules
+
+on:
+  pull_request:
+    types: [opened, reopened, synchronize]
+
+jobs:
+  pre_commit_check:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Fetch head and base refs
+        # This is necessary for pre-commit to check the changes in the PR branch
+        run: |
+          git fetch origin ${{ github.base_ref }}:base_ref
+          git fetch origin pull/${{ github.event.pull_request.number }}/head:pr_ref
+      - name: Set up Python environment
+        uses: actions/setup-python@master
+        with:
+          python-version: v3.7
+      - name: Install python packages
+        run: |
+          pip install pre-commit
+          pre-commit install-hooks
+      - name: Run pre-commit and check for any changes
+        run: |
+          echo "Commits being checked:"
+          git log --oneline --no-decorate base_ref..pr_ref
+          echo ""
+          if ! pre-commit run --from-ref base_ref --to-ref pr_ref --show-diff-on-failure ; then
+            echo ""
+            echo "::notice::It looks like the commits in this PR have been made without having pre-commit hooks installed."
+            echo "::notice::Please see https://docs.espressif.com/projects/esp-idf/en/latest/esp32/contribute/install-pre-commit-hook.html for instructions."
+            echo ""
+            exit 1
+          fi

.github/workflows/python_lint.yml

@@ -1,33 +0,0 @@
-name: Python CI
-
-# This workflow will be triggered when a PR modifies some python relevant files
-on:
-  pull_request:
-    paths:
-      - "**.py"
-      - "requirements.txt"
-
-jobs:
-  python_lint:
-    name: python lint
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: [3.6, 3.7, 3.8]
-
-    steps:
-      - name: Checkout
-        uses: actions/checkout@master
-      - name: Set up Python environment
-        uses: actions/setup-python@master
-        with:
-          python-version: ${{ matrix.python-version }}
-      - name: Install dependencies
-        run: |
-          export IDF_PATH=${GITHUB_WORKSPACE}
-          pip install --upgrade pip
-          pip install -r requirements.txt
-      - name: Lint with flake8
-        run: |
-          pip install flake8
-          flake8 . --config=.flake8 --benchmark

.gitignore

@@ -21,15 +21,20 @@ GPATH
 # MacOS directory files
 .DS_Store
 
+# cache dir
+.cache/
+
 # Components Unit Test Apps files
-components/**/build
+components/**/build/
+components/**/build_*_*/
 components/**/sdkconfig
 components/**/sdkconfig.old
 
 # Example project files
+examples/**/build/
+examples/**/build_esp*_*/
 examples/**/sdkconfig
 examples/**/sdkconfig.old
-examples/**/build
 
 # Doc build artifacts
 docs/_build/
@@ -43,7 +48,7 @@ docs/_static/NotoSansSC-Regular.otf
 tools/unit-test-app/sdkconfig
 tools/unit-test-app/sdkconfig.old
 tools/unit-test-app/build
-tools/unit-test-app/builds
+tools/unit-test-app/build_*_*/
 tools/unit-test-app/output
 tools/unit-test-app/test_configs
 
@@ -51,7 +56,8 @@ tools/unit-test-app/test_configs
 log_ut_cmake
 
 # test application build files
-tools/test_apps/**/build
+tools/test_apps/**/build/
+tools/test_apps/**/build_*_*/
 tools/test_apps/**/sdkconfig
 tools/test_apps/**/sdkconfig.old
 
@@ -91,3 +97,6 @@ dependencies.lock
 
 # managed_components for examples
 managed_components
+
+# pytest log
+pytest_embedded_log/

.gitlab-ci.yml

@@ -1,4 +1,5 @@
 stages:
+  - upload_cache
   - pre_check
   - build
   - assign_test
@@ -14,6 +15,14 @@ workflow:
     # Disable those non-protected push triggered pipelines
     - if: '$CI_COMMIT_REF_NAME != "master" && $CI_COMMIT_BRANCH !~ /^release\/v/ && $CI_COMMIT_TAG !~ /^v\d+\.\d+(\.\d+)?($|-)/ && $CI_PIPELINE_SOURCE == "push"'
       when: never
+    # when running merged result pipelines, it would create a temp commit id. use $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA instead of $CI_COMMIT_SHA.
+    # Please use PIPELINE_COMMIT_SHA at all places that require a commit sha
+    - if: $CI_OPEN_MERGE_REQUESTS != null
+      variables:
+        PIPELINE_COMMIT_SHA: $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA
+    - if: $CI_OPEN_MERGE_REQUESTS == null
+      variables:
+        PIPELINE_COMMIT_SHA: $CI_COMMIT_SHA
     - when: always
 
 variables:
@@ -45,14 +54,23 @@ variables:
   BATCH_BUILD: "1"
   V: "0"
   CHECKOUT_REF_SCRIPT: "$CI_PROJECT_DIR/tools/ci/checkout_project_ref.py"
+  PYTHON_VER: 3.7.10
+
+  CLANG_TIDY_RUNNER_PROJ: 2107  # idf/clang-tidy-runner
+  IDF_BUILD_APPS_PROJ: 2818  # fuhanxi/idf-build-apps
 
   # Docker images
   BOT_DOCKER_IMAGE_TAG: ":latest"
-  ESP_IDF_DOC_ENV_IMAGE: "$CI_DOCKER_REGISTRY/esp-idf-doc-env:v4.4-1-v4"
-  ESP_ENV_IMAGE: "$CI_DOCKER_REGISTRY/esp-env:v4.4-1"
-  AFL_FUZZER_TEST_IMAGE: "$CI_DOCKER_REGISTRY/afl-fuzzer-test:v4.4-1-1"
-  CLANG_STATIC_ANALYSIS_IMAGE: "${CI_DOCKER_REGISTRY}/clang-static-analysis:v4.4-1-2"
+
+  ESP_IDF_DOC_ENV_IMAGE: "$CI_DOCKER_REGISTRY/esp-idf-doc-env-v5.0:2-3"
+  ESP_ENV_IMAGE: "$CI_DOCKER_REGISTRY/esp-env-v5.0:2"
+  AFL_FUZZER_TEST_IMAGE: "$CI_DOCKER_REGISTRY/afl-fuzzer-test-v5.0:2-1"
+  CLANG_STATIC_ANALYSIS_IMAGE: "${CI_DOCKER_REGISTRY}/clang-static-analysis-v5.0:2-1"
+  TARGET_TEST_ENV_IMAGE: "$CI_DOCKER_REGISTRY/target-test-env-v5.0:2"
+  QEMU_IMAGE: "${CI_DOCKER_REGISTRY}/qemu-v5.0:2-20210826"
+
   SONARQUBE_SCANNER_IMAGE: "${CI_DOCKER_REGISTRY}/sonarqube-scanner:3"
+  LINUX_SHELL_IMAGE: "${CI_DOCKER_REGISTRY}/linux-shells-v5.0:2"
 
   # target test config file, used by assign test job
   CI_TARGET_TEST_CONFIG_FILE: "$CI_PROJECT_DIR/.gitlab/ci/target-test.yml"
@@ -60,7 +78,39 @@ variables:
   # target test repo parameters
   TEST_ENV_CONFIG_REPO: "https://gitlab-ci-token:${BOT_TOKEN}@${CI_SERVER_HOST}:${CI_SERVER_PORT}/qa/ci-test-runner-configs.git"
   CI_AUTO_TEST_SCRIPT_REPO_URL: "https://gitlab-ci-token:${BOT_TOKEN}@${CI_SERVER_HOST}:${CI_SERVER_PORT}/qa/auto_test_script.git"
-  CI_AUTO_TEST_SCRIPT_REPO_BRANCH: "ci/v3.1"
+  CI_AUTO_TEST_SCRIPT_REPO_BRANCH: "ci/v4.1"
+
+  # cache python dependencies
+  PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
+
+  # Set this variable to the branch of idf-constraints repo in order to test a custom Python constraint file. The
+  # branch name must be without the remote part ("origin/"). Keep the variable empty in order to use the constraint
+  # file from https://dl.espressif.com/dl/esp-idf.
+  CI_PYTHON_CONSTRAINT_BRANCH: ""
+
+  # Update the filename for a specific ESP-IDF release. It is used only with CI_PYTHON_CONSTRAINT_BRANCH.
+  CI_PYTHON_CONSTRAINT_FILE: "espidf.constraints.v5.1.txt"
+
+  # Set this variable to repository name of a Python tool you wish to install and test in the context of ESP-IDF CI.
+  # Keep the variable empty when not used.
+  CI_PYTHON_TOOL_REPO: ""
+
+  # Set this variable to the branch of a Python tool repo specified in CI_PYTHON_TOOL_REPO. The
+  # branch name must be without the remote part ("origin/"). Keep the variable empty when not used.
+  # This is used only if CI_PYTHON_TOOL_REPO is not empty.
+  CI_PYTHON_TOOL_BRANCH: ""
+
+cache:
+  # pull only for most of the use cases since it's cache dir.
+  # Only set "push" policy for "upload_cache" stage jobs
+  - key: pip-cache
+    paths:
+      - .cache/pip
+    policy: pull
+  - key: submodule-cache
+    paths:
+      - .cache/submodule_archives
+    policy: pull
 
 .setup_tools_unless_target_test: &setup_tools_unless_target_test |
   if [[ -n "$IDF_DONT_USE_MIRRORS" ]]; then
@@ -78,45 +128,139 @@ variables:
     fi
   fi
 
+.download_test_python_contraint_file: &download_test_python_contraint_file |
+  if [[ -n "$CI_PYTHON_CONSTRAINT_BRANCH" ]]; then
+    wget -O /tmp/constraint.txt --header="Authorization:Bearer ${ESPCI_TOKEN}" ${GITLAB_HTTP_SERVER}/api/v4/projects/2581/repository/files/${CI_PYTHON_CONSTRAINT_FILE}/raw?ref=${CI_PYTHON_CONSTRAINT_BRANCH}
+    mkdir -p ~/.espressif
+    mv /tmp/constraint.txt ~/.espressif/${CI_PYTHON_CONSTRAINT_FILE}
+  fi
+
+.download_test_python_tool: &download_test_python_tool |
+  if [[ -n "$CI_PYTHON_TOOL_REPO" ]]; then
+    git clone --quiet --depth=1 -b ${CI_PYTHON_TOOL_BRANCH} https://gitlab-ci-token:${ESPCI_TOKEN}@${GITLAB_HTTPS_HOST}/espressif/${CI_PYTHON_TOOL_REPO}.git
+    pip install ./${CI_PYTHON_TOOL_REPO}
+    rm -rf ${CI_PYTHON_TOOL_REPO}
+  fi
+
+.set_include_nightly_run: &set_include_nightly_run |
+  if echo "$CI_MERGE_REQUEST_LABELS" | egrep "^([^,\n\r]+,)*include_nightly_run(,[^,\n\r]+)*$"; then
+    export INCLUDE_NIGHTLY_RUN="1"
+  fi
+
 before_script:
   - source tools/ci/utils.sh
+  - is_based_on_commits $REQUIRED_ANCESTOR_COMMITS
+  - *set_include_nightly_run
   - source tools/ci/setup_python.sh
   - add_gitlab_ssh_keys
   - source tools/ci/configure_ci_environment.sh
   - *setup_tools_unless_target_test
   - fetch_submodules
+  - *download_test_python_contraint_file
+  - $IDF_PATH/tools/idf_tools.py install-python-env
+  # install esptool globally, TODO: remove this, IDFCI-1207
+  - pip install esptool -c ~/.espressif/${CI_PYTHON_CONSTRAINT_FILE}
+  - *download_test_python_tool
 
 # used for check scripts which we want to run unconditionally
 .before_script_no_sync_submodule:
   before_script:
     - echo "Not setting up GitLab key, not fetching submodules"
     - source tools/ci/utils.sh
+    - is_based_on_commits $REQUIRED_ANCESTOR_COMMITS
+    - *set_include_nightly_run
     - source tools/ci/setup_python.sh
     - source tools/ci/configure_ci_environment.sh
+    - *download_test_python_contraint_file
+    - $IDF_PATH/tools/idf_tools.py install-python-env
 
 .before_script_minimal:
   before_script:
     - echo "Only load utils.sh"
     - source tools/ci/utils.sh
+    - is_based_on_commits $REQUIRED_ANCESTOR_COMMITS
+    - *set_include_nightly_run
+
+.before_script_integration_test:
+  before_script:
+    # need to install esptool, needn't install idf python env
+  - source tools/ci/utils.sh
+  - is_based_on_commits $REQUIRED_ANCESTOR_COMMITS
+  - *set_include_nightly_run
+  - source tools/ci/setup_python.sh
+  - add_gitlab_ssh_keys
+  - fetch_submodules
+  # download the constraint file manually
+  - mkdir -p ~/.espressif
+  - wget -O ~/.espressif/${CI_PYTHON_CONSTRAINT_FILE} --header="Authorization:Bearer ${ESPCI_TOKEN}" ${GITLAB_HTTP_SERVER}/api/v4/projects/2581/repository/files/${CI_PYTHON_CONSTRAINT_FILE}/raw
+  - *download_test_python_contraint_file
+  # TODO: remove this, IDFCI-1207
+  - pip install esptool -c ~/.espressif/${CI_PYTHON_CONSTRAINT_FILE}
+  - *download_test_python_tool
 
 .before_script_macos:
   before_script:
     - source tools/ci/utils.sh
+    - is_based_on_commits $REQUIRED_ANCESTOR_COMMITS
+    - *set_include_nightly_run
     - export IDF_TOOLS_PATH="${HOME}/.espressif_runner_${CI_RUNNER_ID}_${CI_CONCURRENT_ID}"
+    - *download_test_python_contraint_file
     - $IDF_PATH/tools/idf_tools.py install-python-env
     # On macOS, these tools need to be installed
     - $IDF_PATH/tools/idf_tools.py --non-interactive install cmake ninja
     # This adds tools (compilers) and the version-specific Python environment to PATH
     - *setup_tools_unless_target_test
     # Install packages required by CI scripts into IDF Python environment
-    - pip install -r $IDF_PATH/tools/ci/python_packages/ttfw_idf/requirements.txt
+    - pip install -r $IDF_PATH/tools/requirements/requirements.ci.txt
     - source tools/ci/configure_ci_environment.sh
     # Part of tools/ci/setup_python.sh; we don't use pyenv on macOS, so can't run the rest of the script.
     - export PYTHONPATH="$IDF_PATH/tools:$IDF_PATH/tools/ci/python_packages:$PYTHONPATH"
     - fetch_submodules
 
+.before_script_pytest:
+  before_script:
+    - source tools/ci/utils.sh
+    - is_based_on_commits $REQUIRED_ANCESTOR_COMMITS
+    - *set_include_nightly_run
+    - source tools/ci/setup_python.sh
+    - add_gitlab_ssh_keys
+    - source tools/ci/configure_ci_environment.sh
+    - *setup_tools_unless_target_test
+    - fetch_submodules
+    - *download_test_python_contraint_file
+    # only need ci and pytest related packages, ttfw requirements would be mocked if not installed
+    - $IDF_PATH/tools/idf_tools.py install-python-env --features pytest,ci
+    - eval "$($IDF_PATH/tools/idf_tools.py export)"  # use idf venv instead
+
+.before_script_build_jobs:
+  before_script:
+    - source tools/ci/utils.sh
+    - is_based_on_commits $REQUIRED_ANCESTOR_COMMITS
+    - *set_include_nightly_run
+    - source tools/ci/setup_python.sh
+    - add_gitlab_ssh_keys
+    - source tools/ci/configure_ci_environment.sh
+    - *setup_tools_unless_target_test
+    - fetch_submodules
+    - *download_test_python_contraint_file
+    # only need ci and pytest related packages, ttfw requirements would be mocked if not installed
+    - $IDF_PATH/tools/idf_tools.py install-python-env --features pytest,ci
+    - eval "$($IDF_PATH/tools/idf_tools.py export)"  # use idf venv instead
+    - export EXTRA_CFLAGS=${PEDANTIC_CFLAGS}
+    - export EXTRA_CXXFLAGS=${PEDANTIC_CXXFLAGS}
+
+default:
+  retry:
+    max: 2
+    when:
+      # In case of a runner failure we could hop to another one, or a network error could go away.
+      - runner_system_failure
+      # Job execution timeout may be caused by a network issue.
+      - job_execution_timeout
+
 include:
   - '.gitlab/ci/rules.yml'
+  - '.gitlab/ci/upload_cache.yml'
   - '.gitlab/ci/docs.yml'
   - '.gitlab/ci/static-code-analysis.yml'
   - '.gitlab/ci/pre_check.yml'

.gitlab/CODEOWNERS

@@ -47,15 +47,21 @@
 * @esp-idf-codeowners/other
 
 /.*                                   @esp-idf-codeowners/tools
+/.github/workflows/                   @esp-idf-codeowners/ci
 /.gitlab-ci.yml                       @esp-idf-codeowners/ci
 /.gitlab/ci/                          @esp-idf-codeowners/ci
 /.pre-commit-config.yaml              @esp-idf-codeowners/ci
 /.readthedocs.yml                     @esp-idf-codeowners/docs
 /CMakeLists.txt                       @esp-idf-codeowners/build-config
+/CONTRIBUTING.md                      @esp-idf-codeowners/docs
 /Kconfig                              @esp-idf-codeowners/build-config
+/README*.md                           @esp-idf-codeowners/docs
+/SUPPORT_POLICY*.md                   @esp-idf-codeowners/docs
 /add_path.sh                          @esp-idf-codeowners/tools
+/conftest.py                          @esp-idf-codeowners/ci
 /export.*                             @esp-idf-codeowners/tools
 /install.*                            @esp-idf-codeowners/tools
+/pytest.ini                           @esp-idf-codeowners/ci
 /sdkconfig.rename                     @esp-idf-codeowners/build-config
 /sonar-project.properties             @esp-idf-codeowners/ci
 
@@ -63,18 +69,15 @@
 
 /components/app_trace/                @esp-idf-codeowners/tools
 /components/app_update/               @esp-idf-codeowners/system @esp-idf-codeowners/app-utilities
-/components/asio/                     @esp-idf-codeowners/network
 /components/bootloader*/              @esp-idf-codeowners/system @esp-idf-codeowners/security
+/components/bootloader_support/bootloader_flash/    @esp-idf-codeowners/peripherals
 /components/bt/                       @esp-idf-codeowners/bluetooth
-/components/cbor/                     @esp-idf-codeowners/app-utilities
 /components/cmock/                    @esp-idf-codeowners/system
-/components/coap/                     @esp-idf-codeowners/app-utilities
-/components/console/                  @esp-idf-codeowners/system @esp-idf-codeowners/app-utilities
+/components/console/                  @esp-idf-codeowners/system @esp-idf-codeowners/app-utilities/console
 /components/cxx/                      @esp-idf-codeowners/system
 /components/driver/                   @esp-idf-codeowners/peripherals
 /components/efuse/                    @esp-idf-codeowners/system
-/components/esp32*/                   @esp-idf-codeowners/system
-/components/esp_adc_cal/              @esp-idf-codeowners/peripherals
+/components/esp_adc/                  @esp-idf-codeowners/peripherals
 /components/esp_common/               @esp-idf-codeowners/system
 /components/esp_eth/                  @esp-idf-codeowners/network
 /components/esp_event/                @esp-idf-codeowners/system
@@ -84,55 +87,50 @@
 /components/esp_http_server/          @esp-idf-codeowners/app-utilities
 /components/esp_https_ota/            @esp-idf-codeowners/app-utilities
 /components/esp_https_server/         @esp-idf-codeowners/app-utilities
-/components/esp_hw_support/           @esp-idf-codeowners/system
-/components/esp_ipc/                  @esp-idf-codeowners/system
+/components/esp_hw_support/           @esp-idf-codeowners/system @esp-idf-codeowners/peripherals
 /components/esp_lcd/                  @esp-idf-codeowners/peripherals
 /components/esp_local_ctrl/           @esp-idf-codeowners/app-utilities
 /components/esp_netif/                @esp-idf-codeowners/network
 /components/esp_phy/                  @esp-idf-codeowners/bluetooth @esp-idf-codeowners/wifi @esp-idf-codeowners/ieee802154
 /components/esp_pm/                   @esp-idf-codeowners/power-management @esp-idf-codeowners/bluetooth @esp-idf-codeowners/wifi
+/components/esp_psram/                @esp-idf-codeowners/peripherals @esp-idf-codeowners/system
 /components/esp_ringbuf/              @esp-idf-codeowners/system
 /components/esp_rom/                  @esp-idf-codeowners/system @esp-idf-codeowners/bluetooth @esp-idf-codeowners/wifi
 /components/esp_serial_slave_link/    @esp-idf-codeowners/peripherals
 /components/esp_system/               @esp-idf-codeowners/system
 /components/esp_timer/                @esp-idf-codeowners/system
 /components/esp-tls/                  @esp-idf-codeowners/app-utilities
-/components/esp_websocket_client/     @esp-idf-codeowners/network
 /components/esp_wifi/                 @esp-idf-codeowners/wifi
 /components/espcoredump/              @esp-idf-codeowners/tools
 /components/esptool_py/               @esp-idf-codeowners/tools
-/components/expat/                    @esp-idf-codeowners/app-utilities
 /components/fatfs/                    @esp-idf-codeowners/storage
-/components/freemodbus/               @esp-idf-codeowners/peripherals
+/components/fatfs/**/*.py             @esp-idf-codeowners/tools
 /components/freertos/                 @esp-idf-codeowners/system
 /components/hal/                      @esp-idf-codeowners/peripherals
 /components/heap/                     @esp-idf-codeowners/system
+/components/http_parser/              @esp-idf-codeowners/app-utilities
 /components/idf_test/                 @esp-idf-codeowners/ci
 /components/ieee802154/               @esp-idf-codeowners/ieee802154
-/components/jsmn/                     @esp-idf-codeowners/app-utilities
 /components/json/                     @esp-idf-codeowners/app-utilities
-/components/libsodium/                @esp-idf-codeowners/security
+/components/linux/                    @esp-idf-codeowners/system
 /components/log/                      @esp-idf-codeowners/system
 /components/lwip/                     @esp-idf-codeowners/lwip
-/components/mbedtls/                  @esp-idf-codeowners/app-utilities @esp-idf-codeowners/security
-/components/mdns/                     @esp-idf-codeowners/network
+/components/mbedtls/                  @esp-idf-codeowners/app-utilities/mbedtls @esp-idf-codeowners/security
 /components/mqtt/                     @esp-idf-codeowners/network
 /components/newlib/                   @esp-idf-codeowners/system @esp-idf-codeowners/tools
-/components/nghttp/                   @esp-idf-codeowners/app-utilities
 /components/nvs_flash/                @esp-idf-codeowners/storage
-/components/openssl/                  @esp-idf-codeowners/network
 /components/openthread/               @esp-idf-codeowners/ieee802154
 /components/partition_table/          @esp-idf-codeowners/system
 /components/perfmon/                  @esp-idf-codeowners/tools
 /components/protobuf-c/               @esp-idf-codeowners/app-utilities
-/components/protocomm/                @esp-idf-codeowners/app-utilities
+/components/protocomm/                @esp-idf-codeowners/app-utilities/provisioning
 /components/pthread/                  @esp-idf-codeowners/system
+/components/riscv/                    @esp-idf-codeowners/system
 /components/sdmmc/                    @esp-idf-codeowners/storage
 /components/soc/                      @esp-idf-codeowners/peripherals @esp-idf-codeowners/system
 /components/spi_flash/                @esp-idf-codeowners/peripherals
 /components/spiffs/                   @esp-idf-codeowners/storage
 /components/tcp_transport/            @esp-idf-codeowners/network
-/components/tcpip_adapter/            @esp-idf-codeowners/network
 /components/tinyusb/                  @esp-idf-codeowners/peripherals/usb
 /components/touch_element/            @esp-idf-codeowners/peripherals
 /components/ulp/                      @esp-idf-codeowners/system
@@ -140,7 +138,7 @@
 /components/usb/                      @esp-idf-codeowners/peripherals/usb
 /components/vfs/                      @esp-idf-codeowners/storage
 /components/wear_levelling/           @esp-idf-codeowners/storage
-/components/wifi_provisioning/        @esp-idf-codeowners/app-utilities
+/components/wifi_provisioning/        @esp-idf-codeowners/app-utilities/provisioning
 /components/wpa_supplicant/           @esp-idf-codeowners/wifi
 /components/xtensa/                   @esp-idf-codeowners/system
 
@@ -149,14 +147,18 @@
 /docs/**/api-reference/network/         @esp-idf-codeowners/network
 /docs/**/api-reference/peripherals/     @esp-idf-codeowners/peripherals
 /docs/**/api-reference/protocols/       @esp-idf-codeowners/network @esp-idf-codeowners/app-utilities
-/docs/**/api-reference/provisioning/    @esp-idf-codeowners/app-utilities
+/docs/**/api-reference/provisioning/    @esp-idf-codeowners/app-utilities/provisioning
 /docs/**/api-reference/storage/         @esp-idf-codeowners/storage
 /docs/**/api-reference/system/          @esp-idf-codeowners/system
 /docs/**/security/                      @esp-idf-codeowners/security
+/docs/**/migration-guides/              @esp-idf-codeowners/docs @esp-idf-codeowners/all-maintainers
 
+/examples/README.md           @esp-idf-codeowners/docs @esp-idf-codeowners/ci
+/examples/**/*.py             @esp-idf-codeowners/ci @esp-idf-codeowners/tools
 /examples/bluetooth/          @esp-idf-codeowners/bluetooth
 /examples/build_system/       @esp-idf-codeowners/build-config
 /examples/common_components/  @esp-idf-codeowners/system
+/examples/custom_bootloader/  @esp-idf-codeowners/system
 /examples/cxx/                @esp-idf-codeowners/system
 /examples/ethernet/           @esp-idf-codeowners/network
 /examples/get-started/        @esp-idf-codeowners/system
@@ -164,30 +166,28 @@
 /examples/network/            @esp-idf-codeowners/network @esp-idf-codeowners/wifi
 /examples/openthread/         @esp-idf-codeowners/ieee802154
 /examples/peripherals/        @esp-idf-codeowners/peripherals
+/examples/peripherals/usb/    @esp-idf-codeowners/peripherals @esp-idf-codeowners/peripherals/usb
 /examples/protocols/          @esp-idf-codeowners/network @esp-idf-codeowners/app-utilities
-/examples/provisioning/       @esp-idf-codeowners/app-utilities
+/examples/provisioning/       @esp-idf-codeowners/app-utilities/provisioning
 /examples/security/           @esp-idf-codeowners/security
 /examples/storage/            @esp-idf-codeowners/storage
 /examples/system/             @esp-idf-codeowners/system
+/examples/system/ota/         @esp-idf-codeowners/app-utilities
 /examples/wifi/               @esp-idf-codeowners/wifi
-
-/examples/**/*.py             @esp-idf-codeowners/ci @esp-idf-codeowners/tools
-
-/make/    @esp-idf-codeowners/build-config
+/examples/zigbee/             @esp-idf-codeowners/ieee802154
 
 /tools/                       @esp-idf-codeowners/tools
-/tools/*_apps.py              @esp-idf-codeowners/ci
 /tools/ble/                   @esp-idf-codeowners/app-utilities
 /tools/catch/                 @esp-idf-codeowners/ci
 /tools/ci/                    @esp-idf-codeowners/ci
 /tools/cmake/                 @esp-idf-codeowners/build-config
 /tools/esp_prov/              @esp-idf-codeowners/app-utilities
-/tools/find_build_apps/       @esp-idf-codeowners/ci
 /tools/idf_size_yaml/         @esp-idf-codeowners/peripherals
 /tools/kconfig*/              @esp-idf-codeowners/build-config
 /tools/ldgen/                 @esp-idf-codeowners/build-config
 /tools/mass_mfg/              @esp-idf-codeowners/app-utilities
 /tools/mocks/                 @esp-idf-codeowners/system
+/tools/test_apps/README.md    @esp-idf-codeowners/docs @esp-idf-codeowners/ci
 
 ## Note: owners here should be the same as the owners for the same example subdir, above
 /tools/test_apps/build_system/       @esp-idf-codeowners/build-config
@@ -199,13 +199,11 @@
 
 /tools/unit-test-app/         @esp-idf-codeowners/system @esp-idf-codeowners/tools
 
-requirements.txt              @esp-idf-codeowners/tools
-
 # sort-order-reset
 
+/components/**/test_apps/**/*.py     @esp-idf-codeowners/ci @esp-idf-codeowners/tools
+
 # ignore lists
-/tools/ci/check_copyright_ignore.txt                    @esp-idf-codeowners/tools
-/tools/ci/check_copyright_permanent_ignore.txt          @esp-idf-codeowners/all-maintainers
-/tools/ci/check_examples_cmake_make-cmake_ignore.txt    @esp-idf-codeowners/tools
-/tools/ci/check_examples_cmake_make-make_ignore.txt     @esp-idf-codeowners/tools
+/tools/ci/check_copyright_config.yaml                   @esp-idf-codeowners/all-maintainers
+/tools/ci/check_copyright_ignore.txt                    @esp-idf-codeowners/all-maintainers
 /tools/ci/mypy_ignore_list.txt                          @esp-idf-codeowners/tools

.gitlab/ci/README.md

@@ -19,6 +19,13 @@
     - [Functions](#functions)
       - [CI Job Related](#ci-job-related)
       - [Shell Script Related](#shell-script-related)
+  - [Manifest File to Control the Build/Test apps](#manifest-file-to-control-the-buildtest-apps)
+    - [Grammar](#grammar)
+      - [Operands](#operands)
+      - [Operators](#operators)
+      - [Limitation:](#limitation)
+    - [How does it work?](#how-does-it-work)
+    - [Example](#example)
 
 ## General Workflow
 
@@ -45,6 +52,7 @@
 - `custom_test[_esp32/esp32s2/...]`
 - `docker`
 - `docs`
+- `docs_fast`, triggers a fast docs build, not a full build which is the CI default. This skips PDF build as well as doxygen APIs, reducing the build time by 90+%.
 - `example_test[_esp32/esp32s2/...]`
 - `fuzzer_test`
 - `host_test`
@@ -223,3 +231,93 @@ To run these commands in shell script locally, place `source tools/ci/utils.sh`
 - `info`: log in green color
 - `run_cmd`: run the command with duration seconds info
 - `retry_failed`: run the command with duration seconds info, retry when failed
+
+## Manifest File to Control the Build/Test apps
+
+`.build-test-rules.yml` file is a manifest file to control if the CI is running the build and test job or not. The Supported Targets table in `README.md` for apps would be auto-generated by `pre-commit` from the app's `.build-test-rules.yml`.
+
+### Grammar
+
+#### Operands
+
+- Variables start with `SOC_`. The value would be parsed from components/soc/[TARGET]/include/soc/*_caps.h
+- `IDF_TARGET`
+- `INCLUDE_DEFAULT` (The default value of officially supported targets is 1, otherwise is 0)
+- String, must be double-quoted. e.g. `"esp32"`, `"12345"`
+- Integer, support decimal and hex. e.g. `1`, `0xAB`
+- List with String and Integer inside, the type could be mixed. e.g. `["esp32", 1]`
+
+#### Operators
+
+- `==`, `!=`, `>`, `>=`, `<`, `<=`
+- `and`, `or`
+- `in`, `not in` with list
+- parentheses
+
+#### Limitation:
+
+- all operators are binary operator. For more than two operands, you may use nested parentheses trick. For example,
+  - `A == 1 or (B == 2 and C in [1,2,3])`
+  - `(A == 1 and B == 2) or (C not in ["3", "4", 5])`
+
+### How does it work?
+
+By default, we enable build and test jobs for supported targets. In other words, if an app supports all supported targets, it does not need to be added in a manifest file. The manifest files are files that set the violation rules for apps.
+
+three rules (disable rules are calculated after the `enable` rule):
+- enable: run CI build/test jobs for targets that match any of the specified conditions only
+- disable: will not run CI build/test jobs for targets that match any of the specified conditions
+- disable_test: will not run CI test jobs for targets that match any of the specified conditions
+
+Each key is a test folder. Will apply to all folders inside.
+
+If one sub folder is in a special case, you can overwrite the rules for this folder by adding another entry for this folder itself. Each folder's rules are standalone, and will not inherit its parent's rules. (YAML inheritance is too complicated for reading)
+
+For example in the following codeblock, only `disable` rule exists in `examples/foo/bar`. It's unaware of its parent's `enable` rule. 
+
+```yaml
+examples/foo:
+  enable:
+    - if: IDF_TARGET == "esp32"
+
+examples/foo/bar:
+  disable:
+    - if: IDF_TARGET == "esp32s2"
+```
+
+### Example
+
+```yaml
+examples/foo:
+  enable:
+    - if IDF_TARGET in ["esp32", 1, 2, 3]
+    - if IDF_TARGET not in ["4", "5", 6]
+  # should be run under all targets!
+
+examples/bluetooth:
+  disable:  # disable both build and tests jobs
+    - if: SOC_BT_SUPPORTED != 1
+    # reason is optional if there's no `temporary: true`
+  disable_test:
+    - if: IDF_TARGET == "esp32"
+      temporary: true
+      reason: lack of ci runners  # required when `temporary: true`
+
+examples/bluetooth/test_foo:
+  # each folder's settings are standalone
+  disable:
+    - if: IDF_TARGET == "esp32s2"
+      temporary: true
+      reason: no idea
+  # unlike examples/bluetooth, the apps under this folder would not be build nor test for "no idea" under target esp32s2
+
+examples/get-started/hello_world:
+  enable:
+    - if: IDF_TARGET == "linux"
+      reason: this one only supports linux!
+
+examples/get-started/blink:
+  enable:
+    - if: INCLUDE_DEFAULT == 1 or IDF_TARGET == "linux"
+      reason: This one supports all supported targets and linux
+```

.gitlab/ci/assign-test.yml

@@ -1,88 +1,109 @@
-assign_test:
-  extends: .rules:test:any_test
+.assign_test_template:
+  image: $TARGET_TEST_ENV_IMAGE
+  stage: assign_test
   tags:
     - assign_test
-  image: $CI_DOCKER_REGISTRY/ubuntu-test-env$BOT_DOCKER_IMAGE_TAG
-  stage: assign_test
-  # gitlab ci do not support match job with RegEx or wildcard now in dependencies.
-  # we have a lot build example jobs. now we don't use dependencies, just download all artifacts of build stage.
-  dependencies: # Here is not a hard dependency relationship, could be skipped. so we do not use "needs" here.
-    - build_ssc_esp32
-    - build_esp_idf_tests_cmake_esp32
-    - build_esp_idf_tests_cmake_esp32s2
-    - build_esp_idf_tests_cmake_esp32s3
-    - build_esp_idf_tests_cmake_esp32c3
   variables:
-    SUBMODULES_TO_FETCH: "components/esptool_py/esptool"
-    EXAMPLE_TEST_DIR: "${CI_PROJECT_DIR}/examples"
-    CUSTOM_TEST_DIR: "${CI_PROJECT_DIR}/tools/test_apps"
-    UNIT_TEST_DIR: "${CI_PROJECT_DIR}/components/idf_test/unit_test"
-    # COMPONENT_UT_DIRS is set by `set_component_ut_vars` in `utils.sh`
-    COMPONENT_UT_OUTPUT_DIR: "${CI_PROJECT_DIR}/component_ut"
-    INTEGRATION_CONFIG_OUTPUT_PATH: "${CI_PROJECT_DIR}/components/idf_test/integration_test/CIConfigs"
+    SUBMODULES_TO_FETCH: "none"
+  artifacts:
+    paths:
+      - ${TEST_DIR}/test_configs/
+      - artifact_index.json
+    when: always
+    expire_in: 1 week
+  script:
+    - run_cmd python tools/ci/python_packages/ttfw_idf/IDFAssignTest.py $TEST_TYPE $TEST_DIR -c $CI_TARGET_TEST_CONFIG_FILE -o $TEST_DIR/test_configs
+
+assign_example_test:
+  extends:
+    - .assign_test_template
+    - .rules:build:example_test
+  needs:
+    - job: build_examples_cmake_esp32
+      artifacts: false
+      optional: true
+    - job: build_examples_cmake_esp32s2
+      artifacts: false
+      optional: true
+    - job: build_examples_cmake_esp32c2
+      artifacts: false
+      optional: true
+    - job: build_examples_cmake_esp32c3
+      artifacts: false
+      optional: true
+    - job: build_examples_cmake_esp32s3
+      artifacts: false
+      optional: true
+  variables:
+    TEST_TYPE: example_test
+    TEST_DIR: examples
+
+assign_custom_test:
+  extends:
+    - .assign_test_template
+    - .rules:build:custom_test
+  needs:
+    - job: build_test_apps_esp32
+      artifacts: false
+      optional: true
+    - job: build_test_apps_esp32s2
+      artifacts: false
+      optional: true
+    - job: build_test_apps_esp32c2
+      artifacts: false
+      optional: true
+    - job: build_test_apps_esp32c3
+      artifacts: false
+      optional: true
+    - job: build_test_apps_esp32s3
+      artifacts: false
+      optional: true
+  variables:
+    TEST_TYPE: custom_test
+    TEST_DIR: tools/test_apps
+
+assign_unit_test:
+  extends:
+    - .assign_test_template
+    - .rules:build:unit_test
+  needs:
+    - job: build_esp_idf_tests_cmake_esp32
+      optional: true
+    - job: build_esp_idf_tests_cmake_esp32s2
+      optional: true
+    - job: build_esp_idf_tests_cmake_esp32c2
+      optional: true
+    - job: build_esp_idf_tests_cmake_esp32c3
+      optional: true
+    - job: build_esp_idf_tests_cmake_esp32s3
+      optional: true
+  variables:
+    TEST_TYPE: unit_test
+    TEST_DIR: components/idf_test/unit_test
+
+assign_integration_test:
+  extends:
+    - .assign_test_template
+    - .rules:test:integration_test
+  needs:
+    - build_ssc_esp32
+    - build_ssc_esp32c3
+  artifacts:
+    paths:
+      - $TEST_DIR/test_configs
+    expire_in: 1 week
+  variables:
+    TEST_DIR: ${CI_PROJECT_DIR}/tools/ci/integration_test
+    BUILD_DIR: ${CI_PROJECT_DIR}/SSC/ssc_bin
     INTEGRATION_TEST_CASE_PATH: "${CI_PROJECT_DIR}/auto_test_script/TestCaseFiles"
     ASSIGN_TEST_CASE_SCRIPT: "${CI_PROJECT_DIR}/auto_test_script/bin/CIAssignTestCases.py"
     PYTHONPATH: ${CI_PROJECT_DIR}/auto_test_script/packages
-    # auto_test_script only supports python 3.7.x
     PYTHON_VER: 3.7.7
-  artifacts:
-    paths:
-      - components/idf_test/*/CIConfigs
-      - $EXAMPLE_TEST_DIR/test_configs
-      - $CUSTOM_TEST_DIR/test_configs
-      - $COMPONENT_UT_OUTPUT_DIR/test_configs
-      - build_examples/artifact_index.json
-      - build_test_apps/artifact_index.json
-      - build_component_ut/artifact_index.json
-      - tools/unit-test-app/builds/artifact_index.json
-    expire_in: 1 week
   script:
-    - set_component_ut_vars
-    - python tools/ci/python_packages/ttfw_idf/IDFAssignTest.py example_test $EXAMPLE_TEST_DIR -c $CI_TARGET_TEST_CONFIG_FILE -o $EXAMPLE_TEST_DIR/test_configs
-    - python tools/ci/python_packages/ttfw_idf/IDFAssignTest.py custom_test $CUSTOM_TEST_DIR -c $CI_TARGET_TEST_CONFIG_FILE -o $CUSTOM_TEST_DIR/test_configs
-    - python tools/ci/python_packages/ttfw_idf/IDFAssignTest.py component_ut $COMPONENT_UT_DIRS -c $CI_TARGET_TEST_CONFIG_FILE -o $COMPONENT_UT_OUTPUT_DIR/test_configs
-    - python tools/ci/python_packages/ttfw_idf/IDFAssignTest.py unit_test $UNIT_TEST_DIR -c $CI_TARGET_TEST_CONFIG_FILE -o $UNIT_TEST_DIR/CIConfigs
     # clone test script to assign tests
     # can not retry if downing git lfs files failed, so using empty_branch first.
     - retry_failed git clone ${CI_AUTO_TEST_SCRIPT_REPO_URL} -b empty_branch
     - retry_failed git -C auto_test_script checkout -f ${CI_AUTO_TEST_SCRIPT_REPO_BRANCH}
     - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script --customized_only
     # assign integration test cases
-    - python ${ASSIGN_TEST_CASE_SCRIPT} -t ${INTEGRATION_TEST_CASE_PATH} -c $CI_TARGET_TEST_CONFIG_FILE -b $IDF_PATH/SSC/ssc_bin -o $INTEGRATION_CONFIG_OUTPUT_PATH
-
-update_test_cases:
-  extends: .rules:ref:master-schedule
-  stage: assign_test
-  image: $CI_DOCKER_REGISTRY/ubuntu-test-env
-  tags:
-    - deploy_test
-  needs:
-    - build_esp_idf_tests_cmake_esp32
-    - build_esp_idf_tests_cmake_esp32s2
-    - build_esp_idf_tests_cmake_esp32s3
-    - build_esp_idf_tests_cmake_esp32c3
-  artifacts:
-    when: always
-    paths:
-      - ${CI_PROJECT_DIR}/test-management/*.log
-    expire_in: 1 week
-  variables:
-    SUBMODULES_TO_FETCH: "components/esptool_py/esptool"
-    UNIT_TEST_DIR: "${CI_PROJECT_DIR}/components/idf_test/unit_test"
-    BOT_ACCOUNT_CONFIG_FILE: "${CI_PROJECT_DIR}/test-management/Config/Account.local.yml"
-    AUTO_TEST_SCRIPT_PATH: "${CI_PROJECT_DIR}/auto_test_script"
-    PYTHON_VER: 3.7.7
-  script:
-    - export GIT_SHA=$(echo ${CI_COMMIT_SHA} | cut -c 1-8)
-    - retry_failed git clone $TEST_MANAGEMENT_REPO
-    - python $CHECKOUT_REF_SCRIPT test-management test-management
-    - cd test-management
-    - echo $BOT_JIRA_ACCOUNT > ${BOT_ACCOUNT_CONFIG_FILE}
-    # update unit test cases
-    - export UNIT_TEST_CASE_FILES=$(find $UNIT_TEST_DIR -maxdepth 1 -name "*.yml" | xargs)
-    - python ImportTestCase.py $JIRA_TEST_MANAGEMENT_PROJECT unity -d $UNIT_TEST_CASE_FILES -r $GIT_SHA
-    # update example test cases
-    - python ImportTestCase.py $JIRA_TEST_MANAGEMENT_PROJECT tiny_test_fw -d ${CI_PROJECT_DIR}/examples -r $GIT_SHA
-    - python ImportTestCase.py $JIRA_TEST_MANAGEMENT_PROJECT tiny_test_fw -d ${CI_PROJECT_DIR}/tools/test_apps -r $GIT_SHA
-    # organize test cases
-    - python OrganizeTestCases.py $JIRA_TEST_MANAGEMENT_PROJECT
+    - python ${ASSIGN_TEST_CASE_SCRIPT} -t ${INTEGRATION_TEST_CASE_PATH} -c $CI_TARGET_TEST_CONFIG_FILE -b ${BUILD_DIR} -o $TEST_DIR/test_configs

.gitlab/ci/build.yml

@@ -5,15 +5,192 @@
     - build
   variables:
     SIZE_INFO_LOCATION: "$CI_PROJECT_DIR/size_info.txt"
+    # Enable ccache for all build jobs. See configure_ci_environment.sh for more ccache related settings.
+    IDF_CCACHE_ENABLE: "1"
+  after_script:
+    # Show ccache statistics if enabled globally
+    - test "$CI_CCACHE_STATS" == 1 && test -n "$(which ccache)" && ccache --show-stats || true
   dependencies: []
 
+.build_pytest_template:
+  extends:
+    - .build_template
+    - .before_script_build_jobs
+  dependencies:  # set dependencies to null to avoid missing artifacts issue
+  needs:
+    - job: fast_template_app
+      artifacts: false
+  artifacts:
+    paths:
+      - "**/build*/size.json"
+      - "**/build*/build_log.txt"
+      - "**/build*/*.bin"
+      - "**/build*/*.elf"
+      - "**/build*/*.map"
+      - "**/build*/flasher_args.json"
+      - "**/build*/flash_project_args"
+      - "**/build*/config/sdkconfig.json"
+      - "**/build*/bootloader/*.bin"
+      - "**/build*/partition_table/*.bin"
+      - $SIZE_INFO_LOCATION
+    when: always
+    expire_in: 3 days
+  script:
+    # CI specific options start from "--collect-size-info xxx". could ignore when running locally
+    - run_cmd python tools/ci/ci_build_apps.py $TEST_DIR -v
+      -t $IDF_TARGET
+      --pytest-apps
+      --collect-size-info $SIZE_INFO_LOCATION
+      --parallel-count ${CI_NODE_TOTAL:-1}
+      --parallel-index ${CI_NODE_INDEX:-1}
+
+build_pytest_examples_esp32:
+  extends:
+    - .build_pytest_template
+    - .rules:build:example_test-esp32
+  parallel: 3
+  variables:
+    IDF_TARGET: esp32
+    TEST_DIR: examples
+
+build_pytest_examples_esp32s2:
+  extends:
+    - .build_pytest_template
+    - .rules:build:example_test-esp32s2
+  parallel: 3
+  variables:
+    IDF_TARGET: esp32s2
+    TEST_DIR: examples
+
+build_pytest_examples_esp32s3:
+  extends:
+    - .build_pytest_template
+    - .rules:build:example_test-esp32s3
+  parallel: 3
+  variables:
+    IDF_TARGET: esp32s3
+    TEST_DIR: examples
+
+build_pytest_examples_esp32c3:
+  extends:
+    - .build_pytest_template
+    - .rules:build:example_test-esp32c3
+  parallel: 3
+  variables:
+    IDF_TARGET: esp32c3
+    TEST_DIR: examples
+
+build_pytest_examples_esp32c2:
+  extends:
+    - .build_pytest_template
+    - .rules:build:example_test-esp32c2
+  variables:
+    IDF_TARGET: esp32c2
+    TEST_DIR: examples
+
+build_pytest_components_esp32:
+  extends:
+    - .build_pytest_template
+    - .rules:build:component_ut-esp32
+  parallel: 2
+  variables:
+    IDF_TARGET: esp32
+    TEST_DIR: components
+
+build_pytest_components_esp32s2:
+  extends:
+    - .build_pytest_template
+    - .rules:build:component_ut-esp32s2
+  variables:
+    IDF_TARGET: esp32s2
+    TEST_DIR: components
+
+build_pytest_components_esp32s3:
+  extends:
+    - .build_pytest_template
+    - .rules:build:component_ut-esp32s3
+  variables:
+    IDF_TARGET: esp32s3
+    TEST_DIR: components
+
+build_pytest_components_esp32c3:
+  extends:
+    - .build_pytest_template
+    - .rules:build:component_ut-esp32c3
+  variables:
+    IDF_TARGET: esp32c3
+    TEST_DIR: components
+
+build_pytest_components_esp32c2:
+  extends:
+    - .build_pytest_template
+    - .rules:build:component_ut-esp32c2
+  variables:
+    IDF_TARGET: esp32c2
+    TEST_DIR: components
+
+build_non_test_components_apps:
+  extends:
+    - .build_cmake_template
+    - .rules:build:component_ut
+  script:
+    - set_component_ut_vars
+    # CI specific options start from "--collect-size-info xxx". could ignore when running locally
+    - run_cmd python tools/ci/ci_build_apps.py $COMPONENT_UT_DIRS -v
+      -t all
+      --collect-size-info $SIZE_INFO_LOCATION
+      --collect-app-info list_job_${CI_NODE_INDEX:-1}.json
+      --parallel-count ${CI_NODE_TOTAL:-1}
+      --parallel-index ${CI_NODE_INDEX:-1}
+
+build_pytest_test_apps_esp32:
+  extends:
+    - .build_pytest_template
+    - .rules:build:custom_test-esp32
+  variables:
+    IDF_TARGET: esp32
+    TEST_DIR: tools/test_apps
+
+build_pytest_test_apps_esp32s2:
+  extends:
+    - .build_pytest_template
+    - .rules:build:custom_test-esp32s2
+  variables:
+    IDF_TARGET: esp32s2
+    TEST_DIR: tools/test_apps
+
+build_pytest_test_apps_esp32s3:
+  extends:
+    - .build_pytest_template
+    - .rules:build:custom_test-esp32s3
+  variables:
+    IDF_TARGET: esp32s3
+    TEST_DIR: tools/test_apps
+
+build_pytest_test_apps_esp32c3:
+  extends:
+    - .build_pytest_template
+    - .rules:build:custom_test-esp32c3
+  variables:
+    IDF_TARGET: esp32c3
+    TEST_DIR: tools/test_apps
+
+build_pytest_test_apps_esp32c2:
+  extends:
+    - .build_pytest_template
+    - .rules:build:custom_test-esp32c2
+  variables:
+    IDF_TARGET: esp32c2
+    TEST_DIR: tools/test_apps
+
 .build_template_app_template:
-  extends: .build_template
+  extends:
+    - .build_template
+    - .before_script_build_jobs
   variables:
     LOG_PATH: "${CI_PROJECT_DIR}/log_template_app"
     BUILD_PATH: "${CI_PROJECT_DIR}/build_template_app"
     BUILD_DIR: "@t/@w"
-    BUILD_LOG_MAKE: "${LOG_PATH}/make_@t_@w.txt"
     BUILD_LOG_CMAKE: "${LOG_PATH}/cmake_@t_@w.txt"
     BUILD_COMMAND_ARGS: ""
   artifacts:
@@ -22,6 +199,7 @@
       - log_template_app/*
       - size_info.txt
       - build_template_app/**/size.json
+    expire_in: 1 week
   script:
     # Set the variable for 'esp-idf-template' testing
     - ESP_IDF_TEMPLATE_GIT=${ESP_IDF_TEMPLATE_GIT:-"https://github.com/espressif/esp-idf-template.git"}
@@ -30,14 +208,8 @@
     # using on esp-idf. If it doesn't exist then just stick to the default branch
     - python $CHECKOUT_REF_SCRIPT esp-idf-template esp-idf-template
     - export PATH="$IDF_PATH/tools:$PATH"
-    - export EXTRA_CFLAGS=${PEDANTIC_CFLAGS}
-    - export EXTRA_CXXFLAGS=${PEDANTIC_CXXFLAGS}
     # Only do the default cmake build for each target, remaining part are done in the build_template_app job
     - tools/ci/build_template_app.sh ${BUILD_COMMAND_ARGS}
-    # Check if there are any stray printf/ets_printf references in WiFi libs
-    - cd components/esp_wifi/lib
-    - for dir in esp32 esp32s2; do test $(xtensa-esp32-elf-nm $dir/*.a | grep -w printf | wc -l) -eq 0; done;
-    - for dir in esp32 esp32s2; do test $(xtensa-esp32-elf-nm $dir/*.a | grep -w ets_printf | wc -l) -eq 0; done;
 
 # build-related-pre-check-jobs ------------------------------------------------
 # Build at least one project for each target at earliest stage to reduce build cost for obvious failing commits
@@ -76,41 +248,64 @@ build_ssc_esp32s2:
   variables:
     TARGET_NAME: "ESP32S2"
 
+build_ssc_esp32c2:
+  extends: .build_ssc_template
+  parallel: 2
+  variables:
+    TARGET_NAME: "ESP32C2"
+
 build_ssc_esp32c3:
   extends: .build_ssc_template
   parallel: 3
   variables:
     TARGET_NAME: "ESP32C3"
 
+build_ssc_esp32s3:
+  extends: .build_ssc_template
+  parallel: 3
+  variables:
+    TARGET_NAME: "ESP32S3"
+
 .build_esp_idf_tests_cmake_template:
-  extends: .build_template
+  extends:
+    - .build_template
+    - .before_script_build_jobs
   dependencies:  # set dependencies to null to avoid missing artifacts issue
   needs:
     - job: fast_template_app
       artifacts: false
-    - scan_tests
   artifacts:
     paths:
-      - tools/unit-test-app/output/${IDF_TARGET}
-      - tools/unit-test-app/builds/*.json
-      - tools/unit-test-app/builds/${IDF_TARGET}/*/size.json
-      - components/idf_test/unit_test/*.yml
-      - $LOG_PATH
+      - "**/build*/size.json"
+      - "**/build*/build_log.txt"
+      - "**/build*/*.bin"
+      - "**/build*/*.elf"
+      - "**/build*/*.map"
+      - "**/build*/flasher_args.json"
+      - "**/build*/flash_project_args"
+      - "**/build*/config/sdkconfig.json"
+      - "**/build*/sdkconfig"
+      - "**/build*/bootloader/*.bin"
+      - "**/build*/partition_table/*.bin"
+      - list_job_*.json
       - $SIZE_INFO_LOCATION
+      - components/idf_test/unit_test/*.yml
     when: always
     expire_in: 4 days
   variables:
-    LOG_PATH: "$CI_PROJECT_DIR/log_ut_cmake"
-    BUILD_PATH: ${CI_PROJECT_DIR}/tools/unit-test-app/builds
-    OUTPUT_PATH: ${CI_PROJECT_DIR}/tools/unit-test-app/output
-    BUILD_SYSTEM: "cmake"
-    TEST_TYPE: "unit_test"
-    PYTHON_VER: 3.6.13
     LDGEN_CHECK_MAPPING: 1
   script:
-    - ${IDF_PATH}/tools/ci/find_apps_build_apps.sh
-    - cd $CI_PROJECT_DIR/tools/unit-test-app
-    - python tools/UnitTestParser.py ${BUILD_PATH} ${CI_NODE_INDEX:-1}
+    # CI specific options start from "--collect-size-info xxx". could ignore when running locally
+    - run_cmd python tools/ci/ci_build_apps.py tools/unit-test-app -v
+      -t $IDF_TARGET
+      --config "configs/*="
+      --copy-sdkconfig
+      --preserve-all
+      --collect-size-info $SIZE_INFO_LOCATION
+      --collect-app-info list_job_${CI_NODE_INDEX:-1}.json
+      --parallel-count ${CI_NODE_TOTAL:-1}
+      --parallel-index ${CI_NODE_INDEX:-1}
+    - run_cmd python tools/unit-test-app/tools/UnitTestParser.py tools/unit-test-app ${CI_NODE_INDEX:-1}
 
 build_esp_idf_tests_cmake_esp32:
   extends:
@@ -135,6 +330,13 @@ build_esp_idf_tests_cmake_esp32s3:
   variables:
     IDF_TARGET: esp32s3
 
+build_esp_idf_tests_cmake_esp32c2:
+  extends:
+    - .build_esp_idf_tests_cmake_template
+    - .rules:build:unit_test-esp32c2
+  variables:
+    IDF_TARGET: esp32c2
+
 build_esp_idf_tests_cmake_esp32c3:
   extends:
     - .build_esp_idf_tests_cmake_template
@@ -142,175 +344,143 @@ build_esp_idf_tests_cmake_esp32c3:
   variables:
     IDF_TARGET: esp32c3
 
-.build_examples_template:
-  extends: .build_template
-  dependencies:  # set dependencies to null to avoid missing artifacts issue
+.build_cmake_template:
+  extends:
+    - .build_template
+    - .before_script_build_jobs
+  dependencies: # set dependencies to null to avoid missing artifacts issue
   needs:
     - job: fast_template_app
       artifacts: false
-    - scan_tests
   variables:
-    TEST_PREFIX: examples
-    TEST_RELATIVE_DIR: examples
-    SCAN_TEST_JSON: ${CI_PROJECT_DIR}/${TEST_RELATIVE_DIR}/test_configs/scan_${IDF_TARGET}_${BUILD_SYSTEM}.json
-    TEST_TYPE: example_test
-    LOG_PATH: ${CI_PROJECT_DIR}/log_${TEST_PREFIX}
-    BUILD_PATH: ${CI_PROJECT_DIR}/build_${TEST_PREFIX}
-    PYTHON_VER: 3.6.13
     LDGEN_CHECK_MAPPING: 1
+  artifacts:
+    paths:
+      - "**/build*/size.json"
+      - "**/build*/build_log.txt"
+      - "**/build*/*.bin"
+      - "**/build*/*.elf"
+      - "**/build*/*.map"
+      - "**/build*/flasher_args.json"
+      - "**/build*/flash_project_args"
+      - "**/build*/config/sdkconfig.json"
+      - "**/build*/sdkconfig"
+      - "**/build*/bootloader/*.bin"
+      - "**/build*/partition_table/*.bin"
+      - list_job_*.json
+      - $SIZE_INFO_LOCATION
+    when: always
+    expire_in: 4 days
   script:
-    # it's not possible to build 100% out-of-tree and have the "artifacts"
-    # mechanism work, but this is the next best thing
-    - ${IDF_PATH}/tools/ci/find_apps_build_apps.sh
-
-build_examples_make:
-  extends:
-    - .build_examples_template
-    - .rules:build:example_test-esp32
-  stage: host_test
-  # This is a workaround for a rarely encountered issue with building examples in CI.
-  # Probably related to building of Kconfig in 'make clean' stage
-  retry: 1
-  parallel: 8
-  artifacts:
-    paths:
-      - $LOG_PATH
-      - build_${TEST_PREFIX}/*/*/*/build/size.json
-      - $SIZE_INFO_LOCATION
-    when: always
-    expire_in: 4 days
-  variables:
-    BUILD_SYSTEM: make
-    IDF_TARGET: esp32  # currently we only support esp32
-
-# same as above, but for CMake
-.build_examples_cmake_template:
-  extends: .build_examples_template
-  artifacts:
-    paths:
-      - build_${TEST_PREFIX}/list.json
-      - build_${TEST_PREFIX}/list_job_*.json
-      - build_${TEST_PREFIX}/*/*/*/sdkconfig
-      - build_${TEST_PREFIX}/*/*/*/build/size.json
-      - build_${TEST_PREFIX}/*/*/*/build/*.bin
-      - build_${TEST_PREFIX}/*/*/*/build/*.elf
-      - build_${TEST_PREFIX}/*/*/*/build/*.map
-      - build_${TEST_PREFIX}/*/*/*/build/flasher_args.json
-      - build_${TEST_PREFIX}/*/*/*/build/bootloader/*.bin
-      - build_${TEST_PREFIX}/*/*/*/build/partition_table/*.bin
-      - $LOG_PATH
-      - $SIZE_INFO_LOCATION
-    when: always
-    expire_in: 4 days
-  variables:
-    BUILD_SYSTEM: cmake
+    # CI specific options start from "--collect-size-info xxx". could ignore when running locally
+    - run_cmd python tools/ci/ci_build_apps.py $TEST_DIR -v
+      -t $IDF_TARGET
+      --copy-sdkconfig
+      --collect-size-info $SIZE_INFO_LOCATION
+      --collect-app-info list_job_${CI_NODE_INDEX:-1}.json
+      --parallel-count ${CI_NODE_TOTAL:-1}
+      --parallel-index ${CI_NODE_INDEX:-1}
+      --extra-preserve-dirs
+        examples/bluetooth/esp_ble_mesh/ble_mesh_console
+        examples/bluetooth/hci/controller_hci_uart_esp32
+        examples/wifi/iperf
 
 build_examples_cmake_esp32:
   extends:
-    - .build_examples_cmake_template
+    - .build_cmake_template
     - .rules:build:example_test-esp32
-  parallel: 10
+  parallel: 12
   variables:
     IDF_TARGET: esp32
+    TEST_DIR: examples
 
 build_examples_cmake_esp32s2:
   extends:
-    - .build_examples_cmake_template
+    - .build_cmake_template
     - .rules:build:example_test-esp32s2
   parallel: 8
   variables:
     IDF_TARGET: esp32s2
+    TEST_DIR: examples
 
 build_examples_cmake_esp32s3:
   extends:
-    - .build_examples_cmake_template
+    - .build_cmake_template
     - .rules:build:example_test-esp32s3
   parallel: 8
   variables:
     IDF_TARGET: esp32s3
+    TEST_DIR: examples
+
+build_examples_cmake_esp32c2:
+  extends:
+    - .build_cmake_template
+    - .rules:build:example_test-esp32c2
+  parallel: 8
+  variables:
+    IDF_TARGET: esp32c2
+    TEST_DIR: examples
 
 build_examples_cmake_esp32c3:
   extends:
-    - .build_examples_cmake_template
+    - .build_cmake_template
     - .rules:build:example_test-esp32c3
   parallel: 8
   variables:
     IDF_TARGET: esp32c3
+    TEST_DIR: examples
 
-.build_test_apps_template:
-  extends: .build_examples_cmake_template
+build_examples_cmake_esp32h2:
+  extends:
+    - .build_cmake_template
+    - .rules:build:example_test-esp32h2
   variables:
-    TEST_PREFIX: test_apps
-    TEST_RELATIVE_DIR: tools/test_apps
-    TEST_TYPE: custom_test
-  script:
-    - ${IDF_PATH}/tools/ci/find_apps_build_apps.sh
+    IDF_TARGET: esp32h2
+    TEST_DIR: examples
 
 build_test_apps_esp32:
   extends:
-    - .build_test_apps_template
+    - .build_cmake_template
     - .rules:build:custom_test-esp32
-  parallel: 8
+  parallel: 2
   variables:
     IDF_TARGET: esp32
+    TEST_DIR: tools/test_apps
 
 build_test_apps_esp32s2:
   extends:
-    - .build_test_apps_template
+    - .build_cmake_template
     - .rules:build:custom_test-esp32s2
-  parallel: 8
+  parallel: 2
   variables:
     IDF_TARGET: esp32s2
+    TEST_DIR: tools/test_apps
 
 build_test_apps_esp32s3:
   extends:
-    - .build_test_apps_template
+    - .build_cmake_template
     - .rules:build:custom_test-esp32s3
-  parallel: 8
+  parallel: 2
   variables:
     IDF_TARGET: esp32s3
+    TEST_DIR: tools/test_apps
 
 build_test_apps_esp32c3:
   extends:
-    - .build_test_apps_template
+    - .build_cmake_template
     - .rules:build:custom_test-esp32c3
-  parallel: 8
+  parallel: 2
   variables:
     IDF_TARGET: esp32c3
+    TEST_DIR: tools/test_apps
 
-.build_component_ut_template:
-  extends: .build_test_apps_template
-  variables:
-    TEST_PREFIX: component_ut
-    TEST_RELATIVE_DIR: component_ut
-
-build_component_ut_esp32:
+build_test_apps_esp32c2:
   extends:
-    - .build_component_ut_template
-    - .rules:build:component_ut-esp32
+    - .build_cmake_template
+    - .rules:build:custom_test-esp32c2
   variables:
-    IDF_TARGET: esp32
-
-build_component_ut_esp32s2:
-  extends:
-    - .build_component_ut_template
-    - .rules:build:component_ut-esp32s2
-  variables:
-    IDF_TARGET: esp32s2
-
-build_component_ut_esp32s3:
-  extends:
-    - .build_component_ut_template
-    - .rules:build:component_ut-esp32s3
-  variables:
-    IDF_TARGET: esp32s3
-
-build_component_ut_esp32c3:
-  extends:
-    - .build_component_ut_template
-    - .rules:build:component_ut-esp32c3
-  variables:
-    IDF_TARGET: esp32c3
+    IDF_TARGET: esp32c2
+    TEST_DIR: tools/test_apps
 
 .test_build_system_template:
   stage: host_test
@@ -327,11 +497,6 @@ build_component_ut_esp32c3:
     - cd test_build_system
     - ${IDF_PATH}/tools/ci/${SHELL_TEST_SCRIPT}
 
-test_build_system:
-  extends: .test_build_system_template
-  variables:
-    SHELL_TEST_SCRIPT: test_build_system.sh
-
 test_build_system_cmake:
   extends: .test_build_system_template
   variables:
@@ -347,6 +512,11 @@ test_build_system_cmake_macos:
   variables:
     SHELL_TEST_SCRIPT: test_build_system_cmake.sh
 
+test_build_system_spaces:
+  extends: .test_build_system_template
+  variables:
+    SHELL_TEST_SCRIPT: test_build_system_spaces.py
+
 build_docker:
   extends:
     - .before_script_minimal
@@ -361,8 +531,9 @@ build_docker:
   script:
     - export LOCAL_CI_REPOSITORY_URL=$CI_REPOSITORY_URL
     - if [ -n "$LOCAL_GITLAB_HTTPS_HOST" ]; then export LOCAL_CI_REPOSITORY_URL="https://gitlab-ci-token:${CI_JOB_TOKEN}@${LOCAL_GITLAB_HTTPS_HOST}/${CI_PROJECT_PATH}"; fi
+    - if [ -n "$LOCAL_GIT_MIRROR" ]; then export LOCAL_CI_REPOSITORY_URL="${LOCAL_GIT_MIRROR}/${CI_PROJECT_PATH}"; fi
     - echo "Using repository at $LOCAL_CI_REPOSITORY_URL"
-    - export DOCKER_BUILD_ARGS="--build-arg IDF_CLONE_URL=${LOCAL_CI_REPOSITORY_URL} --build-arg IDF_CLONE_BRANCH_OR_TAG=${CI_COMMIT_REF_NAME} --build-arg IDF_CHECKOUT_REF=${CI_COMMIT_TAG:-$CI_COMMIT_SHA}"
+    - export DOCKER_BUILD_ARGS="--build-arg IDF_CLONE_URL=${LOCAL_CI_REPOSITORY_URL} --build-arg IDF_CLONE_BRANCH_OR_TAG=${CI_COMMIT_REF_NAME} --build-arg IDF_CHECKOUT_REF=${CI_COMMIT_TAG:-$PIPELINE_COMMIT_SHA}"
     # Build
     - docker build --tag ${DOCKER_TMP_IMAGE_NAME} ${DOCKER_BUILD_ARGS} tools/docker/
     # We can't mount $PWD/examples/get-started/blink into the container, see https://gitlab.com/gitlab-org/gitlab-ce/issues/41227.
@@ -370,31 +541,6 @@ build_docker:
     # Therefore, build a copy of the example located inside the container.
     - docker run --rm --workdir /opt/esp/idf/examples/get-started/blink ${DOCKER_TMP_IMAGE_NAME} idf.py build
 
-.test-on-windows:
-  extends:
-    - .before_script_minimal
-    - .rules:build:windows
-  stage: host_test
-  needs: []
-  image: $CI_DOCKER_REGISTRY/esp32-toolchain-win-cross
-  tags:
-    - build
-  script:
-    - cd $TEST_DIR
-    - mkdir build
-    - cd build
-    - cmake -DCMAKE_TOOLCHAIN_FILE=../toolchain-i686-w64-mingw32.cmake -DCMAKE_BUILD_TYPE=Release ..
-    - cmake --build .
-
-build_idf_exe:
-  extends: .test-on-windows
-  artifacts:
-    paths:
-      - tools/windows/idf_exe/build/idf-exe-v*.zip
-    expire_in: 4 days
-  variables:
-    TEST_DIR: tools/windows/idf_exe
-
 # This job builds template app with permutations of targets and optimization levels
 build_template_app:
   extends:

.gitlab/ci/dependencies/dependencies.yml

@@ -3,6 +3,8 @@
   - esp32s2
   - esp32s3
   - esp32c3
+  - esp32h2
+  - esp32c2
 
 .target_test: &target_test
   - example_test
@@ -61,6 +63,10 @@
   patterns:
     - build_components
     - build_system
+    - build_target_test
+  included_in:
+    - "build:{0}"
+    - build:target_test
 
 build:integration_test:
   labels:
@@ -68,19 +74,48 @@ build:integration_test:
   patterns:
     - build_components
     - build_system
+  included_in:
+    - build:target_test
 
 # -------------
 # Special Cases
 # -------------
-"build:example_test-esp32c3":  # esp32c3 test is only run by label, but build jobs should always be triggered
+"build:{0}-esp32s3":
+  matrix:
+    - *target_test
   labels:
     - build
-    - example_test
+    - "{0}"
+    - target_test
   patterns:
     - build_components
     - build_system
-    - build-example_test
-    - example_test
+    - build_target_test
+    - "{0}"
+    - "build-{0}"
+  included_in:
+    - "build:{0}"
+    - build:target_test
+
+"build:component_ut-esp32s3":
+  matrix:
+    - *target_test
+  labels:
+    - build
+    - component_ut
+    - unit_test
+    - target_test
+  patterns:
+    - build_components
+    - build_system
+    - build_target_test
+    - component_ut
+    - unit_test
+    - build-component_ut
+    - build-unit_test
+  included_in:
+    - "build:{0}"
+    - build:target_test
 
 ####################
 # Target Test Jobs #
@@ -92,14 +127,14 @@ build:integration_test:
   labels: # For each rule, use labels <test_type> and <test_type>-<target>
     - "{0}"
     - "{0}_{1}"
+    - target_test
   patterns: # For each rule, use patterns <test_type> and build-<test_type>
     - "{0}"
     - "build-{0}"
   included_in: # Parent rules
+    - "build:{0}"
     - "build:{0}-{1}"
     - build:target_test
-    - test:target_test
-    - test:any_test
 
 # -------------
 # Special Cases
@@ -112,42 +147,109 @@ build:integration_test:
     - "component_ut_{0}"
     - unit_test
     - "unit_test_{0}"
+    - target_test
   patterns:
     - component_ut
     - "build-component_ut-{0}"
   included_in:
+    - build:component_ut
     - "build:component_ut-{0}"
     - build:target_test
-    - test:target_test
-    - test:any_test
 
-"test:example_test-esp32c3":  # For esp32c3 we trigger only with label
-
-"labels:example_test-esp32c3":
+# To reduce the specific runners' usage.
+# Do not create these jobs by default patterns on development branches
+# Can be triggered by labels or related changes
+"test:{0}-{1}-{2}":
+  matrix:
+    - *target_test
+    - *all_targets
+    - - bt        # example_test_005
+      - wifi      # example_test_002, example_test*wifi*
+      - ethernet  # example_test*ethernet*
+      - sdio      # UT_044, UT_045
+  patterns:
+    - "{0}-{1}-{2}"
+    - "{0}-{2}"
+    - "target_test-{2}"
   labels:
-    - example_test_esp32c3
+    - "{0}_{1}"
+    - "{0}"
+    - target_test
   included_in:
-    - build:example_test-esp32c3
+    - "build:{0}-{1}"
+    - "build:{0}"
+    - build:target_test
+
+# For example_test*flash_encryption_wifi_high_traffic jobs
+# set `INCLUDE_NIGHTLY_RUN` variable when triggered on development branches
+"test:example_test-{0}-include_nightly_run-rule":
+  matrix:
+    - - esp32
+      - esp32c3
+  specific_rules:
+    - "if-example_test-ota-include_nightly_run-rule"
+  included_in:
+    - "build:example_test-{0}"
+    - "build:example_test"
+    - build:target_test
+
+# due to the lack of runners, c2 tests will only be triggered by label
+"test:{0}-esp32c2":
+  matrix:
+    - *target_test
+  labels:
+    - "{0}_esp32c2"
+  included_in:
+    - "build:{0}"
+    - "build:{0}-esp32c2"
+    - build:target_test
+
+# overwrite the esp32c2 component ut again
+"test:component_ut-esp32c2": # component_ut will trigger by unit_test as well, since now we have 2 kinds of UT
+  labels:
+    - "component_ut_esp32c2"
+    - "unit_test_esp32c2"
+  included_in:
+    - build:component_ut
+    - "build:component_ut-esp32c2"
+    - build:target_test
+
+# due to the lack of runners, s3 tests will only be triggered by label
+"test:{0}-esp32s3":
+  matrix:
+    - *target_test
+  labels:
+    - "{0}_esp32s3"
+  included_in:
+    - "build:{0}"
+    - "build:{0}-esp32s3"
+    - build:target_test
+
+# overwrite the esp32s3 component ut again
+"test:component_ut-esp32s3": # component_ut will trigger by unit_test as well, since now we have 2 kinds of UT
+  labels:
+    - "component_ut_esp32s3"
+    - "unit_test_esp32s3"
+  included_in:
+    - build:component_ut
+    - "build:component_ut-esp32s3"
     - build:target_test
 
 "test:integration_test":
   labels:
-    - "integration_test"
+    - integration_test
+    - target_test
   patterns:
-    - "integration_test"
+    - integration_test
   included_in:
-    - "build:integration_test"
+    - build:integration_test
     - build:target_test
-    - test:target_test
-    - test:any_test
 
 "test:host_test":
   labels:
     - host_test
   patterns:
     - host_test
-  included_in:
-    - test:any_test
 
 "test:submodule":
   labels:
@@ -162,17 +264,17 @@ build:integration_test:
   labels:
     - iperf_stress_test
   included_in:
+    - build:example_test
     - build:example_test-esp32
     - build:target_test
-    - test:any_test
 
 "labels:weekend_test": # custom test
   labels:
     - weekend_test
   included_in:
+    - build:custom_test
     - build:custom_test-esp32
     - build:target_test
-    - test:any_test
 
 "labels:nvs_coverage": # host_test
   labels:
@@ -182,3 +284,13 @@ build:integration_test:
   labels:
     - fuzzer_test
     - weekend_test
+
+"labels-protected:lan8720":  # UT # FIXME: IDFCI-1176 temporary run this on master/release or with label
+  labels:
+    - lan8720
+  included_in:
+    - build:unit_test
+    - build:unit_test-esp32
+    - build:target_test
+    - build:component_ut
+    - build:component_ut-esp32

.gitlab/ci/dependencies/generate_rules.py

@@ -1,18 +1,7 @@
 #!/usr/bin/env python
 #
-# Copyright 2021 Espressif Systems (Shanghai) CO LTD
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Apache-2.0
 
 import argparse
 import inspect
@@ -88,9 +77,12 @@ class RulesWriter:
     RULE_PROTECTED_NO_LABEL = '    - <<: *if-protected-no_label'
     RULE_BUILD_ONLY = '    - <<: *if-label-build-only\n' \
                       '      when: never'
+    RULE_REVERT_BRANCH = '    - <<: *if-revert-branch\n' \
+                         '      when: never'
     RULE_LABEL_TEMPLATE = '    - <<: *if-label-{0}'
     RULE_PATTERN_TEMPLATE = '    - <<: *if-dev-push\n' \
                             '      changes: *patterns-{0}'
+    SPECIFIC_RULE_TEMPLATE = '    - <<: *{0}'
     RULES_TEMPLATE = inspect.cleandoc(r"""
     .rules:{0}:
       rules:
@@ -165,6 +157,8 @@ class RulesWriter:
                 continue
             if 'included_in' in v:
                 for item in _list(v['included_in']):
+                    if 'specific_rules' in v:
+                        res[item]['specific_rules'].update(_list(v['specific_rules']))
                     if 'labels' in v:
                         res[item]['labels'].update(_list(v['labels']))
                     if 'patterns' in v:
@@ -211,15 +205,20 @@ class RulesWriter:
         return '\n\n'.join(res)
 
     def _format_rule(self, name, cfg):  # type: (str, dict) -> str
-        _rules = []
+        _rules = [self.RULE_REVERT_BRANCH]
         if name.endswith('-production'):
             _rules.append(self.RULE_PROTECTED_NO_LABEL)
         else:
             if not (name.endswith('-preview') or name.startswith('labels:')):
                 _rules.append(self.RULE_PROTECTED)
-            # Special case for esp32c3 example_test, for now it only run with label
-            if name.startswith('test:') or name == 'labels:example_test-esp32c3':
+            if name.startswith('test:'):
                 _rules.append(self.RULE_BUILD_ONLY)
+
+            for specific_rule in cfg['specific_rules']:
+                if f'.{specific_rule}' in self.rules_cfg:
+                    _rules.append(self.SPECIFIC_RULE_TEMPLATE.format(specific_rule))
+                else:
+                    print('WARNING: specific_rule {} not exists'.format(specific_rule))
             for label in cfg['labels']:
                 _rules.append(self.RULE_LABEL_TEMPLATE.format(label))
             for pattern in cfg['patterns']:

.gitlab/ci/deploy.yml

@@ -5,57 +5,12 @@
   tags:
     - deploy
 
-push_to_github:
-  extends:
-    - .deploy_job_template
-    - .before_script_minimal
-    - .rules:protected-no_label
-  dependencies: []
-  script:
-    - add_github_ssh_keys
-    - git remote remove github &>/dev/null || true
-    - git remote add github git@github.com:espressif/esp-idf.git
-    - tools/ci/push_to_github.sh
-
-deploy_test_result:
-  extends:
-    - .deploy_job_template
-    - .before_script_minimal
-    - .rules:ref:master-schedule-always
-  image: $CI_DOCKER_REGISTRY/bot-env
-  tags:
-    - deploy_test
-  artifacts:
-    when: always
-    paths:
-      - ${CI_PROJECT_DIR}/test-management/*.log
-      # save all test logs as artifacts, make it easier to track errors
-      - ${CI_PROJECT_DIR}/TEST_LOGS
-    expire_in: 1 mos
-  variables:
-    UNIT_TEST_CASE_FILE: "${CI_PROJECT_DIR}/components/idf_test/unit_test/TestCaseAll.yml"
-    BOT_ACCOUNT_CONFIG_FILE: "${CI_PROJECT_DIR}/test-management/Config/Account.local.yml"
-    TEST_FW_PATH: "$CI_PROJECT_DIR/tools/tiny-test-fw"
-    AUTO_TEST_SCRIPT_PATH: "${CI_PROJECT_DIR}/auto_test_script"
-  script:
-    - add_gitlab_ssh_keys
-    - export GIT_SHA=$(echo ${CI_COMMIT_SHA} | cut -c 1-8)
-    - export REV_COUNT=$(git rev-list --count HEAD)
-    - export SUMMARY="IDF CI test result for $GIT_SHA (r${REV_COUNT})"
-    # artifacts of job update_test_cases creates test-management folder
-    # we need to remove it so we can clone test-management folder again
-    - rm -rf test-management
-    - retry_failed git clone $TEST_MANAGEMENT_REPO
-    - python3 $CHECKOUT_REF_SCRIPT test-management test-management
-    - cd test-management
-    - echo $BOT_JIRA_ACCOUNT > ${BOT_ACCOUNT_CONFIG_FILE}
-    # update test results
-    - python3 ImportTestResult.py -r "$GIT_SHA (r${REV_COUNT})" -j $JIRA_TEST_MANAGEMENT_PROJECT -s "$SUMMARY" -l CI -p ${CI_PROJECT_DIR}/TEST_LOGS --pipeline_url ${CI_PIPELINE_URL}
-
+# Check this before push_to_github
 check_submodule_sync:
   extends:
     - .deploy_job_template
     - .rules:test:submodule
+  stage: test_deploy
   tags:
     - github_sync
   retry: 2
@@ -73,3 +28,55 @@ check_submodule_sync:
     - git config --get-regexp '^submodule\..*\.url$' || true
     - git submodule update --recursive
     - echo "IDF was cloned from ${PUBLIC_IDF_URL} completely"
+
+push_to_github:
+  extends:
+    - .deploy_job_template
+    - .before_script_minimal
+    - .rules:protected-no_label
+  dependencies: []
+  script:
+    - add_github_ssh_keys
+    - git remote remove github &>/dev/null || true
+    - git remote add github git@github.com:espressif/esp-idf.git
+    - tools/ci/push_to_github.sh
+
+deploy_test_result:
+  extends:
+    - .deploy_job_template
+    - .before_script_minimal
+    - .rules:ref:master-always
+  image: $CI_DOCKER_REGISTRY/bot-env:1
+  dependencies: []
+  tags:
+    - deploy_test
+  artifacts:
+    when: always
+    paths:
+      - ${CI_PROJECT_DIR}/test-management/*.log
+    expire_in: 1 week
+  variables:
+    BOT_ACCOUNT_CONFIG_FILE: "${CI_PROJECT_DIR}/test-management/Config/Account.local.yml"
+    TEST_RESULTS_PATH: "${CI_PROJECT_DIR}/TEST_RESULTS"
+  script:
+    - add_gitlab_ssh_keys
+    - export GIT_SHA=$(echo ${CI_COMMIT_SHA} | cut -c 1-8)
+    - export REV_COUNT=$(git rev-list --count ${GIT_SHA} --)
+    - export SUMMARY="IDF CI test result for $GIT_SHA (r${REV_COUNT})"
+    # Download test result
+    - export PYTHONPATH="$IDF_PATH/tools:$IDF_PATH/tools/ci/python_packages:$PYTHONPATH"
+    - python3 ${IDF_PATH}/tools/ci/get_all_test_results.py --path ${TEST_RESULTS_PATH} --include_retried
+    - if [[ -z $(find ${TEST_RESULTS_PATH} -name "*.xml") ]]; then exit 0; fi
+    # Clone test-management repo
+    - retry_failed git clone $TEST_MANAGEMENT_REPO
+    - python3 $CHECKOUT_REF_SCRIPT test-management test-management
+    - cd test-management
+    - echo $BOT_JIRA_ACCOUNT > ${BOT_ACCOUNT_CONFIG_FILE}
+    # Make sure all requirements are installed
+    - pip3 install -r requirements.txt
+    # Update test cases
+    - python3 ImportTestCase.py $JIRA_TEST_MANAGEMENT_PROJECT from_xml -d ${TEST_RESULTS_PATH} -r $GIT_SHA -l IDFCI
+    # update test results
+    - python3 ImportTestResult.py -r "$GIT_SHA (r${REV_COUNT})" -j $JIRA_TEST_MANAGEMENT_PROJECT -s "$SUMMARY" -l IDFCI -p ${TEST_RESULTS_PATH} --pipeline_url ${CI_PIPELINE_URL}
+  # May need a long time to upload all test results.
+  timeout: 4 hours

.gitlab/ci/docs.yml

@@ -23,8 +23,8 @@
 .if-label-docs: &if-label-docs
   if: '$BOT_LABEL_DOCS || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*docs(?:,[^,\n\r]+)*$/i'
 
-.if-label-docs_full: &if-label-docs_full
-  if: '$BOT_LABEL_DOCS_FULL || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*docs_full(?:,[^,\n\r]+)*$/i'
+.if-label-docs_fast: &if-label-docs_fast
+  if: '$BOT_LABEL_DOCS_FAST || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*docs_fast(?:,[^,\n\r]+)*$/i'
 
 .if-dev-push: &if-dev-push
   if: '$CI_COMMIT_REF_NAME != "master" && $CI_COMMIT_BRANCH !~ /^release\/v/ && $CI_COMMIT_TAG !~ /^v\d+\.\d+(\.\d+)?($|-)/ && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event")'
@@ -34,7 +34,7 @@
     - <<: *if-protected
     - <<: *if-label-build_docs
     - <<: *if-label-docs
-    - <<: *if-label-docs_full
+    - <<: *if-label-docs_fast
     - <<: *if-dev-push
       changes: *patterns-docs
 
@@ -57,30 +57,26 @@ check_docs_lang_sync:
 
 .build_docs_template:
   image: $ESP_IDF_DOC_ENV_IMAGE
-  variables:
-    PYTHON_VER: 3.6.13
   tags:
     - build_docs
   dependencies: []
   script:
     - cd docs
-    - pip install -r requirements.txt
+    - pip install -U -r $IDF_PATH/tools/requirements/requirements.docs.txt
     - build-docs -t $DOCTGT -bs $DOC_BUILDERS -l $DOCLANG build
   parallel:
     matrix:
       - DOCLANG: ["en", "zh_CN"]
-        DOCTGT: ["esp32", "esp32s2", "esp32s3", "esp32c3"]
+        DOCTGT: ["esp32", "esp32s2", "esp32s3", "esp32c3", "esp32c2"]
 
 check_docs_gh_links:
   image: $ESP_IDF_DOC_ENV_IMAGE
-  variables:
-    PYTHON_VER: 3.6.13
   extends:
     - .pre_check_job_template
     - .doc-rules:build:docs
   script:
     - cd docs
-    - pip install -r requirements.txt
+    - pip install -U -r $IDF_PATH/tools/requirements/requirements.docs.txt
     - build-docs gh-linkcheck
 
 # stage: build_doc
@@ -101,8 +97,13 @@ build_docs_html_full:
   extends:
     - .build_docs_build_stage_template
   rules:
+    - <<: *if-label-docs_fast
+      when: never
     - <<: *if-protected
-    - <<: *if-label-docs_full
+    - <<: *if-label-build_docs
+    - <<: *if-label-docs
+    - <<: *if-dev-push
+      changes: *patterns-docs
   artifacts:
     when: always
     paths:
@@ -116,12 +117,7 @@ build_docs_html_fast:
   extends:
     - .build_docs_build_stage_template
   rules:
-    - <<: *if-label-docs_full
-      when: never
-    - <<: *if-label-build_docs
-    - <<: *if-label-docs
-    - <<: *if-dev-push
-      changes: *patterns-docs
+    - <<: *if-label-docs_fast
   artifacts:
     when: always
     paths:
@@ -136,8 +132,13 @@ build_docs_pdf:
   extends:
     - .build_docs_build_stage_template
   rules:
+    - <<: *if-label-docs_fast
+      when: never
     - <<: *if-protected
-    - <<: *if-label-docs_full
+    - <<: *if-label-build_docs
+    - <<: *if-label-docs
+    - <<: *if-dev-push
+      changes: *patterns-docs
   artifacts:
     when: always
     paths:
@@ -151,19 +152,16 @@ build_docs_pdf:
     - .before_script_no_sync_submodule
   image: $ESP_IDF_DOC_ENV_IMAGE
   variables:
-    PYTHON_VER: 3.6.13
+    DOCS_BUILD_DIR: "${IDF_PATH}/docs/_build/"
+    PYTHONUNBUFFERED: 1
   stage: test_deploy
   tags:
     - deploy
     - shiny
-  variables:
-    DOCS_BUILD_DIR: "${IDF_PATH}/docs/_build/"
-    PYTHONUNBUFFERED: 1
-  dependencies: []
   script:
     - add_doc_server_ssh_keys $DOCS_DEPLOY_PRIVATEKEY $DOCS_DEPLOY_SERVER $DOCS_DEPLOY_SERVER_USER
-    - export GIT_VER=$(git describe --always)
-    - pip install -r docs/requirements.txt
+    - export GIT_VER=$(git describe --always ${PIPELINE_COMMIT_SHA} --)
+    - pip install -U -r $IDF_PATH/tools/requirements/requirements.docs.txt
     - deploy-docs
 
 # stage: test_deploy
@@ -175,10 +173,13 @@ deploy_docs_preview:
     - <<: *if-label-docs
     - <<: *if-dev-push
       changes: *patterns-docs-preview
-  dependencies:
-    - build_docs_html_fast
-    - build_docs_html_full
-    - build_docs_pdf
+  needs:
+    - job: build_docs_html_fast
+      optional: true
+    - job: build_docs_html_full
+      optional: true
+    - job: build_docs_pdf
+      optional: true
   variables:
     TYPE: "preview"
     # older branches use DOCS_DEPLOY_KEY, DOCS_SERVER, DOCS_SERVER_USER, DOCS_PATH for preview server so we keep these names for 'preview'
@@ -209,6 +210,7 @@ deploy_docs_production:
     DOCS_DEPLOY_SERVER_USER: "$DOCS_PROD_SERVER_USER"
     DOCS_DEPLOY_PATH: "$DOCS_PROD_PATH"
     DOCS_DEPLOY_URL_BASE: "https://docs.espressif.com/projects/esp-idf"
+    DEPLOY_STABLE: 1
 
 check_doc_links:
   extends:
@@ -225,6 +227,5 @@ check_doc_links:
   allow_failure: true
   script:
     - cd docs
-    - pip install -r requirements.txt
-    # At the moment this check will always fail due to multiple known limitations, ignore result
-    - build-docs -t $DOCTGT -l $DOCLANG linkcheck || { echo "THERE ARE ISSUES DUE TO KNOWN LIMITATIONS, PLEASE FIX THEM. Nowadays we're ignored them to pass pipeline."; true; }
+    - pip install -U -r $IDF_PATH/tools/requirements/requirements.docs.txt
+    - build-docs -t $DOCTGT -l $DOCLANG linkcheck

.gitlab/ci/host-test.yml

@@ -2,12 +2,17 @@
   extends: .rules:test:host_test
   stage: host_test
   image: $ESP_ENV_IMAGE
-  variables:
-    PYTHON_VER: 3.6.13
   tags:
     - host_test
   dependencies: []
-  needs: []  # run host_test jobs immediately
+  # run host_test jobs immediately, only after upload cache
+  needs:
+    - job: upload-pip-cache
+      optional: true
+      artifacts: false
+    - job: upload-submodules-cache
+      optional: true
+      artifacts: false
 
 test_nvs_on_host:
   extends: .host_test_template
@@ -57,13 +62,28 @@ test_ldgen_on_host:
   extends: .host_test_template
   script:
     - cd tools/ldgen/test
-    - ./test_fragments.py
-    - ./test_generation.py
-    - ./test_entity.py
-    - ./test_output_commands.py
+    - export PYTHONPATH=$PYTHONPATH:..
+    - python -m unittest
   variables:
     LC_ALL: C.UTF-8
 
+test_reproducible_build:
+  extends: .host_test_template
+  script:
+    - ./tools/ci/test_reproducible_build.sh
+  artifacts:
+    when: on_failure
+    paths:
+      - "**/sdkconfig"
+      - "**/build*/*.bin"
+      - "**/build*/*.elf"
+      - "**/build*/*.map"
+      - "**/build*/flasher_args.json"
+      - "**/build*/*.bin"
+      - "**/build*/bootloader/*.bin"
+      - "**/build*/partition_table/*.bin"
+    expire_in: 1 week
+
 .host_fuzzer_test_template:
   extends:
     - .host_test_template
@@ -84,11 +104,6 @@ test_ldgen_on_host:
     # check no crashes found
     - test -z "$(ls out/crashes/)" || exit 1
 
-test_mdns_fuzzer_on_host:
-  extends: .host_fuzzer_test_template
-  variables:
-    FUZZER_TEST_DIR: components/mdns/test_afl_fuzz_host
-
 test_lwip_dns_fuzzer_on_host:
   extends: .host_fuzzer_test_template
   variables:
@@ -115,6 +130,14 @@ test_spiffs_on_host:
     - cd ../test_spiffsgen
     - ./test_spiffsgen.py
 
+test_fatfsgen_on_host:
+  extends: .host_test_template
+  script:
+    - cd components/fatfs/test_fatfsgen/
+    - ./test_fatfsgen.py
+    - ./test_wl_fatfsgen.py
+    - ./test_fatfsparse.py
+
 test_multi_heap_on_host:
   extends: .host_test_template
   script:
@@ -157,6 +180,7 @@ test_idf_monitor:
       - tools/test_idf_monitor/outputs/*
     expire_in: 1 week
   script:
+    - eval $($IDF_PATH/tools/idf_tools.py export)
     - cd ${IDF_PATH}/tools/test_idf_monitor
     - ./run_test_idf_monitor.py
 
@@ -193,8 +217,7 @@ test_idf_tools:
     - cd ${IDF_PATH}/tools/test_idf_tools
     - ./test_idf_tools.py
     # Test for create virtualenv. It must be invoked from Python, not from virtualenv.
-    - cd ${IDF_PATH}/tools
-    - python3 ./idf_tools.py install-python-env
+    - python3 ./test_idf_tools_python_env.py
 
 .test_efuse_table_on_host_template:
   extends: .host_test_template
@@ -245,10 +268,11 @@ test_espcoredump:
     expire_in: 1 week
   variables:
     IDF_COREDUMP_ELF_REPO: "https://gitlab-ci-token:${BOT_TOKEN}@${CI_SERVER_HOST}:${CI_SERVER_PORT}/idf/idf-coredump-elf.git"
-    IDF_COREDUMP_ELF_TAG: idf-20210910-00
+    IDF_COREDUMP_ELF_TAG: idf-20210915
     # install CMake version specified in tools.json
     SETUP_TOOLS_LIST: "all"
   script:
+    - eval $($IDF_PATH/tools/idf_tools.py export)
     - retry_failed git clone ${IDF_COREDUMP_ELF_REPO} -b $IDF_COREDUMP_ELF_TAG
     - cd ${IDF_PATH}/components/espcoredump/test/
     - ./test_espcoredump.sh ${CI_PROJECT_DIR}/idf-coredump-elf
@@ -293,7 +317,7 @@ test_mkuf2:
 
 test_autocomplete:
   extends: .host_test_template
-  image: $CI_DOCKER_REGISTRY/linux-shells:1
+  image: $LINUX_SHELL_IMAGE
   artifacts:
     when: on_failure
     paths:
@@ -304,7 +328,7 @@ test_autocomplete:
 
 test_detect_python:
   extends: .host_test_template
-  image: $CI_DOCKER_REGISTRY/linux-shells:1
+  image: $LINUX_SHELL_IMAGE
   script:
     - cd ${IDF_PATH}
     - shellcheck -s sh tools/detect_python.sh
@@ -315,6 +339,12 @@ test_detect_python:
     - "zsh -c '. tools/detect_python.sh && echo Our Python: ${ESP_PYTHON?Python is not set}'"
     - "fish -c 'source tools/detect_python.fish && echo Our Python: $ESP_PYTHON'"
 
+test_split_path_by_spaces:
+  extends: .host_test_template
+  script:
+    - cd ${IDF_PATH}/tools
+    - python -m unittest split_paths_by_spaces.py
+
 test_nvs_page:
   extends: .host_test_template
   script:
@@ -322,6 +352,13 @@ test_nvs_page:
     - idf.py build
     - build/test_nvs_page_host.elf
 
+test_mqtt_on_host:
+  extends: .host_test_template
+  script:
+    - cd ${IDF_PATH}/components/mqtt/host_test
+    - idf.py build
+    - LSAN_OPTIONS=verbosity=1:log_threads=1 build/host_mqtt_client_test.elf
+
 test_log:
   extends: .host_test_template
   script:
@@ -364,6 +401,27 @@ test_cxx_gpio:
     - idf.py build
     - build/test_gpio_cxx_host.elf
 
+test_i2c_cxx:
+  extends: .host_test_template
+  script:
+    - cd ${IDF_PATH}/examples/cxx/experimental/experimental_cpp_component/host_test/i2c
+    - idf.py build
+    - build/test_i2c_cxx_host.elf
+
+test_spi_cxx:
+  extends: .host_test_template
+  script:
+    - cd ${IDF_PATH}/examples/cxx/experimental/experimental_cpp_component/host_test/spi
+    - idf.py build
+    - build/test_spi_cxx_host.elf
+
+test_system_cxx:
+  extends: .host_test_template
+  script:
+    - cd ${IDF_PATH}/examples/cxx/experimental/experimental_cpp_component/host_test/system
+    - idf.py build
+    - build/test_system_cxx_host.elf
+
 test_linux_example:
   extends: .host_test_template
   script:
@@ -371,3 +429,21 @@ test_linux_example:
     - idf.py build
     - timeout 5 ./build/linux_host_app.elf >test.log || true
     - grep "Restarting" test.log
+
+test_gen_soc_caps_kconfig:
+  extends: .host_test_template
+  script:
+    - cd ${IDF_PATH}/tools/gen_soc_caps_kconfig/
+    - ./test/test_gen_soc_caps_kconfig.py
+
+test_pytest_qemu:
+  extends:
+    - .host_test_template
+    - .before_script_build_jobs
+  image: $QEMU_IMAGE
+  script:
+    - run_cmd python tools/ci/ci_build_apps.py . -vv
+      --target esp32
+      --pytest-apps
+      -m qemu
+    - pytest --target esp32 -m qemu --embedded-services idf,qemu

.gitlab/ci/pre_check.yml

@@ -22,7 +22,7 @@ check_pre_commit_master_release:
     - .check_pre_commit_template
     - .rules:protected
   script:
-    - git diff-tree --no-commit-id --name-only -r $CI_COMMIT_SHA | xargs pre-commit run --files
+    - git diff-tree --no-commit-id --name-only -r $PIPELINE_COMMIT_SHA | xargs pre-commit run --files
 
 check_pre_commit_MR:
   extends:
@@ -41,11 +41,6 @@ check_version:
     - export IDF_PATH=$PWD
     - tools/ci/check_idf_version.sh
 
-check_examples_cmake_make:
-  extends: .pre_check_job_template
-  script:
-  - python ${IDF_PATH}/tools/ci/check_examples_cmake_make.py
-
 check_rom_api_header:
   extends: .pre_check_job_template
   script:
@@ -56,8 +51,6 @@ check_python_style:
   extends:
     - .pre_check_base_template
     - .rules:patterns:python-files
-  variables:
-    PYTHON_VER: 3.6.13
   artifacts:
     when: on_failure
     paths:
@@ -68,8 +61,6 @@ check_python_style:
 
 test_check_kconfigs:
   extends: .pre_check_job_template
-  variables:
-    PYTHON_VER: 3.6.13
   artifacts:
     when: on_failure
     paths:
@@ -83,15 +74,20 @@ test_check_kconfigs:
   script:
     - python ${IDF_PATH}/tools/ci/test_check_kconfigs.py
 
-check_wifi_lib_md5:
+check_blobs:
   extends: .pre_check_base_template
   tags:
     - build
   variables:
-    SUBMODULES_TO_FETCH: "components/esp_wifi/lib"
+    SUBMODULES_TO_FETCH: "components/esp_wifi/lib;components/esp_phy/lib"
   script:
+    # Check if Wi-Fi library header files match between IDF and the version used when compiling the libraries
     - IDF_TARGET=esp32 $IDF_PATH/components/esp_wifi/test_md5/test_md5.sh
     - IDF_TARGET=esp32s2 $IDF_PATH/components/esp_wifi/test_md5/test_md5.sh
+    - IDF_TARGET=esp32s3 $IDF_PATH/components/esp_wifi/test_md5/test_md5.sh
+    - IDF_TARGET=esp32c3 $IDF_PATH/components/esp_wifi/test_md5/test_md5.sh
+    # Check if Wi-Fi, PHY, BT blobs contain references to specific symbols
+    - bash $IDF_PATH/tools/ci/check_blobs.sh
 
 check_fuzzer_compilation:
    extends: .pre_check_base_template
@@ -101,8 +97,6 @@ check_fuzzer_compilation:
      - make MODE=dhcp_server
      - make MODE=dhcp_client
      - make MODE=dns
-     - cd ${IDF_PATH}/components/mdns/test_afl_fuzz_host
-     - make
 
 check_public_headers:
   extends:
@@ -111,16 +105,28 @@ check_public_headers:
   tags:
     - build
   script:
-    - python tools/ci/check_public_headers.py --jobs 4 --prefix xtensa-esp32-elf-
+    - IDF_TARGET=esp32 python tools/ci/check_public_headers.py --jobs 4 --prefix xtensa-esp32-elf-
+    - IDF_TARGET=esp32s2 python tools/ci/check_public_headers.py --jobs 4 --prefix xtensa-esp32s2-elf-
+    - IDF_TARGET=esp32s3 python tools/ci/check_public_headers.py --jobs 4 --prefix xtensa-esp32s3-elf-
+    - IDF_TARGET=esp32c3 python tools/ci/check_public_headers.py --jobs 4 --prefix riscv32-esp-elf-
+    - IDF_TARGET=esp32c2 python tools/ci/check_public_headers.py --jobs 4 --prefix riscv32-esp-elf-
 
-check_soc_struct_headers:
+check_chip_support_components:
   extends:
     - .pre_check_base_template
     - .rules:build
   tags:
     - build
+  artifacts:
+    when: on_failure
+    paths:
+      - esp_hw_support_part.h
+      - bootloader_support_part.h
+    expire_in: 1 week
   script:
+    - python tools/ci/check_soc_headers_leak.py
     - find ${IDF_PATH}/components/soc/*/include/soc/ -name "*_struct.h" -print0 | xargs -0 -n1 ./tools/ci/check_soc_struct_headers.py
+    - tools/ci/check_esp_memory_utils_headers.sh
 
 check_esp_err_to_name:
   extends:
@@ -138,31 +144,14 @@ check_esp_err_to_name:
     - ./gen_esp_err_to_name.py
     - git diff --exit-code -- ../components/esp_common/src/esp_err_to_name.c || { echo 'Differences found. Please run gen_esp_err_to_name.py and commit the changes.'; exit 1; }
 
-scan_tests:
+check_esp_system:
   extends:
     - .pre_check_base_template
-    - .rules:build:target_test
-  image: $CI_DOCKER_REGISTRY/ubuntu-test-env$BOT_DOCKER_IMAGE_TAG
+    - .rules:build
   tags:
-    - assign_test
-  artifacts:
-    paths:
-      - $EXAMPLE_TEST_OUTPUT_DIR
-      - $TEST_APPS_OUTPUT_DIR
-      - $COMPONENT_UT_OUTPUT_DIR
-  variables:
-    EXAMPLE_TEST_DIR: ${CI_PROJECT_DIR}/examples
-    EXAMPLE_TEST_OUTPUT_DIR: ${CI_PROJECT_DIR}/examples/test_configs
-    TEST_APPS_TEST_DIR: ${CI_PROJECT_DIR}/tools/test_apps
-    TEST_APPS_OUTPUT_DIR: ${CI_PROJECT_DIR}/tools/test_apps/test_configs
-    COMPONENT_UT_OUTPUT_DIR: ${CI_PROJECT_DIR}/component_ut/test_configs
-    CI_SCAN_TESTS_PY: ${CI_PROJECT_DIR}/tools/ci/python_packages/ttfw_idf/CIScanTests.py
+    - build
   script:
-    - set_component_ut_vars
-    - run_cmd python $CI_SCAN_TESTS_PY example_test $EXAMPLE_TEST_DIR -b make --exclude examples/build_system/idf_as_lib -c $CI_TARGET_TEST_CONFIG_FILE -o $EXAMPLE_TEST_OUTPUT_DIR
-    - run_cmd python $CI_SCAN_TESTS_PY example_test $EXAMPLE_TEST_DIR -b cmake --exclude examples/build_system/idf_as_lib -c $CI_TARGET_TEST_CONFIG_FILE -o $EXAMPLE_TEST_OUTPUT_DIR
-    - run_cmd python $CI_SCAN_TESTS_PY test_apps $TEST_APPS_TEST_DIR -c $CI_TARGET_TEST_CONFIG_FILE -o $TEST_APPS_OUTPUT_DIR
-    - run_cmd python $CI_SCAN_TESTS_PY component_ut $COMPONENT_UT_DIRS --exclude $COMPONENT_UT_EXCLUDES -c $CI_TARGET_TEST_CONFIG_FILE -o $COMPONENT_UT_OUTPUT_DIR
+    - python components/esp_system/check_system_init_priorities.py
 
 # For release tag pipelines only, make sure the tag was created with 'git tag -a' so it will update
 # the version returned by 'git describe'
@@ -173,13 +162,6 @@ check_version_tag:
   script:
     - (git cat-file -t $CI_COMMIT_REF_NAME | grep tag) || (echo "ESP-IDF versions must be annotated tags." && exit 1)
 
-check_ut_cmake_make:
-  extends: .pre_check_job_template
-  tags:
-    - build
-  script:
-    - tools/ci/check_ut_cmake_make.sh
-
 check_artifacts_expire_time:
   extends: .pre_check_job_template
   script:
@@ -190,6 +172,13 @@ check_commit_msg:
   extends: .pre_check_job_template
   script:
     - git status
-    - git log -n10 --oneline
+    - git log -n10 --oneline ${PIPELINE_COMMIT_SHA}
     # commit start with "WIP: " need to be squashed before merge
-    - 'git log --pretty=%s master.. -- | grep "^WIP: " && exit 1 || exit 0'
+    - 'git log --pretty=%s origin/master..${PIPELINE_COMMIT_SHA} -- | grep -i "^WIP:" && exit 1 || exit 0'
+
+check_test_scripts_build_test_rules:
+  extends:
+    - .pre_check_job_template
+    - .before_script_build_jobs
+  script:
+    - python tools/ci/check_build_test_rules.py check-test-scripts examples/ tools/test_apps components

.gitlab/ci/static-code-analysis.yml

@@ -10,7 +10,6 @@ clang_tidy_check:
     when: always
     expire_in: 1 day
   variables:
-    CLANG_TIDY_RUNNER_PROJ: 2107  # idf/clang-tidy-runner
     CLANG_TIDY_DIRS_TXT: ${CI_PROJECT_DIR}/tools/ci/clang_tidy_dirs.txt
     RULES_FILE: ${CI_PROJECT_DIR}/tools/ci/static-analysis-rules.yml
     OUTPUT_DIR: ${CI_PROJECT_DIR}/clang_tidy_reports
@@ -82,6 +81,7 @@ check_pylint:
     when: always
     paths:
       - $REPORT_PATTERN
+    expire_in: 1 week
   tags:
     - host_test
   dependencies:  # Here is not a hard dependency relationship, could be skipped when only python files changed. so we do not use "needs" here.
@@ -123,7 +123,7 @@ code_quality_report:
       -Dsonar.branch.name=$CI_COMMIT_REF_NAME
       -Dsonar.cxx.clangtidy.reportPath=$REPORT_PATTERN
       -Dsonar.exclusions=$EXCLUSIONS
-      -Dsonar.gitlab.commit_sha=$CI_COMMIT_SHA
+      -Dsonar.gitlab.commit_sha=$PIPELINE_COMMIT_SHA
       -Dsonar.gitlab.ref_name=$CI_COMMIT_REF_NAME
       -Dsonar.host.url=$SONAR_HOST_URL
       -Dsonar.login=$SONAR_LOGIN

.gitlab/ci/upload_cache.yml

@@ -0,0 +1,53 @@
+# pull only for most of the use cases for cache
+# only set "push" policy for the jobs under this file.
+# The cache would be updated when files matched specified patterns changes.
+
+.upload_cache_template:
+  stage: upload_cache
+  image: $ESP_ENV_IMAGE
+
+upload-pip-cache:
+  extends:
+    - .upload_cache_template
+    - .rules:patterns:python-cache
+  tags:
+    - $GEO
+    - build
+  cache:
+    key: pip-cache
+    paths:
+      - .cache/pip
+    policy: push
+  before_script: []
+  script:
+    - source tools/ci/utils.sh
+    - is_based_on_commits $REQUIRED_ANCESTOR_COMMITS
+    - source tools/ci/setup_python.sh
+    - rm -rf .cache/pip  # clear old packages
+    - $IDF_PATH/tools/idf_tools.py install-python-env --features pytest,ci
+  parallel:
+    matrix:
+      - GEO: [ 'shiny',  'brew' ]
+
+upload-submodules-cache:
+  extends:
+    - .upload_cache_template
+    - .rules:patterns:submodule
+  tags:
+    - $GEO
+    - build
+  cache:
+    key: submodule-cache
+    paths:
+      - .cache/submodule_archives
+    policy: push
+  before_script: []
+  script:
+    - source tools/ci/utils.sh
+    - is_based_on_commits $REQUIRED_ANCESTOR_COMMITS
+    - source tools/ci/setup_python.sh
+    - rm -rf .cache/submodule_archives  # clear old submodule archives
+    - fetch_submodules
+  parallel:
+    matrix:
+      - GEO: [ 'shiny',  'brew' ]

.gitlab/merge_request_templates/Minimal_Template.md

@@ -0,0 +1,15 @@
+<!-- This Template states the absolute minimum for an MR.
+    If you want to have a more elaborate template or know why we have this structure,
+    please use the "Mixed Template" or consult the Wiki. -->
+<!-- Add the CI labels to trigger the appropriate tests (e.g. "unit_test_esp32") --><!-- Mandatory -->
+<!-- Make sure the commit message follows the Wiki about "Commit Messages" --><!-- Mandatory -->
+
+<!-- Add description of the change here --><!-- Mandatory -->
+
+## Related <!-- Optional -->
+<!-- Related Jira issues and Github issues -->
+
+## Release notes <!-- Mandatory -->
+<!-- Either state release notes or write "No release notes" -->
+
+<!-- ## Breaking change notes --><!-- Optional -->

.gitmodules

@@ -3,30 +3,14 @@
 # For Espressif's public projects please use '../../espressif/proj', not a '../proj'
 #
 
-[submodule "components/esptool_py/esptool"]
-	path = components/esptool_py/esptool
-	url = ../../espressif/esptool.git
-
 [submodule "components/bt/controller/lib_esp32"]
 	path = components/bt/controller/lib_esp32
-        url = ../../espressif/esp32-bt-lib.git
+	url = ../../espressif/esp32-bt-lib.git
 
 [submodule "components/bootloader/subproject/components/micro-ecc/micro-ecc"]
 	path = components/bootloader/subproject/components/micro-ecc/micro-ecc
 	url = ../../kmackay/micro-ecc.git
 
-[submodule "components/coap/libcoap"]
-	path = components/coap/libcoap
-	url = ../../obgm/libcoap.git
-
-[submodule "components/nghttp/nghttp2"]
-	path = components/nghttp/nghttp2
-	url = ../../nghttp2/nghttp2.git
-
-[submodule "components/libsodium/libsodium"]
-	path = components/libsodium/libsodium
-	url = ../../jedisct1/libsodium.git
-
 [submodule "components/spiffs/spiffs"]
 	path = components/spiffs/spiffs
 	url = ../../pellepl/spiffs.git
@@ -39,14 +23,6 @@
 	path = components/mbedtls/mbedtls
 	url = ../../espressif/mbedtls.git
 
-[submodule "components/asio/asio"]
-	path = components/asio/asio
-	url = ../../espressif/asio.git
-
-[submodule "components/expat/expat"]
-	path = components/expat/expat
-	url = ../../libexpat/libexpat.git
-
 [submodule "components/lwip/lwip"]
 	path = components/lwip/lwip
 	url = ../../espressif/esp-lwip.git
@@ -63,18 +39,10 @@
 	path = components/unity/unity
 	url = ../../ThrowTheSwitch/Unity.git
 
-[submodule "examples/build_system/cmake/import_lib/main/lib/tinyxml2"]
-	path = examples/build_system/cmake/import_lib/main/lib/tinyxml2
-	url = ../../leethomason/tinyxml2.git
-
 [submodule "components/bt/host/nimble/nimble"]
 	path = components/bt/host/nimble/nimble
 	url = ../../espressif/esp-nimble.git
 
-[submodule "components/cbor/tinycbor"]
-	path = components/cbor/tinycbor
-	url = ../../intel/tinycbor.git
-
 [submodule "components/esp_wifi/lib"]
 	path = components/esp_wifi/lib
 	url = ../../espressif/esp32-wifi-lib.git
@@ -110,3 +78,11 @@
 [submodule "components/ieee802154/lib"]
 	path = components/ieee802154/lib
 	url = ../../espressif/esp-ieee802154-lib.git
+
+[submodule "components/bt/controller/lib_esp32h2/esp32h2-bt-lib"]
+	path = components/bt/controller/lib_esp32h2/esp32h2-bt-lib
+	url = ../../espressif/esp32h2-bt-lib.git
+
+[submodule "components/bt/controller/lib_esp32c2/esp32c2-bt-lib"]
+	path = components/bt/controller/lib_esp32c2/esp32c2-bt-lib
+	url = ../../espressif/esp32c2-bt-lib.git

.pre-commit-config.yaml

@@ -12,12 +12,16 @@ repos:
         # 2 - any file matching *test*/*expected* (for host tests, if possible use this naming pattern always)
         # 3 - any directory named 'testdata'
         # 4 - IDF monitor test data
+        # 5 - protobuf auto-generated files
         exclude: &whitespace_excludes |
           (?x)^(
             .+\.(md|rst|map|bin)|
             .+test.*\/.*expected.*|
             .+\/testdata\/.+|
-            .+test_idf_monitor\/tests\/.+
+            .+test_idf_monitor\/tests\/.+|
+            .*_pb2.py|
+            .*.pb-c.h|
+            .*.pb-c.c
           )$
       - id: end-of-file-fixer
         exclude: *whitespace_excludes
@@ -35,6 +39,10 @@ repos:
     hooks:
       - id: isort
         name: isort (python)
+        exclude: >
+          (?x)^(
+              .*_pb2.py
+          )$
   - repo: local
     hooks:
       - id: check-executables
@@ -71,6 +79,7 @@ repos:
         name: Validate Codeowner File
         entry: tools/ci/check_codeowners.py ci-check
         language: python
+        always_run: true
         files: '\.gitlab/CODEOWNERS'
         pass_filenames: false
       - id: check-rules-yml
@@ -93,21 +102,24 @@ repos:
         name: Check type annotations in python files
         entry: tools/ci/check_type_comments.py
         additional_dependencies:
-          - 'mypy==0.800'
+          - 'mypy==0.940'
           - 'mypy-extensions==0.4.3'
+          - 'types-setuptools==57.4.14'
+          - 'types-PyYAML==0.1.9'
+        exclude: >
+          (?x)^(
+              .*_pb2.py
+          )$
         language: python
         types: [python]
-      - id: check-copyright
-        name: Check copyright notices
-        entry: tools/ci/check_copyright.py --verbose --replace
+      - id: check-requirement-files
+        name: Check requirement files
+        entry: tools/ci/check_requirement_files.py
         additional_dependencies:
-          - 'comment_parser == 1.2.3'
-          - 'thefuzz == 0.19.0'
-          - 'thefuzz[speedup] == 0.19.0; sys_platform != "win32"'
-          # don't depend on python-Levenshtein on Windows, as it requires Microsoft C++ Build Tools to install
+          - 'jsonschema'
         language: python
-        files: \.(py|c|h|cpp|hpp|ld)$
-        require_serial: true
+        files: 'tools/requirements.+|tools/requirements/.+'
+        pass_filenames: false
       - id: check-tools-files-patterns
         name: Check tools dir files patterns
         entry: tools/ci/check_tools_files_patterns.py
@@ -116,8 +128,35 @@ repos:
         additional_dependencies:
           - PyYAML == 5.3.1
         pass_filenames: false
+      - id: check-generated-soc-caps-kconfig
+        name: Check soc caps kconfig files are generated (based on components/soc/IDF_TARGET/include/soc/soc_caps.h)
+        entry: tools/gen_soc_caps_kconfig/gen_soc_caps_kconfig.py -d 'components/soc/*/include/soc/' 'components/esp_rom/*/'
+        language: python
+        files: 'components/soc/.+/include/soc/.+_caps\.h|components/esp_rom/.+/.+_caps\.h|kconfig\.soc_caps.in'
+        pass_filenames: false
+        additional_dependencies:
+          - pyparsing
+      - id: check-all-apps-readmes
+        name: Check if all apps readme files match given .build-test-rules.yml files. Modify the supported target tables
+        entry: tools/ci/check_build_test_rules.py check-readmes
+        language: python
+        files: 'tools/test_apps/.+|examples/.+|components/.+'
+        additional_dependencies:
+          - idf_build_apps
+      - id: sort-build-test-rules-ymls
+        name: sort .build-test-rules.yml files
+        entry: tools/ci/check_build_test_rules.py sort-yaml
+        language: python
+        files: '\.build-test-rules\.yml'
+        additional_dependencies:
+          - ruamel.yaml
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.0.1
     hooks:
       - id: file-contents-sorter
-        files: 'tools\/ci\/(executable-list\.txt|mypy_ignore_list\.txt|check_copyright_ignore\.txt|check_copyright_permanent_ignore\.txt)'
+        files: 'tools\/ci\/(executable-list\.txt|mypy_ignore_list\.txt|check_copyright_ignore\.txt)'
+  - repo: https://github.com/espressif/check-copyright/
+    rev: v1.0.1
+    hooks:
+      - id: check-copyright
+        args: ['--ignore', 'tools/ci/check_copyright_ignore.txt', '--config', 'tools/ci/check_copyright_config.yaml']

.pylintrc

@@ -150,6 +150,9 @@ disable=print-statement,
         too-many-nested-blocks,
         too-many-branches,
         too-many-statements,
+        ungrouped-imports,  # since we have isort in pre-commit
+        no-name-in-module,  # since we have flake8 to check this
+        too-many-instance-attributes,
 
 # Enable the message, report, category or checker with the given id(s). You can
 # either give multiple identifier separated by comma (,) or put this option

CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.5)
+cmake_minimum_required(VERSION 3.16)
 project(esp-idf C CXX ASM)
 
 if(CMAKE_CURRENT_LIST_DIR STREQUAL CMAKE_SOURCE_DIR)
@@ -7,11 +7,8 @@ if(CMAKE_CURRENT_LIST_DIR STREQUAL CMAKE_SOURCE_DIR)
     "again.")
 endif()
 
-unset(compile_options)
-unset(c_compile_options)
-unset(cxx_compile_options)
-unset(compile_definitions)
-unset(link_options)
+# Variables compile_options, c_compile_options, cxx_compile_options, compile_definitions, link_options shall
+# not be unset as they may already contain flags, set by toolchain-TARGET.cmake files.
 
 # Add the following build specifications here, since these seem to be dependent
 # on config values on the root Kconfig.
@@ -65,23 +62,6 @@ if(CONFIG_COMPILER_SAVE_RESTORE_LIBCALLS)
     list(APPEND compile_options "-msave-restore")
 endif()
 
-if(CONFIG_COMPILER_DISABLE_GCC8_WARNINGS)
-    list(APPEND compile_options "-Wno-parentheses"
-                                "-Wno-sizeof-pointer-memaccess"
-                                "-Wno-clobbered")
-
-    list(APPEND compile_options "-Wno-format-overflow"
-                                "-Wno-stringop-truncation"
-                                "-Wno-misleading-indentation"
-                                "-Wno-cast-function-type"
-                                "-Wno-implicit-fallthrough"
-                                "-Wno-unused-const-variable"
-                                "-Wno-switch-unreachable"
-                                "-Wno-format-truncation"
-                                "-Wno-memset-elt-size"
-                                "-Wno-int-in-bool-context")
-endif()
-
 if(CMAKE_C_COMPILER_ID MATCHES "GNU")
     list(APPEND c_compile_options "-Wno-old-style-declaration")
 endif()
@@ -119,6 +99,9 @@ if(CMAKE_C_COMPILER_ID MATCHES "Clang")
     # Multiple cases of attributes unknown to clang, for example
     # __attribute__((optimize("-O3")))
     list(APPEND compile_options "-Wno-unknown-attributes")
+    # Disable Clang warnings for atomic operations with access size
+    # more then 4 bytes
+    list(APPEND compile_options "-Wno-atomic-alignment")
     # Clang also produces many -Wunused-function warnings which GCC doesn't.
     # However these aren't treated as errors.
 endif()
@@ -147,7 +130,41 @@ endif()
 if(NOT ${CMAKE_C_COMPILER_VERSION} VERSION_LESS 8.0.0)
     if(CONFIG_COMPILER_HIDE_PATHS_MACROS)
         list(APPEND compile_options "-fmacro-prefix-map=${CMAKE_SOURCE_DIR}=.")
-        list(APPEND compile_options "-fmacro-prefix-map=${IDF_PATH}=IDF")
+        list(APPEND compile_options "-fmacro-prefix-map=${IDF_PATH}=/IDF")
+    endif()
+
+    if(CONFIG_APP_REPRODUCIBLE_BUILD)
+        idf_build_set_property(DEBUG_PREFIX_MAP_GDBINIT "${BUILD_DIR}/prefix_map_gdbinit")
+
+        list(APPEND compile_options "-fdebug-prefix-map=${IDF_PATH}=/IDF")
+        list(APPEND compile_options "-fdebug-prefix-map=${PROJECT_DIR}=/IDF_PROJECT")
+        list(APPEND compile_options "-fdebug-prefix-map=${BUILD_DIR}=/IDF_BUILD")
+
+        # component dirs
+        idf_build_get_property(python PYTHON)
+        idf_build_get_property(idf_path IDF_PATH)
+        idf_build_get_property(component_dirs BUILD_COMPONENT_DIRS)
+
+        execute_process(
+            COMMAND ${python}
+                "${idf_path}/tools/generate_debug_prefix_map.py"
+                "${BUILD_DIR}"
+                "${component_dirs}"
+            OUTPUT_VARIABLE result
+            RESULT_VARIABLE ret
+        )
+        if(NOT ret EQUAL 0)
+            message(FATAL_ERROR "This is a bug. Please report to https://github.com/espressif/esp-idf/issues")
+        endif()
+
+        spaces2list(result)
+        list(LENGTH component_dirs length)
+        math(EXPR max_index "${length} - 1")
+        foreach(index RANGE ${max_index})
+            list(GET component_dirs ${index} folder)
+            list(GET result ${index} after)
+            list(APPEND compile_options "-fdebug-prefix-map=${folder}=${after}")
+        endforeach()
     endif()
 endif()
 
@@ -165,6 +182,20 @@ endif()
 
 list(APPEND link_options "-fno-lto")
 
+if(CONFIG_IDF_TARGET_LINUX AND CMAKE_HOST_SYSTEM_NAME STREQUAL "Darwin")
+    list(APPEND link_options "-Wl,-dead_strip")
+    list(APPEND link_options "-Wl,-warn_commons")
+else()
+    list(APPEND link_options "-Wl,--gc-sections")
+    list(APPEND link_options "-Wl,--warn-common")
+endif()
+
+# SMP FreeRTOS user provided minimal idle hook. This allows the user to provide
+# their own copy of vApplicationMinimalIdleHook()
+if(CONFIG_FREERTOS_USE_MINIMAL_IDLE_HOOK)
+    list(APPEND link_options "-Wl,--wrap=vApplicationMinimalIdleHook")
+endif()
+
 # Placing jump tables in flash would cause issues with code that required
 # to be placed in IRAM
 list(APPEND compile_options "-fno-jump-tables")
@@ -178,9 +209,20 @@ if(CMAKE_C_COMPILER_ID MATCHES "LLVM")
     list(APPEND compile_options "-fno-use-cxa-atexit")
 endif()
 
+# For the transition period from 32-bit time_t to 64-bit time_t,
+# auto-detect the size of this type and set corresponding variable.
+include(CheckTypeSize)
+check_type_size("time_t" TIME_T_SIZE)
+if(TIME_T_SIZE)
+    idf_build_set_property(TIME_T_SIZE ${TIME_T_SIZE})
+else()
+    message(FATAL_ERROR "Failed to determine sizeof(time_t)")
+endif()
+
 idf_build_set_property(COMPILE_OPTIONS "${compile_options}" APPEND)
 idf_build_set_property(C_COMPILE_OPTIONS "${c_compile_options}" APPEND)
 idf_build_set_property(CXX_COMPILE_OPTIONS "${cxx_compile_options}" APPEND)
+idf_build_set_property(ASM_COMPILE_OPTIONS "${asm_compile_options}" APPEND)
 idf_build_set_property(COMPILE_DEFINITIONS "${compile_definitions}" APPEND)
 idf_build_set_property(LINK_OPTIONS "${link_options}" APPEND)
 

CONTRIBUTING.md

@@ -0,0 +1,6 @@
+# Contributing to ESP-IDF
+
+Contributions to ESP-IDF - fixing bugs, adding features, adding documentation - are welcome! We accept contributions via Github Pull Requests.
+
+Please see the [Contributions Guide](https://docs.espressif.com/projects/esp-idf/en/latest/esp32/contribute/index.html) for more information.
+

CONTRIBUTING.rst

@@ -1,61 +0,0 @@
-Contributions Guide
-===================
-
-We welcome contributions to the esp-idf project!
-
-How to Contribute
------------------
-
-Contributions to esp-idf - fixing bugs, adding features, adding documentation - are welcome. We accept contributions via `Github Pull Requests <https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/about-pull-requests>`_.
-
-Before Contributing
--------------------
-
-Before sending us a Pull Request, please consider this list of points:
-
-* Is the contribution entirely your own work, or already licensed under an Apache License 2.0 compatible Open Source License? If not then we unfortunately cannot accept it.
-
-* Does any new code conform to the esp-idf :doc:`Style Guide <style-guide>`?
-
-* Have you installed the :doc:`pre-commit hook <install-pre-commit-hook>` for esp-idf project?
-
-* Does the code documentation follow requirements in :doc:`documenting-code`?
-
-* Is the code adequately commented for people to understand how it is structured?
-
-* Is there documentation or examples that go with code contributions? There are additional suggestions for writing good examples in :idf:`examples` readme.
-
-* Are comments and documentation written in clear English, with no spelling or grammar errors?
-
-* Example contributions are also welcome. Please check the :doc:`creating-examples` guide for these.
-
-* If the contribution contains multiple commits, are they grouped together into logical changes (one major change per pull request)? Are any commits with names like "fixed typo" `squashed into previous commits <https://eli.thegreenplace.net/2014/02/19/squashing-github-pull-requests-into-a-single-commit/>`_?
-
-* If you're unsure about any of these points, please open the Pull Request anyhow and then ask us for feedback.
-
-Pull Request Process
---------------------
-
-After you open the Pull Request, there will probably be some discussion in the comments field of the request itself.
-
-Once the Pull Request is ready to merge, it will first be merged into our internal git system for in-house automated testing.
-
-If this process passes, it will be merged onto the public github repository.
-
-Legal Part
-----------
-
-Before a contribution can be accepted, you will need to sign our :doc:`contributor-agreement`. You will be prompted for this automatically as part of the Pull Request process.
-
-Related Documents
------------------
-
-.. toctree::
-    :maxdepth: 1
-
-    style-guide
-    install-pre-commit-hook
-    documenting-code
-    creating-examples
-    ../api-reference/template
-    contributor-agreement

Kconfig

@@ -1,12 +1,14 @@
 #
-# For a description of the syntax of this configuration file,
-# see kconfig/kconfig-language.txt.
+# Please run the following command for opening a page with more information about this configuration file:
+# idf.py docs -sp api-reference/kconfig.html
 #
 mainmenu "Espressif IoT Development Framework Configuration"
 
+    orsource "./components/soc/$IDF_TARGET/include/soc/Kconfig.soc_caps.in"
+
     config IDF_CMAKE
         bool
-        option env="IDF_CMAKE"
+        default "y"
 
     config IDF_ENV_FPGA
         # This option is for internal use only
@@ -21,6 +23,11 @@ mainmenu "Espressif IoT Development Framework Configuration"
         bool
         default "n"
 
+    config IDF_TARGET_ARCH
+        string
+        default "riscv" if IDF_TARGET_ARCH_RISCV
+        default "xtensa" if IDF_TARGET_ARCH_XTENSA
+
     config IDF_TARGET
         # This option records the IDF target when sdkconfig is generated the first time.
         # It is not updated if environment variable $IDF_TARGET changes later, and
@@ -29,6 +36,10 @@ mainmenu "Espressif IoT Development Framework Configuration"
         string
         default "$IDF_TARGET"
 
+    config IDF_TARGET_LINUX
+        bool
+        default "y" if IDF_TARGET="linux"
+
     config IDF_TARGET_ESP32
         bool
         default "y" if IDF_TARGET="esp32"
@@ -57,6 +68,29 @@ mainmenu "Espressif IoT Development Framework Configuration"
         select FREERTOS_UNICORE
         select IDF_TARGET_ARCH_RISCV
 
+    choice IDF_TARGET_ESP32H2_BETA_VERSION
+        prompt "ESP32-H2 beta version"
+        depends on IDF_TARGET_ESP32H2
+        default IDF_TARGET_ESP32H2_BETA_VERSION_2
+        help
+            Currently ESP32-H2 has several beta versions for internal use only.
+            Select the one that matches your chip model.
+
+        config IDF_TARGET_ESP32H2_BETA_VERSION_1
+            bool
+            prompt "ESP32-H2 beta1"
+        config IDF_TARGET_ESP32H2_BETA_VERSION_2
+            bool
+            prompt "ESP32-H2 beta2"
+            select ESPTOOLPY_NO_STUB    # TODO: IDF-4288
+    endchoice
+
+    config IDF_TARGET_ESP32C2
+        bool
+        default "y" if IDF_TARGET="esp32c2"
+        select FREERTOS_UNICORE
+        select IDF_TARGET_ARCH_RISCV
+
     config IDF_TARGET_LINUX
         bool
         default "y" if IDF_TARGET="linux"
@@ -67,62 +101,11 @@ mainmenu "Espressif IoT Development Framework Configuration"
         default 0x0002 if IDF_TARGET_ESP32S2
         default 0x0005 if IDF_TARGET_ESP32C3
         default 0x0009 if IDF_TARGET_ESP32S3
-        default 0x000A if IDF_TARGET_ESP32H2 # ESP32H2-TODO: IDF-3475
+        default 0x000C if IDF_TARGET_ESP32C2
+        default 0x000A if IDF_TARGET_ESP32H2_BETA_VERSION_1
+        default 0x000E if IDF_TARGET_ESP32H2_BETA_VERSION_2 # ESP32H2-TODO: IDF-3475
         default 0xFFFF
 
-    menu "SDK tool configuration"
-        config SDK_TOOLPREFIX
-            string "Compiler toolchain path/prefix"
-            default "xtensa-esp32-elf-" if IDF_TARGET_ESP32
-            default "xtensa-esp32s2-elf-" if IDF_TARGET_ESP32S2
-            default "xtensa-esp32s3-elf-" if IDF_TARGET_ESP32S3
-            default "riscv32-esp-elf-" if IDF_TARGET_ESP32C3
-            default "riscv32-esp-elf-" if IDF_TARGET_ESP32H2
-
-            help
-                The prefix/path that is used to call the toolchain. The default setting assumes
-                a crosstool-ng gcc setup that is in your PATH.
-
-        config SDK_PYTHON
-            string "Python interpreter"
-            depends on !IDF_CMAKE
-            default "python"
-            help
-                The executable name/path that is used to run python.
-
-                (Note: This option is used with the legacy GNU Make build system only.)
-
-        config SDK_MAKE_WARN_UNDEFINED_VARIABLES
-            bool "'make' warns on undefined variables"
-            depends on !IDF_CMAKE
-            default "y"
-            help
-                Adds --warn-undefined-variables to MAKEFLAGS. This causes make to
-                print a warning any time an undefined variable is referenced.
-
-                This option helps find places where a variable reference is misspelled
-                or otherwise missing, but it can be unwanted if you have Makefiles which
-                depend on undefined variables expanding to an empty string.
-
-                (Note: this option is used with the legacy GNU Make build system only.)
-
-        config SDK_TOOLCHAIN_SUPPORTS_TIME_WIDE_64_BITS
-            bool "Toolchain supports time_t wide 64-bits"
-            default n
-            help
-                Enable this option in case you have a custom toolchain which supports time_t wide 64-bits.
-                This option checks time_t is 64-bits and disables ROM time functions
-                to use the time functions from the toolchain instead.
-                This option allows resolving the Y2K38 problem.
-                See "Setup Linux Toolchain from Scratch" to build
-                a custom toolchain which supports 64-bits time_t.
-
-                Note: ESP-IDF does not currently come with any pre-compiled toolchain
-                that supports 64-bit wide time_t.
-                This will change in a future major release,
-                but currently 64-bit time_t requires a custom built toolchain.
-
-    endmenu  # SDK tool configuration
 
     menu "Build type"
 
@@ -182,6 +165,7 @@ mainmenu "Espressif IoT Development Framework Configuration"
             config APP_BUILD_TYPE_APP_2NDBOOT
                 bool
                 prompt "Default (binary application + 2nd stage bootloader)"
+                depends on !IDF_TARGET_LINUX
                 select APP_BUILD_GENERATE_BINARIES
                 select APP_BUILD_BOOTLOADER
                 select APP_BUILD_USE_FLASH_SECTIONS
@@ -201,6 +185,63 @@ mainmenu "Espressif IoT Development Framework Configuration"
         config APP_BUILD_USE_FLASH_SECTIONS
             bool # Whether to place code/data into memory-mapped flash sections
 
+        config APP_REPRODUCIBLE_BUILD
+            bool "Enable reproducible build"
+            default n
+            select COMPILER_HIDE_PATHS_MACROS
+            help
+                If enabled, all date, time, and path information would be eliminated. A .gdbinit file would be create
+                automatically. (or will be append if you have one already)
+
+        config APP_NO_BLOBS
+            bool "No Binary Blobs"
+            default n
+            help
+                If enabled, this disables the linking of binary libraries in the application build. Note
+                that after enabling this Wi-Fi/Bluetooth will not work.
+
+        config APP_COMPATIBLE_PRE_V2_1_BOOTLOADERS
+            bool "App compatible with bootloaders before ESP-IDF v2.1"
+            select APP_COMPATIBLE_PRE_V3_1_BOOTLOADERS
+            depends on IDF_TARGET_ESP32
+            default n
+            help
+                Bootloaders before ESP-IDF v2.1 did less initialisation of the
+                system clock. This setting needs to be enabled to build an app
+                which can be booted by these older bootloaders.
+
+                If this setting is enabled, the app can be booted by any bootloader
+                from IDF v1.0 up to the current version.
+
+                If this setting is disabled, the app can only be booted by bootloaders
+                from IDF v2.1 or newer.
+
+                Enabling this setting adds approximately 1KB to the app's IRAM usage.
+
+        config APP_COMPATIBLE_PRE_V3_1_BOOTLOADERS
+            bool "App compatible with bootloader and partition table before ESP-IDF v3.1"
+            depends on IDF_TARGET_ESP32
+            default n
+            help
+                Partition tables before ESP-IDF V3.1 do not contain an MD5 checksum
+                field, and the bootloader before ESP-IDF v3.1 cannot read a partition
+                table that contains an MD5 checksum field.
+
+                Enable this option only if your app needs to boot on a bootloader and/or
+                partition table that was generated from a version *before* ESP-IDF v3.1.
+
+                If this option and Flash Encryption are enabled at the same time, and any
+                data partitions in the partition table are marked Encrypted, then the
+                partition encrypted flag should be manually verified in the app before accessing
+                the partition (see CVE-2021-27926).
+
+        config APP_INIT_CLK
+            bool
+            depends on IDF_TARGET_ESP32
+            default y if APP_COMPATIBLE_PRE_V2_1_BOOTLOADERS
+            default y if APP_BUILD_TYPE_ELF_RAM
+
+
     endmenu # Build type
 
     source "$COMPONENT_KCONFIGS_PROJBUILD_SOURCE_FILE"
@@ -278,6 +319,26 @@ mainmenu "Espressif IoT Development Framework Configuration"
 
         endchoice # assertions
 
+        choice COMPILER_FLOAT_LIB_FROM
+            prompt "Compiler float lib source"
+            default COMPILER_FLOAT_LIB_FROM_RVFPLIB if ESP_ROM_HAS_RVFPLIB
+            default COMPILER_FLOAT_LIB_FROM_GCCLIB
+            help
+                In the soft-fp part of libgcc, riscv version is written in C,
+                and handles all edge cases in IEEE754, which makes it larger
+                and performance is slow.
+
+                RVfplib is an optimized RISC-V library for FP arithmetic on 32-bit
+                integer processors, for single and double-precision FP.
+                RVfplib is "fast", but it has a few exceptions from IEEE 754 compliance.
+
+            config COMPILER_FLOAT_LIB_FROM_GCCLIB
+                bool "libgcc"
+            config COMPILER_FLOAT_LIB_FROM_RVFPLIB
+                depends on ESP_ROM_HAS_RVFPLIB
+                bool "librvfp"
+        endchoice # COMPILER_FLOAT_LIB_FROM
+
         config COMPILER_OPTIMIZATION_ASSERTION_LEVEL
             int
             default 0 if COMPILER_OPTIMIZATION_ASSERTIONS_DISABLE
@@ -297,7 +358,6 @@ mainmenu "Espressif IoT Development Framework Configuration"
         menuconfig COMPILER_HIDE_PATHS_MACROS
             bool "Replace ESP-IDF and project paths in binaries"
             default y
-            depends on IDF_CMAKE
             help
                 When expanding the __FILE__ and __BASE_FILE__ macros, replace paths inside ESP-IDF
                 with paths relative to the placeholder string "IDF", and convert paths inside the
@@ -406,41 +466,14 @@ mainmenu "Espressif IoT Development Framework Configuration"
 
                 This option can be enabled for RISC-V targets only.
 
-        config COMPILER_DISABLE_GCC8_WARNINGS
-            bool "Disable new warnings introduced in GCC 6 - 8"
-            default "n"
-            help
-                Enable this option if using GCC 6 or newer, and wanting to disable warnings which don't appear with
-                GCC 5.
-
         config COMPILER_DUMP_RTL_FILES
             bool "Dump RTL files during compilation"
             help
                 If enabled, RTL files will be produced during compilation. These files
                 can be used by other tools, for example to calculate call graphs.
 
-
     endmenu # Compiler Options
 
     menu "Component config"
         source "$COMPONENT_KCONFIGS_SOURCE_FILE"
     endmenu
-
-    menu "Compatibility options"
-        config LEGACY_INCLUDE_COMMON_HEADERS
-            bool "Include headers across components as before IDF v4.0"
-            default n
-            help
-                Soc, esp32, and driver components, the most common
-                components. Some header of these components are included
-                implicitly by headers of other components before IDF v4.0.
-                It's not required for high-level components, but still
-                included through long header chain everywhere.
-
-                This is harmful to the modularity. So it's changed in IDF
-                v4.0.
-
-                You can still include these headers in a legacy way until it
-                is totally deprecated by enable this option.
-
-    endmenu #Compatibility options

README.md

@@ -4,17 +4,25 @@
 
 ESP-IDF is the development framework for Espressif SoCs supported on Windows, Linux and macOS.
 
+# ESP-IDF Release Support Schedule
+
+![Support Schedule](https://dl.espressif.com/dl/esp-idf/support-periods.svg)
+
+- Please read [the support policy](SUPPORT_POLICY.md) and [the documentation](https://docs.espressif.com/projects/esp-idf/en/latest/esp32/versions.html) for more information about ESP-IDF versions.
+- Please see the [End-of-Life Advisories](https://www.espressif.com/en/support/documents/advisories?keys=&field_type_of_advisory_tid%5B%5D=817) for information about ESP-IDF releases with discontinued support.
+
 # ESP-IDF Release and SoC Compatibility
 
-The following table shows ESP-IDF support of Espressif SoCs where ![alt text][preview] and ![alt text][supported] denote preview status and support, respectively. In preview status the build is not yet enabled and some crucial parts could be missing (like documentation, datasheet). Please use an ESP-IDF release where the desired SoC is already supported.
+The following table shows ESP-IDF support of Espressif SoCs where ![alt text][preview] and ![alt text][supported] denote preview status and support, respectively. The preview support is usually limited in time and intended for beta versions of chips. Please use an ESP-IDF release where the desired SoC is already supported.
 
-|Chip         |         v3.3           |          v4.0          |           v4.1         |          v4.2          |         v4.3           |          v4.4          |                                                            |
-|:----------- |:---------------------: | :---------------------:| :---------------------:| :---------------------:| :---------------------:| :---------------------:|:---------------------------------------------------------- |
-|ESP32        | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] |                                                            |
-|ESP32-S2     |                        |                        |                        | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] |                                                            |
-|ESP32-C3     |                        |                        |                        |                        | ![alt text][supported] | ![alt text][supported] |                                                            |
-|ESP32-S3     |                        |                        |                        |                        | ![alt text][preview]   | ![alt text][supported] | [Announcement](https://www.espressif.com/en/news/ESP32_S3) |
-|ESP32-H2     |                        |                        |                        |                        |                        | ![alt text][preview]   | [Announcement](https://www.espressif.com/en/news/ESP32_H2) |
+|Chip         |          v4.1         |          v4.2          |         v4.3           |          v4.4          |          v5.0          |                                                                                      |
+|:----------- |:---------------------:| :---------------------:| :---------------------:| :---------------------:| :---------------------:|:------------------------------------------------------------------------------------ |
+|ESP32        |![alt text][supported] | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] |                                                                                      |
+|ESP32-S2     |                       | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] |                                                                                      |
+|ESP32-C3     |                       |                        | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] |                                                                                      |
+|ESP32-S3     |                       |                        |                        | ![alt text][supported] | ![alt text][supported] | [Announcement](https://www.espressif.com/en/news/ESP32_S3)                           |
+|ESP32-C2     |                       |                        |                        |                        | ![alt text][supported] | [Announcement](https://blog.espressif.com/esp32-c2-and-why-it-matter-s-bcf4d7d0b2c6) |
+|ESP32-H2     |                       |                        |                        | ![alt text][preview]   | ![alt text][preview]   | [Announcement](https://www.espressif.com/en/news/ESP32_H2)                           |
 
 [supported]: https://img.shields.io/badge/-supported-green "supported"
 [preview]: https://img.shields.io/badge/-preview-orange "preview"
@@ -31,11 +39,9 @@ See https://idf.espressif.com/ for links to detailed instructions on how to set
 
 ### Non-GitHub forks
 
-ESP-IDF uses relative locations as its submodules URLs ([.gitmodules](.gitmodules)). So they link to GitHub.
-If ESP-IDF is forked to a Git repository which is not on GitHub, you will need to run the script
-[tools/set-submodules-to-github.sh](tools/set-submodules-to-github.sh) after git clone.
-The script sets absolute URLs for all submodules, allowing `git submodule update --init --recursive` to complete.
-If cloning ESP-IDF from GitHub, this step is not needed.
+ESP-IDF uses relative locations as its submodules URLs ([.gitmodules](.gitmodules)). So they link to GitHub. If ESP-IDF is forked to a Git repository which is not on GitHub, you will need to run the script [tools/set-submodules-to-github.sh](tools/set-submodules-to-github.sh) after git clone.
+
+The script sets absolute URLs for all submodules, allowing `git submodule update --init --recursive` to complete. If cloning ESP-IDF from GitHub, this step is not needed.
 
 ## Finding a Project
 

README_CN.md

@@ -4,17 +4,25 @@
 
 ESP-IDF 是乐鑫官方推出的物联网开发框架，支持 Windows、Linux 和 macOS 操作系统。
 
+# ESP-IDF 版本支持期限
+
+![支持期限](https://dl.espressif.com/dl/esp-idf/support-periods.svg)
+
+- 请参考 [ESP-IDF 支持政策](SUPPORT_POLICY_CN.md) 以及 [相关文档](https://docs.espressif.com/projects/esp-idf/zh_CN/latest/esp32/versions.html) 了解更多关于 ESP-IDF 版本的信息。
+- 请参考 [ESP-IDF 版本停止维护 (EOL) 公告](https://www.espressif.com/zh-hans/support/documents/advisories?keys=&field_type_of_advisory_tid%5B%5D=817)。
+
 # ESP-IDF 与乐鑫芯片
 
-下表总结了乐鑫芯片在 ESP-IDF 各版本中的支持状态，其中 ![alt text][supported] 代表已支持，![alt text][preview] 代表目前处于预览支持状态。在预览支持阶段，因为新芯片尚未完全添加到构建系统目录，所以一些重要的内容（如文档和技术规格书等）可能会缺失。请确保使用与芯片相匹配的 ESP-IDF 版本。
+下表总结了乐鑫芯片在 ESP-IDF 各版本中的支持状态，其中 ![alt text][supported] 代表已支持，![alt text][preview] 代表目前处于预览支持状态。预览支持状态通常有时间限制，而且仅适用于测试版芯片。请确保使用与芯片相匹配的 ESP-IDF 版本。
 
-|    芯片     |         v3.3           |          v4.0          |           v4.1         |          v4.2          |         v4.3           |          v4.4          |                                                            |
-|:----------- |:---------------------: | :---------------------:| :---------------------:| :---------------------:| :---------------------:| :---------------------:|:---------------------------------------------------------- |
-|ESP32        | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] |                                                            |
-|ESP32-S2     |                        |                        |                        | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] |                                                            |
-|ESP32-C3     |                        |                        |                        |                        | ![alt text][supported] | ![alt text][supported] |                                                            |
-|ESP32-S3     |                        |                        |                        |                        | ![alt text][preview]   | ![alt text][supported] | [芯片发布公告](https://www.espressif.com/en/news/ESP32_S3) |
-|ESP32-H2     |                        |                        |                        |                        |                        | ![alt text][preview]   | [芯片发布公告](https://www.espressif.com/en/news/ESP32_H2) |
+|芯片         |          v4.1         |          v4.2          |         v4.3           |          v4.4          |          v5.0          |                                                                                      |
+|:----------- |:---------------------:| :---------------------:| :---------------------:| :---------------------:| :---------------------:|:------------------------------------------------------------------------------------ |
+|ESP32        |![alt text][supported] | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] |                                                                                      |
+|ESP32-S2     |                       | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] |                                                                                      |
+|ESP32-C3     |                       |                        | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] |                                                                                      |
+|ESP32-S3     |                       |                        |                        | ![alt text][supported] | ![alt text][supported] | [芯片发布公告](https://www.espressif.com/en/news/ESP32_S3)                           |
+|ESP32-C2     |                       |                        |                        |                        | ![alt text][supported] | [芯片发布公告](https://blog.espressif.com/esp32-c2-and-why-it-matter-s-bcf4d7d0b2c6) |
+|ESP32-H2     |                       |                        |                        | ![alt text][preview]   | ![alt text][preview]   | [芯片发布公告](https://www.espressif.com/en/news/ESP32_H2)                           |
 
 [supported]: https://img.shields.io/badge/-%E6%94%AF%E6%8C%81-green "supported"
 [preview]: https://img.shields.io/badge/-%E9%A2%84%E8%A7%88-orange "preview"
@@ -31,11 +39,9 @@ ESP-IDF 是乐鑫官方推出的物联网开发框架，支持 Windows、Linux
 
 ### 非 GitHub 分叉的 ESP-IDF 项目
 
-ESP-IDF 中的子模块采用相对路径（[详见 .gitmodules 文件](.gitmodules)），所以它们会指向 GitHub。
-如果 ESP-IDF 被分叉到的仓库不在 GitHub 上，那么你需要在克隆结束后运行该脚本 [tools/set-submodules-to-github.sh](tools/set-submodules-to-github.sh)。
+ESP-IDF 中的子模块采用相对路径（[详见 .gitmodules 文件](.gitmodules)），所以它们会指向 GitHub。 如果 ESP-IDF 被分叉到的仓库不在 GitHub 上，那么你需要在克隆结束后运行该脚本 [tools/set-submodules-to-github.sh](tools/set-submodules-to-github.sh)。
 
-这个脚本会为所有的子模块设置绝对路径，接着可以通过 `git submodule update --init --recursive` 完成子模块的更新。
-如果 ESP-IDF 是从 GitHub 上克隆得到，则不需要此步骤。
+这个脚本会为所有的子模块设置绝对路径，接着可以通过 `git submodule update --init --recursive` 完成子模块的更新。如果 ESP-IDF 是从 GitHub 上克隆得到，则不需要此步骤。
 
 ## 寻找项目
 

SECURITY.md

@@ -0,0 +1,9 @@
+# Security Policy
+
+## Supported Versions
+
+Please refer to https://docs.espressif.com/projects/esp-idf/en/latest/esp32/versions.html#support-periods for more details on ESP-IDF supported versions and support period policy.
+
+## Reporting a Vulnerability
+
+If you think you have found a security vulnerability in Espressif solutions (including ESP-IDF), then please send an email to our Bug Bounty team at bugbounty@espressif.com. Please do **NOT** create a public GitHub issue.

SUPPORT_POLICY.md

@@ -5,12 +5,9 @@ Support Period Policy
 
 * [中文版](./SUPPORT_POLICY_CN.md)
 
-Each ESP-IDF major and minor release (V4.1, V4.2, etc) is supported for
-30 months after the initial stable release date.
+Each ESP-IDF major and minor release (V4.1, V4.2, etc) is supported for 30 months after the initial stable release date.
 
-Supported means that the ESP-IDF team will continue to apply bug fixes,
-security fixes, etc to the release branch on GitHub, and periodically
-make new bugfix releases as needed.
+Supported means that the ESP-IDF team will continue to apply bug fixes, security fixes, etc to the release branch on GitHub, and periodically make new bugfix releases as needed.
 
 Support period is divided into "Service" and "Maintenance" period:
 
@@ -19,58 +16,36 @@ Support period is divided into "Service" and "Maintenance" period:
 | Service     | 12 months    | Yes                                   |
 | Maintenance | 18 months    | No                                    |
 
-During the Service period, bugfixes releases are more frequent. In some cases,
-support for new features may be added during the Service period (this is
-reserved for features which are needed to meet particular regulatory
-requirements or standards for new products, and which carry a very low risk of
-introducing regressions.)
+During the Service period, bugfixes releases are more frequent. In some cases, support for new features may be added during the Service period (this is reserved for features which are needed to meet particular regulatory requirements or standards for new products, and which carry a very low risk of introducing regressions.)
 
-During the Maintenance period, the version is still supported but only bugfixes
-for high severity issues or security issues will be applied.
+During the Maintenance period, the version is still supported but only bugfixes for high severity issues or security issues will be applied.
 
 Using an “In Service” version is recommended when starting a new project.
 
-Users are encouraged to upgrade all projects to a newer ESP-IDF release before
-the support period finishes and the release becomes End of Life (EOL). It is our
-policy to not continue fixing bugs in End of Life releases.
+Users are encouraged to upgrade all projects to a newer ESP-IDF release before the support period finishes and the release becomes End of Life (EOL). It is our policy to not continue fixing bugs in End of Life releases.
 
-Pre-release versions (betas, previews, `-rc` and `-dev` versions, etc)
-are not covered by any support period. Sometimes a particular feature is
-marked as "Preview" in a release, which means it is also not covered
-by the support period.
+Pre-release versions (betas, previews, `-rc` and `-dev` versions, etc) are not covered by any support period. Sometimes a particular feature is marked as "Preview" in a release, which means it is also not covered by the support period.
 
-ESP-IDF should be used in an up-to-date software environment. The operating system
-and other third-party tools should be supported by their maintainers.
-ESP-IDF cannot keep compatibility with unsupported third-party tools.
+ESP-IDF should be used in an up-to-date software environment. The operating system and other third-party tools should be supported by their maintainers. ESP-IDF cannot keep compatibility with unsupported third-party tools.
 
-The ESP-IDF Programming Guide has information about the
-[different versions of ESP-IDF](https://docs.espressif.com/projects/esp-idf/en/latest/versions.html)
-(major, minor, bugfix, etc).
+The ESP-IDF Programming Guide has information about the [different versions of ESP-IDF](https://docs.espressif.com/projects/esp-idf/en/latest/versions.html) (major, minor, bugfix, etc).
 
 Example
 -------
 
-ESP-IDF V3.3 was released in September 2019. It will be supported for 30 months
-until February 2022.
+ESP-IDF V3.3 was released in September 2019. It was supported for 30 months until February 2022.
 
 - The first V3.3 release was `v3.3` in September 2019.
-- The ESP-IDF team continues to backport bug fixes, security fixes,
-  etc to the release branch `release/v3.3`.
-- Periodically stable bugfix releases are created from the release
-  branch. For example `v3.3.1`, `v3.3.2`, etc. Users are encouraged to
-  always update to the latest bugfix release.
-- V3.3 bugfix releases continue until February 2022, when all V3.3.x
-  releases become End of Life.
+- The ESP-IDF team continues to backport bug fixes, security fixes, etc to the release branch `release/v3.3`.
+- Periodically stable bugfix releases are created from the release branch. For example `v3.3.1`, `v3.3.2`, etc. Users are encouraged to always update to the latest bugfix release.
+- V3.3 bugfix releases continue until February 2022, when all V3.3.x releases become End of Life.
 
 Existing Releases
 -----------------
 
-ESP-IDF release V4.1 and all newer releases will follow this support period
-policy. The support period for each release will be announced when the release
-is made.
+ESP-IDF release V4.1 and all newer releases will follow this support period policy. The support period for each release will be announced when the release is made.
 
-For releases made before the current support period policy was announced, the
-original support periods apply:
+For releases made before the current support period policy was announced, the original support periods apply:
 
 * ESP-IDF V4.0.x will be supported until October 2021
 * ESP-IDF V3.3.x will be supported until February 2022
@@ -79,8 +54,5 @@ original support periods apply:
 Policy History
 --------------
 
-* September 2019. This policy split ESP-IDF releases into Standard and Long Term
-  Support.
-* July 2020. All releases from now will have the same support period, which is
-  equal to the previous Long Term Support period. Added “In Service” period,
-  during which versions will receive more updates.
+* September 2019. This policy splits ESP-IDF releases into Standard and Long Term Support.
+* July 2020. All releases from now will have the same support period, which is equal to the previous Long Term Support period. Added “In Service” period, during which versions will receive more updates.

SUPPORT_POLICY_CN.md

@@ -3,6 +3,8 @@
 支持期限政策
 =====================
 
+* [英文版](./SUPPORT_POLICY.md)
+
 ESP-IDF 的每个主要版本和次要版本（如 V4.1、V4.2 等）自其首次稳定版本发布之日起将支持 30 个月。
 
 支持意味着 ESP-IDF 团队将会对 GitHub 上的发布分支继续进行 bug 修复、安全修补等，并根据需求定期发布新的 bugfix 版本。
@@ -18,12 +20,14 @@ ESP-IDF 的每个主要版本和次要版本（如 V4.1、V4.2 等）自其首
 
 在版本的维护周期内，版本仍然会继续支持，但仅会对严重问题或安全问题进行 bug 修复。
 
-建议在开始新项目的时候使用处于服务周期的版本。 
+建议在开始新项目的时候使用处于服务周期的版本。
 
 在某一版本支持期限结束，停止更新维护 (EOL) 前，建议用户升级所有项目至较新的 ESP-IDF 版本。根据《支持期限政策》，我们将停止对 EOL 版本进行 bug 修复。
 
 《支持期限政策》不适用于预发布版本（包括 beta、preview、-rc 和 -dev 版本等）。有时，在发布的版本中存在被标记为 "Preview" 的特定功能，则该功能也不在支持期限内。
 
+请确保在最新的软件环境中使用 ESP-IDF。操作系统和其他第三方工具应得到其维护者的支持。ESP-IDF 无法兼容不受支持的第三方工具。
+
 有关 [ESP-IDF 不同版本](https://docs.espressif.com/projects/esp-idf/zh_CN/latest/esp32/versions.html)（主要版本、次要版本、bugfix 版本等）信息，可参阅《ESP-IDF 编程指南》。
 
 示例
@@ -45,11 +49,10 @@ ESP-IDF V4.1 及所有后续更新版本都将遵守该《支持期限政策》
 
 * ESP-IDF V4.0.x 将支持至 2021 年 10 月。
 * ESP-IDF V3.3.x 将支持至 2022 年 2 月。
-* ESP-IDF V3.1.x 和 V3.2.x 都将在 2020 年 10 月停止支持。
-* ESP-IDF v3.1.x 之前的版本已经停止支持。
-  
+* ESP-IDF v3.3 之前的版本已经停止支持。
+
 政策历史
 --------------
 
-* 2019 年 9 月。该政策将 ESP-IDF 版本发布分为标准支持版本和长期支持版本。 
-* 2020 年 7 月。从现在开始，所有发布的版本都将有相同的支持周期，等同于之前的长期支持周期。新增了”服务周期“，期间版本的更新将较为频繁。
+* 2019 年 9 月。该政策将 ESP-IDF 版本发布分为标准支持版本和长期支持版本。
+* 2020 年 7 月。从现在开始，所有发布的版本都将有相同的支持周期，即之前的长期支持周期。新增了”服务周期“，期间版本的更新将较为频繁。

components/README.md

@@ -150,11 +150,10 @@ The current implementation in `esp_system` can then just register the interrupt
 #### `esp_mac.h`, `esp_chip_info.h`, `esp_random.h` (`esp_hw_support`)
 
 The functions in these headers used to be in `esp_system.h`, but have been split-off.
-However, to maintain backward compatibility, `esp_system.h` includes these headers.
 
 The remaining functions in `esp_system.h` are those that deal with system behavior, such
 as `esp_register_shutdown_handler`, or are proxy for other system components's APIs such as
 `esp_get_free_heap_size`.
 
 The functions split-off from `esp_system.h` are much more hardware manipulation oriented such as:
-`esp_read_mac`, `esp_random` and `esp_chip_info`.
+`esp_read_mac`, `esp_random` and `esp_chip_info`.

components/app_trace/CMakeLists.txt

@@ -21,6 +21,8 @@ if(CONFIG_APPTRACE_MEMBUFS_APPTRACE_PROTO_ENABLE)
             "port/riscv/port.c")
     endif()
 endif()
+list(APPEND srcs
+    "port/port_uart.c")
 
 if(CONFIG_APPTRACE_SV_ENABLE)
     list(APPEND include_dirs
@@ -47,7 +49,9 @@ endif()
 idf_component_register(SRCS "${srcs}"
                        INCLUDE_DIRS "${include_dirs}"
                        PRIV_INCLUDE_DIRS "${priv_include_dirs}"
-                       PRIV_REQUIRES soc esp_ipc
+                       # Requires "driver" for GPTimer in "SEGGER_SYSVIEW_Config_FreeRTOS.c"
+                       PRIV_REQUIRES soc driver
+                       REQUIRES esp_timer
                        LDFRAGMENTS linker.lf)
 
 # disable --coverage for this component, as it is used as transport
@@ -57,3 +61,16 @@ target_compile_options(${COMPONENT_LIB} PRIVATE "-fno-profile-arcs" "-fno-test-c
 # Force app_trace to also appear later than gcov in link line
 idf_component_get_property(app_trace app_trace COMPONENT_LIB)
 target_link_libraries(${COMPONENT_LIB} INTERFACE $<TARGET_FILE:${app_trace}> gcov $<TARGET_FILE:${app_trace}> c)
+
+# This function adds a dependency on the given component if the component is included into the build.
+function(maybe_add_component component_name)
+    idf_build_get_property(components BUILD_COMPONENTS)
+    if(${component_name} IN_LIST components)
+        idf_component_get_property(lib_name ${component_name} COMPONENT_LIB)
+        target_link_libraries(${COMPONENT_LIB} PUBLIC ${lib_name})
+    endif()
+endfunction()
+
+if(CONFIG_APPTRACE_DEST_UART0 OR CONFIG_APPTRACE_DEST_UART1 OR CONFIG_APPTRACE_DEST_UART2)
+    maybe_add_component(driver)
+endif()

components/app_trace/Kconfig

@@ -1,7 +1,7 @@
 menu "Application Level Tracing"
 
-    choice APPTRACE_DESTINATION
-        prompt "Data Destination"
+    choice APPTRACE_DESTINATION1
+        prompt "Data Destination 1"
         default APPTRACE_DEST_NONE
         help
             Select destination for application trace: JTAG or none (to disable).
@@ -16,13 +16,133 @@ menu "Application Level Tracing"
             bool "None"
     endchoice
 
+    config APPTRACE_DEST_UART
+        bool
+
+    config APPTRACE_DEST_UART_NOUSB
+        bool
+
+    choice APPTRACE_DESTINATION2
+        prompt "Data Destination 2"
+        default APPTRACE_DEST_UART_NONE
+        help
+            Select destination for application trace: UART(XX) or none (to disable).
+
+        config APPTRACE_DEST_UART0
+            bool "UART0"
+            select APPTRACE_ENABLE
+            select APPTRACE_DEST_UART
+            select APPTRACE_DEST_UART_NOUSB
+            depends on (ESP_CONSOLE_UART_NUM !=0)
+
+        config APPTRACE_DEST_UART1
+            bool "UART1"
+            select APPTRACE_ENABLE
+            select APPTRACE_DEST_UART
+            select APPTRACE_DEST_UART_NOUSB
+            depends on (ESP_CONSOLE_UART_NUM !=1)
+
+        config APPTRACE_DEST_UART2
+            bool "UART2"
+            select APPTRACE_ENABLE
+            select APPTRACE_DEST_UART
+            select APPTRACE_DEST_UART_NOUSB
+            depends on (ESP_CONSOLE_UART_NUM !=2) && !IDF_TARGET_ESP32C3 && !IDF_TARGET_ESP32S2 && !IDF_TARGET_ESP32H2
+
+        config APPTRACE_DEST_USB_CDC
+            bool "USB_CDC"
+            select APPTRACE_ENABLE
+            select APPTRACE_DEST_UART
+            depends on !ESP_CONSOLE_USB_CDC && (IDF_TARGET_ESP32C3 || IDF_TARGET_ESP32S3) && !USB_ENABLED
+
+        config APPTRACE_DEST_UART_NONE
+            bool "None"
+    endchoice
+
+    config APPTRACE_UART_TX_GPIO
+        int "UART TX on GPIO#"
+        depends on APPTRACE_DEST_UART_NOUSB
+        range 0 46
+        default 12 if IDF_TARGET_ESP32
+        default 12 if IDF_TARGET_ESP32C3
+        default 12
+        help
+            This GPIO is used for UART TX pin.
+
+    config APPTRACE_UART_RX_GPIO
+        int "UART RX on GPIO#"
+        depends on APPTRACE_DEST_UART_NOUSB
+        range 0 46
+        default 13 if IDF_TARGET_ESP32
+        default 13 if IDF_TARGET_ESP32C3
+        default 13
+        help
+            This GPIO is used for UART RX pin.
+
+    config APPTRACE_UART_BAUDRATE
+        int
+        prompt "UART baud rate" if APPTRACE_DEST_UART
+        depends on APPTRACE_DEST_UART
+        default 1000000
+        range 1200 8000000
+        range 1200 1000000
+        help
+            This baud rate is used for UART.
+
+            The app's maximum baud rate depends on the UART clock source. If Power Management is disabled,
+            the UART clock source is the APB clock and all baud rates in the available range will be sufficiently
+            accurate. If Power Management is enabled, REF_TICK clock source is used so the baud rate is divided
+            from 1MHz. Baud rates above 1Mbps are not possible and values between 500Kbps and 1Mbps may not be
+            accurate.
+
+    config APPTRACE_UART_RX_BUFF_SIZE
+        int
+        prompt "UART RX ring buffer size" if APPTRACE_DEST_UART
+        depends on APPTRACE_DEST_UART
+        default 128
+        range 64 32768
+        help
+            Size of the UART input ring buffer.
+            This size related to the baudrate, system tick frequency and amount of data to transfer.
+            The data placed to this buffer before sent out to the interface.
+
+    config APPTRACE_UART_TX_BUFF_SIZE
+        int
+        prompt "UART TX ring buffer size" if APPTRACE_DEST_UART
+        depends on APPTRACE_DEST_UART
+        default 4096
+        range 2048 32768
+        help
+            Size of the UART output ring buffer.
+            This size related to the baudrate, system tick frequency and amount of data to transfer.
+
+    config APPTRACE_UART_TX_MSG_SIZE
+        int
+        prompt "UART TX message size" if APPTRACE_DEST_UART
+        depends on APPTRACE_DEST_UART
+        default 128
+        range 64 32768
+        help
+            Maximum size of the single message to transfer.
+
+    config APPTRACE_UART_TASK_PRIO
+        int
+        prompt "UART Task Priority" if APPTRACE_DEST_UART
+        default 1
+        range 1 32
+        help
+            UART task priority. In case of high events rate,
+            this parameter could be changed up to (configMAX_PRIORITIES-1).
+
     config APPTRACE_DEST_TRAX
         bool
-        depends on IDF_TARGET_ARCH_XTENSA && !ESP32_TRAX && !ESP32S2_TRAX
+        depends on IDF_TARGET_ARCH_XTENSA && !ESP32_TRAX && !ESP32S2_TRAX && !ESP32S3_TRAX
         select ESP32_MEMMAP_TRACEMEM
         select ESP32S2_MEMMAP_TRACEMEM
+        select ESP32S3_MEMMAP_TRACEMEM
         select ESP32_MEMMAP_TRACEMEM_TWOBANKS
         select ESP32S2_MEMMAP_TRACEMEM_TWOBANKS
+        select ESP32S3_MEMMAP_TRACEMEM_TWOBANKS
         default n
         help
             Enables/disable TRAX tracing HW.
@@ -88,11 +208,52 @@ menu "Application Level Tracing"
             help
                 Enables supporrt for SEGGER SystemView tracing functionality.
 
+        choice APPTRACE_SV_DEST
+            prompt "SystemView destination"
+            depends on APPTRACE_SV_ENABLE
+            default APPTRACE_SV_DEST_JTAG
+            help
+                SystemView witt transfer data trough defined interface.
+
+            config APPTRACE_SV_DEST_JTAG
+                bool "Data destination JTAG"
+                depends on  !PM_ENABLE && !APPTRACE_DEST_NONE
+                help
+                    Send SEGGER SystemView events through JTAG interface.
+
+            config APPTRACE_SV_DEST_UART
+                bool "Data destination UART"
+                depends on APPTRACE_DEST_UART
+                help
+                    Send SEGGER SystemView events through UART interface.
+
+        endchoice
+
+        choice APPTRACE_SV_CPU
+            prompt "CPU to trace"
+            depends on APPTRACE_SV_DEST_UART && !FREERTOS_UNICORE
+            default APPTRACE_SV_DEST_CPU_0
+            help
+                Define the CPU to trace by SystemView.
+
+            config APPTRACE_SV_DEST_CPU_0
+                bool "CPU0"
+                help
+                    Send SEGGER SystemView events for Pro CPU.
+
+            config APPTRACE_SV_DEST_CPU_1
+                bool "CPU1"
+                help
+                    Send SEGGER SystemView events for App CPU.
+
+        endchoice
+
+
         choice APPTRACE_SV_TS_SOURCE
             prompt "Timer to use as timestamp source"
             depends on APPTRACE_SV_ENABLE
             default APPTRACE_SV_TS_SOURCE_CCOUNT if FREERTOS_UNICORE && !PM_ENABLE && !IDF_TARGET_ESP32C3
-            default APPTRACE_SV_TS_SOURCE_TIMER_00 if !FREERTOS_UNICORE && !PM_ENABLE && !IDF_TARGET_ESP32C3
+            default APPTRACE_SV_TS_SOURCE_GPTIMER if !FREERTOS_UNICORE && !PM_ENABLE && !IDF_TARGET_ESP32C3
             default APPTRACE_SV_TS_SOURCE_ESP_TIMER if PM_ENABLE || IDF_TARGET_ESP32C3
             help
                 SystemView needs to use a hardware timer as the source of timestamps
@@ -102,20 +263,8 @@ menu "Application Level Tracing"
                 bool "CPU cycle counter (CCOUNT)"
                 depends on FREERTOS_UNICORE && !PM_ENABLE && !IDF_TARGET_ESP32C3
 
-            config APPTRACE_SV_TS_SOURCE_TIMER_00
-                bool "Timer 0, Group 0"
-                depends on !PM_ENABLE && !IDF_TARGET_ESP32C3
-
-            config APPTRACE_SV_TS_SOURCE_TIMER_01
-                bool "Timer 1, Group 0"
-                depends on !PM_ENABLE && !IDF_TARGET_ESP32C3
-
-            config APPTRACE_SV_TS_SOURCE_TIMER_10
-                bool "Timer 0, Group 1"
-                depends on !PM_ENABLE && !IDF_TARGET_ESP32C3
-
-            config APPTRACE_SV_TS_SOURCE_TIMER_11
-                bool "Timer 1, Group 1"
+            config APPTRACE_SV_TS_SOURCE_GPTIMER
+                bool "General Purpose Timer (Timer Group)"
                 depends on !PM_ENABLE && !IDF_TARGET_ESP32C3
 
             config APPTRACE_SV_TS_SOURCE_ESP_TIMER

components/app_trace/app_trace.c

@@ -5,24 +5,38 @@
  */
 
 #include <string.h>
+#include "esp_cpu.h"
 #include "esp_log.h"
 #include "esp_app_trace.h"
 #include "esp_app_trace_port.h"
+#include "esp_private/startup_internal.h"
 
-#define ESP_APPTRACE_MAX_VPRINTF_ARGS           256
-#define ESP_APPTRACE_HOST_BUF_SIZE              256
+#ifdef CONFIG_APPTRACE_DEST_UART0
+#define ESP_APPTRACE_DEST_UART_NUM 0
+#elif CONFIG_APPTRACE_DEST_UART1
+#define ESP_APPTRACE_DEST_UART_NUM 1
+#elif CONFIG_APPTRACE_DEST_UART2
+#define ESP_APPTRACE_DEST_UART_NUM 2
+#elif CONFIG_APPTRACE_DEST_USB_CDC
+#define ESP_APPTRACE_DEST_UART_NUM 10
+#else
+#define ESP_APPTRACE_DEST_UART_NUM 0
+#endif
 
-#define ESP_APPTRACE_PRINT_LOCK                 0
+#define ESP_APPTRACE_MAX_VPRINTF_ARGS 256
+#define ESP_APPTRACE_HOST_BUF_SIZE 256
+
+#define ESP_APPTRACE_PRINT_LOCK 0
 
 const static char *TAG = "esp_apptrace";
 
 /** tracing module internal data */
 typedef struct {
-    esp_apptrace_hw_t * hw;
-    void *              hw_data;
+    esp_apptrace_hw_t *hw;
+    void               *hw_data;
 } esp_apptrace_channel_t;
 
-static esp_apptrace_channel_t   s_trace_channels[ESP_APPTRACE_DEST_NUM];
+static esp_apptrace_channel_t   s_trace_channels[ESP_APPTRACE_DEST_MAX];
 static bool s_inited;
 
 esp_err_t esp_apptrace_init(void)
@@ -32,7 +46,7 @@ esp_err_t esp_apptrace_init(void)
     void *hw_data = NULL;
 
     // 'esp_apptrace_init()' is called on every core, so ensure to do main initialization only once
-    if (cpu_hal_get_core_id() == 0) {
+    if (esp_cpu_get_core_id() == 0) {
         memset(&s_trace_channels, 0, sizeof(s_trace_channels));
         hw = esp_apptrace_jtag_hw_get(&hw_data);
         ESP_APPTRACE_LOGD("HW interface %p", hw);
@@ -40,16 +54,16 @@ esp_err_t esp_apptrace_init(void)
             s_trace_channels[ESP_APPTRACE_DEST_JTAG].hw = hw;
             s_trace_channels[ESP_APPTRACE_DEST_JTAG].hw_data = hw_data;
         }
-        hw = esp_apptrace_uart_hw_get(0, &hw_data);
+        hw = esp_apptrace_uart_hw_get(ESP_APPTRACE_DEST_UART_NUM, &hw_data);
         if (hw != NULL) {
-            s_trace_channels[ESP_APPTRACE_DEST_UART0].hw = hw;
-            s_trace_channels[ESP_APPTRACE_DEST_UART0].hw_data = hw_data;
+            s_trace_channels[ESP_APPTRACE_DEST_UART].hw = hw;
+            s_trace_channels[ESP_APPTRACE_DEST_UART].hw_data = hw_data;
         }
         s_inited = true;
     }
 
     // esp_apptrace_init() is called on every core, so initialize trace channel on every core
-    for (int i = 0; i < sizeof(s_trace_channels)/sizeof(s_trace_channels[0]); i++) {
+    for (int i = 0; i < sizeof(s_trace_channels) / sizeof(s_trace_channels[0]); i++) {
         esp_apptrace_channel_t *ch = &s_trace_channels[i];
         if (ch->hw) {
             res = ch->hw->init(ch->hw_data);
@@ -63,6 +77,11 @@ esp_err_t esp_apptrace_init(void)
     return ESP_OK;
 }
 
+ESP_SYSTEM_INIT_FN(esp_apptrace_init, ESP_SYSTEM_INIT_ALL_CORES, 115)
+{
+    return esp_apptrace_init();
+}
+
 void esp_apptrace_down_buffer_config(uint8_t *buf, uint32_t size)
 {
     esp_apptrace_channel_t *ch;
@@ -73,15 +92,21 @@ void esp_apptrace_down_buffer_config(uint8_t *buf, uint32_t size)
     // currently down buffer is supported for JTAG interface only
     // TODO: one more argument should be added to this function to specify HW inteface: JTAG, UART0 etc
     ch = &s_trace_channels[ESP_APPTRACE_DEST_JTAG];
-    if (ch->hw == NULL) {
-        ESP_APPTRACE_LOGE("Trace destination not supported!");
-        return;
+    if (ch->hw != NULL) {
+        if (ch->hw->down_buffer_config != NULL) {
+            ch->hw->down_buffer_config(ch->hw_data, buf, size);
+        }
+    } else {
+        ESP_APPTRACE_LOGD("Trace destination for JTAG not supported!");
     }
-    if (ch->hw->down_buffer_config == NULL) {
-        return;
+    ch = &s_trace_channels[ESP_APPTRACE_DEST_UART];
+    if (ch->hw != NULL) {
+        if (ch->hw->down_buffer_config != NULL) {
+            ch->hw->down_buffer_config(ch->hw_data, buf, size);
+        }
+    } else {
+        ESP_APPTRACE_LOGD("Trace destination for UART not supported!");
     }
-
-    ch->hw->down_buffer_config(ch->hw_data, buf, size);
 }
 
 uint8_t *esp_apptrace_down_buffer_get(esp_apptrace_dest_t dest, uint32_t *size, uint32_t user_tmo)
@@ -169,7 +194,7 @@ esp_err_t esp_apptrace_read(esp_apptrace_dest_t dest, void *buf, uint32_t *size,
     esp_apptrace_tmo_init(&tmo, user_tmo);
     uint32_t act_sz = *size;
     *size = 0;
-    uint8_t * ptr = ch->hw->get_down_buffer(ch->hw_data, &act_sz, &tmo);
+    uint8_t *ptr = ch->hw->get_down_buffer(ch->hw_data, &act_sz, &tmo);
     if (ptr && act_sz > 0) {
         ESP_APPTRACE_LOGD("Read %d bytes from host", act_sz);
         memcpy(buf, ptr, act_sz);
@@ -419,3 +444,10 @@ bool esp_apptrace_host_is_connected(esp_apptrace_dest_t dest)
 
     return ch->hw->host_is_connected(ch->hw_data);
 }
+
+#if !CONFIG_APPTRACE_DEST_JTAG
+esp_apptrace_hw_t *esp_apptrace_jtag_hw_get(void **data)
+{
+    return NULL;
+}
+#endif

components/app_trace/app_trace_membufs_proto.c

@@ -1,7 +1,14 @@
+/*
+ * SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
 #include <sys/param.h>
 #include <string.h>
 #include "sdkconfig.h"
 #include "esp_log.h"
+#include "esp_cpu.h"
 #include "esp_app_trace_membufs_proto.h"
 
 /** Trace data header. Every user data chunk is prepended with this header.
@@ -93,7 +100,8 @@ static esp_err_t esp_apptrace_membufs_swap(esp_apptrace_membufs_proto_data_t *pr
     if (proto->hw->host_data_pending() && hdr->block_sz > 0) {
         // TODO: add support for multiple blocks from host, currently there is no need for that
         uint8_t *p = proto->blocks[new_block_num].start + proto->blocks[new_block_num].sz;
-        ESP_APPTRACE_LOGD("Recvd %d bytes from host [%x %x %x %x %x %x %x %x .. %x %x %x %x %x %x %x %x]", hdr->block_sz,
+        ESP_APPTRACE_LOGD("Recvd %d bytes from host (@ 0x%x) [%x %x %x %x %x %x %x %x .. %x %x %x %x %x %x %x %x]",
+            hdr->block_sz, proto->blocks[new_block_num].start,
             *(proto->blocks[new_block_num].start+0), *(proto->blocks[new_block_num].start+1),
             *(proto->blocks[new_block_num].start+2), *(proto->blocks[new_block_num].start+3),
             *(proto->blocks[new_block_num].start+4), *(proto->blocks[new_block_num].start+5),
@@ -252,8 +260,8 @@ static inline uint8_t *esp_apptrace_membufs_wait4buf(esp_apptrace_membufs_proto_
 
 static inline uint8_t *esp_apptrace_membufs_pkt_start(uint8_t *ptr, uint16_t size)
 {
-    // it is safe to use cpu_hal_get_core_id() in macro call because arg is used only once inside it
-    ((esp_tracedata_hdr_t *)ptr)->block_sz = ESP_APPTRACE_USR_BLOCK_CORE(cpu_hal_get_core_id()) | size;
+    // it is safe to use esp_cpu_get_core_id() in macro call because arg is used only once inside it
+    ((esp_tracedata_hdr_t *)ptr)->block_sz = ESP_APPTRACE_USR_BLOCK_CORE(esp_cpu_get_core_id()) | size;
     ((esp_tracedata_hdr_t *)ptr)->wr_sz = 0;
     return ptr + sizeof(esp_tracedata_hdr_t);
 }

components/app_trace/app_trace_util.c

@@ -60,22 +60,24 @@ esp_err_t esp_apptrace_lock_take(esp_apptrace_lock_t *lock, esp_apptrace_tmo_t *
     int res;
 
     while (1) {
+        //Todo: Replace the current locking mechanism and int_state with portTRY_ENTER_CRITICAL() instead.
         // do not overwrite lock->int_state before we actually acquired the mux
-        unsigned int_state = portENTER_CRITICAL_NESTED();
-        // FIXME: if mux is busy it is not good idea to loop during the whole tmo with disabled IRQs.
-        // So we check mux state using zero tmo, restore IRQs and let others tasks/IRQs to run on this CPU
-        // while we are doing our own tmo check.
-#ifdef CONFIG_FREERTOS_PORTMUX_DEBUG
-        bool success = vPortCPUAcquireMutexTimeout(&lock->mux, 0, __FUNCTION__, __LINE__);
+#if CONFIG_FREERTOS_SMP
+        unsigned int_state = portDISABLE_INTERRUPTS();
 #else
-        bool success = vPortCPUAcquireMutexTimeout(&lock->mux, 0);
+        unsigned int_state = portSET_INTERRUPT_MASK_FROM_ISR();
 #endif
+        bool success = spinlock_acquire(&lock->mux, 0);
         if (success) {
             lock->int_state = int_state;
             return ESP_OK;
         }
-        portEXIT_CRITICAL_NESTED(int_state);
-        // we can be preempted from this place till the next call (above) to portENTER_CRITICAL_NESTED()
+#if CONFIG_FREERTOS_SMP
+        portRESTORE_INTERRUPTS(int_state);
+#else
+        portCLEAR_INTERRUPT_MASK_FROM_ISR(int_state);
+#endif
+        // we can be preempted from this place till the next call (above) to portSET_INTERRUPT_MASK_FROM_ISR()
         res = esp_apptrace_tmo_check(tmo);
         if (res != ESP_OK) {
             break;
@@ -90,12 +92,12 @@ esp_err_t esp_apptrace_lock_give(esp_apptrace_lock_t *lock)
     unsigned int_state = lock->int_state;
     // after call to the following func we can not be sure that lock->int_state
     // is not overwritten by other CPU who has acquired the mux just after we released it. See esp_apptrace_lock_take().
-#ifdef CONFIG_FREERTOS_PORTMUX_DEBUG
-    vPortCPUReleaseMutex(&lock->mux, __FUNCTION__, __LINE__);
+    spinlock_release(&lock->mux);
+#if CONFIG_FREERTOS_SMP
+    portRESTORE_INTERRUPTS(int_state);
 #else
-    vPortCPUReleaseMutex(&lock->mux);
+    portCLEAR_INTERRUPT_MASK_FROM_ISR(int_state);
 #endif
-    portEXIT_CRITICAL_NESTED(int_state);
     return ESP_OK;
 }
 

components/app_trace/component.mk

@@ -1,39 +0,0 @@
-#
-# Component Makefile
-#
-
-COMPONENT_SRCDIRS := .
-
-ifdef CONFIG_APPTRACE_MEMBUFS_APPTRACE_PROTO_ENABLE
-COMPONENT_SRCDIRS += port/xtensa
-endif
-
-COMPONENT_ADD_INCLUDEDIRS = include
-
-COMPONENT_PRIV_INCLUDEDIRS = private_include \
-	port/include
-
-COMPONENT_ADD_LDFLAGS = -lapp_trace
-
-# do not produce gcov info for this module, it is used as transport for gcov
-CFLAGS := $(subst --coverage,,$(CFLAGS))
-
-ifdef CONFIG_APPTRACE_SV_ENABLE
-
-COMPONENT_ADD_INCLUDEDIRS += \
-	sys_view/Config \
-	sys_view/SEGGER \
-	sys_view/Sample/OS
-
-COMPONENT_SRCDIRS += \
-	gcov \
-	sys_view/SEGGER \
-	sys_view/Sample/OS \
-	sys_view/Sample/Config \
-	sys_view/esp \
-	sys_view/ext
-else
-COMPONENT_SRCDIRS += gcov
-endif
-
-COMPONENT_ADD_LDFRAGMENTS += linker.lf

components/app_trace/gcov/gcov_rtio.c

@@ -11,7 +11,6 @@
 #include "freertos/FreeRTOS.h"
 #include "freertos/task.h"
 #include "freertos/semphr.h"
-#include "soc/cpu.h"
 #include "soc/timer_periph.h"
 #include "esp_app_trace.h"
 #include "esp_freertos_hooks.h"
@@ -52,18 +51,11 @@ void gcov_dump_task(void *pvParameter)
     }
     ESP_EARLY_LOGV(TAG, "Config apptrace down buf");
     esp_apptrace_down_buffer_config(down_buf, ESP_GCOV_DOWN_BUF_SIZE);
-    /* we are directing the std outputs to the fake ones in order to reduce stack usage */
-    FILE *old_stderr = stderr;
-    FILE *old_stdout = stdout;
-    stderr = (FILE *) &__sf_fake_stderr;
-    stdout = (FILE *) &__sf_fake_stdout;
     ESP_EARLY_LOGV(TAG, "Dump data...");
     __gcov_dump();
     // reset dump status to allow incremental data accumulation
     __gcov_reset();
     free(down_buf);
-    stderr = old_stderr;
-    stdout = old_stdout;
     ESP_EARLY_LOGV(TAG, "Finish file transfer session");
     dump_result = esp_apptrace_fstop(ESP_APPTRACE_DEST_TRAX);
     if (dump_result != ESP_OK) {

components/app_trace/include/esp_app_trace.h

@@ -20,8 +20,8 @@ extern "C" {
 typedef enum {
     ESP_APPTRACE_DEST_JTAG = 1,                         ///< JTAG destination
     ESP_APPTRACE_DEST_TRAX = ESP_APPTRACE_DEST_JTAG, 	///< xxx_TRAX name is obsolete, use more common xxx_JTAG
-    ESP_APPTRACE_DEST_UART0,	                        ///< UART0 destination
-    ESP_APPTRACE_DEST_MAX = ESP_APPTRACE_DEST_UART0,
+    ESP_APPTRACE_DEST_UART,	                            ///< UART destination
+    ESP_APPTRACE_DEST_MAX = ESP_APPTRACE_DEST_UART+1,
     ESP_APPTRACE_DEST_NUM
 } esp_apptrace_dest_t;
 
@@ -36,7 +36,7 @@ esp_err_t esp_apptrace_init(void);
 
 /**
  * @brief Configures down buffer.
- *        @note Needs to be called before initiating any data transfer using esp_apptrace_buffer_get and esp_apptrace_write.
+ *        @note Needs to be called before attempting to receive any data using esp_apptrace_down_buffer_get and esp_apptrace_read.
  *              This function does not protect internal data by lock.
  *
  * @param buf Address of buffer to use for down channel (host to target) data.
@@ -46,7 +46,7 @@ void esp_apptrace_down_buffer_config(uint8_t *buf, uint32_t size);
 
 /**
  * @brief Allocates buffer for trace data.
- *        After data in buffer are ready to be sent off esp_apptrace_buffer_put must be called to indicate it.
+ *        Once the data in the buffer is ready to be sent, esp_apptrace_buffer_put must be called to indicate it.
  *
  * @param dest Indicates HW interface to send data.
  * @param size Size of data to write to trace buffer.
@@ -57,8 +57,8 @@ void esp_apptrace_down_buffer_config(uint8_t *buf, uint32_t size);
 uint8_t *esp_apptrace_buffer_get(esp_apptrace_dest_t dest, uint32_t size, uint32_t tmo);
 
 /**
- * @brief Indicates that the data in buffer are ready to be sent off.
- *        This function is a counterpart of and must be preceeded by esp_apptrace_buffer_get.
+ * @brief Indicates that the data in the buffer is ready to be sent.
+ *        This function is a counterpart of and must be preceded by esp_apptrace_buffer_get.
  *
  * @param dest Indicates HW interface to send data. Should be identical to the same parameter in call to esp_apptrace_buffer_get.
  * @param ptr  Address of trace buffer to release. Should be the value returned by call to esp_apptrace_buffer_get.
@@ -81,7 +81,7 @@ esp_err_t esp_apptrace_buffer_put(esp_apptrace_dest_t dest, uint8_t *ptr, uint32
 esp_err_t esp_apptrace_write(esp_apptrace_dest_t dest, const void *data, uint32_t size, uint32_t tmo);
 
 /**
- * @brief vprintf-like function to sent log messages to host via specified HW interface.
+ * @brief vprintf-like function to send log messages to host via specified HW interface.
  *
  * @param dest Indicates HW interface to send data.
  * @param tmo  Timeout for operation (in us). Use ESP_APPTRACE_TMO_INFINITE to wait indefinitely.
@@ -93,7 +93,7 @@ esp_err_t esp_apptrace_write(esp_apptrace_dest_t dest, const void *data, uint32_
 int esp_apptrace_vprintf_to(esp_apptrace_dest_t dest, uint32_t tmo, const char *fmt, va_list ap);
 
 /**
- * @brief vprintf-like function to sent log messages to host.
+ * @brief vprintf-like function to send log messages to host.
  *
  * @param fmt  Address of format string.
  * @param ap   List of arguments.
@@ -114,7 +114,7 @@ esp_err_t esp_apptrace_flush(esp_apptrace_dest_t dest, uint32_t tmo);
 
 /**
  * @brief Flushes remaining data in trace buffer to host without locking internal data.
- *        This is special version of esp_apptrace_flush which should be called from panic handler.
+ *        This is a special version of esp_apptrace_flush which should be called from panic handler.
  *
  * @param dest   Indicates HW interface to flush data on.
  * @param min_sz Threshold for flushing data. If current filling level is above this value, data will be flushed. TRAX destinations only.
@@ -138,7 +138,7 @@ esp_err_t esp_apptrace_read(esp_apptrace_dest_t dest, void *data, uint32_t *size
 
 /**
  * @brief Retrieves incoming data buffer if any.
- *        After data in buffer are processed esp_apptrace_down_buffer_put must be called to indicate it.
+ *        Once data in the buffer is processed, esp_apptrace_down_buffer_put must be called to indicate it.
  *
  * @param dest Indicates HW interface to receive data.
  * @param size Address to store size of available data in down buffer. Must be initialized with requested value.
@@ -149,8 +149,8 @@ esp_err_t esp_apptrace_read(esp_apptrace_dest_t dest, void *data, uint32_t *size
 uint8_t *esp_apptrace_down_buffer_get(esp_apptrace_dest_t dest, uint32_t *size, uint32_t tmo);
 
 /**
- * @brief Indicates that the data in down buffer are processed.
- *        This function is a counterpart of and must be preceeded by esp_apptrace_down_buffer_get.
+ * @brief Indicates that the data in the down buffer is processed.
+ *        This function is a counterpart of and must be preceded by esp_apptrace_down_buffer_get.
  *
  * @param dest Indicates HW interface to receive data. Should be identical to the same parameter in call to esp_apptrace_down_buffer_get.
  * @param ptr  Address of trace buffer to release. Should be the value returned by call to esp_apptrace_down_buffer_get.
@@ -245,7 +245,7 @@ int esp_apptrace_fseek(esp_apptrace_dest_t dest, void *stream, long offset, int
 int esp_apptrace_ftell(esp_apptrace_dest_t dest, void *stream);
 
 /**
- * @brief Indicates to the host that all file operations are completed.
+ * @brief Indicates to the host that all file operations are complete.
  *		  This function should be called after all file operations are finished and
  *		  indicate to the host that it can perform cleanup operations (close open files etc.).
  *

components/app_trace/include/esp_app_trace_util.h

@@ -57,7 +57,7 @@ static inline uint32_t esp_apptrace_tmo_remaining_us(esp_apptrace_tmo_t *tmo)
 
 /** Tracing module synchronization lock */
 typedef struct {
-    portMUX_TYPE mux;
+    spinlock_t mux;
     unsigned int_state;
 } esp_apptrace_lock_t;
 
@@ -68,7 +68,7 @@ typedef struct {
  */
 static inline void esp_apptrace_lock_init(esp_apptrace_lock_t *lock)
 {
-    vPortCPUInitializeMutex(&lock->mux);
+    portMUX_INITIALIZE(&lock->mux);
     lock->int_state = 0;
 }
 

components/app_trace/linker.lf

@@ -2,7 +2,12 @@
 archive: libapp_trace.a
 entries:
     app_trace (noflash)
+    port_uart (noflash)
     app_trace_util (noflash)
+    if APPTRACE_MEMBUFS_APPTRACE_PROTO_ENABLE:
+      app_trace_membufs_proto (noflash)
+    if APPTRACE_DEST_JTAG = y:
+      port (noflash)
     if APPTRACE_SV_ENABLE = y:
       SEGGER_SYSVIEW (noflash)
       SEGGER_RTT_esp (noflash)
@@ -12,8 +17,7 @@ entries:
 [mapping:app_trace_driver]
 archive: libdriver.a
 entries:
-    if APPTRACE_SV_TS_SOURCE_TIMER_00 = y || APPTRACE_SV_TS_SOURCE_TIMER_01 = y
-       || APPTRACE_SV_TS_SOURCE_TIMER_10 = y || APPTRACE_SV_TS_SOURCE_TIMER_11 = y:
-      timer (noflash)
+    if APPTRACE_SV_TS_SOURCE_GPTIMER = y:
+      gptimer (noflash)
     else:
       * (default)

components/app_trace/port/include/esp_app_trace_port.h

@@ -1,16 +1,9 @@
-// Copyright 2020 Espressif Systems (Shanghai) PTE LTD
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
+/*
+ * SPDX-FileCopyrightText: 2020-2022 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
 #ifndef ESP_APP_TRACE_PORT_H_
 #define ESP_APP_TRACE_PORT_H_
 

components/app_trace/port/port_uart.c

@@ -0,0 +1,348 @@
+/*
+ * SPDX-FileCopyrightText: 2017-2021 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#include "soc/soc.h"
+#include "esp_log.h"
+#include "esp_cpu.h"
+#include "esp_app_trace_port.h"
+
+#include "driver/uart.h"
+#include "hal/uart_ll.h"
+#include "string.h"
+#include "driver/gpio.h"
+
+
+#define APPTRACE_DEST_UART (CONFIG_APPTRACE_DEST_UART0 | CONFIG_APPTRACE_DEST_UART1 | CONFIG_APPTRACE_DEST_UART2)
+
+#define APP_TRACE_MAX_TX_BUFF_UART          CONFIG_APPTRACE_UART_TX_BUFF_SIZE
+#define APP_TRACE_MAX_TX_MSG_UART           CONFIG_APPTRACE_UART_TX_MSG_SIZE
+
+/** UART HW transport data */
+typedef struct {
+    uint8_t                             inited;
+#if CONFIG_APPTRACE_LOCK_ENABLE
+    esp_apptrace_lock_t                 lock;   // sync lock
+#endif
+    uart_port_t port_num;
+// TX data ring buffer
+    uint8_t *tx_data_buff;
+    int32_t tx_data_buff_in;
+    int32_t tx_data_buff_out;
+// TX message buffer
+    uint8_t *tx_msg_buff;
+    uint32_t tx_msg_buff_size;
+
+// RX message buffer
+    uint8_t *down_buffer;
+    uint32_t down_buffer_size;
+// Buffer overflow flags
+    bool message_buff_overflow;
+    bool circular_buff_overflow;
+} esp_apptrace_uart_data_t;
+
+#if APPTRACE_DEST_UART
+static esp_err_t esp_apptrace_uart_init(esp_apptrace_uart_data_t *hw_data);
+static esp_err_t esp_apptrace_uart_flush(esp_apptrace_uart_data_t *hw_data, esp_apptrace_tmo_t *tmo);
+static esp_err_t esp_apptrace_uart_flush_nolock(esp_apptrace_uart_data_t *hw_data, uint32_t min_sz, esp_apptrace_tmo_t *tmo);
+static uint8_t *esp_apptrace_uart_up_buffer_get(esp_apptrace_uart_data_t *hw_data, uint32_t size, esp_apptrace_tmo_t *tmo);
+static esp_err_t esp_apptrace_uart_up_buffer_put(esp_apptrace_uart_data_t *hw_data, uint8_t *ptr, esp_apptrace_tmo_t *tmo);
+static void esp_apptrace_uart_down_buffer_config(esp_apptrace_uart_data_t *hw_data, uint8_t *buf, uint32_t size);
+static uint8_t *esp_apptrace_uart_down_buffer_get(esp_apptrace_uart_data_t *hw_data, uint32_t *size, esp_apptrace_tmo_t *tmo);
+static esp_err_t esp_apptrace_uart_down_buffer_put(esp_apptrace_uart_data_t *hw_data, uint8_t *ptr, esp_apptrace_tmo_t *tmo);
+static bool esp_apptrace_uart_host_is_connected(esp_apptrace_uart_data_t *hw_data);
+
+#endif // APPTRACE_DEST_UART
+const static char *TAG = "esp_apptrace_uart";
+
+esp_apptrace_hw_t *esp_apptrace_uart_hw_get(int num, void **data)
+{
+    ESP_LOGD(TAG,"esp_apptrace_uart_hw_get - %i", num);
+#if APPTRACE_DEST_UART
+    static esp_apptrace_uart_data_t s_uart_hw_data = {
+    };
+    static esp_apptrace_hw_t s_uart_hw = {
+        .init = (esp_err_t (*)(void *))esp_apptrace_uart_init,
+        .get_up_buffer = (uint8_t *(*)(void *, uint32_t, esp_apptrace_tmo_t *))esp_apptrace_uart_up_buffer_get,
+        .put_up_buffer = (esp_err_t (*)(void *, uint8_t *, esp_apptrace_tmo_t *))esp_apptrace_uart_up_buffer_put,
+        .flush_up_buffer_nolock = (esp_err_t (*)(void *, uint32_t, esp_apptrace_tmo_t *))esp_apptrace_uart_flush_nolock,
+        .flush_up_buffer = (esp_err_t (*)(void *, esp_apptrace_tmo_t *))esp_apptrace_uart_flush,
+        .down_buffer_config = (void (*)(void *, uint8_t *, uint32_t ))esp_apptrace_uart_down_buffer_config,
+        .get_down_buffer = (uint8_t *(*)(void *, uint32_t *, esp_apptrace_tmo_t *))esp_apptrace_uart_down_buffer_get,
+        .put_down_buffer = (esp_err_t (*)(void *, uint8_t *, esp_apptrace_tmo_t *))esp_apptrace_uart_down_buffer_put,
+        .host_is_connected = (bool (*)(void *))esp_apptrace_uart_host_is_connected,
+    };
+    s_uart_hw_data.port_num = num;
+    *data = &s_uart_hw_data;
+    return &s_uart_hw;
+#else
+    return NULL;
+#endif
+}
+
+#if APPTRACE_DEST_UART
+
+static esp_err_t esp_apptrace_uart_lock(esp_apptrace_uart_data_t *hw_data, esp_apptrace_tmo_t *tmo)
+{
+#if CONFIG_APPTRACE_LOCK_ENABLE
+    esp_err_t ret = esp_apptrace_lock_take(&hw_data->lock, tmo);
+    if (ret != ESP_OK) {
+        return ESP_FAIL;
+    }
+#endif
+    return ESP_OK;
+}
+
+static esp_err_t esp_apptrace_uart_unlock(esp_apptrace_uart_data_t *hw_data)
+{
+    esp_err_t ret = ESP_OK;
+#if CONFIG_APPTRACE_LOCK_ENABLE
+    ret = esp_apptrace_lock_give(&hw_data->lock);
+#endif
+    return ret;
+}
+
+static inline void esp_apptrace_uart_hw_init(void)
+{
+    ESP_APPTRACE_LOGI("Initialized UART on CPU%d", esp_cpu_get_core_id());
+}
+
+
+/*****************************************************************************************/
+/***************************** Apptrace HW iface *****************************************/
+/*****************************************************************************************/
+
+static esp_err_t esp_apptrace_send_uart_data(esp_apptrace_uart_data_t *hw_data, const char *data, uint32_t size, esp_apptrace_tmo_t *tmo)
+{
+    esp_err_t res = esp_apptrace_uart_lock(hw_data, tmo);
+    if (res != ESP_OK) {
+        return res;
+    }
+    // We store current out position to handle it without lock
+    volatile int32_t out_position = hw_data->tx_data_buff_out;
+
+    int len_free = APP_TRACE_MAX_TX_BUFF_UART - (hw_data->tx_data_buff_in - out_position);
+    if (out_position > hw_data->tx_data_buff_in) {
+        len_free = out_position - hw_data->tx_data_buff_in;
+    }
+    int check_len = APP_TRACE_MAX_TX_BUFF_UART - hw_data->tx_data_buff_in;
+    if (size <= len_free)
+    {
+        if ( check_len >= size) {
+            memcpy(&hw_data->tx_data_buff[hw_data->tx_data_buff_in], data, size);
+            hw_data->tx_data_buff_in += size;
+        } else {
+            memcpy(&hw_data->tx_data_buff[hw_data->tx_data_buff_in], data, APP_TRACE_MAX_TX_BUFF_UART - hw_data->tx_data_buff_in);
+            memcpy(&hw_data->tx_data_buff[0], &data[APP_TRACE_MAX_TX_BUFF_UART - hw_data->tx_data_buff_in], size - (APP_TRACE_MAX_TX_BUFF_UART - hw_data->tx_data_buff_in));
+            hw_data->tx_data_buff_in = size - (APP_TRACE_MAX_TX_BUFF_UART - hw_data->tx_data_buff_in);
+        }
+        if (hw_data->tx_data_buff_in >= APP_TRACE_MAX_TX_BUFF_UART) {
+            hw_data->tx_data_buff_in = 0;
+        }
+    } else {
+        hw_data->circular_buff_overflow = true;
+    }
+
+    if (esp_apptrace_uart_unlock(hw_data) != ESP_OK) {
+        assert(false && "Failed to unlock apptrace data!");
+    }
+
+    return ESP_OK;
+}
+
+static void send_buff_data(esp_apptrace_uart_data_t *hw_data, esp_apptrace_tmo_t *tmo)
+{
+    if (hw_data->tx_data_buff_in == hw_data->tx_data_buff_out) {
+        return;
+    }
+    // We store current in position to handle it without lock
+    volatile int32_t in_position = hw_data->tx_data_buff_in;
+    if (in_position > hw_data->tx_data_buff_out) {
+        int bytes_sent = uart_write_bytes(hw_data->port_num, &hw_data->tx_data_buff[hw_data->tx_data_buff_out], in_position - hw_data->tx_data_buff_out);
+        hw_data->tx_data_buff_out += bytes_sent;
+    } else {
+        int bytes_sent = uart_write_bytes(hw_data->port_num, &hw_data->tx_data_buff[hw_data->tx_data_buff_out], APP_TRACE_MAX_TX_BUFF_UART - hw_data->tx_data_buff_out);
+        hw_data->tx_data_buff_out += bytes_sent;
+        if (hw_data->tx_data_buff_out >= APP_TRACE_MAX_TX_BUFF_UART) {
+            hw_data->tx_data_buff_out = 0;
+        }
+    }
+}
+
+#define APP_TRACE_UART_STOP_WAIT_TMO    1000000 //us
+
+static void esp_apptrace_send_uart_tx_task(void *arg)
+{
+    esp_apptrace_uart_data_t *hw_data = (esp_apptrace_uart_data_t *)arg;
+    esp_apptrace_tmo_t tmo;
+    esp_apptrace_tmo_init(&tmo, APP_TRACE_UART_STOP_WAIT_TMO);
+
+    vTaskDelay(10);
+    while (1) {
+        send_buff_data(hw_data, &tmo);
+        vTaskDelay(10);
+        if (hw_data->circular_buff_overflow == true)
+        {
+            hw_data->circular_buff_overflow = false;
+            ESP_LOGE(TAG, "Buffer overflow. Please increase UART baudrate, or increase UART TX ring buffer size in menuconfig.");
+        }
+        if (hw_data->message_buff_overflow == true)
+        {
+            hw_data->message_buff_overflow = false;
+            ESP_LOGE(TAG, "Message size more then message buffer!");
+        }
+    }
+}
+
+static const int APP_TRACE_UART_RX_BUF_SIZE = 4024;
+
+static esp_err_t esp_apptrace_uart_init(esp_apptrace_uart_data_t *hw_data)
+{
+    int core_id = esp_cpu_get_core_id();
+    if (core_id == 0) {
+        hw_data->tx_data_buff = (uint8_t *)heap_caps_malloc(APP_TRACE_MAX_TX_BUFF_UART, MALLOC_CAP_INTERNAL|MALLOC_CAP_8BIT);
+        if (hw_data->tx_data_buff == NULL){
+            return ESP_ERR_NO_MEM;
+        }
+        hw_data->tx_data_buff_in = 0;
+        hw_data->tx_data_buff_out = 0;
+        hw_data->tx_msg_buff =  (uint8_t *)heap_caps_malloc(APP_TRACE_MAX_TX_MSG_UART, MALLOC_CAP_INTERNAL|MALLOC_CAP_8BIT);
+        if (hw_data->tx_msg_buff == NULL)
+        {
+            return ESP_ERR_NO_MEM;
+        }
+        hw_data->tx_msg_buff_size = 0;
+        hw_data->down_buffer_size = 0;
+        hw_data->message_buff_overflow = false;
+        hw_data->circular_buff_overflow = false;
+
+
+        const uart_config_t uart_config = {
+            .baud_rate = CONFIG_APPTRACE_UART_BAUDRATE,
+            .data_bits = UART_DATA_8_BITS,
+            .parity = UART_PARITY_DISABLE,
+            .stop_bits = UART_STOP_BITS_1,
+            .flow_ctrl = UART_HW_FLOWCTRL_DISABLE,
+            .source_clk = UART_SCLK_DEFAULT,
+        };
+        ESP_LOGI(TAG, "UART baud rate: %i", CONFIG_APPTRACE_UART_BAUDRATE);
+        // We won't use a buffer for sending data.
+        esp_err_t err = uart_driver_install(hw_data->port_num, APP_TRACE_UART_RX_BUF_SIZE, APP_TRACE_UART_RX_BUF_SIZE, 0, NULL, 0);
+        assert((err == ESP_OK) && "Not possible to install UART. Please check and change menuconfig parameters!");
+        err = uart_param_config(hw_data->port_num, &uart_config);
+        assert((err == ESP_OK) && "Not possible to configure UART. Please check and change menuconfig parameters!");
+        err = uart_set_pin(hw_data->port_num, CONFIG_APPTRACE_UART_TX_GPIO, CONFIG_APPTRACE_UART_RX_GPIO, UART_PIN_NO_CHANGE, UART_PIN_NO_CHANGE);
+        assert((err == ESP_OK) && "Not possible to configure UART RX/TX pins. Please check and change menuconfig parameters!");
+
+        int uart_prio = CONFIG_APPTRACE_UART_TASK_PRIO;
+        if (uart_prio >= (configMAX_PRIORITIES-1)) uart_prio = configMAX_PRIORITIES - 1;
+        err = xTaskCreate(esp_apptrace_send_uart_tx_task, "app_trace_uart_tx_task", 2500, hw_data, uart_prio, NULL);
+        assert((err == pdPASS) && "Not possible to configure UART. Not possible to create task!");
+
+#if CONFIG_APPTRACE_LOCK_ENABLE
+        esp_apptrace_lock_init(&hw_data->lock);
+#endif
+    }
+    // init UART on this CPU
+    esp_apptrace_uart_hw_init();
+    hw_data->inited |= 1 << core_id;
+
+    return ESP_OK;
+}
+
+static uint8_t *esp_apptrace_uart_up_buffer_get(esp_apptrace_uart_data_t *hw_data, uint32_t size, esp_apptrace_tmo_t *tmo)
+{
+    uint8_t *ptr;
+    if (size > APP_TRACE_MAX_TX_MSG_UART) {
+        hw_data->message_buff_overflow = true;
+        return NULL;
+    }
+    if (hw_data->tx_msg_buff_size != 0)
+    {
+        // A previous message was not sent.
+        return NULL;
+    }
+    esp_err_t res = esp_apptrace_uart_lock(hw_data, tmo);
+    if (res != ESP_OK) {
+        return NULL;
+    }
+    ptr = hw_data->tx_msg_buff;
+    hw_data->tx_msg_buff_size = size;
+
+    // now we can safely unlock apptrace to allow other tasks/ISRs to get other buffers and write their data
+    if (esp_apptrace_uart_unlock(hw_data) != ESP_OK) {
+        assert(false && "Failed to unlock apptrace data!");
+    }
+    return ptr;
+}
+
+static esp_err_t esp_apptrace_uart_up_buffer_put(esp_apptrace_uart_data_t *hw_data, uint8_t *ptr, esp_apptrace_tmo_t *tmo)
+{
+    esp_err_t res = esp_apptrace_send_uart_data(hw_data, (const char *)ptr, hw_data->tx_msg_buff_size, tmo);
+    // Clear size to indicate that we've sent data
+    hw_data->tx_msg_buff_size = 0;
+    return res;
+}
+
+static void esp_apptrace_uart_down_buffer_config(esp_apptrace_uart_data_t *hw_data, uint8_t *buf, uint32_t size)
+{
+    hw_data->down_buffer = (uint8_t *)malloc(size);
+    if (hw_data->down_buffer == NULL){
+        assert(false && "Failed to allocate apptrace uart down buffer!");
+    }
+    hw_data->down_buffer_size = size;
+}
+
+static uint8_t *esp_apptrace_uart_down_buffer_get(esp_apptrace_uart_data_t *hw_data, uint32_t *size, esp_apptrace_tmo_t *tmo)
+{
+    uint8_t *ptr = NULL;
+
+    if (*size > hw_data->down_buffer_size) {
+        return NULL;
+    }
+
+    esp_err_t res = esp_apptrace_uart_lock(hw_data, tmo);
+    if (res != ESP_OK) {
+        return NULL;
+    }
+
+    size_t uart_fifolen = 0;
+    uart_get_buffered_data_len(hw_data->port_num, &uart_fifolen);
+    if (uart_fifolen > 0) {
+        if (*size < uart_fifolen) {
+            uart_fifolen = *size;
+        }
+        *size = uart_fifolen;
+        ptr = hw_data->down_buffer;
+        *size =uart_read_bytes(hw_data->port_num, ptr, uart_fifolen, 0);
+    }
+
+    if (esp_apptrace_uart_unlock(hw_data) != ESP_OK) {
+        assert(false && "Failed to unlock apptrace data!");
+    }
+    return ptr;
+}
+
+static esp_err_t esp_apptrace_uart_down_buffer_put(esp_apptrace_uart_data_t *hw_data, uint8_t *ptr, esp_apptrace_tmo_t *tmo)
+{
+    return ESP_OK;
+}
+
+static bool esp_apptrace_uart_host_is_connected(esp_apptrace_uart_data_t *hw_data)
+{
+    return hw_data->inited & 1;
+}
+
+static esp_err_t esp_apptrace_uart_flush_nolock(esp_apptrace_uart_data_t *hw_data, uint32_t min_sz, esp_apptrace_tmo_t *tmo)
+{
+    return ESP_OK;
+}
+
+static esp_err_t esp_apptrace_uart_flush(esp_apptrace_uart_data_t *hw_data, esp_apptrace_tmo_t *tmo)
+{
+    return ESP_OK;
+}
+
+#endif // APPTRACE_DEST_UART

components/app_trace/port/riscv/port.c

@@ -1,6 +1,14 @@
+/*
+ * SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#include "esp_cpu.h"
 #include "esp_log.h"
 #include "esp_app_trace_membufs_proto.h"
 #include "esp_app_trace_port.h"
+#include "riscv/semihosting.h"
 
 /** RISCV HW transport data */
 typedef struct {
@@ -20,8 +28,6 @@ typedef struct {
     esp_apptrace_mem_block_t *  mem_blocks;
 } esp_apptrace_riscv_ctrl_block_t;
 
-#define RISCV_APPTRACE_SYSNR    0x64
-
 #define ESP_APPTRACE_RISCV_BLOCK_LEN_MSK         0x7FFFUL
 #define ESP_APPTRACE_RISCV_BLOCK_LEN(_l_)        ((_l_) & ESP_APPTRACE_RISCV_BLOCK_LEN_MSK)
 #define ESP_APPTRACE_RISCV_BLOCK_LEN_GET(_v_)    ((_v_) & ESP_APPTRACE_RISCV_BLOCK_LEN_MSK)
@@ -31,7 +37,7 @@ typedef struct {
 #define ESP_APPTRACE_RISCV_HOST_DATA             (1 << 22)
 #define ESP_APPTRACE_RISCV_HOST_CONNECT          (1 << 23)
 
-#define ESP_APPTRACE_RISCV_INITED(_hw_)          ((_hw_)->inited & (1 << 0/*cpu_hal_get_core_id()*/))
+#define ESP_APPTRACE_RISCV_INITED(_hw_)          ((_hw_)->inited & (1 << 0/*esp_cpu_get_core_id()*/))
 
 static esp_err_t esp_apptrace_riscv_init(esp_apptrace_riscv_data_t *hw_data);
 static esp_err_t esp_apptrace_riscv_flush(esp_apptrace_riscv_data_t *hw_data, esp_apptrace_tmo_t *tmo);
@@ -52,11 +58,6 @@ const static char *TAG = "esp_apptrace";
 
 static esp_apptrace_riscv_ctrl_block_t s_tracing_ctrl[portNUM_PROCESSORS];
 
-esp_apptrace_hw_t *esp_apptrace_uart_hw_get(int num, void **data)
-{
-    return NULL;
-}
-
 esp_apptrace_hw_t *esp_apptrace_jtag_hw_get(void **data)
 {
 #if CONFIG_APPTRACE_DEST_JTAG
@@ -94,25 +95,10 @@ esp_apptrace_hw_t *esp_apptrace_jtag_hw_get(void **data)
    e.g. OpenOCD flasher stub use own implementation of it. */
 __attribute__((weak)) int esp_apptrace_advertise_ctrl_block(void *ctrl_block_addr)
 {
-    register int sys_nr = RISCV_APPTRACE_SYSNR;
-    register int host_ret = 0;
-
-    if (!esp_cpu_in_ocd_debug_mode()) {
+    if (!esp_cpu_dbgr_is_attached()) {
         return 0;
     }
-    __asm__ volatile ( \
-        ".option push\n" \
-        ".option norvc\n" \
-        "mv a0, %[sys_nr]\n" \
-        "mv a1, %[arg1]\n" \
-        "slli    zero,zero,0x1f\n" \
-        "ebreak\n" \
-        "srai    zero,zero,0x7\n" \
-        "mv %[host_ret], a0\n" \
-    	".option pop\n" \
-        :[host_ret]"=r"(host_ret)
-        :[sys_nr]"r"(sys_nr),[arg1]"r"(ctrl_block_addr):"a0","a1");
-    return host_ret;
+    return (int) semihosting_call_noerrno(ESP_SEMIHOSTING_SYS_APPTRACE_INIT, (long*)ctrl_block_addr);
 }
 
 /* Returns up buffers config.
@@ -154,7 +140,7 @@ static esp_err_t esp_apptrace_riscv_unlock(esp_apptrace_riscv_data_t *hw_data)
 
 static esp_err_t esp_apptrace_riscv_init(esp_apptrace_riscv_data_t *hw_data)
 {
-    int core_id = cpu_hal_get_core_id();
+    int core_id = esp_cpu_get_core_id();
 
     if (hw_data->inited == 0) {
         esp_apptrace_mem_block_t mem_blocks_cfg[2];
@@ -268,7 +254,7 @@ static bool esp_apptrace_riscv_host_is_connected(esp_apptrace_riscv_data_t *hw_d
     if (!ESP_APPTRACE_RISCV_INITED(hw_data)) {
         return false;
     }
-    return s_tracing_ctrl[cpu_hal_get_core_id()].ctrl & ESP_APPTRACE_RISCV_HOST_CONNECT ? true : false;
+    return s_tracing_ctrl[esp_cpu_get_core_id()].ctrl & ESP_APPTRACE_RISCV_HOST_CONNECT ? true : false;
 }
 
 static esp_err_t esp_apptrace_riscv_flush_nolock(esp_apptrace_riscv_data_t *hw_data, uint32_t min_sz, esp_apptrace_tmo_t *tmo)
@@ -312,13 +298,13 @@ static inline void esp_apptrace_riscv_buffer_swap_lock(void)
     // HACK: in this case host will set breakpoint just after ESP_APPTRACE_RISCV_CTRL_REG update,
     // here we set address to set bp at
     // enter ERI update critical section
-    s_tracing_ctrl[cpu_hal_get_core_id()].stat = (uint32_t)&__esp_apptrace_riscv_updated;
+    s_tracing_ctrl[esp_cpu_get_core_id()].stat = (uint32_t)&__esp_apptrace_riscv_updated;
 }
 
 static __attribute__((noinline)) void esp_apptrace_riscv_buffer_swap_unlock(void)
 {
     // exit ERI update critical section
-    s_tracing_ctrl[cpu_hal_get_core_id()].stat = 0;
+    s_tracing_ctrl[esp_cpu_get_core_id()].stat = 0;
     // TODO: currently host sets breakpoint, use break instruction to stop;
     // it will allow to use ESP_APPTRACE_RISCV_STAT_REG for other purposes
     asm volatile (
@@ -332,13 +318,13 @@ static esp_err_t esp_apptrace_riscv_buffer_swap_start(uint32_t curr_block_id)
 
     esp_apptrace_riscv_buffer_swap_lock();
 
-    uint32_t ctrl_reg = s_tracing_ctrl[cpu_hal_get_core_id()].ctrl;
+    uint32_t ctrl_reg = s_tracing_ctrl[esp_cpu_get_core_id()].ctrl;
     uint32_t host_connected = ESP_APPTRACE_RISCV_HOST_CONNECT & ctrl_reg;
     if (host_connected) {
         uint32_t acked_block = ESP_APPTRACE_RISCV_BLOCK_ID_GET(ctrl_reg);
         uint32_t host_to_read = ESP_APPTRACE_RISCV_BLOCK_LEN_GET(ctrl_reg);
         if (host_to_read != 0 || acked_block != (curr_block_id & ESP_APPTRACE_RISCV_BLOCK_ID_MSK)) {
-            ESP_APPTRACE_LOGD("[%d]: Can not switch %x %d %x %x/%lx", cpu_hal_get_core_id(), ctrl_reg, host_to_read, acked_block,
+            ESP_APPTRACE_LOGD("[%d]: Can not switch %x %d %x %x/%lx", esp_cpu_get_core_id(), ctrl_reg, host_to_read, acked_block,
                 curr_block_id & ESP_APPTRACE_RISCV_BLOCK_ID_MSK, curr_block_id);
             res = ESP_ERR_NO_MEM;
             goto _on_err;
@@ -352,9 +338,9 @@ _on_err:
 
 static esp_err_t esp_apptrace_riscv_buffer_swap_end(uint32_t new_block_id, uint32_t prev_block_len)
 {
-    uint32_t ctrl_reg = s_tracing_ctrl[cpu_hal_get_core_id()].ctrl;
+    uint32_t ctrl_reg = s_tracing_ctrl[esp_cpu_get_core_id()].ctrl;
     uint32_t host_connected = ESP_APPTRACE_RISCV_HOST_CONNECT & ctrl_reg;
-    s_tracing_ctrl[cpu_hal_get_core_id()].ctrl = ESP_APPTRACE_RISCV_BLOCK_ID(new_block_id) |
+    s_tracing_ctrl[esp_cpu_get_core_id()].ctrl = ESP_APPTRACE_RISCV_BLOCK_ID(new_block_id) |
               host_connected | ESP_APPTRACE_RISCV_BLOCK_LEN(prev_block_len);
     esp_apptrace_riscv_buffer_swap_unlock();
     return ESP_OK;
@@ -368,7 +354,7 @@ static esp_err_t esp_apptrace_riscv_buffer_swap(uint32_t new_block_id)
 
 static bool esp_apptrace_riscv_host_data_pending(void)
 {
-    uint32_t ctrl_reg = s_tracing_ctrl[cpu_hal_get_core_id()].ctrl;
+    uint32_t ctrl_reg = s_tracing_ctrl[esp_cpu_get_core_id()].ctrl;
     // ESP_APPTRACE_LOGV("%s() 0x%x", __func__, ctrl_reg);
     return (ctrl_reg & ESP_APPTRACE_RISCV_HOST_DATA) ? true : false;
 }

components/app_trace/port/xtensa/port.c

@@ -1,3 +1,9 @@
+/*
+ * SPDX-FileCopyrightText: 2015-2021 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
 //
 // How It Works
 // ************
@@ -142,30 +148,18 @@
 // time exceeds specified timeout value operation is canceled and ESP_ERR_TIMEOUT code is returned.
 #include "sdkconfig.h"
 #include "soc/soc.h"
-#include "soc/dport_access.h"
-#if CONFIG_IDF_TARGET_ESP32
 #include "soc/dport_reg.h"
-#elif CONFIG_IDF_TARGET_ESP32S2
+#include "soc/tracemem_config.h"
+#if CONFIG_IDF_TARGET_ESP32S2 || CONFIG_IDF_TARGET_ESP32S3
 #include "soc/sensitive_reg.h"
 #endif
 #include "eri.h"
-#include "trax.h"
+#include "esp_private/trax.h"
+#include "esp_cpu.h"
 #include "esp_log.h"
 #include "esp_app_trace_membufs_proto.h"
 #include "esp_app_trace_port.h"
 
-// TODO: move these (and same definitions in trax.c to dport_reg.h)
-#if CONFIG_IDF_TARGET_ESP32
-#define TRACEMEM_MUX_PROBLK0_APPBLK1            0
-#define TRACEMEM_MUX_BLK0_ONLY                  1
-#define TRACEMEM_MUX_BLK1_ONLY                  2
-#define TRACEMEM_MUX_PROBLK1_APPBLK0            3
-#elif CONFIG_IDF_TARGET_ESP32S2
-#define TRACEMEM_MUX_BLK0_NUM                   19
-#define TRACEMEM_MUX_BLK1_NUM                   20
-#define TRACEMEM_BLK_NUM2ADDR(_n_)              (0x3FFB8000UL + 0x4000UL*((_n_)-4))
-#endif
-
 // TRAX is disabled, so we use its registers for our own purposes
 // | 31..XXXXXX..24 | 23 .(host_connect). 23 | 22 .(host_data). 22| 21..(block_id)..15 | 14..(block_len)..0 |
 #define ESP_APPTRACE_TRAX_CTRL_REG              ERI_TRAX_DELAYCNT
@@ -180,19 +174,7 @@
 #define ESP_APPTRACE_TRAX_HOST_DATA             (1 << 22)
 #define ESP_APPTRACE_TRAX_HOST_CONNECT          (1 << 23)
 
-#define ESP_APPTRACE_TRAX_INITED(_hw_)          ((_hw_)->inited & (1 << cpu_hal_get_core_id()))
-
-#if CONFIG_IDF_TARGET_ESP32
-static uint8_t * const s_trax_blocks[] = {
-    (uint8_t *) 0x3FFFC000,
-    (uint8_t *) 0x3FFF8000
-};
-#elif CONFIG_IDF_TARGET_ESP32S2
-static uint8_t * const s_trax_blocks[] = {
-    (uint8_t *)TRACEMEM_BLK_NUM2ADDR(TRACEMEM_MUX_BLK0_NUM),
-    (uint8_t *)TRACEMEM_BLK_NUM2ADDR(TRACEMEM_MUX_BLK1_NUM)
-};
-#endif
+#define ESP_APPTRACE_TRAX_INITED(_hw_)          ((_hw_)->inited & (1 << esp_cpu_get_core_id()))
 
 #define ESP_APPTRACE_TRAX_BLOCK_SIZE            (0x4000UL)
 
@@ -223,10 +205,10 @@ static bool esp_apptrace_trax_host_data_pending(void);
 
 const static char *TAG = "esp_apptrace";
 
-esp_apptrace_hw_t *esp_apptrace_uart_hw_get(int num, void **data)
-{
-    return NULL;
-}
+static uint8_t * const s_trax_blocks[] = {
+    (uint8_t *)TRACEMEM_BLK0_ADDR,
+    (uint8_t *)TRACEMEM_BLK1_ADDR
+};
 
 esp_apptrace_hw_t *esp_apptrace_jtag_hw_get(void **data)
 {
@@ -290,7 +272,7 @@ static inline void esp_apptrace_trax_hw_init(void)
     // must be read by host before any transfer using TRAX
     eri_write(ESP_APPTRACE_TRAX_STAT_REG, 0);
 
-    ESP_APPTRACE_LOGI("Initialized TRAX on CPU%d", cpu_hal_get_core_id());
+    ESP_APPTRACE_LOGI("Initialized TRAX on CPU%d", esp_cpu_get_core_id());
 }
 
 static inline void esp_apptrace_trax_select_memory_block(int block_num)
@@ -300,6 +282,14 @@ static inline void esp_apptrace_trax_select_memory_block(int block_num)
     DPORT_WRITE_PERI_REG(DPORT_TRACEMEM_MUX_MODE_REG, block_num ? TRACEMEM_MUX_BLK0_ONLY : TRACEMEM_MUX_BLK1_ONLY);
 #elif CONFIG_IDF_TARGET_ESP32S2
     WRITE_PERI_REG(DPORT_PMS_OCCUPY_3_REG, block_num ? BIT(TRACEMEM_MUX_BLK0_NUM-4) : BIT(TRACEMEM_MUX_BLK1_NUM-4));
+#elif CONFIG_IDF_TARGET_ESP32S3
+    // select memory block to be exposed to the TRAX module (accessed by host)
+    uint32_t block_bits = block_num ? TRACEMEM_CORE0_MUX_BLK_BITS(TRACEMEM_MUX_BLK0_NUM)
+                        : TRACEMEM_CORE0_MUX_BLK_BITS(TRACEMEM_MUX_BLK1_NUM);
+    block_bits |= block_num ? TRACEMEM_CORE1_MUX_BLK_BITS(TRACEMEM_MUX_BLK0_NUM)
+                        : TRACEMEM_CORE1_MUX_BLK_BITS(TRACEMEM_MUX_BLK1_NUM);
+    ESP_EARLY_LOGV(TAG, "Select block %d @ %p (bits 0x%x)", block_num, s_trax_blocks[block_num], block_bits);
+    DPORT_WRITE_PERI_REG(SENSITIVE_INTERNAL_SRAM_USAGE_2_REG, block_bits);
 #endif
 }
 
@@ -321,7 +311,7 @@ static inline void esp_apptrace_trax_memory_enable(void)
 
 static esp_err_t esp_apptrace_trax_init(esp_apptrace_trax_data_t *hw_data)
 {
-    int core_id = cpu_hal_get_core_id();
+    int core_id = esp_cpu_get_core_id();
 
     // 'esp_apptrace_trax_init()' is called on every core, so ensure to do main initialization only once
     if (core_id == 0) {
@@ -508,7 +498,7 @@ static esp_err_t esp_apptrace_trax_buffer_swap_start(uint32_t curr_block_id)
         uint32_t acked_block = ESP_APPTRACE_TRAX_BLOCK_ID_GET(ctrl_reg);
         uint32_t host_to_read = ESP_APPTRACE_TRAX_BLOCK_LEN_GET(ctrl_reg);
         if (host_to_read != 0 || acked_block != (curr_block_id & ESP_APPTRACE_TRAX_BLOCK_ID_MSK)) {
-            ESP_APPTRACE_LOGD("HC[%d]: Can not switch %x %d %x %x/%lx", cpu_hal_get_core_id(), ctrl_reg, host_to_read, acked_block,
+            ESP_APPTRACE_LOGD("HC[%d]: Can not switch %x %d %x %x/%lx", esp_cpu_get_core_id(), ctrl_reg, host_to_read, acked_block,
                 curr_block_id & ESP_APPTRACE_TRAX_BLOCK_ID_MSK, curr_block_id);
             res = ESP_ERR_NO_MEM;
             goto _on_err;

components/app_trace/private_include/esp_app_trace_membufs_proto.h

@@ -1,16 +1,9 @@
-// Copyright 2020 Espressif Systems (Shanghai) PTE LTD
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
+/*
+ * SPDX-FileCopyrightText: 2020-2022 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
 #ifndef ESP_APP_TRACE_MEMBUFS_PROTO_H_
 #define ESP_APP_TRACE_MEMBUFS_PROTO_H_
 

components/app_trace/project_include.cmake

@@ -2,7 +2,7 @@
 #
 # Create coverage report.
 function(idf_create_coverage_report report_dir)
-    set(gcov_tool ${CONFIG_SDK_TOOLPREFIX}gcov)
+    set(gcov_tool ${_CMAKE_TOOLCHAIN_PREFIX}gcov)
     idf_build_get_property(project_name PROJECT_NAME)
 
     add_custom_target(pre-cov-report

components/app_trace/sdkconfig.rename

@@ -14,10 +14,6 @@ CONFIG_ESP32_GCOV_ENABLE								CONFIG_APPTRACE_GCOV_ENABLE
 CONFIG_SYSVIEW_ENABLE							        CONFIG_APPTRACE_SV_ENABLE
 CONFIG_SYSVIEW_TS_SOURCE							    CONFIG_APPTRACE_SV_TS_SOURCE
 CONFIG_SYSVIEW_TS_SOURCE_CCOUNT							CONFIG_APPTRACE_SV_TS_SOURCE_CCOUNT
-CONFIG_SYSVIEW_TS_SOURCE_TIMER_00						CONFIG_APPTRACE_SV_TS_SOURCE_TIMER_00
-CONFIG_SYSVIEW_TS_SOURCE_TIMER_01						CONFIG_APPTRACE_SV_TS_SOURCE_TIMER_01
-CONFIG_SYSVIEW_TS_SOURCE_TIMER_10						CONFIG_APPTRACE_SV_TS_SOURCE_TIMER_10
-CONFIG_SYSVIEW_TS_SOURCE_TIMER_11						CONFIG_APPTRACE_SV_TS_SOURCE_TIMER_11
 CONFIG_SYSVIEW_TS_SOURCE_ESP_TIMER						CONFIG_APPTRACE_SV_TS_SOURCE_ESP_TIMER
 CONFIG_SYSVIEW_MAX_TASKS							    CONFIG_APPTRACE_SV_MAX_TASKS
 CONFIG_SYSVIEW_BUF_WAIT_TMO							    CONFIG_APPTRACE_SV_BUF_WAIT_TMO

components/app_trace/sys_view/Config/Global.h

@@ -1,3 +1,8 @@
+/*
+ * SPDX-FileCopyrightText: 2015-2017 SEGGER Microcontroller GmbH & Co. KG
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
 /*********************************************************************
 *                SEGGER Microcontroller GmbH & Co. KG                *
 *                        The Embedded Experts                        *

components/app_trace/sys_view/Config/SEGGER_RTT_Conf.h

@@ -1,3 +1,8 @@
+/*
+ * SPDX-FileCopyrightText: 2015-2017 SEGGER Microcontroller GmbH & Co. KG
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
 /*********************************************************************
 *                SEGGER Microcontroller GmbH & Co. KG                *
 *                        The Embedded Experts                        *

components/app_trace/sys_view/Config/SEGGER_SYSVIEW_Conf.h

@@ -1,3 +1,8 @@
+/*
+ * SPDX-FileCopyrightText: 2015-2017 SEGGER Microcontroller GmbH & Co. KG
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
 /*********************************************************************
 *                SEGGER Microcontroller GmbH & Co. KG                *
 *                        The Embedded Experts                        *

components/app_trace/sys_view/SEGGER/SEGGER.h

@@ -1,3 +1,8 @@
+/*
+ * SPDX-FileCopyrightText: 2015-2017 SEGGER Microcontroller GmbH & Co. KG
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
 /*********************************************************************
 *                SEGGER Microcontroller GmbH & Co. KG                *
 *                        The Embedded Experts                        *

components/app_trace/sys_view/SEGGER/SEGGER_RTT.h

@@ -1,3 +1,8 @@
+/*
+ * SPDX-FileCopyrightText: 2015-2017 SEGGER Microcontroller GmbH & Co. KG
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
 /*********************************************************************
 *                SEGGER Microcontroller GmbH & Co. KG                *
 *                        The Embedded Experts                        *

components/app_trace/sys_view/SEGGER/SEGGER_SYSVIEW.c

@@ -1,3 +1,8 @@
+/*
+ * SPDX-FileCopyrightText: 2015-2017 SEGGER Microcontroller GmbH & Co. KG
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
 /*********************************************************************
 *                SEGGER Microcontroller GmbH & Co. KG                *
 *                        The Embedded Experts                        *
@@ -1681,7 +1686,8 @@ void SEGGER_SYSVIEW_Start(void) {
 void SEGGER_SYSVIEW_Stop(void) {
   U8* pPayloadStart;
   RECORD_START(SEGGER_SYSVIEW_INFO_SIZE);
-  //
+  // We should send answer for the Stop command in any case.
+  _SYSVIEW_Globals.EnableState = 1;
   if (_SYSVIEW_Globals.EnableState) {
     _SendPacket(pPayloadStart, pPayloadStart, SYSVIEW_EVTID_TRACE_STOP);
     _SYSVIEW_Globals.EnableState = 0;
@@ -1801,6 +1807,10 @@ void SEGGER_SYSVIEW_SendSysDesc(const char *sSysDesc) {
 */
 void SEGGER_SYSVIEW_RecordSystime(void) {
   U64 Systime;
+    // This code requeued because SystemView expect the answer from the device.
+    // If there is no answer, then communication will be broken.
+    U8 old_en = _SYSVIEW_Globals.EnableState;
+    _SYSVIEW_Globals.EnableState = 1;
 
   if (_SYSVIEW_Globals.pOSAPI && _SYSVIEW_Globals.pOSAPI->pfGetTime) {
     Systime = _SYSVIEW_Globals.pOSAPI->pfGetTime();
@@ -1810,6 +1820,7 @@ void SEGGER_SYSVIEW_RecordSystime(void) {
   } else {
     SEGGER_SYSVIEW_RecordU32(SYSVIEW_EVTID_SYSTIME_CYCLES, SEGGER_SYSVIEW_GET_TIMESTAMP());
   }
+    _SYSVIEW_Globals.EnableState = old_en;
 }
 
 /*********************************************************************

components/app_trace/sys_view/SEGGER/SEGGER_SYSVIEW.h

@@ -1,3 +1,8 @@
+/*
+ * SPDX-FileCopyrightText: 2015-2017 SEGGER Microcontroller GmbH & Co. KG
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
 /*********************************************************************
 *                SEGGER Microcontroller GmbH & Co. KG                *
 *                        The Embedded Experts                        *

components/app_trace/sys_view/SEGGER/SEGGER_SYSVIEW_ConfDefaults.h

@@ -1,3 +1,8 @@
+/*
+ * SPDX-FileCopyrightText: 2015-2017 SEGGER Microcontroller GmbH & Co. KG
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
 /*********************************************************************
 *                SEGGER Microcontroller GmbH & Co. KG                *
 *                        The Embedded Experts                        *

components/app_trace/sys_view/SEGGER/SEGGER_SYSVIEW_Int.h

@@ -1,3 +1,8 @@
+/*
+ * SPDX-FileCopyrightText: 2015-2017 SEGGER Microcontroller GmbH & Co. KG
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
 /*********************************************************************
 *                SEGGER Microcontroller GmbH & Co. KG                *
 *                        The Embedded Experts                        *

components/app_trace/sys_view/Sample/Config/SEGGER_SYSVIEW_Config_FreeRTOS.c

@@ -1,3 +1,10 @@
+/*
+ * SPDX-FileCopyrightText: 2015-2017 SEGGER Microcontroller GmbH & Co. KG
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ *
+ * SPDX-FileContributor: 2017-2022 Espressif Systems (Shanghai) CO LTD
+ */
 /*********************************************************************
 *                SEGGER Microcontroller GmbH & Co. KG                *
 *                        The Embedded Experts                        *
@@ -69,14 +76,7 @@ Revision: $Rev: 3734 $
 #include "esp_intr_alloc.h"
 #include "soc/soc.h"
 #include "soc/interrupts.h"
-#if CONFIG_IDF_TARGET_ESP32
-#include "esp32/clk.h"
-#elif CONFIG_IDF_TARGET_ESP32S2
-#include "esp32s2/clk.h"
-#elif CONFIG_IDF_TARGET_ESP32C3
-#include "esp32c3/clk.h"
-#endif
-
+#include "esp_private/esp_clk.h"
 
 extern const SEGGER_SYSVIEW_OS_API SYSVIEW_X_OS_TraceAPI;
 
@@ -108,26 +108,16 @@ extern const SEGGER_SYSVIEW_OS_API SYSVIEW_X_OS_TraceAPI;
 #endif
 
 #if TS_USE_TIMERGROUP
-#include "driver/timer.h"
+#include "driver/gptimer.h"
 
 // Timer group timer divisor
 #define SYSVIEW_TIMER_DIV       2
 
-// Frequency of the timestamp.
+// Frequency of the timestamp, using APB as GPTimer source clock
 #define SYSVIEW_TIMESTAMP_FREQ  (esp_clk_apb_freq() / SYSVIEW_TIMER_DIV)
 
-// Timer ID and group ID
-#if defined(CONFIG_APPTRACE_SV_TS_SOURCE_TIMER_00) || defined(CONFIG_APPTRACE_SV_TS_SOURCE_TIMER_10)
-#define TS_TIMER_ID 0
-#else
-#define TS_TIMER_ID 1
-#endif // TIMER_00 || TIMER_01
-
-#if defined(CONFIG_APPTRACE_SV_TS_SOURCE_TIMER_00) || defined(CONFIG_APPTRACE_SV_TS_SOURCE_TIMER_01)
-#define TS_TIMER_GROUP 0
-#else
-#define TS_TIMER_GROUP 1
-#endif // TIMER_00 || TIMER_10
+// GPTimer handle
+gptimer_handle_t s_sv_gptimer;
 
 #endif // TS_USE_TIMERGROUP
 
@@ -138,11 +128,7 @@ extern const SEGGER_SYSVIEW_OS_API SYSVIEW_X_OS_TraceAPI;
 
 #if TS_USE_CCOUNT
 // CCOUNT is incremented at CPU frequency
-#if CONFIG_IDF_TARGET_ESP32
-#define SYSVIEW_TIMESTAMP_FREQ  (CONFIG_ESP32_DEFAULT_CPU_FREQ_MHZ * 1000000)
-#elif CONFIG_IDF_TARGET_ESP32S2
-#define SYSVIEW_TIMESTAMP_FREQ  (CONFIG_ESP32S2_DEFAULT_CPU_FREQ_MHZ * 1000000)
-#endif
+#define SYSVIEW_TIMESTAMP_FREQ  (CONFIG_ESP_DEFAULT_CPU_FREQ_MHZ * 1000000)
 #endif // TS_USE_CCOUNT
 
 // System Frequency.
@@ -204,19 +190,16 @@ static void SEGGER_SYSVIEW_TS_Init(void)
      * esp_timer and ccount can be used as is.
      */
 #if TS_USE_TIMERGROUP
-    timer_config_t config = {
-        .alarm_en = 0,
-        .auto_reload = 0,
-        .counter_dir = TIMER_COUNT_UP,
-        .divider = SYSVIEW_TIMER_DIV,
-        .counter_en = 0
+    gptimer_config_t config = {
+        .clk_src = GPTIMER_CLK_SRC_DEFAULT,
+        .direction = GPTIMER_COUNT_UP,
+        .resolution_hz = SYSVIEW_TIMESTAMP_FREQ,
     };
-    /* Configure timer */
-    timer_init(TS_TIMER_GROUP, TS_TIMER_ID, &config);
-    /* Load counter value */
-    timer_set_counter_value(TS_TIMER_GROUP, TS_TIMER_ID, 0x00000000ULL);
+    // pick any free GPTimer instance
+    ESP_ERROR_CHECK(gptimer_new_timer(&config, &s_sv_gptimer));
     /* Start counting */
-    timer_start(TS_TIMER_GROUP, TS_TIMER_ID);
+    gptimer_enable(s_sv_gptimer);
+    gptimer_start(s_sv_gptimer);
 #endif // TS_USE_TIMERGROUP
 }
 
@@ -274,7 +257,7 @@ U32 SEGGER_SYSVIEW_X_GetTimestamp(void)
 {
 #if TS_USE_TIMERGROUP
     uint64_t ts = 0;
-    timer_get_counter_value(TS_TIMER_GROUP, TS_TIMER_ID, &ts);
+    gptimer_get_raw_count(s_sv_gptimer, &ts);
     return (U32) ts; // return lower part of counter value
 #elif TS_USE_CCOUNT
     return portGET_RUN_TIME_COUNTER_VALUE();

components/app_trace/sys_view/Sample/OS/SEGGER_SYSVIEW_FreeRTOS.c

@@ -1,3 +1,8 @@
+/*
+ * SPDX-FileCopyrightText: 2015-2017 SEGGER Microcontroller GmbH & Co. KG
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
 /*********************************************************************
 *                SEGGER Microcontroller GmbH & Co. KG                *
 *                        The Embedded Experts                        *

components/app_trace/sys_view/Sample/OS/SEGGER_SYSVIEW_FreeRTOS.h

@@ -1,3 +1,8 @@
+/*
+ * SPDX-FileCopyrightText: 2015-2017 SEGGER Microcontroller GmbH & Co. KG
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
 /*********************************************************************
 *                SEGGER Microcontroller GmbH & Co. KG                *
 *                        The Embedded Experts                        *
@@ -76,9 +81,6 @@ Notes:
 *
 **********************************************************************
 */
-#ifndef portSTACK_GROWTH
-  #define portSTACK_GROWTH              ( -1 )
-#endif
 
 #define SYSVIEW_FREERTOS_MAX_NOF_TASKS  CONFIG_APPTRACE_SV_MAX_TASKS
 
@@ -209,6 +211,56 @@ Notes:
 #define apiID_VEVENTGROUPDELETE                   (72u)
 #define apiID_UXEVENTGROUPGETNUMBER               (73u)
 
+#ifdef CONFIG_FREERTOS_SMP
+/*
+FreeRTOS SMP has diverged from ESP-IDF FreeRTOS source quite a bit, thus Sysview is out of sync. For now, we just
+define away all of the tracing macros.
+*/
+#define traceTASK_NOTIFY_TAKE( uxIndexToWait )
+#define traceTASK_DELAY()
+#define traceTASK_DELAY_UNTIL( xTimeToWake )
+#define traceTASK_DELETE( pxTCB )
+#define traceTASK_NOTIFY_GIVE_FROM_ISR( uxIndexToNotify )
+#define traceTASK_PRIORITY_INHERIT( pxTCB, uxPriority )
+#define traceTASK_RESUME( pxTCB )
+#define traceINCREASE_TICK_COUNT( xTicksToJump )
+#define traceTASK_SUSPEND( pxTCB )
+#define traceTASK_PRIORITY_DISINHERIT( pxTCB, uxBasePriority )
+#define traceTASK_RESUME_FROM_ISR( pxTCB )
+#define traceTASK_NOTIFY( uxIndexToNotify )
+#define traceTASK_NOTIFY_FROM_ISR( uxIndexToNotify )
+#define traceTASK_NOTIFY_WAIT( uxIndexToWait )
+#define traceQUEUE_CREATE( pxNewQueue )
+#define traceQUEUE_DELETE( pxQueue )
+#define traceQUEUE_PEEK( pxQueue )
+#define traceQUEUE_PEEK_FROM_ISR( pxQueue )
+#define traceQUEUE_PEEK_FROM_ISR_FAILED( pxQueue )
+#define traceQUEUE_RECEIVE( pxQueue )
+#define traceQUEUE_RECEIVE_FAILED( pxQueue )
+#define traceQUEUE_SEMAPHORE_RECEIVE( pxQueue )
+#define traceQUEUE_RECEIVE_FROM_ISR( pxQueue )
+#define traceQUEUE_RECEIVE_FROM_ISR_FAILED( pxQueue )
+#define traceQUEUE_REGISTRY_ADD( xQueue, pcQueueName )
+#define traceQUEUE_SEND( pxQueue )
+#define traceQUEUE_SEND_FAILED( pxQueue )
+#define traceQUEUE_SEND_FROM_ISR( pxQueue )
+#define traceQUEUE_SEND_FROM_ISR_FAILED( pxQueue )
+#define traceQUEUE_GIVE_FROM_ISR( pxQueue )
+#define traceQUEUE_GIVE_FROM_ISR_FAILED( pxQueue )
+#define traceTASK_CREATE(pxNewTCB)
+#define traceTASK_PRIORITY_SET(pxTask, uxNewPriority)
+#define traceTASK_SWITCHED_IN()
+#define traceMOVED_TASK_TO_READY_STATE(pxTCB)
+#define traceREADDED_TASK_TO_READY_STATE(pxTCB)
+#define traceMOVED_TASK_TO_DELAYED_LIST()
+#define traceMOVED_TASK_TO_OVERFLOW_DELAYED_LIST()
+#define traceMOVED_TASK_TO_SUSPENDED_LIST(pxTCB)
+#define traceISR_EXIT_TO_SCHEDULER()
+#define traceISR_EXIT()
+#define traceISR_ENTER(_n_)
+
+#else // CONFIG_FREERTOS_SMP
+
 #define traceTASK_NOTIFY_TAKE( uxIndexToWait )                        SEGGER_SYSVIEW_RecordU32x2(apiFastID_OFFSET + apiID_ULTASKNOTIFYTAKE, xClearCountOnExit, xTicksToWait)
 #define traceTASK_DELAY()                                             SEGGER_SYSVIEW_RecordU32(apiFastID_OFFSET + apiID_VTASKDELAY, xTicksToDelay)
 #define traceTASK_DELAY_UNTIL()                                       SEGGER_SYSVIEW_RecordVoid(apiFastID_OFFSET + apiID_VTASKDELAYUNTIL)
@@ -290,12 +342,12 @@ Notes:
   #define traceTASK_SWITCHED_IN()                   if(prvGetTCBFromHandle(NULL) == xTaskGetIdleTaskHandle()) {           \
                                                       SEGGER_SYSVIEW_OnIdle();                                          \
                                                     } else {                                                            \
-                                                      SEGGER_SYSVIEW_OnTaskStartExec((U32)pxCurrentTCB[cpu_hal_get_core_id()]); \
+                                                      SEGGER_SYSVIEW_OnTaskStartExec((U32)pxCurrentTCB[esp_cpu_get_core_id()]); \
                                                     }
 #else
   #define traceTASK_SWITCHED_IN()                   {                                                                   \
-                                                      if (memcmp(pxCurrentTCB[cpu_hal_get_core_id()]->pcTaskName, "IDLE", 5) != 0) { \
-                                                        SEGGER_SYSVIEW_OnTaskStartExec((U32)pxCurrentTCB[cpu_hal_get_core_id()]);    \
+                                                      if (memcmp(pxCurrentTCB[esp_cpu_get_core_id()]->pcTaskName, "IDLE", 5) != 0) { \
+                                                        SEGGER_SYSVIEW_OnTaskStartExec((U32)pxCurrentTCB[esp_cpu_get_core_id()]);    \
                                                       } else {                                                          \
                                                         SEGGER_SYSVIEW_OnIdle();                                        \
                                                       }                                                                 \
@@ -305,14 +357,15 @@ Notes:
 #define traceMOVED_TASK_TO_READY_STATE(pxTCB)       SEGGER_SYSVIEW_OnTaskStartReady((U32)pxTCB)
 #define traceREADDED_TASK_TO_READY_STATE(pxTCB)
 
-#define traceMOVED_TASK_TO_DELAYED_LIST()           SEGGER_SYSVIEW_OnTaskStopReady((U32)pxCurrentTCB[cpu_hal_get_core_id()],  (1u << 2))
-#define traceMOVED_TASK_TO_OVERFLOW_DELAYED_LIST()  SEGGER_SYSVIEW_OnTaskStopReady((U32)pxCurrentTCB[cpu_hal_get_core_id()],  (1u << 2))
+#define traceMOVED_TASK_TO_DELAYED_LIST()           SEGGER_SYSVIEW_OnTaskStopReady((U32)pxCurrentTCB[esp_cpu_get_core_id()],  (1u << 2))
+#define traceMOVED_TASK_TO_OVERFLOW_DELAYED_LIST()  SEGGER_SYSVIEW_OnTaskStopReady((U32)pxCurrentTCB[esp_cpu_get_core_id()],  (1u << 2))
 #define traceMOVED_TASK_TO_SUSPENDED_LIST(pxTCB)    SEGGER_SYSVIEW_OnTaskStopReady((U32)pxTCB,         ((3u << 3) | 3))
 
 #define traceISR_EXIT_TO_SCHEDULER()                SEGGER_SYSVIEW_RecordExitISRToScheduler()
 #define traceISR_EXIT()                             SEGGER_SYSVIEW_RecordExitISR()
 #define traceISR_ENTER(_n_)                         SEGGER_SYSVIEW_RecordEnterISR(_n_)
 
+#endif // CONFIG_FREERTOS_SMP
 
 /*********************************************************************
 *

components/app_trace/sys_view/esp/SEGGER_RTT_esp.c

@@ -12,6 +12,8 @@
 
 #include "esp_app_trace.h"
 #include "esp_log.h"
+#include "esp_cpu.h"
+#include "esp_private/startup_internal.h"
 
 const static char *TAG = "segger_rtt";
 
@@ -30,6 +32,19 @@ static uint8_t s_events_buf[SYSVIEW_EVENTS_BUF_SZ];
 static uint16_t s_events_buf_filled;
 static uint8_t s_down_buf[SYSVIEW_DOWN_BUF_SIZE];
 
+#if CONFIG_APPTRACE_SV_DEST_UART
+
+#define ESP_APPTRACE_DEST_SYSVIEW ESP_APPTRACE_DEST_UART
+#if CONFIG_APPTRACE_SV_DEST_CPU_0 || CONFIG_FREERTOS_UNICORE
+#define APPTRACE_SV_DEST_CPU 0
+#else
+#define APPTRACE_SV_DEST_CPU 1
+#endif // CONFIG_APPTRACE_SV_DEST_CPU_0
+
+#elif CONFIG_APPTRACE_SV_DEST_JTAG || (CONFIG_APPTRACE_ENABLE && CONFIG_APPTRACE_DEST_UART_NONE)
+#define ESP_APPTRACE_DEST_SYSVIEW ESP_APPTRACE_DEST_TRAX
+#endif
+
 /*********************************************************************
 *
 *       Public code
@@ -55,13 +70,13 @@ void SEGGER_RTT_ESP_FlushNoLock(unsigned long min_sz, unsigned long tmo)
 {
     esp_err_t res;
     if (s_events_buf_filled > 0) {
-      res = esp_apptrace_write(ESP_APPTRACE_DEST_TRAX, s_events_buf, s_events_buf_filled, tmo);
+      res = esp_apptrace_write(ESP_APPTRACE_DEST_SYSVIEW, s_events_buf, s_events_buf_filled, tmo);
       if (res != ESP_OK) {
         ESP_LOGE(TAG, "Failed to flush buffered events (%d)!\n", res);
       }
     }
     // flush even if we failed to write buffered events, because no new events will be sent after STOP
-    res = esp_apptrace_flush_nolock(ESP_APPTRACE_DEST_TRAX, min_sz, tmo);
+    res = esp_apptrace_flush_nolock(ESP_APPTRACE_DEST_SYSVIEW, min_sz, tmo);
     if (res != ESP_OK) {
       ESP_LOGE(TAG, "Failed to flush apptrace data (%d)!\n", res);
     }
@@ -108,7 +123,7 @@ void SEGGER_RTT_ESP_Flush(unsigned long min_sz, unsigned long tmo)
 */
 unsigned SEGGER_RTT_ReadNoLock(unsigned BufferIndex, void* pData, unsigned BufferSize) {
   uint32_t size = BufferSize;
-  esp_err_t res = esp_apptrace_read(ESP_APPTRACE_DEST_TRAX, pData, &size, 0);
+  esp_err_t res = esp_apptrace_read(ESP_APPTRACE_DEST_SYSVIEW, pData, &size, 0);
   if (res != ESP_OK) {
     return 0;
   }
@@ -142,12 +157,40 @@ unsigned SEGGER_RTT_ReadNoLock(unsigned BufferIndex, void* pData, unsigned Buffe
 unsigned SEGGER_RTT_WriteSkipNoLock(unsigned BufferIndex, const void* pBuffer, unsigned NumBytes) {
   uint8_t *pbuf = (uint8_t *)pBuffer;
   uint8_t event_id = *pbuf;
+#if CONFIG_APPTRACE_SV_DEST_UART
+  if (
+    (APPTRACE_SV_DEST_CPU != esp_cpu_get_core_id()) &&
+    (
+      (event_id == SYSVIEW_EVTID_ISR_ENTER) ||
+      (event_id == SYSVIEW_EVTID_ISR_EXIT) ||
+      (event_id == SYSVIEW_EVTID_TASK_START_EXEC) ||
+      (event_id == SYSVIEW_EVTID_TASK_STOP_EXEC) ||
+      (event_id == SYSVIEW_EVTID_TASK_START_READY) ||
+      (event_id == SYSVIEW_EVTID_TASK_STOP_READY) ||
+      (event_id == SYSVIEW_EVTID_USER_START) ||
+      (event_id == SYSVIEW_EVTID_USER_STOP) ||
+      (event_id == SYSVIEW_EVTID_TIMER_ENTER) ||
+      (event_id == SYSVIEW_EVTID_TIMER_EXIT) ||
+      (event_id == SYSVIEW_EVTID_STACK_INFO) ||
+      (event_id == SYSVIEW_EVTID_MODULEDESC)
+    )
+  ){
+    return NumBytes;
+  }
+
+// This is workaround for SystemView!
+// Without this line SystemView will hangs on when heap tracing enabled.
+  if(event_id == SYSVIEW_EVTID_MODULEDESC){
+    return NumBytes;
+  }
+#endif // CONFIG_APPTRACE_SV_DEST_UART
 
   if (NumBytes > SYSVIEW_EVENTS_BUF_SZ) {
       ESP_LOGE(TAG, "Too large event %u bytes!", NumBytes);
       return 0;
   }
-  if (cpu_hal_get_core_id()) { // dual core specific code
+#if CONFIG_APPTRACE_SV_DEST_JTAG
+  if (esp_cpu_get_core_id()) { // dual core specific code
     // use the highest - 1 bit of event ID to indicate core ID
     // the highest bit can not be used due to event ID encoding method
     // this reduces supported ID range to [0..63] (for 1 byte IDs) plus [128..16383] (for 2 bytes IDs)
@@ -157,16 +200,31 @@ unsigned SEGGER_RTT_WriteSkipNoLock(unsigned BufferIndex, const void* pBuffer, u
       *pbuf |= (1 << 6);
     }
   }
+#endif // CONFIG_APPTRACE_SV_DEST_JTAG
+#if CONFIG_APPTRACE_SV_DEST_JTAG
   if (s_events_buf_filled + NumBytes > SYSVIEW_EVENTS_BUF_SZ) {
-    esp_err_t res = esp_apptrace_write(ESP_APPTRACE_DEST_TRAX, s_events_buf, s_events_buf_filled, SEGGER_HOST_WAIT_TMO);
+
+    esp_err_t res = esp_apptrace_write(ESP_APPTRACE_DEST_SYSVIEW, s_events_buf, s_events_buf_filled, SEGGER_HOST_WAIT_TMO);
     if (res != ESP_OK) {
       return 0; // skip current data buffer only, accumulated events are kept
     }
     s_events_buf_filled = 0;
   }
+#endif
   memcpy(&s_events_buf[s_events_buf_filled], pBuffer, NumBytes);
   s_events_buf_filled += NumBytes;
-  if (event_id == SYSVIEW_EVTID_TRACE_STOP) {
+
+#if CONFIG_APPTRACE_SV_DEST_UART
+  esp_err_t res = esp_apptrace_write(ESP_APPTRACE_DEST_SYSVIEW, pBuffer, NumBytes, SEGGER_HOST_WAIT_TMO);
+  if (res != ESP_OK)
+  {
+    return 0; // skip current data buffer only, accumulated events are kept
+  }
+  s_events_buf_filled = 0;
+#endif
+
+  if (event_id == SYSVIEW_EVTID_TRACE_STOP)
+  {
     SEGGER_RTT_ESP_FlushNoLock(0, SEGGER_STOP_WAIT_TMO);
   }
   return NumBytes;
@@ -232,4 +290,17 @@ int SEGGER_RTT_ConfigDownBuffer(unsigned BufferIndex, const char* sName, void* p
   return 0;
 }
 
+/*************************** Init hook ****************************
+ *
+ * This init function is placed here because this port file will be
+ * linked whenever SystemView is used.
+ */
+
+ESP_SYSTEM_INIT_FN(sysview_init, BIT(0), 120)
+{
+    SEGGER_SYSVIEW_Conf();
+    return ESP_OK;
+}
+
+
 /*************************** End of file ****************************/

components/app_trace/test/CMakeLists.txt

@@ -1,3 +1,4 @@
 idf_component_register(SRC_DIRS "."
                     PRIV_INCLUDE_DIRS "."
                     PRIV_REQUIRES cmock driver)
+target_compile_options(${COMPONENT_LIB} PRIVATE "-Wno-format")

components/app_trace/test/component.mk

@@ -1,5 +0,0 @@
-#
-#Component Makefile
-#
-
-COMPONENT_ADD_LDFLAGS = -Wl,--whole-archive -l$(COMPONENT_NAME) -Wl,--no-whole-archive

components/app_trace/test/test_trace.c

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: 2021 Espressif Systems (Shanghai) CO LTD
+ * SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
  *
  * SPDX-License-Identifier: Apache-2.0
  */
@@ -9,9 +9,10 @@
 #include <stdio.h>
 #include <stdarg.h>
 #include "unity.h"
-#include "driver/timer.h"
+#include "driver/gptimer.h"
+#include "esp_intr_alloc.h"
 #include "esp_rom_sys.h"
-#include "soc/cpu.h"
+#include "esp_cpu.h"
 #include "freertos/FreeRTOS.h"
 #include "freertos/semphr.h"
 #include "freertos/task.h"
@@ -65,29 +66,6 @@ const static char *TAG = "esp_apptrace_test";
 #define ESP_APPTRACE_TEST_LOGV( format, ... )  ESP_APPTRACE_TEST_LOG_LEVEL(V, ESP_LOG_VERBOSE, format, ##__VA_ARGS__)
 #define ESP_APPTRACE_TEST_LOGO( format, ... )  ESP_APPTRACE_TEST_LOG_LEVEL(E, ESP_LOG_NONE, format, ##__VA_ARGS__)
 
-static void esp_apptrace_test_timer_init(int timer_group, int timer_idx, uint32_t period)
-{
-    timer_config_t config;
-    uint64_t alarm_val = (period * (TIMER_BASE_CLK / 1000000UL)) / 2;
-
-    config.alarm_en = 1;
-    config.auto_reload = 1;
-    config.counter_dir = TIMER_COUNT_UP;
-    config.divider = 2;     //Range is 2 to 65536
-    config.intr_type = TIMER_INTR_LEVEL;
-    config.counter_en = TIMER_PAUSE;
-    /*Configure timer*/
-    timer_init(timer_group, timer_idx, &config);
-    /*Stop timer counter*/
-    timer_pause(timer_group, timer_idx);
-    /*Load counter value */
-    timer_set_counter_value(timer_group, timer_idx, 0x00000000ULL);
-    /*Set alarm value*/
-    timer_set_alarm_value(timer_group, timer_idx, alarm_val);
-    /*Enable timer interrupt*/
-    timer_enable_intr(timer_group, timer_idx);
-}
-
 #if CONFIG_APPTRACE_SV_ENABLE == 0
 #define ESP_APPTRACE_TEST_WRITE(_b_, _s_)            esp_apptrace_write(ESP_APPTRACE_DEST_TRAX, _b_, _s_, ESP_APPTRACE_TMO_INFINITE)
 #define ESP_APPTRACE_TEST_WRITE_FROM_ISR(_b_, _s_)   esp_apptrace_write(ESP_APPTRACE_DEST_TRAX, _b_, _s_, 0UL)
@@ -103,9 +81,8 @@ typedef struct {
 } esp_apptrace_test_gen_data_t;
 
 typedef struct {
-    int  group;
-    int  id;
-    void (*isr_func)(void *);
+    gptimer_handle_t gptimer;
+    bool (*isr_func)(gptimer_handle_t timer, const gptimer_alarm_event_data_t *edata, void *user_ctx);
     esp_apptrace_test_gen_data_t data;
 } esp_apptrace_test_timer_arg_t;
 
@@ -117,7 +94,6 @@ typedef struct {
     esp_apptrace_test_gen_data_t data;
     volatile int stop;
     SemaphoreHandle_t done;
-
     uint32_t timers_num;
     esp_apptrace_test_timer_arg_t *timers;
 } esp_apptrace_test_task_arg_t;
@@ -133,103 +109,87 @@ static SemaphoreHandle_t s_print_lock;
 
 static uint64_t esp_apptrace_test_ts_get(void);
 
-static void esp_apptrace_test_timer_isr(void *arg)
+static bool esp_apptrace_test_timer_isr(gptimer_handle_t timer, const gptimer_alarm_event_data_t *edata, void *user_ctx)
 {
-    esp_apptrace_test_timer_arg_t *tim_arg = (esp_apptrace_test_timer_arg_t *)arg;
+    esp_apptrace_test_timer_arg_t *tim_arg = (esp_apptrace_test_timer_arg_t *)user_ctx;
 
     uint32_t *ts = (uint32_t *)(tim_arg->data.buf + sizeof(uint32_t));
     *ts = (uint32_t)esp_apptrace_test_ts_get();
     memset(tim_arg->data.buf + 2 * sizeof(uint32_t), tim_arg->data.wr_cnt & tim_arg->data.mask, tim_arg->data.buf_sz - 2 * sizeof(uint32_t));
     int res = ESP_APPTRACE_TEST_WRITE_FROM_ISR(tim_arg->data.buf, tim_arg->data.buf_sz);
-    if (res != ESP_OK) {
-    } else {
-        if (0) {
-            esp_rom_printf("tim-%d-%d: Written chunk%d %d bytes, %x\n",
-                       tim_arg->group, tim_arg->id, tim_arg->data.wr_cnt, tim_arg->data.buf_sz, tim_arg->data.wr_cnt & tim_arg->data.mask);
-        }
+    if (res == ESP_OK) {
         tim_arg->data.wr_err = 0;
     }
 
     tim_arg->data.wr_cnt++;
-    timer_group_clr_intr_status_in_isr(tim_arg->group, tim_arg->id);
-    timer_group_enable_alarm_in_isr(tim_arg->group, tim_arg->id);
+    return true;
 }
 
-static void esp_apptrace_test_timer_isr_crash(void *arg)
+static bool esp_apptrace_test_timer_isr_crash(gptimer_handle_t timer, const gptimer_alarm_event_data_t *edata, void *user_ctx)
 {
-    esp_apptrace_test_timer_arg_t *tim_arg = (esp_apptrace_test_timer_arg_t *)arg;
+    esp_apptrace_test_timer_arg_t *tim_arg = (esp_apptrace_test_timer_arg_t *)user_ctx;
 
-    timer_group_clr_intr_status_in_isr(tim_arg->group, tim_arg->id);
-    timer_group_enable_alarm_in_isr(tim_arg->group, tim_arg->id);
     if (tim_arg->data.wr_cnt < ESP_APPTRACE_TEST_BLOCKS_BEFORE_CRASH) {
         uint32_t *ts = (uint32_t *)(tim_arg->data.buf + sizeof(uint32_t));
         *ts = (uint32_t)esp_apptrace_test_ts_get();//xthal_get_ccount();//xTaskGetTickCount();
         memset(tim_arg->data.buf + 2 * sizeof(uint32_t), tim_arg->data.wr_cnt & tim_arg->data.mask, tim_arg->data.buf_sz - 2 * sizeof(uint32_t));
         int res = ESP_APPTRACE_TEST_WRITE_FROM_ISR(tim_arg->data.buf, tim_arg->data.buf_sz);
         if (res != ESP_OK) {
-            esp_rom_printf("tim-%d-%d: Failed to write trace %d %x!\n", tim_arg->group, tim_arg->id, res, tim_arg->data.wr_cnt & tim_arg->data.mask);
+            esp_rom_printf("tim-%x: Failed to write trace %d %x!\n", tim_arg->gptimer, res, tim_arg->data.wr_cnt & tim_arg->data.mask);
         } else {
-            esp_rom_printf("tim-%d-%d: Written chunk%d %d bytes, %x\n",
-                       tim_arg->group, tim_arg->id, tim_arg->data.wr_cnt, tim_arg->data.buf_sz, tim_arg->data.wr_cnt & tim_arg->data.mask);
+            esp_rom_printf("tim-%x: Written chunk%d %d bytes, %x\n",
+                           timer, tim_arg->data.wr_cnt, tim_arg->data.buf_sz, tim_arg->data.wr_cnt & tim_arg->data.mask);
             tim_arg->data.wr_cnt++;
         }
     } else {
         uint32_t *ptr = 0;
         *ptr = 1000;
     }
+    return true;
 }
 
 static void esp_apptrace_dummy_task(void *p)
 {
     esp_apptrace_test_task_arg_t *arg = (esp_apptrace_test_task_arg_t *) p;
-    int res, flags = 0, i;
-    timer_isr_handle_t *inth = NULL;
     TickType_t tmo_ticks = arg->data.period / (1000 * portTICK_PERIOD_MS);
 
     ESP_APPTRACE_TEST_LOGI("%x: run dummy task (period %u us, %u timers)", xTaskGetCurrentTaskHandle(), arg->data.period, arg->timers_num);
 
-    if (arg->timers_num > 0) {
-        inth = pvPortMalloc(arg->timers_num * sizeof(timer_isr_handle_t));
-        if (!inth) {
-            ESP_APPTRACE_TEST_LOGE("Failed to alloc timer ISR handles!");
-            goto on_fail;
-        }
-        memset(inth, 0, arg->timers_num * sizeof(timer_isr_handle_t));
-        for (int i = 0; i < arg->timers_num; i++) {
-            esp_apptrace_test_timer_init(arg->timers[i].group, arg->timers[i].id, arg->timers[i].data.period);
-            res = timer_isr_register(arg->timers[i].group, arg->timers[i].id, arg->timers[i].isr_func, &arg->timers[i], flags, &inth[i]);
-            if (res != ESP_OK) {
-                ESP_APPTRACE_TEST_LOGE("Failed to timer_isr_register (%d)!", res);
-                goto on_fail;
-            }
-            *(uint32_t *)arg->timers[i].data.buf = (uint32_t)inth[i] | (1 << 31);
-            ESP_APPTRACE_TEST_LOGI("%x: start timer %x period %u us", xTaskGetCurrentTaskHandle(), inth[i], arg->timers[i].data.period);
-            res = timer_start(arg->timers[i].group, arg->timers[i].id);
-            if (res != ESP_OK) {
-                ESP_APPTRACE_TEST_LOGE("Failed to timer_start (%d)!", res);
-                goto on_fail;
-            }
-        }
+    for (int i = 0; i < arg->timers_num; i++) {
+        gptimer_config_t timer_config = {
+            .clk_src = GPTIMER_CLK_SRC_DEFAULT,
+            .direction = GPTIMER_COUNT_UP,
+            .resolution_hz = 1000000,
+        };
+        TEST_ESP_OK(gptimer_new_timer(&timer_config, &arg->timers[i].gptimer));
+        *(uint32_t *)arg->timers[i].data.buf = (uint32_t)arg->timers[i].gptimer | (1 << 31);
+        ESP_APPTRACE_TEST_LOGI("%x: start timer %x period %u us", xTaskGetCurrentTaskHandle(), arg->timers[i].gptimer, arg->timers[i].data.period);
+        gptimer_alarm_config_t alarm_config = {
+            .reload_count = 0,
+            .alarm_count = arg->timers[i].data.period,
+            .flags.auto_reload_on_alarm = true,
+        };
+        gptimer_event_callbacks_t cbs = {
+            .on_alarm = arg->timers[i].isr_func,
+        };
+        TEST_ESP_OK(gptimer_register_event_callbacks(arg->timers[i].gptimer, &cbs, &arg->timers[i]));
+        TEST_ESP_OK(gptimer_enable(arg->timers[i].gptimer));
+        TEST_ESP_OK(gptimer_set_alarm_action(arg->timers[i].gptimer, &alarm_config));
+        TEST_ESP_OK(gptimer_start(arg->timers[i].gptimer));
     }
 
-    i = 0;
+    int i = 0;
     while (!arg->stop) {
-        ESP_APPTRACE_TEST_LOGD("%x: dummy task work %d.%d", xTaskGetCurrentTaskHandle(), cpu_hal_get_core_id(), i++);
+        ESP_APPTRACE_TEST_LOGD("%x: dummy task work %d.%d", xTaskGetCurrentTaskHandle(), esp_cpu_get_core_id(), i++);
         if (tmo_ticks) {
             vTaskDelay(tmo_ticks);
         }
     }
 
-on_fail:
-    if (inth) {
-        for (int i = 0; i < arg->timers_num; i++) {
-            timer_pause(arg->timers[i].group, arg->timers[i].id);
-            timer_disable_intr(arg->timers[i].group, arg->timers[i].id);
-            if (inth[i]) {
-                esp_intr_free(inth[i]);
-            }
-        }
-        vPortFree(inth);
+    for (int i = 0; i < arg->timers_num; i++) {
+        TEST_ESP_OK(gptimer_stop(arg->timers[i].gptimer));
+        TEST_ESP_OK(gptimer_disable(arg->timers[i].gptimer));
+        TEST_ESP_OK(gptimer_del_timer(arg->timers[i].gptimer));
     }
     xSemaphoreGive(arg->done);
     vTaskDelay(1);
@@ -239,37 +199,35 @@ on_fail:
 static void esp_apptrace_test_task(void *p)
 {
     esp_apptrace_test_task_arg_t *arg = (esp_apptrace_test_task_arg_t *) p;
-    int res, flags = 0;
-    timer_isr_handle_t *inth = NULL;
+    int res;
     TickType_t tmo_ticks = arg->data.period / (1000 * portTICK_PERIOD_MS);
 
     ESP_APPTRACE_TEST_LOGI("%x: run (period %u us, stamp mask %x, %u timers)", xTaskGetCurrentTaskHandle(), arg->data.period, arg->data.mask, arg->timers_num);
 
-    if (arg->timers_num > 0) {
-        inth = pvPortMalloc(arg->timers_num * sizeof(timer_isr_handle_t));
-        if (!inth) {
-            ESP_APPTRACE_TEST_LOGE("Failed to alloc timer ISR handles!");
-            goto on_fail;
-        }
-        memset(inth, 0, arg->timers_num * sizeof(timer_isr_handle_t));
-        for (int i = 0; i < arg->timers_num; i++) {
-            esp_apptrace_test_timer_init(arg->timers[i].group, arg->timers[i].id, arg->timers[i].data.period);
-            res = timer_isr_register(arg->timers[i].group, arg->timers[i].id, arg->timers[i].isr_func, &arg->timers[i], flags, &inth[i]);
-            if (res != ESP_OK) {
-                ESP_APPTRACE_TEST_LOGE("Failed to timer_isr_register (%d)!", res);
-                goto on_fail;
-            }
-            *(uint32_t *)arg->timers[i].data.buf = ((uint32_t)inth[i]) | (1 << 31) | (cpu_hal_get_core_id() ? 0x1 : 0);
-            ESP_APPTRACE_TEST_LOGI("%x: start timer %x period %u us", xTaskGetCurrentTaskHandle(), inth[i], arg->timers[i].data.period);
-            res = timer_start(arg->timers[i].group, arg->timers[i].id);
-            if (res != ESP_OK) {
-                ESP_APPTRACE_TEST_LOGE("Failed to timer_start (%d)!", res);
-                goto on_fail;
-            }
-        }
+    for (int i = 0; i < arg->timers_num; i++) {
+        gptimer_config_t timer_config = {
+            .clk_src = GPTIMER_CLK_SRC_DEFAULT,
+            .direction = GPTIMER_COUNT_UP,
+            .resolution_hz = 1000000,
+        };
+        TEST_ESP_OK(gptimer_new_timer(&timer_config, &arg->timers[i].gptimer));
+        *(uint32_t *)arg->timers[i].data.buf = ((uint32_t)arg->timers[i].gptimer) | (1 << 31) | (esp_cpu_get_core_id() ? 0x1 : 0);
+        ESP_APPTRACE_TEST_LOGI("%x: start timer %x period %u us", xTaskGetCurrentTaskHandle(), arg->timers[i].gptimer, arg->timers[i].data.period);
+        gptimer_alarm_config_t alarm_config = {
+            .reload_count = 0,
+            .alarm_count = arg->timers[i].data.period,
+            .flags.auto_reload_on_alarm = true,
+        };
+        gptimer_event_callbacks_t cbs = {
+            .on_alarm = arg->timers[i].isr_func,
+        };
+        TEST_ESP_OK(gptimer_register_event_callbacks(arg->timers[i].gptimer, &cbs, &arg->timers[i]));
+        TEST_ESP_OK(gptimer_enable(arg->timers[i].gptimer));
+        TEST_ESP_OK(gptimer_set_alarm_action(arg->timers[i].gptimer, &alarm_config));
+        TEST_ESP_OK(gptimer_start(arg->timers[i].gptimer));
     }
 
-    *(uint32_t *)arg->data.buf = (uint32_t)xTaskGetCurrentTaskHandle() | (cpu_hal_get_core_id() ? 0x1 : 0);
+    *(uint32_t *)arg->data.buf = (uint32_t)xTaskGetCurrentTaskHandle() | (esp_cpu_get_core_id() ? 0x1 : 0);
     arg->data.wr_cnt = 0;
     arg->data.wr_err = 0;
     while (!arg->stop) {
@@ -283,7 +241,7 @@ static void esp_apptrace_test_task(void *p)
             res = ESP_APPTRACE_TEST_WRITE(arg->data.buf, arg->data.buf_sz);
         }
         if (res) {
-            if (1){//arg->data.wr_err++ < ESP_APPTRACE_TEST_PRN_WRERR_MAX) {
+            if (1) { //arg->data.wr_err++ < ESP_APPTRACE_TEST_PRN_WRERR_MAX) {
                 ESP_APPTRACE_TEST_LOGE("%x: Failed to write trace %d %x!", xTaskGetCurrentTaskHandle(), res, arg->data.wr_cnt & arg->data.mask);
                 if (arg->data.wr_err == ESP_APPTRACE_TEST_PRN_WRERR_MAX) {
                     ESP_APPTRACE_TEST_LOGE("\n");
@@ -301,16 +259,10 @@ static void esp_apptrace_test_task(void *p)
         }
     }
 
-on_fail:
-    if (inth) {
-        for (int i = 0; i < arg->timers_num; i++) {
-            timer_pause(arg->timers[i].group, arg->timers[i].id);
-            timer_disable_intr(arg->timers[i].group, arg->timers[i].id);
-            if (inth[i]) {
-                esp_intr_free(inth[i]);
-            }
-        }
-        vPortFree(inth);
+    for (int i = 0; i < arg->timers_num; i++) {
+        TEST_ESP_OK(gptimer_stop(arg->timers[i].gptimer));
+        TEST_ESP_OK(gptimer_disable(arg->timers[i].gptimer));
+        TEST_ESP_OK(gptimer_del_timer(arg->timers[i].gptimer));
     }
     xSemaphoreGive(arg->done);
     vTaskDelay(1);
@@ -320,17 +272,16 @@ on_fail:
 static void esp_apptrace_test_task_crash(void *p)
 {
     esp_apptrace_test_task_arg_t *arg = (esp_apptrace_test_task_arg_t *) p;
-    int res, i;
 
     ESP_APPTRACE_TEST_LOGE("%x: run (period %u us, stamp mask %x, %u timers)", xTaskGetCurrentTaskHandle(), arg->data.period, arg->data.mask, arg->timers_num);
 
     arg->data.wr_cnt = 0;
     *(uint32_t *)arg->data.buf = (uint32_t)xTaskGetCurrentTaskHandle();
-    for (i = 0; i < ESP_APPTRACE_TEST_BLOCKS_BEFORE_CRASH; i++) {
+    for (int i = 0; i < ESP_APPTRACE_TEST_BLOCKS_BEFORE_CRASH; i++) {
         uint32_t *ts = (uint32_t *)(arg->data.buf + sizeof(uint32_t));
         *ts = (uint32_t)esp_apptrace_test_ts_get();
         memset(arg->data.buf + sizeof(uint32_t), arg->data.wr_cnt & arg->data.mask, arg->data.buf_sz - sizeof(uint32_t));
-        res = ESP_APPTRACE_TEST_WRITE(arg->data.buf, arg->data.buf_sz);
+        int res = ESP_APPTRACE_TEST_WRITE(arg->data.buf, arg->data.buf_sz);
         if (res) {
             ESP_APPTRACE_TEST_LOGE("%x: Failed to write trace %d %x!", xTaskGetCurrentTaskHandle(), res, arg->data.wr_cnt & arg->data.mask);
         } else {
@@ -347,66 +298,49 @@ static void esp_apptrace_test_task_crash(void *p)
     vTaskDelete(NULL);
 }
 
-static int s_ts_timer_group, s_ts_timer_idx;
+static gptimer_handle_t ts_gptimer;
 
 static uint64_t esp_apptrace_test_ts_get(void)
 {
     uint64_t ts = 0;
-    timer_get_counter_value(s_ts_timer_group, s_ts_timer_idx, &ts);
+    gptimer_get_raw_count(ts_gptimer, &ts);
     return ts;
 }
 
-static void esp_apptrace_test_ts_init(int timer_group, int timer_idx)
+static void esp_apptrace_test_ts_init(void)
 {
-    timer_config_t config;
-    //uint64_t alarm_val = period * (TIMER_BASE_CLK / 1000000UL);
-
-    ESP_APPTRACE_TEST_LOGI("Use timer%d.%d for TS", timer_group, timer_idx);
-
-    s_ts_timer_group = timer_group;
-    s_ts_timer_idx = timer_idx;
-
-    config.alarm_en = 0;
-    config.auto_reload = 0;
-    config.counter_dir = TIMER_COUNT_UP;
-    config.divider = 2;     //Range is 2 to 65536
-    config.counter_en = 0;
-    /*Configure timer*/
-    timer_init(timer_group, timer_idx, &config);
-    /*Load counter value */
-    timer_set_counter_value(timer_group, timer_idx, 0x00000000ULL);
-    /*Enable timer interrupt*/
-    timer_start(timer_group, timer_idx);
+    gptimer_config_t timer_config = {
+        .clk_src = GPTIMER_CLK_SRC_DEFAULT,
+        .direction = GPTIMER_COUNT_UP,
+        .resolution_hz = 10000000,
+    };
+    TEST_ESP_OK(gptimer_new_timer(&timer_config, &ts_gptimer));
+    ESP_APPTRACE_TEST_LOGI("Use timer %x for TS", ts_gptimer);
+    TEST_ESP_OK(gptimer_enable(ts_gptimer));
+    TEST_ESP_OK(gptimer_start(ts_gptimer));
 }
 
 static void esp_apptrace_test_ts_cleanup(void)
 {
-    timer_config_t config;
-
-    config.alarm_en = 0;
-    config.auto_reload = 0;
-    config.counter_dir = TIMER_COUNT_UP;
-    config.divider = 2;     //Range is 2 to 65536
-    config.counter_en = 0;
-    /*Configure timer*/
-    timer_init(s_ts_timer_group, s_ts_timer_idx, &config);
+    TEST_ESP_OK(gptimer_stop(ts_gptimer));
+    TEST_ESP_OK(gptimer_disable(ts_gptimer));
+    TEST_ESP_OK(gptimer_del_timer(ts_gptimer));
+    ts_gptimer = NULL;
 }
 
 static void esp_apptrace_test(esp_apptrace_test_cfg_t *test_cfg)
 {
-    int i, k;
-    int tims_in_use[TIMER_GROUP_MAX][TIMER_MAX] = {{0, 0}, {0, 0}};
-    esp_apptrace_test_task_arg_t dummy_task_arg[1];
+    esp_apptrace_test_task_arg_t dummy_task_arg = {
+        .core = 0,
+        .prio = 3,
+        .task_func = esp_apptrace_test_task_crash,
+        .data.buf = NULL,
+        .data.buf_sz = 0,
+        .data.period = 500000,
+        .timers_num = 0,
+        .timers = NULL,
+    };
 
-    memset(dummy_task_arg, 0, sizeof(dummy_task_arg));
-    dummy_task_arg[0].core = 0;
-    dummy_task_arg[0].prio = 3;
-    dummy_task_arg[0].task_func = esp_apptrace_test_task_crash;
-    dummy_task_arg[0].data.buf = NULL;
-    dummy_task_arg[0].data.buf_sz = 0;
-    dummy_task_arg[0].data.period = 500000;
-    dummy_task_arg[0].timers_num = 0;
-    dummy_task_arg[0].timers = NULL;
 #if ESP_APPTRACE_TEST_USE_PRINT_LOCK == 1
     s_print_lock = xSemaphoreCreateBinary();
     if (!s_print_lock) {
@@ -417,38 +351,17 @@ static void esp_apptrace_test(esp_apptrace_test_cfg_t *test_cfg)
 #else
 #endif
 
-    for (i = 0; i < test_cfg->tasks_num; i++) {
+    for (int i = 0; i < test_cfg->tasks_num; i++) {
         test_cfg->tasks[i].data.mask = 0xFF;
         test_cfg->tasks[i].stop = 0;
         test_cfg->tasks[i].done = xSemaphoreCreateBinary();
-        if (!test_cfg->tasks[i].done) {
-            ESP_APPTRACE_TEST_LOGE("Failed to create task completion semaphore!");
-            goto on_fail;
-        }
-        for (k = 0; k < test_cfg->tasks[i].timers_num; k++) {
+        TEST_ASSERT_NOT_NULL(test_cfg->tasks[i].done);
+        for (int k = 0; k < test_cfg->tasks[i].timers_num; k++) {
             test_cfg->tasks[i].timers[k].data.mask = 0xFF;
-            tims_in_use[test_cfg->tasks[i].timers[k].group][test_cfg->tasks[i].timers[k].id] = 1;
         }
     }
 
-    int found = 0;
-    for (i = 0; i < TIMER_GROUP_MAX; i++) {
-        for (k = 0; k < TIMER_MAX; k++) {
-            if (!tims_in_use[i][k]) {
-                ESP_APPTRACE_TEST_LOGD("Found timer%d.%d", i, k);
-                found = 1;
-                break;
-            }
-        }
-        if (found) {
-            break;
-        }
-    }
-    if (!found) {
-        ESP_APPTRACE_TEST_LOGE("No free timer for TS!");
-        goto on_fail;
-    }
-    esp_apptrace_test_ts_init(i, k);
+    esp_apptrace_test_ts_init();
 
     for (int i = 0; i < test_cfg->tasks_num; i++) {
         char name[30];
@@ -457,16 +370,15 @@ static void esp_apptrace_test(esp_apptrace_test_cfg_t *test_cfg)
         xTaskCreatePinnedToCore(test_cfg->tasks[i].task_func, name, 2048, &test_cfg->tasks[i], test_cfg->tasks[i].prio, &thnd, test_cfg->tasks[i].core);
         ESP_APPTRACE_TEST_LOGI("Created task %x", thnd);
     }
-    xTaskCreatePinnedToCore(esp_apptrace_dummy_task, "dummy0", 2048, &dummy_task_arg[0], dummy_task_arg[0].prio, NULL, 0);
+    xTaskCreatePinnedToCore(esp_apptrace_dummy_task, "dummy0", 2048, &dummy_task_arg, dummy_task_arg.prio, NULL, 0);
 #if CONFIG_FREERTOS_UNICORE == 0
-    xTaskCreatePinnedToCore(esp_apptrace_dummy_task, "dummy1", 2048, &dummy_task_arg[0], dummy_task_arg[0].prio, NULL, 1);
+    xTaskCreatePinnedToCore(esp_apptrace_dummy_task, "dummy1", 2048, &dummy_task_arg, dummy_task_arg.prio, NULL, 1);
 #endif
     for (int i = 0; i < test_cfg->tasks_num; i++) {
         //arg1.stop = 1;
         xSemaphoreTake(test_cfg->tasks[i].done, portMAX_DELAY);
     }
 
-on_fail:
     for (int i = 0; i < test_cfg->tasks_num; i++) {
         if (test_cfg->tasks[i].done) {
             vSemaphoreDelete(test_cfg->tasks[i].done);
@@ -494,8 +406,6 @@ TEST_CASE("App trace test (1 task + 1 crashed timer ISR @ 1 core)", "[trace][ign
     memset(s_test_timers, 0, sizeof(s_test_timers));
     memset(s_test_tasks, 0, sizeof(s_test_tasks));
 
-    s_test_timers[0].group = TIMER_GROUP_0;
-    s_test_timers[0].id = TIMER_0;
     s_test_timers[0].isr_func = esp_apptrace_test_timer_isr_crash;
     s_test_timers[0].data.buf = s_bufs[0];
     s_test_timers[0].data.buf_sz = sizeof(s_bufs[0]);
@@ -548,15 +458,11 @@ TEST_CASE("App trace test (2 tasks + 1 timer @ each core", "[trace][ignore]")
     memset(s_test_tasks, 0, sizeof(s_test_tasks));
     memset(s_test_timers, 0, sizeof(s_test_timers));
 
-    s_test_timers[0].group = TIMER_GROUP_0;
-    s_test_timers[0].id = TIMER_0;
     s_test_timers[0].isr_func = esp_apptrace_test_timer_isr;
     s_test_timers[0].data.buf = s_bufs[0];
     s_test_timers[0].data.buf_sz = sizeof(s_bufs[0]);
     s_test_timers[0].data.period = 150;
 
-    s_test_timers[1].group = TIMER_GROUP_1;
-    s_test_timers[1].id = TIMER_0;
     s_test_timers[1].isr_func = esp_apptrace_test_timer_isr;
     s_test_timers[1].data.buf = s_bufs[1];
     s_test_timers[1].data.buf_sz = sizeof(s_bufs[1]);
@@ -612,8 +518,6 @@ TEST_CASE("App trace test (1 task + 1 timer @ 1 core)", "[trace][ignore]")
     memset(s_test_timers, 0, sizeof(s_test_timers));
     memset(s_test_tasks, 0, sizeof(s_test_tasks));
 
-    s_test_timers[0].group = TIMER_GROUP_0;
-    s_test_timers[0].id = TIMER_0;
     s_test_timers[0].isr_func = esp_apptrace_test_timer_isr;
     s_test_timers[0].data.buf = s_bufs[0];
     s_test_timers[0].data.buf_sz = sizeof(s_bufs[0]);
@@ -749,7 +653,7 @@ static void esp_logtrace_task(void *p)
         ESP_LOGI(TAG, "%p: sample print 4 %c", xTaskGetCurrentTaskHandle(), ((i & 0xFF) % 95) + 32);
         ESP_LOGI(TAG, "%p: sample print 5 %f", xTaskGetCurrentTaskHandle(), 1.0);
         ESP_LOGI(TAG, "%p: sample print 6 %f", xTaskGetCurrentTaskHandle(), 3.45);
-        ESP_LOGI(TAG, "%p: logtrace task work %d.%d", xTaskGetCurrentTaskHandle(), cpu_hal_get_core_id(), i);
+        ESP_LOGI(TAG, "%p: logtrace task work %d.%d", xTaskGetCurrentTaskHandle(), esp_cpu_get_core_id(), i);
         if (++i == 10000) {
             break;
         }
@@ -792,11 +696,11 @@ TEST_CASE("Log trace test (2 tasks)", "[trace][ignore]")
     vSemaphoreDelete(arg2.done);
 }
 
-#else
+#else // #if CONFIG_APPTRACE_SV_ENABLE == 0
 
 typedef struct {
-    int group;
-    int timer;
+    gptimer_handle_t gptimer;
+    uint32_t period;
     int flags;
     uint32_t id;
 } esp_sysviewtrace_timer_arg_t;
@@ -810,50 +714,47 @@ typedef struct {
     uint32_t id;
 } esp_sysviewtrace_task_arg_t;
 
-static void esp_sysview_test_timer_isr(void *arg)
+static bool esp_sysview_test_timer_isr(gptimer_handle_t timer, const gptimer_alarm_event_data_t *edata, void *user_ctx)
 {
-    esp_sysviewtrace_timer_arg_t *tim_arg = (esp_sysviewtrace_timer_arg_t *)arg;
-
-    //ESP_APPTRACE_TEST_LOGI("tim-%d: IRQ %d/%d\n", tim_arg->id, tim_arg->group, tim_arg->timer);
-
-    timer_group_clr_intr_status_in_isr(tim_arg->group, tim_arg->id);
-    timer_group_enable_alarm_in_isr(tim_arg->group, tim_arg->id);
+    esp_sysviewtrace_timer_arg_t *tim_arg = (esp_sysviewtrace_timer_arg_t *)user_ctx;
+    return false;
 }
 
 static void esp_sysviewtrace_test_task(void *p)
 {
     esp_sysviewtrace_task_arg_t *arg = (esp_sysviewtrace_task_arg_t *) p;
     volatile uint32_t tmp = 0;
-    timer_isr_handle_t inth;
-
     printf("%x: run sysview task\n", (uint32_t)xTaskGetCurrentTaskHandle());
 
     if (arg->timer) {
-        esp_err_t res = timer_isr_register(arg->timer->group, arg->timer->timer, esp_sysview_test_timer_isr, arg->timer, arg->timer->flags, &inth);
-        if (res != ESP_OK) {
-            printf("%x: failed to register timer ISR\n", (uint32_t)xTaskGetCurrentTaskHandle());
-        }
-        else {
-            res = timer_start(arg->timer->group, arg->timer->timer);
-            if (res != ESP_OK) {
-                printf("%x: failed to start timer\n", (uint32_t)xTaskGetCurrentTaskHandle());
-            }
-        }
+        gptimer_alarm_config_t alarm_config = {
+            .reload_count = 0,
+            .alarm_count = arg->timer->period,
+            .flags.auto_reload_on_alarm = true,
+        };
+        gptimer_event_callbacks_t cbs = {
+            .on_alarm = esp_sysview_test_timer_isr,
+        };
+        TEST_ESP_OK(gptimer_register_event_callbacks(arg->timer->gptimer, &cbs, arg->timer));
+        TEST_ESP_OK(gptimer_enable(arg->timer->gptimer));
+        TEST_ESP_OK(gptimer_set_alarm_action(arg->timer->gptimer, &alarm_config));
+        TEST_ESP_OK(gptimer_start(arg->timer->gptimer));
     }
 
     int i = 0;
     while (1) {
         static uint32_t count;
         printf("%d", arg->id);
-        if((++count % 80) == 0)
+        if ((++count % 80) == 0) {
             printf("\n");
+        }
         if (arg->sync) {
             xSemaphoreTake(*arg->sync, portMAX_DELAY);
         }
         for (uint32_t k = 0; k < arg->work_count; k++) {
             tmp++;
         }
-        vTaskDelay(arg->sleep_tmo/portTICK_PERIOD_MS);
+        vTaskDelay(arg->sleep_tmo / portTICK_PERIOD_MS);
         i++;
         if (arg->sync) {
             xSemaphoreGive(*arg->sync);
@@ -871,10 +772,9 @@ TEST_CASE("SysView trace test 1", "[trace][ignore]")
     TaskHandle_t thnd;
 
     esp_sysviewtrace_timer_arg_t tim_arg1 = {
-        .group = TIMER_GROUP_1,
-        .timer = TIMER_1,
         .flags = ESP_INTR_FLAG_SHARED,
         .id = 0,
+        .period = 500,
     };
     esp_sysviewtrace_task_arg_t arg1 = {
         .done = xSemaphoreCreateBinary(),
@@ -885,10 +785,9 @@ TEST_CASE("SysView trace test 1", "[trace][ignore]")
         .id = 0,
     };
     esp_sysviewtrace_timer_arg_t tim_arg2 = {
-        .group = TIMER_GROUP_1,
-        .timer = TIMER_0,
         .flags = 0,
         .id = 1,
+        .period = 100,
     };
     esp_sysviewtrace_task_arg_t arg2 = {
         .done = xSemaphoreCreateBinary(),
@@ -899,8 +798,15 @@ TEST_CASE("SysView trace test 1", "[trace][ignore]")
         .id = 1,
     };
 
-    esp_apptrace_test_timer_init(TIMER_GROUP_1, TIMER_1, 500);
-    esp_apptrace_test_timer_init(TIMER_GROUP_1, TIMER_0, 100);
+    gptimer_config_t timer_config = {
+        .clk_src = GPTIMER_CLK_SRC_DEFAULT,
+        .direction = GPTIMER_COUNT_UP,
+        .resolution_hz = 1000000,
+    };
+    timer_config.flags.intr_shared = (tim_arg1.flags & ESP_INTR_FLAG_SHARED) == ESP_INTR_FLAG_SHARED;
+    TEST_ESP_OK(gptimer_new_timer(&timer_config, &tim_arg1.gptimer));
+    timer_config.flags.intr_shared = (tim_arg2.flags & ESP_INTR_FLAG_SHARED) == ESP_INTR_FLAG_SHARED;
+    TEST_ESP_OK(gptimer_new_timer(&timer_config, &tim_arg2.gptimer));
 
     xTaskCreatePinnedToCore(esp_sysviewtrace_test_task, "svtrace0", 2048, &arg1, 3, &thnd, 0);
     ESP_APPTRACE_TEST_LOGI("Created task %x", thnd);
@@ -915,6 +821,12 @@ TEST_CASE("SysView trace test 1", "[trace][ignore]")
     vSemaphoreDelete(arg1.done);
     xSemaphoreTake(arg2.done, portMAX_DELAY);
     vSemaphoreDelete(arg2.done);
+    TEST_ESP_OK(gptimer_stop(tim_arg1.gptimer));
+    TEST_ESP_OK(gptimer_disable(tim_arg1.gptimer));
+    TEST_ESP_OK(gptimer_del_timer(tim_arg1.gptimer));
+    TEST_ESP_OK(gptimer_stop(tim_arg2.gptimer));
+    TEST_ESP_OK(gptimer_disable(tim_arg2.gptimer));
+    TEST_ESP_OK(gptimer_del_timer(tim_arg2.gptimer));
 }
 
 TEST_CASE("SysView trace test 2", "[trace][ignore]")
@@ -922,10 +834,9 @@ TEST_CASE("SysView trace test 2", "[trace][ignore]")
     TaskHandle_t thnd;
 
     esp_sysviewtrace_timer_arg_t tim_arg1 = {
-        .group = TIMER_GROUP_1,
-        .timer = TIMER_1,
         .flags = ESP_INTR_FLAG_SHARED,
         .id = 0,
+        .period = 500,
     };
     esp_sysviewtrace_task_arg_t arg1 = {
         .done = xSemaphoreCreateBinary(),
@@ -936,10 +847,9 @@ TEST_CASE("SysView trace test 2", "[trace][ignore]")
         .id = 0,
     };
     esp_sysviewtrace_timer_arg_t tim_arg2 = {
-        .group = TIMER_GROUP_1,
-        .timer = TIMER_0,
         .flags = 0,
         .id = 1,
+        .period = 100,
     };
     esp_sysviewtrace_task_arg_t arg2 = {
         .done = xSemaphoreCreateBinary(),
@@ -969,8 +879,15 @@ TEST_CASE("SysView trace test 2", "[trace][ignore]")
         .id = 3,
     };
 
-    esp_apptrace_test_timer_init(TIMER_GROUP_1, TIMER_1, 500);
-    esp_apptrace_test_timer_init(TIMER_GROUP_1, TIMER_0, 100);
+    gptimer_config_t timer_config = {
+        .clk_src = GPTIMER_CLK_SRC_DEFAULT,
+        .direction = GPTIMER_COUNT_UP,
+        .resolution_hz = 1000000,
+    };
+    timer_config.flags.intr_shared = (tim_arg1.flags & ESP_INTR_FLAG_SHARED) == ESP_INTR_FLAG_SHARED;
+    TEST_ESP_OK(gptimer_new_timer(&timer_config, &tim_arg1.gptimer));
+    timer_config.flags.intr_shared = (tim_arg2.flags & ESP_INTR_FLAG_SHARED) == ESP_INTR_FLAG_SHARED;
+    TEST_ESP_OK(gptimer_new_timer(&timer_config, &tim_arg2.gptimer));
 
     xTaskCreatePinnedToCore(esp_sysviewtrace_test_task, "svtrace0", 2048, &arg1, 3, &thnd, 0);
     printf("Created task %x\n", (uint32_t)thnd);
@@ -999,6 +916,12 @@ TEST_CASE("SysView trace test 2", "[trace][ignore]")
     xSemaphoreTake(arg4.done, portMAX_DELAY);
     vSemaphoreDelete(arg4.done);
     vSemaphoreDelete(test_sync);
+    TEST_ESP_OK(gptimer_stop(tim_arg1.gptimer));
+    TEST_ESP_OK(gptimer_disable(tim_arg1.gptimer));
+    TEST_ESP_OK(gptimer_del_timer(tim_arg1.gptimer));
+    TEST_ESP_OK(gptimer_stop(tim_arg2.gptimer));
+    TEST_ESP_OK(gptimer_disable(tim_arg2.gptimer));
+    TEST_ESP_OK(gptimer_del_timer(tim_arg2.gptimer));
 }
-#endif
-#endif
+#endif // #if CONFIG_APPTRACE_SV_ENABLE == 0
+#endif // #if CONFIG_APPTRACE_ENABLE == 1

components/app_update/CMakeLists.txt

@@ -48,34 +48,37 @@ if(NOT BOOTLOADER_BUILD)
         add_dependencies(flash blank_ota_data)
         add_dependencies(encrypted-flash blank_ota_data)
 
-        set(otatool_py ${python} ${COMPONENT_DIR}/otatool.py)
+        set(otatool_py "${python}" "${COMPONENT_DIR}/otatool.py")
+
+        set(esptool_args --esptool-args before=${CONFIG_ESPTOOLPY_BEFORE} after=${CONFIG_ESPTOOLPY_AFTER})
+        set(otatool_args --partition-table-file ${PARTITION_CSV_PATH})
+        list(APPEND otatool_args --partition-table-offset ${PARTITION_TABLE_OFFSET})
 
-        set(esptool_args "--esptool-args;before=${CONFIG_ESPTOOLPY_BEFORE};after=${CONFIG_ESPTOOLPY_AFTER}")
-        set(otatool_args "--partition-table-file;${PARTITION_CSV_PATH}"
-                         "--partition-table-offset;${PARTITION_TABLE_OFFSET}")
         idf_component_get_property(esptool_py_dir esptool_py COMPONENT_DIR)
 
         add_custom_target(read-otadata DEPENDS "${PARTITION_CSV_PATH}"
             COMMAND ${CMAKE_COMMAND}
-            -D IDF_PATH="${idf_path}"
-            -D SERIAL_TOOL="${otatool_py}"
-            -D SERIAL_TOOL_ARGS="${esptool_args};${otatool_args};read_otadata"
-            -D WORKING_DIRECTORY="${build_dir}"
+            -D "IDF_PATH=${idf_path}"
+            -D "SERIAL_TOOL=${otatool_py}"
+            -D "SERIAL_TOOL_ARGS=${esptool_args};${otatool_args};read_otadata"
+            -D "WORKING_DIRECTORY=${build_dir}"
             -P ${esptool_py_dir}/run_serial_tool.cmake
-            WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}
+            WORKING_DIRECTORY "${build_dir}"
             USES_TERMINAL
+            VERBATIM
             )
         add_deprecated_target_alias(read_otadata read-otadata)
 
         add_custom_target(erase-otadata DEPENDS "${PARTITION_CSV_PATH}"
             COMMAND ${CMAKE_COMMAND}
-            -D IDF_PATH="${idf_path}"
-            -D SERIAL_TOOL="${otatool_py}"
-            -D SERIAL_TOOL_ARGS="${esptool_args};${otatool_args};erase_otadata"
-            -D WORKING_DIRECTORY="${build_dir}"
+            -D "IDF_PATH=${idf_path}"
+            -D "SERIAL_TOOL=${otatool_py}"
+            -D "SERIAL_TOOL_ARGS=${esptool_args};${otatool_args};erase_otadata"
+            -D "WORKING_DIRECTORY=${build_dir}"
             -P ${esptool_py_dir}/run_serial_tool.cmake
-            WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}
+            WORKING_DIRECTORY "${build_dir}"
             USES_TERMINAL
+            VERBATIM
             )
         add_deprecated_target_alias(erase_otadata erase-otadata)
 

components/app_update/Makefile.projbuild

@@ -1,54 +0,0 @@
-# Generate partition binary
-#
-.PHONY: blank_ota_data erase_otadata read_otadata
-
-OTATOOL_PY := $(PYTHON) $(COMPONENT_PATH)/otatool.py
-PARTTOOL_PY := $(PYTHON) $(IDF_PATH)/components/partition_table/parttool.py
-
-# Generate blank partition file
-BLANK_OTA_DATA_FILE = $(BUILD_DIR_BASE)/ota_data_initial.bin
-
-# Copy PARTITION_TABLE_CSV_PATH definition here from $IDF_PATH/components/partition_table/Makefile.projbuild
-# to avoid undefined variables warning for PARTITION_TABLE_CSV_PATH
-ifndef PARTITION_TABLE_CSV_PATH
-PARTITION_TABLE_ROOT := $(call dequote,$(if $(CONFIG_PARTITION_TABLE_CUSTOM),$(PROJECT_PATH),$(IDF_PATH)/components/partition_table))
-PARTITION_TABLE_CSV_PATH := $(call dequote,$(abspath $(PARTITION_TABLE_ROOT)/$(call dequote,$(CONFIG_PARTITION_TABLE_FILENAME))))
-endif
-
-$(BLANK_OTA_DATA_FILE): partition_table_get_info $(PARTITION_TABLE_CSV_PATH) | check_python_dependencies
-	$(shell if [ "$(OTA_DATA_OFFSET)" != "" ] && [ "$(OTA_DATA_SIZE)" != "" ]; then \
-				$(PYTHON) $(IDF_PATH)/components/partition_table/gen_empty_partition.py $(OTA_DATA_SIZE) $(BLANK_OTA_DATA_FILE); \
-			fi; )
-	$(eval BLANK_OTA_DATA_FILE = $(shell if [ "$(OTA_DATA_OFFSET)" != "" ] && [ "$(OTA_DATA_SIZE)" != "" ]; then \
-			echo $(BLANK_OTA_DATA_FILE); else echo " "; fi) )
-
-blank_ota_data: $(BLANK_OTA_DATA_FILE)
-
-# If there is no otadata partition, both OTA_DATA_OFFSET and BLANK_OTA_DATA_FILE
-# expand to empty values.
-ESPTOOL_ALL_FLASH_ARGS += $(OTA_DATA_OFFSET) $(BLANK_OTA_DATA_FILE)
-
-ESPTOOL_ARGS := --esptool-args port=$(CONFIG_ESPTOOLPY_PORT) baud=$(CONFIG_ESPTOOLPY_BAUD) before=$(CONFIG_ESPTOOLPY_BEFORE) after=$(CONFIG_ESPTOOLPY_AFTER)
-
-erase_otadata: $(PARTITION_TABLE_CSV_PATH) partition_table_get_info | check_python_dependencies
-	$(OTATOOL_PY) $(ESPTOOL_ARGS) --partition-table-file $(PARTITION_TABLE_CSV_PATH) \
-	--partition-table-offset $(PARTITION_TABLE_OFFSET) \
-	erase_otadata
-
-read_otadata: $(PARTITION_TABLE_CSV_PATH) partition_table_get_info | check_python_dependencies
-	$(OTATOOL_PY) $(ESPTOOL_ARGS) --partition-table-file $(PARTITION_TABLE_CSV_PATH) \
-	--partition-table-offset $(partition_table_offset) \
-	read_otadata
-
-all: blank_ota_data
-flash: blank_ota_data
-ifdef CONFIG_SECURE_FLASH_ENCRYPTION_MODE_DEVELOPMENT
-encrypted-flash: blank_ota_data
-endif
-
-TMP_DEFINES := $(BUILD_DIR_BASE)/app_update/tmp_cppflags.txt
-export TMP_DEFINES
-
-clean:
-	rm -f $(BLANK_OTA_DATA_FILE)
-	rm -f $(TMP_DEFINES)

components/app_update/component.mk

@@ -1,56 +0,0 @@
-#
-# Component Makefile
-#
-# (Uses default behaviour of compiling all source files in directory, adding 'include' to include path.)
-
-# esp_app_desc structure is added as an undefined symbol because otherwise the
-# linker will ignore this structure as it has no other files depending on it.
-COMPONENT_ADD_LDFLAGS += -u esp_app_desc
-
-ifndef IS_BOOTLOADER_BUILD
-    # If ``CONFIG_APP_PROJECT_VER_FROM_CONFIG`` option is set, the value of ``CONFIG_APP_PROJECT_VER`` will be used
-    # Else, if ``PROJECT_VER`` variable set in project Makefile file, its value will be used.
-    # Else, if the ``$PROJECT_PATH/version.txt`` exists, its contents will be used as ``PROJECT_VER``.
-    # Else, if the project is located inside a Git repository, the output of git describe will be used.
-    # Otherwise, ``PROJECT_VER`` will be "1".
-    ifdef CONFIG_APP_PROJECT_VER_FROM_CONFIG
-        PROJECT_VER:= $(CONFIG_APP_PROJECT_VER)
-    else
-        ifneq ("${PROJECT_VER}", "")
-            PROJECT_VER:= $(PROJECT_VER)
-        else
-            ifneq ("$(wildcard ${PROJECT_PATH}/version.txt)","")
-                PROJECT_VER := $(shell cat ${PROJECT_PATH}/version.txt)
-            else
-                GIT_PROJECT_VER := $(shell cd ${PROJECT_PATH} && git describe --always --tags --dirty 2> /dev/null)
-                ifeq ("${GIT_PROJECT_VER}", "")
-                    PROJECT_VER := "1"
-                    $(info Project is not inside a git repository, or git repository has no commits)
-                    $(info will not use 'git describe' to determine PROJECT_VER.)
-                else
-                    PROJECT_VER:= $(GIT_PROJECT_VER)
-                endif # a git repository
-            endif # version.txt
-        endif # PROJECT_VER
-    endif # CONFIG_APP_PROJECT_VER_FROM_CONFIG
-
-    # cut PROJECT_VER and PROJECT_NAME to required 32 characters.
-    PROJECT_VER_CUT  := $(shell echo "$(PROJECT_VER)"  | cut -c 1-31)
-    PROJECT_NAME_CUT := $(shell echo "$(PROJECT_NAME)" | cut -c 1-31)
-
-    $(info App "$(PROJECT_NAME_CUT)" version: $(PROJECT_VER_CUT))
-
-    NEW_DEFINES:= "$(PROJECT_VER_CUT) $(PROJECT_NAME_CUT) $(IDF_VER)"
-    ifeq ("$(wildcard ${TMP_DEFINES})","")
-        OLD_DEFINES:= ""
-    else
-        OLD_DEFINES:= "$(shell cat $(TMP_DEFINES))"
-    endif
-
-    # If NEW_DEFINES (PROJECT_VER, PROJECT_NAME) were changed then rebuild only esp_app_desc.
-    ifneq (${NEW_DEFINES}, ${OLD_DEFINES})
-        $(shell echo $(NEW_DEFINES) > $(TMP_DEFINES); rm -f esp_app_desc.o;)
-    endif
-
-    esp_app_desc.o: CPPFLAGS += -D PROJECT_VER=\""$(PROJECT_VER_CUT)"\" -D PROJECT_NAME=\""$(PROJECT_NAME_CUT)"\"
-endif # IS_BOOTLOADER_BUILD

components/app_update/esp_app_desc.c

@@ -32,7 +32,7 @@ const __attribute__((section(".rodata_desc"))) esp_app_desc_t esp_app_desc = {
     .secure_version = 0,
 #endif
 
-#ifdef CONFIG_APP_COMPILE_TIME_DATE
+#if defined(CONFIG_APP_COMPILE_TIME_DATE) && !defined(CONFIG_APP_REPRODUCIBLE_BUILD)
     .time = __TIME__,
     .date = __DATE__,
 #else

components/app_update/esp_ota_ops.c

@@ -14,11 +14,10 @@
 
 #include "esp_err.h"
 #include "esp_partition.h"
-#include "esp_spi_flash.h"
+#include "spi_flash_mmap.h"
 #include "esp_image_format.h"
 #include "esp_secure_boot.h"
 #include "esp_flash_encrypt.h"
-#include "esp_spi_flash.h"
 #include "sdkconfig.h"
 
 #include "esp_ota_ops.h"
@@ -31,6 +30,20 @@
 #include "esp_efuse.h"
 #include "esp_attr.h"
 
+#if CONFIG_IDF_TARGET_ESP32
+#include "esp32/rom/secure_boot.h"
+#elif CONFIG_IDF_TARGET_ESP32S2
+#include "esp32s2/rom/secure_boot.h"
+#elif CONFIG_IDF_TARGET_ESP32C3
+#include "esp32c3/rom/secure_boot.h"
+#elif CONFIG_IDF_TARGET_ESP32S3
+#include "esp32s3/rom/secure_boot.h"
+#elif CONFIG_IDF_TARGET_ESP32H2
+#include "esp32h2/rom/secure_boot.h"
+#elif CONFIG_IDF_TARGET_ESP32C2
+#include "esp32c2/rom/secure_boot.h"
+#endif
+
 #define SUB_TYPE_ID(i) (i & 0x0F)
 
 /* Partial_data is word aligned so no reallocation is necessary for encrypted flash write */
@@ -899,7 +912,7 @@ esp_err_t esp_ota_revoke_secure_boot_public_key(esp_ota_secure_boot_public_key_i
     }
 
     esp_err_t ret;
-    ets_secure_boot_key_digests_t trusted_keys;
+    esp_secure_boot_key_digests_t trusted_keys;
     ret = esp_secure_boot_read_key_digests(&trusted_keys);
     if (ret != ESP_OK) {
         ESP_LOGE(TAG, "Could not read the secure boot key digests from efuse. Aborting..");

components/app_update/include/esp_ota_ops.h

@@ -112,17 +112,17 @@ esp_err_t esp_ota_begin(const esp_partition_t* partition, size_t image_size, esp
 esp_err_t esp_ota_write(esp_ota_handle_t handle, const void* data, size_t size);
 
 /**
- * @brief   Write OTA update data to partition
+ * @brief   Write OTA update data to partition at an offset
  *
- * This function can write data in non contiguous manner.
- * If flash encryption is enabled, data should be 16 byte aligned.
+ * This function can write data in non-contiguous manner.
+ * If flash encryption is enabled, data should be 16 bytes aligned.
  *
  * @param handle  Handle obtained from esp_ota_begin
  * @param data    Data buffer to write
  * @param size    Size of data buffer in bytes
  * @param offset  Offset in flash partition
  *
- * @note While performing OTA, if the packets arrive out of order, esp_ota_write_with_offset() can be used to write data in non contiguous manner.
+ * @note While performing OTA, if the packets arrive out of order, esp_ota_write_with_offset() can be used to write data in non-contiguous manner.
  *       Use of esp_ota_write_with_offset() in combination with esp_ota_write() is not recommended.
  *
  * @return
@@ -327,9 +327,9 @@ typedef enum {
 /**
  * @brief Revokes the old signature digest. To be called in the application after the rollback logic.
  *
- * Relevant for Secure boot v2 on ESP32-S2 where upto 3 key digests can be stored (Key #N-1, Key #N, Key #N+1).
- * When key #N-1 used to sign an app is invalidated, an OTA update is to be sent with an app signed with key #N-1 & Key #N.
- * After successfully booting the OTA app should call this function to revoke Key #N-1.
+ * Relevant for Secure boot v2 on ESP32-S2, ESP32-S3, ESP32-C3, ESP32-H2 where upto 3 key digests can be stored (Key \#N-1, Key \#N, Key \#N+1).
+ * When key \#N-1 used to sign an app is invalidated, an OTA update is to be sent with an app signed with key \#N-1 & Key \#N.
+ * After successfully booting the OTA app should call this function to revoke Key \#N-1.
  *
  * @param index - The index of the signature block to be revoked
  *

components/app_update/test/CMakeLists.txt

@@ -1,4 +1,5 @@
 idf_component_register(SRC_DIRS "."
                        PRIV_INCLUDE_DIRS "."
-                       PRIV_REQUIRES cmock test_utils app_update bootloader_support nvs_flash
+                       PRIV_REQUIRES cmock test_utils app_update bootloader_support nvs_flash driver
                       )
+target_compile_options(${COMPONENT_LIB} PRIVATE "-Wno-format")

components/app_update/test/component.mk

@@ -1,5 +0,0 @@
-#
-#Component Makefile
-#
-
-COMPONENT_ADD_LDFLAGS = -Wl,--whole-archive -l$(COMPONENT_NAME) -Wl,--no-whole-archive

components/app_update/test/test_switch_ota.c

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: 2021 Espressif Systems (Shanghai) CO LTD
+ * SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
  *
  * SPDX-License-Identifier: Apache-2.0
  */
@@ -12,11 +12,7 @@
 #include "string.h"
 #include "sdkconfig.h"
 
-#if CONFIG_IDF_TARGET_ESP32
-#include "esp32/rom/spi_flash.h"
-#elif CONFIG_IDF_TARGET_ESP32S2
-#include "esp32s2/rom/spi_flash.h"
-#endif
+#include "esp_rom_spiflash.h"
 
 #include "freertos/FreeRTOS.h"
 #include "freertos/task.h"
@@ -25,7 +21,7 @@
 #include "unity.h"
 
 #include "bootloader_common.h"
-#include "../include_bootloader/bootloader_flash_priv.h"
+#include "../bootloader_flash/include/bootloader_flash_priv.h"
 
 #include "esp_log.h"
 #include "esp_ota_ops.h"
@@ -36,9 +32,13 @@
 
 #include "driver/gpio.h"
 #include "esp_sleep.h"
+#include "test_utils.h"
 
 
+#if !TEMPORARY_DISABLED_FOR_TARGETS(ESP32C2)
+//IDF-5131
 RTC_DATA_ATTR static int boot_count = 0;
+
 static const char *TAG = "ota_test";
 
 /* @brief Copies a current app to next partition using handle.
@@ -824,3 +824,27 @@ static void test_flow6(void)
 // 2 Stage: run factory -> check it -> copy factory to OTA0             -> reboot  --//--
 // 3 Stage: run OTA0    -> check it -> erase OTA_DATA for next tests    -> PASS
 TEST_CASE_MULTIPLE_STAGES("Switching between factory, OTA0 using esp_ota_write_with_offset", "[app_update][timeout=90][reset=DEEPSLEEP_RESET, DEEPSLEEP_RESET]", start_test, test_flow6, test_flow6);
+
+//IDF-5145
+TEST_CASE("Test bootloader_common_get_sha256_of_partition returns ESP_ERR_IMAGE_INVALID when image is ivalid", "[partitions]")
+{
+    const esp_partition_t *cur_app = esp_ota_get_running_partition();
+    ESP_LOGI(TAG, "copy current app to next part");
+    const esp_partition_t *other_app = get_next_update_partition();
+    copy_current_app_to_next_part(cur_app, other_app);
+    erase_ota_data();
+
+    uint8_t sha_256_cur_app[32];
+    uint8_t sha_256_other_app[32];
+    TEST_ESP_OK(bootloader_common_get_sha256_of_partition(cur_app->address, cur_app->size, cur_app->type, sha_256_cur_app));
+    TEST_ESP_OK(bootloader_common_get_sha256_of_partition(other_app->address, other_app->size, other_app->type, sha_256_other_app));
+
+    TEST_ASSERT_EQUAL_MEMORY_MESSAGE(sha_256_cur_app, sha_256_other_app, sizeof(sha_256_cur_app), "must be the same");
+
+    uint32_t data = 0;
+    bootloader_flash_write(other_app->address + 0x50, &data, sizeof(data), false);
+
+    TEST_ESP_ERR(ESP_ERR_IMAGE_INVALID, bootloader_common_get_sha256_of_partition(other_app->address, other_app->size, other_app->type, sha_256_other_app));
+    TEST_ASSERT_EQUAL_MEMORY_MESSAGE(sha_256_cur_app, sha_256_other_app, sizeof(sha_256_cur_app), "must be the same");
+}
+#endif //!TEMPORARY_DISABLED_FOR_TARGETS(ESP32C2)

components/asio/CMakeLists.txt

@@ -1,41 +0,0 @@
-if(NOT CONFIG_LWIP_IPV6 AND NOT CMAKE_BUILD_EARLY_EXPANSION)
-    # note: the component is still included in the build so it can become visible again in config
-    # without needing to re-run CMake. However no source or header files are built.
-    message(STATUS "IPV6 support is disabled so the asio component will not be built")
-    idf_component_register()
-    return()
-endif()
-
-set(asio_sources "asio/asio/src/asio.cpp")
-
-if(CONFIG_ASIO_SSL_SUPPORT)
-    if(CONFIG_ASIO_USE_ESP_OPENSSL)
-        list(APPEND asio_sources
-                "asio/asio/src/asio_ssl.cpp"
-                "port/src/esp_asio_openssl_stubs.c")
-    endif()
-
-    if(CONFIG_ASIO_USE_ESP_WOLFSSL)
-        list(APPEND asio_sources
-                "asio/asio/src/asio_ssl.cpp")
-    endif()
-endif()
-
-idf_component_register(SRCS ${asio_sources}
-                    INCLUDE_DIRS "asio/asio/include" "port/include"
-                    REQUIRES lwip)
-
-if(CONFIG_ASIO_SSL_SUPPORT)
-    if(CONFIG_ASIO_USE_ESP_WOLFSSL)
-        idf_component_get_property(wolflib esp-wolfssl COMPONENT_LIB)
-        idf_component_get_property(wolfdir esp-wolfssl COMPONENT_DIR)
-
-        target_link_libraries(${COMPONENT_LIB} PUBLIC ${wolflib})
-        target_include_directories(${COMPONENT_LIB} PUBLIC ${wolfdir}/wolfssl/wolfssl)
-    endif()
-
-    if(CONFIG_ASIO_USE_ESP_OPENSSL)
-        idf_component_get_property(esp_openssl openssl COMPONENT_LIB)
-        target_link_libraries(${COMPONENT_LIB} PUBLIC ${esp_openssl})
-    endif()
-endif()

components/asio/Kconfig

@@ -1,27 +0,0 @@
-menu "ESP-ASIO"
-    visible if LWIP_IPV6
-
-    config ASIO_SSL_SUPPORT
-        bool "Enable SSL/TLS support of ASIO"
-        default n
-        help
-            Enable support for basic SSL/TLS features, available for mbedTLS/OpenSSL
-            as well as wolfSSL TLS library.
-
-    choice ASIO_SSL_LIBRARY_CHOICE
-        prompt "Choose SSL/TLS library for ESP-TLS (See help for more Info)"
-        default ASIO_USE_ESP_OPENSSL
-        depends on ASIO_SSL_SUPPORT
-        help
-            The ASIO support multiple backend TLS libraries. Currently the mbedTLS with a thin ESP-OpenSSL
-            port layer (default choice) and WolfSSL are supported.
-            Different TLS libraries may support different features and have different resource
-            usage. Consult the ESP-TLS documentation in ESP-IDF Programming guide for more details.
-        config ASIO_USE_ESP_OPENSSL
-            bool "esp-openssl"
-        config ASIO_USE_ESP_WOLFSSL
-            depends on TLS_STACK_WOLFSSL
-            bool "wolfSSL (License info in wolfSSL directory README)"
-    endchoice
-
-endmenu

components/asio/component.mk

@@ -1,13 +0,0 @@
-ifdef CONFIG_LWIP_IPV6
-
-COMPONENT_ADD_INCLUDEDIRS := asio/asio/include port/include
-COMPONENT_PRIV_INCLUDEDIRS := private_include
-COMPONENT_SRCDIRS := asio/asio/src port/src
-
-ifeq ($(CONFIG_ASIO_SSL_SUPPORT), )
-COMPONENT_OBJEXCLUDE := asio/asio/src/asio_ssl.o port/src/esp_asio_openssl_stubs.o
-endif
-
-COMPONENT_SUBMODULES += asio
-
-endif # CONFIG_LWIP_IPV6

components/asio/port/include/esp_asio_config.h

@@ -1,47 +0,0 @@
-/*
- * SPDX-FileCopyrightText: 2018-2021 Espressif Systems (Shanghai) CO LTD
- *
- * SPDX-License-Identifier: Apache-2.0
- */
-#ifndef _ESP_ASIO_CONFIG_H_
-#define _ESP_ASIO_CONFIG_H_
-
-//
-// Enabling exceptions only when they are enabled in menuconfig
-//
-# include <sdkconfig.h>
-# ifndef CONFIG_COMPILER_CXX_EXCEPTIONS
-#  define ASIO_NO_EXCEPTIONS
-# endif   // CONFIG_COMPILER_CXX_EXCEPTIONS
-
-# ifndef CONFIG_COMPILER_RTTI
-#  define ASIO_NO_TYPEID
-# endif   // CONFIG_COMPILER_RTTI
-
-//
-// Supress OpenSSL deprecation warning, when building ASIO
-//
-#define ESP_OPENSSL_SUPPRESS_LEGACY_WARNING
-
-//
-// LWIP compatibility inet and address macros/functions
-//
-# define LWIP_COMPAT_SOCKET_INET 1
-# define LWIP_COMPAT_SOCKET_ADDR 1
-
-//
-// Specific ASIO feature flags
-//
-# define ASIO_DISABLE_SERIAL_PORT
-# define ASIO_SEPARATE_COMPILATION
-# define ASIO_STANDALONE
-# define ASIO_HAS_PTHREADS
-
-# ifdef CONFIG_ASIO_USE_ESP_OPENSSL
-#  define ASIO_USE_ESP_OPENSSL
-#  define OPENSSL_NO_ENGINE
-# elif CONFIG_ASIO_USE_ESP_WOLFSSL
-#  define ASIO_USE_WOLFSSL
-# endif   // CONFIG_ASIO_USE_ESP_OPENSSL
-
-#endif // _ESP_ASIO_CONFIG_H_

components/asio/port/include/esp_exception.h

@@ -1,31 +0,0 @@
-
-/*
- * SPDX-FileCopyrightText: 2018-2021 Espressif Systems (Shanghai) CO LTD
- *
- * SPDX-License-Identifier: Apache-2.0
- */
-#ifndef _ESP_EXCEPTION_H_
-#define _ESP_EXCEPTION_H_
-
-//
-// This exception stub is enabled only if exceptions are disabled in menuconfig
-//
-#if !defined(CONFIG_COMPILER_CXX_EXCEPTIONS) && defined (ASIO_NO_EXCEPTIONS)
-
-#include "esp_log.h"
-
-//
-// asio exception stub
-//
-namespace asio {
-namespace detail {
-template <typename Exception>
-void throw_exception(const Exception& e)
-{
-  ESP_LOGE("esp32_asio_exception", "Caught exception: %s!", e.what());
-  abort();
-}
-}}
-#endif // CONFIG_COMPILER_CXX_EXCEPTIONS==1 && defined (ASIO_NO_EXCEPTIONS)
-
-#endif // _ESP_EXCEPTION_H_

components/asio/port/include/openssl/conf.h

@@ -1,18 +0,0 @@
-/*
- * SPDX-FileCopyrightText: 2020-2021 Espressif Systems (Shanghai) CO LTD
- *
- * SPDX-License-Identifier: Apache-2.0
- */
-
-#ifndef _ESP_ASIO_OPENSSL_CONF_H
-#define _ESP_ASIO_OPENSSL_CONF_H
-#include "esp_asio_config.h"
-#include "openssl/esp_asio_openssl_stubs.h"
-
-#if defined(ASIO_USE_WOLFSSL)
-// SSLv3 Methods not present in current wolfSSL library
-#define OPENSSL_NO_SSL3
-#include_next "openssl/conf.h"
-#endif // ASIO_USE_WOLFSSL
-
-#endif // _ESP_ASIO_OPENSSL_CONF_H

components/asio/port/include/openssl/dh.h

@@ -1,15 +0,0 @@
-/*
- * SPDX-FileCopyrightText: 2020-2021 Espressif Systems (Shanghai) CO LTD
- *
- * SPDX-License-Identifier: Apache-2.0
- */
-
-#ifndef _ESP_ASIO_OPENSSL_DH_STUB_H
-#define _ESP_ASIO_OPENSSL_DH_STUB_H
-// Dummy header needed for ASIO compilation with esp-openssl
-
-#if defined(ASIO_USE_WOLFSSL)
-#include_next "openssl/dh.h"
-#endif // ASIO_USE_WOLFSSL
-
-#endif // _ESP_ASIO_OPENSSL_DH_STUB_H

components/asio/port/include/openssl/esp_asio_openssl_stubs.h

@@ -1,201 +0,0 @@
-/*
- * SPDX-FileCopyrightText: 2020-2021 Espressif Systems (Shanghai) CO LTD
- *
- * SPDX-License-Identifier: Apache-2.0
- */
-
-#ifndef _ESP_ASIO_OPENSSL_STUBS_H
-#define _ESP_ASIO_OPENSSL_STUBS_H
-
-/**
- * @note This header contains openssl API which are NOT implemented, and are only provided
- * as stubs or no-operations to get the ASIO library compiled and working with most
- * practical use cases as an embedded application on ESP platform
- */
-
-#if defined(ASIO_USE_WOLFSSL)
-
-#include "wolfssl/ssl.h"
-// esp-wolfssl disables filesystem by default, but the ssl filesystem functions are needed for the ASIO to compile
-//  - so we could either configure wolfSSL to use filesystem
-//  - or use the default wolfSSL and declare the filesystem functions -- preferred option, as whenever
-//    the filesystem functions are used from app code (potential security impact if private keys in a filesystem)
-//    compilation fails with linking errors.
-
-#if defined(NO_FILESYSTEM)
-// WolfSSL methods that are not included in standard esp-wolfssl config, must be defined here
-// as function stubs,  so ASIO compiles, but would get link errors, if these functions were used.
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-typedef struct WOLFSSL_CTX      WOLFSSL_CTX;
-
-void wolfSSL_CTX_set_verify_depth(WOLFSSL_CTX *ctx,int depth);
-int SSL_CTX_load_verify_locations(WOLFSSL_CTX*, const char*, const char*);
-int SSL_CTX_use_certificate_file(WOLFSSL_CTX*, const char*, int);
-int SSL_CTX_use_certificate_chain_file(WOLFSSL_CTX*, const char*);
-int SSL_CTX_use_PrivateKey_file(WOLFSSL_CTX*, const char*, int);
-int SSL_CTX_use_RSAPrivateKey_file(WOLFSSL_CTX*, const char*, int);
-
-#if defined(__cplusplus)
-} /* extern C */
-#endif
-
-#endif // NO_FILESYSTEM
-
-#elif defined(ASIO_USE_ESP_OPENSSL)
-
-#include "internal/ssl_x509.h"
-#include "internal/ssl_pkey.h"
-#include "mbedtls/pem.h"
-#include <stdint.h>
-
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-
-// The most applicable OpenSSL version wrtt ASIO usage
-#define OPENSSL_VERSION_NUMBER 0x10100001L
-// SSLv2 methods not supported
-// OpenSSL port supports: TLS_ANY, TLS_1, TLS_1_1, TLS_1_2, SSL_3
-#define OPENSSL_NO_SSL2
-#define SSL2_VERSION 0x0002
-
-#define SSL_R_SHORT_READ 219
-#define SSL_OP_ALL 0
-#define SSL_OP_SINGLE_DH_USE 0
-#define SSL_OP_NO_COMPRESSION 0
-// Translates mbedTLS PEM parse error, used by ASIO
-#define PEM_R_NO_START_LINE -MBEDTLS_ERR_PEM_NO_HEADER_FOOTER_PRESENT
-
-#define SSL_OP_NO_SSLv2					0x01000000L
-#define SSL_OP_NO_SSLv3					0x02000000L
-#define SSL_OP_NO_TLSv1					0x04000000L
-
-#define X509_FILETYPE_PEM	1
-#define X509_FILETYPE_ASN1	2
-#define SSL_FILETYPE_ASN1	X509_FILETYPE_ASN1
-#define SSL_FILETYPE_PEM	X509_FILETYPE_PEM
-
-#define NID_subject_alt_name 85
-
-
-#define GEN_DNS		2
-#define GEN_IPADD	7
-#define V_ASN1_OCTET_STRING		4
-#define V_ASN1_IA5STRING		22
-#define NID_commonName 13
-
-#define SSL_CTX_get_app_data(ctx) ((void*)SSL_CTX_get_ex_data(ctx, 0))
-
-/**
-* @brief Frees DH object -- not implemented
-*
-* Current implementation calls SSL_ASSERT
-*
-* @param r DH object
-*/
-void DH_free(DH *r);
-
-/**
- * @brief Frees GENERAL_NAMES -- not implemented
- *
- * Current implementation calls SSL_ASSERT
- *
- * @param r GENERAL_NAMES object
- */
-void GENERAL_NAMES_free(GENERAL_NAMES * gens);
-
-/**
- * @brief Returns subject name from X509 -- not implemented
- *
- * Current implementation calls SSL_ASSERT
- *
- * @param r X509 object
- */
-X509_NAME *X509_get_subject_name(X509 *a);
-
-/**
- * @brief API provaded as declaration only
- *
- */
-int	X509_STORE_CTX_get_error_depth(X509_STORE_CTX *ctx);
-
-/**
- * @brief API provaded as declaration only
- *
- */
-int X509_NAME_get_index_by_NID(X509_NAME *name, int nid, int lastpos);
-
-/**
- * @brief API provaded as declaration only
- *
- */
-X509_NAME_ENTRY *X509_NAME_get_entry(X509_NAME *name, int loc);
-
-/**
- * @brief API provaded as declaration only
- *
- */
-ASN1_STRING *X509_NAME_ENTRY_get_data(X509_NAME_ENTRY *ne);
-
-/**
- * @brief API provaded as declaration only
- *
- */
-void *X509_get_ext_d2i(X509 *x, int nid, int *crit, int *idx);
-
-/**
- * @brief API provaded as declaration only
- *
- */
-X509 *	X509_STORE_CTX_get_current_cert(X509_STORE_CTX *ctx);
-
-/**
- * @brief Reads DH params from a bio object -- not implemented
- *
- * Current implementation calls SSL_ASSERT
- */
-DH *PEM_read_bio_DHparams(BIO *bp, DH **x, pem_password_cb *cb, void *u);
-
-/**
- * @brief API provaded as declaration only
- *
- */
-void *	X509_STORE_CTX_get_ex_data(X509_STORE_CTX *ctx,int idx);
-
-/**
- * @brief Sets DH params to ssl ctx -- not implemented
- *
- * Current implementation calls SSL_ASSERT
- */
-int SSL_CTX_set_tmp_dh(SSL_CTX *ctx, const DH *dh);
-
-/**
- * @brief API provaded as declaration only
- *
- */
-void SSL_CTX_set_default_passwd_cb_userdata(SSL_CTX *ctx, void *data);
-
-/**
- * @brief API provaded as declaration only
- *
- */
-void SSL_CTX_set_default_passwd_cb(SSL_CTX *ctx, pem_password_cb *cb);
-
-/**
- * @brief Clears any existing chain associated with the current certificate of ctx.
- *
- */
-int SSL_CTX_clear_chain_certs(SSL_CTX *ctx);
-
-#if defined(__cplusplus)
-} /* extern C */
-#endif
-
-#endif /* ASIO_USE_ESP_OPENSSL, ASIO_USE_WOLFSSL */
-#endif /* _ESP_ASIO_OPENSSL_STUBS_H */

components/asio/port/include/openssl/rsa.h

@@ -1,15 +0,0 @@
-/*
- * SPDX-FileCopyrightText: 2020-2021 Espressif Systems (Shanghai) CO LTD
- *
- * SPDX-License-Identifier: Apache-2.0
- */
-
-#ifndef _ESP_ASIO_OPENSSL_RSA_STUB_H
-#define _ESP_ASIO_OPENSSL_RSA_STUB_H
-// Dummy header needed for ASIO compilation with esp-openssl
-
-#if defined(ASIO_USE_WOLFSSL)
-#include_next "openssl/rsa.h"
-#endif // ASIO_USE_WOLFSSL
-
-#endif // _ESP_ASIO_OPENSSL_RSA_STUB_H

components/asio/port/include/openssl/x509v3.h

@@ -1,15 +0,0 @@
-/*
- * SPDX-FileCopyrightText: 2020-2021 Espressif Systems (Shanghai) CO LTD
- *
- * SPDX-License-Identifier: Apache-2.0
- */
-
-#ifndef _ESP_ASIO_OPENSSL_X509V3_STUB_H
-#define _ESP_ASIO_OPENSSL_X509V3_STUB_H
-// Dummy header needed for ASIO compilation with esp-openssl
-
-#if defined(ASIO_USE_WOLFSSL)
-#include_next "openssl/x509v3.h"
-#endif // ASIO_USE_WOLFSSL
-
-#endif // _ESP_ASIO_OPENSSL_X509V3_STUB_H

components/asio/port/src/esp_asio_openssl_stubs.c

@@ -1,47 +0,0 @@
-/*
- * SPDX-FileCopyrightText: 2020-2021 Espressif Systems (Shanghai) CO LTD
- *
- * SPDX-License-Identifier: Apache-2.0
- */
-//
-
-#include "esp_asio_config.h"
-#include "internal/ssl_dbg.h"
-#include "openssl/esp_asio_openssl_stubs.h"
-
-// Unsupported features as macros to make the assertions more readable
-#define ESP_OPENSSL_DH_IS_SUPPORTED 0
-#define ESP_OPENSSL_GENERAL_NAMES_IS_SUPPORTED 0
-
-void DH_free (DH *r)
-{
-    SSL_ASSERT3(ESP_OPENSSL_DH_IS_SUPPORTED);
-}
-
-DH *PEM_read_bio_DHparams(BIO *bp, DH **x, pem_password_cb *cb, void *u)
-{
-    SSL_ASSERT2(ESP_OPENSSL_DH_IS_SUPPORTED);
-    return NULL;
-}
-
-int SSL_CTX_set_tmp_dh(SSL_CTX *ctx, const DH *dh)
-{
-    SSL_ASSERT1(ESP_OPENSSL_DH_IS_SUPPORTED);
-    return -1;
-}
-
-void GENERAL_NAMES_free(GENERAL_NAMES * gens)
-{
-    SSL_ASSERT3(ESP_OPENSSL_GENERAL_NAMES_IS_SUPPORTED);
-}
-
-X509_NAME *X509_get_subject_name(X509 *a)
-{
-    SSL_ASSERT2(ESP_OPENSSL_GENERAL_NAMES_IS_SUPPORTED);
-    return NULL;
-}
-
-int SSL_CTX_clear_chain_certs(SSL_CTX *ctx)
-{
-    return 1;
-}

components/bootloader/Kconfig.projbuild

@@ -185,6 +185,20 @@ menu "Bootloader config"
             (factory or OTA[x]).
             Note that GPIO34-39 do not have an internal pullup and an external one must be provided.
 
+    choice BOOTLOADER_APP_TEST_PIN_LEVEL
+        bool "App test GPIO level"
+        depends on BOOTLOADER_APP_TEST
+        default BOOTLOADER_APP_TEST_PIN_LOW
+        help
+            Pin level for app test, can be triggered on low or high.
+
+        config BOOTLOADER_APP_TEST_PIN_LOW
+            bool "Enter test app on GPIO low"
+
+        config BOOTLOADER_APP_TEST_PIN_HIGH
+            bool "Enter test app on GPIO high"
+    endchoice
+
     config BOOTLOADER_HOLD_TIME_GPIO
         int "Hold time of GPIO for reset/test mode (seconds)"
         depends on BOOTLOADER_FACTORY_RESET || BOOTLOADER_APP_TEST
@@ -193,6 +207,14 @@ menu "Bootloader config"
             The GPIO must be held low continuously for this period of time after reset
             before a factory reset or test partition boot (as applicable) is performed.
 
+    config BOOTLOADER_REGION_PROTECTION_ENABLE
+        bool "Enable protection for unmapped memory regions"
+        default y
+        help
+            Protects the unmapped memory regions of the entire address space from unintended accesses.
+            This will ensure that an exception will be triggered whenever the CPU performs a memory
+            operation on unmapped regions of the address space.
+
     config BOOTLOADER_WDT_ENABLE
         bool "Use RTC watchdog in start code"
         default y
@@ -204,7 +226,7 @@ menu "Bootloader config"
             source for slow_clk - and ends calling app_main.
             Re-set timeout is needed due to WDT uses a SLOW_CLK clock source. After changing a frequency slow_clk a
             time of WDT needs to re-set for new frequency.
-            slow_clk depends on ESP32_RTC_CLK_SRC (INTERNAL_RC or EXTERNAL_CRYSTAL).
+            slow_clk depends on RTC_CLK_SRC (INTERNAL_RC or EXTERNAL_CRYSTAL).
 
     config BOOTLOADER_WDT_DISABLE_IN_USER_CODE
         bool "Allows RTC watchdog disable in user code"
@@ -273,6 +295,8 @@ menu "Bootloader config"
         depends on BOOTLOADER_APP_ANTI_ROLLBACK
         range 1 32 if IDF_TARGET_ESP32
         default 32 if IDF_TARGET_ESP32
+        range 1 4 if IDF_TARGET_ESP32C2
+        default 4 if IDF_TARGET_ESP32C2
         range 1 16
         default 16
         help
@@ -299,7 +323,7 @@ menu "Bootloader config"
         # options, allowing to turn on "allow insecure options" and have secure boot with
         # "skip validation when existing deep sleep". Keeping this to avoid a breaking change,
         # but - as noted in help - it invalidates the integrity of Secure Boot checks
-        depends on (SECURE_BOOT && SECURE_BOOT_INSECURE) || !SECURE_BOOT
+        depends on SOC_RTC_FAST_MEM_SUPPORTED && ((SECURE_BOOT && SECURE_BOOT_INSECURE) || !SECURE_BOOT)
         default n
         help
             This option disables the normal validation of an image coming out of
@@ -355,6 +379,7 @@ menu "Bootloader config"
 
     config BOOTLOADER_RESERVE_RTC_SIZE
         hex
+        depends on SOC_RTC_FAST_MEM_SUPPORTED
         default 0x10 if BOOTLOADER_SKIP_VALIDATE_IN_DEEP_SLEEP || BOOTLOADER_CUSTOM_RESERVE_RTC
         default 0
         help
@@ -366,6 +391,7 @@ menu "Bootloader config"
 
     config BOOTLOADER_CUSTOM_RESERVE_RTC
         bool "Reserve RTC FAST memory for custom purposes"
+        depends on SOC_RTC_FAST_MEM_SUPPORTED
         default n
         help
             This option allows the customer to place data in the RTC FAST memory,
@@ -421,16 +447,39 @@ menu "Security features"
         select MBEDTLS_ECDSA_C
         depends on SECURE_SIGNED_ON_BOOT || SECURE_SIGNED_ON_UPDATE
 
-    config SECURE_BOOT_SUPPORTS_RSA
+    config SECURE_BOOT_V2_RSA_SUPPORTED
         bool
         default y
-        depends on ESP32_REV_MIN_3 || IDF_TARGET_ESP32S2 || IDF_TARGET_ESP32C3 || IDF_TARGET_ESP32S3
+        # RSA secure boot is supported in ESP32 revision >= ECO3
+        depends on (IDF_TARGET_ESP32 && ESP32_REV_MIN >= 3) || SOC_SECURE_BOOT_V2_RSA
 
-    config SECURE_TARGET_HAS_SECURE_ROM_DL_MODE
+    config SECURE_BOOT_V2_ECC_SUPPORTED
         bool
         default y
-        depends on IDF_TARGET_ESP32S2 || IDF_TARGET_ESP32C3 || IDF_TARGET_ESP32S3
+        depends on SOC_SECURE_BOOT_V2_ECC
 
+    config SECURE_BOOT_V1_SUPPORTED
+        bool
+        default y
+        depends on SOC_SECURE_BOOT_V1
+
+    config SECURE_BOOT_V2_PREFERRED
+        bool
+        default y
+        depends on ESP32_REV_MIN >= 3
+
+    config SECURE_BOOT_V2_ECDSA_ENABLED
+        bool
+        default y if SECURE_BOOT_V2_ENABLED && SECURE_BOOT_V2_ECC_SUPPORTED
+
+    config SECURE_BOOT_V2_RSA_ENABLED
+        bool
+        default y if SECURE_BOOT_V2_ENABLED && SECURE_BOOT_V2_RSA_SUPPORTED
+
+    config SECURE_BOOT_FLASH_ENC_KEYS_BURN_TOGETHER
+        bool
+        default y if SOC_EFUSE_CONSISTS_OF_ONE_KEY_BLOCK && SECURE_BOOT && SECURE_FLASH_ENC_ENABLED
+        # ESP32-C2 has one key block for SB and FE keys. These keys must be burned at the same time.
 
     config SECURE_SIGNED_APPS_NO_SECURE_BOOT
         bool "Require signed app images"
@@ -447,29 +496,66 @@ menu "Security features"
         bool "App Signing Scheme"
         depends on SECURE_BOOT || SECURE_SIGNED_APPS_NO_SECURE_BOOT
         default SECURE_SIGNED_APPS_ECDSA_SCHEME if SECURE_BOOT_V1_ENABLED
-        default SECURE_SIGNED_APPS_RSA_SCHEME if SECURE_BOOT_V2_ENABLED
+        default SECURE_SIGNED_APPS_RSA_SCHEME if SECURE_BOOT_V2_RSA_SUPPORTED
+        default SECURE_SIGNED_APPS_ECDSA_V2_SCHEME if SECURE_BOOT_V2_ECC_SUPPORTED
         help
             Select the Secure App signing scheme. Depends on the Chip Revision.
-            There are two options:
-            1. ECDSA based secure boot scheme. (Only choice for Secure Boot V1)
-            Supported in ESP32 and ESP32-ECO3.
-            2. The RSA based secure boot scheme. (Only choice for Secure Boot V2)
-            Supported in ESP32-ECO3 (ESP32 Chip Revision 3 onwards), ESP32-S2, ESP32-C3, ESP32-S3.
+            There are two secure boot versions:
+
+            1. Secure boot V1
+                - Legacy custom secure boot scheme. Supported in ESP32 SoC.
+
+            2. Secure boot V2
+                - RSA based secure boot scheme.
+                  Supported in ESP32-ECO3 (ESP32 Chip Revision 3 onwards), ESP32-S2, ESP32-C3, ESP32-S3 SoCs.
+
+                - ECDSA based secure boot scheme. Supported in ESP32-C2 SoC.
 
         config SECURE_SIGNED_APPS_ECDSA_SCHEME
             bool "ECDSA"
-            depends on IDF_TARGET_ESP32 && (SECURE_SIGNED_APPS_NO_SECURE_BOOT || SECURE_BOOT_V1_ENABLED)
+            depends on SECURE_BOOT_V1_SUPPORTED && (SECURE_SIGNED_APPS_NO_SECURE_BOOT || SECURE_BOOT_V1_ENABLED)
             help
                 Embeds the ECDSA public key in the bootloader and signs the application with an ECDSA key.
-
                 Refer to the documentation before enabling.
 
         config SECURE_SIGNED_APPS_RSA_SCHEME
             bool "RSA"
-            depends on SECURE_BOOT_SUPPORTS_RSA && (SECURE_SIGNED_APPS_NO_SECURE_BOOT || SECURE_BOOT_V2_ENABLED)
+            depends on SECURE_BOOT_V2_RSA_SUPPORTED && (SECURE_SIGNED_APPS_NO_SECURE_BOOT || SECURE_BOOT_V2_ENABLED)
             help
                 Appends the RSA-3072 based Signature block to the application.
                 Refer to <Secure Boot Version 2 documentation link> before enabling.
+
+        config SECURE_SIGNED_APPS_ECDSA_V2_SCHEME
+            bool "ECDSA (V2)"
+            depends on SECURE_BOOT_V2_ECC_SUPPORTED && (SECURE_SIGNED_APPS_NO_SECURE_BOOT || SECURE_BOOT_V2_ENABLED)
+            help
+                For Secure boot V2 (e.g., ESP32-C2 SoC), appends ECDSA based signature block to the application.
+                Refer to documentation before enabling.
+
+    endchoice
+
+    choice SECURE_BOOT_ECDSA_KEY_LEN_SIZE
+        bool "ECDSA key size"
+        depends on SECURE_SIGNED_APPS_ECDSA_V2_SCHEME
+        default SECURE_BOOT_ECDSA_KEY_LEN_256_BITS
+        help
+            Select the ECDSA key size. Two key sizes are supported
+
+            - 192 bit key using NISTP192 curve
+            - 256 bit key using NISTP256 curve (Recommended)
+
+            The advantage of using 256 bit key is the extra randomness which makes it difficult to be
+            bruteforced compared to 192 bit key.
+            At present, both key sizes are practically implausible to bruteforce.
+
+        config SECURE_BOOT_ECDSA_KEY_LEN_192_BITS
+            bool "Using ECC curve NISTP192"
+            depends on SECURE_SIGNED_APPS_ECDSA_V2_SCHEME
+
+        config SECURE_BOOT_ECDSA_KEY_LEN_256_BITS
+            bool "Using ECC curve NISTP256 (Recommended)"
+            depends on SECURE_SIGNED_APPS_ECDSA_V2_SCHEME
+
     endchoice
 
     config SECURE_SIGNED_ON_BOOT_NO_SECURE_BOOT
@@ -501,7 +587,8 @@ menu "Security features"
     config SECURE_BOOT
         bool "Enable hardware Secure Boot in bootloader (READ DOCS FIRST)"
         default n
-        depends on IDF_TARGET_ESP32 || IDF_TARGET_ESP32S2 || ESP32C3_REV_MIN_3 || IDF_TARGET_ESP32S3
+        # Secure boot is not supported for ESP32-C3 revision < ECO3
+        depends on SOC_SECURE_BOOT_SUPPORTED && !(IDF_TARGET_ESP32C3 && ESP32C3_REV_MIN < 3)
         select ESPTOOLPY_NO_STUB if !IDF_TARGET_ESP32 && !IDF_TARGET_ESP32S2
         help
             Build a bootloader which enables Secure Boot on first boot.
@@ -514,25 +601,31 @@ menu "Security features"
 
     choice SECURE_BOOT_VERSION
         bool "Select secure boot version"
-        default SECURE_BOOT_V2_ENABLED if ESP32_REV_MIN_3
+        default SECURE_BOOT_V2_ENABLED if SECURE_BOOT_V2_PREFERRED
         depends on SECURE_BOOT
         help
             Select the Secure Boot Version. Depends on the Chip Revision.
-            Secure Boot V2 is the new RSA based secure boot scheme.
-            Supported in ESP32-ECO3 (ESP32 Chip Revision 3 onwards), ESP32-S2, ESP32-C3 ECO3.
-            Secure Boot V1 is the AES based secure boot scheme.
-            Supported in ESP32 and ESP32-ECO3.
+            Secure Boot V2 is the new RSA / ECDSA based secure boot scheme.
+
+                - RSA based scheme is supported in ESP32 (Revision 3 onwards), ESP32-S2, ESP32-C3 (ECO3), ESP32-S3.
+                - ECDSA based scheme is supported in ESP32-C2 SoC.
+
+            Please note that, RSA or ECDSA secure boot is property of specific SoC based on its HW design, supported
+            crypto accelerators, die-size, cost and similar parameters. Please note that RSA scheme has requirement
+            for bigger key sizes but at the same time it is comparatively faster than ECDSA verification.
+
+            Secure Boot V1 is the AES based (custom) secure boot scheme supported in ESP32 SoC.
 
         config SECURE_BOOT_V1_ENABLED
             bool "Enable Secure Boot version 1"
-            depends on IDF_TARGET_ESP32
+            depends on SECURE_BOOT_V1_SUPPORTED
             help
                 Build a bootloader which enables secure boot version 1 on first boot.
                 Refer to the Secure Boot section of the ESP-IDF Programmer's Guide for this version before enabling.
 
         config SECURE_BOOT_V2_ENABLED
             bool "Enable Secure Boot version 2"
-            depends on SECURE_BOOT_SUPPORTS_RSA
+            depends on SECURE_BOOT_V2_RSA_SUPPORTED || SECURE_BOOT_V2_ECC_SUPPORTED
             help
                 Build a bootloader which enables Secure Boot version 2 on first boot.
                 Refer to Secure Boot V2 section of the ESP-IDF Programmer's Guide for this version before enabling.
@@ -614,6 +707,22 @@ menu "Security features"
 
             Refer to the Secure Boot section of the ESP-IDF Programmer's Guide for this version before enabling.
 
+    config SECURE_BOOT_ENABLE_AGGRESSIVE_KEY_REVOKE
+        bool "Enable Aggressive key revoke strategy"
+        depends on SECURE_BOOT && SOC_SUPPORT_SECURE_BOOT_REVOKE_KEY
+        default N
+        help
+            If this option is set, ROM bootloader will revoke the public key digest burned in efuse block
+            if it fails to verify the signature of software bootloader with it.
+            Revocation of keys does not happen when enabling secure boot. Once secure boot is enabled,
+            key revocation checks will be done on subsequent boot-up, while verifying the software bootloader
+
+            This feature provides a strong resistance against physical attacks on the device.
+
+            NOTE: Once a digest slot is revoked, it can never be used again to verify an image
+            This can lead to permanent bricking of the device, in case all keys are revoked
+            because of signature verification failure.
+
     choice SECURE_BOOTLOADER_KEY_ENCODING
         bool "Hardware Key Encoding"
         depends on SECURE_BOOTLOADER_REFLASHABLE
@@ -665,21 +774,29 @@ menu "Security features"
     choice SECURE_FLASH_ENCRYPTION_KEYSIZE
         bool "Size of generated AES-XTS key"
         default SECURE_FLASH_ENCRYPTION_AES128
-        depends on (IDF_TARGET_ESP32S2 || IDF_TARGET_ESP32S3) && SECURE_FLASH_ENC_ENABLED
+        depends on SOC_FLASH_ENCRYPTION_XTS_AES_OPTIONS && SECURE_FLASH_ENC_ENABLED
         help
             Size of generated AES-XTS key.
 
-            AES-128 uses a 256-bit key (32 bytes) which occupies one Efuse key block.
-            AES-256 uses a 512-bit key (64 bytes) which occupies two Efuse key blocks.
+            - AES-128 uses a 256-bit key (32 bytes) derived from 128 bits (16 bytes) burned in half Efuse key block.
+              Internally, it calculates SHA256(128 bits)
+            - AES-128 uses a 256-bit key (32 bytes) which occupies one Efuse key block.
+            - AES-256 uses a 512-bit key (64 bytes) which occupies two Efuse key blocks.
 
             This setting is ignored if either type of key is already burned to Efuse before the first boot.
             In this case, the pre-burned key is used and no new key is generated.
 
+        config SECURE_FLASH_ENCRYPTION_AES128_DERIVED
+            bool "AES-128 key derived from 128 bits (SHA256(128 bits))"
+            depends on SOC_FLASH_ENCRYPTION_XTS_AES_128_DERIVED
+
         config SECURE_FLASH_ENCRYPTION_AES128
             bool "AES-128 (256-bit key)"
+            depends on SOC_FLASH_ENCRYPTION_XTS_AES_128 && !(IDF_TARGET_ESP32C2 && SECURE_BOOT)
 
         config SECURE_FLASH_ENCRYPTION_AES256
             bool "AES-256 (512-bit key)"
+            depends on SOC_FLASH_ENCRYPTION_XTS_AES_256
     endchoice
 
     choice SECURE_FLASH_ENCRYPTION_MODE
@@ -703,12 +820,12 @@ menu "Security features"
 
         config SECURE_FLASH_ENCRYPTION_MODE_RELEASE
             bool "Release"
-            select PARTITION_TABLE_MD5 if !ESP32_COMPATIBLE_PRE_V3_1_BOOTLOADERS
+            select PARTITION_TABLE_MD5 if !APP_COMPATIBLE_PRE_V3_1_BOOTLOADERS
 
     endchoice
 
     menu "Potentially insecure options"
-        visible if SECURE_FLASH_ENCRYPTION_MODE_DEVELOPMENT || SECURE_BOOT_INSECURE
+        visible if SECURE_FLASH_ENCRYPTION_MODE_DEVELOPMENT || SECURE_BOOT_INSECURE || SECURE_SIGNED_ON_UPDATE_NO_SECURE_BOOT # NOERROR
 
         # NOTE: Options in this menu NEED to have SECURE_BOOT_INSECURE
         # and/or SECURE_FLASH_ENCRYPTION_MODE_DEVELOPMENT in "depends on", as the menu
@@ -745,7 +862,7 @@ menu "Security features"
 
         config SECURE_BOOT_ALLOW_SHORT_APP_PARTITION
             bool "Allow app partition length not 64KB aligned"
-            depends on SECURE_BOOT_INSECURE
+            depends on SECURE_BOOT_INSECURE || SECURE_SIGNED_ON_UPDATE_NO_SECURE_BOOT
             help
                 If not set (default), app partition size must be a multiple of 64KB. App images are padded to 64KB
                 length, and the bootloader checks any trailing bytes after the signature (before the next 64KB
@@ -779,7 +896,7 @@ menu "Security features"
 
         config SECURE_BOOT_ALLOW_UNUSED_DIGEST_SLOTS
             bool "Leave unused digest slots available (not revoke)"
-            depends on SECURE_BOOT_INSECURE && !IDF_TARGET_ESP32
+            depends on SECURE_BOOT_INSECURE && SOC_EFUSE_REVOKE_BOOT_KEY_DIGESTS
             default N
             help
                 If not set (default), during startup in the app all unused digest slots will be revoked.
@@ -846,9 +963,13 @@ menu "Security features"
 
             If not set, the app does not care if the flash encryption eFuse bit is set or not.
 
+    config SECURE_ROM_DL_MODE_ENABLED
+        bool
+        default y if SOC_SUPPORTS_SECURE_DL_MODE && !SECURE_FLASH_ENCRYPTION_MODE_DEVELOPMENT
+
     choice SECURE_UART_ROM_DL_MODE
         bool "UART ROM download mode"
-        default SECURE_ENABLE_SECURE_ROM_DL_MODE if SECURE_TARGET_HAS_SECURE_ROM_DL_MODE && !SECURE_FLASH_ENCRYPTION_MODE_DEVELOPMENT # NOERROR
+        default SECURE_ENABLE_SECURE_ROM_DL_MODE if SECURE_ROM_DL_MODE_ENABLED # NOERROR
         default SECURE_INSECURE_ALLOW_DL_MODE
         depends on SECURE_BOOT_V2_ENABLED || SECURE_FLASH_ENC_ENABLED
         depends on !IDF_TARGET_ESP32 || ESP32_REV_MIN_3
@@ -870,7 +991,7 @@ menu "Security features"
 
         config SECURE_ENABLE_SECURE_ROM_DL_MODE
             bool "UART ROM download mode (Permanently switch to Secure mode (recommended))"
-            depends on SECURE_TARGET_HAS_SECURE_ROM_DL_MODE
+            depends on SOC_SUPPORTS_SECURE_DL_MODE
             select ESPTOOLPY_NO_STUB
             help
                 If set, during startup the app will burn an eFuse bit to permanently switch the UART ROM

components/bootloader/Makefile.projbuild

@@ -1,167 +0,0 @@
-# Bootloader component (top-level project parts)
-#
-# The bootloader is not a real component that gets linked into the project.
-# Instead it is an entire standalone project (in subproject/) that gets
-# built in the upper project's build directory. This Makefile.projbuild provides
-# the glue to build the bootloader project from the original project. It
-# basically runs Make in the subproject/ directory but it needs to
-# zero some variables the ESP-IDF project.mk makefile exports first, to not
-# let them interfere.
-#
-BOOTLOADER_COMPONENT_PATH := $(COMPONENT_PATH)
-BOOTLOADER_BUILD_DIR=$(abspath $(BUILD_DIR_BASE)/bootloader)
-BOOTLOADER_BIN=$(BOOTLOADER_BUILD_DIR)/bootloader.bin
-
-# signing key path is resolved relative to the project directory
-CONFIG_SECURE_BOOT_SIGNING_KEY ?=
-SECURE_BOOT_SIGNING_KEY=$(abspath $(call dequote,$(CONFIG_SECURE_BOOT_SIGNING_KEY)))
-export SECURE_BOOT_SIGNING_KEY  # used by bootloader_support component
-
-BOOTLOADER_SIGNED_BIN ?=
-
-# Has a matching value in bootloader_support esp_flash_partitions.h
-BOOTLOADER_OFFSET := 0x1000
-
-# Custom recursive make for bootloader sub-project
-#
-# NB: Some variables are cleared in the environment, not
-# overriden, because they need to be re-defined in the child
-# project.
-#
-# Pass PROJECT_PATH variable, it will let the subproject look
-# for user defined bootloader component(s).
-BOOTLOADER_MAKE= +\
-	PROJECT_PATH= \
-	COMPONENT_DIRS= \
-	$(MAKE) -C $(BOOTLOADER_COMPONENT_PATH)/subproject \
-	V=$(V) \
-	BUILD_DIR_BASE=$(BOOTLOADER_BUILD_DIR) \
-	TEST_COMPONENTS= \
-	TESTS_ALL= \
-	EXCLUDE_COMPONENTS= \
-	PROJECT_SOURCE_DIR=$(PROJECT_PATH)
-
-.PHONY: bootloader-clean bootloader-flash bootloader-list-components bootloader $(BOOTLOADER_BIN)
-
-$(BOOTLOADER_BIN): $(SDKCONFIG_MAKEFILE)
-	$(BOOTLOADER_MAKE) $@
-
-clean: bootloader-clean
-
-bootloader-list-components:
-	$(BOOTLOADER_MAKE) list-components
-
-ifndef CONFIG_SECURE_BOOT
-# If secure boot disabled, bootloader flashing is integrated
-# with 'make flash' and no warnings are printed.
-
-bootloader: $(BOOTLOADER_BIN) | check_python_dependencies
-	@echo $(SEPARATOR)
-	@echo "Bootloader built. Default flash command is:"
-	@echo "$(ESPTOOLPY_WRITE_FLASH) $(BOOTLOADER_OFFSET) $^"
-
-ESPTOOL_ALL_FLASH_ARGS += $(BOOTLOADER_OFFSET) $(BOOTLOADER_BIN)
-UF2_ADD_BINARIES += $(BOOTLOADER_OFFSET) $(BOOTLOADER_BIN)
-
-bootloader-flash: $(BOOTLOADER_BIN) $(call prereq_if_explicit,erase_flash) | check_python_dependencies
-	$(ESPTOOLPY_WRITE_FLASH) 0x1000 $^
-
-else ifdef CONFIG_SECURE_BOOTLOADER_ONE_TIME_FLASH
-
-# One time flashing requires user to run esptool.py command themselves,
-# and warning is printed about inability to reflash.
-#
-# The flashing command is deliberately printed without an auto-reset
-# step, so the device doesn't immediately reset to flash itself.
-
-bootloader: $(BOOTLOADER_BIN) | check_python_dependencies
-	@echo $(SEPARATOR)
-	@echo "Bootloader built. One-time flash command is:"
-	@echo "$(subst hard_reset,no_reset,$(ESPTOOLPY_WRITE_FLASH)) $(BOOTLOADER_OFFSET) $(BOOTLOADER_BIN)"
-	@echo $(SEPARATOR)
-	@echo "* IMPORTANT: After first boot, BOOTLOADER CANNOT BE RE-FLASHED on same device"
-
-else ifdef CONFIG_SECURE_BOOTLOADER_REFLASHABLE
-# Reflashable secure bootloader
-# generates a digest binary (bootloader + digest)
-
-ifdef CONFIG_SECURE_BOOTLOADER_KEY_ENCODING_192BIT
-KEY_DIGEST_LEN=192
-else
-KEY_DIGEST_LEN=256
-endif
-
-BOOTLOADER_DIGEST_BIN := $(BOOTLOADER_BUILD_DIR)/bootloader-reflash-digest.bin
-SECURE_BOOTLOADER_KEY := $(BOOTLOADER_BUILD_DIR)/secure-bootloader-key-$(KEY_DIGEST_LEN).bin
-
-ifdef CONFIG_SECURE_BOOT_BUILD_SIGNED_BINARIES
-$(SECURE_BOOTLOADER_KEY): $(SECURE_BOOT_SIGNING_KEY) | check_python_dependencies
-	$(ESPSECUREPY) digest_private_key --keylen $(KEY_DIGEST_LEN) -k $< $@
-else
-$(SECURE_BOOTLOADER_KEY):
-	@echo "No pre-generated key for a reflashable secure bootloader is available, due to signing configuration."
-	@echo "To generate one, you can use this command:"
-	@echo "espsecure.py generate_flash_encryption_key $@"
-	@echo "then re-run make."
-	exit 1
-endif
-
-bootloader: $(BOOTLOADER_DIGEST_BIN)
-	@echo $(SEPARATOR)
-	@echo "Bootloader built and secure digest generated. First time flash command is:"
-	@echo "$(ESPEFUSEPY) burn_key secure_boot_v1 $(SECURE_BOOTLOADER_KEY)"
-	@echo "$(ESPTOOLPY_WRITE_FLASH) $(BOOTLOADER_OFFSET) $(BOOTLOADER_BIN)"
-	@echo $(SEPARATOR)
-	@echo "To reflash the bootloader after initial flash:"
-	@echo "$(ESPTOOLPY_WRITE_FLASH) 0x0 $(BOOTLOADER_DIGEST_BIN)"
-	@echo $(SEPARATOR)
-	@echo "* After first boot, only re-flashes of this kind (with same key) will be accepted."
-	@echo "* Not recommended to re-use the same secure boot keyfile on multiple production devices."
-
-$(BOOTLOADER_DIGEST_BIN): $(BOOTLOADER_BIN) $(SECURE_BOOTLOADER_KEY) | check_python_dependencies
-	@echo "DIGEST $(notdir $@)"
-	$(ESPSECUREPY) digest_secure_bootloader -k $(SECURE_BOOTLOADER_KEY) -o $@ $<
-
-else ifdef CONFIG_SECURE_BOOT_V2_ENABLED
-BOOTLOADER_SIGNED_BIN := $(BOOTLOADER_BUILD_DIR)/bootloader-signed.bin
-ifdef CONFIG_SECURE_BOOT_BUILD_SIGNED_BINARIES
-bootloader: $(BOOTLOADER_BIN) $(SDKCONFIG_MAKEFILE) | check_python_dependencies
-	$(ESPSECUREPY) sign_data --keyfile $(SECURE_BOOT_SIGNING_KEY) --version 2 \
-	-o $(BOOTLOADER_SIGNED_BIN) $(BOOTLOADER_BIN)
-else
-bootloader: $(BOOTLOADER_BIN) $(SDKCONFIG_MAKEFILE) | check_python_dependencies
-	@echo "Bootloader not signed. Sign the bootloader before flashing."
-	@echo "To sign the bootloader, you can use this command:"
-	@echo "espsecure.py sign_data --keyfile SECURE_BOOT_SIGNING_KEY --version 2 $(BOOTLOADER_BIN)"
-endif
-	@echo $(SEPARATOR)
-	@echo "Use the following command to flash the bootloader:"
-ifdef CONFIG_SECURE_BOOT_BUILD_SIGNED_BINARIES
-	@echo "$(ESPTOOLPY_WRITE_FLASH) $(BOOTLOADER_OFFSET) $(BOOTLOADER_SIGNED_BIN)"
-else
-	@echo "$(ESPTOOLPY_WRITE_FLASH) $(BOOTLOADER_OFFSET) $(BOOTLOADER_BIN)"
-endif
-	@echo $(SEPARATOR)
-
-else # CONFIG_SECURE_BOOT && !CONFIG_SECURE_BOOTLOADER_REFLASHABLE \
-&& !CONFIG_SECURE_BOOTLOADER_ONE_TIME_FLASH && !CONFIG_SECURE_SIGNED_APPS_RSA_SCHEME
-bootloader:
-	@echo "Invalid bootloader target: bad sdkconfig?"
-	@exit 1
-endif
-
-ifndef CONFIG_SECURE_BOOT
-# don't build bootloader by default if secure boot is enabled
-all_binaries: $(BOOTLOADER_BIN)
-endif
-
-bootloader-clean: $(SDKCONFIG_MAKEFILE)
-	$(BOOTLOADER_MAKE) app-clean
-ifdef CONFIG_SECURE_BOOTLOADER_REFLASHABLE
-	rm -f $(SECURE_BOOTLOADER_KEY) $(BOOTLOADER_DIGEST_BIN)
-endif
-ifdef CONFIG_SECURE_SIGNED_APPS_RSA_SCHEME
-ifdef CONFIG_SECURE_BOOT_BUILD_SIGNED_BINARIES
-	rm -f $(BOOTLOADER_SIGNED_BIN)
-endif
-endif

components/bootloader/component.mk

@@ -1,6 +0,0 @@
-# bootloader component is special, as bootloader is also a project.
-#
-# This top-level component is only configuration files for the IDF project.
-#
-# See Makefile.projbuild for the targets which actually build the bootloader.
-COMPONENT_CONFIG_ONLY := 1

components/bootloader/project_include.cmake

@@ -30,9 +30,9 @@ if(CONFIG_SECURE_SIGNED_APPS)
     add_custom_target(gen_secure_boot_keys)
 
     if(CONFIG_SECURE_SIGNED_APPS_ECDSA_SCHEME)
-        set(secure_apps_signing_scheme "1")
-    elseif(CONFIG_SECURE_SIGNED_APPS_RSA_SCHEME)
-        set(secure_apps_signing_scheme "2")
+        set(secure_apps_signing_version "1")
+    elseif(CONFIG_SECURE_SIGNED_APPS_RSA_SCHEME OR CONFIG_SECURE_SIGNED_APPS_ECDSA_V2_SCHEME)
+        set(secure_apps_signing_version "2")
     endif()
 
     if(CONFIG_SECURE_BOOT_V1_ENABLED)
@@ -64,10 +64,22 @@ if(CONFIG_SECURE_SIGNED_APPS)
             # If the signing key is not found, create a phony gen_secure_boot_signing_key target that
             # fails the build. fail_at_build_time causes a cmake run next time
             # (to pick up a new signing key if one exists, etc.)
-            fail_at_build_time(gen_secure_boot_signing_key
-                "Secure Boot Signing Key ${CONFIG_SECURE_BOOT_SIGNING_KEY} does not exist. Generate using:"
-                "\tespsecure.py generate_signing_key --version ${secure_apps_signing_scheme} \
-                ${CONFIG_SECURE_BOOT_SIGNING_KEY}")
+            if(CONFIG_SECURE_SIGNED_APPS_RSA_SCHEME)
+                fail_at_build_time(gen_secure_boot_signing_key
+                    "Secure Boot Signing Key ${CONFIG_SECURE_BOOT_SIGNING_KEY} does not exist. Generate using:"
+                    "\tespsecure.py generate_signing_key --version ${secure_apps_signing_version} \
+                    ${CONFIG_SECURE_BOOT_SIGNING_KEY}")
+            else()
+                if(CONFIG_SECURE_BOOT_ECDSA_KEY_LEN_192_BITS)
+                    set(scheme "ecdsa192")
+                elseif(CONFIG_SECURE_BOOT_ECDSA_KEY_LEN_256_BITS)
+                    set(scheme "ecdsa256")
+                endif()
+                fail_at_build_time(gen_secure_boot_signing_key
+                    "Secure Boot Signing Key ${CONFIG_SECURE_BOOT_SIGNING_KEY} does not exist. Generate using:"
+                    "\tespsecure.py generate_signing_key --version ${secure_apps_signing_version} \
+                    --scheme ${scheme} ${CONFIG_SECURE_BOOT_SIGNING_KEY}")
+            endif()
         else()
             add_custom_target(gen_secure_boot_signing_key)
         endif()
@@ -119,10 +131,6 @@ externalproject_add(bootloader
                 -DEXTRA_COMPONENT_DIRS=${CMAKE_CURRENT_LIST_DIR}
                 -DPROJECT_SOURCE_DIR=${PROJECT_SOURCE_DIR}
                 ${sign_key_arg} ${ver_key_arg}
-                # LEGACY_INCLUDE_COMMON_HEADERS has to be passed in via cache variable since
-                # the bootloader common component requirements depends on this and
-                # config variables are not available before project() call.
-                -DLEGACY_INCLUDE_COMMON_HEADERS=${CONFIG_LEGACY_INCLUDE_COMMON_HEADERS}
                 ${extra_cmake_args}
     INSTALL_COMMAND ""
     BUILD_ALWAYS 1  # no easy way around this...

components/bootloader/subproject/CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.5)
+cmake_minimum_required(VERSION 3.16)
 
 if(NOT SDKCONFIG)
     message(FATAL_ERROR "Bootloader subproject expects the SDKCONFIG variable to be passed "
@@ -37,7 +37,9 @@ set(COMPONENTS
 # Make EXTRA_COMPONENT_DIRS variable to point to the bootloader_components directory
 # of the project being compiled
 set(PROJECT_EXTRA_COMPONENTS "${PROJECT_SOURCE_DIR}/bootloader_components")
-list(APPEND EXTRA_COMPONENT_DIRS "${PROJECT_EXTRA_COMPONENTS}")
+if(EXISTS ${PROJECT_EXTRA_COMPONENTS})
+    list(APPEND EXTRA_COMPONENT_DIRS "${PROJECT_EXTRA_COMPONENTS}")
+endif()
 
 # Consider each directory in project's bootloader_components as a component to be compiled
 file(GLOB proj_components RELATIVE ${PROJECT_EXTRA_COMPONENTS} ${PROJECT_EXTRA_COMPONENTS}/*)
@@ -50,10 +52,7 @@ endforeach()
 
 set(BOOTLOADER_BUILD 1)
 include("${IDF_PATH}/tools/cmake/project.cmake")
-set(common_req log esp_rom esp_common esp_hw_support hal newlib)
-if(LEGACY_INCLUDE_COMMON_HEADERS)
-    list(APPEND common_req soc hal)
-endif()
+set(common_req log esp_rom esp_common esp_hw_support newlib)
 idf_build_set_property(__COMPONENT_REQUIRES_COMMON "${common_req}")
 idf_build_set_property(__OUTPUT_SDKCONFIG 0)
 project(bootloader)

components/bootloader/subproject/Makefile

@@ -1,35 +0,0 @@
-#
-# This is a project Makefile. It is assumed the directory this Makefile resides in is a
-# project subdirectory.
-#
-ifeq ("$(MAKELEVEL)","0")
-$(error Bootloader makefile expects to be run as part of 'make bootloader' from a top-level project.)
-endif
-
-PROJECT_NAME := bootloader
-
-COMPONENTS := esp_hw_support esptool_py bootloader_support log spi_flash micro-ecc soc main efuse esp_rom hal xtensa
-
-# Clear C and CXX from top level project
-CFLAGS =
-CXXFLAGS =
-
-#We cannot include the some components like idf_target, esp_common directly but we need their includes.
-CFLAGS += -I $(IDF_PATH)/components/$(IDF_TARGET)/include
-CFLAGS += -I $(IDF_PATH)/components/esp_common/include
-CFLAGS += -I $(IDF_PATH)/components/newlib/platform_include
-CFLAGS += -I $(IDF_PATH)/components/xtensa/include -I $(IDF_PATH)/components/xtensa/$(IDF_TARGET)/include
-
-# The bootloader pseudo-component is also included in this build, for its Kconfig.projbuild to be included.
-#
-# IS_BOOTLOADER_BUILD tells the component Makefile.projbuild to be a no-op
-IS_BOOTLOADER_BUILD := 1
-export IS_BOOTLOADER_BUILD
-
-# BOOTLOADER_BUILD macro is the same, for source file changes
-CFLAGS += -D BOOTLOADER_BUILD=1
-
-# include the top-level "project" include directory, for sdkconfig.h
-CFLAGS += -I$(BUILD_DIR_BASE)/../include
-
-include $(IDF_PATH)/make/project.mk

components/bootloader/subproject/components/micro-ecc/component.mk

@@ -1,6 +0,0 @@
-# only compile the "uECC_verify_antifault.c" file which includes the "micro-ecc/uECC.c" source file
-COMPONENT_SRCDIRS := .
-
-COMPONENT_ADD_INCLUDEDIRS := . micro-ecc
-
-COMPONENT_SUBMODULES := micro-ecc