change(version): Update version to 5.0.6

fix(ble): Fixed Bluetooth not waking up due to clock drift(v5.0)

See merge request espressif/esp-idf!28723

.editorconfig

@@ -19,14 +19,6 @@ trim_trailing_whitespace = false
 indent_style = tab
 indent_size = 2
 
-[*/freertos/**]
-indent_style = tab
-indent_size = 4
-
-[{*/freertos/**.S,**/FreeRTOSConfig.h}]
-indent_style = space
-indent_size = 4
-
 [*.pem]
 insert_final_newline = false
 
@@ -38,6 +30,8 @@ indent_style = space
 indent_size = 4
 max_line_length = 120
 
-[{*.sh,*.yml}]
-indent_style = space
+[{*.sh,*.yml,*.yaml}]
+indent_size = 2
+
+[*.ini]
 indent_size = 2

.flake8

@@ -143,23 +143,22 @@ exclude =
         components/bootloader/subproject/components/micro-ecc/micro-ecc,
         components/bt/host/nimble/nimble,
         components/cmock/CMock,
-        components/esptool_py/esptool,
-        components/expat/expat,
         components/json/cJSON,
-        components/libsodium/libsodium,
         components/mbedtls/mbedtls,
-        components/nghttp/nghttp2,
-        components/tinyusb,
+        components/openthread/openthread,
         components/unity/unity,
-        examples/build_system/cmake/import_lib/main/lib/tinyxml2,
-        examples/peripherals/secure_element/atecc608_ecdsa/components/esp-cryptoauthlib,
+        components/spiffs/spiffs,
     # autogenerated scripts
         components/protocomm/python/constants_pb2.py,
         components/protocomm/python/sec0_pb2.py,
         components/protocomm/python/sec1_pb2.py,
+        components/protocomm/python/sec2_pb2.py,
         components/protocomm/python/session_pb2.py,
         components/wifi_provisioning/python/wifi_scan_pb2.py,
         components/wifi_provisioning/python/wifi_config_pb2.py,
         components/wifi_provisioning/python/wifi_constants_pb2.py,
         components/esp_local_ctrl/python/esp_local_ctrl_pb2.py,
-        examples/provisioning/legacy/custom_config/components/custom_provisioning/python/custom_config_pb2.py,
+
+per-file-ignores =
+    # Sphinx conf.py files use star imports to setup config variables
+        docs/conf_common.py: F405

.github/ISSUE_TEMPLATE/bug_report.md

@@ -24,7 +24,7 @@ If the issue cannot be solved after the steps before, please follow these instru
 
 1. Fill in all the fields under **Environment** marked with [ ] by picking the correct option for you in each case and deleting the others.
 2. Describe your problem.
-3. Include [debug logs from the "monitor" tool](https://docs.espressif.com/projects/esp-idf/en/latest/get-started/idf-monitor.html#automatically-decoding-addresses), or [coredumps](https://docs.espressif.com/projects/esp-idf/en/latest/api-guides/core_dump.html).
+3. Include [debug logs from the "monitor" tool](https://docs.espressif.com/projects/esp-idf/en/latest/api-guides/tools/idf-monitor.html), or [coredumps](https://docs.espressif.com/projects/esp-idf/en/latest/api-guides/core_dump.html).
 4. Providing as much information as possible under **Other items if possible** will help us locate and fix the problem.
 5. Use [Markdown](https://guides.github.com/features/mastering-markdown/) (see formatting buttons above) and the Preview tab to check what the issue will look like.
 6. Delete these instructions from the above to the below marker lines before submitting this issue.

.github/workflows/docker.yml

@@ -0,0 +1,87 @@
+name: docker
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+
+on:
+  push:
+    branches:
+      - 'master'
+      - 'release/*'
+    tags:
+      - 'v*.*'
+
+env:
+  # Platforms to build the image for
+  BUILD_PLATFORMS: linux/amd64,linux/arm64
+  DOCKERHUB_REPO: ${{ github.repository_owner }}/idf
+
+jobs:
+  docker:
+    # Disable the job in forks
+    if: ${{ github.repository_owner == 'espressif' }}
+
+    runs-on: ubuntu-latest
+    steps:
+      # Depending on the branch/tag, set CLONE_BRANCH_OR_TAG variable (used in the Dockerfile
+      # as a build arg) and TAG_NAME (used when tagging the image).
+      #
+      # The following 3 steps cover the alternatives (tag, release branch, master branch):
+      - name: Set variables (tags)
+        if: ${{ github.ref_type == 'tag' }}
+        run: |
+          echo "CLONE_BRANCH_OR_TAG=$GITHUB_REF_NAME" >> $GITHUB_ENV
+          echo "TAG_NAME=$GITHUB_REF_NAME" >> $GITHUB_ENV
+      - name: Set variables (release branches)
+        if: ${{ github.ref_type == 'branch' && startsWith(github.ref_name, 'release/') }}
+        run: |
+          echo "CLONE_BRANCH_OR_TAG=$GITHUB_REF_NAME" >> $GITHUB_ENV
+          echo "TAG_NAME=release-${GITHUB_REF_NAME##release/}" >> $GITHUB_ENV
+      - name: Set variables (main branch)
+        if: ${{ github.ref_type == 'branch' && github.ref_name == 'master' }}
+        run: |
+          echo "CLONE_BRANCH_OR_TAG=master" >> $GITHUB_ENV
+          echo "TAG_NAME=latest" >> $GITHUB_ENV
+
+      # Display the variables set above, just in case.
+      - name: Check variables
+        run: |
+          echo "CLONE_BRANCH_OR_TAG: $CLONE_BRANCH_OR_TAG"
+          echo "CHECKOUT_REF: $CHECKOUT_REF"
+          echo "TAG_NAME: $TAG_NAME"
+
+      # The following steps are the standard boilerplate from
+      # https://github.com/marketplace/actions/build-and-push-docker-images
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Set up QEMU for multiarch builds
+        uses: docker/setup-qemu-action@v2
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      - name: Build and push
+        uses: docker/build-push-action@v3
+        with:
+          context: tools/docker
+          push: true
+          tags: ${{ env.DOCKERHUB_REPO }}:${{ env.TAG_NAME }}
+          platforms: ${{ env.BUILD_PLATFORMS }}
+          build-args: |
+            IDF_CLONE_URL=${{ github.server_url }}/${{ github.repository }}.git
+            IDF_CLONE_BRANCH_OR_TAG=${{ env.CLONE_BRANCH_OR_TAG }}
+
+      - name: Update Docker Hub repository description (master branch)
+        if: ${{ github.ref_type == 'branch' && github.ref_name == 'master' }}
+        uses: peter-evans/dockerhub-description@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          # Token based authentication is not supported here:
+          # https://github.com/peter-evans/dockerhub-description/issues/10
+          # https://github.com/docker/roadmap/issues/115#issuecomment-891694974
+          password: ${{ secrets.DOCKERHUB_PASSWORD }}
+          repository: ${{ env.DOCKERHUB_REPO }}
+          readme-filepath: ./tools/docker/README.md

.github/workflows/issue_comment.yml

@@ -3,12 +3,16 @@ name: Sync issue comments to JIRA
 # This workflow will be triggered when new issue comment is created (including PR comments)
 on: issue_comment
 
+# Limit to single concurrent run for workflows which can create Jira issues.
+# Same concurrency group is used in new_issues.yml
+concurrency: jira_issues
+
 jobs:
   sync_issue_comments_to_jira:
     name: Sync Issue Comments to Jira
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@master
+      - uses: actions/checkout@v2
       - name: Sync issue comments to JIRA
         uses: espressif/github-actions/sync_issues_to_jira@master
         env:

.github/workflows/new_issues.yml

@@ -3,12 +3,16 @@ name: Sync issues to Jira
 # This workflow will be triggered when a new issue is opened
 on: issues
 
+# Limit to single concurrent run for workflows which can create Jira issues.
+# Same concurrency group is used in issue_comment.yml
+concurrency: jira_issues
+
 jobs:
   sync_issues_to_jira:
     name: Sync issues to Jira
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@master
+      - uses: actions/checkout@v2
       - name: Sync GitHub issues to Jira project
         uses: espressif/github-actions/sync_issues_to_jira@master
         env:

.github/workflows/new_prs.yml

@@ -6,12 +6,16 @@ on:
   schedule:
     - cron: "0 * * * *"
 
+# Limit to single concurrent run for workflows which can create Jira issues.
+# Same concurrency group is used in issue_comment.yml
+concurrency: jira_issues
+
 jobs:
   sync_prs_to_jira:
     name: Sync PRs to Jira
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@master
+      - uses: actions/checkout@v2
       - name: Sync PRs to Jira project
         uses: espressif/github-actions/sync_issues_to_jira@master
         with:

.github/workflows/pr_approved.yml

@@ -0,0 +1,23 @@
+name: Sync approved PRs to internal codebase
+on:
+  pull_request_target:
+    types: [labeled]
+
+jobs:
+  sync_prs_to_internal_codebase:
+    name: GitHub PR to Internal Codebase Sync
+    runs-on: ubuntu-latest
+    if: (github.event.label.name == 'PR-Sync-Merge')   ||
+        (github.event.label.name == 'PR-Sync-Rebase')  ||
+        (github.event.label.name == 'PR-Sync-Update')
+    steps:
+      - uses: actions/checkout@v2
+      - name: Sync approved PRs to internal codebase
+        uses: espressif/github-actions/github_pr_to_internal_pr@master
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GITLAB_URL: ${{ secrets.GITLAB_URL }}
+          GITLAB_TOKEN: ${{ secrets.GITLAB_TOKEN }}
+          GIT_CONFIG_NAME: ${{ secrets.GIT_CONFIG_NAME }}
+          GIT_CONFIG_EMAIL: ${{ secrets.GIT_CONFIG_EMAIL }}
+          JIRA_PROJECT: IDFGH

.github/workflows/pre_commit_check.yml

@@ -0,0 +1,37 @@
+name: Check pre-commit rules
+
+on:
+  pull_request:
+    types: [opened, reopened, synchronize]
+
+jobs:
+  pre_commit_check:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Fetch head and base refs
+        # This is necessary for pre-commit to check the changes in the PR branch
+        run: |
+          git fetch origin ${{ github.base_ref }}:base_ref
+          git fetch origin pull/${{ github.event.pull_request.number }}/head:pr_ref
+      - name: Set up Python environment
+        uses: actions/setup-python@master
+        with:
+          python-version: v3.7
+      - name: Install python packages
+        run: |
+          pip install pre-commit
+          pre-commit install-hooks
+      - name: Run pre-commit and check for any changes
+        run: |
+          echo "Commits being checked:"
+          git log --oneline --no-decorate base_ref..pr_ref
+          echo ""
+          if ! pre-commit run --from-ref base_ref --to-ref pr_ref --show-diff-on-failure ; then
+            echo ""
+            echo "::notice::It looks like the commits in this PR have been made without having pre-commit hooks installed."
+            echo "::notice::Please see https://docs.espressif.com/projects/esp-idf/en/latest/esp32/contribute/install-pre-commit-hook.html for instructions."
+            echo ""
+            exit 1
+          fi

.github/workflows/python_lint.yml

@@ -1,33 +0,0 @@
-name: Python CI
-
-# This workflow will be triggered when a PR modifies some python relevant files
-on:
-  pull_request:
-    paths:
-      - "**.py"
-      - "requirements.txt"
-
-jobs:
-  python_lint:
-    name: python lint
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: [2.7, 3.5, 3.6, 3.7, 3.8]
-
-    steps:
-      - name: Checkout
-        uses: actions/checkout@master
-      - name: Set up Python environment
-        uses: actions/setup-python@master
-        with:
-          python-version: ${{ matrix.python-version }}
-      - name: Install dependencies
-        run: |
-          export IDF_PATH=${GITHUB_WORKSPACE}
-          pip install --upgrade pip
-          pip install -r requirements.txt
-      - name: Lint with flake8
-        run: |
-          pip install flake8
-          flake8 . --config=.flake8 --benchmark

.gitignore

@@ -21,15 +21,20 @@ GPATH
 # MacOS directory files
 .DS_Store
 
+# cache dir
+.cache/
+
 # Components Unit Test Apps files
-components/**/build
+components/**/build/
+components/**/build_*_*/
 components/**/sdkconfig
 components/**/sdkconfig.old
 
 # Example project files
+examples/**/build/
+examples/**/build_esp*_*/
 examples/**/sdkconfig
 examples/**/sdkconfig.old
-examples/**/build
 
 # Doc build artifacts
 docs/_build/
@@ -43,7 +48,7 @@ docs/_static/NotoSansSC-Regular.otf
 tools/unit-test-app/sdkconfig
 tools/unit-test-app/sdkconfig.old
 tools/unit-test-app/build
-tools/unit-test-app/builds
+tools/unit-test-app/build_*_*/
 tools/unit-test-app/output
 tools/unit-test-app/test_configs
 
@@ -51,7 +56,8 @@ tools/unit-test-app/test_configs
 log_ut_cmake
 
 # test application build files
-tools/test_apps/**/build
+tools/test_apps/**/build/
+tools/test_apps/**/build_*_*/
 tools/test_apps/**/sdkconfig
 tools/test_apps/**/sdkconfig.old
 
@@ -91,3 +97,6 @@ dependencies.lock
 
 # managed_components for examples
 managed_components
+
+# pytest log
+pytest_embedded_log/

.gitlab-ci.yml

@@ -1,24 +1,28 @@
 stages:
+  - upload_cache
   - pre_check
   - build
   - assign_test
-  - host_test
+  - build_doc
   - target_test
+  - host_test
   - test_deploy
-  - post_check
   - deploy
   - post_deploy
 
-# pipelines will not be created in such two cases:
-# 1. MR push
-# 2. push not on "master/release" branches, and not tagged
-# This behavior could be changed after the `rules: changes` feature is implemented
 workflow:
   rules:
-    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
-      when: never
-    - if: '$CI_COMMIT_REF_NAME != "master" && $CI_COMMIT_BRANCH !~ /^release\/v/ && $CI_COMMIT_TAG !~ /^v\d+\.\d+(\.\d+)?($|-)/ && $CI_PIPELINE_SOURCE == "push"'
+    # Disable those non-protected push triggered pipelines
+    - if: '$CI_COMMIT_REF_NAME != "master" && $CI_COMMIT_BRANCH !~ /^release\/v/ && $CI_COMMIT_TAG !~ /^v\d+\.\d+(\.\d+)?($|-)/ && $CI_COMMIT_TAG !~ /^qa-test/ && $CI_PIPELINE_SOURCE == "push"'
       when: never
+    # when running merged result pipelines, it would create a temp commit id. use $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA instead of $CI_COMMIT_SHA.
+    # Please use PIPELINE_COMMIT_SHA at all places that require a commit sha
+    - if: $CI_OPEN_MERGE_REQUESTS != null
+      variables:
+        PIPELINE_COMMIT_SHA: $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA
+    - if: $CI_OPEN_MERGE_REQUESTS == null
+      variables:
+        PIPELINE_COMMIT_SHA: $CI_COMMIT_SHA
     - when: always
 
 variables:
@@ -50,90 +54,178 @@ variables:
   BATCH_BUILD: "1"
   V: "0"
   CHECKOUT_REF_SCRIPT: "$CI_PROJECT_DIR/tools/ci/checkout_project_ref.py"
+  PYTHON_VER: 3.7.10
+
+  CLANG_TIDY_RUNNER_PROJ: 2107  # idf/clang-tidy-runner
+  IDF_BUILD_APPS_PROJ: 2818  # espressif/idf-build-apps
 
   # Docker images
   BOT_DOCKER_IMAGE_TAG: ":latest"
 
-  # target test config file, used by assign test job
-  CI_TARGET_TEST_CONFIG_FILE: "$CI_PROJECT_DIR/tools/ci/config/target-test.yml"
+  ESP_ENV_IMAGE: "$CI_DOCKER_REGISTRY/esp-env-v5.0:4"
+  AFL_FUZZER_TEST_IMAGE: "$CI_DOCKER_REGISTRY/afl-fuzzer-test-v5.0:4-2"
+  CLANG_STATIC_ANALYSIS_IMAGE: "${CI_DOCKER_REGISTRY}/clang-static-analysis-v5.0:4-2"
+  ESP_IDF_DOC_ENV_IMAGE: "$CI_DOCKER_REGISTRY/esp-idf-doc-env-v5.0:4-4"
+  QEMU_IMAGE: "${CI_DOCKER_REGISTRY}/qemu-v5.0:4-20220802"
+  TARGET_TEST_ENV_IMAGE: "$CI_DOCKER_REGISTRY/target-test-env-v5.0:3"
+
+  SONARQUBE_SCANNER_IMAGE: "${CI_DOCKER_REGISTRY}/sonarqube-scanner:3"
+
+  PRE_COMMIT_IMAGE: "$CI_DOCKER_REGISTRY/esp-idf-pre-commit:1"
 
   # target test repo parameters
   TEST_ENV_CONFIG_REPO: "https://gitlab-ci-token:${BOT_TOKEN}@${CI_SERVER_HOST}:${CI_SERVER_PORT}/qa/ci-test-runner-configs.git"
-  CI_AUTO_TEST_SCRIPT_REPO_URL: "https://gitlab-ci-token:${BOT_TOKEN}@${CI_SERVER_HOST}:${CI_SERVER_PORT}/qa/auto_test_script.git"
-  CI_AUTO_TEST_SCRIPT_REPO_BRANCH: "ci/v3.1"
 
-  # Versioned esp-idf-doc env image to use for all document building jobs
-  ESP_IDF_DOC_ENV_IMAGE: "$CI_DOCKER_REGISTRY/esp-idf-doc-env:v10"
+  # cache python dependencies
+  PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
+
+  # Set this variable to the branch of idf-constraints repo in order to test a custom Python constraint file. The
+  # branch name must be without the remote part ("origin/"). Keep the variable empty in order to use the constraint
+  # file from https://dl.espressif.com/dl/esp-idf.
+  CI_PYTHON_CONSTRAINT_BRANCH: ""
+
+  # Update the filename for a specific ESP-IDF release. It is used only with CI_PYTHON_CONSTRAINT_BRANCH.
+  CI_PYTHON_CONSTRAINT_FILE: "espidf.constraints.v5.0.txt"
+
+  # Set this variable to repository name of a Python tool you wish to install and test in the context of ESP-IDF CI.
+  # Keep the variable empty when not used.
+  CI_PYTHON_TOOL_REPO: ""
+
+  # Set this variable to the branch of a Python tool repo specified in CI_PYTHON_TOOL_REPO. The
+  # branch name must be without the remote part ("origin/"). Keep the variable empty when not used.
+  # This is used only if CI_PYTHON_TOOL_REPO is not empty.
+  CI_PYTHON_TOOL_BRANCH: ""
+
+  IDF_CI_BUILD: 1
+
+cache:
+  # pull only for most of the use cases since it's cache dir.
+  # Only set "push" policy for "upload_cache" stage jobs
+  - key: pip-cache
+    paths:
+      - .cache/pip
+    policy: pull
+  - key: submodule-cache
+    paths:
+      - .cache/submodule_archives
+    policy: pull
+
+.common_before_scripts: &common-before_scripts |
+  source tools/ci/utils.sh
+  is_based_on_commits $REQUIRED_ANCESTOR_COMMITS
 
-.setup_tools_unless_target_test: &setup_tools_unless_target_test |
   if [[ -n "$IDF_DONT_USE_MIRRORS" ]]; then
-  export IDF_MIRROR_PREFIX_MAP=
+    export IDF_MIRROR_PREFIX_MAP=
   fi
-  if [[ "$SETUP_TOOLS" == "1" || "$CI_JOB_STAGE" != "target_test" ]]; then
-  tools/idf_tools.py --non-interactive install && eval "$(tools/idf_tools.py --non-interactive export)" || exit 1
+
+  if echo "$CI_MERGE_REQUEST_LABELS" | egrep "^([^,\n\r]+,)*include_nightly_run(,[^,\n\r]+)*$"; then
+    export INCLUDE_NIGHTLY_RUN="1"
+  fi
+
+  # configure cmake related flags
+  source tools/ci/configure_ci_environment.sh
+
+  # add extra python packages
+  export PYTHONPATH="$IDF_PATH/tools:$IDF_PATH/tools/esp_app_trace:$IDF_PATH/components/partition_table:$IDF_PATH/tools/ci/python_packages:$PYTHONPATH"
+
+.setup_tools_and_idf_python_venv: &setup_tools_and_idf_python_venv |
+  # Since the version 3.21 CMake passes source files and include dirs to ninja using absolute paths.
+  $IDF_PATH/tools/idf_tools.py --non-interactive install cmake
+
+  # download constraint file for dev
+  if [[ -n "$CI_PYTHON_CONSTRAINT_BRANCH" ]]; then
+    wget -O /tmp/constraint.txt --header="Authorization:Bearer ${ESPCI_TOKEN}" ${GITLAB_HTTP_SERVER}/api/v4/projects/2581/repository/files/${CI_PYTHON_CONSTRAINT_FILE}/raw?ref=${CI_PYTHON_CONSTRAINT_BRANCH}
+    mkdir -p ~/.espressif
+    mv /tmp/constraint.txt ~/.espressif/${CI_PYTHON_CONSTRAINT_FILE}
+  fi
+
+  # Mirror
+  if [[ -n "$IDF_DONT_USE_MIRRORS" ]]; then
+    export IDF_MIRROR_PREFIX_MAP=
+  fi
+
+  # install latest python packages
+  # target test jobs
+  if [[ "${CI_JOB_STAGE}" == "target_test" ]]; then
+    # ttfw jobs
+    if ! echo "${CI_JOB_NAME}" | egrep ".+_pytest_.+"; then
+      run_cmd bash install.sh --enable-ci --enable-ttfw
+    else
+      run_cmd bash install.sh --enable-ci --enable-pytest
+    fi
+  elif [[ "${CI_JOB_STAGE}" == "build_doc" ]]; then
+    run_cmd bash install.sh --enable-ci --enable-docs
+  elif [[ "${CI_JOB_STAGE}" == "build" ]]; then
+    run_cmd bash install.sh --enable-ci --enable-pytest
+  else
+    run_cmd bash install.sh --enable-ci
+  fi
+
+  source ./export.sh
+
+  # Custom OpenOCD
+  if [[ ! -z "$OOCD_DISTRO_URL" && "$CI_JOB_STAGE" == "target_test" ]]; then
+    echo "Using custom OpenOCD from ${OOCD_DISTRO_URL}"
+    wget $OOCD_DISTRO_URL
+    ARCH_NAME=$(basename $OOCD_DISTRO_URL)
+    tar -x -f $ARCH_NAME
+    export OPENOCD_SCRIPTS=$PWD/openocd-esp32/share/openocd/scripts
+    export PATH=$PWD/openocd-esp32/bin:$PATH
+  fi
+
+  if [[ -n "$CI_PYTHON_TOOL_REPO" ]]; then
+    git clone --quiet --depth=1 -b ${CI_PYTHON_TOOL_BRANCH} https://gitlab-ci-token:${ESPCI_TOKEN}@${GITLAB_HTTPS_HOST}/espressif/${CI_PYTHON_TOOL_REPO}.git
+    pip install ./${CI_PYTHON_TOOL_REPO}
+    rm -rf ${CI_PYTHON_TOOL_REPO}
   fi
 
 before_script:
-  - source tools/ci/utils.sh
-  - source tools/ci/setup_python.sh
-  - apply_bot_filter
+  - *common-before_scripts
+  - *setup_tools_and_idf_python_venv
   - add_gitlab_ssh_keys
-  - source tools/ci/configure_ci_environment.sh
-  - *setup_tools_unless_target_test
   - fetch_submodules
 
-# used for check scripts which we want to run unconditionally
-.before_script_lesser_nofilter:
+.before_script_minimal:
   before_script:
-    - echo "Not setting up GitLab key, not fetching submodules, not applying bot filter"
-    - source tools/ci/utils.sh
-    - source tools/ci/setup_python.sh
-    - source tools/ci/configure_ci_environment.sh
-
-# used for everything else where we want to do no prep, except for bot filter
-.before_script_lesser:
-  before_script:
-    - echo "Not setting up GitLab key, not fetching submodules"
-    - source tools/ci/utils.sh
-    - source tools/ci/setup_python.sh
-    - apply_bot_filter
-    - source tools/ci/configure_ci_environment.sh
-
-.before_script_slim:
-  before_script:
-    - echo "Only load utils.sh inside"
-    - source tools/ci/utils.sh
+    - *common-before_scripts
 
 .before_script_macos:
   before_script:
-    - source tools/ci/utils.sh
-    - apply_bot_filter
-    - export IDF_TOOLS_PATH="${HOME}/.espressif_runner_${CI_RUNNER_ID}_${CI_CONCURRENT_ID}"
-    - $IDF_PATH/tools/idf_tools.py install-python-env
+    - *common-before_scripts
     # On macOS, these tools need to be installed
+    - export IDF_TOOLS_PATH="${HOME}/.espressif_runner_${CI_RUNNER_ID}_${CI_CONCURRENT_ID}"
     - $IDF_PATH/tools/idf_tools.py --non-interactive install cmake ninja
     # This adds tools (compilers) and the version-specific Python environment to PATH
-    - *setup_tools_unless_target_test
-    # Install packages required by CI scripts into IDF Python environment
-    - pip install -r $IDF_PATH/tools/ci/python_packages/ttfw_idf/requirements.txt
-    - source tools/ci/configure_ci_environment.sh
-    # Part of tools/ci/setup_python.sh; we don't use pyenv on macOS, so can't run the rest of the script.
-    - export PYTHONPATH="$IDF_PATH/tools:$IDF_PATH/tools/ci/python_packages:$PYTHONPATH"
+    - *setup_tools_and_idf_python_venv
     - fetch_submodules
 
+.before_script_build_jobs:
+  before_script:
+    - *common-before_scripts
+    - *setup_tools_and_idf_python_venv
+    - add_gitlab_ssh_keys
+    - fetch_submodules
+    - export EXTRA_CFLAGS=${PEDANTIC_CFLAGS}
+    - export EXTRA_CXXFLAGS=${PEDANTIC_CXXFLAGS}
+
 default:
   retry:
     max: 2
-    # In case of a runner failure we could hop to another one, or a network error could go away.
-    when: runner_system_failure
+    when:
+      # In case of a runner failure we could hop to another one, or a network error could go away.
+      - runner_system_failure
+      # Job execution timeout may be caused by a network issue.
+      - job_execution_timeout
 
 include:
-  - '/tools/ci/config/rules.yml'
-  - '/tools/ci/config/pre_check.yml'
-  - '/tools/ci/config/build.yml'
-  - '/tools/ci/config/assign-test.yml'
-  - '/tools/ci/config/host-test.yml'
-  - '/tools/ci/config/target-test.yml'
-  - '/tools/ci/config/post_check.yml'
-  - '/tools/ci/config/deploy.yml'
-  - '/tools/ci/config/post_deploy.yml'
+  - '.gitlab/ci/rules.yml'
+  - '.gitlab/ci/upload_cache.yml'
+  - '.gitlab/ci/docs.yml'
+  - '.gitlab/ci/static-code-analysis.yml'
+  - '.gitlab/ci/pre_check.yml'
+  - '.gitlab/ci/build.yml'
+  - '.gitlab/ci/assign-test.yml'
+  - '.gitlab/ci/integration_test.yml'
+  - '.gitlab/ci/host-test.yml'
+  - '.gitlab/ci/target-test.yml'
+  - '.gitlab/ci/deploy.yml'

.gitlab/ci/README.md

@@ -0,0 +1,319 @@
+# IDF CI
+
+- [IDF CI](#idf-ci)
+  - [General Workflow](#general-workflow)
+  - [What if Expected Jobs ARE NOT Created?](#what-if-expected-jobs-are-not-created)
+  - [MR labels for additional jobs](#mr-labels-for-additional-jobs)
+    - [Supported MR Labels](#supported-mr-labels)
+    - [How to trigger a `detached` pipeline without pushing new commits?](#how-to-trigger-a-detached-pipeline-without-pushing-new-commits)
+  - [How to Develop With `rules.yml`?](#how-to-develop-with-rulesyml)
+    - [General Concepts](#general-concepts)
+    - [How to Add a New `Job`?](#how-to-add-a-new-job)
+    - [How to Add a New `Rules` Template?](#how-to-add-a-new-rules-template)
+    - [How to Add a New `if` Anchor?](#how-to-add-a-new-if-anchor)
+    - [Naming Rules](#naming-rules)
+      - [Common Naming Rules](#common-naming-rules)
+      - [`if` Anchors Naming Rules](#if-anchors-naming-rules)
+      - [`rules` Template Naming Rules](#rules-template-naming-rules)
+  - [Reusable Shell Script `tools/ci/utils.sh`](#reusable-shell-script-toolsciutilssh)
+    - [Functions](#functions)
+      - [CI Job Related](#ci-job-related)
+      - [Shell Script Related](#shell-script-related)
+  - [Manifest File to Control the Build/Test apps](#manifest-file-to-control-the-buildtest-apps)
+    - [Grammar](#grammar)
+
+## General Workflow
+
+1. Push to a remote branch
+2. Create an MR, choose related labels (not required)
+3. A `detached` pipeline will be created.
+4. if you push a new commit, a new pipeline will be created automatically.
+
+## What if Expected Jobs ARE NOT Created?
+
+1. check the file patterns
+
+   If you found a job that is not running as expected with some file changes, a git commit to improve the `pattern` will be appreciated.
+
+2. please add MR labels to run additional tests, currently we have to do this only for `target-test` jobs, please use it as few as possible. Our final goal is to remove all the labels and let the file changes decide everything!
+
+## MR labels for additional jobs
+
+### Supported MR Labels
+
+- `build`
+- `build_docs`
+- `component_ut[_esp32/esp32s2/...]`
+- `custom_test[_esp32/esp32s2/...]`
+- `docker`
+- `docs`
+- `docs_fast`, triggers a fast docs build, not a full build which is the CI default. This skips PDF build as well as doxygen APIs, reducing the build time by 90+%.
+- `example_test[_esp32/esp32s2/...]`
+- `fuzzer_test`
+- `host_test`
+- `integration_test`
+- `iperf_stress_test`
+- `macos`
+- `macos_test`
+- `nvs_coverage`
+- `submodule`
+- `unit_test[_esp32/esp32s2/...]`
+- `weekend_test`
+- `windows`
+
+There are two general labels (not recommended since these two labels will trigger a lot of jobs)
+
+- `target_test`: includes all target for `example_test`, `custom_test`, `component_ut`, `unit_test`, `integration_test`
+- `all_test`: includes all test labels
+
+### How to trigger a `detached` pipeline without pushing new commits?
+
+Go to MR web page -> `Pipelines` tab -> click `Run pipeline` button.
+
+In very rare case, this tab will not show up because no merge_request pipeline is created before. Please use web API then.
+
+```shell
+curl -X POST --header "PRIVATE-TOKEN: [YOUR PERSONAL ACCESS TOKEN]" [GITLAB_SERVER]/api/v4/projects/103/merge_requests/[MERGE_REQUEST_IID]/pipelines
+```
+
+## How to Develop With `rules.yml`?
+
+### General Concepts
+
+- `pattern`: Defined in an array. A GitLab job will be created if the changed files in this MR matched one of the patterns. For example:
+
+    ```yaml
+    .patterns-python-files: &patterns-python-files
+      - "**/*.py"
+    ```
+
+- `label`: Defined in an if clause, similar as the previous bot command. A GitLab job will be created if the pipeline variables contains variables in `BOT_LABEL_xxx` format (DEPRECATED) or included in the MR labels. For example:
+
+    ```yaml
+    .if-label-build_docs: &if-label-build_docs
+      if: '$BOT_LABEL_BUILD_DOCS || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*build_docs(?:,[^,\n\r]+)*$/i'
+    ```
+
+- `rule`: A combination of various patterns, and labels. It will be used by GitLab YAML `extends` keyword to tell GitLab in what conditions will this job be created. For example:
+
+    ```yaml
+    .rules:build:docs:
+      rules:
+        - <<: *if-protected
+        - <<: *if-label-build_docs
+        - <<: *if-label-docs
+        - <<: *if-dev-push
+          changes: *patterns-docs
+    ```
+
+  An example for GitLab job on how to use extends:
+
+    ```yaml
+    check_docs_lang_sync:
+      extends:
+        - .pre_check_template
+        - .rules:build:docs
+      script:
+        - cd docs
+        - ./check_lang_folder_sync.sh
+    ```
+
+### How to Add a New `Job`?
+
+check if there's a suitable `.rules:<rules-you-need>` template
+
+1. if there is, put this in the job `extends`. All done, now you can close this window. (`extends` could be array or string)
+2. if there isn't
+    1. check [How to Add a New `Rules` Template?](#how-to-add-a-new-rules-template), create a suitable one
+    2. follow step 1
+
+### How to Add a New `Rules` Template?
+
+check if this rule is related to `labels`, `patterns`
+
+1. if it is, please refer to [dependencies/README.md](./dependencies/README.md) and add new rules by auto-generating
+2. if it isn't, please continue reading
+
+check if there's a suitable `.if-<if-anchor-you-need>` anchor
+
+1. if there is, create a rule following [`rules` Template Naming Rules](#rules-template-naming-rules).For detail information, please refer to [GitLab Documentation `rules-if`](https://docs.gitlab.com/ee/ci/yaml/README.html#rulesif). Here's an example.
+
+    ```yaml
+    .rules:patterns:python-files:
+      rules:
+        - <<: *if-protected
+        - <<: *if-dev-push
+          changes: *patterns-python-files
+    ```
+
+2. if there isn't
+
+    1. check [How to Add a New `if` Anchor?](#how-to-add-a-new-if-anchor), create a suitable one
+    2. follow step 1
+
+### How to Add a New `if` Anchor?
+
+Create an `if` anchor following [`if` Anchors Naming Rules](#if-anchors-naming-rules). For detailed information about how to write the condition clause, please refer to [GitLab Documentation `only/except (advanced)](https://docs.gitlab.com/ee/ci/yaml/README.html#onlyexcept-advanced). Here's an example.
+
+```yaml
+.if-schedule: &if-schedule:
+  if: '$CI_PIPELINE_SOURCE == "schedule"'
+```
+
+### Naming Rules
+
+#### Common Naming Rules
+
+if a phrase has multi words, use `_` to concatenate them.
+
+> e.g. `regular_test`
+
+if a name has multi phrases, use `-` to concatenate them.
+
+> e.g. `regular_test-example_test`
+
+#### `if` Anchors Naming Rules
+
+- if it's a label: `.if-label-<label_name>`
+- if it's a ref: `.if-ref-<ref_name>`
+- if it's a branch: `.if-branch-<branch_name>`
+- if it's a tag: `.if-tag-<tag_name>`
+- if it's multi-type combination: `.if-ref-<release_name>-branch-<branch_name>`
+
+  **Common Phrases/Abbreviations**
+
+    - `no_label`
+
+      `$BOT_TRIGGER_WITH_LABEL == null`
+
+    - `protected`
+
+      `($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_BRANCH =~ /^release\/v/ || $CI_COMMIT_TAG =~ /^v\d+\.\d+(\.\d+)?($|-)/)`
+
+    - `target_test`
+
+      a combination of `example_test`, `custom_test`, `unit_test`, `component_ut`, `integration_test` and all targets
+
+#### `rules` Template Naming Rules
+
+- if it's tag related: `.rules:tag:<tag_1>-<tag_2>`
+- if it's label related: `.rules:labels:<label_1>-<label_2>`
+- if it's test related: `.rules:test:<test_type>`
+- if it's build related: `.rules:build:<build_type>`
+- if it's pattern related: `.rules:patterns:<patterns>`
+
+## Reusable Shell Script `tools/ci/utils.sh`
+
+It is used to put all the reusable shell scripts as small functions. If you want to set `before_script: []` for you job, now you can set `extends: .before_script_slim` instead. it will only run `source tools/ci/utils.sh`
+
+If you're developing CI shell scripts, you can use these functions without `source` them. They're already included in all `before_script`
+
+To run these commands in shell script locally, place `source tools/ci/utils.sh` at the very beginning.
+
+### Functions
+
+#### CI Job Related
+
+- `add_gitlab_ssh_keys`
+- `add_github_ssh_keys`
+- `add_doc_server_ssh_keys`
+- `fetch_submodules`
+- `get_all_submodules`
+
+#### Shell Script Related
+
+- `error`: log in red color
+- `warning`: log in orange color
+- `info`: log in green color
+- `run_cmd`: run the command with duration seconds info
+- `retry_failed`: run the command with duration seconds info, retry when failed
+
+## Manifest File to Control the Build/Test apps
+
+`.build-test-rules.yml` file is a manifest file to control if the CI is running the build and test job or not. The Supported Targets table in `README.md` for apps would be auto-generated by `pre-commit` from the app's `.build-test-rules.yml`.
+
+### Grammar
+
+#### Operands
+
+- Variables start with `SOC_`. The value would be parsed from components/soc/[TARGET]/include/soc/*_caps.h
+- `IDF_TARGET`
+- `INCLUDE_DEFAULT` (The default value of officially supported targets is 1, otherwise is 0)
+- String, must be double-quoted. e.g. `"esp32"`, `"12345"`
+- Integer, support decimal and hex. e.g. `1`, `0xAB`
+- List with String and Integer inside, the type could be mixed. e.g. `["esp32", 1]`
+
+#### Operators
+
+- `==`, `!=`, `>`, `>=`, `<`, `<=`
+- `and`, `or`
+- `in`, `not in` with list
+- parentheses
+
+#### Limitation:
+
+- all operators are binary operator. For more than two operands, you may use nested parentheses trick. For example,
+  - `A == 1 or (B == 2 and C in [1,2,3])`
+  - `(A == 1 and B == 2) or (C not in ["3", "4", 5])`
+
+### How does it work?
+
+By default, we enable build and test jobs for supported targets. In other words, if an app supports all supported targets, it does not need to be added in a manifest file. The manifest files are files that set the violation rules for apps.
+
+three rules (disable rules are calculated after the `enable` rule):
+- enable: run CI build/test jobs for targets that match any of the specified conditions only
+- disable: will not run CI build/test jobs for targets that match any of the specified conditions
+- disable_test: will not run CI test jobs for targets that match any of the specified conditions
+
+Each key is a test folder. Will apply to all folders inside.
+
+If one sub folder is in a special case, you can overwrite the rules for this folder by adding another entry for this folder itself. Each folder's rules are standalone, and will not inherit its parent's rules. (YAML inheritance is too complicated for reading)
+
+For example in the following codeblock, only `disable` rule exists in `examples/foo/bar`. It's unaware of its parent's `enable` rule. 
+
+```yaml
+examples/foo:
+  enable:
+    - if: IDF_TARGET == "esp32"
+
+examples/foo/bar:
+  disable:
+    - if: IDF_TARGET == "esp32s2"
+```
+
+### Example
+
+```yaml
+examples/foo:
+  enable:
+    - if IDF_TARGET in ["esp32", 1, 2, 3]
+    - if IDF_TARGET not in ["4", "5", 6]
+  # should be run under all targets!
+
+examples/bluetooth:
+  disable:  # disable both build and tests jobs
+    - if: SOC_BT_SUPPORTED != 1
+    # reason is optional if there's no `temporary: true`
+  disable_test:
+    - if: IDF_TARGET == "esp32"
+      temporary: true
+      reason: lack of ci runners  # required when `temporary: true`
+
+examples/bluetooth/test_foo:
+  # each folder's settings are standalone
+  disable:
+    - if: IDF_TARGET == "esp32s2"
+      temporary: true
+      reason: no idea
+  # unlike examples/bluetooth, the apps under this folder would not be build nor test for "no idea" under target esp32s2
+
+examples/get-started/hello_world:
+  enable:
+    - if: IDF_TARGET == "linux"
+      reason: this one only supports linux!
+
+examples/get-started/blink:
+  enable:
+    - if: INCLUDE_DEFAULT == 1 or IDF_TARGET == "linux"
+      reason: This one supports all supported targets and linux
+```

.gitlab/ci/assign-test.yml

@@ -0,0 +1,83 @@
+.assign_test_template:
+  image: $TARGET_TEST_ENV_IMAGE
+  stage: assign_test
+  tags:
+    - assign_test
+  variables:
+    SUBMODULES_TO_FETCH: "none"
+    CI_TARGET_TEST_CONFIG_FILE: "$CI_PROJECT_DIR/.gitlab/ci/target-test.yml"
+  artifacts:
+    paths:
+      - ${TEST_DIR}/test_configs/
+      - artifact_index.json
+    when: always
+    expire_in: 1 week
+  script:
+    - run_cmd python tools/ci/python_packages/ttfw_idf/IDFAssignTest.py $TEST_TYPE $TEST_DIR -c $CI_TARGET_TEST_CONFIG_FILE -o $TEST_DIR/test_configs
+
+assign_example_test:
+  extends:
+    - .assign_test_template
+    - .rules:build:example_test
+  needs:
+    - job: build_examples_cmake_esp32
+      artifacts: false
+      optional: true
+    - job: build_examples_cmake_esp32s2
+      artifacts: false
+      optional: true
+    - job: build_examples_cmake_esp32c2
+      artifacts: false
+      optional: true
+    - job: build_examples_cmake_esp32c3
+      artifacts: false
+      optional: true
+    - job: build_examples_cmake_esp32s3
+      artifacts: false
+      optional: true
+  variables:
+    TEST_TYPE: example_test
+    TEST_DIR: examples
+
+assign_custom_test:
+  extends:
+    - .assign_test_template
+    - .rules:build:custom_test
+  needs:
+    - job: build_test_apps_esp32
+      artifacts: false
+      optional: true
+    - job: build_test_apps_esp32s2
+      artifacts: false
+      optional: true
+    - job: build_test_apps_esp32c2
+      artifacts: false
+      optional: true
+    - job: build_test_apps_esp32c3
+      artifacts: false
+      optional: true
+    - job: build_test_apps_esp32s3
+      artifacts: false
+      optional: true
+  variables:
+    TEST_TYPE: custom_test
+    TEST_DIR: tools/test_apps
+
+assign_unit_test:
+  extends:
+    - .assign_test_template
+    - .rules:build:unit_test
+  needs:
+    - job: build_esp_idf_tests_cmake_esp32
+      optional: true
+    - job: build_esp_idf_tests_cmake_esp32s2
+      optional: true
+    - job: build_esp_idf_tests_cmake_esp32c2
+      optional: true
+    - job: build_esp_idf_tests_cmake_esp32c3
+      optional: true
+    - job: build_esp_idf_tests_cmake_esp32s3
+      optional: true
+  variables:
+    TEST_TYPE: unit_test
+    TEST_DIR: components/idf_test/unit_test

.gitlab/ci/build.yml

@@ -0,0 +1,556 @@
+.build_template:
+  stage: build
+  image: $ESP_ENV_IMAGE
+  tags:
+    - build
+  variables:
+    # Enable ccache for all build jobs. See configure_ci_environment.sh for more ccache related settings.
+    IDF_CCACHE_ENABLE: "1"
+  after_script:
+    # Show ccache statistics if enabled globally
+    - test "$CI_CCACHE_STATS" == 1 && test -n "$(which ccache)" && ccache --show-stats || true
+  dependencies: []
+
+.build_cmake_template:
+  extends:
+    - .build_template
+    - .before_script_build_jobs
+  dependencies: # set dependencies to null to avoid missing artifacts issue
+  needs:
+    - job: fast_template_app
+      artifacts: false
+  tags:
+    - build
+    # build only on shiny servers since shiny storage server is at the same location
+    - shiny
+  artifacts:
+    paths:
+      - "**/build*/size.json"
+      - "**/build*/build_log.txt"
+      - "**/build*/*.bin"
+      # upload to s3 server to save the artifacts size
+      # - "**/build*/*.map"
+      # ttfw tests require elf files
+      - "**/build*/*.elf"
+      - "**/build*/flasher_args.json"
+      - "**/build*/flash_project_args"
+      - "**/build*/config/sdkconfig.json"
+      # ttfw tests require sdkconfig file
+      - "**/build*/sdkconfig"
+      - "**/build*/bootloader/*.bin"
+      - "**/build*/partition_table/*.bin"
+      - list_job_*.json
+      - size_info.txt
+      # unit test specific
+      - components/idf_test/unit_test/*.yml
+    when: always
+    expire_in: 4 days
+  after_script:
+    # Show ccache statistics if enabled globally
+    - test "$CI_CCACHE_STATS" == 1 && test -n "$(which ccache)" && ccache --show-stats || true
+    # upload the binary files to s3 server
+    - echo -e "\e[0Ksection_start:`date +%s`:upload_binaries_to_s3_server[collapsed=true]\r\e[0KUploading binaries to s3 Server"
+    - shopt -s globstar
+    # use || true to bypass the no-file error
+    - zip ${CI_JOB_ID}.zip **/build*/*.bin || true
+    - zip ${CI_JOB_ID}.zip **/build*/*.elf || true
+    - zip ${CI_JOB_ID}.zip **/build*/*.map || true
+    - zip ${CI_JOB_ID}.zip **/build*/flasher_args.json || true
+    - zip ${CI_JOB_ID}.zip **/build*/flash_project_args || true
+    - zip ${CI_JOB_ID}.zip **/build*/config/sdkconfig.json || true
+    - zip ${CI_JOB_ID}.zip **/build*/sdkconfig || true
+    - zip ${CI_JOB_ID}.zip **/build*/bootloader/*.bin || true
+    - zip ${CI_JOB_ID}.zip **/build*/partition_table/*.bin || true
+    - mc alias set shiny-s3 ${SHINY_S3_SERVER} ${SHINY_S3_ACCESS_KEY} ${SHINY_S3_SECRET_KEY}
+    - mc cp ${CI_JOB_ID}.zip shiny-s3/idf-artifacts/${CI_PIPELINE_ID}/${CI_JOB_ID}.zip
+    - echo -e "\e[0Ksection_end:`date +%s`:upload_binaries_to_s3_server\r\e[0K"
+    - echo "Please download the full binary files (including *.elf and *.map files) from the following share link"
+    # would be clean up after 4 days
+    - mc share download shiny-s3/idf-artifacts/${CI_PIPELINE_ID}/${CI_JOB_ID}.zip --expire=96h
+  script:
+    # CI specific options start from "--collect-size-info xxx". could ignore when running locally
+    - run_cmd python tools/ci/ci_build_apps.py $TEST_DIR -v
+      -t $IDF_TARGET
+      --copy-sdkconfig
+      --collect-size-info size_info.txt
+      --collect-app-info list_job_${CI_NODE_INDEX:-1}.json
+      --parallel-count ${CI_NODE_TOTAL:-1}
+      --parallel-index ${CI_NODE_INDEX:-1}
+      --extra-preserve-dirs
+        examples/bluetooth/esp_ble_mesh/ble_mesh_console
+        examples/bluetooth/hci/controller_hci_uart_esp32
+        examples/wifi/iperf
+
+.build_pytest_template:
+  extends:
+    - .build_cmake_template
+    - .before_script_build_jobs
+  artifacts:
+    paths:
+      - "**/build*/size.json"
+      - "**/build*/build_log.txt"
+      - "**/build*/*.bin"
+      # upload to s3 server to save the artifacts size
+      # - "**/build*/*.map"
+      # - "**/build*/*.elf"
+      - "**/build*/flasher_args.json"
+      - "**/build*/flash_project_args"
+      - "**/build*/config/sdkconfig.json"
+      - "**/build*/bootloader/*.bin"
+      - "**/build*/bootloader/*.elf"
+      - "**/build*/partition_table/*.bin"
+      - "**/build*/project_description.json"
+      - list_job_*.json
+      - size_info.txt
+    when: always
+    expire_in: 4 days
+  script:
+    # CI specific options start from "--collect-size-info xxx". could ignore when running locally
+    - run_cmd python tools/ci/ci_build_apps.py $TEST_DIR -v
+      -t $IDF_TARGET
+      --pytest-apps
+      --collect-size-info size_info.txt
+      --parallel-count ${CI_NODE_TOTAL:-1}
+      --parallel-index ${CI_NODE_INDEX:-1}
+
+build_pytest_examples_esp32:
+  extends:
+    - .build_pytest_template
+    - .rules:build:example_test-esp32
+  parallel: 4
+  variables:
+    IDF_TARGET: esp32
+    TEST_DIR: examples
+
+build_pytest_examples_esp32s2:
+  extends:
+    - .build_pytest_template
+    - .rules:build:example_test-esp32s2
+  parallel: 3
+  variables:
+    IDF_TARGET: esp32s2
+    TEST_DIR: examples
+
+build_pytest_examples_esp32s3:
+  extends:
+    - .build_pytest_template
+    - .rules:build:example_test-esp32s3
+  parallel: 4
+  variables:
+    IDF_TARGET: esp32s3
+    TEST_DIR: examples
+
+build_pytest_examples_esp32c3:
+  extends:
+    - .build_pytest_template
+    - .rules:build:example_test-esp32c3
+  parallel: 3
+  variables:
+    IDF_TARGET: esp32c3
+    TEST_DIR: examples
+
+build_pytest_examples_esp32c2:
+  extends:
+    - .build_pytest_template
+    - .rules:build:example_test-esp32c2
+  variables:
+    IDF_TARGET: esp32c2
+    TEST_DIR: examples
+
+build_pytest_components_esp32:
+  extends:
+    - .build_pytest_template
+    - .rules:build:component_ut-esp32
+  parallel: 2
+  variables:
+    IDF_TARGET: esp32
+    TEST_DIR: components
+
+build_pytest_components_esp32s2:
+  extends:
+    - .build_pytest_template
+    - .rules:build:component_ut-esp32s2
+  parallel: 2
+  variables:
+    IDF_TARGET: esp32s2
+    TEST_DIR: components
+
+build_pytest_components_esp32s3:
+  extends:
+    - .build_pytest_template
+    - .rules:build:component_ut-esp32s3
+  parallel: 2
+  variables:
+    IDF_TARGET: esp32s3
+    TEST_DIR: components
+
+build_pytest_components_esp32c3:
+  extends:
+    - .build_pytest_template
+    - .rules:build:component_ut-esp32c3
+  parallel: 2
+  variables:
+    IDF_TARGET: esp32c3
+    TEST_DIR: components
+
+build_pytest_components_esp32c2:
+  extends:
+    - .build_pytest_template
+    - .rules:build:component_ut-esp32c2
+  variables:
+    IDF_TARGET: esp32c2
+    TEST_DIR: components
+
+build_non_test_components_apps:
+  extends:
+    - .build_cmake_template
+    - .rules:build:component_ut
+  script:
+    - set_component_ut_vars
+    # CI specific options start from "--collect-size-info xxx". could ignore when running locally
+    - run_cmd python tools/ci/ci_build_apps.py $COMPONENT_UT_DIRS -v
+      -t all
+      --collect-size-info size_info.txt
+      --collect-app-info list_job_${CI_NODE_INDEX:-1}.json
+      --parallel-count ${CI_NODE_TOTAL:-1}
+      --parallel-index ${CI_NODE_INDEX:-1}
+
+.build_pytest_test_apps_template:
+  extends: .build_pytest_template
+  artifacts:
+    paths:
+      - "**/build*/size.json"
+      - "**/build*/build_log.txt"
+      - "**/build*/*.bin"
+      # upload to s3 server to save the artifacts size
+      # - "**/build*/*.map"
+      # pytest test apps requires elf files for coredump tests
+      - "**/build*/*.elf"
+      - "**/build*/flasher_args.json"
+      - "**/build*/flash_project_args"
+      - "**/build*/config/sdkconfig.json"
+      - "**/build*/bootloader/*.bin"
+      - "**/build*/partition_table/*.bin"
+      - list_job_*.json
+      - size_info.txt
+    when: always
+    expire_in: 4 days
+
+build_pytest_test_apps_esp32:
+  extends:
+    - .build_pytest_test_apps_template
+    - .rules:build:custom_test-esp32
+  variables:
+    IDF_TARGET: esp32
+    TEST_DIR: tools/test_apps
+
+build_pytest_test_apps_esp32s2:
+  extends:
+    - .build_pytest_test_apps_template
+    - .rules:build:custom_test-esp32s2
+  variables:
+    IDF_TARGET: esp32s2
+    TEST_DIR: tools/test_apps
+
+build_pytest_test_apps_esp32s3:
+  extends:
+    - .build_pytest_test_apps_template
+    - .rules:build:custom_test-esp32s3
+  variables:
+    IDF_TARGET: esp32s3
+    TEST_DIR: tools/test_apps
+
+build_pytest_test_apps_esp32c3:
+  extends:
+    - .build_pytest_test_apps_template
+    - .rules:build:custom_test-esp32c3
+  variables:
+    IDF_TARGET: esp32c3
+    TEST_DIR: tools/test_apps
+
+build_pytest_test_apps_esp32c2:
+  extends:
+    - .build_pytest_test_apps_template
+    - .rules:build:custom_test-esp32c2
+  variables:
+    IDF_TARGET: esp32c2
+    TEST_DIR: tools/test_apps
+
+.build_template_app_template:
+  extends:
+    - .build_template
+    - .before_script_build_jobs
+  variables:
+    LOG_PATH: "${CI_PROJECT_DIR}/log_template_app"
+    BUILD_PATH: "${CI_PROJECT_DIR}/build_template_app"
+    BUILD_DIR: "@t/@w"
+    BUILD_LOG_CMAKE: "${LOG_PATH}/cmake_@t_@w.txt"
+    BUILD_COMMAND_ARGS: ""
+  artifacts:
+    when: always
+    paths:
+      - log_template_app/*
+      - size_info.txt
+      - build_template_app/**/size.json
+    expire_in: 1 week
+  script:
+    # Set the variable for 'esp-idf-template' testing
+    - ESP_IDF_TEMPLATE_GIT=${ESP_IDF_TEMPLATE_GIT:-"https://github.com/espressif/esp-idf-template.git"}
+    - retry_failed git clone ${ESP_IDF_TEMPLATE_GIT}
+    # Try to use the same branch name for esp-idf-template that we're
+    # using on esp-idf. If it doesn't exist then just stick to the default branch
+    - python $CHECKOUT_REF_SCRIPT esp-idf-template esp-idf-template
+    - export PATH="$IDF_PATH/tools:$PATH"
+    # Only do the default cmake build for each target, remaining part are done in the build_template_app job
+    - tools/ci/build_template_app.sh ${BUILD_COMMAND_ARGS}
+
+# build-related-pre-check-jobs ------------------------------------------------
+# Build at least one project for each target at earliest stage to reduce build cost for obvious failing commits
+fast_template_app:
+  extends:
+    - .build_template_app_template
+    - .rules:build:target_test
+  stage: pre_check
+  variables:
+    BUILD_COMMAND_ARGS: "-p"
+#------------------------------------------------------------------------------
+
+.build_esp_idf_tests_cmake_template:
+  extends:
+    - .build_cmake_template
+    - .before_script_build_jobs
+  artifacts:
+    paths:
+      - "**/build*/size.json"
+      - "**/build*/build_log.txt"
+      - "**/build*/*.bin"
+      # upload to s3 server to save the artifacts size
+      # - "**/build*/*.map"
+      # ttfw tests require elf files
+      - "**/build*/*.elf"
+      - "**/build*/flasher_args.json"
+      - "**/build*/flash_project_args"
+      - "**/build*/config/sdkconfig.json"
+      - "**/build*/sdkconfig"
+      - "**/build*/bootloader/*.bin"
+      - "**/build*/partition_table/*.bin"
+      - list_job_*.json
+      - size_info.txt
+      - components/idf_test/unit_test/*.yml
+    when: always
+    expire_in: 4 days
+  script:
+    # CI specific options start from "--collect-size-info xxx". could ignore when running locally
+    - run_cmd python tools/ci/ci_build_apps.py tools/unit-test-app -v
+      -t $IDF_TARGET
+      --config "configs/*="
+      --copy-sdkconfig
+      --preserve-all
+      --collect-size-info size_info.txt
+      --collect-app-info list_job_${CI_NODE_INDEX:-1}.json
+      --parallel-count ${CI_NODE_TOTAL:-1}
+      --parallel-index ${CI_NODE_INDEX:-1}
+    - run_cmd python tools/unit-test-app/tools/UnitTestParser.py tools/unit-test-app ${CI_NODE_INDEX:-1}
+
+build_esp_idf_tests_cmake_esp32:
+  extends:
+    - .build_esp_idf_tests_cmake_template
+    - .rules:build:unit_test-esp32
+  parallel: 2
+  variables:
+    IDF_TARGET: esp32
+
+build_esp_idf_tests_cmake_esp32s2:
+  extends:
+    - .build_esp_idf_tests_cmake_template
+    - .rules:build:unit_test-esp32s2
+  parallel: 2
+  variables:
+    IDF_TARGET: esp32s2
+
+build_esp_idf_tests_cmake_esp32s3:
+  extends:
+    - .build_esp_idf_tests_cmake_template
+    - .rules:build:unit_test-esp32s3
+  variables:
+    IDF_TARGET: esp32s3
+
+build_esp_idf_tests_cmake_esp32c2:
+  extends:
+    - .build_esp_idf_tests_cmake_template
+    - .rules:build:unit_test-esp32c2
+  variables:
+    IDF_TARGET: esp32c2
+
+build_esp_idf_tests_cmake_esp32c3:
+  extends:
+    - .build_esp_idf_tests_cmake_template
+    - .rules:build:unit_test-esp32c3
+  variables:
+    IDF_TARGET: esp32c3
+
+build_examples_cmake_esp32:
+  extends:
+    - .build_cmake_template
+    - .rules:build:example_test-esp32
+  parallel: 9
+  variables:
+    IDF_TARGET: esp32
+    TEST_DIR: examples
+
+build_examples_cmake_esp32s2:
+  extends:
+    - .build_cmake_template
+    - .rules:build:example_test-esp32s2
+  parallel: 6
+  variables:
+    IDF_TARGET: esp32s2
+    TEST_DIR: examples
+
+build_examples_cmake_esp32s3:
+  extends:
+    - .build_cmake_template
+    - .rules:build:example_test-esp32s3
+  parallel: 7
+  variables:
+    IDF_TARGET: esp32s3
+    TEST_DIR: examples
+
+build_examples_cmake_esp32c2:
+  extends:
+    - .build_cmake_template
+    - .rules:build:example_test-esp32c2
+  parallel: 4
+  variables:
+    IDF_TARGET: esp32c2
+    TEST_DIR: examples
+
+build_examples_cmake_esp32c3:
+  extends:
+    - .build_cmake_template
+    - .rules:build:example_test-esp32c3
+  parallel: 6
+  variables:
+    IDF_TARGET: esp32c3
+    TEST_DIR: examples
+
+build_examples_cmake_esp32h2:
+  extends:
+    - .build_cmake_template
+    - .rules:build:example_test-esp32h2
+  variables:
+    IDF_TARGET: esp32h2
+    TEST_DIR: examples
+
+build_test_apps_esp32:
+  extends:
+    - .build_cmake_template
+    - .rules:build:custom_test-esp32
+  parallel: 2
+  variables:
+    IDF_TARGET: esp32
+    TEST_DIR: tools/test_apps
+
+build_test_apps_esp32s2:
+  extends:
+    - .build_cmake_template
+    - .rules:build:custom_test-esp32s2
+  parallel: 2
+  variables:
+    IDF_TARGET: esp32s2
+    TEST_DIR: tools/test_apps
+
+build_test_apps_esp32s3:
+  extends:
+    - .build_cmake_template
+    - .rules:build:custom_test-esp32s3
+  parallel: 2
+  variables:
+    IDF_TARGET: esp32s3
+    TEST_DIR: tools/test_apps
+
+build_test_apps_esp32c3:
+  extends:
+    - .build_cmake_template
+    - .rules:build:custom_test-esp32c3
+  parallel: 2
+  variables:
+    IDF_TARGET: esp32c3
+    TEST_DIR: tools/test_apps
+
+build_test_apps_esp32c2:
+  extends:
+    - .build_cmake_template
+    - .rules:build:custom_test-esp32c2
+  variables:
+    IDF_TARGET: esp32c2
+    TEST_DIR: tools/test_apps
+
+.test_build_system_template:
+  stage: host_test
+  extends:
+    - .build_template
+    - .rules:build
+  needs:
+    - job: fast_template_app
+      artifacts: false
+  script:
+    - ${IDF_PATH}/tools/ci/test_configure_ci_environment.sh
+    - rm -rf test_build_system
+    - mkdir test_build_system
+    - cd test_build_system
+    - ${IDF_PATH}/tools/ci/${SHELL_TEST_SCRIPT}
+
+test_build_system_cmake:
+  extends: .test_build_system_template
+  variables:
+    SHELL_TEST_SCRIPT: test_build_system_cmake.sh
+
+test_build_system_cmake_macos:
+  extends:
+    - .test_build_system_template
+    - .before_script_macos
+    - .rules:build:macos
+  tags:
+    - macos_shell
+  variables:
+    SHELL_TEST_SCRIPT: test_build_system_cmake.sh
+
+test_build_system_spaces:
+  extends: .test_build_system_template
+  variables:
+    SHELL_TEST_SCRIPT: test_build_system_spaces.py
+
+build_docker:
+  extends:
+    - .before_script_minimal
+    - .rules:build:docker
+  stage: host_test
+  needs: []
+  image: espressif/docker-builder:1
+  tags:
+    - build_docker_amd64_brno
+  variables:
+    DOCKER_TMP_IMAGE_NAME: "idf_tmp_image"
+  script:
+    - export LOCAL_CI_REPOSITORY_URL=$CI_REPOSITORY_URL
+    - if [ -n "$LOCAL_GITLAB_HTTPS_HOST" ]; then export LOCAL_CI_REPOSITORY_URL="https://gitlab-ci-token:${CI_JOB_TOKEN}@${LOCAL_GITLAB_HTTPS_HOST}/${CI_PROJECT_PATH}"; fi
+    - if [ -n "$LOCAL_GIT_MIRROR" ]; then export LOCAL_CI_REPOSITORY_URL="${LOCAL_GIT_MIRROR}/${CI_PROJECT_PATH}"; fi
+    - echo "Using repository at $LOCAL_CI_REPOSITORY_URL"
+    - export DOCKER_BUILD_ARGS="--build-arg IDF_CLONE_URL=${LOCAL_CI_REPOSITORY_URL} --build-arg IDF_CLONE_BRANCH_OR_TAG=${CI_COMMIT_REF_NAME} --build-arg IDF_CHECKOUT_REF=${CI_COMMIT_TAG:-$PIPELINE_COMMIT_SHA}"
+    # Build
+    - docker build --tag ${DOCKER_TMP_IMAGE_NAME} ${DOCKER_BUILD_ARGS} tools/docker/
+    # We can't mount $PWD/examples/get-started/blink into the container, see https://gitlab.com/gitlab-org/gitlab-ce/issues/41227.
+    # The workaround mentioned there works, but leaves around directories which need to be cleaned up manually.
+    # Therefore, build a copy of the example located inside the container.
+    - docker run --rm --workdir /opt/esp/idf/examples/get-started/blink ${DOCKER_TMP_IMAGE_NAME} idf.py build
+
+# This job builds template app with permutations of targets and optimization levels
+build_template_app:
+  extends:
+    - .build_template_app_template
+    - .rules:build
+  stage: host_test
+  needs:
+    - job: fast_template_app
+      artifacts: false

.gitlab/ci/dependencies/README.md

@@ -0,0 +1,100 @@
+# How the `generate_rules.py` works
+
+## Functionalities
+
+This script can do only two things:
+
+1. Auto-generate some labels/rules we need and update them in `rules.yml`
+2. Generate a dependency tree graph
+
+## Schema
+
+This file only used basic YAML grammar and has nothing to do with the GitLab version YAML file.
+
+It has five custom keywords:
+
+- `matrix`: An array of sub-arrays, used to replicate rules by formatting strings. You can use the format string everywhere, it will be formatted recursively
+- `labels`: An array of `labels`.
+- `patterns`: An array of `patterns`. Patterns that not included
+- `included_in`: An array of other `rule` names. It indicates the `labels` and `patterns` will be included in all specified `rules` as well
+- `deploy`: An array of strings, used to replicate rules by adding postfix `-<item in deploy array>`. It indicates the extra `label` used in `rules`, which will explain later.
+
+## How to use this file to generate `rules.yml`
+
+Let's take a complicated example to help understand the process
+
+```yaml
+"test-{0}-{1}":
+    matrix:
+        - [a, b]
+        - [c, d]
+    labels:
+        - "{0}-{1}"
+    patterns:
+        - "{0}"
+        - pattern-not-exist
+    included_in:
+        - build-{0}
+```
+
+1. expand the mapping dicts defined by `matrix`
+
+   After this step, it will turn into 4 dicts:
+
+   | key      | labels | patterns | included_in |
+   | -------- | ------ | -------- | ----------- |
+   | test-a-c | a-c    | a        | build-a     |
+   | test-a-d | a-d    | a        | build-a     |
+   | test-b-c | b-c    | b        | build-b     |
+   | test-b-d | b-d    | b        | build-b     |
+
+   **Advanced Usage: You can overwrite a mapping by declaring it again later**, For example:
+
+   If we concatenate this part to the previous example,
+
+    ```yaml
+    # ... The same as the previous example
+    
+    test-a-c:
+        labels:
+            - overwrite
+    ```
+
+   `rule` `test-a-c` will be turned into:
+
+   | key      | labels    |
+   | -------- | --------- |
+   | test-a-c | overwrite |
+
+   **Mappings with the keyword `deploy` will also replicate by adding a postfix `-<item in deploy array>` to the mapping key**
+
+2. create rules by `included_in`
+
+   After this step, it will turn into 6 mapping dicts:
+
+   | key      | labels   | patterns |
+   | -------- | -------- | -------- |
+   | test-a-c | a-c      | a        |
+   | test-a-d | a-d      | a        |
+   | test-b-c | b-c      | b        |
+   | test-b-d | b-d      | b        |
+   | build-a  | a-c, a-d | a        |
+   | build-b  | b-c, b-d | b        |
+
+3. replace the auto-generated region in `rules.yml` with `labels`, and `rules`. Each mapping will generate a `rule` and all the required labels. `patterns` are pre-defined in `rules.yml` and could not be generated automatically. If a mapping is using a `pattern` undefined, the `pattern` will be ignored.
+
+    - If a mapping key has postfix `-preview`, no `if-protected-xxx` clause will be added
+    - else if a mapping key has postfix `-production`, an `if-protected-no_label` clause will be added
+    - else, an `if-protected` clause will be added
+
+## Graph
+
+All `label` nodes are in green, `pattern` nodes are in cyan, `rule` nodes are in blue
+
+### Requirements
+
+There are a few extra dependencies while generating the dependency tree graph, please refer to [pygraphviz](https://github.com/pygraphviz/pygraphviz/blob/master/INSTALL.txt) documentation to install both `graphviz` and `pygraphviz`
+
+### CLI usage
+
+`python generate_rules.py --graph OUTPUT_PATH`

.gitlab/ci/dependencies/dependencies.yml

@@ -0,0 +1,197 @@
+.all_targets: &all_targets
+  - esp32
+  - esp32s2
+  - esp32s3
+  - esp32c3
+  - esp32h2
+  - esp32c2
+
+.target_test: &target_test
+  - example_test
+  - custom_test
+  - unit_test
+  - component_ut
+
+##############
+# Build Jobs #
+##############
+"build":
+  labels:
+    - build
+  patterns:
+    - build_components
+    - build_system
+  included_in:
+    - build:target_test
+
+# -------------------
+# Specific Build Jobs
+# -------------------
+"build:docker":
+  labels:
+    - build
+    - docker
+  patterns:
+    - docker
+
+"build:windows":
+  labels:
+    - build
+    - windows
+  patterns:
+    - build_system
+    - windows
+
+"build:macos":
+  labels:
+    - build
+    - macos
+    - macos_test  # for backward compatibility
+  patterns:
+    - build_system
+    - macos
+
+# ---------------
+# Build Test Jobs
+# ---------------
+"build:{0}-{1}":
+  matrix:
+    - *target_test
+    - *all_targets
+  labels:
+    - build
+  patterns:
+    - build_components
+    - build_system
+    - build_target_test
+  included_in:
+    - "build:{0}"
+    - build:target_test
+
+####################
+# Target Test Jobs #
+####################
+"test:{0}-{1}":
+  matrix:
+    - *target_test
+    - *all_targets
+  labels: # For each rule, use labels <test_type> and <test_type>-<target>
+    - "{0}"
+    - "{0}_{1}"
+    - target_test
+  patterns: # For each rule, use patterns <test_type> and build-<test_type>
+    - "{0}"
+    - "build-{0}"
+  included_in: # Parent rules
+    - "build:{0}"
+    - "build:{0}-{1}"
+    - build:target_test
+
+# -------------
+# Special Cases
+# -------------
+"test:component_ut-{0}": # component_ut will trigger by unit_test as well, since now we have 2 kinds of UT
+  matrix:
+    - *all_targets
+  labels:
+    - component_ut
+    - "component_ut_{0}"
+    - unit_test
+    - "unit_test_{0}"
+    - target_test
+  patterns:
+    - component_ut
+    - "build-component_ut-{0}"
+  included_in:
+    - build:component_ut
+    - "build:component_ut-{0}"
+    - build:target_test
+
+# To reduce the specific runners' usage.
+# Do not create these jobs by default patterns on development branches
+# Can be triggered by labels or related changes
+"test:{0}-{1}-{2}":
+  matrix:
+    - *target_test
+    - *all_targets
+    - - bt        # example_test_005
+      - wifi      # example_test_002, example_test*wifi*
+      - ethernet  # example_test*ethernet*
+      - sdio      # UT_044, UT_045
+      - usb       # USB Device & Host tests
+      - adc       # component_ut_pytest_esp32x_adc
+  patterns:
+    - "{0}-{1}-{2}"
+    - "{0}-{2}"
+    - "target_test-{2}"
+  labels:
+    - "{0}_{1}"
+    - "{0}"
+    - target_test
+  included_in:
+    - "build:{0}-{1}"
+    - "build:{0}"
+    - build:target_test
+
+# For example_test*flash_encryption_wifi_high_traffic jobs
+# set `INCLUDE_NIGHTLY_RUN` variable when triggered on development branches
+"test:example_test-{0}-include_nightly_run-rule":
+  matrix:
+    - - esp32
+      - esp32c3
+  specific_rules:
+    - "if-example_test-ota-include_nightly_run-rule"
+  included_in:
+    - "build:example_test-{0}"
+    - "build:example_test"
+    - build:target_test
+
+"test:host_test":
+  labels:
+    - host_test
+  patterns:
+    - host_test
+
+"test:submodule":
+  labels:
+    - submodule
+  patterns:
+    - submodule
+
+#################################
+# Triggered Only By Labels Jobs #
+#################################
+"labels:iperf_stress_test": # example_test
+  labels:
+    - iperf_stress_test
+  included_in:
+    - build:example_test
+    - build:example_test-esp32
+    - build:target_test
+
+"labels:weekend_test": # custom test
+  labels:
+    - weekend_test
+  included_in:
+    - build:custom_test
+    - build:custom_test-esp32
+    - build:target_test
+
+"labels:nvs_coverage": # host_test
+  labels:
+    - nvs_coverage
+
+"labels:fuzzer_test-weekend_test": # host test
+  labels:
+    - fuzzer_test
+    - weekend_test
+
+"labels-protected:lan8720":  # UT # FIXME: IDFCI-1176 temporary run this on master/release or with label
+  labels:
+    - lan8720
+  included_in:
+    - build:unit_test
+    - build:unit_test-esp32
+    - build:target_test
+    - build:component_ut
+    - build:component_ut-esp32

.gitlab/ci/dependencies/generate_rules.py

@@ -0,0 +1,304 @@
+#!/usr/bin/env python
+#
+# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Apache-2.0
+
+import argparse
+import inspect
+import os
+import sys
+from collections import defaultdict
+from itertools import product
+
+import yaml
+
+try:
+    import pygraphviz as pgv
+except ImportError:  # used when pre-commit, skip generating image
+    pass
+
+try:
+    from typing import Union
+except ImportError:  # used for type hint
+    pass
+
+IDF_PATH = os.path.abspath(os.getenv('IDF_PATH', os.path.join(os.path.dirname(__file__), '..', '..', '..')))
+
+
+def _list(str_or_list):  # type: (Union[str, list]) -> list
+    if isinstance(str_or_list, str):
+        return [str_or_list]
+    elif isinstance(str_or_list, list):
+        return str_or_list
+    else:
+        raise ValueError('Wrong type: {}. Only supports str or list.'.format(type(str_or_list)))
+
+
+def _format_nested_dict(_dict, f_tuple):  # type: (dict[str, dict], tuple[str, ...]) -> dict[str, dict]
+    res = {}
+    for k, v in _dict.items():
+        k = k.split('__')[0]
+        if isinstance(v, dict):
+            v = _format_nested_dict(v, f_tuple)
+        elif isinstance(v, list):
+            v = _format_nested_list(v, f_tuple)
+        elif isinstance(v, str):
+            v = v.format(*f_tuple)
+        res[k.format(*f_tuple)] = v
+    return res
+
+
+def _format_nested_list(_list, f_tuple):  # type: (list[str], tuple[str, ...]) -> list[str]
+    res = []
+    for item in _list:
+        if isinstance(item, list):
+            item = _format_nested_list(item, f_tuple)
+        elif isinstance(item, dict):
+            item = _format_nested_dict(item, f_tuple)
+        elif isinstance(item, str):
+            item = item.format(*f_tuple)
+        res.append(item)
+    return res
+
+
+class RulesWriter:
+    AUTO_GENERATE_MARKER = inspect.cleandoc(r'''
+    ##################
+    # Auto Generated #
+    ##################
+    ''')
+
+    LABEL_TEMPLATE = inspect.cleandoc(r'''
+    .if-label-{0}: &if-label-{0}
+      if: '$BOT_LABEL_{1} || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*{0}(?:,[^,\n\r]+)*$/i'
+    ''')
+
+    RULE_PROTECTED = '    - <<: *if-protected'
+    RULE_PROTECTED_NO_LABEL = '    - <<: *if-protected-no_label'
+    RULE_BUILD_ONLY = '    - <<: *if-label-build-only\n' \
+                      '      when: never'
+    RULE_REVERT_BRANCH = '    - <<: *if-revert-branch\n' \
+                         '      when: never'
+    RULE_LABEL_TEMPLATE = '    - <<: *if-label-{0}'
+    RULE_PATTERN_TEMPLATE = '    - <<: *if-dev-push\n' \
+                            '      changes: *patterns-{0}'
+    SPECIFIC_RULE_TEMPLATE = '    - <<: *{0}'
+    RULES_TEMPLATE = inspect.cleandoc(r"""
+    .rules:{0}:
+      rules:
+    {1}
+    """)
+
+    KEYWORDS = ['labels', 'patterns']
+
+    def __init__(self, rules_yml, depend_yml):  # type: (str, str) -> None
+        self.rules_yml = rules_yml
+        self.rules_cfg = yaml.load(open(rules_yml), Loader=yaml.FullLoader)
+
+        self.full_cfg = yaml.load(open(depend_yml), Loader=yaml.FullLoader)
+        self.cfg = {k: v for k, v in self.full_cfg.items() if not k.startswith('.')}
+        self.cfg = self.expand_matrices()
+        self.rules = self.expand_rules()
+
+        self.graph = None
+
+    def expand_matrices(self):  # type: () -> dict
+        """
+        Expand the matrix into different rules
+        """
+        res = {}
+        for k, v in self.cfg.items():
+            res.update(self._expand_matrix(k, v))
+
+        for k, v in self.cfg.items():
+            if not v:
+                continue
+            deploy = v.get('deploy')
+            if deploy:
+                for item in _list(deploy):
+                    res['{}-{}'.format(k, item)] = v
+        return res
+
+    @staticmethod
+    def _expand_matrix(name, cfg):  # type: (str, dict) -> dict
+        """
+        Expand matrix into multi keys
+        :param cfg: single rule dict
+        :return:
+        """
+        default = {name: cfg}
+        if not cfg:
+            return default
+        matrices = cfg.pop('matrix', None)
+        if not matrices:
+            return default
+
+        res = {}
+        for comb in product(*_list(matrices)):
+            res.update(_format_nested_dict(default, comb))
+        return res
+
+    def expand_rules(self):  # type: () -> dict[str, dict[str, list]]
+        res = defaultdict(lambda: defaultdict(set))  # type: dict[str, dict[str, set]]
+        for k, v in self.cfg.items():
+            if not v:
+                continue
+            for vk, vv in v.items():
+                if vk in self.KEYWORDS:
+                    res[k][vk] = set(_list(vv))
+                else:
+                    res[k][vk] = vv
+            for key in self.KEYWORDS:  # provide empty set for missing field
+                if key not in res[k]:
+                    res[k][key] = set()
+
+        for k, v in self.cfg.items():
+            if not v:
+                continue
+            if 'included_in' in v:
+                for item in _list(v['included_in']):
+                    if 'specific_rules' in v:
+                        res[item]['specific_rules'].update(_list(v['specific_rules']))
+                    if 'labels' in v:
+                        res[item]['labels'].update(_list(v['labels']))
+                    if 'patterns' in v:
+                        for _pat in _list(v['patterns']):
+                            # Patterns must be pre-defined
+                            if '.patterns-{}'.format(_pat) not in self.rules_cfg:
+                                print('WARNING: pattern {} not exists'.format(_pat))
+                                continue
+                            res[item]['patterns'].add(_pat)
+
+        sorted_res = defaultdict(lambda: defaultdict(list))  # type: dict[str, dict[str, list]]
+        for k, v in res.items():
+            for vk, vv in v.items():
+                sorted_res[k][vk] = sorted(vv)
+        return sorted_res
+
+    def new_labels_str(self):  # type: () -> str
+        _labels = set([])
+        for k, v in self.cfg.items():
+            if not v:
+                continue  # shouldn't be possible
+            labels = v.get('labels')
+            if not labels:
+                continue
+            _labels.update(_list(labels))
+        labels = sorted(_labels)
+
+        res = ''
+        res += '\n\n'.join([self._format_label(_label) for _label in labels])
+        return res
+
+    @classmethod
+    def _format_label(cls, label):  # type: (str) -> str
+        return cls.LABEL_TEMPLATE.format(label, cls.bot_label_str(label))
+
+    @staticmethod
+    def bot_label_str(label):  # type: (str) -> str
+        return label.upper().replace('-', '_')
+
+    def new_rules_str(self):  # type: () -> str
+        res = []
+        for k, v in sorted(self.rules.items()):
+            res.append(self.RULES_TEMPLATE.format(k, self._format_rule(k, v)))
+        return '\n\n'.join(res)
+
+    def _format_rule(self, name, cfg):  # type: (str, dict) -> str
+        _rules = [self.RULE_REVERT_BRANCH]
+        if name.endswith('-production'):
+            _rules.append(self.RULE_PROTECTED_NO_LABEL)
+        else:
+            if not (name.endswith('-preview') or name.startswith('labels:')):
+                _rules.append(self.RULE_PROTECTED)
+            if name.startswith('test:'):
+                _rules.append(self.RULE_BUILD_ONLY)
+
+            for specific_rule in cfg['specific_rules']:
+                if f'.{specific_rule}' in self.rules_cfg:
+                    _rules.append(self.SPECIFIC_RULE_TEMPLATE.format(specific_rule))
+                else:
+                    print('WARNING: specific_rule {} not exists'.format(specific_rule))
+            for label in cfg['labels']:
+                _rules.append(self.RULE_LABEL_TEMPLATE.format(label))
+            for pattern in cfg['patterns']:
+                if '.patterns-{}'.format(pattern) in self.rules_cfg:
+                    _rules.append(self.RULE_PATTERN_TEMPLATE.format(pattern))
+                else:
+                    print('WARNING: pattern {} not exists'.format(pattern))
+        return '\n'.join(_rules)
+
+    def update_rules_yml(self):  # type: () -> bool
+        with open(self.rules_yml) as fr:
+            file_str = fr.read()
+
+        auto_generate_str = '\n{}\n\n{}\n'.format(self.new_labels_str(), self.new_rules_str())
+        rest, marker, old = file_str.partition(self.AUTO_GENERATE_MARKER)
+        if old == auto_generate_str:
+            return False
+        else:
+            print(self.rules_yml, 'has been modified. Please check')
+            with open(self.rules_yml, 'w') as fw:
+                fw.write(rest + marker + auto_generate_str)
+            return True
+
+
+LABEL_COLOR = 'green'
+PATTERN_COLOR = 'cyan'
+RULE_COLOR = 'blue'
+
+
+def build_graph(rules_dict):  # type: (dict[str, dict[str, list]]) -> pgv.AGraph
+    graph = pgv.AGraph(directed=True, rankdir='LR', concentrate=True)
+
+    for k, v in rules_dict.items():
+        if not v:
+            continue
+        included_in = v.get('included_in')
+        if included_in:
+            for item in _list(included_in):
+                graph.add_node(k, color=RULE_COLOR)
+                graph.add_node(item, color=RULE_COLOR)
+                graph.add_edge(k, item, color=RULE_COLOR)
+        labels = v.get('labels')
+        if labels:
+            for _label in labels:
+                graph.add_node('label:{}'.format(_label), color=LABEL_COLOR)
+                graph.add_edge('label:{}'.format(_label), k, color=LABEL_COLOR)
+        patterns = v.get('patterns')
+        if patterns:
+            for _pat in patterns:
+                graph.add_node('pattern:{}'.format(_pat), color=PATTERN_COLOR)
+                graph.add_edge('pattern:{}'.format(_pat), k, color=PATTERN_COLOR)
+
+    return graph
+
+
+def output_graph(graph, output_path='output.png'):  # type: (pgv.AGraph, str) -> None
+    graph.layout('dot')
+    if output_path.endswith('.png'):
+        img_path = output_path
+    else:
+        img_path = os.path.join(output_path, 'output.png')
+    graph.draw(img_path)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('rules_yml', nargs='?', default=os.path.join(IDF_PATH, '.gitlab', 'ci', 'rules.yml'),
+                        help='rules.yml file path')
+    parser.add_argument('dependencies_yml', nargs='?', default=os.path.join(IDF_PATH, '.gitlab', 'ci', 'dependencies',
+                                                                            'dependencies.yml'),
+                        help='dependencies.yml file path')
+    parser.add_argument('--graph',
+                        help='Specify PNG image output path. Use this argument to generate dependency graph')
+    args = parser.parse_args()
+
+    writer = RulesWriter(args.rules_yml, args.dependencies_yml)
+    file_modified = writer.update_rules_yml()
+
+    if args.graph:
+        dep_tree_graph = build_graph(writer.rules)
+        output_graph(dep_tree_graph)
+
+    sys.exit(file_modified)

.gitlab/ci/deploy.yml

@@ -0,0 +1,42 @@
+.deploy_job_template:
+  stage: deploy
+  image: $ESP_ENV_IMAGE
+  tags:
+    - deploy
+
+# Check this before push_to_github
+check_submodule_sync:
+  extends:
+    - .deploy_job_template
+    - .rules:test:submodule
+  stage: test_deploy
+  tags:
+    - github_sync
+  retry: 2
+  variables:
+    GIT_STRATEGY: clone
+    SUBMODULES_TO_FETCH: "none"
+    PUBLIC_IDF_URL: "https://github.com/espressif/esp-idf.git"
+  dependencies: []
+  script:
+    - git submodule deinit --force .
+    # setting the default remote URL to the public one, to resolve relative location URLs
+    - git config remote.origin.url ${PUBLIC_IDF_URL}
+    # check if all submodules are correctly synced to public repository
+    - git submodule init
+    - git config --get-regexp '^submodule\..*\.url$' || true
+    - git submodule update --recursive
+    - echo "IDF was cloned from ${PUBLIC_IDF_URL} completely"
+
+push_to_github:
+  extends:
+    - .deploy_job_template
+    - .before_script_minimal
+    - .rules:push_to_github
+  needs:
+    - check_submodule_sync
+  script:
+    - add_github_ssh_keys
+    - git remote remove github &>/dev/null || true
+    - git remote add github git@github.com:espressif/esp-idf.git
+    - tools/ci/push_to_github.sh

.gitlab/ci/docs.yml

@@ -0,0 +1,230 @@
+.patterns-docs: &patterns-docs
+  - ".gitlab/ci/docs.yml"
+  - "docs/**/*"
+  - "components/**/*.h"
+  - "components/**/Kconfig*"
+  - "components/**/CMakeList.txt"
+  - "components/**/sdkconfig*"
+  - "tools/kconfig_new/**/*"
+  - "CONTRIBUTING.rst"
+
+.patterns-docs-preview: &patterns-docs-preview
+  - "docs/**/*"
+
+.if-protected: &if-protected
+  if: '($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_BRANCH =~ /^release\/v/ || $CI_COMMIT_TAG =~ /^v\d+\.\d+(\.\d+)?($|-)/)'
+
+.if-protected-no_label: &if-protected-no_label
+  if: '($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_BRANCH =~ /^release\/v/ || $CI_COMMIT_TAG =~ /^v\d+\.\d+(\.\d+)?($|-)/) && $BOT_TRIGGER_WITH_LABEL == null'
+
+.if-qa-test-tag: &if-qa-test-tag
+  if: '$CI_COMMIT_TAG =~ /^qa-test/'
+
+.if-label-build_docs: &if-label-build_docs
+  if: '$BOT_LABEL_BUILD_DOCS || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*build_docs(?:,[^,\n\r]+)*$/i'
+
+.if-label-docs: &if-label-docs
+  if: '$BOT_LABEL_DOCS || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*docs(?:,[^,\n\r]+)*$/i'
+
+.if-label-docs_fast: &if-label-docs_fast
+  if: '$BOT_LABEL_DOCS_FAST || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*docs_fast(?:,[^,\n\r]+)*$/i'
+
+.if-dev-push: &if-dev-push
+  if: '$CI_COMMIT_REF_NAME != "master" && $CI_COMMIT_BRANCH !~ /^release\/v/ && $CI_COMMIT_TAG !~ /^v\d+\.\d+(\.\d+)?($|-)/ && $CI_COMMIT_TAG !~ /^qa-test/ && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event")'
+
+.doc-rules:build:docs:
+  rules:
+    - <<: *if-qa-test-tag
+      when: never
+    - <<: *if-protected
+    - <<: *if-label-build_docs
+    - <<: *if-label-docs
+    - <<: *if-label-docs_fast
+    - <<: *if-dev-push
+      changes: *patterns-docs
+
+# stage: pre_check
+check_readme_links:
+  extends:
+    - .pre_check_template
+  tags: ["build", "amd64", "internet"]
+  allow_failure: true
+  script:
+    - python ${IDF_PATH}/tools/ci/check_readme_links.py
+
+check_docs_lang_sync:
+  extends:
+    - .pre_check_template
+    - .doc-rules:build:docs
+  script:
+    - cd docs
+    - ./check_lang_folder_sync.sh
+
+.build_docs_template:
+  image: $ESP_IDF_DOC_ENV_IMAGE
+  tags:
+    - build_docs
+  dependencies: []
+  script:
+    - cd docs
+    - build-docs -t $DOCTGT -bs $DOC_BUILDERS -l $DOCLANG build
+  parallel:
+    matrix:
+      - DOCLANG: ["en", "zh_CN"]
+        DOCTGT: ["esp32", "esp32s2", "esp32s3", "esp32c3", "esp32c2"]
+
+check_docs_gh_links:
+  image: $ESP_IDF_DOC_ENV_IMAGE
+  extends:
+    - .pre_check_template
+    - .doc-rules:build:docs
+  script:
+    - cd docs
+    - build-docs gh-linkcheck
+
+# stage: build_doc
+# Add this stage to let the build_docs job run in parallel with build
+.build_docs_build_stage_template:
+  extends:
+    - .build_docs_template
+  stage: build_doc
+  needs:
+    - job: check_docs_lang_sync
+      artifacts: false
+    - job: check_docs_gh_links
+      artifacts: false
+
+# Doc jobs have a lot of special cases, we specify rules here directly instead
+# in dependencies.yml to simplify things
+build_docs_html_full:
+  extends:
+    - .build_docs_build_stage_template
+  rules:
+    - <<: *if-label-docs_fast
+      when: never
+    - <<: *if-protected
+    - <<: *if-label-build_docs
+    - <<: *if-label-docs
+    - <<: *if-dev-push
+      changes: *patterns-docs
+  artifacts:
+    when: always
+    paths:
+      - docs/_build/*/*/*.txt
+      - docs/_build/*/*/html/*
+    expire_in: 4 days
+  variables:
+    DOC_BUILDERS: "html"
+
+build_docs_html_fast:
+  extends:
+    - .build_docs_build_stage_template
+  rules:
+    - <<: *if-label-docs_fast
+  artifacts:
+    when: always
+    paths:
+      - docs/_build/*/*/*.txt
+      - docs/_build/*/*/html/*
+    expire_in: 4 days
+  variables:
+    DOC_BUILDERS: "html"
+    DOCS_FAST_BUILD: "yes"
+
+build_docs_pdf:
+  extends:
+    - .build_docs_build_stage_template
+  rules:
+    - <<: *if-label-docs_fast
+      when: never
+    - <<: *if-protected
+    - <<: *if-label-build_docs
+    - <<: *if-label-docs
+    - <<: *if-dev-push
+      changes: *patterns-docs
+  artifacts:
+    when: always
+    paths:
+      - docs/_build/*/*/latex/*
+    expire_in: 4 days
+  variables:
+    DOC_BUILDERS: "latex"
+
+.deploy_docs_template:
+  image: $ESP_IDF_DOC_ENV_IMAGE
+  variables:
+    DOCS_BUILD_DIR: "${IDF_PATH}/docs/_build/"
+    PYTHONUNBUFFERED: 1
+  stage: test_deploy
+  tags:
+    - deploy
+    - shiny
+  script:
+    - add_doc_server_ssh_keys $DOCS_DEPLOY_PRIVATEKEY $DOCS_DEPLOY_SERVER $DOCS_DEPLOY_SERVER_USER
+    - export GIT_VER=$(git describe --always ${PIPELINE_COMMIT_SHA} --)
+    - deploy-docs
+
+# stage: test_deploy
+deploy_docs_preview:
+  extends:
+    - .deploy_docs_template
+  rules:
+    - <<: *if-label-build_docs
+    - <<: *if-label-docs
+    - <<: *if-dev-push
+      changes: *patterns-docs-preview
+  needs:
+    - job: build_docs_html_fast
+      optional: true
+    - job: build_docs_html_full
+      optional: true
+    - job: build_docs_pdf
+      optional: true
+  variables:
+    TYPE: "preview"
+    # older branches use DOCS_DEPLOY_KEY, DOCS_SERVER, DOCS_SERVER_USER, DOCS_PATH for preview server so we keep these names for 'preview'
+    DOCS_DEPLOY_PRIVATEKEY: "$DOCS_DEPLOY_KEY"
+    DOCS_DEPLOY_SERVER: "$DOCS_SERVER"
+    DOCS_DEPLOY_SERVER_USER: "$DOCS_SERVER_USER"
+    DOCS_DEPLOY_PATH: "$DOCS_PATH"
+    DOCS_DEPLOY_URL_BASE: "https://$DOCS_PREVIEW_SERVER_URL/docs/esp-idf"
+
+# stage: post_deploy
+deploy_docs_production:
+  # The DOCS_PROD_* variables used by this job are "Protected" so these branches must all be marked "Protected" in Gitlab settings
+  extends:
+    - .deploy_docs_template
+  rules:
+    - <<: *if-protected-no_label
+  stage: post_deploy
+  dependencies:  # set dependencies to null to avoid missing artifacts issue
+  needs: # ensure runs after push_to_github succeeded
+    - build_docs_html_full
+    - build_docs_pdf
+    - job: push_to_github
+      artifacts: false
+  variables:
+    TYPE: "preview"
+    DOCS_DEPLOY_PRIVATEKEY: "$DOCS_PROD_DEPLOY_KEY"
+    DOCS_DEPLOY_SERVER: "$DOCS_PROD_SERVER"
+    DOCS_DEPLOY_SERVER_USER: "$DOCS_PROD_SERVER_USER"
+    DOCS_DEPLOY_PATH: "$DOCS_PROD_PATH"
+    DOCS_DEPLOY_URL_BASE: "https://docs.espressif.com/projects/esp-idf"
+    DEPLOY_STABLE: 1
+
+check_doc_links:
+  extends:
+    - .build_docs_template
+    - .rules:protected
+  stage: post_deploy
+  tags: ["build", "amd64", "internet"]
+  artifacts:
+    when: always
+    paths:
+      - docs/_build/*/*/*.txt
+      - docs/_build/*/*/linkcheck/*.txt
+    expire_in: 1 week
+  allow_failure: true
+  script:
+    - cd docs
+    - build-docs -t $DOCTGT -l $DOCLANG linkcheck

.gitlab/ci/host-test.yml

@@ -0,0 +1,410 @@
+.host_test_template:
+  extends: .rules:test:host_test
+  stage: host_test
+  image: $ESP_ENV_IMAGE
+  tags:
+    - host_test
+  dependencies: []
+  # run host_test jobs immediately, only after upload cache
+  needs:
+    - job: upload-pip-cache
+      optional: true
+      artifacts: false
+    - job: upload-submodules-cache
+      optional: true
+      artifacts: false
+
+test_nvs_on_host:
+  extends: .host_test_template
+  script:
+    - cd components/nvs_flash/test_nvs_host
+    - make test
+
+test_nvs_coverage:
+  extends:
+    - .host_test_template
+    - .rules:labels:nvs_coverage
+  artifacts:
+    paths:
+      - components/nvs_flash/test_nvs_host/coverage_report
+    expire_in: 1 week
+  script:
+    - cd components/nvs_flash/test_nvs_host
+    - make coverage_report
+  # the 'long' host tests take approx 11 hours on our current runners. Adding some margin here for possible CPU contention
+  timeout: 18 hours
+
+test_partition_table_on_host:
+  extends: .host_test_template
+  tags:
+    - build
+  script:
+    - cd components/partition_table/test_gen_esp32part_host
+    - ./gen_esp32part_tests.py
+
+test_wl_on_host:
+  extends: .host_test_template
+  artifacts:
+    paths:
+      - components/wear_levelling/test_wl_host/coverage_report.zip
+    expire_in: 1 week
+  script:
+    - cd components/wear_levelling/test_wl_host
+    - make test
+
+test_fatfs_on_host:
+  extends: .host_test_template
+  script:
+    - cd components/fatfs/test_fatfs_host/
+    - make test
+
+test_ldgen_on_host:
+  extends: .host_test_template
+  script:
+    - cd tools/ldgen/test
+    - export PYTHONPATH=$PYTHONPATH:..
+    - python -m unittest
+  variables:
+    LC_ALL: C.UTF-8
+
+test_reproducible_build:
+  extends: .host_test_template
+  script:
+    - ./tools/ci/test_reproducible_build.sh
+  artifacts:
+    when: on_failure
+    paths:
+      - "**/sdkconfig"
+      - "**/build*/*.bin"
+      - "**/build*/*.elf"
+      - "**/build*/*.map"
+      - "**/build*/flasher_args.json"
+      - "**/build*/*.bin"
+      - "**/build*/bootloader/*.bin"
+      - "**/build*/partition_table/*.bin"
+    expire_in: 1 week
+
+.host_fuzzer_test_template:
+  extends:
+    - .host_test_template
+    - .rules:labels:fuzzer_test-weekend_test
+  image: $AFL_FUZZER_TEST_IMAGE
+  artifacts:
+    when: always
+    paths:
+      - ${FUZZER_TEST_DIR}/out/crashes
+      - ${FUZZER_TEST_DIR}/fuzz_output.log
+    expire_in: 1 week
+  script:
+    - export AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES=1 && export AFL_SKIP_CPUFREQ=1
+    - cd ${FUZZER_TEST_DIR}
+    # run AFL fuzzer for one hour
+    - ( ( make ${FUZZER_PARAMS} fuzz | tee fuzz_output.log | grep -v '\(Fuzzing test case\|Entering queue cycle\)' ) || pkill sleep ) &
+    - ( sleep 3600 || mkdir -p out/crashes/env_failed ) && pkill afl-fuz
+    # check no crashes found
+    - test -z "$(ls out/crashes/)" || exit 1
+
+test_lwip_dns_fuzzer_on_host:
+  extends: .host_fuzzer_test_template
+  variables:
+    FUZZER_TEST_DIR: components/lwip/test_afl_host
+    FUZZER_PARAMS: MODE=dns
+
+test_lwip_dhcp_fuzzer_on_host:
+  extends: .host_fuzzer_test_template
+  variables:
+    FUZZER_TEST_DIR: components/lwip/test_afl_host
+    FUZZER_PARAMS: MODE=dhcp_client
+
+test_lwip_dhcps_fuzzer_on_host:
+  extends: .host_fuzzer_test_template
+  variables:
+    FUZZER_TEST_DIR: components/lwip/test_afl_host
+    FUZZER_PARAMS: MODE=dhcp_server
+
+test_spiffs_on_host:
+  extends: .host_test_template
+  script:
+    - cd components/spiffs/test_spiffs_host/
+    - make test
+    - cd ../test_spiffsgen
+    - ./test_spiffsgen.py
+
+test_fatfsgen_on_host:
+  extends: .host_test_template
+  script:
+    - cd components/fatfs/test_fatfsgen/
+    - ./test_fatfsgen.py
+    - ./test_wl_fatfsgen.py
+    - ./test_fatfsparse.py
+
+test_multi_heap_on_host:
+  extends: .host_test_template
+  script:
+    - cd components/heap/test_multi_heap_host
+    - ./test_all_configs.sh
+
+test_certificate_bundle_on_host:
+  extends: .host_test_template
+  tags:
+    - build
+  script:
+    - cd components/mbedtls/esp_crt_bundle/test_gen_crt_bundle/
+    - ./test_gen_crt_bundle.py
+
+test_confserver:
+  extends: .host_test_template
+  script:
+    - cd tools/kconfig_new/test/confserver
+    - ./test_confserver.py
+
+test_gen_kconfig_doc:
+  extends: .host_test_template
+  script:
+    - cd tools/kconfig_new/test/gen_kconfig_doc/
+    - ./test_target_visibility.py
+    - ./test_kconfig_out.py
+
+test_confgen:
+  extends: .host_test_template
+  script:
+    - cd tools/kconfig_new/test/confgen/
+    - ./test_confgen.py
+
+test_idf_monitor:
+  extends: .host_test_template
+  artifacts:
+    # save artifacts always in order to access results which were retried without consequent failure
+    when: always
+    paths:
+      - tools/test_idf_monitor/outputs/*
+    expire_in: 1 week
+  script:
+    - eval $($IDF_PATH/tools/idf_tools.py export)
+    - cd ${IDF_PATH}/tools/test_idf_monitor
+    - ./run_test_idf_monitor.py
+
+test_idf_size:
+  extends: .host_test_template
+  artifacts:
+    when: on_failure
+    paths:
+      - tools/test_idf_size/output
+      - tools/test_idf_size/.coverage
+    expire_in: 1 week
+  script:
+    - cd ${IDF_PATH}/tools/test_idf_size
+    - ./test.sh
+
+test_idf_py:
+  extends: .host_test_template
+  variables:
+    LC_ALL: C.UTF-8
+  script:
+    - cd ${IDF_PATH}/tools/test_idf_py
+    - ./test_idf_py.py
+
+# Test for create virtualenv. It must be invoked from Python, not from virtualenv.
+# Use docker image system python without any extra dependencies
+test_idf_tools:
+  extends:
+    - .host_test_template
+    - .before_script_minimal
+  artifacts:
+    when: on_failure
+    paths:
+      - tools/tools.new.json
+    expire_in: 1 week
+  image:
+    name: $ESP_ENV_IMAGE
+    entrypoint: [""]  # use system python3. no extra pip package installed
+  script:
+    # Tools must be downloaded for testing
+    - python3 ${IDF_PATH}/tools/idf_tools.py download
+    - cd ${IDF_PATH}/tools/test_idf_tools
+    - python3 -m pip install jsonschema
+    - python3 ./test_idf_tools.py
+    - python3 ./test_idf_tools_python_env.py
+
+.test_efuse_table_on_host_template:
+  extends: .host_test_template
+  variables:
+    IDF_TARGET: "esp32"
+  artifacts:
+    when: on_failure
+    paths:
+      - components/efuse/${IDF_TARGET}/esp_efuse_table.c
+    expire_in: 1 week
+  script:
+    - cd ${IDF_PATH}/components/efuse/
+    - ./efuse_table_gen.py -t "${IDF_TARGET}" ${IDF_PATH}/components/efuse/${IDF_TARGET}/esp_efuse_table.csv
+    - git diff --exit-code -- ${IDF_TARGET}/esp_efuse_table.c || { echo 'Differences found for ${IDF_TARGET} target. Please run make efuse_common_table or idf.py efuse-common-table and commit the changes.'; exit 1; }
+    - cd ${IDF_PATH}/components/efuse/test_efuse_host
+    - ./efuse_tests.py
+
+test_efuse_table_on_host_esp32:
+  extends: .test_efuse_table_on_host_template
+
+test_efuse_table_on_host_esp32s2:
+  extends: .test_efuse_table_on_host_template
+  variables:
+    IDF_TARGET: esp32s2
+
+test_efuse_table_on_host_esp32s3:
+  extends: .test_efuse_table_on_host_template
+  variables:
+    IDF_TARGET: esp32s3
+
+test_efuse_table_on_host_esp32c3:
+  extends: .test_efuse_table_on_host_template
+  variables:
+    IDF_TARGET: esp32c3
+
+test_efuse_table_on_host_esp32h2:
+  extends: .test_efuse_table_on_host_template
+  variables:
+    IDF_TARGET: esp32h2
+
+test_logtrace_proc:
+  extends: .host_test_template
+  artifacts:
+    when: on_failure
+    paths:
+      - tools/esp_app_trace/test/logtrace/output
+      - tools/esp_app_trace/test/logtrace/.coverage
+    expire_in: 1 week
+  script:
+    - cd ${IDF_PATH}/tools/esp_app_trace/test/logtrace
+    - ./test.sh
+
+test_sysviewtrace_proc:
+  extends: .host_test_template
+  artifacts:
+    when: on_failure
+    paths:
+      - tools/esp_app_trace/test/sysview/output
+      - tools/esp_app_trace/test/sysview/.coverage
+    expire_in: 1 week
+  script:
+    - cd ${IDF_PATH}/tools/esp_app_trace/test/sysview
+    - ./test.sh
+
+test_mkdfu:
+  extends: .host_test_template
+  variables:
+    LC_ALL: C.UTF-8
+  script:
+    - cd ${IDF_PATH}/tools/test_mkdfu
+    - ./test_mkdfu.py
+
+test_mkuf2:
+  extends: .host_test_template
+  script:
+    - cd ${IDF_PATH}/tools/test_mkuf2
+    - ./test_mkuf2.py
+
+test_autocomplete:
+  extends:
+    - .host_test_template
+  artifacts:
+    when: on_failure
+    paths:
+      - ${IDF_PATH}/*.out
+    expire_in: 1 week
+  script:
+    - ${IDF_PATH}/tools/ci/test_autocomplete.py
+
+test_detect_python:
+  extends:
+    - .host_test_template
+  script:
+    - cd ${IDF_PATH}
+    - shellcheck -s sh tools/detect_python.sh
+    - shellcheck -s bash tools/detect_python.sh
+    - shellcheck -s dash tools/detect_python.sh
+    - "bash -c '. tools/detect_python.sh && echo Our Python: ${ESP_PYTHON?Python is not set}'"
+    - "dash -c '. tools/detect_python.sh && echo Our Python: ${ESP_PYTHON?Python is not set}'"
+    - "zsh -c '. tools/detect_python.sh && echo Our Python: ${ESP_PYTHON?Python is not set}'"
+    - "fish -c 'source tools/detect_python.fish && echo Our Python: $ESP_PYTHON'"
+
+test_split_path_by_spaces:
+  extends: .host_test_template
+  script:
+    - cd ${IDF_PATH}/tools
+    - python -m unittest split_paths_by_spaces.py
+
+test_nvs_page:
+  extends: .host_test_template
+  script:
+    - cd ${IDF_PATH}/components/nvs_flash/host_test/nvs_page_test
+    - idf.py build
+    - build/test_nvs_page_host.elf
+
+test_mqtt_on_host:
+  extends: .host_test_template
+  script:
+    - cd ${IDF_PATH}/components/mqtt/host_test
+    - idf.py build
+    - LSAN_OPTIONS=verbosity=1:log_threads=1 build/host_mqtt_client_test.elf
+
+test_log:
+  extends: .host_test_template
+  script:
+    - cd ${IDF_PATH}/components/log/host_test/log_test
+    - idf.py build
+    - build/test_log_host.elf
+
+test_esp_event:
+  extends: .host_test_template
+  script:
+    - cd ${IDF_PATH}/components/esp_event/host_test/esp_event_unit_test
+    - idf.py build
+    - build/test_esp_event_host.elf
+
+test_eh_frame_parser:
+  extends: .host_test_template
+  script:
+    - cd ${IDF_PATH}/components/esp_system/test_eh_frame_parser
+    - make
+    - ./eh_frame_test
+
+test_rom_on_linux_works:
+  extends: .host_test_template
+  script:
+    - cd ${IDF_PATH}/components/esp_rom/host_test/rom_test
+    - idf.py build
+    - build/test_rom_host.elf
+
+test_linux_example:
+  extends: .host_test_template
+  script:
+    - cd ${IDF_PATH}/examples/build_system/cmake/linux_host_app
+    - idf.py build
+    - timeout 5 ./build/linux_host_app.elf >test.log || true
+    - grep "Restarting" test.log
+
+test_partition_api_host:
+  extends: .host_test_template
+  script:
+    - cd ${IDF_PATH}/components/esp_partition/host_test/partition_api_test
+    - idf.py build
+    - timeout 5 ./build/partition_api_test.elf >test.log
+    - grep " 0 Failures" test.log
+
+test_gen_soc_caps_kconfig:
+  extends: .host_test_template
+  script:
+    - cd ${IDF_PATH}/tools/gen_soc_caps_kconfig/
+    - ./test/test_gen_soc_caps_kconfig.py
+
+test_pytest_qemu:
+  extends:
+    - .host_test_template
+    - .before_script_build_jobs
+  image: $QEMU_IMAGE
+  script:
+    - run_cmd python tools/ci/ci_build_apps.py . -vv
+      --target esp32
+      --pytest-apps
+      -m qemu
+    - pytest --target esp32 -m qemu --embedded-services idf,qemu

.gitlab/ci/integration_test.yml

@@ -0,0 +1,69 @@
+# generate dynamic integration pipeline by `idf-integration-ci` project
+
+.patterns-integration_test: &patterns-integration_test
+  # add all possible patterns to make sure `gen_integration_pipeline` can be triggered.
+  # fine-grained control will be done while generating the pipeline
+  # find `patterns` in `idf-integration-ci` project
+  - "components/**/*"
+  - "tools/**/*"
+  - ".gitlab-ci.yml"
+  - ".gitlab/ci/common.yml"
+  - ".gitlab/ci/integration_test.yml"
+  - ".gitmodules"
+  - "CMakeLists.txt"
+  - "install.sh"
+  - "export.sh"
+  - "Kconfig"
+  - "sdkconfig.rename"
+
+# Simplify the rules
+.integration_test_rules:
+  rules:
+    - if: '$CI_PIPELINE_SOURCE != "merge_request_event"'
+    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
+      changes: *patterns-integration_test
+    # support trigger by ci labels
+    - if: '$CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*target_test(?:,[^,\n\r]+)*$/i'
+    - if: '$CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*integration_test(?:,[^,\n\r]+)*$/i'
+    - if: '$CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*build(?:,[^,\n\r]+)*$/i'
+
+gen_integration_pipeline:
+  extends:
+    - .before_script_minimal
+    - .integration_test_rules
+  image: ${CI_INTEGRATION_ASSIGN_ENV}
+  stage: assign_test
+  cache: []
+  tags:
+    - assign_test
+  variables:
+    SUBMODULES_TO_FETCH: "none"
+    GIT_LFS_SKIP_SMUDGE: 1
+  needs:
+    - job: fast_template_app
+      artifacts: false
+      optional: true
+  artifacts:
+    paths:
+      - idf-integration-ci/child_pipeline/
+    expire_in: 2 weeks
+  script:
+    - add_gitlab_ssh_keys
+    - retry_failed git clone ${CI_GEN_INTEGRATION_PIPELINE_REPO} idf-integration-ci
+    - python $CHECKOUT_REF_SCRIPT idf-integration-ci idf-integration-ci
+    - cd idf-integration-ci
+    - python tools/generate_child_pipeline.py -o child_pipeline/
+
+child_integration_test_pipeline:
+  extends:
+    - .integration_test_rules
+  stage: assign_test
+  needs:
+    - gen_integration_pipeline
+  trigger:
+    include:
+      - artifact: idf-integration-ci/child_pipeline/pipeline.yml
+        job: gen_integration_pipeline
+    forward:
+      yaml_variables: false
+    strategy: depend

.gitlab/ci/pre_check.yml

@@ -0,0 +1,170 @@
+.pre_check_template:
+  stage: pre_check
+  image: $ESP_ENV_IMAGE
+  tags:
+    - host_test
+  dependencies: []
+
+.check_pre_commit_template:
+  extends:
+    - .pre_check_template
+    - .before_script_minimal
+  image: $PRE_COMMIT_IMAGE
+
+check_pre_commit_master_release:
+  extends:
+    - .check_pre_commit_template
+    - .rules:protected
+  script:
+    - git diff-tree --no-commit-id --name-only -r $PIPELINE_COMMIT_SHA | xargs pre-commit run --files
+    - pre-commit run --hook-stage post-commit validate-sbom-manifest
+
+check_pre_commit_MR:
+  extends:
+    - .check_pre_commit_template
+    - .rules:mr
+  script:
+    - python ${CI_PROJECT_DIR}/tools/ci/ci_get_mr_info.py files ${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME} | xargs pre-commit run --files
+    - pre-commit run --hook-stage post-commit validate-sbom-manifest
+
+check_version:
+  # Don't run this for feature/bugfix branches, so that it is possible to modify
+  # esp_idf_version.h in a branch before tagging the next version.
+  extends:
+    - .pre_check_template
+    - .rules:protected
+  script:
+    - export IDF_PATH=$PWD
+    - tools/ci/check_idf_version.sh
+
+check_rom_api_header:
+  extends: .pre_check_template
+  script:
+    - tools/ci/check_examples_rom_header.sh
+    - tools/ci/check_api_violation.sh
+
+test_check_kconfigs:
+  extends: .pre_check_template
+  artifacts:
+    when: on_failure
+    paths:
+      - components/*/Kconfig*.new
+      - examples/*/*/*/Kconfig*.new
+      - examples/*/*/*/*/Kconfig*.new
+      - tools/*/Kconfig*.new
+      - tools/*/*/Kconfig*.new
+      - tools/*/*/*/Kconfig*.new
+    expire_in: 1 week
+  script:
+    - python ${IDF_PATH}/tools/ci/test_check_kconfigs.py
+
+check_blobs:
+  extends: .pre_check_template
+  tags:
+    - build
+  variables:
+    SUBMODULES_TO_FETCH: "components/esp_wifi/lib;components/esp_phy/lib"
+  script:
+    # Check if Wi-Fi library header files match between IDF and the version used when compiling the libraries
+    - IDF_TARGET=esp32 $IDF_PATH/components/esp_wifi/test_md5/test_md5.sh
+    - IDF_TARGET=esp32s2 $IDF_PATH/components/esp_wifi/test_md5/test_md5.sh
+    - IDF_TARGET=esp32s3 $IDF_PATH/components/esp_wifi/test_md5/test_md5.sh
+    - IDF_TARGET=esp32c3 $IDF_PATH/components/esp_wifi/test_md5/test_md5.sh
+    # Check if Wi-Fi, PHY, BT blobs contain references to specific symbols
+    - bash $IDF_PATH/tools/ci/check_blobs.sh
+
+check_fuzzer_compilation:
+   extends: .pre_check_template
+   image: $AFL_FUZZER_TEST_IMAGE
+   script:
+     - cd ${IDF_PATH}/components/lwip/test_afl_host
+     - make MODE=dhcp_server
+     - make MODE=dhcp_client
+     - make MODE=dns
+
+check_public_headers:
+  extends:
+    - .pre_check_template
+    - .rules:build
+  tags:
+    - build
+  script:
+    - IDF_TARGET=esp32 python tools/ci/check_public_headers.py --jobs 4 --prefix xtensa-esp32-elf-
+    - IDF_TARGET=esp32s2 python tools/ci/check_public_headers.py --jobs 4 --prefix xtensa-esp32s2-elf-
+    - IDF_TARGET=esp32s3 python tools/ci/check_public_headers.py --jobs 4 --prefix xtensa-esp32s3-elf-
+    - IDF_TARGET=esp32c3 python tools/ci/check_public_headers.py --jobs 4 --prefix riscv32-esp-elf-
+    - IDF_TARGET=esp32c2 python tools/ci/check_public_headers.py --jobs 4 --prefix riscv32-esp-elf-
+
+check_chip_support_components:
+  extends:
+    - .pre_check_template
+    - .rules:build
+  tags:
+    - build
+  artifacts:
+    when: on_failure
+    paths:
+      - esp_hw_support_part.h
+      - bootloader_support_part.h
+    expire_in: 1 week
+  script:
+    - python tools/ci/check_soc_headers_leak.py
+    - find ${IDF_PATH}/components/soc/*/include/soc/ -name "*_struct.h" -print0 | xargs -0 -n1 ./tools/ci/check_soc_struct_headers.py
+    - tools/ci/check_esp_memory_utils_headers.sh
+
+check_esp_err_to_name:
+  extends:
+    - .pre_check_template
+    - .rules:build
+  tags:
+    - build
+  artifacts:
+    when: on_failure
+    paths:
+      - components/esp_common/esp_err_to_name.c
+    expire_in: 1 week
+  script:
+    - cd ${IDF_PATH}/tools/
+    - ./gen_esp_err_to_name.py
+    - git diff --exit-code -- ../components/esp_common/src/esp_err_to_name.c || { echo 'Differences found. Please run gen_esp_err_to_name.py and commit the changes.'; exit 1; }
+
+check_esp_system:
+  extends:
+    - .pre_check_template
+    - .rules:build
+  tags:
+    - build
+  script:
+    - python components/esp_system/check_system_init_priorities.py
+
+# For release tag pipelines only, make sure the tag was created with 'git tag -a' so it will update
+# the version returned by 'git describe'
+check_version_tag:
+  extends:
+    - .pre_check_template
+    - .rules:tag:release
+  script:
+    - (git cat-file -t $CI_COMMIT_REF_NAME | grep tag) || (echo "ESP-IDF versions must be annotated tags." && exit 1)
+
+check_artifacts_expire_time:
+  extends: .pre_check_template
+  script:
+    # check if we have set expire time for all artifacts
+    - python tools/ci/check_artifacts_expire_time.py
+
+check_commit_msg:
+  extends: .pre_check_template
+  script:
+    - git status
+    - git log -n10 --oneline ${PIPELINE_COMMIT_SHA}
+    # commit start with "WIP: " need to be squashed before merge
+    - 'git log --pretty=%s origin/master..${PIPELINE_COMMIT_SHA} -- | grep -i "^WIP:" && exit 1 || exit 0'
+
+check_test_scripts_build_test_rules:
+  extends:
+    - .pre_check_template
+    - .before_script_build_jobs
+  script:
+    # required pytest related packages
+    - run_cmd bash install.sh --enable-pytest
+    - python tools/ci/check_build_test_rules.py check-test-scripts examples/ tools/test_apps components

.gitlab/ci/static-code-analysis.yml

@@ -0,0 +1,129 @@
+# pre_check stage
+clang_tidy_check:
+  extends:
+    - .pre_check_template
+    - .rules:patterns:clang_tidy
+  image: ${CLANG_STATIC_ANALYSIS_IMAGE}
+  artifacts:
+    paths:
+      - $OUTPUT_DIR
+    when: always
+    expire_in: 1 day
+  variables:
+    CLANG_TIDY_DIRS_TXT: ${CI_PROJECT_DIR}/tools/ci/clang_tidy_dirs.txt
+    RULES_FILE: ${CI_PROJECT_DIR}/tools/ci/static-analysis-rules.yml
+    OUTPUT_DIR: ${CI_PROJECT_DIR}/clang_tidy_reports
+  script:
+    - internal_pip_install $CLANG_TIDY_RUNNER_PROJ pyclang
+    - export PATH=$PATH:$(python -c "import sys; print(sys.executable.rsplit('/', 1)[0])")
+    - dirs=$(cat ${CLANG_TIDY_DIRS_TXT} | while read line; do echo ${CI_PROJECT_DIR}/${line}; done | xargs)
+    - run_cmd idf_clang ${dirs}
+      --output-path ${OUTPUT_DIR}
+      --limit-file ${RULES_FILE}
+      --xtensa-include-dir
+      --run-clang-tidy-py ${RUN_CLANG_TIDY_PY}
+
+check_pylint:
+  extends:
+    - .pre_check_template
+    - .rules:patterns:python-files
+    - .before_script_minimal
+  image: $SONARQUBE_SCANNER_IMAGE
+  artifacts:
+    when: always
+    paths:
+      - pylint-report.txt
+    expire_in: 1 week
+  script:
+    - export PYTHONPATH="$IDF_PATH/tools:$IDF_PATH/tools/ci/python_packages:$PYTHONPATH"
+    - |
+      if [ -n "$CI_MERGE_REQUEST_IID" ]; then
+        export files=$(python ${CI_PROJECT_DIR}/tools/ci/ci_get_mr_info.py files ${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME} | grep ".py");
+      else
+        export files=$(find . -iname "*.py" -print);
+      fi
+    - pylint --rcfile=.pylintrc $files -r n --output-format=parseable > pylint-report.txt || exit 0
+
+# build stage
+# Sonarqube related jobs put here for this reason:
+# Here we have two jobs. code_quality_check and code_quality_report.
+#
+# code_quality_check will analyze the code changes between your MR and
+# code repo stored in sonarqube server. The analysis result is only shown in
+# the comments under this MR and won't be transferred to the server.
+#
+# code_quality_report will analyze and transfer both of the newly added code
+# and the analysis result to the server.
+#
+# Put in the front to ensure that the newly merged code can be stored in
+# sonarqube server ASAP, in order to avoid reporting unrelated code issues
+.sonar_scan_template:
+  stage: build
+  image:
+    name: $SONARQUBE_SCANNER_IMAGE
+  before_script:
+    - source tools/ci/utils.sh
+    - export PYTHONPATH="$CI_PROJECT_DIR/tools:$CI_PROJECT_DIR/tools/ci/python_packages:$PYTHONPATH"
+    - fetch_submodules
+    # Exclude the submodules, all paths ends with /**
+    - submodules=$(get_all_submodules)
+    # get all exclude paths specified in tools/ci/sonar_exclude_list.txt | ignore lines start with # | xargs | replace all <space> to <comma>
+    - custom_excludes=$(cat $CI_PROJECT_DIR/tools/ci/sonar_exclude_list.txt | grep -v '^#' | xargs | sed -e 's/ /,/g')
+    # Exclude the report dir as well
+    - export EXCLUSIONS="$custom_excludes,$submodules"
+    - export SONAR_SCANNER_OPTS="-Xmx2048m"
+
+  variables:
+    GIT_DEPTH: 0
+    REPORT_PATTERN: clang_tidy_reports/*.txt
+  artifacts:
+    when: always
+    paths:
+      - $REPORT_PATTERN
+    expire_in: 1 week
+  tags:
+    - host_test
+  dependencies:  # Here is not a hard dependency relationship, could be skipped when only python files changed. so we do not use "needs" here.
+    - clang_tidy_check
+    - check_pylint
+
+code_quality_check:
+  extends:
+    - .sonar_scan_template
+    - .rules:patterns:static-code-analysis-preview
+  allow_failure: true  # since now it's using exit code to indicate the code analysis result,
+                       # we don't want to block ci when critical issues founded
+  script:
+    - export CI_MERGE_REQUEST_COMMITS=$(python ${CI_PROJECT_DIR}/tools/ci/ci_get_mr_info.py commits ${CI_COMMIT_REF_NAME} | tr '\n' ',')
+    # test if this branch have merge request, if not, exit 0
+    - test -n "$CI_MERGE_REQUEST_IID" || exit 0
+    - test -n "$CI_MERGE_REQUEST_COMMITS" || exit 0
+    - sonar-scanner
+      -Dsonar.analysis.mode=preview
+      -Dsonar.branch.name=$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME
+      -Dsonar.cxx.clangtidy.reportPath=$REPORT_PATTERN
+      -Dsonar.exclusions=$EXCLUSIONS
+      -Dsonar.gitlab.ci_merge_request_iid=$CI_MERGE_REQUEST_IID
+      -Dsonar.gitlab.commit_sha=$CI_MERGE_REQUEST_COMMITS
+      -Dsonar.gitlab.merge_request_discussion=true
+      -Dsonar.gitlab.ref_name=$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME
+      -Dsonar.host.url=$SONAR_HOST_URL
+      -Dsonar.login=$SONAR_LOGIN
+      -Dsonar.python.pylint.reportPath=pylint-report.txt
+
+code_quality_report:
+  extends:
+    - .sonar_scan_template
+    - .rules:protected
+  allow_failure: true  # since now it's using exit code to indicate the code analysis result,
+                       # we don't want to block ci when critical issues founded
+  script:
+    - sonar-scanner
+      -Dsonar.branch.name=$CI_COMMIT_REF_NAME
+      -Dsonar.cxx.clangtidy.reportPath=$REPORT_PATTERN
+      -Dsonar.exclusions=$EXCLUSIONS
+      -Dsonar.gitlab.commit_sha=$PIPELINE_COMMIT_SHA
+      -Dsonar.gitlab.ref_name=$CI_COMMIT_REF_NAME
+      -Dsonar.host.url=$SONAR_HOST_URL
+      -Dsonar.login=$SONAR_LOGIN
+      -Dsonar.python.pylint.reportPath=pylint-report.txt

.gitlab/ci/upload_cache.yml

@@ -0,0 +1,50 @@
+# pull only for most of the use cases for cache
+# only set "push" policy for the jobs under this file.
+# The cache would be updated when files matched specified patterns changes.
+
+.upload_cache_template:
+  stage: upload_cache
+  image: $ESP_ENV_IMAGE
+
+upload-pip-cache:
+  extends:
+    - .upload_cache_template
+    - .before_script_minimal
+    - .rules:patterns:python-cache
+  tags:
+    - $GEO
+    - cache
+  cache:
+    key: pip-cache
+    paths:
+      - .cache/pip
+    policy: push
+  script:
+    - rm -rf .cache/pip  # clear old packages
+    - bash install.sh --enable-ci --enable-pytest
+  parallel:
+    matrix:
+      - GEO: [ 'shiny',  'brew' ]
+
+upload-submodules-cache:
+  extends:
+    - .upload_cache_template
+    - .before_script_minimal
+    - .rules:patterns:submodule
+  tags:
+    - $GEO
+    - cache
+  cache:
+    key: submodule-cache
+    paths:
+      - .cache/submodule_archives
+    policy: push
+  script:
+    # use the default gitlab server
+    - unset LOCAL_GITLAB_HTTPS_HOST
+    - rm -rf .cache/submodule_archives  # clear old submodule archives
+    - add_gitlab_ssh_keys
+    - fetch_submodules
+  parallel:
+    matrix:
+      - GEO: [ 'shiny',  'brew' ]

.gitlab/merge_request_templates/Minimal_Template.md

@@ -0,0 +1,15 @@
+<!-- This Template states the absolute minimum for an MR.
+    If you want to have a more elaborate template or know why we have this structure,
+    please use the "Mixed Template" or consult the Wiki. -->
+<!-- Add the CI labels to trigger the appropriate tests (e.g. "unit_test_esp32") --><!-- Mandatory -->
+<!-- Make sure the commit message follows the Wiki about "Commit Messages" --><!-- Mandatory -->
+
+<!-- Add description of the change here --><!-- Mandatory -->
+
+## Related <!-- Optional -->
+<!-- Related Jira issues and Github issues -->
+
+## Release notes <!-- Mandatory -->
+<!-- Either state release notes or write "No release notes" -->
+
+<!-- ## Breaking change notes --><!-- Optional -->

.gitlab/merge_request_templates/Mixed Template.md

@@ -17,7 +17,6 @@ _This entire section can be deleted if all items are checked._
 * [ ] Release note entry if this is a new public feature, or a fix for an issue introduced in the previous release.
 * [ ] The commit log is clean and ready to merge.
 * [ ] All relevant CI jobs have been run, i.e. jobs which cover the code changed by the MR.
-* [ ] Pipeline with Python 3 has been successfully run, i.e. `@bot ...; with Python3`, if the MR adds or changes Python code.
 
 ---
 

.gitmodules

@@ -2,10 +2,26 @@
 # All the relative URL paths are intended to be GitHub ones
 # For Espressif's public projects please use '../../espressif/proj', not a '../proj'
 #
-
-[submodule "components/esptool_py/esptool"]
-	path = components/esptool_py/esptool
-	url = ../../espressif/esptool.git
+# Submodules SBOM information
+# ---------------------------
+# Submodules, which are used directly and not forked into espressif namespace should
+# contain SBOM information here. Other submodules should have the SBOM manifest file
+# included in the root of their project's repository.
+#
+# The sbom-hash entry records the submodule's checkout SHA as presented in git-tree
+# commit object. For example spiffs submodule
+#
+# $ git ls-tree HEAD components/spiffs/spiffs
+# 160000 commit 0dbb3f71c5f6fae3747a9d935372773762baf852	components/spiffs/spiffs
+#
+# The hash can be also obtained with git submodule command
+#
+# $ git submodule status components/spiffs/spiffs
+# 0dbb3f71c5f6fae3747a9d935372773762baf852 components/spiffs/spiffs (0.2-255-g0dbb3f71c5f6)
+#
+# The submodule SHA recorded here has to match with SHA, which is presented in git-tree.
+# This is checked by CI. Also please don't forget to update the submodule version
+# if you are changing the sbom-hash. This is important for SBOM generation.
 
 [submodule "components/bt/controller/lib_esp32"]
 	path = components/bt/controller/lib_esp32
@@ -14,39 +30,36 @@
 [submodule "components/bootloader/subproject/components/micro-ecc/micro-ecc"]
 	path = components/bootloader/subproject/components/micro-ecc/micro-ecc
 	url = ../../kmackay/micro-ecc.git
-
-[submodule "components/coap/libcoap"]
-	path = components/coap/libcoap
-	url = ../../obgm/libcoap.git
-
-[submodule "components/nghttp/nghttp2"]
-	path = components/nghttp/nghttp2
-	url = ../../nghttp2/nghttp2.git
-
-[submodule "components/libsodium/libsodium"]
-	path = components/libsodium/libsodium
-	url = ../../jedisct1/libsodium.git
+	sbom-version = 1.1
+	sbom-cpe = cpe:2.3:a:micro-ecc_project:micro-ecc:{}:*:*:*:*:*:*:*
+	sbom-supplier = Person: Ken MacKay
+	sbom-url = https://github.com/kmackay/micro-ecc
+	sbom-description = A small and fast ECDH and ECDSA implementation for 8-bit, 32-bit, and 64-bit processors
+	sbom-hash = 24c60e243580c7868f4334a1ba3123481fe1aa48
 
 [submodule "components/spiffs/spiffs"]
 	path = components/spiffs/spiffs
 	url = ../../pellepl/spiffs.git
+	sbom-version = 0.2-255-g0dbb3f71c5f6
+	sbom-supplier = Person: Peter Andersson
+	sbom-url = https://github.com/pellepl/spiffs
+	sbom-description = Wear-leveled SPI flash file system for embedded devices
+	sbom-hash = 0dbb3f71c5f6fae3747a9d935372773762baf852
 
 [submodule "components/json/cJSON"]
 	path = components/json/cJSON
 	url = ../../DaveGamble/cJSON.git
+	sbom-version = 1.7.17
+	sbom-cpe = cpe:2.3:a:cjson_project:cjson:{}:*:*:*:*:*:*:*
+	sbom-supplier = Person: Dave Gamble
+	sbom-url = https://github.com/DaveGamble/cJSON
+	sbom-description = Ultralightweight JSON parser in ANSI C
+	sbom-hash = 87d8f0961a01bf09bef98ff89bae9fdec42181ee
 
 [submodule "components/mbedtls/mbedtls"]
 	path = components/mbedtls/mbedtls
 	url = ../../espressif/mbedtls.git
 
-[submodule "components/asio/asio"]
-	path = components/asio/asio
-	url = ../../espressif/asio.git
-
-[submodule "components/expat/expat"]
-	path = components/expat/expat
-	url = ../../libexpat/libexpat.git
-
 [submodule "components/lwip/lwip"]
 	path = components/lwip/lwip
 	url = ../../espressif/esp-lwip.git
@@ -58,39 +71,67 @@
 [submodule "components/protobuf-c/protobuf-c"]
 	path = components/protobuf-c/protobuf-c
 	url = ../../protobuf-c/protobuf-c.git
+	sbom-version = 1.4.1
+	sbom-cpe = cpe:2.3:a:protobuf-c_project:protobuf-c:{}:*:*:*:*:*:*:*
+	sbom-supplier = Organization: protobuf-c community <https://groups.google.com/g/protobuf-c>
+	sbom-url = https://github.com/protobuf-c/protobuf-c
+	sbom-description = Protocol Buffers implementation in C
+	sbom-hash = abc67a11c6db271bedbb9f58be85d6f4e2ea8389
 
 [submodule "components/unity/unity"]
 	path = components/unity/unity
 	url = ../../ThrowTheSwitch/Unity.git
-
-[submodule "examples/build_system/cmake/import_lib/main/lib/tinyxml2"]
-	path = examples/build_system/cmake/import_lib/main/lib/tinyxml2
-	url = ../../leethomason/tinyxml2.git
+	sbom-version = v2.4.3-51-g7d2bf62b7e6a
+	sbom-supplier = Organization: ThrowTheSwitch community <http://www.throwtheswitch.org>
+	sbom-url = https://github.com/ThrowTheSwitch/Unity
+	sbom-description = Simple Unit Testing for C
+	sbom-hash = 7d2bf62b7e6afaf38153041a9d53c21aeeca9a25
 
 [submodule "components/bt/host/nimble/nimble"]
 	path = components/bt/host/nimble/nimble
 	url = ../../espressif/esp-nimble.git
 
-[submodule "components/cbor/tinycbor"]
-	path = components/cbor/tinycbor
-	url = ../../intel/tinycbor.git
-
 [submodule "components/esp_wifi/lib"]
 	path = components/esp_wifi/lib
 	url = ../../espressif/esp32-wifi-lib.git
 
-[submodule "components/tinyusb/tinyusb"]
-	path = components/tinyusb/tinyusb
-	url = ../../espressif/tinyusb.git
-
-[submodule "examples/peripherals/secure_element/atecc608_ecdsa/components/esp-cryptoauthlib"]
-	path = examples/peripherals/secure_element/atecc608_ecdsa/components/esp-cryptoauthlib
-	url = ../../espressif/esp-cryptoauthlib.git
-
 [submodule "components/cmock/CMock"]
 	path = components/cmock/CMock
 	url = ../../ThrowTheSwitch/CMock.git
+	sbom-version = v2.5.2-2-geeecc49ce8af
+	sbom-supplier = Organization: ThrowTheSwitch community <http://www.throwtheswitch.org>
+	sbom-url = https://github.com/ThrowTheSwitch/CMock
+	sbom-description = CMock - Mock/stub generator for C
+	sbom-hash = eeecc49ce8af123cf8ad40efdb9673e37b56230f
+
+[submodule "components/openthread/openthread"]
+	path = components/openthread/openthread
+	url = ../../espressif/openthread.git
 
 [submodule "components/bt/controller/lib_esp32c3_family"]
 	path = components/bt/controller/lib_esp32c3_family
 	url = ../../espressif/esp32c3-bt-lib.git
+
+[submodule "components/esp_phy/lib"]
+	path = components/esp_phy/lib
+	url = ../../espressif/esp-phy-lib.git
+
+[submodule "components/openthread/lib"]
+	path = components/openthread/lib
+	url = ../../espressif/esp-thread-lib.git
+
+[submodule "components/ieee802154/lib"]
+	path = components/ieee802154/lib
+	url = ../../espressif/esp-ieee802154-lib.git
+
+[submodule "components/bt/controller/lib_esp32h2/esp32h2-bt-lib"]
+	path = components/bt/controller/lib_esp32h2/esp32h2-bt-lib
+	url = ../../espressif/esp32h2-bt-lib.git
+
+[submodule "components/bt/controller/lib_esp32c2/esp32c2-bt-lib"]
+	path = components/bt/controller/lib_esp32c2/esp32c2-bt-lib
+	url = ../../espressif/esp32c2-bt-lib.git
+
+[submodule "components/heap/tlsf"]
+	path = components/heap/tlsf
+	url = ../../espressif/tlsf.git

.mypy.ini

@@ -0,0 +1,26 @@
+[mypy]
+
+# Specifies the Python version used to parse and check the target program
+python_version = 3.9
+
+# Disallows defining functions without type annotations or with incomplete type annotations
+# True => enforce type annotation in all function definitions
+disallow_untyped_defs = True
+
+# Shows a warning when returning a value with type Any from a function declared with a non- Any return type
+warn_return_any = True
+
+# Shows errors for missing return statements on some execution paths
+warn_no_return = True
+
+# Suppress error messages about imports that cannot be resolved
+# True => ignore all import errors
+ignore_missing_imports = True
+
+# Disallows defining functions with incomplete type annotations
+disallow_incomplete_defs = False
+
+# Directs what to do with imports when the imported module is found as a .py file and not part of the files,
+# modules and packages provided on the command line.
+# SKIP -> mypy checks only single file, not included imports
+follow_imports = skip

.pre-commit-config.yaml

@@ -1,9 +1,11 @@
 # See https://pre-commit.com for more information
 # See https://pre-commit.com/hooks.html for more hooks
 
+default_stages: [commit]
+
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v3.4.0
+    rev: v4.0.1
     hooks:
       - id: trailing-whitespace
         # note: whitespace exclusions use multiline regex, see https://pre-commit.com/#regular-expressions
@@ -12,31 +14,38 @@ repos:
         # 2 - any file matching *test*/*expected* (for host tests, if possible use this naming pattern always)
         # 3 - any directory named 'testdata'
         # 4 - IDF monitor test data
+        # 5 - protobuf auto-generated files
         exclude: &whitespace_excludes |
-            (?x)^(
-                .+\.(md|rst|map|bin)|
-                .+test.*\/.*expected.*|
-                .+\/testdata\/.+|
-                .+test_idf_monitor\/tests\/.+
-            )$
+          (?x)^(
+            .+\.(md|rst|map|bin)|
+            .+test.*\/.*expected.*|
+            .+\/testdata\/.+|
+            .+test_idf_monitor\/tests\/.+|
+            .*_pb2.py|
+            .*.pb-c.h|
+            .*.pb-c.c|
+            .*.yuv
+          )$
       - id: end-of-file-fixer
         exclude: *whitespace_excludes
       - id: check-executables-have-shebangs
-      - id: file-contents-sorter
-        files: 'tools/ci/executable-list.txt'
       - id: mixed-line-ending
         args: ['-f=lf']
       - id: double-quote-string-fixer
-  - repo: https://gitlab.com/pycqa/flake8
-    rev: 3.8.4
+  - repo: https://github.com/PyCQA/flake8
+    rev: 5.0.4
     hooks:
       - id: flake8
         args: ['--config=.flake8', '--tee', '--benchmark']
   - repo: https://github.com/pycqa/isort
-    rev: 5.6.4
+    rev: 5.11.5  # python 3.7 compatible
     hooks:
       - id: isort
         name: isort (python)
+        exclude: >
+          (?x)^(
+              .*_pb2.py
+          )$
   - repo: local
     hooks:
       - id: check-executables
@@ -73,5 +82,90 @@ repos:
         name: Validate Codeowner File
         entry: tools/ci/check_codeowners.py ci-check
         language: python
+        always_run: true
         files: '\.gitlab/CODEOWNERS'
         pass_filenames: false
+      - id: check-rules-yml
+        name: Check rules.yml all rules have at lease one job applied, all rules needed exist
+        entry: tools/ci/check_rules_yml.py
+        language: python
+        files: '\.gitlab/ci/.+\.yml|\.gitlab-ci.yml'
+        pass_filenames: false
+        additional_dependencies:
+          - PyYAML == 5.3.1
+      - id: check-generated-rules
+        name: Check rules are generated (based on .gitlab/ci/dependencies/dependencies.yml)
+        entry: .gitlab/ci/dependencies/generate_rules.py
+        language: python
+        files: '\.gitlab/ci/dependencies/.+|\.gitlab/ci/rules\.yml'
+        pass_filenames: false
+        additional_dependencies:
+          - PyYAML == 5.3.1
+      - id: mypy-check
+        name: Check type annotations in python files
+        entry: tools/ci/check_type_comments.py
+        additional_dependencies:
+          - 'mypy==0.940'
+          - 'mypy-extensions==0.4.3'
+          - 'types-setuptools==57.4.14'
+          - 'types-PyYAML==0.1.9'
+        exclude: >
+          (?x)^(
+              .*_pb2.py
+          )$
+        language: python
+        types: [python]
+      - id: check-requirement-files
+        name: Check requirement files
+        entry: tools/ci/check_requirement_files.py
+        additional_dependencies:
+          - 'jsonschema'
+        language: python
+        files: 'tools/requirements.+|tools/requirements/.+'
+        pass_filenames: false
+      - id: check-tools-files-patterns
+        name: Check tools dir files patterns
+        entry: tools/ci/check_tools_files_patterns.py
+        language: python
+        files: '^tools/.+'
+        additional_dependencies:
+          - PyYAML == 5.3.1
+        pass_filenames: false
+      - id: check-generated-soc-caps-kconfig
+        name: Check soc caps kconfig files are generated (based on components/soc/IDF_TARGET/include/soc/soc_caps.h)
+        entry: tools/gen_soc_caps_kconfig/gen_soc_caps_kconfig.py -d 'components/soc/*/include/soc/' 'components/esp_rom/*/'
+        language: python
+        files: 'components/soc/.+/include/soc/.+_caps\.h|components/esp_rom/.+/.+_caps\.h|kconfig\.soc_caps.in'
+        pass_filenames: false
+        additional_dependencies:
+          - pyparsing
+      - id: check-all-apps-readmes
+        name: Check if all apps readme files match given .build-test-rules.yml files. Modify the supported target tables
+        entry: tools/ci/check_build_test_rules.py check-readmes
+        language: python
+        require_serial: true
+        files: 'tools/test_apps/.+|examples/.+|components/.+'
+        additional_dependencies:
+          - idf_build_apps
+      - id: sort-build-test-rules-ymls
+        name: sort .build-test-rules.yml files
+        entry: tools/ci/check_build_test_rules.py sort-yaml
+        language: python
+        files: '\.build-test-rules\.yml'
+        additional_dependencies:
+          - ruamel.yaml
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.0.1
+    hooks:
+      - id: file-contents-sorter
+        files: 'tools\/ci\/(executable-list\.txt|mypy_ignore_list\.txt|check_copyright_ignore\.txt)'
+  - repo: https://github.com/espressif/check-copyright/
+    rev: v1.0.3
+    hooks:
+      - id: check-copyright
+        args: ['--ignore', 'tools/ci/check_copyright_ignore.txt', '--config', 'tools/ci/check_copyright_config.yaml']
+  - repo: https://github.com/espressif/esp-idf-sbom.git
+    rev: v0.13.0
+    hooks:
+      - id: validate-sbom-manifest
+        stages: [post-commit]

.pylintrc

@@ -147,6 +147,12 @@ disable=print-statement,
         too-few-public-methods,
         too-many-locals,
         bad-super-call,  # since we still haven't drop python2 support
+        too-many-nested-blocks,
+        too-many-branches,
+        too-many-statements,
+        ungrouped-imports,  # since we have isort in pre-commit
+        no-name-in-module,  # since we have flake8 to check this
+        too-many-instance-attributes,
 
 # Enable the message, report, category or checker with the given id(s). You can
 # either give multiple identifier separated by comma (,) or put this option
@@ -244,7 +250,7 @@ missing-member-hint-distance=1
 missing-member-max-choices=1
 
 # List of decorators that change the signature of a decorated function.
-signature-mutators=
+signature-mutators=ttfw_idf.idf_example_test,ttfw_idf.idf_unit_test,ttfw_idf.idf_custom_test,ttfw_idf.idf_component_unit_test
 
 
 [SPELLING]

CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.5)
+cmake_minimum_required(VERSION 3.16)
 project(esp-idf C CXX ASM)
 
 if(CMAKE_CURRENT_LIST_DIR STREQUAL CMAKE_SOURCE_DIR)
@@ -7,11 +7,8 @@ if(CMAKE_CURRENT_LIST_DIR STREQUAL CMAKE_SOURCE_DIR)
     "again.")
 endif()
 
-unset(compile_options)
-unset(c_compile_options)
-unset(cxx_compile_options)
-unset(compile_definitions)
-unset(link_options)
+# Variables compile_options, c_compile_options, cxx_compile_options, compile_definitions, link_options shall
+# not be unset as they may already contain flags, set by toolchain-TARGET.cmake files.
 
 # Add the following build specifications here, since these seem to be dependent
 # on config values on the root Kconfig.
@@ -20,7 +17,9 @@ if(NOT BOOTLOADER_BUILD)
 
     if(CONFIG_COMPILER_OPTIMIZATION_SIZE)
         list(APPEND compile_options "-Os")
-        list(APPEND compile_options "-freorder-blocks")
+        if(CMAKE_C_COMPILER_ID MATCHES "GNU")
+            list(APPEND compile_options "-freorder-blocks")
+        endif()
     elseif(CONFIG_COMPILER_OPTIMIZATION_DEFAULT)
         list(APPEND compile_options "-Og")
     elseif(CONFIG_COMPILER_OPTIMIZATION_NONE)
@@ -33,7 +32,9 @@ else()  # BOOTLOADER_BUILD
 
     if(CONFIG_BOOTLOADER_COMPILER_OPTIMIZATION_SIZE)
         list(APPEND compile_options "-Os")
-        list(APPEND compile_options "-freorder-blocks")
+        if(CMAKE_C_COMPILER_ID MATCHES "GNU")
+            list(APPEND compile_options "-freorder-blocks")
+        endif()
     elseif(CONFIG_BOOTLOADER_COMPILER_OPTIMIZATION_DEBUG)
         list(APPEND compile_options "-Og")
     elseif(CONFIG_BOOTLOADER_COMPILER_OPTIMIZATION_NONE)
@@ -57,23 +58,55 @@ else()
     list(APPEND link_options "-fno-rtti")           # used to invoke correct multilib variant (no-rtti) during linking
 endif()
 
-if(CONFIG_COMPILER_DISABLE_GCC8_WARNINGS)
-    list(APPEND compile_options "-Wno-parentheses"
-                                "-Wno-sizeof-pointer-memaccess"
-                                "-Wno-clobbered")
-
-    list(APPEND compile_options "-Wno-format-overflow"
-                                "-Wno-stringop-truncation"
-                                "-Wno-misleading-indentation"
-                                "-Wno-cast-function-type"
-                                "-Wno-implicit-fallthrough"
-                                "-Wno-unused-const-variable"
-                                "-Wno-switch-unreachable"
-                                "-Wno-format-truncation"
-                                "-Wno-memset-elt-size"
-                                "-Wno-int-in-bool-context")
+if(CONFIG_COMPILER_SAVE_RESTORE_LIBCALLS)
+    list(APPEND compile_options "-msave-restore")
 endif()
 
+if(CMAKE_C_COMPILER_ID MATCHES "GNU")
+    list(APPEND c_compile_options "-Wno-old-style-declaration")
+endif()
+
+# Clang finds some warnings in IDF code which GCC doesn't.
+# All these warnings should be fixed before Clang is presented
+# as a toolchain choice for users.
+if(CMAKE_C_COMPILER_ID MATCHES "Clang")
+    # Clang checks Doxygen comments for being in sync with function prototype.
+    # There are some inconsistencies, especially in ROM headers.
+    list(APPEND compile_options "-Wno-documentation")
+    # GCC allows repeated typedefs when the source and target types are the same.
+    # Clang doesn't allow this. This occurs in many components due to forward
+    # declarations.
+    list(APPEND compile_options "-Wno-typedef-redefinition")
+    # This issue is seemingly related to newlib's char type functions.
+    # Fix is not clear yet.
+    list(APPEND compile_options "-Wno-char-subscripts")
+    # Clang seems to notice format string issues which GCC doesn't.
+    list(APPEND compile_options "-Wno-format-security")
+    # Logic bug in essl component
+    list(APPEND compile_options "-Wno-tautological-overlap-compare")
+    # Some pointer checks in mDNS component check addresses which can't be NULL
+    list(APPEND compile_options "-Wno-tautological-pointer-compare")
+    # Similar to the above, in tcp_transport
+    list(APPEND compile_options "-Wno-pointer-bool-conversion")
+    # mbedTLS md5.c triggers this warning in md5_test_buf (false positive)
+    list(APPEND compile_options "-Wno-string-concatenation")
+    # multiple cases of implict convertions between unrelated enum types
+    list(APPEND compile_options "-Wno-enum-conversion")
+    # When IRAM_ATTR is specified both in function declaration and definition,
+    # it produces different section names, since section names include __COUNTER__.
+    # Occurs in multiple places.
+    list(APPEND compile_options "-Wno-section")
+    # Multiple cases of attributes unknown to clang, for example
+    # __attribute__((optimize("-O3")))
+    list(APPEND compile_options "-Wno-unknown-attributes")
+    # Disable Clang warnings for atomic operations with access size
+    # more then 4 bytes
+    list(APPEND compile_options "-Wno-atomic-alignment")
+    # Clang also produces many -Wunused-function warnings which GCC doesn't.
+    # However these aren't treated as errors.
+endif()
+# More warnings may exist in unit tests and example projects.
+
 if(CONFIG_COMPILER_WARN_WRITE_STRINGS)
     list(APPEND compile_options "-Wwrite-strings")
 endif()
@@ -97,7 +130,41 @@ endif()
 if(NOT ${CMAKE_C_COMPILER_VERSION} VERSION_LESS 8.0.0)
     if(CONFIG_COMPILER_HIDE_PATHS_MACROS)
         list(APPEND compile_options "-fmacro-prefix-map=${CMAKE_SOURCE_DIR}=.")
-        list(APPEND compile_options "-fmacro-prefix-map=${IDF_PATH}=IDF")
+        list(APPEND compile_options "-fmacro-prefix-map=${IDF_PATH}=/IDF")
+    endif()
+
+    if(CONFIG_APP_REPRODUCIBLE_BUILD)
+        idf_build_set_property(DEBUG_PREFIX_MAP_GDBINIT "${BUILD_DIR}/prefix_map_gdbinit")
+
+        list(APPEND compile_options "-fdebug-prefix-map=${IDF_PATH}=/IDF")
+        list(APPEND compile_options "-fdebug-prefix-map=${PROJECT_DIR}=/IDF_PROJECT")
+        list(APPEND compile_options "-fdebug-prefix-map=${BUILD_DIR}=/IDF_BUILD")
+
+        # component dirs
+        idf_build_get_property(python PYTHON)
+        idf_build_get_property(idf_path IDF_PATH)
+        idf_build_get_property(component_dirs BUILD_COMPONENT_DIRS)
+
+        execute_process(
+            COMMAND ${python}
+                "${idf_path}/tools/generate_debug_prefix_map.py"
+                "${BUILD_DIR}"
+                "${component_dirs}"
+            OUTPUT_VARIABLE result
+            RESULT_VARIABLE ret
+        )
+        if(NOT ret EQUAL 0)
+            message(FATAL_ERROR "This is a bug. Please report to https://github.com/espressif/esp-idf/issues")
+        endif()
+
+        spaces2list(result)
+        list(LENGTH component_dirs length)
+        math(EXPR max_index "${length} - 1")
+        foreach(index RANGE ${max_index})
+            list(GET component_dirs ${index} folder)
+            list(GET result ${index} after)
+            list(APPEND compile_options "-fdebug-prefix-map=${folder}=${after}")
+        endforeach()
     endif()
 endif()
 
@@ -108,11 +175,54 @@ if(CMAKE_C_COMPILER_ID STREQUAL "GNU")
                                 )
 endif()
 
+if(CONFIG_ESP_SYSTEM_USE_EH_FRAME)
+  list(APPEND compile_options "-fasynchronous-unwind-tables")
+  list(APPEND link_options "-Wl,--eh-frame-hdr")
+endif()
+
 list(APPEND link_options "-fno-lto")
 
+if(CONFIG_IDF_TARGET_LINUX AND CMAKE_HOST_SYSTEM_NAME STREQUAL "Darwin")
+    list(APPEND link_options "-Wl,-dead_strip")
+    list(APPEND link_options "-Wl,-warn_commons")
+else()
+    list(APPEND link_options "-Wl,--gc-sections")
+    list(APPEND link_options "-Wl,--warn-common")
+endif()
+
+# SMP FreeRTOS user provided minimal idle hook. This allows the user to provide
+# their own copy of vApplicationMinimalIdleHook()
+if(CONFIG_FREERTOS_USE_MINIMAL_IDLE_HOOK)
+    list(APPEND link_options "-Wl,--wrap=vApplicationMinimalIdleHook")
+endif()
+
+# Placing jump tables in flash would cause issues with code that required
+# to be placed in IRAM
+list(APPEND compile_options "-fno-jump-tables")
+if(CMAKE_C_COMPILER_ID MATCHES "GNU")
+    # This flag is GCC-specific.
+    # Not clear yet if some other flag should be used for Clang.
+    list(APPEND compile_options "-fno-tree-switch-conversion")
+endif()
+
+if(CMAKE_C_COMPILER_ID MATCHES "LLVM")
+    list(APPEND compile_options "-fno-use-cxa-atexit")
+endif()
+
+# For the transition period from 32-bit time_t to 64-bit time_t,
+# auto-detect the size of this type and set corresponding variable.
+include(CheckTypeSize)
+check_type_size("time_t" TIME_T_SIZE)
+if(TIME_T_SIZE)
+    idf_build_set_property(TIME_T_SIZE ${TIME_T_SIZE})
+else()
+    message(FATAL_ERROR "Failed to determine sizeof(time_t)")
+endif()
+
 idf_build_set_property(COMPILE_OPTIONS "${compile_options}" APPEND)
 idf_build_set_property(C_COMPILE_OPTIONS "${c_compile_options}" APPEND)
 idf_build_set_property(CXX_COMPILE_OPTIONS "${cxx_compile_options}" APPEND)
+idf_build_set_property(ASM_COMPILE_OPTIONS "${asm_compile_options}" APPEND)
 idf_build_set_property(COMPILE_DEFINITIONS "${compile_definitions}" APPEND)
 idf_build_set_property(LINK_OPTIONS "${link_options}" APPEND)
 

CONTRIBUTING.md

@@ -0,0 +1,6 @@
+# Contributing to ESP-IDF
+
+Contributions to ESP-IDF - fixing bugs, adding features, adding documentation - are welcome! We accept contributions via Github Pull Requests.
+
+Please see the [Contributions Guide](https://docs.espressif.com/projects/esp-idf/en/latest/esp32/contribute/index.html) for more information.
+

CONTRIBUTING.rst

@@ -1,62 +0,0 @@
-Contributions Guide
-===================
-
-We welcome contributions to the esp-idf project!
-
-How to Contribute
------------------
-
-Contributions to esp-idf - fixing bugs, adding features, adding documentation - are welcome. We accept contributions via `Github Pull Requests <https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/about-pull-requests>`_.
-
-Before Contributing
--------------------
-
-Before sending us a Pull Request, please consider this list of points:
-
-* Is the contribution entirely your own work, or already licensed under an Apache License 2.0 compatible Open Source License? If not then we unfortunately cannot accept it.
-
-* Does any new code conform to the esp-idf :doc:`Style Guide <style-guide>`?
-
-* Have you installed the :doc:`pre-commit hook <install-pre-commit-hook>` for esp-idf project?
-
-* Does the code documentation follow requirements in :doc:`documenting-code`?
-
-* Is the code adequately commented for people to understand how it is structured?
-
-* Is there documentation or examples that go with code contributions? There are additional suggestions for writing good examples in :idf:`examples` readme.
-
-* Are comments and documentation written in clear English, with no spelling or grammar errors?
-
-* Example contributions are also welcome. Please check the :doc:`creating-examples` guide for these.
-
-* If the contribution contains multiple commits, are they grouped together into logical changes (one major change per pull request)? Are any commits with names like "fixed typo" `squashed into previous commits <https://eli.thegreenplace.net/2014/02/19/squashing-github-pull-requests-into-a-single-commit/>`_?
-
-* If you're unsure about any of these points, please open the Pull Request anyhow and then ask us for feedback.
-
-Pull Request Process
---------------------
-
-After you open the Pull Request, there will probably be some discussion in the comments field of the request itself.
-
-Once the Pull Request is ready to merge, it will first be merged into our internal git system for in-house automated testing.
-
-If this process passes, it will be merged onto the public github repository.
-
-Legal Part
-----------
-
-Before a contribution can be accepted, you will need to sign our :doc:`contributor-agreement`. You will be prompted for this automatically as part of the Pull Request process.
-
-Related Documents
------------------
-
-.. toctree::
-    :maxdepth: 1
-
-    style-guide
-    install-pre-commit-hook
-    documenting-code
-    add-ons-reference
-    creating-examples
-    ../api-reference/template
-    contributor-agreement

Kconfig

@@ -1,18 +1,24 @@
 #
-# For a description of the syntax of this configuration file,
-# see kconfig/kconfig-language.txt.
+# Please run the following command for opening a page with more information about this configuration file:
+# idf.py docs -sp api-reference/kconfig.html
 #
 mainmenu "Espressif IoT Development Framework Configuration"
 
+    orsource "./components/soc/$IDF_TARGET/include/soc/Kconfig.soc_caps.in"
+
     config IDF_CMAKE
         bool
-        option env="IDF_CMAKE"
+        default "y"
 
     config IDF_ENV_FPGA
         # This option is for internal use only
         bool
         option env="IDF_ENV_FPGA"
 
+    config IDF_CI_BUILD
+        bool
+        default y if "$(IDF_CI_BUILD)" = "y" || "$(IDF_CI_BUILD)" = 1
+
     config IDF_TARGET_ARCH_RISCV
         bool
         default "n"
@@ -21,6 +27,11 @@ mainmenu "Espressif IoT Development Framework Configuration"
         bool
         default "n"
 
+    config IDF_TARGET_ARCH
+        string
+        default "riscv" if IDF_TARGET_ARCH_RISCV
+        default "xtensa" if IDF_TARGET_ARCH_XTENSA
+
     config IDF_TARGET
         # This option records the IDF target when sdkconfig is generated the first time.
         # It is not updated if environment variable $IDF_TARGET changes later, and
@@ -29,6 +40,10 @@ mainmenu "Espressif IoT Development Framework Configuration"
         string
         default "$IDF_TARGET"
 
+    config IDF_TARGET_LINUX
+        bool
+        default "y" if IDF_TARGET="linux"
+
     config IDF_TARGET_ESP32
         bool
         default "y" if IDF_TARGET="esp32"
@@ -45,85 +60,55 @@ mainmenu "Espressif IoT Development Framework Configuration"
         default "y" if IDF_TARGET="esp32s3"
         select IDF_TARGET_ARCH_XTENSA
 
-    choice IDF_TARGET_ESP32S3_BETA_VERSION
-        prompt "ESP32-S3 beta version"
-        depends on IDF_TARGET_ESP32S3
-        default IDF_TARGET_ESP32S3_BETA_VERSION_2
-        help
-            Currently ESP32-S3 has several beta versions for internal use only.
-            Select the one that matches your chip model.
-
-        config IDF_TARGET_ESP32S3_BETA_VERSION_2
-            bool
-            prompt "ESP32-S3 beta2"
-    endchoice
-
     config IDF_TARGET_ESP32C3
         bool
         default "y" if IDF_TARGET="esp32c3"
         select FREERTOS_UNICORE
         select IDF_TARGET_ARCH_RISCV
 
+    config IDF_TARGET_ESP32H2
+        bool
+        default "y" if IDF_TARGET="esp32h2"
+        select FREERTOS_UNICORE
+        select IDF_TARGET_ARCH_RISCV
+
+    choice IDF_TARGET_ESP32H2_BETA_VERSION
+        prompt "ESP32-H2 beta version"
+        depends on IDF_TARGET_ESP32H2
+        default IDF_TARGET_ESP32H2_BETA_VERSION_2
+        help
+            Currently ESP32-H2 has several beta versions for internal use only.
+            Select the one that matches your chip model.
+
+        config IDF_TARGET_ESP32H2_BETA_VERSION_1
+            bool
+            prompt "ESP32-H2 beta1"
+        config IDF_TARGET_ESP32H2_BETA_VERSION_2
+            bool
+            prompt "ESP32-H2 beta2"
+    endchoice
+
+    config IDF_TARGET_ESP32C2
+        bool
+        default "y" if IDF_TARGET="esp32c2"
+        select FREERTOS_UNICORE
+        select IDF_TARGET_ARCH_RISCV
+
+    config IDF_TARGET_LINUX
+        bool
+        default "y" if IDF_TARGET="linux"
+
     config IDF_FIRMWARE_CHIP_ID
         hex
         default 0x0000 if IDF_TARGET_ESP32
         default 0x0002 if IDF_TARGET_ESP32S2
-        default 0x0004 if IDF_TARGET_ESP32S3
         default 0x0005 if IDF_TARGET_ESP32C3
+        default 0x0009 if IDF_TARGET_ESP32S3
+        default 0x000C if IDF_TARGET_ESP32C2
+        default 0x000A if IDF_TARGET_ESP32H2_BETA_VERSION_1
+        default 0x000E if IDF_TARGET_ESP32H2_BETA_VERSION_2 # ESP32H2-TODO: IDF-3475
         default 0xFFFF
 
-    menu "SDK tool configuration"
-        config SDK_TOOLPREFIX
-            string "Compiler toolchain path/prefix"
-            default "xtensa-esp32-elf-" if IDF_TARGET_ESP32
-            default "xtensa-esp32s2-elf-" if IDF_TARGET_ESP32S2
-            default "xtensa-esp32s3-elf-" if IDF_TARGET_ESP32S3
-            default "riscv32-esp-elf-" if IDF_TARGET_ESP32C3
-
-            help
-                The prefix/path that is used to call the toolchain. The default setting assumes
-                a crosstool-ng gcc setup that is in your PATH.
-
-        config SDK_PYTHON
-            string "Python interpreter"
-            depends on !IDF_CMAKE
-            default "python"
-            help
-                The executable name/path that is used to run python.
-
-                (Note: This option is used with the legacy GNU Make build system only.)
-
-        config SDK_MAKE_WARN_UNDEFINED_VARIABLES
-            bool "'make' warns on undefined variables"
-            depends on !IDF_CMAKE
-            default "y"
-            help
-                Adds --warn-undefined-variables to MAKEFLAGS. This causes make to
-                print a warning any time an undefined variable is referenced.
-
-                This option helps find places where a variable reference is misspelled
-                or otherwise missing, but it can be unwanted if you have Makefiles which
-                depend on undefined variables expanding to an empty string.
-
-                (Note: this option is used with the legacy GNU Make build system only.)
-
-        config SDK_TOOLCHAIN_SUPPORTS_TIME_WIDE_64_BITS
-            bool "Toolchain supports time_t wide 64-bits"
-            default n
-            help
-                Enable this option in case you have a custom toolchain which supports time_t wide 64-bits.
-                This option checks time_t is 64-bits and disables ROM time functions
-                to use the time functions from the toolchain instead.
-                This option allows resolving the Y2K38 problem.
-                See "Setup Linux Toolchain from Scratch" to build
-                a custom toolchain which supports 64-bits time_t.
-
-                Note: ESP-IDF does not currently come with any pre-compiled toolchain
-                that supports 64-bit wide time_t.
-                This will change in a future major release,
-                but currently 64-bit time_t requires a custom built toolchain.
-
-    endmenu  # SDK tool configuration
 
     menu "Build type"
 
@@ -183,6 +168,7 @@ mainmenu "Espressif IoT Development Framework Configuration"
             config APP_BUILD_TYPE_APP_2NDBOOT
                 bool
                 prompt "Default (binary application + 2nd stage bootloader)"
+                depends on !IDF_TARGET_LINUX
                 select APP_BUILD_GENERATE_BINARIES
                 select APP_BUILD_BOOTLOADER
                 select APP_BUILD_USE_FLASH_SECTIONS
@@ -202,6 +188,63 @@ mainmenu "Espressif IoT Development Framework Configuration"
         config APP_BUILD_USE_FLASH_SECTIONS
             bool # Whether to place code/data into memory-mapped flash sections
 
+        config APP_REPRODUCIBLE_BUILD
+            bool "Enable reproducible build"
+            default n
+            select COMPILER_HIDE_PATHS_MACROS
+            help
+                If enabled, all date, time, and path information would be eliminated. A .gdbinit file would be create
+                automatically. (or will be append if you have one already)
+
+        config APP_NO_BLOBS
+            bool "No Binary Blobs"
+            default n
+            help
+                If enabled, this disables the linking of binary libraries in the application build. Note
+                that after enabling this Wi-Fi/Bluetooth will not work.
+
+        config APP_COMPATIBLE_PRE_V2_1_BOOTLOADERS
+            bool "App compatible with bootloaders before ESP-IDF v2.1"
+            select APP_COMPATIBLE_PRE_V3_1_BOOTLOADERS
+            depends on IDF_TARGET_ESP32
+            default n
+            help
+                Bootloaders before ESP-IDF v2.1 did less initialisation of the
+                system clock. This setting needs to be enabled to build an app
+                which can be booted by these older bootloaders.
+
+                If this setting is enabled, the app can be booted by any bootloader
+                from IDF v1.0 up to the current version.
+
+                If this setting is disabled, the app can only be booted by bootloaders
+                from IDF v2.1 or newer.
+
+                Enabling this setting adds approximately 1KB to the app's IRAM usage.
+
+        config APP_COMPATIBLE_PRE_V3_1_BOOTLOADERS
+            bool "App compatible with bootloader and partition table before ESP-IDF v3.1"
+            depends on IDF_TARGET_ESP32
+            default n
+            help
+                Partition tables before ESP-IDF V3.1 do not contain an MD5 checksum
+                field, and the bootloader before ESP-IDF v3.1 cannot read a partition
+                table that contains an MD5 checksum field.
+
+                Enable this option only if your app needs to boot on a bootloader and/or
+                partition table that was generated from a version *before* ESP-IDF v3.1.
+
+                If this option and Flash Encryption are enabled at the same time, and any
+                data partitions in the partition table are marked Encrypted, then the
+                partition encrypted flag should be manually verified in the app before accessing
+                the partition (see CVE-2021-27926).
+
+        config APP_INIT_CLK
+            bool
+            depends on IDF_TARGET_ESP32
+            default y if APP_COMPATIBLE_PRE_V2_1_BOOTLOADERS
+            default y if APP_BUILD_TYPE_ELF_RAM
+
+
     endmenu # Build type
 
     source "$COMPONENT_KCONFIGS_PROJBUILD_SOURCE_FILE"
@@ -279,6 +322,57 @@ mainmenu "Espressif IoT Development Framework Configuration"
 
         endchoice # assertions
 
+        choice COMPILER_FLOAT_LIB_FROM
+            prompt "Compiler float lib source"
+            default COMPILER_FLOAT_LIB_FROM_RVFPLIB if ESP_ROM_HAS_RVFPLIB
+            default COMPILER_FLOAT_LIB_FROM_GCCLIB
+            help
+                In the soft-fp part of libgcc, riscv version is written in C,
+                and handles all edge cases in IEEE754, which makes it larger
+                and performance is slow.
+
+                RVfplib is an optimized RISC-V library for FP arithmetic on 32-bit
+                integer processors, for single and double-precision FP.
+                RVfplib is "fast", but it has a few exceptions from IEEE 754 compliance.
+
+            config COMPILER_FLOAT_LIB_FROM_GCCLIB
+                bool "libgcc"
+            config COMPILER_FLOAT_LIB_FROM_RVFPLIB
+                depends on ESP_ROM_HAS_RVFPLIB
+                bool "librvfp"
+        endchoice # COMPILER_FLOAT_LIB_FROM
+
+        config COMPILER_OPTIMIZATION_ASSERTION_LEVEL
+            int
+            default 0 if COMPILER_OPTIMIZATION_ASSERTIONS_DISABLE
+            default 1 if COMPILER_OPTIMIZATION_ASSERTIONS_SILENT
+            default 2 if COMPILER_OPTIMIZATION_ASSERTIONS_ENABLE
+
+        config COMPILER_OPTIMIZATION_CHECKS_SILENT
+            bool "Disable messages in ESP_RETURN_ON_* and ESP_EXIT_ON_* macros"
+            default n
+            help
+                If enabled, the error messages will be discarded in following check macros:
+                - ESP_RETURN_ON_ERROR
+                - ESP_EXIT_ON_ERROR
+                - ESP_RETURN_ON_FALSE
+                - ESP_EXIT_ON_FALSE
+
+        menuconfig COMPILER_HIDE_PATHS_MACROS
+            bool "Replace ESP-IDF and project paths in binaries"
+            default y
+            help
+                When expanding the __FILE__ and __BASE_FILE__ macros, replace paths inside ESP-IDF
+                with paths relative to the placeholder string "IDF", and convert paths inside the
+                project directory to relative paths.
+
+                This allows building the project with assertions or other code that embeds file paths,
+                without the binary containing the exact path to the IDF or project directories.
+
+                This option passes -fmacro-prefix-map options to the GCC command line. To replace additional
+                paths in your binaries, modify the project CMakeLists.txt file to pass custom -fmacro-prefix-map or
+                -ffile-prefix-map arguments.
+
         menuconfig COMPILER_CXX_EXCEPTIONS
             bool "Enable C++ exceptions"
             default n
@@ -331,6 +425,7 @@ mainmenu "Espressif IoT Development Framework Configuration"
 
                 - coverage: NORMAL < STRONG < OVERALL
 
+                The performance impact includes increasing the amount of stack memory required for each task.
 
             config COMPILER_STACK_CHECK_MODE_NONE
                 bool "None"
@@ -362,12 +457,17 @@ mainmenu "Espressif IoT Development Framework Configuration"
                 For C++, this warns about the deprecated conversion from string
                 literals to ``char *``.
 
-        config COMPILER_DISABLE_GCC8_WARNINGS
-            bool "Disable new warnings introduced in GCC 6 - 8"
-            default "n"
+        config COMPILER_SAVE_RESTORE_LIBCALLS
+            bool "Enable -msave-restore flag to reduce code size"
+            depends on IDF_TARGET_ARCH_RISCV
             help
-                Enable this option if using GCC 6 or newer, and wanting to disable warnings which don't appear with
-                GCC 5.
+                Adds -msave-restore to C/C++ compilation flags.
+
+                When this flag is enabled, compiler will call library functions to
+                save/restore registers in function prologues/epilogues. This results
+                in lower overall code size, at the expense of slightly reduced performance.
+
+                This option can be enabled for RISC-V targets only.
 
         config COMPILER_DUMP_RTL_FILES
             bool "Dump RTL files during compilation"
@@ -375,28 +475,8 @@ mainmenu "Espressif IoT Development Framework Configuration"
                 If enabled, RTL files will be produced during compilation. These files
                 can be used by other tools, for example to calculate call graphs.
 
-
     endmenu # Compiler Options
 
     menu "Component config"
         source "$COMPONENT_KCONFIGS_SOURCE_FILE"
     endmenu
-
-    menu "Compatibility options"
-        config LEGACY_INCLUDE_COMMON_HEADERS
-            bool "Include headers across components as before IDF v4.0"
-            default n
-            help
-                Soc, esp32, and driver components, the most common
-                components. Some header of these components are included
-                implicitly by headers of other components before IDF v4.0.
-                It's not required for high-level components, but still
-                included through long header chain everywhere.
-
-                This is harmful to the modularity. So it's changed in IDF
-                v4.0.
-
-                You can still include these headers in a legacy way until it
-                is totally deprecated by enable this option.
-
-    endmenu #Compatibility options

README.md

@@ -2,7 +2,32 @@
 
 * [中文版](./README_CN.md)
 
-ESP-IDF is the development framework for Espressif SoCs (released after 2016<sup>[1](#fn1)</sup>) provided for Windows, Linux and macOS.
+ESP-IDF is the development framework for Espressif SoCs supported on Windows, Linux and macOS.
+
+# ESP-IDF Release Support Schedule
+
+![Support Schedule](https://dl.espressif.com/dl/esp-idf/support-periods.svg)
+
+- Please read [the support policy](SUPPORT_POLICY.md) and [the documentation](https://docs.espressif.com/projects/esp-idf/en/latest/esp32/versions.html) for more information about ESP-IDF versions.
+- Please see the [End-of-Life Advisories](https://www.espressif.com/en/support/documents/advisories?keys=&field_type_of_advisory_tid%5B%5D=817) for information about ESP-IDF releases with discontinued support.
+
+# ESP-IDF Release and SoC Compatibility
+
+The following table shows ESP-IDF support of Espressif SoCs where ![alt text][preview] and ![alt text][supported] denote preview status and support, respectively. The preview support is usually limited in time and intended for beta versions of chips. Please use an ESP-IDF release where the desired SoC is already supported.
+
+|Chip         |          v4.1         |          v4.2          |         v4.3           |          v4.4          |          v5.0          |                                                                                      |
+|:----------- |:---------------------:| :---------------------:| :---------------------:| :---------------------:| :---------------------:|:------------------------------------------------------------------------------------ |
+|ESP32        |![alt text][supported] | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] |                                                                                      |
+|ESP32-S2     |                       | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] |                                                                                      |
+|ESP32-C3     |                       |                        | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] |                                                                                      |
+|ESP32-S3     |                       |                        |                        | ![alt text][supported] | ![alt text][supported] | [Announcement](https://www.espressif.com/en/news/ESP32_S3)                           |
+|ESP32-C2     |                       |                        |                        |                        | ![alt text][supported] | [Announcement](https://blog.espressif.com/esp32-c2-and-why-it-matter-s-bcf4d7d0b2c6) |
+|ESP32-H2     |                       |                        |                        | ![alt text][preview]   | ![alt text][preview]   | [Announcement](https://www.espressif.com/en/news/ESP32_H2)                           |
+
+[supported]: https://img.shields.io/badge/-supported-green "supported"
+[preview]: https://img.shields.io/badge/-preview-orange "preview"
+
+Espressif SoCs released before 2016 (ESP8266 and ESP8285) are supported by [RTOS SDK](https://github.com/espressif/ESP8266_RTOS_SDK) instead.
 
 # Developing With ESP-IDF
 
@@ -14,11 +39,9 @@ See https://idf.espressif.com/ for links to detailed instructions on how to set
 
 ### Non-GitHub forks
 
-ESP-IDF uses relative locations as its submodules URLs ([.gitmodules](.gitmodules)). So they link to GitHub.
-If ESP-IDF is forked to a Git repository which is not on GitHub, you will need to run the script
-[tools/set-submodules-to-github.sh](tools/set-submodules-to-github.sh) after git clone.
-The script sets absolute URLs for all submodules, allowing `git submodule update --init --recursive` to complete.
-If cloning ESP-IDF from GitHub, this step is not needed.
+ESP-IDF uses relative locations as its submodules URLs ([.gitmodules](.gitmodules)). So they link to GitHub. If ESP-IDF is forked to a Git repository which is not on GitHub, you will need to run the script [tools/set-submodules-to-github.sh](tools/set-submodules-to-github.sh) after git clone.
+
+The script sets absolute URLs for all submodules, allowing `git submodule update --init --recursive` to complete. If cloning ESP-IDF from GitHub, this step is not needed.
 
 ## Finding a Project
 
@@ -86,9 +109,9 @@ After the initial flash, you may just want to build and flash just your app, not
 
 ## Erasing Flash
 
-The `idf.py flash` target does not erase the entire flash contents. However it is sometimes useful to set the device back to a totally erased state, particularly when making partition table changes or OTA app updates. To erase the entire flash, run `idf.py erase_flash`.
+The `idf.py flash` target does not erase the entire flash contents. However it is sometimes useful to set the device back to a totally erased state, particularly when making partition table changes or OTA app updates. To erase the entire flash, run `idf.py erase-flash`.
 
-This can be combined with other targets, ie `idf.py -p PORT erase_flash flash` will erase everything and then re-flash the new app, bootloader and partition table.
+This can be combined with other targets, ie `idf.py -p PORT erase-flash flash` will erase everything and then re-flash the new app, bootloader and partition table.
 
 # Resources
 
@@ -99,8 +122,3 @@ This can be combined with other targets, ie `idf.py -p PORT erase_flash flash` w
 * [Check the Issues section on github](https://github.com/espressif/esp-idf/issues) if you find a bug or have a feature request. Please check existing Issues before opening a new one.
 
 * If you're interested in contributing to ESP-IDF, please check the [Contributions Guide](https://docs.espressif.com/projects/esp-idf/en/latest/contribute/index.html).
-
-
-________
-
-<a name="fn1">1</a>: ESP8266 and ESP8285 are not supported in ESP-IDF. See [RTOS SDK](https://github.com/espressif/ESP8266_RTOS_SDK) instead.

README_CN.md

@@ -2,23 +2,46 @@
 
 * [English Version](./README.md)
 
-ESP-IDF 是由乐鑫官方针对乐鑫各系列芯片产品（发布于 2016 年后<sup>[1](#fn1)</sup>）推出的开发框架，支持 Windows、Linux 和 macOS 操作系统。
+ESP-IDF 是乐鑫官方推出的物联网开发框架，支持 Windows、Linux 和 macOS 操作系统。
+
+# ESP-IDF 版本支持期限
+
+![支持期限](https://dl.espressif.com/dl/esp-idf/support-periods.svg)
+
+- 请参考 [ESP-IDF 支持政策](SUPPORT_POLICY_CN.md) 以及 [相关文档](https://docs.espressif.com/projects/esp-idf/zh_CN/latest/esp32/versions.html) 了解更多关于 ESP-IDF 版本的信息。
+- 请参考 [ESP-IDF 版本停止维护 (EOL) 公告](https://www.espressif.com/zh-hans/support/documents/advisories?keys=&field_type_of_advisory_tid%5B%5D=817)。
+
+# ESP-IDF 与乐鑫芯片
+
+下表总结了乐鑫芯片在 ESP-IDF 各版本中的支持状态，其中 ![alt text][supported] 代表已支持，![alt text][preview] 代表目前处于预览支持状态。预览支持状态通常有时间限制，而且仅适用于测试版芯片。请确保使用与芯片相匹配的 ESP-IDF 版本。
+
+|芯片         |          v4.1         |          v4.2          |         v4.3           |          v4.4          |          v5.0          |                                                                                      |
+|:----------- |:---------------------:| :---------------------:| :---------------------:| :---------------------:| :---------------------:|:------------------------------------------------------------------------------------ |
+|ESP32        |![alt text][supported] | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] |                                                                                      |
+|ESP32-S2     |                       | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] |                                                                                      |
+|ESP32-C3     |                       |                        | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] |                                                                                      |
+|ESP32-S3     |                       |                        |                        | ![alt text][supported] | ![alt text][supported] | [芯片发布公告](https://www.espressif.com/en/news/ESP32_S3)                           |
+|ESP32-C2     |                       |                        |                        |                        | ![alt text][supported] | [芯片发布公告](https://blog.espressif.com/esp32-c2-and-why-it-matter-s-bcf4d7d0b2c6) |
+|ESP32-H2     |                       |                        |                        | ![alt text][preview]   | ![alt text][preview]   | [芯片发布公告](https://www.espressif.com/en/news/ESP32_H2)                           |
+
+[supported]: https://img.shields.io/badge/-%E6%94%AF%E6%8C%81-green "supported"
+[preview]: https://img.shields.io/badge/-%E9%A2%84%E8%A7%88-orange "preview"
+
+对于 2016 年之前发布的乐鑫芯片（包括 ESP8266 和 ESP8285），请参考 [RTOS SDK](https://github.com/espressif/ESP8266_RTOS_SDK)。
 
 # 使用 ESP-IDF 进行开发
 
 ## 搭建 ESP-IDF 开发环境
 
-关于不同芯片如何搭建 ESP-IDF 的开发环境，请参考 https://idf.espressif.com/。
+关于不同芯片如何搭建 ESP-IDF 的开发环境，请参考 https://idf.espressif.com/ 。
 
 **注意：** 不同系列芯片和不同 ESP-IDF 版本都有其对应的文档。请参阅[版本](https://docs.espressif.com/projects/esp-idf/zh_CN/latest/esp32/versions.html)部分，获得关于如何查找文档以及如何检出 ESP-IDF 的特定发行版的详细信息。
 
 ### 非 GitHub 分叉的 ESP-IDF 项目
 
-ESP-IDF 中的子模块采用相对路径（[详见 .gitmodules 文件](.gitmodules)），所以它们会指向 GitHub。
-如果 ESP-IDF 被分叉到的仓库不在 GitHub 上，那么你需要在克隆结束后运行该脚本 [tools/set-submodules-to-github.sh](tools/set-submodules-to-github.sh)。
+ESP-IDF 中的子模块采用相对路径（[详见 .gitmodules 文件](.gitmodules)），所以它们会指向 GitHub。 如果 ESP-IDF 被分叉到的仓库不在 GitHub 上，那么你需要在克隆结束后运行该脚本 [tools/set-submodules-to-github.sh](tools/set-submodules-to-github.sh)。
 
-这个脚本会为所有的子模块设置绝对路径，接着可以通过 `git submodule update --init --recursive` 完成子模块的更新。
-如果 ESP-IDF 是从 GitHub 上克隆得到，则不需要此步骤。
+这个脚本会为所有的子模块设置绝对路径，接着可以通过 `git submodule update --init --recursive` 完成子模块的更新。如果 ESP-IDF 是从 GitHub 上克隆得到，则不需要此步骤。
 
 ## 寻找项目
 
@@ -39,7 +62,7 @@ ESP-IDF 中的子模块采用相对路径（[详见 .gitmodules 文件](.gitmodu
 * 在主机中安装入门指南中提到的构建所依赖的工具。
 * 运行安装脚本来设置构建环境。可为 Windows shell 选择 `install.bat` 或 `install.ps1`，为 Unix shell 选择 `install.sh` 和 `install.fish`。
 * 在使用 ESP-IDF 之前，需要在 shell 中运行导出脚本。Windows 下可运行 `export.bat`，Unix 下可运行 `source export.sh`。
-  
+
 ## 配置项目
 
 * `idf.py set-target <chip_name>` 可将项目的目标芯片设置为 `<chip_name>`。运行 `idf.py set-target`，不用带任何参数，可查看所有支持的目标芯片列表。
@@ -86,21 +109,16 @@ ESP-IDF 中的子模块采用相对路径（[详见 .gitmodules 文件](.gitmodu
 
 ## 擦除 Flash
 
-`idf.py flash` 并不会擦除 flash 上所有的内容，但是有时候我们需要设备恢复到完全擦除的状态，尤其是分区表发生了变化或者 OTA 应用升级时。要擦除整块 flash 请运行 `idf.py erase_flash`。
+`idf.py flash` 并不会擦除 flash 上所有的内容，但是有时候我们需要设备恢复到完全擦除的状态，尤其是分区表发生了变化或者 OTA 应用升级时。要擦除整块 flash 请运行 `idf.py erase-flash`。
 
-这条命令还可以和其余命令整合在一起，`idf.py -p PORT erase_flash flash` 会擦除一切然后重新烧写新的应用程序、引导程序和分区表。
+这条命令还可以和其余命令整合在一起，`idf.py -p PORT erase-flash flash` 会擦除一切然后重新烧写新的应用程序、引导程序和分区表。
 
 # 其它参考资源
 
-* 最新版的文档：https://docs.espressif.com/projects/esp-idf/，该文档是由本仓库 [docs 目录](docs) 构建得到。
+* 最新版的文档：https://docs.espressif.com/projects/esp-idf/ ，该文档是由本仓库 [docs 目录](docs) 构建得到。
 
 * 可以前往 [esp32.com 论坛](https://esp32.com/) 提问，挖掘社区资源。
 
 * 如果你在使用中发现了错误或者需要新的功能，请先[查看 GitHub Issues](https://github.com/espressif/esp-idf/issues)，确保该问题没有重复提交。
 
 * 如果你有兴趣为 ESP-IDF 作贡献，请先阅读[贡献指南](https://docs.espressif.com/projects/esp-idf/en/latest/contribute/index.html)。
-
-
-__________
-
-<a name="fn1">1</a>: ESP-IDF 不支持 ESP8266 和 ESP8285。如有需要，请参考 [RTOS SDK](https://github.com/espressif/ESP8266_RTOS_SDK)。

SECURITY.md

@@ -0,0 +1,9 @@
+# Security Policy
+
+## Supported Versions
+
+Please refer to https://docs.espressif.com/projects/esp-idf/en/latest/esp32/versions.html#support-periods for more details on ESP-IDF supported versions and support period policy.
+
+## Reporting a Vulnerability
+
+If you think you have found a security vulnerability in Espressif solutions (including ESP-IDF), then please send an email to our Bug Bounty team at bugbounty@espressif.com. Please do **NOT** create a public GitHub issue.

SUPPORT_POLICY.md

@@ -5,12 +5,9 @@ Support Period Policy
 
 * [中文版](./SUPPORT_POLICY_CN.md)
 
-Each ESP-IDF major and minor release (V4.1, V4.2, etc) is supported for
-30 months after the initial stable release date.
+Each ESP-IDF major and minor release (V4.1, V4.2, etc) is supported for 30 months after the initial stable release date.
 
-Supported means that the ESP-IDF team will continue to apply bug fixes,
-security fixes, etc to the release branch on GitHub, and periodically
-make new bugfix releases as needed.
+Supported means that the ESP-IDF team will continue to apply bug fixes, security fixes, etc to the release branch on GitHub, and periodically make new bugfix releases as needed.
 
 Support period is divided into "Service" and "Maintenance" period:
 
@@ -19,65 +16,43 @@ Support period is divided into "Service" and "Maintenance" period:
 | Service     | 12 months    | Yes                                   |
 | Maintenance | 18 months    | No                                    |
 
-During the Service period, bugfixes releases are more frequent. In some cases,
-support for new features may be added during the Service period (this is
-reserved for features which are needed to meet particular regulatory
-requirements or standards for new products, and which carry a very low risk of
-introducing regressions.)
+During the Service period, bugfixes releases are more frequent. In some cases, support for new features may be added during the Service period (this is reserved for features which are needed to meet particular regulatory requirements or standards for new products, and which carry a very low risk of introducing regressions.)
 
-During the Maintenance period, the version is still supported but only bugfixes
-for high severity issues or security issues will be applied.
+During the Maintenance period, the version is still supported but only bugfixes for high severity issues or security issues will be applied.
 
 Using an “In Service” version is recommended when starting a new project.
 
-Users are encouraged to upgrade all projects to a newer ESP-IDF release before
-the support period finishes and the release becomes End of Life (EOL). It is our
-policy to not continue fixing bugs in End of Life releases.
+Users are encouraged to upgrade all projects to a newer ESP-IDF release before the support period finishes and the release becomes End of Life (EOL). It is our policy to not continue fixing bugs in End of Life releases.
 
-Pre-release versions (betas, previews, `-rc` and `-dev` versions, etc)
-are not covered by any support period. Sometimes a particular feature is
-marked as "Preview" in a release, which means it is also not covered
-by the support period.
+Pre-release versions (betas, previews, `-rc` and `-dev` versions, etc) are not covered by any support period. Sometimes a particular feature is marked as "Preview" in a release, which means it is also not covered by the support period.
 
-The ESP-IDF Programming Guide has information about the
-[different versions of ESP-IDF](https://docs.espressif.com/projects/esp-idf/en/latest/versions.html)
-(major, minor, bugfix, etc).
+ESP-IDF should be used in an up-to-date software environment. The operating system and other third-party tools should be supported by their maintainers. ESP-IDF cannot keep compatibility with unsupported third-party tools.
+
+The ESP-IDF Programming Guide has information about the [different versions of ESP-IDF](https://docs.espressif.com/projects/esp-idf/en/latest/versions.html) (major, minor, bugfix, etc).
 
 Example
 -------
 
-ESP-IDF V3.3 was released in September 2019. It will be supported for 30 months
-until February 2022.
+ESP-IDF V3.3 was released in September 2019. It was supported for 30 months until February 2022.
 
 - The first V3.3 release was `v3.3` in September 2019.
-- The ESP-IDF team continues to backport bug fixes, security fixes,
-  etc to the release branch `release/v3.3`.
-- Periodically stable bugfix releases are created from the release
-  branch. For example `v3.3.1`, `v3.3.2`, etc. Users are encouraged to
-  always update to the latest bugfix release.
-- V3.3 bugfix releases continue until February 2022, when all V3.3.x
-  releases become End of Life.
+- The ESP-IDF team continues to backport bug fixes, security fixes, etc to the release branch `release/v3.3`.
+- Periodically stable bugfix releases are created from the release branch. For example `v3.3.1`, `v3.3.2`, etc. Users are encouraged to always update to the latest bugfix release.
+- V3.3 bugfix releases continue until February 2022, when all V3.3.x releases become End of Life.
 
 Existing Releases
 -----------------
 
-ESP-IDF release V4.1 and all newer releases will follow this support period
-policy. The support period for each release will be announced when the release
-is made.
+ESP-IDF release V4.1 and all newer releases will follow this support period policy. The support period for each release will be announced when the release is made.
 
-For releases made before the current support period policy was announced, the
-original support periods apply:
+For releases made before the current support period policy was announced, the original support periods apply:
 
 * ESP-IDF V4.0.x will be supported until October 2021
 * ESP-IDF V3.3.x will be supported until February 2022
-* ESP-IDF V3.1.x and V3.2.x will both be supported until October 2020.
-* ESP-IDF versions before V3.1 are already End of Life.
+* ESP-IDF versions before V3.3 are already End of Life.
 
 Policy History
 --------------
 
-* September 2019. This policy split ESP-IDF releases into Standard and Long Term
-  Support.
-* July 2020. All releases from now will have the same support period, which is
-  equal to the previous Long Term Support period. Added “In Service” period,
-  during which versions will receive more updates.
+* September 2019. This policy splits ESP-IDF releases into Standard and Long Term Support.
+* July 2020. All releases from now will have the same support period, which is equal to the previous Long Term Support period. Added “In Service” period, during which versions will receive more updates.

SUPPORT_POLICY_CN.md

@@ -3,6 +3,8 @@
 支持期限政策
 =====================
 
+* [英文版](./SUPPORT_POLICY.md)
+
 ESP-IDF 的每个主要版本和次要版本（如 V4.1、V4.2 等）自其首次稳定版本发布之日起将支持 30 个月。
 
 支持意味着 ESP-IDF 团队将会对 GitHub 上的发布分支继续进行 bug 修复、安全修补等，并根据需求定期发布新的 bugfix 版本。
@@ -18,12 +20,14 @@ ESP-IDF 的每个主要版本和次要版本（如 V4.1、V4.2 等）自其首
 
 在版本的维护周期内，版本仍然会继续支持，但仅会对严重问题或安全问题进行 bug 修复。
 
-建议在开始新项目的时候使用处于服务周期的版本。 
+建议在开始新项目的时候使用处于服务周期的版本。
 
 在某一版本支持期限结束，停止更新维护 (EOL) 前，建议用户升级所有项目至较新的 ESP-IDF 版本。根据《支持期限政策》，我们将停止对 EOL 版本进行 bug 修复。
 
 《支持期限政策》不适用于预发布版本（包括 beta、preview、-rc 和 -dev 版本等）。有时，在发布的版本中存在被标记为 "Preview" 的特定功能，则该功能也不在支持期限内。
 
+请确保在最新的软件环境中使用 ESP-IDF。操作系统和其他第三方工具应得到其维护者的支持。ESP-IDF 无法兼容不受支持的第三方工具。
+
 有关 [ESP-IDF 不同版本](https://docs.espressif.com/projects/esp-idf/zh_CN/latest/esp32/versions.html)（主要版本、次要版本、bugfix 版本等）信息，可参阅《ESP-IDF 编程指南》。
 
 示例
@@ -45,11 +49,10 @@ ESP-IDF V4.1 及所有后续更新版本都将遵守该《支持期限政策》
 
 * ESP-IDF V4.0.x 将支持至 2021 年 10 月。
 * ESP-IDF V3.3.x 将支持至 2022 年 2 月。
-* ESP-IDF V3.1.x 和 V3.2.x 都将在 2020 年 10 月停止支持。
-* ESP-IDF v3.1.x 之前的版本已经停止支持。
-  
+* ESP-IDF v3.3 之前的版本已经停止支持。
+
 政策历史
 --------------
 
-* 2019 年 9 月。该政策将 ESP-IDF 版本发布分为标准支持版本和长期支持版本。 
-* 2020 年 7 月。从现在开始，所有发布的版本都将有相同的支持周期，等同于之前的长期支持周期。新增了”服务周期“，期间版本的更新将较为频繁。
+* 2019 年 9 月。该政策将 ESP-IDF 版本发布分为标准支持版本和长期支持版本。
+* 2020 年 7 月。从现在开始，所有发布的版本都将有相同的支持周期，即之前的长期支持周期。新增了”服务周期“，期间版本的更新将较为频繁。

components/README.md

@@ -0,0 +1,159 @@
+# Core Components
+
+## Overview
+
+This document contains details about what the core components are, what they contain, and how they are organized.
+
+## Organization
+
+The core components are organized into two groups.
+
+The first group (referred to as `G0` from now on) contains `hal`, `xtensa` and `riscv` (referred to as `arch` components from now on), `esp_rom`, `esp_common`, and `soc`. This
+group contain information about and low-level access to underlying hardware; or in the case of `esp_common`, hardware-agnostic code and utilities.
+These components can depend on each other, but as much as possible have no dependencies outside the group. The reason for this is that, due to the
+nature of what these components contain, the likelihood is high that a lot of other components will require these. Ideally, then, the dependency
+relationship only goes one way. This makes it easier for these components, as a group, to be usable in another project. One can conceivably implement
+a competing SDK to ESP-IDF on top of these components.
+
+The second group (referred to as `G1` from now on) sits at a higher level than the first group. This group contains the components `esp_hw_support`, `esp_system`, `newlib`, `spi_flash`,
+`freertos`, `log`, and `heap`. Like the first group, circular dependencies within the group are allowed; and being at a higher level, dependency on the first group
+is allowed. These components represent software mechanisms essential to building other components.
+
+## Descriptions
+
+The following is a short description of the components mentioned above.
+
+### `G0` Components
+
+#### `hal`
+
+Contains the hardware abstraction layer and low-level operation implementations for the various peripherals. The low-level functions assign meaningful names to register-level manipulations; the hardware abstraction provide operations one level above this, grouping these low-level functions
+into routines that achieve a meaningful action or state of the peripheral.
+
+Example:
+
+- `spi_flash_ll_set_address` is a low-level function part of the hardware abstraction `spi_flash_hal_read_block`
+
+#### `arch`
+
+Contains low-level architecture operations and definitions, including those for customizations (can be thought of on the same level as the low-level functions of `hal`).
+This can also contain files provided by the architecture vendor.
+
+Example:
+
+- `xt_set_exception_handler`
+- `rv_utils_intr_enable`
+- `ERI_PERFMON_MAX`
+
+#### `esp_common`
+
+Contains hardware-agnostic definitions, constants, macros, utilities, 'pure' and/or algorithmic functions that is useable by all other components (that is, barring there being a more appropriate component to put them in).
+
+Example:
+
+- `BIT(nr)` and other bit manipulation utilities in the future
+- `IDF_DEPRECATED(REASON)`
+- `ESP_IDF_VERSION_MAJOR`
+
+#### `soc`
+
+Contains description of the underlying hardware: register structure, addresses, pins, capabilities, etc.
+
+Example:
+
+- `DR_REG_DPORT_BASE`
+- `SOC_MCPWM_SUPPORTED`
+- `uart_dev_s`
+
+#### `esp_rom`
+
+Contains headers, linker scripts, abstraction layer, patches, and other related files to ROM functions.
+
+Example:
+
+- `esp32.rom.eco3.ld`
+- `rom/aes.h`
+
+### `G1` Components
+
+#### `spi_flash`
+
+SPI flash device access implementation.
+
+#### `freertos`
+
+FreeRTOS port to targets supported by ESP-IDF.
+
+#### `log`
+
+Logging library.
+
+#### `heap`
+
+Heap implementation.
+
+#### `newlib`
+
+Some functions n the standard library are implemented here, especially those needing other `G1` components.
+
+Example:
+
+- `malloc` is implemented in terms of the component `heap`'s functions
+- `gettimeofday` is implemented in terms of system time in `esp_system`
+
+#### `esp_system`
+
+Contains implementation of system services and controls system behavior. The implementations
+here may take hardware resources and/or decide on a hardware state needed for support of a system service/feature/mechanism.
+Currently, this encompasses the following, but not limited to:
+
+- Startup and initialization
+- Panic and debug
+- Reset and reset reason
+- Task and interrupt watchdogs
+
+#### `esp_hw_support`
+
+Contains implementations that provide hardware operations, arbitration, or resource sharing, especially those that
+is used in the system. Unlike `esp_system`, implementations here do not decide on a hardware state or takes hardware resource, acting
+merely as facilitator to hardware access. Currently, this encompasses the following, but not limited to:
+
+- Interrupt allocation
+- Sleep functions
+- Memory functions (external SPIRAM, async memory, etc.)
+- Clock and clock control
+- Random generation
+- CPU utilities
+- MAC settings
+
+### `esp_hw_support` vs `esp_system`
+
+This section details list some implementations and the reason for placing it in either `esp_hw_support` or `esp_system`.
+
+#### `task_wdt.c` (`esp_system`) vs  `intr_alloc.c` (`esp_hw_support`)
+
+The task watchdog fits the definition of taking and configuring hardware resources (wdt, interrupt) for implementation of a system service/mechanism.
+
+This is in contrast with interrupt allocation that merely facilitates access to the underlying hardware for other implementations -
+drivers, user code, and even the task watchdog mentioned previously!
+
+#### `crosscore_int.c` (`esp_system`)
+
+The current implementation of crosscore interrupts is tightly coupled with a number of interrupt reasons
+associated with system services/mechanisms: REASON_YIELD (scheduler), REASON_FREQ_SWITCH (power management)
+REASON_PRINT_BACKTRACE (panic and debug).
+
+However, if an implementation exists that makes it possible to register an arbitrary interrupt reason - a
+lower level inter-processor call if you will, then this implementation is a good candidate for `esp_hw_support`.
+The current implementation in `esp_system` can then just register the interrupt reasons mentioned above.
+
+#### `esp_mac.h`, `esp_chip_info.h`, `esp_random.h` (`esp_hw_support`)
+
+The functions in these headers used to be in `esp_system.h`, but have been split-off.
+
+The remaining functions in `esp_system.h` are those that deal with system behavior, such
+as `esp_register_shutdown_handler`, or are proxy for other system components's APIs such as
+`esp_get_free_heap_size`.
+
+The functions split-off from `esp_system.h` are much more hardware manipulation oriented such as:
+`esp_read_mac`, `esp_random` and `esp_chip_info`.

components/app_trace/CMakeLists.txt

@@ -6,7 +6,25 @@ set(srcs
 
 set(include_dirs "include")
 
-if(CONFIG_SYSVIEW_ENABLE)
+set(priv_include_dirs "private_include" "port/include")
+
+if(CONFIG_APPTRACE_MEMBUFS_APPTRACE_PROTO_ENABLE)
+    list(APPEND srcs
+        "app_trace_membufs_proto.c")
+
+    if(CONFIG_IDF_TARGET_ARCH_XTENSA)
+        list(APPEND srcs
+            "port/xtensa/port.c")
+    endif()
+    if(CONFIG_IDF_TARGET_ARCH_RISCV)
+        list(APPEND srcs
+            "port/riscv/port.c")
+    endif()
+endif()
+list(APPEND srcs
+    "port/port_uart.c")
+
+if(CONFIG_APPTRACE_SV_ENABLE)
     list(APPEND include_dirs
         sys_view/Config
         sys_view/SEGGER
@@ -16,7 +34,7 @@ if(CONFIG_SYSVIEW_ENABLE)
         "sys_view/SEGGER/SEGGER_SYSVIEW.c"
         "sys_view/Sample/Config/SEGGER_SYSVIEW_Config_FreeRTOS.c"
         "sys_view/Sample/OS/SEGGER_SYSVIEW_FreeRTOS.c"
-        "sys_view/esp32/SEGGER_RTT_esp32.c"
+        "sys_view/esp/SEGGER_RTT_esp.c"
         "sys_view/ext/heap_trace_module.c"
         "sys_view/ext/logging.c")
 endif()
@@ -30,17 +48,31 @@ endif()
 
 idf_component_register(SRCS "${srcs}"
                        INCLUDE_DIRS "${include_dirs}"
-                       PRIV_REQUIRES soc esp_ipc
+                       PRIV_INCLUDE_DIRS "${priv_include_dirs}"
+                       # Requires "driver" for GPTimer in "SEGGER_SYSVIEW_Config_FreeRTOS.c"
+                       PRIV_REQUIRES soc driver
+                       REQUIRES esp_timer
                        LDFRAGMENTS linker.lf)
 
-
 if(CONFIG_APPTRACE_GCOV_ENABLE)
     # disable --coverage for this component, as it is used as transport
     # for gcov
     target_compile_options(${COMPONENT_LIB} PRIVATE "-fno-profile-arcs" "-fno-test-coverage")
-
     # Force app_trace to also appear later than gcov in link line
     idf_component_get_property(app_trace app_trace COMPONENT_LIB)
     target_link_libraries(${COMPONENT_LIB} INTERFACE
         "-Wl,--undefined=gcov_rtio_atexit" $<TARGET_FILE:${app_trace}> gcov $<TARGET_FILE:${app_trace}> c)
 endif()
+
+# This function adds a dependency on the given component if the component is included into the build.
+function(maybe_add_component component_name)
+    idf_build_get_property(components BUILD_COMPONENTS)
+    if(${component_name} IN_LIST components)
+        idf_component_get_property(lib_name ${component_name} COMPONENT_LIB)
+        target_link_libraries(${COMPONENT_LIB} PUBLIC ${lib_name})
+    endif()
+endfunction()
+
+if(CONFIG_APPTRACE_DEST_UART0 OR CONFIG_APPTRACE_DEST_UART1 OR CONFIG_APPTRACE_DEST_UART2)
+    maybe_add_component(driver)
+endif()

components/app_trace/Kconfig

@@ -1,32 +1,167 @@
 menu "Application Level Tracing"
 
-    choice APPTRACE_DESTINATION
-        prompt "Data Destination"
+    choice APPTRACE_DESTINATION1
+        prompt "Data Destination 1"
         default APPTRACE_DEST_NONE
         help
-            Select destination for application trace: trace memory or none (to disable).
+            Select destination for application trace: JTAG or none (to disable).
 
-        config APPTRACE_DEST_TRAX
-            bool "Trace memory"
+        config APPTRACE_DEST_JTAG
+            bool "JTAG"
+            select APPTRACE_DEST_TRAX if IDF_TARGET_ARCH_XTENSA
+            select APPTRACE_MEMBUFS_APPTRACE_PROTO_ENABLE
             select APPTRACE_ENABLE
+
         config APPTRACE_DEST_NONE
             bool "None"
     endchoice
 
-    config APPTRACE_ENABLE
+    config APPTRACE_DEST_UART
         bool
-        depends on !ESP32_TRAX && !ESP32S2_TRAX
+
+    config APPTRACE_DEST_UART_NOUSB
+        bool
+
+    choice APPTRACE_DESTINATION2
+        prompt "Data Destination 2"
+        default APPTRACE_DEST_UART_NONE
+        help
+            Select destination for application trace: UART(XX) or none (to disable).
+
+        config APPTRACE_DEST_UART0
+            bool "UART0"
+            select APPTRACE_ENABLE
+            select APPTRACE_DEST_UART
+            select APPTRACE_DEST_UART_NOUSB
+            depends on (ESP_CONSOLE_UART_NUM !=0)
+
+        config APPTRACE_DEST_UART1
+            bool "UART1"
+            select APPTRACE_ENABLE
+            select APPTRACE_DEST_UART
+            select APPTRACE_DEST_UART_NOUSB
+            depends on (ESP_CONSOLE_UART_NUM !=1)
+
+        config APPTRACE_DEST_UART2
+            bool "UART2"
+            select APPTRACE_ENABLE
+            select APPTRACE_DEST_UART
+            select APPTRACE_DEST_UART_NOUSB
+            depends on (ESP_CONSOLE_UART_NUM !=2) && !IDF_TARGET_ESP32C3 && !IDF_TARGET_ESP32S2 && !IDF_TARGET_ESP32H2
+
+        config APPTRACE_DEST_USB_CDC
+            bool "USB_CDC"
+            select APPTRACE_ENABLE
+            select APPTRACE_DEST_UART
+            depends on !ESP_CONSOLE_USB_CDC && (IDF_TARGET_ESP32C3 || IDF_TARGET_ESP32S3) && !USB_ENABLED
+
+        config APPTRACE_DEST_UART_NONE
+            bool "None"
+    endchoice
+
+    config APPTRACE_UART_TX_GPIO
+        int "UART TX on GPIO#"
+        depends on APPTRACE_DEST_UART_NOUSB
+        range 0 46
+        default 12 if IDF_TARGET_ESP32
+        default 12 if IDF_TARGET_ESP32C3
+        default 12
+        help
+            This GPIO is used for UART TX pin.
+
+    config APPTRACE_UART_RX_GPIO
+        int "UART RX on GPIO#"
+        depends on APPTRACE_DEST_UART_NOUSB
+        range 0 46
+        default 13 if IDF_TARGET_ESP32
+        default 13 if IDF_TARGET_ESP32C3
+        default 13
+        help
+            This GPIO is used for UART RX pin.
+
+    config APPTRACE_UART_BAUDRATE
+        int
+        prompt "UART baud rate" if APPTRACE_DEST_UART
+        depends on APPTRACE_DEST_UART
+        default 1000000
+        range 1200 8000000
+        range 1200 1000000
+        help
+            This baud rate is used for UART.
+
+            The app's maximum baud rate depends on the UART clock source. If Power Management is disabled,
+            the UART clock source is the APB clock and all baud rates in the available range will be sufficiently
+            accurate. If Power Management is enabled, REF_TICK clock source is used so the baud rate is divided
+            from 1MHz. Baud rates above 1Mbps are not possible and values between 500Kbps and 1Mbps may not be
+            accurate.
+
+    config APPTRACE_UART_RX_BUFF_SIZE
+        int
+        prompt "UART RX ring buffer size" if APPTRACE_DEST_UART
+        depends on APPTRACE_DEST_UART
+        default 128
+        range 64 32768
+        help
+            Size of the UART input ring buffer.
+            This size related to the baudrate, system tick frequency and amount of data to transfer.
+            The data placed to this buffer before sent out to the interface.
+
+    config APPTRACE_UART_TX_BUFF_SIZE
+        int
+        prompt "UART TX ring buffer size" if APPTRACE_DEST_UART
+        depends on APPTRACE_DEST_UART
+        default 4096
+        range 2048 32768
+        help
+            Size of the UART output ring buffer.
+            This size related to the baudrate, system tick frequency and amount of data to transfer.
+
+    config APPTRACE_UART_TX_MSG_SIZE
+        int
+        prompt "UART TX message size" if APPTRACE_DEST_UART
+        depends on APPTRACE_DEST_UART
+        default 128
+        range 64 32768
+        help
+            Maximum size of the single message to transfer.
+
+    config APPTRACE_UART_TASK_PRIO
+        int
+        prompt "UART Task Priority" if APPTRACE_DEST_UART
+        default 1
+        range 1 32
+        help
+            UART task priority. In case of high events rate,
+            this parameter could be changed up to (configMAX_PRIORITIES-1).
+
+    config APPTRACE_DEST_TRAX
+        bool
+        depends on IDF_TARGET_ARCH_XTENSA && !ESP32_TRAX && !ESP32S2_TRAX && !ESP32S3_TRAX
         select ESP32_MEMMAP_TRACEMEM
         select ESP32S2_MEMMAP_TRACEMEM
+        select ESP32S3_MEMMAP_TRACEMEM
         select ESP32_MEMMAP_TRACEMEM_TWOBANKS
         select ESP32S2_MEMMAP_TRACEMEM_TWOBANKS
+        select ESP32S3_MEMMAP_TRACEMEM_TWOBANKS
+        default n
+        help
+            Enables/disable TRAX tracing HW.
+
+    config APPTRACE_MEMBUFS_APPTRACE_PROTO_ENABLE
+        bool
+        default n
+        help
+            Enables/disable swapping memory buffers tracing protocol.
+
+    config APPTRACE_ENABLE
+        bool
         default n
         help
             Enables/disable application tracing module.
 
     config APPTRACE_LOCK_ENABLE
         bool
-        default !SYSVIEW_ENABLE
+        default !APPTRACE_SV_ENABLE
         help
             Enables/disable application tracing module internal sync lock.
 
@@ -41,16 +176,23 @@ menu "Application Level Tracing"
 
     config APPTRACE_POSTMORTEM_FLUSH_THRESH
         int "Threshold for flushing last trace data to host on panic"
-        depends on APPTRACE_DEST_TRAX
+        depends on APPTRACE_ENABLE
         range 0 16384
         default 0
         help
             Threshold for flushing last trace data to host on panic in post-mortem mode.
             This is minimal amount of data needed to perform flush. In bytes.
 
+    config APPTRACE_BUF_SIZE
+        int "Size of the apptrace buffer"
+        depends on APPTRACE_MEMBUFS_APPTRACE_PROTO_ENABLE && !APPTRACE_DEST_TRAX
+        default 16384
+        help
+            Size of the memory buffer for trace data in bytes.
+
     config APPTRACE_PENDING_DATA_SIZE_MAX
         int "Size of the pending data buffer"
-        depends on APPTRACE_DEST_TRAX
+        depends on APPTRACE_MEMBUFS_APPTRACE_PROTO_ENABLE
         default 0
         help
             Size of the buffer for events in bytes. It is useful for buffering events from
@@ -59,151 +201,180 @@ menu "Application Level Tracing"
 
     menu "FreeRTOS SystemView Tracing"
         depends on APPTRACE_ENABLE
-        config SYSVIEW_ENABLE
+        config APPTRACE_SV_ENABLE
             bool "SystemView Tracing Enable"
             depends on APPTRACE_ENABLE
             default n
             help
                 Enables supporrt for SEGGER SystemView tracing functionality.
 
-        choice SYSVIEW_TS_SOURCE
+        choice APPTRACE_SV_DEST
+            prompt "SystemView destination"
+            depends on APPTRACE_SV_ENABLE
+            default APPTRACE_SV_DEST_JTAG
+            help
+                SystemView witt transfer data trough defined interface.
+
+            config APPTRACE_SV_DEST_JTAG
+                bool "Data destination JTAG"
+                depends on  !PM_ENABLE && !APPTRACE_DEST_NONE
+                help
+                    Send SEGGER SystemView events through JTAG interface.
+
+            config APPTRACE_SV_DEST_UART
+                bool "Data destination UART"
+                depends on APPTRACE_DEST_UART
+                help
+                    Send SEGGER SystemView events through UART interface.
+
+        endchoice
+
+        choice APPTRACE_SV_CPU
+            prompt "CPU to trace"
+            depends on APPTRACE_SV_DEST_UART && !FREERTOS_UNICORE
+            default APPTRACE_SV_DEST_CPU_0
+            help
+                Define the CPU to trace by SystemView.
+
+            config APPTRACE_SV_DEST_CPU_0
+                bool "CPU0"
+                help
+                    Send SEGGER SystemView events for Pro CPU.
+
+            config APPTRACE_SV_DEST_CPU_1
+                bool "CPU1"
+                help
+                    Send SEGGER SystemView events for App CPU.
+
+        endchoice
+
+
+        choice APPTRACE_SV_TS_SOURCE
             prompt "Timer to use as timestamp source"
-            depends on SYSVIEW_ENABLE
-            default SYSVIEW_TS_SOURCE_CCOUNT if FREERTOS_UNICORE && !PM_ENABLE
-            default SYSVIEW_TS_SOURCE_TIMER_00 if !FREERTOS_UNICORE && !PM_ENABLE
-            default SYSVIEW_TS_SOURCE_ESP_TIMER if PM_ENABLE
+            depends on APPTRACE_SV_ENABLE
+            default APPTRACE_SV_TS_SOURCE_CCOUNT if FREERTOS_UNICORE && !PM_ENABLE && !IDF_TARGET_ESP32C3
+            default APPTRACE_SV_TS_SOURCE_GPTIMER if !FREERTOS_UNICORE && !PM_ENABLE && !IDF_TARGET_ESP32C3
+            default APPTRACE_SV_TS_SOURCE_ESP_TIMER if PM_ENABLE || IDF_TARGET_ESP32C3
             help
                 SystemView needs to use a hardware timer as the source of timestamps
                 when tracing. This option selects the timer for it.
 
-            config SYSVIEW_TS_SOURCE_CCOUNT
+            config APPTRACE_SV_TS_SOURCE_CCOUNT
                 bool "CPU cycle counter (CCOUNT)"
-                depends on FREERTOS_UNICORE && !PM_ENABLE
+                depends on FREERTOS_UNICORE && !PM_ENABLE && !IDF_TARGET_ESP32C3
 
-            config SYSVIEW_TS_SOURCE_TIMER_00
-                bool "Timer 0, Group 0"
-                depends on !PM_ENABLE
+            config APPTRACE_SV_TS_SOURCE_GPTIMER
+                bool "General Purpose Timer (Timer Group)"
+                depends on !PM_ENABLE && !IDF_TARGET_ESP32C3
 
-            config SYSVIEW_TS_SOURCE_TIMER_01
-                bool "Timer 1, Group 0"
-                depends on !PM_ENABLE
-
-            config SYSVIEW_TS_SOURCE_TIMER_10
-                bool "Timer 0, Group 1"
-                depends on !PM_ENABLE
-
-            config SYSVIEW_TS_SOURCE_TIMER_11
-                bool "Timer 1, Group 1"
-                depends on !PM_ENABLE
-
-            config SYSVIEW_TS_SOURCE_ESP_TIMER
+            config APPTRACE_SV_TS_SOURCE_ESP_TIMER
                 bool "esp_timer high resolution timer"
 
         endchoice
 
-        config SYSVIEW_MAX_TASKS
+        config APPTRACE_SV_MAX_TASKS
             int "Maximum supported tasks"
-            depends on SYSVIEW_ENABLE
+            depends on APPTRACE_SV_ENABLE
             range 1 64
             default 16
             help
                 Configures maximum supported tasks in sysview debug
 
-        config SYSVIEW_BUF_WAIT_TMO
+        config APPTRACE_SV_BUF_WAIT_TMO
             int "Trace buffer wait timeout"
-            depends on SYSVIEW_ENABLE
+            depends on APPTRACE_SV_ENABLE
             default 500
             help
                 Configures timeout (in us) to wait for free space in trace buffer.
                 Set to -1 to wait forever and avoid lost events.
 
-        config SYSVIEW_EVT_OVERFLOW_ENABLE
+        config APPTRACE_SV_EVT_OVERFLOW_ENABLE
             bool "Trace Buffer Overflow Event"
-            depends on SYSVIEW_ENABLE
+            depends on APPTRACE_SV_ENABLE
             default y
             help
                 Enables "Trace Buffer Overflow" event.
 
-        config SYSVIEW_EVT_ISR_ENTER_ENABLE
+        config APPTRACE_SV_EVT_ISR_ENTER_ENABLE
             bool "ISR Enter Event"
-            depends on SYSVIEW_ENABLE
+            depends on APPTRACE_SV_ENABLE
             default y
             help
                 Enables "ISR Enter" event.
 
-        config SYSVIEW_EVT_ISR_EXIT_ENABLE
+        config APPTRACE_SV_EVT_ISR_EXIT_ENABLE
             bool "ISR Exit Event"
-            depends on SYSVIEW_ENABLE
+            depends on APPTRACE_SV_ENABLE
             default y
             help
                 Enables "ISR Exit" event.
 
-        config SYSVIEW_EVT_ISR_TO_SCHEDULER_ENABLE
+        config APPTRACE_SV_EVT_ISR_TO_SCHED_ENABLE
             bool "ISR Exit to Scheduler Event"
-            depends on SYSVIEW_ENABLE
+            depends on APPTRACE_SV_ENABLE
             default y
             help
                 Enables "ISR to Scheduler" event.
 
-        config SYSVIEW_EVT_TASK_START_EXEC_ENABLE
+        config APPTRACE_SV_EVT_TASK_START_EXEC_ENABLE
             bool "Task Start Execution Event"
-            depends on SYSVIEW_ENABLE
+            depends on APPTRACE_SV_ENABLE
             default y
             help
                 Enables "Task Start Execution" event.
 
-        config SYSVIEW_EVT_TASK_STOP_EXEC_ENABLE
+        config APPTRACE_SV_EVT_TASK_STOP_EXEC_ENABLE
             bool "Task Stop Execution Event"
-            depends on SYSVIEW_ENABLE
+            depends on APPTRACE_SV_ENABLE
             default y
             help
                 Enables "Task Stop Execution" event.
 
-        config SYSVIEW_EVT_TASK_START_READY_ENABLE
+        config APPTRACE_SV_EVT_TASK_START_READY_ENABLE
             bool "Task Start Ready State Event"
-            depends on SYSVIEW_ENABLE
+            depends on APPTRACE_SV_ENABLE
             default y
             help
                 Enables "Task Start Ready State" event.
 
-        config SYSVIEW_EVT_TASK_STOP_READY_ENABLE
+        config APPTRACE_SV_EVT_TASK_STOP_READY_ENABLE
             bool "Task Stop Ready State Event"
-            depends on SYSVIEW_ENABLE
+            depends on APPTRACE_SV_ENABLE
             default y
             help
                 Enables "Task Stop Ready State" event.
 
-        config SYSVIEW_EVT_TASK_CREATE_ENABLE
+        config APPTRACE_SV_EVT_TASK_CREATE_ENABLE
             bool "Task Create Event"
-            depends on SYSVIEW_ENABLE
+            depends on APPTRACE_SV_ENABLE
             default y
             help
                 Enables "Task Create" event.
 
-        config SYSVIEW_EVT_TASK_TERMINATE_ENABLE
+        config APPTRACE_SV_EVT_TASK_TERMINATE_ENABLE
             bool "Task Terminate Event"
-            depends on SYSVIEW_ENABLE
+            depends on APPTRACE_SV_ENABLE
             default y
             help
                 Enables "Task Terminate" event.
 
-        config SYSVIEW_EVT_IDLE_ENABLE
+        config APPTRACE_SV_EVT_IDLE_ENABLE
             bool "System Idle Event"
-            depends on SYSVIEW_ENABLE
+            depends on APPTRACE_SV_ENABLE
             default y
             help
                 Enables "System Idle" event.
 
-        config SYSVIEW_EVT_TIMER_ENTER_ENABLE
+        config APPTRACE_SV_EVT_TIMER_ENTER_ENABLE
             bool "Timer Enter Event"
-            depends on SYSVIEW_ENABLE
+            depends on APPTRACE_SV_ENABLE
             default y
             help
                 Enables "Timer Enter" event.
 
-        config SYSVIEW_EVT_TIMER_EXIT_ENABLE
+        config APPTRACE_SV_EVT_TIMER_EXIT_ENABLE
             bool "Timer Exit Event"
-            depends on SYSVIEW_ENABLE
+            depends on APPTRACE_SV_ENABLE
             default y
             help
                 Enables "Timer Exit" event.
@@ -212,10 +383,17 @@ menu "Application Level Tracing"
 
     config APPTRACE_GCOV_ENABLE
         bool "GCOV to Host Enable"
-        depends on APPTRACE_ENABLE && !SYSVIEW_ENABLE
+        depends on APPTRACE_ENABLE && !APPTRACE_SV_ENABLE
         select ESP_DEBUG_STUBS_ENABLE
         default n
         help
             Enables support for GCOV data transfer to host.
 
+    config APPTRACE_GCOV_DUMP_TASK_STACK_SIZE
+        int "Gcov dump task stack size"
+        depends on APPTRACE_GCOV_ENABLE
+        default 2048
+        help
+            Configures stack size of Gcov dump task
+
 endmenu

components/app_trace/app_trace_membufs_proto.c

@@ -0,0 +1,372 @@
+/*
+ * SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#include <sys/param.h>
+#include <string.h>
+#include "sdkconfig.h"
+#include "esp_log.h"
+#include "esp_cpu.h"
+#include "esp_app_trace_membufs_proto.h"
+
+/** Trace data header. Every user data chunk is prepended with this header.
+ * User allocates block with esp_apptrace_buffer_get and then fills it with data,
+ * in multithreading environment it can happen that tasks gets buffer and then gets interrupted,
+ * so it is possible that user data are incomplete when  memory block is exposed to the host.
+ * In this case host SW will see that wr_sz < block_sz and will report error.
+ */
+typedef struct {
+#if CONFIG_APPTRACE_SV_ENABLE
+    uint8_t   block_sz; // size of allocated block for user data
+    uint8_t   wr_sz;    // size of actually written data
+#else
+    uint16_t   block_sz; // size of allocated block for user data
+    uint16_t   wr_sz;    // size of actually written data
+#endif
+} esp_tracedata_hdr_t;
+
+/** TODO: docs
+ */
+typedef struct {
+    uint16_t   block_sz; // size of allocated block for user data
+} esp_hostdata_hdr_t;
+
+#if CONFIG_APPTRACE_SV_ENABLE
+#define ESP_APPTRACE_USR_BLOCK_CORE(_cid_)          (0)
+#define ESP_APPTRACE_USR_BLOCK_LEN(_v_)             (_v_)
+#define ESP_APPTRACE_USR_DATA_LEN_MAX(_hw_data_)    255UL
+#else
+#define ESP_APPTRACE_USR_BLOCK_CORE(_cid_)      ((_cid_) << 15)
+#define ESP_APPTRACE_USR_BLOCK_LEN(_v_)         (~(1 << 15) & (_v_))
+#define ESP_APPTRACE_USR_DATA_LEN_MAX(_hw_data_)       (ESP_APPTRACE_INBLOCK(_hw_data_)->sz - sizeof(esp_tracedata_hdr_t))
+#endif
+#define ESP_APPTRACE_USR_BLOCK_RAW_SZ(_s_)     ((_s_) + sizeof(esp_tracedata_hdr_t))
+
+#define ESP_APPTRACE_INBLOCK_MARKER(_hw_data_)          ((_hw_data_)->state.markers[(_hw_data_)->state.in_block % 2])
+#define ESP_APPTRACE_INBLOCK_MARKER_UPD(_hw_data_, _v_)   do {(_hw_data_)->state.markers[(_hw_data_)->state.in_block % 2] += (_v_);}while(0)
+#define ESP_APPTRACE_INBLOCK(_hw_data_)             (&(_hw_data_)->blocks[(_hw_data_)->state.in_block % 2])
+
+const static char *TAG = "esp_apptrace";
+
+static uint32_t esp_apptrace_membufs_down_buffer_write_nolock(esp_apptrace_membufs_proto_data_t *proto, uint8_t *data, uint32_t size);
+
+
+esp_err_t esp_apptrace_membufs_init(esp_apptrace_membufs_proto_data_t *proto, const esp_apptrace_mem_block_t blocks_cfg[2])
+{
+    // disabled by default
+    esp_apptrace_rb_init(&proto->rb_down, NULL, 0);
+    // membufs proto init
+    for (unsigned i = 0; i < 2; i++) {
+        proto->blocks[i].start = blocks_cfg[i].start;
+        proto->blocks[i].sz = blocks_cfg[i].sz;
+        proto->state.markers[i] = 0;
+    }
+    proto->state.in_block = 0;
+#if CONFIG_APPTRACE_PENDING_DATA_SIZE_MAX > 0
+    esp_apptrace_rb_init(&proto->rb_pend, proto->pending_data,
+                        sizeof(proto->pending_data));
+#endif
+    return ESP_OK;
+}
+
+void esp_apptrace_membufs_down_buffer_config(esp_apptrace_membufs_proto_data_t *data, uint8_t *buf, uint32_t size)
+{
+    esp_apptrace_rb_init(&data->rb_down, buf, size);
+}
+
+// assumed to be protected by caller from multi-core/thread access
+static esp_err_t esp_apptrace_membufs_swap(esp_apptrace_membufs_proto_data_t *proto)
+{
+    int prev_block_num = proto->state.in_block % 2;
+    int new_block_num = prev_block_num ? (0) : (1);
+    esp_err_t res = ESP_OK;
+
+    res = proto->hw->swap_start(proto->state.in_block);
+    if (res != ESP_OK) {
+        return res;
+    }
+
+    proto->state.markers[new_block_num] = 0;
+    // switch to new block
+    proto->state.in_block++;
+
+    proto->hw->swap(new_block_num);
+
+    // handle data from host
+    esp_hostdata_hdr_t *hdr = (esp_hostdata_hdr_t *)proto->blocks[new_block_num].start;
+    // ESP_APPTRACE_LOGV("Host data %d, sz %d @ %p", proto->hw->host_data_pending(), hdr->block_sz, hdr);
+    if (proto->hw->host_data_pending() && hdr->block_sz > 0) {
+        // TODO: add support for multiple blocks from host, currently there is no need for that
+        uint8_t *p = proto->blocks[new_block_num].start + proto->blocks[new_block_num].sz;
+        ESP_APPTRACE_LOGD("Recvd %d bytes from host (@ 0x%x) [%x %x %x %x %x %x %x %x .. %x %x %x %x %x %x %x %x]",
+            hdr->block_sz, proto->blocks[new_block_num].start,
+            *(proto->blocks[new_block_num].start+0), *(proto->blocks[new_block_num].start+1),
+            *(proto->blocks[new_block_num].start+2), *(proto->blocks[new_block_num].start+3),
+            *(proto->blocks[new_block_num].start+4), *(proto->blocks[new_block_num].start+5),
+            *(proto->blocks[new_block_num].start+6), *(proto->blocks[new_block_num].start+7),
+            *(p-8), *(p-7), *(p-6), *(p-5), *(p-4), *(p-3), *(p-2), *(p-1));
+        uint32_t sz = esp_apptrace_membufs_down_buffer_write_nolock(proto, (uint8_t *)(hdr+1), hdr->block_sz);
+        if (sz != hdr->block_sz) {
+            ESP_APPTRACE_LOGE("Failed to write %d bytes to down buffer (%d %d)!", hdr->block_sz - sz, hdr->block_sz, sz);
+        }
+        hdr->block_sz = 0;
+    }
+#if CONFIG_APPTRACE_PENDING_DATA_SIZE_MAX > 0
+    // copy pending data to  block if any
+    while (proto->state.markers[new_block_num] < proto->blocks[new_block_num].sz) {
+        uint32_t read_sz = esp_apptrace_rb_read_size_get(&proto->rb_pend);
+        if (read_sz == 0) {
+            break; // no more data in pending buffer
+        }
+        if (read_sz > proto->blocks[new_block_num].sz - proto->state.markers[new_block_num]) {
+            read_sz = proto->blocks[new_block_num].sz - proto->state.markers[new_block_num];
+        }
+        uint8_t *ptr = esp_apptrace_rb_consume(&proto->rb_pend, read_sz);
+        if (!ptr) {
+            assert(false && "Failed to consume pended bytes!!");
+            break;
+        }
+        ESP_APPTRACE_LOGD("Pump %d pend bytes [%x %x %x %x : %x %x %x %x : %x %x %x %x : %x %x...%x %x]",
+            read_sz, *(ptr+0), *(ptr+1), *(ptr+2), *(ptr+3), *(ptr+4),
+            *(ptr+5), *(ptr+6), *(ptr+7), *(ptr+8), *(ptr+9), *(ptr+10), *(ptr+11), *(ptr+12), *(ptr+13), *(ptr+read_sz-2), *(ptr+read_sz-1));
+        memcpy(proto->blocks[new_block_num].start + proto->state.markers[new_block_num], ptr, read_sz);
+        proto->state.markers[new_block_num] += read_sz;
+    }
+#endif
+    proto->hw->swap_end(proto->state.in_block, proto->state.markers[prev_block_num]);
+    return res;
+}
+
+static esp_err_t esp_apptrace_membufs_swap_waitus(esp_apptrace_membufs_proto_data_t *proto, esp_apptrace_tmo_t *tmo)
+{
+    int res;
+
+    while ((res = esp_apptrace_membufs_swap(proto)) != ESP_OK) {
+        res = esp_apptrace_tmo_check(tmo);
+        if (res != ESP_OK) {
+            break;
+        }
+    }
+    return res;
+}
+
+uint8_t *esp_apptrace_membufs_down_buffer_get(esp_apptrace_membufs_proto_data_t *proto, uint32_t *size, esp_apptrace_tmo_t *tmo)
+{
+    uint8_t *ptr = NULL;
+
+    while (1) {
+        uint32_t sz = esp_apptrace_rb_read_size_get(&proto->rb_down);
+        if (sz != 0) {
+            *size = MIN(*size, sz);
+            ptr = esp_apptrace_rb_consume(&proto->rb_down, *size);
+            if (!ptr) {
+                assert(false && "Failed to consume bytes from down buffer!");
+            }
+            break;
+        }
+        // may need to flush
+        if (proto->hw->host_data_pending()) {
+            ESP_APPTRACE_LOGD("force flush");
+            int res = esp_apptrace_membufs_swap_waitus(proto, tmo);
+            if (res != ESP_OK) {
+                ESP_APPTRACE_LOGE("Failed to switch to another block to recv data from host!");
+                /*do not return error because data can be in down buffer already*/
+            }
+        } else {
+            // check tmo only if there is no data from host
+            int res = esp_apptrace_tmo_check(tmo);
+            if (res != ESP_OK) {
+                return NULL;
+            }
+        }
+    }
+    return ptr;
+}
+
+esp_err_t esp_apptrace_membufs_down_buffer_put(esp_apptrace_membufs_proto_data_t *proto, uint8_t *ptr, esp_apptrace_tmo_t *tmo)
+{
+    /* nothing todo */
+    return ESP_OK;
+}
+
+static uint32_t esp_apptrace_membufs_down_buffer_write_nolock(esp_apptrace_membufs_proto_data_t *proto, uint8_t *data, uint32_t size)
+{
+    uint32_t total_sz = 0;
+
+    while (total_sz < size) {
+        ESP_APPTRACE_LOGD("esp_apptrace_trax_down_buffer_write_nolock WRS %d-%d-%d %d", proto->rb_down.wr, proto->rb_down.rd,
+            proto->rb_down.cur_size, size);
+        uint32_t wr_sz = esp_apptrace_rb_write_size_get(&proto->rb_down);
+        if (wr_sz == 0) {
+            break;
+        }
+
+        if (wr_sz > size - total_sz) {
+            wr_sz = size - total_sz;
+        }
+        ESP_APPTRACE_LOGD("esp_apptrace_trax_down_buffer_write_nolock wr %d", wr_sz);
+        uint8_t *ptr = esp_apptrace_rb_produce(&proto->rb_down, wr_sz);
+        if (!ptr) {
+            assert(false && "Failed to produce bytes to down buffer!");
+        }
+        ESP_APPTRACE_LOGD("esp_apptrace_trax_down_buffer_write_nolock wr %d to 0x%x from 0x%x", wr_sz, ptr, data + total_sz + wr_sz);
+        memcpy(ptr, data + total_sz, wr_sz);
+        total_sz += wr_sz;
+        ESP_APPTRACE_LOGD("esp_apptrace_trax_down_buffer_write_nolock wr %d/%d", wr_sz, total_sz);
+    }
+    return total_sz;
+}
+
+static inline uint8_t *esp_apptrace_membufs_wait4buf(esp_apptrace_membufs_proto_data_t *proto, uint16_t size, esp_apptrace_tmo_t *tmo, int *pended)
+{
+    uint8_t *ptr = NULL;
+
+    int res = esp_apptrace_membufs_swap_waitus(proto, tmo);
+    if (res != ESP_OK) {
+        return NULL;
+    }
+#if CONFIG_APPTRACE_PENDING_DATA_SIZE_MAX > 0
+    // check if we still have pending data
+    if (esp_apptrace_rb_read_size_get(&proto->rb_pend) > 0) {
+        // if after block switch we still have pending data (not all pending data have been pumped to block)
+        // alloc new pending buffer
+        *pended = 1;
+        ptr = esp_apptrace_rb_produce(&proto->rb_pend, size);
+        if (!ptr) {
+            ESP_APPTRACE_LOGE("Failed to alloc pend buf 1: w-r-s %d-%d-%d!", proto->rb_pend.wr, proto->rb_pend.rd, proto->rb_pend.cur_size);
+        }
+    } else
+#endif
+    {
+        // update block pointers
+        if (ESP_APPTRACE_INBLOCK_MARKER(proto) + size > ESP_APPTRACE_INBLOCK(proto)->sz) {
+#if CONFIG_APPTRACE_PENDING_DATA_SIZE_MAX > 0
+            *pended = 1;
+            ptr = esp_apptrace_rb_produce(&proto->rb_pend, size);
+            if (ptr == NULL) {
+                ESP_APPTRACE_LOGE("Failed to alloc pend buf 2: w-r-s %d-%d-%d!", proto->rb_pend.wr, proto->rb_pend.rd, proto->rb_pend.cur_size);
+            }
+#endif
+        } else {
+            *pended = 0;
+            ptr = ESP_APPTRACE_INBLOCK(proto)->start + ESP_APPTRACE_INBLOCK_MARKER(proto);
+        }
+    }
+
+    return ptr;
+}
+
+static inline uint8_t *esp_apptrace_membufs_pkt_start(uint8_t *ptr, uint16_t size)
+{
+    // it is safe to use esp_cpu_get_core_id() in macro call because arg is used only once inside it
+    ((esp_tracedata_hdr_t *)ptr)->block_sz = ESP_APPTRACE_USR_BLOCK_CORE(esp_cpu_get_core_id()) | size;
+    ((esp_tracedata_hdr_t *)ptr)->wr_sz = 0;
+    return ptr + sizeof(esp_tracedata_hdr_t);
+}
+
+static inline void esp_apptrace_membufs_pkt_end(uint8_t *ptr)
+{
+    esp_tracedata_hdr_t *hdr = (esp_tracedata_hdr_t *)(ptr - sizeof(esp_tracedata_hdr_t));
+    // update written size
+    hdr->wr_sz = hdr->block_sz;
+}
+
+uint8_t *esp_apptrace_membufs_up_buffer_get(esp_apptrace_membufs_proto_data_t *proto, uint32_t size, esp_apptrace_tmo_t *tmo)
+{
+    uint8_t *buf_ptr = NULL;
+
+    if (size > ESP_APPTRACE_USR_DATA_LEN_MAX(proto)) {
+        ESP_APPTRACE_LOGE("Too large user data size %d!", size);
+        return NULL;
+    }
+
+    // check for data in the pending buffer
+#if CONFIG_APPTRACE_PENDING_DATA_SIZE_MAX > 0
+    if (esp_apptrace_rb_read_size_get(&proto->rb_pend) > 0) {
+        // if we have buffered data try to switch  block
+        esp_apptrace_membufs_swap(proto);
+        // if switch was successful, part or all pended data have been copied to  block
+    }
+    if (esp_apptrace_rb_read_size_get(&proto->rb_pend) > 0) {
+        // if we have buffered data alloc new pending buffer
+        ESP_APPTRACE_LOGD("Get %d bytes from PEND buffer", size);
+        buf_ptr = esp_apptrace_rb_produce(&proto->rb_pend, ESP_APPTRACE_USR_BLOCK_RAW_SZ(size));
+        if (buf_ptr == NULL) {
+            int pended_buf;
+            buf_ptr = esp_apptrace_membufs_wait4buf(proto, ESP_APPTRACE_USR_BLOCK_RAW_SZ(size), tmo, &pended_buf);
+            if (buf_ptr && !pended_buf) {
+                ESP_APPTRACE_LOGD("Get %d bytes from block", size);
+                // update cur block marker
+                ESP_APPTRACE_INBLOCK_MARKER_UPD(proto, ESP_APPTRACE_USR_BLOCK_RAW_SZ(size));
+            }
+        }
+    } else {
+#else
+    if (1) {
+#endif
+        if (ESP_APPTRACE_INBLOCK_MARKER(proto) + ESP_APPTRACE_USR_BLOCK_RAW_SZ(size) > ESP_APPTRACE_INBLOCK(proto)->sz) {
+            #if CONFIG_APPTRACE_PENDING_DATA_SIZE_MAX > 0
+            ESP_APPTRACE_LOGD("Block full. Get %d bytes from PEND buffer", size);
+            buf_ptr = esp_apptrace_rb_produce(&proto->rb_pend, ESP_APPTRACE_USR_BLOCK_RAW_SZ(size));
+            #endif
+            if (buf_ptr == NULL) {
+                int pended_buf;
+                ESP_APPTRACE_LOGD(" full. Get %d bytes from pend buffer", size);
+                buf_ptr = esp_apptrace_membufs_wait4buf(proto, ESP_APPTRACE_USR_BLOCK_RAW_SZ(size), tmo, &pended_buf);
+                if (buf_ptr && !pended_buf) {
+                    ESP_APPTRACE_LOGD("Got %d bytes from block", size);
+                    // update cur block marker
+                    ESP_APPTRACE_INBLOCK_MARKER_UPD(proto, ESP_APPTRACE_USR_BLOCK_RAW_SZ(size));
+                }
+            }
+        } else {
+            ESP_APPTRACE_LOGD("Get %d bytes from  buffer", size);
+            // fit to curr  nlock
+            buf_ptr = ESP_APPTRACE_INBLOCK(proto)->start + ESP_APPTRACE_INBLOCK_MARKER(proto);
+            // update cur block marker
+            ESP_APPTRACE_INBLOCK_MARKER_UPD(proto, ESP_APPTRACE_USR_BLOCK_RAW_SZ(size));
+        }
+    }
+    if (buf_ptr) {
+        buf_ptr = esp_apptrace_membufs_pkt_start(buf_ptr, size);
+    }
+
+    return buf_ptr;
+}
+
+esp_err_t esp_apptrace_membufs_up_buffer_put(esp_apptrace_membufs_proto_data_t *proto, uint8_t *ptr, esp_apptrace_tmo_t *tmo)
+{
+    esp_apptrace_membufs_pkt_end(ptr);
+    // TODO: mark block as busy in order not to re-use it for other tracing calls until it is completely written
+    // TODO: avoid potential situation when all memory is consumed by low prio tasks which can not complete writing due to
+    // higher prio tasks and the latter can not allocate buffers at all
+    // this is abnormal situation can be detected on host which will receive only uncompleted buffers
+    // workaround: use own memcpy which will kick-off dead tracing calls
+    return ESP_OK;
+}
+
+esp_err_t esp_apptrace_membufs_flush_nolock(esp_apptrace_membufs_proto_data_t *proto, uint32_t min_sz, esp_apptrace_tmo_t *tmo)
+{
+    int res = ESP_OK;
+
+    if (ESP_APPTRACE_INBLOCK_MARKER(proto) < min_sz) {
+        ESP_APPTRACE_LOGI("Ignore flush request for min %d bytes. Bytes in  block: %d.", min_sz, ESP_APPTRACE_INBLOCK_MARKER(proto));
+        return ESP_OK;
+    }
+    // switch  block while size of data (including that in pending buffer) is more than min size
+    while (ESP_APPTRACE_INBLOCK_MARKER(proto) > min_sz) {
+        ESP_APPTRACE_LOGD("Try to flush %d bytes. Wait until block switch for %lld us", ESP_APPTRACE_INBLOCK_MARKER(proto), tmo->tmo);
+        res = esp_apptrace_membufs_swap_waitus(proto, tmo);
+        if (res != ESP_OK) {
+            if (tmo->tmo != ESP_APPTRACE_TMO_INFINITE)
+                ESP_APPTRACE_LOGW("Failed to switch to another block in %lld us!", tmo->tmo);
+            else
+                ESP_APPTRACE_LOGE("Failed to switch to another block in %lld us!", tmo->tmo);
+            return res;
+        }
+    }
+
+    return res;
+}

components/app_trace/app_trace_util.c

@@ -1,48 +1,49 @@
-// Copyright 2017 Espressif Systems (Shanghai) PTE LTD
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
+/*
+ * SPDX-FileCopyrightText: 2017-2021 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
 //
 #include "freertos/FreeRTOS.h"
 #include "freertos/task.h"
 #include "esp_app_trace_util.h"
 #include "sdkconfig.h"
-#if CONFIG_IDF_TARGET_ESP32
-#include "esp32/clk.h"
-#elif CONFIG_IDF_TARGET_ESP32S2
-#include "esp32s2/clk.h"
-#elif CONFIG_IDF_TARGET_ESP32S3
-#include "esp32s3/clk.h"
-#elif CONFIG_IDF_TARGET_ESP32C3
-#include "esp32c3/clk.h"
+
+///////////////////////////////////////////////////////////////////////////////
+///////////////////////////////// Locks /////////////////////////////////////
+///////////////////////////////////////////////////////////////////////////////
+
+#if ESP_APPTRACE_PRINT_LOCK
+static esp_apptrace_lock_t s_log_lock = {.irq_stat = 0, .portmux = portMUX_INITIALIZER_UNLOCKED};
 #endif
 
+int esp_apptrace_log_lock(void)
+{
+#if ESP_APPTRACE_PRINT_LOCK
+    esp_apptrace_tmo_t tmo;
+    esp_apptrace_tmo_init(&tmo, ESP_APPTRACE_TMO_INFINITE);
+    int ret = esp_apptrace_lock_take(&s_log_lock, &tmo);
+    return ret;
+#else
+    return 0;
+#endif
+}
+
+void esp_apptrace_log_unlock(void)
+{
+ #if ESP_APPTRACE_PRINT_LOCK
+    esp_apptrace_lock_give(&s_log_lock);
+#endif
+}
+
 ///////////////////////////////////////////////////////////////////////////////
 ///////////////////////////////// TIMEOUT /////////////////////////////////////
 ///////////////////////////////////////////////////////////////////////////////
 
-#define ESP_APPTRACE_CPUTICKS2US(_t_, _cpu_freq_)       ((_t_)/(_cpu_freq_/1000000))
-#define ESP_APPTRACE_US2CPUTICKS(_t_, _cpu_freq_)       ((_t_)*(_cpu_freq_/1000000))
-
 esp_err_t esp_apptrace_tmo_check(esp_apptrace_tmo_t *tmo)
 {
-    int cpu_freq = esp_clk_cpu_freq();
-    if (tmo->tmo != ESP_APPTRACE_TMO_INFINITE) {
-        unsigned cur = portGET_RUN_TIME_COUNTER_VALUE();
-        if (tmo->start <= cur) {
-            tmo->elapsed = ESP_APPTRACE_CPUTICKS2US(cur - tmo->start, cpu_freq);
-        } else {
-            tmo->elapsed = ESP_APPTRACE_CPUTICKS2US(0xFFFFFFFF - tmo->start + cur, cpu_freq);
-        }
+    if (tmo->tmo != (int64_t)-1) {
+        tmo->elapsed = esp_timer_get_time() - tmo->start;
         if (tmo->elapsed >= tmo->tmo) {
             return ESP_ERR_TIMEOUT;
         }
@@ -59,22 +60,24 @@ esp_err_t esp_apptrace_lock_take(esp_apptrace_lock_t *lock, esp_apptrace_tmo_t *
     int res;
 
     while (1) {
+        //Todo: Replace the current locking mechanism and int_state with portTRY_ENTER_CRITICAL() instead.
         // do not overwrite lock->int_state before we actually acquired the mux
-        unsigned int_state = portENTER_CRITICAL_NESTED();
-        // FIXME: if mux is busy it is not good idea to loop during the whole tmo with disabled IRQs.
-        // So we check mux state using zero tmo, restore IRQs and let others tasks/IRQs to run on this CPU
-        // while we are doing our own tmo check.
-#ifdef CONFIG_FREERTOS_PORTMUX_DEBUG
-        bool success = vPortCPUAcquireMutexTimeout(&lock->mux, 0, __FUNCTION__, __LINE__);
+#if CONFIG_FREERTOS_SMP
+        unsigned int_state = portDISABLE_INTERRUPTS();
 #else
-        bool success = vPortCPUAcquireMutexTimeout(&lock->mux, 0);
+        unsigned int_state = portSET_INTERRUPT_MASK_FROM_ISR();
 #endif
+        bool success = spinlock_acquire(&lock->mux, 0);
         if (success) {
             lock->int_state = int_state;
             return ESP_OK;
         }
-        portEXIT_CRITICAL_NESTED(int_state);
-        // we can be preempted from this place till the next call (above) to portENTER_CRITICAL_NESTED()
+#if CONFIG_FREERTOS_SMP
+        portRESTORE_INTERRUPTS(int_state);
+#else
+        portCLEAR_INTERRUPT_MASK_FROM_ISR(int_state);
+#endif
+        // we can be preempted from this place till the next call (above) to portSET_INTERRUPT_MASK_FROM_ISR()
         res = esp_apptrace_tmo_check(tmo);
         if (res != ESP_OK) {
             break;
@@ -89,12 +92,12 @@ esp_err_t esp_apptrace_lock_give(esp_apptrace_lock_t *lock)
     unsigned int_state = lock->int_state;
     // after call to the following func we can not be sure that lock->int_state
     // is not overwritten by other CPU who has acquired the mux just after we released it. See esp_apptrace_lock_take().
-#ifdef CONFIG_FREERTOS_PORTMUX_DEBUG
-    vPortCPUReleaseMutex(&lock->mux, __FUNCTION__, __LINE__);
+    spinlock_release(&lock->mux);
+#if CONFIG_FREERTOS_SMP
+    portRESTORE_INTERRUPTS(int_state);
 #else
-    vPortCPUReleaseMutex(&lock->mux);
+    portCLEAR_INTERRUPT_MASK_FROM_ISR(int_state);
 #endif
-    portEXIT_CRITICAL_NESTED(int_state);
     return ESP_OK;
 }
 

components/app_trace/component.mk

@@ -1,33 +0,0 @@
-#
-# Component Makefile
-#
-
-COMPONENT_SRCDIRS := .
-
-COMPONENT_ADD_INCLUDEDIRS = include
-
-COMPONENT_ADD_LDFLAGS = -lapp_trace
-
-ifdef CONFIG_SYSVIEW_ENABLE
-COMPONENT_ADD_INCLUDEDIRS += \
-	sys_view/Config \
-	sys_view/SEGGER \
-	sys_view/Sample/OS
-
-COMPONENT_SRCDIRS += \
-	gcov \
-	sys_view/SEGGER \
-	sys_view/Sample/OS \
-	sys_view/Sample/Config \
-	sys_view/esp32 \
-	sys_view/ext
-else
-ifdef CONFIG_APPTRACE_GCOV_ENABLE
-# do not produce gcov info for this module, it is used as transport for gcov
-CFLAGS := $(subst --coverage,,$(CFLAGS))
-COMPONENT_ADD_LDFLAGS += -Wl,--undefined=gcov_rtio_atexit
-COMPONENT_SRCDIRS += gcov
-endif
-endif
-
-COMPONENT_ADD_LDFRAGMENTS += linker.lf

components/app_trace/gcov/gcov_rtio.c

@@ -1,16 +1,8 @@
-// Copyright 2017 Espressif Systems (Shanghai) PTE LTD
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
+/*
+ * SPDX-FileCopyrightText: 2017-2021 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
 
 // This module implements runtime file I/O API for GCOV.
 
@@ -19,7 +11,6 @@
 #include "freertos/FreeRTOS.h"
 #include "freertos/task.h"
 #include "freertos/semphr.h"
-#include "soc/cpu.h"
 #include "soc/timer_periph.h"
 #include "esp_app_trace.h"
 #include "esp_freertos_hooks.h"
@@ -36,7 +27,6 @@
 
 #define ESP_GCOV_DOWN_BUF_SIZE  4200
 
-#define LOG_LOCAL_LEVEL CONFIG_LOG_DEFAULT_LEVEL
 #include "esp_log.h"
 const static char *TAG = "esp_gcov_rtio";
 static volatile bool s_create_gcov_task = false;
@@ -61,18 +51,11 @@ void gcov_dump_task(void *pvParameter)
     }
     ESP_EARLY_LOGV(TAG, "Config apptrace down buf");
     esp_apptrace_down_buffer_config(down_buf, ESP_GCOV_DOWN_BUF_SIZE);
-    /* we are directing the std outputs to the fake ones in order to reduce stack usage */
-    FILE *old_stderr = stderr;
-    FILE *old_stdout = stdout;
-    stderr = (FILE *) &__sf_fake_stderr;
-    stdout = (FILE *) &__sf_fake_stdout;
     ESP_EARLY_LOGV(TAG, "Dump data...");
     __gcov_dump();
     // reset dump status to allow incremental data accumulation
     __gcov_reset();
     free(down_buf);
-    stderr = old_stderr;
-    stdout = old_stdout;
     ESP_EARLY_LOGV(TAG, "Finish file transfer session");
     dump_result = esp_apptrace_fstop(ESP_APPTRACE_DEST_TRAX);
     if (dump_result != ESP_OK) {
@@ -93,7 +76,8 @@ gcov_exit:
 void gcov_create_task(void *arg)
 {
     ESP_EARLY_LOGV(TAG, "%s", __FUNCTION__);
-    xTaskCreatePinnedToCore(&gcov_dump_task, "gcov_dump_task", 2048, (void *)&s_gcov_task_running, configMAX_PRIORITIES - 1, NULL, 0);
+    xTaskCreatePinnedToCore(&gcov_dump_task, "gcov_dump_task", CONFIG_APPTRACE_GCOV_DUMP_TASK_STACK_SIZE,
+		(void *)&s_gcov_task_running, configMAX_PRIORITIES - 1, NULL, 0);
 }
 
 void gcov_create_task_tick_hook(void)

components/app_trace/heap_trace_tohost.c

@@ -1,23 +1,15 @@
-// Copyright 2018 Espressif Systems (Shanghai) PTE LTD
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
+/*
+ * SPDX-FileCopyrightText: 2018-2021 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
 #include <sdkconfig.h>
 
 #define HEAP_TRACE_SRCFILE /* don't warn on inclusion here */
 #include "esp_heap_trace.h"
 #undef HEAP_TRACE_SRCFILE
 
-#if CONFIG_SYSVIEW_ENABLE
+#if CONFIG_APPTRACE_SV_ENABLE
 #include "esp_app_trace.h"
 #include "esp_sysview_trace.h"
 #endif
@@ -26,7 +18,7 @@
 
 #ifdef CONFIG_HEAP_TRACING_TOHOST
 
-#if !CONFIG_SYSVIEW_ENABLE
+#if !CONFIG_APPTRACE_SV_ENABLE
 #error None of the heap tracing backends is enabled! You must enable SystemView compatible tracing to use this feature.
 #endif
 
@@ -42,7 +34,7 @@ esp_err_t heap_trace_init_tohost(void)
 
 esp_err_t heap_trace_start(heap_trace_mode_t mode_param)
 {
-#if CONFIG_SYSVIEW_ENABLE
+#if CONFIG_APPTRACE_SV_ENABLE
     esp_err_t ret = esp_sysview_heap_trace_start((uint32_t)-1);
     if (ret != ESP_OK) {
         return ret;
@@ -55,7 +47,7 @@ esp_err_t heap_trace_start(heap_trace_mode_t mode_param)
 esp_err_t heap_trace_stop(void)
 {
     esp_err_t ret = ESP_ERR_NOT_SUPPORTED;
-#if CONFIG_SYSVIEW_ENABLE
+#if CONFIG_APPTRACE_SV_ENABLE
     ret = esp_sysview_heap_trace_stop();
 #endif
     s_tracing = false;
@@ -88,7 +80,7 @@ static IRAM_ATTR void record_allocation(const heap_trace_record_t *record)
     if (!s_tracing) {
         return;
     }
-#if CONFIG_SYSVIEW_ENABLE
+#if CONFIG_APPTRACE_SV_ENABLE
     esp_sysview_heap_trace_alloc(record->address, record->size, record->alloced_by);
 #endif
 }
@@ -103,7 +95,7 @@ static IRAM_ATTR void record_free(void *p, void **callers)
     if (!s_tracing) {
         return;
     }
-#if CONFIG_SYSVIEW_ENABLE
+#if CONFIG_APPTRACE_SV_ENABLE
     esp_sysview_heap_trace_free(p, callers);
 #endif
 }

components/app_trace/host_file_io.c

@@ -1,16 +1,8 @@
-// Copyright 2017 Espressif Systems (Shanghai) PTE LTD
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
+/*
+ * SPDX-FileCopyrightText: 2017-2021 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
 //
 // Hot It Works
 // ************
@@ -27,7 +19,6 @@
 
 #if CONFIG_APPTRACE_ENABLE
 
-#define LOG_LOCAL_LEVEL CONFIG_LOG_DEFAULT_LEVEL
 #include "esp_log.h"
 const static char *TAG = "esp_host_file_io";
 

components/app_trace/include/esp_app_trace.h

@@ -1,16 +1,8 @@
-// Copyright 2017 Espressif Systems (Shanghai) PTE LTD
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
+/*
+ * SPDX-FileCopyrightText: 2017-2021 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
 #ifndef ESP_APP_TRACE_H_
 #define ESP_APP_TRACE_H_
 
@@ -26,8 +18,11 @@ extern "C" {
  * Application trace data destinations bits.
  */
 typedef enum {
-    ESP_APPTRACE_DEST_TRAX = 0x1,	///< JTAG destination
-    ESP_APPTRACE_DEST_UART0 = 0x2,	///< UART destination
+    ESP_APPTRACE_DEST_JTAG = 1,                         ///< JTAG destination
+    ESP_APPTRACE_DEST_TRAX = ESP_APPTRACE_DEST_JTAG, 	///< xxx_TRAX name is obsolete, use more common xxx_JTAG
+    ESP_APPTRACE_DEST_UART,	                            ///< UART destination
+    ESP_APPTRACE_DEST_MAX = ESP_APPTRACE_DEST_UART+1,
+    ESP_APPTRACE_DEST_NUM
 } esp_apptrace_dest_t;
 
 /**
@@ -41,7 +36,7 @@ esp_err_t esp_apptrace_init(void);
 
 /**
  * @brief Configures down buffer.
- *        @note Needs to be called before initiating any data transfer using esp_apptrace_buffer_get and esp_apptrace_write.
+ *        @note Needs to be called before attempting to receive any data using esp_apptrace_down_buffer_get and esp_apptrace_read.
  *              This function does not protect internal data by lock.
  *
  * @param buf Address of buffer to use for down channel (host to target) data.
@@ -51,7 +46,7 @@ void esp_apptrace_down_buffer_config(uint8_t *buf, uint32_t size);
 
 /**
  * @brief Allocates buffer for trace data.
- *        After data in buffer are ready to be sent off esp_apptrace_buffer_put must be called to indicate it.
+ *        Once the data in the buffer is ready to be sent, esp_apptrace_buffer_put must be called to indicate it.
  *
  * @param dest Indicates HW interface to send data.
  * @param size Size of data to write to trace buffer.
@@ -62,8 +57,8 @@ void esp_apptrace_down_buffer_config(uint8_t *buf, uint32_t size);
 uint8_t *esp_apptrace_buffer_get(esp_apptrace_dest_t dest, uint32_t size, uint32_t tmo);
 
 /**
- * @brief Indicates that the data in buffer are ready to be sent off.
- *        This function is a counterpart of and must be preceeded by esp_apptrace_buffer_get.
+ * @brief Indicates that the data in the buffer is ready to be sent.
+ *        This function is a counterpart of and must be preceded by esp_apptrace_buffer_get.
  *
  * @param dest Indicates HW interface to send data. Should be identical to the same parameter in call to esp_apptrace_buffer_get.
  * @param ptr  Address of trace buffer to release. Should be the value returned by call to esp_apptrace_buffer_get.
@@ -86,7 +81,7 @@ esp_err_t esp_apptrace_buffer_put(esp_apptrace_dest_t dest, uint8_t *ptr, uint32
 esp_err_t esp_apptrace_write(esp_apptrace_dest_t dest, const void *data, uint32_t size, uint32_t tmo);
 
 /**
- * @brief vprintf-like function to sent log messages to host via specified HW interface.
+ * @brief vprintf-like function to send log messages to host via specified HW interface.
  *
  * @param dest Indicates HW interface to send data.
  * @param tmo  Timeout for operation (in us). Use ESP_APPTRACE_TMO_INFINITE to wait indefinitely.
@@ -98,7 +93,7 @@ esp_err_t esp_apptrace_write(esp_apptrace_dest_t dest, const void *data, uint32_
 int esp_apptrace_vprintf_to(esp_apptrace_dest_t dest, uint32_t tmo, const char *fmt, va_list ap);
 
 /**
- * @brief vprintf-like function to sent log messages to host.
+ * @brief vprintf-like function to send log messages to host.
  *
  * @param fmt  Address of format string.
  * @param ap   List of arguments.
@@ -119,7 +114,7 @@ esp_err_t esp_apptrace_flush(esp_apptrace_dest_t dest, uint32_t tmo);
 
 /**
  * @brief Flushes remaining data in trace buffer to host without locking internal data.
- *        This is special version of esp_apptrace_flush which should be called from panic handler.
+ *        This is a special version of esp_apptrace_flush which should be called from panic handler.
  *
  * @param dest   Indicates HW interface to flush data on.
  * @param min_sz Threshold for flushing data. If current filling level is above this value, data will be flushed. TRAX destinations only.
@@ -143,7 +138,7 @@ esp_err_t esp_apptrace_read(esp_apptrace_dest_t dest, void *data, uint32_t *size
 
 /**
  * @brief Retrieves incoming data buffer if any.
- *        After data in buffer are processed esp_apptrace_down_buffer_put must be called to indicate it.
+ *        Once data in the buffer is processed, esp_apptrace_down_buffer_put must be called to indicate it.
  *
  * @param dest Indicates HW interface to receive data.
  * @param size Address to store size of available data in down buffer. Must be initialized with requested value.
@@ -154,8 +149,8 @@ esp_err_t esp_apptrace_read(esp_apptrace_dest_t dest, void *data, uint32_t *size
 uint8_t *esp_apptrace_down_buffer_get(esp_apptrace_dest_t dest, uint32_t *size, uint32_t tmo);
 
 /**
- * @brief Indicates that the data in down buffer are processed.
- *        This function is a counterpart of and must be preceeded by esp_apptrace_down_buffer_get.
+ * @brief Indicates that the data in the down buffer is processed.
+ *        This function is a counterpart of and must be preceded by esp_apptrace_down_buffer_get.
  *
  * @param dest Indicates HW interface to receive data. Should be identical to the same parameter in call to esp_apptrace_down_buffer_get.
  * @param ptr  Address of trace buffer to release. Should be the value returned by call to esp_apptrace_down_buffer_get.
@@ -250,7 +245,7 @@ int esp_apptrace_fseek(esp_apptrace_dest_t dest, void *stream, long offset, int
 int esp_apptrace_ftell(esp_apptrace_dest_t dest, void *stream);
 
 /**
- * @brief Indicates to the host that all file operations are completed.
+ * @brief Indicates to the host that all file operations are complete.
  *		  This function should be called after all file operations are finished and
  *		  indicate to the host that it can perform cleanup operations (close open files etc.).
  *

components/app_trace/include/esp_app_trace_util.h

@@ -1,16 +1,8 @@
-// Copyright 2017 Espressif Systems (Shanghai) PTE LTD
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
+/*
+ * SPDX-FileCopyrightText: 2017-2021 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
 #ifndef ESP_APP_TRACE_UTIL_H_
 #define ESP_APP_TRACE_UTIL_H_
 
@@ -20,6 +12,7 @@ extern "C" {
 
 #include "freertos/FreeRTOS.h"
 #include "esp_err.h"
+#include "esp_timer.h"
 
 /** Infinite waiting timeout */
 #define ESP_APPTRACE_TMO_INFINITE               ((uint32_t)-1)
@@ -30,9 +23,9 @@ extern "C" {
  *  periodically to check timeout for expiration.
  */
 typedef struct {
-    uint32_t   start;   ///< time interval start (in CPU ticks)
-    uint32_t   tmo;     ///< timeout value (in us)
-    uint32_t   elapsed; ///< elapsed time (in us)
+    int64_t   start;   ///< time interval start (in us)
+    int64_t   tmo;     ///< timeout value (in us)
+    int64_t   elapsed; ///< elapsed time (in us)
 } esp_apptrace_tmo_t;
 
 /**
@@ -43,28 +36,28 @@ typedef struct {
 */
 static inline void esp_apptrace_tmo_init(esp_apptrace_tmo_t *tmo, uint32_t user_tmo)
 {
-    tmo->start = portGET_RUN_TIME_COUNTER_VALUE();
-    tmo->tmo = user_tmo;
+    tmo->start = esp_timer_get_time();
+    tmo->tmo = user_tmo == ESP_APPTRACE_TMO_INFINITE ? (int64_t)-1 : (int64_t)user_tmo;
     tmo->elapsed = 0;
 }
 
 /**
  * @brief Checks timeout for expiration.
  *
- * @param tmo Pointer to timeout structure to be initialized.
+ * @param tmo Pointer to timeout structure.
  *
- * @return ESP_OK on success, otherwise \see esp_err_t
+ * @return number of remaining us till tmo.
  */
 esp_err_t esp_apptrace_tmo_check(esp_apptrace_tmo_t *tmo);
 
 static inline uint32_t esp_apptrace_tmo_remaining_us(esp_apptrace_tmo_t *tmo)
 {
-    return tmo->tmo != ESP_APPTRACE_TMO_INFINITE ? (tmo->elapsed - tmo->tmo) : ESP_APPTRACE_TMO_INFINITE;
+    return tmo->tmo != (int64_t)-1 ? (tmo->elapsed - tmo->tmo) : ESP_APPTRACE_TMO_INFINITE;
 }
 
 /** Tracing module synchronization lock */
 typedef struct {
-    portMUX_TYPE mux;
+    spinlock_t mux;
     unsigned int_state;
 } esp_apptrace_lock_t;
 
@@ -75,7 +68,7 @@ typedef struct {
  */
 static inline void esp_apptrace_lock_init(esp_apptrace_lock_t *lock)
 {
-    vPortCPUInitializeMutex(&lock->mux);
+    portMUX_INITIALIZE(&lock->mux);
     lock->int_state = 0;
 }
 
@@ -168,6 +161,30 @@ uint32_t esp_apptrace_rb_read_size_get(esp_apptrace_rb_t *rb);
  */
 uint32_t esp_apptrace_rb_write_size_get(esp_apptrace_rb_t *rb);
 
+int esp_apptrace_log_lock(void);
+void esp_apptrace_log_unlock(void);
+
+#define ESP_APPTRACE_LOG( format, ... )   \
+    do { \
+        esp_apptrace_log_lock(); \
+        esp_rom_printf(format, ##__VA_ARGS__); \
+        esp_apptrace_log_unlock(); \
+    } while(0)
+
+#define ESP_APPTRACE_LOG_LEV( _L_, level, format, ... )   \
+    do { \
+        if (LOG_LOCAL_LEVEL >= level) { \
+            ESP_APPTRACE_LOG(LOG_FORMAT(_L_, format), esp_log_early_timestamp(), TAG, ##__VA_ARGS__); \
+        } \
+    } while(0)
+
+#define ESP_APPTRACE_LOGE( format, ... )  ESP_APPTRACE_LOG_LEV(E, ESP_LOG_ERROR, format, ##__VA_ARGS__)
+#define ESP_APPTRACE_LOGW( format, ... )  ESP_APPTRACE_LOG_LEV(W, ESP_LOG_WARN, format, ##__VA_ARGS__)
+#define ESP_APPTRACE_LOGI( format, ... )  ESP_APPTRACE_LOG_LEV(I, ESP_LOG_INFO, format, ##__VA_ARGS__)
+#define ESP_APPTRACE_LOGD( format, ... )  ESP_APPTRACE_LOG_LEV(D, ESP_LOG_DEBUG, format, ##__VA_ARGS__)
+#define ESP_APPTRACE_LOGV( format, ... )  ESP_APPTRACE_LOG_LEV(V, ESP_LOG_VERBOSE, format, ##__VA_ARGS__)
+#define ESP_APPTRACE_LOGO( format, ... )  ESP_APPTRACE_LOG_LEV(E, ESP_LOG_NONE, format, ##__VA_ARGS__)
+
 #ifdef __cplusplus
 }
 #endif

components/app_trace/include/esp_sysview_trace.h

@@ -1,16 +1,8 @@
-// Copyright 2018 Espressif Systems (Shanghai) PTE LTD
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
+/*
+ * SPDX-FileCopyrightText: 2018-2021 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
 #ifndef ESP_SYSVIEW_TRACE_H_
 #define ESP_SYSVIEW_TRACE_H_
 
@@ -20,7 +12,7 @@ extern "C" {
 
 #include <stdarg.h>
 #include "esp_err.h"
-#include "SEGGER_RTT.h" // SEGGER_RTT_ESP32_Flush
+#include "SEGGER_RTT.h" // SEGGER_RTT_ESP_Flush
 #include "esp_app_trace_util.h" // ESP_APPTRACE_TMO_INFINITE
 
 /**
@@ -32,7 +24,7 @@ extern "C" {
  */
 static inline esp_err_t esp_sysview_flush(uint32_t tmo)
 {
-    SEGGER_RTT_ESP32_Flush(0, tmo);
+    SEGGER_RTT_ESP_Flush(0, tmo);
     return ESP_OK;
 }
 

components/app_trace/linker.lf

@@ -2,18 +2,22 @@
 archive: libapp_trace.a
 entries:
     app_trace (noflash)
+    port_uart (noflash)
     app_trace_util (noflash)
-    if SYSVIEW_ENABLE = y:
+    if APPTRACE_MEMBUFS_APPTRACE_PROTO_ENABLE:
+      app_trace_membufs_proto (noflash)
+    if APPTRACE_DEST_JTAG = y:
+      port (noflash)
+    if APPTRACE_SV_ENABLE = y:
       SEGGER_SYSVIEW (noflash)
-      SEGGER_RTT_esp32 (noflash)
+      SEGGER_RTT_esp (noflash)
       SEGGER_SYSVIEW_Config_FreeRTOS (noflash)
       SEGGER_SYSVIEW_FreeRTOS (noflash)
 
 [mapping:app_trace_driver]
 archive: libdriver.a
 entries:
-    if SYSVIEW_TS_SOURCE_TIMER_00 = y || SYSVIEW_TS_SOURCE_TIMER_01 = y
-       || SYSVIEW_TS_SOURCE_TIMER_10 = y || SYSVIEW_TS_SOURCE_TIMER_11 = y:
-      timer (noflash)
+    if APPTRACE_SV_TS_SOURCE_GPTIMER = y:
+      gptimer (noflash)
     else:
       * (default)

components/app_trace/port/include/esp_app_trace_port.h

@@ -0,0 +1,36 @@
+/*
+ * SPDX-FileCopyrightText: 2020-2022 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#ifndef ESP_APP_TRACE_PORT_H_
+#define ESP_APP_TRACE_PORT_H_
+
+#include "esp_app_trace_util.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/** Apptrace HW interface. */
+typedef struct {
+    esp_err_t (*init)(void *hw_data);
+    uint8_t *(*get_up_buffer)(void *hw_data, uint32_t, esp_apptrace_tmo_t *);
+    esp_err_t (*put_up_buffer)(void *hw_data, uint8_t *, esp_apptrace_tmo_t *);
+    esp_err_t (*flush_up_buffer_nolock)(void *hw_data, uint32_t, esp_apptrace_tmo_t *);
+    esp_err_t (*flush_up_buffer)(void *hw_data, esp_apptrace_tmo_t *);
+    void (*down_buffer_config)(void *hw_data, uint8_t *buf, uint32_t size);
+    uint8_t *(*get_down_buffer)(void *hw_data, uint32_t *, esp_apptrace_tmo_t *);
+    esp_err_t (*put_down_buffer)(void *hw_data, uint8_t *, esp_apptrace_tmo_t *);
+    bool (*host_is_connected)(void *hw_data);
+} esp_apptrace_hw_t;
+
+esp_apptrace_hw_t *esp_apptrace_jtag_hw_get(void **data);
+esp_apptrace_hw_t *esp_apptrace_uart_hw_get(int num, void **data);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

components/app_trace/port/port_uart.c

@@ -0,0 +1,348 @@
+/*
+ * SPDX-FileCopyrightText: 2017-2021 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#include "soc/soc.h"
+#include "esp_log.h"
+#include "esp_cpu.h"
+#include "esp_app_trace_port.h"
+
+#include "driver/uart.h"
+#include "hal/uart_ll.h"
+#include "string.h"
+#include "driver/gpio.h"
+
+
+#define APPTRACE_DEST_UART (CONFIG_APPTRACE_DEST_UART0 | CONFIG_APPTRACE_DEST_UART1 | CONFIG_APPTRACE_DEST_UART2)
+
+#define APP_TRACE_MAX_TX_BUFF_UART          CONFIG_APPTRACE_UART_TX_BUFF_SIZE
+#define APP_TRACE_MAX_TX_MSG_UART           CONFIG_APPTRACE_UART_TX_MSG_SIZE
+
+/** UART HW transport data */
+typedef struct {
+    uint8_t                             inited;
+#if CONFIG_APPTRACE_LOCK_ENABLE
+    esp_apptrace_lock_t                 lock;   // sync lock
+#endif
+    uart_port_t port_num;
+// TX data ring buffer
+    uint8_t *tx_data_buff;
+    int32_t tx_data_buff_in;
+    int32_t tx_data_buff_out;
+// TX message buffer
+    uint8_t *tx_msg_buff;
+    uint32_t tx_msg_buff_size;
+
+// RX message buffer
+    uint8_t *down_buffer;
+    uint32_t down_buffer_size;
+// Buffer overflow flags
+    bool message_buff_overflow;
+    bool circular_buff_overflow;
+} esp_apptrace_uart_data_t;
+
+#if APPTRACE_DEST_UART
+static esp_err_t esp_apptrace_uart_init(esp_apptrace_uart_data_t *hw_data);
+static esp_err_t esp_apptrace_uart_flush(esp_apptrace_uart_data_t *hw_data, esp_apptrace_tmo_t *tmo);
+static esp_err_t esp_apptrace_uart_flush_nolock(esp_apptrace_uart_data_t *hw_data, uint32_t min_sz, esp_apptrace_tmo_t *tmo);
+static uint8_t *esp_apptrace_uart_up_buffer_get(esp_apptrace_uart_data_t *hw_data, uint32_t size, esp_apptrace_tmo_t *tmo);
+static esp_err_t esp_apptrace_uart_up_buffer_put(esp_apptrace_uart_data_t *hw_data, uint8_t *ptr, esp_apptrace_tmo_t *tmo);
+static void esp_apptrace_uart_down_buffer_config(esp_apptrace_uart_data_t *hw_data, uint8_t *buf, uint32_t size);
+static uint8_t *esp_apptrace_uart_down_buffer_get(esp_apptrace_uart_data_t *hw_data, uint32_t *size, esp_apptrace_tmo_t *tmo);
+static esp_err_t esp_apptrace_uart_down_buffer_put(esp_apptrace_uart_data_t *hw_data, uint8_t *ptr, esp_apptrace_tmo_t *tmo);
+static bool esp_apptrace_uart_host_is_connected(esp_apptrace_uart_data_t *hw_data);
+
+#endif // APPTRACE_DEST_UART
+const static char *TAG = "esp_apptrace_uart";
+
+esp_apptrace_hw_t *esp_apptrace_uart_hw_get(int num, void **data)
+{
+    ESP_LOGD(TAG,"esp_apptrace_uart_hw_get - %i", num);
+#if APPTRACE_DEST_UART
+    static esp_apptrace_uart_data_t s_uart_hw_data = {
+    };
+    static esp_apptrace_hw_t s_uart_hw = {
+        .init = (esp_err_t (*)(void *))esp_apptrace_uart_init,
+        .get_up_buffer = (uint8_t *(*)(void *, uint32_t, esp_apptrace_tmo_t *))esp_apptrace_uart_up_buffer_get,
+        .put_up_buffer = (esp_err_t (*)(void *, uint8_t *, esp_apptrace_tmo_t *))esp_apptrace_uart_up_buffer_put,
+        .flush_up_buffer_nolock = (esp_err_t (*)(void *, uint32_t, esp_apptrace_tmo_t *))esp_apptrace_uart_flush_nolock,
+        .flush_up_buffer = (esp_err_t (*)(void *, esp_apptrace_tmo_t *))esp_apptrace_uart_flush,
+        .down_buffer_config = (void (*)(void *, uint8_t *, uint32_t ))esp_apptrace_uart_down_buffer_config,
+        .get_down_buffer = (uint8_t *(*)(void *, uint32_t *, esp_apptrace_tmo_t *))esp_apptrace_uart_down_buffer_get,
+        .put_down_buffer = (esp_err_t (*)(void *, uint8_t *, esp_apptrace_tmo_t *))esp_apptrace_uart_down_buffer_put,
+        .host_is_connected = (bool (*)(void *))esp_apptrace_uart_host_is_connected,
+    };
+    s_uart_hw_data.port_num = num;
+    *data = &s_uart_hw_data;
+    return &s_uart_hw;
+#else
+    return NULL;
+#endif
+}
+
+#if APPTRACE_DEST_UART
+
+static esp_err_t esp_apptrace_uart_lock(esp_apptrace_uart_data_t *hw_data, esp_apptrace_tmo_t *tmo)
+{
+#if CONFIG_APPTRACE_LOCK_ENABLE
+    esp_err_t ret = esp_apptrace_lock_take(&hw_data->lock, tmo);
+    if (ret != ESP_OK) {
+        return ESP_FAIL;
+    }
+#endif
+    return ESP_OK;
+}
+
+static esp_err_t esp_apptrace_uart_unlock(esp_apptrace_uart_data_t *hw_data)
+{
+    esp_err_t ret = ESP_OK;
+#if CONFIG_APPTRACE_LOCK_ENABLE
+    ret = esp_apptrace_lock_give(&hw_data->lock);
+#endif
+    return ret;
+}
+
+static inline void esp_apptrace_uart_hw_init(void)
+{
+    ESP_APPTRACE_LOGI("Initialized UART on CPU%d", esp_cpu_get_core_id());
+}
+
+
+/*****************************************************************************************/
+/***************************** Apptrace HW iface *****************************************/
+/*****************************************************************************************/
+
+static esp_err_t esp_apptrace_send_uart_data(esp_apptrace_uart_data_t *hw_data, const char *data, uint32_t size, esp_apptrace_tmo_t *tmo)
+{
+    esp_err_t res = esp_apptrace_uart_lock(hw_data, tmo);
+    if (res != ESP_OK) {
+        return res;
+    }
+    // We store current out position to handle it without lock
+    volatile int32_t out_position = hw_data->tx_data_buff_out;
+
+    int len_free = APP_TRACE_MAX_TX_BUFF_UART - (hw_data->tx_data_buff_in - out_position);
+    if (out_position > hw_data->tx_data_buff_in) {
+        len_free = out_position - hw_data->tx_data_buff_in;
+    }
+    int check_len = APP_TRACE_MAX_TX_BUFF_UART - hw_data->tx_data_buff_in;
+    if (size <= len_free)
+    {
+        if ( check_len >= size) {
+            memcpy(&hw_data->tx_data_buff[hw_data->tx_data_buff_in], data, size);
+            hw_data->tx_data_buff_in += size;
+        } else {
+            memcpy(&hw_data->tx_data_buff[hw_data->tx_data_buff_in], data, APP_TRACE_MAX_TX_BUFF_UART - hw_data->tx_data_buff_in);
+            memcpy(&hw_data->tx_data_buff[0], &data[APP_TRACE_MAX_TX_BUFF_UART - hw_data->tx_data_buff_in], size - (APP_TRACE_MAX_TX_BUFF_UART - hw_data->tx_data_buff_in));
+            hw_data->tx_data_buff_in = size - (APP_TRACE_MAX_TX_BUFF_UART - hw_data->tx_data_buff_in);
+        }
+        if (hw_data->tx_data_buff_in >= APP_TRACE_MAX_TX_BUFF_UART) {
+            hw_data->tx_data_buff_in = 0;
+        }
+    } else {
+        hw_data->circular_buff_overflow = true;
+    }
+
+    if (esp_apptrace_uart_unlock(hw_data) != ESP_OK) {
+        assert(false && "Failed to unlock apptrace data!");
+    }
+
+    return ESP_OK;
+}
+
+static void send_buff_data(esp_apptrace_uart_data_t *hw_data, esp_apptrace_tmo_t *tmo)
+{
+    if (hw_data->tx_data_buff_in == hw_data->tx_data_buff_out) {
+        return;
+    }
+    // We store current in position to handle it without lock
+    volatile int32_t in_position = hw_data->tx_data_buff_in;
+    if (in_position > hw_data->tx_data_buff_out) {
+        int bytes_sent = uart_write_bytes(hw_data->port_num, &hw_data->tx_data_buff[hw_data->tx_data_buff_out], in_position - hw_data->tx_data_buff_out);
+        hw_data->tx_data_buff_out += bytes_sent;
+    } else {
+        int bytes_sent = uart_write_bytes(hw_data->port_num, &hw_data->tx_data_buff[hw_data->tx_data_buff_out], APP_TRACE_MAX_TX_BUFF_UART - hw_data->tx_data_buff_out);
+        hw_data->tx_data_buff_out += bytes_sent;
+        if (hw_data->tx_data_buff_out >= APP_TRACE_MAX_TX_BUFF_UART) {
+            hw_data->tx_data_buff_out = 0;
+        }
+    }
+}
+
+#define APP_TRACE_UART_STOP_WAIT_TMO    1000000 //us
+
+static void esp_apptrace_send_uart_tx_task(void *arg)
+{
+    esp_apptrace_uart_data_t *hw_data = (esp_apptrace_uart_data_t *)arg;
+    esp_apptrace_tmo_t tmo;
+    esp_apptrace_tmo_init(&tmo, APP_TRACE_UART_STOP_WAIT_TMO);
+
+    vTaskDelay(10);
+    while (1) {
+        send_buff_data(hw_data, &tmo);
+        vTaskDelay(10);
+        if (hw_data->circular_buff_overflow == true)
+        {
+            hw_data->circular_buff_overflow = false;
+            ESP_LOGE(TAG, "Buffer overflow. Please increase UART baudrate, or increase UART TX ring buffer size in menuconfig.");
+        }
+        if (hw_data->message_buff_overflow == true)
+        {
+            hw_data->message_buff_overflow = false;
+            ESP_LOGE(TAG, "Message size more then message buffer!");
+        }
+    }
+}
+
+static const int APP_TRACE_UART_RX_BUF_SIZE = 4024;
+
+static esp_err_t esp_apptrace_uart_init(esp_apptrace_uart_data_t *hw_data)
+{
+    int core_id = esp_cpu_get_core_id();
+    if (core_id == 0) {
+        hw_data->tx_data_buff = (uint8_t *)heap_caps_malloc(APP_TRACE_MAX_TX_BUFF_UART, MALLOC_CAP_INTERNAL|MALLOC_CAP_8BIT);
+        if (hw_data->tx_data_buff == NULL){
+            return ESP_ERR_NO_MEM;
+        }
+        hw_data->tx_data_buff_in = 0;
+        hw_data->tx_data_buff_out = 0;
+        hw_data->tx_msg_buff =  (uint8_t *)heap_caps_malloc(APP_TRACE_MAX_TX_MSG_UART, MALLOC_CAP_INTERNAL|MALLOC_CAP_8BIT);
+        if (hw_data->tx_msg_buff == NULL)
+        {
+            return ESP_ERR_NO_MEM;
+        }
+        hw_data->tx_msg_buff_size = 0;
+        hw_data->down_buffer_size = 0;
+        hw_data->message_buff_overflow = false;
+        hw_data->circular_buff_overflow = false;
+
+
+        const uart_config_t uart_config = {
+            .baud_rate = CONFIG_APPTRACE_UART_BAUDRATE,
+            .data_bits = UART_DATA_8_BITS,
+            .parity = UART_PARITY_DISABLE,
+            .stop_bits = UART_STOP_BITS_1,
+            .flow_ctrl = UART_HW_FLOWCTRL_DISABLE,
+            .source_clk = UART_SCLK_DEFAULT,
+        };
+        ESP_LOGI(TAG, "UART baud rate: %i", CONFIG_APPTRACE_UART_BAUDRATE);
+        // We won't use a buffer for sending data.
+        esp_err_t err = uart_driver_install(hw_data->port_num, APP_TRACE_UART_RX_BUF_SIZE, APP_TRACE_UART_RX_BUF_SIZE, 0, NULL, 0);
+        assert((err == ESP_OK) && "Not possible to install UART. Please check and change menuconfig parameters!");
+        err = uart_param_config(hw_data->port_num, &uart_config);
+        assert((err == ESP_OK) && "Not possible to configure UART. Please check and change menuconfig parameters!");
+        err = uart_set_pin(hw_data->port_num, CONFIG_APPTRACE_UART_TX_GPIO, CONFIG_APPTRACE_UART_RX_GPIO, UART_PIN_NO_CHANGE, UART_PIN_NO_CHANGE);
+        assert((err == ESP_OK) && "Not possible to configure UART RX/TX pins. Please check and change menuconfig parameters!");
+
+        int uart_prio = CONFIG_APPTRACE_UART_TASK_PRIO;
+        if (uart_prio >= (configMAX_PRIORITIES-1)) uart_prio = configMAX_PRIORITIES - 1;
+        err = xTaskCreate(esp_apptrace_send_uart_tx_task, "app_trace_uart_tx_task", 2500, hw_data, uart_prio, NULL);
+        assert((err == pdPASS) && "Not possible to configure UART. Not possible to create task!");
+
+#if CONFIG_APPTRACE_LOCK_ENABLE
+        esp_apptrace_lock_init(&hw_data->lock);
+#endif
+    }
+    // init UART on this CPU
+    esp_apptrace_uart_hw_init();
+    hw_data->inited |= 1 << core_id;
+
+    return ESP_OK;
+}
+
+static uint8_t *esp_apptrace_uart_up_buffer_get(esp_apptrace_uart_data_t *hw_data, uint32_t size, esp_apptrace_tmo_t *tmo)
+{
+    uint8_t *ptr;
+    if (size > APP_TRACE_MAX_TX_MSG_UART) {
+        hw_data->message_buff_overflow = true;
+        return NULL;
+    }
+    if (hw_data->tx_msg_buff_size != 0)
+    {
+        // A previous message was not sent.
+        return NULL;
+    }
+    esp_err_t res = esp_apptrace_uart_lock(hw_data, tmo);
+    if (res != ESP_OK) {
+        return NULL;
+    }
+    ptr = hw_data->tx_msg_buff;
+    hw_data->tx_msg_buff_size = size;
+
+    // now we can safely unlock apptrace to allow other tasks/ISRs to get other buffers and write their data
+    if (esp_apptrace_uart_unlock(hw_data) != ESP_OK) {
+        assert(false && "Failed to unlock apptrace data!");
+    }
+    return ptr;
+}
+
+static esp_err_t esp_apptrace_uart_up_buffer_put(esp_apptrace_uart_data_t *hw_data, uint8_t *ptr, esp_apptrace_tmo_t *tmo)
+{
+    esp_err_t res = esp_apptrace_send_uart_data(hw_data, (const char *)ptr, hw_data->tx_msg_buff_size, tmo);
+    // Clear size to indicate that we've sent data
+    hw_data->tx_msg_buff_size = 0;
+    return res;
+}
+
+static void esp_apptrace_uart_down_buffer_config(esp_apptrace_uart_data_t *hw_data, uint8_t *buf, uint32_t size)
+{
+    hw_data->down_buffer = (uint8_t *)malloc(size);
+    if (hw_data->down_buffer == NULL){
+        assert(false && "Failed to allocate apptrace uart down buffer!");
+    }
+    hw_data->down_buffer_size = size;
+}
+
+static uint8_t *esp_apptrace_uart_down_buffer_get(esp_apptrace_uart_data_t *hw_data, uint32_t *size, esp_apptrace_tmo_t *tmo)
+{
+    uint8_t *ptr = NULL;
+
+    if (*size > hw_data->down_buffer_size) {
+        return NULL;
+    }
+
+    esp_err_t res = esp_apptrace_uart_lock(hw_data, tmo);
+    if (res != ESP_OK) {
+        return NULL;
+    }
+
+    size_t uart_fifolen = 0;
+    uart_get_buffered_data_len(hw_data->port_num, &uart_fifolen);
+    if (uart_fifolen > 0) {
+        if (*size < uart_fifolen) {
+            uart_fifolen = *size;
+        }
+        *size = uart_fifolen;
+        ptr = hw_data->down_buffer;
+        *size =uart_read_bytes(hw_data->port_num, ptr, uart_fifolen, 0);
+    }
+
+    if (esp_apptrace_uart_unlock(hw_data) != ESP_OK) {
+        assert(false && "Failed to unlock apptrace data!");
+    }
+    return ptr;
+}
+
+static esp_err_t esp_apptrace_uart_down_buffer_put(esp_apptrace_uart_data_t *hw_data, uint8_t *ptr, esp_apptrace_tmo_t *tmo)
+{
+    return ESP_OK;
+}
+
+static bool esp_apptrace_uart_host_is_connected(esp_apptrace_uart_data_t *hw_data)
+{
+    return hw_data->inited & 1;
+}
+
+static esp_err_t esp_apptrace_uart_flush_nolock(esp_apptrace_uart_data_t *hw_data, uint32_t min_sz, esp_apptrace_tmo_t *tmo)
+{
+    return ESP_OK;
+}
+
+static esp_err_t esp_apptrace_uart_flush(esp_apptrace_uart_data_t *hw_data, esp_apptrace_tmo_t *tmo)
+{
+    return ESP_OK;
+}
+
+#endif // APPTRACE_DEST_UART

components/app_trace/port/riscv/port.c

@@ -0,0 +1,360 @@
+/*
+ * SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#include "esp_cpu.h"
+#include "esp_log.h"
+#include "esp_app_trace_membufs_proto.h"
+#include "esp_app_trace_port.h"
+#include "riscv/semihosting.h"
+
+/** RISCV HW transport data */
+typedef struct {
+    uint8_t                             inited; // initialization state flags for every core
+#if CONFIG_APPTRACE_LOCK_ENABLE
+    esp_apptrace_lock_t                 lock;   // sync lock
+#endif
+    esp_apptrace_membufs_proto_data_t   membufs;
+} esp_apptrace_riscv_data_t;
+
+/** RISCV memory host iface control block */
+typedef struct {
+    uint32_t                    ctrl;
+    // - Guard field. If this register is not zero then CPU is changing this struct and
+    //   this guard field holds address of the instruction which application will execute when CPU finishes with those modifications.
+    uint32_t                    stat;
+    esp_apptrace_mem_block_t *  mem_blocks;
+} esp_apptrace_riscv_ctrl_block_t;
+
+#define ESP_APPTRACE_RISCV_BLOCK_LEN_MSK         0x7FFFUL
+#define ESP_APPTRACE_RISCV_BLOCK_LEN(_l_)        ((_l_) & ESP_APPTRACE_RISCV_BLOCK_LEN_MSK)
+#define ESP_APPTRACE_RISCV_BLOCK_LEN_GET(_v_)    ((_v_) & ESP_APPTRACE_RISCV_BLOCK_LEN_MSK)
+#define ESP_APPTRACE_RISCV_BLOCK_ID_MSK          0x7FUL
+#define ESP_APPTRACE_RISCV_BLOCK_ID(_id_)        (((_id_) & ESP_APPTRACE_RISCV_BLOCK_ID_MSK) << 15)
+#define ESP_APPTRACE_RISCV_BLOCK_ID_GET(_v_)     (((_v_) >> 15) & ESP_APPTRACE_RISCV_BLOCK_ID_MSK)
+#define ESP_APPTRACE_RISCV_HOST_DATA             (1 << 22)
+#define ESP_APPTRACE_RISCV_HOST_CONNECT          (1 << 23)
+
+#define ESP_APPTRACE_RISCV_INITED(_hw_)          ((_hw_)->inited & (1 << 0/*esp_cpu_get_core_id()*/))
+
+static esp_err_t esp_apptrace_riscv_init(esp_apptrace_riscv_data_t *hw_data);
+static esp_err_t esp_apptrace_riscv_flush(esp_apptrace_riscv_data_t *hw_data, esp_apptrace_tmo_t *tmo);
+static esp_err_t esp_apptrace_riscv_flush_nolock(esp_apptrace_riscv_data_t *hw_data, uint32_t min_sz, esp_apptrace_tmo_t *tmo);
+static uint8_t *esp_apptrace_riscv_up_buffer_get(esp_apptrace_riscv_data_t *hw_data, uint32_t size, esp_apptrace_tmo_t *tmo);
+static esp_err_t esp_apptrace_riscv_up_buffer_put(esp_apptrace_riscv_data_t *hw_data, uint8_t *ptr, esp_apptrace_tmo_t *tmo);
+static void esp_apptrace_riscv_down_buffer_config(esp_apptrace_riscv_data_t *hw_data, uint8_t *buf, uint32_t size);
+static uint8_t *esp_apptrace_riscv_down_buffer_get(esp_apptrace_riscv_data_t *hw_data, uint32_t *size, esp_apptrace_tmo_t *tmo);
+static esp_err_t esp_apptrace_riscv_down_buffer_put(esp_apptrace_riscv_data_t *hw_data, uint8_t *ptr, esp_apptrace_tmo_t *tmo);
+static bool esp_apptrace_riscv_host_is_connected(esp_apptrace_riscv_data_t *hw_data);
+static esp_err_t esp_apptrace_riscv_buffer_swap_start(uint32_t curr_block_id);
+static esp_err_t esp_apptrace_riscv_buffer_swap(uint32_t new_block_id);
+static esp_err_t esp_apptrace_riscv_buffer_swap_end(uint32_t new_block_id, uint32_t prev_block_len);
+static bool esp_apptrace_riscv_host_data_pending(void);
+
+
+const static char *TAG = "esp_apptrace";
+
+static esp_apptrace_riscv_ctrl_block_t s_tracing_ctrl[portNUM_PROCESSORS];
+
+esp_apptrace_hw_t *esp_apptrace_jtag_hw_get(void **data)
+{
+#if CONFIG_APPTRACE_DEST_JTAG
+    static esp_apptrace_membufs_proto_hw_t s_trace_proto_hw = {
+        .swap_start = esp_apptrace_riscv_buffer_swap_start,
+        .swap = esp_apptrace_riscv_buffer_swap,
+        .swap_end = esp_apptrace_riscv_buffer_swap_end,
+        .host_data_pending = esp_apptrace_riscv_host_data_pending,
+    };
+    static esp_apptrace_riscv_data_t s_trace_hw_data = {
+        .membufs = {
+            .hw = &s_trace_proto_hw,
+        },
+    };
+    static esp_apptrace_hw_t s_trace_hw = {
+        .init = (esp_err_t (*)(void *))esp_apptrace_riscv_init,
+        .get_up_buffer = (uint8_t *(*)(void *, uint32_t, esp_apptrace_tmo_t *))esp_apptrace_riscv_up_buffer_get,
+        .put_up_buffer = (esp_err_t (*)(void *, uint8_t *, esp_apptrace_tmo_t *))esp_apptrace_riscv_up_buffer_put,
+        .flush_up_buffer_nolock = (esp_err_t (*)(void *, uint32_t, esp_apptrace_tmo_t *))esp_apptrace_riscv_flush_nolock,
+        .flush_up_buffer = (esp_err_t (*)(void *, esp_apptrace_tmo_t *))esp_apptrace_riscv_flush,
+        .down_buffer_config = (void (*)(void *, uint8_t *, uint32_t ))esp_apptrace_riscv_down_buffer_config,
+        .get_down_buffer = (uint8_t *(*)(void *, uint32_t *, esp_apptrace_tmo_t *))esp_apptrace_riscv_down_buffer_get,
+        .put_down_buffer = (esp_err_t (*)(void *, uint8_t *, esp_apptrace_tmo_t *))esp_apptrace_riscv_down_buffer_put,
+        .host_is_connected = (bool (*)(void *))esp_apptrace_riscv_host_is_connected,
+    };
+    *data = &s_trace_hw_data;
+    return &s_trace_hw;
+#else
+    return NULL;
+#endif
+}
+
+/* Advertises apptrace control block address to host.
+   This function can be overriden with custom implementation,
+   e.g. OpenOCD flasher stub use own implementation of it. */
+__attribute__((weak)) int esp_apptrace_advertise_ctrl_block(void *ctrl_block_addr)
+{
+    if (!esp_cpu_dbgr_is_attached()) {
+        return 0;
+    }
+    return (int) semihosting_call_noerrno(ESP_SEMIHOSTING_SYS_APPTRACE_INIT, (long*)ctrl_block_addr);
+}
+
+/* Returns up buffers config.
+   This function can be overriden with custom implementation,
+   e.g. OpenOCD flasher stub use own implementation of it. */
+__attribute__((weak)) void esp_apptrace_get_up_buffers(esp_apptrace_mem_block_t mem_blocks_cfg[2])
+{
+    static uint8_t s_mem_blocks[2][CONFIG_APPTRACE_BUF_SIZE];
+
+    mem_blocks_cfg[0].start = s_mem_blocks[0];
+    mem_blocks_cfg[0].sz = CONFIG_APPTRACE_BUF_SIZE;
+    mem_blocks_cfg[1].start = s_mem_blocks[1];
+    mem_blocks_cfg[1].sz = CONFIG_APPTRACE_BUF_SIZE;
+}
+
+static esp_err_t esp_apptrace_riscv_lock(esp_apptrace_riscv_data_t *hw_data, esp_apptrace_tmo_t *tmo)
+{
+#if CONFIG_APPTRACE_LOCK_ENABLE
+    esp_err_t ret = esp_apptrace_lock_take(&hw_data->lock, tmo);
+    if (ret != ESP_OK) {
+        return ESP_FAIL;
+    }
+#endif
+    return ESP_OK;
+}
+
+static esp_err_t esp_apptrace_riscv_unlock(esp_apptrace_riscv_data_t *hw_data)
+{
+    esp_err_t ret = ESP_OK;
+#if CONFIG_APPTRACE_LOCK_ENABLE
+    ret = esp_apptrace_lock_give(&hw_data->lock);
+#endif
+    return ret;
+}
+
+/*****************************************************************************************/
+/***************************** Apptrace HW iface *****************************************/
+/*****************************************************************************************/
+
+static esp_err_t esp_apptrace_riscv_init(esp_apptrace_riscv_data_t *hw_data)
+{
+    int core_id = esp_cpu_get_core_id();
+
+    if (hw_data->inited == 0) {
+        esp_apptrace_mem_block_t mem_blocks_cfg[2];
+        esp_apptrace_get_up_buffers(mem_blocks_cfg);
+        esp_err_t res = esp_apptrace_membufs_init(&hw_data->membufs, mem_blocks_cfg);
+        if (res != ESP_OK) {
+            ESP_APPTRACE_LOGE("Failed to init membufs proto (%d)!", res);
+            return res;
+        }
+#if CONFIG_APPTRACE_LOCK_ENABLE
+        esp_apptrace_lock_init(&hw_data->lock);
+#endif
+    }
+    hw_data->inited |= 1 << core_id;
+    ESP_APPTRACE_LOGI("Apptrace initialized on CPU%d. Tracing control block @ %p.", core_id, &s_tracing_ctrl[core_id]);
+    s_tracing_ctrl[core_id].mem_blocks = hw_data->membufs.blocks;
+    for (int i = 0; i < 2; i++) {
+        ESP_APPTRACE_LOGD("Mem buf[%d] %d bytes @ %p (%p/%p)", i,
+            s_tracing_ctrl[core_id].mem_blocks[i].sz, s_tracing_ctrl[core_id].mem_blocks[i].start,
+            &(s_tracing_ctrl[core_id].mem_blocks[i].start), &(s_tracing_ctrl[core_id].mem_blocks[i].sz));
+    }
+    // notify host about control block address
+    int res = esp_apptrace_advertise_ctrl_block(&s_tracing_ctrl[core_id]);
+    assert(res == 0 && "Falied to send config to host!");
+
+    return ESP_OK;
+}
+
+static uint8_t *esp_apptrace_riscv_up_buffer_get(esp_apptrace_riscv_data_t *hw_data, uint32_t size, esp_apptrace_tmo_t *tmo)
+{
+    uint8_t *ptr;
+
+    if (!ESP_APPTRACE_RISCV_INITED(hw_data)) {
+        return NULL;
+    }
+    esp_err_t res = esp_apptrace_riscv_lock(hw_data, tmo);
+    if (res != ESP_OK) {
+        return NULL;
+    }
+
+    ptr = esp_apptrace_membufs_up_buffer_get(&hw_data->membufs, size, tmo);
+
+    // now we can safely unlock apptrace to allow other tasks/ISRs to get other buffers and write their data
+    if (esp_apptrace_riscv_unlock(hw_data) != ESP_OK) {
+        assert(false && "Failed to unlock apptrace data!");
+    }
+    return ptr;
+}
+
+static esp_err_t esp_apptrace_riscv_up_buffer_put(esp_apptrace_riscv_data_t *hw_data, uint8_t *ptr, esp_apptrace_tmo_t *tmo)
+{
+    if (!ESP_APPTRACE_RISCV_INITED(hw_data)) {
+        return ESP_ERR_INVALID_STATE;
+    }
+    // Can avoid locking because esp_apptrace_membufs_up_buffer_put() just modifies buffer's header
+    esp_err_t res = esp_apptrace_membufs_up_buffer_put(&hw_data->membufs, ptr, tmo);
+    return res;
+}
+
+static void esp_apptrace_riscv_down_buffer_config(esp_apptrace_riscv_data_t *hw_data, uint8_t *buf, uint32_t size)
+{
+    if (!ESP_APPTRACE_RISCV_INITED(hw_data)) {
+        return;
+    }
+    esp_apptrace_membufs_down_buffer_config(&hw_data->membufs, buf, size);
+}
+
+static uint8_t *esp_apptrace_riscv_down_buffer_get(esp_apptrace_riscv_data_t *hw_data, uint32_t *size, esp_apptrace_tmo_t *tmo)
+{
+    uint8_t *ptr;
+
+    if (!ESP_APPTRACE_RISCV_INITED(hw_data)) {
+        return NULL;
+    }
+    esp_err_t res = esp_apptrace_riscv_lock(hw_data, tmo);
+    if (res != ESP_OK) {
+        return NULL;
+    }
+
+    ptr = esp_apptrace_membufs_down_buffer_get(&hw_data->membufs, size, tmo);
+
+    // now we can safely unlock apptrace to allow other tasks/ISRs to get other buffers and write their data
+    if (esp_apptrace_riscv_unlock(hw_data) != ESP_OK) {
+        assert(false && "Failed to unlock apptrace data!");
+    }
+    return ptr;
+}
+
+static esp_err_t esp_apptrace_riscv_down_buffer_put(esp_apptrace_riscv_data_t *hw_data, uint8_t *ptr, esp_apptrace_tmo_t *tmo)
+{
+    if (!ESP_APPTRACE_RISCV_INITED(hw_data)) {
+        return ESP_ERR_INVALID_STATE;
+    }
+    // Can avoid locking because esp_apptrace_membufs_down_buffer_put() does nothing
+    /*esp_err_t res = esp_apptrace_riscv_lock(hw_data, tmo);
+    if (res != ESP_OK) {
+        return res;
+    }*/
+
+    esp_err_t res = esp_apptrace_membufs_down_buffer_put(&hw_data->membufs, ptr, tmo);
+
+    // now we can safely unlock apptrace to allow other tasks/ISRs to get other buffers and write their data
+    /*if (esp_apptrace_riscv_unlock(hw_data) != ESP_OK) {
+        assert(false && "Failed to unlock apptrace data!");
+    }*/
+    return res;
+}
+
+static bool esp_apptrace_riscv_host_is_connected(esp_apptrace_riscv_data_t *hw_data)
+{
+    if (!ESP_APPTRACE_RISCV_INITED(hw_data)) {
+        return false;
+    }
+    return s_tracing_ctrl[esp_cpu_get_core_id()].ctrl & ESP_APPTRACE_RISCV_HOST_CONNECT ? true : false;
+}
+
+static esp_err_t esp_apptrace_riscv_flush_nolock(esp_apptrace_riscv_data_t *hw_data, uint32_t min_sz, esp_apptrace_tmo_t *tmo)
+{
+    if (!ESP_APPTRACE_RISCV_INITED(hw_data)) {
+        return ESP_ERR_INVALID_STATE;
+    }
+    return esp_apptrace_membufs_flush_nolock(&hw_data->membufs, min_sz, tmo);
+}
+
+static esp_err_t esp_apptrace_riscv_flush(esp_apptrace_riscv_data_t *hw_data, esp_apptrace_tmo_t *tmo)
+{
+    if (!ESP_APPTRACE_RISCV_INITED(hw_data)) {
+        return ESP_ERR_INVALID_STATE;
+    }
+    esp_err_t res = esp_apptrace_riscv_lock(hw_data, tmo);
+    if (res != ESP_OK) {
+        return res;
+    }
+
+    res = esp_apptrace_membufs_flush_nolock(&hw_data->membufs, 0, tmo);
+
+    // now we can safely unlock apptrace to allow other tasks/ISRs to get other buffers and write their data
+    if (esp_apptrace_riscv_unlock(hw_data) != ESP_OK) {
+        assert(false && "Failed to unlock apptrace data!");
+    }
+    return res;
+}
+
+/*****************************************************************************************/
+/************************** Membufs proto HW iface ***************************************/
+/*****************************************************************************************/
+
+static inline void esp_apptrace_riscv_buffer_swap_lock(void)
+{
+    extern uint32_t __esp_apptrace_riscv_updated;
+
+    // indicate to host that we are about to update.
+    // this is used only to place CPU into streaming mode at tracing startup
+    // before starting streaming host can halt us after we read  ESP_APPTRACE_RISCV_CTRL_REG and before we updated it
+    // HACK: in this case host will set breakpoint just after ESP_APPTRACE_RISCV_CTRL_REG update,
+    // here we set address to set bp at
+    // enter ERI update critical section
+    s_tracing_ctrl[esp_cpu_get_core_id()].stat = (uint32_t)&__esp_apptrace_riscv_updated;
+}
+
+static __attribute__((noinline)) void esp_apptrace_riscv_buffer_swap_unlock(void)
+{
+    // exit ERI update critical section
+    s_tracing_ctrl[esp_cpu_get_core_id()].stat = 0;
+    // TODO: currently host sets breakpoint, use break instruction to stop;
+    // it will allow to use ESP_APPTRACE_RISCV_STAT_REG for other purposes
+    asm volatile (
+        "    .global     __esp_apptrace_riscv_updated\n"
+        "__esp_apptrace_riscv_updated:\n"); // host will set bp here to resolve collision at streaming start
+}
+
+static esp_err_t esp_apptrace_riscv_buffer_swap_start(uint32_t curr_block_id)
+{
+    esp_err_t res = ESP_OK;
+
+    esp_apptrace_riscv_buffer_swap_lock();
+
+    uint32_t ctrl_reg = s_tracing_ctrl[esp_cpu_get_core_id()].ctrl;
+    uint32_t host_connected = ESP_APPTRACE_RISCV_HOST_CONNECT & ctrl_reg;
+    if (host_connected) {
+        uint32_t acked_block = ESP_APPTRACE_RISCV_BLOCK_ID_GET(ctrl_reg);
+        uint32_t host_to_read = ESP_APPTRACE_RISCV_BLOCK_LEN_GET(ctrl_reg);
+        if (host_to_read != 0 || acked_block != (curr_block_id & ESP_APPTRACE_RISCV_BLOCK_ID_MSK)) {
+            ESP_APPTRACE_LOGD("[%d]: Can not switch %x %d %x %x/%lx", esp_cpu_get_core_id(), ctrl_reg, host_to_read, acked_block,
+                curr_block_id & ESP_APPTRACE_RISCV_BLOCK_ID_MSK, curr_block_id);
+            res = ESP_ERR_NO_MEM;
+            goto _on_err;
+        }
+    }
+    return ESP_OK;
+_on_err:
+    esp_apptrace_riscv_buffer_swap_unlock();
+    return res;
+}
+
+static esp_err_t esp_apptrace_riscv_buffer_swap_end(uint32_t new_block_id, uint32_t prev_block_len)
+{
+    uint32_t ctrl_reg = s_tracing_ctrl[esp_cpu_get_core_id()].ctrl;
+    uint32_t host_connected = ESP_APPTRACE_RISCV_HOST_CONNECT & ctrl_reg;
+    s_tracing_ctrl[esp_cpu_get_core_id()].ctrl = ESP_APPTRACE_RISCV_BLOCK_ID(new_block_id) |
+              host_connected | ESP_APPTRACE_RISCV_BLOCK_LEN(prev_block_len);
+    esp_apptrace_riscv_buffer_swap_unlock();
+    return ESP_OK;
+}
+
+static esp_err_t esp_apptrace_riscv_buffer_swap(uint32_t new_block_id)
+{
+    /* do nothing */
+    return ESP_OK;
+}
+
+static bool esp_apptrace_riscv_host_data_pending(void)
+{
+    uint32_t ctrl_reg = s_tracing_ctrl[esp_cpu_get_core_id()].ctrl;
+    // ESP_APPTRACE_LOGV("%s() 0x%x", __func__, ctrl_reg);
+    return (ctrl_reg & ESP_APPTRACE_RISCV_HOST_DATA) ? true : false;
+}

components/app_trace/port/xtensa/port.c

@@ -0,0 +1,533 @@
+/*
+ * SPDX-FileCopyrightText: 2015-2021 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+//
+// How It Works
+// ************
+
+// 1. Components Overview
+// ======================
+
+// Xtensa has useful feature: TRAX debug module. It allows recording program execution flow at run-time without disturbing CPU.
+// Exectution flow data are written to configurable Trace RAM block. Besides accessing Trace RAM itself TRAX module also allows to read/write
+// trace memory via its registers by means of JTAG, APB or ERI transactions.
+// ESP32 has two Xtensa cores with separate TRAX modules on them and provides two special memory regions to be used as trace memory.
+// Chip allows muxing access to those trace memory blocks in such a way that while one block is accessed by CPUs another one can be accessed by host
+// by means of reading/writing TRAX registers via JTAG. Blocks muxing is configurable at run-time and allows switching trace memory blocks between
+// accessors in round-robin fashion so they can read/write separate memory blocks without disturbing each other.
+// This module implements application tracing feature based on above mechanisms. It allows to transfer arbitrary user data to/from
+// host via JTAG with minimal impact on system performance. This module is implied to be used in the following tracing scheme.
+
+//                                                        ------>------                                         ----- (host components) -----
+//                                                        |           |                                         |                           |
+// -------------------   -----------------------     -----------------------     ----------------    ------     ---------   -----------------
+// |trace data source|-->|target tracing module|<--->|TRAX_MEM0 | TRAX_MEM1|---->|TRAX_DATA_REGS|<-->|JTAG|<--->|OpenOCD|-->|trace data sink|
+// -------------------   -----------------------     -----------------------     ----------------    ------     ---------   -----------------
+//                                 |                      |           |                                |
+//                                 |                      ------<------          ----------------      |
+//                                 |<------------------------------------------->|TRAX_CTRL_REGS|<---->|
+//                                                                               ----------------
+
+// In general tracing goes in the following way. User application requests tracing module to send some data by calling esp_apptrace_buffer_get(),
+// module allocates necessary buffer in current input trace block. Then user fills received buffer with data and calls esp_apptrace_buffer_put().
+// When current input trace block is filled with app data it is exposed to host and the second block becomes input one and buffer filling restarts.
+// While target application fills one TRAX block host reads another one via JTAG.
+// This module also allows communication in the opposite direction: from host to target. As it was said ESP32 and host can access different TRAX blocks
+// simultaneously, so while target writes trace data to one block host can write its own data (e.g. tracing commands) to another one then when
+// blocks are switched host receives trace data and target receives data written by host application. Target user application can read host data
+// by calling esp_apptrace_read() API.
+// To control buffer switching and for other communication purposes this implementation uses some TRAX registers. It is safe since HW TRAX tracing
+// can not be used along with application tracing feature so these registers are freely readable/writeable via JTAG from host and via ERI from ESP32 cores.
+// Overhead of this implementation on target CPU is produced only by allocating/managing buffers and copying of data.
+// On the host side special OpenOCD command must be used to read trace data.
+
+// 2. TRAX Registers layout
+// ========================
+
+// This module uses two TRAX HW registers to communicate with host SW (OpenOCD).
+//  - Control register uses TRAX_DELAYCNT as storage. Only lower 24 bits of TRAX_DELAYCNT are writable. Control register has the following bitfields:
+//   | 31..XXXXXX..24 | 23 .(host_connect). 23| 22..(block_id)..15 | 14..(block_len)..0 |
+//    14..0  bits - actual length of user data in trace memory block. Target updates it every time it fills memory block and exposes it to host.
+//                  Host writes zero to this field when it finishes reading exposed block;
+//    21..15 bits - trace memory block transfer ID. Block counter. It can overflow. Updated by target, host should not modify it. Actually can be 2 bits;
+//    22     bit  - 'host data present' flag. If set to one there is data from host, otherwise - no host data;
+//    23     bit  - 'host connected' flag. If zero then host is not connected and tracing module works in post-mortem mode, otherwise in streaming mode;
+// - Status register uses TRAX_TRIGGERPC as storage. If this register is not zero then current CPU is changing TRAX registers and
+//   this register holds address of the instruction which application will execute when it finishes with those registers modifications.
+//   See 'Targets Connection' setion for details.
+
+// 3. Modes of operation
+// =====================
+
+// This module supports two modes of operation:
+//  - Post-mortem mode. This is the default mode. In this mode application tracing module does not check whether host has read all the data from block
+//    exposed to it and switches block in any case. The mode does not need host interaction for operation and so can be useful when only the latest
+//    trace data are necessary, e.g. for analyzing crashes. On panic the latest data from current input block are exposed to host and host can read them.
+//    It can happen that system panic occurs when there are very small amount of data which are not exposed to host yet (e.g. crash just after the
+//    TRAX block switch). In this case the previous 16KB of collected data will be dropped and host will see the latest, but very small piece of trace.
+//    It can be insufficient to diagnose the problem. To avoid such situations there is menuconfig option
+//    CONFIG_APPTRACE_POSTMORTEM_FLUSH_THRESH
+//    which controls the threshold for flushing data in case of panic.
+//  - Streaming mode. Tracing module enters this mode when host connects to target and sets respective bits in control registers (per core).
+//    In this mode before switching the block tracing module waits for the host to read all the data from the previously exposed block.
+//    On panic tracing module also waits (timeout is configured via menuconfig via CONFIG_APPTRACE_ONPANIC_HOST_FLUSH_TMO) for the host to read all data.
+
+// 4. Communication Protocol
+// =========================
+
+// 4.1 Trace Memory Blocks
+// -----------------------
+
+// Communication is controlled via special register. Host periodically polls control register on each core to find out if there are any data available.
+// When current input memory block is filled it is exposed to host and 'block_len' and 'block_id' fields are updated in the control register.
+// Host reads new register value and according to it's value starts reading data from exposed block. Meanwhile target starts filling another trace block.
+// When host finishes reading the block it clears 'block_len' field in control register indicating to the target that it is ready to accept the next one.
+// If the host has some data to transfer to the target it writes them to trace memory block before clearing 'block_len' field. Then it sets
+// 'host_data_present' bit and clears 'block_len' field in control register. Upon every block switch target checks 'host_data_present' bit and if it is set
+// reads them to down buffer before writing any trace data to switched TRAX block.
+
+// 4.2 User Data Chunks Level
+// --------------------------
+
+// Since trace memory block is shared between user data chunks and data copying is performed on behalf of the API user (in its normal context) in
+// multithreading environment it can happen that task/ISR which copies data is preempted by another high prio task/ISR. So it is possible situation
+// that task/ISR will fail to complete filling its data chunk before the whole trace block is exposed to the host. To handle such conditions tracing
+// module prepends all user data chunks with header which contains allocated buffer size and actual data length within it. OpenOCD command
+// which reads application traces reports error when it reads incomplete user data block.
+// Data which are transffered from host to target are also prepended with a header. Down channel data header is simple and consists of one two bytes field
+// containing length of host data following the header.
+
+// 4.3 Data Buffering
+// ------------------
+
+// It takes some time for the host to read TRAX memory block via JTAG. In streaming mode it can happen that target has filled its TRAX block, but host
+// has not completed reading of the previous one yet. So in this case time critical tracing calls (which can not be delayed for too long time due to
+// the lack of free memory in TRAX block) can be dropped. To avoid such scenarios tracing module implements data buffering. Buffered data will be sent
+// to the host later when TRAX block switch occurs. The maximum size of the buffered data is controlled by menuconfig option
+// CONFIG_APPTRACE_PENDING_DATA_SIZE_MAX.
+
+// 4.4 Target Connection/Disconnection
+// -----------------------------------
+
+// When host is going to start tracing in streaming mode it needs to put both ESP32 cores into initial state when 'host connected' bit is set
+// on both cores. To accomplish this host halts both cores and sets this bit in TRAX registers. But target code can be halted in state when it has read control
+// register but has not updated its value. To handle such situations target code indicates to the host that it is updating control register by writing
+// non-zero value to status register. Actually it writes address of the instruction which it will execute when it finishes with
+// the registers update. When target is halted during control register update host sets breakpoint at the address from status register and resumes CPU.
+// After target code finishes with register update it is halted on breakpoint, host detects it and safely sets 'host connected' bit. When both cores
+// are set up they are resumed. Tracing starts without further intrusion into CPUs work.
+// When host is going to stop tracing in streaming mode it needs to disconnect targets. Disconnection process is done using the same algorithm
+// as for connecting, but 'host connected' bits are cleared on ESP32 cores.
+
+// 5. Module Access Synchronization
+// ================================
+
+// Access to internal module's data is synchronized with custom mutex. Mutex is a wrapper for portMUX_TYPE and uses almost the same sync mechanism as in
+// vPortCPUAcquireMutex/vPortCPUReleaseMutex. The mechanism uses S32C1I Xtensa instruction to implement exclusive access to module's data from tasks and
+// ISRs running on both cores. Also custom mutex allows specifying timeout for locking operation. Locking routine checks underlaying mutex in cycle until
+// it gets its ownership or timeout expires. The differences of application tracing module's mutex implementation from vPortCPUAcquireMutex/vPortCPUReleaseMutex are:
+// - Support for timeouts.
+// - Local IRQs for CPU which owns the mutex are disabled till the call to unlocking routine. This is made to avoid possible task's prio inversion.
+//   When low prio task takes mutex and enables local IRQs gets preempted by high prio task which in its turn can try to acquire mutex using infinite timeout.
+//   So no local task switch occurs when mutex is locked. But this does not apply to tasks on another CPU.
+//   WARNING: Priority inversion can happen when low prio task works on one CPU and medium and high prio tasks work on another.
+// WARNING: Care must be taken when selecting timeout values for trace calls from ISRs. Tracing module does not care about watchdogs when waiting
+// on internal locks and for host to complete previous block reading, so if timeout value exceeds watchdog's one it can lead to the system reboot.
+
+// 6. Timeouts
+// ===========
+
+// Timeout mechanism is based on xthal_get_ccount() routine and supports timeout values in microseconds.
+// There are two situations when task/ISR can be delayed by tracing API call. Timeout mechanism takes into account both conditions:
+// - Trace data are locked by another task/ISR. When wating on trace data lock.
+// - Current TRAX memory input block is full when working in streaming mode (host is connected). When waiting for host to complete previous block reading.
+// When wating for any of above conditions xthal_get_ccount() is called periodically to calculate time elapsed from trace API routine entry. When elapsed
+// time exceeds specified timeout value operation is canceled and ESP_ERR_TIMEOUT code is returned.
+#include "sdkconfig.h"
+#include "soc/soc.h"
+#include "soc/dport_reg.h"
+#include "soc/tracemem_config.h"
+#if CONFIG_IDF_TARGET_ESP32S2 || CONFIG_IDF_TARGET_ESP32S3
+#include "soc/sensitive_reg.h"
+#endif
+#include "eri.h"
+#include "esp_private/trax.h"
+#include "esp_cpu.h"
+#include "esp_log.h"
+#include "esp_app_trace_membufs_proto.h"
+#include "esp_app_trace_port.h"
+
+// TRAX is disabled, so we use its registers for our own purposes
+// | 31..XXXXXX..24 | 23 .(host_connect). 23 | 22 .(host_data). 22| 21..(block_id)..15 | 14..(block_len)..0 |
+#define ESP_APPTRACE_TRAX_CTRL_REG              ERI_TRAX_DELAYCNT
+#define ESP_APPTRACE_TRAX_STAT_REG              ERI_TRAX_TRIGGERPC
+
+#define ESP_APPTRACE_TRAX_BLOCK_LEN_MSK         0x7FFFUL
+#define ESP_APPTRACE_TRAX_BLOCK_LEN(_l_)        ((_l_) & ESP_APPTRACE_TRAX_BLOCK_LEN_MSK)
+#define ESP_APPTRACE_TRAX_BLOCK_LEN_GET(_v_)    ((_v_) & ESP_APPTRACE_TRAX_BLOCK_LEN_MSK)
+#define ESP_APPTRACE_TRAX_BLOCK_ID_MSK          0x7FUL
+#define ESP_APPTRACE_TRAX_BLOCK_ID(_id_)        (((_id_) & ESP_APPTRACE_TRAX_BLOCK_ID_MSK) << 15)
+#define ESP_APPTRACE_TRAX_BLOCK_ID_GET(_v_)     (((_v_) >> 15) & ESP_APPTRACE_TRAX_BLOCK_ID_MSK)
+#define ESP_APPTRACE_TRAX_HOST_DATA             (1 << 22)
+#define ESP_APPTRACE_TRAX_HOST_CONNECT          (1 << 23)
+
+#define ESP_APPTRACE_TRAX_INITED(_hw_)          ((_hw_)->inited & (1 << esp_cpu_get_core_id()))
+
+#define ESP_APPTRACE_TRAX_BLOCK_SIZE            (0x4000UL)
+
+/** TRAX HW transport data */
+typedef struct {
+    uint8_t                             inited;
+#if CONFIG_APPTRACE_LOCK_ENABLE
+    esp_apptrace_lock_t                 lock;   // sync lock
+#endif
+    esp_apptrace_membufs_proto_data_t   membufs;
+} esp_apptrace_trax_data_t;
+
+
+static esp_err_t esp_apptrace_trax_init(esp_apptrace_trax_data_t *hw_data);
+static esp_err_t esp_apptrace_trax_flush(esp_apptrace_trax_data_t *hw_data, esp_apptrace_tmo_t *tmo);
+static esp_err_t esp_apptrace_trax_flush_nolock(esp_apptrace_trax_data_t *hw_data, uint32_t min_sz, esp_apptrace_tmo_t *tmo);
+static uint8_t *esp_apptrace_trax_up_buffer_get(esp_apptrace_trax_data_t *hw_data, uint32_t size, esp_apptrace_tmo_t *tmo);
+static esp_err_t esp_apptrace_trax_up_buffer_put(esp_apptrace_trax_data_t *hw_data, uint8_t *ptr, esp_apptrace_tmo_t *tmo);
+static void esp_apptrace_trax_down_buffer_config(esp_apptrace_trax_data_t *hw_data, uint8_t *buf, uint32_t size);
+static uint8_t *esp_apptrace_trax_down_buffer_get(esp_apptrace_trax_data_t *hw_data, uint32_t *size, esp_apptrace_tmo_t *tmo);
+static esp_err_t esp_apptrace_trax_down_buffer_put(esp_apptrace_trax_data_t *hw_data, uint8_t *ptr, esp_apptrace_tmo_t *tmo);
+static bool esp_apptrace_trax_host_is_connected(esp_apptrace_trax_data_t *hw_data);
+static esp_err_t esp_apptrace_trax_buffer_swap_start(uint32_t curr_block_id);
+static esp_err_t esp_apptrace_trax_buffer_swap(uint32_t new_block_id);
+static esp_err_t esp_apptrace_trax_buffer_swap_end(uint32_t new_block_id, uint32_t prev_block_len);
+static bool esp_apptrace_trax_host_data_pending(void);
+
+
+const static char *TAG = "esp_apptrace";
+
+static uint8_t * const s_trax_blocks[] = {
+    (uint8_t *)TRACEMEM_BLK0_ADDR,
+    (uint8_t *)TRACEMEM_BLK1_ADDR
+};
+
+esp_apptrace_hw_t *esp_apptrace_jtag_hw_get(void **data)
+{
+#if CONFIG_APPTRACE_DEST_JTAG
+    static esp_apptrace_membufs_proto_hw_t s_trax_proto_hw = {
+        .swap_start = esp_apptrace_trax_buffer_swap_start,
+        .swap = esp_apptrace_trax_buffer_swap,
+        .swap_end = esp_apptrace_trax_buffer_swap_end,
+        .host_data_pending = esp_apptrace_trax_host_data_pending,
+    };
+    static esp_apptrace_trax_data_t s_trax_hw_data = {
+        .membufs = {
+            .hw = &s_trax_proto_hw,
+        },
+    };
+    static esp_apptrace_hw_t s_trax_hw = {
+        .init = (esp_err_t (*)(void *))esp_apptrace_trax_init,
+        .get_up_buffer = (uint8_t *(*)(void *, uint32_t, esp_apptrace_tmo_t *))esp_apptrace_trax_up_buffer_get,
+        .put_up_buffer = (esp_err_t (*)(void *, uint8_t *, esp_apptrace_tmo_t *))esp_apptrace_trax_up_buffer_put,
+        .flush_up_buffer_nolock = (esp_err_t (*)(void *, uint32_t, esp_apptrace_tmo_t *))esp_apptrace_trax_flush_nolock,
+        .flush_up_buffer = (esp_err_t (*)(void *, esp_apptrace_tmo_t *))esp_apptrace_trax_flush,
+        .down_buffer_config = (void (*)(void *, uint8_t *, uint32_t ))esp_apptrace_trax_down_buffer_config,
+        .get_down_buffer = (uint8_t *(*)(void *, uint32_t *, esp_apptrace_tmo_t *))esp_apptrace_trax_down_buffer_get,
+        .put_down_buffer = (esp_err_t (*)(void *, uint8_t *, esp_apptrace_tmo_t *))esp_apptrace_trax_down_buffer_put,
+        .host_is_connected = (bool (*)(void *))esp_apptrace_trax_host_is_connected,
+    };
+    *data = &s_trax_hw_data;
+    return &s_trax_hw;
+#else
+    return NULL;
+#endif
+}
+
+static esp_err_t esp_apptrace_trax_lock(esp_apptrace_trax_data_t *hw_data, esp_apptrace_tmo_t *tmo)
+{
+#if CONFIG_APPTRACE_LOCK_ENABLE
+    esp_err_t ret = esp_apptrace_lock_take(&hw_data->lock, tmo);
+    if (ret != ESP_OK) {
+        return ESP_FAIL;
+    }
+#endif
+    return ESP_OK;
+}
+
+static esp_err_t esp_apptrace_trax_unlock(esp_apptrace_trax_data_t *hw_data)
+{
+    esp_err_t ret = ESP_OK;
+#if CONFIG_APPTRACE_LOCK_ENABLE
+    ret = esp_apptrace_lock_give(&hw_data->lock);
+#endif
+    return ret;
+}
+
+static inline void esp_apptrace_trax_hw_init(void)
+{
+    // Stop trace, if any (on the current CPU)
+    eri_write(ERI_TRAX_TRAXCTRL, TRAXCTRL_TRSTP);
+    eri_write(ERI_TRAX_TRAXCTRL, TRAXCTRL_TMEN);
+    eri_write(ESP_APPTRACE_TRAX_CTRL_REG, ESP_APPTRACE_TRAX_BLOCK_ID(0));
+    // this is for OpenOCD to let him know where stub entries vector is resided
+    // must be read by host before any transfer using TRAX
+    eri_write(ESP_APPTRACE_TRAX_STAT_REG, 0);
+
+    ESP_APPTRACE_LOGI("Initialized TRAX on CPU%d", esp_cpu_get_core_id());
+}
+
+static inline void esp_apptrace_trax_select_memory_block(int block_num)
+{
+    // select memory block to be exposed to the TRAX module (accessed by host)
+#if CONFIG_IDF_TARGET_ESP32
+    DPORT_WRITE_PERI_REG(DPORT_TRACEMEM_MUX_MODE_REG, block_num ? TRACEMEM_MUX_BLK0_ONLY : TRACEMEM_MUX_BLK1_ONLY);
+#elif CONFIG_IDF_TARGET_ESP32S2
+    WRITE_PERI_REG(DPORT_PMS_OCCUPY_3_REG, block_num ? BIT(TRACEMEM_MUX_BLK0_NUM-4) : BIT(TRACEMEM_MUX_BLK1_NUM-4));
+#elif CONFIG_IDF_TARGET_ESP32S3
+    // select memory block to be exposed to the TRAX module (accessed by host)
+    uint32_t block_bits = block_num ? TRACEMEM_CORE0_MUX_BLK_BITS(TRACEMEM_MUX_BLK0_NUM)
+                        : TRACEMEM_CORE0_MUX_BLK_BITS(TRACEMEM_MUX_BLK1_NUM);
+    block_bits |= block_num ? TRACEMEM_CORE1_MUX_BLK_BITS(TRACEMEM_MUX_BLK0_NUM)
+                        : TRACEMEM_CORE1_MUX_BLK_BITS(TRACEMEM_MUX_BLK1_NUM);
+    ESP_EARLY_LOGV(TAG, "Select block %d @ %p (bits 0x%x)", block_num, s_trax_blocks[block_num], block_bits);
+    DPORT_WRITE_PERI_REG(SENSITIVE_INTERNAL_SRAM_USAGE_2_REG, block_bits);
+#endif
+}
+
+static inline void esp_apptrace_trax_memory_enable(void)
+{
+#if CONFIG_IDF_TARGET_ESP32
+        /* Enable trace memory on PRO CPU */
+        DPORT_WRITE_PERI_REG(DPORT_PRO_TRACEMEM_ENA_REG, DPORT_PRO_TRACEMEM_ENA_M);
+#if CONFIG_FREERTOS_UNICORE == 0
+        /* Enable trace memory on APP CPU */
+        DPORT_WRITE_PERI_REG(DPORT_APP_TRACEMEM_ENA_REG, DPORT_APP_TRACEMEM_ENA_M);
+#endif
+#endif
+}
+
+/*****************************************************************************************/
+/***************************** Apptrace HW iface *****************************************/
+/*****************************************************************************************/
+
+static esp_err_t esp_apptrace_trax_init(esp_apptrace_trax_data_t *hw_data)
+{
+    int core_id = esp_cpu_get_core_id();
+
+    // 'esp_apptrace_trax_init()' is called on every core, so ensure to do main initialization only once
+    if (core_id == 0) {
+        esp_apptrace_mem_block_t mem_blocks_cfg[2] = {
+            {
+                .start = s_trax_blocks[0],
+                .sz = ESP_APPTRACE_TRAX_BLOCK_SIZE
+            },
+            {
+                .start = s_trax_blocks[1],
+                .sz = ESP_APPTRACE_TRAX_BLOCK_SIZE
+            },
+        };
+        esp_err_t res = esp_apptrace_membufs_init(&hw_data->membufs, mem_blocks_cfg);
+        if (res != ESP_OK) {
+            ESP_APPTRACE_LOGE("Failed to init membufs proto (%d)!", res);
+            return res;
+        }
+#if CONFIG_APPTRACE_LOCK_ENABLE
+        esp_apptrace_lock_init(&hw_data->lock);
+#endif
+        esp_apptrace_trax_memory_enable();
+        esp_apptrace_trax_select_memory_block(0);
+    }
+    // init TRAX on this CPU
+    esp_apptrace_trax_hw_init();
+    hw_data->inited |= 1 << core_id;
+
+    return ESP_OK;
+}
+
+static uint8_t *esp_apptrace_trax_up_buffer_get(esp_apptrace_trax_data_t *hw_data, uint32_t size, esp_apptrace_tmo_t *tmo)
+{
+    uint8_t *ptr;
+
+    if (!ESP_APPTRACE_TRAX_INITED(hw_data)) {
+        return NULL;
+    }
+    esp_err_t res = esp_apptrace_trax_lock(hw_data, tmo);
+    if (res != ESP_OK) {
+        return NULL;
+    }
+
+    ptr = esp_apptrace_membufs_up_buffer_get(&hw_data->membufs, size, tmo);
+
+    // now we can safely unlock apptrace to allow other tasks/ISRs to get other buffers and write their data
+    if (esp_apptrace_trax_unlock(hw_data) != ESP_OK) {
+        assert(false && "Failed to unlock apptrace data!");
+    }
+    return ptr;
+}
+
+static esp_err_t esp_apptrace_trax_up_buffer_put(esp_apptrace_trax_data_t *hw_data, uint8_t *ptr, esp_apptrace_tmo_t *tmo)
+{
+    if (!ESP_APPTRACE_TRAX_INITED(hw_data)) {
+        return ESP_ERR_INVALID_STATE;
+    }
+    // Can avoid locking because esp_apptrace_membufs_up_buffer_put() just modifies buffer's header
+    esp_err_t res = esp_apptrace_membufs_up_buffer_put(&hw_data->membufs, ptr, tmo);
+    return res;
+}
+
+static void esp_apptrace_trax_down_buffer_config(esp_apptrace_trax_data_t *hw_data, uint8_t *buf, uint32_t size)
+{
+    if (!ESP_APPTRACE_TRAX_INITED(hw_data)) {
+        return;
+    }
+    esp_apptrace_membufs_down_buffer_config(&hw_data->membufs, buf, size);
+}
+
+static uint8_t *esp_apptrace_trax_down_buffer_get(esp_apptrace_trax_data_t *hw_data, uint32_t *size, esp_apptrace_tmo_t *tmo)
+{
+    uint8_t *ptr;
+
+    if (!ESP_APPTRACE_TRAX_INITED(hw_data)) {
+        return NULL;
+    }
+    esp_err_t res = esp_apptrace_trax_lock(hw_data, tmo);
+    if (res != ESP_OK) {
+        return NULL;
+    }
+
+    ptr = esp_apptrace_membufs_down_buffer_get(&hw_data->membufs, size, tmo);
+
+    // now we can safely unlock apptrace to allow other tasks/ISRs to get other buffers and write their data
+    if (esp_apptrace_trax_unlock(hw_data) != ESP_OK) {
+        assert(false && "Failed to unlock apptrace data!");
+    }
+    return ptr;
+}
+
+static esp_err_t esp_apptrace_trax_down_buffer_put(esp_apptrace_trax_data_t *hw_data, uint8_t *ptr, esp_apptrace_tmo_t *tmo)
+{
+    if (!ESP_APPTRACE_TRAX_INITED(hw_data)) {
+        return ESP_ERR_INVALID_STATE;
+    }
+    // Can avoid locking because esp_apptrace_membufs_down_buffer_put() does nothing
+    /*esp_err_t res = esp_apptrace_trax_lock(hw_data, tmo);
+    if (res != ESP_OK) {
+        return res;
+    }*/
+
+    esp_err_t res = esp_apptrace_membufs_down_buffer_put(&hw_data->membufs, ptr, tmo);
+
+    // now we can safely unlock apptrace to allow other tasks/ISRs to get other buffers and write their data
+    /*if (esp_apptrace_trax_unlock(hw_data) != ESP_OK) {
+        assert(false && "Failed to unlock apptrace data!");
+    }*/
+    return res;
+}
+
+static bool esp_apptrace_trax_host_is_connected(esp_apptrace_trax_data_t *hw_data)
+{
+    if (!ESP_APPTRACE_TRAX_INITED(hw_data)) {
+        return false;
+    }
+    return eri_read(ESP_APPTRACE_TRAX_CTRL_REG) & ESP_APPTRACE_TRAX_HOST_CONNECT ? true : false;
+}
+
+static esp_err_t esp_apptrace_trax_flush_nolock(esp_apptrace_trax_data_t *hw_data, uint32_t min_sz, esp_apptrace_tmo_t *tmo)
+{
+    if (!ESP_APPTRACE_TRAX_INITED(hw_data)) {
+        return ESP_ERR_INVALID_STATE;
+    }
+    return esp_apptrace_membufs_flush_nolock(&hw_data->membufs, min_sz, tmo);
+}
+
+static esp_err_t esp_apptrace_trax_flush(esp_apptrace_trax_data_t *hw_data, esp_apptrace_tmo_t *tmo)
+{
+    if (!ESP_APPTRACE_TRAX_INITED(hw_data)) {
+        return ESP_ERR_INVALID_STATE;
+    }
+    esp_err_t res = esp_apptrace_trax_lock(hw_data, tmo);
+    if (res != ESP_OK) {
+        return res;
+    }
+
+    res = esp_apptrace_membufs_flush_nolock(&hw_data->membufs, 0, tmo);
+
+    // now we can safely unlock apptrace to allow other tasks/ISRs to get other buffers and write their data
+    if (esp_apptrace_trax_unlock(hw_data) != ESP_OK) {
+        assert(false && "Failed to unlock apptrace data!");
+    }
+    return res;
+}
+
+/*****************************************************************************************/
+/************************** Membufs proto HW iface ***************************************/
+/*****************************************************************************************/
+
+static inline void esp_apptrace_trax_buffer_swap_lock(void)
+{
+    extern uint32_t __esp_apptrace_trax_eri_updated;
+
+    // indicate to host that we are about to update.
+    // this is used only to place CPU into streaming mode at tracing startup
+    // before starting streaming host can halt us after we read  ESP_APPTRACE_TRAX_CTRL_REG and before we updated it
+    // HACK: in this case host will set breakpoint just after ESP_APPTRACE_TRAX_CTRL_REG update,
+    // here we set address to set bp at
+    // enter ERI update critical section
+    eri_write(ESP_APPTRACE_TRAX_STAT_REG, (uint32_t)&__esp_apptrace_trax_eri_updated);
+}
+
+static __attribute__((noinline)) void esp_apptrace_trax_buffer_swap_unlock(void)
+{
+    // exit ERI update critical section
+    eri_write(ESP_APPTRACE_TRAX_STAT_REG, 0x0);
+    // TODO: currently host sets breakpoint, use break instruction to stop;
+    // it will allow to use ESP_APPTRACE_TRAX_STAT_REG for other purposes
+    asm volatile (
+        "    .global     __esp_apptrace_trax_eri_updated\n"
+        "__esp_apptrace_trax_eri_updated:\n"); // host will set bp here to resolve collision at streaming start
+}
+
+static esp_err_t esp_apptrace_trax_buffer_swap_start(uint32_t curr_block_id)
+{
+    esp_err_t res = ESP_OK;
+
+    esp_apptrace_trax_buffer_swap_lock();
+
+    uint32_t ctrl_reg = eri_read(ESP_APPTRACE_TRAX_CTRL_REG);
+    uint32_t host_connected = ESP_APPTRACE_TRAX_HOST_CONNECT & ctrl_reg;
+    if (host_connected) {
+        uint32_t acked_block = ESP_APPTRACE_TRAX_BLOCK_ID_GET(ctrl_reg);
+        uint32_t host_to_read = ESP_APPTRACE_TRAX_BLOCK_LEN_GET(ctrl_reg);
+        if (host_to_read != 0 || acked_block != (curr_block_id & ESP_APPTRACE_TRAX_BLOCK_ID_MSK)) {
+            ESP_APPTRACE_LOGD("HC[%d]: Can not switch %x %d %x %x/%lx", esp_cpu_get_core_id(), ctrl_reg, host_to_read, acked_block,
+                curr_block_id & ESP_APPTRACE_TRAX_BLOCK_ID_MSK, curr_block_id);
+            res = ESP_ERR_NO_MEM;
+            goto _on_err;
+        }
+    }
+    return ESP_OK;
+_on_err:
+    esp_apptrace_trax_buffer_swap_unlock();
+    return res;
+}
+
+static esp_err_t esp_apptrace_trax_buffer_swap_end(uint32_t new_block_id, uint32_t prev_block_len)
+{
+    uint32_t ctrl_reg = eri_read(ESP_APPTRACE_TRAX_CTRL_REG);
+    uint32_t host_connected = ESP_APPTRACE_TRAX_HOST_CONNECT & ctrl_reg;
+    eri_write(ESP_APPTRACE_TRAX_CTRL_REG, ESP_APPTRACE_TRAX_BLOCK_ID(new_block_id) |
+              host_connected | ESP_APPTRACE_TRAX_BLOCK_LEN(prev_block_len));
+    esp_apptrace_trax_buffer_swap_unlock();
+    return ESP_OK;
+}
+
+static esp_err_t esp_apptrace_trax_buffer_swap(uint32_t new_block_id)
+{
+    esp_apptrace_trax_select_memory_block(new_block_id);
+    return ESP_OK;
+}
+
+static bool esp_apptrace_trax_host_data_pending(void)
+{
+    uint32_t ctrl_reg = eri_read(ESP_APPTRACE_TRAX_CTRL_REG);
+    return (ctrl_reg & ESP_APPTRACE_TRAX_HOST_DATA) ? true : false;
+}

components/app_trace/private_include/esp_app_trace_membufs_proto.h

@@ -0,0 +1,63 @@
+/*
+ * SPDX-FileCopyrightText: 2020-2022 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#ifndef ESP_APP_TRACE_MEMBUFS_PROTO_H_
+#define ESP_APP_TRACE_MEMBUFS_PROTO_H_
+
+#include "esp_app_trace_util.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/** TRAX HW transport state */
+typedef struct {
+    uint32_t                   in_block;     // input block ID
+    // TODO: change to uint16_t
+    uint32_t                   markers[2];   // block filling level markers
+} esp_apptrace_membufs_state_t;
+
+/** memory block parameters,
+ * should be packed, because it is read from the host */
+typedef struct {
+    uint8_t   *start;   // start address
+    uint32_t   sz;      // size
+} esp_apptrace_mem_block_t;
+
+typedef struct {
+    esp_err_t (*swap_start)(uint32_t curr_block_id);
+    esp_err_t (*swap)(uint32_t new_block_id);
+    esp_err_t (*swap_end)(uint32_t new_block_id, uint32_t prev_block_len);
+    bool (*host_data_pending)(void);
+} esp_apptrace_membufs_proto_hw_t;
+
+typedef struct {
+    esp_apptrace_membufs_proto_hw_t *       hw;
+    volatile esp_apptrace_membufs_state_t   state;       // state
+    esp_apptrace_mem_block_t                blocks[2];   // memory blocks
+#if CONFIG_APPTRACE_PENDING_DATA_SIZE_MAX > 0
+    // ring buffer control struct for pending user blocks
+    esp_apptrace_rb_t                   rb_pend;
+    // storage for pending user blocks
+    uint8_t                             pending_data[CONFIG_APPTRACE_PENDING_DATA_SIZE_MAX + 1];
+#endif
+    // ring buffer control struct for data from host (down buffer)
+    esp_apptrace_rb_t                       rb_down;
+} esp_apptrace_membufs_proto_data_t;
+
+esp_err_t esp_apptrace_membufs_init(esp_apptrace_membufs_proto_data_t *proto, const esp_apptrace_mem_block_t blocks_cfg[2]);
+void esp_apptrace_membufs_down_buffer_config(esp_apptrace_membufs_proto_data_t *data, uint8_t *buf, uint32_t size);
+uint8_t *esp_apptrace_membufs_down_buffer_get(esp_apptrace_membufs_proto_data_t *proto, uint32_t *size, esp_apptrace_tmo_t *tmo);
+esp_err_t esp_apptrace_membufs_down_buffer_put(esp_apptrace_membufs_proto_data_t *proto, uint8_t *ptr, esp_apptrace_tmo_t *tmo);
+uint8_t *esp_apptrace_membufs_up_buffer_get(esp_apptrace_membufs_proto_data_t *proto, uint32_t size, esp_apptrace_tmo_t *tmo);
+esp_err_t esp_apptrace_membufs_up_buffer_put(esp_apptrace_membufs_proto_data_t *proto, uint8_t *ptr, esp_apptrace_tmo_t *tmo);
+esp_err_t esp_apptrace_membufs_flush_nolock(esp_apptrace_membufs_proto_data_t *proto, uint32_t min_sz, esp_apptrace_tmo_t *tmo);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

components/app_trace/project_include.cmake

@@ -2,7 +2,7 @@
 #
 # Create coverage report.
 function(idf_create_coverage_report report_dir)
-    set(gcov_tool ${CONFIG_SDK_TOOLPREFIX}gcov)
+    set(gcov_tool ${_CMAKE_TOOLCHAIN_PREFIX}gcov)
     idf_build_get_property(project_name PROJECT_NAME)
 
     add_custom_target(pre-cov-report

components/app_trace/sdkconfig.rename

@@ -3,10 +3,30 @@
 
 CONFIG_ESP32_APPTRACE_DESTINATION      					CONFIG_APPTRACE_DESTINATION
 CONFIG_ESP32_APPTRACE_DEST_NONE							CONFIG_APPTRACE_DEST_NONE
-CONFIG_ESP32_APPTRACE_DEST_TRAX							CONFIG_APPTRACE_DEST_TRAX
+CONFIG_ESP32_APPTRACE_DEST_TRAX							CONFIG_APPTRACE_DEST_JTAG
 CONFIG_ESP32_APPTRACE_ENABLE							CONFIG_APPTRACE_ENABLE
 CONFIG_ESP32_APPTRACE_LOCK_ENABLE						CONFIG_APPTRACE_LOCK_ENABLE
 CONFIG_ESP32_APPTRACE_ONPANIC_HOST_FLUSH_TMO			CONFIG_APPTRACE_ONPANIC_HOST_FLUSH_TMO
 CONFIG_ESP32_APPTRACE_POSTMORTEM_FLUSH_TRAX_THRESH      CONFIG_APPTRACE_POSTMORTEM_FLUSH_THRESH
 CONFIG_ESP32_APPTRACE_PENDING_DATA_SIZE_MAX				CONFIG_APPTRACE_PENDING_DATA_SIZE_MAX
 CONFIG_ESP32_GCOV_ENABLE								CONFIG_APPTRACE_GCOV_ENABLE
+
+CONFIG_SYSVIEW_ENABLE							        CONFIG_APPTRACE_SV_ENABLE
+CONFIG_SYSVIEW_TS_SOURCE							    CONFIG_APPTRACE_SV_TS_SOURCE
+CONFIG_SYSVIEW_TS_SOURCE_CCOUNT							CONFIG_APPTRACE_SV_TS_SOURCE_CCOUNT
+CONFIG_SYSVIEW_TS_SOURCE_ESP_TIMER						CONFIG_APPTRACE_SV_TS_SOURCE_ESP_TIMER
+CONFIG_SYSVIEW_MAX_TASKS							    CONFIG_APPTRACE_SV_MAX_TASKS
+CONFIG_SYSVIEW_BUF_WAIT_TMO							    CONFIG_APPTRACE_SV_BUF_WAIT_TMO
+CONFIG_SYSVIEW_EVT_OVERFLOW_ENABLE						CONFIG_APPTRACE_SV_EVT_OVERFLOW_ENABLE
+CONFIG_SYSVIEW_EVT_ISR_ENTER_ENABLE						CONFIG_APPTRACE_SV_EVT_ISR_ENTER_ENABLE
+CONFIG_SYSVIEW_EVT_ISR_EXIT_ENABLE						CONFIG_APPTRACE_SV_EVT_ISR_EXIT_ENABLE
+CONFIG_SYSVIEW_EVT_ISR_TO_SCHEDULER_ENABLE				CONFIG_APPTRACE_SV_EVT_ISR_TO_SCHED_ENABLE
+CONFIG_SYSVIEW_EVT_TASK_START_EXEC_ENABLE				CONFIG_APPTRACE_SV_EVT_TASK_START_EXEC_ENABLE
+CONFIG_SYSVIEW_EVT_TASK_STOP_EXEC_ENABLE				CONFIG_APPTRACE_SV_EVT_TASK_STOP_EXEC_ENABLE
+CONFIG_SYSVIEW_EVT_TASK_START_READY_ENABLE				CONFIG_APPTRACE_SV_EVT_TASK_START_READY_ENABLE
+CONFIG_SYSVIEW_EVT_TASK_STOP_READY_ENABLE				CONFIG_APPTRACE_SV_EVT_TASK_STOP_READY_ENABLE
+CONFIG_SYSVIEW_EVT_TASK_CREATE_ENABLE					CONFIG_APPTRACE_SV_EVT_TASK_CREATE_ENABLE
+CONFIG_SYSVIEW_EVT_TASK_TERMINATE_ENABLE				CONFIG_APPTRACE_SV_EVT_TASK_TERMINATE_ENABLE
+CONFIG_SYSVIEW_EVT_IDLE_ENABLE							CONFIG_APPTRACE_SV_EVT_IDLE_ENABLE
+CONFIG_SYSVIEW_EVT_TIMER_ENTER_ENABLE					CONFIG_APPTRACE_SV_EVT_TIMER_ENTER_ENABLE
+CONFIG_SYSVIEW_EVT_TIMER_EXIT_ENABLE					CONFIG_APPTRACE_SV_EVT_TIMER_EXIT_ENABLE

components/app_trace/sys_view/Config/Global.h

@@ -1,3 +1,8 @@
+/*
+ * SPDX-FileCopyrightText: 2015-2017 SEGGER Microcontroller GmbH & Co. KG
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
 /*********************************************************************
 *                SEGGER Microcontroller GmbH & Co. KG                *
 *                        The Embedded Experts                        *

components/app_trace/sys_view/Config/SEGGER_RTT_Conf.h

@@ -1,3 +1,8 @@
+/*
+ * SPDX-FileCopyrightText: 2015-2017 SEGGER Microcontroller GmbH & Co. KG
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
 /*********************************************************************
 *                SEGGER Microcontroller GmbH & Co. KG                *
 *                        The Embedded Experts                        *

components/app_trace/sys_view/Config/SEGGER_SYSVIEW_Conf.h

@@ -1,3 +1,8 @@
+/*
+ * SPDX-FileCopyrightText: 2015-2017 SEGGER Microcontroller GmbH & Co. KG
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
 /*********************************************************************
 *                SEGGER Microcontroller GmbH & Co. KG                *
 *                        The Embedded Experts                        *

components/app_trace/sys_view/SEGGER/SEGGER.h

@@ -1,3 +1,8 @@
+/*
+ * SPDX-FileCopyrightText: 2015-2017 SEGGER Microcontroller GmbH & Co. KG
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
 /*********************************************************************
 *                SEGGER Microcontroller GmbH & Co. KG                *
 *                        The Embedded Experts                        *

components/app_trace/sys_view/SEGGER/SEGGER_RTT.h

@@ -1,3 +1,8 @@
+/*
+ * SPDX-FileCopyrightText: 2015-2017 SEGGER Microcontroller GmbH & Co. KG
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
 /*********************************************************************
 *                SEGGER Microcontroller GmbH & Co. KG                *
 *                        The Embedded Experts                        *
@@ -159,8 +164,8 @@ unsigned     SEGGER_RTT_WriteNoLock             (unsigned BufferIndex, const voi
 unsigned     SEGGER_RTT_WriteSkipNoLock         (unsigned BufferIndex, const void* pBuffer, unsigned NumBytes);
 unsigned     SEGGER_RTT_WriteString             (unsigned BufferIndex, const char* s);
 void         SEGGER_RTT_WriteWithOverwriteNoLock(unsigned BufferIndex, const void* pBuffer, unsigned NumBytes);
-void         SEGGER_RTT_ESP32_FlushNoLock       (unsigned long min_sz, unsigned long tmo);
-void         SEGGER_RTT_ESP32_Flush             (unsigned long min_sz, unsigned long tmo);
+void         SEGGER_RTT_ESP_FlushNoLock       (unsigned long min_sz, unsigned long tmo);
+void         SEGGER_RTT_ESP_Flush             (unsigned long min_sz, unsigned long tmo);
 //
 // Function macro for performance optimization
 //

components/app_trace/sys_view/SEGGER/SEGGER_SYSVIEW.c

@@ -1,3 +1,8 @@
+/*
+ * SPDX-FileCopyrightText: 2015-2017 SEGGER Microcontroller GmbH & Co. KG
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
 /*********************************************************************
 *                SEGGER Microcontroller GmbH & Co. KG                *
 *                        The Embedded Experts                        *
@@ -69,7 +74,7 @@ Additional information:
 
     Packets with IDs 24..31 are standard packets with extendible
     structure and contain a length field.
-    <ID><Lenght><Data><TimeStampDelta>
+    <ID><Length><Data><TimeStampDelta>
 
     Packets with IDs >= 32 always contain a length field.
     <ID><Length><Data><TimeStampDelta>
@@ -1681,7 +1686,8 @@ void SEGGER_SYSVIEW_Start(void) {
 void SEGGER_SYSVIEW_Stop(void) {
   U8* pPayloadStart;
   RECORD_START(SEGGER_SYSVIEW_INFO_SIZE);
-  //
+  // We should send answer for the Stop command in any case.
+  _SYSVIEW_Globals.EnableState = 1;
   if (_SYSVIEW_Globals.EnableState) {
     _SendPacket(pPayloadStart, pPayloadStart, SYSVIEW_EVTID_TRACE_STOP);
     _SYSVIEW_Globals.EnableState = 0;
@@ -1801,6 +1807,10 @@ void SEGGER_SYSVIEW_SendSysDesc(const char *sSysDesc) {
 */
 void SEGGER_SYSVIEW_RecordSystime(void) {
   U64 Systime;
+    // This code requeued because SystemView expect the answer from the device.
+    // If there is no answer, then communication will be broken.
+    U8 old_en = _SYSVIEW_Globals.EnableState;
+    _SYSVIEW_Globals.EnableState = 1;
 
   if (_SYSVIEW_Globals.pOSAPI && _SYSVIEW_Globals.pOSAPI->pfGetTime) {
     Systime = _SYSVIEW_Globals.pOSAPI->pfGetTime();
@@ -1810,6 +1820,7 @@ void SEGGER_SYSVIEW_RecordSystime(void) {
   } else {
     SEGGER_SYSVIEW_RecordU32(SYSVIEW_EVTID_SYSTIME_CYCLES, SEGGER_SYSVIEW_GET_TIMESTAMP());
   }
+    _SYSVIEW_Globals.EnableState = old_en;
 }
 
 /*********************************************************************

components/app_trace/sys_view/SEGGER/SEGGER_SYSVIEW.h

@@ -1,3 +1,8 @@
+/*
+ * SPDX-FileCopyrightText: 2015-2017 SEGGER Microcontroller GmbH & Co. KG
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
 /*********************************************************************
 *                SEGGER Microcontroller GmbH & Co. KG                *
 *                        The Embedded Experts                        *

components/app_trace/sys_view/SEGGER/SEGGER_SYSVIEW_ConfDefaults.h

@@ -1,3 +1,8 @@
+/*
+ * SPDX-FileCopyrightText: 2015-2017 SEGGER Microcontroller GmbH & Co. KG
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
 /*********************************************************************
 *                SEGGER Microcontroller GmbH & Co. KG                *
 *                        The Embedded Experts                        *

components/app_trace/sys_view/SEGGER/SEGGER_SYSVIEW_Int.h

@@ -1,3 +1,8 @@
+/*
+ * SPDX-FileCopyrightText: 2015-2017 SEGGER Microcontroller GmbH & Co. KG
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
 /*********************************************************************
 *                SEGGER Microcontroller GmbH & Co. KG                *
 *                        The Embedded Experts                        *

components/app_trace/sys_view/Sample/Config/SEGGER_SYSVIEW_Config_FreeRTOS.c

@@ -1,3 +1,10 @@
+/*
+ * SPDX-FileCopyrightText: 2015-2017 SEGGER Microcontroller GmbH & Co. KG
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ *
+ * SPDX-FileContributor: 2017-2022 Espressif Systems (Shanghai) CO LTD
+ */
 /*********************************************************************
 *                SEGGER Microcontroller GmbH & Co. KG                *
 *                        The Embedded Experts                        *
@@ -69,12 +76,7 @@ Revision: $Rev: 3734 $
 #include "esp_intr_alloc.h"
 #include "soc/soc.h"
 #include "soc/interrupts.h"
-#if CONFIG_IDF_TARGET_ESP32
-#include "esp32/clk.h"
-#elif CONFIG_IDF_TARGET_ESP32S2
-#include "esp32s2/clk.h"
-#endif
-
+#include "esp_private/esp_clk.h"
 
 extern const SEGGER_SYSVIEW_OS_API SYSVIEW_X_OS_TraceAPI;
 
@@ -89,37 +91,33 @@ extern const SEGGER_SYSVIEW_OS_API SYSVIEW_X_OS_TraceAPI;
 
 // The target device name
 #define SYSVIEW_DEVICE_NAME     CONFIG_IDF_TARGET
+// The target core name
+#if CONFIG_IDF_TARGET_ARCH_XTENSA
+#define SYSVIEW_CORE_NAME       "xtensa"
+#elif CONFIG_IDF_TARGET_ARCH_RISCV
+#define SYSVIEW_CORE_NAME       "riscv"
+#endif
 
 // Determine which timer to use as timestamp source
-#if CONFIG_SYSVIEW_TS_SOURCE_CCOUNT
+#if CONFIG_APPTRACE_SV_TS_SOURCE_CCOUNT
 #define TS_USE_CCOUNT 1
-#elif CONFIG_SYSVIEW_TS_SOURCE_ESP_TIMER
+#elif CONFIG_APPTRACE_SV_TS_SOURCE_ESP_TIMER
 #define TS_USE_ESP_TIMER 1
 #else
 #define TS_USE_TIMERGROUP 1
 #endif
 
 #if TS_USE_TIMERGROUP
-#include "driver/timer.h"
+#include "driver/gptimer.h"
 
 // Timer group timer divisor
 #define SYSVIEW_TIMER_DIV       2
 
-// Frequency of the timestamp.
+// Frequency of the timestamp, using APB as GPTimer source clock
 #define SYSVIEW_TIMESTAMP_FREQ  (esp_clk_apb_freq() / SYSVIEW_TIMER_DIV)
 
-// Timer ID and group ID
-#if defined(CONFIG_SYSVIEW_TS_SOURCE_TIMER_00) || defined(CONFIG_SYSVIEW_TS_SOURCE_TIMER_01)
-#define TS_TIMER_ID 0
-#else
-#define TS_TIMER_ID 1
-#endif // TIMER_00 || TIMER_01
-
-#if defined(CONFIG_SYSVIEW_TS_SOURCE_TIMER_00) || defined(CONFIG_SYSVIEW_TS_SOURCE_TIMER_10)
-#define TS_TIMER_GROUP 0
-#else
-#define TS_TIMER_GROUP 1
-#endif // TIMER_00 || TIMER_10
+// GPTimer handle
+gptimer_handle_t s_sv_gptimer;
 
 #endif // TS_USE_TIMERGROUP
 
@@ -130,11 +128,7 @@ extern const SEGGER_SYSVIEW_OS_API SYSVIEW_X_OS_TraceAPI;
 
 #if TS_USE_CCOUNT
 // CCOUNT is incremented at CPU frequency
-#if CONFIG_IDF_TARGET_ESP32
-#define SYSVIEW_TIMESTAMP_FREQ  (CONFIG_ESP32_DEFAULT_CPU_FREQ_MHZ * 1000000)
-#elif CONFIG_IDF_TARGET_ESP32S2
-#define SYSVIEW_TIMESTAMP_FREQ  (CONFIG_ESP32S2_DEFAULT_CPU_FREQ_MHZ * 1000000)
-#endif
+#define SYSVIEW_TIMESTAMP_FREQ  (CONFIG_ESP_DEFAULT_CPU_FREQ_MHZ * 1000000)
 #endif // TS_USE_CCOUNT
 
 // System Frequency.
@@ -143,6 +137,7 @@ extern const SEGGER_SYSVIEW_OS_API SYSVIEW_X_OS_TraceAPI;
 // The lowest RAM address used for IDs (pointers)
 #define SYSVIEW_RAM_BASE        (SOC_DROM_LOW)
 
+#ifdef CONFIG_FREERTOS_TICK_SUPPORT_CORETIMER
 #if CONFIG_FREERTOS_CORETIMER_0
     #define SYSTICK_INTR_ID (ETS_INTERNAL_TIMER0_INTR_SOURCE+ETS_INTERNAL_INTR_SOURCE_OFF)
 #endif
@@ -150,6 +145,10 @@ extern const SEGGER_SYSVIEW_OS_API SYSVIEW_X_OS_TraceAPI;
     #define SYSTICK_INTR_ID (ETS_INTERNAL_TIMER1_INTR_SOURCE+ETS_INTERNAL_INTR_SOURCE_OFF)
 #endif
 
+#elif CONFIG_FREERTOS_SYSTICK_USES_SYSTIMER
+    #define SYSTICK_INTR_ID (ETS_SYSTIMER_TARGET0_EDGE_INTR_SOURCE)
+#endif // CONFIG_FREERTOS_TICK_SUPPORT_CORETIMER
+
 // SystemView is single core specific: it implies that SEGGER_SYSVIEW_LOCK()
 // disables IRQs (disables rescheduling globally). So we can not use finite timeouts for locks and return error
 // in case of expiration, because error will not be handled and SEGGER's code will go further implying that
@@ -167,11 +166,13 @@ static esp_apptrace_lock_t s_sys_view_lock = {.mux = portMUX_INITIALIZER_UNLOCKE
 */
 static void _cbSendSystemDesc(void) {
     char irq_str[32];
-    SEGGER_SYSVIEW_SendSysDesc("N="SYSVIEW_APP_NAME",D="SYSVIEW_DEVICE_NAME",C=Xtensa,O=FreeRTOS");
+    SEGGER_SYSVIEW_SendSysDesc("N="SYSVIEW_APP_NAME",D="SYSVIEW_DEVICE_NAME",C="SYSVIEW_CORE_NAME",O=FreeRTOS");
     snprintf(irq_str, sizeof(irq_str), "I#%d=SysTick", SYSTICK_INTR_ID);
     SEGGER_SYSVIEW_SendSysDesc(irq_str);
     size_t isr_count = sizeof(esp_isr_names)/sizeof(esp_isr_names[0]);
     for (size_t i = 0; i < isr_count; ++i) {
+        if (esp_isr_names[i] == NULL || (ETS_INTERNAL_INTR_SOURCE_OFF + i) == SYSTICK_INTR_ID)
+            continue;
         snprintf(irq_str, sizeof(irq_str), "I#%d=%s", ETS_INTERNAL_INTR_SOURCE_OFF + i, esp_isr_names[i]);
         SEGGER_SYSVIEW_SendSysDesc(irq_str);
     }
@@ -189,19 +190,16 @@ static void SEGGER_SYSVIEW_TS_Init(void)
      * esp_timer and ccount can be used as is.
      */
 #if TS_USE_TIMERGROUP
-    timer_config_t config = {
-        .alarm_en = 0,
-        .auto_reload = 0,
-        .counter_dir = TIMER_COUNT_UP,
-        .divider = SYSVIEW_TIMER_DIV,
-        .counter_en = 0
+    gptimer_config_t config = {
+        .clk_src = GPTIMER_CLK_SRC_DEFAULT,
+        .direction = GPTIMER_COUNT_UP,
+        .resolution_hz = SYSVIEW_TIMESTAMP_FREQ,
     };
-    /* Configure timer */
-    timer_init(TS_TIMER_GROUP, TS_TIMER_ID, &config);
-    /* Load counter value */
-    timer_set_counter_value(TS_TIMER_GROUP, TS_TIMER_ID, 0x00000000ULL);
+    // pick any free GPTimer instance
+    ESP_ERROR_CHECK(gptimer_new_timer(&config, &s_sv_gptimer));
     /* Start counting */
-    timer_start(TS_TIMER_GROUP, TS_TIMER_ID);
+    gptimer_enable(s_sv_gptimer);
+    gptimer_start(s_sv_gptimer);
 #endif // TS_USE_TIMERGROUP
 }
 
@@ -213,43 +211,43 @@ void SEGGER_SYSVIEW_Conf(void) {
                         &SYSVIEW_X_OS_TraceAPI, _cbSendSystemDesc);
     SEGGER_SYSVIEW_SetRAMBase(SYSVIEW_RAM_BASE);
 
-#if !CONFIG_SYSVIEW_EVT_OVERFLOW_ENABLE
+#if !CONFIG_APPTRACE_SV_EVT_OVERFLOW_ENABLE
     disable_evts |= SYSVIEW_EVTMASK_OVERFLOW;
 #endif
-#if !CONFIG_SYSVIEW_EVT_ISR_ENTER_ENABLE
+#if !CONFIG_APPTRACE_SV_EVT_ISR_ENTER_ENABLE
     disable_evts |= SYSVIEW_EVTMASK_ISR_ENTER;
 #endif
-#if !CONFIG_SYSVIEW_EVT_ISR_EXIT_ENABLE
+#if !CONFIG_APPTRACE_SV_EVT_ISR_EXIT_ENABLE
     disable_evts |= SYSVIEW_EVTMASK_ISR_EXIT;
 #endif
-#if !CONFIG_SYSVIEW_EVT_TASK_START_EXEC_ENABLE
+#if !CONFIG_APPTRACE_SV_EVT_TASK_START_EXEC_ENABLE
     disable_evts |= SYSVIEW_EVTMASK_TASK_START_EXEC;
 #endif
-#if !CONFIG_SYSVIEW_EVT_TASK_STOP_EXEC_ENABLE
+#if !CONFIG_APPTRACE_SV_EVT_TASK_STOP_EXEC_ENABLE
     disable_evts |= SYSVIEW_EVTMASK_TASK_STOP_EXEC;
 #endif
-#if !CONFIG_SYSVIEW_EVT_TASK_START_READY_ENABLE
+#if !CONFIG_APPTRACE_SV_EVT_TASK_START_READY_ENABLE
     disable_evts |= SYSVIEW_EVTMASK_TASK_START_READY;
 #endif
-#if !CONFIG_SYSVIEW_EVT_TASK_STOP_READY_ENABLE
+#if !CONFIG_APPTRACE_SV_EVT_TASK_STOP_READY_ENABLE
     disable_evts |= SYSVIEW_EVTMASK_TASK_STOP_READY;
 #endif
-#if !CONFIG_SYSVIEW_EVT_TASK_CREATE_ENABLE
+#if !CONFIG_APPTRACE_SV_EVT_TASK_CREATE_ENABLE
     disable_evts |= SYSVIEW_EVTMASK_TASK_CREATE;
 #endif
-#if !CONFIG_SYSVIEW_EVT_TASK_TERMINATE_ENABLE
+#if !CONFIG_APPTRACE_SV_EVT_TASK_TERMINATE_ENABLE
     disable_evts |= SYSVIEW_EVTMASK_TASK_TERMINATE;
 #endif
-#if !CONFIG_SYSVIEW_EVT_IDLE_ENABLE
+#if !CONFIG_APPTRACE_SV_EVT_IDLE_ENABLE
     disable_evts |= SYSVIEW_EVTMASK_IDLE;
 #endif
-#if !CONFIG_SYSVIEW_EVT_ISR_TO_SCHEDULER_ENABLE
+#if !CONFIG_APPTRACE_SV_EVT_ISR_TO_SCHED_ENABLE
     disable_evts |= SYSVIEW_EVTMASK_ISR_TO_SCHEDULER;
 #endif
-#if !CONFIG_SYSVIEW_EVT_TIMER_ENTER_ENABLE
+#if !CONFIG_APPTRACE_SV_EVT_TIMER_ENTER_ENABLE
     disable_evts |= SYSVIEW_EVTMASK_TIMER_ENTER;
 #endif
-#if !CONFIG_SYSVIEW_EVT_TIMER_EXIT_ENABLE
+#if !CONFIG_APPTRACE_SV_EVT_TIMER_EXIT_ENABLE
     disable_evts |= SYSVIEW_EVTMASK_TIMER_EXIT;
 #endif
   SEGGER_SYSVIEW_DisableEvents(disable_evts);
@@ -259,7 +257,7 @@ U32 SEGGER_SYSVIEW_X_GetTimestamp(void)
 {
 #if TS_USE_TIMERGROUP
     uint64_t ts = 0;
-    timer_get_counter_value(TS_TIMER_GROUP, TS_TIMER_ID, &ts);
+    gptimer_get_raw_count(s_sv_gptimer, &ts);
     return (U32) ts; // return lower part of counter value
 #elif TS_USE_CCOUNT
     return portGET_RUN_TIME_COUNTER_VALUE();

components/app_trace/sys_view/Sample/OS/SEGGER_SYSVIEW_FreeRTOS.c

@@ -1,3 +1,8 @@
+/*
+ * SPDX-FileCopyrightText: 2015-2017 SEGGER Microcontroller GmbH & Co. KG
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
 /*********************************************************************
 *                SEGGER Microcontroller GmbH & Co. KG                *
 *                        The Embedded Experts                        *
@@ -137,7 +142,11 @@ static U64 _cbGetTime(void) {
 void SYSVIEW_AddTask(U32 xHandle, const char* pcTaskName, unsigned uxCurrentPriority, U32  pxStack, unsigned uStackHighWaterMark) {
   unsigned n;
 
-  if (memcmp(pcTaskName, "IDLE", 5) == 0) {
+  /* On multi-core we have several idle tasks with 'IDLEx' names
+     Not best solution, because we can filter out user tasks starting with 'IDLE'.
+     But we can not use 'xTaskGetIdleTaskHandle' because at the moment when this
+     function is called array of idle tasks handles are not initialized yet. */
+  if (memcmp(pcTaskName, "IDLE", 4) == 0) {
     return;
   }
 
@@ -171,7 +180,11 @@ void SYSVIEW_AddTask(U32 xHandle, const char* pcTaskName, unsigned uxCurrentPrio
 void SYSVIEW_UpdateTask(U32 xHandle, const char* pcTaskName, unsigned uxCurrentPriority, U32 pxStack, unsigned uStackHighWaterMark) {
   unsigned n;
 
-  if (memcmp(pcTaskName, "IDLE", 5) == 0) {
+  /* On multi-core we have several idle tasks with 'IDLEx' names
+     Not best solution, because we can filter out user tasks starting with 'IDLE'.
+     But we can not use 'xTaskGetIdleTaskHandle' because at the moment when this
+     function is called array of idle tasks handles are not initialized yet. */
+  if (memcmp(pcTaskName, "IDLE", 4) == 0) {
     return;
   }
 

components/app_trace/sys_view/Sample/OS/SEGGER_SYSVIEW_FreeRTOS.h

@@ -1,3 +1,8 @@
+/*
+ * SPDX-FileCopyrightText: 2015-2017 SEGGER Microcontroller GmbH & Co. KG
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
 /*********************************************************************
 *                SEGGER Microcontroller GmbH & Co. KG                *
 *                        The Embedded Experts                        *
@@ -76,11 +81,8 @@ Notes:
 *
 **********************************************************************
 */
-#ifndef portSTACK_GROWTH
-  #define portSTACK_GROWTH              ( -1 )
-#endif
 
-#define SYSVIEW_FREERTOS_MAX_NOF_TASKS  CONFIG_SYSVIEW_MAX_TASKS
+#define SYSVIEW_FREERTOS_MAX_NOF_TASKS  CONFIG_APPTRACE_SV_MAX_TASKS
 
 /*********************************************************************
 *
@@ -209,24 +211,38 @@ Notes:
 #define apiID_VEVENTGROUPDELETE                   (72u)
 #define apiID_UXEVENTGROUPGETNUMBER               (73u)
 
-#define traceTASK_NOTIFY_TAKE()                                       SEGGER_SYSVIEW_RecordU32x2(apiFastID_OFFSET + apiID_ULTASKNOTIFYTAKE, xClearCountOnExit, xTicksToWait)
+#ifdef CONFIG_FREERTOS_SMP
+
+#define traceQUEUE_SEND( pxQueue )                  SYSVIEW_RecordU32x4(apiFastID_OFFSET + apiID_XQUEUEGENERICSEND, SEGGER_SYSVIEW_ShrinkId((U32)pxQueue), 0, 0, 0)
+
+#else // CONFIG_FREERTOS_SMP
+
+#if ( configUSE_QUEUE_SETS != 1 )
+  #define traceQUEUE_SEND( pxQueue )                SYSVIEW_RecordU32x4(apiFastID_OFFSET + apiID_XQUEUEGENERICSEND, SEGGER_SYSVIEW_ShrinkId((U32)pxQueue), (U32)pvItemToQueue, xTicksToWait, xCopyPosition)
+#else
+  #define traceQUEUE_SEND( pxQueue )                SYSVIEW_RecordU32x4(apiFastID_OFFSET + apiID_XQUEUEGENERICSEND, SEGGER_SYSVIEW_ShrinkId((U32)pxQueue), 0, 0, xCopyPosition)
+#endif
+
+#endif // CONFIG_FREERTOS_SMP
+
+#define traceTASK_NOTIFY_TAKE( uxIndexToWait )                        SEGGER_SYSVIEW_RecordU32x2(apiFastID_OFFSET + apiID_ULTASKNOTIFYTAKE, xClearCountOnExit, xTicksToWait)
 #define traceTASK_DELAY()                                             SEGGER_SYSVIEW_RecordU32(apiFastID_OFFSET + apiID_VTASKDELAY, xTicksToDelay)
-#define traceTASK_DELAY_UNTIL()                                       SEGGER_SYSVIEW_RecordVoid(apiFastID_OFFSET + apiID_VTASKDELAYUNTIL)
-#define traceTASK_DELETE( pxTCB )                                     if (pxTCB != NULL) {                                              \
-						                                                SEGGER_SYSVIEW_RecordU32(apiFastID_OFFSET + apiID_VTASKDELETE,  \
-						                                                                      SEGGER_SYSVIEW_ShrinkId((U32)pxTCB)); 	\
-					                                                    SYSVIEW_DeleteTask((U32)pxTCB);                                 \
-					                                                  }
-#define traceTASK_NOTIFY_GIVE_FROM_ISR()                              SEGGER_SYSVIEW_RecordU32x2(apiFastID_OFFSET + apiID_VTASKNOTIFYGIVEFROMISR, SEGGER_SYSVIEW_ShrinkId((U32)pxTCB), (U32)pxHigherPriorityTaskWoken)
+#define traceTASK_DELAY_UNTIL( xTimeToWake )                          SEGGER_SYSVIEW_RecordVoid(apiFastID_OFFSET + apiID_VTASKDELAYUNTIL)
+#define traceTASK_DELETE( pxTCB )                                     do {                                              \
+                                                                        SEGGER_SYSVIEW_RecordU32(apiFastID_OFFSET + apiID_VTASKDELETE,  \
+                                                                                              SEGGER_SYSVIEW_ShrinkId((U32)pxTCB)); 	\
+                                                                        SYSVIEW_DeleteTask((U32)pxTCB);                                 \
+                                                                      } while(0)
+#define traceTASK_NOTIFY_GIVE_FROM_ISR( uxIndexToNotify )             SEGGER_SYSVIEW_RecordU32x2(apiFastID_OFFSET + apiID_VTASKNOTIFYGIVEFROMISR, SEGGER_SYSVIEW_ShrinkId((U32)pxTCB), (U32)pxHigherPriorityTaskWoken)
 #define traceTASK_PRIORITY_INHERIT( pxTCB, uxPriority )               SEGGER_SYSVIEW_RecordU32(apiFastID_OFFSET + apiID_VTASKPRIORITYINHERIT, (U32)pxMutexHolder)
 #define traceTASK_RESUME( pxTCB )                                     SEGGER_SYSVIEW_RecordU32(apiFastID_OFFSET + apiID_VTASKRESUME, SEGGER_SYSVIEW_ShrinkId((U32)pxTCB))
 #define traceINCREASE_TICK_COUNT( xTicksToJump )                      SEGGER_SYSVIEW_RecordU32(apiFastID_OFFSET + apiID_VTASKSTEPTICK, xTicksToJump)
 #define traceTASK_SUSPEND( pxTCB )                                    SEGGER_SYSVIEW_RecordU32(apiFastID_OFFSET + apiID_VTASKSUSPEND, SEGGER_SYSVIEW_ShrinkId((U32)pxTCB))
 #define traceTASK_PRIORITY_DISINHERIT( pxTCB, uxBasePriority )        SEGGER_SYSVIEW_RecordU32(apiFastID_OFFSET + apiID_XTASKPRIORITYDISINHERIT, (U32)pxMutexHolder)
 #define traceTASK_RESUME_FROM_ISR( pxTCB )                            SEGGER_SYSVIEW_RecordU32(apiFastID_OFFSET + apiID_XTASKRESUMEFROMISR, SEGGER_SYSVIEW_ShrinkId((U32)pxTCB))
-#define traceTASK_NOTIFY()                                            SYSVIEW_RecordU32x4(apiFastID_OFFSET + apiID_XTASKGENERICNOTIFY, SEGGER_SYSVIEW_ShrinkId((U32)pxTCB), ulValue, eAction, (U32)pulPreviousNotificationValue)
-#define traceTASK_NOTIFY_FROM_ISR()                                   SYSVIEW_RecordU32x5(apiFastID_OFFSET + apiID_XTASKGENERICNOTIFYFROMISR, SEGGER_SYSVIEW_ShrinkId((U32)pxTCB), ulValue, eAction, (U32)pulPreviousNotificationValue, (U32)pxHigherPriorityTaskWoken)
-#define traceTASK_NOTIFY_WAIT()                                       SYSVIEW_RecordU32x4(apiFastID_OFFSET + apiID_XTASKNOTIFYWAIT, ulBitsToClearOnEntry, ulBitsToClearOnExit, (U32)pulNotificationValue, xTicksToWait)
+#define traceTASK_NOTIFY( uxIndexToNotify )                           SYSVIEW_RecordU32x4(apiFastID_OFFSET + apiID_XTASKGENERICNOTIFY, SEGGER_SYSVIEW_ShrinkId((U32)pxTCB), ulValue, eAction, (U32)pulPreviousNotificationValue)
+#define traceTASK_NOTIFY_FROM_ISR( uxIndexToNotify )                  SYSVIEW_RecordU32x5(apiFastID_OFFSET + apiID_XTASKGENERICNOTIFYFROMISR, SEGGER_SYSVIEW_ShrinkId((U32)pxTCB), ulValue, eAction, (U32)pulPreviousNotificationValue, (U32)pxHigherPriorityTaskWoken)
+#define traceTASK_NOTIFY_WAIT( uxIndexToWait )                        SYSVIEW_RecordU32x4(apiFastID_OFFSET + apiID_XTASKNOTIFYWAIT, ulBitsToClearOnEntry, ulBitsToClearOnExit, (U32)pulNotificationValue, xTicksToWait)
 
 #define traceQUEUE_CREATE( pxNewQueue )                               SEGGER_SYSVIEW_RecordU32x3(apiFastID_OFFSET + apiID_XQUEUEGENERICCREATE, uxQueueLength, uxItemSize, ucQueueType)
 #define traceQUEUE_DELETE( pxQueue )                                  SEGGER_SYSVIEW_RecordU32(apiFastID_OFFSET + apiID_VQUEUEDELETE, SEGGER_SYSVIEW_ShrinkId((U32)pxQueue))
@@ -239,11 +255,6 @@ Notes:
 #define traceQUEUE_RECEIVE_FROM_ISR( pxQueue )                        SEGGER_SYSVIEW_RecordU32x3(apiFastID_OFFSET + apiID_XQUEUERECEIVEFROMISR, SEGGER_SYSVIEW_ShrinkId((U32)pxQueue), SEGGER_SYSVIEW_ShrinkId((U32)pvBuffer), (U32)pxHigherPriorityTaskWoken)
 #define traceQUEUE_RECEIVE_FROM_ISR_FAILED( pxQueue )                 SEGGER_SYSVIEW_RecordU32x3(apiFastID_OFFSET + apiID_XQUEUERECEIVEFROMISR, SEGGER_SYSVIEW_ShrinkId((U32)pxQueue), SEGGER_SYSVIEW_ShrinkId((U32)pvBuffer), (U32)pxHigherPriorityTaskWoken)
 #define traceQUEUE_REGISTRY_ADD( xQueue, pcQueueName )                SEGGER_SYSVIEW_RecordU32x2(apiFastID_OFFSET + apiID_VQUEUEADDTOREGISTRY, SEGGER_SYSVIEW_ShrinkId((U32)xQueue), (U32)pcQueueName)
-#if ( configUSE_QUEUE_SETS != 1 )
-  #define traceQUEUE_SEND( pxQueue )                                    SYSVIEW_RecordU32x4(apiFastID_OFFSET + apiID_XQUEUEGENERICSEND, SEGGER_SYSVIEW_ShrinkId((U32)pxQueue), (U32)pvItemToQueue, xTicksToWait, xCopyPosition)
-#else
-  #define traceQUEUE_SEND( pxQueue )                                    SYSVIEW_RecordU32x4(apiFastID_OFFSET + apiID_XQUEUEGENERICSEND, SEGGER_SYSVIEW_ShrinkId((U32)pxQueue), 0, 0, xCopyPosition)
-#endif
 #define traceQUEUE_SEND_FAILED( pxQueue )                             SYSVIEW_RecordU32x4(apiFastID_OFFSET + apiID_XQUEUEGENERICSEND, SEGGER_SYSVIEW_ShrinkId((U32)pxQueue), (U32)pvItemToQueue, xTicksToWait, xCopyPosition)
 #define traceQUEUE_SEND_FROM_ISR( pxQueue )                           SEGGER_SYSVIEW_RecordU32x4(apiFastID_OFFSET + apiID_XQUEUEGENERICSENDFROMISR, SEGGER_SYSVIEW_ShrinkId((U32)pxQueue), (U32)pvItemToQueue, (U32)pxHigherPriorityTaskWoken, xCopyPosition)
 #define traceQUEUE_SEND_FROM_ISR_FAILED( pxQueue )                    SEGGER_SYSVIEW_RecordU32x4(apiFastID_OFFSET + apiID_XQUEUEGENERICSENDFROMISR, SEGGER_SYSVIEW_ShrinkId((U32)pxQueue), (U32)pvItemToQueue, (U32)pxHigherPriorityTaskWoken, xCopyPosition)
@@ -286,27 +297,31 @@ Notes:
 //
 // Define INCLUDE_xTaskGetIdleTaskHandle as 1 in FreeRTOSConfig.h to allow identification of Idle state.
 //
-#if ( INCLUDE_xTaskGetIdleTaskHandle == 1 )
+// SMP FreeRTOS uses unpinned IDLE tasks, so sometimes IDEL0 runs on CPU1, IDLE1 runs on CPU0 and so on.
+// So IDLE state detection based on 'xTaskGetIdleTaskHandle' does not work for SMP kernel.
+// We could compare current task handle with every element of the array returned by 'xTaskGetIdleTaskHandle',
+// but it deos not seem to be efficient enough to be worth of making code more complex and less readabl.
+// So always use task name comparison mechanism for SMP kernel.
+#if ( INCLUDE_xTaskGetIdleTaskHandle == 1  && !defined(CONFIG_FREERTOS_SMP))
   #define traceTASK_SWITCHED_IN()                   if(prvGetTCBFromHandle(NULL) == xTaskGetIdleTaskHandle()) {           \
                                                       SEGGER_SYSVIEW_OnIdle();                                          \
                                                     } else {                                                            \
-                                                      SEGGER_SYSVIEW_OnTaskStartExec((U32)pxCurrentTCB[xPortGetCoreID()]); \
+                                                      SEGGER_SYSVIEW_OnTaskStartExec((U32)prvGetTCBFromHandle(NULL)); \
                                                     }
 #else
   #define traceTASK_SWITCHED_IN()                   {                                                                   \
-                                                      if (memcmp(pxCurrentTCB[xPortGetCoreID()]->pcTaskName, "IDLE", 5) != 0) { \
-                                                        SEGGER_SYSVIEW_OnTaskStartExec((U32)pxCurrentTCB[xPortGetCoreID()]);    \
+                                                      if (memcmp(prvGetTCBFromHandle(NULL)->pcTaskName, "IDLE", 4) != 0) { \
+                                                        SEGGER_SYSVIEW_OnTaskStartExec((U32)prvGetTCBFromHandle(NULL));    \
                                                       } else {                                                          \
                                                         SEGGER_SYSVIEW_OnIdle();                                        \
                                                       }                                                                 \
                                                     }
 #endif
-
 #define traceMOVED_TASK_TO_READY_STATE(pxTCB)       SEGGER_SYSVIEW_OnTaskStartReady((U32)pxTCB)
 #define traceREADDED_TASK_TO_READY_STATE(pxTCB)
 
-#define traceMOVED_TASK_TO_DELAYED_LIST()           SEGGER_SYSVIEW_OnTaskStopReady((U32)pxCurrentTCB[xPortGetCoreID()],  (1u << 2))
-#define traceMOVED_TASK_TO_OVERFLOW_DELAYED_LIST()  SEGGER_SYSVIEW_OnTaskStopReady((U32)pxCurrentTCB[xPortGetCoreID()],  (1u << 2))
+#define traceMOVED_TASK_TO_DELAYED_LIST()           SEGGER_SYSVIEW_OnTaskStopReady((U32)prvGetTCBFromHandle(NULL)],  (1u << 2))
+#define traceMOVED_TASK_TO_OVERFLOW_DELAYED_LIST()  SEGGER_SYSVIEW_OnTaskStopReady((U32)prvGetTCBFromHandle(NULL)],  (1u << 2))
 #define traceMOVED_TASK_TO_SUSPENDED_LIST(pxTCB)    SEGGER_SYSVIEW_OnTaskStopReady((U32)pxTCB,         ((3u << 3) | 3))
 
 #define traceISR_EXIT_TO_SCHEDULER()                SEGGER_SYSVIEW_RecordExitISRToScheduler()

components/app_trace/sys_view/esp/SEGGER_RTT_esp.c

@@ -0,0 +1,306 @@
+/*
+ * SPDX-FileCopyrightText: 2017-2021 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#include "string.h"
+#include "freertos/FreeRTOS.h"
+#include "SEGGER_RTT.h"
+#include "SEGGER_SYSVIEW.h"
+#include "SEGGER_SYSVIEW_Conf.h"
+
+#include "esp_app_trace.h"
+#include "esp_log.h"
+#include "esp_cpu.h"
+#include "esp_private/startup_internal.h"
+
+const static char *TAG = "segger_rtt";
+
+#define SYSVIEW_EVENTS_BUF_SZ         255U
+
+// size of down channel data buf
+#define SYSVIEW_DOWN_BUF_SIZE   32
+#define SEGGER_STOP_WAIT_TMO    1000000 //us
+#if CONFIG_APPTRACE_SV_BUF_WAIT_TMO == -1
+#define SEGGER_HOST_WAIT_TMO    ESP_APPTRACE_TMO_INFINITE
+#else
+#define SEGGER_HOST_WAIT_TMO    CONFIG_APPTRACE_SV_BUF_WAIT_TMO
+#endif
+
+static uint8_t s_events_buf[SYSVIEW_EVENTS_BUF_SZ];
+static uint16_t s_events_buf_filled;
+static uint8_t s_down_buf[SYSVIEW_DOWN_BUF_SIZE];
+
+#if CONFIG_APPTRACE_SV_DEST_UART
+
+#define ESP_APPTRACE_DEST_SYSVIEW ESP_APPTRACE_DEST_UART
+#if CONFIG_APPTRACE_SV_DEST_CPU_0 || CONFIG_FREERTOS_UNICORE
+#define APPTRACE_SV_DEST_CPU 0
+#else
+#define APPTRACE_SV_DEST_CPU 1
+#endif // CONFIG_APPTRACE_SV_DEST_CPU_0
+
+#elif CONFIG_APPTRACE_SV_DEST_JTAG || (CONFIG_APPTRACE_ENABLE && CONFIG_APPTRACE_DEST_UART_NONE)
+#define ESP_APPTRACE_DEST_SYSVIEW ESP_APPTRACE_DEST_TRAX
+#endif
+
+/*********************************************************************
+*
+*       Public code
+*
+**********************************************************************
+*/
+
+/*********************************************************************
+*
+*       SEGGER_RTT_ESP_FlushNoLock()
+*
+*  Function description
+*    Flushes buffered events.
+*
+*  Parameters
+*    min_sz  Threshold for flushing data. If current filling level is above this value, data will be flushed. TRAX destinations only.
+*    tmo     Timeout for operation (in us). Use ESP_APPTRACE_TMO_INFINITE to wait indefinetly.
+*
+*  Return value
+*    None.
+*/
+void SEGGER_RTT_ESP_FlushNoLock(unsigned long min_sz, unsigned long tmo)
+{
+    esp_err_t res;
+    if (s_events_buf_filled > 0) {
+      res = esp_apptrace_write(ESP_APPTRACE_DEST_SYSVIEW, s_events_buf, s_events_buf_filled, tmo);
+      if (res != ESP_OK) {
+        ESP_LOGE(TAG, "Failed to flush buffered events (%d)!\n", res);
+      }
+    }
+    // flush even if we failed to write buffered events, because no new events will be sent after STOP
+    res = esp_apptrace_flush_nolock(ESP_APPTRACE_DEST_SYSVIEW, min_sz, tmo);
+    if (res != ESP_OK) {
+      ESP_LOGE(TAG, "Failed to flush apptrace data (%d)!\n", res);
+    }
+    s_events_buf_filled = 0;
+}
+
+/*********************************************************************
+*
+*       SEGGER_RTT_ESP_Flush()
+*
+*  Function description
+*    Flushes buffered events.
+*
+*  Parameters
+*    min_sz  Threshold for flushing data. If current filling level is above this value, data will be flushed. TRAX destinations only.
+*    tmo     Timeout for operation (in us). Use ESP_APPTRACE_TMO_INFINITE to wait indefinetly.
+*
+*  Return value
+*    None.
+*/
+void SEGGER_RTT_ESP_Flush(unsigned long min_sz, unsigned long tmo)
+{
+    SEGGER_SYSVIEW_LOCK();
+    SEGGER_RTT_ESP_FlushNoLock(min_sz, tmo);
+    SEGGER_SYSVIEW_UNLOCK();
+}
+
+/*********************************************************************
+*
+*       SEGGER_RTT_ReadNoLock()
+*
+*  Function description
+*    Reads characters from SEGGER real-time-terminal control block
+*    which have been previously stored by the host.
+*    Do not lock against interrupts and multiple access.
+*
+*  Parameters
+*    BufferIndex  Index of Down-buffer to be used (e.g. 0 for "Terminal").
+*    pBuffer      Pointer to buffer provided by target application, to copy characters from RTT-down-buffer to.
+*    BufferSize   Size of the target application buffer.
+*
+*  Return value
+*    Number of bytes that have been read.
+*/
+unsigned SEGGER_RTT_ReadNoLock(unsigned BufferIndex, void* pData, unsigned BufferSize) {
+  uint32_t size = BufferSize;
+  esp_err_t res = esp_apptrace_read(ESP_APPTRACE_DEST_SYSVIEW, pData, &size, 0);
+  if (res != ESP_OK) {
+    return 0;
+  }
+  return size;
+}
+
+/*********************************************************************
+*
+*       SEGGER_RTT_WriteSkipNoLock
+*
+*  Function description
+*    Stores a specified number of characters in SEGGER RTT
+*    control block which is then read by the host.
+*    SEGGER_RTT_WriteSkipNoLock does not lock the application and
+*    skips all data, if the data does not fit into the buffer.
+*
+*  Parameters
+*    BufferIndex  Index of "Up"-buffer to be used (e.g. 0 for "Terminal").
+*    pBuffer      Pointer to character array. Does not need to point to a \0 terminated string.
+*    NumBytes     Number of bytes to be stored in the SEGGER RTT control block.
+*
+*  Return value
+*    Number of bytes which have been stored in the "Up"-buffer.
+*
+*  Notes
+*    (1) If there is not enough space in the "Up"-buffer, all data is dropped.
+*    (2) For performance reasons this function does not call Init()
+*        and may only be called after RTT has been initialized.
+*        Either by calling SEGGER_RTT_Init() or calling another RTT API function first.
+*/
+unsigned SEGGER_RTT_WriteSkipNoLock(unsigned BufferIndex, const void* pBuffer, unsigned NumBytes) {
+  uint8_t *pbuf = (uint8_t *)pBuffer;
+  uint8_t event_id = *pbuf;
+#if CONFIG_APPTRACE_SV_DEST_UART
+  if (
+    (APPTRACE_SV_DEST_CPU != esp_cpu_get_core_id()) &&
+    (
+      (event_id == SYSVIEW_EVTID_ISR_ENTER) ||
+      (event_id == SYSVIEW_EVTID_ISR_EXIT) ||
+      (event_id == SYSVIEW_EVTID_TASK_START_EXEC) ||
+      (event_id == SYSVIEW_EVTID_TASK_STOP_EXEC) ||
+      (event_id == SYSVIEW_EVTID_TASK_START_READY) ||
+      (event_id == SYSVIEW_EVTID_TASK_STOP_READY) ||
+      (event_id == SYSVIEW_EVTID_USER_START) ||
+      (event_id == SYSVIEW_EVTID_USER_STOP) ||
+      (event_id == SYSVIEW_EVTID_TIMER_ENTER) ||
+      (event_id == SYSVIEW_EVTID_TIMER_EXIT) ||
+      (event_id == SYSVIEW_EVTID_STACK_INFO) ||
+      (event_id == SYSVIEW_EVTID_MODULEDESC)
+    )
+  ){
+    return NumBytes;
+  }
+
+// This is workaround for SystemView!
+// Without this line SystemView will hangs on when heap tracing enabled.
+  if(event_id == SYSVIEW_EVTID_MODULEDESC){
+    return NumBytes;
+  }
+#endif // CONFIG_APPTRACE_SV_DEST_UART
+
+  if (NumBytes > SYSVIEW_EVENTS_BUF_SZ) {
+      ESP_LOGE(TAG, "Too large event %u bytes!", NumBytes);
+      return 0;
+  }
+#if CONFIG_APPTRACE_SV_DEST_JTAG
+  if (esp_cpu_get_core_id()) { // dual core specific code
+    // use the highest - 1 bit of event ID to indicate core ID
+    // the highest bit can not be used due to event ID encoding method
+    // this reduces supported ID range to [0..63] (for 1 byte IDs) plus [128..16383] (for 2 bytes IDs)
+    if (*pbuf & 0x80) { // 2 bytes ID
+      *(pbuf + 1) |= (1 << 6);
+    } else if (NumBytes != 10 || *pbuf != 0) { // ignore sync sequence
+      *pbuf |= (1 << 6);
+    }
+  }
+#endif // CONFIG_APPTRACE_SV_DEST_JTAG
+#if CONFIG_APPTRACE_SV_DEST_JTAG
+  if (s_events_buf_filled + NumBytes > SYSVIEW_EVENTS_BUF_SZ) {
+
+    esp_err_t res = esp_apptrace_write(ESP_APPTRACE_DEST_SYSVIEW, s_events_buf, s_events_buf_filled, SEGGER_HOST_WAIT_TMO);
+    if (res != ESP_OK) {
+      return 0; // skip current data buffer only, accumulated events are kept
+    }
+    s_events_buf_filled = 0;
+  }
+#endif
+  memcpy(&s_events_buf[s_events_buf_filled], pBuffer, NumBytes);
+  s_events_buf_filled += NumBytes;
+
+#if CONFIG_APPTRACE_SV_DEST_UART
+  esp_err_t res = esp_apptrace_write(ESP_APPTRACE_DEST_SYSVIEW, pBuffer, NumBytes, SEGGER_HOST_WAIT_TMO);
+  if (res != ESP_OK)
+  {
+    return 0; // skip current data buffer only, accumulated events are kept
+  }
+  s_events_buf_filled = 0;
+#endif
+
+  if (event_id == SYSVIEW_EVTID_TRACE_STOP)
+  {
+    SEGGER_RTT_ESP_FlushNoLock(0, SEGGER_STOP_WAIT_TMO);
+  }
+  return NumBytes;
+}
+
+/*********************************************************************
+*
+*       SEGGER_RTT_ConfigUpBuffer
+*
+*  Function description
+*    Run-time configuration of a specific up-buffer (T->H).
+*    Buffer to be configured is specified by index.
+*    This includes: Buffer address, size, name, flags, ...
+*
+*  Parameters
+*    BufferIndex  Index of the buffer to configure.
+*    sName        Pointer to a constant name string.
+*    pBuffer      Pointer to a buffer to be used.
+*    BufferSize   Size of the buffer.
+*    Flags        Operating modes. Define behavior if buffer is full (not enough space for entire message).
+*
+*  Return value
+*    >= 0 - O.K.
+*     < 0 - Error
+*
+*  Additional information
+*    Buffer 0 is configured on compile-time.
+*    May only be called once per buffer.
+*    Buffer name and flags can be reconfigured using the appropriate functions.
+*/
+int SEGGER_RTT_ConfigUpBuffer(unsigned BufferIndex, const char* sName, void* pBuffer, unsigned BufferSize, unsigned Flags) {
+  s_events_buf_filled = 0;
+  return 0;
+}
+
+/*********************************************************************
+*
+*       SEGGER_RTT_ConfigDownBuffer
+*
+*  Function description
+*    Run-time configuration of a specific down-buffer (H->T).
+*    Buffer to be configured is specified by index.
+*    This includes: Buffer address, size, name, flags, ...
+*
+*  Parameters
+*    BufferIndex  Index of the buffer to configure.
+*    sName        Pointer to a constant name string.
+*    pBuffer      Pointer to a buffer to be used.
+*    BufferSize   Size of the buffer.
+*    Flags        Operating modes. Define behavior if buffer is full (not enough space for entire message).
+*
+*  Return value
+*    >= 0  O.K.
+*     < 0  Error
+*
+*  Additional information
+*    Buffer 0 is configured on compile-time.
+*    May only be called once per buffer.
+*    Buffer name and flags can be reconfigured using the appropriate functions.
+*/
+int SEGGER_RTT_ConfigDownBuffer(unsigned BufferIndex, const char* sName, void* pBuffer, unsigned BufferSize, unsigned Flags) {
+  esp_apptrace_down_buffer_config(s_down_buf, sizeof(s_down_buf));
+  return 0;
+}
+
+/*************************** Init hook ****************************
+ *
+ * This init function is placed here because this port file will be
+ * linked whenever SystemView is used.
+ */
+
+ESP_SYSTEM_INIT_FN(sysview_init, BIT(0), 120)
+{
+    SEGGER_SYSVIEW_Conf();
+    return ESP_OK;
+}
+
+
+/*************************** End of file ****************************/

components/app_trace/sys_view/esp32/SEGGER_RTT_esp32.c

@@ -1,243 +0,0 @@
-// Copyright 2017 Espressif Systems (Shanghai) PTE LTD
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "string.h"
-#include "freertos/FreeRTOS.h"
-#include "SEGGER_RTT.h"
-#include "SEGGER_SYSVIEW.h"
-#include "SEGGER_SYSVIEW_Conf.h"
-
-#include "esp_app_trace.h"
-#include "esp_log.h"
-
-const static char *TAG = "segger_rtt";
-
-#define SYSVIEW_EVENTS_BUF_SZ         255U
-
-// size of down channel data buf
-#define SYSVIEW_DOWN_BUF_SIZE   32
-#define SEGGER_STOP_WAIT_TMO    1000000 //us
-#if CONFIG_SYSVIEW_BUF_WAIT_TMO == -1
-#define SEGGER_HOST_WAIT_TMO    ESP_APPTRACE_TMO_INFINITE
-#else
-#define SEGGER_HOST_WAIT_TMO    CONFIG_SYSVIEW_BUF_WAIT_TMO
-#endif
-
-static uint8_t s_events_buf[SYSVIEW_EVENTS_BUF_SZ];
-static uint16_t s_events_buf_filled;
-static uint8_t s_down_buf[SYSVIEW_DOWN_BUF_SIZE];
-
-/*********************************************************************
-*
-*       Public code
-*
-**********************************************************************
-*/
-
-/*********************************************************************
-*
-*       SEGGER_RTT_ESP32_FlushNoLock()
-*
-*  Function description
-*    Flushes buffered events.
-*
-*  Parameters
-*    min_sz  Threshold for flushing data. If current filling level is above this value, data will be flushed. TRAX destinations only.
-*    tmo     Timeout for operation (in us). Use ESP_APPTRACE_TMO_INFINITE to wait indefinetly.
-*
-*  Return value
-*    None.
-*/
-void SEGGER_RTT_ESP32_FlushNoLock(unsigned long min_sz, unsigned long tmo)
-{
-    esp_err_t res;
-    if (s_events_buf_filled > 0) {
-      res = esp_apptrace_write(ESP_APPTRACE_DEST_TRAX, s_events_buf, s_events_buf_filled, tmo);
-      if (res != ESP_OK) {
-        ESP_LOGE(TAG, "Failed to flush buffered events (%d)!\n", res);
-      }
-    }
-    // flush even if we failed to write buffered events, because no new events will be sent after STOP
-    res = esp_apptrace_flush_nolock(ESP_APPTRACE_DEST_TRAX, min_sz, tmo);
-    if (res != ESP_OK) {
-      ESP_LOGE(TAG, "Failed to flush apptrace data (%d)!\n", res);
-    }
-    s_events_buf_filled = 0;
-}
-
-/*********************************************************************
-*
-*       SEGGER_RTT_ESP32_Flush()
-*
-*  Function description
-*    Flushes buffered events.
-*
-*  Parameters
-*    min_sz  Threshold for flushing data. If current filling level is above this value, data will be flushed. TRAX destinations only.
-*    tmo     Timeout for operation (in us). Use ESP_APPTRACE_TMO_INFINITE to wait indefinetly.
-*
-*  Return value
-*    None.
-*/
-void SEGGER_RTT_ESP32_Flush(unsigned long min_sz, unsigned long tmo)
-{
-    SEGGER_SYSVIEW_LOCK();
-    SEGGER_RTT_ESP32_FlushNoLock(min_sz, tmo);
-    SEGGER_SYSVIEW_UNLOCK();
-}
-
-/*********************************************************************
-*
-*       SEGGER_RTT_ReadNoLock()
-*
-*  Function description
-*    Reads characters from SEGGER real-time-terminal control block
-*    which have been previously stored by the host.
-*    Do not lock against interrupts and multiple access.
-*
-*  Parameters
-*    BufferIndex  Index of Down-buffer to be used (e.g. 0 for "Terminal").
-*    pBuffer      Pointer to buffer provided by target application, to copy characters from RTT-down-buffer to.
-*    BufferSize   Size of the target application buffer.
-*
-*  Return value
-*    Number of bytes that have been read.
-*/
-unsigned SEGGER_RTT_ReadNoLock(unsigned BufferIndex, void* pData, unsigned BufferSize) {
-  uint32_t size = BufferSize;
-  esp_err_t res = esp_apptrace_read(ESP_APPTRACE_DEST_TRAX, pData, &size, 0);
-  if (res != ESP_OK) {
-    return 0;
-  }
-  return size;
-}
-
-/*********************************************************************
-*
-*       SEGGER_RTT_WriteSkipNoLock
-*
-*  Function description
-*    Stores a specified number of characters in SEGGER RTT
-*    control block which is then read by the host.
-*    SEGGER_RTT_WriteSkipNoLock does not lock the application and
-*    skips all data, if the data does not fit into the buffer.
-*
-*  Parameters
-*    BufferIndex  Index of "Up"-buffer to be used (e.g. 0 for "Terminal").
-*    pBuffer      Pointer to character array. Does not need to point to a \0 terminated string.
-*    NumBytes     Number of bytes to be stored in the SEGGER RTT control block.
-*
-*  Return value
-*    Number of bytes which have been stored in the "Up"-buffer.
-*
-*  Notes
-*    (1) If there is not enough space in the "Up"-buffer, all data is dropped.
-*    (2) For performance reasons this function does not call Init()
-*        and may only be called after RTT has been initialized.
-*        Either by calling SEGGER_RTT_Init() or calling another RTT API function first.
-*/
-unsigned SEGGER_RTT_WriteSkipNoLock(unsigned BufferIndex, const void* pBuffer, unsigned NumBytes) {
-  uint8_t *pbuf = (uint8_t *)pBuffer;
-  uint8_t event_id = *pbuf;
-
-  if (NumBytes > SYSVIEW_EVENTS_BUF_SZ) {
-      ESP_LOGE(TAG, "Too large event %u bytes!", NumBytes);
-      return 0;
-  }
-  if (xPortGetCoreID()) { // dual core specific code
-    // use the highest - 1 bit of event ID to indicate core ID
-    // the highest bit can not be used due to event ID encoding method
-    // this reduces supported ID range to [0..63] (for 1 byte IDs) plus [128..16383] (for 2 bytes IDs)
-    if (*pbuf & 0x80) { // 2 bytes ID
-      *(pbuf + 1) |= (1 << 6);
-    } else if (NumBytes != 10 || *pbuf != 0) { // ignore sync sequence
-      *pbuf |= (1 << 6);
-    }
-  }
-  if (s_events_buf_filled + NumBytes > SYSVIEW_EVENTS_BUF_SZ) {
-    esp_err_t res = esp_apptrace_write(ESP_APPTRACE_DEST_TRAX, s_events_buf, s_events_buf_filled, SEGGER_HOST_WAIT_TMO);
-    if (res != ESP_OK) {
-      return 0; // skip current data buffer only, accumulated events are kept
-    }
-    s_events_buf_filled = 0;
-  }
-  memcpy(&s_events_buf[s_events_buf_filled], pBuffer, NumBytes);
-  s_events_buf_filled += NumBytes;
-  if (event_id == SYSVIEW_EVTID_TRACE_STOP) {
-    SEGGER_RTT_ESP32_FlushNoLock(0, SEGGER_STOP_WAIT_TMO);
-  }
-  return NumBytes;
-}
-
-/*********************************************************************
-*
-*       SEGGER_RTT_ConfigUpBuffer
-*
-*  Function description
-*    Run-time configuration of a specific up-buffer (T->H).
-*    Buffer to be configured is specified by index.
-*    This includes: Buffer address, size, name, flags, ...
-*
-*  Parameters
-*    BufferIndex  Index of the buffer to configure.
-*    sName        Pointer to a constant name string.
-*    pBuffer      Pointer to a buffer to be used.
-*    BufferSize   Size of the buffer.
-*    Flags        Operating modes. Define behavior if buffer is full (not enough space for entire message).
-*
-*  Return value
-*    >= 0 - O.K.
-*     < 0 - Error
-*
-*  Additional information
-*    Buffer 0 is configured on compile-time.
-*    May only be called once per buffer.
-*    Buffer name and flags can be reconfigured using the appropriate functions.
-*/
-int SEGGER_RTT_ConfigUpBuffer(unsigned BufferIndex, const char* sName, void* pBuffer, unsigned BufferSize, unsigned Flags) {
-  s_events_buf_filled = 0;
-  return 0;
-}
-
-/*********************************************************************
-*
-*       SEGGER_RTT_ConfigDownBuffer
-*
-*  Function description
-*    Run-time configuration of a specific down-buffer (H->T).
-*    Buffer to be configured is specified by index.
-*    This includes: Buffer address, size, name, flags, ...
-*
-*  Parameters
-*    BufferIndex  Index of the buffer to configure.
-*    sName        Pointer to a constant name string.
-*    pBuffer      Pointer to a buffer to be used.
-*    BufferSize   Size of the buffer.
-*    Flags        Operating modes. Define behavior if buffer is full (not enough space for entire message).
-*
-*  Return value
-*    >= 0  O.K.
-*     < 0  Error
-*
-*  Additional information
-*    Buffer 0 is configured on compile-time.
-*    May only be called once per buffer.
-*    Buffer name and flags can be reconfigured using the appropriate functions.
-*/
-int SEGGER_RTT_ConfigDownBuffer(unsigned BufferIndex, const char* sName, void* pBuffer, unsigned BufferSize, unsigned Flags) {
-  esp_apptrace_down_buffer_config(s_down_buf, sizeof(s_down_buf));
-  return 0;
-}
-
-/*************************** End of file ****************************/

components/app_trace/sys_view/ext/heap_trace_module.c

@@ -1,16 +1,8 @@
-// Copyright 2018 Espressif Systems (Shanghai) PTE LTD
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
+/*
+ * SPDX-FileCopyrightText: 2018-2021 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
 #include <stdint.h>
 #include <sdkconfig.h>
 #include "SEGGER_SYSVIEW.h"
@@ -60,7 +52,7 @@ esp_err_t esp_sysview_heap_trace_start(uint32_t tmo)
 esp_err_t esp_sysview_heap_trace_stop(void)
 {
     ESP_EARLY_LOGV(TAG, "%s", __func__);
-    SEGGER_RTT_ESP32_Flush(0, ESP_APPTRACE_TMO_INFINITE);
+    SEGGER_RTT_ESP_Flush(0, ESP_APPTRACE_TMO_INFINITE);
     return ESP_OK;
 }
 

components/app_trace/sys_view/ext/logging.c

@@ -1,16 +1,8 @@
-// Copyright 2018 Espressif Systems (Shanghai) PTE LTD
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
+/*
+ * SPDX-FileCopyrightText: 2018-2021 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
 #include <stdio.h>
 #include <stdarg.h>
 #include <sdkconfig.h>

components/app_trace/test/CMakeLists.txt

@@ -1,3 +1,4 @@
 idf_component_register(SRC_DIRS "."
                     PRIV_INCLUDE_DIRS "."
-                    PRIV_REQUIRES cmock)
+                    PRIV_REQUIRES cmock driver)
+target_compile_options(${COMPONENT_LIB} PRIVATE "-Wno-format")

components/app_trace/test/component.mk

@@ -1,5 +0,0 @@
-#
-#Component Makefile
-#
-
-COMPONENT_ADD_LDFLAGS = -Wl,--whole-archive -l$(COMPONENT_NAME) -Wl,--no-whole-archive

components/app_trace/test/test_trace.c

@@ -1,12 +1,18 @@
+/*
+ * SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
 #include <stddef.h>
 #include <stdint.h>
 #include <string.h>
 #include <stdio.h>
 #include <stdarg.h>
 #include "unity.h"
-#include "driver/timer.h"
+#include "driver/gptimer.h"
+#include "esp_intr_alloc.h"
 #include "esp_rom_sys.h"
-#include "soc/cpu.h"
+#include "esp_cpu.h"
 #include "freertos/FreeRTOS.h"
 #include "freertos/semphr.h"
 #include "freertos/task.h"
@@ -60,30 +66,7 @@ const static char *TAG = "esp_apptrace_test";
 #define ESP_APPTRACE_TEST_LOGV( format, ... )  ESP_APPTRACE_TEST_LOG_LEVEL(V, ESP_LOG_VERBOSE, format, ##__VA_ARGS__)
 #define ESP_APPTRACE_TEST_LOGO( format, ... )  ESP_APPTRACE_TEST_LOG_LEVEL(E, ESP_LOG_NONE, format, ##__VA_ARGS__)
 
-static void esp_apptrace_test_timer_init(int timer_group, int timer_idx, uint32_t period)
-{
-    timer_config_t config;
-    uint64_t alarm_val = (period * (TIMER_BASE_CLK / 1000000UL)) / 2;
-
-    config.alarm_en = 1;
-    config.auto_reload = 1;
-    config.counter_dir = TIMER_COUNT_UP;
-    config.divider = 2;     //Range is 2 to 65536
-    config.intr_type = TIMER_INTR_LEVEL;
-    config.counter_en = TIMER_PAUSE;
-    /*Configure timer*/
-    timer_init(timer_group, timer_idx, &config);
-    /*Stop timer counter*/
-    timer_pause(timer_group, timer_idx);
-    /*Load counter value */
-    timer_set_counter_value(timer_group, timer_idx, 0x00000000ULL);
-    /*Set alarm value*/
-    timer_set_alarm_value(timer_group, timer_idx, alarm_val);
-    /*Enable timer interrupt*/
-    timer_enable_intr(timer_group, timer_idx);
-}
-
-#if CONFIG_SYSVIEW_ENABLE == 0
+#if CONFIG_APPTRACE_SV_ENABLE == 0
 #define ESP_APPTRACE_TEST_WRITE(_b_, _s_)            esp_apptrace_write(ESP_APPTRACE_DEST_TRAX, _b_, _s_, ESP_APPTRACE_TMO_INFINITE)
 #define ESP_APPTRACE_TEST_WRITE_FROM_ISR(_b_, _s_)   esp_apptrace_write(ESP_APPTRACE_DEST_TRAX, _b_, _s_, 0UL)
 #define ESP_APPTRACE_TEST_WRITE_NOWAIT(_b_, _s_)     esp_apptrace_write(ESP_APPTRACE_DEST_TRAX, _b_, _s_, 0)
@@ -98,9 +81,8 @@ typedef struct {
 } esp_apptrace_test_gen_data_t;
 
 typedef struct {
-    int  group;
-    int  id;
-    void (*isr_func)(void *);
+    gptimer_handle_t gptimer;
+    bool (*isr_func)(gptimer_handle_t timer, const gptimer_alarm_event_data_t *edata, void *user_ctx);
     esp_apptrace_test_gen_data_t data;
 } esp_apptrace_test_timer_arg_t;
 
@@ -112,7 +94,6 @@ typedef struct {
     esp_apptrace_test_gen_data_t data;
     volatile int stop;
     SemaphoreHandle_t done;
-
     uint32_t timers_num;
     esp_apptrace_test_timer_arg_t *timers;
 } esp_apptrace_test_task_arg_t;
@@ -128,103 +109,87 @@ static SemaphoreHandle_t s_print_lock;
 
 static uint64_t esp_apptrace_test_ts_get(void);
 
-static void esp_apptrace_test_timer_isr(void *arg)
+static bool esp_apptrace_test_timer_isr(gptimer_handle_t timer, const gptimer_alarm_event_data_t *edata, void *user_ctx)
 {
-    esp_apptrace_test_timer_arg_t *tim_arg = (esp_apptrace_test_timer_arg_t *)arg;
+    esp_apptrace_test_timer_arg_t *tim_arg = (esp_apptrace_test_timer_arg_t *)user_ctx;
 
     uint32_t *ts = (uint32_t *)(tim_arg->data.buf + sizeof(uint32_t));
     *ts = (uint32_t)esp_apptrace_test_ts_get();
     memset(tim_arg->data.buf + 2 * sizeof(uint32_t), tim_arg->data.wr_cnt & tim_arg->data.mask, tim_arg->data.buf_sz - 2 * sizeof(uint32_t));
     int res = ESP_APPTRACE_TEST_WRITE_FROM_ISR(tim_arg->data.buf, tim_arg->data.buf_sz);
-    if (res != ESP_OK) {
-    } else {
-        if (0) {
-            esp_rom_printf("tim-%d-%d: Written chunk%d %d bytes, %x\n",
-                       tim_arg->group, tim_arg->id, tim_arg->data.wr_cnt, tim_arg->data.buf_sz, tim_arg->data.wr_cnt & tim_arg->data.mask);
-        }
+    if (res == ESP_OK) {
         tim_arg->data.wr_err = 0;
     }
 
     tim_arg->data.wr_cnt++;
-    timer_group_clr_intr_status_in_isr(tim_arg->group, tim_arg->id);
-    timer_group_enable_alarm_in_isr(tim_arg->group, tim_arg->id);
+    return true;
 }
 
-static void esp_apptrace_test_timer_isr_crash(void *arg)
+static bool esp_apptrace_test_timer_isr_crash(gptimer_handle_t timer, const gptimer_alarm_event_data_t *edata, void *user_ctx)
 {
-    esp_apptrace_test_timer_arg_t *tim_arg = (esp_apptrace_test_timer_arg_t *)arg;
+    esp_apptrace_test_timer_arg_t *tim_arg = (esp_apptrace_test_timer_arg_t *)user_ctx;
 
-    timer_group_clr_intr_status_in_isr(tim_arg->group, tim_arg->id);
-    timer_group_enable_alarm_in_isr(tim_arg->group, tim_arg->id);
     if (tim_arg->data.wr_cnt < ESP_APPTRACE_TEST_BLOCKS_BEFORE_CRASH) {
         uint32_t *ts = (uint32_t *)(tim_arg->data.buf + sizeof(uint32_t));
         *ts = (uint32_t)esp_apptrace_test_ts_get();//xthal_get_ccount();//xTaskGetTickCount();
         memset(tim_arg->data.buf + 2 * sizeof(uint32_t), tim_arg->data.wr_cnt & tim_arg->data.mask, tim_arg->data.buf_sz - 2 * sizeof(uint32_t));
         int res = ESP_APPTRACE_TEST_WRITE_FROM_ISR(tim_arg->data.buf, tim_arg->data.buf_sz);
         if (res != ESP_OK) {
-            esp_rom_printf("tim-%d-%d: Failed to write trace %d %x!\n", tim_arg->group, tim_arg->id, res, tim_arg->data.wr_cnt & tim_arg->data.mask);
+            esp_rom_printf("tim-%x: Failed to write trace %d %x!\n", tim_arg->gptimer, res, tim_arg->data.wr_cnt & tim_arg->data.mask);
         } else {
-            esp_rom_printf("tim-%d-%d: Written chunk%d %d bytes, %x\n",
-                       tim_arg->group, tim_arg->id, tim_arg->data.wr_cnt, tim_arg->data.buf_sz, tim_arg->data.wr_cnt & tim_arg->data.mask);
+            esp_rom_printf("tim-%x: Written chunk%d %d bytes, %x\n",
+                           timer, tim_arg->data.wr_cnt, tim_arg->data.buf_sz, tim_arg->data.wr_cnt & tim_arg->data.mask);
             tim_arg->data.wr_cnt++;
         }
     } else {
         uint32_t *ptr = 0;
         *ptr = 1000;
     }
+    return true;
 }
 
 static void esp_apptrace_dummy_task(void *p)
 {
     esp_apptrace_test_task_arg_t *arg = (esp_apptrace_test_task_arg_t *) p;
-    int res, flags = 0, i;
-    timer_isr_handle_t *inth = NULL;
     TickType_t tmo_ticks = arg->data.period / (1000 * portTICK_PERIOD_MS);
 
     ESP_APPTRACE_TEST_LOGI("%x: run dummy task (period %u us, %u timers)", xTaskGetCurrentTaskHandle(), arg->data.period, arg->timers_num);
 
-    if (arg->timers_num > 0) {
-        inth = pvPortMalloc(arg->timers_num * sizeof(timer_isr_handle_t));
-        if (!inth) {
-            ESP_APPTRACE_TEST_LOGE("Failed to alloc timer ISR handles!");
-            goto on_fail;
-        }
-        memset(inth, 0, arg->timers_num * sizeof(timer_isr_handle_t));
-        for (int i = 0; i < arg->timers_num; i++) {
-            esp_apptrace_test_timer_init(arg->timers[i].group, arg->timers[i].id, arg->timers[i].data.period);
-            res = timer_isr_register(arg->timers[i].group, arg->timers[i].id, arg->timers[i].isr_func, &arg->timers[i], flags, &inth[i]);
-            if (res != ESP_OK) {
-                ESP_APPTRACE_TEST_LOGE("Failed to timer_isr_register (%d)!", res);
-                goto on_fail;
-            }
-            *(uint32_t *)arg->timers[i].data.buf = (uint32_t)inth[i] | (1 << 31);
-            ESP_APPTRACE_TEST_LOGI("%x: start timer %x period %u us", xTaskGetCurrentTaskHandle(), inth[i], arg->timers[i].data.period);
-            res = timer_start(arg->timers[i].group, arg->timers[i].id);
-            if (res != ESP_OK) {
-                ESP_APPTRACE_TEST_LOGE("Failed to timer_start (%d)!", res);
-                goto on_fail;
-            }
-        }
+    for (int i = 0; i < arg->timers_num; i++) {
+        gptimer_config_t timer_config = {
+            .clk_src = GPTIMER_CLK_SRC_DEFAULT,
+            .direction = GPTIMER_COUNT_UP,
+            .resolution_hz = 1000000,
+        };
+        TEST_ESP_OK(gptimer_new_timer(&timer_config, &arg->timers[i].gptimer));
+        *(uint32_t *)arg->timers[i].data.buf = (uint32_t)arg->timers[i].gptimer | (1 << 31);
+        ESP_APPTRACE_TEST_LOGI("%x: start timer %x period %u us", xTaskGetCurrentTaskHandle(), arg->timers[i].gptimer, arg->timers[i].data.period);
+        gptimer_alarm_config_t alarm_config = {
+            .reload_count = 0,
+            .alarm_count = arg->timers[i].data.period,
+            .flags.auto_reload_on_alarm = true,
+        };
+        gptimer_event_callbacks_t cbs = {
+            .on_alarm = arg->timers[i].isr_func,
+        };
+        TEST_ESP_OK(gptimer_register_event_callbacks(arg->timers[i].gptimer, &cbs, &arg->timers[i]));
+        TEST_ESP_OK(gptimer_enable(arg->timers[i].gptimer));
+        TEST_ESP_OK(gptimer_set_alarm_action(arg->timers[i].gptimer, &alarm_config));
+        TEST_ESP_OK(gptimer_start(arg->timers[i].gptimer));
     }
 
-    i = 0;
+    int i = 0;
     while (!arg->stop) {
-        ESP_APPTRACE_TEST_LOGD("%x: dummy task work %d.%d", xTaskGetCurrentTaskHandle(), xPortGetCoreID(), i++);
+        ESP_APPTRACE_TEST_LOGD("%x: dummy task work %d.%d", xTaskGetCurrentTaskHandle(), esp_cpu_get_core_id(), i++);
         if (tmo_ticks) {
             vTaskDelay(tmo_ticks);
         }
     }
 
-on_fail:
-    if (inth) {
-        for (int i = 0; i < arg->timers_num; i++) {
-            timer_pause(arg->timers[i].group, arg->timers[i].id);
-            timer_disable_intr(arg->timers[i].group, arg->timers[i].id);
-            if (inth[i]) {
-                esp_intr_free(inth[i]);
-            }
-        }
-        vPortFree(inth);
+    for (int i = 0; i < arg->timers_num; i++) {
+        TEST_ESP_OK(gptimer_stop(arg->timers[i].gptimer));
+        TEST_ESP_OK(gptimer_disable(arg->timers[i].gptimer));
+        TEST_ESP_OK(gptimer_del_timer(arg->timers[i].gptimer));
     }
     xSemaphoreGive(arg->done);
     vTaskDelay(1);
@@ -234,37 +199,35 @@ on_fail:
 static void esp_apptrace_test_task(void *p)
 {
     esp_apptrace_test_task_arg_t *arg = (esp_apptrace_test_task_arg_t *) p;
-    int res, flags = 0;
-    timer_isr_handle_t *inth = NULL;
+    int res;
     TickType_t tmo_ticks = arg->data.period / (1000 * portTICK_PERIOD_MS);
 
     ESP_APPTRACE_TEST_LOGI("%x: run (period %u us, stamp mask %x, %u timers)", xTaskGetCurrentTaskHandle(), arg->data.period, arg->data.mask, arg->timers_num);
 
-    if (arg->timers_num > 0) {
-        inth = pvPortMalloc(arg->timers_num * sizeof(timer_isr_handle_t));
-        if (!inth) {
-            ESP_APPTRACE_TEST_LOGE("Failed to alloc timer ISR handles!");
-            goto on_fail;
-        }
-        memset(inth, 0, arg->timers_num * sizeof(timer_isr_handle_t));
-        for (int i = 0; i < arg->timers_num; i++) {
-            esp_apptrace_test_timer_init(arg->timers[i].group, arg->timers[i].id, arg->timers[i].data.period);
-            res = timer_isr_register(arg->timers[i].group, arg->timers[i].id, arg->timers[i].isr_func, &arg->timers[i], flags, &inth[i]);
-            if (res != ESP_OK) {
-                ESP_APPTRACE_TEST_LOGE("Failed to timer_isr_register (%d)!", res);
-                goto on_fail;
-            }
-            *(uint32_t *)arg->timers[i].data.buf = ((uint32_t)inth[i]) | (1 << 31) | (xPortGetCoreID() ? 0x1 : 0);
-            ESP_APPTRACE_TEST_LOGI("%x: start timer %x period %u us", xTaskGetCurrentTaskHandle(), inth[i], arg->timers[i].data.period);
-            res = timer_start(arg->timers[i].group, arg->timers[i].id);
-            if (res != ESP_OK) {
-                ESP_APPTRACE_TEST_LOGE("Failed to timer_start (%d)!", res);
-                goto on_fail;
-            }
-        }
+    for (int i = 0; i < arg->timers_num; i++) {
+        gptimer_config_t timer_config = {
+            .clk_src = GPTIMER_CLK_SRC_DEFAULT,
+            .direction = GPTIMER_COUNT_UP,
+            .resolution_hz = 1000000,
+        };
+        TEST_ESP_OK(gptimer_new_timer(&timer_config, &arg->timers[i].gptimer));
+        *(uint32_t *)arg->timers[i].data.buf = ((uint32_t)arg->timers[i].gptimer) | (1 << 31) | (esp_cpu_get_core_id() ? 0x1 : 0);
+        ESP_APPTRACE_TEST_LOGI("%x: start timer %x period %u us", xTaskGetCurrentTaskHandle(), arg->timers[i].gptimer, arg->timers[i].data.period);
+        gptimer_alarm_config_t alarm_config = {
+            .reload_count = 0,
+            .alarm_count = arg->timers[i].data.period,
+            .flags.auto_reload_on_alarm = true,
+        };
+        gptimer_event_callbacks_t cbs = {
+            .on_alarm = arg->timers[i].isr_func,
+        };
+        TEST_ESP_OK(gptimer_register_event_callbacks(arg->timers[i].gptimer, &cbs, &arg->timers[i]));
+        TEST_ESP_OK(gptimer_enable(arg->timers[i].gptimer));
+        TEST_ESP_OK(gptimer_set_alarm_action(arg->timers[i].gptimer, &alarm_config));
+        TEST_ESP_OK(gptimer_start(arg->timers[i].gptimer));
     }
 
-    *(uint32_t *)arg->data.buf = (uint32_t)xTaskGetCurrentTaskHandle() | (xPortGetCoreID() ? 0x1 : 0);
+    *(uint32_t *)arg->data.buf = (uint32_t)xTaskGetCurrentTaskHandle() | (esp_cpu_get_core_id() ? 0x1 : 0);
     arg->data.wr_cnt = 0;
     arg->data.wr_err = 0;
     while (!arg->stop) {
@@ -278,7 +241,7 @@ static void esp_apptrace_test_task(void *p)
             res = ESP_APPTRACE_TEST_WRITE(arg->data.buf, arg->data.buf_sz);
         }
         if (res) {
-            if (1){//arg->data.wr_err++ < ESP_APPTRACE_TEST_PRN_WRERR_MAX) {
+            if (1) { //arg->data.wr_err++ < ESP_APPTRACE_TEST_PRN_WRERR_MAX) {
                 ESP_APPTRACE_TEST_LOGE("%x: Failed to write trace %d %x!", xTaskGetCurrentTaskHandle(), res, arg->data.wr_cnt & arg->data.mask);
                 if (arg->data.wr_err == ESP_APPTRACE_TEST_PRN_WRERR_MAX) {
                     ESP_APPTRACE_TEST_LOGE("\n");
@@ -296,16 +259,10 @@ static void esp_apptrace_test_task(void *p)
         }
     }
 
-on_fail:
-    if (inth) {
-        for (int i = 0; i < arg->timers_num; i++) {
-            timer_pause(arg->timers[i].group, arg->timers[i].id);
-            timer_disable_intr(arg->timers[i].group, arg->timers[i].id);
-            if (inth[i]) {
-                esp_intr_free(inth[i]);
-            }
-        }
-        vPortFree(inth);
+    for (int i = 0; i < arg->timers_num; i++) {
+        TEST_ESP_OK(gptimer_stop(arg->timers[i].gptimer));
+        TEST_ESP_OK(gptimer_disable(arg->timers[i].gptimer));
+        TEST_ESP_OK(gptimer_del_timer(arg->timers[i].gptimer));
     }
     xSemaphoreGive(arg->done);
     vTaskDelay(1);
@@ -315,17 +272,16 @@ on_fail:
 static void esp_apptrace_test_task_crash(void *p)
 {
     esp_apptrace_test_task_arg_t *arg = (esp_apptrace_test_task_arg_t *) p;
-    int res, i;
 
     ESP_APPTRACE_TEST_LOGE("%x: run (period %u us, stamp mask %x, %u timers)", xTaskGetCurrentTaskHandle(), arg->data.period, arg->data.mask, arg->timers_num);
 
     arg->data.wr_cnt = 0;
     *(uint32_t *)arg->data.buf = (uint32_t)xTaskGetCurrentTaskHandle();
-    for (i = 0; i < ESP_APPTRACE_TEST_BLOCKS_BEFORE_CRASH; i++) {
+    for (int i = 0; i < ESP_APPTRACE_TEST_BLOCKS_BEFORE_CRASH; i++) {
         uint32_t *ts = (uint32_t *)(arg->data.buf + sizeof(uint32_t));
         *ts = (uint32_t)esp_apptrace_test_ts_get();
         memset(arg->data.buf + sizeof(uint32_t), arg->data.wr_cnt & arg->data.mask, arg->data.buf_sz - sizeof(uint32_t));
-        res = ESP_APPTRACE_TEST_WRITE(arg->data.buf, arg->data.buf_sz);
+        int res = ESP_APPTRACE_TEST_WRITE(arg->data.buf, arg->data.buf_sz);
         if (res) {
             ESP_APPTRACE_TEST_LOGE("%x: Failed to write trace %d %x!", xTaskGetCurrentTaskHandle(), res, arg->data.wr_cnt & arg->data.mask);
         } else {
@@ -342,66 +298,49 @@ static void esp_apptrace_test_task_crash(void *p)
     vTaskDelete(NULL);
 }
 
-static int s_ts_timer_group, s_ts_timer_idx;
+static gptimer_handle_t ts_gptimer;
 
 static uint64_t esp_apptrace_test_ts_get(void)
 {
     uint64_t ts = 0;
-    timer_get_counter_value(s_ts_timer_group, s_ts_timer_idx, &ts);
+    gptimer_get_raw_count(ts_gptimer, &ts);
     return ts;
 }
 
-static void esp_apptrace_test_ts_init(int timer_group, int timer_idx)
+static void esp_apptrace_test_ts_init(void)
 {
-    timer_config_t config;
-    //uint64_t alarm_val = period * (TIMER_BASE_CLK / 1000000UL);
-
-    ESP_APPTRACE_TEST_LOGI("Use timer%d.%d for TS", timer_group, timer_idx);
-
-    s_ts_timer_group = timer_group;
-    s_ts_timer_idx = timer_idx;
-
-    config.alarm_en = 0;
-    config.auto_reload = 0;
-    config.counter_dir = TIMER_COUNT_UP;
-    config.divider = 2;     //Range is 2 to 65536
-    config.counter_en = 0;
-    /*Configure timer*/
-    timer_init(timer_group, timer_idx, &config);
-    /*Load counter value */
-    timer_set_counter_value(timer_group, timer_idx, 0x00000000ULL);
-    /*Enable timer interrupt*/
-    timer_start(timer_group, timer_idx);
+    gptimer_config_t timer_config = {
+        .clk_src = GPTIMER_CLK_SRC_DEFAULT,
+        .direction = GPTIMER_COUNT_UP,
+        .resolution_hz = 10000000,
+    };
+    TEST_ESP_OK(gptimer_new_timer(&timer_config, &ts_gptimer));
+    ESP_APPTRACE_TEST_LOGI("Use timer %x for TS", ts_gptimer);
+    TEST_ESP_OK(gptimer_enable(ts_gptimer));
+    TEST_ESP_OK(gptimer_start(ts_gptimer));
 }
 
 static void esp_apptrace_test_ts_cleanup(void)
 {
-    timer_config_t config;
-
-    config.alarm_en = 0;
-    config.auto_reload = 0;
-    config.counter_dir = TIMER_COUNT_UP;
-    config.divider = 2;     //Range is 2 to 65536
-    config.counter_en = 0;
-    /*Configure timer*/
-    timer_init(s_ts_timer_group, s_ts_timer_idx, &config);
+    TEST_ESP_OK(gptimer_stop(ts_gptimer));
+    TEST_ESP_OK(gptimer_disable(ts_gptimer));
+    TEST_ESP_OK(gptimer_del_timer(ts_gptimer));
+    ts_gptimer = NULL;
 }
 
 static void esp_apptrace_test(esp_apptrace_test_cfg_t *test_cfg)
 {
-    int i, k;
-    int tims_in_use[TIMER_GROUP_MAX][TIMER_MAX] = {{0, 0}, {0, 0}};
-    esp_apptrace_test_task_arg_t dummy_task_arg[1];
+    esp_apptrace_test_task_arg_t dummy_task_arg = {
+        .core = 0,
+        .prio = 3,
+        .task_func = esp_apptrace_test_task_crash,
+        .data.buf = NULL,
+        .data.buf_sz = 0,
+        .data.period = 500000,
+        .timers_num = 0,
+        .timers = NULL,
+    };
 
-    memset(dummy_task_arg, 0, sizeof(dummy_task_arg));
-    dummy_task_arg[0].core = 0;
-    dummy_task_arg[0].prio = 3;
-    dummy_task_arg[0].task_func = esp_apptrace_test_task_crash;
-    dummy_task_arg[0].data.buf = NULL;
-    dummy_task_arg[0].data.buf_sz = 0;
-    dummy_task_arg[0].data.period = 500000;
-    dummy_task_arg[0].timers_num = 0;
-    dummy_task_arg[0].timers = NULL;
 #if ESP_APPTRACE_TEST_USE_PRINT_LOCK == 1
     s_print_lock = xSemaphoreCreateBinary();
     if (!s_print_lock) {
@@ -412,38 +351,17 @@ static void esp_apptrace_test(esp_apptrace_test_cfg_t *test_cfg)
 #else
 #endif
 
-    for (i = 0; i < test_cfg->tasks_num; i++) {
+    for (int i = 0; i < test_cfg->tasks_num; i++) {
         test_cfg->tasks[i].data.mask = 0xFF;
         test_cfg->tasks[i].stop = 0;
         test_cfg->tasks[i].done = xSemaphoreCreateBinary();
-        if (!test_cfg->tasks[i].done) {
-            ESP_APPTRACE_TEST_LOGE("Failed to create task completion semaphore!");
-            goto on_fail;
-        }
-        for (k = 0; k < test_cfg->tasks[i].timers_num; k++) {
+        TEST_ASSERT_NOT_NULL(test_cfg->tasks[i].done);
+        for (int k = 0; k < test_cfg->tasks[i].timers_num; k++) {
             test_cfg->tasks[i].timers[k].data.mask = 0xFF;
-            tims_in_use[test_cfg->tasks[i].timers[k].group][test_cfg->tasks[i].timers[k].id] = 1;
         }
     }
 
-    int found = 0;
-    for (i = 0; i < TIMER_GROUP_MAX; i++) {
-        for (k = 0; k < TIMER_MAX; k++) {
-            if (!tims_in_use[i][k]) {
-                ESP_APPTRACE_TEST_LOGD("Found timer%d.%d", i, k);
-                found = 1;
-                break;
-            }
-        }
-        if (found) {
-            break;
-        }
-    }
-    if (!found) {
-        ESP_APPTRACE_TEST_LOGE("No free timer for TS!");
-        goto on_fail;
-    }
-    esp_apptrace_test_ts_init(i, k);
+    esp_apptrace_test_ts_init();
 
     for (int i = 0; i < test_cfg->tasks_num; i++) {
         char name[30];
@@ -452,16 +370,15 @@ static void esp_apptrace_test(esp_apptrace_test_cfg_t *test_cfg)
         xTaskCreatePinnedToCore(test_cfg->tasks[i].task_func, name, 2048, &test_cfg->tasks[i], test_cfg->tasks[i].prio, &thnd, test_cfg->tasks[i].core);
         ESP_APPTRACE_TEST_LOGI("Created task %x", thnd);
     }
-    xTaskCreatePinnedToCore(esp_apptrace_dummy_task, "dummy0", 2048, &dummy_task_arg[0], dummy_task_arg[0].prio, NULL, 0);
+    xTaskCreatePinnedToCore(esp_apptrace_dummy_task, "dummy0", 2048, &dummy_task_arg, dummy_task_arg.prio, NULL, 0);
 #if CONFIG_FREERTOS_UNICORE == 0
-    xTaskCreatePinnedToCore(esp_apptrace_dummy_task, "dummy1", 2048, &dummy_task_arg[0], dummy_task_arg[0].prio, NULL, 1);
+    xTaskCreatePinnedToCore(esp_apptrace_dummy_task, "dummy1", 2048, &dummy_task_arg, dummy_task_arg.prio, NULL, 1);
 #endif
     for (int i = 0; i < test_cfg->tasks_num; i++) {
         //arg1.stop = 1;
         xSemaphoreTake(test_cfg->tasks[i].done, portMAX_DELAY);
     }
 
-on_fail:
     for (int i = 0; i < test_cfg->tasks_num; i++) {
         if (test_cfg->tasks[i].done) {
             vSemaphoreDelete(test_cfg->tasks[i].done);
@@ -489,8 +406,6 @@ TEST_CASE("App trace test (1 task + 1 crashed timer ISR @ 1 core)", "[trace][ign
     memset(s_test_timers, 0, sizeof(s_test_timers));
     memset(s_test_tasks, 0, sizeof(s_test_tasks));
 
-    s_test_timers[0].group = TIMER_GROUP_0;
-    s_test_timers[0].id = TIMER_0;
     s_test_timers[0].isr_func = esp_apptrace_test_timer_isr_crash;
     s_test_timers[0].data.buf = s_bufs[0];
     s_test_timers[0].data.buf_sz = sizeof(s_bufs[0]);
@@ -543,15 +458,11 @@ TEST_CASE("App trace test (2 tasks + 1 timer @ each core", "[trace][ignore]")
     memset(s_test_tasks, 0, sizeof(s_test_tasks));
     memset(s_test_timers, 0, sizeof(s_test_timers));
 
-    s_test_timers[0].group = TIMER_GROUP_0;
-    s_test_timers[0].id = TIMER_0;
     s_test_timers[0].isr_func = esp_apptrace_test_timer_isr;
     s_test_timers[0].data.buf = s_bufs[0];
     s_test_timers[0].data.buf_sz = sizeof(s_bufs[0]);
     s_test_timers[0].data.period = 150;
 
-    s_test_timers[1].group = TIMER_GROUP_1;
-    s_test_timers[1].id = TIMER_0;
     s_test_timers[1].isr_func = esp_apptrace_test_timer_isr;
     s_test_timers[1].data.buf = s_bufs[1];
     s_test_timers[1].data.buf_sz = sizeof(s_bufs[1]);
@@ -607,8 +518,6 @@ TEST_CASE("App trace test (1 task + 1 timer @ 1 core)", "[trace][ignore]")
     memset(s_test_timers, 0, sizeof(s_test_timers));
     memset(s_test_tasks, 0, sizeof(s_test_tasks));
 
-    s_test_timers[0].group = TIMER_GROUP_0;
-    s_test_timers[0].id = TIMER_0;
     s_test_timers[0].isr_func = esp_apptrace_test_timer_isr;
     s_test_timers[0].data.buf = s_bufs[0];
     s_test_timers[0].data.buf_sz = sizeof(s_bufs[0]);
@@ -744,7 +653,7 @@ static void esp_logtrace_task(void *p)
         ESP_LOGI(TAG, "%p: sample print 4 %c", xTaskGetCurrentTaskHandle(), ((i & 0xFF) % 95) + 32);
         ESP_LOGI(TAG, "%p: sample print 5 %f", xTaskGetCurrentTaskHandle(), 1.0);
         ESP_LOGI(TAG, "%p: sample print 6 %f", xTaskGetCurrentTaskHandle(), 3.45);
-        ESP_LOGI(TAG, "%p: logtrace task work %d.%d", xTaskGetCurrentTaskHandle(), xPortGetCoreID(), i);
+        ESP_LOGI(TAG, "%p: logtrace task work %d.%d", xTaskGetCurrentTaskHandle(), esp_cpu_get_core_id(), i);
         if (++i == 10000) {
             break;
         }
@@ -787,11 +696,11 @@ TEST_CASE("Log trace test (2 tasks)", "[trace][ignore]")
     vSemaphoreDelete(arg2.done);
 }
 
-#else
+#else // #if CONFIG_APPTRACE_SV_ENABLE == 0
 
 typedef struct {
-    int group;
-    int timer;
+    gptimer_handle_t gptimer;
+    uint32_t period;
     int flags;
     uint32_t id;
 } esp_sysviewtrace_timer_arg_t;
@@ -805,50 +714,47 @@ typedef struct {
     uint32_t id;
 } esp_sysviewtrace_task_arg_t;
 
-static void esp_sysview_test_timer_isr(void *arg)
+static bool esp_sysview_test_timer_isr(gptimer_handle_t timer, const gptimer_alarm_event_data_t *edata, void *user_ctx)
 {
-    esp_sysviewtrace_timer_arg_t *tim_arg = (esp_sysviewtrace_timer_arg_t *)arg;
-
-    //ESP_APPTRACE_TEST_LOGI("tim-%d: IRQ %d/%d\n", tim_arg->id, tim_arg->group, tim_arg->timer);
-
-    timer_group_clr_intr_status_in_isr(tim_arg->group, tim_arg->id);
-    timer_group_enable_alarm_in_isr(tim_arg->group, tim_arg->id);
+    esp_sysviewtrace_timer_arg_t *tim_arg = (esp_sysviewtrace_timer_arg_t *)user_ctx;
+    return false;
 }
 
 static void esp_sysviewtrace_test_task(void *p)
 {
     esp_sysviewtrace_task_arg_t *arg = (esp_sysviewtrace_task_arg_t *) p;
     volatile uint32_t tmp = 0;
-    timer_isr_handle_t inth;
-
     printf("%x: run sysview task\n", (uint32_t)xTaskGetCurrentTaskHandle());
 
     if (arg->timer) {
-        esp_err_t res = timer_isr_register(arg->timer->group, arg->timer->timer, esp_sysview_test_timer_isr, arg->timer, arg->timer->flags, &inth);
-        if (res != ESP_OK) {
-            printf("%x: failed to register timer ISR\n", (uint32_t)xTaskGetCurrentTaskHandle());
-        }
-        else {
-            res = timer_start(arg->timer->group, arg->timer->timer);
-            if (res != ESP_OK) {
-                printf("%x: failed to start timer\n", (uint32_t)xTaskGetCurrentTaskHandle());
-            }
-        }
+        gptimer_alarm_config_t alarm_config = {
+            .reload_count = 0,
+            .alarm_count = arg->timer->period,
+            .flags.auto_reload_on_alarm = true,
+        };
+        gptimer_event_callbacks_t cbs = {
+            .on_alarm = esp_sysview_test_timer_isr,
+        };
+        TEST_ESP_OK(gptimer_register_event_callbacks(arg->timer->gptimer, &cbs, arg->timer));
+        TEST_ESP_OK(gptimer_enable(arg->timer->gptimer));
+        TEST_ESP_OK(gptimer_set_alarm_action(arg->timer->gptimer, &alarm_config));
+        TEST_ESP_OK(gptimer_start(arg->timer->gptimer));
     }
 
     int i = 0;
     while (1) {
         static uint32_t count;
         printf("%d", arg->id);
-        if((++count % 80) == 0)
+        if ((++count % 80) == 0) {
             printf("\n");
+        }
         if (arg->sync) {
             xSemaphoreTake(*arg->sync, portMAX_DELAY);
         }
         for (uint32_t k = 0; k < arg->work_count; k++) {
             tmp++;
         }
-        vTaskDelay(arg->sleep_tmo/portTICK_PERIOD_MS);
+        vTaskDelay(arg->sleep_tmo / portTICK_PERIOD_MS);
         i++;
         if (arg->sync) {
             xSemaphoreGive(*arg->sync);
@@ -866,10 +772,9 @@ TEST_CASE("SysView trace test 1", "[trace][ignore]")
     TaskHandle_t thnd;
 
     esp_sysviewtrace_timer_arg_t tim_arg1 = {
-        .group = TIMER_GROUP_1,
-        .timer = TIMER_1,
         .flags = ESP_INTR_FLAG_SHARED,
         .id = 0,
+        .period = 500,
     };
     esp_sysviewtrace_task_arg_t arg1 = {
         .done = xSemaphoreCreateBinary(),
@@ -880,10 +785,9 @@ TEST_CASE("SysView trace test 1", "[trace][ignore]")
         .id = 0,
     };
     esp_sysviewtrace_timer_arg_t tim_arg2 = {
-        .group = TIMER_GROUP_1,
-        .timer = TIMER_0,
         .flags = 0,
         .id = 1,
+        .period = 100,
     };
     esp_sysviewtrace_task_arg_t arg2 = {
         .done = xSemaphoreCreateBinary(),
@@ -894,8 +798,15 @@ TEST_CASE("SysView trace test 1", "[trace][ignore]")
         .id = 1,
     };
 
-    esp_apptrace_test_timer_init(TIMER_GROUP_1, TIMER_1, 500);
-    esp_apptrace_test_timer_init(TIMER_GROUP_1, TIMER_0, 100);
+    gptimer_config_t timer_config = {
+        .clk_src = GPTIMER_CLK_SRC_DEFAULT,
+        .direction = GPTIMER_COUNT_UP,
+        .resolution_hz = 1000000,
+    };
+    timer_config.flags.intr_shared = (tim_arg1.flags & ESP_INTR_FLAG_SHARED) == ESP_INTR_FLAG_SHARED;
+    TEST_ESP_OK(gptimer_new_timer(&timer_config, &tim_arg1.gptimer));
+    timer_config.flags.intr_shared = (tim_arg2.flags & ESP_INTR_FLAG_SHARED) == ESP_INTR_FLAG_SHARED;
+    TEST_ESP_OK(gptimer_new_timer(&timer_config, &tim_arg2.gptimer));
 
     xTaskCreatePinnedToCore(esp_sysviewtrace_test_task, "svtrace0", 2048, &arg1, 3, &thnd, 0);
     ESP_APPTRACE_TEST_LOGI("Created task %x", thnd);
@@ -910,6 +821,12 @@ TEST_CASE("SysView trace test 1", "[trace][ignore]")
     vSemaphoreDelete(arg1.done);
     xSemaphoreTake(arg2.done, portMAX_DELAY);
     vSemaphoreDelete(arg2.done);
+    TEST_ESP_OK(gptimer_stop(tim_arg1.gptimer));
+    TEST_ESP_OK(gptimer_disable(tim_arg1.gptimer));
+    TEST_ESP_OK(gptimer_del_timer(tim_arg1.gptimer));
+    TEST_ESP_OK(gptimer_stop(tim_arg2.gptimer));
+    TEST_ESP_OK(gptimer_disable(tim_arg2.gptimer));
+    TEST_ESP_OK(gptimer_del_timer(tim_arg2.gptimer));
 }
 
 TEST_CASE("SysView trace test 2", "[trace][ignore]")
@@ -917,10 +834,9 @@ TEST_CASE("SysView trace test 2", "[trace][ignore]")
     TaskHandle_t thnd;
 
     esp_sysviewtrace_timer_arg_t tim_arg1 = {
-        .group = TIMER_GROUP_1,
-        .timer = TIMER_1,
         .flags = ESP_INTR_FLAG_SHARED,
         .id = 0,
+        .period = 500,
     };
     esp_sysviewtrace_task_arg_t arg1 = {
         .done = xSemaphoreCreateBinary(),
@@ -931,10 +847,9 @@ TEST_CASE("SysView trace test 2", "[trace][ignore]")
         .id = 0,
     };
     esp_sysviewtrace_timer_arg_t tim_arg2 = {
-        .group = TIMER_GROUP_1,
-        .timer = TIMER_0,
         .flags = 0,
         .id = 1,
+        .period = 100,
     };
     esp_sysviewtrace_task_arg_t arg2 = {
         .done = xSemaphoreCreateBinary(),
@@ -964,8 +879,15 @@ TEST_CASE("SysView trace test 2", "[trace][ignore]")
         .id = 3,
     };
 
-    esp_apptrace_test_timer_init(TIMER_GROUP_1, TIMER_1, 500);
-    esp_apptrace_test_timer_init(TIMER_GROUP_1, TIMER_0, 100);
+    gptimer_config_t timer_config = {
+        .clk_src = GPTIMER_CLK_SRC_DEFAULT,
+        .direction = GPTIMER_COUNT_UP,
+        .resolution_hz = 1000000,
+    };
+    timer_config.flags.intr_shared = (tim_arg1.flags & ESP_INTR_FLAG_SHARED) == ESP_INTR_FLAG_SHARED;
+    TEST_ESP_OK(gptimer_new_timer(&timer_config, &tim_arg1.gptimer));
+    timer_config.flags.intr_shared = (tim_arg2.flags & ESP_INTR_FLAG_SHARED) == ESP_INTR_FLAG_SHARED;
+    TEST_ESP_OK(gptimer_new_timer(&timer_config, &tim_arg2.gptimer));
 
     xTaskCreatePinnedToCore(esp_sysviewtrace_test_task, "svtrace0", 2048, &arg1, 3, &thnd, 0);
     printf("Created task %x\n", (uint32_t)thnd);
@@ -994,6 +916,12 @@ TEST_CASE("SysView trace test 2", "[trace][ignore]")
     xSemaphoreTake(arg4.done, portMAX_DELAY);
     vSemaphoreDelete(arg4.done);
     vSemaphoreDelete(test_sync);
+    TEST_ESP_OK(gptimer_stop(tim_arg1.gptimer));
+    TEST_ESP_OK(gptimer_disable(tim_arg1.gptimer));
+    TEST_ESP_OK(gptimer_del_timer(tim_arg1.gptimer));
+    TEST_ESP_OK(gptimer_stop(tim_arg2.gptimer));
+    TEST_ESP_OK(gptimer_disable(tim_arg2.gptimer));
+    TEST_ESP_OK(gptimer_del_timer(tim_arg2.gptimer));
 }
-#endif
-#endif
+#endif // #if CONFIG_APPTRACE_SV_ENABLE == 0
+#endif // #if CONFIG_APPTRACE_ENABLE == 1

components/app_update/CMakeLists.txt

@@ -1,29 +1,7 @@
-idf_component_register(SRCS "esp_ota_ops.c"
-                            "esp_app_desc.c"
+idf_component_register(SRCS "esp_ota_ops.c" "esp_ota_app_desc.c"
                     INCLUDE_DIRS "include"
-                    REQUIRES spi_flash partition_table bootloader_support)
-
-# esp_app_desc structure is added as an undefined symbol because otherwise the
-# linker will ignore this structure as it has no other files depending on it.
-target_link_libraries(${COMPONENT_LIB} INTERFACE "-u esp_app_desc")
-
-if(CONFIG_APP_PROJECT_VER_FROM_CONFIG)
-    # Ignore current PROJECT_VER (which was set in __project_get_revision()).
-    # Gets the version from the CONFIG_APP_PROJECT_VER.
-    idf_build_set_property(PROJECT_VER "${CONFIG_APP_PROJECT_VER}")
-endif()
-
-# cut PROJECT_VER and PROJECT_NAME to required 32 characters.
-idf_build_get_property(project_ver PROJECT_VER)
-idf_build_get_property(project_name PROJECT_NAME)
-string(SUBSTRING "${project_ver}" 0 31 PROJECT_VER_CUT)
-string(SUBSTRING "${project_name}" 0 31 PROJECT_NAME_CUT)
-message(STATUS "App \"${PROJECT_NAME_CUT}\" version: ${PROJECT_VER_CUT}")
-
-set_source_files_properties(
-    SOURCE "esp_app_desc.c"
-    PROPERTIES COMPILE_DEFINITIONS
-    "PROJECT_VER=\"${PROJECT_VER_CUT}\"; PROJECT_NAME=\"${PROJECT_NAME_CUT}\"")
+                    REQUIRES partition_table bootloader_support esp_app_format esp_partition
+                    PRIV_REQUIRES esptool_py efuse spi_flash)
 
 if(NOT BOOTLOADER_BUILD)
     partition_table_get_partition_info(otadata_offset "--partition-type data --partition-subtype ota" "offset")
@@ -45,35 +23,41 @@ if(NOT BOOTLOADER_BUILD)
 
         add_custom_target(blank_ota_data ALL DEPENDS ${blank_otadata_file})
         add_dependencies(flash blank_ota_data)
+        add_dependencies(encrypted-flash blank_ota_data)
 
-        set(otatool_py ${python} ${COMPONENT_DIR}/otatool.py)
+        set(otatool_py "${python}" "${COMPONENT_DIR}/otatool.py")
+
+        set(esptool_args --esptool-args before=${CONFIG_ESPTOOLPY_BEFORE} after=${CONFIG_ESPTOOLPY_AFTER})
+        set(otatool_args --partition-table-file ${PARTITION_CSV_PATH})
+        list(APPEND otatool_args --partition-table-offset ${PARTITION_TABLE_OFFSET})
 
-        set(esptool_args "--esptool-args;before=${CONFIG_ESPTOOLPY_BEFORE};after=${CONFIG_ESPTOOLPY_AFTER}")
-        set(otatool_args "--partition-table-file;${PARTITION_CSV_PATH}"
-                         "--partition-table-offset;${PARTITION_TABLE_OFFSET}")
         idf_component_get_property(esptool_py_dir esptool_py COMPONENT_DIR)
 
-        add_custom_target(read_otadata DEPENDS "${PARTITION_CSV_PATH}"
+        add_custom_target(read-otadata DEPENDS "${PARTITION_CSV_PATH}"
             COMMAND ${CMAKE_COMMAND}
-            -D IDF_PATH="${idf_path}"
-            -D SERIAL_TOOL="${otatool_py}"
-            -D SERIAL_TOOL_ARGS="${esptool_args};${otatool_args};read_otadata"
-            -D WORKING_DIRECTORY="${build_dir}"
+            -D "IDF_PATH=${idf_path}"
+            -D "SERIAL_TOOL=${otatool_py}"
+            -D "SERIAL_TOOL_ARGS=${esptool_args};${otatool_args};read_otadata"
+            -D "WORKING_DIRECTORY=${build_dir}"
             -P ${esptool_py_dir}/run_serial_tool.cmake
-            WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}
+            WORKING_DIRECTORY "${build_dir}"
             USES_TERMINAL
+            VERBATIM
             )
+        add_deprecated_target_alias(read_otadata read-otadata)
 
-        add_custom_target(erase_otadata DEPENDS "${PARTITION_CSV_PATH}"
+        add_custom_target(erase-otadata DEPENDS "${PARTITION_CSV_PATH}"
             COMMAND ${CMAKE_COMMAND}
-            -D IDF_PATH="${idf_path}"
-            -D SERIAL_TOOL="${otatool_py}"
-            -D SERIAL_TOOL_ARGS="${esptool_args};${otatool_args};erase_otadata"
-            -D WORKING_DIRECTORY="${build_dir}"
+            -D "IDF_PATH=${idf_path}"
+            -D "SERIAL_TOOL=${otatool_py}"
+            -D "SERIAL_TOOL_ARGS=${esptool_args};${otatool_args};erase_otadata"
+            -D "WORKING_DIRECTORY=${build_dir}"
             -P ${esptool_py_dir}/run_serial_tool.cmake
-            WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}
+            WORKING_DIRECTORY "${build_dir}"
             USES_TERMINAL
+            VERBATIM
             )
+        add_deprecated_target_alias(erase_otadata erase-otadata)
 
         idf_component_get_property(main_args esptool_py FLASH_ARGS)
         idf_component_get_property(sub_args esptool_py FLASH_SUB_ARGS)

components/app_update/Makefile.projbuild

@@ -1,54 +0,0 @@
-# Generate partition binary
-#
-.PHONY: blank_ota_data erase_otadata read_otadata
-
-OTATOOL_PY := $(PYTHON) $(COMPONENT_PATH)/otatool.py
-PARTTOOL_PY := $(PYTHON) $(IDF_PATH)/components/partition_table/parttool.py
-
-# Generate blank partition file
-BLANK_OTA_DATA_FILE = $(BUILD_DIR_BASE)/ota_data_initial.bin
-
-# Copy PARTITION_TABLE_CSV_PATH definition here from $IDF_PATH/components/partition_table/Makefile.projbuild
-# to avoid undefined variables warning for PARTITION_TABLE_CSV_PATH
-ifndef PARTITION_TABLE_CSV_PATH
-PARTITION_TABLE_ROOT := $(call dequote,$(if $(CONFIG_PARTITION_TABLE_CUSTOM),$(PROJECT_PATH),$(IDF_PATH)/components/partition_table))
-PARTITION_TABLE_CSV_PATH := $(call dequote,$(abspath $(PARTITION_TABLE_ROOT)/$(call dequote,$(CONFIG_PARTITION_TABLE_FILENAME))))
-endif
-
-$(BLANK_OTA_DATA_FILE): partition_table_get_info $(PARTITION_TABLE_CSV_PATH) | check_python_dependencies
-	$(shell if [ "$(OTA_DATA_OFFSET)" != "" ] && [ "$(OTA_DATA_SIZE)" != "" ]; then \
-				$(PYTHON) $(IDF_PATH)/components/partition_table/gen_empty_partition.py $(OTA_DATA_SIZE) $(BLANK_OTA_DATA_FILE); \
-			fi; )
-	$(eval BLANK_OTA_DATA_FILE = $(shell if [ "$(OTA_DATA_OFFSET)" != "" ] && [ "$(OTA_DATA_SIZE)" != "" ]; then \
-			echo $(BLANK_OTA_DATA_FILE); else echo " "; fi) )
-
-blank_ota_data: $(BLANK_OTA_DATA_FILE)
-
-# If there is no otadata partition, both OTA_DATA_OFFSET and BLANK_OTA_DATA_FILE
-# expand to empty values.
-ESPTOOL_ALL_FLASH_ARGS += $(OTA_DATA_OFFSET) $(BLANK_OTA_DATA_FILE)
-
-ESPTOOL_ARGS := --esptool-args port=$(CONFIG_ESPTOOLPY_PORT) baud=$(CONFIG_ESPTOOLPY_BAUD) before=$(CONFIG_ESPTOOLPY_BEFORE) after=$(CONFIG_ESPTOOLPY_AFTER)
-
-erase_otadata: $(PARTITION_TABLE_CSV_PATH) partition_table_get_info | check_python_dependencies
-	$(OTATOOL_PY) $(ESPTOOL_ARGS) --partition-table-file $(PARTITION_TABLE_CSV_PATH) \
-	--partition-table-offset $(PARTITION_TABLE_OFFSET) \
-	erase_otadata
-
-read_otadata: $(PARTITION_TABLE_CSV_PATH) partition_table_get_info | check_python_dependencies
-	$(OTATOOL_PY) $(ESPTOOL_ARGS) --partition-table-file $(PARTITION_TABLE_CSV_PATH) \
-	--partition-table-offset $(partition_table_offset) \
-	read_otadata
-
-all: blank_ota_data
-flash: blank_ota_data
-ifdef CONFIG_SECURE_FLASH_ENCRYPTION_MODE_DEVELOPMENT
-encrypted-flash: blank_ota_data
-endif
-
-TMP_DEFINES := $(BUILD_DIR_BASE)/app_update/tmp_cppflags.txt
-export TMP_DEFINES
-
-clean:
-	rm -f $(BLANK_OTA_DATA_FILE)
-	rm -f $(TMP_DEFINES)

components/app_update/component.mk

@@ -1,56 +0,0 @@
-#
-# Component Makefile
-#
-# (Uses default behaviour of compiling all source files in directory, adding 'include' to include path.)
-
-# esp_app_desc structure is added as an undefined symbol because otherwise the
-# linker will ignore this structure as it has no other files depending on it.
-COMPONENT_ADD_LDFLAGS += -u esp_app_desc
-
-ifndef IS_BOOTLOADER_BUILD
-    # If ``CONFIG_APP_PROJECT_VER_FROM_CONFIG`` option is set, the value of ``CONFIG_APP_PROJECT_VER`` will be used
-    # Else, if ``PROJECT_VER`` variable set in project Makefile file, its value will be used.
-    # Else, if the ``$PROJECT_PATH/version.txt`` exists, its contents will be used as ``PROJECT_VER``.
-    # Else, if the project is located inside a Git repository, the output of git describe will be used.
-    # Otherwise, ``PROJECT_VER`` will be "1".
-    ifdef CONFIG_APP_PROJECT_VER_FROM_CONFIG
-        PROJECT_VER:= $(CONFIG_APP_PROJECT_VER)
-    else
-        ifneq ("${PROJECT_VER}", "")
-            PROJECT_VER:= $(PROJECT_VER)
-        else
-            ifneq ("$(wildcard ${PROJECT_PATH}/version.txt)","")
-                PROJECT_VER := $(shell cat ${PROJECT_PATH}/version.txt)
-            else
-                GIT_PROJECT_VER := $(shell cd ${PROJECT_PATH} && git describe --always --tags --dirty 2> /dev/null)
-                ifeq ("${GIT_PROJECT_VER}", "")
-                    PROJECT_VER := "1"
-                    $(info Project is not inside a git repository, or git repository has no commits)
-                    $(info will not use 'git describe' to determine PROJECT_VER.)
-                else
-                    PROJECT_VER:= $(GIT_PROJECT_VER)
-                endif # a git repository
-            endif # version.txt
-        endif # PROJECT_VER
-    endif # CONFIG_APP_PROJECT_VER_FROM_CONFIG
-
-    # cut PROJECT_VER and PROJECT_NAME to required 32 characters.
-    PROJECT_VER_CUT  := $(shell echo "$(PROJECT_VER)"  | cut -c 1-31)
-    PROJECT_NAME_CUT := $(shell echo "$(PROJECT_NAME)" | cut -c 1-31)
-
-    $(info App "$(PROJECT_NAME_CUT)" version: $(PROJECT_VER_CUT))
-
-    NEW_DEFINES:= "$(PROJECT_VER_CUT) $(PROJECT_NAME_CUT) $(IDF_VER)"
-    ifeq ("$(wildcard ${TMP_DEFINES})","")
-        OLD_DEFINES:= ""
-    else
-        OLD_DEFINES:= "$(shell cat $(TMP_DEFINES))"
-    endif
-
-    # If NEW_DEFINES (PROJECT_VER, PROJECT_NAME) were changed then rebuild only esp_app_desc.
-    ifneq (${NEW_DEFINES}, ${OLD_DEFINES})
-        $(shell echo $(NEW_DEFINES) > $(TMP_DEFINES); rm -f esp_app_desc.o;)
-    endif
-
-    esp_app_desc.o: CPPFLAGS += -D PROJECT_VER=\""$(PROJECT_VER_CUT)"\" -D PROJECT_NAME=\""$(PROJECT_NAME_CUT)"\"
-endif # IS_BOOTLOADER_BUILD

components/app_update/esp_app_desc.c

@@ -1,110 +0,0 @@
-// Copyright 2017-2018 Espressif Systems (Shanghai) PTE LTD
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include <assert.h>
-#include <sys/param.h>
-#include "esp_ota_ops.h"
-#include "esp_attr.h"
-#include "sdkconfig.h"
-
-// Application version info
-const __attribute__((section(".rodata_desc"))) esp_app_desc_t esp_app_desc = {
-    .magic_word = ESP_APP_DESC_MAGIC_WORD,
-#ifdef CONFIG_APP_EXCLUDE_PROJECT_VER_VAR
-    .version = "",
-#else
-    .version = PROJECT_VER,
-#endif
-
-#ifdef CONFIG_APP_EXCLUDE_PROJECT_NAME_VAR
-    .project_name = "",
-#else
-    .project_name = PROJECT_NAME,
-#endif
-    .idf_ver = IDF_VER,
-
-#ifdef CONFIG_BOOTLOADER_APP_SECURE_VERSION
-    .secure_version = CONFIG_BOOTLOADER_APP_SECURE_VERSION,
-#else
-    .secure_version = 0,
-#endif
-
-#ifdef CONFIG_APP_COMPILE_TIME_DATE
-    .time = __TIME__,
-    .date = __DATE__,
-#else
-    .time = "",
-    .date = "",
-#endif
-};
-
-
-#ifndef CONFIG_APP_EXCLUDE_PROJECT_VER_VAR
-_Static_assert(sizeof(PROJECT_VER) <= sizeof(esp_app_desc.version), "PROJECT_VER is longer than version field in structure");
-#endif
-_Static_assert(sizeof(IDF_VER) <= sizeof(esp_app_desc.idf_ver), "IDF_VER is longer than idf_ver field in structure");
-#ifndef CONFIG_APP_EXCLUDE_PROJECT_NAME_VAR
-_Static_assert(sizeof(PROJECT_NAME) <= sizeof(esp_app_desc.project_name), "PROJECT_NAME is longer than project_name field in structure");
-#endif
-
-const esp_app_desc_t *esp_ota_get_app_description(void)
-{
-    return &esp_app_desc;
-}
-
-/* The following two functions may be called from the panic handler
- * or core dump, hence IRAM_ATTR.
- */
-
-static inline char IRAM_ATTR to_hex_digit(unsigned val)
-{
-    return (val < 10) ? ('0' + val) : ('a' + val - 10);
-}
-
-__attribute__((constructor)) void esp_ota_init_app_elf_sha256(void)
-{
-    esp_ota_get_app_elf_sha256(NULL, 0);
-}
-
-/* The esp_app_desc.app_elf_sha256 should be possible to print in panic handler during cache is disabled.
- * But because the cache is disabled the reading esp_app_desc.app_elf_sha256 is not right and
- * can lead to a complete lock-up of the CPU.
- * For this reason we do a reading of esp_app_desc.app_elf_sha256 while start up in esp_ota_init_app_elf_sha256()
- * and keep it in the static s_app_elf_sha256 value.
- */
-int IRAM_ATTR esp_ota_get_app_elf_sha256(char* dst, size_t size)
-{
-    static char s_app_elf_sha256[CONFIG_APP_RETRIEVE_LEN_ELF_SHA / 2];
-    static bool first_call = true;
-    if (first_call) {
-        first_call = false;
-        // At -O2 optimization level, GCC optimizes out the copying of the first byte of the app_elf_sha256,
-        // because it is zero at compile time, and only modified afterwards by esptool.
-        // Casting to volatile disables the optimization.
-        const volatile uint8_t* src = (const volatile uint8_t*)esp_app_desc.app_elf_sha256;
-        for (size_t i = 0; i < sizeof(s_app_elf_sha256); ++i) {
-            s_app_elf_sha256[i] = src[i];
-        }
-    }
-    if (dst == NULL || size == 0) {
-        return 0;
-    }
-    size_t n = MIN((size - 1) / 2, sizeof(s_app_elf_sha256));
-    for (size_t i = 0; i < n; ++i) {
-        dst[2*i] = to_hex_digit(s_app_elf_sha256[i] >> 4);
-        dst[2*i + 1] = to_hex_digit(s_app_elf_sha256[i] & 0xf);
-    }
-    dst[2*n] = 0;
-    return 2*n + 1;
-}

components/app_update/esp_ota_app_desc.c

@@ -0,0 +1,21 @@
+/*
+ * SPDX-FileCopyrightText: 2017-2022 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#include <assert.h>
+#include <sys/param.h>
+#include "esp_ota_ops.h"
+#include "esp_attr.h"
+#include "sdkconfig.h"
+
+const esp_app_desc_t *esp_ota_get_app_description(void)
+{
+    return esp_app_get_description();
+}
+
+int IRAM_ATTR esp_ota_get_app_elf_sha256(char* dst, size_t size)
+{
+   return esp_app_get_elf_sha256(dst, size);
+}

components/app_update/esp_ota_ops.c

@@ -1,16 +1,8 @@
-// Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
+/*
+ * SPDX-FileCopyrightText: 2015-2021 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
 
 #include <stdint.h>
 #include <stdbool.h>
@@ -19,16 +11,12 @@
 #include <stdio.h>
 #include <string.h>
 #include <assert.h>
-#include <freertos/FreeRTOS.h>
-#include <freertos/task.h>
 
 #include "esp_err.h"
 #include "esp_partition.h"
-#include "esp_spi_flash.h"
 #include "esp_image_format.h"
 #include "esp_secure_boot.h"
 #include "esp_flash_encrypt.h"
-#include "esp_spi_flash.h"
 #include "sdkconfig.h"
 
 #include "esp_ota_ops.h"
@@ -41,6 +29,20 @@
 #include "esp_efuse.h"
 #include "esp_attr.h"
 
+#if CONFIG_IDF_TARGET_ESP32
+#include "esp32/rom/secure_boot.h"
+#elif CONFIG_IDF_TARGET_ESP32S2
+#include "esp32s2/rom/secure_boot.h"
+#elif CONFIG_IDF_TARGET_ESP32C3
+#include "esp32c3/rom/secure_boot.h"
+#elif CONFIG_IDF_TARGET_ESP32S3
+#include "esp32s3/rom/secure_boot.h"
+#elif CONFIG_IDF_TARGET_ESP32H2
+#include "esp32h2/rom/secure_boot.h"
+#elif CONFIG_IDF_TARGET_ESP32C2
+#include "esp32c2/rom/secure_boot.h"
+#endif
+
 #define SUB_TYPE_ID(i) (i & 0x0F)
 
 /* Partial_data is word aligned so no reallocation is necessary for encrypted flash write */
@@ -81,16 +83,16 @@ static const esp_partition_t *read_otadata(esp_ota_select_entry_t *two_otadata)
         return NULL;
     }
 
-    spi_flash_mmap_handle_t ota_data_map;
+    esp_partition_mmap_handle_t ota_data_map;
     const void *result = NULL;
-    esp_err_t err = esp_partition_mmap(otadata_partition, 0, otadata_partition->size, SPI_FLASH_MMAP_DATA, &result, &ota_data_map);
+    esp_err_t err = esp_partition_mmap(otadata_partition, 0, otadata_partition->size, ESP_PARTITION_MMAP_DATA, &result, &ota_data_map);
     if (err != ESP_OK) {
         ESP_LOGE(TAG, "mmap otadata filed. Err=0x%8x", err);
         return NULL;
     } else {
         memcpy(&two_otadata[0], result, sizeof(esp_ota_select_entry_t));
         memcpy(&two_otadata[1], result + SPI_FLASH_SEC_SIZE, sizeof(esp_ota_select_entry_t));
-        spi_flash_munmap(ota_data_map);
+        esp_partition_munmap(ota_data_map);
     }
     return otadata_partition;
 }
@@ -191,13 +193,18 @@ esp_err_t esp_ota_write(esp_ota_handle_t handle, const void *data, size_t size)
         return ESP_ERR_INVALID_ARG;
     }
 
+    if (size == 0) {
+        ESP_LOGD(TAG, "write data size is 0");
+        return ESP_OK;
+    }
+
     // find ota handle in linked list
     for (it = LIST_FIRST(&s_ota_ops_entries_head); it != NULL; it = LIST_NEXT(it, entries)) {
         if (it->handle == handle) {
             if (it->need_erase) {
                 // must erase the partition before writing to it
-                uint32_t first_sector = it->wrote_size / SPI_FLASH_SEC_SIZE;
-                uint32_t last_sector = (it->wrote_size + size) / SPI_FLASH_SEC_SIZE;
+                uint32_t first_sector = it->wrote_size / SPI_FLASH_SEC_SIZE; // first affected sector
+                uint32_t last_sector = (it->wrote_size + size - 1) / SPI_FLASH_SEC_SIZE; // last affected sector
 
                 ret = ESP_OK;
                 if ((it->wrote_size % SPI_FLASH_SEC_SIZE) == 0) {
@@ -381,7 +388,7 @@ static esp_err_t rewrite_ota_seq(esp_ota_select_entry_t *two_otadata, uint32_t s
     }
 }
 
-static uint8_t get_ota_partition_count(void)
+uint8_t esp_ota_get_app_partition_count(void)
 {
     uint16_t ota_app_count = 0;
     while (esp_partition_find_first(ESP_PARTITION_TYPE_APP, ESP_PARTITION_SUBTYPE_APP_OTA_MIN + ota_app_count, NULL) != NULL) {
@@ -399,7 +406,7 @@ static esp_err_t esp_rewrite_ota_data(esp_partition_subtype_t subtype)
         return ESP_ERR_NOT_FOUND;
     }
 
-    uint8_t ota_app_count = get_ota_partition_count();
+    uint8_t ota_app_count = esp_ota_get_app_partition_count();
     if (SUB_TYPE_ID(subtype) >= ota_app_count) {
         return ESP_ERR_INVALID_ARG;
     }
@@ -517,7 +524,7 @@ const esp_partition_t *esp_ota_get_boot_partition(void)
         return NULL;
     }
 
-    int ota_app_count = get_ota_partition_count();
+    int ota_app_count = esp_ota_get_app_partition_count();
     ESP_LOGD(TAG, "found ota app max = %d", ota_app_count);
 
     if ((bootloader_common_ota_select_invalid(&otadata[0]) &&
@@ -648,8 +655,14 @@ esp_err_t esp_ota_get_partition_description(const esp_partition_t *partition, es
 
 #ifdef CONFIG_BOOTLOADER_APP_ANTI_ROLLBACK
 static esp_err_t esp_ota_set_anti_rollback(void) {
-    const esp_app_desc_t *app_desc = esp_ota_get_app_description();
-    return esp_efuse_update_secure_version(app_desc->secure_version);
+    const esp_partition_t* partition = esp_ota_get_running_partition();
+    esp_app_desc_t app_desc = {0};
+
+    esp_err_t err = esp_ota_get_partition_description(partition, &app_desc);
+    if (err == ESP_OK) {
+        return esp_efuse_update_secure_version(app_desc.secure_version);
+    }
+    return err;
 }
 #endif
 
@@ -662,7 +675,7 @@ bool esp_ota_check_rollback_is_possible(void)
         return false;
     }
 
-    int ota_app_count = get_ota_partition_count();
+    int ota_app_count = esp_ota_get_app_partition_count();
     if (ota_app_count == 0) {
         return false;
     }
@@ -720,7 +733,7 @@ static esp_err_t esp_ota_current_ota_is_workable(bool valid)
     }
 
     int active_otadata = bootloader_common_get_active_otadata(otadata);
-    if (active_otadata != -1 && get_ota_partition_count() != 0) {
+    if (active_otadata != -1 && esp_ota_get_app_partition_count() != 0) {
         if (valid == true && otadata[active_otadata].ota_state != ESP_OTA_IMG_VALID) {
             otadata[active_otadata].ota_state = ESP_OTA_IMG_VALID;
             ESP_LOGD(TAG, "OTA[current] partition is marked as VALID");
@@ -789,7 +802,7 @@ const esp_partition_t* esp_ota_get_last_invalid_partition(void)
 
     int invalid_otadata = get_last_invalid_otadata(otadata);
 
-    int ota_app_count = get_ota_partition_count();
+    int ota_app_count = esp_ota_get_app_partition_count();
     if (invalid_otadata != -1 && ota_app_count != 0) {
         int ota_slot = (otadata[invalid_otadata].ota_seq - 1) % ota_app_count;
         ESP_LOGD(TAG, "Find invalid ota_%d app", ESP_PARTITION_SUBTYPE_APP_OTA_MIN + ota_slot);
@@ -817,7 +830,7 @@ esp_err_t esp_ota_get_state_partition(const esp_partition_t *partition, esp_ota_
     }
 
     esp_ota_select_entry_t otadata[2];
-    int ota_app_count = get_ota_partition_count();
+    int ota_app_count = esp_ota_get_app_partition_count();
     if (read_otadata(otadata) == NULL || ota_app_count == 0) {
         return ESP_ERR_NOT_FOUND;
     }
@@ -849,7 +862,7 @@ esp_err_t esp_ota_erase_last_boot_app_partition(void)
     }
 
     int active_otadata = bootloader_common_get_active_otadata(otadata);
-    int ota_app_count = get_ota_partition_count();
+    int ota_app_count = esp_ota_get_app_partition_count();
     if (active_otadata == -1 || ota_app_count == 0) {
         return ESP_FAIL;
     }
@@ -887,8 +900,8 @@ esp_err_t esp_ota_erase_last_boot_app_partition(void)
 }
 
 #if SOC_EFUSE_SECURE_BOOT_KEY_DIGESTS > 1 && CONFIG_SECURE_BOOT_V2_ENABLED
-esp_err_t esp_ota_revoke_secure_boot_public_key(esp_ota_secure_boot_public_key_index_t index)
-{
+esp_err_t esp_ota_revoke_secure_boot_public_key(esp_ota_secure_boot_public_key_index_t index) {
+
     if (!esp_secure_boot_enabled()) {
         ESP_LOGE(TAG, "Secure boot v2 has not been enabled.");
         return ESP_FAIL;
@@ -908,9 +921,10 @@ esp_err_t esp_ota_revoke_secure_boot_public_key(esp_ota_secure_boot_public_key_i
         return ESP_FAIL;
     }
 
-    ets_secure_boot_key_digests_t trusted_keys;
-    int ets_status = ets_secure_boot_read_key_digests(&trusted_keys);
-    if (ets_status != ETS_OK) {
+    esp_err_t ret;
+    esp_secure_boot_key_digests_t trusted_keys;
+    ret = esp_secure_boot_read_key_digests(&trusted_keys);
+    if (ret != ESP_OK) {
         ESP_LOGE(TAG, "Could not read the secure boot key digests from efuse. Aborting..");
         return ESP_FAIL;
     }

components/app_update/include/esp_ota_ops.h

@@ -1,16 +1,8 @@
-// Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
+/*
+ * SPDX-FileCopyrightText: 2015-2023 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
 
 #ifndef _OTA_OPS_H
 #define _OTA_OPS_H
@@ -20,8 +12,9 @@
 #include <stddef.h>
 #include "esp_err.h"
 #include "esp_partition.h"
-#include "esp_image_format.h"
+#include "esp_app_desc.h"
 #include "esp_flash_partitions.h"
+#include "soc/soc_caps.h"
 
 #ifdef __cplusplus
 extern "C"
@@ -51,20 +44,27 @@ typedef uint32_t esp_ota_handle_t;
 /**
  * @brief   Return esp_app_desc structure. This structure includes app version.
  *
+ * @note This API is present for backward compatibility reasons. Alternative function
+ * with the same functionality is `esp_app_get_description`
+ *
  * Return description for running app.
  * @return Pointer to esp_app_desc structure.
  */
-const esp_app_desc_t *esp_ota_get_app_description(void);
+const esp_app_desc_t *esp_ota_get_app_description(void) __attribute__((deprecated("Please use esp_app_get_description instead")));
 
 /**
  * @brief   Fill the provided buffer with SHA256 of the ELF file, formatted as hexadecimal, null-terminated.
  * If the buffer size is not sufficient to fit the entire SHA256 in hex plus a null terminator,
  * the largest possible number of bytes will be written followed by a null.
+ *
+* @note This API is present for backward compatibility reasons. Alternative function
+ * with the same functionality is `esp_app_get_elf_sha256`
+ *
  * @param dst   Destination buffer
  * @param size  Size of the buffer
  * @return      Number of bytes written to dst (including null terminator)
  */
-int esp_ota_get_app_elf_sha256(char* dst, size_t size);
+int esp_ota_get_app_elf_sha256(char* dst, size_t size) __attribute__((deprecated("Please use esp_app_get_elf_sha256 instead")));
 
 /**
  * @brief   Commence an OTA update writing to the specified partition.
@@ -110,7 +110,7 @@ esp_err_t esp_ota_begin(const esp_partition_t* partition, size_t image_size, esp
  * @param size    Size of data buffer in bytes.
  *
  * @return
- *    - ESP_OK: Data was written to flash successfully.
+ *    - ESP_OK: Data was written to flash successfully, or size = 0
  *    - ESP_ERR_INVALID_ARG: handle is invalid.
  *    - ESP_ERR_OTA_VALIDATE_FAILED: First byte of image contains invalid app image magic byte.
  *    - ESP_ERR_FLASH_OP_TIMEOUT or ESP_ERR_FLASH_OP_FAIL: Flash write failed.
@@ -119,17 +119,17 @@ esp_err_t esp_ota_begin(const esp_partition_t* partition, size_t image_size, esp
 esp_err_t esp_ota_write(esp_ota_handle_t handle, const void* data, size_t size);
 
 /**
- * @brief   Write OTA update data to partition
+ * @brief   Write OTA update data to partition at an offset
  *
- * This function can write data in non contiguous manner.
- * If flash encryption is enabled, data should be 16 byte aligned.
+ * This function can write data in non-contiguous manner.
+ * If flash encryption is enabled, data should be 16 bytes aligned.
  *
  * @param handle  Handle obtained from esp_ota_begin
  * @param data    Data buffer to write
  * @param size    Size of data buffer in bytes
  * @param offset  Offset in flash partition
  *
- * @note While performing OTA, if the packets arrive out of order, esp_ota_write_with_offset() can be used to write data in non contiguous manner.
+ * @note While performing OTA, if the packets arrive out of order, esp_ota_write_with_offset() can be used to write data in non-contiguous manner.
  *       Use of esp_ota_write_with_offset() in combination with esp_ota_write() is not recommended.
  *
  * @return
@@ -251,6 +251,14 @@ const esp_partition_t* esp_ota_get_next_update_partition(const esp_partition_t *
  */
 esp_err_t esp_ota_get_partition_description(const esp_partition_t *partition, esp_app_desc_t *app_desc);
 
+/**
+ * @brief Returns number of ota partitions provided in partition table.
+ *
+ * @return
+ *  - Number of OTA partitions
+ */
+uint8_t esp_ota_get_app_partition_count(void);
+
 /**
  * @brief This function is called to indicate that the running app is working well.
  *
@@ -326,9 +334,9 @@ typedef enum {
 /**
  * @brief Revokes the old signature digest. To be called in the application after the rollback logic.
  *
- * Relevant for Secure boot v2 on targets where upto 3 key digests can be stored (Key N-1, Key N, Key N+1).
- * When key N-1 used to sign an app is invalidated, an OTA update is to be sent with an app signed with key N-1 & Key N.
- * After successfully booting the OTA app should call this function to revoke Key N-1.
+ * Relevant for Secure boot v2 on ESP32-S2, ESP32-S3, ESP32-C3, ESP32-H2 where upto 3 key digests can be stored (Key \#N-1, Key \#N, Key \#N+1).
+ * When key \#N-1 used to sign an app is invalidated, an OTA update is to be sent with an app signed with key \#N-1 & Key \#N.
+ * After successfully booting the OTA app should call this function to revoke Key \#N-1.
  *
  * @param index - The index of the signature block to be revoked
  *

components/app_update/otatool.py

@@ -3,19 +3,8 @@
 # otatool is used to perform ota-level operations - flashing ota partition
 # erasing ota partition and switching ota partition
 #
-# Copyright 2018 Espressif Systems (Shanghai) PTE LTD
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http:#www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# SPDX-FileCopyrightText: 2018-2021 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Apache-2.0
 from __future__ import division, print_function
 
 import argparse
@@ -245,10 +234,6 @@ def _erase_ota_partition(target, ota_id):
 
 
 def main():
-    if sys.version_info[0] < 3:
-        print('WARNING: Support for Python 2 is deprecated and will be removed in future versions.', file=sys.stderr)
-    elif sys.version_info[0] == 3 and sys.version_info[1] < 6:
-        print('WARNING: Python 3 versions older than 3.6 are not supported.', file=sys.stderr)
     global quiet
 
     parser = argparse.ArgumentParser('ESP-IDF OTA Partitions Tool')

components/app_update/test/CMakeLists.txt

@@ -1,4 +1,5 @@
 idf_component_register(SRC_DIRS "."
                        PRIV_INCLUDE_DIRS "."
-                       PRIV_REQUIRES cmock test_utils app_update bootloader_support nvs_flash
+                       PRIV_REQUIRES cmock test_utils app_update bootloader_support nvs_flash driver spi_flash
                       )
+target_compile_options(${COMPONENT_LIB} PRIVATE "-Wno-format")

components/app_update/test/component.mk

@@ -1,5 +0,0 @@
-#
-#Component Makefile
-#
-
-COMPONENT_ADD_LDFLAGS = -Wl,--whole-archive -l$(COMPONENT_NAME) -Wl,--no-whole-archive

components/app_update/test/test_app_desc.c

@@ -1,50 +0,0 @@
-#include <string.h>
-#include "esp_ota_ops.h"
-#include "unity.h"
-
-TEST_CASE("esp_ota_get_app_elf_sha256 test", "[esp_app_desc]")
-{
-    const int sha256_hex_len = CONFIG_APP_RETRIEVE_LEN_ELF_SHA;
-    char dst[sha256_hex_len + 2];
-    const char fill = 0xcc;
-    int res;
-    size_t len;
-
-    char ref_sha256[sha256_hex_len + 1];
-    const esp_app_desc_t* desc = esp_ota_get_app_description();
-    for (int i = 0; i < sizeof(ref_sha256) / 2; ++i) {
-        snprintf(ref_sha256 + 2*i, 3, "%02x", desc->app_elf_sha256[i]);
-    }
-    ref_sha256[sha256_hex_len] = 0;
-
-    printf("Ref: %s\n", ref_sha256);
-
-    memset(dst, fill, sizeof(dst));
-    len = sizeof(dst);
-    res = esp_ota_get_app_elf_sha256(dst, len);
-    printf("%d: %s (%d)\n", len, dst, res);
-    TEST_ASSERT_EQUAL(sha256_hex_len + 1, res);
-    TEST_ASSERT_EQUAL(0, memcmp(dst, ref_sha256, res - 1));
-    TEST_ASSERT_EQUAL_HEX(0, dst[sha256_hex_len]);
-    TEST_ASSERT_EQUAL_HEX(fill, dst[sha256_hex_len + 1]);
-
-    memset(dst, fill, sizeof(dst));
-    len = 9;
-    res = esp_ota_get_app_elf_sha256(dst, len);
-    printf("%d: %s (%d)\n", len, dst, res);
-    TEST_ASSERT_EQUAL(9, res);
-    TEST_ASSERT_EQUAL(0, memcmp(dst, ref_sha256, res - 1));
-    TEST_ASSERT_EQUAL_HEX(0, dst[8]);
-    TEST_ASSERT_EQUAL_HEX(fill, dst[9]);
-
-    memset(dst, fill, sizeof(dst));
-    len = 8;
-    res = esp_ota_get_app_elf_sha256(dst, len);
-    printf("%d: %s (%d)\n", len, dst, res);
-    // should output even number of characters plus '\0'
-    TEST_ASSERT_EQUAL(7, res);
-    TEST_ASSERT_EQUAL(0, memcmp(dst, ref_sha256, res - 1));
-    TEST_ASSERT_EQUAL_HEX(0, dst[6]);
-    TEST_ASSERT_EQUAL_HEX(fill, dst[7]);
-    TEST_ASSERT_EQUAL_HEX(fill, dst[8]);
-}

components/app_update/test/test_ota_ops.c

@@ -1,3 +1,8 @@
+/*
+ * SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
@@ -8,7 +13,6 @@
 #include <unity.h>
 #include <test_utils.h>
 #include <esp_ota_ops.h>
-#include "bootloader_common.h"
 
 /* These OTA tests currently don't assume an OTA partition exists
    on the device, so they're a bit limited
@@ -86,6 +90,8 @@ TEST_CASE("esp_ota_get_next_update_partition logic", "[ota]")
 
 TEST_CASE("esp_ota_get_partition_description", "[ota]")
 {
+    extern esp_err_t bootloader_common_get_partition_description(const esp_partition_pos_t *partition, esp_app_desc_t *app_desc);
+
     const esp_partition_t *running = esp_ota_get_running_partition();
     TEST_ASSERT_NOT_NULL(running);
     esp_app_desc_t app_desc1, app_desc2;

components/app_update/test/test_switch_ota.c

@@ -1,3 +1,8 @@
+/*
+ * SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
 /*
  * Tests for switching between partitions: factory, OTAx, test.
  */
@@ -7,13 +12,7 @@
 #include "string.h"
 #include "sdkconfig.h"
 
-#if CONFIG_IDF_TARGET_ESP32
-#include "esp32/rom/spi_flash.h"
-#include "esp32/rom/rtc.h"
-#elif CONFIG_IDF_TARGET_ESP32S2
-#include "esp32s2/rom/spi_flash.h"
-#include "esp32s2/rom/rtc.h"
-#endif
+#include "esp_rom_spiflash.h"
 
 #include "freertos/FreeRTOS.h"
 #include "freertos/task.h"
@@ -22,7 +21,7 @@
 #include "unity.h"
 
 #include "bootloader_common.h"
-#include "../include_bootloader/bootloader_flash_priv.h"
+#include "../bootloader_flash/include/bootloader_flash_priv.h"
 
 #include "esp_log.h"
 #include "esp_ota_ops.h"
@@ -33,9 +32,13 @@
 
 #include "driver/gpio.h"
 #include "esp_sleep.h"
+#include "test_utils.h"
 
 
+#if !TEMPORARY_DISABLED_FOR_TARGETS(ESP32C2)
+//IDF-5131
 RTC_DATA_ATTR static int boot_count = 0;
+
 static const char *TAG = "ota_test";
 
 /* @brief Copies a current app to next partition using handle.
@@ -46,11 +49,11 @@ static const char *TAG = "ota_test";
 static void copy_app_partition(esp_ota_handle_t update_handle, const esp_partition_t *curr_app)
 {
     const void *partition_bin = NULL;
-    spi_flash_mmap_handle_t  data_map;
+    esp_partition_mmap_handle_t data_map;
     ESP_LOGI(TAG, "start the copy process");
-    TEST_ESP_OK(esp_partition_mmap(curr_app, 0, curr_app->size, SPI_FLASH_MMAP_DATA, &partition_bin, &data_map));
+    TEST_ESP_OK(esp_partition_mmap(curr_app, 0, curr_app->size, ESP_PARTITION_MMAP_DATA, &partition_bin, &data_map));
     TEST_ESP_OK(esp_ota_write(update_handle, (const void *)partition_bin, curr_app->size));
-    spi_flash_munmap(data_map);
+    esp_partition_munmap(data_map);
     ESP_LOGI(TAG, "finish the copy process");
 }
 
@@ -62,15 +65,15 @@ static void copy_app_partition(esp_ota_handle_t update_handle, const esp_partiti
 static void copy_app_partition_with_offset(esp_ota_handle_t update_handle, const esp_partition_t *curr_app)
 {
     const void *partition_bin = NULL;
-    spi_flash_mmap_handle_t  data_map;
+    esp_partition_mmap_handle_t  data_map;
     ESP_LOGI(TAG, "start the copy process");
     uint32_t offset = 0, bytes_to_write = curr_app->size;
     uint32_t write_bytes;
     while (bytes_to_write > 0) {
         write_bytes = (bytes_to_write > (4 * 1024)) ? (4 * 1024) : bytes_to_write;
-        TEST_ESP_OK(esp_partition_mmap(curr_app, offset, write_bytes, SPI_FLASH_MMAP_DATA, &partition_bin, &data_map));
+        TEST_ESP_OK(esp_partition_mmap(curr_app, offset, write_bytes, ESP_PARTITION_MMAP_DATA, &partition_bin, &data_map));
         TEST_ESP_OK(esp_ota_write_with_offset(update_handle, (const void *)partition_bin, write_bytes, offset));
-        spi_flash_munmap(data_map);
+        esp_partition_munmap(data_map);
         bytes_to_write -= write_bytes;
         offset += write_bytes;
     }
@@ -87,11 +90,11 @@ static void copy_app_partition_with_offset(esp_ota_handle_t update_handle, const
 static void copy_partition(const esp_partition_t *dst_partition, const esp_partition_t *src_partition)
 {
     const void *partition_bin = NULL;
-    spi_flash_mmap_handle_t data_map;
-    TEST_ESP_OK(esp_partition_mmap(src_partition, 0, src_partition->size, SPI_FLASH_MMAP_DATA, &partition_bin, &data_map));
+    esp_partition_mmap_handle_t data_map;
+    TEST_ESP_OK(esp_partition_mmap(src_partition, 0, src_partition->size, ESP_PARTITION_MMAP_DATA, &partition_bin, &data_map));
     TEST_ESP_OK(esp_partition_erase_range(dst_partition, 0, dst_partition->size));
     TEST_ESP_OK(esp_partition_write(dst_partition, 0, (const void *)partition_bin, dst_partition->size));
-    spi_flash_munmap(data_map);
+    esp_partition_munmap(data_map);
 }
 #endif
 
@@ -821,3 +824,27 @@ static void test_flow6(void)
 // 2 Stage: run factory -> check it -> copy factory to OTA0             -> reboot  --//--
 // 3 Stage: run OTA0    -> check it -> erase OTA_DATA for next tests    -> PASS
 TEST_CASE_MULTIPLE_STAGES("Switching between factory, OTA0 using esp_ota_write_with_offset", "[app_update][timeout=90][reset=DEEPSLEEP_RESET, DEEPSLEEP_RESET]", start_test, test_flow6, test_flow6);
+
+//IDF-5145
+TEST_CASE("Test bootloader_common_get_sha256_of_partition returns ESP_ERR_IMAGE_INVALID when image is ivalid", "[partitions]")
+{
+    const esp_partition_t *cur_app = esp_ota_get_running_partition();
+    ESP_LOGI(TAG, "copy current app to next part");
+    const esp_partition_t *other_app = get_next_update_partition();
+    copy_current_app_to_next_part(cur_app, other_app);
+    erase_ota_data();
+
+    uint8_t sha_256_cur_app[32];
+    uint8_t sha_256_other_app[32];
+    TEST_ESP_OK(bootloader_common_get_sha256_of_partition(cur_app->address, cur_app->size, cur_app->type, sha_256_cur_app));
+    TEST_ESP_OK(bootloader_common_get_sha256_of_partition(other_app->address, other_app->size, other_app->type, sha_256_other_app));
+
+    TEST_ASSERT_EQUAL_MEMORY_MESSAGE(sha_256_cur_app, sha_256_other_app, sizeof(sha_256_cur_app), "must be the same");
+
+    uint32_t data = 0;
+    bootloader_flash_write(other_app->address + 0x50, &data, sizeof(data), false);
+
+    TEST_ESP_ERR(ESP_ERR_IMAGE_INVALID, bootloader_common_get_sha256_of_partition(other_app->address, other_app->size, other_app->type, sha_256_other_app));
+    TEST_ASSERT_EQUAL_MEMORY_MESSAGE(sha_256_cur_app, sha_256_other_app, sizeof(sha_256_cur_app), "must be the same");
+}
+#endif //!TEMPORARY_DISABLED_FOR_TARGETS(ESP32C2)

components/asio/CMakeLists.txt

@@ -1,41 +0,0 @@
-if(NOT CONFIG_LWIP_IPV6 AND NOT CMAKE_BUILD_EARLY_EXPANSION)
-    # note: the component is still included in the build so it can become visible again in config
-    # without needing to re-run CMake. However no source or header files are built.
-    message(STATUS "IPV6 support is disabled so the asio component will not be built")
-    idf_component_register()
-    return()
-endif()
-
-set(asio_sources "asio/asio/src/asio.cpp")
-
-if(CONFIG_ASIO_SSL_SUPPORT)
-    if(CONFIG_ASIO_USE_ESP_OPENSSL)
-        list(APPEND asio_sources
-                "asio/asio/src/asio_ssl.cpp"
-                "port/src/esp_asio_openssl_stubs.c")
-    endif()
-
-    if(CONFIG_ASIO_USE_ESP_WOLFSSL)
-        list(APPEND asio_sources
-                "asio/asio/src/asio_ssl.cpp")
-    endif()
-endif()
-
-idf_component_register(SRCS ${asio_sources}
-                    INCLUDE_DIRS "asio/asio/include" "port/include"
-                    REQUIRES lwip)
-
-if(CONFIG_ASIO_SSL_SUPPORT)
-    if(CONFIG_ASIO_USE_ESP_WOLFSSL)
-        idf_component_get_property(wolflib esp-wolfssl COMPONENT_LIB)
-        idf_component_get_property(wolfdir esp-wolfssl COMPONENT_DIR)
-
-        target_link_libraries(${COMPONENT_LIB} PUBLIC ${wolflib})
-        target_include_directories(${COMPONENT_LIB} PUBLIC ${wolfdir}/wolfssl/wolfssl)
-    endif()
-
-    if(CONFIG_ASIO_USE_ESP_OPENSSL)
-        idf_component_get_property(esp_openssl openssl COMPONENT_LIB)
-        target_link_libraries(${COMPONENT_LIB} PUBLIC ${esp_openssl})
-    endif()
-endif()

components/asio/Kconfig

@@ -1,27 +0,0 @@
-menu "ESP-ASIO"
-    visible if LWIP_IPV6
-
-    config ASIO_SSL_SUPPORT
-        bool "Enable SSL/TLS support of ASIO"
-        default n
-        help
-            Enable support for basic SSL/TLS features, available for mbedTLS/OpenSSL
-            as well as wolfSSL TLS library.
-
-    choice ASIO_SSL_LIBRARY_CHOICE
-        prompt "Choose SSL/TLS library for ESP-TLS (See help for more Info)"
-        default ASIO_USE_ESP_OPENSSL
-        depends on ASIO_SSL_SUPPORT
-        help
-            The ASIO support multiple backend TLS libraries. Currently the mbedTLS with a thin ESP-OpenSSL
-            port layer (default choice) and WolfSSL are supported.
-            Different TLS libraries may support different features and have different resource
-            usage. Consult the ESP-TLS documentation in ESP-IDF Programming guide for more details.
-        config ASIO_USE_ESP_OPENSSL
-            bool "esp-openssl"
-        config ASIO_USE_ESP_WOLFSSL
-            depends on TLS_STACK_WOLFSSL
-            bool "wolfSSL (License info in wolfSSL directory README)"
-    endchoice
-
-endmenu

components/asio/component.mk

@@ -1,13 +0,0 @@
-ifdef CONFIG_LWIP_IPV6
-
-COMPONENT_ADD_INCLUDEDIRS := asio/asio/include port/include
-COMPONENT_PRIV_INCLUDEDIRS := private_include
-COMPONENT_SRCDIRS := asio/asio/src port/src
-
-ifeq ($(CONFIG_ASIO_SSL_SUPPORT), )
-COMPONENT_OBJEXCLUDE := asio/asio/src/asio_ssl.o port/src/esp_asio_openssl_stubs.o
-endif
-
-COMPONENT_SUBMODULES += asio
-
-endif # CONFIG_LWIP_IPV6

components/asio/port/include/esp_asio_config.h

@@ -1,50 +0,0 @@
-// Copyright 2018 Espressif Systems (Shanghai) PTE LTD
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-#ifndef _ESP_ASIO_CONFIG_H_
-#define _ESP_ASIO_CONFIG_H_
-
-//
-// Enabling exceptions only when they are enabled in menuconfig
-//
-# include <sdkconfig.h>
-# ifndef CONFIG_COMPILER_CXX_EXCEPTIONS
-#  define ASIO_NO_EXCEPTIONS
-# endif   // CONFIG_COMPILER_CXX_EXCEPTIONS
-
-# ifndef CONFIG_COMPILER_RTTI
-#  define ASIO_NO_TYPEID
-# endif   // CONFIG_COMPILER_RTTI
-
-//
-// LWIP compatibility inet and address macros/functions
-//
-# define LWIP_COMPAT_SOCKET_INET 1
-# define LWIP_COMPAT_SOCKET_ADDR 1
-
-//
-// Specific ASIO feature flags
-//
-# define ASIO_DISABLE_SERIAL_PORT
-# define ASIO_SEPARATE_COMPILATION
-# define ASIO_STANDALONE
-# define ASIO_HAS_PTHREADS
-
-# ifdef CONFIG_ASIO_USE_ESP_OPENSSL
-#  define ASIO_USE_ESP_OPENSSL
-#  define OPENSSL_NO_ENGINE
-# elif CONFIG_ASIO_USE_ESP_WOLFSSL
-#  define ASIO_USE_WOLFSSL
-# endif   // CONFIG_ASIO_USE_ESP_OPENSSL
-
-#endif // _ESP_ASIO_CONFIG_H_