change(version): Update version to 5.2.1

ci: use full fetch while checking git tags (v5.2)

See merge request espressif/esp-idf!29427

.codespellrc

@@ -1,4 +0,0 @@
-[codespell]
-skip = build,*.yuv,components/fatfs/src/*,alice.txt,*.rgb,components/wpa_supplicant/*,components/esp_wifi/*,*.pem
-ignore-words-list = ser,dout,rsource,fram,inout,shs,ans,aci,unstall,unstalling,hart,wheight,wel,ot,fane,assertIn,registr,oen,parms
-write-changes = true

.github/dangerjs/.gitignore

@@ -0,0 +1,5 @@
+# Transpiled JavaScript (if any)
+dist
+
+# Installed dependencies
+node_modules

.github/dangerjs/README.md

@@ -0,0 +1,47 @@
+# DangerJS pull request automatic review tool - GitHub
+
+## Implementation
+The main development is done in Espressif Gitlab project.
+Espressif [GitHub project espressif/esp-idf](https://github.com/espressif/esp-idf) is only a public mirror. 
+
+Therefore, all changes and updates to DangerJS files (`.github/dangerjs`) must be made via MR in the **Gitlab** repository by Espressif engineer. 
+
+When adding a new Danger rule or updating existing one, might be a good idea to test it on the developer's fork of GitHub project. This way, the new feature can be tested using a GitHub action without concern of damaging Espressif's GitHub repository.
+
+Danger for Espressif GitHub is implemented in TypeScript. This makes the code more readable and robust than plain JavaScript. 
+Compilation to JavaScript code (using `tsc`) is not necessary; Danger handles TypeScript natively.
+
+A good practice is to store each Danger rule in a separate module, and then import these modules into the main Danger file `.github/dangerjs/dangerfile.ts` (see how this is done for currently present modules when adding a new one).
+
+If the Danger module (new check/rule) uses an external NPM module (e.g. `axios`), be sure to add this dependency to `.github/dangerjs/package.json` and also update `.github/dangerjs/package-lock.json`.
+
+In the GitHub action, `danger` is not installed globally (nor are its dependencies) and the `npx` call is used to start the `danger` checks in CI.
+
+
+## Adding new Danger rule
+For local development you can use following strategy
+
+#### Install dependencies
+```sh
+cd .github/dangerjs
+npm install
+```
+(If the IDE still shows compiler/typing errors, reload the IDE window.)
+
+#### Add new code as needed or make updates
+
+#### Test locally
+Danger rules can be tested locally (without running the GitHub action pipeline). 
+To do this, you have to first export the ENV variables used by Danger in the local terminal:
+
+```sh
+export GITHUB_TOKEN='**************************************'
+```
+
+Then you can call Danger by:
+```sh
+cd .github/dangerjs
+
+danger pr https://github.com/espressif/esp-idf/pull/<number_of_pull_request> 
+```
+The result will be displayed in your terminal.

.github/dangerjs/dangerfile.ts

@@ -0,0 +1,48 @@
+import { DangerResults } from "danger";
+declare const results: DangerResults;
+declare const message: (message: string, results?: DangerResults) => void;
+declare const markdown: (message: string, results?: DangerResults) => void;
+
+// Import modules with danger rules
+// (Modules with checks are stored in ".github/dangerjs/<module_name>.ts". To import them, use path relative to "dangerfile.ts")
+import prCommitsTooManyCommits from "./prCommitsTooManyCommits";
+import prDescription from "./prDescription";
+import prTargetBranch from "./prTargetBranch";
+import prInfoContributor from "./prInfoContributor";
+import prCommitMessage from "./prCommitMessage";
+
+async function runDangerRules(): Promise<void> {
+    // Message to contributor about review and merge process
+    const prInfoContributorMessage: string = await prInfoContributor();
+    markdown(prInfoContributorMessage);
+
+    // Run danger checks
+    prCommitsTooManyCommits();
+    prDescription();
+    prTargetBranch();
+    prCommitMessage();
+
+    // Add success log if no issues
+    const dangerFails: number = results.fails.length;
+    const dangerWarns: number = results.warnings.length;
+    const dangerInfos: number = results.messages.length;
+    if (!dangerFails && !dangerWarns && !dangerInfos) {
+        return message("Good Job! All checks are passing!");
+    }
+
+    // Add retry link
+    addRetryLink();
+}
+
+runDangerRules();
+
+function addRetryLink(): void {
+    const serverUrl: string | undefined = process.env.GITHUB_SERVER_URL;
+    const repoName: string | undefined = process.env.GITHUB_REPOSITORY;
+    const runId: string | undefined = process.env.GITHUB_RUN_ID;
+
+    const retryLinkUrl: string = `${serverUrl}/${repoName}/actions/runs/${runId}`;
+    const retryLink: string = `<sub>:repeat: You can re-run automatic PR checks by retrying the <a href="${retryLinkUrl}">DangerJS action</a></sub>`;
+
+    markdown(retryLink);
+}

.github/dangerjs/package.json

@@ -0,0 +1,18 @@
+{
+    "name": "dangerjs-github",
+    "description": "GitHub PR reviewing with DangerJS",
+    "main": "dangerfile.ts",
+    "keywords": [],
+    "author": "",
+    "license": "ISC",
+    "dependencies": {
+        "axios": "^1.3.3",
+        "danger": "^11.2.3",
+        "request": "^2.88.2",
+        "sync-request": "^6.1.0",
+        "typescript": "^5.0.3"
+    },
+    "devDependencies": {
+        "@types/node": "^18.15.11"
+    }
+}

.github/dangerjs/prCommitMessage.ts

@@ -0,0 +1,67 @@
+import { DangerDSLType, DangerResults } from "danger";
+declare const danger: DangerDSLType;
+declare const warn: (message: string, results?: DangerResults) => void;
+
+interface Commit {
+    message: string;
+}
+
+/**
+ * Check if commit messages are sufficiently descriptive (not too short).
+ *
+ * Search for commit messages that appear to be automatically generated or temporary messages and report them.
+ *
+ * @dangerjs WARN
+ */
+export default function (): void {
+    const prCommits: Commit[] = danger.git.commits;
+
+    const detectRegexes: RegExp[] = [
+        /^Merge pull request #\d+ from .*/i, // Automatically generated message by GitHub
+        /^Merged .+:.+ into .+/i, // Automatically generated message by GitHub
+        /^Automatic merge by GitHub Action/i, // Automatically generated message by GitHub
+        /^Merge branch '.*' of .+ into .+/i, // Automatically generated message by GitHub
+        /^Create\s[a-zA-Z0-9_.-]+(\.[a-zA-Z0-9]{1,4})?(?=\s|$)/, // Automatically generated message by GitHub using UI
+        /^Delete\s[a-zA-Z0-9_.-]+(\.[a-zA-Z0-9]{1,4})?(?=\s|$)/, // Automatically generated message by GitHub using UI
+        /^Update\s[a-zA-Z0-9_.-]+(\.[a-zA-Z0-9]{1,4})?(?=\s|$)/, // Automatically generated message by GitHub using UI
+        /^Initial commit/i, // Automatically generated message by GitHub
+        /^WIP.*/i, // Message starts with prefix "WIP"
+        /^Cleaned.*/i, // Message starts "Cleaned", , probably temporary
+        /^Test:.*/i, // Message starts with "test" prefix, probably temporary
+        /clean ?up/i, // Message contains "clean up", probably temporary
+        /^[^A-Za-z0-9\s].*/, // Message starts with special characters
+    ];
+
+    let partMessages: string[] = [];
+
+    for (const commit of prCommits) {
+        const commitMessage: string = commit.message;
+        const commitMessageTitle: string = commit.message.split("\n")[0];
+
+        // Check if the commit message matches any regex from "detectRegexes"
+        if (detectRegexes.some((regex) => commitMessage.match(regex))) {
+            partMessages.push(
+                `- the commit message \`${commitMessageTitle}\` appears to be a temporary or automatically generated message`
+            );
+            continue;
+        }
+
+        // Check if the commit message is not too short
+        const shortCommitMessageThreshold: number = 20; // commit message is considered too short below this number of characters
+        if (commitMessage.length < shortCommitMessageThreshold) {
+            partMessages.push(
+                `- the commit message \`${commitMessageTitle}\` may not be sufficiently descriptive`
+            );
+        }
+    }
+
+    // Create report
+    if (partMessages.length) {
+        partMessages.sort();
+        let dangerMessage = `\nSome issues found for the commit messages in this MR:\n${partMessages.join(
+            "\n"
+        )}
+		\nPlease consider updating these commit messages.`;
+        warn(dangerMessage);
+    }
+}

.github/dangerjs/prCommitsTooManyCommits.ts

@@ -0,0 +1,19 @@
+import { DangerDSLType, DangerResults } from "danger";
+declare const danger: DangerDSLType;
+declare const message: (message: string, results?: DangerResults) => void;
+
+/**
+ * Check if pull request has not an excessive numbers of commits (if squashed)
+ *
+ * @dangerjs INFO
+ */
+export default function (): void {
+    const tooManyCommitThreshold: number = 2; // above this number of commits, squash commits is suggested
+    const prCommits: number = danger.github.commits.length;
+
+    if (prCommits > tooManyCommitThreshold) {
+        return message(
+            `You might consider squashing your ${prCommits} commits (simplifying branch history).`
+        );
+    }
+}

.github/dangerjs/prDescription.ts

@@ -0,0 +1,19 @@
+import { DangerDSLType, DangerResults } from "danger";
+declare const danger: DangerDSLType;
+declare const warn: (message: string, results?: DangerResults) => void;
+
+/**
+ * Check if pull request has has a sufficiently accurate description
+ *
+ * @dangerjs WARN
+ */
+export default function (): void {
+    const prDescription: string = danger.github.pr.body;
+    const shortPrDescriptionThreshold: number = 100; // Description is considered too short below this number of characters
+
+    if (prDescription.length < shortPrDescriptionThreshold) {
+        return warn(
+            "The PR description looks very brief, please check if more details can be added."
+        );
+    }
+}

.github/dangerjs/prInfoContributor.ts

@@ -0,0 +1,58 @@
+import { DangerDSLType } from "danger";
+declare const danger: DangerDSLType;
+
+interface Contributor {
+    login?: string;
+}
+
+const authorLogin = danger.github.pr.user.login;
+const messageKnownContributor: string = `
+***
+👋 **Hi ${authorLogin}**, thank you for your another contribution to \`espressif/esp-idf\` project!
+
+If the change is approved and passes the tests in our internal git repository, it will appear in this public Github repository on the next sync.
+***
+`;
+
+const messageFirstContributor: string = `
+***
+👋 **Welcome ${authorLogin}**, thank you for your first contribution to \`espressif/esp-idf\` project!
+
+📘 Please check [Contributions Guide](https://docs.espressif.com/projects/esp-idf/en/latest/esp32/contribute/index.html#contributions-guide) for the contribution checklist, information regarding code and documentation style, testing and other topics.
+
+🖊️ Please also make sure you have **read and signed** the [Contributor License Agreement for espressif/esp-idf project](https://cla-assistant.io/espressif/esp-idf).
+
+#### Pull request review and merge process you can expect
+Espressif develops the ESP-IDF project in an internal repository (Gitlab). We do welcome contributions in the form of bug reports, feature requests and pull requests via this public GitHub repository.
+
+1. An internal issue has been created for the PR, we assign it to the relevant engineer
+2. They review the PR and either approve it or ask you for changes or clarifications
+3. Once the Github PR is approved, we synchronize it into our internal git repository
+4. In the internal git repository we do the final review, collect approvals from core owners and make sure all the automated tests are passing
+    - At this point we may do some adjustments to the proposed change, or extend it by adding tests or documentation.
+5. If the change is approved and passes the tests it is merged into the \`master\` branch
+6. On next sync from the internal git repository merged change will appear in this public Github repository
+
+***
+`;
+
+/**
+ * Check whether the author of the pull request is known or a first-time contributor, and add a message to the PR with information about the review and merge process.
+ */
+export default async function (): Promise<string> {
+    const contributors = await danger.github.api.repos.listContributors({
+        owner: danger.github.thisPR.owner,
+        repo: danger.github.thisPR.repo,
+    });
+
+    const contributorsData: Contributor[] = contributors.data;
+    const knownContributors: (string | undefined)[] = contributorsData.map(
+        (contributor: Contributor) => contributor.login
+    );
+
+    if (knownContributors.includes(authorLogin)) {
+        return messageKnownContributor;
+    } else {
+        return messageFirstContributor;
+    }
+}

.github/dangerjs/prTargetBranch.ts

@@ -0,0 +1,19 @@
+import { DangerDSLType, DangerResults } from "danger";
+declare const danger: DangerDSLType;
+declare const fail: (message: string, results?: DangerResults) => void;
+
+/**
+ * Check if the target branch is "master"
+ *
+ * @dangerjs FAIL
+ */
+export default function (): void {
+    const prTargetBranch: string = danger.github?.pr?.base?.ref;
+
+    if (prTargetBranch !== "master") {
+        return fail(`
+        The target branch for this pull request should be \`master\`.\n
+        If you would like to add this feature to the release branch, please state this in the PR description and we will consider backporting it.
+        `);
+    }
+}

.github/dangerjs/tsconfig.json

@@ -0,0 +1,17 @@
+{
+    "compilerOptions": {
+        "module": "commonjs",
+        "moduleResolution": "node",
+        "esModuleInterop": true,
+        "target": "es6",
+        "noImplicitAny": true,
+        "noUnusedParameters": true,
+        "strictNullChecks": true,
+        "sourceMap": true,
+        "removeComments": true,
+        "outDir": "./dist"
+    },
+    "include": [
+        "./*.ts"
+    ]
+}

.github/dependabot.yml

@@ -0,0 +1,15 @@
+version: 2
+updates:
+  - package-ecosystem: "all"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    ignore:
+      - directory: ".gitlab/dangerjs"
+        patterns:
+          - "package-lock.json"
+      - directory: ".github/dangerjs"
+        patterns:
+          - "package-lock.json"
+    # Disable "version updates" (keep only "security updates")
+    open-pull-requests-limit: 0

.github/workflows/dangerjs.yml

@@ -9,19 +9,28 @@ permissions:
   contents: write
 
 jobs:
-  pull-request-style-linter:
+  danger-check:
     runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: .github/dangerjs
     steps:
     - name: Check out PR head
-      uses: actions/checkout@v4
+      uses: actions/checkout@v3
       with:
         ref: ${{ github.event.pull_request.head.sha }}
 
-    - name: DangerJS pull request linter
-      uses: espressif/shared-github-dangerjs@v1
+    - name: Setup NodeJS environment
+      uses: actions/setup-node@v3
+      with:
+        node-version: 18
+        cache: npm
+        cache-dependency-path: .github/dangerjs/package-lock.json
+
+    - name: Install DangerJS dependencies
+      run: npm install
+
+    - name: Run DangerJS
+      run: npx danger ci --failOnErrors -v
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      with:
-        instructions-gitlab-mirror: 'true'
-        instructions-contributions-file: 'CONTRIBUTING.md'
-        instructions-cla-link: 'https://cla-assistant.io/espressif/esp-idf'

.github/workflows/issue_comment.yml

@@ -12,7 +12,7 @@ jobs:
     name: Sync Issue Comments to Jira
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v2
       - name: Sync issue comments to JIRA
         uses: espressif/github-actions/sync_issues_to_jira@master
         env:

.github/workflows/new_issues.yml

@@ -12,7 +12,7 @@ jobs:
     name: Sync issues to Jira
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v2
       - name: Sync GitHub issues to Jira project
         uses: espressif/github-actions/sync_issues_to_jira@master
         env:

.github/workflows/new_prs.yml

@@ -15,7 +15,7 @@ jobs:
     name: Sync PRs to Jira
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v2
       - name: Sync PRs to Jira project
         uses: espressif/github-actions/sync_issues_to_jira@master
         with:

.github/workflows/pr_approved.yml

@@ -11,7 +11,7 @@ jobs:
         (github.event.label.name == 'PR-Sync-Rebase')  ||
         (github.event.label.name == 'PR-Sync-Update')
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v2
       - name: Sync approved PRs to internal codebase
         uses: espressif/github-actions/github_pr_to_internal_pr@master
         env:

.github/workflows/pre_commit_check.yml

@@ -14,7 +14,7 @@ jobs:
       SKIP: "cleanup-ignore-lists"  # Comma-separated string of ignored pre-commit check IDs
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v2
       - name: Fetch head and base refs
         # This is necessary for pre-commit to check the changes in the PR branch
         run: |
@@ -40,5 +40,3 @@ jobs:
             echo ""
             exit 1
           fi
-          # Run pre-commit for PowerShell scripts check
-          pre-commit run --hook-stage manual check-powershell-scripts --from-ref base_ref --to-ref pr_ref --show-diff-on-failure

.github/workflows/vulnerability_scan.yml

@@ -1,34 +0,0 @@
-name: Vulnerability scan
-
-on:
-  schedule:
-    - cron: '0 0 * * *'
-  workflow_dispatch:
-
-jobs:
-  vulnerability-scan:
-    strategy:
-      # We don't want to run all jobs in parallel, because this would
-      # overload NVD and we would get 503
-      max-parallel: 1
-      matrix:
-        # References/branches which should be scanned for vulnerabilities are
-        # defined in the VULNERABILITY_SCAN_REFS variable as json list.
-        # For example: ['master', 'release/v5.2', 'release/v5.1', 'release/v5.0', 'release/v4.4']
-        ref: ${{ fromJSON(vars.VULNERABILITY_SCAN_REFS) }}
-    name: Vulnerability scan
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-        with:
-          submodules: recursive
-          ref: ${{ matrix.ref }}
-
-      - name: Vulnerability scan
-        env:
-          SBOM_MATTERMOST_WEBHOOK: ${{ secrets.SBOM_MATTERMOST_WEBHOOK }}
-          NVDAPIKEY: ${{ secrets.NVDAPIKEY }}
-        uses: espressif/esp-idf-sbom-action@master
-        with:
-          ref: ${{ matrix.ref }}

.gitignore

@@ -24,14 +24,6 @@ GPATH
 # cache dir
 .cache/
 
-# Doc build artifacts
-docs/_build/
-docs/doxygen_sqlite3.db
-
-# Downloaded font files
-docs/_static/DejaVuSans.ttf
-docs/_static/NotoSansSC-Regular.otf
-
 # Components Unit Test Apps files
 components/**/build/
 components/**/build_*_*/
@@ -40,15 +32,28 @@ components/**/sdkconfig.old
 
 # Example project files
 examples/**/build/
-examples/**/build_*_*/
+examples/**/build_esp*_*/
 examples/**/sdkconfig
 examples/**/sdkconfig.old
 
+# Doc build artifacts
+docs/_build/
+docs/doxygen_sqlite3.db
+
+# Downloaded font files
+docs/_static/DejaVuSans.ttf
+docs/_static/NotoSansSC-Regular.otf
+
 # Unit test app files
-tools/unit-test-app/build
-tools/unit-test-app/build_*_*/
 tools/unit-test-app/sdkconfig
 tools/unit-test-app/sdkconfig.old
+tools/unit-test-app/build
+tools/unit-test-app/build_*_*/
+tools/unit-test-app/output
+tools/unit-test-app/test_configs
+
+# Unit Test CMake compile log folder
+log_ut_cmake
 
 # test application build files
 tools/test_apps/**/build/
@@ -56,8 +61,7 @@ tools/test_apps/**/build_*_*/
 tools/test_apps/**/sdkconfig
 tools/test_apps/**/sdkconfig.old
 
-TEST_LOGS/
-build_summary_*.xml
+TEST_LOGS
 
 # gcov coverage reports
 *.gcda
@@ -96,13 +100,9 @@ dependencies.lock
 managed_components
 
 # pytest log
-pytest-embedded/
-# legacy one
 pytest_embedded_log/
-list_job*.txt
-size_info*.txt
-XUNIT_RESULT*.xml
-.manifest_sha
+list_job_*.txt
+size_info.txt
 
 # clang config (for LSP)
 .clangd

.gitlab-ci.yml

@@ -18,18 +18,14 @@ workflow:
 # Place the default settings in `.gitlab/ci/common.yml` instead
 
 include:
-  - '.gitlab/ci/danger.yml'
   - '.gitlab/ci/common.yml'
   - '.gitlab/ci/rules.yml'
   - '.gitlab/ci/upload_cache.yml'
   - '.gitlab/ci/docs.yml'
   - '.gitlab/ci/static-code-analysis.yml'
-  - '.gitlab/ci/pre_commit.yml'
   - '.gitlab/ci/pre_check.yml'
   - '.gitlab/ci/build.yml'
   - '.gitlab/ci/integration_test.yml'
   - '.gitlab/ci/host-test.yml'
+  - '.gitlab/ci/target-test.yml'
   - '.gitlab/ci/deploy.yml'
-  - '.gitlab/ci/post_deploy.yml'
-  - '.gitlab/ci/retry_failed_jobs.yml'
-  - '.gitlab/ci/test-win.yml'

.gitlab/CODEOWNERS

@@ -2,248 +2,5 @@
 #
 #    https://docs.gitlab.com/ee/user/project/code_owners.html#the-syntax-of-code-owners-files
 #
-# If more than one rule matches a given file, the latest rule is used.
-# The file should be generally kept sorted, except when it is necessary
-# to use a different order due to the fact above. In that case, use
-# '# sort-order-reset' comment line to reset the sort order.
-#
-# Recipes for a few common cases:
-#
-# 1. Specific directory with all its contents:
-#
-#    /components/app_trace/
-#
-#    Note the trailing slash!
-#
-# 2. File with certain extension in any subdirectory of a certain directory:
-#
-#    /examples/**/*.py
-#
-#    This includes an *.py files in /examples/ directory as well.
-#
-# 3. Contents of a directory with a certain name, anywhere in the tree:
-#
-#    test_*_host/
-#
-#    Will match everything under components/efuse/test_efuse_host/,
-#    components/heap/test_multi_heap_host/, components/lwip/test_afl_host/, etc.
-#
-# 4. Same as above, except limited to a specific place in the tree:
-#
-#    /components/esp32*/
-#
-#    Matches everything under /components/esp32, /components/esp32s2, etc.
-#    Doesn't match /tools/some-test/components/esp32s5.
-#
-# 5. Specific file:
-#
-#    /tools/tools.json
-#
-# 6. File with a certain name anywhere in the tree
-#
-#    .gitignore
-#
 
-* @esp-idf-codeowners/other
-
-/.*                                   @esp-idf-codeowners/tools
-/.codespellrc                         @esp-idf-codeowners/ci
-/.github/workflows/                   @esp-idf-codeowners/ci
-/.gitlab-ci.yml                       @esp-idf-codeowners/ci
-/.gitlab/ci/                          @esp-idf-codeowners/ci
-/.pre-commit-config.yaml              @esp-idf-codeowners/ci
-/.readthedocs.yml                     @esp-idf-codeowners/docs
-/.vale.ini                            @esp-idf-codeowners/docs
-/CMakeLists.txt                       @esp-idf-codeowners/build-config
-/COMPATIBILITY*.md                    @esp-idf-codeowners/peripherals
-/CONTRIBUTING.md                      @esp-idf-codeowners/docs
-/Kconfig                              @esp-idf-codeowners/build-config
-/README*.md                           @esp-idf-codeowners/docs
-/ROADMAP*.md                          @esp-idf-codeowners/docs
-/SUPPORT_POLICY*.md                   @esp-idf-codeowners/docs
-/add_path.sh                          @esp-idf-codeowners/tools
-/conftest.py                          @esp-idf-codeowners/ci
-/export.*                             @esp-idf-codeowners/tools
-/install.*                            @esp-idf-codeowners/tools
-/pytest.ini                           @esp-idf-codeowners/ci
-/sdkconfig.rename                     @esp-idf-codeowners/build-config
-/sonar-project.properties             @esp-idf-codeowners/ci
-
-# sort-order-reset
-
-/components/app_trace/                @esp-idf-codeowners/debugging
-/components/app_update/               @esp-idf-codeowners/system @esp-idf-codeowners/app-utilities
-/components/bootloader*/              @esp-idf-codeowners/system @esp-idf-codeowners/security
-/components/bootloader_support/bootloader_flash/    @esp-idf-codeowners/peripherals
-/components/bt/                       @esp-idf-codeowners/bluetooth
-/components/cmock/                    @esp-idf-codeowners/system
-/components/console/                  @esp-idf-codeowners/system @esp-idf-codeowners/app-utilities/console
-/components/cxx/                      @esp-idf-codeowners/system
-/components/driver/                   @esp-idf-codeowners/peripherals
-/components/efuse/                    @esp-idf-codeowners/system
-/components/esp_adc/                  @esp-idf-codeowners/peripherals
-/components/esp_app_format/           @esp-idf-codeowners/system @esp-idf-codeowners/app-utilities
-/components/esp_bootloader_format/    @esp-idf-codeowners/system @esp-idf-codeowners/app-utilities
-/components/esp_coex/                 @esp-idf-codeowners/wifi @esp-idf-codeowners/bluetooth @esp-idf-codeowners/ieee802154
-/components/esp_common/               @esp-idf-codeowners/system
-/components/esp_driver_*/             @esp-idf-codeowners/peripherals
-/components/esp_driver_sdmmc/         @esp-idf-codeowners/peripherals @esp-idf-codeowners/storage
-/components/esp_eth/                  @esp-idf-codeowners/network
-/components/esp_event/                @esp-idf-codeowners/system
-/components/esp_gdbstub/              @esp-idf-codeowners/debugging
-/components/esp_hid/                  @esp-idf-codeowners/bluetooth
-/components/esp_http_client/          @esp-idf-codeowners/app-utilities
-/components/esp_http_server/          @esp-idf-codeowners/app-utilities
-/components/esp_https_ota/            @esp-idf-codeowners/app-utilities
-/components/esp_https_server/         @esp-idf-codeowners/app-utilities
-/components/esp_hw_support/           @esp-idf-codeowners/system @esp-idf-codeowners/peripherals
-/components/esp_hw_support/lowpower/  @esp-idf-codeowners/power-management
-/components/esp_lcd/                  @esp-idf-codeowners/peripherals
-/components/esp_local_ctrl/           @esp-idf-codeowners/app-utilities
-/components/esp_mm/                   @esp-idf-codeowners/peripherals
-/components/esp_netif/                @esp-idf-codeowners/network
-/components/esp_netif_stack/          @esp-idf-codeowners/network
-/components/esp_partition/            @esp-idf-codeowners/storage
-/components/esp_phy/                  @esp-idf-codeowners/bluetooth @esp-idf-codeowners/wifi @esp-idf-codeowners/ieee802154
-/components/esp_pm/                   @esp-idf-codeowners/power-management @esp-idf-codeowners/bluetooth @esp-idf-codeowners/wifi @esp-idf-codeowners/system
-/components/esp_psram/                @esp-idf-codeowners/peripherals @esp-idf-codeowners/system
-/components/esp_ringbuf/              @esp-idf-codeowners/system
-/components/esp_rom/                  @esp-idf-codeowners/system @esp-idf-codeowners/bluetooth @esp-idf-codeowners/wifi
-/components/esp_security/             @esp-idf-codeowners/security
-/components/esp_system/               @esp-idf-codeowners/system
-/components/esp_timer/                @esp-idf-codeowners/system
-/components/esp-tls/                  @esp-idf-codeowners/app-utilities
-/components/esp_vfs_*/                @esp-idf-codeowners/storage
-/components/esp_vfs_console/          @esp-idf-codeowners/storage @esp-idf-codeowners/system
-/components/esp_wifi/                 @esp-idf-codeowners/wifi
-/components/espcoredump/              @esp-idf-codeowners/debugging
-/components/esptool_py/               @esp-idf-codeowners/tools
-/components/fatfs/                    @esp-idf-codeowners/storage
-/components/freertos/                 @esp-idf-codeowners/system
-/components/hal/                      @esp-idf-codeowners/peripherals
-/components/hal/test_apps/crypto/     @esp-idf-codeowners/peripherals @esp-idf-codeowners/security
-/components/heap/                     @esp-idf-codeowners/system
-/components/http_parser/              @esp-idf-codeowners/app-utilities
-/components/idf_test/                 @esp-idf-codeowners/ci
-/components/ieee802154/               @esp-idf-codeowners/ieee802154
-/components/json/                     @esp-idf-codeowners/app-utilities
-/components/linux/                    @esp-idf-codeowners/system
-/components/log/                      @esp-idf-codeowners/system
-/components/lwip/                     @esp-idf-codeowners/lwip
-/components/mbedtls/                  @esp-idf-codeowners/app-utilities/mbedtls @esp-idf-codeowners/security
-/components/mqtt/                     @esp-idf-codeowners/network
-/components/newlib/                   @esp-idf-codeowners/system @esp-idf-codeowners/toolchain
-/components/nvs_flash/                @esp-idf-codeowners/storage
-/components/nvs_sec_provider/         @esp-idf-codeowners/storage @esp-idf-codeowners/security
-/components/openthread/               @esp-idf-codeowners/ieee802154
-/components/partition_table/          @esp-idf-codeowners/system
-/components/perfmon/                  @esp-idf-codeowners/debugging
-/components/protobuf-c/               @esp-idf-codeowners/app-utilities
-/components/protocomm/                @esp-idf-codeowners/app-utilities/provisioning
-/components/pthread/                  @esp-idf-codeowners/system
-/components/riscv/                    @esp-idf-codeowners/system
-/components/rt/                       @esp-idf-codeowners/system
-/components/sdmmc/                    @esp-idf-codeowners/storage
-/components/soc/                      @esp-idf-codeowners/peripherals @esp-idf-codeowners/system
-/components/spi_flash/                @esp-idf-codeowners/peripherals
-/components/spiffs/                   @esp-idf-codeowners/storage
-/components/tcp_transport/            @esp-idf-codeowners/network
-/components/touch_element/            @esp-idf-codeowners/peripherals
-/components/ulp/                      @esp-idf-codeowners/system
-/components/unity/                    @esp-idf-codeowners/ci
-/components/usb/                      @esp-idf-codeowners/peripherals/usb
-/components/vfs/                      @esp-idf-codeowners/storage
-/components/wear_levelling/           @esp-idf-codeowners/storage
-/components/wifi_provisioning/        @esp-idf-codeowners/app-utilities/provisioning
-/components/wpa_supplicant/           @esp-idf-codeowners/wifi @esp-idf-codeowners/app-utilities/mbedtls
-/components/xtensa/                   @esp-idf-codeowners/system
-
-/docs/                                      @esp-idf-codeowners/docs
-/docs/docs_not_updated/                     @esp-idf-codeowners/all-maintainers
-/docs/**/api-guides/tools/                  @esp-idf-codeowners/tools
-/docs/en/api-guides/core_dump.rst           @esp-idf-codeowners/debugging
-/docs/**/api-guides/wifi*                   @esp-idf-codeowners/wifi
-/docs/**/api-guides/esp-wifi-mesh.rst       @esp-idf-codeowners/wifi
-/docs/en/api-guides/jtag-debugging/         @esp-idf-codeowners/debugging
-/docs/**/api-reference/bluetooth/           @esp-idf-codeowners/bluetooth
-/docs/**/api-reference/network/             @esp-idf-codeowners/network @esp-idf-codeowners/wifi
-/docs/**/api-reference/peripherals/         @esp-idf-codeowners/peripherals
-/docs/**/api-reference/peripherals/usb*     @esp-idf-codeowners/peripherals @esp-idf-codeowners/peripherals/usb
-/docs/**/api-reference/peripherals/usb*/    @esp-idf-codeowners/peripherals @esp-idf-codeowners/peripherals/usb
-/docs/**/api-reference/protocols/           @esp-idf-codeowners/network @esp-idf-codeowners/app-utilities
-/docs/**/api-reference/provisioning/        @esp-idf-codeowners/app-utilities/provisioning
-/docs/**/api-reference/storage/             @esp-idf-codeowners/storage
-/docs/**/api-reference/system/              @esp-idf-codeowners/system
-/docs/**/security/                          @esp-idf-codeowners/security
-/docs/**/migration-guides/                  @esp-idf-codeowners/docs @esp-idf-codeowners/all-maintainers
-/docs/**/contribute/install-pre-commit-hook.rst @esp-idf-codeowners/ci @esp-idf-codeowners/tools
-
-/examples/README.md           @esp-idf-codeowners/docs @esp-idf-codeowners/ci
-/examples/**/*.py             @esp-idf-codeowners/ci @esp-idf-codeowners/tools
-/examples/bluetooth/          @esp-idf-codeowners/bluetooth
-/examples/build_system/       @esp-idf-codeowners/build-config
-/examples/common_components/  @esp-idf-codeowners/system @esp-idf-codeowners/wifi @esp-idf-codeowners/lwip @esp-idf-codeowners/network
-/examples/custom_bootloader/  @esp-idf-codeowners/system
-/examples/cxx/                @esp-idf-codeowners/system
-/examples/ethernet/           @esp-idf-codeowners/network
-/examples/get-started/        @esp-idf-codeowners/system
-/examples/ieee802154/         @esp-idf-codeowners/ieee802154
-/examples/mesh/               @esp-idf-codeowners/wifi
-/examples/network/            @esp-idf-codeowners/network @esp-idf-codeowners/wifi
-/examples/openthread/         @esp-idf-codeowners/ieee802154
-/examples/peripherals/        @esp-idf-codeowners/peripherals
-/examples/peripherals/usb/    @esp-idf-codeowners/peripherals @esp-idf-codeowners/peripherals/usb
-/examples/phy/                @esp-idf-codeowners/bluetooth @esp-idf-codeowners/wifi @esp-idf-codeowners/ieee802154
-/examples/protocols/          @esp-idf-codeowners/network @esp-idf-codeowners/app-utilities
-/examples/provisioning/       @esp-idf-codeowners/app-utilities/provisioning
-/examples/security/           @esp-idf-codeowners/security
-/examples/storage/            @esp-idf-codeowners/storage
-/examples/system/             @esp-idf-codeowners/system
-/examples/system/ota/         @esp-idf-codeowners/app-utilities
-/examples/wifi/               @esp-idf-codeowners/wifi
-/examples/zigbee/             @esp-idf-codeowners/ieee802154
-
-/tools/                       @esp-idf-codeowners/tools
-/tools/ble/                   @esp-idf-codeowners/app-utilities
-/tools/catch/                 @esp-idf-codeowners/ci
-/tools/ci/                    @esp-idf-codeowners/ci
-/tools/cmake/                 @esp-idf-codeowners/build-config
-/tools/cmake/toolchain-*.cmake      @esp-idf-codeowners/toolchain
-/tools/esp_app_trace/         @esp-idf-codeowners/debugging
-/tools/esp_prov/              @esp-idf-codeowners/app-utilities
-/tools/gdb_panic_server.py    @esp-idf-codeowners/debugging
-/tools/kconfig*/              @esp-idf-codeowners/build-config
-/tools/ldgen/                 @esp-idf-codeowners/build-config
-/tools/mass_mfg/              @esp-idf-codeowners/app-utilities
-/tools/mocks/                 @esp-idf-codeowners/system
-
-/tools/test_apps/                    @esp-idf-codeowners/ci
-/tools/test_apps/README.md           @esp-idf-codeowners/docs @esp-idf-codeowners/ci
-
-## Note: owners here should be the same as the owners for the same example subdir, above
-/tools/test_apps/build_system/       @esp-idf-codeowners/build-config
-/tools/test_apps/configs/            @esp-idf-codeowners/system
-/tools/test_apps/linux_compatible/   @esp-idf-codeowners/system
-/tools/test_apps/peripherals/        @esp-idf-codeowners/peripherals
-/tools/test_apps/phy/                @esp-idf-codeowners/bluetooth @esp-idf-codeowners/wifi @esp-idf-codeowners/ieee802154
-/tools/test_apps/protocols/          @esp-idf-codeowners/network @esp-idf-codeowners/app-utilities
-/tools/test_apps/security/           @esp-idf-codeowners/security
-/tools/test_apps/storage/            @esp-idf-codeowners/storage
-/tools/test_apps/system/             @esp-idf-codeowners/system
-
-/tools/test_apps/**/*.py             @esp-idf-codeowners/ci @esp-idf-codeowners/tools
-
-/tools/test_build_system/     @esp-idf-codeowners/tools @esp-idf-codeowners/build-config
-
-/tools/tools.json             @esp-idf-codeowners/tools @esp-idf-codeowners/toolchain @esp-idf-codeowners/debugging
-
-/tools/unit-test-app/         @esp-idf-codeowners/system @esp-idf-codeowners/tools
-
-# sort-order-reset
-
-/components/**/test_apps/**/*.py     @esp-idf-codeowners/ci @esp-idf-codeowners/tools
-
-# ignore lists
-/tools/ci/check_copyright_config.yaml                   @esp-idf-codeowners/all-maintainers
-/tools/ci/check_copyright_ignore.txt                    @esp-idf-codeowners/all-maintainers
-/tools/ci/mypy_ignore_list.txt                          @esp-idf-codeowners/tools
+* @esp-idf-codeowners/all-maintainers

.gitlab/ci/build.yml

@@ -1,7 +1,7 @@
 .build_template:
   stage: build
   extends:
-    - .after_script:build:ccache-show-stats:upload-failed-job-logs
+    - .after_script:build:ccache
   image: $ESP_ENV_IMAGE
   tags:
     - build
@@ -12,11 +12,11 @@
     IDF_CCACHE_ENABLE: "1"
   dependencies: []
 
-.build_cmake_clang_template:
+.build_cmake_template:
   extends:
     - .build_template
     - .before_script:build
-    - .after_script:build:ccache-show-stats
+    - .after_script:build:ccache
   dependencies: # set dependencies to null to avoid missing artifacts issue
   needs:
     - job: fast_template_app
@@ -32,13 +32,8 @@
       # keep the size info to help track the binary size
       - size_info.txt
       - "**/build*/size.json"
-    expire_in: 1 week
     when: always
-  variables:
-    IDF_TOOLCHAIN: clang
-    TEST_BUILD_OPTS_EXTRA: ""
-    TEST_DIR: tools/test_apps/system/clang_build_test
-    PYTEST_IGNORE_COLLECT_IMPORT_ERROR: "1"
+    expire_in: 4 days
   script:
     # CI specific options start from "--parallel-count xxx". could ignore when running locally
     - run_cmd python tools/ci/ci_build_apps.py $TEST_DIR -v
@@ -46,14 +41,304 @@
       --copy-sdkconfig
       --parallel-count ${CI_NODE_TOTAL:-1}
       --parallel-index ${CI_NODE_INDEX:-1}
-      --modified-components ${MR_MODIFIED_COMPONENTS}
-      --modified-files ${MR_MODIFIED_FILES}
+      --extra-preserve-dirs
+        examples/bluetooth/esp_ble_mesh/ble_mesh_console
+        examples/bluetooth/hci/controller_hci_uart_esp32
+        examples/wifi/iperf
+      --modified-components ${MODIFIED_COMPONENTS}
+      --modified-files ${MODIFIED_FILES}
+    # for detailed documents, please refer to .gitlab/ci/README.md#uploaddownload-artifacts-to-internal-minio-server
+    - python tools/ci/artifacts_handler.py upload
+
+.build_cmake_clang_template:
+  extends:
+    - .build_cmake_template
+  variables:
+    IDF_TOOLCHAIN: clang
+    TEST_BUILD_OPTS_EXTRA: ""
+    TEST_DIR: tools/test_apps/system/cxx_pthread_bluetooth
+  script:
+    # CI specific options start from "--parallel-count xxx". could ignore when running locally
+    - run_cmd python tools/ci/ci_build_apps.py $TEST_DIR -v
+      -t $IDF_TARGET
+      --copy-sdkconfig
+      --parallel-count ${CI_NODE_TOTAL:-1}
+      --parallel-index ${CI_NODE_INDEX:-1}
+      --modified-components ${MODIFIED_COMPONENTS}
+      --modified-files ${MODIFIED_FILES}
       $TEST_BUILD_OPTS_EXTRA
     - python tools/ci/artifacts_handler.py upload
 
-######################
-# build_template_app #
-######################
+.build_pytest_template:
+  extends:
+    - .build_cmake_template
+  script:
+    # CI specific options start from "--parallel-count xxx". could ignore when running locally
+    - run_cmd python tools/ci/ci_build_apps.py $TEST_DIR -v
+      -t $IDF_TARGET
+      -m \"not host_test\"
+      --pytest-apps
+      --parallel-count ${CI_NODE_TOTAL:-1}
+      --parallel-index ${CI_NODE_INDEX:-1}
+      --collect-app-info "list_job_${CI_JOB_NAME_SLUG}.txt"
+      --modified-components ${MODIFIED_COMPONENTS}
+      --modified-files ${MODIFIED_FILES}
+    - python tools/ci/artifacts_handler.py upload
+
+.build_pytest_no_jtag_template:
+  extends:
+    - .build_cmake_template
+  script:
+    # CI specific options start from "--parallel-count xxx". could ignore when running locally
+    - run_cmd python tools/ci/ci_build_apps.py $TEST_DIR -v
+      -t $IDF_TARGET
+      -m \"not host_test and not jtag\"
+      --pytest-apps
+      --parallel-count ${CI_NODE_TOTAL:-1}
+      --parallel-index ${CI_NODE_INDEX:-1}
+      --collect-app-info "list_job_${CI_JOB_NAME_SLUG}.txt"
+      --modified-components ${MODIFIED_COMPONENTS}
+      --modified-files ${MODIFIED_FILES}
+    - python tools/ci/artifacts_handler.py upload
+
+.build_pytest_jtag_template:
+  extends:
+    - .build_cmake_template
+  script:
+    # CI specific options start from "--parallel-count xxx". could ignore when running locally
+    - run_cmd python tools/ci/ci_build_apps.py $TEST_DIR -v
+      -t $IDF_TARGET
+      -m \"not host_test and jtag\"
+      --pytest-apps
+      --parallel-count ${CI_NODE_TOTAL:-1}
+      --parallel-index ${CI_NODE_INDEX:-1}
+      --collect-app-info "list_job_${CI_JOB_NAME_SLUG}.txt"
+      --modified-components ${MODIFIED_COMPONENTS}
+      --modified-files ${MODIFIED_FILES}
+    - python tools/ci/artifacts_handler.py upload
+
+build_pytest_examples_esp32:
+  extends:
+    - .build_pytest_no_jtag_template
+    - .rules:build:example_test-esp32
+  parallel: 6
+  variables:
+    IDF_TARGET: esp32
+    TEST_DIR: examples
+
+build_pytest_examples_esp32s2:
+  extends:
+    - .build_pytest_no_jtag_template
+    - .rules:build:example_test-esp32s2
+  parallel: 3
+  variables:
+    IDF_TARGET: esp32s2
+    TEST_DIR: examples
+
+build_pytest_examples_esp32s3:
+  extends:
+    - .build_pytest_no_jtag_template
+    - .rules:build:example_test-esp32s3
+  parallel: 4
+  variables:
+    IDF_TARGET: esp32s3
+    TEST_DIR: examples
+
+build_pytest_examples_esp32c3:
+  extends:
+    - .build_pytest_no_jtag_template
+    - .rules:build:example_test-esp32c3
+  parallel: 4
+  variables:
+    IDF_TARGET: esp32c3
+    TEST_DIR: examples
+
+build_pytest_examples_esp32c2:
+  extends:
+    - .build_pytest_no_jtag_template
+    - .rules:build:example_test-esp32c2
+  parallel: 2
+  variables:
+    IDF_TARGET: esp32c2
+    TEST_DIR: examples
+
+build_pytest_examples_esp32c6:
+  extends:
+    - .build_pytest_no_jtag_template
+    - .rules:build:example_test-esp32c6
+  parallel: 2
+  variables:
+    IDF_TARGET: esp32c6
+    TEST_DIR: examples
+
+build_pytest_examples_esp32h2:
+  extends:
+    - .build_pytest_no_jtag_template
+    - .rules:build:example_test-esp32h2
+  parallel: 2
+  variables:
+    IDF_TARGET: esp32h2
+    TEST_DIR: examples
+
+build_pytest_examples_jtag:  # for all targets
+  extends:
+    - .build_pytest_jtag_template
+    - .rules:build:example_test
+  variables:
+    IDF_TARGET: all
+    TEST_DIR: examples
+
+build_pytest_components_esp32:
+  extends:
+    - .build_pytest_template
+    - .rules:build:component_ut-esp32
+  parallel: 5
+  variables:
+    IDF_TARGET: esp32
+    TEST_DIR: components
+
+build_pytest_components_esp32s2:
+  extends:
+    - .build_pytest_template
+    - .rules:build:component_ut-esp32s2
+  parallel: 4
+  variables:
+    IDF_TARGET: esp32s2
+    TEST_DIR: components
+
+build_pytest_components_esp32s3:
+  extends:
+    - .build_pytest_template
+    - .rules:build:component_ut-esp32s3
+  parallel: 4
+  variables:
+    IDF_TARGET: esp32s3
+    TEST_DIR: components
+
+build_pytest_components_esp32c3:
+  extends:
+    - .build_pytest_template
+    - .rules:build:component_ut-esp32c3
+  parallel: 4
+  variables:
+    IDF_TARGET: esp32c3
+    TEST_DIR: components
+
+build_pytest_components_esp32c2:
+  extends:
+    - .build_pytest_template
+    - .rules:build:component_ut-esp32c2
+  parallel: 3
+  variables:
+    IDF_TARGET: esp32c2
+    TEST_DIR: components
+
+build_pytest_components_esp32c6:
+  extends:
+    - .build_pytest_template
+    - .rules:build:component_ut-esp32c6
+  parallel: 3
+  variables:
+    IDF_TARGET: esp32c6
+    TEST_DIR: components
+
+build_pytest_components_esp32h2:
+  extends:
+    - .build_pytest_template
+    - .rules:build:component_ut-esp32h2
+  parallel: 4
+  variables:
+    IDF_TARGET: esp32h2
+    TEST_DIR: components
+
+build_only_components_apps:
+  extends:
+    - .build_cmake_template
+    - .rules:build:component_ut
+  parallel: 5
+  script:
+    - set_component_ut_vars
+    # CI specific options start from "--parallel-count xxx". could ignore when running locally
+    - run_cmd python tools/ci/ci_build_apps.py $COMPONENT_UT_DIRS -v
+      -t all
+      --parallel-count ${CI_NODE_TOTAL:-1}
+      --parallel-index ${CI_NODE_INDEX:-1}
+      --modified-components ${MODIFIED_COMPONENTS}
+      --modified-files ${MODIFIED_FILES}
+    - python tools/ci/artifacts_handler.py upload
+
+build_pytest_test_apps_esp32:
+  extends:
+    - .build_pytest_template
+    - .rules:build:custom_test-esp32
+  variables:
+    IDF_TARGET: esp32
+    TEST_DIR: tools/test_apps
+
+build_pytest_test_apps_esp32s2:
+  extends:
+    - .build_pytest_template
+    - .rules:build:custom_test-esp32s2
+  variables:
+    IDF_TARGET: esp32s2
+    TEST_DIR: tools/test_apps
+
+build_pytest_test_apps_esp32s3:
+  extends:
+    - .build_pytest_template
+    - .rules:build:custom_test-esp32s3
+  parallel: 2
+  variables:
+    IDF_TARGET: esp32s3
+    TEST_DIR: tools/test_apps
+
+build_pytest_test_apps_esp32c3:
+  extends:
+    - .build_pytest_template
+    - .rules:build:custom_test-esp32c3
+  variables:
+    IDF_TARGET: esp32c3
+    TEST_DIR: tools/test_apps
+
+build_pytest_test_apps_esp32c2:
+  extends:
+    - .build_pytest_template
+    - .rules:build:custom_test-esp32c2
+  variables:
+    IDF_TARGET: esp32c2
+    TEST_DIR: tools/test_apps
+
+build_pytest_test_apps_esp32c6:
+  extends:
+    - .build_pytest_template
+    - .rules:build:custom_test-esp32c6
+  variables:
+    IDF_TARGET: esp32c6
+    TEST_DIR: tools/test_apps
+
+build_pytest_test_apps_esp32h2:
+  extends:
+    - .build_pytest_template
+    - .rules:build:custom_test-esp32h2
+  variables:
+    IDF_TARGET: esp32h2
+    TEST_DIR: tools/test_apps
+
+build_only_tools_test_apps:
+  extends:
+    - .build_cmake_template
+    - .rules:build:custom_test
+  parallel: 9
+  script:
+    # CI specific options start from "--parallel-count xxx". could ignore when running locally
+    - run_cmd python tools/ci/ci_build_apps.py tools/test_apps -v
+      -t all
+      --parallel-count ${CI_NODE_TOTAL:-1}
+      --parallel-index ${CI_NODE_INDEX:-1}
+      --modified-components ${MODIFIED_COMPONENTS}
+      --modified-files ${MODIFIED_FILES}
+    - python tools/ci/artifacts_handler.py upload
+
 .build_template_app_template:
   extends:
     - .build_template
@@ -65,12 +350,12 @@
     BUILD_LOG_CMAKE: "${LOG_PATH}/cmake_@t_@w.txt"
     BUILD_COMMAND_ARGS: ""
   artifacts:
+    when: always
     paths:
       - log_template_app/*
       - size_info.txt
       - build_template_app/**/size.json
     expire_in: 1 week
-    when: always
   script:
     # Set the variable for 'esp-idf-template' testing
     - ESP_IDF_TEMPLATE_GIT=${ESP_IDF_TEMPLATE_GIT:-"https://github.com/espressif/esp-idf-template.git"}
@@ -89,48 +374,100 @@ fast_template_app:
     - .build_template_app_template
     - .rules:build:target_test
   stage: pre_check
-  tags: [fast_run, shiny]
   variables:
     BUILD_COMMAND_ARGS: "-p"
 #------------------------------------------------------------------------------
 
-#######################
-# gnu_static_analyzer #
-#######################
-gcc_static_analyzer:
+build_examples_cmake_esp32:
   extends:
-    - .build_template_app_template
-    - .rules:build:target_test
-  stage: pre_check
-  tags: [build, shiny]
+    - .build_cmake_template
+    - .rules:build:example_test-esp32
+  parallel: 8
   variables:
-    CI_CCACHE_DISABLE: 1
-    ANALYZING_APP: "examples/get-started/hello_world"
-  script:
-    - echo "CONFIG_COMPILER_STATIC_ANALYZER=y" >> ${ANALYZING_APP}/sdkconfig.defaults
-    - python -m idf_build_apps build -v -p ${ANALYZING_APP} -t all
+    IDF_TARGET: esp32
+    TEST_DIR: examples
+
+build_examples_cmake_esp32s2:
+  extends:
+    - .build_cmake_template
+    - .rules:build:example_test-esp32s2
+  parallel: 7
+  variables:
+    IDF_TARGET: esp32s2
+    TEST_DIR: examples
+
+build_examples_cmake_esp32s3:
+  extends:
+    - .build_cmake_template
+    - .rules:build:example_test-esp32s3
+  parallel: 11
+  variables:
+    IDF_TARGET: esp32s3
+    TEST_DIR: examples
+
+build_examples_cmake_esp32c2:
+  extends:
+    - .build_cmake_template
+    - .rules:build:example_test-esp32c2
+  parallel: 7
+  variables:
+    IDF_TARGET: esp32c2
+    TEST_DIR: examples
+
+build_examples_cmake_esp32c3:
+  extends:
+    - .build_cmake_template
+    - .rules:build:example_test-esp32c3
+  parallel: 9
+  variables:
+    IDF_TARGET: esp32c3
+    TEST_DIR: examples
+
+build_examples_cmake_esp32c6:
+  extends:
+    - .build_cmake_template
+    - .rules:build:example_test-esp32c6
+  parallel: 11
+  variables:
+    IDF_TARGET: esp32c6
+    TEST_DIR: examples
+
+build_examples_cmake_esp32h2:
+  extends:
+    - .build_cmake_template
+    - .rules:build:example_test-esp32h2
+  parallel: 9
+  variables:
+    IDF_TARGET: esp32h2
+    TEST_DIR: examples
+
+build_examples_cmake_esp32p4:
+  extends:
+    - .build_cmake_template
+    - .rules:build:example_test-esp32p4
+  parallel: 4
+  variables:
+    IDF_TARGET: esp32p4
+    TEST_DIR: examples
 
-########################################
-# Clang Build Apps Without Tests Cases #
-########################################
 build_clang_test_apps_esp32:
   extends:
     - .build_cmake_clang_template
-    - .rules:build
+    - .rules:build:custom_test-esp32
   variables:
     IDF_TARGET: esp32
 
 build_clang_test_apps_esp32s2:
   extends:
     - .build_cmake_clang_template
-    - .rules:build
+    - .rules:build:custom_test-esp32s2
   variables:
     IDF_TARGET: esp32s2
 
 build_clang_test_apps_esp32s3:
   extends:
     - .build_cmake_clang_template
-    - .rules:build
+    - .rules:build:custom_test-esp32s3
   variables:
     IDF_TARGET: esp32s3
 
@@ -138,106 +475,81 @@ build_clang_test_apps_esp32s3:
   extends:
     - .build_cmake_clang_template
   variables:
-    # https://reviews.llvm.org/D90108.
-    # GNU 'as' lets .weak override .globl since binutils-gdb
-    # https://github.com/bminor/binutils-gdb/commit/5ca547dc2399a0a5d9f20626d4bf5547c3ccfddd (1996)
-    # while MC lets the last directive win (PR38921).
-    # For RISCV chips we use integrated assembler by default, so suppress this warning to pass CI pipeline.
-    TEST_BUILD_OPTS_EXTRA: "--ignore-warning-str 'changed binding to STB_WEAK'"
+    # For RISCV clang generates '.linker-options' sections of type 'llvm_linker_options' in asm files.
+    # See (https://llvm.org/docs/Extensions.html#linker-options-section-linker-options).
+    # Binutils gas ignores them with warning.
+    # TODO: LLVM-112, Use integrated assembler.
+    TEST_BUILD_OPTS_EXTRA: "--ignore-warning-str 'Warning: unrecognized section type'"
 
 build_clang_test_apps_esp32c3:
   extends:
     - .build_clang_test_apps_riscv
-    - .rules:build
+    - .rules:build:custom_test-esp32c3
   variables:
     IDF_TARGET: esp32c3
 
 build_clang_test_apps_esp32c2:
   extends:
     - .build_clang_test_apps_riscv
-    - .rules:build
+    - .rules:build:custom_test-esp32c2
   variables:
     IDF_TARGET: esp32c2
 
 build_clang_test_apps_esp32c6:
   extends:
     - .build_clang_test_apps_riscv
-    - .rules:build
+    - .rules:build:custom_test-esp32c6
+  # TODO: c6 builds fail in master due to missing headers
+  allow_failure: true
   variables:
     IDF_TARGET: esp32c6
 
-build_clang_test_apps_esp32c5:
-  extends:
-    - .build_clang_test_apps_riscv
-    - .rules:build
-  variables:
-    IDF_TARGET: esp32c5
-
-build_clang_test_apps_esp32h2:
-  extends:
-    - .build_clang_test_apps_riscv
-    - .rules:build
-  variables:
-    IDF_TARGET: esp32h2
-
-build_clang_test_apps_esp32p4:
-  extends:
-    - .build_clang_test_apps_riscv
-    - .rules:build
-  variables:
-    IDF_TARGET: esp32p4
-
-######################
-# Build System Tests #
-######################
 .test_build_system_template:
   stage: host_test
   extends:
     - .build_template
     - .rules:build:check
-  dependencies: # set dependencies to null to avoid missing artifacts issue
   needs:
     - job: fast_template_app
       artifacts: false
       optional: true
-  artifacts:
-    reports:
-      junit: XUNIT_RESULT.xml
-    paths:
-      - XUNIT_RESULT.xml
-      - test_build_system
-    expire_in: 1 week
-    when: always
   script:
     - ${IDF_PATH}/tools/ci/test_configure_ci_environment.sh
     - cd ${IDF_PATH}/tools/test_build_system
     - python ${IDF_PATH}/tools/ci/get_known_failure_cases_file.py
-    - pytest
-      --cleanup-idf-copy
-      --parallel-count ${CI_NODE_TOTAL:-1}
-      --parallel-index ${CI_NODE_INDEX:-1}
-      --work-dir ${CI_PROJECT_DIR}/test_build_system
-      --junitxml ${CI_PROJECT_DIR}/XUNIT_RESULT.xml
+    - pytest --parallel-count ${CI_NODE_TOTAL:-1} --parallel-index ${CI_NODE_INDEX:-1}
+      --work-dir ${CI_PROJECT_DIR}/test_build_system --junitxml=${CI_PROJECT_DIR}/XUNIT_RESULT.xml
       --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME}
 
 pytest_build_system:
   extends: .test_build_system_template
   parallel: 3
+  artifacts:
+    paths:
+      - XUNIT_RESULT.xml
+      - test_build_system
+    when: always
+    expire_in: 2 days
+    reports:
+      junit: XUNIT_RESULT.xml
 
 pytest_build_system_macos:
   extends:
     - .test_build_system_template
     - .before_script:build:macos
-    - .after_script:build:macos:upload-failed-job-logs:ccache-show-stats
     - .rules:build:macos
   tags:
     - macos_shell
   parallel: 3
-  variables:
-    PYENV_VERSION: "3.9"
-    # CCACHE_DIR: "/cache/idf_ccache". On macOS, you cannot write to this folder due to insufficient permissions.
-    CCACHE_DIR: "" # ccache will use "$HOME/Library/Caches/ccache".
-    CCACHE_MAXSIZE: "5G" # To preserve the limited Macbook storage. CCACHE automatically prunes old caches to fit the set limit.
+  artifacts:
+    paths:
+      - XUNIT_RESULT.xml
+      - test_build_system
+    when: always
+    expire_in: 2 days
+    reports:
+      junit: XUNIT_RESULT.xml
+
 build_docker:
   extends:
     - .before_script:minimal
@@ -268,57 +580,6 @@ build_template_app:
     - .build_template_app_template
     - .rules:build
   stage: host_test
-  dependencies: # set dependencies to null to avoid missing artifacts issue
   needs:
     - job: fast_template_app
       artifacts: false
-
-####################
-# Dynamic Pipeline #
-####################
-generate_build_child_pipeline:
-  extends:
-    - .build_template
-  tags: [fast_run, shiny]
-  dependencies: # set dependencies to null to avoid missing artifacts issue
-  needs:
-    - pipeline_variables
-    - job: baseline_manifest_sha
-      optional: true
-  artifacts:
-    paths:
-      - build_child_pipeline.yml
-      - test_related_apps.txt
-      - non_test_related_apps.txt
-    expire_in: 1 week
-    when: always
-  variables:
-    PYTEST_IGNORE_COLLECT_IMPORT_ERROR: "1"
-  script:
-    # requires basic pytest dependencies
-    - run_cmd bash install.sh --enable-pytest
-    - run_cmd python tools/ci/dynamic_pipelines/scripts/generate_build_child_pipeline.py
-
-build_child_pipeline:
-  stage: build
-  needs:
-    - job: fast_template_app
-      optional: true
-      artifacts: false
-    - pipeline_variables
-    - generate_build_child_pipeline
-  variables:
-    IS_MR_PIPELINE: $IS_MR_PIPELINE
-    MR_MODIFIED_COMPONENTS: $MR_MODIFIED_COMPONENTS
-    MR_MODIFIED_FILES: $MR_MODIFIED_FILES
-    PARENT_PIPELINE_ID: $CI_PIPELINE_ID
-    BUILD_AND_TEST_ALL_APPS: $BUILD_AND_TEST_ALL_APPS
-    REPORT_EXIT_CODE: $REPORT_EXIT_CODE
-  # https://gitlab.com/gitlab-org/gitlab/-/issues/214340
-  inherit:
-    variables: false
-  trigger:
-    include:
-      - artifact: build_child_pipeline.yml
-        job: generate_build_child_pipeline
-    strategy: depend

.gitlab/ci/common.yml

@@ -12,7 +12,6 @@ stages:
   - test_deploy
   - deploy
   - post_deploy
-  - retry_failed_jobs
 
 variables:
 # System environment
@@ -40,7 +39,7 @@ variables:
   GIT_FETCH_EXTRA_FLAGS: "--no-recurse-submodules --prune --prune-tags"
   # we're using .cache folder for caches
   GIT_CLEAN_FLAGS: -ffdx -e .cache/
-  LATEST_GIT_TAG: v5.5-dev
+  LATEST_GIT_TAG: v5.2.1
 
   SUBMODULE_FETCH_TOOL: "tools/ci/ci_fetch_submodule.py"
   # by default we will fetch all submodules
@@ -53,14 +52,20 @@ variables:
   IDF_PATH: "$CI_PROJECT_DIR"
   V: "0"
   CHECKOUT_REF_SCRIPT: "$CI_PROJECT_DIR/tools/ci/checkout_project_ref.py"
+  PYTHON_VER: 3.8.17
 
   # Docker images
-  ESP_ENV_IMAGE: "${CI_DOCKER_REGISTRY}/esp-env-v5.5:2"
-  ESP_IDF_DOC_ENV_IMAGE: "${CI_DOCKER_REGISTRY}/esp-idf-doc-env-v5.5:2-1"
-  TARGET_TEST_ENV_IMAGE: "${CI_DOCKER_REGISTRY}/target-test-env-v5.5:2"
+  ESP_ENV_IMAGE: "${CI_DOCKER_REGISTRY}/esp-env-v5.2:2"
+  ESP_IDF_DOC_ENV_IMAGE: "${CI_DOCKER_REGISTRY}/esp-idf-doc-env-v5.2:2-1"
+  QEMU_IMAGE: "${CI_DOCKER_REGISTRY}/qemu-v5.2:2-20230522"
+  TARGET_TEST_ENV_IMAGE: "${CI_DOCKER_REGISTRY}/target-test-env-v5.2:2"
+
   SONARQUBE_SCANNER_IMAGE: "${CI_DOCKER_REGISTRY}/sonarqube-scanner:5"
   PRE_COMMIT_IMAGE: "${CI_DOCKER_REGISTRY}/esp-idf-pre-commit:1"
 
+  # target test repo parameters
+  TEST_ENV_CONFIG_REPO: "https://gitlab-ci-token:${BOT_TOKEN}@${CI_SERVER_HOST}:${CI_SERVER_PORT}/qa/ci-test-runner-configs.git"
+
   # cache python dependencies
   PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
 
@@ -70,7 +75,7 @@ variables:
   CI_PYTHON_CONSTRAINT_BRANCH: ""
 
   # Update the filename for a specific ESP-IDF release. It is used only with CI_PYTHON_CONSTRAINT_BRANCH.
-  CI_PYTHON_CONSTRAINT_FILE: "espidf.constraints.v5.5.txt"
+  CI_PYTHON_CONSTRAINT_FILE: "espidf.constraints.v5.2.txt"
 
   # Set this variable to repository name of a Python tool you wish to install and test in the context of ESP-IDF CI.
   # Keep the variable empty when not used.
@@ -81,33 +86,16 @@ variables:
   # This is used only if CI_PYTHON_TOOL_REPO is not empty.
   CI_PYTHON_TOOL_BRANCH: ""
 
-  # Set this variable to Clang toolchain distro URL to be used.
-  # NOTE: We have separate toolchains for Xtensa and RISCV, therefore jobs for one arch will fail.
-  # This is OK as far as we use CI_CLANG_DISTRO_URL for pre-release tests purposes only.
-  # Keep the variable empty when not used.
-  CI_CLANG_DISTRO_URL: ""
-
   # Set this variable to specify the file name for the known failure cases.
-  KNOWN_FAILURE_CASES_FILE_NAME: "master.txt"
+  KNOWN_FAILURE_CASES_FILE_NAME: "5.2.txt"
 
   IDF_CI_BUILD: 1
 
-  # ccache settings
-  # some settings need to set in .gitlab-ci.yml as it takes effect while start-up the job
-  # https://ccache.dev/manual/latest.html#_configuring_ccache
-
-  # host mapping volume to share ccache between runner concurrent jobs
-  CCACHE_DIR: "/cache/idf_ccache"
-  CCACHE_MAXSIZE: "50G"
-
-  FF_USE_NEW_BASH_EVAL_STRATEGY: "true"
-
 ################################################
 # `before_script` and `after_script` Templates #
 ################################################
 .common_before_scripts: &common-before_scripts |
   source tools/ci/utils.sh
-
   is_based_on_commits $REQUIRED_ANCESTOR_COMMITS
 
   if [[ -n "$IDF_DONT_USE_MIRRORS" ]]; then
@@ -141,48 +129,35 @@ variables:
     export IDF_MIRROR_PREFIX_MAP=
   fi
 
-  if [[ "${CI_JOB_STAGE}" != "target_test" ]]; then
-    section_start "running_install_sh" "Running install.sh"
-    if [[ "${CI_JOB_STAGE}" == "build_doc" ]]; then
-      run_cmd bash install.sh --enable-ci --enable-docs
-    elif [[ "${CI_JOB_STAGE}" == "build" ]]; then
+  # install latest python packages
+  # target test jobs
+  if [[ "${CI_JOB_STAGE}" == "target_test" ]]; then
+    run_cmd bash install.sh --enable-ci --enable-pytest
+  elif [[ "${CI_JOB_STAGE}" == "build_doc" ]]; then
+    run_cmd bash install.sh --enable-ci --enable-docs
+  elif [[ "${CI_JOB_STAGE}" == "build" ]]; then
+    run_cmd bash install.sh --enable-ci --enable-pytest
+  else
+    if ! echo "${CI_JOB_NAME}" | egrep ".*pytest.*"; then
       run_cmd bash install.sh --enable-ci
     else
-      if ! echo "${CI_JOB_NAME}" | egrep ".*pytest.*"; then
-        run_cmd bash install.sh --enable-ci
-      else
-        run_cmd bash install.sh --enable-ci --enable-pytest --enable-test-specific
-      fi
+      run_cmd bash install.sh --enable-ci --enable-pytest
     fi
-    section_end "running_install_sh"
-  else
-    section_start "install_python_env" "Install Python environment"
-    run_cmd python tools/idf_tools.py install-python-env --features ci,pytest,test-specific
-    section_end "install_python_env"
   fi
 
-  if [[ ! -z "$INSTALL_EXTRA_TOOLS" ]]; then
-    section_start "installing_optional_tools" "Install optional tools ${INSTALL_EXTRA_TOOLS}"
-    $IDF_PATH/tools/idf_tools.py --non-interactive install $INSTALL_EXTRA_TOOLS
-    section_end "installing_optional_tools"
-  fi
-
-  # Install esp-clang if necessary (esp-clang is separately installed)
-  if [[ "$IDF_TOOLCHAIN" == "clang" && -z "$CI_CLANG_DISTRO_URL" ]]; then
+  # Install esp-clang if necessary
+  if [[ "$IDF_TOOLCHAIN" == "clang" ]]; then
     $IDF_PATH/tools/idf_tools.py --non-interactive install esp-clang
   fi
 
-  if [[ "${CI_JOB_STAGE}" == "target_test" ]]; then
-    section_start "IDF_SKIP_TOOLS_CHECK" "Skip required tools check"
-    export IDF_SKIP_TOOLS_CHECK=1
-    section_end "IDF_SKIP_TOOLS_CHECK"
-  fi
-  section_start "source_export" "Source export.sh"
-  source ./export.sh
-  section_end "source_export"
+  # Since the version 3.21 CMake passes source files and include dirs to ninja using absolute paths.
+  # Needed for pytest junit reports.
+  $IDF_PATH/tools/idf_tools.py --non-interactive install cmake
 
-  # Custom clang toolchain
-  if [[ "$IDF_TOOLCHAIN" == "clang" && ! -z "$CI_CLANG_DISTRO_URL" ]]; then
+  source ./export.sh
+
+  # Custom clang
+  if [[ ! -z "$CI_CLANG_DISTRO_URL" ]]; then
     echo "Using custom clang from ${CI_CLANG_DISTRO_URL}"
     wget $CI_CLANG_DISTRO_URL
     ARCH_NAME=$(basename $CI_CLANG_DISTRO_URL)
@@ -206,11 +181,9 @@ variables:
     rm -rf ${CI_PYTHON_TOOL_REPO}
   fi
 
-  info "setup tools and python venv done"
-
 .show_ccache_statistics: &show_ccache_statistics |
   # Show ccache statistics if enabled globally
-  test "$CI_CCACHE_STATS" == 1 && test -n "$(which ccache)" && ccache --show-stats -vv || true
+  test "$CI_CCACHE_STATS" == 1 && test -n "$(which ccache)" && ccache --show-stats || true
 
 .upload_failed_job_log_artifacts: &upload_failed_job_log_artifacts |
   if [ $CI_JOB_STATUS = "failed" ]; then
@@ -223,29 +196,13 @@ variables:
 
 .before_script:build:macos:
   before_script:
-    # macos is running shell executor, which means it would use
-    # the system installed /usr/local/bin/python3 by default.
-    # Ensure pyenv and PYENV_VERSION installed
-    - eval "$(pyenv init -)"
     - *common-before_scripts
     # On macOS, these tools need to be installed
     - export IDF_TOOLS_PATH="${HOME}/.espressif_runner_${CI_RUNNER_ID}_${CI_CONCURRENT_ID}"
-    # remove idf-env.json, since it may contains enabled "features"
-    - rm -f $IDF_TOOLS_PATH/idf-env.json
+    - $IDF_PATH/tools/idf_tools.py --non-interactive install cmake ninja
     # This adds tools (compilers) and the version-specific Python environment to PATH
     - *setup_tools_and_idf_python_venv
     - fetch_submodules
-  variables:
-    INSTALL_EXTRA_TOOLS: cmake ninja
-
-.after_script:build:macos:upload-failed-job-logs:ccache-show-stats:
-  after_script:
-    # macos is running shell executor, which means it would use
-    # the system installed /usr/local/bin/python3 by default.
-    # Ensure pyenv and PYENV_VERSION installed
-    - eval "$(pyenv init -)"
-    - *upload_failed_job_log_artifacts
-    - *show_ccache_statistics
 
 .before_script:build:
   before_script:
@@ -256,11 +213,7 @@ variables:
     - export EXTRA_CFLAGS=${PEDANTIC_CFLAGS}
     - export EXTRA_CXXFLAGS=${PEDANTIC_CXXFLAGS}
 
-.after_script:build:ccache-show-stats:
-  after_script:
-    - *show_ccache_statistics
-
-.after_script:build:ccache-show-stats:upload-failed-job-logs:
+.after_script:build:ccache:
   after_script:
     - *show_ccache_statistics
     - *upload_failed_job_log_artifacts
@@ -299,8 +252,8 @@ variables:
     git remote add origin "${CI_REPOSITORY_URL}"
   fi
 
-.git_checkout_ci_commit_sha: &git_checkout_ci_commit_sha |
-  git checkout $CI_COMMIT_SHA
+.git_checkout_fetch_head: &git_checkout_fetch_head |
+  git checkout FETCH_HEAD
   git clean ${GIT_CLEAN_FLAGS}
 
 # git diff requires two commits, with different CI env var
@@ -319,34 +272,27 @@ variables:
     - *git_init
     - *git_fetch_from_mirror_url_if_exists
     - |
-      # Store the diff output in a temporary file
-      TEMP_FILE=$(mktemp)
       # merged results pipelines, by default
       if [[ -n $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA ]]; then
         git fetch origin $CI_MERGE_REQUEST_DIFF_BASE_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
         git fetch origin $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
-        git diff --name-only $CI_MERGE_REQUEST_DIFF_BASE_SHA $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA > "$TEMP_FILE"
-        GIT_DIFF_OUTPUT=$(cat "$TEMP_FILE")
-        git fetch origin $CI_COMMIT_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
+        export GIT_DIFF_OUTPUT=$(git diff --name-only $CI_MERGE_REQUEST_DIFF_BASE_SHA $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA)
       # merge request pipelines, when the mr got conflicts
       elif [[ -n $CI_MERGE_REQUEST_DIFF_BASE_SHA ]]; then
         git fetch origin $CI_MERGE_REQUEST_DIFF_BASE_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
         git fetch origin $CI_COMMIT_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
-        git diff --name-only $CI_MERGE_REQUEST_DIFF_BASE_SHA $CI_COMMIT_SHA > "$TEMP_FILE"
-        GIT_DIFF_OUTPUT=$(cat "$TEMP_FILE")
+        export GIT_DIFF_OUTPUT=$(git diff --name-only $CI_MERGE_REQUEST_DIFF_BASE_SHA $CI_COMMIT_SHA)
       # other pipelines, like the protected branches pipelines
       elif [[ "$CI_COMMIT_BEFORE_SHA" != "0000000000000000000000000000000000000000" ]]; then
         git fetch origin $CI_COMMIT_BEFORE_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
         git fetch origin $CI_COMMIT_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
-        git diff --name-only $CI_COMMIT_BEFORE_SHA $CI_COMMIT_SHA > "$TEMP_FILE"
-        GIT_DIFF_OUTPUT=$(cat "$TEMP_FILE")
+        export GIT_DIFF_OUTPUT=$(git diff --name-only $CI_COMMIT_BEFORE_SHA $CI_COMMIT_SHA)
       else
         # pipeline source could be web, scheduler, etc.
         git fetch origin $CI_COMMIT_SHA --depth=2 ${GIT_FETCH_EXTRA_FLAGS}
-        git diff --name-only $CI_COMMIT_SHA~1 $CI_COMMIT_SHA > "$TEMP_FILE"
-        GIT_DIFF_OUTPUT=$(cat "$TEMP_FILE")
+        export GIT_DIFF_OUTPUT=$(git diff --name-only $CI_COMMIT_SHA~1 $CI_COMMIT_SHA)
       fi
-    - *git_checkout_ci_commit_sha
+    - *git_checkout_fetch_head
     - *common-before_scripts
     - *setup_tools_and_idf_python_venv
     - add_gitlab_ssh_keys
@@ -360,7 +306,7 @@ variables:
     - *git_init
     - *git_fetch_from_mirror_url_if_exists
     - git fetch origin "${CI_COMMIT_SHA}" --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
-    - *git_checkout_ci_commit_sha
+    - *git_checkout_fetch_head
     - *common-before_scripts
     - *setup_tools_and_idf_python_venv
     - add_gitlab_ssh_keys
@@ -373,15 +319,11 @@ default:
   cache:
     # pull only for most of the use cases since it's cache dir.
     # Only set "push" policy for "upload_cache" stage jobs
-    - key: pip-cache-${LATEST_GIT_TAG}
-      fallback_keys:
-        - pip-cache
+    - key: pip-cache
       paths:
         - .cache/pip
       policy: pull
-    - key: submodule-cache-${LATEST_GIT_TAG}
-      fallback_keys:
-        - submodule-cache
+    - key: submodule-cache
       paths:
         - .cache/submodule_archives
       policy: pull
@@ -390,12 +332,6 @@ default:
     - *setup_tools_and_idf_python_venv
     - add_gitlab_ssh_keys
     - fetch_submodules
-# gitlab bug, setting them here doesn't work
-# - expire_in: https://gitlab.com/gitlab-org/gitlab/-/issues/404563
-# - when: https://gitlab.com/gitlab-org/gitlab/-/issues/440672
-#  artifacts:
-#    expire_in: 1 week
-#    when: always
   retry:
     max: 2
     when:

.gitlab/ci/danger.yml

@@ -1,17 +0,0 @@
-# External DangerJS
-include:
-  - project: espressif/shared-ci-dangerjs
-    ref: master
-    file: danger.yaml
-
-run-danger-mr-linter:
-  stage: pre_check
-  variables:
-    GIT_STRATEGY: none # no repo checkout
-    ENABLE_CHECK_AREA_LABELS: 'true'
-    ENABLE_CHECK_DOCS_TRANSLATION: 'true'
-    ENABLE_CHECK_UPDATED_CHANGELOG: 'false'
-  before_script: []
-  cache: []
-  tags:
-    - dangerjs

.gitlab/ci/default-build-test-rules.yml

@@ -7,11 +7,10 @@
 #
 # This file should ONLY be used during bringup. Should be reset to empty after the bringup process
 extra_default_build_targets:
-  - esp32c5
-  - esp32c61
+ - esp32p4
 
 bypass_check_test_targets:
-
+ - esp32p4
 #
 # These lines would
 # - enable the README.md check for esp32c6. Don't forget to add the build jobs in .gitlab/ci/build.yml

.gitlab/ci/dependencies/dependencies.yml

@@ -5,7 +5,6 @@
   - esp32c3
   - esp32c2
   - esp32c6
-  - esp32c5
   - esp32h2
   - esp32p4
 
@@ -66,6 +65,104 @@
   included_in:
     - build:check
 
+# ---------------
+# Build Test Jobs
+# ---------------
+"build:{0}-{1}":
+  matrix:
+    - *target_test
+    - *all_targets
+  labels:
+    - build
+  patterns:
+    - build_components
+    - build_system
+    - build_target_test
+    - downloadable-tools
+  included_in:
+    - "build:{0}"
+    - build:target_test
+
+####################
+# Target Test Jobs #
+####################
+"test:{0}-{1}":
+  matrix:
+    - *target_test
+    - *all_targets
+  labels: # For each rule, use labels <test_type> and <test_type>-<target>
+    - "{0}"
+    - "{0}_{1}"
+    - target_test
+  patterns: # For each rule, use patterns <test_type> and build-<test_type>
+    - "{0}"
+    - "build-{0}"
+  included_in: # Parent rules
+    - "build:{0}"
+    - "build:{0}-{1}"
+    - build:target_test
+
+# -------------
+# Special Cases
+# -------------
+
+# To reduce the specific runners' usage.
+# Do not create these jobs by default patterns on development branches
+# Can be triggered by labels or related changes
+"test:{0}-{1}-{2}":
+  matrix:
+    - *target_test
+    - *all_targets
+    - - wifi      # pytest*wifi*
+      - ethernet  # pytest*ethernet*
+      - sdio      # pytest*sdio*
+      - usb       # USB Device & Host tests
+      - adc       # pytest*adc*
+      - i154
+      - flash_multi
+      - ecdsa
+      - nvs_encr_hmac
+  patterns:
+    - "{0}-{1}-{2}"
+    - "{0}-{2}"
+    - "target_test-{2}"
+  labels:
+    - "{0}_{1}"
+    - "{0}"
+    - target_test
+  included_in:
+    - "build:{0}-{1}"
+    - "build:{0}"
+    - build:target_test
+
+# For example_test*flash_encryption_wifi_high_traffic jobs
+# set `INCLUDE_NIGHTLY_RUN` variable when triggered on development branches
+"test:example_test-{0}-include_nightly_run-rule":
+  matrix:
+    - - esp32
+      - esp32c3
+  specific_rules:
+    - "if-example_test-ota-include_nightly_run-rule"
+  included_in:
+    - "build:example_test-{0}"
+    - "build:example_test"
+    - build:target_test
+
+# For i154 runners
+"test:example_test-i154":
+  patterns:
+    - "example_test-i154"
+    - "target_test-i154"
+  labels:
+    - target_test
+    - example_test
+  included_in:
+    - "build:example_test-esp32s3"
+    - "build:example_test-esp32c6"
+    - "build:example_test-esp32h2"
+    - "build:example_test"
+    - build:target_test
+
 "test:host_test":
   labels:
     - host_test
@@ -84,9 +181,3 @@
 "labels:nvs_coverage": # host_test
   labels:
     - nvs_coverage
-
-"labels:windows_pytest_build_system":
-  labels:
-    - windows
-  specific_rules:
-    - if-schedule-test-build-system-windows

.gitlab/ci/docs.yml

@@ -96,13 +96,10 @@ check_docs_lang_sync:
     - if [ -n "${BREATHE_ALT_INSTALL_URL}" ]; then pip uninstall -y breathe && pip install -U ${BREATHE_ALT_INSTALL_URL}; fi
     - cd docs
     - build-docs -t $DOCTGT -bs $DOC_BUILDERS -l $DOCLANG build
-  artifacts:
-    expire_in: 4 days
-    when: always
   parallel:
     matrix:
       - DOCLANG: ["en", "zh_CN"]
-        DOCTGT: ["esp32", "esp32s2", "esp32s3", "esp32c3", "esp32c2", "esp32c6", "esp32c61", "esp32c5","esp32h2", "esp32p4"]
+        DOCTGT: ["esp32", "esp32s2", "esp32s3", "esp32c3", "esp32c2", "esp32c6", "esp32h2", "esp32p4"]
 
 check_docs_gh_links:
   image: $ESP_IDF_DOC_ENV_IMAGE
@@ -124,9 +121,11 @@ build_docs_html_full:
       artifacts: false
       optional: true
   artifacts:
+    when: always
     paths:
       - docs/_build/*/*/*.txt
       - docs/_build/*/*/html/*
+    expire_in: 4 days
   variables:
     DOC_BUILDERS: "html"
 
@@ -136,9 +135,11 @@ build_docs_html_full_prod:
     - .doc-rules:build:docs-full-prod
   dependencies: [] # Stop build_docs jobs from downloading all previous job's artifacts
   artifacts:
+    when: always
     paths:
       - docs/_build/*/*/*.txt
       - docs/_build/*/*/html/*
+    expire_in: 4 days
   variables:
     DOC_BUILDERS: "html"
 
@@ -151,9 +152,11 @@ build_docs_html_partial:
       artifacts: false
       optional: true
   artifacts:
+    when: always
     paths:
       - docs/_build/*/*/*.txt
       - docs/_build/*/*/html/*
+    expire_in: 4 days
   variables:
     DOC_BUILDERS: "html"
   parallel:
@@ -171,10 +174,11 @@ build_docs_pdf:
     - job: fast_template_app
       artifacts: false
       optional: true
-  allow_failure: true # TODO IDFCI-2216
   artifacts:
+    when: always
     paths:
       - docs/_build/*/*/latex/*
+    expire_in: 4 days
   variables:
     DOC_BUILDERS: "latex"
 
@@ -183,10 +187,11 @@ build_docs_pdf_prod:
     - .build_docs_template
     - .doc-rules:build:docs-full-prod
   dependencies: [] # Stop build_docs jobs from downloading all previous job's artifacts
-  allow_failure: true # TODO IDFCI-2216
   artifacts:
+    when: always
     paths:
       - docs/_build/*/*/latex/*
+    expire_in: 4 days
   variables:
     DOC_BUILDERS: "latex"
 
@@ -261,9 +266,11 @@ check_doc_links:
       artifacts: false
   tags: ["build", "amd64", "internet"]
   artifacts:
+    when: always
     paths:
       - docs/_build/*/*/*.txt
       - docs/_build/*/*/linkcheck/*.txt
+    expire_in: 1 week
   allow_failure: true
   script:
     - cd docs

.gitlab/ci/host-test.yml

@@ -14,25 +14,12 @@
       optional: true
       artifacts: false
     - pipeline_variables
-  artifacts:
-    expire_in: 1 week
-    when: always
 
-check_public_headers:
-  extends:
-    - .host_test_template
-    - .rules:build:check
+test_nvs_on_host:
+  extends: .host_test_template
   script:
-    - IDF_TARGET=esp32 python tools/ci/check_public_headers.py --jobs 4 --prefix xtensa-esp32-elf-
-    - IDF_TARGET=esp32s2 python tools/ci/check_public_headers.py --jobs 4 --prefix xtensa-esp32s2-elf-
-    - IDF_TARGET=esp32s3 python tools/ci/check_public_headers.py --jobs 4 --prefix xtensa-esp32s3-elf-
-    - IDF_TARGET=esp32c3 python tools/ci/check_public_headers.py --jobs 4 --prefix riscv32-esp-elf-
-    - IDF_TARGET=esp32c2 python tools/ci/check_public_headers.py --jobs 4 --prefix riscv32-esp-elf-
-    - IDF_TARGET=esp32c6 python tools/ci/check_public_headers.py --jobs 4 --prefix riscv32-esp-elf-
-    - IDF_TARGET=esp32c5 python tools/ci/check_public_headers.py --jobs 4 --prefix riscv32-esp-elf-
-    - IDF_TARGET=esp32h2 python tools/ci/check_public_headers.py --jobs 4 --prefix riscv32-esp-elf-
-    - IDF_TARGET=esp32p4 python tools/ci/check_public_headers.py --jobs 4 --prefix riscv32-esp-elf-
-    - IDF_TARGET=esp32c61 python tools/ci/check_public_headers.py --jobs 4 --prefix riscv32-esp-elf-
+    - cd components/nvs_flash/test_nvs_host
+    - make test
 
 test_nvs_coverage:
   extends:
@@ -40,10 +27,11 @@ test_nvs_coverage:
     - .rules:labels:nvs_coverage
   artifacts:
     paths:
-      - components/nvs_flash/host_test/nvs_host_test/coverage_report
+      - components/nvs_flash/test_nvs_host/coverage_report
+    expire_in: 1 week
   script:
-    - cd components/nvs_flash/host_test/nvs_host_test
-    - idf.py build coverage
+    - cd components/nvs_flash/test_nvs_host
+    - make coverage_report
   # the 'long' host tests take approx 11 hours on our current runners. Adding some margin here for possible CPU contention
   timeout: 18 hours
 
@@ -62,6 +50,23 @@ test_ldgen_on_host:
   variables:
     LC_ALL: C.UTF-8
 
+test_reproducible_build:
+  extends: .host_test_template
+  script:
+    - ./tools/ci/test_reproducible_build.sh
+  artifacts:
+    when: on_failure
+    paths:
+      - "**/sdkconfig"
+      - "**/build*/*.bin"
+      - "**/build*/*.elf"
+      - "**/build*/*.map"
+      - "**/build*/flasher_args.json"
+      - "**/build*/*.bin"
+      - "**/build*/bootloader/*.bin"
+      - "**/build*/partition_table/*.bin"
+    expire_in: 1 week
+
 test_spiffs_on_host:
   extends: .host_test_template
   script:
@@ -94,9 +99,18 @@ test_gdbstub_on_host:
     - cd components/esp_gdbstub/test_gdbstub_host
     - make test
 
+test_idf_py:
+  extends: .host_test_template
+  variables:
+    LC_ALL: C.UTF-8
+  script:
+    - cd ${IDF_PATH}/tools/test_idf_py
+    - ./test_idf_py.py
+    - ./test_hints.py
+
 # Test for create virtualenv. It must be invoked from Python, not from virtualenv.
 # Use docker image system python without any extra dependencies
-test_cli_installer:
+test_idf_tools:
   extends:
     - .host_test_template
     - .before_script:minimal
@@ -105,19 +119,17 @@ test_cli_installer:
     paths:
       - tools/tools.new.json
       - tools/test_idf_tools/test_python_env_logs.txt
+    expire_in: 1 week
   image:
     name: $ESP_ENV_IMAGE
     entrypoint: [""]  # use system python3. no extra pip package installed
   script:
     # Tools must be downloaded for testing
-    # We could use "idf_tools.py download all", but we don't want to install clang because of its huge size
-    - python3 ${IDF_PATH}/tools/idf_tools.py download required qemu-riscv32 qemu-xtensa cmake
+    - python3 ${IDF_PATH}/tools/idf_tools.py download required qemu-riscv32 qemu-xtensa
     - cd ${IDF_PATH}/tools/test_idf_tools
     - python3 -m pip install jsonschema
     - python3 ./test_idf_tools.py -v
     - python3 ./test_idf_tools_python_env.py
-    # It runs at the end because it modifies dependencies
-    - IDF_TEST_MAY_BREAK_DEPENDENCIES=1 python3 ./test_idf_tools.py -v TestSystemDependencies.test_commands_when_nodeps
 
 .test_efuse_table_on_host_template:
   extends: .host_test_template
@@ -127,6 +139,7 @@ test_cli_installer:
     when: on_failure
     paths:
       - components/efuse/${IDF_TARGET}/esp_efuse_table.c
+    expire_in: 1 week
   script:
     - cd ${IDF_PATH}/components/efuse/
     - ./efuse_table_gen.py -t "${IDF_TARGET}" ${IDF_PATH}/components/efuse/${IDF_TARGET}/esp_efuse_table.csv
@@ -169,6 +182,7 @@ test_logtrace_proc:
     paths:
       - tools/esp_app_trace/test/logtrace/output
       - tools/esp_app_trace/test/logtrace/.coverage
+    expire_in: 1 week
   script:
     - cd ${IDF_PATH}/tools/esp_app_trace/test/logtrace
     - ./test.sh
@@ -180,43 +194,42 @@ test_sysviewtrace_proc:
     paths:
       - tools/esp_app_trace/test/sysview/output
       - tools/esp_app_trace/test/sysview/.coverage
+    expire_in: 1 week
   script:
     - cd ${IDF_PATH}/tools/esp_app_trace/test/sysview
     - ./test.sh
 
-test_tools:
+test_mkdfu:
+  extends: .host_test_template
+  variables:
+    LC_ALL: C.UTF-8
+  script:
+    - cd ${IDF_PATH}/tools/test_mkdfu
+    - ./test_mkdfu.py
+
+test_autocomplete:
   extends:
     - .host_test_template
   artifacts:
+    when: on_failure
     paths:
       - ${IDF_PATH}/*.out
-      - ${IDF_PATH}/XUNIT_*.xml
-    reports:
-      junit: ${IDF_PATH}/XUNIT_*.xml
-  variables:
-    LC_ALL: C.UTF-8
-    INSTALL_EXTRA_TOOLS: "qemu-xtensa qemu-riscv32"  # for test_idf_qemu.py
+    expire_in: 1 week
+  script:
+    - ${IDF_PATH}/tools/ci/test_autocomplete.py
+
+test_detect_python:
+  extends:
+    - .host_test_template
   script:
-    - stat=0
-    - cd ${IDF_PATH}/tools/ci/test_autocomplete
-    - pytest --noconftest test_autocomplete.py --junitxml=${IDF_PATH}/XUNIT_AUTOCOMP.xml                  || stat=1
-    - cd ${IDF_PATH}/tools/test_idf_py
-    - pytest --noconftest test_idf_py.py       --junitxml=${IDF_PATH}/XUNIT_IDF_PY.xml                    || stat=1
-    - pytest --noconftest test_hints.py        --junitxml=${IDF_PATH}/XUNIT_HINTS.xml                     || stat=1
-    - pytest --noconftest test_idf_qemu.py     --junitxml=${IDF_PATH}/XUNIT_IDF_PY_QEMU.xml               || stat=1
-    - cd ${IDF_PATH}/tools/test_mkdfu
-    - pytest --noconftest test_mkdfu.py        --junitxml=${IDF_PATH}/XUNIT_MKDFU.xml                     || stat=1
-    - cd ${IDF_PATH}/tools/test_idf_size
-    - pytest --noconftest test_idf_size.py     --junitxml=${IDF_PATH}/XUNIT_IDF_SIZE.xml                  || stat=1
     - cd ${IDF_PATH}
-    - shellcheck -s sh tools/detect_python.sh                                                             || stat=1
-    - shellcheck -s bash tools/detect_python.sh                                                           || stat=1
-    - shellcheck -s dash tools/detect_python.sh                                                           || stat=1
+    - shellcheck -s sh tools/detect_python.sh
+    - shellcheck -s bash tools/detect_python.sh
+    - shellcheck -s dash tools/detect_python.sh
     - "bash -c '. tools/detect_python.sh && echo Our Python: ${ESP_PYTHON?Python is not set}'"
     - "dash -c '. tools/detect_python.sh && echo Our Python: ${ESP_PYTHON?Python is not set}'"
     - "zsh -c '. tools/detect_python.sh && echo Our Python: ${ESP_PYTHON?Python is not set}'"
     - "fish -c 'source tools/detect_python.fish && echo Our Python: $ESP_PYTHON'"
-    - exit "$stat"
 
 test_split_path_by_spaces:
   extends: .host_test_template
@@ -273,146 +286,61 @@ test_pytest_qemu:
   extends:
     - .host_test_template
     - .before_script:build
+  image: $QEMU_IMAGE
   artifacts:
+    when: always
     paths:
       - XUNIT_RESULT.xml
-      - pytest-embedded/
-      - "**/build*/*.bin"
+      - pytest_embedded_log/
     reports:
       junit: XUNIT_RESULT.xml
+    expire_in: 1 week
+  allow_failure: true # IDFCI-1752
   parallel:
     matrix:
-      - IDF_TARGET: "esp32"
-        INSTALL_EXTRA_TOOLS: "qemu-xtensa"
-      - IDF_TARGET: "esp32c3"
-        INSTALL_EXTRA_TOOLS: "qemu-riscv32"
+      - IDF_TARGET: [esp32, esp32c3]
   script:
-    - run_cmd python tools/ci/ci_build_apps.py . -v
+    - run_cmd python tools/ci/ci_build_apps.py . -vv
       --target $IDF_TARGET
       --pytest-apps
       -m qemu
       --collect-app-info "list_job_${CI_JOB_NAME_SLUG}.txt"
-      --modified-components ${MR_MODIFIED_COMPONENTS}
-      --modified-files ${MR_MODIFIED_FILES}
+      --modified-components ${MODIFIED_COMPONENTS}
+      --modified-files ${MODIFIED_FILES}
     - python tools/ci/get_known_failure_cases_file.py
     - run_cmd pytest
       --target $IDF_TARGET
-      --log-cli-level DEBUG
       -m qemu
       --embedded-services idf,qemu
       --junitxml=XUNIT_RESULT.xml
       --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME}
       --app-info-filepattern \"list_job_*.txt\"
-      --qemu-extra-args \"-global driver=timer.$IDF_TARGET.timg,property=wdt_disable,value=true\"
 
 test_pytest_linux:
   extends:
     - .host_test_template
     - .before_script:build
   artifacts:
+    when: always
     paths:
       - XUNIT_RESULT.xml
-      - pytest-embedded/
+      - pytest_embedded_log/
       - "**/build*/build_log.txt"
     reports:
       junit: XUNIT_RESULT.xml
+    expire_in: 1 week
   script:
-    - run_cmd python tools/ci/ci_build_apps.py components examples tools/test_apps -v
+    - run_cmd python tools/ci/ci_build_apps.py components examples tools/test_apps -vv
       --target linux
       --pytest-apps
       -m host_test
       --collect-app-info "list_job_${CI_JOB_NAME_SLUG}.txt"
-      --modified-components ${MR_MODIFIED_COMPONENTS}
-      --modified-files ${MR_MODIFIED_FILES}
+      --modified-components ${MODIFIED_COMPONENTS}
+      --modified-files ${MODIFIED_FILES}
     - python tools/ci/get_known_failure_cases_file.py
     - run_cmd pytest
       --target linux
       -m host_test
-      --embedded-services idf
       --junitxml=XUNIT_RESULT.xml
       --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME}
       --app-info-filepattern \"list_job_*.txt\"
-
-test_pytest_macos:
-  extends:
-    - .host_test_template
-    - .before_script:build:macos
-  tags:
-    - macos_shell
-  artifacts:
-    paths:
-      - XUNIT_RESULT.xml
-      - pytest-embedded/
-      - "**/build*/build_log.txt"
-    reports:
-      junit: XUNIT_RESULT.xml
-  variables:
-    PYENV_VERSION: "3.9"
-    PYTEST_IGNORE_COLLECT_IMPORT_ERROR: "1"
-  script:
-    - run_cmd python tools/ci/ci_build_apps.py components examples tools/test_apps -v
-      --target linux
-      --pytest-apps
-      -m \"host_test and macos_shell\"
-      --collect-app-info "list_job_${CI_JOB_NAME_SLUG}.txt"
-      --modified-components ${MR_MODIFIED_COMPONENTS}
-      --modified-files ${MR_MODIFIED_FILES}
-    - python tools/ci/get_known_failure_cases_file.py
-    - run_cmd pytest
-      --target linux
-      -m \"host_test and macos_shell\"
-      --junitxml=XUNIT_RESULT.xml
-      --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME}
-      --app-info-filepattern \"list_job_*.txt\"
-
-test_idf_pytest_plugin:
-  extends:
-    - .host_test_template
-    - .rules:patterns:idf-pytest-plugin
-  variables:
-    SUBMODULES_TO_FETCH: "none"
-  artifacts:
-    reports:
-      junit: XUNIT_RESULT.xml
-  script:
-    - cd ${IDF_PATH}/tools/ci/dynamic_pipelines/tests/test_report_generator
-    - python -m unittest test_report_generator.py
-    - cd ${IDF_PATH}/tools/ci/idf_pytest
-    - pytest --junitxml=${CI_PROJECT_DIR}/XUNIT_RESULT.xml
-
-test_idf_build_apps_load_soc_caps:
-  extends: .host_test_template
-  script:
-    - python tools/ci/check_soc_headers_load_in_idf_build_apps.py
-
-test_nvs_gen_check:
-  extends: .host_test_template
-  artifacts:
-    paths:
-      - XUNIT_RESULT.xml
-      - components/nvs_flash/nvs_partition_tool
-    reports:
-      junit: XUNIT_RESULT.xml
-  variables:
-    LC_ALL: C.UTF-8
-  script:
-    - cd ${IDF_PATH}/components/nvs_flash/nvs_partition_tool
-    - pytest --noconftest test_nvs_gen_check.py --junitxml=XUNIT_RESULT.xml
-
-make_sure_soc_caps_compatible_in_idf_build_apps:
-    extends:
-      - .host_test_template
-      - .rules:dev-push
-    artifacts:
-      paths:
-        - new.json
-        - base.json
-      when: always
-    when: manual
-    script:
-      - python tools/ci/idf_build_apps_dump_soc_caps.py new.json
-      - git fetch --depth=1 origin $CI_MERGE_REQUEST_DIFF_BASE_SHA
-      - git checkout -f $CI_MERGE_REQUEST_DIFF_BASE_SHA
-      - git checkout $CI_COMMIT_SHA -- tools/ci/idf_build_apps_dump_soc_caps.py
-      - python tools/ci/idf_build_apps_dump_soc_caps.py base.json
-      - diff new.json base.json

.gitlab/ci/integration_test.yml

@@ -34,7 +34,8 @@ gen_integration_pipeline:
   image: ${CI_INTEGRATION_ASSIGN_ENV}
   stage: assign_test
   cache: []
-  tags: [fast_run, shiny]
+  tags:
+    - assign_test
   variables:
     SUBMODULES_TO_FETCH: "none"
     GIT_LFS_SKIP_SMUDGE: 1
@@ -46,7 +47,6 @@ gen_integration_pipeline:
     paths:
       - idf-integration-ci/child_pipeline/
     expire_in: 2 weeks
-    when: always
   script:
     - add_gitlab_ssh_keys
     - retry_failed git clone ${CI_GEN_INTEGRATION_PIPELINE_REPO} idf-integration-ci

.gitlab/ci/post_deploy.yml

@@ -1,13 +0,0 @@
-generate_failed_jobs_report:
-  stage: post_deploy
-  tags: [build, shiny]
-  image: $ESP_ENV_IMAGE
-  when: always
-  dependencies: []  # Do not download artifacts from the previous stages
-  artifacts:
-    expire_in: 1 week
-    when: always
-    paths:
-      - job_report.html
-  script:
-    - python tools/ci/dynamic_pipelines/scripts/generate_report.py --report-type job

.gitlab/ci/pre_check.yml

@@ -3,7 +3,46 @@
   image: $ESP_ENV_IMAGE
   tags:
     - host_test
-  dependencies: # set dependencies to null to avoid missing artifacts issue
+  dependencies: []
+
+check_pre_commit:
+  extends:
+    - .pre_check_template
+    - .before_script:minimal
+  image: $PRE_COMMIT_IMAGE
+  needs:
+    - pipeline_variables
+  script:
+    - fetch_submodules
+    - pre-commit run --files $MODIFIED_FILES
+    - pre-commit run --hook-stage post-commit validate-sbom-manifest
+
+check_MR_style_dangerjs:
+  extends:
+    - .pre_check_template
+  image: node:18.15.0-alpine3.16
+  variables:
+    DANGER_GITLAB_API_TOKEN: ${ESPCI_TOKEN}
+    DANGER_GITLAB_HOST: ${GITLAB_HTTP_SERVER}
+    DANGER_GITLAB_API_BASE_URL: ${GITLAB_HTTP_SERVER}/api/v4
+    DANGER_JIRA_USER: ${DANGER_JIRA_USER}
+    DANGER_JIRA_PASSWORD: ${DANGER_JIRA_PASSWORD}
+  cache:
+    # pull only for most of the use cases since it's cache dir.
+    # Only set "push" policy for "upload_cache" stage jobs
+    key:
+      files:
+        - .gitlab/dangerjs/package-lock.json
+    paths:
+      - .gitlab/dangerjs/node_modules/
+    policy: pull
+  before_script:
+    - cd .gitlab/dangerjs
+    - npm install --no-progress --no-update-notifier  # Install danger dependencies
+  script:
+    - npx danger ci --failOnErrors -v
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
 
 check_version:
   # Don't run this for feature/bugfix branches, so that it is possible to modify
@@ -25,8 +64,7 @@ check_version:
 check_api_usage:
   extends: .pre_check_template
   script:
-    - python -m pip install ast-grep-cli # use ast-grep to describe customized lint rules
-    - ast-grep scan
+    - tools/ci/check_examples_rom_header.sh
     - tools/ci/check_api_violation.sh
     - tools/ci/check_examples_extra_component_dirs.sh
 
@@ -44,9 +82,6 @@ check_blobs:
     - IDF_TARGET=esp32c2 $IDF_PATH/components/esp_wifi/test_md5/test_md5.sh
     - IDF_TARGET=esp32c3 $IDF_PATH/components/esp_wifi/test_md5/test_md5.sh
     - IDF_TARGET=esp32c6 $IDF_PATH/components/esp_wifi/test_md5/test_md5.sh
-    - IDF_TARGET=esp32c5 $IDF_PATH/components/esp_wifi/test_md5/test_md5.sh
-    - IDF_TARGET=esp32c61 $IDF_PATH/components/esp_wifi/test_md5/test_md5.sh
-    - IDF_TARGET=esp32_host $IDF_PATH/components/esp_wifi/test_md5/test_md5.sh
     # Check if Coexistence library header files match between IDF and the version used when compiling the libraries
     - IDF_TARGET=esp32 $IDF_PATH/components/esp_coex/test_md5/test_md5.sh
     - IDF_TARGET=esp32s2 $IDF_PATH/components/esp_coex/test_md5/test_md5.sh
@@ -55,11 +90,23 @@ check_blobs:
     - IDF_TARGET=esp32c3 $IDF_PATH/components/esp_coex/test_md5/test_md5.sh
     - IDF_TARGET=esp32c6 $IDF_PATH/components/esp_coex/test_md5/test_md5.sh
     - IDF_TARGET=esp32h2 $IDF_PATH/components/esp_coex/test_md5/test_md5.sh
-    - IDF_TARGET=esp32c5 $IDF_PATH/components/esp_coex/test_md5/test_md5.sh
-    - IDF_TARGET=esp32c61 $IDF_PATH/components/esp_coex/test_md5/test_md5.sh
     # Check if Wi-Fi, PHY, BT blobs contain references to specific symbols
     - bash $IDF_PATH/tools/ci/check_blobs.sh
 
+check_public_headers:
+  extends:
+    - .pre_check_template
+    - .rules:build:check
+  script:
+    - IDF_TARGET=esp32 python tools/ci/check_public_headers.py --jobs 4 --prefix xtensa-esp32-elf-
+    - IDF_TARGET=esp32s2 python tools/ci/check_public_headers.py --jobs 4 --prefix xtensa-esp32s2-elf-
+    - IDF_TARGET=esp32s3 python tools/ci/check_public_headers.py --jobs 4 --prefix xtensa-esp32s3-elf-
+    - IDF_TARGET=esp32c3 python tools/ci/check_public_headers.py --jobs 4 --prefix riscv32-esp-elf-
+    - IDF_TARGET=esp32c2 python tools/ci/check_public_headers.py --jobs 4 --prefix riscv32-esp-elf-
+    - IDF_TARGET=esp32c6 python tools/ci/check_public_headers.py --jobs 4 --prefix riscv32-esp-elf-
+    - IDF_TARGET=esp32h2 python tools/ci/check_public_headers.py --jobs 4 --prefix riscv32-esp-elf-
+    - IDF_TARGET=esp32p4 python tools/ci/check_public_headers.py --jobs 4 --prefix riscv32-esp-elf-
+
 check_chip_support_components:
   extends:
     - .pre_check_template
@@ -72,7 +119,7 @@ check_chip_support_components:
     expire_in: 1 week
   script:
     - python tools/ci/check_soc_headers_leak.py
-    - find ${IDF_PATH}/components/soc/**/include/soc/ -name "*_struct.h" -print0 | xargs -0 -n1 ./tools/ci/check_soc_struct_headers.py
+    - find ${IDF_PATH}/components/soc/*/include/soc/ -name "*_struct.h" -print0 | xargs -0 -n1 ./tools/ci/check_soc_struct_headers.py
     - tools/ci/check_esp_memory_utils_headers.sh
 
 check_esp_err_to_name:
@@ -113,14 +160,18 @@ check_version_tag:
   script:
     - (git cat-file -t $CI_COMMIT_REF_NAME | grep tag) || (echo "ESP-IDF versions must be annotated tags." && exit 1)
 
+check_artifacts_expire_time:
+  extends: .pre_check_template
+  script:
+    # check if we have set expire time for all artifacts
+    - python tools/ci/check_artifacts_expire_time.py
+
 check_test_scripts_build_test_rules:
   extends:
     - .pre_check_template
     - .before_script:build
-  variables:
-    PYTEST_IGNORE_COLLECT_IMPORT_ERROR: "1"
   script:
-    # requires basic pytest dependencies
+    # required pytest related packages
     - run_cmd bash install.sh --enable-pytest
     - python tools/ci/check_build_test_rules.py check-test-scripts examples/ tools/test_apps components
 
@@ -137,85 +188,19 @@ pipeline_variables:
   extends:
     - .pre_check_template
     - .before_script:fetch:git_diff
-  tags: [fast_run, shiny]
+  tags:
+    - build
   script:
-    # MODIFIED_FILES is a list of files that changed, could be used everywhere
     - MODIFIED_FILES=$(echo "$GIT_DIFF_OUTPUT" | xargs)
     - echo "MODIFIED_FILES=$MODIFIED_FILES" >> pipeline.env
-    - echo "REPORT_EXIT_CODE=0" >> pipeline.env
-    # MR_MODIFIED_FILES and MR_MODIFIED_COMPONENTS are semicolon separated lists that is used in MR only
-    # for non MR pipeline, these are empty lists
-    - |
-      if [ $IS_MR_PIPELINE == "0" ]; then
-        echo "MR_MODIFIED_FILES=\"\"" >> pipeline.env
-        echo "MR_MODIFIED_COMPONENTS=\"\"" >> pipeline.env
-      else
-        MR_MODIFIED_FILES=$(echo "$GIT_DIFF_OUTPUT" | tr '\n' ';')
-        echo "MR_MODIFIED_FILES=\"$MR_MODIFIED_FILES\"" >> pipeline.env
-
-        MR_MODIFIED_COMPONENTS=$(run_cmd python tools/ci/ci_get_mr_info.py components --modified-files $MODIFIED_FILES | tr '\n' ';')
-        echo "MR_MODIFIED_COMPONENTS=\"$MR_MODIFIED_COMPONENTS\"" >> pipeline.env
-      fi
+    - echo "MODIFIED_COMPONENTS=$(run_cmd python tools/ci/ci_get_mr_info.py components --modified-files $MODIFIED_FILES | xargs)" >> pipeline.env
     - |
       if echo "$CI_MERGE_REQUEST_LABELS" | egrep "(^|,)BUILD_AND_TEST_ALL_APPS(,|$)"; then
         echo "BUILD_AND_TEST_ALL_APPS=1" >> pipeline.env
       fi
-    # run full pipeline if testing constraint branch
-    - |
-      if [ -n "$CI_PYTHON_CONSTRAINT_BRANCH" ]; then
-        echo "BUILD_AND_TEST_ALL_APPS=1" >> pipeline.env
-      fi
-    - python tools/ci/ci_process_description.py
     - cat pipeline.env
     - python tools/ci/artifacts_handler.py upload --type modified_files_and_components_report
   artifacts:
     reports:
       dotenv: pipeline.env
-    paths:
-      - pipeline.env
-    expire_in: 1 week
-    when: always
-
-baseline_manifest_sha:
-  extends:
-    - .pre_check_template
-    - .rules:dev-push
-  tags: [fast_run, shiny]
-  script:
-    - |
-      # merged results pipelines, by default
-      # diff between target-branch-head and merged-result-head
-      if [ -n "$CI_MERGE_REQUEST_TARGET_BRANCH_SHA" ]; then
-        git fetch origin $CI_MERGE_REQUEST_TARGET_BRANCH_SHA --depth=1
-        git checkout FETCH_HEAD
-        idf-build-apps dump-manifest-sha \
-          --manifest-files $(find . -name ".build-test-rules.yml" | xargs) \
-          --output .manifest_sha
-      # merge request pipelines, when the mr got conflicts
-      # diff between diff-base-sha and merge-request-head
-      elif [ -n "$CI_MERGE_REQUEST_DIFF_BASE_SHA" ]; then
-        git fetch origin $CI_MERGE_REQUEST_DIFF_BASE_SHA --depth=1
-        git checkout FETCH_HEAD
-        idf-build-apps dump-manifest-sha \
-          --manifest-files $(find . -name ".build-test-rules.yml" | xargs) \
-          --output .manifest_sha
-      # other pipelines, like the protected branches pipelines
-      # not triggered in this job
-      fi
-  artifacts:
-    paths:
-      - .manifest_sha
-    expire_in: 1 week
-    when: always
-
-redundant_pass_job:
-  stage: pre_check
-  tags: [shiny, fast_run]
-  image: $ESP_ENV_IMAGE
-  dependencies: null
-  before_script: []
-  cache: []
-  extends: []
-  script:
-    - echo "This job is redundant to ensure the 'retry_failed_jobs' job can exist and not be skipped"
-  when: always
+    expire_in: 4 days

.gitlab/ci/pre_commit.yml

@@ -1,80 +0,0 @@
-.check_pre_commit_template:
-  extends:
-    - .before_script:minimal
-  stage: pre_check
-  image: $PRE_COMMIT_IMAGE
-  tags: [cache, shiny]
-  needs:
-    - pipeline_variables
-  variables:
-    # cache pre_commit
-    PRE_COMMIT_HOME: "$CI_PROJECT_DIR/.cache/pre-commit"
-  script:
-    - fetch_submodules
-    - pre-commit run --files $MODIFIED_FILES
-    - pre-commit run --hook-stage post-commit validate-sbom-manifest
-
-check_pre_commit_upload_cache:
-  extends:
-    - .check_pre_commit_template
-  rules:
-    - if: '($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_BRANCH =~ /^release\/v/) && $CI_PIPELINE_SOURCE == "push"'
-  cache:
-    - key: pre_commit-cache-${LATEST_GIT_TAG}
-      paths:
-        - .cache/pre-commit
-      policy: pull-push
-    - key: submodule-cache-${LATEST_GIT_TAG}
-      paths:
-        - .cache/submodule_archives
-      policy: pull
-
-check_pre_commit:
-  extends:
-    - .check_pre_commit_template
-  rules:
-    - if: '($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_BRANCH =~ /^release\/v/) && $CI_PIPELINE_SOURCE == "push"'
-      when: never
-    - when: on_success
-  cache:
-    - key: pre_commit-cache-${LATEST_GIT_TAG}
-      paths:
-        - .cache/pre-commit
-      policy: pull
-    - key: submodule-cache-${LATEST_GIT_TAG}
-      paths:
-        - .cache/submodule_archives
-      policy: pull
-
-check_powershell:
-  extends:
-    - .before_script:minimal
-  stage: pre_check
-  image: docker:latest
-  services:
-    - docker:dind
-  tags:
-    - dind
-    - amd64
-  needs:
-    - pipeline_variables
-  variables:
-    # cache pre_commit
-    PRE_COMMIT_HOME: "$CI_PROJECT_DIR/.cache/pre-commit"
-  rules:
-    - changes:
-        - "*.ps1"
-  script:
-    - apk add python3
-    - apk add py3-pip
-    - pip install pre-commit --break-system-packages
-    - pre-commit run --hook-stage manual check-powershell-scripts --files $MODIFIED_FILES
-  cache:
-    - key: pre_commit-cache-${LATEST_GIT_TAG}
-      paths:
-        - .cache/pre-commit
-      policy: pull
-    - key: submodule-cache-${LATEST_GIT_TAG}
-      paths:
-        - .cache/submodule_archives
-      policy: pull

.gitlab/ci/retry_failed_jobs.yml

@@ -1,15 +0,0 @@
-retry_failed_jobs:
-  stage: retry_failed_jobs
-  tags: [shiny, fast_run]
-  allow_failure: true
-  image: $ESP_ENV_IMAGE
-  dependencies: null
-  before_script: []
-  cache: []
-  extends: []
-  script:
-    - echo "Retrieving and retrying all failed jobs for the pipeline..."
-    - python tools/ci/python_packages/gitlab_api.py retry_failed_jobs $CI_MERGE_REQUEST_PROJECT_ID --pipeline_id $CI_PIPELINE_ID
-  when: manual
-  needs:
-    - redundant_pass_job

.gitlab/ci/static-code-analysis.yml

@@ -6,8 +6,8 @@ clang_tidy_check:
   artifacts:
     paths:
       - clang_tidy_reports/
-    expire_in: 1 week
     when: always
+    expire_in: 1 day
   variables:
     IDF_TOOLCHAIN: clang
   script:
@@ -23,12 +23,10 @@ check_pylint:
   needs:
     - pipeline_variables
   artifacts:
+    when: always
     reports:
       codequality: pylint.json
-    paths:
-      - pylint.json
     expire_in: 1 week
-    when: always
   script:
     - |
       if [ -n "$CI_MERGE_REQUEST_IID" ]; then
@@ -38,84 +36,82 @@ check_pylint:
       fi
     - if [ -z "$files" ]; then echo "No python files found"; exit 0; fi
     - run_cmd pylint --exit-zero --load-plugins=pylint_gitlab --output-format=gitlab-codeclimate:pylint.json $files
+
+# build stage
+# Sonarqube related jobs put here for this reason:
+# Here we have two jobs. code_quality_check and code_quality_report.
 #
-## build stage
-## Sonarqube related jobs put here for this reason:
-## Here we have two jobs. code_quality_check and code_quality_report.
-##
-## code_quality_check will analyze the code changes between your MR and
-## code repo stored in sonarqube server. The analysis result is only shown in
-## the comments under this MR and won't be transferred to the server.
-##
-## code_quality_report will analyze and transfer both of the newly added code
-## and the analysis result to the server.
-##
-## Put in the front to ensure that the newly merged code can be stored in
-## sonarqube server ASAP, in order to avoid reporting unrelated code issues
-#.sonar_scan_template:
-#  stage: build
-#  extends: .pre_check_template
-#  # full clone since this image does not support fetch --shallow-since-cutoff
-#  # shiny runners are used for full clone
-#  tags: [build, shiny]
-#  image: $SONARQUBE_SCANNER_IMAGE
-#  before_script:
-#    - source tools/ci/utils.sh
-#    - export PYTHONPATH="$CI_PROJECT_DIR/tools:$CI_PROJECT_DIR/tools/ci/python_packages:$PYTHONPATH"
-#    - fetch_submodules
-#    # Exclude the submodules, all paths ends with /**
-#    - submodules=$(get_all_submodules)
-#    # get all exclude paths specified in tools/ci/sonar_exclude_list.txt | ignore lines start with # | xargs | replace all <space> to <comma>
-#    - custom_excludes=$(cat $CI_PROJECT_DIR/tools/ci/sonar_exclude_list.txt | grep -v '^#' | xargs | sed -e 's/ /,/g')
-#    # Exclude the report dir as well
-#    - export EXCLUSIONS="$custom_excludes,$submodules"
-#    - export SONAR_SCANNER_OPTS="-Xmx2048m"
-#  variables:
-#    GIT_DEPTH: 0
-#    REPORT_PATTERN: clang_tidy_reports/**/*.txt
-#  artifacts:
-#    paths:
-#      - $REPORT_PATTERN
-#    expire_in: 1 week
-#    when: always
-#  dependencies:  # Here is not a hard dependency relationship, could be skipped when only python files changed. so we do not use "needs" here.
-#    - clang_tidy_check
+# code_quality_check will analyze the code changes between your MR and
+# code repo stored in sonarqube server. The analysis result is only shown in
+# the comments under this MR and won't be transferred to the server.
 #
-#code_quality_check:
-#  extends:
-#    - .sonar_scan_template
-#    - .rules:patterns:static-code-analysis-preview
-#  allow_failure: true  # it's using exit code to indicate the code analysis result,
-#                       # we don't want to block ci when critical issues founded
-#  script:
-#    - export CI_MERGE_REQUEST_COMMITS=$(python ${CI_PROJECT_DIR}/tools/ci/ci_get_mr_info.py commits --src-branch ${CI_COMMIT_REF_NAME} | tr '\n' ',')
-#    # test if this branch have merge request, if not, exit 0
-#    - test -n "$CI_MERGE_REQUEST_IID" || exit 0
-#    - test -n "$CI_MERGE_REQUEST_COMMITS" || exit 0
-#    - sonar-scanner
-#      -Dsonar.analysis.mode=preview
-#      -Dsonar.branch.name=$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME
-#      -Dsonar.cxx.clangtidy.reportPath=$REPORT_PATTERN
-#      -Dsonar.exclusions=$EXCLUSIONS
-#      -Dsonar.gitlab.ci_merge_request_iid=$CI_MERGE_REQUEST_IID
-#      -Dsonar.gitlab.commit_sha=$CI_MERGE_REQUEST_COMMITS
-#      -Dsonar.gitlab.merge_request_discussion=true
-#      -Dsonar.gitlab.ref_name=$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME
-#      -Dsonar.host.url=$SONAR_HOST_URL
-#      -Dsonar.login=$SONAR_LOGIN
+# code_quality_report will analyze and transfer both of the newly added code
+# and the analysis result to the server.
 #
-#code_quality_report:
-#  extends:
-#    - .sonar_scan_template
-#    - .rules:protected
-#  allow_failure: true  # it's using exit code to indicate the code analysis result,
-#                       # we don't want to block ci when critical issues founded
-#  script:
-#    - sonar-scanner
-#      -Dsonar.branch.name=$CI_COMMIT_REF_NAME
-#      -Dsonar.cxx.clangtidy.reportPath=$REPORT_PATTERN
-#      -Dsonar.exclusions=$EXCLUSIONS
-#      -Dsonar.gitlab.commit_sha=$PIPELINE_COMMIT_SHA
-#      -Dsonar.gitlab.ref_name=$CI_COMMIT_REF_NAME
-#      -Dsonar.host.url=$SONAR_HOST_URL
-#      -Dsonar.login=$SONAR_LOGIN
+# Put in the front to ensure that the newly merged code can be stored in
+# sonarqube server ASAP, in order to avoid reporting unrelated code issues
+.sonar_scan_template:
+  stage: build
+  extends: .pre_check_template
+  image:
+    name: $SONARQUBE_SCANNER_IMAGE
+  before_script:
+    - source tools/ci/utils.sh
+    - export PYTHONPATH="$CI_PROJECT_DIR/tools:$CI_PROJECT_DIR/tools/ci/python_packages:$PYTHONPATH"
+    - fetch_submodules
+    # Exclude the submodules, all paths ends with /**
+    - submodules=$(get_all_submodules)
+    # get all exclude paths specified in tools/ci/sonar_exclude_list.txt | ignore lines start with # | xargs | replace all <space> to <comma>
+    - custom_excludes=$(cat $CI_PROJECT_DIR/tools/ci/sonar_exclude_list.txt | grep -v '^#' | xargs | sed -e 's/ /,/g')
+    # Exclude the report dir as well
+    - export EXCLUSIONS="$custom_excludes,$submodules"
+    - export SONAR_SCANNER_OPTS="-Xmx2048m"
+  variables:
+    GIT_DEPTH: 0
+    REPORT_PATTERN: clang_tidy_reports/*.txt
+  artifacts:
+    when: always
+    paths:
+      - $REPORT_PATTERN
+    expire_in: 1 week
+  dependencies:  # Here is not a hard dependency relationship, could be skipped when only python files changed. so we do not use "needs" here.
+    - clang_tidy_check
+
+code_quality_check:
+  extends:
+    - .sonar_scan_template
+    - .rules:patterns:static-code-analysis-preview
+  allow_failure: true  # since now it's using exit code to indicate the code analysis result,
+                       # we don't want to block ci when critical issues founded
+  script:
+    - export CI_MERGE_REQUEST_COMMITS=$(python ${CI_PROJECT_DIR}/tools/ci/ci_get_mr_info.py commits --src-branch ${CI_COMMIT_REF_NAME} | tr '\n' ',')
+    # test if this branch have merge request, if not, exit 0
+    - test -n "$CI_MERGE_REQUEST_IID" || exit 0
+    - test -n "$CI_MERGE_REQUEST_COMMITS" || exit 0
+    - sonar-scanner
+      -Dsonar.analysis.mode=preview
+      -Dsonar.branch.name=$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME
+      -Dsonar.cxx.clangtidy.reportPath=$REPORT_PATTERN
+      -Dsonar.exclusions=$EXCLUSIONS
+      -Dsonar.gitlab.ci_merge_request_iid=$CI_MERGE_REQUEST_IID
+      -Dsonar.gitlab.commit_sha=$CI_MERGE_REQUEST_COMMITS
+      -Dsonar.gitlab.merge_request_discussion=true
+      -Dsonar.gitlab.ref_name=$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME
+      -Dsonar.host.url=$SONAR_HOST_URL
+      -Dsonar.login=$SONAR_LOGIN
+
+code_quality_report:
+  extends:
+    - .sonar_scan_template
+    - .rules:protected
+  allow_failure: true  # since now it's using exit code to indicate the code analysis result,
+                       # we don't want to block ci when critical issues founded
+  script:
+    - sonar-scanner
+      -Dsonar.branch.name=$CI_COMMIT_REF_NAME
+      -Dsonar.cxx.clangtidy.reportPath=$REPORT_PATTERN
+      -Dsonar.exclusions=$EXCLUSIONS
+      -Dsonar.gitlab.commit_sha=$PIPELINE_COMMIT_SHA
+      -Dsonar.gitlab.ref_name=$CI_COMMIT_REF_NAME
+      -Dsonar.host.url=$SONAR_HOST_URL
+      -Dsonar.login=$SONAR_LOGIN

.gitlab/ci/test-win.yml

@@ -1,96 +0,0 @@
-# Host tests
-.host_test_win_template:
-  extends: .rules:test:host_test
-  stage: host_test
-  image: $ESP_ENV_IMAGE
-  tags:
-    - windows-build
-  dependencies: # set dependencies to null to avoid missing artifacts issue
-  # run host_test jobs immediately, only after upload cache
-  needs:
-    - job: upload-pip-cache
-      optional: true
-      artifacts: false
-    - job: upload-submodules-cache
-      optional: true
-      artifacts: false
-  before_script: []
-  after_script: []
-
-test_cli_installer_win:
-  extends:
-    - .host_test_win_template
-    - .rules:labels:windows_pytest_build_system
-  artifacts:
-    when: on_failure
-    paths:
-      - tools/tools.new.json
-      - tools/test_idf_tools/test_python_env_logs.txt
-    expire_in: 1 week
-  variables:
-    IDF_PATH: "$CI_PROJECT_DIR"
-  script:
-    # Tools must be downloaded for testing
-    - python ${IDF_PATH}\tools\idf_tools.py download required qemu-riscv32 qemu-xtensa cmake
-    - cd ${IDF_PATH}\tools\test_idf_tools
-    - python -m pip install jsonschema
-    - python .\test_idf_tools.py
-    - python .\test_idf_tools_python_env.py
-
-test_tools_win:
-  extends:
-    - .host_test_win_template
-    - .rules:labels:windows_pytest_build_system
-  artifacts:
-    paths:
-      - ${IDF_PATH}/*.out
-      - ${IDF_PATH}/XUNIT_*.xml
-    reports:
-      junit: ${IDF_PATH}/XUNIT_*.xml
-    expire_in: 1 week
-    when: always
-  variables:
-    LC_ALL: C.UTF-8
-    PYTHONPATH: "$PYTHONPATH;$IDF_PATH\\tools;$IDF_PATH\\tools\\esp_app_trace;$IDF_PATH\\components\\partition_table;$IDF_PATH\\tools\\ci\\python_packages"
-  script:
-    - python -m pip install jsonschema
-    - .\install.ps1 --enable-ci --enable-pytest
-    - .\export.ps1
-    - python "${SUBMODULE_FETCH_TOOL}" -s "all"
-    - cd ${IDF_PATH}/tools/test_idf_py
-    - pytest --noconftest test_idf_py.py       --junitxml=${IDF_PATH}/XUNIT_IDF_PY.xml
-    - pytest --noconftest test_hints.py        --junitxml=${IDF_PATH}/XUNIT_HINTS.xml
-
-# Build tests
-.test_build_system_template_win:
-  stage: host_test
-  variables:
-  # Enable ccache for all build jobs. See configure_ci_environment.sh for more ccache related settings.
-    IDF_CCACHE_ENABLE: "1"
-    PYTHONPATH: "$PYTHONPATH;$IDF_PATH\\tools;$IDF_PATH\\tools\\esp_app_trace;$IDF_PATH\\components\\partition_table;$IDF_PATH\\tools\\ci\\python_packages"
-  before_script: []
-  after_script: []
-  timeout: 4 hours
-  script:
-    - .\install.ps1 --enable-ci --enable-pytest
-    - . .\export.ps1
-    - python "${SUBMODULE_FETCH_TOOL}" -s "all"
-    - cd ${IDF_PATH}\tools\test_build_system
-    - pytest --parallel-count ${CI_NODE_TOTAL} --parallel-index ${CI_NODE_INDEX} --junitxml=${CI_PROJECT_DIR}\XUNIT_RESULT.xml
-
-pytest_build_system_win:
-  extends:
-    - .test_build_system_template_win
-    - .rules:labels:windows_pytest_build_system
-  parallel: 2
-  needs: []
-  tags:
-    - windows-build
-  artifacts:
-    paths:
-      - XUNIT_RESULT.xml
-      - test_build_system
-    expire_in: 2 days
-    reports:
-      junit: XUNIT_RESULT.xml
-    when: always

.gitlab/ci/upload_cache.yml

@@ -10,12 +10,12 @@ upload-pip-cache:
   extends:
     - .upload_cache_template
     - .before_script:minimal
-    - .rules:upload-python-cache
+    - .rules:patterns:python-cache
   tags:
     - $GEO
     - cache
   cache:
-    key: pip-cache-${LATEST_GIT_TAG}
+    key: pip-cache
     paths:
       - .cache/pip
     policy: push
@@ -30,12 +30,12 @@ upload-submodules-cache:
   extends:
     - .upload_cache_template
     - .before_script:minimal
-    - .rules:upload-submodule-cache
+    - .rules:patterns:submodule
   tags:
     - $GEO
     - cache
   cache:
-    key: submodule-cache-${LATEST_GIT_TAG}
+    key: submodule-cache
     paths:
       - .cache/submodule_archives
     policy: push
@@ -48,3 +48,27 @@ upload-submodules-cache:
   parallel:
     matrix:
       - GEO: [ 'shiny',  'brew' ]
+
+upload-danger-npm-cache:
+  stage: upload_cache
+  image: node:18.15.0-alpine3.16
+  extends:
+    - .rules:patterns:dangerjs
+  tags:
+    - $GEO
+    - cache
+  cache:
+    key:
+      files:
+        - .gitlab/dangerjs/package-lock.json
+    paths:
+      - .gitlab/dangerjs/node_modules/
+    policy: push
+  before_script:
+    - echo "Skip before scripts ...."
+  script:
+    - cd .gitlab/dangerjs
+    - npm install --no-progress --no-update-notifier
+  parallel:
+    matrix:
+      - GEO: [ 'shiny', 'brew' ]

.gitlab/dangerjs/aiGenerateGitMessage.js

@@ -0,0 +1,172 @@
+const {
+    minimumSummaryChars,
+    maximumSummaryChars,
+    maximumBodyLineChars,
+    allowedTypes,
+} = require("./mrCommitsConstants.js");
+const { gptStandardModelTokens } = require("./mrCommitsConstants.js");
+
+const { ChatPromptTemplate } = require("langchain/prompts");
+const { SystemMessagePromptTemplate } = require("langchain/prompts");
+const { LLMChain } = require("langchain/chains");
+const { ChatOpenAI } = require("langchain/chat_models/openai");
+const openAiTokenCount = require("openai-gpt-token-counter");
+
+module.exports = async function () {
+    let outputDangerMessage = `\n\nPerhaps you could use an AI-generated suggestion for your commit message. Here is one `;
+
+    let mrDiff = await getMrGitDiff(danger.git.modified_files);
+    const mrCommitMessages = getCommitMessages(danger.gitlab.commits);
+    const inputPrompt = getInputPrompt();
+    const inputLlmTokens = getInputLlmTokens(
+        inputPrompt,
+        mrDiff,
+        mrCommitMessages
+    );
+    console.log(`Input tokens for LLM: ${inputLlmTokens}`);
+
+    if (inputLlmTokens >= gptStandardModelTokens) {
+        mrDiff = ""; // If the input mrDiff is larger than 16k model, don't use mrDiff, use only current commit messages
+        outputDangerMessage += `(based only on your current commit messages, git-diff of this MR is too big (${inputLlmTokens} tokens) for the AI models):\n\n`;
+    } else {
+        outputDangerMessage += `(based on your MR git-diff and your current commit messages):\n\n`;
+    }
+
+    // Generate AI commit message
+    let generatedCommitMessage = "";
+    try {
+        const rawCommitMessage = await createAiGitMessage(
+            inputPrompt,
+            mrDiff,
+            mrCommitMessages
+        );
+        generatedCommitMessage = postProcessCommitMessage(rawCommitMessage);
+    } catch (error) {
+        console.error("Error in generating AI commit message: ", error);
+        outputDangerMessage +=
+            "\nCould not generate commit message due to an error.\n";
+    }
+
+    // Append closing statements ("Closes https://github.com/espressif/esp-idf/issues/XXX") to the generated commit message
+    let closingStatements = extractClosingStatements(mrCommitMessages);
+    if (closingStatements.length > 0) {
+        generatedCommitMessage += "\n\n" + closingStatements;
+    }
+
+    // Add the generated git message, format to the markdown code block
+    outputDangerMessage += `\n\`\`\`\n${generatedCommitMessage}\n\`\`\`\n`;
+    outputDangerMessage +=
+        "\n**NOTE: AI-generated suggestions may not always be correct, please review the suggestion before using it.**"; // Add disclaimer
+    return outputDangerMessage;
+};
+
+async function getMrGitDiff(mrModifiedFiles) {
+    const fileDiffs = await Promise.all(
+        mrModifiedFiles.map((file) => danger.git.diffForFile(file))
+    );
+    return fileDiffs.map((fileDiff) => fileDiff.diff.trim()).join(" ");
+}
+
+function getCommitMessages(mrCommits) {
+    return mrCommits.map((commit) => commit.message);
+}
+
+function getInputPrompt() {
+    return `You are a helpful assistant that creates suggestions for single git commit message, that user can use to describe all the changes in their merge request.
+    Use git diff: {mrDiff} and users current commit messages: {mrCommitMessages} to get the changes made in the commit.
+
+    Output should be git commit message following the conventional commit format.
+
+    Output only git commit message in desired format, without comments and other text.
+
+    Do not include the closing statements ("Closes https://....") in the output.
+
+    Here are the strict rules you must follow:
+
+    - Avoid mentioning any JIRA tickets (e.g., "Closes JIRA-123").
+    - Be specific. Don't use vague terms (e.g., "some checks", "add new ones", "few changes").
+    - The commit message structure should be:  <type><(scope/component)>: <summary>
+    - Types allowed: ${allowedTypes.join(", ")}
+    - If 'scope/component' is used, it must start with a lowercase letter.
+    - The 'summary' must NOT end with a period.
+    - The 'summary' must be between ${minimumSummaryChars} and ${maximumSummaryChars} characters long.
+
+    If a 'body' of commit message is used:
+
+    - Each line must be no longer than ${maximumBodyLineChars} characters.
+    - It must be separated from the 'summary' by a blank line.
+
+    Examples of correct commit messages:
+
+        - With scope and body:
+            fix(freertos): Fix startup timeout issue
+
+            This is a text of commit message body...
+            - adds support for wifi6
+            - adds validations for logging script
+
+        - Without scope and body:
+            ci: added target test job for ESP32-Wifi6`;
+}
+
+function getInputLlmTokens(inputPrompt, mrDiff, mrCommitMessages) {
+    const mrCommitMessagesTokens = openAiTokenCount(mrCommitMessages.join(" "));
+    const gitDiffTokens = openAiTokenCount(mrDiff);
+    const promptTokens = openAiTokenCount(inputPrompt);
+    return mrCommitMessagesTokens + gitDiffTokens + promptTokens;
+}
+
+async function createAiGitMessage(inputPrompt, mrDiff, mrCommitMessages) {
+    const chat = new ChatOpenAI({ engine: "gpt-3.5-turbo", temperature: 0 });
+    const chatPrompt = ChatPromptTemplate.fromPromptMessages([
+        SystemMessagePromptTemplate.fromTemplate(inputPrompt),
+    ]);
+    const chain = new LLMChain({ prompt: chatPrompt, llm: chat });
+
+    const response = await chain.call({
+        mrDiff: mrDiff,
+        mrCommitMessages: mrCommitMessages,
+    });
+    return response.text;
+}
+
+function postProcessCommitMessage(rawCommitMessage) {
+    // Split the result into lines
+    let lines = rawCommitMessage.split("\n");
+
+    // Format each line
+    for (let i = 0; i < lines.length; i++) {
+        let line = lines[i].trim();
+
+        // If the line is longer than maximumBodyLineChars, split it into multiple lines
+        if (line.length > maximumBodyLineChars) {
+            let newLines = [];
+            while (line.length > maximumBodyLineChars) {
+                let lastSpaceIndex = line.lastIndexOf(
+                    " ",
+                    maximumBodyLineChars
+                );
+                newLines.push(line.substring(0, lastSpaceIndex));
+                line = line.substring(lastSpaceIndex + 1);
+            }
+            newLines.push(line);
+            lines[i] = newLines.join("\n");
+        }
+    }
+
+    // Join the lines back into a single string with a newline between each one
+    return lines.join("\n");
+}
+
+function extractClosingStatements(mrCommitMessages) {
+    let closingStatements = [];
+    mrCommitMessages.forEach((message) => {
+        const lines = message.split("\n");
+        lines.forEach((line) => {
+            if (line.startsWith("Closes")) {
+                closingStatements.push(line);
+            }
+        });
+    });
+    return closingStatements.join("\n");
+}

.gitlab/dangerjs/configParameters.js

@@ -0,0 +1,56 @@
+let outputStatuses = [];
+
+/**
+ * Logs the status of a rule with padded formatting and stores it in the `outputStatuses` array.
+ * If the rule already exists in the array, its status is updated.
+ * @param message The name of the rule
+ * @param status The output (exit) status of the rule
+ */
+function recordRuleExitStatus(message, status) {
+    // Check if the rule already exists in the array
+    const existingRecord = outputStatuses.find(
+        (rule) => rule.message === message
+    );
+
+    if (existingRecord) {
+        // Update the status of the existing rule
+        existingRecord.status = status;
+    } else {
+        // If the rule doesn't exist, add it to the array
+        outputStatuses.push({ message, status });
+    }
+}
+
+/**
+ * Displays all the rule output statuses stored in the `outputStatuses` array.
+ * Filters out any empty lines, sorts them alphabetically, and prints the statuses
+ * with a header and separator.
+ * These statuses are later displayed in CI job tracelog.
+ */
+function displayAllOutputStatuses() {
+    const lineLength = 100;
+    const sortedStatuses = outputStatuses.sort((a, b) =>
+        a.message.localeCompare(b.message)
+    );
+
+    const formattedLines = sortedStatuses.map((statusObj) => {
+        const paddingLength =
+            lineLength - statusObj.message.length - statusObj.status.length;
+        const paddedMessage = statusObj.message.padEnd(
+            statusObj.message.length + paddingLength,
+            "."
+        );
+        return `${paddedMessage} ${statusObj.status}`;
+    });
+
+    console.log(
+        "DangerJS checks (rules) output states:\n" + "=".repeat(lineLength + 2)
+    );
+    console.log(formattedLines.join("\n"));
+    console.log("=".repeat(lineLength + 2));
+}
+
+module.exports = {
+    displayAllOutputStatuses,
+    recordRuleExitStatus,
+};

.gitlab/dangerjs/dangerfile.js

@@ -0,0 +1,51 @@
+const { displayAllOutputStatuses } = require("./configParameters.js");
+
+/*
+ * Modules with checks are stored in ".gitlab/dangerjs/<module_name>". To import them, use path relative to "dangerfile.js"
+ */
+async function runChecks() {
+    // Checks for merge request title
+    require("./mrTitleNoDraftOrWip.js")();
+
+    // Checks for merge request description
+    require("./mrDescriptionLongEnough.js")();
+    require("./mrDescriptionReleaseNotes.js")();
+    await require("./mrDescriptionJiraLinks.js")();
+
+    // Checks for documentation
+    await require("./mrDocsTranslation.js")();
+
+    // Checks for MR commits
+    require("./mrCommitsTooManyCommits.js")();
+    await require("./mrCommitsCommitMessage.js")();
+    require("./mrCommitsEmail.js")();
+
+    // Checks for MR code
+    require("./mrSizeTooLarge.js")();
+
+    // Checks for MR area labels
+    await require("./mrAreaLabels.js")();
+
+    // Checks for Source branch name
+    require("./mrSourceBranchName.js")();
+
+    // Show DangerJS individual checks statuses - visible in CI job tracelog
+    displayAllOutputStatuses();
+
+    // Add success log if no issues
+    if (
+        results.fails.length === 0 &&
+        results.warnings.length === 0 &&
+        results.messages.length === 0
+    ) {
+        return message("🎉 Good Job! All checks are passing!");
+    }
+}
+
+runChecks();
+
+// Add retry link
+const retryLink = `${process.env.DANGER_GITLAB_HOST}/${process.env.CI_PROJECT_PATH}/-/jobs/${process.env.CI_JOB_ID}`;
+markdown(
+    `***\n#### :repeat: You can enforce automatic MR checks by retrying the [DangerJS job](${retryLink})\n***`
+);

.gitlab/dangerjs/mrAreaLabels.js

@@ -0,0 +1,27 @@
+const { recordRuleExitStatus } = require("./configParameters.js");
+
+/**
+ * Check if MR has area labels (light blue labels)
+ *
+ * @dangerjs WARN
+ */
+module.exports = async function () {
+    const ruleName = "Merge request area labels";
+    const projectId = 103; // ESP-IDF
+    const areaLabelColor = /^#d2ebfa$/i; // match color code (case-insensitive)
+    const projectLabels = await danger.gitlab.api.Labels.all(projectId); // Get all project labels
+    const areaLabels = projectLabels
+        .filter((label) => areaLabelColor.test(label.color))
+        .map((label) => label.name); // Filter only area labels
+    const mrLabels = danger.gitlab.mr.labels; // Get MR labels
+
+    if (!mrLabels.some((label) => areaLabels.includes(label))) {
+        recordRuleExitStatus(ruleName, "Failed");
+        return warn(
+            `Please add some [area labels](${process.env.DANGER_GITLAB_HOST}/espressif/esp-idf/-/labels) to this MR.`
+        );
+    }
+
+    // At this point, the rule has passed
+    recordRuleExitStatus(ruleName, "Passed");
+};

.gitlab/dangerjs/mrCommitsCommitMessage.js

@@ -0,0 +1,165 @@
+const {
+    minimumSummaryChars,
+    maximumSummaryChars,
+    maximumBodyLineChars,
+    allowedTypes,
+} = require("./mrCommitsConstants.js");
+const { recordRuleExitStatus } = require("./configParameters.js");
+
+/**
+ * Check that commit messages are based on the Espressif ESP-IDF project's rules for git commit messages.
+ *
+ * @dangerjs WARN
+ */
+module.exports = async function () {
+    const ruleName = "Commit messages style";
+    const mrCommits = danger.gitlab.commits;
+    const lint = require("@commitlint/lint").default;
+
+    const lintingRules = {
+        // rule definition: [(0-1 = off/on), (always/never = must be/mustn't be), (value)]
+        "body-max-line-length": [1, "always", maximumBodyLineChars], // Max length of the body line
+        "footer-leading-blank": [1, "always"], // Always have a blank line before the footer section
+        "footer-max-line-length": [1, "always", maximumBodyLineChars], // Max length of the footer line
+        "subject-max-length": [1, "always", maximumSummaryChars], // Max length of the "Summary"
+        "subject-min-length": [1, "always", minimumSummaryChars], // Min length of the "Summary"
+        "scope-case": [1, "always", "lower-case"], // "scope/component" must start with lower-case
+        "subject-full-stop": [1, "never", "."], // "Summary" must not end with a full stop (period)
+        "subject-empty": [1, "never"], // "Summary" is mandatory
+        "type-case": [1, "always", "lower-case"], // "type/action" must start with lower-case
+        "type-empty": [1, "never"], // "type/action" is mandatory
+        "type-enum": [1, "always", allowedTypes], // "type/action" must be one of the allowed types
+        "body-leading-blank": [1, "always"], // Always have a blank line before the body section
+    };
+
+    // Switcher for AI suggestions (for poor messages)
+    let generateAISuggestion = false;
+
+    // Search for the messages in each commit
+    let issuesAllCommitMessages = [];
+
+    for (const commit of mrCommits) {
+        const commitMessage = commit.message;
+        const commitMessageTitle = commit.title;
+
+        let issuesSingleCommitMessage = [];
+        let reportSingleCommitMessage = "";
+
+        // Check if the commit message contains any Jira ticket references
+        const jiraTicketRegex = /[A-Z0-9]+-[0-9]+/g;
+        const jiraTicketMatches = commitMessage.match(jiraTicketRegex);
+        if (jiraTicketMatches) {
+            const jiraTicketNames = jiraTicketMatches.join(", ");
+            issuesSingleCommitMessage.push(
+                `- probably contains Jira ticket reference (\`${jiraTicketNames}\`). Please remove Jira tickets from commit messages.`
+            );
+        }
+
+        // Lint commit messages with @commitlint (Conventional Commits style)
+        const result = await lint(commit.message, lintingRules);
+
+        for (const warning of result.warnings) {
+            // Custom messages for each rule with terminology used by Espressif conventional commits guide
+            switch (warning.name) {
+                case "subject-max-length":
+                    issuesSingleCommitMessage.push(
+                        `- *summary* appears to be too long`
+                    );
+                    break;
+                case "type-empty":
+                    issuesSingleCommitMessage.push(
+                        `- *type/action* looks empty`
+                    );
+                    break;
+                case "type-case":
+                    issuesSingleCommitMessage.push(
+                        `- *type/action* should start with a lowercase letter`
+                    );
+
+                    break;
+                case "scope-empty":
+                    issuesSingleCommitMessage.push(
+                        `- *scope/component* looks empty`
+                    );
+                    break;
+                case "scope-case":
+                    issuesSingleCommitMessage.push(
+                        `- *scope/component* should be lowercase without whitespace, allowed special characters are \`_\` \`/\` \`.\` \`,\` \`*\` \`-\` \`.\``
+                    );
+                    break;
+                case "subject-empty":
+                    issuesSingleCommitMessage.push(`- *summary* looks empty`);
+                    generateAISuggestion = true;
+                    break;
+                case "subject-min-length":
+                    issuesSingleCommitMessage.push(
+                        `- *summary* looks too short`
+                    );
+                    generateAISuggestion = true;
+                    break;
+                case "subject-case":
+                    issuesSingleCommitMessage.push(
+                        `- *summary* should start with a capital letter`
+                    );
+                    break;
+                case "subject-full-stop":
+                    issuesSingleCommitMessage.push(
+                        `- *summary* should not end with a period (full stop)`
+                    );
+                    break;
+                case "type-enum":
+                    issuesSingleCommitMessage.push(
+                        `- *type/action* should be one of [${allowedTypes
+                            .map((type) => `\`${type}\``)
+                            .join(", ")}]`
+                    );
+                    break;
+
+                default:
+                    issuesSingleCommitMessage.push(`- ${warning.message}`);
+            }
+        }
+
+        if (issuesSingleCommitMessage.length) {
+            reportSingleCommitMessage = `- the commit message \`"${commitMessageTitle}"\`:\n${issuesSingleCommitMessage
+                .map((message) => `  ${message}`) // Indent each issue by 2 spaces
+                .join("\n")}`;
+            issuesAllCommitMessages.push(reportSingleCommitMessage);
+        }
+    }
+
+    // Create report
+    if (issuesAllCommitMessages.length) {
+        issuesAllCommitMessages.sort();
+        const basicTips = [
+            `- correct format of commit message should be: \`<type/action>(<scope/component>): <summary>\`, for example \`fix(esp32): Fixed startup timeout issue\``,
+            `- allowed types are: \`${allowedTypes}\``,
+            `- sufficiently descriptive message summary should be between ${minimumSummaryChars} to ${maximumSummaryChars} characters and start with upper case letter`,
+            `- avoid Jira references in commit messages (unavailable/irrelevant for our customers)`,
+            `- follow this [commit messages guide](${process.env.DANGER_GITLAB_HOST}/espressif/esp-idf/-/wikis/dev-proc/Commit-messages)`,
+        ];
+        let dangerMessage = `\n**Some issues found for the commit messages in this MR:**\n${issuesAllCommitMessages.join(
+            "\n"
+        )}
+			\n***
+			\n**Please consider updating these commit messages** - here are some basic tips:\n${basicTips.join(
+                "\n"
+            )}
+            \n \`TIP:\` You can install commit-msg pre-commit hook (\`pre-commit install -t pre-commit -t commit-msg\`) to run this check when committing.
+            \n***
+            `;
+
+        if (generateAISuggestion) {
+            // Create AI generated suggestion for git commit message based of gitDiff and current commit messages
+            const AImessageSuggestion =
+                await require("./aiGenerateGitMessage.js")();
+            dangerMessage += AImessageSuggestion;
+        }
+
+        recordRuleExitStatus(ruleName, "Failed");
+        return warn(dangerMessage);
+    }
+
+    // At this point, the rule has passed
+    recordRuleExitStatus(ruleName, "Passed");
+};

.gitlab/dangerjs/mrCommitsConstants.js

@@ -0,0 +1,16 @@
+module.exports = {
+    gptStandardModelTokens: 4096,
+    minimumSummaryChars: 20,
+    maximumSummaryChars: 72,
+    maximumBodyLineChars: 100,
+    allowedTypes: [
+        "change",
+        "ci",
+        "docs",
+        "feat",
+        "fix",
+        "refactor",
+        "remove",
+        "revert",
+    ],
+};

.gitlab/dangerjs/mrCommitsEmail.js

@@ -0,0 +1,23 @@
+const { recordRuleExitStatus } = require("./configParameters.js");
+
+/**
+ * Check if the author is accidentally making a commit using a personal email
+ *
+ * @dangerjs INFO
+ */
+module.exports = function () {
+    const ruleName = 'Commits from outside Espressif';
+    const mrCommitAuthorEmails = danger.gitlab.commits.map(commit => commit.author_email);
+    const mrCommitCommitterEmails = danger.gitlab.commits.map(commit => commit.committer_email);
+	const emailPattern = /.*@espressif\.com/;
+	const filteredEmails = [...mrCommitAuthorEmails, ...mrCommitCommitterEmails].filter((email) => !emailPattern.test(email));
+	if (filteredEmails.length) {
+        recordRuleExitStatus(ruleName, "Failed");
+		return message(
+			`Some of the commits were authored or committed by developers outside Espressif: ${filteredEmails.join(', ')}. Please check if this is expected.`
+		);
+	}
+
+    // At this point, the rule has passed
+	recordRuleExitStatus(ruleName, 'Passed');
+};

.gitlab/dangerjs/mrCommitsTooManyCommits.js

@@ -0,0 +1,22 @@
+const { recordRuleExitStatus } = require("./configParameters.js");
+
+/**
+ * Check if MR has not an excessive numbers of commits (if squashed)
+ *
+ * @dangerjs INFO
+ */
+module.exports = function () {
+	const ruleName = 'Number of commits in merge request';
+    const tooManyCommitThreshold = 2; // above this number of commits, squash commits is suggested
+    const mrCommits = danger.gitlab.commits;
+
+    if (mrCommits.length > tooManyCommitThreshold) {
+        recordRuleExitStatus(ruleName, "Passed (with suggestions)");
+        return message(
+            `You might consider squashing your ${mrCommits.length} commits (simplifying branch history).`
+        );
+    }
+
+    // At this point, the rule has passed
+	recordRuleExitStatus(ruleName, 'Passed');
+};

.gitlab/dangerjs/mrDescriptionJiraLinks.js

@@ -0,0 +1,238 @@
+const { recordRuleExitStatus } = require("./configParameters.js");
+
+/** Check that there are valid JIRA links in MR description.
+ *
+ * This check extracts the "Related" section from the MR description and
+ * searches for JIRA ticket references in the format "Closes [JIRA ticket key]".
+ *
+ * It then extracts the closing GitHub links from the corresponding JIRA tickets and
+ * checks if the linked GitHub issues are still in open state.
+ *
+ * Finally, it checks if the required GitHub closing links are present in the MR's commit messages.
+ *
+ */
+module.exports = async function () {
+	const ruleName = 'Jira ticket references';
+    const axios = require("axios");
+    const mrDescription = danger.gitlab.mr.description;
+    const mrCommitMessages = danger.gitlab.commits.map(
+        (commit) => commit.message
+    );
+    const jiraTicketRegex = /[A-Z0-9]+-[0-9]+/;
+
+    let partMessages = []; // Create a blank field for future records of individual issues
+
+    // Parse section "Related" from MR Description
+    const sectionRelated = extractSectionRelated(mrDescription);
+
+    if (
+        !sectionRelated.header || // No section Related in MR description or ...
+        !jiraTicketRegex.test(sectionRelated.content) // no Jira links in section Related
+    ) {
+        recordRuleExitStatus(ruleName, 'Passed (with suggestions)');
+        return message(
+            "Please consider adding references to JIRA issues in the `Related` section of the MR description."
+        );
+    }
+
+    // Get closing (only) JIRA tickets
+    const jiraTickets = findClosingJiraTickets(sectionRelated.content);
+
+    for (const ticket of jiraTickets) {
+        ticket.jiraUIUrl = `https://jira.espressif.com:8443/browse/${ticket.ticketName}`;
+
+        if (!ticket.correctFormat) {
+            partMessages.push(
+                `- closing ticket \`${ticket.record}\` seems to be in the wrong format (or inaccessible to Jira DangerBot).. The correct format is for example \`- Closes JIRA-123\`.`
+            );
+        }
+
+        // Get closing GitHub issue links from JIRA tickets
+        const closingGithubLink = await getGitHubClosingLink(ticket.ticketName);
+        if (closingGithubLink) {
+            ticket.closingGithubLink = closingGithubLink;
+        } else if (closingGithubLink === null) {
+            partMessages.push(
+                `- the Jira issue number [\`${ticket.ticketName}\`](${ticket.jiraUIUrl}) seems to be invalid (please check if the ticket number is correct)`
+            );
+            continue; // Handle unreachable JIRA tickets; skip the following checks
+        } else {
+            continue; // Jira ticket have no GitHub closing link; skip the following checks
+        }
+
+        // Get still open GitHub issues
+        const githubIssueStatusOpen = await isGithubIssueOpen(
+            ticket.closingGithubLink
+        );
+        ticket.isOpen = githubIssueStatusOpen;
+        if (githubIssueStatusOpen === null) {
+            // Handle unreachable GitHub issues
+            partMessages.push(
+                `- the GitHub issue [\`${ticket.closingGithubLink}\`](${ticket.closingGithubLink}) does not seem to exist on GitHub (referenced from JIRA ticket [\`${ticket.ticketName}\`](${ticket.jiraUIUrl}) )`
+            );
+            continue; // skip the following checks
+        }
+
+        // Search in commit message if there are all GitHub closing links (from Related section) for still open GH issues
+        if (ticket.isOpen) {
+            if (
+                !mrCommitMessages.some((item) =>
+                    item.includes(`Closes ${ticket.closingGithubLink}`)
+                )
+            ) {
+                partMessages.push(
+                    `- please add \`Closes ${ticket.closingGithubLink}\` to the commit message`
+                );
+            }
+        }
+    }
+
+    // Create report / DangerJS check feedback if issues with Jira links found
+    if (partMessages.length) {
+        createReport();
+    }
+
+    // At this point, the rule has passed
+	recordRuleExitStatus(ruleName, 'Passed');
+
+    // ---------------------------------------------------------------
+
+    /**
+     * This function takes in a string mrDescription which contains a Markdown-formatted text
+     * related to a Merge Request (MR) in a GitLab repository. It searches for a section titled "Related"
+     * and extracts the content of that section. If the section is not found, it returns an object
+     * indicating that the header and content are null. If the section is found but empty, it returns
+     * an object indicating that the header is present but the content is null. If the section is found
+     * with content, it returns an object indicating that the header is present and the content of the
+     * "Related" section.
+     *
+     * @param {string} mrDescription - The Markdown-formatted text related to the Merge Request.
+     * @returns {{
+     *  header: string | boolean | null,
+     * 	content: string | null
+     * 	}} - An object containing the header and content of the "Related" section, if present.
+     */
+
+    function extractSectionRelated(mrDescription) {
+        const regexSectionRelated = /## Related([\s\S]*?)(?=## |$)/;
+        const sectionRelated = mrDescription.match(regexSectionRelated);
+        if (!sectionRelated) {
+            return { header: null, content: null }; // Section "Related" is missing
+        }
+
+        const content = sectionRelated[1].replace(/(\r\n|\n|\r)/gm, ""); // Remove empty lines
+        if (!content.length) {
+            return { header: true, content: null }; // Section "Related" is present, but empty
+        }
+
+        return { header: true, content: sectionRelated[1] }; // Found section "Related" with content
+    }
+
+    /**
+     * Finds all JIRA tickets that are being closed in the given sectionRelatedcontent.
+     * The function searches for lines that start with - Closes and have the format Closes [uppercase letters]-[numbers].
+     * @param {string} sectionRelatedcontent - A string that contains lines with mentions of JIRA tickets
+     * @returns {Array} An array of objects with ticketName property that has the correct format
+     */
+
+    function findClosingJiraTickets(sectionRelatedcontent) {
+        let closingTickets = [];
+        const lines = sectionRelatedcontent.split("\n");
+        for (const line of lines) {
+            if (!line.startsWith("- Closes")) {
+                continue; // Not closing-type ticket, skip
+            }
+
+            const correctJiraClosingLinkFormat = new RegExp(
+                `^- Closes ${jiraTicketRegex.source}$`
+            );
+            const matchedJiraTicket = line.match(jiraTicketRegex);
+            if (matchedJiraTicket) {
+                if (!correctJiraClosingLinkFormat.test(line)) {
+                    closingTickets.push({
+                        record: line,
+                        ticketName: matchedJiraTicket[0],
+                        correctFormat: false,
+                    });
+                } else {
+                    closingTickets.push({
+                        record: line,
+                        ticketName: matchedJiraTicket[0],
+                        correctFormat: true,
+                    });
+                }
+            }
+        }
+        return closingTickets;
+    }
+
+    /**
+     * This function takes a JIRA issue key and retrieves the description from JIRA's API.
+     * It then searches the description for a GitHub closing link in the format "Closes https://github.com/owner/repo/issues/123".
+     * If a GitHub closing link is found, it is returned. If no GitHub closing link is found, it returns null.
+     * @param {string} jiraIssueKey - The key of the JIRA issue to search for the GitHub closing link.
+     * @returns {Promise<string|null>} - A promise that resolves to a string containing the GitHub closing link if found,
+     * or null if not found.
+     */
+    async function getGitHubClosingLink(jiraIssueKey) {
+        let jiraDescription = "";
+
+        // Get JIRA ticket description content
+        try {
+            const response = await axios({
+                url: `https://jira.espressif.com:8443/rest/api/latest/issue/${jiraIssueKey}`,
+                auth: {
+                    username: process.env.DANGER_JIRA_USER,
+                    password: process.env.DANGER_JIRA_PASSWORD,
+                },
+            });
+            jiraDescription = response.data.fields.description
+                ? response.data.fields.description
+                : ""; // if the Jira ticket has an unfilled Description, the ".description" property is missing in API response - in that case set "jiraDescription" to an empty string
+        } catch (error) {
+            return null;
+        }
+
+        // Find GitHub closing link in description
+        const regexClosingGhLink =
+            /Closes\s+(https:\/\/github.com\/\S+\/\S+\/issues\/\d+)/;
+        const closingGithubLink = jiraDescription.match(regexClosingGhLink);
+
+        if (closingGithubLink) {
+            return closingGithubLink[1];
+        } else {
+            return false; // Jira issue has no GitHub closing link in description
+        }
+    }
+
+    /**
+     * Check if a GitHub issue linked in a merge request is still open.
+     *
+     * @param {string} link - The link to the GitHub issue.
+     * @returns {Promise<boolean>} A promise that resolves to a boolean indicating if the issue is open.
+     * @throws {Error} If the link is invalid or if there was an error fetching the issue.
+     */
+    async function isGithubIssueOpen(link) {
+        const parsedUrl = new URL(link);
+        const [owner, repo] = parsedUrl.pathname.split("/").slice(1, 3);
+        const issueNumber = parsedUrl.pathname.split("/").slice(-1)[0];
+
+        try {
+            const response = await axios.get(
+                `https://api.github.com/repos/${owner}/${repo}/issues/${issueNumber}`
+            );
+            return response.data.state === "open"; // return True if GitHub issue is open
+        } catch (error) {
+            return null; // GET request to issue fails
+        }
+    }
+
+    function createReport() {
+        partMessages.sort();
+        let dangerMessage = `Some issues found for the related JIRA tickets in this MR:\n${partMessages.join(
+            "\n"
+        )}`;
+        recordRuleExitStatus(ruleName, "Failed");
+        return warn(dangerMessage);
+    }
+};

.gitlab/dangerjs/mrDescriptionLongEnough.js

@@ -0,0 +1,24 @@
+const { recordRuleExitStatus } = require("./configParameters.js");
+
+/**
+ * Check if MR Description has accurate description".
+ *
+ * @dangerjs WARN
+ */
+module.exports = function () {
+    const ruleName = "Merge request sufficient description";
+    const mrDescription = danger.gitlab.mr.description;
+    const descriptionChunk = mrDescription.match(/^([^#]*)/)[1].trim(); // Extract all text before the first section header (i.e., the text before the "## Release notes")
+
+    const shortMrDescriptionThreshold = 50; // Description is considered too short below this number of characters
+
+    if (descriptionChunk.length < shortMrDescriptionThreshold) {
+        recordRuleExitStatus(ruleName, "Failed");
+        return warn(
+            "The MR description looks very brief, please check if more details can be added."
+        );
+    }
+
+    // At this point, the rule has passed
+    recordRuleExitStatus(ruleName, "Passed");
+};

.gitlab/dangerjs/mrDescriptionReleaseNotes.js

@@ -0,0 +1,103 @@
+const { recordRuleExitStatus } = require("./configParameters.js");
+
+/**
+ * Check if MR Description contains mandatory section "Release notes"
+ *
+ * Extracts the content of the "Release notes" section from the GitLab merge request description.
+ *
+ * @dangerjs WARN (if section missing, is empty or wrong markdown format)
+ */
+module.exports = function () {
+    const ruleName = 'Merge request Release Notes section';
+    const mrDescription = danger.gitlab.mr.description;
+    const wiki_link = `${process.env.DANGER_GITLAB_HOST}/espressif/esp-idf/-/wikis/rfc/How-to-write-release-notes-properly`;
+
+    const regexSectionReleaseNotes = /## Release notes([\s\S]*?)(?=## |$)/;
+    const regexValidEntry = /^\s*[-*+]\s+.+/;
+    const regexNoReleaseNotes = /no release note/i;
+
+    const sectionReleaseNotes = mrDescription.match(regexSectionReleaseNotes);
+    if (!sectionReleaseNotes) {
+        recordRuleExitStatus(ruleName, "Failed");
+        return warn(`The \`Release Notes\` section seems to be missing. Please check if the section header in MR description is present and in the correct markdown format ("## Release Notes").\n\nSee [Release Notes Format Rules](${wiki_link}).`);
+    }
+
+    const releaseNotesLines = sectionReleaseNotes[1].replace(/<!--[\s\S]*?-->/g, '')
+
+    const lines = releaseNotesLines.split("\n").filter(s => s.trim().length > 0);
+    let valid_entries_found = 0;
+    let no_release_notes_found = false;
+    let violations = [];
+
+    lines.forEach((line) => {
+        if (line.match(regexValidEntry)) {
+            valid_entries_found++;
+            const error_msg = check_entry(line);
+            if (error_msg) {
+                violations.push(error_msg);
+            }
+        } else if (line.match(regexNoReleaseNotes)) {
+            no_release_notes_found = true;
+        }
+    });
+
+    let error_output = [];
+    if (violations.length > 0) {
+        error_output = [...error_output, 'Invalid release note entries:', violations.join('\n')];
+    }
+    if (no_release_notes_found) {
+        if (valid_entries_found > 0) {
+            error_output.push('`No release notes` comment shows up when there is valid entry. Remove bullets before comments in release notes section.');
+        }
+    } else {
+        if (!valid_entries_found) {
+            error_output.push('The `Release Notes` section seems to have no valid entries. Add bullets before valid entries, or add `No release notes` comment to suppress this error if you mean to have no release notes.');
+        }
+    }
+
+    if (error_output.length > 0) {
+        // Paragraphs joined by double `\n`s.
+        error_output = [...error_output, `See [Release Notes Format Guide](${wiki_link}).`].join('\n\n');
+        recordRuleExitStatus(ruleName, "Failed");
+        return warn(error_output);
+    }
+
+    // At this point, the rule has passed
+	recordRuleExitStatus(ruleName, 'Passed');
+};
+
+function check_entry(entry) {
+    const entry_str = `- \`${entry}\``;
+    const indent = "  ";
+
+    if (entry.match(/no\s+release\s+note/i)) {
+        return [entry_str, `${indent}- \`No release notes\` comment shouldn't start with bullet.`].join('\n');
+    }
+
+    // Remove a leading escaping backslash of the special characters, https://www.markdownguide.org/basic-syntax/#characters-you-can-escape
+    const escapeCharRegex = /\\([\\`*_{}[\]<>()+#-.!|])/g;
+    entry = entry.replace(escapeCharRegex, '$1');
+
+    const regex = /^(\s*)[-*+]\s+\[([^\]]+)\]\s+(.*)$/;
+    const match = regex.exec(entry);
+    if (!match) {
+        return [entry_str, `${indent}- Please specify the [area] to which the change belongs (see guide). If this line is just a comment, remove the bullet.`].join('\n');
+    }
+
+    // area is in match[2]
+    const description = match[3].trim();
+    let violations = [];
+
+    if (match[1]) {
+        violations.push(`${indent}- Release note entry should start from the beginning of line. (Nested release note not allowed.)`);
+    }
+
+    if (!/^[A-Z0-9]/.test(description)) {
+        violations.push(`${indent}- Release note statement should start with a capital letter or digit.`);
+    }
+
+    if (violations.length > 0) {
+        return [entry_str, ...violations].join('\n');
+    }
+    return null;
+}

.gitlab/dangerjs/mrDocsTranslation.js

@@ -0,0 +1,280 @@
+const { recordRuleExitStatus } = require("./configParameters.js");
+
+/**
+ * Check the documentation files in this MR.
+ *
+ * Generate an object with all docs/ files found in this MR with paths to their EN/CN versions.
+ *
+ * For common files (both language versions exist in this MR), compare the lines of both files.
+ * Ignore if the CN file is only a single line file with an "include" reference to the EN version.
+ *
+ * For files that only have a CN version in this MR, add a message to the message that an EN file also needs to be created.
+ *
+ * For a file that only has an EN version in this MR, try loading its CN version from the target Gitlab branch.
+ * If its CN version doesn't exist in the repository or it does exist,
+ * but its contents are larger than just an "include" link to the EN version (it's a full-size file),
+ * add a message to the report
+ *
+ * Create a compiled report with the docs/ files issues found and set its severity (WARN/INFO).
+ * Severity is based on the presence of "needs translation: ??" labels in this MR
+ *
+ * @dangerjs WARN (if docs translation issues in the MR)
+ * @dangerjs INFO (if docs translation issues in the MR and the user has already added translation labels).
+ * Adding translation labels "needs translation: XX" automatically notifies the Documentation team
+ *
+ * @dangerjs WARN (if there are no docs issues in MR, but translation labels have been added anyway)
+ *
+ */
+module.exports = async function () {
+    const ruleName = 'Documentation translation';
+    let partMessages = []; // Create a blank field for future records of individual issues
+    const pathProject = "espressif/esp-idf";
+    const regexIncludeLink = /\.\.\sinclude::\s((\.\.\/)+)en\//;
+    const allMrFiles = [
+        ...danger.git.modified_files,
+        ...danger.git.created_files,
+        ...danger.git.deleted_files,
+    ];
+
+    const docsFilesMR = parseMrDocsFiles(allMrFiles); // Create single object of all doc files in MR with names, paths and groups
+
+    // Both versions (EN and CN) of document found changed in this MR
+    for (const file of docsFilesMR.bothFilesInMr) {
+        file.contentEn = await getContentFileInMR(file.fileEnPath); // Get content of English file
+        file.linesEn = file.contentEn.split("\n").length; // Get number of lines of English file
+
+        file.contentCn = await getContentFileInMR(file.fileCnPath); // Get content of Chinese file
+        file.linesCn = file.contentCn.split("\n").length; // Get number of lines of English file
+
+        // Compare number of lines in both versions
+        if (file.linesEn !== file.linesCn) {
+            // Check if CN file is only link to EN file
+            if (!regexIncludeLink.test(file.contentCn)) {
+                // if not just a link ...
+                partMessages.push(
+                    `- please synchronize the EN and CN version of \`${file.fileName}\`. [\`${file.fileEnPath}\`](${file.fileUrlRepoEN}) has ${file.linesEn} lines; [\`${file.fileCnPath}\`](${file.fileUrlRepoCN}) has ${file.linesCn} lines.`
+                );
+            }
+        }
+    }
+
+    // Only Chinese version of document found changed in this MR
+    for (const file of docsFilesMR.onlyCnFilesInMr) {
+        partMessages.push(
+            `- file \`${file.fileEnPath}\` doesn't exist in this MR or in the GitLab repo. Please add \`${file.fileEnPath}\` into this MR.`
+        );
+    }
+
+    // Only English version of document found in this MR
+    for (const file of docsFilesMR.onlyEnFilesInMr) {
+        const targetBranch = danger.gitlab.mr.target_branch;
+        file.contentCn = await getContentFileInGitlab(
+            file.fileCnPath,
+            targetBranch
+        ); // Try to fetch CN file from target branch of Gitlab repository and store content
+
+        if (file.contentCn) {
+            // File found on target branch in Gitlab repository
+            if (!regexIncludeLink.test(file.contentCn)) {
+                // File on Gitlab master is NOT just an ..include:: link to ENG version
+                file.fileUrlRepoMasterCN = `${process.env.DANGER_GITLAB_HOST}/${pathProject}/-/blob/${targetBranch}/${file.fileCnPath}`;
+                partMessages.push(
+                    `- file \`${file.fileCnPath}\` was not updated in this MR, but found unchanged full document (not just link to EN) in target branch of Gitlab repository [\`${file.fileCnPath}\`](${file.fileUrlRepoMasterCN}). Please update \`${file.fileCnPath}\` into this MR.`
+                );
+            }
+        } else {
+            // File failed to fetch, probably does not exist in the target branch
+            partMessages.push(
+                `- file \`${file.fileCnPath}\` probably doesn't exist in this MR or in the GitLab repo. Please add \`${file.fileCnPath}\` into this MR.`
+            );
+        }
+    }
+
+    // Create a report with found issues with documents in MR
+    createReport();
+
+    // At this point, the rule has passed
+	recordRuleExitStatus(ruleName, 'Passed');
+
+    /**
+     * Generates an object that represents the relationships between files in two different languages found in this MR.
+     *
+     * @param {string[]} docsFilesEN - An array of file paths for documents in English.
+     * @param {string[]} docsFilesCN - An array of file paths for documents in Chinese.
+     * @returns {Object} An object with the following properties:
+     * - bothFilesInMr: An array of objects that represent files that found in MR in both languages. Each object has the following properties:
+     *   - fileName: The name of the file.
+     *   - fileEnPath: The path to the file in English.
+     *   - fileCnPath: The path to the file in Chinese.
+     *   - fileUrlRepoEN: The URL link to MR branch path to the file in English.
+     *   - fileUrlRepoCN: The URL link to MR branch path to the file in Chinese.
+     * - onlyCnFilesInMr: An array of objects that represent files that only found in MR in English. Each object has the following properties:
+     *   - fileName: The name of the file.
+     *   - fileEnPath: The path to the file in English.
+     * 	 - fileCnPath: The FUTURE path to the file in Chinese.
+     *   - fileUrlRepoEN: The URL link to MR branch path to the file in English.
+     *   - fileUrlRepoCN: The URL link to MR branch path to the file in Chinese.
+     * - onlyEnFilesInMr: An array of objects that represent files that only found in MR in Chinese. Each object has the following properties:
+     *   - fileName: The name of the file.
+     *   - fileEnPath: The FUTURE path to the file in English.
+     *   - fileCnPath: The path to the file in Chinese.
+     *   - fileUrlRepoEN: The URL link to MR branch path to the file in English.
+     *   - fileUrlRepoCN: The URL link to MR branch path to the file in Chinese.
+     */
+    function parseMrDocsFiles(allMrFiles) {
+        const path = require("path");
+        const mrBranch = danger.gitlab.mr.source_branch;
+
+        const docsEnFilesMrPath = allMrFiles.filter((file) =>
+            file.startsWith("docs/en")
+        ); // Filter all English doc files in MR
+        const docsCnFilesMrPath = allMrFiles.filter((file) =>
+            file.startsWith("docs/zh_CN")
+        ); // Filter all Chinese doc files in MR
+
+        const docsEnFileNames = docsEnFilesMrPath.map((filePath) =>
+            path.basename(filePath)
+        ); // Get (base) file names for English docs
+        const docsCnFileNames = docsCnFilesMrPath.map((filePath) =>
+            path.basename(filePath)
+        ); // Get (base) file names for Chinese docs
+
+        const bothFileNames = docsEnFileNames.filter((fileName) =>
+            docsCnFileNames.includes(fileName)
+        ); // Get file names that are common to both English and Chinese docs
+        const onlyEnFileNames = docsEnFileNames.filter(
+            (fileName) => !docsCnFileNames.includes(fileName)
+        ); // Get file names that are only present in English version
+        const onlyCnFileNames = docsCnFileNames.filter(
+            (fileName) => !docsEnFileNames.includes(fileName)
+        ); // Get file names that are only present in Chinese version
+
+        return {
+            bothFilesInMr: bothFileNames.map((fileName) => {
+                const fileEnPath =
+                    docsEnFilesMrPath[docsEnFileNames.indexOf(fileName)];
+                const fileCnPath =
+                    docsCnFilesMrPath[docsCnFileNames.indexOf(fileName)];
+
+                return {
+                    fileName,
+                    fileEnPath,
+                    fileCnPath,
+                    fileUrlRepoEN: `${process.env.DANGER_GITLAB_HOST}/${pathProject}/-/blob/${mrBranch}/${fileEnPath}`,
+                    fileUrlRepoCN: `${process.env.DANGER_GITLAB_HOST}/${pathProject}/-/blob/${mrBranch}/${fileCnPath}`,
+                };
+            }),
+            onlyEnFilesInMr: onlyEnFileNames.map((fileName) => {
+                const fileEnPath =
+                    docsEnFilesMrPath[docsEnFileNames.indexOf(fileName)];
+                const fileCnPath = fileEnPath.replace("en", "zh_CN"); // Generate future CN file path
+
+                return {
+                    fileName,
+                    fileEnPath,
+                    fileCnPath,
+                    fileUrlRepoEN: `${process.env.DANGER_GITLAB_HOST}/${pathProject}/-/blob/${mrBranch}/${fileEnPath}`,
+                    fileUrlRepoCN: `${process.env.DANGER_GITLAB_HOST}/${pathProject}/-/blob/${mrBranch}/${fileCnPath}`,
+                };
+            }),
+            onlyCnFilesInMr: onlyCnFileNames.map((fileName) => {
+                const fileCnPath =
+                    docsCnFilesMrPath[docsCnFileNames.indexOf(fileName)];
+                const fileEnPath = fileCnPath.replace("zh_CN", "en"); // Generate future EN file path
+
+                return {
+                    fileName,
+                    fileEnPath,
+                    fileCnPath,
+                    fileUrlRepoEN: `${process.env.DANGER_GITLAB_HOST}/${pathProject}/-/blob/${mrBranch}/${fileEnPath}`,
+                    fileUrlRepoCN: `${process.env.DANGER_GITLAB_HOST}/${pathProject}/-/blob/${mrBranch}/${fileCnPath}`,
+                };
+            }),
+        };
+    }
+
+    /**
+     * Retrieves the contents of a file from GitLab using the GitLab API.
+     *
+     * @param {string} filePath - The path of the file to retrieve.
+     * @param {string} branch - The branch where the file is located.
+     * @returns {string|null} - The contents of the file, with any trailing new lines trimmed, or null if the file cannot be retrieved.
+     */
+    async function getContentFileInGitlab(filePath, branch) {
+        const axios = require("axios");
+
+        const encFilePath = encodeURIComponent(filePath);
+        const encBranch = encodeURIComponent(branch);
+        const urlApi = `${process.env.DANGER_GITLAB_API_BASE_URL}/projects/${danger.gitlab.mr.project_id}/repository/files/${encFilePath}/raw?ref=${encBranch}`;
+
+        try {
+            const response = await axios.get(urlApi, {
+                headers: {
+                    "Private-Token": process.env.DANGER_GITLAB_API_TOKEN,
+                },
+            });
+            return response.data.trim(); // Trim trailing  new line
+        } catch (error) {
+            return null;
+        }
+    }
+
+    /**
+     * Retrieves the contents of a file in a DangerJS merge request object.
+     *
+     * @param {string} filePath - The path of the file to retrieve.
+     * @returns {string|null} - The contents of the file, with any trailing new lines trimmed, or null if the file cannot be retrieved.
+     */
+    async function getContentFileInMR(filePath) {
+        try {
+            const content = await danger.git.diffForFile(filePath);
+            const fileContentAfter = content.after.trim(); // Trim trailing new lines
+            return fileContentAfter;
+        } catch (error) {
+            console.error(`Error while getting file content MR: ${error}`);
+            return null;
+        }
+    }
+
+    /**
+     * Creates a compiled report for found documentation issues in the current MR and alerts the Documentation team if there are any "needs translation" labels present.
+     *
+     * Report if documentation labels have been added by mistake.
+     */
+    function createReport() {
+        const mrLabels = danger.gitlab.mr.labels; // Get MR labels
+        const regexTranslationLabel = /needs translation:/i;
+
+        const translationLabelsPresent = mrLabels.some((label) =>
+            regexTranslationLabel.test(label)
+        ); // Check if any of MR labels are "needs translation: XX"
+
+        // No docs issues found in MR, but translation labels have been added anyway
+        if (!partMessages.length && translationLabelsPresent) {
+            recordRuleExitStatus(ruleName, "Failed");
+            return warn(
+                `Please remove the \`needs translation: XX\` labels. For documents that need to translate from scratch, Doc team will translate them in the future. For the current stage, we only focus on updating exiting EN and CN translation to make them in sync.`
+            );
+        }
+
+        // Docs issues found in this MR
+        partMessages.sort();
+        let dangerMessage = `Some of the documentation files in this MR seem to have translations issues:\n${partMessages.join(
+            "\n"
+        )}\n`;
+
+        if (partMessages.length) {
+            if (!translationLabelsPresent) {
+                dangerMessage += `
+				\nWhen synchronizing the EN and CN versions, please follow the [Documentation Code](https://docs.espressif.com/projects/esp-idf/zh_CN/latest/esp32/contribute/documenting-code.html#standardize-document-format). The total number of lines of EN and CN should be same.\n
+				\nIf you have difficulty in providing translation, you can contact Documentation team by adding <kbd>needs translation: CN</kbd> or <kbd>needs translation: EN</kbd> labels into this MR and retrying Danger CI job. The documentation team will be automatically notified and will help you with the translations before the merge.\n`;
+                recordRuleExitStatus(ruleName, "Failed");
+                return warn(dangerMessage); // no "needs translation: XX" labels in MR; report issues as warn
+            } else {
+				dangerMessage += `\nTranslation labels <kbd>needs translation: CN</kbd> or <kbd>needs translation: EN</kbd> were added - this will automatically notify the Documentation team to help you with translation issues.`;
+                recordRuleExitStatus(ruleName, 'Passed (with suggestions)');
+                return message(dangerMessage); // "needs translation: XX" labels were found in MR and Docs team was notified; report issues as info
+            }
+        }
+    }
+};

.gitlab/dangerjs/mrSizeTooLarge.js

@@ -0,0 +1,22 @@
+const { recordRuleExitStatus } = require("./configParameters.js");
+
+/**
+ * Check if MR is too large (more than 1000 lines of changes)
+ *
+ * @dangerjs INFO
+ */
+module.exports = async function () {
+    const ruleName = "Merge request size (number of changed lines)";
+    const bigMrLinesOfCodeThreshold = 1000;
+    const totalLines = await danger.git.linesOfCode();
+
+    if (totalLines > bigMrLinesOfCodeThreshold) {
+        recordRuleExitStatus(ruleName, "Passed (with suggestions)");
+        return message(
+            `This MR seems to be quite large (total lines of code: ${totalLines}), you might consider splitting it into smaller MRs`
+        );
+    }
+
+    // At this point, the rule has passed
+    recordRuleExitStatus(ruleName, "Passed");
+};

.gitlab/dangerjs/mrSourceBranchName.js

@@ -0,0 +1,31 @@
+const { recordRuleExitStatus } = require("./configParameters.js");
+
+/**
+ * Throw Danger WARN if branch name contains more than one slash or uppercase letters
+ *
+ * @dangerjs INFO
+ */
+module.exports = function () {
+    const ruleName = "Source branch name";
+    const sourceBranch = danger.gitlab.mr.source_branch;
+
+    // Check if the source branch name contains more than one slash
+    const slashCount = (sourceBranch.match(/\//g) || []).length;
+    if (slashCount > 1) {
+        recordRuleExitStatus(ruleName, "Failed");
+        return warn(
+            `The source branch name \`${sourceBranch}\` contains more than one slash. This can cause troubles with git sync. Please rename the branch.`
+        );
+    }
+
+    // Check if the source branch name contains any uppercase letters
+    if (sourceBranch !== sourceBranch.toLowerCase()) {
+        recordRuleExitStatus(ruleName, "Failed");
+        return warn(
+            `The source branch name \`${sourceBranch}\` contains uppercase letters. This can cause troubles on case-insensitive file systems (macOS). Please use only lowercase letters.`
+        );
+    }
+
+    // At this point, the rule has passed
+    recordRuleExitStatus(ruleName, "Passed");
+};

.gitlab/dangerjs/mrTitleNoDraftOrWip.js

@@ -0,0 +1,31 @@
+const { recordRuleExitStatus } = require("./configParameters.js");
+
+/**
+ * Check if MR Title contains prefix "WIP: ...".
+ *
+ * @dangerjs WARN
+ */
+module.exports = function () {
+    const ruleName = 'Merge request not in Draft or WIP state';
+    const mrTitle = danger.gitlab.mr.title;
+    const regexes = [
+        { prefix: "WIP", regex: /^WIP:/i },
+        { prefix: "W.I.P", regex: /^W\.I\.P/i },
+        { prefix: "[WIP]", regex: /^\[WIP/i },
+        { prefix: "[W.I.P]", regex: /^\[W\.I\.P/i },
+        { prefix: "(WIP)", regex: /^\(WIP/i },
+        { prefix: "(W.I.P)", regex: /^\(W\.I\.P/i },
+    ];
+
+    for (const item of regexes) {
+        if (item.regex.test(mrTitle)) {
+            recordRuleExitStatus(ruleName, "Failed");
+            return warn(
+                `Please remove the \`${item.prefix}\` prefix from the MR name before merging this MR.`
+            );
+        }
+    }
+
+    // At this point, the rule has passed
+    recordRuleExitStatus(ruleName, "Passed");
+};

.gitlab/dangerjs/package.json

@@ -0,0 +1,12 @@
+{
+    "name": "dangerjs-esp-idf",
+    "description": "Merge request automatic linter",
+    "main": "dangerfile.js",
+    "dependencies": {
+        "danger": "^11.2.3",
+        "axios": "^1.3.3",
+        "langchain": "^0.0.53",
+        "openai-gpt-token-counter": "^1.0.3",
+        "@commitlint/lint": "^13.1.0"
+    }
+}

.gitlab/merge_request_templates/Minimal_Template.md

@@ -13,17 +13,3 @@
 <!-- Either state release notes or write "No release notes" -->
 
 <!-- ## Breaking change notes --><!-- Optional -->
-
-<!-- ## Dynamic Pipeline Configuration
-```yaml
-Test Case Filters:
-    # Only run tests that match the given substring expression (modified files/components will be ignored):
-    # Please use a list of strings. 
-    # This will run the test cases filtered like `pytest -k "(<list_item_1>) or (<list_item_2>) or ...`
-    # The fast pipeline will fail at the final stage.
-    # For example:
-      - test_sdm and not sdmmc
-      - test_hello_world
-    # This example will include all tests containing 'test_hello_world' in the name,
-    # and include all tests containing 'test_sdm' but not 'sdmmc' in the name.
-``` --><!-- Optional -->

.gitlab/merge_request_templates/Mixed Template.md

@@ -10,7 +10,8 @@ _This entire section can be deleted if all items are checked._
 * [ ] All related links, including JIRA, backport, submodule MR, are mentioned in the `Related` subsection.
 * [ ] Any GitHub issues are linked inside the git commit message and corresponding release notes
 * [ ] Add label for the area this MR is part of
-* [ ] For documentation updates, check if label `needs translation:CN` or `needs translation:EN` have been added when the other language version still needs the update. Don't add such labels if the document updated has never been translated before and requires translation from scratch.
+* [ ] For documentation updates, check if label `Docs` and `needs translation:CN` or `needs translation:EN`  have been added when the other language version still needs the update. Skip adding the label if the document is not yet translated.
+* [ ] Check if documents requiring translation fall under get-started section. If yes, add the labels mentioned above. Then the documentation team will assign a translator for you. Please inform the translator to prepare translation once your MR is ready to merge. The translation should be included in your MR to get it merged. For more information, see documentation workflow in Wiki.
 * [ ] Any necessary "needs backport" labels are added
 * [ ] Check if this is a breaking change. If it is, add notes to the `Breaking change notes` subsection below
 * [ ] Release note entry if this is a new public feature, or a fix for an issue introduced in the previous release.
@@ -41,7 +42,7 @@ _For other small/non-public changes, which are not expected to be in the release
 
 _Don't touch the subsection titles below, they will be parsed by scripts._
 
-## Release notes <!-- Mandatory -->
+## Release notes (Mandatory)
 
 _Changes made in this MR that should go into the **Release Notes** should be listed here. Please use **past tense** and *specify the area (see maintainers page of IDF internal wiki)*. If there is a subscope, include it and separate with slash (`/`). Minor changes can go to the descriptions above without a release notes entry._
 

.gitmodules

@@ -49,13 +49,12 @@
 [submodule "components/json/cJSON"]
 	path = components/json/cJSON
 	url = ../../DaveGamble/cJSON.git
-	sbom-version = 1.7.18
+	sbom-version = 1.7.17
 	sbom-cpe = cpe:2.3:a:cjson_project:cjson:{}:*:*:*:*:*:*:*
 	sbom-supplier = Person: Dave Gamble
 	sbom-url = https://github.com/DaveGamble/cJSON
 	sbom-description = Ultralightweight JSON parser in ANSI C
-	sbom-hash = acc76239bee01d8e9c858ae2cab296704e52d916
-	sbom-cve-exclude-list = CVE-2024-31755 Resolved in v1.7.18
+	sbom-hash = 87d8f0961a01bf09bef98ff89bae9fdec42181ee
 
 [submodule "components/mbedtls/mbedtls"]
 	path = components/mbedtls/mbedtls
@@ -82,11 +81,11 @@
 [submodule "components/unity/unity"]
 	path = components/unity/unity
 	url = ../../ThrowTheSwitch/Unity.git
-	sbom-version = v2.6.0-RC1
+	sbom-version = v2.4.3-51-g7d2bf62b7e6a
 	sbom-supplier = Organization: ThrowTheSwitch community <http://www.throwtheswitch.org>
 	sbom-url = https://github.com/ThrowTheSwitch/Unity
 	sbom-description = Simple Unit Testing for C
-	sbom-hash = bf560290f6020737eafaa8b5cbd2177c3956c03f
+	sbom-hash = 7d2bf62b7e6afaf38153041a9d53c21aeeca9a25
 
 [submodule "components/bt/host/nimble/nimble"]
 	path = components/bt/host/nimble/nimble
@@ -133,10 +132,6 @@
 	path = components/bt/controller/lib_esp32c6/esp32c6-bt-lib
 	url = ../../espressif/esp32c6-bt-lib.git
 
-[submodule "components/bt/controller/lib_esp32c5/esp32c5-bt-lib"]
-	path = components/bt/controller/lib_esp32c5/esp32c5-bt-lib
-	url = ../../espressif/esp32c5-bt-lib.git
-
 [submodule "components/heap/tlsf"]
 	path = components/heap/tlsf
 	url = ../../espressif/tlsf.git

.pre-commit-config.yaml

@@ -5,7 +5,7 @@ default_stages: [commit]
 
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.5.0
+    rev: v4.0.1
     hooks:
       - id: trailing-whitespace
         # note: whitespace exclusions use multiline regex, see https://pre-commit.com/#regular-expressions
@@ -22,8 +22,7 @@ repos:
             .*_pb2.py|
             .*.pb-c.h|
             .*.pb-c.c|
-            .*.yuv|
-            .*.rgb
+            .*.yuv
           )$
       - id: end-of-file-fixer
         exclude: *whitespace_excludes
@@ -52,10 +51,6 @@ repos:
           (?x)^(
               .*_pb2.py
           )$
-  - repo: https://github.com/codespell-project/codespell
-    rev: v2.3.0
-    hooks:
-      - id: codespell
   - repo: local
     hooks:
       - id: check-executables
@@ -70,6 +65,13 @@ repos:
         language: python
         pass_filenames: false
         always_run: true
+      - id: check-kconfigs
+        name: Validate Kconfig files
+        entry: tools/ci/check_kconfigs.py
+        language: python
+        additional_dependencies:
+          - esp-idf-kconfig>=1.4.2,<2.0.0
+        files: '^Kconfig$|Kconfig.*$'
       - id: check-deprecated-kconfigs-options
         name: Check if any Kconfig Options Deprecated
         entry: tools/ci/check_deprecated_kconfigs.py
@@ -90,24 +92,30 @@ repos:
         always_run: true
         files: '\.gitlab/CODEOWNERS'
         pass_filenames: false
+      - id: check-rules-yml
+        name: Check rules.yml all rules have at lease one job applied, all rules needed exist
+        entry: tools/ci/check_rules_yml.py
+        language: python
+        files: '\.gitlab/ci/.+\.yml|\.gitlab-ci.yml|\.gitmodules'
+        pass_filenames: false
+        additional_dependencies:
+          - PyYAML == 5.3.1
       - id: check-generated-rules
         name: Check rules are generated (based on .gitlab/ci/dependencies/dependencies.yml)
         entry: tools/ci/generate_rules.py
         language: python
-        files: '\.gitlab/ci/dependencies/.+|\.gitlab/ci/.*\.yml|.gitlab-ci.yml'
+        files: '\.gitlab/ci/dependencies/.+|\.gitlab/ci/.*\.yml'
         pass_filenames: false
-        require_serial: true
         additional_dependencies:
           - PyYAML == 5.3.1
       - id: mypy-check
         name: Check type annotations in python files
         entry: tools/ci/check_type_comments.py
         additional_dependencies:
-          - 'mypy'
-          - 'mypy-extensions'
-          - 'types-setuptools'
-          - 'types-PyYAML'
-          - 'types-requests'
+          - 'mypy==0.940'
+          - 'mypy-extensions==0.4.3'
+          - 'types-setuptools==57.4.14'
+          - 'types-PyYAML==0.1.9'
         exclude: >
           (?x)^(
               .*_pb2.py
@@ -154,20 +162,14 @@ repos:
         require_serial: true
         additional_dependencies:
           - PyYAML == 5.3.1
-          - idf-build-apps~=2.0
-      - id: sort-yaml-files
-        name: sort yaml files
-        entry: tools/ci/sort_yaml.py
+          - idf_build_apps~=1.0
+      - id: sort-build-test-rules-ymls
+        name: sort .build-test-rules.yml files
+        entry: tools/ci/check_build_test_rules.py sort-yaml
         language: python
-        files: '\.build-test-rules\.yml$|known_generate_test_child_pipeline_warnings\.yml$'
-        additional_dependencies:
-          - ruamel.yaml
-      - id: sort-yaml-test
-        name: sort yaml test
-        entry: python -m unittest tools/ci/sort_yaml.py
-        language: python
-        files: 'tools/ci/sort_yaml\.py$'
+        files: '\.build-test-rules\.yml'
         additional_dependencies:
+          - PyYAML == 5.3.1
           - ruamel.yaml
       - id: check-build-test-rules-path-exists
         name: check path in .build-test-rules.yml exists
@@ -184,14 +186,6 @@ repos:
         language: python
         always_run: true
         require_serial: true
-      - id: gitlab-yaml-linter
-        name: Check gitlab yaml files
-        entry: tools/ci/gitlab_yaml_linter.py
-        language: python
-        files: '\.gitlab-ci\.yml|\.gitlab/ci/.+\.yml|\.gitmodules'
-        pass_filenames: false
-        additional_dependencies:
-          - PyYAML == 5.3.1
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.0.1
     hooks:
@@ -203,7 +197,7 @@ repos:
       - id: check-copyright
         args: ['--ignore', 'tools/ci/check_copyright_ignore.txt', '--config', 'tools/ci/check_copyright_config.yaml']
   - repo: https://github.com/espressif/conventional-precommit-linter
-    rev: v1.7.0
+    rev: v1.2.1
     hooks:
       - id: conventional-precommit-linter
         stages: [commit-msg]
@@ -224,23 +218,8 @@ repos:
       name: shellcheck dash (export.sh)
       args: ['--shell', 'dash', '-x']
       files: 'export.sh'
-  - repo: https://github.com/espressif/esp-pwsh-check
-    rev: v1.0.1
-    hooks:
-    - id: check-powershell-scripts
-      stages: [manual]
   - repo: https://github.com/espressif/esp-idf-sbom.git
     rev: v0.13.0
     hooks:
       - id: validate-sbom-manifest
         stages: [post-commit]
-  - repo: https://github.com/sphinx-contrib/sphinx-lint
-    rev: v0.9.1
-    hooks:
-      - id: 'sphinx-lint'
-        name: Lint rST files in docs folder using Sphinx Lint
-        files: ^(docs/en|docs/zh_CN)/.*\.(rst|inc)$
-  - repo: https://github.com/espressif/esp-idf-kconfig.git
-    rev: v2.3.0
-    hooks:
-      - id: check-kconfig-files

.readthedocs.yml

@@ -15,7 +15,7 @@ python:
   install:
     - requirements: docs/requirements.txt
 
-# We need to list all the submodules included in documenation build by Doxygen
+# We need to list all the submodules included in documenation build by DOxygen
 submodules:
   include:
     - components/mqtt/esp-mqtt

CMakeLists.txt

@@ -16,11 +16,7 @@ endif()
 if(NOT BOOTLOADER_BUILD)
 
     if(CONFIG_COMPILER_OPTIMIZATION_SIZE)
-        if(CMAKE_C_COMPILER_ID MATCHES "Clang")
-            list(APPEND compile_options "-Oz")
-        else()
-            list(APPEND compile_options "-Os")
-        endif()
+        list(APPEND compile_options "-Os")
         if(CMAKE_C_COMPILER_ID MATCHES "GNU")
             list(APPEND compile_options "-freorder-blocks")
         endif()
@@ -38,11 +34,7 @@ if(NOT BOOTLOADER_BUILD)
 else()  # BOOTLOADER_BUILD
 
     if(CONFIG_BOOTLOADER_COMPILER_OPTIMIZATION_SIZE)
-        if(CMAKE_C_COMPILER_ID MATCHES "Clang")
-            list(APPEND compile_options "-Oz")
-        else()
-            list(APPEND compile_options "-Os")
-        endif()
+        list(APPEND compile_options "-Os")
         if(CMAKE_C_COMPILER_ID MATCHES "GNU")
             list(APPEND compile_options "-freorder-blocks")
         endif()
@@ -104,7 +96,7 @@ if(CMAKE_C_COMPILER_ID MATCHES "Clang")
     list(APPEND compile_options "-Wno-pointer-bool-conversion")
     # mbedTLS md5.c triggers this warning in md5_test_buf (false positive)
     list(APPEND compile_options "-Wno-string-concatenation")
-    # multiple cases of implicit conversions between unrelated enum types
+    # multiple cases of implict convertions between unrelated enum types
     list(APPEND compile_options "-Wno-enum-conversion")
     # When IRAM_ATTR is specified both in function declaration and definition,
     # it produces different section names, since section names include __COUNTER__.
@@ -129,10 +121,8 @@ if(CMAKE_C_COMPILER_ID MATCHES "Clang")
     list(APPEND compile_options "-Wno-c2x-extensions")
     # warning on xMPU_SETTINGS for esp32s2 has size 0 for C and 1 for C++
     list(APPEND compile_options "-Wno-extern-c-compat")
-    if(NOT (CONFIG_IDF_TARGET_LINUX AND CMAKE_HOST_SYSTEM_NAME STREQUAL "Darwin"))
-        # warning: implicit truncation from 'int' to a one-bit wide bit-field changes value from 1 to -1
-        list(APPEND compile_options "-Wno-single-bit-bitfield-constant-conversion")
-    endif()
+    # warning: implicit truncation from 'int' to a one-bit wide bit-field changes value from 1 to -1
+    list(APPEND compile_options "-Wno-single-bit-bitfield-constant-conversion")
 endif()
 # More warnings may exist in unit tests and example projects.
 
@@ -144,10 +134,6 @@ if(CONFIG_COMPILER_OPTIMIZATION_ASSERTIONS_DISABLE)
     list(APPEND compile_definitions "-DNDEBUG")
 endif()
 
-if(CONFIG_COMPILER_NO_MERGE_CONSTANTS)
-    list(APPEND compile_options "-fno-merge-constants")
-endif()
-
 if(CONFIG_COMPILER_STACK_CHECK_MODE_NORM)
     list(APPEND compile_options "-fstack-protector")
 elseif(CONFIG_COMPILER_STACK_CHECK_MODE_STRONG)
@@ -160,8 +146,46 @@ if(CONFIG_COMPILER_DUMP_RTL_FILES)
     list(APPEND compile_options "-fdump-rtl-expand")
 endif()
 
-__generate_prefix_map(prefix_map_compile_options)
-list(APPEND compile_options ${prefix_map_compile_options})
+if(NOT ${CMAKE_C_COMPILER_VERSION} VERSION_LESS 8.0.0)
+    if(CONFIG_COMPILER_HIDE_PATHS_MACROS)
+        list(APPEND compile_options "-fmacro-prefix-map=${CMAKE_SOURCE_DIR}=.")
+        list(APPEND compile_options "-fmacro-prefix-map=${IDF_PATH}=/IDF")
+    endif()
+
+    if(CONFIG_APP_REPRODUCIBLE_BUILD)
+        idf_build_set_property(DEBUG_PREFIX_MAP_GDBINIT "${BUILD_DIR}/prefix_map_gdbinit")
+
+        list(APPEND compile_options "-fdebug-prefix-map=${IDF_PATH}=/IDF")
+        list(APPEND compile_options "-fdebug-prefix-map=${PROJECT_DIR}=/IDF_PROJECT")
+        list(APPEND compile_options "-fdebug-prefix-map=${BUILD_DIR}=/IDF_BUILD")
+
+        # component dirs
+        idf_build_get_property(python PYTHON)
+        idf_build_get_property(idf_path IDF_PATH)
+        idf_build_get_property(component_dirs BUILD_COMPONENT_DIRS)
+
+        execute_process(
+            COMMAND ${python}
+                "${idf_path}/tools/generate_debug_prefix_map.py"
+                "${BUILD_DIR}"
+                "${component_dirs}"
+            OUTPUT_VARIABLE result
+            RESULT_VARIABLE ret
+        )
+        if(NOT ret EQUAL 0)
+            message(FATAL_ERROR "This is a bug. Please report to https://github.com/espressif/esp-idf/issues")
+        endif()
+
+        spaces2list(result)
+        list(LENGTH component_dirs length)
+        math(EXPR max_index "${length} - 1")
+        foreach(index RANGE ${max_index})
+            list(GET component_dirs ${index} folder)
+            list(GET result ${index} after)
+            list(APPEND compile_options "-fdebug-prefix-map=${folder}=${after}")
+        endforeach()
+    endif()
+endif()
 
 if(CONFIG_COMPILER_DISABLE_GCC12_WARNINGS)
     list(APPEND compile_options "-Wno-address"
@@ -175,22 +199,10 @@ if(CONFIG_COMPILER_DISABLE_GCC13_WARNINGS)
                                     "-Wno-dangling-reference")
 endif()
 
-if(CONFIG_COMPILER_DISABLE_GCC14_WARNINGS)
-    list(APPEND compile_options "-Wno-calloc-transposed-args")
-endif()
-
-if(CONFIG_COMPILER_DISABLE_DEFAULT_ERRORS)
-    if(NOT CMAKE_C_COMPILER_ID MATCHES "Clang")
-        idf_build_replace_option_from_property(COMPILE_OPTIONS "-Werror" "-Werror=all")
-    endif()
-endif()
-
 # GCC-specific options
 if(CMAKE_C_COMPILER_ID STREQUAL "GNU")
-    list(APPEND compile_options "-fstrict-volatile-bitfields")
-    if(CONFIG_COMPILER_STATIC_ANALYZER)
-        list(APPEND compile_options "-fanalyzer")
-    endif()
+    list(APPEND compile_options "-fstrict-volatile-bitfields"
+                                )
 endif()
 
 if(CONFIG_ESP_SYSTEM_USE_EH_FRAME)
@@ -201,44 +213,17 @@ endif()
 list(APPEND link_options "-fno-lto")
 
 if(CONFIG_IDF_TARGET_LINUX AND CMAKE_HOST_SYSTEM_NAME STREQUAL "Darwin")
-    # Not all versions of the MacOS linker support the -warn_commons flag.
-    # ld version 1053.12 (and above) have been tested to support it.
-    # Hence, we extract the version string from the linker output
-    # before including the flag.
-
-    # Get the ld version, capturing both stdout and stderr
-    execute_process(
-        COMMAND ${CMAKE_LINKER} -v
-        OUTPUT_VARIABLE LD_VERSION_OUTPUT
-        ERROR_VARIABLE LD_VERSION_ERROR
-        OUTPUT_STRIP_TRAILING_WHITESPACE
-        ERROR_STRIP_TRAILING_WHITESPACE
-    )
-
-    # Combine stdout and stderr
-    set(LD_VERSION_OUTPUT "${LD_VERSION_OUTPUT}\n${LD_VERSION_ERROR}")
-
-    # Extract the version string
-    string(REGEX MATCH "PROJECT:(ld|dyld)-([0-9]+)\\.([0-9]+)" LD_VERSION_MATCH "${LD_VERSION_OUTPUT}")
-    set(LD_VERSION_MAJOR_MINOR "${CMAKE_MATCH_2}.${CMAKE_MATCH_3}")
-
-    message(STATUS "Linker Version: ${LD_VERSION_MAJOR_MINOR}")
-
-    # Compare the version with 1053.12
-    if(LD_VERSION_MAJOR_MINOR VERSION_GREATER_EQUAL "1053.12")
-        list(APPEND link_options "-Wl,-warn_commons")
-    endif()
-
     list(APPEND link_options "-Wl,-dead_strip")
+    list(APPEND link_options "-Wl,-warn_commons")
 else()
     list(APPEND link_options "-Wl,--gc-sections")
     list(APPEND link_options "-Wl,--warn-common")
 endif()
 
 # SMP FreeRTOS user provided minimal idle hook. This allows the user to provide
-# their own copy of vApplicationPassiveIdleHook()
-if(CONFIG_FREERTOS_USE_PASSIVE_IDLE_HOOK)
-    list(APPEND link_options "-Wl,--wrap=vApplicationPassiveIdleHook")
+# their own copy of vApplicationMinimalIdleHook()
+if(CONFIG_FREERTOS_USE_MINIMAL_IDLE_HOOK)
+    list(APPEND link_options "-Wl,--wrap=vApplicationMinimalIdleHook")
 endif()
 
 # Placing jump tables in flash would cause issues with code that required
@@ -251,15 +236,22 @@ if(CMAKE_C_COMPILER_ID MATCHES "GNU")
 endif()
 
 if(CMAKE_C_COMPILER_ID MATCHES "Clang")
-    list(APPEND compile_options "-fno-use-cxa-atexit")  # TODO IDF-10934
-else()
-    list(APPEND cxx_compile_options "-fuse-cxa-atexit")
+    list(APPEND compile_options "-fno-use-cxa-atexit")
 endif()
 
 if(COMPILER_RT_LIB_NAME)
     list(APPEND link_options "-rtlib=${CONFIG_COMPILER_RT_LIB_NAME}")
 endif()
 
+# For the transition period from 32-bit time_t to 64-bit time_t,
+# auto-detect the size of this type and set corresponding variable.
+include(CheckTypeSize)
+check_type_size("time_t" TIME_T_SIZE)
+if(TIME_T_SIZE)
+    idf_build_set_property(TIME_T_SIZE ${TIME_T_SIZE})
+else()
+    message(FATAL_ERROR "Failed to determine sizeof(time_t)")
+endif()
 
 idf_build_set_property(COMPILE_OPTIONS "${compile_options}" APPEND)
 idf_build_set_property(C_COMPILE_OPTIONS "${c_compile_options}" APPEND)

COMPATIBILITY.md

@@ -52,62 +52,47 @@ Supported since ESP-IDF v4.2.
 
 | Release branch         | Recommended | Required |
 |------------------------|-------------|----------|
-| release/v4.2           | v4.2.3+     | v4.2     |
-| release/v4.3           | v4.3.3+     | v4.3     |
-| release/v4.4           | v4.4.6+     | v4.4     |
-| release/v5.0           | v5.0.4+     | v5.0     |
-| release/v5.1           | v5.1.2+     | v5.1     |
-| release/v5.2 and above | v5.2+       | v5.2     |
+| release/v4.2           | v4.2.3      | v4.2.3   |
+| release/v4.3           | v4.3.3      | v4.3.3   |
+| release/v4.4           | v4.4.6      | v4.4.1   |
+| release/v5.0           | v5.0.4      | v5.0     |
+| release/v5.1           | v5.1.2      | v5.1     |
+| release/v5.2 and above | v5.2        | v5.2     |
 
 ### ESP32-C3
 
-#### v0.2 - v0.4
+#### v0.2, v0.3
 
 Supported since ESP-IDF v4.3.
 
-#### v1.1
+#### v0.4
 
-| Release branch         | Recommended | Required |
-|------------------------|-------------|----------|
-| release/v4.2           | EOL         | EOL      |
-| release/v4.3           | v4.3.7+     | v4.3.7   |
-| release/v4.4           | v4.4.7+     | v4.4.7   |
-| release/v5.0           | v5.0.5+     | v5.0.5   |
-| release/v5.1           | v5.1.3+     | v5.1.3   |
-| release/v5.2 and above | v5.2+       | v5.2     |
+To be added.
 
 ### ESP32-S3
 
-#### v0.1, v0.2
+#### v0.1
 
 Supported since ESP-IDF v4.4.
 
+#### v0.2
+
+To be added.
+
 ### ESP32-C2 & ESP8684
 
-#### v1.0, v1.1
+#### v1.0
 
 Supported since ESP-IDF v5.0.
 
+#### v1.1
+
+To be added.
+
 #### v1.2
 
-| Release branch         | Recommended | Required |
-|------------------------|-------------|----------|
-| release/v5.0           | v5.0.7+     | v5.0     |
-| release/v5.1           | v5.1.4+     | v5.1     |
-| release/v5.2           | v5.2.2+     | v5.2     |
-| release/v5.3 and above | v5.3+       | v5.3     |
+To be added.
 
-### ESP32-C6
-
-#### v0.0, v0.1
-
-Supported since ESP-IDF v5.1.
-
-### ESP32-H2
-
-#### v0.1, v0.2
-
-Supported since ESP-IDF v5.1.
 
 ## What If the ESP-IDF Version Is Lower than the `Required` Version?
 

COMPATIBILITY_CN.md

@@ -50,64 +50,48 @@
 
 #### v1.0
 
-| 发布分支                | 推荐版本     | 需求版本 |
+| 发布分支               | 推荐版本    | 需求版本 |
 |------------------------|-------------|----------|
-| release/v4.2           | v4.2.3+     | v4.2     |
-| release/v4.3           | v4.3.3+     | v4.3     |
-| release/v4.4           | v4.4.6+     | v4.4     |
-| release/v5.0           | v5.0.4+     | v5.0     |
-| release/v5.1           | v5.1.2+     | v5.1     |
-| release/v5.2 及以上     | v5.2+       | v5.2     |
+| release/v4.2           | v4.2.3      | v4.2.3   |
+| release/v4.3           | v4.3.3      | v4.3.3   |
+| release/v4.4           | v4.4.6      | v4.4.1   |
+| release/v5.0           | v5.0.4      | v5.0     |
+| release/v5.1           | v5.1.2      | v5.1     |
+| release/v5.2 及以上     | v5.2        | v5.2     |
 
 ### ESP32-C3
 
-#### v0.2 - v0.4
+#### v0.2 和 v0.3
 
 从 ESP-IDF v4.3 开始支持。
 
-#### v1.1
+#### v0.4
 
-| 发布分支                | 推荐版本      | 需求版本  |
-|------------------------|-------------|----------|
-| release/v4.2           | EOL         | EOL      |
-| release/v4.3           | v4.3.7+     | v4.3.7   |
-| release/v4.4           | v4.4.7+     | v4.4.7   |
-| release/v5.0           | v5.0.5+     | v5.0.5   |
-| release/v5.1           | v5.1.3+     | v5.1.3   |
-| release/v5.2 及以上     | v5.2+       | v5.2     |
+待更新。
 
 ### ESP32-S3
 
-#### v0.1, v0.2
+#### v0.1
 
 从 ESP-IDF v4.4 开始支持。
 
+#### v0.2
+
+待更新。
+
 ### ESP32-C2 & ESP8684
 
-#### v1.0, v1.1
+#### v1.0
 
 从 ESP-IDF v5.0 开始支持。
 
+#### v1.1
+
+待更新。
+
 #### v1.2
 
-| 发布分支                | 推荐版本      | 需求版本  |
-|------------------------|-------------|----------|
-| release/v5.0           | v5.0.7+     | v5.0     |
-| release/v5.1           | v5.1.4+     | v5.1     |
-| release/v5.2           | v5.2.2+     | v5.1     |
-| release/v5.3 及以上     | v5.3+       | v5.3     |
-
-### ESP32-C6
-
-#### v0.0, v0.1
-
-从 ESP-IDF v5.1 开始支持。
-
-### ESP32-H2
-
-#### v0.1, v0.2
-
-从 ESP-IDF v5.1 开始支持。
+待更新。
 
 
 ## 如果 ESP-IDF 版本低于 `需求版本` 会出现什么情况？

Kconfig

@@ -22,6 +22,7 @@ mainmenu "Espressif IoT Development Framework Configuration"
 
     config IDF_ENV_BRINGUP
         bool
+        default "y" if IDF_TARGET_ESP32P4
         help
             - This option is ONLY used when doing new chip bringup.
             - This option will only enable necessary hw / sw settings for running
@@ -48,10 +49,6 @@ mainmenu "Espressif IoT Development Framework Configuration"
         bool
         default "y" if IDF_TOOLCHAIN="clang"
 
-    config IDF_TOOLCHAIN_GCC
-        bool
-        default "y" if IDF_TOOLCHAIN="gcc"
-
     config IDF_TARGET_ARCH_RISCV
         bool
         default "n"
@@ -79,6 +76,10 @@ mainmenu "Espressif IoT Development Framework Configuration"
         string
         default "$IDF_INIT_VERSION"
 
+    config IDF_TARGET_LINUX
+        bool
+        default "y" if IDF_TARGET="linux"
+
     config IDF_TARGET_ESP32
         bool
         default "y" if IDF_TARGET="esp32"
@@ -113,12 +114,6 @@ mainmenu "Espressif IoT Development Framework Configuration"
         select FREERTOS_UNICORE
         select IDF_TARGET_ARCH_RISCV
 
-    config IDF_TARGET_ESP32C5
-        bool
-        default "y" if IDF_TARGET="esp32c5"
-        select FREERTOS_UNICORE
-        select IDF_TARGET_ARCH_RISCV
-
     config IDF_TARGET_ESP32P4
         bool
         default "y" if IDF_TARGET="esp32p4"
@@ -130,20 +125,6 @@ mainmenu "Espressif IoT Development Framework Configuration"
         select FREERTOS_UNICORE
         select IDF_TARGET_ARCH_RISCV
 
-    config IDF_TARGET_ESP32C61
-        bool
-        default "y" if IDF_TARGET="esp32c61"
-        select FREERTOS_UNICORE
-        select IDF_TARGET_ARCH_RISCV
-
-    config IDF_TARGET_ESP32H21
-        bool
-        default "y" if IDF_TARGET="esp32h21"
-        select FREERTOS_UNICORE
-        select IDF_TARGET_ARCH_RISCV
-        select IDF_ENV_FPGA
-        select IDF_ENV_BRINGUP
-
     config IDF_TARGET_LINUX
         bool
         default "y" if IDF_TARGET="linux"
@@ -158,9 +139,6 @@ mainmenu "Espressif IoT Development Framework Configuration"
         default 0x000D if IDF_TARGET_ESP32C6
         default 0x0010 if IDF_TARGET_ESP32H2
         default 0x0012 if IDF_TARGET_ESP32P4
-        default 0x0017 if IDF_TARGET_ESP32C5
-        default 0x0014 if IDF_TARGET_ESP32C61
-        default 0x0019 if IDF_TARGET_ESP32H21
         default 0xFFFF
 
 
@@ -332,8 +310,8 @@ mainmenu "Espressif IoT Development Framework Configuration"
             help
                 This option sets compiler optimization level (gcc -O argument) for the app.
 
-                - The "Debug" setting will add the -Og flag to CFLAGS.
-                - The "Size" setting will add the -Os flag to CFLAGS (-Oz with Clang).
+                - The "Debug" setting will add the -0g flag to CFLAGS.
+                - The "Size" setting will add the -0s flag to CFLAGS.
                 - The "Performance" setting will add the -O2 flag to CFLAGS.
                 - The "None" setting will add the -O0 flag to CFLAGS.
 
@@ -354,7 +332,7 @@ mainmenu "Espressif IoT Development Framework Configuration"
             config COMPILER_OPTIMIZATION_DEBUG
                 bool "Debug (-Og)"
             config COMPILER_OPTIMIZATION_SIZE
-                bool "Optimize for size (-Os with GCC, -Oz with Clang)"
+                bool "Optimize for size (-Os)"
             config COMPILER_OPTIMIZATION_PERF
                 bool "Optimize for performance (-O2)"
             config COMPILER_OPTIMIZATION_NONE
@@ -397,21 +375,6 @@ mainmenu "Espressif IoT Development Framework Configuration"
 
         endchoice # assertions
 
-        config COMPILER_ASSERT_NDEBUG_EVALUATE
-            bool "Enable the evaluation of the expression inside assert(X) when NDEBUG is set"
-            default y
-            help
-                When NDEBUG is set, assert(X) will not cause code to trigger an assertion.
-                With this option set, assert(X) will still evaluate the expression X, though
-                the result will never cause an assertion. This means that if X is a function
-                then the function will be called.
-
-                This is not according to the standard, which states that the assert(X) should
-                be replaced with ((void)0) if NDEBUG is defined.
-
-                In ESP-IDF v6.0 the default behavior will change to "no" to be in line with the
-                standard.
-
         choice COMPILER_FLOAT_LIB_FROM
             prompt "Compiler float lib source"
             default COMPILER_FLOAT_LIB_FROM_RVFPLIB if ESP_ROM_HAS_RVFPLIB
@@ -533,15 +496,6 @@ mainmenu "Espressif IoT Development Framework Configuration"
             help
                 Stack smashing protection.
 
-        config COMPILER_NO_MERGE_CONSTANTS
-            bool "Disable merging const sections"
-            depends on IDF_TOOLCHAIN_GCC
-            help
-                Disable merging identical constants (string/floating-point) across compilation units.
-                This helps in better size analysis of the application binary as the rodata section
-                distribution is more uniform across libraries. On downside, it may increase
-                the binary size and hence should be used during development phase only.
-
         config COMPILER_WARN_WRITE_STRINGS
             bool "Enable -Wwrite-strings warning flag"
             default "n"
@@ -568,20 +522,6 @@ mainmenu "Espressif IoT Development Framework Configuration"
 
                 This option can be enabled for RISC-V targets only.
 
-        config COMPILER_DISABLE_DEFAULT_ERRORS
-            bool "Disable errors for default warnings"
-            default "y"
-            help
-                Enable this option if you do not want default warnings to be considered as errors,
-                especially when updating IDF.
-
-                This is a temporary flag that could help to allow upgrade while having
-                some time to address the warnings raised by those default warnings.
-                Alternatives are:
-                1) fix code (preferred),
-                2) remove specific warnings,
-                3) do not consider specific warnings as error.
-
         config COMPILER_DISABLE_GCC12_WARNINGS
             bool "Disable new warnings introduced in GCC 12"
             default "n"
@@ -596,13 +536,6 @@ mainmenu "Espressif IoT Development Framework Configuration"
                 Enable this option if use GCC 13 or newer, and want to disable warnings which don't appear with
                 GCC 12.
 
-        config COMPILER_DISABLE_GCC14_WARNINGS
-            bool "Disable new warnings introduced in GCC 14"
-            default "n"
-            help
-                Enable this option if use GCC 14 or newer, and want to disable warnings which don't appear with
-                GCC 13.
-
         config COMPILER_DUMP_RTL_FILES
             bool "Dump RTL files during compilation"
             help
@@ -637,34 +570,6 @@ mainmenu "Espressif IoT Development Framework Configuration"
             default "gcc" if COMPILER_RT_LIB_GCCLIB
             default "" if COMPILER_RT_LIB_HOST
 
-        choice COMPILER_ORPHAN_SECTIONS
-            prompt "Orphan sections handling"
-            default COMPILER_ORPHAN_SECTIONS_WARNING
-            depends on !IDF_TARGET_LINUX
-            help
-                If the linker finds orphan sections, it attempts to place orphan sections after sections of the same
-                attribute such as code vs data, loadable vs non-loadable, etc.
-                That means that orphan sections could placed between sections defined in IDF linker scripts.
-                This could lead to corruption of the binary image. Configure the linker action here.
-
-            config COMPILER_ORPHAN_SECTIONS_WARNING
-                bool "Place with warning"
-                help
-                    Places orphan sections with a warning message.
-
-            config COMPILER_ORPHAN_SECTIONS_PLACE
-                bool "Place silently"
-                help
-                    Places orphan sections without a warning/error message.
-        endchoice
-
-        config COMPILER_STATIC_ANALYZER
-            bool "Enable compiler static analyzer"
-            default "n"
-            depends on IDF_TOOLCHAIN_GCC
-            help
-                Enable compiler static analyzer. This may produce false-positive results and increases compile time.
-
     endmenu # Compiler Options
 
     menu "Component config"
@@ -685,7 +590,4 @@ mainmenu "Espressif IoT Development Framework Configuration"
             - CONFIG_ESPTOOLPY_FLASHFREQ_120M && CONFIG_ESPTOOLPY_FLASH_SAMPLE_MODE_DTR
             - CONFIG_SPIRAM_SPEED_120M && CONFIG_SPIRAM_MODE_OCT
             - CONFIG_BOOTLOADER_CACHE_32BIT_ADDR_QUAD_FLASH
-            - CONFIG_ESP_WIFI_EAP_TLS1_3
-            - CONFIG_ESP_WIFI_ENABLE_ROAMING_APP
-            - CONFIG_USB_HOST_EXT_PORT_SUPPORT_LS
-            - CONFIG_USB_HOST_EXT_PORT_RESET_ATTEMPTS
+            - CONFIG_MBEDTLS_USE_CRYPTO_ROM_IMPL

README.md

@@ -15,18 +15,16 @@ ESP-IDF is the development framework for Espressif SoCs supported on Windows, Li
 
 The following table shows ESP-IDF support of Espressif SoCs where ![alt text][preview] and ![alt text][supported] denote preview status and support, respectively. The preview support is usually limited in time and intended for beta versions of chips. Please use an ESP-IDF release where the desired SoC is already supported.
 
-|Chip         |          v5.0          |         v5.1           |         v5.2           |         v5.3           |          v5.4           |                                                                     |
-|:----------- | :---------------------:| :--------------------: | :--------------------: | :--------------------: |  :--------------------: |:------------------------------------------------------------------- |
-|ESP32        | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] |  ![alt text][supported] |                                                                     |
-|ESP32-S2     | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] |  ![alt text][supported] |                                                                     |
-|ESP32-C3     | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] |  ![alt text][supported] |                                                                     |
-|ESP32-S3     | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] |  ![alt text][supported] |[Announcement](https://www.espressif.com/en/news/ESP32_S3)           |
-|ESP32-C2     | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] |  ![alt text][supported] |[Announcement](https://www.espressif.com/en/news/ESP32-C2)           |
-|ESP32-C6     |                        | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] |  ![alt text][supported] |[Announcement](https://www.espressif.com/en/news/ESP32_C6)           |
-|ESP32-H2     |                        | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] |  ![alt text][supported] |[Announcement](https://www.espressif.com/en/news/ESP32_H2)           |
-|ESP32-P4     |                        |                        |                        | ![alt text][supported] |  ![alt text][supported] |[Announcement](https://www.espressif.com/en/news/ESP32-P4)           |
-|ESP32-C5     |                        |                        |                        |                        |  ![alt text][preview]   |[Announcement](https://www.espressif.com/en/news/ESP32-C5)           |
-|ESP32-C61    |                        |                        |                        |                        |  ![alt text][preview]   |[Announcement](https://www.espressif.com/en/products/socs/esp32-c61) |
+|Chip         |         v4.3           |          v4.4          |          v5.0          |         v5.1           |         v5.2           |                                                            |
+|:----------- | :---------------------:| :---------------------:| :---------------------:| :--------------------: | :--------------------: | :----------------------------------------------------------|
+|ESP32        | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] |                                                            |
+|ESP32-S2     | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] |                                                            |
+|ESP32-C3     | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] |                                                            |
+|ESP32-S3     |                        | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] | [Announcement](https://www.espressif.com/en/news/ESP32_S3) |
+|ESP32-C2     |                        |                        | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] | [Announcement](https://www.espressif.com/en/news/ESP32-C2) |
+|ESP32-C6     |                        |                        |                        | ![alt text][supported] | ![alt text][supported] | [Announcement](https://www.espressif.com/en/news/ESP32_C6) |
+|ESP32-H2     |                        |                        |                        | ![alt text][supported] | ![alt text][supported] | [Announcement](https://www.espressif.com/en/news/ESP32_H2) |
+|ESP32-P4     |                        |                        |                        |                        | ![alt text][preview]   | [Announcement](https://www.espressif.com/en/news/ESP32-P4) |
 
 [supported]: https://img.shields.io/badge/-supported-green "supported"
 [preview]: https://img.shields.io/badge/-preview-orange "preview"

README_CN.md

@@ -15,18 +15,16 @@ ESP-IDF 是乐鑫官方推出的物联网开发框架，支持 Windows、Linux
 
 下表总结了乐鑫芯片在 ESP-IDF 各版本中的支持状态，其中 ![alt text][supported] 代表已支持，![alt text][preview] 代表目前处于预览支持状态。预览支持状态通常有时间限制，而且仅适用于测试版芯片。请确保使用与芯片相匹配的 ESP-IDF 版本。
 
-|芯片         |          v5.0          |          v5.1          |          v5.2          |          v5.3          |          v5.4          |                                                                           |
-|:----------- | :---------------------:| :--------------------: | :--------------------: | :--------------------: | :--------------------: | :------------------------------------------------------------------------ |
-|ESP32        | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] |                                                                           |
-|ESP32-S2     | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] |                                                                           |
-|ESP32-C3     | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] |                                                                           |
-|ESP32-S3     | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] | [芯片发布公告](https://www.espressif.com/zh-hans/news/ESP32_S3)           |
-|ESP32-C2     | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] | [芯片发布公告](https://www.espressif.com/zh-hans/news/ESP32-C2)           |
-|ESP32-C6     |                        | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] | [芯片发布公告](https://www.espressif.com/zh-hans/news/ESP32_C6)           |
-|ESP32-H2     |                        | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] | [芯片发布公告](https://www.espressif.com/zh-hans/news/ESP32_H2)           |
-|ESP32-P4     |                        |                        |                        | ![alt text][supported] | ![alt text][supported] | [芯片发布公告](https://www.espressif.com/zh-hans/news/ESP32-P4)           |
-|ESP32-C5     |                        |                        |                        |                        | ![alt text][preview]   | [芯片发布公告](https://www.espressif.com/zh-hans/news/ESP32-C5)           |
-|ESP32-C61    |                        |                        |                        |                        | ![alt text][preview]   | [芯片发布公告](https://www.espressif.com/zh-hans/products/socs/esp32-c61) |
+|芯片         |         v4.3           |          v4.4          |          v5.0          |          v5.1          |          v5.2          |                                                                 |
+|:----------- | :---------------------:| :---------------------:| :---------------------:| :--------------------: | :--------------------: | :-------------------------------------------------------------- |
+|ESP32        | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] |                                                                 |
+|ESP32-S2     | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] |                                                                 |
+|ESP32-C3     | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] |                                                                 |
+|ESP32-S3     |                        | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] | [芯片发布公告](https://www.espressif.com/zh-hans/news/ESP32_S3) |
+|ESP32-C2     |                        |                        | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] | [芯片发布公告](https://www.espressif.com/zh-hans/news/ESP32-C2) |
+|ESP32-C6     |                        |                        |                        | ![alt text][supported] | ![alt text][supported] | [芯片发布公告](https://www.espressif.com/zh-hans/news/ESP32_C6) |
+|ESP32-H2     |                        |                        |                        | ![alt text][supported] | ![alt text][supported] | [芯片发布公告](https://www.espressif.com/zh-hans/news/ESP32_H2) |
+|ESP32-P4     |                        |                        |                        |                        | ![alt text][preview]   | [芯片发布公告](https://www.espressif.com/en/news/ESP32-P4)      |
 
 [supported]: https://img.shields.io/badge/-%E6%94%AF%E6%8C%81-green "supported"
 [preview]: https://img.shields.io/badge/-%E9%A2%84%E8%A7%88-orange "preview"
@@ -123,7 +121,7 @@ ESP-IDF 中的子模块采用相对路径（[详见 .gitmodules 文件](.gitmodu
 
 * 最新版的文档：https://docs.espressif.com/projects/esp-idf/ ，该文档是由本仓库 [docs 目录](docs) 构建得到。
 
-* [初学者指南：主要概念和资源](https://www.bilibili.com/video/BV1114y1r7du/)
+* [初学者指南：主要概念和资源](https://www.bilibili.com/video/BV1114y1r7du/) 
 
 * 可以前往 [esp32.com 论坛](https://esp32.com/) 提问，挖掘社区资源。
 

ROADMAP.md

@@ -1,123 +0,0 @@
-# ESP-IDF Project Roadmap 2024
-
-This document outlines the goals of ESP-IDF project and is shared for the convenience of our customers. It is important to clarify that this document is not a binding commitment to our customers. Instead, its primary purpose is to offer a clear roadmap and direction for the project's development. By openly sharing this information, we aim to enhance our customers' understanding, promote transparency and ensure alignment with the overarching objectives of the ESP-IDF project.
-
-## Project Overview
-
-### Project Goals
-
-In both minor and major releases, we integrate new chip support to enhance our product range. By expanding the chip matrix, we broaden the scope of our offerings, catering to a wider audience with diverse needs. This proactive approach ensures that our products remain at the forefront of technological advancements, consistently meeting and exceeding customer expectations.
-
-Furthermore, we prioritize bugfix releases for active branches, focusing on improving the stability and performance of products already in production. By addressing bugs promptly, we aim to enhance the overall user experience and provide tangible benefits to customers relying on our solutions. This proactive maintenance strategy reflects our commitment to delivering reliable, high-quality products to our valued customer base.
-
-Below are the main objectives that ESP-IDF project/teams would like to implement in 2024.
-
--   New Chip Support
-
-    -   Add support for ESP32-P4
-    -   Add support for ESP32-C5
-    -   Add support for ESP32-C61
-
--   More Minor Releases
-
-    -   Release IDF v5.3 in 2024
-    -   Release IDF v5.4 at the start of 2025
-
--   More Bugfix Releases
-
-    -   Release v4.4.8 for IDF v4.4 before ESP-IDF v4.4 goes End of Life in July 2024
-    -   Do more bugfix releases for IDF v5.1 before release/5.1 enters maintenance period in June 2024
-    -   Do more bug fixes releases for release/5.2 and release/5.3, and push the two releases to be more stable and production-ready
-
--   Updates of Libraries
-
-    -   Update GDB to 14.2
-    -   Update LLVM to 18.1.2
-    -   Update MbedTLS to 3.6 (LTS)
-    -   Update LWIP to 2.2.0
-    -   Change minimal Python requirement to 3.9
-
-Please note that support status of previous silicones could be found on [ESP-IDF Release and SoC Compatibility](https://github.com/espressif/esp-idf#esp-idf-release-and-soc-compatibility).
-
-### Roadmap Details
-
-The ESP-IDF project prioritizes consistent maintenance and updates to ensure our customers remain at the forefront of technological advancements. Our commitment to ongoing development ensures that customers continuously benefit from the latest innovations in the field.
-
-Moreover, we are dedicated to empowering our customers to leverage newly implemented features and enhanced functionalities through iterative improvements. Our steadfast commitment to pushing boundaries ensures that clients not only keep pace with evolving technology but also extract optimal value from the cutting-edge capabilities of our products.
-
-Below are the main roadmap details for functional areas inside ESP-IDF.
-
--   New Chip Support
-
-    -   Add full support for ESP32-P4 in ESP-IDF v5.3, refer to [ESP32-P4 Support Status](https://github.com/espressif/esp-idf/issues/12996)
-    -   Add preview support for ESP32-C5 in ESP-IDF v5.3 and full support for ESP32-C5 in ESP-IDF v5.4, refer to [ESP32-C5 Support Status](https://github.com/espressif/esp-idf/issues/14021)
-    -   Add preview support for the early samples of ESP32-C61 in ESP-IDF v5.4 and full support for mass production version in ESP-IDF v5.4.x. Refer to [ESP32-C61 Support Status](https://developer.espressif.com/pages/chip-support-status/esp32c61/#esp-idf)
-
--   Bugfix releases
-
-    -   Do bugfix release IDF v4.4.8 and stop maintaining ESP-IDF v4.4 in July 2024
-    -   Release bugfix IDF v5.0.6 and IDF v5.0.7 in 2024 (maintenance period)
-    -   Release bugfix IDF v5.1.3 and IDF v5.1.4 in H1 of 2024, and release IDF v5.1.5 in H2 of 2024
-    -   Push release/5.1 to maintenance period from June 2024
-    -   Do more bug fixes releases for release/5.2 (IDF v5.2.1, IDF v5.2.2, IDF v5.2.3) and release/5.3 (IDF v5.3.1, IDF v5.3.2), and push releases to be more stable and more production-ready
-
-## ESP-IDF Planning information
-
-For the full list of ESP-IDF releases, please visit https://github.com/espressif/esp-idf/releases
-
-### ESP-IDF Major Releases
-
-No Major Releases planned
-
-### ESP-IDF Minor Releases
-
-#### Quarter One
-
--   v5.2-RC1, estimate release date: 2024/02/02
--   v5.2 final release, estimate release date: 2024/02/08
-
-#### Quarter Two
-
--   v5.3-beta1, estimate release date:: 2024/05/13
--   v5.3-beta2, estimate release date:: 2024/05/31
--   v5.3-RC1, estimate release date:: 2024/07/08
-
-#### Quarter Three
-
--   v5.3-RC2, estimate release date:: 2024/07/26
--   v5.3 final release, estimate release date:: 2024/08/02
-
-#### Quarter Four
-
--   v5.4-beta1, estimate release date:: 2024/11/08
--   v5.4-beta2, estimate release date:: 2024/11/29
--   v5.4-RC1, estimate release date:: 2025/01/09
--   v5.4-RC2, estimate release date:: 2025/01/29
--   v5.4 final release, estimate release date: 2025/02/05
-
-### ESP-IDF Bugfix Releases
-
-#### Quarter One
-
--   v5.1.3, estimate release date: 2024/02/08
--   v5.0.6, estimate release date: 2024/02/18
--   v4.4.7, estimate release date:: 2024/03/19
--   v5.2.1, estimate release date:: 2024/03/31
-
-#### Quarter Two
-
--   v5.1.4, estimate release date:: 2024/05/06
--   v5.2.2, estimate release date:: 2024/06/17
--   v4.4.8, estimate release date:: 2024/07/19
-
-#### Quarter Three
-
--   v5.0.7, estimate release date: 2024/08/22
--   v5.3.1, estimate release date: 2024/09/16
--   v5.2.3, estimate release date: 2024/10/10
-
-#### Quarter Four
-
--   v5.1.5, estimate release date: 2024/11/04  (Maintenance period since June 2024)
--   v5.3.2, estimate release date: 2024/12/31
--   v5.2.4, estimate release date: 2025/02/20 (Service period ends Feb. 2025)

ROADMAP_CN.md

@@ -1,123 +0,0 @@
-# ESP-IDF 项目路线图 2024 (v1.0)
-
-本文档概述了 ESP-IDF 项目的年度计划，方便客户据此规划自己的项目周期。需要说明的是该文档并不是我们对客户的约束性承诺。相反，其主要目的是为客户提供 ESP-IDF 项目开发的路线图和方向。通过公开这些信息，我们希望增进客户对 ESP-IDF 项目的理解，提高透明度，并确保与 ESP-IDF 项目的总体目标保持一致。
-
-## 项目总览
-
-### 项目目标
-
-在 ESP-IDF 的主要版本和次要版本中，我们一般会增加对新芯片的支持，以扩展我们的产品线。通过扩展芯片矩阵，拓宽我们的产品范围，并满足广泛受众的各种需求。这样便能保证我们的产品始终处于技术进步的前沿，不断满足客户的需求并超越客户的期望。
-
-此外，ESP-IDF 各活跃分支的 Bugfix 版本发布也是我们项目的重中之重，着力提升已量产产品的稳定性和性能。通过及时解决问题，我们期待提升用户的整体体验，切实惠及使用乐鑫解决方案的客户。通过积极维护 ESP-IDF 的各活跃分支，我们践行了对宝贵的客户群提供可靠、高质量产品的承诺。
-
-以下是 ESP-IDF 项目在 2024 年计划实现的主要目标。
-
-* 新芯片支持
-
-	* 增加对 ESP32-P4 芯片的支持
-	* 增加对 ESP32-C5 芯片的支持
-	* 增加对 ESP32-C61 芯片的支持
-
-* 发布更多的次要版本
-
-	* 在 2024 年发布 IDF v5.3 
-	* 在 2025 年初发布 IDF v5.4
-
-* 发布更多 Bugfix 版本
-
-	* 在 2024 年 7 月底 IDF v4.4 停止维护之前，发布 IDF v4.4.8
-	* 在 2024 年 6 月底，release/5.1 分支进入维护周期之前，发布更多 Bugfix 版本
-	* release/5.2 分支和 release/5.3 分支发布更多 Bugfix 版本，使这两个分支更加稳定和产品化
-
-* 上游库与工具链的更新
-
-	* 将 GDB 升级至 14.2
-	* 将 LLVM 升级至 18.1.2
-	* 更新 MbedTLS 至 3.6（LTS）
-	* 更新 LWIP 至 2.2.0
-	* 将最低 Python 要求更改为 3.9
-
-请注意，获取之前芯片的支持状态，请参阅 [ESP-IDF 发布和 SoC 兼容性](https://github.com/espressif/esp-idf/blob/master/README_CN.md#esp-idf-与乐鑫芯片)。
-
-### 路线图细节
-
-ESP-IDF 项目重视持续维护和更新，确保我们的客户始终处于技术进步的前沿。我们承诺持续进行开发，并将该领域的最新创新成果呈现给客户。
-
-此外，我们也在给客户赋能，客户通过迭代改进便能接触到新开发的功能和更高的性能。我们在突破技术界限方面的坚定承诺，使客户不仅能接触到最新的技术，还能从我们产品的尖端功能中获取最大价值。
-
-以下是 ESP-IDF 路线图的主要信息。
-
-* 新芯片支持
-
-	* 在 ESP-IDF v5.3 中为 ESP32-P4 提供完整支持，参考 [ESP32-P4 支持状态](https://github.com/espressif/esp-idf/issues/12996)
-	* 在 ESP-IDF v5.3 中为 ESP32-C5 提供预览支持，并在 ESP-IDF v5.4 中为 ESP32-C5 提供完整支持，参考 [ESP32-C5 支持状态](https://github.com/espressif/esp-idf/issues/14021)
-	* 在 ESP-IDF v5.4 中增加对 ESP32-C61 早期样品的预览支持，并在 IDF v5.4.x 中增加对 ESP32-C61 量产版本的完整支持，参考 [ESP32-C61 支持状态](https://developer.espressif.com/pages/chip-support-status/esp32c61/#esp-idf)
-
-* Bugfix 版本发布
-
-	* 发布 Bugfix 版本 IDF v4.4.8，并在 2024 年 7 月底停止维护 ESP-IDF v4.4
-	* 2024 年，release/5.0 分支已处于维护周期，发布 IDF v5.0.6 和 IDF v5.0.7
-	* 2024 年上半年发布 Bugfix 版本 IDF v5.1.3 和 IDF v5.1.4，并在 2024 年下半年发布 IDF v5.1.5
-	* 自 2024 年 6 月起，release/5.1 分支进入到维护周期
-	* release/5.2 分支发布更多 Bugfix 版本，包括 IDF v5.2.1、IDF v5.2.2、IDF v5.2.3；release/5.3 分支发布更多 Bugfix 版本，包括 IDF v5.3.1、IDF v5.3.2。通过发布这些 Bugfix 版本，使 release/5.2 分支和 release/5.3 分支更加稳定和产品化。
-
-## ESP-IDF 发布计划
-
-获取 ESP-IDF 的完整发布列表，请访问 https://github.com/espressif/esp-idf/releases
-
-### ESP-IDF 主要版本发布
-
-2024 年，无主要版本发布计划
-
-### ESP-IDF 次要版本发布
-
-#### 第一季度
-
-* v5.2-RC1，预计发布日期：2024/02/02
-* v5.2 正式发布，预计发布日期：2024/02/08
-
-#### 第二季度
-
-* v5.3-beta1，预计发布日期：2024/05/13
-* v5.3-beta2，预计发布日期：2024/05/31
-* v5.3-RC1，预计发布日期：2024/07/08
-
-#### 第三季度
-
-* v5.3-RC2，预计发布日期：2024/07/26
-* v5.3 正式发布，预计发布日期：2024/08/02
-
-#### 第四季度
-
-* v5.4-beta1，预计发布日期：2024/11/08
-* v5.4-beta2，预计发布日期：2024/11/29
-* v5.4-RC1，预计发布日期：2025/01/09
-* v5.4-RC2，预计发布日期：2025/01/29
-* v5.4 正式发布，预计发布日期：2025/02/05
-
-### ESP-IDF Bugfix 版本发布
-
-#### 第一季度
-
-* v5.1.3，预计发布日期：2024/02/08
-* v5.0.6，预计发布日期：2024/02/18
-* v4.4.7，预计发布日期：2024/03/19
-* v5.2.1，预计发布日期：2024/03/31
-
-#### 第二季度
-
-* v5.1.4，预计发布日期：2024/05/06
-* v5.2.2，预计发布日期：2024/06/17
-* v4.4.8，预计发布日期：2024/07/19
-
-#### 第三季度
-
-* v5.0.7，预计发布日期：2024/08/22
-* v5.3.1，预计发布日期：2024/09/16
-* v5.2.3，预计发布日期：2024/10/10
-
-#### 第四季度
-
-* v5.1.5，预计发布日期：2024/11/04（自 2024 年 6 月进入维护周期）
-* v5.3.2，预计发布日期：2024/12/31
-* v5.2.4，预计发布日期：2025/02/20（服务周期截止至 2025 年 2 月）

components/app_trace/CMakeLists.txt

@@ -9,11 +9,6 @@ set(srcs
     "app_trace_util.c"
     "host_file_io.c")
 
-if(CONFIG_ESP_DEBUG_STUBS_ENABLE)
-    list(APPEND srcs
-        "debug_stubs.c")
-endif()
-
 if(CONFIG_APPTRACE_GCOV_ENABLE)
     if("${CMAKE_C_COMPILER_ID}" STREQUAL "GNU")
         list(APPEND srcs
@@ -68,7 +63,8 @@ endif()
 idf_component_register(SRCS "${srcs}"
                        INCLUDE_DIRS "${include_dirs}"
                        PRIV_INCLUDE_DIRS "${priv_include_dirs}"
-                       PRIV_REQUIRES esp_driver_gptimer esp_driver_gpio esp_driver_uart
+                       # Requires "driver" for GPTimer in "SEGGER_SYSVIEW_Config_FreeRTOS.c"
+                       PRIV_REQUIRES soc driver
                        REQUIRES esp_timer
                        LDFRAGMENTS linker.lf)
 
@@ -121,3 +117,16 @@ if(CONFIG_APPTRACE_GCOV_ENABLE)
 else()
     target_link_libraries(${COMPONENT_LIB} INTERFACE $<TARGET_FILE:${app_trace}> c)
 endif()
+
+# This function adds a dependency on the given component if the component is included into the build.
+function(maybe_add_component component_name)
+    idf_build_get_property(components BUILD_COMPONENTS)
+    if(${component_name} IN_LIST components)
+        idf_component_get_property(lib_name ${component_name} COMPONENT_LIB)
+        target_link_libraries(${COMPONENT_LIB} PUBLIC ${lib_name})
+    endif()
+endfunction()
+
+if(CONFIG_APPTRACE_DEST_UART0 OR CONFIG_APPTRACE_DEST_UART1 OR CONFIG_APPTRACE_DEST_UART2)
+    maybe_add_component(driver)
+endif()

components/app_trace/Kconfig

@@ -14,7 +14,6 @@ menu "Application Level Tracing"
 
         config APPTRACE_DEST_NONE
             bool "None"
-
     endchoice
 
     config APPTRACE_DEST_UART
@@ -61,7 +60,7 @@ menu "Application Level Tracing"
     endchoice
 
     config APPTRACE_UART_TX_GPIO
-        int "UART TX on GPIO<num>"
+        int "UART TX on GPIO#"
         depends on APPTRACE_DEST_UART_NOUSB
         range 0 46
         default 12 if IDF_TARGET_ESP32
@@ -71,7 +70,7 @@ menu "Application Level Tracing"
             This GPIO is used for UART TX pin.
 
     config APPTRACE_UART_RX_GPIO
-        int "UART RX on GPIO<num>"
+        int "UART RX on GPIO#"
         depends on APPTRACE_DEST_UART_NOUSB
         range 0 46
         default 13 if IDF_TARGET_ESP32
@@ -214,7 +213,7 @@ menu "Application Level Tracing"
             depends on APPTRACE_SV_ENABLE
             default APPTRACE_SV_DEST_JTAG
             help
-                SystemView will transfer data through the defined interface.
+                SystemView witt transfer data trough defined interface.
 
             config APPTRACE_SV_DEST_JTAG
                 bool "Data destination JTAG"
@@ -232,7 +231,7 @@ menu "Application Level Tracing"
 
         choice APPTRACE_SV_CPU
             prompt "CPU to trace"
-            depends on APPTRACE_SV_DEST_UART && !ESP_SYSTEM_SINGLE_CORE_MODE
+            depends on APPTRACE_SV_DEST_UART && !FREERTOS_UNICORE
             default APPTRACE_SV_DEST_CPU_0
             help
                 Define the CPU to trace by SystemView.
@@ -253,8 +252,8 @@ menu "Application Level Tracing"
         choice APPTRACE_SV_TS_SOURCE
             prompt "Timer to use as timestamp source"
             depends on APPTRACE_SV_ENABLE
-            default APPTRACE_SV_TS_SOURCE_CCOUNT if ESP_SYSTEM_SINGLE_CORE_MODE && !PM_ENABLE && !IDF_TARGET_ESP32C3
-            default APPTRACE_SV_TS_SOURCE_GPTIMER if !ESP_SYSTEM_SINGLE_CORE_MODE && !PM_ENABLE && !IDF_TARGET_ESP32C3
+            default APPTRACE_SV_TS_SOURCE_CCOUNT if FREERTOS_UNICORE && !PM_ENABLE && !IDF_TARGET_ESP32C3
+            default APPTRACE_SV_TS_SOURCE_GPTIMER if !FREERTOS_UNICORE && !PM_ENABLE && !IDF_TARGET_ESP32C3
             default APPTRACE_SV_TS_SOURCE_ESP_TIMER if PM_ENABLE || IDF_TARGET_ESP32C3
             help
                 SystemView needs to use a hardware timer as the source of timestamps
@@ -262,7 +261,7 @@ menu "Application Level Tracing"
 
             config APPTRACE_SV_TS_SOURCE_CCOUNT
                 bool "CPU cycle counter (CCOUNT)"
-                depends on ESP_SYSTEM_SINGLE_CORE_MODE && !PM_ENABLE && !IDF_TARGET_ESP32C3
+                depends on FREERTOS_UNICORE && !PM_ENABLE && !IDF_TARGET_ESP32C3
 
             config APPTRACE_SV_TS_SOURCE_GPTIMER
                 bool "General Purpose Timer (Timer Group)"

components/app_trace/app_trace.c

@@ -1,7 +1,7 @@
 /*
- * SPDX-FileCopyrightText: 2017-2024 Espressif Systems (Shanghai) CO LTD
+ * SPDX-FileCopyrightText: 2017-2021 Espressif Systems (Shanghai) CO LTD
  *
- * SPDX-License-Identifier: Apache-2.0 OR MIT
+ * SPDX-License-Identifier: Apache-2.0
  */
 
 #include <string.h>
@@ -26,6 +26,8 @@
 #define ESP_APPTRACE_MAX_VPRINTF_ARGS 256
 #define ESP_APPTRACE_HOST_BUF_SIZE 256
 
+#define ESP_APPTRACE_PRINT_LOCK 0
+
 const static char *TAG = "esp_apptrace";
 
 /** tracing module internal data */
@@ -75,7 +77,7 @@ esp_err_t esp_apptrace_init(void)
     return ESP_OK;
 }
 
-ESP_SYSTEM_INIT_FN(esp_apptrace_init, SECONDARY, ESP_SYSTEM_INIT_ALL_CORES, 115)
+ESP_SYSTEM_INIT_FN(esp_apptrace_init, ESP_SYSTEM_INIT_ALL_CORES, 115)
 {
     return esp_apptrace_init();
 }
@@ -88,7 +90,7 @@ void esp_apptrace_down_buffer_config(uint8_t *buf, uint32_t size)
         return;
     }
     // currently down buffer is supported for JTAG interface only
-    // TODO: one more argument should be added to this function to specify HW interface: JTAG, UART0 etc
+    // TODO: one more argument should be added to this function to specify HW inteface: JTAG, UART0 etc
     ch = &s_trace_channels[ESP_APPTRACE_DEST_JTAG];
     if (ch->hw != NULL) {
         if (ch->hw->down_buffer_config != NULL) {
@@ -194,7 +196,7 @@ esp_err_t esp_apptrace_read(esp_apptrace_dest_t dest, void *buf, uint32_t *size,
     *size = 0;
     uint8_t *ptr = ch->hw->get_down_buffer(ch->hw_data, &act_sz, &tmo);
     if (ptr && act_sz > 0) {
-        ESP_APPTRACE_LOGD("Read %" PRIu32 " bytes from host", act_sz);
+        ESP_APPTRACE_LOGD("Read %d bytes from host", act_sz);
         memcpy(buf, ptr, act_sz);
         res = ch->hw->put_down_buffer(ch->hw_data, ptr, &tmo);
         *size = act_sz;
@@ -327,7 +329,7 @@ int esp_apptrace_vprintf_to(esp_apptrace_dest_t dest, uint32_t user_tmo, const c
     }
 
     esp_apptrace_tmo_init(&tmo, user_tmo);
-    ESP_APPTRACE_LOGD("fmt %p", fmt);
+    ESP_APPTRACE_LOGD("fmt %x", fmt);
     while ((p = (uint8_t *)strchr((char *)p, '%')) && nargs < ESP_APPTRACE_MAX_VPRINTF_ARGS) {
         p++;
         if (*p != '%' && *p != 0) {
@@ -353,7 +355,7 @@ int esp_apptrace_vprintf_to(esp_apptrace_dest_t dest, uint32_t user_tmo, const c
         uint32_t arg = va_arg(ap, uint32_t);
         *(uint32_t *)pout = arg;
         pout += sizeof(uint32_t);
-        ESP_APPTRACE_LOGD("arg %" PRIx32, arg);
+        ESP_APPTRACE_LOGD("arg %x", arg);
     }
 
     int ret = ch->hw->put_up_buffer(ch->hw_data, p, &tmo);

components/app_trace/app_trace_membufs_proto.c

@@ -1,7 +1,7 @@
 /*
- * SPDX-FileCopyrightText: 2021-2024 Espressif Systems (Shanghai) CO LTD
+ * SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
  *
- * SPDX-License-Identifier: Apache-2.0 OR MIT
+ * SPDX-License-Identifier: Apache-2.0
  */
 
 #include <sys/param.h>
@@ -100,7 +100,7 @@ static esp_err_t esp_apptrace_membufs_swap(esp_apptrace_membufs_proto_data_t *pr
     if (proto->hw->host_data_pending() && hdr->block_sz > 0) {
         // TODO: add support for multiple blocks from host, currently there is no need for that
         uint8_t *p = proto->blocks[new_block_num].start + proto->blocks[new_block_num].sz;
-        ESP_APPTRACE_LOGD("Recvd %" PRIu16 " bytes from host (@ %p) [%x %x %x %x %x %x %x %x .. %x %x %x %x %x %x %x %x]",
+        ESP_APPTRACE_LOGD("Recvd %d bytes from host (@ 0x%x) [%x %x %x %x %x %x %x %x .. %x %x %x %x %x %x %x %x]",
             hdr->block_sz, proto->blocks[new_block_num].start,
             *(proto->blocks[new_block_num].start+0), *(proto->blocks[new_block_num].start+1),
             *(proto->blocks[new_block_num].start+2), *(proto->blocks[new_block_num].start+3),
@@ -109,7 +109,7 @@ static esp_err_t esp_apptrace_membufs_swap(esp_apptrace_membufs_proto_data_t *pr
             *(p-8), *(p-7), *(p-6), *(p-5), *(p-4), *(p-3), *(p-2), *(p-1));
         uint32_t sz = esp_apptrace_membufs_down_buffer_write_nolock(proto, (uint8_t *)(hdr+1), hdr->block_sz);
         if (sz != hdr->block_sz) {
-            ESP_APPTRACE_LOGE("Failed to write %" PRIu32 " bytes to down buffer (%" PRIu16 " %" PRIu32 ")!", hdr->block_sz - sz, hdr->block_sz, sz);
+            ESP_APPTRACE_LOGE("Failed to write %d bytes to down buffer (%d %d)!", hdr->block_sz - sz, hdr->block_sz, sz);
         }
         hdr->block_sz = 0;
     }
@@ -196,7 +196,7 @@ static uint32_t esp_apptrace_membufs_down_buffer_write_nolock(esp_apptrace_membu
     uint32_t total_sz = 0;
 
     while (total_sz < size) {
-        ESP_APPTRACE_LOGD("esp_apptrace_trax_down_buffer_write_nolock WRS %" PRIu32 "-%" PRIu32 "-%" PRIu32 " %" PRIu32, proto->rb_down.wr, proto->rb_down.rd,
+        ESP_APPTRACE_LOGD("esp_apptrace_trax_down_buffer_write_nolock WRS %d-%d-%d %d", proto->rb_down.wr, proto->rb_down.rd,
             proto->rb_down.cur_size, size);
         uint32_t wr_sz = esp_apptrace_rb_write_size_get(&proto->rb_down);
         if (wr_sz == 0) {
@@ -206,15 +206,15 @@ static uint32_t esp_apptrace_membufs_down_buffer_write_nolock(esp_apptrace_membu
         if (wr_sz > size - total_sz) {
             wr_sz = size - total_sz;
         }
-        ESP_APPTRACE_LOGD("esp_apptrace_trax_down_buffer_write_nolock wr %" PRIu32, wr_sz);
+        ESP_APPTRACE_LOGD("esp_apptrace_trax_down_buffer_write_nolock wr %d", wr_sz);
         uint8_t *ptr = esp_apptrace_rb_produce(&proto->rb_down, wr_sz);
         if (!ptr) {
             assert(false && "Failed to produce bytes to down buffer!");
         }
-        ESP_APPTRACE_LOGD("esp_apptrace_trax_down_buffer_write_nolock wr %" PRIu32 " to %p from %p", wr_sz, ptr, data + total_sz + wr_sz);
+        ESP_APPTRACE_LOGD("esp_apptrace_trax_down_buffer_write_nolock wr %d to 0x%x from 0x%x", wr_sz, ptr, data + total_sz + wr_sz);
         memcpy(ptr, data + total_sz, wr_sz);
         total_sz += wr_sz;
-        ESP_APPTRACE_LOGD("esp_apptrace_trax_down_buffer_write_nolock wr %" PRIu32 "/%" PRIu32 "", wr_sz, total_sz);
+        ESP_APPTRACE_LOGD("esp_apptrace_trax_down_buffer_write_nolock wr %d/%d", wr_sz, total_sz);
     }
     return total_sz;
 }
@@ -278,7 +278,7 @@ uint8_t *esp_apptrace_membufs_up_buffer_get(esp_apptrace_membufs_proto_data_t *p
     uint8_t *buf_ptr = NULL;
 
     if (size > ESP_APPTRACE_USR_DATA_LEN_MAX(proto)) {
-        ESP_APPTRACE_LOGE("Too large user data size %" PRIu32 "!", size);
+        ESP_APPTRACE_LOGE("Too large user data size %d!", size);
         return NULL;
     }
 
@@ -308,21 +308,21 @@ uint8_t *esp_apptrace_membufs_up_buffer_get(esp_apptrace_membufs_proto_data_t *p
 #endif
         if (ESP_APPTRACE_INBLOCK_MARKER(proto) + ESP_APPTRACE_USR_BLOCK_RAW_SZ(size) > ESP_APPTRACE_INBLOCK(proto)->sz) {
             #if CONFIG_APPTRACE_PENDING_DATA_SIZE_MAX > 0
-            ESP_APPTRACE_LOGD("Block full. Get %" PRIu32 " bytes from PEND buffer", size);
+            ESP_APPTRACE_LOGD("Block full. Get %d bytes from PEND buffer", size);
             buf_ptr = esp_apptrace_rb_produce(&proto->rb_pend, ESP_APPTRACE_USR_BLOCK_RAW_SZ(size));
             #endif
             if (buf_ptr == NULL) {
                 int pended_buf;
-                ESP_APPTRACE_LOGD(" full. Get %" PRIu32 " bytes from pend buffer", size);
+                ESP_APPTRACE_LOGD(" full. Get %d bytes from pend buffer", size);
                 buf_ptr = esp_apptrace_membufs_wait4buf(proto, ESP_APPTRACE_USR_BLOCK_RAW_SZ(size), tmo, &pended_buf);
                 if (buf_ptr && !pended_buf) {
-                    ESP_APPTRACE_LOGD("Got %" PRIu32 " bytes from block", size);
+                    ESP_APPTRACE_LOGD("Got %d bytes from block", size);
                     // update cur block marker
                     ESP_APPTRACE_INBLOCK_MARKER_UPD(proto, ESP_APPTRACE_USR_BLOCK_RAW_SZ(size));
                 }
             }
         } else {
-            ESP_APPTRACE_LOGD("Get %" PRIu32 " bytes from  buffer", size);
+            ESP_APPTRACE_LOGD("Get %d bytes from  buffer", size);
             // fit to curr  nlock
             buf_ptr = ESP_APPTRACE_INBLOCK(proto)->start + ESP_APPTRACE_INBLOCK_MARKER(proto);
             // update cur block marker
@@ -352,12 +352,12 @@ esp_err_t esp_apptrace_membufs_flush_nolock(esp_apptrace_membufs_proto_data_t *p
     int res = ESP_OK;
 
     if (ESP_APPTRACE_INBLOCK_MARKER(proto) < min_sz) {
-        ESP_APPTRACE_LOGI("Ignore flush request for min %" PRIu32 " bytes. Bytes in  block: %" PRIu32, min_sz, ESP_APPTRACE_INBLOCK_MARKER(proto));
+        ESP_APPTRACE_LOGI("Ignore flush request for min %d bytes. Bytes in  block: %d.", min_sz, ESP_APPTRACE_INBLOCK_MARKER(proto));
         return ESP_OK;
     }
     // switch  block while size of data (including that in pending buffer) is more than min size
     while (ESP_APPTRACE_INBLOCK_MARKER(proto) > min_sz) {
-        ESP_APPTRACE_LOGD("Try to flush %" PRIu32 " bytes. Wait until block switch for %" PRIi64 " us", ESP_APPTRACE_INBLOCK_MARKER(proto), tmo->tmo);
+        ESP_APPTRACE_LOGD("Try to flush %d bytes. Wait until block switch for %lld us", ESP_APPTRACE_INBLOCK_MARKER(proto), tmo->tmo);
         res = esp_apptrace_membufs_swap_waitus(proto, tmo);
         if (res != ESP_OK) {
             if (tmo->tmo != ESP_APPTRACE_TMO_INFINITE)

components/app_trace/app_trace_util.c

@@ -1,7 +1,7 @@
 /*
- * SPDX-FileCopyrightText: 2017-2024 Espressif Systems (Shanghai) CO LTD
+ * SPDX-FileCopyrightText: 2017-2021 Espressif Systems (Shanghai) CO LTD
  *
- * SPDX-License-Identifier: Apache-2.0 OR MIT
+ * SPDX-License-Identifier: Apache-2.0
  */
 //
 #include "freertos/FreeRTOS.h"
@@ -9,14 +9,12 @@
 #include "esp_app_trace_util.h"
 #include "sdkconfig.h"
 
-#define ESP_APPTRACE_PRINT_LOCK 0
-
 ///////////////////////////////////////////////////////////////////////////////
 ///////////////////////////////// Locks /////////////////////////////////////
 ///////////////////////////////////////////////////////////////////////////////
 
 #if ESP_APPTRACE_PRINT_LOCK
-static esp_apptrace_lock_t s_log_lock = { .mux = portMUX_INITIALIZER_UNLOCKED };
+static esp_apptrace_lock_t s_log_lock = {.irq_stat = 0, .portmux = portMUX_INITIALIZER_UNLOCKED};
 #endif
 
 int esp_apptrace_log_lock(void)
@@ -33,7 +31,7 @@ int esp_apptrace_log_lock(void)
 
 void esp_apptrace_log_unlock(void)
 {
-#if ESP_APPTRACE_PRINT_LOCK
+ #if ESP_APPTRACE_PRINT_LOCK
     esp_apptrace_lock_give(&s_log_lock);
 #endif
 }
@@ -89,7 +87,7 @@ esp_err_t esp_apptrace_lock_give(esp_apptrace_lock_t *lock)
 uint8_t *esp_apptrace_rb_produce(esp_apptrace_rb_t *rb, uint32_t size)
 {
     uint8_t *ptr = rb->data + rb->wr;
-    // check for available space
+    // check for avalable space
     if (rb->rd <= rb->wr) {
         // |?R......W??|
         if (rb->wr + size >= rb->size) {

components/app_trace/gcov/gcov_rtio.c

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: 2017-2024 Espressif Systems (Shanghai) CO LTD
+ * SPDX-FileCopyrightText: 2017-2023 Espressif Systems (Shanghai) CO LTD
  *
  * SPDX-License-Identifier: Apache-2.0
  */
@@ -14,10 +14,14 @@
 #include "soc/timer_periph.h"
 #include "esp_app_trace.h"
 #include "esp_freertos_hooks.h"
-#include "dbg_stubs.h"
-#include "esp_private/esp_ipc.h"
-#include "esp_attr.h"
+#include "esp_private/dbg_stubs.h"
+#include "esp_ipc.h"
 #include "hal/wdt_hal.h"
+#if CONFIG_IDF_TARGET_ESP32
+#include "esp32/rom/libc_stubs.h"
+#elif CONFIG_IDF_TARGET_ESP32S2
+#include "esp32s2/rom/libc_stubs.h"
+#endif
 
 #if CONFIG_APPTRACE_GCOV_ENABLE
 
@@ -76,11 +80,11 @@ void gcov_create_task(void *arg)
 		(void *)&s_gcov_task_running, configMAX_PRIORITIES - 1, NULL, 0);
 }
 
-static IRAM_ATTR
 void gcov_create_task_tick_hook(void)
 {
+    extern esp_err_t esp_ipc_start_gcov_from_isr(uint32_t cpu_id, esp_ipc_func_t func, void* arg);
     if (s_create_gcov_task) {
-        if (esp_ipc_call_nonblocking(xPortGetCoreID(), &gcov_create_task, NULL) == ESP_OK) {
+        if (esp_ipc_start_gcov_from_isr(xPortGetCoreID(), &gcov_create_task, NULL) == ESP_OK) {
             s_create_gcov_task = false;
         }
     }
@@ -102,16 +106,12 @@ static int esp_dbg_stub_gcov_entry(void)
 
 void gcov_rtio_init(void)
 {
-    uint32_t stub_entry = 0;
+    uint32_t capabilities = 0;
     ESP_EARLY_LOGV(TAG, "%s", __FUNCTION__);
-    assert(esp_dbg_stub_entry_get(ESP_DBG_STUB_ENTRY_GCOV, &stub_entry) == ESP_OK);
-    if (stub_entry != 0) {
-        /* "__gcov_init()" can be called several times. We must avoid multiple tick hook registration */
-        return;
-    }
     esp_dbg_stub_entry_set(ESP_DBG_STUB_ENTRY_GCOV, (uint32_t)&esp_dbg_stub_gcov_entry);
-    assert(esp_dbg_stub_entry_get(ESP_DBG_STUB_ENTRY_CAPABILITIES, &stub_entry) == ESP_OK);
-    esp_dbg_stub_entry_set(ESP_DBG_STUB_ENTRY_CAPABILITIES, stub_entry | ESP_DBG_STUB_CAP_GCOV_TASK);
+    if (esp_dbg_stub_entry_get(ESP_DBG_STUB_ENTRY_CAPABILITIES, &capabilities) == ESP_OK) {
+        esp_dbg_stub_entry_set(ESP_DBG_STUB_ENTRY_CAPABILITIES, capabilities | ESP_DBG_STUB_CAP_GCOV_TASK);
+    }
     esp_register_freertos_tick_hook(gcov_create_task_tick_hook);
 }
 

components/app_trace/host_file_io.c

@@ -122,7 +122,7 @@ static esp_err_t esp_apptrace_file_rsp_recv(esp_apptrace_dest_t dest, uint8_t *b
             ESP_EARLY_LOGE(TAG, "Failed to read (%d)!", ret);
             return ret;
         }
-        ESP_EARLY_LOGV(TAG, "%s read %" PRIu32 " bytes", __FUNCTION__, rd_size);
+        ESP_EARLY_LOGV(TAG, "%s read %d bytes", __FUNCTION__, rd_size);
         tot_rd += rd_size;
     }
 

components/app_trace/linker.lf

@@ -15,7 +15,7 @@ entries:
       SEGGER_SYSVIEW_FreeRTOS (noflash)
 
 [mapping:app_trace_driver]
-archive: libesp_driver_gptimer.a
+archive: libdriver.a
 entries:
     if APPTRACE_SV_TS_SOURCE_GPTIMER = y:
       gptimer (noflash)

components/app_trace/port/port_uart.c

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: 2017-2024 Espressif Systems (Shanghai) CO LTD
+ * SPDX-FileCopyrightText: 2017-2021 Espressif Systems (Shanghai) CO LTD
  *
  * SPDX-License-Identifier: Apache-2.0
  */
@@ -218,13 +218,6 @@ static esp_err_t esp_apptrace_uart_init(esp_apptrace_uart_data_t *hw_data)
         hw_data->message_buff_overflow = false;
         hw_data->circular_buff_overflow = false;
 
-        assert((hw_data->port_num <= SOC_UART_NUM) && "Not possible to configure UART. Please check selected UART port");
-
-        int source_clk = UART_SCLK_DEFAULT;
-#if SOC_UART_LP_NUM > 0
-        if (hw_data->port_num >= SOC_UART_HP_NUM)
-            source_clk = LP_UART_SCLK_DEFAULT;
-#endif
 
         const uart_config_t uart_config = {
             .baud_rate = CONFIG_APPTRACE_UART_BAUDRATE,
@@ -232,7 +225,7 @@ static esp_err_t esp_apptrace_uart_init(esp_apptrace_uart_data_t *hw_data)
             .parity = UART_PARITY_DISABLE,
             .stop_bits = UART_STOP_BITS_1,
             .flow_ctrl = UART_HW_FLOWCTRL_DISABLE,
-            .source_clk = source_clk,
+            .source_clk = UART_SCLK_DEFAULT,
         };
         ESP_LOGI(TAG, "UART baud rate: %i", CONFIG_APPTRACE_UART_BAUDRATE);
         // We won't use a buffer for sending data.

components/app_trace/port/riscv/port.c

@@ -1,10 +1,9 @@
 /*
- * SPDX-FileCopyrightText: 2021-2024 Espressif Systems (Shanghai) CO LTD
+ * SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
  *
- * SPDX-License-Identifier: Apache-2.0 OR MIT
+ * SPDX-License-Identifier: Apache-2.0
  */
 
-#include "sdkconfig.h"
 #include "esp_cpu.h"
 #include "esp_log.h"
 #include "esp_app_trace_membufs_proto.h"
@@ -57,13 +56,7 @@ static bool esp_apptrace_riscv_host_data_pending(void);
 
 const static char *TAG = "esp_apptrace";
 
-#if SOC_CACHE_INTERNAL_MEM_VIA_L1CACHE
-#define APPTRACE_DRAM_ATTR TCM_DRAM_ATTR
-#else
-#define APPTRACE_DRAM_ATTR
-#endif
-
-static APPTRACE_DRAM_ATTR esp_apptrace_riscv_ctrl_block_t s_tracing_ctrl[CONFIG_FREERTOS_NUMBER_OF_CORES];
+static esp_apptrace_riscv_ctrl_block_t s_tracing_ctrl[portNUM_PROCESSORS];
 
 esp_apptrace_hw_t *esp_apptrace_jtag_hw_get(void **data)
 {
@@ -98,7 +91,7 @@ esp_apptrace_hw_t *esp_apptrace_jtag_hw_get(void **data)
 }
 
 /* Advertises apptrace control block address to host.
-   This function can be overridden with custom implementation,
+   This function can be overriden with custom implementation,
    e.g. OpenOCD flasher stub use own implementation of it. */
 __attribute__((weak)) int esp_apptrace_advertise_ctrl_block(void *ctrl_block_addr)
 {
@@ -109,7 +102,7 @@ __attribute__((weak)) int esp_apptrace_advertise_ctrl_block(void *ctrl_block_add
 }
 
 /* Returns up buffers config.
-   This function can be overridden with custom implementation,
+   This function can be overriden with custom implementation,
    e.g. OpenOCD flasher stub use own implementation of it. */
 __attribute__((weak)) void esp_apptrace_get_up_buffers(esp_apptrace_mem_block_t mem_blocks_cfg[2])
 {
@@ -165,13 +158,13 @@ static esp_err_t esp_apptrace_riscv_init(esp_apptrace_riscv_data_t *hw_data)
     ESP_APPTRACE_LOGI("Apptrace initialized on CPU%d. Tracing control block @ %p.", core_id, &s_tracing_ctrl[core_id]);
     s_tracing_ctrl[core_id].mem_blocks = hw_data->membufs.blocks;
     for (int i = 0; i < 2; i++) {
-        ESP_APPTRACE_LOGD("Mem buf[%d] %" PRIu32 " bytes @ %p (%p/%p)", i,
+        ESP_APPTRACE_LOGD("Mem buf[%d] %d bytes @ %p (%p/%p)", i,
             s_tracing_ctrl[core_id].mem_blocks[i].sz, s_tracing_ctrl[core_id].mem_blocks[i].start,
             &(s_tracing_ctrl[core_id].mem_blocks[i].start), &(s_tracing_ctrl[core_id].mem_blocks[i].sz));
     }
     // notify host about control block address
     int res = esp_apptrace_advertise_ctrl_block(&s_tracing_ctrl[core_id]);
-    assert(res == 0 && "Failed to send config to host!");
+    assert(res == 0 && "Falied to send config to host!");
 
     return ESP_OK;
 }
@@ -331,7 +324,7 @@ static esp_err_t esp_apptrace_riscv_buffer_swap_start(uint32_t curr_block_id)
         uint32_t acked_block = ESP_APPTRACE_RISCV_BLOCK_ID_GET(ctrl_reg);
         uint32_t host_to_read = ESP_APPTRACE_RISCV_BLOCK_LEN_GET(ctrl_reg);
         if (host_to_read != 0 || acked_block != (curr_block_id & ESP_APPTRACE_RISCV_BLOCK_ID_MSK)) {
-            ESP_APPTRACE_LOGD("[%d]: Can not switch %" PRIx32 " %" PRIu32 " %" PRIx32 " %" PRIx32 "/%" PRIx32, esp_cpu_get_core_id(), ctrl_reg, host_to_read, acked_block,
+            ESP_APPTRACE_LOGD("[%d]: Can not switch %x %d %x %x/%lx", esp_cpu_get_core_id(), ctrl_reg, host_to_read, acked_block,
                 curr_block_id & ESP_APPTRACE_RISCV_BLOCK_ID_MSK, curr_block_id);
             res = ESP_ERR_NO_MEM;
             goto _on_err;

components/app_trace/port/xtensa/port.c

@@ -1,7 +1,7 @@
 /*
- * SPDX-FileCopyrightText: 2015-2024 Espressif Systems (Shanghai) CO LTD
+ * SPDX-FileCopyrightText: 2015-2021 Espressif Systems (Shanghai) CO LTD
  *
- * SPDX-License-Identifier: Apache-2.0 OR MIT
+ * SPDX-License-Identifier: Apache-2.0
  */
 
 //
@@ -288,7 +288,7 @@ static inline void esp_apptrace_trax_select_memory_block(int block_num)
                         : TRACEMEM_CORE0_MUX_BLK_BITS(TRACEMEM_MUX_BLK1_NUM);
     block_bits |= block_num ? TRACEMEM_CORE1_MUX_BLK_BITS(TRACEMEM_MUX_BLK0_NUM)
                         : TRACEMEM_CORE1_MUX_BLK_BITS(TRACEMEM_MUX_BLK1_NUM);
-    ESP_EARLY_LOGV(TAG, "Select block %d @ %p (bits 0x%" PRIx32 ")", block_num, s_trax_blocks[block_num], block_bits);
+    ESP_EARLY_LOGV(TAG, "Select block %d @ %p (bits 0x%x)", block_num, s_trax_blocks[block_num], block_bits);
     DPORT_WRITE_PERI_REG(SENSITIVE_INTERNAL_SRAM_USAGE_2_REG, block_bits);
 #endif
 }
@@ -298,7 +298,7 @@ static inline void esp_apptrace_trax_memory_enable(void)
 #if CONFIG_IDF_TARGET_ESP32
         /* Enable trace memory on PRO CPU */
         DPORT_WRITE_PERI_REG(DPORT_PRO_TRACEMEM_ENA_REG, DPORT_PRO_TRACEMEM_ENA_M);
-#if CONFIG_ESP_SYSTEM_SINGLE_CORE_MODE == 0
+#if CONFIG_FREERTOS_UNICORE == 0
         /* Enable trace memory on APP CPU */
         DPORT_WRITE_PERI_REG(DPORT_APP_TRACEMEM_ENA_REG, DPORT_APP_TRACEMEM_ENA_M);
 #endif
@@ -498,7 +498,7 @@ static esp_err_t esp_apptrace_trax_buffer_swap_start(uint32_t curr_block_id)
         uint32_t acked_block = ESP_APPTRACE_TRAX_BLOCK_ID_GET(ctrl_reg);
         uint32_t host_to_read = ESP_APPTRACE_TRAX_BLOCK_LEN_GET(ctrl_reg);
         if (host_to_read != 0 || acked_block != (curr_block_id & ESP_APPTRACE_TRAX_BLOCK_ID_MSK)) {
-            ESP_APPTRACE_LOGD("HC[%d]: Can not switch %" PRIx32 " %" PRIu32 " %" PRIx32 " %" PRIx32 "/%" PRIx32, esp_cpu_get_core_id(), ctrl_reg, host_to_read, acked_block,
+            ESP_APPTRACE_LOGD("HC[%d]: Can not switch %x %d %x %x/%lx", esp_cpu_get_core_id(), ctrl_reg, host_to_read, acked_block,
                 curr_block_id & ESP_APPTRACE_TRAX_BLOCK_ID_MSK, curr_block_id);
             res = ESP_ERR_NO_MEM;
             goto _on_err;

components/app_trace/private_include/dbg_stubs.h

@@ -1,69 +0,0 @@
-/*
- * SPDX-FileCopyrightText: 2017-2024 Espressif Systems (Shanghai) CO LTD
- *
- * SPDX-License-Identifier: Apache-2.0
- */
-
-#ifndef ESP_DBG_STUBS_H_
-#define ESP_DBG_STUBS_H_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include "esp_err.h"
-
-/**
- * Debug stubs entries IDs
- */
-typedef enum {
-    ESP_DBG_STUB_MAGIC_NUM,
-    ESP_DBG_STUB_TABLE_SIZE,
-    ESP_DBG_STUB_CONTROL_DATA,   ///< stubs descriptor entry
-    ESP_DBG_STUB_ENTRY_FIRST,
-    ESP_DBG_STUB_ENTRY_GCOV ///< GCOV entry
-    = ESP_DBG_STUB_ENTRY_FIRST,
-    ESP_DBG_STUB_ENTRY_CAPABILITIES,
-    ESP_DBG_STUB_ENTRY_MAX
-} esp_dbg_stub_id_t;
-
-#define ESP_DBG_STUB_MAGIC_NUM_VAL      0xFEEDBEEF
-#define ESP_DBG_STUB_CAP_GCOV_TASK      (1 << 0)
-
-/**
- * @brief  Initializes debug stubs.
- *
- * @note   Must be called after esp_apptrace_init() if app tracing is enabled.
- */
-void esp_dbg_stubs_init(void);
-
-/**
- * @brief  Initializes application tracing module.
- *
- * @note   Should be called before any esp_apptrace_xxx call.
- *
- * @param id    Stub ID.
- * @param entry Stub entry. Usually it is stub entry function address,
- *              but can be any value meaningfull for OpenOCD command/code
- *              such as capabilities
- * @return ESP_OK on success, otherwise see esp_err_t
- */
-esp_err_t esp_dbg_stub_entry_set(esp_dbg_stub_id_t id, uint32_t entry);
-
-/**
- * @brief   Retrives the corresponding stub entry
- *
- * @param id    Stub ID.
- * @param entry Stub entry. Usually it is stub entry function address,
- *              but can be any value meaningfull for OpenOCD command/code
- *              such as capabilities
- *
- * @return ESP_OK on success, otherwise see esp_err_t
- */
-esp_err_t esp_dbg_stub_entry_get(esp_dbg_stub_id_t id, uint32_t *entry);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif // ESP_DBG_STUBS_H_

components/app_trace/project_include.cmake

@@ -5,25 +5,21 @@ function(idf_create_coverage_report report_dir)
     set(gcov_tool ${_CMAKE_TOOLCHAIN_PREFIX}gcov)
     idf_build_get_property(project_name PROJECT_NAME)
 
-    file(TO_NATIVE_PATH "${report_dir}" _report_dir)
-    file(TO_NATIVE_PATH "${project_dir}" _project_dir)
-    file(TO_NATIVE_PATH "${report_dir}/html/index.html" _index_path)
-
     add_custom_target(pre-cov-report
-        COMMENT "Generating coverage report in: ${_report_dir}"
+        COMMENT "Generating coverage report in: ${report_dir}"
         COMMAND ${CMAKE_COMMAND} -E echo "Using gcov: ${gcov_tool}"
-        COMMAND ${CMAKE_COMMAND} -E make_directory ${_report_dir}/html
+        COMMAND ${CMAKE_COMMAND} -E make_directory ${report_dir}/html
         )
 
     add_custom_target(lcov-report
         COMMENT "WARNING: lcov-report is deprecated. Please use gcovr-report instead."
-        COMMAND lcov --gcov-tool ${gcov_tool} -c -d ${CMAKE_CURRENT_BINARY_DIR} -o ${_report_dir}/${project_name}.info
-        COMMAND genhtml -o ${_report_dir}/html ${_report_dir}/${project_name}.info
+        COMMAND lcov --gcov-tool ${gcov_tool} -c -d ${CMAKE_CURRENT_BINARY_DIR} -o ${report_dir}/${project_name}.info
+        COMMAND genhtml -o ${report_dir}/html ${report_dir}/${project_name}.info
         DEPENDS pre-cov-report
         )
 
     add_custom_target(gcovr-report
-        COMMAND gcovr -r ${_project_dir} --gcov-executable ${gcov_tool} -s --html-details ${_index_path}
+        COMMAND gcovr -r ${project_dir} --gcov-executable ${gcov_tool} -s --html-details ${report_dir}/html/index.html
         WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
         DEPENDS pre-cov-report
         )
@@ -33,9 +29,7 @@ endfunction()
 #
 # Clean coverage report.
 function(idf_clean_coverage_report report_dir)
-    file(TO_CMAKE_PATH "${report_dir}" _report_dir)
-
     add_custom_target(cov-data-clean
-        COMMENT "Clean coverage report in: ${_report_dir}"
-        COMMAND ${CMAKE_COMMAND} -E remove_directory ${_report_dir})
+        COMMENT "Clean coverage report in: ${report_dir}"
+        COMMAND ${CMAKE_COMMAND} -E remove_directory ${report_dir})
 endfunction()

components/app_trace/sys_view/SEGGER/sbom.yml

@@ -1,6 +0,0 @@
-name: 'SystemView'
-version: '3.42'
-cpe: cpe:2.3:a:segger:systemview:{}:*:*:*:*:*:*:*
-supplier: 'Organization: Espressif Systems (Shanghai) CO LTD'
-originator: 'Organization: SEGGER Microcontroller GmbH'
-description: Real-time recording and visualization tool for embedded systems.

components/app_trace/sys_view/Sample/FreeRTOSV10.4/Config/esp/SEGGER_SYSVIEW_Config_FreeRTOS.c

@@ -3,7 +3,7 @@
  *
  * SPDX-License-Identifier: BSD-1-Clause
  *
- * SPDX-FileContributor: 2017-2024 Espressif Systems (Shanghai) CO LTD
+ * SPDX-FileContributor: 2017-2022 Espressif Systems (Shanghai) CO LTD
  */
 /*********************************************************************
 *                    SEGGER Microcontroller GmbH                     *
@@ -58,7 +58,6 @@ File    : SEGGER_SYSVIEW_Config_FreeRTOS.c
 Purpose : Sample setup configuration of SystemView with FreeRTOS.
 Revision: $Rev: 7745 $
 */
-#include <string.h>
 #include "sdkconfig.h"
 #include "freertos/FreeRTOS.h"
 #include "SEGGER_SYSVIEW.h"
@@ -157,16 +156,15 @@ static esp_apptrace_lock_t s_sys_view_lock = {.mux = portMUX_INITIALIZER_UNLOCKE
 *    Sends SystemView description strings.
 */
 static void _cbSendSystemDesc(void) {
-    char irq_str[32] = "I#";
+    char irq_str[32];
     SEGGER_SYSVIEW_SendSysDesc("N="SYSVIEW_APP_NAME",D="SYSVIEW_DEVICE_NAME",C="SYSVIEW_CORE_NAME",O=FreeRTOS");
-    strcat(itoa(SYSTICK_INTR_ID, irq_str + 2, 10), "=SysTick");
+    snprintf(irq_str, sizeof(irq_str), "I#%d=SysTick", SYSTICK_INTR_ID);
     SEGGER_SYSVIEW_SendSysDesc(irq_str);
     size_t isr_count = sizeof(esp_isr_names)/sizeof(esp_isr_names[0]);
     for (size_t i = 0; i < isr_count; ++i) {
         if (esp_isr_names[i] == NULL || (ETS_INTERNAL_INTR_SOURCE_OFF + i) == SYSTICK_INTR_ID)
             continue;
-        strcat(itoa(ETS_INTERNAL_INTR_SOURCE_OFF + i, irq_str + 2, 10), "=");
-        strncat(irq_str, esp_isr_names[i], sizeof(irq_str) - strlen(irq_str) - 1);
+        snprintf(irq_str, sizeof(irq_str), "I#%d=%s", ETS_INTERNAL_INTR_SOURCE_OFF + i, esp_isr_names[i]);
         SEGGER_SYSVIEW_SendSysDesc(irq_str);
     }
 }

components/app_trace/sys_view/esp/SEGGER_RTT_esp.c

@@ -35,7 +35,7 @@ static uint8_t s_down_buf[SYSVIEW_DOWN_BUF_SIZE];
 #if CONFIG_APPTRACE_SV_DEST_UART
 
 #define ESP_APPTRACE_DEST_SYSVIEW ESP_APPTRACE_DEST_UART
-#if CONFIG_APPTRACE_SV_DEST_CPU_0 || CONFIG_ESP_SYSTEM_SINGLE_CORE_MODE
+#if CONFIG_APPTRACE_SV_DEST_CPU_0 || CONFIG_FREERTOS_UNICORE
 #define APPTRACE_SV_DEST_CPU 0
 #else
 #define APPTRACE_SV_DEST_CPU 1
@@ -296,7 +296,7 @@ int SEGGER_RTT_ConfigDownBuffer(unsigned BufferIndex, const char* sName, void* p
  * linked whenever SystemView is used.
  */
 
-ESP_SYSTEM_INIT_FN(sysview_init, SECONDARY, BIT(0), 120)
+ESP_SYSTEM_INIT_FN(sysview_init, BIT(0), 120)
 {
     SEGGER_SYSVIEW_Conf();
     return ESP_OK;

components/app_trace/test_apps/.build-test-rules.yml

@@ -7,7 +7,3 @@ components/app_trace/test_apps:
     - soc
     - driver
     - esp_hw_support
-  disable:
-    - if: IDF_TARGET in ["esp32c5", "esp32c61"]
-      temporary: true
-      reason: not support yet # TODO: [ESP32C5] IDF-8705, [ESP32C61] IDF-9306

components/app_trace/test_apps/main/CMakeLists.txt

@@ -1,4 +1,4 @@
 idf_component_register(SRCS "test_app_trace_main.c" "test_trace.c"
                     INCLUDE_DIRS "."
-                    PRIV_REQUIRES app_trace unity esp_driver_gptimer
+                    PRIV_REQUIRES app_trace unity driver
                     WHOLE_ARCHIVE)

components/app_trace/test_apps/main/test_trace.c

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: 2021-2024 Espressif Systems (Shanghai) CO LTD
+ * SPDX-FileCopyrightText: 2021-2023 Espressif Systems (Shanghai) CO LTD
  *
  * SPDX-License-Identifier: Apache-2.0
  */
@@ -135,9 +135,9 @@ static bool esp_apptrace_test_timer_isr_crash(gptimer_handle_t timer, const gpti
         memset(tim_arg->data.buf + 2 * sizeof(uint32_t), tim_arg->data.wr_cnt & tim_arg->data.mask, tim_arg->data.buf_sz - 2 * sizeof(uint32_t));
         int res = ESP_APPTRACE_TEST_WRITE_FROM_ISR(tim_arg->data.buf, tim_arg->data.buf_sz);
         if (res != ESP_OK) {
-            esp_rom_printf("tim-%p: Failed to write trace %d %" PRIx32 "!\n", tim_arg->gptimer, res, tim_arg->data.wr_cnt & tim_arg->data.mask);
+            esp_rom_printf("tim-%x: Failed to write trace %d %x!\n", tim_arg->gptimer, res, tim_arg->data.wr_cnt & tim_arg->data.mask);
         } else {
-            esp_rom_printf("tim-%p: Written chunk%" PRIu32 " %" PRIu32 " bytes, %" PRIx32 "\n",
+            esp_rom_printf("tim-%x: Written chunk%d %d bytes, %x\n",
                            timer, tim_arg->data.wr_cnt, tim_arg->data.buf_sz, tim_arg->data.wr_cnt & tim_arg->data.mask);
             tim_arg->data.wr_cnt++;
         }
@@ -153,7 +153,7 @@ static void esp_apptrace_dummy_task(void *p)
     esp_apptrace_test_task_arg_t *arg = (esp_apptrace_test_task_arg_t *) p;
     TickType_t tmo_ticks = arg->data.period / (1000 * portTICK_PERIOD_MS);
 
-    ESP_APPTRACE_TEST_LOGI("%p: run dummy task (period %" PRIu32 " us, %" PRIu32 " timers)", xTaskGetCurrentTaskHandle(), arg->data.period, arg->timers_num);
+    ESP_APPTRACE_TEST_LOGI("%x: run dummy task (period %u us, %u timers)", xTaskGetCurrentTaskHandle(), arg->data.period, arg->timers_num);
 
     for (int i = 0; i < arg->timers_num; i++) {
         gptimer_config_t timer_config = {
@@ -163,7 +163,7 @@ static void esp_apptrace_dummy_task(void *p)
         };
         TEST_ESP_OK(gptimer_new_timer(&timer_config, &arg->timers[i].gptimer));
         *(uint32_t *)arg->timers[i].data.buf = (uint32_t)arg->timers[i].gptimer | (1 << 31);
-        ESP_APPTRACE_TEST_LOGI("%p: start timer %p period %" PRIu32 " us", xTaskGetCurrentTaskHandle(), arg->timers[i].gptimer, arg->timers[i].data.period);
+        ESP_APPTRACE_TEST_LOGI("%x: start timer %x period %u us", xTaskGetCurrentTaskHandle(), arg->timers[i].gptimer, arg->timers[i].data.period);
         gptimer_alarm_config_t alarm_config = {
             .reload_count = 0,
             .alarm_count = arg->timers[i].data.period,
@@ -180,7 +180,7 @@ static void esp_apptrace_dummy_task(void *p)
 
     int i = 0;
     while (!arg->stop) {
-        ESP_APPTRACE_TEST_LOGD("%p: dummy task work %d.%d", xTaskGetCurrentTaskHandle(), esp_cpu_get_core_id(), i++);
+        ESP_APPTRACE_TEST_LOGD("%x: dummy task work %d.%d", xTaskGetCurrentTaskHandle(), esp_cpu_get_core_id(), i++);
         if (tmo_ticks) {
             vTaskDelay(tmo_ticks);
         }
@@ -202,7 +202,7 @@ static void esp_apptrace_test_task(void *p)
     int res;
     TickType_t tmo_ticks = arg->data.period / (1000 * portTICK_PERIOD_MS);
 
-    ESP_APPTRACE_TEST_LOGI("%p: run (period %" PRIu32 " us, stamp mask %" PRIx8 ", %" PRIu32 " timers)", xTaskGetCurrentTaskHandle(), arg->data.period, arg->data.mask, arg->timers_num);
+    ESP_APPTRACE_TEST_LOGI("%x: run (period %u us, stamp mask %x, %u timers)", xTaskGetCurrentTaskHandle(), arg->data.period, arg->data.mask, arg->timers_num);
 
     for (int i = 0; i < arg->timers_num; i++) {
         gptimer_config_t timer_config = {
@@ -212,7 +212,7 @@ static void esp_apptrace_test_task(void *p)
         };
         TEST_ESP_OK(gptimer_new_timer(&timer_config, &arg->timers[i].gptimer));
         *(uint32_t *)arg->timers[i].data.buf = ((uint32_t)arg->timers[i].gptimer) | (1 << 31) | (esp_cpu_get_core_id() ? 0x1 : 0);
-        ESP_APPTRACE_TEST_LOGI("%p: start timer %p period %" PRIu32 " us", xTaskGetCurrentTaskHandle(), arg->timers[i].gptimer, arg->timers[i].data.period);
+        ESP_APPTRACE_TEST_LOGI("%x: start timer %x period %u us", xTaskGetCurrentTaskHandle(), arg->timers[i].gptimer, arg->timers[i].data.period);
         gptimer_alarm_config_t alarm_config = {
             .reload_count = 0,
             .alarm_count = arg->timers[i].data.period,
@@ -242,14 +242,14 @@ static void esp_apptrace_test_task(void *p)
         }
         if (res) {
             if (1) { //arg->data.wr_err++ < ESP_APPTRACE_TEST_PRN_WRERR_MAX) {
-                ESP_APPTRACE_TEST_LOGE("%p: Failed to write trace %d %" PRIx32 "!", xTaskGetCurrentTaskHandle(), res, arg->data.wr_cnt & arg->data.mask);
+                ESP_APPTRACE_TEST_LOGE("%x: Failed to write trace %d %x!", xTaskGetCurrentTaskHandle(), res, arg->data.wr_cnt & arg->data.mask);
                 if (arg->data.wr_err == ESP_APPTRACE_TEST_PRN_WRERR_MAX) {
                     ESP_APPTRACE_TEST_LOGE("\n");
                 }
             }
         } else {
             if (0) {
-                ESP_APPTRACE_TEST_LOGD("%p:%" PRIx32 ": Written chunk%" PRIu32 " %" PRIu32 " bytes, %" PRIx32, xTaskGetCurrentTaskHandle(), *ts, arg->data.wr_cnt, arg->data.buf_sz, arg->data.wr_cnt & arg->data.mask);
+                ESP_APPTRACE_TEST_LOGD("%x:%x: Written chunk%d %d bytes, %x", xTaskGetCurrentTaskHandle(), *ts, arg->data.wr_cnt, arg->data.buf_sz, arg->data.wr_cnt & arg->data.mask);
             }
             arg->data.wr_err = 0;
         }
@@ -273,7 +273,7 @@ static void esp_apptrace_test_task_crash(void *p)
 {
     esp_apptrace_test_task_arg_t *arg = (esp_apptrace_test_task_arg_t *) p;
 
-    ESP_APPTRACE_TEST_LOGE("%p: run (period %" PRIu32 " us, stamp mask %" PRIx8 ", %" PRIu32 " timers)", xTaskGetCurrentTaskHandle(), arg->data.period, arg->data.mask, arg->timers_num);
+    ESP_APPTRACE_TEST_LOGE("%x: run (period %u us, stamp mask %x, %u timers)", xTaskGetCurrentTaskHandle(), arg->data.period, arg->data.mask, arg->timers_num);
 
     arg->data.wr_cnt = 0;
     *(uint32_t *)arg->data.buf = (uint32_t)xTaskGetCurrentTaskHandle();
@@ -283,9 +283,9 @@ static void esp_apptrace_test_task_crash(void *p)
         memset(arg->data.buf + sizeof(uint32_t), arg->data.wr_cnt & arg->data.mask, arg->data.buf_sz - sizeof(uint32_t));
         int res = ESP_APPTRACE_TEST_WRITE(arg->data.buf, arg->data.buf_sz);
         if (res) {
-            ESP_APPTRACE_TEST_LOGE("%p: Failed to write trace %d %" PRIx32 "!", xTaskGetCurrentTaskHandle(), res, arg->data.wr_cnt & arg->data.mask);
+            ESP_APPTRACE_TEST_LOGE("%x: Failed to write trace %d %x!", xTaskGetCurrentTaskHandle(), res, arg->data.wr_cnt & arg->data.mask);
         } else {
-            ESP_APPTRACE_TEST_LOGD("%p: Written chunk%" PRIu32 " %" PRIu32 " bytes, %" PRIx32, xTaskGetCurrentTaskHandle(), arg->data.wr_cnt, arg->data.buf_sz, arg->data.wr_cnt & arg->data.mask);
+            ESP_APPTRACE_TEST_LOGD("%x: Written chunk%d %d bytes, %x", xTaskGetCurrentTaskHandle(), arg->data.wr_cnt, arg->data.buf_sz, arg->data.wr_cnt & arg->data.mask);
         }
         arg->data.wr_cnt++;
     }
@@ -315,7 +315,7 @@ static void esp_apptrace_test_ts_init(void)
         .resolution_hz = 10000000,
     };
     TEST_ESP_OK(gptimer_new_timer(&timer_config, &ts_gptimer));
-    ESP_APPTRACE_TEST_LOGI("Use timer %p for TS", ts_gptimer);
+    ESP_APPTRACE_TEST_LOGI("Use timer %x for TS", ts_gptimer);
     TEST_ESP_OK(gptimer_enable(ts_gptimer));
     TEST_ESP_OK(gptimer_start(ts_gptimer));
 }
@@ -368,7 +368,7 @@ static void esp_apptrace_test(esp_apptrace_test_cfg_t *test_cfg)
         TaskHandle_t thnd;
         sprintf(name, "apptrace_test%d", i);
         xTaskCreatePinnedToCore(test_cfg->tasks[i].task_func, name, 2048, &test_cfg->tasks[i], test_cfg->tasks[i].prio, &thnd, test_cfg->tasks[i].core);
-        ESP_APPTRACE_TEST_LOGI("Created task %p", thnd);
+        ESP_APPTRACE_TEST_LOGI("Created task %x", thnd);
     }
     xTaskCreatePinnedToCore(esp_apptrace_dummy_task, "dummy0", 2048, &dummy_task_arg, dummy_task_arg.prio, NULL, 0);
 #if CONFIG_FREERTOS_UNICORE == 0
@@ -641,7 +641,7 @@ static void esp_logtrace_task(void *p)
 {
     esp_logtrace_task_t *arg = (esp_logtrace_task_t *) p;
 
-    ESP_APPTRACE_TEST_LOGI("%p: run log test task", xTaskGetCurrentTaskHandle());
+    ESP_APPTRACE_TEST_LOGI("%x: run log test task", xTaskGetCurrentTaskHandle());
 
     int i = 0;
     while (1) {
@@ -662,7 +662,7 @@ static void esp_logtrace_task(void *p)
         ESP_APPTRACE_TEST_LOGE("Failed to flush printf buf (%d)!", ret);
     }
 
-    ESP_APPTRACE_TEST_LOGI("%p: finished", xTaskGetCurrentTaskHandle());
+    ESP_APPTRACE_TEST_LOGI("%x: finished", xTaskGetCurrentTaskHandle());
 
     xSemaphoreGive(arg->done);
     vTaskDelay(1);
@@ -681,13 +681,13 @@ TEST_CASE("Log trace test (2 tasks)", "[trace][ignore]")
     };
 
     xTaskCreatePinnedToCore(esp_logtrace_task, "logtrace0", 2048, &arg1, 3, &thnd, 0);
-    ESP_APPTRACE_TEST_LOGI("Created task %p", thnd);
+    ESP_APPTRACE_TEST_LOGI("Created task %x", thnd);
 #if CONFIG_FREERTOS_UNICORE == 0
     xTaskCreatePinnedToCore(esp_logtrace_task, "logtrace1", 2048, &arg2, 3, &thnd, 1);
 #else
     xTaskCreatePinnedToCore(esp_logtrace_task, "logtrace1", 2048, &arg2, 3, &thnd, 0);
 #endif
-    ESP_APPTRACE_TEST_LOGI("Created task %p", thnd);
+    ESP_APPTRACE_TEST_LOGI("Created task %x", thnd);
 
     xSemaphoreTake(arg1.done, portMAX_DELAY);
     vSemaphoreDelete(arg1.done);
@@ -760,7 +760,7 @@ static void esp_sysviewtrace_test_task(void *p)
             xSemaphoreGive(*arg->sync);
         }
     }
-    ESP_APPTRACE_TEST_LOGI("%p: finished", xTaskGetCurrentTaskHandle());
+    ESP_APPTRACE_TEST_LOGI("%x: finished", xTaskGetCurrentTaskHandle());
 
     xSemaphoreGive(arg->done);
     vTaskDelay(1);
@@ -809,13 +809,13 @@ TEST_CASE("SysView trace test 1", "[trace][ignore]")
     TEST_ESP_OK(gptimer_new_timer(&timer_config, &tim_arg2.gptimer));
 
     xTaskCreatePinnedToCore(esp_sysviewtrace_test_task, "svtrace0", 2048, &arg1, 3, &thnd, 0);
-    ESP_APPTRACE_TEST_LOGI("Created task %p", thnd);
+    ESP_APPTRACE_TEST_LOGI("Created task %x", thnd);
 #if CONFIG_FREERTOS_UNICORE == 0
     xTaskCreatePinnedToCore(esp_sysviewtrace_test_task, "svtrace1", 2048, &arg2, 5, &thnd, 1);
 #else
     xTaskCreatePinnedToCore(esp_sysviewtrace_test_task, "svtrace1", 2048, &arg2, 5, &thnd, 0);
 #endif
-    ESP_APPTRACE_TEST_LOGI("Created task %p", thnd);
+    ESP_APPTRACE_TEST_LOGI("Created task %x", thnd);
 
     xSemaphoreTake(arg1.done, portMAX_DELAY);
     vSemaphoreDelete(arg1.done);

components/app_update/esp_ota_ops.c

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: 2015-2024 Espressif Systems (Shanghai) CO LTD
+ * SPDX-FileCopyrightText: 2015-2023 Espressif Systems (Shanghai) CO LTD
  *
  * SPDX-License-Identifier: Apache-2.0
  */
@@ -17,7 +17,6 @@
 #include "esp_image_format.h"
 #include "esp_secure_boot.h"
 #include "esp_flash_encrypt.h"
-#include "spi_flash_mmap.h"
 #include "sdkconfig.h"
 
 #include "esp_ota_ops.h"
@@ -31,19 +30,31 @@
 #include "esp_attr.h"
 #include "esp_bootloader_desc.h"
 #include "esp_flash.h"
-#include "esp_flash_internal.h"
+
+#if CONFIG_IDF_TARGET_ESP32
+#include "esp32/rom/secure_boot.h"
+#elif CONFIG_IDF_TARGET_ESP32S2
+#include "esp32s2/rom/secure_boot.h"
+#elif CONFIG_IDF_TARGET_ESP32C3
+#include "esp32c3/rom/secure_boot.h"
+#elif CONFIG_IDF_TARGET_ESP32S3
+#include "esp32s3/rom/secure_boot.h"
+#elif CONFIG_IDF_TARGET_ESP32C2
+#include "esp32c2/rom/secure_boot.h"
+#elif CONFIG_IDF_TARGET_ESP32C6
+#include "esp32c6/rom/secure_boot.h"
+#elif CONFIG_IDF_TARGET_ESP32H2
+#include "esp32h2/rom/secure_boot.h"
+#elif CONFIG_IDF_TARGET_ESP32P4
+#include "esp32p4/rom/secure_boot.h"
+#endif
 
 #define SUB_TYPE_ID(i) (i & 0x0F)
-#define ALIGN_UP(num, align) (((num) + ((align) - 1)) & ~((align) - 1))
 
 /* Partial_data is word aligned so no reallocation is necessary for encrypted flash write */
 typedef struct ota_ops_entry_ {
     uint32_t handle;
-    struct {
-        const esp_partition_t *staging;      /*!< New image will be downloaded in this staging partition. */
-        const esp_partition_t *final;        /*!< Final destination partition which is intended to be updated. Its type/subtype shall be used for verification. */
-        bool finalize_with_copy;             /*!< Flag to copy the image from staging partition to the final partition at the end of OTA update */
-    } partition;
+    const esp_partition_t *part;
     bool need_erase;
     uint32_t wrote_size;
     uint8_t partial_bytes;
@@ -58,8 +69,6 @@ static uint32_t s_ota_ops_last_handle = 0;
 
 const static char *TAG = "esp_ota_ops";
 
-static ota_ops_entry_t *get_ota_ops_entry(esp_ota_handle_t handle);
-
 /* Return true if this is an OTA app partition */
 static bool is_ota_partition(const esp_partition_t *p)
 {
@@ -88,7 +97,7 @@ static const esp_partition_t *read_otadata(esp_ota_select_entry_t *two_otadata)
         return NULL;
     } else {
         memcpy(&two_otadata[0], result, sizeof(esp_ota_select_entry_t));
-        memcpy(&two_otadata[1], result + otadata_partition->erase_size, sizeof(esp_ota_select_entry_t));
+        memcpy(&two_otadata[1], result + SPI_FLASH_SEC_SIZE, sizeof(esp_ota_select_entry_t));
         esp_partition_munmap(ota_data_map);
     }
     return otadata_partition;
@@ -122,6 +131,8 @@ static esp_ota_img_states_t set_new_state_otadata(void)
 esp_err_t esp_ota_begin(const esp_partition_t *partition, size_t image_size, esp_ota_handle_t *out_handle)
 {
     ota_ops_entry_t *new_entry;
+    esp_err_t ret = ESP_OK;
+
     if ((partition == NULL) || (out_handle == NULL)) {
         return ESP_ERR_INVALID_ARG;
     }
@@ -131,89 +142,49 @@ esp_err_t esp_ota_begin(const esp_partition_t *partition, size_t image_size, esp
         return ESP_ERR_NOT_FOUND;
     }
 
-    if (partition->type == ESP_PARTITION_TYPE_APP) {
-        // The staging partition cannot be of type Factory, but the final partition can be.
-        if (!is_ota_partition(partition)) {
-           return ESP_ERR_INVALID_ARG;
-        }
-
-        const esp_partition_t* running_partition = esp_ota_get_running_partition();
-        if (partition == running_partition) {
-            return ESP_ERR_OTA_PARTITION_CONFLICT;
-        }
-
-#ifdef CONFIG_BOOTLOADER_APP_ROLLBACK_ENABLE
-        esp_ota_img_states_t ota_state_running_part;
-        if (esp_ota_get_state_partition(running_partition, &ota_state_running_part) == ESP_OK) {
-            if (ota_state_running_part == ESP_OTA_IMG_PENDING_VERIFY) {
-                ESP_LOGE(TAG, "Running app has not confirmed state (ESP_OTA_IMG_PENDING_VERIFY)");
-                return ESP_ERR_OTA_ROLLBACK_INVALID_STATE;
-            }
-        }
-#endif
+    if (!is_ota_partition(partition)) {
+        return ESP_ERR_INVALID_ARG;
     }
 
-    new_entry = (ota_ops_entry_t *) calloc(1, sizeof(ota_ops_entry_t));
+    const esp_partition_t* running_partition = esp_ota_get_running_partition();
+    if (partition == running_partition) {
+        return ESP_ERR_OTA_PARTITION_CONFLICT;
+    }
+
+#ifdef CONFIG_BOOTLOADER_APP_ROLLBACK_ENABLE
+    esp_ota_img_states_t ota_state_running_part;
+    if (esp_ota_get_state_partition(running_partition, &ota_state_running_part) == ESP_OK) {
+        if (ota_state_running_part == ESP_OTA_IMG_PENDING_VERIFY) {
+            ESP_LOGE(TAG, "Running app has not confirmed state (ESP_OTA_IMG_PENDING_VERIFY)");
+            return ESP_ERR_OTA_ROLLBACK_INVALID_STATE;
+        }
+    }
+#endif
+
+    if (image_size != OTA_WITH_SEQUENTIAL_WRITES) {
+        // If input image size is 0 or OTA_SIZE_UNKNOWN, erase entire partition
+        if ((image_size == 0) || (image_size == OTA_SIZE_UNKNOWN)) {
+            ret = esp_partition_erase_range(partition, 0, partition->size);
+        } else {
+            const int aligned_erase_size = (image_size + SPI_FLASH_SEC_SIZE - 1) & ~(SPI_FLASH_SEC_SIZE - 1);
+            ret = esp_partition_erase_range(partition, 0, aligned_erase_size);
+        }
+        if (ret != ESP_OK) {
+            return ret;
+        }
+    }
+
+    new_entry = (ota_ops_entry_t *) calloc(sizeof(ota_ops_entry_t), 1);
     if (new_entry == NULL) {
         return ESP_ERR_NO_MEM;
     }
 
     LIST_INSERT_HEAD(&s_ota_ops_entries_head, new_entry, entries);
 
-    new_entry->partition.staging = partition;
-    new_entry->partition.final = partition;
-    new_entry->partition.finalize_with_copy = false;
+    new_entry->part = partition;
     new_entry->handle = ++s_ota_ops_last_handle;
     new_entry->need_erase = (image_size == OTA_WITH_SEQUENTIAL_WRITES);
     *out_handle = new_entry->handle;
-
-    if (partition->type == ESP_PARTITION_TYPE_BOOTLOADER) {
-        esp_image_bootloader_offset_set(partition->address);
-    }
-    if (partition->type == ESP_PARTITION_TYPE_BOOTLOADER || partition->type == ESP_PARTITION_TYPE_PARTITION_TABLE) {
-        esp_flash_set_dangerous_write_protection(esp_flash_default_chip, false);
-    }
-
-    if (image_size != OTA_WITH_SEQUENTIAL_WRITES) {
-        // If input image size is 0 or OTA_SIZE_UNKNOWN, erase entire partition
-        size_t erase_size;
-        if ((image_size == 0) || (image_size == OTA_SIZE_UNKNOWN)) {
-            erase_size = partition->size;
-        } else {
-            erase_size = ALIGN_UP(image_size, partition->erase_size);
-        }
-        return esp_partition_erase_range(partition, 0, erase_size);
-    }
-    return ESP_OK;
-}
-
-esp_err_t esp_ota_set_final_partition(esp_ota_handle_t handle, const esp_partition_t *final, bool finalize_with_copy)
-{
-    ota_ops_entry_t *it = get_ota_ops_entry(handle);
-    if (final == NULL) {
-        return ESP_ERR_INVALID_ARG;
-    }
-    if (it == NULL) {
-        return ESP_ERR_NOT_FOUND;
-    }
-    if (it->wrote_size != 0) {
-        return ESP_ERR_INVALID_STATE;
-    }
-    if (it->partition.staging != final) {
-        const esp_partition_t* final_partition = esp_partition_verify(final);
-        if (final_partition == NULL) {
-            return ESP_ERR_NOT_FOUND;
-        }
-        ESP_LOGI(TAG,"Staging partition - <%s>. Final partition - <%s>.", it->partition.staging->label, final_partition->label);
-        it->partition.final = final_partition;
-        it->partition.finalize_with_copy = finalize_with_copy;
-        if (final_partition->type == ESP_PARTITION_TYPE_BOOTLOADER) {
-            esp_image_bootloader_offset_set(it->partition.staging->address);
-        }
-        if (final_partition->type == ESP_PARTITION_TYPE_BOOTLOADER || final_partition->type == ESP_PARTITION_TYPE_PARTITION_TABLE) {
-            esp_flash_set_dangerous_write_protection(esp_flash_default_chip, false);
-        }
-    }
     return ESP_OK;
 }
 
@@ -238,33 +209,23 @@ esp_err_t esp_ota_write(esp_ota_handle_t handle, const void *data, size_t size)
         if (it->handle == handle) {
             if (it->need_erase) {
                 // must erase the partition before writing to it
-                uint32_t first_sector = it->wrote_size / it->partition.staging->erase_size; // first affected sector
-                uint32_t last_sector = (it->wrote_size + size - 1) / it->partition.staging->erase_size; // last affected sector
+                uint32_t first_sector = it->wrote_size / SPI_FLASH_SEC_SIZE; // first affected sector
+                uint32_t last_sector = (it->wrote_size + size - 1) / SPI_FLASH_SEC_SIZE; // last affected sector
 
                 ret = ESP_OK;
-                if ((it->wrote_size % it->partition.staging->erase_size) == 0) {
-                    ret = esp_partition_erase_range(it->partition.staging, it->wrote_size, ((last_sector - first_sector) + 1) * it->partition.staging->erase_size);
+                if ((it->wrote_size % SPI_FLASH_SEC_SIZE) == 0) {
+                    ret = esp_partition_erase_range(it->part, it->wrote_size, ((last_sector - first_sector) + 1) * SPI_FLASH_SEC_SIZE);
                 } else if (first_sector != last_sector) {
-                    ret = esp_partition_erase_range(it->partition.staging, (first_sector + 1) * it->partition.staging->erase_size, (last_sector - first_sector) * it->partition.staging->erase_size);
+                    ret = esp_partition_erase_range(it->part, (first_sector + 1) * SPI_FLASH_SEC_SIZE, (last_sector - first_sector) * SPI_FLASH_SEC_SIZE);
                 }
                 if (ret != ESP_OK) {
                     return ret;
                 }
             }
 
-            if (it->wrote_size == 0 && it->partial_bytes == 0 && size > 0) {
-                if (it->partition.final->type == ESP_PARTITION_TYPE_APP || it->partition.final->type == ESP_PARTITION_TYPE_BOOTLOADER) {
-                    if (data_bytes[0] != ESP_IMAGE_HEADER_MAGIC) {
-                        ESP_LOGE(TAG, "OTA image has invalid magic byte (expected 0xE9, saw 0x%02x)", data_bytes[0]);
-                        return ESP_ERR_OTA_VALIDATE_FAILED;
-                    }
-
-                } else if (it->partition.final->type == ESP_PARTITION_TYPE_PARTITION_TABLE) {
-                    if (*(uint16_t*)data_bytes != (uint16_t)ESP_PARTITION_MAGIC) {
-                        ESP_LOGE(TAG, "Partition table image has invalid magic word (expected 0x50AA, saw 0x%04x)", *(uint16_t*)data_bytes);
-                        return ESP_ERR_OTA_VALIDATE_FAILED;
-                    }
-                }
+            if (it->wrote_size == 0 && it->partial_bytes == 0 && size > 0 && data_bytes[0] != ESP_IMAGE_HEADER_MAGIC) {
+                ESP_LOGE(TAG, "OTA image has invalid magic byte (expected 0xE9, saw 0x%02x)", data_bytes[0]);
+                return ESP_ERR_OTA_VALIDATE_FAILED;
             }
 
             if (esp_flash_encryption_enabled()) {
@@ -280,7 +241,7 @@ esp_err_t esp_ota_write(esp_ota_handle_t handle, const void *data, size_t size)
                         return ESP_OK; /* nothing to write yet, just filling buffer */
                     }
                     /* write 16 byte to partition */
-                    ret = esp_partition_write(it->partition.staging, it->wrote_size, it->partial_data, 16);
+                    ret = esp_partition_write(it->part, it->wrote_size, it->partial_data, 16);
                     if (ret != ESP_OK) {
                         return ret;
                     }
@@ -299,7 +260,7 @@ esp_err_t esp_ota_write(esp_ota_handle_t handle, const void *data, size_t size)
                 }
             }
 
-            ret = esp_partition_write(it->partition.staging, it->wrote_size, data_bytes, size);
+            ret = esp_partition_write(it->part, it->wrote_size, data_bytes, size);
             if(ret == ESP_OK){
                 it->wrote_size += size;
             }
@@ -336,7 +297,7 @@ esp_err_t esp_ota_write_with_offset(esp_ota_handle_t handle, const void *data, s
                 ESP_LOGE(TAG, "Size should be 16byte aligned for flash encryption case");
                 return ESP_ERR_INVALID_ARG;
             }
-            ret = esp_partition_write(it->partition.staging, offset, data_bytes, size);
+            ret = esp_partition_write(it->part, offset, data_bytes, size);
             if (ret == ESP_OK) {
                 it->wrote_size += size;
             }
@@ -372,34 +333,6 @@ esp_err_t esp_ota_abort(esp_ota_handle_t handle)
     return ESP_OK;
 }
 
-static esp_err_t ota_verify_partition(ota_ops_entry_t *ota_ops)
-{
-    esp_err_t ret = ESP_OK;
-    if (ota_ops->partition.final->type == ESP_PARTITION_TYPE_APP || ota_ops->partition.final->type == ESP_PARTITION_TYPE_BOOTLOADER) {
-        esp_image_metadata_t data;
-        const esp_partition_pos_t part_pos = {
-            .offset = ota_ops->partition.staging->address,
-            .size = ota_ops->partition.staging->size,
-        };
-        if (esp_image_verify(ESP_IMAGE_VERIFY, &part_pos, &data) != ESP_OK) {
-            return ESP_ERR_OTA_VALIDATE_FAILED;
-        }
-    } else if (ota_ops->partition.final->type == ESP_PARTITION_TYPE_PARTITION_TABLE) {
-        const esp_partition_info_t *partition_table = NULL;
-        esp_partition_mmap_handle_t partition_table_map;
-        ret = esp_partition_mmap(ota_ops->partition.staging, 0, ESP_PARTITION_TABLE_MAX_LEN, ESP_PARTITION_MMAP_DATA, (const void**)&partition_table, &partition_table_map);
-        if (ret == ESP_OK) {
-            int num_partitions;
-            if (esp_partition_table_verify(partition_table, true, &num_partitions) != ESP_OK) {
-                esp_partition_munmap(partition_table_map);
-                return ESP_ERR_OTA_VALIDATE_FAILED;
-            }
-            esp_partition_munmap(partition_table_map);
-        }
-    }
-    return ret;
-}
-
 esp_err_t esp_ota_end(esp_ota_handle_t handle)
 {
     ota_ops_entry_t *it = get_ota_ops_entry(handle);
@@ -419,7 +352,7 @@ esp_err_t esp_ota_end(esp_ota_handle_t handle)
 
     if (it->partial_bytes > 0) {
         /* Write out last 16 bytes, if necessary */
-        ret = esp_partition_write(it->partition.staging, it->wrote_size, it->partial_data, 16);
+        ret = esp_partition_write(it->part, it->wrote_size, it->partial_data, 16);
         if (ret != ESP_OK) {
             ret = ESP_ERR_INVALID_STATE;
             goto cleanup;
@@ -428,21 +361,18 @@ esp_err_t esp_ota_end(esp_ota_handle_t handle)
         it->partial_bytes = 0;
     }
 
-    ret = ota_verify_partition(it);
-    if (ret != ESP_OK) {
-        ESP_LOGE(TAG, "New image failed verification");
-    } else {
-        if (it->partition.finalize_with_copy) {
-            ESP_LOGI(TAG, "Copy from <%s> staging partition to <%s>...", it->partition.staging->label, it->partition.final->label);
-            ret = esp_partition_copy(it->partition.final, 0, it->partition.staging, 0, it->partition.final->size);
-        }
+    esp_image_metadata_t data;
+    const esp_partition_pos_t part_pos = {
+      .offset = it->part->address,
+      .size = it->part->size,
+    };
+
+    if (esp_image_verify(ESP_IMAGE_VERIFY, &part_pos, &data) != ESP_OK) {
+        ret = ESP_ERR_OTA_VALIDATE_FAILED;
+        goto cleanup;
     }
 
  cleanup:
-    if (it->partition.final->type == ESP_PARTITION_TYPE_BOOTLOADER) {
-        // In esp_ota_begin, bootloader offset was updated, here we return it to default.
-        esp_image_bootloader_offset_set(ESP_PRIMARY_BOOTLOADER_OFFSET);
-    }
     LIST_REMOVE(it, entries);
     free(it);
     return ret;
@@ -456,11 +386,11 @@ static esp_err_t rewrite_ota_seq(esp_ota_select_entry_t *two_otadata, uint32_t s
 
     two_otadata[sec_id].ota_seq = seq;
     two_otadata[sec_id].crc = bootloader_common_ota_select_crc(&two_otadata[sec_id]);
-    esp_err_t ret = esp_partition_erase_range(ota_data_partition, sec_id * ota_data_partition->erase_size, ota_data_partition->erase_size);
+    esp_err_t ret = esp_partition_erase_range(ota_data_partition, sec_id * SPI_FLASH_SEC_SIZE, SPI_FLASH_SEC_SIZE);
     if (ret != ESP_OK) {
         return ret;
     } else {
-        return esp_partition_write(ota_data_partition, ota_data_partition->erase_size * sec_id, &two_otadata[sec_id], sizeof(esp_ota_select_entry_t));
+        return esp_partition_write(ota_data_partition, SPI_FLASH_SEC_SIZE * sec_id, &two_otadata[sec_id], sizeof(esp_ota_select_entry_t));
     }
 }
 
@@ -993,7 +923,7 @@ esp_err_t esp_ota_erase_last_boot_app_partition(void)
     }
 
     int sec_id = inactive_otadata;
-    err = esp_partition_erase_range(ota_data_partition, sec_id * ota_data_partition->erase_size, ota_data_partition->erase_size);
+    err = esp_partition_erase_range(ota_data_partition, sec_id * SPI_FLASH_SEC_SIZE, SPI_FLASH_SEC_SIZE);
     if (err != ESP_OK) {
         return err;
     }

components/app_update/include/esp_ota_ops.h

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: 2015-2024 Espressif Systems (Shanghai) CO LTD
+ * SPDX-FileCopyrightText: 2015-2023 Espressif Systems (Shanghai) CO LTD
  *
  * SPDX-License-Identifier: Apache-2.0
  */
@@ -82,11 +82,7 @@ int esp_ota_get_app_elf_sha256(char* dst, size_t size) __attribute__((deprecated
  * it will lead to the ESP_ERR_OTA_ROLLBACK_INVALID_STATE error. Confirm the running app before to run download a new app,
  * use esp_ota_mark_app_valid_cancel_rollback() function for it (this should be done as early as possible when you first download a new application).
  *
- * Note: Rollback is applicable only for app type partitions.
- *
- * @param partition  Pointer to info for partition which will receive the OTA update. Required.
- *                   This is considered as the staging partition (where OTA is downloaded), be default this also considered as the final partition which supposed to be updated.
- *                   The final partition can be overwritten using esp_ota_set_final_partition() after calling esp_ota_begin() to relocate contents to the final destination partition.
+ * @param partition Pointer to info for partition which will receive the OTA update. Required.
  * @param image_size Size of new OTA app image. Partition will be erased in order to receive this size of image. If 0 or OTA_SIZE_UNKNOWN, the entire partition is erased.
  * @param out_handle On success, returns a handle which should be used for subsequent esp_ota_write() and esp_ota_end() calls.
 
@@ -103,31 +99,6 @@ int esp_ota_get_app_elf_sha256(char* dst, size_t size) __attribute__((deprecated
  */
 esp_err_t esp_ota_begin(const esp_partition_t* partition, size_t image_size, esp_ota_handle_t* out_handle);
 
-/**
- * @brief Set the final destination partition for OTA update
- *
- * This function configures the specified final partition as the destination for the OTA update.
- * It also allows setting a flag to indicate if the image should be copied from the staging
- * partition to the final partition after the OTA update completes. Otherwise, copying will need
- * to be handled by custom code using esp_partition_copy().
- *
- * @note This can be called after esp_ota_begin() and before the OTA update has started (before esp_ota_write()).
- *
- * @param handle OTA update handle obtained from esp_ota_begin().
- * @param final Pointer to the final destination partition where the new image will be verified and potentially finalized.
- *              This partition must not be NULL.
- * @param finalize_with_copy Boolean flag indicating if the downloaded image should be copied
- *                           from the staging partition to the final partition upon completion.
- *                           Set to False if you intend to perform the final copy process manually later.
- *
- * @return
- *      - ESP_OK: final destination partition set successfully.
- *      - ESP_ERR_INVALID_STATE: Once the OTA update has started, changing the final destination partition is prohibited.
- *      - ESP_ERR_INVALID_ARG: Invalid arguments were passed (e.g., final partition is NULL).
- *      - ESP_ERR_NOT_FOUND: OTA handle not found or final partition verification failed.
- */
-esp_err_t esp_ota_set_final_partition(esp_ota_handle_t handle, const esp_partition_t *final, bool finalize_with_copy);
-
 /**
  * @brief   Write OTA update data to partition
  *
@@ -142,10 +113,9 @@ esp_err_t esp_ota_set_final_partition(esp_ota_handle_t handle, const esp_partiti
  * @return
  *    - ESP_OK: Data was written to flash successfully, or size = 0
  *    - ESP_ERR_INVALID_ARG: handle is invalid.
- *    - ESP_ERR_OTA_VALIDATE_FAILED: First byte of image contains invalid image magic byte.
+ *    - ESP_ERR_OTA_VALIDATE_FAILED: First byte of image contains invalid app image magic byte.
  *    - ESP_ERR_FLASH_OP_TIMEOUT or ESP_ERR_FLASH_OP_FAIL: Flash write failed.
- *    - ESP_ERR_INVALID_SIZE: if write would go out of bounds of the partition
- *    - or one of error codes from lower-level flash driver.
+ *    - ESP_ERR_OTA_SELECT_INFO_INVALID: OTA data partition has invalid contents
  */
 esp_err_t esp_ota_write(esp_ota_handle_t handle, const void* data, size_t size);
 
@@ -166,7 +136,9 @@ esp_err_t esp_ota_write(esp_ota_handle_t handle, const void* data, size_t size);
  * @return
  *    - ESP_OK: Data was written to flash successfully.
  *    - ESP_ERR_INVALID_ARG: handle is invalid.
+ *    - ESP_ERR_OTA_VALIDATE_FAILED: First byte of image contains invalid app image magic byte.
  *    - ESP_ERR_FLASH_OP_TIMEOUT or ESP_ERR_FLASH_OP_FAIL: Flash write failed.
+ *    - ESP_ERR_OTA_SELECT_INFO_INVALID: OTA data partition has invalid contents
  */
 esp_err_t esp_ota_write_with_offset(esp_ota_handle_t handle, const void *data, size_t size, uint32_t offset);
 
@@ -176,11 +148,6 @@ esp_err_t esp_ota_write_with_offset(esp_ota_handle_t handle, const void *data, s
  * @param handle  Handle obtained from esp_ota_begin().
  *
  * @note After calling esp_ota_end(), the handle is no longer valid and any memory associated with it is freed (regardless of result).
- * @note If either the final or staging partitions were for the bootloader, then at the end of this function,
- *       the bootloader is reset to its default offset: esp_image_bootloader_offset_set(ESP_PRIMARY_BOOTLOADER_OFFSET)
- *
- * If the finalize_with_copy option is set, the staging partition will be copied to the final partition at the end of this function.
- * Otherwise, copying will need to be handled by custom code using esp_partition_copy().
  *
  * @return
  *    - ESP_OK: Newly written OTA app image is valid.
@@ -289,7 +256,7 @@ esp_err_t esp_ota_get_partition_description(const esp_partition_t *partition, es
  * @brief Returns the description structure of the bootloader.
  *
  * @param[in] bootloader_partition Pointer to bootloader partition.
- *                                 If NULL, then the PRIMARY bootloader is used (the default location).
+ *                                 If NULL, then the current bootloader is used (the default location).
  *                                 offset = CONFIG_BOOTLOADER_OFFSET_IN_FLASH,
  *                                 size = CONFIG_PARTITION_TABLE_OFFSET - CONFIG_BOOTLOADER_OFFSET_IN_FLASH,
  * @param[out] desc     Structure of info about bootloader.

components/app_update/test_apps/.build-test-rules.yml

@@ -0,0 +1,7 @@
+# Documentation: .gitlab/ci/README.md#manifest-file-to-control-the-buildtest-apps
+
+components/app_update/test_apps:
+  disable:
+    - if: IDF_TARGET in ["esp32c6", "esp32h2", "esp32p4"]
+      temporary: true
+      reason: target esp32c6, esp32h2, esp32p4 is not supported yet # TODO: IDF-8068

components/app_update/test_apps/README.md

@@ -0,0 +1,2 @@
+| Supported Targets | ESP32 | ESP32-C2 | ESP32-C3 | ESP32-S2 | ESP32-S3 |
+| ----------------- | ----- | -------- | -------- | -------- | -------- |

components/app_update/test_apps/main/CMakeLists.txt

@@ -0,0 +1,4 @@
+idf_component_register(SRC_DIRS "."
+                       PRIV_INCLUDE_DIRS "."
+                       PRIV_REQUIRES cmock test_utils app_update bootloader_support nvs_flash driver spi_flash
+                      WHOLE_ARCHIVE)

components/app_update/test_apps/main/test_ota_ops.c

@@ -6,7 +6,6 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
-#include "esp_log.h"
 #include <freertos/FreeRTOS.h>
 #include <freertos/task.h>
 #include <freertos/semphr.h>
@@ -114,11 +113,3 @@ TEST_CASE("esp_ota_get_partition_description", "[ota]")
     };
     TEST_ESP_ERR(ESP_ERR_NOT_FOUND, bootloader_common_get_partition_description(&not_app_pos, &app_desc1));
 }
-
-TEST_CASE("esp_ota_get_running_partition points to correct address", "[spi_flash]")
-{
-    const esp_partition_t *factory = esp_partition_find_first(ESP_PARTITION_TYPE_APP, ESP_PARTITION_SUBTYPE_ANY, "factory");
-    const esp_partition_t* part = esp_ota_get_running_partition();
-    ESP_LOGI("running bin", "0x%p", (void*)part->address);
-    TEST_ASSERT_EQUAL_HEX32(factory->address, part->address);
-}

components/app_update/test_apps/main/test_switch_ota.c

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: 2021-2024 Espressif Systems (Shanghai) CO LTD
+ * SPDX-FileCopyrightText: 2021-2023 Espressif Systems (Shanghai) CO LTD
  *
  * SPDX-License-Identifier: Apache-2.0
  */
@@ -43,12 +43,7 @@ static const char *TAG = "ota_test";
 static void set_boot_count_in_nvs(uint8_t boot_count)
 {
     nvs_handle_t boot_count_handle;
-    esp_err_t err = nvs_open(BOOT_COUNT_NAMESPACE, NVS_READWRITE, &boot_count_handle);
-    if (err != ESP_OK) {
-        TEST_ESP_OK(nvs_flash_erase());
-        TEST_ESP_OK(nvs_flash_init());
-        TEST_ESP_OK(nvs_open(BOOT_COUNT_NAMESPACE, NVS_READWRITE, &boot_count_handle));
-    }
+    TEST_ESP_OK(nvs_open(BOOT_COUNT_NAMESPACE, NVS_READWRITE, &boot_count_handle));
     TEST_ESP_OK(nvs_set_u8(boot_count_handle, "boot_count", boot_count));
     TEST_ESP_OK(nvs_commit(boot_count_handle));
     nvs_close(boot_count_handle);
@@ -106,6 +101,24 @@ static void copy_app_partition_with_offset(esp_ota_handle_t update_handle, const
     ESP_LOGI(TAG, "finish the copy process");
 }
 
+#if defined(CONFIG_BOOTLOADER_FACTORY_RESET) || defined(CONFIG_BOOTLOADER_APP_TEST)
+/* @brief Copies partition from source partition to destination partition.
+ *
+ * Partitions can be of any types and subtypes.
+ * @param[in] dst_partition - Destination partition
+ * @param[in] src_partition - Source partition
+ */
+static void copy_partition(const esp_partition_t *dst_partition, const esp_partition_t *src_partition)
+{
+    const void *partition_bin = NULL;
+    esp_partition_mmap_handle_t data_map;
+    TEST_ESP_OK(esp_partition_mmap(src_partition, 0, src_partition->size, ESP_PARTITION_MMAP_DATA, &partition_bin, &data_map));
+    TEST_ESP_OK(esp_partition_erase_range(dst_partition, 0, dst_partition->size));
+    TEST_ESP_OK(esp_partition_write(dst_partition, 0, (const void *)partition_bin, dst_partition->size));
+    esp_partition_munmap(data_map);
+}
+#endif
+
 /* @brief Get the next partition of OTA for the update.
  *
  * @return The next partition of OTA(OTA0-15).
@@ -164,7 +177,7 @@ static void erase_ota_data(void)
 {
     const esp_partition_t *data_partition = esp_partition_find_first(ESP_PARTITION_TYPE_DATA, ESP_PARTITION_SUBTYPE_DATA_OTA, NULL);
     TEST_ASSERT_NOT_EQUAL(NULL, data_partition);
-    TEST_ESP_OK(esp_partition_erase_range(data_partition, 0, 2 * data_partition->erase_size));
+    TEST_ESP_OK(esp_partition_erase_range(data_partition, 0, 2 * SPI_FLASH_SEC_SIZE));
 }
 
 /* @brief Reboots ESP using mode deep sleep. This mode guaranty that RTC_DATA_ATTR variables is not reset.
@@ -238,7 +251,7 @@ static void get_ota_data(const esp_partition_t *otadata_partition, esp_ota_selec
         TEST_ASSERT_NOT_EQUAL(NULL, ota_select_map);
 
         memcpy(ota_data_0, ota_select_map, sizeof(esp_ota_select_entry_t));
-        memcpy(ota_data_1, (uint8_t *)ota_select_map + otadata_partition->erase_size, sizeof(esp_ota_select_entry_t));
+        memcpy(ota_data_1, (uint8_t *)ota_select_map + SPI_FLASH_SEC_SIZE, sizeof(esp_ota_select_entry_t));
         bootloader_munmap(ota_select_map);
     }
 }
@@ -251,7 +264,7 @@ static void get_ota_data(const esp_partition_t *otadata_partition, esp_ota_selec
  */
 static void write_ota_data(const esp_partition_t *otadata_partition, esp_ota_select_entry_t *ota_data, int sec_id)
 {
-    esp_partition_write(otadata_partition, otadata_partition->erase_size * sec_id, &ota_data[sec_id], sizeof(esp_ota_select_entry_t));
+    esp_partition_write(otadata_partition, SPI_FLASH_SEC_SIZE * sec_id, &ota_data[sec_id], sizeof(esp_ota_select_entry_t));
 }
 
 /* @brief Makes a corrupt of ota_data.
@@ -517,7 +530,7 @@ static void test_flow5(void)
             ESP_LOGI(TAG, "Factory");
             TEST_ASSERT_EQUAL(ESP_PARTITION_SUBTYPE_APP_FACTORY, cur_app->subtype);
             set_output_pin(CONFIG_BOOTLOADER_NUM_PIN_APP_TEST);
-            esp_partition_copy(esp_partition_find_first(ESP_PARTITION_TYPE_APP, ESP_PARTITION_SUBTYPE_APP_TEST, NULL), 0, cur_app, 0, cur_app->size);
+            copy_partition(esp_partition_find_first(ESP_PARTITION_TYPE_APP, ESP_PARTITION_SUBTYPE_APP_TEST, NULL), cur_app);
             esp_restart();
             break;
         case 3:
@@ -828,7 +841,8 @@ static void test_flow6(void)
 // 3 Stage: run OTA0    -> check it -> erase OTA_DATA for next tests    -> PASS
 TEST_CASE_MULTIPLE_STAGES("Switching between factory, OTA0 using esp_ota_write_with_offset", "[app_update][timeout=90][reset=DEEPSLEEP_RESET, DEEPSLEEP_RESET]", start_test, test_flow6, test_flow6);
 
-TEST_CASE("Test bootloader_common_get_sha256_of_partition returns ESP_ERR_IMAGE_INVALID when image is invalid", "[partitions]")
+//IDF-5145
+TEST_CASE("Test bootloader_common_get_sha256_of_partition returns ESP_ERR_IMAGE_INVALID when image is ivalid", "[partitions]")
 {
     const esp_partition_t *cur_app = esp_ota_get_running_partition();
     ESP_LOGI(TAG, "copy current app to next part");

components/app_update/test_apps/partition_table_unit_test_two_ota.csv

@@ -7,3 +7,5 @@ factory,    0,    0,       ,        0xB0000
 ota_0,      0,    ota_0,   ,        0xB0000
 ota_1,      0,    ota_1,   ,        0xB0000
 test,       0,    test,    ,        0xB0000
+# flash_test partition used for SPI flash tests, WL FAT tests, and SPIFFS tests
+flash_test, data, fat,     ,        528K

components/app_update/test_apps/partition_table_unit_test_two_ota_2m.csv

@@ -7,3 +7,5 @@ factory,    0,    0,       ,        0x70000
 ota_0,      0,    ota_0,   ,        0x70000
 ota_1,      0,    ota_1,   ,        0x70000
 test,       0,    test,    ,        0x70000
+# flash_test partition used for SPI flash tests, WL FAT tests, and SPIFFS tests
+flash_test, data, fat,     ,        128K