Merge pull request #372 from espressif-abhikroy/components/console_simple_init

feat(common): Added simple component for console initialization

.flake8

@@ -137,4 +137,4 @@ show_source = True
 statistics = True
 
 exclude =
-        components/asio/docs/conf_common.py
+        docs/asio/conf_common.py

.github/ISSUE_TEMPLATE/bug-report.yml

@@ -8,7 +8,7 @@ body:
       label: Answers checklist.
       description: Before submitting a new issue, please follow the checklist and try to find the answer.
       options:
-        - label: I have read the documentation for [esp-protocols components](https://espressif.github.io/esp-protocols/) and the issue is not addressed there.
+        - label: I have read the documentation for esp-protocols [components](https://github.com/espressif/esp-protocols#readme) and the issue is not addressed there.
           required: true
         - label: I have updated my esp-protocols branch (master or release) to the latest version and checked that the issue is present there.
           required: true

.github/ISSUE_TEMPLATE/config.yml

@@ -1,7 +1,7 @@
 blank_issues_enabled: false
 contact_links:
-  - name: esp-protocols documentation
+  - name: esp-protocols documentation links
-    url: https://espressif.github.io/esp-protocols/
+    url: https://github.com/espressif/esp-protocols#readme
     about: Documenation for esp-protocols components.
   - name: ESP-IDF Programming Guide
     url: https://docs.espressif.com/projects/esp-idf/en/latest/

.github/ISSUE_TEMPLATE/other-issue.yml

@@ -7,7 +7,7 @@ body:
       label: Answers checklist.
       description: Before submitting a new issue, please follow the checklist and try to find the answer.
       options:
-        - label: I have read the documentation for [esp-protocols components](https://espressif.github.io/esp-protocols/) and the issue is not addressed there.
+        - label: I have read the documentation for esp-protocols [components](https://github.com/espressif/esp-protocols#readme) and the issue is not addressed there.
           required: true
         - label: I have updated my esp-protocols branch (master or release) to the latest version and checked that the issue is present there.
           required: true

.github/workflows/asio__build-target-test.yml

@@ -13,7 +13,7 @@ jobs:
     name: Build
     strategy:
       matrix:
-        idf_ver: ["latest", "release-v5.0"]
+        idf_ver: ["latest", "release-v5.0", "release-v5.1"]
         idf_target: ["esp32", "esp32s2"]
         example: ["asio_chat", "async_request", "socks4", "ssl_client_server", "tcp_echo_server", "udp_echo_server"]
     runs-on: ubuntu-20.04

.github/workflows/console_simple_init__build.yml

@@ -0,0 +1,32 @@
+name: "console_simple_init: build-tests"
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    types: [opened, synchronize, reopened, labeled]
+
+jobs:
+  build_console_simple_init:
+    if: contains(github.event.pull_request.labels.*.name, 'console') || github.event_name == 'push'
+    name: Build
+    strategy:
+      matrix:
+        idf_ver: ["latest", "release-v5.0"]
+        idf_target: ["esp32"]
+        test: [ { app: example, path: "components/console_simple_init/examples" }]
+    runs-on: ubuntu-20.04
+    container: espressif/idf:${{ matrix.idf_ver }}
+    steps:
+      - name: Checkout esp-protocols
+        uses: actions/checkout@v3
+        with:
+          submodules: recursive
+      - name: Build ${{ matrix.test.app }} with IDF-${{ matrix.idf_ver }} for ${{ matrix.idf_target }}
+        shell: bash
+        working-directory: ${{matrix.test.path}}
+        run: |
+          ${IDF_PATH}/install.sh --enable-pytest
+          . ${IDF_PATH}/export.sh
+          python $IDF_PATH/tools/ci/ci_build_apps.py . --target ${{ matrix.idf_target }} -vv --preserve-all --pytest-app

.github/workflows/examples_build-host-test.yml

@@ -0,0 +1,48 @@
+name: "examples: build/host-tests"
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    types: [opened, synchronize, reopened, labeled]
+
+jobs:
+  build_all_examples:
+    if: contains(github.event.pull_request.labels.*.name, 'examples') || github.event_name == 'push'
+    name: Build examples
+    strategy:
+      matrix:
+        idf_ver: ["latest", "release-v5.1"]
+    runs-on: ubuntu-20.04
+    container: espressif/idf:${{ matrix.idf_ver }}
+    steps:
+      - name: Checkout esp-protocols
+        uses: actions/checkout@v3
+      - name: Build with IDF-${{ matrix.idf_ver }}
+        shell: bash
+        run: |
+          . ${IDF_PATH}/export.sh
+          python -m pip install idf-build-apps
+          # Build default configs for all targets
+          python ./ci/build_apps.py examples -m examples/.build-test-rules.yml -d -c
+
+  build_and_run_on_host:
+    if: contains(github.event.pull_request.labels.*.name, 'examples') || github.event_name == 'push'
+    name: Build and run examples on linux
+    strategy:
+      matrix:
+        idf_ver: ["latest"]
+    runs-on: ubuntu-20.04
+    container: espressif/idf:${{ matrix.idf_ver }}
+    steps:
+      - name: Checkout esp-protocols
+        uses: actions/checkout@v3
+      - name: Build with IDF-${{ matrix.idf_ver }}
+        shell: bash
+        run: |
+          . ${IDF_PATH}/export.sh
+          python -m pip install idf-build-apps
+          python ./ci/build_apps.py examples/mqtt -l -t linux
+          timeout 5 ./examples/mqtt/build_linux_default/esp_mqtt_demo.elf | tee test.log  || true
+          grep 'MQTT_EVENT_DATA' test.log

.github/workflows/mdns__build-target-test.yml

@@ -13,122 +13,33 @@ jobs:
     name: Build
     strategy:
       matrix:
-        idf_ver: ["latest"]
+        idf_ver: ["latest", "release-v5.0"]
-        idf_target: ["esp32", "esp32s2", "esp32c3"]
+        test: [ { app: example, path: "examples" }, { app: unit_test, path: "tests/unit_test" }, { app: test_app, path: "tests/test_apps" } ]
-        test: [ { app: example, path: "components/mdns/examples" }, { app: unit_test, path: "components/mdns/tests/unit_test" } ]
     runs-on: ubuntu-20.04
     container: espressif/idf:${{ matrix.idf_ver }}
     steps:
       - name: Checkout esp-protocols
         uses: actions/checkout@v3
-        with:
+      - name: Build ${{ matrix.test.app }} with IDF-${{ matrix.idf_ver }}
-          submodules: recursive
-      - name: Build ${{ matrix.test.app }} with IDF-${{ matrix.idf_ver }} for ${{ matrix.idf_target }}
         shell: bash
-        working-directory: ${{ matrix.test.path }}
         run: |
-          ${IDF_PATH}/install.sh --enable-pytest
           . ${IDF_PATH}/export.sh
-          python $IDF_PATH/tools/ci/ci_build_apps.py . --target ${{ matrix.idf_target }} -vv --preserve-all --pytest-app
+          python -m pip install idf-build-apps
-          for dir in `ls -d build_*`; do
+          # Build default configs for all targets
+          python ./ci/build_apps.py components/mdns/${{ matrix.test.path }} -r default -m components/mdns/.build-test-rules.yml -d
+          # Build specific configs for test targets
+          python ./ci/build_apps.py components/mdns/${{ matrix.test.path }}
+          cd components/mdns/${{ matrix.test.path }}
+          for dir in `ls -d build_esp32_*`; do
           $GITHUB_WORKSPACE/ci/clean_build_artifacts.sh `pwd`/$dir
           zip -qur artifacts.zip $dir
           done
       - uses: actions/upload-artifact@v3
-        if: ${{ matrix.idf_target }} == "esp32"
         with:
-          name: mdns_bin_${{ matrix.idf_target }}_${{ matrix.idf_ver }}_${{ matrix.test.app }}
+          name: mdns_bin_esp32_${{ matrix.idf_ver }}_${{ matrix.test.app }}
-          path: ${{ matrix.test.path }}/artifacts.zip
+          path: components/mdns/${{ matrix.test.path }}/artifacts.zip
           if-no-files-found: error
 
-  build_mdns_app:
-    if: contains(github.event.pull_request.labels.*.name, 'mdns') || github.event_name == 'push'
-    name: Build Test Apps
-    strategy:
-      matrix:
-        idf_ver: ["latest"]
-        idf_target: ["esp32", "esp32s2", "esp32c3"]
-    runs-on: ubuntu-20.04
-    container: espressif/idf:${{ matrix.idf_ver }}
-    steps:
-      - name: Checkout esp-protocols
-        uses: actions/checkout@v3
-        with:
-          submodules: recursive
-      - name: Setup Build Environment
-        working-directory: components/mdns/tests/test_apps
-        run: |
-          . ${IDF_PATH}/export.sh
-          pip install -r $GITHUB_WORKSPACE/ci/requirements.txt
-      - name: Build ${{ matrix.example }} with IDF-${{ matrix.idf_ver }} for ${{ matrix.idf_target }}
-        env:
-          IDF_TARGET: ${{ matrix.idf_target }}
-        shell: bash
-        working-directory: components/mdns/tests/test_apps
-        run: |
-          . ${IDF_PATH}/export.sh
-          python $IDF_PATH/tools/ci/ci_build_apps.py . --target ${{ matrix.idf_target }} -vv --pytest-apps
-      - name: Merge binaries with IDF-${{ matrix.idf_ver }}
-        working-directory: components/mdns/tests/test_apps/build_${{ matrix.idf_target }}_default
-        env:
-          IDF_TARGET: ${{ matrix.idf_target }}
-        shell: bash
-        run: |
-          . ${IDF_PATH}/export.sh
-          esptool.py --chip ${{ matrix.idf_target }} merge_bin --fill-flash-size 4MB -o flash_image.bin @flash_args
-      - uses: actions/upload-artifact@v2
-        with:
-          name: examples_app_bin_${{ matrix.idf_target }}_${{ matrix.idf_ver }}
-          path: |
-            components/mdns/tests/test_apps/build_${{ matrix.idf_target }}_default/bootloader/bootloader.bin
-            components/mdns/tests/test_apps/build_${{ matrix.idf_target }}_default/partition_table/partition-table.bin
-            components/mdns/tests/test_apps/build_${{ matrix.idf_target }}_default/*.bin
-            components/mdns/tests/test_apps/build_${{ matrix.idf_target }}_default/*.elf
-            components/mdns/tests/test_apps/build_${{ matrix.idf_target }}_default/flasher_args.json
-            components/mdns/tests/test_apps/build_${{ matrix.idf_target }}_default/config/sdkconfig.h
-            components/mdns/tests/test_apps/build_${{ matrix.idf_target }}_default/config/sdkconfig.json
-          if-no-files-found: error
-
-  target_test_apps_mdns:
-    # Skip running on forks since it won't have access to secrets
-    if: |
-      github.repository == 'espressif/esp-protocols' &&
-      ( contains(github.event.pull_request.labels.*.name, 'mdns') || github.event_name == 'push' )
-    name: Target Test Apps
-    strategy:
-      matrix:
-        idf_ver: ["latest"]
-        idf_target: ["esp32"]
-    needs: build_mdns_app
-    runs-on:
-      - self-hosted
-      - ESP32-ETHERNET-KIT
-    steps:
-      - name: Clear repository
-        run: sudo rm -fr $GITHUB_WORKSPACE && mkdir $GITHUB_WORKSPACE
-      - uses: actions/checkout@v3
-      - uses: actions/download-artifact@v2
-        with:
-          name: examples_app_bin_${{ matrix.idf_target }}_${{ matrix.idf_ver }}
-          path: components/mdns/tests/test_apps/build
-      - name: Install Python packages
-        env:
-          PIP_EXTRA_INDEX_URL: "https://www.piwheels.org/simple"
-        run: |
-            sudo apt-get install -y dnsutils
-      - name: Download Test apps to target
-        run: |
-            python -m esptool --chip ${{ matrix.idf_target }} write_flash 0x0 components/mdns/tests/test_apps/build/flash_image.bin
-      - name: Run Example Test on target
-        working-directory: components/mdns/tests/test_apps
-        run: |
-            python -m pytest --log-cli-level DEBUG --junit-xml=./examples_results_${{ matrix.idf_target }}_${{ matrix.idf_ver }}.xml --target=${{matrix.idf_target}}
-      - uses: actions/upload-artifact@v2
-        if: always()
-        with:
-          name: test_apps_results_${{ matrix.idf_target }}_${{ matrix.idf_ver }}
-          path: components/mdns/tests/test_apps/*.xml
-
   target_tests_mdns:
     # Skip running on forks since it won't have access to secrets
     if: |
@@ -139,7 +50,7 @@ jobs:
       matrix:
         idf_ver: ["latest"]
         idf_target: ["esp32"]
-        test: [ { app: example, path: "components/mdns/examples" }, { app: unit_test, path: "components/mdns/tests/unit_test" } ]
+        test: [ { app: example, path: "examples" }, { app: unit_test, path: "tests/unit_test" }, { app: test_app, path: "tests/test_apps" } ]
     needs: build_mdns
     runs-on:
       - self-hosted
@@ -151,14 +62,14 @@ jobs:
       - uses: actions/download-artifact@v3
         with:
           name: mdns_bin_${{ matrix.idf_target }}_${{ matrix.idf_ver }}_${{ matrix.test.app }}
-          path: ${{ matrix.test.path }}/ci/
+          path: components/mdns/${{ matrix.test.path }}/ci/
       - name: Install Python packages
         env:
           PIP_EXTRA_INDEX_URL: "https://www.piwheels.org/simple"
         run: |
             sudo apt-get install -y dnsutils
       - name: Run ${{ matrix.test.app }} application on ${{ matrix.idf_target }}
-        working-directory: ${{ matrix.test.path }}
+        working-directory: components/mdns/${{ matrix.test.path }}
         run: |
             unzip ci/artifacts.zip -d ci
             for dir in `ls -d ci/build_*`; do
@@ -170,4 +81,4 @@ jobs:
         if: always()
         with:
           name: results_${{ matrix.test.app }}_${{ matrix.idf_target }}_${{ matrix.idf_ver }}.xml
-          path: ${{ matrix.test.path }}/*.xml
+          path: components/mdns/${{ matrix.test.path }}/*.xml

.github/workflows/mdns__host-tests.yml

@@ -12,7 +12,7 @@ jobs:
     if: contains(github.event.pull_request.labels.*.name, 'mdns') || github.event_name == 'push'
     name: Host test
     runs-on: ubuntu-20.04
-    container: espressif/idf:latest
+    container: espressif/idf:release-v5.1
 
     steps:
       - name: Checkout esp-protocols

.github/workflows/modem__build-host-tests.yml

@@ -22,6 +22,8 @@ jobs:
             example: modem_tcp_client
           - idf_ver: "release-v4.3"
             example: modem_tcp_client
+          - idf_ver: "release-v4.4"
+            example: modem_tcp_client
         include:
           - idf_ver: "release-v4.2"
             skip_config: usb
@@ -48,7 +50,7 @@ jobs:
           . ${IDF_PATH}/export.sh
           python -m pip install idf-build-apps
           cd $GITHUB_WORKSPACE/protocols
-          python ./ci/build_apps.py components/esp_modem/examples/${{ matrix.example }}
+          python ./ci/build_apps.py components/esp_modem/examples/${{ matrix.example }} -m components/esp_modem/examples/.build-test-rules.yml
 
   host_test_esp_modem:
     if: contains(github.event.pull_request.labels.*.name, 'modem') || github.event_name == 'push'
@@ -62,7 +64,7 @@ jobs:
       COMP_DIR: esp-protocols/components/esp_modem
     steps:
       - name: Checkout esp-protocols
-        uses: actions/checkout@master
+        uses: actions/checkout@v3
         with:
           path: esp-protocols
 
@@ -91,6 +93,8 @@ jobs:
     if: contains(github.event.pull_request.labels.*.name, 'modem') || github.event_name == 'push'
     name: Run gcovr on esp modem host test
     runs-on: ubuntu-22.04
+    permissions:
+      contents: write
     container: espressif/idf:release-v4.3
     env:
       lwip: lwip-2.1.2
@@ -102,11 +106,11 @@ jobs:
         uses: actions/checkout@v3
         with:
           path: esp-protocols
+          repository: ''
           persist-credentials: false
       - name: Build and Test
         shell: bash
         run: |
-          apt-get update
           apt-get update && apt-get install -y gcc-8 g++-8 python3-pip
           apt-get install -y rsync
           update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-8 800 --slave /usr/bin/g++ g++ /usr/bin/g++-8
@@ -126,10 +130,9 @@ jobs:
           cd $GITHUB_WORKSPACE/${{ env.COMP_DIR }}
           gcov-8 `find . -name "esp_modem*gcda" -printf '%h\n' | head -n 1`/*
           gcovr --gcov-ignore-parse-errors -g -k -r . --html index.html  -x esp_modem_coverage.xml
-          mkdir docs_gcovr
+          mkdir modem_coverage_report
-          cp $GITHUB_WORKSPACE/${{ env.COMP_DIR }}/index.html docs_gcovr
+          cp $GITHUB_WORKSPACE/${{ env.COMP_DIR }}/index.html modem_coverage_report
-          touch docs_gcovr/.nojekyll
+          cp -rf modem_coverage_report $GITHUB_WORKSPACE
-
       - name: Code Coverage Summary Report
         uses: irongut/CodeCoverageSummary@v1.3.0
         with:
@@ -142,7 +145,6 @@ jobs:
           indicators: true
           output: both
           thresholds: '60 80'
-
       - name: Write to Job Summary
         run: cat code-coverage-results.md >> $GITHUB_STEP_SUMMARY
 
@@ -150,28 +152,7 @@ jobs:
         uses: actions/upload-artifact@v3
         if: always()
         with:
-          name: docs_gcovr
+          name: modem_coverage_report
           path: |
-            ${{ env.COMP_DIR }}/docs_gcovr
+            ${{ env.COMP_DIR }}/modem_coverage_report
           if-no-files-found: error
-
-#  show_report_data:
-#    name: Publish-Results
-#    if: github.ref == 'refs/heads/master' || github.repository != 'espressif/esp-protocols'
-#    runs-on: ubuntu-22.04
-#    needs: gcovr_analyzer_esp_modem
-#    steps:
-#      - name: Checkout 🛎️
-#        uses: actions/checkout@v3
-#        with:
-#          persist-credentials: false
-#      - name: Download Artifacts
-#        uses: actions/download-artifact@v1
-#        with:
-#          name: docs_gcovr
-#
-#      - name: Deploy generated docs
-#        uses: JamesIves/github-pages-deploy-action@v4
-#        with:
-#          branch: gh-pages
-#          folder: 'docs_gcovr'

.github/workflows/modem__target-test.yml

@@ -15,7 +15,7 @@ jobs:
       matrix:
         idf_ver: ["latest"]
         idf_target: ["esp32c3"]
-        test: [ { app: pppd, path: test/target }, { app: sim800_c3, path: examples/pppos_client } ]
+        test: [ { app: pppd, path: test/target }, { app: sim800_c3, path: examples/pppos_client }, { app: sim800_cmux, path: examples/simple_cmux_client } ]
     runs-on: ubuntu-20.04
     container: espressif/idf:${{ matrix.idf_ver }}
     env:
@@ -54,11 +54,11 @@ jobs:
       matrix:
         idf_ver: ["latest"]
         idf_target: ["esp32c3"]
-        test: [ { app: pppd, path: test/target }, { app: sim800_c3, path: examples/pppos_client } ]
+        test: [ { app: pppd, path: test/target }, { app: sim800_c3, path: examples/pppos_client }, { app: sim800_cmux, path: examples/simple_cmux_client } ]
     needs: build_esp_modem_tests
     runs-on:
       - self-hosted
-      - BrnoRPI-GH006
+      - modem
     env:
       TEST_DIR: components/esp_modem/${{ matrix.test.path }}
     steps:

.github/workflows/pre_commit_check.yml

@@ -12,12 +12,17 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.base_ref }}
+          fetch-depth: 20
       - name: Fetch head and base refs
-        # This is necessary for pre-commit to check the changes in the PR branch
+        # This is necessary for pre-commit to check the changes in the PR branch (and to set origin/HEAD and HEAD refs)
         run: |
-          git fetch origin ${{ github.base_ref }}:base_ref
+          git fetch origin ${{ github.event.pull_request.head.sha }}:pr_ref
-          git fetch origin pull/${{ github.event.pull_request.number }}/head:pr_ref
+          git checkout pr_ref
+          git remote set-head origin --auto
+          git merge-base origin/HEAD HEAD || ( echo "Your PR is far behind origin/HEAD, please rebase" && exit 1 )
       - name: Set up Python environment
         uses: actions/setup-python@master
         with:
@@ -26,13 +31,12 @@ jobs:
         run: |
           pip install pre-commit
           pre-commit install-hooks
-          pre-commit install --hook-type commit-msg --hook-type pre-push
       - name: Run pre-commit and check for any changes
         run: |
           echo "Commits being checked:"
-          git log --oneline --no-decorate base_ref..pr_ref
+          git log --oneline --no-decorate origin/HEAD..HEAD
           echo ""
-          if ! pre-commit run --from-ref base_ref --to-ref pr_ref --show-diff-on-failure ; then
+          if ! pre-commit run --from-ref origin/HEAD --to-ref HEAD --hook-stage manual --show-diff-on-failure ; then
             echo ""
             echo "::notice::It looks like the commits in this PR have been made without having pre-commit hooks installed."
             echo "::notice::Please see https://github.com/espressif/esp-protocols/CONTRIBUTING.md for instructions."

.github/workflows/publish-coverage-report.yml

@@ -0,0 +1,45 @@
+name: Publish coverage report to Github Pages
+
+on:
+  workflow_run:
+    workflows: ["websocket: build/host-tests", "esp-modem: build/host-tests"]
+    types:
+      - completed
+    branches:
+      - master
+
+jobs:
+  publish_github_pages:
+    runs-on: ubuntu-latest
+    if: github.repository == 'espressif/esp-protocols'
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+      - name: Download Websocket Artifact
+        uses: dawidd6/action-download-artifact@v2
+        with:
+          workflow: websocket__build-host-tests.yml
+          workflow_conclusion: success
+          name: websocket_coverage_report
+          path: websocket_coverage_report_artifact
+      - name: Download Modem Artifact
+        uses: dawidd6/action-download-artifact@v2
+        with:
+          workflow: modem__build-host-tests.yml
+          workflow_conclusion: success
+          name: modem_coverage_report
+          path: modem_coverage_report_artifact
+      - name: Merge HTML files
+        run: |
+          echo "<html><body>" > index.html
+          cat modem_coverage_report_artifact/index.html >> index.html
+          cat websocket_coverage_report_artifact/index.html >> index.html
+          echo "</body></html>" >> index.html
+          mkdir coverage_report
+          mv index.html coverage_report
+
+      - name: Deploy generated docs
+        uses: JamesIves/github-pages-deploy-action@4.1.5
+        with:
+          branch: gh-pages
+          folder: coverage_report

.github/workflows/publish-docs-component.yml

@@ -5,66 +5,90 @@ on:
     branches:
       - master
 
+env:
+  DOCS_DEPLOY_URL_BASE: ${{ secrets.DOCS_DEPLOY_URL_BASE }}
+  DOCS_DEPLOY_SERVER: ${{ secrets.DOCS_DEPLOY_SERVER }}
+  DOCS_DEPLOY_SERVER_USER: ${{ secrets.DOCS_DEPLOY_SERVER_USER }}
+  DOCS_DEPLOY_KEY: ${{ secrets.DOCS_DEPLOY_PRIVATEKEY }}
+  DOCS_DEPLOY_PATH_ORIG : ${{ secrets.DOCS_DEPLOY_PATH }}
+
 jobs:
-  docs_build:
+  publish:
-    name: Docs-Build-And-Upload
+    name: Publish Tag, Release, Docs, Component
     runs-on: ubuntu-latest
     # Skip running on forks since it won't have access to secrets
     if: github.repository == 'espressif/esp-protocols'
-
     steps:
       - name: Checkout esp-protocols
-        uses: actions/checkout@master
+        uses: actions/checkout@v3
         with:
           persist-credentials: false
           fetch-depth: 0
           submodules: recursive
-
+          token: "${{ secrets.GITHUB_TOKEN }}"
-      - name: Generate docs
+      - name: Check for version update
-        run: |
+        shell: bash
-          sudo apt-get update
+        run: ./ci/detect_component_bump
-          sudo apt-get -y install doxygen clang python3-pip
+      - name: Tag merge commit
-          python -m pip install breathe recommonmark esp-docs
+        if: env.BUMP_VERSION != ''
-          cd $GITHUB_WORKSPACE/components/esp_modem/docs
+        uses: anothrNick/github-tag-action@1.61.0
-          ./generate_docs
+        env:
-          mkdir -p $GITHUB_WORKSPACE/docs/esp_modem
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          cp -r html/. $GITHUB_WORKSPACE/docs/esp_modem
+          CUSTOM_TAG: ${{ env.BUMP_TAG }}
-
+      - name: Create Release
-          cd $GITHUB_WORKSPACE/components/esp_websocket_client/docs
+        if: env.BUMP_VERSION != ''
-          ./generate_docs
+        uses: softprops/action-gh-release@v1
-          mkdir -p $GITHUB_WORKSPACE/docs/esp_websocket_client
-          cp -r html/. $GITHUB_WORKSPACE/docs/esp_websocket_client
-
-          cd $GITHUB_WORKSPACE/components/mdns/docs
-          ./generate_docs
-          mkdir -p $GITHUB_WORKSPACE/docs/mdns/en
-          mkdir -p $GITHUB_WORKSPACE/docs/mdns/zh_CN
-          cp -r html_en/. $GITHUB_WORKSPACE/docs/mdns/en
-          cp -r html_zh_CN/. $GITHUB_WORKSPACE/docs/mdns/zh_CN
-
-          cd $GITHUB_WORKSPACE/components/asio/docs
-          ./generate_docs
-          mkdir -p $GITHUB_WORKSPACE/docs/asio
-          cp -r html/. $GITHUB_WORKSPACE/docs/asio
-
-          cd $GITHUB_WORKSPACE/docs
-          touch .nojekyll
-          echo '<a href="esp_modem/index.html">esp-modem</a><br>' > index.html
-          echo '<a href="esp_websocket_client/index.html">esp-websocket-client</a><br>' >> index.html
-          echo '<a href="asio/index.html">ASIO</a><br>' >> index.html
-          echo '<a href="mdns/en/index.html">mDNS_en</a><br>' >> index.html
-          echo '<a href="mdns/zh_CN/index.html">mDNS_zh_CN</a><br>' >> index.html
-
-
-      - name: Upload components to component service
-        uses: espressif/github-actions/upload_components@master
         with:
-          directories: "components/esp_modem;components/esp_websocket_client;components/mdns;components/asio"
+          body_path: "release_notes.md"
+          tag_name: ${{ env.BUMP_TAG }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - name: Generate docs
+        shell: bash
+        run: |
+            sudo apt-get update
+            sudo apt-get -y install doxygen clang python3-pip
+            python -m pip install breathe recommonmark esp-docs==1.4.1
+            for comp in  `ls components`; do
+              cd $GITHUB_WORKSPACE/docs/${comp}
+              if [[ "${{ env.BUMP_COMPONENT }}" == "${comp}" ]]; then
+                echo "Building specific version of ${comp} (${{ env.BUMP_VERSION }})"
+                ./generate_docs ${{ env.BUMP_VERSION }}
+              else
+                echo "Building latest version of ${comp}"
+                ./generate_docs
+              fi
+            done
+      - name: Deploying generated docs
+        shell: bash
+        run: |
+            source $GITHUB_WORKSPACE/docs/utils.sh
+            add_doc_server_ssh_keys $DOCS_DEPLOY_KEY $DOCS_DEPLOY_SERVER $DOCS_DEPLOY_SERVER_USER
+            export GIT_VER=$(git describe --always)
+            export GITHUB_REF_NAME=latest
+            for comp in  `ls components`; do
+              echo "Deploying latest of ${comp}"
+              export DOCS_BUILD_DIR=$GITHUB_WORKSPACE/docs/${comp}
+              export DOCS_DEPLOY_PATH=$DOCS_DEPLOY_PATH_ORIG/${comp}
+              cd $GITHUB_WORKSPACE/docs/${comp}
+              deploy-docs
+            done;
+            # Deploy docs with version path
+            if [[ "${{ env.BUMP_VERSION }}" != "" ]]; then
+                echo "Deploying specific version of ${comp} (${{ env.BUMP_VERSION }})"
+                cd $GITHUB_WORKSPACE/docs/${{ env.BUMP_COMPONENT }}
+                export GITHUB_REF_NAME=${{ env.BUMP_VERSION }}
+                deploy-docs
+            fi
+      - name: Upload components to component service
+        uses: espressif/upload-components-ci-action@v1
+        with:
+          directories: >
+            components/asio;
+            components/esp_modem;
+            components/esp_mqtt_cxx;
+            components/esp_websocket_client;
+            components/mdns;
+            components/console_simple_init;
           namespace: "espressif"
           api_token: ${{ secrets.IDF_COMPONENT_API_TOKEN }}
-
-      - name: Deploy generated docs
-        uses: JamesIves/github-pages-deploy-action@4.1.5
-        with:
-          branch: gh-pages
-          folder: docs

.github/workflows/run-host-tests.yml

@@ -0,0 +1,86 @@
+name: Run on host
+
+on:
+    workflow_call:
+        inputs:
+            idf_version:
+                required: true
+                type: string
+            app_name:
+                type: string
+                required: true
+            app_path:
+                type: string
+                required: true
+            component_path:
+                type: string
+                required: true
+            upload_artifacts:
+                type: boolean
+                required: true
+
+jobs:
+  build:
+      name: Build App
+      runs-on: ubuntu-20.04
+      permissions:
+        contents: write
+      container: espressif/idf:${{inputs.idf_version}}
+      steps:
+        - name: Checkout esp-protocols
+          uses: actions/checkout@v3
+          with:
+            path: esp-protocols
+        - name: Build ${{ inputs.app_name }} with IDF-${{ inputs.idf_version }}
+          shell: bash
+          run: |
+            . ${IDF_PATH}/export.sh
+            cd $GITHUB_WORKSPACE/${{inputs.app_path}}
+            rm -rf sdkconfig sdkconfig.defaults build
+            cp sdkconfig.ci.linux sdkconfig.defaults
+            idf.py build
+            ./build/${{inputs.app_name}}.elf
+        - name: Build with Coverage Enabled
+          shell: bash
+          run: |
+            . ${IDF_PATH}/export.sh
+            cd $GITHUB_WORKSPACE/${{inputs.app_path}}
+            rm -rf build sdkconfig sdkconfig.defaults
+            cp sdkconfig.ci.coverage sdkconfig.defaults
+            idf.py fullclean
+            idf.py build
+            ./build/${{inputs.app_name}}.elf
+        - name: Run Coverage
+          shell: bash
+          run: |
+            apt-get update && apt-get install -y python3-pip rsync
+            python -m pip install gcovr
+            cd $GITHUB_WORKSPACE/${{inputs.component_path}}
+            gcov `find . -name "*gcda"`
+            gcovr --gcov-ignore-parse-errors -g -k -r . --html index.html -x ${{inputs.app_name}}_coverage.xml
+            mkdir ${{inputs.app_name}}_coverage_report
+            touch ${{inputs.app_name}}_coverage_report/.nojekyll
+            cp index.html ${{inputs.app_name}}_coverage_report
+            cp -rf ${{inputs.app_name}}_coverage_report ${{inputs.app_name}}_coverage.xml $GITHUB_WORKSPACE
+        - name: Code Coverage Summary Report
+          uses: irongut/CodeCoverageSummary@v1.3.0
+          with:
+            filename: esp-protocols/**/${{inputs.app_name}}_coverage.xml
+            badge: true
+            fail_below_min: false
+            format: markdown
+            hide_branch_rate: false
+            hide_complexity: false
+            indicators: true
+            output: both
+            thresholds: '60 80'
+        - name: Write to Job Summary
+          run: cat code-coverage-results.md >> $GITHUB_STEP_SUMMARY
+        - name: Upload files to artifacts for run-target job
+          uses: actions/upload-artifact@v3
+          if: ${{inputs.upload_artifacts}}
+          with:
+            name: ${{inputs.app_name}}_coverage_report
+            path: |
+              ${{inputs.component_path}}/${{inputs.app_name}}_coverage_report
+            if-no-files-found: error

.github/workflows/websocket__build-host-tests.yml

@@ -0,0 +1,20 @@
+name: "websocket: build/host-tests"
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    types: [opened, synchronize, reopened, labeled]
+
+
+jobs:
+  host_test_websocket:
+    if: contains(github.event.pull_request.labels.*.name, 'websocket') || github.event_name == 'push'
+    uses: "./.github/workflows/run-host-tests.yml"
+    with:
+        idf_version: "latest"
+        app_name: "websocket"
+        app_path: "esp-protocols/components/esp_websocket_client/examples/linux"
+        component_path: "esp-protocols/components/esp_websocket_client"
+        upload_artifacts: true

.github/workflows/websocket__build-target-test.yml

@@ -13,45 +13,32 @@ jobs:
     name: Build
     strategy:
       matrix:
-        idf_ver: ["release-v5.0", "latest"]
+        idf_ver: ["release-v5.0", "release-v5.1", "latest"]
-        idf_target: ["esp32"]
+        test: [ { app: example, path: "examples/target" }, { app: unit_test, path: "test" } ]
     runs-on: ubuntu-20.04
     container: espressif/idf:${{ matrix.idf_ver }}
     env:
-      TEST_DIR: components/esp_websocket_client/examples
+      TEST_DIR: components/esp_websocket_client/${{ matrix.test.path }}
     steps:
       - name: Checkout esp-protocols
         uses: actions/checkout@v3
         with:
           submodules: recursive
       - name: Build ${{ matrix.example }} with IDF-${{ matrix.idf_ver }} for ${{ matrix.idf_target }}
-        env:
-          IDF_TARGET: ${{ matrix.idf_target }}
-        shell: bash
-        working-directory: ${{ env.TEST_DIR }}
-        run: |
-          . ${IDF_PATH}/export.sh
-          cat sdkconfig.ci >> sdkconfig.defaults
-          idf.py build
-      - name: Merge binaries
-        working-directory: ${{ env.TEST_DIR }}/build
-        env:
-          IDF_TARGET: ${{ matrix.idf_target }}
         shell: bash
         run: |
           . ${IDF_PATH}/export.sh
-          esptool.py --chip ${{ matrix.idf_target }} merge_bin --fill-flash-size 4MB -o flash_image.bin @flash_args
+          python -m pip install idf-build-apps
+          python ./ci/build_apps.py ${TEST_DIR}
+          cd ${TEST_DIR}
+          for dir in `ls -d build_esp32_*`; do
+          $GITHUB_WORKSPACE/ci/clean_build_artifacts.sh `pwd`/$dir
+          zip -qur artifacts.zip $dir
+          done
       - uses: actions/upload-artifact@v3
         with:
-          name: examples_app_bin_${{ matrix.idf_target }}_${{ matrix.idf_ver }}
+          name: websocket_bin_esp32_${{ matrix.idf_ver }}_${{ matrix.test.app }}
-          path: |
+          path: ${{ env.TEST_DIR }}/artifacts.zip
-            ${{ env.TEST_DIR }}/build/bootloader/bootloader.bin
-            ${{ env.TEST_DIR }}/build/partition_table/partition-table.bin
-            ${{ env.TEST_DIR }}/build/*.bin
-            ${{ env.TEST_DIR }}/build/*.elf
-            ${{ env.TEST_DIR }}/build/flasher_args.json
-            ${{ env.TEST_DIR }}/build/config/sdkconfig.h
-            ${{ env.TEST_DIR }}/build/config/sdkconfig.json
           if-no-files-found: error
 
   run-target-websocket:
@@ -64,32 +51,36 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        idf_ver: ["release-v5.0", "latest"]
+        idf_ver: ["release-v5.0", "release-v5.1", "latest"]
         idf_target: ["esp32"]
+        test: [ { app: example, path: "examples/target" }, { app: unit_test, path: "test" } ]
     runs-on:
       - self-hosted
       - ESP32-ETHERNET-KIT
     env:
-      TEST_DIR: components/esp_websocket_client/examples
+      TEST_DIR: components/esp_websocket_client/${{ matrix.test.path }}
     steps:
       - uses: actions/checkout@v3
       - uses: actions/download-artifact@v3
         with:
-          name: examples_app_bin_${{ matrix.idf_target }}_${{ matrix.idf_ver }}
+          name: websocket_bin_esp32_${{ matrix.idf_ver }}_${{ matrix.test.app }}
-          path: ${{ env.TEST_DIR }}/build/
+          path: ${{ env.TEST_DIR }}/ci/
       - name: Install Python packages
         env:
           PIP_EXTRA_INDEX_URL: "https://www.piwheels.org/simple"
         run: |
           pip install --only-binary cryptography --extra-index-url https://dl.espressif.com/pypi/ -r $GITHUB_WORKSPACE/ci/requirements.txt
-      - name: Download Example Test to target
-        run: python -m esptool --chip ${{ matrix.idf_target }} write_flash 0x0 components/esp_websocket_client/examples/build/flash_image.bin
       - name: Run Example Test on target
         working-directory: ${{ env.TEST_DIR }}
         run: |
-          python -m pytest --log-cli-level DEBUG --junit-xml=./examples_results_${{ matrix.idf_target }}_${{ matrix.idf_ver }}.xml --target=${{ matrix.idf_target }}
+          unzip ci/artifacts.zip -d ci
+          for dir in `ls -d ci/build_*`; do
+          rm -rf build sdkconfig.defaults
+          mv $dir build
+          python -m pytest --log-cli-level DEBUG --junit-xml=./results_${{ matrix.test.app }}_${{ matrix.idf_target }}_${{ matrix.idf_ver }}_${dir#"ci/build_"}.xml --target=${{ matrix.idf_target }}
+          done
       - uses: actions/upload-artifact@v3
         if: always()
         with:
-          name: examples_results_${{ matrix.idf_target }}_${{ matrix.idf_ver }}
+          name: results_${{ matrix.test.app }}_${{ matrix.idf_target }}_${{ matrix.idf_ver }}.xml
-          path: ${{ env.TEST_DIR }}/*.xml
+          path: components/esp_websocket_client/${{ matrix.test.path }}/*.xml

.pre-commit-config.yaml

@@ -43,7 +43,7 @@ repos:
     hooks:
       - id: eradicate
   - repo: https://github.com/espressif/check-copyright/
-    rev: v1.0.1
+    rev: v1.0.3
     hooks:
       - id: check-copyright
         args: ['--ignore', 'ci/check_copyright_ignore.txt', '--config', 'ci/check_copyright_config.yaml']
@@ -57,12 +57,12 @@ repos:
     hooks:
       - id: commitizen
       - id: commitizen-branch
-        stages: [push]
+        stages: [push, manual]
   - repo: local
     hooks:
       - id: commit message scopes
-        name: "commit message must be scoped with: mdns, modem, websocket, asio, mqtt_cxx, common"
+        name: "commit message must be scoped with: mdns, modem, websocket, asio, mqtt_cxx, console, common"
-        entry: '\A(?!(feat|fix|ci|bump|test|docs)\((mdns|modem|common|websocket|asio|mqtt_cxx)\)\:)'
+        entry: '\A(?!(feat|fix|ci|bump|test|docs)\((mdns|modem|common|console|websocket|asio|mqtt_cxx|examples)\)\:)'
         language: pygrep
         args: [--multiline]
         stages: [commit-msg]

CONTRIBUTING.md

@@ -24,8 +24,8 @@ When releasing a new component version we have to:
 * Deploy the component to component registry
 * Update the documentation
 
-This process is not fully automated, the first three steps need to be performed manually by project maintainers running the `bump` command (from within this repository, rather than forks, to publish the release `tag`). Release procedure is as follows:
+This process is not fully automated, the first step needs to be performed manually by project maintainers running the `bump` command. Release procedure is as follows:
-* Create a branch in this repository (not from fork)
+* Run `ci/bump [component] [version]` (version number is optional, `cz` would automatically increment it if not present)
-* Run `cz bump [version]` (version number is optional, `cz` would automatically increment it if not present)
+* Check the updated `CHANGELOG.md` and the generated bump commit message
-* Check the updated `CHANGELOG.md`
+* Create a PR
-* Create and merge the branch to master
+Once the PR is merged, the CI job tags the merge commit, creates a new release, builds and deploys documentation and the new component to the component registry

README.md

@@ -1,26 +1,35 @@
 # Collection of protocol components for ESP-IDF
 
-[Documentation of esp-protocol](https://espressif.github.io/esp-protocols)
+## How to use
+
+The [ESP-Protocols](https://github.com/espressif/esp-protocols) repository contains a collection of protocol components for [ESP-IDF](https://github.com/espressif/esp-idf).
+Additionally, each component is available in [IDF Component Registry](https://components.espressif.com).
+Please refer to instructions in [ESP-IDF](https://github.com/espressif/esp-idf)
 
 ## Components
 
 ### esp_modem
 
 * Brief introduction [README](components/esp_modem/README.md)
-* Full html [documentation](https://espressif.github.io/esp-protocols/esp_modem/index.html)
+* Full html [documentation](https://docs.espressif.com/projects/esp-protocols/esp_modem/docs/latest/index.html)
 
 ### mDNS
 
 * Brief introduction [README](components/mdns/README.md)
-* Full html [documentation(English)](https://espressif.github.io/esp-protocols/mdns/en/index.html)
+* Full html [documentation(English)](https://docs.espressif.com/projects/esp-protocols/mdns/docs/latest/en/index.html)
-* Full html [documentation(Chinese)](https://espressif.github.io/esp-protocols/mdns/zh_CN/index.html)
+* Full html [documentation(Chinese)](https://docs.espressif.com/projects/esp-protocols/mdns/docs/latest/zh_CN/index.html)
 
 ### esp_websocket_client
 
 * Brief introduction [README](components/esp_websocket_client/README.md)
-* Full html [documentation](https://espressif.github.io/esp-protocols/esp_websocket_client/index.html)
+* Full html [documentation](https://docs.espressif.com/projects/esp-protocols/esp_websocket_client/docs/latest/index.html)
 
 ### ASIO port
 
 * Brief introduction [README](components/asio/README.md)
-* Full html [documentation](https://espressif.github.io/esp-protocols/asio/index.html)
+* Full html [documentation](https://docs.espressif.com/projects/esp-protocols/asio/docs/latest/index.html)
+
+### esp_mqtt_cxx
+
+* Brief introduction [README](components/esp_mqtt_cxx/README.md)
+* Full html [documentation](https://docs.espressif.com/projects/esp-protocols/esp_mqtt_cxx/docs/latest/index.html)

ci/build_apps.py

@@ -7,7 +7,6 @@ This file is used in CI for esp-protocols build tests
 import argparse
 import os
 import sys
-from pathlib import Path
 
 from idf_build_apps import build_apps, find_apps, setup_logging
 from idf_build_apps.constants import SUPPORTED_TARGETS
@@ -24,27 +23,35 @@ if __name__ == '__main__':
         default='all',
         help='Build apps for given target',
     )
+    parser.add_argument('-r', '--rules', nargs='*', default=['sdkconfig.ci=default', 'sdkconfig.ci.*=', '=default'], help='Rules how to treat configs')
+    parser.add_argument('-m', '--manifests', nargs='*', default=[], help='list of manifest files')
+    parser.add_argument('-d', '--delete', action='store_true', help='Delete build artifacts')
+    parser.add_argument('-c', '--recursive', action='store_true', help='Build recursively')
+    parser.add_argument('-l', '--linux', action='store_true', help='Include linux build (dont check warnings)')
     args = parser.parse_args()
 
     IDF_PATH = os.environ['IDF_PATH']
 
-    print(args.paths)
+    # Compose the ignore warning strings from the global list and from the environment
+    ignore_warning_file = os.path.join(os.path.dirname(os.path.realpath(__file__)),'ignore_build_warnings.txt')
+    ignore_warning = open(ignore_warning_file).read().rstrip('\n').split('\n')
+    if 'EXPECTED_WARNING' in os.environ:
+        ignore_warning += os.environ['EXPECTED_WARNING'].split('\n')
+    if args.linux:
+        SUPPORTED_TARGETS.append('linux')
+        ignore_warning = 'warning: '  # Ignore all common warnings on linux builds
     setup_logging(2)
     apps = find_apps(
         args.paths,
-        recursive=True,
+        recursive=args.recursive,
         target=args.target,
         build_dir='build_@t_@w',
-        config_rules_str=[
+        config_rules_str=args.rules,
-            'sdkconfig.ci=default', 'sdkconfig.ci.*=', '=default'
-        ],
         build_log_path='build_log.txt',
-        size_json_path='size.json',
+        size_json_path='size.json' if not args.linux else None,
         check_warnings=True,
-        preserve=True,
+        preserve=not args.delete,
-        manifest_files=[
+        manifest_files=args.manifests,
-            str(p) for p in Path('.').glob('**/.build-test-rules.yml')
-        ],
         default_build_targets=SUPPORTED_TARGETS,
         manifest_rootpath='.',
     )
@@ -56,5 +63,5 @@ if __name__ == '__main__':
         build_apps(apps,
                    dry_run=False,
                    keep_going=False,
-                   ignore_warning_strs=os.environ['EXPECTED_WARNING']
+                   ignore_warning_strs=ignore_warning)
-                   if 'EXPECTED_WARNING' in os.environ else None))
+    )

ci/bump

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+if [ -z "$1" ]; then
+    echo "Usage: bump component [version]"
+    exit 1;
+fi
+
+comp=$1; shift;
+
+cd components/${comp}
+if ! cz bump --dry-run; then
+    echo "Commitizen bump commad failed!"
+    exit 1;
+fi
+
+cz_bump_out=`cz bump --files-only "$@"`
+commit_title=`echo "${cz_bump_out}" | head -1`
+commit_body=`cat ../../release_notes.txt`
+
+git add -u .
+git commit -m $"${commit_title}
+
+${commit_body}"

ci/changelog.py

@@ -93,6 +93,13 @@ def main():
         updated_changelog.write(orig_items)
     git.add(filename)
 
+    # write the current changelog entry to a local text file (removing links, captions and extra newlines)
+    changelog = re.sub(r'\[([^\]]+)\]\([^\)]+\)', r'\1', changelog)
+    changelog = re.sub(r'\#\#[\#\s]*(.+)', r'\1', changelog)
+    changelog = re.sub(r'\n\n', '\n', changelog)
+    with open(os.path.join(root_path, 'release_notes.txt'), 'w') as release_notes:
+        release_notes.write(changelog)
+
 
 if __name__ == '__main__':
     main()

ci/check_copyright_config.yaml

@@ -29,6 +29,7 @@ DEFAULT:
 # this section sets the default license for examples and unit tests of components
 examples_and_unit_tests:
   include:
+   - 'examples/**'
    - 'components/**/examples/**'
    - 'components/**/test/**'
    - 'components/**/tests/**'
@@ -57,7 +58,7 @@ slim_modem_examples:
 ignore:
   perform_check: no
   include:
-   - 'components/**/docs/**'
+   - '**/docs/**'
    - 'components/esp_modem/port/linux/**'
    - 'components/asio/examples/**'
    - 'components/mdns/**/esp_system_protocols_linux/**'

ci/detect_component_bump

@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+
+set -e
+
+if ! git show -s | grep -q '^Merge'; then
+    echo "Not a merge commit"
+    exit 0;
+fi
+
+for comp in  `ls components`; do
+if git log -1 -m --name-only --pretty="" | grep -q components/${comp}/idf_component.yml; then
+    echo "${comp}: Component version file has changed"
+    version=`grep version: components/${comp}/.cz.yaml`
+    version=${version#*version: }
+
+    tag_format=`grep tag_format: components/${comp}/.cz.yaml`
+    tag_format=${tag_format#*tag_format: }
+
+    eval tag=$tag_format
+    # check if the tag is already created
+    if [ $(git tag -l "$tag") ]; then
+        echo "${comp}: version (${tag}) already exits"
+    else
+        echo "${comp}: Component version has been updated to ${version}"
+        # creates release notes from the last entry (between first two "## sections")
+        awk '/^## \[/{a++};{if(a==1){print}}' components/${comp}/CHANGELOG.md > release_notes.md
+
+        echo "BUMP_VERSION=${version}"
+        echo "BUMP_COMPONENT=${comp}"
+        echo "BUMP_TAG=${tag}"
+
+        # export the findings to github env, so it could be used in other jobs
+        echo "BUMP_VERSION=${version}" >> "$GITHUB_ENV"
+        echo "BUMP_COMPONENT=${comp}" >> "$GITHUB_ENV"
+        echo "BUMP_TAG=${tag}" >> "$GITHUB_ENV"
+
+        exit 0;
+    fi
+fi
+done
+echo "No changes in component version file"

ci/ignore_build_warnings.txt

@@ -0,0 +1 @@
+DeprecationWarning: pkg_resources is deprecated as an API

common_components/linux_compat/esp_timer/esp_timer_linux.c

@@ -7,6 +7,7 @@
 #include "esp_timer.h"
 #include "freertos/FreeRTOS.h"
 #include "freertos/task.h"
+#include <pthread.h>
 
 void *create_tt(esp_timer_cb_t cb);
 
@@ -37,3 +38,10 @@ esp_err_t esp_timer_delete(esp_timer_handle_t timer)
     destroy_tt(timer);
     return ESP_OK;
 }
+
+int64_t esp_timer_get_time(void)
+{
+    struct timespec spec;
+    clock_gettime(CLOCK_REALTIME, &spec);
+    return spec.tv_nsec / 1000 + spec.tv_sec * 1000000;
+}

common_components/linux_compat/esp_timer/include/esp_timer.h

@@ -9,6 +9,11 @@
 #include <stdint.h>
 #include "bsd/string.h"
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
 typedef struct esp_timer *esp_timer_handle_t;
 
 typedef void (*esp_timer_cb_t)(void *arg);
@@ -32,3 +37,9 @@ esp_err_t esp_timer_start_periodic(esp_timer_handle_t timer, uint64_t period);
 esp_err_t esp_timer_stop(esp_timer_handle_t timer);
 
 esp_err_t esp_timer_delete(esp_timer_handle_t timer);
+
+int64_t esp_timer_get_time(void);
+
+#ifdef __cplusplus
+}
+#endif

common_components/linux_compat/freertos/CMakeLists.txt

@@ -1,4 +1,5 @@
-idf_component_register(SRCS freertos_linux.c queue_unique_ptr.cpp
+idf_component_register(SRCS freertos_linux.c
+                            osal/queue.cpp osal/event_group.cpp osal/mutex.cpp
                        INCLUDE_DIRS include)
 
 set(THREADS_PREFER_PTHREAD_FLAG ON)

common_components/linux_compat/freertos/freertos_linux.c

@@ -0,0 +1,272 @@
+/*
+ * SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#include <unistd.h>
+#include "freertos/FreeRTOS.h"
+#include "freertos/task.h"
+#include <pthread.h>
+#include <assert.h>
+#include <stdlib.h>
+#include <string.h>
+#include "osal/osal_api.h"
+
+static uint64_t s_semaphore_data = 0;
+
+typedef enum queue_type_tag {
+    MUTEX_REC,
+    MUTEX,
+    SEMA,
+    QUEUE,
+} queue_type_t;
+
+struct generic_queue_handle {
+    queue_type_t type;
+    size_t item_size;
+    void *q;
+};
+
+static struct generic_queue_handle *create_generic_queue(queue_type_t type, uint32_t len, uint32_t item_size)
+{
+    struct generic_queue_handle *h = calloc(1, sizeof(struct generic_queue_handle));
+    h->item_size = len;
+    h->type = type;
+    switch (type) {
+    default:
+    case QUEUE:
+    case SEMA:
+        h->q = osal_queue_create();
+        break;
+
+    case MUTEX:
+    case MUTEX_REC:
+        h->q = osal_mutex_create();
+        break;
+    }
+    return h;
+}
+
+QueueHandle_t xQueueCreate(uint32_t uxQueueLength, uint32_t uxItemSize )
+{
+    return (QueueHandle_t)create_generic_queue(QUEUE, uxQueueLength, uxItemSize);
+}
+
+uint32_t xQueueSend(QueueHandle_t xQueue, const void *pvItemToQueue, TickType_t xTicksToWait)
+{
+    struct generic_queue_handle *h = xQueue;
+    return osal_queue_send(h->q, (uint8_t *)pvItemToQueue, h->item_size) ? pdTRUE : pdFAIL;
+}
+
+uint32_t xQueueSendToBack(QueueHandle_t xQueue, const void *pvItemToQueue, TickType_t xTicksToWait )
+{
+    return xQueueSend(xQueue, pvItemToQueue, xTicksToWait);
+}
+
+uint32_t xQueueReceive(QueueHandle_t xQueue, void *pvBuffer, TickType_t xTicksToWait)
+{
+    struct generic_queue_handle *h = xQueue;
+    return osal_queue_recv(h->q, (uint8_t *)pvBuffer, h->item_size, xTicksToWait) ? pdTRUE : pdFAIL;
+}
+
+BaseType_t xSemaphoreGive( QueueHandle_t xQueue)
+{
+    struct generic_queue_handle *h = xQueue;
+    if (h->type == MUTEX) {
+        osal_mutex_give(h->q);
+        return pdTRUE;
+    }
+    return xQueueSend(xQueue, &s_semaphore_data, portMAX_DELAY);
+}
+
+BaseType_t xSemaphoreGiveRecursive( QueueHandle_t xQueue)
+{
+    struct generic_queue_handle *h = xQueue;
+    if (h->type == MUTEX_REC) {
+        osal_mutex_give(h->q);
+        return pdTRUE;
+    }
+    return pdFALSE;
+}
+BaseType_t xSemaphoreTake( QueueHandle_t xQueue, TickType_t pvTask )
+{
+    struct generic_queue_handle *h = xQueue;
+    if (h->type == MUTEX) {
+        osal_mutex_take(h->q);
+        return pdTRUE;
+    }
+    return xQueueReceive(xQueue, &s_semaphore_data, portMAX_DELAY);
+}
+
+
+BaseType_t xSemaphoreTakeRecursive( QueueHandle_t xQueue, TickType_t pvTask )
+{
+    struct generic_queue_handle *h = xQueue;
+    if (h->type == MUTEX_REC) {
+        osal_mutex_take(h->q);
+        return pdTRUE;
+    }
+    return pdFALSE;
+}
+
+
+
+
+void vQueueDelete( QueueHandle_t xQueue )
+{
+    struct generic_queue_handle *h = xQueue;
+    if (h->q) {
+        if (h->type == MUTEX || h->type == MUTEX_REC) {
+            osal_mutex_delete(h->q);
+        } else {
+            osal_queue_delete(h->q);
+        }
+    }
+    free(xQueue);
+}
+
+QueueHandle_t xSemaphoreCreateBinary(void)
+{
+    QueueHandle_t sempaphore =  xQueueCreate(1, 1);
+    return sempaphore;
+}
+
+QueueHandle_t xSemaphoreCreateMutex(void)
+{
+    return (QueueHandle_t)create_generic_queue(MUTEX, 1, 1);
+}
+
+QueueHandle_t xSemaphoreCreateRecursiveMutex(void)
+{
+    return (QueueHandle_t)create_generic_queue(MUTEX_REC, 1, 1);
+}
+
+
+void vTaskDelete(TaskHandle_t *task)
+{
+    if (task == NULL) {
+        pthread_exit(0);
+    }
+    void *thread_rval = NULL;
+    pthread_join((pthread_t)task, &thread_rval);
+}
+
+void vTaskSuspend(void *task)
+{
+    vTaskDelete(task);
+}
+
+TickType_t xTaskGetTickCount( void )
+{
+    struct timespec spec;
+    clock_gettime(CLOCK_REALTIME, &spec);
+    return spec.tv_nsec / 1000000 + spec.tv_sec * 1000;
+}
+
+void vTaskDelay( const TickType_t xTicksToDelay )
+{
+    usleep(xTicksToDelay * 1000);
+}
+
+void *pthread_task(void *params)
+{
+    struct {
+        void *const param;
+        TaskFunction_t task;
+        bool started;
+    } *pthread_params = params;
+
+    void *const param = pthread_params->param;
+    TaskFunction_t task = pthread_params->task;
+    pthread_params->started = true;
+
+    task(param);
+
+    return NULL;
+}
+
+BaseType_t xTaskCreatePinnedToCore( TaskFunction_t pvTaskCode,
+                                    const char *const pcName,
+                                    const uint32_t usStackDepth,
+                                    void *const pvParameters,
+                                    UBaseType_t uxPriority,
+                                    TaskHandle_t *const pvCreatedTask,
+                                    const BaseType_t xCoreID)
+{
+    xTaskCreate(pvTaskCode, pcName, usStackDepth, pvParameters, uxPriority, pvCreatedTask);
+    return pdTRUE;
+}
+
+
+BaseType_t xTaskCreate(TaskFunction_t pvTaskCode, const char *const pcName, const uint32_t usStackDepth, void *const pvParameters, UBaseType_t uxPriority, TaskHandle_t *const pvCreatedTask)
+{
+    pthread_t new_thread = (pthread_t)NULL;
+    pthread_attr_t attr;
+    struct {
+        void *const param;
+        TaskFunction_t task;
+        bool started;
+    } pthread_params = { .param = pvParameters, .task = pvTaskCode};
+    int res = pthread_attr_init(&attr);
+    assert(res == 0);
+    res = pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE);
+    assert(res == 0);
+    res = pthread_create(&new_thread, &attr, pthread_task, &pthread_params);
+    assert(res == 0);
+
+    if (pvCreatedTask) {
+        *pvCreatedTask = (void *)new_thread;
+    }
+
+    // just wait till the task started so we can unwind params from the stack
+    while (pthread_params.started == false) {
+        usleep(1000);
+    }
+    return pdTRUE;
+}
+
+void xTaskNotifyGive(TaskHandle_t task)
+{
+
+}
+
+BaseType_t xTaskNotifyWait(uint32_t bits_entry_clear, uint32_t bits_exit_clear, uint32_t *value, TickType_t wait_time )
+{
+    return true;
+}
+
+TaskHandle_t xTaskGetCurrentTaskHandle(void)
+{
+    return NULL;
+}
+
+EventGroupHandle_t xEventGroupCreate( void )
+{
+    return osal_signal_create();
+}
+
+void vEventGroupDelete( EventGroupHandle_t xEventGroup )
+{
+    osal_signal_delete(xEventGroup);
+}
+
+EventBits_t xEventGroupClearBits( EventGroupHandle_t xEventGroup, const EventBits_t uxBitsToClear )
+{
+    return osal_signal_clear(xEventGroup, uxBitsToClear);
+}
+
+EventBits_t xEventGroupGetBits( EventGroupHandle_t xEventGroup)
+{
+    return osal_signal_get(xEventGroup);
+}
+
+EventBits_t xEventGroupSetBits( EventGroupHandle_t xEventGroup, const EventBits_t uxBitsToSet )
+{
+    return osal_signal_set(xEventGroup, uxBitsToSet);
+}
+
+EventBits_t xEventGroupWaitBits( EventGroupHandle_t xEventGroup, const EventBits_t uxBitsToWaitFor, const BaseType_t xClearOnExit, const BaseType_t xWaitForAllBits, TickType_t xTicksToWait )
+{
+    return osal_signal_wait(xEventGroup, uxBitsToWaitFor, xWaitForAllBits, xTicksToWait);
+}

common_components/linux_compat/freertos/include/esp_task.h

@@ -10,3 +10,4 @@
 
 #define ESP_TASK_PRIO_MAX 25
 #define ESP_TASKD_EVENT_PRIO 5
+#define ESP_TASKD_EVENT_STACK 1024

common_components/linux_compat/freertos/include/freertos/FreeRTOS.h

@@ -16,7 +16,9 @@
 typedef void *SemaphoreHandle_t;
 typedef void *QueueHandle_t;
 typedef void *TaskHandle_t;
+typedef void *EventGroupHandle_t;
 typedef uint32_t TickType_t;
+typedef TickType_t               EventBits_t;
 
 typedef void (*TaskFunction_t)( void * );
 typedef unsigned int    UBaseType_t;
@@ -30,6 +32,5 @@ typedef int             BaseType_t;
 
 #define pdMS_TO_TICKS(tick)    (tick)
 
-uint32_t esp_get_free_heap_size(void);
 uint32_t esp_random(void);
 void vTaskSuspendAll(void);

common_components/linux_compat/freertos/include/freertos/event_groups.h

@@ -0,0 +1,6 @@
+/*
+ * SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+#pragma once

common_components/linux_compat/freertos/include/freertos/queue.h

@@ -3,11 +3,4 @@
  *
  * SPDX-License-Identifier: Unlicense OR CC0-1.0
  */
-//
+#pragma once
-// Created by david on 1/13/23.
-//
-
-#ifndef _QUEUE_H_
-#define _QUEUE_H_
-
-#endif //_QUEUE_H_

common_components/linux_compat/freertos/include/freertos/semphr.h

@@ -3,11 +3,4 @@
  *
  * SPDX-License-Identifier: Unlicense OR CC0-1.0
  */
-//
+#pragma once
-// Created by david on 1/13/23.
-//
-
-#ifndef _SEMAPHR_H_
-#define _SEMAPHR_H_
-
-#endif //_SEMAPHR_H_

common_components/linux_compat/freertos/include/freertos/task.h

@@ -7,6 +7,10 @@
 
 #include "freertos/FreeRTOS.h"
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 #define TaskHandle_t TaskHandle_t
 #define vSemaphoreDelete( xSemaphore ) vQueueDelete( ( QueueHandle_t ) ( xSemaphore ) )
 
@@ -26,7 +30,7 @@ BaseType_t xTaskCreatePinnedToCore( TaskFunction_t pvTaskCode,
                                     TaskHandle_t *const pvCreatedTask,
                                     const BaseType_t xCoreID);
 
-void xTaskCreate(TaskFunction_t pvTaskCode, const char *const pcName, const uint32_t usStackDepth, void *const pvParameters, UBaseType_t uxPriority, TaskHandle_t *const pvCreatedTask);
+BaseType_t xTaskCreate(TaskFunction_t pvTaskCode, const char *const pcName, const uint32_t usStackDepth, void *const pvParameters, UBaseType_t uxPriority, TaskHandle_t *const pvCreatedTask);
 
 TickType_t xTaskGetTickCount( void );
 
@@ -35,11 +39,16 @@ void vQueueDelete( QueueHandle_t xQueue );
 QueueHandle_t xSemaphoreCreateBinary(void);
 
 QueueHandle_t xSemaphoreCreateMutex(void);
+QueueHandle_t xSemaphoreCreateRecursiveMutex(void);
 
 BaseType_t xSemaphoreGive( QueueHandle_t xQueue);
 
 BaseType_t xSemaphoreTake( QueueHandle_t xQueue, TickType_t pvTask );
 
+BaseType_t xSemaphoreGiveRecursive( QueueHandle_t xQueue);
+
+BaseType_t xSemaphoreTakeRecursive( QueueHandle_t xQueue, TickType_t pvTask );
+
 void vTaskDelete(TaskHandle_t *task);
 
 QueueHandle_t xQueueCreate( uint32_t uxQueueLength,
@@ -48,3 +57,27 @@ QueueHandle_t xQueueCreate( uint32_t uxQueueLength,
 uint32_t xQueueSend(QueueHandle_t xQueue, const void *pvItemToQueue, TickType_t xTicksToWait);
 
 uint32_t xQueueReceive(QueueHandle_t xQueue, void *pvBuffer, TickType_t xTicksToWait);
+
+void vTaskSuspend(void *task);
+
+EventGroupHandle_t xEventGroupCreate( void );
+void vEventGroupDelete( EventGroupHandle_t xEventGroup );
+EventBits_t xEventGroupClearBits( EventGroupHandle_t xEventGroup,
+                                  const EventBits_t uxBitsToClear );
+
+EventBits_t xEventGroupWaitBits( EventGroupHandle_t xEventGroup,
+                                 const EventBits_t uxBitsToWaitFor,
+                                 const BaseType_t xClearOnExit,
+                                 const BaseType_t xWaitForAllBits,
+                                 TickType_t xTicksToWait );
+
+EventBits_t xEventGroupGetBits( EventGroupHandle_t xEventGroup);
+
+EventBits_t xEventGroupSetBits( EventGroupHandle_t xEventGroup,
+                                const EventBits_t uxBitsToSet );
+
+uint32_t xQueueSendToBack(QueueHandle_t xQueue, const void *pvItemToQueue, TickType_t xTicksToWait );
+
+#ifdef __cplusplus
+}
+#endif //__cplusplus

common_components/linux_compat/freertos/osal/event_group.cpp

@@ -0,0 +1,109 @@
+/*
+ * SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+#include <mutex>
+#include <condition_variable>
+#include "osal_api.h"
+
+
+class SignalGroup {
+
+    struct SignalGroupInternal {
+        std::condition_variable notify;
+        std::mutex m;
+        uint32_t flags{ 0 };
+    };
+
+    using SignalT = std::unique_ptr<SignalGroupInternal>;
+
+public:
+
+    void set(uint32_t bits)
+    {
+        std::unique_lock<std::mutex> lock(event_group->m);
+        event_group->flags |= bits;
+        event_group->notify.notify_all();
+    }
+
+    uint32_t get()
+    {
+        return event_group->flags;
+    }
+
+    void clear(uint32_t bits)
+    {
+        std::unique_lock<std::mutex> lock(event_group->m);
+        event_group->flags &= ~bits;
+        event_group->notify.notify_all();
+    }
+
+    // waiting for all and clearing if set
+    bool wait(uint32_t flags, uint32_t time_ms)
+    {
+        std::unique_lock<std::mutex> lock(event_group->m);
+        return event_group->notify.wait_for(lock, std::chrono::milliseconds(time_ms), [&] {
+            if ((flags & event_group->flags) == flags)
+            {
+                event_group->flags &= ~flags;
+                return true;
+            }
+            return false;
+        });
+    }
+
+
+    // waiting for any bit, not clearing them
+    bool wait_any(uint32_t flags, uint32_t time_ms)
+    {
+        std::unique_lock<std::mutex> lock(event_group->m);
+        return event_group->notify.wait_for(lock, std::chrono::milliseconds(time_ms), [&] { return flags & event_group->flags; });
+    }
+
+private:
+    SignalT event_group{std::make_unique<SignalGroupInternal>()};
+};
+
+
+void *osal_signal_create()
+{
+    auto signal = new SignalGroup;
+    return signal;
+}
+
+void osal_signal_delete(void *s)
+{
+    delete static_cast<SignalGroup *>(s);
+}
+
+uint32_t osal_signal_clear(void *s, uint32_t bits)
+{
+    auto signal = static_cast<SignalGroup *>(s);
+    signal->clear(bits);
+    return signal->get();
+}
+
+uint32_t osal_signal_set(void *s, uint32_t bits)
+{
+    auto signal = static_cast<SignalGroup *>(s);
+    signal->set(bits);
+    return signal->get();
+}
+
+uint32_t osal_signal_get(void *s)
+{
+    auto signal = static_cast<SignalGroup *>(s);
+    return signal->get();
+}
+
+uint32_t osal_signal_wait(void *s, uint32_t flags, bool all, uint32_t timeout)
+{
+    auto signal = static_cast<SignalGroup *>(s);
+    if (all) {
+        signal->wait(flags, timeout);
+    } else {
+        signal->wait_any(flags, timeout);
+    }
+    return signal->get();
+}

common_components/linux_compat/freertos/osal/mutex.cpp

@@ -0,0 +1,31 @@
+/*
+ * SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+#include <queue>
+#include <mutex>
+#include "osal_api.h"
+
+void *osal_mutex_create()
+{
+    auto mut = new std::recursive_mutex();
+    return mut;
+}
+
+void osal_mutex_delete(void *mut)
+{
+    delete static_cast<std::recursive_mutex *>(mut);
+}
+
+void osal_mutex_take(void *m)
+{
+    auto mut = static_cast<std::recursive_mutex *>(m);
+    mut->lock();
+}
+
+void osal_mutex_give(void *m)
+{
+    auto mut = static_cast<std::recursive_mutex *>(m);
+    mut->unlock();
+}

common_components/linux_compat/freertos/osal/osal_api.h

@@ -0,0 +1,34 @@
+/*
+ * SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+#pragma once
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// queue api
+void *osal_queue_create(void);
+void osal_queue_delete(void *q);
+bool osal_queue_send(void *q, uint8_t *data, size_t len);
+bool osal_queue_recv(void *q, uint8_t *data, size_t len, uint32_t ms);
+
+// mutex api
+void *osal_mutex_create(void);
+void osal_mutex_delete(void *m);
+void osal_mutex_take(void *m);
+void osal_mutex_give(void *m);
+
+// event groups
+void *osal_signal_create(void);
+void osal_signal_delete(void *s);
+uint32_t osal_signal_clear(void *s, uint32_t bits);
+uint32_t osal_signal_set(void *s, uint32_t bits);
+uint32_t osal_signal_get(void *s);
+uint32_t osal_signal_wait(void *s, uint32_t flags, bool all, uint32_t timeout);
+
+#ifdef __cplusplus
+}
+#endif

common_components/linux_compat/freertos/osal/queue.cpp

@@ -0,0 +1,78 @@
+/*
+ * SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#include <cstdint>
+#include <vector>
+#include <cstring>
+#include <queue>
+#include <mutex>
+#include <condition_variable>
+#include <atomic>
+#include "osal_api.h"
+
+template <class T>
+class Queue {
+public:
+    void send(std::unique_ptr<T> t)
+    {
+        std::lock_guard<std::mutex> lock(m);
+        q.push(std::move(t));
+        c.notify_one();
+    }
+
+    std::unique_ptr<T> receive(std::chrono::milliseconds ms)
+    {
+        std::unique_lock<std::mutex> lock(m);
+        while (q.empty()) {
+            if (c.wait_for(lock, ms) == std::cv_status::timeout) {
+                return nullptr;
+            }
+        }
+        std::unique_ptr<T> val = std::move(q.front());
+        q.pop();
+        return val;
+    }
+
+private:
+    std::queue<std::unique_ptr<T>> q{};
+    mutable std::mutex m{};
+    std::condition_variable c{};
+};
+
+using item_t = std::vector<uint8_t>;
+
+void *osal_queue_create(void)
+{
+    auto *q = new Queue<item_t>();
+    return q;
+}
+
+void osal_queue_delete(void *q)
+{
+    auto *queue = static_cast<Queue<item_t> *>(q);
+    delete (queue);
+}
+
+bool osal_queue_send(void *q, uint8_t *data, size_t len)
+{
+    auto v = std::make_unique<item_t>(len);
+    v->assign(data, data + len);
+    auto queue = static_cast<Queue<item_t> *>(q);
+    queue->send(std::move(v));
+    return true;
+}
+
+
+bool osal_queue_recv(void *q, uint8_t *data, size_t len, uint32_t ms)
+{
+    auto queue = static_cast<Queue<item_t> *>(q);
+    auto v = queue->receive(std::chrono::milliseconds(ms));
+    if (v != nullptr) {
+        memcpy(data, (void *)v->data(), len);
+        return true;
+    }
+    return false;
+}

components/asio/CMakeLists.txt

@@ -7,39 +7,40 @@ if(NOT CONFIG_LWIP_IPV6 AND NOT CMAKE_BUILD_EARLY_EXPANSION)
 endif()
 
 set(asio_sources "asio/asio/src/asio.cpp")
+set(asio_requires lwip)
 
 if(CONFIG_ASIO_SSL_SUPPORT)
-    if(CONFIG_ASIO_USE_ESP_OPENSSL)
+    list(APPEND asio_sources
-        list(APPEND asio_sources
+            "port/src/asio_ssl_impl.cpp"
-                "port/src/asio_ssl_impl.cpp"
+            "port/mbedtls/src/mbedtls_context.cpp"
-                "port/mbedtls/src/mbedtls_context.cpp"
+            "port/mbedtls/src/mbedtls_engine.cpp")
-                "port/mbedtls/src/mbedtls_engine.cpp")
+    set(asio_priv_includes "port/mbedtls/include")
-        set(asio_priv_includes "port/mbedtls/include")
-    endif()
-
-    if(CONFIG_ASIO_USE_ESP_WOLFSSL)
-        list(APPEND asio_sources
-                "asio/asio/src/asio_ssl.cpp")
-    endif()
 endif()
 
 idf_component_register(SRCS ${asio_sources}
                     INCLUDE_DIRS "asio/asio/include" "port/include"
                     PRIV_INCLUDE_DIRS ${asio_priv_includes}
-                    REQUIRES lwip)
+                    PRIV_REQUIRES ${asio_requires})
-target_compile_options(${COMPONENT_LIB} PRIVATE "-Wno-format")
+
+target_compile_definitions(${COMPONENT_LIB} PUBLIC SA_RESTART=0x01
+                                                   SA_NOCLDSTOP=0x2
+                                                   SA_NOCLDWAIT=0x4
+                                                   ASIO_DISABLE_SERIAL_PORT
+                                                   ASIO_SEPARATE_COMPILATION
+                                                   ASIO_STANDALONE
+                                                   ASIO_HAS_PTHREADS
+                                                   OPENSSL_NO_ENGINE
+)
+
+if(NOT CONFIG_COMPILER_CXX_EXCEPTIONS)
+    target_compile_definitions(${COMPONENT_LIB} PUBLIC ASIO_NO_EXCEPTIONS)
+endif()
+
+if(NOT CONFIG_COMPILER_RTTI)
+    target_compile_definitions(${COMPONENT_LIB} PUBLIC ASIO_NO_TYPEID)
+endif()
 
 if(CONFIG_ASIO_SSL_SUPPORT)
-    if(CONFIG_ASIO_USE_ESP_WOLFSSL)
-        idf_component_get_property(wolflib esp-wolfssl COMPONENT_LIB)
-        idf_component_get_property(wolfdir esp-wolfssl COMPONENT_DIR)
-
-        target_link_libraries(${COMPONENT_LIB} PUBLIC ${wolflib})
-        target_include_directories(${COMPONENT_LIB} PUBLIC ${wolfdir}/wolfssl/wolfssl)
-    endif()
-
-    if(CONFIG_ASIO_USE_ESP_OPENSSL)
         idf_component_get_property(mbedtls mbedtls COMPONENT_LIB)
         target_link_libraries(${COMPONENT_LIB} PUBLIC ${mbedtls})
-    endif()
 endif()

components/asio/Kconfig

@@ -6,23 +6,6 @@ menu "ESP-ASIO"
         default n
         help
             Enable support for basic SSL/TLS features, available for mbedTLS/OpenSSL
-            as well as wolfSSL TLS library.
-
-    choice ASIO_SSL_LIBRARY_CHOICE
-        prompt "Choose SSL/TLS library for ESP-TLS (See help for more Info)"
-        default ASIO_USE_ESP_OPENSSL
-        depends on ASIO_SSL_SUPPORT
-        help
-            The ASIO support multiple backend TLS libraries. Currently the mbedTLS with a thin ESP-OpenSSL
-            port layer (default choice) and WolfSSL are supported.
-            Different TLS libraries may support different features and have different resource
-            usage. Consult the ESP-TLS documentation in ESP-IDF Programming guide for more details.
-        config ASIO_USE_ESP_OPENSSL
-            bool "esp-openssl"
-        config ASIO_USE_ESP_WOLFSSL
-            depends on TLS_STACK_WOLFSSL
-            bool "wolfSSL (License info in wolfSSL directory README)"
-    endchoice
 
     config ASIO_SSL_BIO_SIZE
         int "Size of BIO object"

components/asio/README.md

@@ -6,8 +6,8 @@ Asio is a cross-platform C++ library, see https://think-async.com/Asio/. It prov
 
 ## Examples
 
-Get started with example test :example:`examples <examples/..>`:
+Get started with examples [examples](https://github.com/espressif/esp-protocols/tree/master/components/asio/examples)
 
 ## Documentation
 
-* View the full [html documentation](https://espressif.github.io/esp-protocols/asio/index.html)
+* View the full [html documentation](https://docs.espressif.com/projects/esp-protocols/asio/docs/latest/index.html)

components/asio/docs/generate_docs

@@ -1,26 +0,0 @@
-build-docs --target esp32 --language en
-
-cp -rf _build/en/esp32/html .
-rm -rf _build __pycache__
-
-# Modifes some version and target fields of index.html
-echo "<script type="text/javascript">
-window.onload =(function() {
-    var myAnchor = document.getElementById('version-select');
-    var mySpan = document.createElement('input');
-    mySpan.setAttribute('type', 'text');
-    mySpan.setAttribute('maxLength', '10');
-    mySpan.value = 'latest';
-    mySpan.setAttribute('disabled', true);
-    myAnchor.parentNode.replaceChild(mySpan, myAnchor);
-
-    var myAnchor = document.getElementById('target-select');
-    var mySpan = document.createElement('input');
-    mySpan.setAttribute('type', 'text');
-    mySpan.setAttribute('maxLength', '10');
-    mySpan.value = 'all targets';
-    mySpan.setAttribute('disabled', true);
-    myAnchor.parentNode.replaceChild(mySpan, myAnchor);
-
-})();
-</script>" >> html/index.html

components/asio/examples/asio_chat/main/idf_component.yml

@@ -0,0 +1,6 @@
+dependencies:
+  ## Required IDF version
+  idf: ">=5.0"
+  espressif/asio:
+    version: "^1.14.1"
+    override_path: '../../../'

components/asio/examples/async_request/main/idf_component.yml

@@ -0,0 +1,6 @@
+dependencies:
+  ## Required IDF version
+  idf: ">=5.0"
+  espressif/asio:
+    version: "^1.14.1"
+    override_path: '../../../'

components/asio/examples/async_request/sdkconfig.defaults

@@ -0,0 +1,8 @@
+CONFIG_ESP_MAIN_TASK_STACK_SIZE=8192
+CONFIG_COMPILER_CXX_EXCEPTIONS=y
+
+#
+# Partition Table
+#
+# Leave some room for larger apps without needing to reduce other features
+CONFIG_PARTITION_TABLE_SINGLE_APP_LARGE=y

components/asio/examples/socks4/main/idf_component.yml

@@ -0,0 +1,6 @@
+dependencies:
+  ## Required IDF version
+  idf: ">=5.0"
+  espressif/asio:
+    version: "^1.14.1"
+    override_path: '../../../'

components/asio/examples/ssl_client_server/main/idf_component.yml

@@ -0,0 +1,6 @@
+dependencies:
+  ## Required IDF version
+  idf: ">=5.0"
+  espressif/asio:
+    version: "^1.14.1"
+    override_path: '../../../'

components/asio/examples/ssl_client_server/sdkconfig.defaults

@@ -2,6 +2,7 @@ CONFIG_ASIO_SSL_SUPPORT=y
 CONFIG_PARTITION_TABLE_CUSTOM=y
 CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="partitions.csv"
 CONFIG_PARTITION_TABLE_FILENAME="partitions.csv"
+CONFIG_COMPILER_CXX_EXCEPTIONS=y
 
 #
 # Partition Table

components/asio/examples/tcp_echo_server/main/idf_component.yml

@@ -0,0 +1,6 @@
+dependencies:
+  ## Required IDF version
+  idf: ">=5.0"
+  espressif/asio:
+    version: "^1.14.1"
+    override_path: '../../../'

components/asio/examples/tcp_echo_server/sdkconfig.defaults

@@ -1,4 +1,5 @@
 CONFIG_ESP_MAIN_TASK_STACK_SIZE=8192
+CONFIG_COMPILER_CXX_EXCEPTIONS=y
 
 #
 # Partition Table

components/asio/examples/udp_echo_server/main/idf_component.yml

@@ -0,0 +1,6 @@
+dependencies:
+  ## Required IDF version
+  idf: ">=5.0"
+  espressif/asio:
+    version: "^1.14.1"
+    override_path: '../../../'

components/asio/examples/udp_echo_server/sdkconfig.defaults

@@ -1,4 +1,5 @@
 CONFIG_ESP_MAIN_TASK_STACK_SIZE=8192
+CONFIG_COMPILER_CXX_EXCEPTIONS=y
 
 #
 # Partition Table

components/asio/port/include/esp_asio_config.h

@@ -1,45 +1,12 @@
 /*
- * SPDX-FileCopyrightText: 2018-2022 Espressif Systems (Shanghai) CO LTD
+ * SPDX-FileCopyrightText: 2018-2023 Espressif Systems (Shanghai) CO LTD
  *
  * SPDX-License-Identifier: Apache-2.0
  */
 #ifndef _ESP_ASIO_CONFIG_H_
 #define _ESP_ASIO_CONFIG_H_
 
-//
+#define ASIO_SSL_DETAIL_OPENSSL_TYPES_HPP
-// Enabling exceptions only when they are enabled in menuconfig
+#include "openssl_stub.hpp"
-//
-# include <sdkconfig.h>
-# ifndef CONFIG_COMPILER_CXX_EXCEPTIONS
-#  define ASIO_NO_EXCEPTIONS
-# endif   // CONFIG_COMPILER_CXX_EXCEPTIONS
-
-# ifndef CONFIG_COMPILER_RTTI
-#  define ASIO_NO_TYPEID
-# endif   // CONFIG_COMPILER_RTTI
-
-//
-// Use system sockets
-//
-# include "sys/socket.h"
-
-//
-// Specific ASIO feature flags
-//
-# define ASIO_DISABLE_SERIAL_PORT
-# define ASIO_SEPARATE_COMPILATION
-# define ASIO_STANDALONE
-# define ASIO_HAS_PTHREADS
-# define ASIO_DISABLE_CONCEPTS
-
-# ifdef CONFIG_ASIO_USE_ESP_OPENSSL
-#  define ASIO_USE_ESP_OPENSSL
-#  define OPENSSL_NO_ENGINE
-#  define ASIO_SSL_DETAIL_OPENSSL_TYPES_HPP
-#  include "openssl_stub.hpp"
-
-# elif CONFIG_ASIO_USE_ESP_WOLFSSL
-#  define ASIO_USE_WOLFSSL
-# endif   // CONFIG_ASIO_USE_ESP_OPENSSL
 
 #endif // _ESP_ASIO_CONFIG_H_

components/asio/port/include/esp_exception.h

@@ -1,31 +0,0 @@
-/*
- * SPDX-FileCopyrightText: 2018-2021 Espressif Systems (Shanghai) CO LTD
- *
- * SPDX-License-Identifier: Apache-2.0
- */
-#ifndef _ESP_EXCEPTION_H_
-#define _ESP_EXCEPTION_H_
-
-//
-// This exception stub is enabled only if exceptions are disabled in menuconfig
-//
-#if !defined(CONFIG_COMPILER_CXX_EXCEPTIONS) && defined (ASIO_NO_EXCEPTIONS)
-
-#include "esp_log.h"
-
-//
-// asio exception stub
-//
-namespace asio {
-namespace detail {
-template <typename Exception>
-void throw_exception(const Exception &e)
-{
-    ESP_LOGE("esp32_asio_exception", "Caught exception: %s!", e.what());
-    abort();
-}
-}
-}
-#endif // CONFIG_COMPILER_CXX_EXCEPTIONS==1 && defined (ASIO_NO_EXCEPTIONS)
-
-#endif // _ESP_EXCEPTION_H_

components/console_simple_init/CMakeLists.txt

@@ -0,0 +1,3 @@
+idf_component_register(SRCS "console_simple_init.c"
+                    INCLUDE_DIRS "."
+                    PRIV_REQUIRES console)

components/console_simple_init/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

components/console_simple_init/README.md

@@ -0,0 +1,37 @@
+# Simple Console Initializer
+The component provides a simple api's to initialize and start the esp console.
+It also provides an api to register an user provided command.
+
+## API
+
+### Steps to enable console in an example code:
+1. Add this component to your project using ```idf.py add-dependency``` command.
+2. In the main file of the example, add the following line:
+    ```c
+    #include "console_simple_init.h"
+    ```
+3. Ensure NVS flash is initialized and default event loop is created in your app_main():
+    ```c
+    ESP_ERROR_CHECK(esp_event_loop_create_default());
+    esp_err_t ret = nvs_flash_init();   //Initialize NVS
+    if (ret == ESP_ERR_NVS_NO_FREE_PAGES || ret == ESP_ERR_NVS_NEW_VERSION_FOUND) {
+        ESP_ERROR_CHECK(nvs_flash_erase());
+        ret = nvs_flash_init();
+    }
+    ESP_ERROR_CHECK(ret);
+    ```
+4. In your app_main() function, add the following line as the last line:
+    ```c
+    ESP_ERROR_CHECK(console_cmd_init());     // Initialize console
+
+    // Define the function prototype for do_user_cmd
+    // It's a function that takes an integer (argc) and a pointer to a pointer to char (argv)
+    int do_user_cmd(int argc, char **argv);
+
+    // Register the do_user_cmd function as a command callback function for "user" command
+    // This allows you to execute the do_user_cmd function when the "user" command is invoked
+    ESP_ERROR_CHECK(console_cmd_user_register("user", do_user_cmd));
+
+
+    ESP_ERROR_CHECK(console_cmd_start());    // Start console
+    ```

components/console_simple_init/console_simple_init.c

@@ -0,0 +1,79 @@
+/*
+ * SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+#include <stdio.h>
+#include "esp_console.h"
+#include "esp_log.h"
+#include "console_simple_init.h"
+
+static esp_console_repl_t *repl = NULL;
+static const char *TAG = "console_simple_init";
+
+/**
+ * @brief Initializes the esp console
+ * @return
+ *          - esp_err_t
+ */
+esp_err_t console_cmd_init(void)
+{
+    esp_console_repl_config_t repl_config = ESP_CONSOLE_REPL_CONFIG_DEFAULT();
+
+    // install console REPL environment
+#if defined(CONFIG_ESP_CONSOLE_UART_DEFAULT) || defined(CONFIG_ESP_CONSOLE_UART_CUSTOM)
+    esp_console_dev_uart_config_t hw_config = ESP_CONSOLE_DEV_UART_CONFIG_DEFAULT();
+    return esp_console_new_repl_uart(&hw_config, &repl_config, &repl);
+
+#elif defined(CONFIG_ESP_CONSOLE_USB_CDC)
+    esp_console_dev_usb_cdc_config_t hw_config = ESP_CONSOLE_DEV_CDC_CONFIG_DEFAULT();
+    return esp_console_new_repl_usb_cdc(&hw_config, &repl_config, &repl);
+
+#elif defined(CONFIG_ESP_CONSOLE_USB_SERIAL_JTAG)
+    esp_console_dev_usb_serial_jtag_config_t hw_config = ESP_CONSOLE_DEV_USB_SERIAL_JTAG_CONFIG_DEFAULT();
+    return esp_console_new_repl_usb_serial_jtag(&hw_config, &repl_config, &repl);
+
+#else
+#error Unsupported console type
+#endif
+
+}
+
+/**
+ * @brief Initialize Ethernet driver based on Espressif IoT Development Framework Configuration
+ *
+ * @param[in] cmd string that is the user defined command
+ * @param[in] do_user_cmd Function pointer for a user-defined command callback function
+ *
+ * @return
+ *          - esp_err_t
+ */
+esp_err_t console_cmd_user_register(char *cmd, esp_console_cmd_func_t do_user_cmd)
+{
+    esp_err_t ret;
+
+    const esp_console_cmd_t user_cmd = {
+        .command = cmd,
+        .help = "User defined command",
+        .hint = NULL,
+        .func = do_user_cmd,
+    };
+
+    ret = esp_console_cmd_register(&user_cmd);
+    if (ret) {
+        ESP_LOGE(TAG, "Unable to register user cmd");
+    }
+
+    return ret;
+}
+
+/**
+ * @brief Starts the esp console
+ * @return
+ *          - esp_err_t
+ */
+esp_err_t console_cmd_start(void)
+{
+    // start console REPL
+    return esp_console_start_repl(repl);
+}

components/console_simple_init/console_simple_init.h

@@ -0,0 +1,43 @@
+/*
+ * SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#pragma once
+
+#include "esp_err.h"
+#include "esp_console.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * @brief Initializes the esp console
+ * @return
+ *          - esp_err_t
+ */
+esp_err_t console_cmd_init(void);
+
+/**
+ * @brief Initialize Ethernet driver based on Espressif IoT Development Framework Configuration
+ *
+ * @param[in] cmd string that is the user defined command
+ * @param[in] do_user_cmd Function pointer for a user-defined command callback function
+ *
+ * @return
+ *          - esp_err_t
+ */
+esp_err_t console_cmd_user_register(char *user_cmd, esp_console_cmd_func_t do_user_cmd);
+
+/**
+ * @brief Starts the esp console
+ * @return
+ *          - esp_err_t
+ */
+esp_err_t console_cmd_start(void);
+
+#ifdef __cplusplus
+}
+#endif

components/console_simple_init/examples/console_basic/CMakeLists.txt

@@ -0,0 +1,8 @@
+# For more information about build system see
+# https://docs.espressif.com/projects/esp-idf/en/latest/api-guides/build-system.html
+# The following five lines of boilerplate have to be in your project's
+# CMakeLists in this exact order for cmake to work correctly
+cmake_minimum_required(VERSION 3.16)
+
+include($ENV{IDF_PATH}/tools/cmake/project.cmake)
+project(console_basic)

components/console_simple_init/examples/console_basic/main/CMakeLists.txt

@@ -0,0 +1,2 @@
+idf_component_register(SRCS "console_basic.c"
+                    INCLUDE_DIRS ".")

components/console_simple_init/examples/console_basic/main/console_basic.c

@@ -0,0 +1,36 @@
+/*
+ * SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Unlicense OR CC0-1.0
+ */
+#include <stdio.h>
+#include "nvs_flash.h"
+#include "esp_event.h"
+#include "console_simple_init.h"
+
+int do_user_cmd(int argc, char **argv)
+{
+    printf("Hello from user command\n");
+    return 0;
+}
+
+void app_main(void)
+{
+    ESP_ERROR_CHECK(esp_event_loop_create_default());
+    esp_err_t ret = nvs_flash_init();   //Initialize NVS
+    if (ret == ESP_ERR_NVS_NO_FREE_PAGES || ret == ESP_ERR_NVS_NEW_VERSION_FOUND) {
+        ESP_ERROR_CHECK(nvs_flash_erase());
+        ret = nvs_flash_init();
+    }
+    ESP_ERROR_CHECK(ret);
+
+    // Initialize console REPL
+    ESP_ERROR_CHECK(console_cmd_init());
+
+    // Register user command
+    ESP_ERROR_CHECK(console_cmd_user_register("user", do_user_cmd));
+
+    // start console REPL
+    ESP_ERROR_CHECK(console_cmd_start());
+
+}

components/console_simple_init/examples/console_basic/main/idf_component.yml

@@ -0,0 +1,6 @@
+dependencies:
+  idf:
+    version: '*'
+  console_simple_init:
+    version: "*"
+    override_path: '../../../'

components/console_simple_init/examples/console_basic/pytest_console_basic.py

@@ -0,0 +1,13 @@
+# SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Unlicense OR CC0-1.0
+
+# -*- coding: utf-8 -*-
+import pytest
+
+
+@pytest.mark.esp32
+def test_examples_user_command(dut):
+    dut.expect('esp>', timeout=30)
+    dut.write('user')
+    dut.expect('Hello from user command', timeout=30)
+    pass

components/console_simple_init/idf_component.yml

@@ -0,0 +1,6 @@
+version: 1.0.0
+url: https://github.com/espressif/esp-protocols/tree/master/components/console_simple_init
+description: The component provides helper functions for easy initialization and start of esp console.
+dependencies:
+  idf:
+    version: '>=5.0'

components/esp_modem/.cz.yaml

@@ -3,6 +3,6 @@ commitizen:
   bump_message: 'bump(modem): $current_version -> $new_version'
   pre_bump_hooks: python ../../ci/changelog.py esp_modem
   tag_format: modem-v$version
-  version: 1.0.0
+  version: 1.0.3
   version_files:
   - idf_component.yml

components/esp_modem/CHANGELOG.md

@@ -1,5 +1,44 @@
 # Changelog
 
+## [1.0.3](https://github.com/espressif/esp-protocols/commits/modem-v1.0.3)
+
+### Bug Fixes
+
+- Fix to allow MSC frame (SIM800 CMUX) after v1.0.2 ([8d5947e](https://github.com/espressif/esp-protocols/commit/8d5947e), [#366](https://github.com/espressif/esp-protocols/issues/366))
+- Add CMUX example to target tests ([4f2ebaa](https://github.com/espressif/esp-protocols/commit/4f2ebaa))
+
+## [1.0.2](https://github.com/espressif/esp-protocols/commits/modem-v1.0.2)
+
+### Features
+
+- Add factory method for simple creation of custom DCEs ([4cf9e50](https://github.com/espressif/esp-protocols/commit/4cf9e50))
+- Support custom transport in AT TCP client example ([ae629ed](https://github.com/espressif/esp-protocols/commit/ae629ed))
+
+### Bug Fixes
+
+- Fix host test race with async read and d'structor ([c89d42f](https://github.com/espressif/esp-protocols/commit/c89d42f))
+- More error handling in cmux protocol ([8edbac6](https://github.com/espressif/esp-protocols/commit/8edbac6))
+- Fix netif to set PPP config in C++ way ([5287432](https://github.com/espressif/esp-protocols/commit/5287432))
+- Fix vfs terminal not to reset read_cb() automatically ([25a35e2](https://github.com/espressif/esp-protocols/commit/25a35e2))
+- Fix netif data race causing PPP startup delays ([c8c0507](https://github.com/espressif/esp-protocols/commit/c8c0507), [#308](https://github.com/espressif/esp-protocols/issues/308))
+- Added support for inflatable buffer ([cb6e03a](https://github.com/espressif/esp-protocols/commit/cb6e03a), [#272](https://github.com/espressif/esp-protocols/issues/272))
+- Fix LoadProhibited after failed CMUX initialization (IDFGH-10845) ([60c87dd](https://github.com/espressif/esp-protocols/commit/60c87dd))
+
+## [1.0.1](https://github.com/espressif/esp-protocols/commits/modem-v1.0.1)
+
+### Bug Fixes
+
+- Support UART clean build with IDF v5.2 ([e620eb5](https://github.com/espressif/esp-protocols/commit/e620eb5))
+- enable code coverage publishing to github pages ([4910e89](https://github.com/espressif/esp-protocols/commit/4910e89))
+- fix esp_modem build issue ([ab94566](https://github.com/espressif/esp-protocols/commit/ab94566))
+- Example to use 1.0.0 ([afb6930](https://github.com/espressif/esp-protocols/commit/afb6930))
+- Changelog to correctly pick references ([423e965](https://github.com/espressif/esp-protocols/commit/423e965))
+
+### Updated
+
+- docs(esp_modem): updated documents to show missed topics ([0534853](https://github.com/espressif/esp-protocols/commit/0534853))
+- docs(common): improving documentation ([ca3fce0](https://github.com/espressif/esp-protocols/commit/ca3fce0))
+
 ## [1.0.0](https://github.com/espressif/esp-protocols/commits/modem-v1.0.0)
 
 ### Major changes

components/esp_modem/CMakeLists.txt

@@ -47,15 +47,3 @@ if(${target} STREQUAL "linux")
     # This is needed for ESP_LOGx() macros, as integer formats differ on ESP32(..) and x64
     set_target_properties(${COMPONENT_LIB} PROPERTIES COMPILE_FLAGS -Wno-format)
 endif()
-
-if(CONFIG_GCOV_ENABLED)
-    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} --coverage -fprofile-arcs -ftest-coverage")
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} --coverage -fprofile-arcs -ftest-coverage")
-
-
-    MARK_AS_ADVANCED(
-        CMAKE_CXX_FLAGS_COVERAGE
-        CMAKE_C_FLAGS_COVERAGE
-        CMAKE_EXE_LINKER_FLAGS_COVERAGE
-        CMAKE_SHARED_LINKER_FLAGS_COVERAGE )
-endif()

components/esp_modem/Kconfig

@@ -16,6 +16,17 @@ menu "esp-modem"
             in command mode might come fragmented in rare cases so might need to retry
             AT commands.
 
+    config ESP_MODEM_USE_INFLATABLE_BUFFER_IF_NEEDED
+        bool "Use inflatable buffer in DCE"
+        default n
+        help
+            If enabled we will process the ongoing AT command by growing the current
+            buffer (if we've run out the preconfigured buffer).
+            If disabled, we simply report a failure.
+            Use this if additional allocation is not a problem and you need to reliably process
+            all commands, usually with sporadically longer responses than the configured buffer.
+            Could be also used to defragment AT replies in CMUX mode if CMUX_DEFRAGMENT_PAYLOAD=n
+
     config ESP_MODEM_CMUX_DELAY_AFTER_DLCI_SETUP
         int "Delay in ms to wait before creating another virtual terminal"
         default 0
@@ -25,4 +36,13 @@ menu "esp-modem"
             The typical reason for failing SABM request without a delay is that
             some devices (SIM800) send MSC requests just after opening a new DLCI.
 
+    config ESP_MODEM_CMUX_USE_SHORT_PAYLOADS_ONLY
+        bool "CMUX to support only short payloads (<128 bytes)"
+        default n
+        help
+            If enabled, the CMUX protocol would only use 1 byte size field.
+            You can use this option for devices that support only short CMUX payloads
+            to make the protocol more robust on noisy environments or when underlying
+            transport gets corrupted often (for example by Rx buffer overflows)
+
 endmenu

components/esp_modem/README.md

@@ -17,5 +17,5 @@ Get started with one of the examples:
 
 ## Documentation
 
-* Continue with esp-modem [brief overview](docs/README.md)
+* Continue with esp-modem [brief overview](https://github.com/espressif/esp-protocols/tree/master/docs/esp_modem/en/README.rst)
-* View the full [html documentation](https://espressif.github.io/esp-protocols/esp_modem/index.html)
+* View the full [html documentation](https://docs.espressif.com/projects/esp-protocols/esp_modem/docs/latest/index.html)

components/esp_modem/docs/README.md

@@ -1,92 +0,0 @@
-# ESP MODEM
-
-This component is used to communicate with modems in the command mode (using AT commands), as well as the data mode
-(over PPPoS protocol).
-The modem device is modeled with a DCE (Data Communication Equipment) object, which is composed of:
-* DTE (Data Terminal Equipment), which abstracts the terminal (currently only UART implemented).
-* PPP Netif representing a network interface communicating with the DTE using PPP protocol.
-* Module abstracting the specific device model and its commands.
-
-```
-   +-----+
-   | DTE |--+
-   +-----+  |   +-------+
-            +-->|   DCE |
-   +-------+    |       |o--- set_mode(command/data)
-   | Module|--->|       |
-   +-------+    |       |o--- send_commands
-             +->|       |
-   +------+  |  +-------+
-   | PPP  |--+
-   | netif|------------------> network events
-   +------+
-```
-
-## Modem components
-### DCE
-
-This is the basic operational unit of the esp_modem component, abstracting a specific module in software,
-which is basically configured by
-* the I/O communication media (UART), defined by the DTE configuration
-* the specific command library supported by the device model, defined with the module type
-* network interface configuration (PPPoS config in lwip)
-
-After the object is created, the application interaction with the DCE is in
-* issuing specific commands to the modem
-* switching between data and command mode
-
-### DTE
-Is an abstraction of the physical interface connected to the modem. Current implementation supports only UART
-
-### PPP netif
-
-Is used to attach the specific network interface to a network communication protocol used by the modem. Currently implementation supports only PPPoS protocol.
-
-### Module
-
-Abstraction of the specific modem device. Currently the component supports SIM800, BG96, SIM7600.
-
-## Use cases
-
-Users interact with the esp-modem using the DCE's interface, to basically
-* Switch between command and data mode to connect to the internet via cellular network.
-* Send various commands to the device (e.g. send SMS)
-
-The applications typically register handlers for network events to receive notification on the network availability and
-IP address changes.
-
-Common use cases of the esp-modem are also listed as the examples:
-* `examples/pppos_client` -- simple client which reads some module properties and switches to the data mode to connect to a public mqtt broker.
-* `examples/modem_console` -- is an example to exercise all possible module commands in a console application.
-* `examples/ap_to_pppos` -- this example focuses on the network connectivity of the esp-modem and provides a WiFi AP that forwards packets (and uses NAT) to and from the PPPoS connection.
-
-## Extensibility
-
-### CMUX
-
-Implementation of virtual terminals is an experimental feature, which allows users to also issue commands in the data mode,
-after creating multiple virtual terminals, designating some of them solely to data mode, others solely to command mode.
-
-### DTE's
-
-Currently, we support only UART (and USB as a preview feature), but modern modules support other communication interfaces, such as USB, SPI.
-
-### Other devices
-
-Adding a new device is a must-have requirement for the esp-modem component. Different modules support different commands,
-or some commands might have a different implementation. Adding a new device means to provide a new implementation
-as a class derived from `GenericModule`, where we could add new commands or modify the existing ones.
-
-## Configuration
-
-Modem abstraction is configurable both compile-time and run-time.
-
-### Component Kconfig
-
-Compile-time configuration is provided using menuconfig. Please check the description for the CMUX mode configuration options.
-
-### Runtime configuration
-
-Is defined using standard configuration structures for `DTE` and `DCE` objects separately. Please find documentation of
-* :cpp:class:`esp_modem_dte_config_t`
-* :cpp:class:`esp_modem_dce_config_t`

components/esp_modem/docs/generate_docs

@@ -1,23 +0,0 @@
-# Cleanup the generated html
-rm -rf html
-
-# Cleans example and test build dirs (to reduce the component size before upload)
-rm -rf ../examples/ap_to_pppos/build/ ../examples/simple_cmux_client/build/ ../examples/pppos_client/build/ ../examples/linux_modem/build/ ../examples/modem_console/build ../test/host_test/build/ ../test/target/build/
-
-# Generate C++ API header of the DCE
-cat ../include/generate/esp_modem_command_declare.inc | clang++ -E -P -CC  -xc++ -I../include -DGENERATE_DOCS  - | sed -n '1,/DCE command documentation/!p' > esp_modem_dce.hpp
-
-# Generate C API header of the modem_api.h
-cat ../include/generate/esp_modem_command_declare.inc | clang -E -P -CC  -xc -I../include -DGENERATE_DOCS  - | sed -n '1,/DCE command documentation/!p' > esp_modem_api_commands.h
-
-# RST with links to C++ API
-cat ../include/generate/esp_modem_command_declare.inc | clang -E -P  -xc -I../include -DGENERATE_DOCS -DGENERATE_RST_LINKS - | sed 's/NL/\n/g' > cxx_api_links.rst
-
-# Run doxygen
-doxygen
-
-# Generate the docs
-python -u -m sphinx.cmd.build -b html  . html
-
-# Cleanup the doxygen xml's and temporary headers
-rm -rf xml esp_modem_api_commands.h esp_modem_dce.hpp cxx_api_links.rst

components/esp_modem/docs/internal_design.md

@@ -1,36 +0,0 @@
-# Internal design
-
-## Design decisions
-
-* Use C++ with additional C API
-
-* Use exceptions
-  - Use macro wrapper over `try-catch` blocks when exceptions off (use `abort()` if `THROW()`)
-
-* Initializes and allocates in the constructor (might throw)
-  - easier code with exceptions ON, with exceptions OFF alloc/init failures are not treated as runtime error (program aborts)
-  - break down long initialization in constructor into more private methods
-
-* Implements different devices using inheritance from `GenericModule`, which is the most general implementation of a common modem
-  - Internally uses templates with device specialization (modeled as `DCE<SpecificModule>`) which could be used as well for some special cases,
-  such as implantation of a minimal device (ModuleIf), add new AT commands (oOnly in compile time), or using the Module with DTE only (no DCE, no Netif) for sending AT commands without network
-
-## DCE collaboration model
-
-The diagram describes how the DCE class collaborates with DTE, PPP and the device abstraction
-
-![DCE_architecture](DCE_DTE_collaboration.png)
-
-## Terminal inheritance
-
-Terminal is a class which can read or write data, and can handle callbacks when data are available. UART specialization
-is provided implementing these method using the uart driver.
-
-## CMUX terminal
-
-The below diagram depicts the idea of using CMUX terminal mode using the CMuxInstance class which is a terminal
-(it implements the basic read/write methods) interfacing arbitrary number of virtual terminals,
-but at the same time it is also composed of CMux class, which consumes the original terminal and uses its read/write methods
-to multiplex the terminal.
-
-![CMUX Terminal](CMux_collaboration.png)

components/esp_modem/examples/modem_console/main/CMakeLists.txt

@@ -5,4 +5,3 @@ idf_component_register(SRCS "modem_console_main.cpp"
                             "ping_handle.c"
                        REQUIRES console esp_http_client nvs_flash
                        INCLUDE_DIRS ".")
-target_compile_options(${COMPONENT_LIB} PRIVATE "-Wno-format")

components/esp_modem/examples/modem_console/main/modem_console_main.cpp

@@ -173,7 +173,7 @@ extern "C" void app_main(void)
         ESP_LOGI(TAG, "Waiting for USB device connection...");
         auto dte = create_usb_dte(&dte_config);
         dte->set_error_cb([&](terminal_error err) {
-            ESP_LOGI(TAG, "error handler %d", err);
+            ESP_LOGI(TAG, "error handler %d", (int)err);
             if (err == terminal_error::DEVICE_GONE) {
                 exit_signal.set(1);
             }
@@ -482,7 +482,7 @@ extern "C" void app_main(void)
     // wait for exit
     exit_signal.wait_any(1, UINT32_MAX);
     s_repl->del(s_repl);
-    ESP_LOGI(TAG, "Exiting...%d", esp_get_free_heap_size());
+    ESP_LOGI(TAG, "Exiting...%" PRIu32, esp_get_free_heap_size());
 #if defined(CONFIG_EXAMPLE_SERIAL_CONFIG_USB)
     // USB example runs in a loop to demonstrate hot-plugging and sudden disconnection features.
 } // while (1)

components/esp_modem/examples/modem_console/main/ping_handle.c

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD
+ * SPDX-FileCopyrightText: 2022-2023 Espressif Systems (Shanghai) CO LTD
  *
  * SPDX-License-Identifier: Unlicense OR CC0-1.0
  */
@@ -28,7 +28,7 @@ static void cmd_ping_on_ping_success(esp_ping_handle_t hdl, void *args)
     esp_ping_get_profile(hdl, ESP_PING_PROF_IPADDR, &target_addr, sizeof(target_addr));
     esp_ping_get_profile(hdl, ESP_PING_PROF_SIZE, &recv_len, sizeof(recv_len));
     esp_ping_get_profile(hdl, ESP_PING_PROF_TIMEGAP, &elapsed_time, sizeof(elapsed_time));
-    ESP_LOGI(TAG, "%d bytes from %s icmp_seq=%d ttl=%d time=%d ms\n",
+    ESP_LOGI(TAG, "%" PRIu32 " bytes from %s icmp_seq=%d ttl=%d time=%" PRIu32 " ms\n",
              recv_len, inet_ntoa(target_addr.u_addr.ip4), seqno, ttl, elapsed_time);
 }
 
@@ -57,7 +57,7 @@ static void cmd_ping_on_ping_end(esp_ping_handle_t hdl, void *args)
     } else {
         ESP_LOGI(TAG, "\n--- %s ping statistics ---\n", inet6_ntoa(*ip_2_ip6(&target_addr)));
     }
-    ESP_LOGI(TAG, "%d packets transmitted, %d received, %d%% packet loss, time %dms\n",
+    ESP_LOGI(TAG, "%" PRIu32 " packets transmitted, %" PRIu32 " received, %" PRIu32 " packet loss, time %" PRIu32 "ms\n",
              transmitted, received, loss, total_time_ms);
     // delete the ping sessions, so that we clean up all resources and can create a new ping session
     // we don't have to call delete function in the callback, instead we can call delete function from other tasks

components/esp_modem/examples/modem_psm/main/CMakeLists.txt

@@ -1,3 +1,2 @@
-
 idf_component_register(SRCS "modem_psm.c"
                     INCLUDE_DIRS ".")

components/esp_modem/examples/modem_tcp_client/CMakeLists.txt

@@ -1,7 +1,6 @@
 # The following lines of boilerplate have to be in your project's CMakeLists
 # in this exact order for cmake to work correctly
 cmake_minimum_required(VERSION 3.8)
-set(CMAKE_CXX_STANDARD 17)
 
 set(EXTRA_COMPONENT_DIRS "../..")
 

components/esp_modem/examples/modem_tcp_client/README.md

@@ -5,6 +5,10 @@
 ## Overview
 This example demonstrates how to act as a MQTT client using modem's TCP commands (provided, the device supports "socket" related commands)
 
+This example could be used in two different configurations:
+1) Custom TCP transport: Implements a TCP transport in form of AT commands and uses it as custom transport for mqtt client.
+2) Localhost listener: Uses standard transports to connect and forwards socket layer data from the client to the modem using AT commands.
+
 ### Supported IDF versions
 
-This example is supported from IDF `v4.4`.
+This example is supported from IDF `v5.0`.

components/esp_modem/examples/modem_tcp_client/components/extra_tcp_transports/CMakeLists.txt

@@ -0,0 +1,4 @@
+idf_component_register(SRCS mbedtls_wrap.cpp
+                            tls_transport.cpp
+                       INCLUDE_DIRS include
+                       REQUIRES tcp_transport)

components/esp_modem/examples/modem_tcp_client/components/extra_tcp_transports/README.md

@@ -0,0 +1,9 @@
+# Custom transports
+
+This component is a placeholder of custom transports. It contains mbedTLS cxx wrapper which could be used to create a custom TLS layer on any kind of IO function.
+
+# List of Transports
+
+## TLS Transport
+
+TLS layer on top of any custom transport. Very similar to `ssl_transport` (standard IDF transport), but this is customizable and could work on any kind of transport, not only  the BSD socket based tcp transport (like `ssl_transport`).

components/esp_modem/examples/modem_tcp_client/components/extra_tcp_transports/include/mbedtls_wrap.hpp

@@ -0,0 +1,52 @@
+/*
+ * SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+#pragma once
+
+#include <utility>
+#include <span>
+#include "mbedtls/ssl.h"
+#include "mbedtls/entropy.h"
+#include "mbedtls/ctr_drbg.h"
+#include "mbedtls/error.h"
+
+using  const_buf = std::span<const unsigned char>;
+
+class Tls {
+public:
+    enum class is_server : bool {};
+    enum class do_verify : bool {};
+
+    Tls();
+    virtual ~Tls();
+    bool init(is_server server, do_verify verify);
+    int handshake();
+    int write(const unsigned char *buf, size_t len);
+    int read(unsigned char *buf, size_t len);
+    [[nodiscard]] bool set_own_cert(const_buf crt, const_buf key);
+    [[nodiscard]] bool set_ca_cert(const_buf crt);
+    virtual int send(const unsigned char *buf, size_t len) = 0;
+    virtual int recv(unsigned char *buf, size_t len) = 0;
+    size_t get_available_bytes();
+
+protected:
+    mbedtls_ssl_context ssl_{};
+    mbedtls_x509_crt public_cert_{};
+    mbedtls_pk_context pk_key_{};
+    mbedtls_x509_crt ca_cert_{};
+    mbedtls_ssl_config conf_{};
+    mbedtls_ctr_drbg_context ctr_drbg_{};
+    mbedtls_entropy_context entropy_{};
+    virtual void delay() {}
+
+private:
+    static void print_error(const char *function, int error_code);
+    static int bio_write(void *ctx, const unsigned char *buf, size_t len);
+    static int bio_read(void *ctx, unsigned char *buf, size_t len);
+    int mbedtls_pk_parse_key( mbedtls_pk_context *ctx,
+                              const unsigned char *key, size_t keylen,
+                              const unsigned char *pwd, size_t pwdlen);
+
+};

components/esp_modem/examples/modem_tcp_client/components/extra_tcp_transports/include/tcp_transport_mbedtls.h

@@ -0,0 +1,16 @@
+/*
+ * SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+#pragma once
+
+#include "esp_transport.h"
+
+/**
+ * @brief Initializes TLS transport based on mbetls wrapper
+ *
+ * @param parent Transport on top of which we run TLS layer
+ * @return Transport handle on success
+ */
+esp_transport_handle_t esp_transport_tls_init(esp_transport_handle_t parent);

components/esp_modem/examples/modem_tcp_client/components/extra_tcp_transports/mbedtls_wrap.cpp

@@ -0,0 +1,134 @@
+/*
+ * SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+#include "mbedtls/ctr_drbg.h"
+#include "mbedtls/ssl.h"
+#include "mbedtls_wrap.hpp"
+
+bool Tls::init(is_server server, do_verify verify)
+{
+    const char pers[] = "mbedtls_wrapper";
+    mbedtls_entropy_init(&entropy_);
+    mbedtls_ctr_drbg_seed(&ctr_drbg_, mbedtls_entropy_func, &entropy_, (const unsigned char *)pers, sizeof(pers));
+    int ret = mbedtls_ssl_config_defaults(&conf_, server == is_server{true} ? MBEDTLS_SSL_IS_SERVER : MBEDTLS_SSL_IS_CLIENT, MBEDTLS_SSL_TRANSPORT_STREAM, MBEDTLS_SSL_PRESET_DEFAULT);
+    if (ret) {
+        print_error("mbedtls_ssl_config_defaults", ret);
+        return false;
+    }
+    mbedtls_ssl_conf_rng(&conf_, mbedtls_ctr_drbg_random, &ctr_drbg_);
+    mbedtls_ssl_conf_authmode(&conf_, verify == do_verify{true} ? MBEDTLS_SSL_VERIFY_REQUIRED : MBEDTLS_SSL_VERIFY_NONE);
+    ret = mbedtls_ssl_conf_own_cert(&conf_, &public_cert_, &pk_key_);
+    if (ret) {
+        print_error("mbedtls_ssl_conf_own_cert", ret);
+        return false;
+    }
+    if (verify == do_verify{true}) {
+        mbedtls_ssl_conf_ca_chain(&conf_, &ca_cert_, nullptr);
+    }
+    ret = mbedtls_ssl_setup(&ssl_, &conf_);
+    if (ret) {
+        print_error("mbedtls_ssl_setup", ret);
+        return false;
+    }
+    return true;
+}
+
+void Tls::print_error(const char *function, int error_code)
+{
+    static char error_buf[100];
+    mbedtls_strerror(error_code, error_buf, sizeof(error_buf));
+
+    printf("%s() returned -0x%04X\n", function, -error_code);
+    printf("-0x%04X: %s\n", -error_code, error_buf);
+}
+
+int Tls::handshake()
+{
+    int ret = 0;
+    mbedtls_ssl_set_bio(&ssl_, this, bio_write, bio_read, nullptr);
+
+    while ( ( ret = mbedtls_ssl_handshake( &ssl_ ) ) != 0 ) {
+        if ( ret != MBEDTLS_ERR_SSL_WANT_READ && ret != MBEDTLS_ERR_SSL_WANT_WRITE ) {
+            print_error( "mbedtls_ssl_handshake returned", ret );
+            return -1;
+        }
+        delay();
+    }
+    return ret;
+}
+
+int Tls::bio_write(void *ctx, const unsigned char *buf, size_t len)
+{
+    auto s = static_cast<Tls *>(ctx);
+    return s->send(buf, len);
+}
+
+int Tls::bio_read(void *ctx, unsigned char *buf, size_t len)
+{
+    auto s = static_cast<Tls *>(ctx);
+    return s->recv(buf, len);
+}
+
+int Tls::write(const unsigned char *buf, size_t len)
+{
+    return mbedtls_ssl_write( &ssl_, buf, len );
+}
+
+int Tls::read(unsigned char *buf, size_t len)
+{
+    return mbedtls_ssl_read( &ssl_, buf, len );
+}
+
+bool Tls::set_own_cert(const_buf crt, const_buf key)
+{
+    int ret = mbedtls_x509_crt_parse(&public_cert_, crt.data(), crt.size());
+    if (ret < 0) {
+        print_error("mbedtls_x509_crt_parse", ret);
+        return false;
+    }
+    ret = mbedtls_pk_parse_key(&pk_key_, key.data(), key.size(), nullptr, 0);
+    if (ret < 0) {
+        print_error("mbedtls_pk_parse_keyfile", ret);
+        return false;
+    }
+    return true;
+}
+
+bool Tls::set_ca_cert(const_buf crt)
+{
+    int ret = mbedtls_x509_crt_parse(&ca_cert_, crt.data(), crt.size());
+    if (ret < 0) {
+        print_error("mbedtls_x509_crt_parse", ret);
+        return false;
+    }
+    return true;
+}
+
+Tls::Tls()
+{
+    mbedtls_x509_crt_init(&public_cert_);
+    mbedtls_pk_init(&pk_key_);
+    mbedtls_x509_crt_init(&ca_cert_);
+}
+
+int Tls::mbedtls_pk_parse_key(mbedtls_pk_context *ctx, const unsigned char *key, size_t keylen, const unsigned char *pwd, size_t pwdlen)
+{
+
+    return ::mbedtls_pk_parse_key(ctx, key, keylen, pwd, pwdlen, nullptr, nullptr);
+}
+
+size_t Tls::get_available_bytes()
+{
+    return ::mbedtls_ssl_get_bytes_avail(&ssl_);
+}
+
+Tls::~Tls()
+{
+    ::mbedtls_ssl_config_free(&conf_);
+    ::mbedtls_ssl_free(&ssl_);
+    ::mbedtls_pk_free(&pk_key_);
+    ::mbedtls_x509_crt_free(&public_cert_);
+    ::mbedtls_x509_crt_free(&ca_cert_);
+}

components/esp_modem/examples/modem_tcp_client/components/extra_tcp_transports/tls_transport.cpp

@@ -0,0 +1,137 @@
+/*
+ * SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+#include "freertos/FreeRTOS.h"
+#include "freertos/task.h"
+#include "esp_log.h"
+#include "esp_transport.h"
+#include "mbedtls_wrap.hpp"
+
+static const char *TAG = "tls_transport";
+
+class TlsTransport: public Tls {
+public:
+    explicit TlsTransport(esp_transport_handle_t parent) : Tls(), transport_(parent) {}
+    int send(const unsigned char *buf, size_t len) override;
+    int recv(unsigned char *buf, size_t len) override;
+    static bool set_func(esp_transport_handle_t tls_transport);
+
+private:
+    esp_transport_handle_t transport_{};
+    int connect(const char *host, int port, int timeout_ms);
+    void delay() override;
+
+    struct transport {
+        static int connect(esp_transport_handle_t t, const char *host, int port, int timeout_ms);
+        static int read(esp_transport_handle_t t, char *buffer, int len, int timeout_ms);
+        static int write(esp_transport_handle_t t, const char *buffer, int len, int timeout_ms);
+        static int close(esp_transport_handle_t t);
+        static int poll_read(esp_transport_handle_t t, int timeout_ms);
+        static int poll_write(esp_transport_handle_t t, int timeout_ms);
+        static int destroy(esp_transport_handle_t t);
+    };
+};
+
+esp_transport_handle_t esp_transport_tls_init(esp_transport_handle_t parent)
+{
+    esp_transport_handle_t ssl = esp_transport_init();
+    auto *tls = new TlsTransport(parent);
+    esp_transport_set_context_data(ssl, tls);
+    TlsTransport::set_func(ssl);
+    return ssl;
+}
+
+int TlsTransport::send(const unsigned char *buf, size_t len)
+{
+    return esp_transport_write(transport_, reinterpret_cast<const char *>(buf), len, 0);
+}
+
+int TlsTransport::recv(unsigned char *buf, size_t len)
+{
+    int ret = esp_transport_read(transport_, reinterpret_cast<char *>(buf), len, 0);
+
+    if (ret == ERR_TCP_TRANSPORT_CONNECTION_TIMEOUT) {
+        return MBEDTLS_ERR_SSL_WANT_READ;
+    }
+    return ret == ERR_TCP_TRANSPORT_CONNECTION_CLOSED_BY_FIN ? 0 : ret;
+}
+
+bool TlsTransport::set_func(esp_transport_handle_t tls_transport)
+{
+    return esp_transport_set_func(tls_transport, TlsTransport::transport::connect, TlsTransport::transport::read, TlsTransport::transport::write, TlsTransport::transport::close, TlsTransport::transport::poll_read, TlsTransport::transport::poll_write, TlsTransport::transport::destroy) == ESP_OK;
+}
+
+int TlsTransport::connect(const char *host, int port, int timeout_ms)
+{
+    return esp_transport_connect(transport_, host, port, timeout_ms);
+}
+
+void TlsTransport::delay()
+{
+    vTaskDelay(pdMS_TO_TICKS(500));
+}
+
+int TlsTransport::transport::connect(esp_transport_handle_t t, const char *host, int port, int timeout_ms)
+{
+    auto tls = static_cast<TlsTransport *>(esp_transport_get_context_data(t));
+    tls->init(is_server{false}, do_verify{false});
+
+    auto ret = tls->connect(host, port, timeout_ms);
+    if (ret < 0) {
+        return ret;
+    }
+    return tls->handshake();
+}
+
+int TlsTransport::transport::read(esp_transport_handle_t t, char *buffer, int len, int timeout_ms)
+{
+    auto tls = static_cast<TlsTransport *>(esp_transport_get_context_data(t));
+    if (tls->get_available_bytes() <= 0) {
+        int poll = esp_transport_poll_read(t, timeout_ms);
+        if (poll == -1) {
+            return ERR_TCP_TRANSPORT_CONNECTION_FAILED;
+        }
+        if (poll == 0) {
+            return ERR_TCP_TRANSPORT_CONNECTION_TIMEOUT;
+        }
+    }
+    return tls->read(reinterpret_cast<unsigned char *>(buffer), len);
+}
+
+int TlsTransport::transport::write(esp_transport_handle_t t, const char *buffer, int len, int timeout_ms)
+{
+    int poll;
+    if ((poll = esp_transport_poll_write(t, timeout_ms)) <= 0) {
+        ESP_LOGW(TAG, "Poll timeout or error timeout_ms=%d", timeout_ms);
+        return poll;
+    }
+
+    auto tls = static_cast<TlsTransport *>(esp_transport_get_context_data(t));
+    return tls->write(reinterpret_cast<const unsigned char *>(buffer), len);
+}
+
+int TlsTransport::transport::close(esp_transport_handle_t t)
+{
+    auto tls = static_cast<TlsTransport *>(esp_transport_get_context_data(t));
+    return esp_transport_close(tls->transport_);
+}
+
+int TlsTransport::transport::poll_read(esp_transport_handle_t t, int timeout_ms)
+{
+    auto tls = static_cast<TlsTransport *>(esp_transport_get_context_data(t));
+    return esp_transport_poll_read(tls->transport_, timeout_ms);
+}
+
+int TlsTransport::transport::poll_write(esp_transport_handle_t t, int timeout_ms)
+{
+    auto tls = static_cast<TlsTransport *>(esp_transport_get_context_data(t));
+    return esp_transport_poll_write(tls->transport_, timeout_ms);
+}
+
+int TlsTransport::transport::destroy(esp_transport_handle_t t)
+{
+    auto tls = static_cast<TlsTransport *>(esp_transport_get_context_data(t));
+    return esp_transport_destroy(tls->transport_);
+}

components/esp_modem/examples/modem_tcp_client/main/CMakeLists.txt

@@ -6,6 +6,6 @@ endif()
 
 idf_component_register(SRCS "modem_client.cpp"
                             "sock_dce.cpp"
+                            "tcp_transport_at.cpp"
                             "${device_srcs}"
                        INCLUDE_DIRS ".")
-target_compile_options(${COMPONENT_LIB} PRIVATE "-Wno-format")

components/esp_modem/examples/modem_tcp_client/main/Kconfig.projbuild

@@ -1,5 +1,10 @@
 menu "Example Configuration"
 
+    config EXAMPLE_CUSTOM_TCP_TRANSPORT
+        bool "Custom TCP transport"
+        help
+            Use custom TCP transport to connect to MQTT broker
+
     choice EXAMPLE_MODEM_DEVICE
         prompt "Choose supported modem device (DCE)"
         default EXAMPLE_MODEM_DEVICE_BG96

components/esp_modem/examples/modem_tcp_client/main/modem_client.cpp

@@ -15,14 +15,17 @@
 #include "freertos/FreeRTOS.h"
 #include "freertos/event_groups.h"
 #include "esp_netif.h"
-#include "esp_netif_ppp.h"
 #include "mqtt_client.h"
 #include "esp_modem_config.h"
 #include "cxx_include/esp_modem_api.hpp"
 #include "sock_dce.hpp"
 #include "esp_log.h"
+#include "tcp_transport_mbedtls.h"
+#include "tcp_transport_at.h"
 
 #define BROKER_URL "mqtt.eclipseprojects.io"
+#define BROKER_PORT 8883
+
 
 static const char *TAG = "modem_client";
 static EventGroupHandle_t event_group = NULL;
@@ -31,7 +34,7 @@ static const int GOT_DATA_BIT = BIT2;
 
 static void mqtt_event_handler(void *handler_args, esp_event_base_t base, int32_t event_id, void *event_data)
 {
-    ESP_LOGD(TAG, "Event dispatched from event loop base=%s, event_id=%d", base, event_id);
+    ESP_LOGD(TAG, "Event dispatched from event loop base=%s, event_id=%" PRId32, base, event_id);
     esp_mqtt_event_handle_t event = (esp_mqtt_event_handle_t)event_data;
     esp_mqtt_client_handle_t client = event->client;
     int msg_id;
@@ -70,6 +73,7 @@ static void mqtt_event_handler(void *handler_args, esp_event_base_t base, int32_
     }
 }
 
+
 extern "C" void app_main(void)
 {
 
@@ -101,24 +105,29 @@ extern "C" void app_main(void)
 
     /* create the DCE and initialize network manually (using AT commands) */
     auto dce = sock_dce::create(&dce_config, std::move(dte));
-    if (!dce->init_network()) {
+    if (!dce->init()) {
         ESP_LOGE(TAG,  "Failed to setup network");
         return;
     }
 
-    dce->init_sock(8883);
     esp_mqtt_client_config_t mqtt_config = {};
-#if ESP_IDF_VERSION >= ESP_IDF_VERSION_VAL(5, 0, 0)
+    mqtt_config.broker.address.port = BROKER_PORT;
-    mqtt_config.broker.address.uri = "mqtts://127.0.0.1";
     mqtt_config.session.message_retransmit_timeout = 10000;
+#ifndef CONFIG_EXAMPLE_CUSTOM_TCP_TRANSPORT
+    mqtt_config.broker.address.uri = "mqtts://127.0.0.1";
+    dce->start_listening(BROKER_PORT);
 #else
-    mqtt_config.uri = "mqtt://127.0.0.1";
+    mqtt_config.broker.address.uri = "mqtt://" BROKER_URL;
-    mqtt_config.message_retransmit_timeout = 10000;
+    esp_transport_handle_t at = esp_transport_at_init(dce.get());
+    esp_transport_handle_t ssl = esp_transport_tls_init(at);
+
+    mqtt_config.network.transport = ssl;
 #endif
     esp_mqtt_client_handle_t mqtt_client = esp_mqtt_client_init(&mqtt_config);
     esp_mqtt_client_register_event(mqtt_client, static_cast<esp_mqtt_event_id_t>(ESP_EVENT_ANY_ID), mqtt_event_handler, NULL);
     esp_mqtt_client_start(mqtt_client);
-    if (!dce->start(BROKER_URL, 8883)) {
+#ifndef CONFIG_EXAMPLE_CUSTOM_TCP_TRANSPORT
+    if (!dce->connect(BROKER_URL, BROKER_PORT)) {
         ESP_LOGE(TAG, "Failed to start DCE");
         return;
     }
@@ -128,13 +137,16 @@ extern "C" void app_main(void)
         }
         ESP_LOGE(TAG, "Loop exit.. retrying");
         // handle disconnections errors
-        if (!dce->init_network()) {
+        if (!dce->init()) {
             ESP_LOGE(TAG, "Failed to reinit network");
             return;
         }
-        if (!dce->start("test.mosquitto.org", 1883)) {
+        if (!dce->connect(BROKER_URL, BROKER_PORT)) {
             ESP_LOGI(TAG, "Network reinitialized, retrying");
         }
     }
+#else
+    vTaskDelay(portMAX_DELAY);
+#endif
 
 }

components/esp_modem/examples/modem_tcp_client/main/sock_commands.hpp

@@ -13,8 +13,6 @@
 
 namespace sock_commands {
 
-//using namespace esp_modem;
-
 #define ESP_MODEM_DECLARE_DCE_COMMAND(name, return_type, num, ...) \
         esp_modem::return_type name(esp_modem::CommandableIf *t, ## __VA_ARGS__);
 

components/esp_modem/examples/modem_tcp_client/main/sock_commands_bg96.cpp

@@ -6,7 +6,6 @@
 
 #include <charconv>
 #include <cstring>
-#include <sys/socket.h>
 #include "sock_commands.hpp"
 #include "cxx_include/esp_modem_command_library_utils.hpp"
 #include "sock_dce.hpp"
@@ -132,7 +131,7 @@ Responder::ret Responder::recv(uint8_t *data, size_t len)
     if (data_to_recv == 0) {
         const std::string_view head = "+QIRD: ";
         auto head_pos = std::search(recv_data, recv_data + len, head.begin(), head.end());
-        if (head_pos == nullptr) {
+        if (head_pos == recv_data + len) {
             return ret::FAIL;
         }
 
@@ -160,17 +159,17 @@ Responder::ret Responder::recv(uint8_t *data, size_t len)
         recv_data = next_nl + 1;
         auto first_data_len = len - (recv_data - (char *)data) /* minus size of the command marker */;
         if (actual_len > first_data_len) {
-            ::send(sock, recv_data, first_data_len, 0);
+            on_read(recv_data, first_data_len);
             data_to_recv = actual_len - first_data_len;
             return ret::NEED_MORE_DATA;
         }
-        ::send(sock, recv_data, actual_len, 0);
+        on_read(recv_data, actual_len);
     } else if (data_to_recv > len) {    // continue sending
-        ::send(sock, recv_data, len, 0);
+        on_read(recv_data, len);
         data_to_recv -= len;
         return ret::NEED_MORE_DATA;
     } else if (data_to_recv <= len) {    // last read -> looking for "OK" marker
-        ::send(sock, recv_data, data_to_recv, 0);
+        on_read(recv_data, data_to_recv);
         actual_len = data_to_recv;
     }
 

components/esp_modem/examples/modem_tcp_client/main/sock_commands_sim7600.cpp

@@ -6,7 +6,6 @@
 
 #include <charconv>
 #include <cstring>
-#include <sys/socket.h>
 #include "sock_commands.hpp"
 #include "cxx_include/esp_modem_command_library_utils.hpp"
 #include "sock_dce.hpp"
@@ -218,7 +217,7 @@ Responder::ret Responder::recv(uint8_t *data, size_t len)
     if (data_to_recv == 0) {
         static constexpr std::string_view head = "+CIPRXGET: 2,0,";
         auto head_pos = std::search(recv_data, recv_data + len, head.begin(), head.end());
-        if (head_pos == nullptr) {
+        if (head_pos == recv_data + len) {
             return ret::FAIL;
         }
 
@@ -246,7 +245,7 @@ Responder::ret Responder::recv(uint8_t *data, size_t len)
             ESP_LOGE(TAG, "TOO BIG");
             return ret::FAIL;
         }
-        size_t total_len = 0;
+        total_len = 0;
         if (std::from_chars(next_comma + 1, next_nl - 1, total_len).ec == std::errc::invalid_argument) {
             ESP_LOGE(TAG, "cannot convert");
             return ret::FAIL;
@@ -255,17 +254,17 @@ Responder::ret Responder::recv(uint8_t *data, size_t len)
         recv_data = next_nl + 1;
         auto first_data_len = len - (recv_data - (char *)data) /* minus size of the command marker */;
         if (actual_len > first_data_len) {
-            ::send(sock, recv_data, first_data_len, 0);
+            on_read(recv_data, first_data_len);
             data_to_recv = actual_len - first_data_len;
             return ret::NEED_MORE_DATA;
         }
-        ::send(sock, recv_data, actual_len, 0);
+        on_read(recv_data, actual_len);
     } else if (data_to_recv > len) {    // continue sending
-        ::send(sock, recv_data, len, 0);
+        on_read(recv_data, len);
         data_to_recv -= len;
         return ret::NEED_MORE_DATA;
     } else if (data_to_recv <= len) {    // last read -> looking for "OK" marker
-        ::send(sock, recv_data, data_to_recv, 0);
+        on_read(recv_data, data_to_recv);
         actual_len = data_to_recv;
     }
 

components/esp_modem/examples/modem_tcp_client/main/sock_dce.cpp

@@ -120,14 +120,14 @@ bool DCE::at_to_sock()
 {
     uint64_t data;
     read(data_ready_fd, &data, sizeof(data));
-    ESP_LOGD(TAG, "select read: modem data available %x", data);
+    ESP_LOGD(TAG, "select read: modem data available %" PRIu64, data);
     if (!signal.wait(IDLE, 1000)) {
         ESP_LOGE(TAG, "Failed to get idle");
         close_sock();
         return false;
     }
     if (state != status::IDLE) {
-        ESP_LOGE(TAG, "Unexpected state %d", state);
+        ESP_LOGE(TAG, "Unexpected state %d", static_cast<int>(state));
         close_sock();
         return false;
     }
@@ -145,7 +145,7 @@ bool DCE::sock_to_at()
         return false;
     }
     if (state != status::IDLE) {
-        ESP_LOGE(TAG,  "Unexpected state %d", state);
+        ESP_LOGE(TAG,  "Unexpected state %d", static_cast<int>(state));
         close_sock();
         return false;
     }
@@ -192,14 +192,8 @@ bool DCE::accept_sock()
     return false;
 }
 
-void DCE::init_sock(int port)
+void DCE::start_listening(int port)
 {
-    esp_vfs_eventfd_config_t config = ESP_VFS_EVENTD_CONFIG_DEFAULT();
-    esp_vfs_eventfd_register(&config);
-
-    data_ready_fd = eventfd(0, EFD_SUPPORT_ISR);
-    assert(data_ready_fd > 0);
-
     listen_sock = socket(AF_INET, SOCK_STREAM, IPPROTO_IP);
     if (listen_sock < 0) {
         ESP_LOGE(TAG, "Unable to create socket: errno %d", errno);
@@ -228,7 +222,7 @@ void DCE::init_sock(int port)
 
 }
 
-bool DCE::start(std::string host, int port)
+bool DCE::connect(std::string host, int port)
 {
     dte->on_read(nullptr);
     tcp_close();
@@ -245,8 +239,14 @@ bool DCE::start(std::string host, int port)
     return true;
 }
 
-bool DCE::init_network()
+bool DCE::init()
 {
+    esp_vfs_eventfd_config_t config = ESP_VFS_EVENTD_CONFIG_DEFAULT();
+    esp_vfs_eventfd_register(&config);
+
+    data_ready_fd = eventfd(0, EFD_SUPPORT_ISR);
+    assert(data_ready_fd > 0);
+
     dte->on_read(nullptr);
     const int retries = 5;
     int i = 0;

components/esp_modem/examples/modem_tcp_client/main/sock_dce.hpp

@@ -9,6 +9,7 @@
 #include <cxx_include/esp_modem_dce_factory.hpp>
 #include "socket_commands.inc"
 #include "sock_commands.hpp"
+#include <sys/socket.h>
 
 #pragma  once
 
@@ -46,9 +47,36 @@ public:
     {
         return buffer_size;
     }
+
+    void clear_offsets()
+    {
+        actual_read = 0;
+    }
+
+    size_t data_available()
+    {
+        return actual_read;
+    }
+
+    size_t has_data()
+    {
+        return total_len;
+    }
+
 private:
     static constexpr size_t buffer_size = 512;
 
+    bool on_read(char *data, size_t len)
+    {
+#ifndef CONFIG_EXAMPLE_CUSTOM_TCP_TRANSPORT
+        ::send(sock, data, len, 0);
+#else
+        ::memcpy(&buffer[actual_read], data, len);
+        actual_read += len;
+#endif
+        return true;
+    }
+
     ret recv(uint8_t *data, size_t len);
     ret send(uint8_t *data, size_t len);
     ret send(std::string_view response);
@@ -59,6 +87,9 @@ private:
     }
     std::array<uint8_t, buffer_size> buffer;
     size_t data_to_recv = 0;
+    size_t actual_read = 0;
+    size_t total_len = 0;
+
     bool read_again = false;
     int &sock;
     int &data_ready_fd;
@@ -78,13 +109,101 @@ esp_modem::return_type name(__VA_ARGS__);
 
 #undef ESP_MODEM_DECLARE_DCE_COMMAND
 
-    bool init_network();
+    bool init();
-    bool start(std::string host, int port);
+    bool connect(std::string host, int port);
 
-    void init_sock(int port);
+    void start_listening(int port);
 
     bool perform_sock();
 
+    void set_idle()
+    {
+        signal.set(IDLE);
+    }
+
+    bool wait_to_idle(uint32_t ms)
+    {
+        if (!signal.wait(IDLE, ms)) {
+            ESP_LOGE("dce", "Failed to get idle");
+            return false;
+        }
+        if (state != status::IDLE) {
+            ESP_LOGE("dce", "Unexpected state %d", static_cast<int>(state));
+            return false;
+        }
+        return true;
+    }
+
+    int sync_recv(char *buffer, int len, int timeout_ms)
+    {
+        if (!wait_to_idle(timeout_ms)) {
+            return 0;
+        }
+        at.clear_offsets();
+        state = status::RECEIVING;
+        uint64_t data;
+        read(data_ready_fd, &data, sizeof(data));
+        int max_len = std::min(len, (int)at.get_buf_len());
+        at.start_receiving(max_len);
+        if (!signal.wait(IDLE, 500 + timeout_ms)) {
+            return 0;
+        }
+        int ret = at.data_available();
+        if (ret > 0) {
+            memcpy(buffer, at.get_buf(), ret);
+        }
+        set_idle();
+        return ret;
+    }
+
+    int sync_send(const char *buffer, size_t len, int timeout_ms)
+    {
+        int len_to_send = std::min(len, at.get_buf_len());
+        if (!wait_to_idle(timeout_ms)) {
+            return -1;
+        }
+        state = status::SENDING;
+        memcpy(at.get_buf(), buffer, len_to_send);
+        ESP_LOG_BUFFER_HEXDUMP("dce", at.get_buf(), len, ESP_LOG_VERBOSE);
+        at.start_sending(len_to_send);
+        if (!signal.wait(IDLE, timeout_ms + 1000)) {
+            if (state == status::PENDING) {
+                state = status::IDLE;
+            } else {
+                return -1;
+            }
+        }
+        set_idle();
+        return len_to_send;
+    }
+
+    int wait_to_read(uint32_t ms)
+    {
+        if (at.has_data() > 0) {
+            ESP_LOGD("dce",  "Data buffered in modem (len=%d)", at.has_data());
+            return 1;
+        }
+        struct timeval tv = {
+            .tv_sec = static_cast<time_t>(ms / 1000),
+            .tv_usec = 0,
+        };
+        fd_set fdset;
+        FD_ZERO(&fdset);
+        FD_SET(data_ready_fd, &fdset);
+        int s = select(data_ready_fd + 1, &fdset, nullptr, nullptr, &tv);
+        if (s == 0) {
+            return 0;
+        } else if (s < 0) {
+            ESP_LOGE("dce",  "select error %d", errno);
+            return -1;
+        }
+        if (FD_ISSET(data_ready_fd, &fdset)) {
+            ESP_LOGD("dce", "select read: modem data available");
+            return 1;
+        }
+        return -1;
+    }
+
 private:
     esp_modem::SignalGroup signal;