esp_modem: Move common C definitions in to separate header

Separate header for C API created.
It can be used by C DTE extensions.

.github/workflows/publish-docs-component.yml

@@ -9,6 +9,8 @@ jobs:
   docs_build:
     name: Docs-Build-And-Upload
     runs-on: ubuntu-latest
+    # Skip running on forks since it won't have access to secrets
+    if: github.repository == 'espressif/esp-protocols'
 
     steps:
       - name: Checkout esp-protocols

.github/workflows/target-test.yml

@@ -23,7 +23,7 @@ jobs:
     container: espressif/idf:${{ matrix.idf_ver }}
     steps:
       - name: Checkout esp-protocols
-        uses: actions/checkout@v3
+        uses: actions/checkout@master
         with:
           path: esp-protocols
       - name: Build ${{ matrix.example }} with IDF-${{ matrix.idf_ver }} for ${{ matrix.idf_target }}
@@ -39,8 +39,8 @@ jobs:
     strategy: 
       matrix: 
         idf_ver: ["latest"] 
-        idf_target: ["esp32", "esp32s2", "esp32c3"] 
- 
+        idf_target: ["esp32", "esp32s2", "esp32c3"]
+        config: ["eth_custom_netif", "eth_def", "eth_no_ipv6", "eth_socket"]
     runs-on: ubuntu-20.04 
     container: espressif/idf:${{ matrix.idf_ver }} 
     steps: 
@@ -48,36 +48,41 @@ jobs:
         uses: actions/checkout@v3 
         with: 
           submodules: recursive 
-          path: esp-protocols 
-      - name: Build ${{ matrix.example }} with IDF-${{ matrix.idf_ver }} for ${{ matrix.idf_target }} 
+      - name: Build ${{ matrix.example }} with IDF-${{ matrix.idf_ver }} for ${{ matrix.idf_target }} for ${{ matrix.config }} 
         env: 
           IDF_TARGET: ${{ matrix.idf_target }} 
         shell: bash 
+        working-directory: components/mdns/examples/
         run: | 
           . ${IDF_PATH}/export.sh 
-          cd $GITHUB_WORKSPACE/esp-protocols/components/mdns/examples/ 
-          idf.py set-target ${{ matrix.idf_target }}
-          echo "Building for default configuration"
-          idf.py build
-          rm sdkconfig
-          echo "Building for sdkconfig.ci.eth_def"
-          cat sdkconfig.ci.eth_def >> sdkconfig.defaults 
-          idf.py build 
-          rm sdkconfig sdkconfig.defaults
-          echo "Building for sdkconfig.ci.eth_custom_netif"
-          cat sdkconfig.ci.eth_custom_netif >> sdkconfig.defaults 
-          idf.py build 
-          rm sdkconfig sdkconfig.defaults
-          echo "Building for sdkconfig.ci.eth_socket"
-          cat sdkconfig.ci.eth_socket >> sdkconfig.defaults 
-          idf.py build 
-          rm sdkconfig sdkconfig.defaults
-          echo "Building for sdkconfig.ci.eth_no_ipv6"
-          cat sdkconfig.ci.eth_no_ipv6 >> sdkconfig.defaults
-          idf.py build
-          cd $GITHUB_WORKSPACE/esp-protocols/components/mdns/tests/test_apps/ 
+          rm -rf sdkconfig sdkconfig.defaults build build_${{ matrix.config }}
+          cat sdkconfig.ci.${{ matrix.config }} >> sdkconfig.defaults  
           idf.py set-target ${{ matrix.idf_target }}  
           idf.py build 
+          mv build build_${{ matrix.config }}
+      - name: Merge binaries with IDF-${{ matrix.idf_ver }} for ${{ matrix.config }}
+        working-directory: components/mdns/examples
+        env:
+          IDF_TARGET: ${{ matrix.idf_target }}
+        shell: bash
+        run: |
+          . ${IDF_PATH}/export.sh
+          cd build_${{ matrix.config }}
+          esptool.py --chip ${{ matrix.idf_target }} merge_bin --fill-flash-size 4MB -o flash_image.bin @flash_args
+          cd ../
+          cat build_${{ matrix.config }}/config/sdkconfig.json
+      - uses: actions/upload-artifact@v2
+        with:
+          name: examples_app_bin_${{ matrix.idf_target }}_${{ matrix.idf_ver }}_${{ matrix.config }}
+          path: |
+            components/mdns/examples/build_${{ matrix.config }}/bootloader/bootloader.bin
+            components/mdns/examples/build_${{ matrix.config }}/partition_table/partition-table.bin
+            components/mdns/examples/build_${{ matrix.config }}/*.bin
+            components/mdns/examples/build_${{ matrix.config }}/*.elf
+            components/mdns/examples/build_${{ matrix.config }}/flasher_args.json
+            components/mdns/examples/build_${{ matrix.config }}/config/sdkconfig.h
+            components/mdns/examples/build_${{ matrix.config }}/config/sdkconfig.json
+          if-no-files-found: error
 
   build_asio:
     strategy:
@@ -85,7 +90,6 @@ jobs:
         idf_ver: ["latest"]
         idf_target: ["esp32", "esp32s2"]
         example: ["asio_chat", "async_request", "socks4", "ssl_client_server", "tcp_echo_server", "udp_echo_server"]
-
     runs-on: ubuntu-20.04
     container: espressif/idf:${{ matrix.idf_ver }}
     steps:
@@ -93,23 +97,43 @@ jobs:
         uses: actions/checkout@v3
         with:
           submodules: recursive 
-          path: esp-protocols
       - name: Build ${{ matrix.example }} with IDF-${{ matrix.idf_ver }} for ${{ matrix.idf_target }}
+        working-directory: components/asio/examples/${{ matrix.example }}
         env:
           IDF_TARGET: ${{ matrix.idf_target }}
         shell: bash
         run: |
           . ${IDF_PATH}/export.sh
-          cd $GITHUB_WORKSPACE/esp-protocols/components/asio/examples/${{ matrix.example }}
+          test -f sdkconfig.ci && cat sdkconfig.ci >> sdkconfig.defaults || echo "No sdkconfig.ci"
           idf.py set-target ${{ matrix.idf_target }}
           idf.py build
+      - name: Merge binaries with IDF-${{ matrix.idf_ver }} for ${{ matrix.idf_target }} for ${{ matrix.example }}
+        working-directory: components/asio/examples/${{ matrix.example }}/build
+        env:
+          IDF_TARGET: ${{ matrix.idf_target }}
+        shell: bash
+        run: |
+          . ${IDF_PATH}/export.sh
+          esptool.py --chip ${{ matrix.idf_target }} merge_bin --fill-flash-size 4MB -o flash_image.bin @flash_args
+      - uses: actions/upload-artifact@v3
+        with:
+          name: examples_app_bin_${{ matrix.idf_target }}_${{ matrix.idf_ver }}_${{ matrix.example }}
+          path: |
+            components/asio/examples/${{ matrix.example }}/build/bootloader/bootloader.bin
+            components/asio/examples/${{ matrix.example }}/build//partition_table/partition-table.bin
+            components/asio/examples/${{ matrix.example }}/build/*.bin
+            components/asio/examples/${{ matrix.example }}/build/*.elf
+            components/asio/examples/${{ matrix.example }}/build/flasher_args.json
+            components/asio/examples/${{ matrix.example }}/build/config/sdkconfig.h
+            components/asio/examples/${{ matrix.example }}/build/config/sdkconfig.json
+          if-no-files-found: error
+
 
   build_websocket:
     strategy:
       matrix:
         idf_ver: ["latest"]
         idf_target: ["esp32"]
-
     runs-on: ubuntu-20.04
     container: espressif/idf:${{ matrix.idf_ver }}
     steps:
@@ -137,7 +161,14 @@ jobs:
       - uses: actions/upload-artifact@v2
         with:
           name: examples_app_bin_${{ matrix.idf_target }}_${{ matrix.idf_ver }}
-          path: components/esp_websocket_client/examples/build/
+          path: |
+            components/esp_websocket_client/examples/build/bootloader/bootloader.bin
+            components/esp_websocket_client/examples/build/partition_table/partition-table.bin
+            components/esp_websocket_client/examples/build/*.bin
+            components/esp_websocket_client/examples/build/*.elf
+            components/esp_websocket_client/examples/build/flasher_args.json
+            components/esp_websocket_client/examples/build/config/sdkconfig.h
+            components/esp_websocket_client/examples/build/config/sdkconfig.json
           if-no-files-found: error
 
   run-target:
@@ -151,6 +182,8 @@ jobs:
     runs-on:
       - self-hosted
       - ESP32-ETHERNET-KIT
+    # Skip running on forks since it won't have access to secrets
+    if: github.repository == 'espressif/esp-protocols'
     container:
       image: python:3.7-buster
       options: --privileged # Privileged mode has access to serial ports
@@ -161,6 +194,8 @@ jobs:
           name: examples_app_bin_${{ matrix.idf_target }}_${{ matrix.idf_ver }}
           path: components/esp_websocket_client/examples/build/
       - name: Install Python packages
+        env:
+          PIP_EXTRA_INDEX_URL: "https://www.piwheels.org/simple"
         run: |
           pip install --only-binary cryptography --extra-index-url https://dl.espressif.com/pypi/ -r $GITHUB_WORKSPACE/components/esp_websocket_client/examples/requirements.txt
       - name: Download Example Test to target
@@ -169,9 +204,91 @@ jobs:
         working-directory: components/esp_websocket_client/examples
         run: |
           cp sdkconfig.ci sdkconfig.defaults
-          pytest --log-cli-level DEBUG --junit-xml=./test_app_results_${{ matrix.idf_target }}_${{ matrix.idf_ver }}.xml --target=${{ matrix.idf_target }}
+          pytest --log-cli-level DEBUG --junit-xml=./examples_results_${{ matrix.idf_target }}_${{ matrix.idf_ver }}.xml --target=${{ matrix.idf_target }}
       - uses: actions/upload-artifact@v2
         if: always()
         with:
           name: examples_results_${{ matrix.idf_target }}_${{ matrix.idf_ver }}
-          path: examples/*.xml
+          path: components/esp_websocket_client/examples/*.xml
+
+  run-target-mdns:
+    strategy: 
+      matrix: 
+        idf_ver: ["latest"] 
+        idf_target: ["esp32"]
+        config: ["eth_custom_netif", "eth_def", "eth_no_ipv6", "eth_socket"]
+    name: Run mDNS Example Test on target
+    needs: build_mdns
+    runs-on:
+      - self-hosted
+      - ESP32-ETHERNET-KIT
+    # Skip running on forks since it won't have access to secrets
+    if: github.repository == 'espressif/esp-protocols'
+    steps:
+      - name: Clear repository
+        run: sudo rm -fr $GITHUB_WORKSPACE && mkdir $GITHUB_WORKSPACE
+      - uses: actions/checkout@v3
+      - uses: actions/download-artifact@v2
+        with:
+          name: examples_app_bin_${{ matrix.idf_target }}_${{ matrix.idf_ver }}_${{ matrix.config }}
+          path: components/mdns/examples/build_${{ matrix.config }}
+      - name: Install Python packages
+        env:
+          PIP_EXTRA_INDEX_URL: "https://www.piwheels.org/simple"
+        run: |
+            sudo apt-get install -y dnsutils
+      - name: Download Example Test to target ${{ matrix.config }}
+        run: |
+            python -m esptool --chip ${{ matrix.idf_target }} write_flash 0x0 components/mdns/examples/build_${{ matrix.config }}/flash_image.bin
+      - name: Run Example Test on target ${{ matrix.config }}
+        working-directory: components/mdns/examples
+        run: |
+            rm -rf build
+            mv build_${{ matrix.config }} build
+            cat sdkconfig.ci.${{ matrix.config }} >> sdkconfig.defaults  
+            python -m pytest --log-cli-level DEBUG --junit-xml=./examples_results_${{ matrix.idf_target }}_${{ matrix.idf_ver }}_${{ matrix.config }}.xml --target=${{ matrix.idf_target }}
+            rm -rf build sdkconfig.defaults
+      - uses: actions/upload-artifact@v2
+        if: always()
+        with:
+          name: examples_results_${{ matrix.idf_target }}_${{ matrix.idf_ver }}_${{ matrix.config }}
+          path: components/mdns/examples/*.xml
+
+  run-target-asio:
+    strategy: 
+      matrix: 
+        idf_ver: ["latest"] 
+        idf_target: ["esp32"]
+        example: ["asio_chat", "tcp_echo_server", "udp_echo_server", "ssl_client_server"]
+    name: Run ASIO Example Test on target
+    needs: build_asio
+    runs-on:
+      - self-hosted
+      - ESP32-ETHERNET-KIT
+    # Skip running on forks since it won't have access to secrets
+    if: github.repository == 'espressif/esp-protocols'
+    steps:
+      - name: Clear repository
+        run: sudo rm -fr $GITHUB_WORKSPACE && mkdir $GITHUB_WORKSPACE
+      - uses: actions/checkout@v3
+      - uses: actions/download-artifact@v2
+        with:
+          name: examples_app_bin_${{ matrix.idf_target }}_${{ matrix.idf_ver }}_${{ matrix.example }}
+          path: components/asio/examples/${{ matrix.example }}/build
+      - name: Install Python packages
+        env:
+          PIP_EXTRA_INDEX_URL: "https://www.piwheels.org/simple"
+        run: |
+            sudo apt-get install -y dnsutils
+      - name: Download Example Test to target ${{ matrix.config }}
+        run: |
+            python -m esptool --chip ${{ matrix.idf_target }} write_flash 0x0 components/asio/examples/${{ matrix.example }}/build/flash_image.bin
+      - name: Run Example Test ${{ matrix.example }} on target
+        working-directory: components/asio/examples/${{ matrix.example }}
+        run: |
+            python -m pytest --log-cli-level DEBUG --junit-xml=./examples_results_${{ matrix.idf_target }}_${{ matrix.idf_ver }}_${{ matrix.config }}.xml --target=${{ matrix.idf_target }}
+      - uses: actions/upload-artifact@v2
+        if: always()
+        with:
+          name: examples_results_${{ matrix.idf_target }}_${{ matrix.idf_ver }}_${{ matrix.example }}
+          path: components/asio/examples/${{ matrix.example }}/*.xml

components/asio/examples/asio_chat/example_test.py

@@ -1,29 +0,0 @@
-# This example code is in the Public Domain (or CC0 licensed, at your option.)
-
-# Unless required by applicable law or agreed to in writing, this
-# software is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
-# CONDITIONS OF ANY KIND, either express or implied.
-
-# -*- coding: utf-8 -*-
-
-from __future__ import print_function, unicode_literals
-
-import re
-
-import ttfw_idf
-
-
-@ttfw_idf.idf_example_test(env_tag='Example_GENERIC')
-def test_examples_asio_chat(env, _):  # type: (ttfw_idf.TinyFW.Env, None) -> None
-    msg = 'asio-chat: received hi'
-    dut = env.get_dut('asio_chat', 'examples/protocols/asio/asio_chat')
-    # start the test and expect the client to receive back it's original data
-    dut.start_app()
-    dut.expect(re.compile(r'{}'.format('Waiting for input')), timeout=30)
-    dut.write(msg)
-    dut.write('exit')
-    dut.expect(re.compile(r'{}'.format(msg)), timeout=30)
-
-
-if __name__ == '__main__':
-    test_examples_asio_chat()

components/asio/examples/asio_chat/pytest.ini

@@ -0,0 +1,24 @@
+[pytest]
+# only the files with prefix `pytest_` would be recognized as pytest test scripts.
+python_files = pytest_*.py
+
+addopts =
+  -s
+  --embedded-services esp,idf
+  -W ignore::_pytest.warning_types.PytestExperimentalApiWarning
+  --tb short
+
+# ignore DeprecationWarning
+filterwarnings =
+  ignore:Call to deprecated create function (.*)\(\):DeprecationWarning
+
+# log related
+log_cli = True
+log_cli_level = INFO
+log_cli_format = %(asctime)s %(levelname)s %(message)s
+log_cli_date_format = %Y-%m-%d %H:%M:%S
+
+log_file = test.log
+log_file_level = INFO
+log_file_format = %(asctime)s %(levelname)s %(message)s
+log_file_date_format = %Y-%m-%d %H:%M:%S

components/asio/examples/asio_chat/pytest_example_test.py

@@ -0,0 +1,21 @@
+# This example code is in the Public Domain (or CC0 licensed, at your option.)
+
+# Unless required by applicable law or agreed to in writing, this
+# software is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied.
+
+# -*- coding: utf-8 -*-
+
+import re
+import pytest
+from pytest_embedded import Dut
+
+
+def test_examples_asio_chat(dut):
+    msg = 'asio-chat: received hi'
+    # start the test and expect the client to receive back it's original data
+    dut.expect(re.compile(b'Waiting for input'), timeout=30)
+    dut.write(msg)
+    dut.write('exit')
+    msg = bytes(msg, encoding='utf8')
+    dut.expect(re.compile(msg), timeout=30)

components/asio/examples/ssl_client_server/example_test.py

@@ -1,16 +0,0 @@
-from __future__ import unicode_literals
-
-import ttfw_idf
-
-
-@ttfw_idf.idf_example_test(env_tag='Example_GENERIC', target=['esp32', 'esp32c3'])
-def test_examples_asio_ssl(env, extra_data):
-
-    dut = env.get_dut('asio_ssl_client_server', 'examples/protocols/asio/ssl_client_server')
-    dut.start_app()
-
-    dut.expect('Reply: GET / HTTP/1.1')
-
-
-if __name__ == '__main__':
-    test_examples_asio_ssl()

components/asio/examples/ssl_client_server/pytest.ini

@@ -0,0 +1,24 @@
+[pytest]
+# only the files with prefix `pytest_` would be recognized as pytest test scripts.
+python_files = pytest_*.py
+
+addopts =
+  -s
+  --embedded-services esp,idf
+  -W ignore::_pytest.warning_types.PytestExperimentalApiWarning
+  --tb short
+
+# ignore DeprecationWarning
+filterwarnings =
+  ignore:Call to deprecated create function (.*)\(\):DeprecationWarning
+
+# log related
+log_cli = True
+log_cli_level = INFO
+log_cli_format = %(asctime)s %(levelname)s %(message)s
+log_cli_date_format = %Y-%m-%d %H:%M:%S
+
+log_file = test.log
+log_file_level = INFO
+log_file_format = %(asctime)s %(levelname)s %(message)s
+log_file_date_format = %Y-%m-%d %H:%M:%S

components/asio/examples/ssl_client_server/pytest_example_test.py

@@ -0,0 +1,8 @@
+from __future__ import unicode_literals
+
+import pytest
+from pytest_embedded import Dut
+
+
+def test_examples_asio_ssl(dut):
+    dut.expect('Reply: GET / HTTP/1.1')

components/asio/examples/tcp_echo_server/pytest.ini

@@ -0,0 +1,24 @@
+[pytest]
+# only the files with prefix `pytest_` would be recognized as pytest test scripts.
+python_files = pytest_*.py
+
+addopts =
+  -s
+  --embedded-services esp,idf
+  -W ignore::_pytest.warning_types.PytestExperimentalApiWarning
+  --tb short
+
+# ignore DeprecationWarning
+filterwarnings =
+  ignore:Call to deprecated create function (.*)\(\):DeprecationWarning
+
+# log related
+log_cli = True
+log_cli_level = INFO
+log_cli_format = %(asctime)s %(levelname)s %(message)s
+log_cli_date_format = %Y-%m-%d %H:%M:%S
+
+log_file = test.log
+log_file_level = INFO
+log_file_format = %(asctime)s %(levelname)s %(message)s
+log_file_date_format = %Y-%m-%d %H:%M:%S

components/asio/examples/tcp_echo_server/pytest_asio_tcp_server_test.py

@@ -1,12 +1,10 @@
 import os
 import re
 import socket
-
-import ttfw_idf
+from pytest_embedded import Dut
 
 
-@ttfw_idf.idf_example_test(env_tag='Example_WIFI_Protocols')
-def test_examples_protocol_asio_tcp_server(env, extra_data):
+def test_examples_protocol_asio_tcp_server(dut):
     """
     steps: |
       1. join AP
@@ -16,20 +14,13 @@ def test_examples_protocol_asio_tcp_server(env, extra_data):
       5. Test evaluates received test message on server stdout
     """
     test_msg = b'echo message from client to server'
-    dut1 = env.get_dut('tcp_echo_server', 'examples/protocols/asio/tcp_echo_server', dut_class=ttfw_idf.ESP32DUT)
-    # check and log bin size
-    binary_file = os.path.join(dut1.app.binary_path, 'asio_tcp_echo_server.bin')
-    bin_size = os.path.getsize(binary_file)
-    ttfw_idf.log_performance('asio_tcp_echo_server_bin_size', '{}KB'.format(bin_size // 1024))
-    # 1. start test
-    dut1.start_app()
     # 2. get the server IP address
-    data = dut1.expect(re.compile(r' IPv4 address: ([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+)'), timeout=30)
+    data = dut.expect(re.compile(b' IPv4 address: ([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+)'), timeout=30).group(1).decode()
     # 3. create tcp client and connect to server
-    dut1.expect('ASIO engine is up and running', timeout=1)
+    dut.expect('ASIO engine is up and running', timeout=1)
     cli = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
     cli.settimeout(30)
-    cli.connect((data[0], 2222))
+    cli.connect((data, 2222))
     cli.send(test_msg)
     data = cli.recv(1024)
     # 4. check the message received back from the server
@@ -40,8 +31,5 @@ def test_examples_protocol_asio_tcp_server(env, extra_data):
         print('Failure!')
         raise ValueError('Wrong data received from asi tcp server: {} (expected:{})'.format(data, test_msg))
     # 5. check the client message appears also on server terminal
-    dut1.expect(test_msg.decode())
+    dut.expect(test_msg.decode())
 
-
-if __name__ == '__main__':
-    test_examples_protocol_asio_tcp_server()

components/asio/examples/tcp_echo_server/sdkconfig.ci

@@ -0,0 +1,2 @@
+CONFIG_EXAMPLE_CONNECT_WIFI=n
+CONFIG_EXAMPLE_CONNECT_ETHERNET=y

components/asio/examples/udp_echo_server/asio_udp_server_test.py

@@ -1,47 +0,0 @@
-import os
-import re
-import socket
-
-import ttfw_idf
-
-
-@ttfw_idf.idf_example_test(env_tag='Example_WIFI_Protocols')
-def test_examples_protocol_asio_udp_server(env, extra_data):
-    """
-    steps: |
-      1. join AP
-      2. Start server
-      3. Test connects to server and sends a test message
-      4. Test evaluates received test message from server
-      5. Test evaluates received test message on server stdout
-    """
-    test_msg = b'echo message from client to server'
-    dut1 = env.get_dut('udp_echo_server', 'examples/protocols/asio/udp_echo_server', dut_class=ttfw_idf.ESP32DUT)
-    # check and log bin size
-    binary_file = os.path.join(dut1.app.binary_path, 'asio_udp_echo_server.bin')
-    bin_size = os.path.getsize(binary_file)
-    ttfw_idf.log_performance('asio_udp_echo_server_bin_size', '{}KB'.format(bin_size // 1024))
-    # 1. start test
-    dut1.start_app()
-    # 2. get the server IP address
-    data = dut1.expect(re.compile(r' IPv4 address: ([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+)'), timeout=30)
-    # 3. create tcp client and connect to server
-    dut1.expect('ASIO engine is up and running', timeout=1)
-    cli = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
-    cli.settimeout(30)
-    cli.connect((data[0], 2222))
-    cli.send(test_msg)
-    data = cli.recv(1024)
-    # 4. check the message received back from the server
-    if (data == test_msg):
-        print('PASS: Received correct message')
-        pass
-    else:
-        print('Failure!')
-        raise ValueError('Wrong data received from asio udp server: {} (expected:{})'.format(data, test_msg))
-    # 5. check the client message appears also on server terminal
-    dut1.expect(test_msg.decode())
-
-
-if __name__ == '__main__':
-    test_examples_protocol_asio_udp_server()

components/asio/examples/udp_echo_server/pytest.ini

@@ -0,0 +1,24 @@
+[pytest]
+# only the files with prefix `pytest_` would be recognized as pytest test scripts.
+python_files = pytest_*.py
+
+addopts =
+  -s
+  --embedded-services esp,idf
+  -W ignore::_pytest.warning_types.PytestExperimentalApiWarning
+  --tb short
+
+# ignore DeprecationWarning
+filterwarnings =
+  ignore:Call to deprecated create function (.*)\(\):DeprecationWarning
+
+# log related
+log_cli = True
+log_cli_level = INFO
+log_cli_format = %(asctime)s %(levelname)s %(message)s
+log_cli_date_format = %Y-%m-%d %H:%M:%S
+
+log_file = test.log
+log_file_level = INFO
+log_file_format = %(asctime)s %(levelname)s %(message)s
+log_file_date_format = %Y-%m-%d %H:%M:%S

components/asio/examples/udp_echo_server/pytest_asio_udp_server_test.py

@@ -0,0 +1,33 @@
+import os
+import re
+import socket
+
+import pytest
+from pytest_embedded import Dut
+
+def test_examples_protocol_asio_udp_server(dut):
+    """
+    steps: |
+      1. join AP
+      2. Start server
+      3. Test connects to server and sends a test message
+      4. Test evaluates received test message from server
+      5. Test evaluates received test message on server stdout
+    """
+    test_msg = b'echo message from client to server'
+    data = dut.expect(re.compile(b' IPv4 address: ([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+)'), timeout=30).group(1).decode()
+    dut.expect('ASIO engine is up and running', timeout=1)
+    cli = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+    cli.settimeout(30)
+    cli.connect((data, 2222))
+    cli.send(test_msg)
+    data = cli.recv(1024)
+    # 4. check the message received back from the server
+    if (data == test_msg):
+        print('PASS: Received correct message')
+        pass
+    else:
+        print('Failure!')
+        raise ValueError('Wrong data received from asio udp server: {} (expected:{})'.format(data, test_msg))
+    # 5. check the client message appears also on server terminal
+    dut.expect(test_msg.decode())

components/asio/examples/udp_echo_server/sdkconfig.ci

@@ -0,0 +1,2 @@
+CONFIG_EXAMPLE_CONNECT_WIFI=n
+CONFIG_EXAMPLE_CONNECT_ETHERNET=y

components/esp_modem/examples/modem_console/main/Kconfig.projbuild

@@ -14,4 +14,163 @@ menu "Example Configuration"
             help
                 Connect to modem via USB (CDC-ACM class). For IDF version >= 4.4.
     endchoice
+
+    choice EXAMPLE_MODEM_DEVICE
+        prompt "Choose supported modem device (DCE)"
+        default EXAMPLE_MODEM_DEVICE_BG96
+        help
+            Select modem device connected to the ESP DTE.
+        config EXAMPLE_MODEM_DEVICE_SHINY
+            bool "SHINY"
+            help
+                SHINY is a GSM/GPRS module.
+                It supports SHINY.
+        config EXAMPLE_MODEM_DEVICE_SIM800
+            bool "SIM800"
+            help
+                SIMCom SIM800L is a GSM/GPRS module.
+                It supports Quad-band 850/900/1800/1900MHz.
+        config EXAMPLE_MODEM_DEVICE_BG96
+            bool "BG96"
+            help
+                Quectel BG96 is a series of LTE Cat M1/Cat NB1/EGPRS module.
+        config EXAMPLE_MODEM_DEVICE_SIM7000
+            bool "SIM7000"
+            help
+                SIM7000 is a Multi-Band LTE-FDD and GSM/GPRS/EDGE module.
+        config EXAMPLE_MODEM_DEVICE_SIM7070
+            bool "SIM7070"
+            help
+                SIM7070 is Multi-Band CAT M and NB IoT module.
+        config EXAMPLE_MODEM_DEVICE_SIM7600
+            bool "SIM7600"
+            help
+                SIM7600 is a Multi-Band LTE-TDD/LTE-FDD/HSPA+ and GSM/GPRS/EDGE module.
+    endchoice
+
+    config EXAMPLE_MODEM_PPP_APN
+        string "Set MODEM APN"
+        default "internet"
+        help
+            Set APN (Access Point Name), a logical name to choose data network
+
+    config EXAMPLE_MODEM_PPP_AUTH_USERNAME
+        string "Set username for authentication"
+        default "espressif"
+        depends on !EXAMPLE_MODEM_PPP_AUTH_NONE
+        help
+            Set username for PPP Authentication.
+
+    config EXAMPLE_MODEM_PPP_AUTH_PASSWORD
+        string "Set password for authentication"
+        default "esp32"
+        depends on !EXAMPLE_MODEM_PPP_AUTH_NONE
+        help
+            Set password for PPP Authentication.
+
+    config EXAMPLE_MODEM_PPP_AUTH_NONE
+        bool "Skip PPP authentication"
+        default n
+        help
+            Set to true for the PPP client to skip authentication
+
+    config EXAMPLE_SEND_MSG
+        bool "Short message (SMS)"
+        default n
+        help
+            Select this, the modem will send a short message before power off.
+
+    if EXAMPLE_SEND_MSG
+        config EXAMPLE_SEND_MSG_PEER_PHONE_NUMBER
+            string "Peer Phone Number (with area code)"
+            default "+8610086"
+            help
+                Enter the peer phone number that you want to send message to.
+    endif
+
+    config EXAMPLE_NEED_SIM_PIN
+        bool "SIM PIN needed"
+        default n
+        help
+            Enable to set SIM PIN before starting the example
+
+    config EXAMPLE_SIM_PIN
+        string "Set SIM PIN"
+        default "1234"
+        depends on EXAMPLE_NEED_SIM_PIN
+        help
+            Pin to unlock the SIM
+
+    menu "UART Configuration"
+        config EXAMPLE_MODEM_UART_TX_PIN
+            int "TXD Pin Number"
+            default 25
+            range 0 31
+            help
+                Pin number of UART TX.
+
+        config EXAMPLE_MODEM_UART_RX_PIN
+            int "RXD Pin Number"
+            default 26
+            range 0 31
+            help
+                Pin number of UART RX.
+
+        config EXAMPLE_MODEM_UART_RTS_PIN
+            int "RTS Pin Number"
+            default 27
+            range 0 31
+            help
+                Pin number of UART RTS.
+
+        config EXAMPLE_MODEM_UART_CTS_PIN
+            int "CTS Pin Number"
+            default 23
+            range 0 31
+            help
+                Pin number of UART CTS.
+
+        config EXAMPLE_MODEM_UART_EVENT_TASK_STACK_SIZE
+            int "UART Event Task Stack Size"
+            range 2000 6000
+            default 2048
+            help
+                Stack size of UART event task.
+
+        config EXAMPLE_MODEM_UART_EVENT_TASK_PRIORITY
+            int "UART Event Task Priority"
+            range 3 22
+            default 5
+            help
+                Priority of UART event task.
+
+        config EXAMPLE_MODEM_UART_EVENT_QUEUE_SIZE
+            int "UART Event Queue Size"
+            range 10 40
+            default 30
+            help
+                Length of UART event queue.
+
+        config EXAMPLE_MODEM_UART_PATTERN_QUEUE_SIZE
+            int "UART Pattern Queue Size"
+            range 10 40
+            default 20
+            help
+                Length of UART pattern queue.
+
+        config EXAMPLE_MODEM_UART_TX_BUFFER_SIZE
+            int "UART TX Buffer Size"
+            range 256 2048
+            default 512
+            help
+                Buffer size of UART TX buffer.
+
+        config EXAMPLE_MODEM_UART_RX_BUFFER_SIZE
+            int "UART RX Buffer Size"
+            range 256 2048
+            default 1024
+            help
+                Buffer size of UART RX buffer.
+    endmenu
+
 endmenu

components/esp_modem/examples/modem_console/main/modem_console_main.cpp

@@ -64,8 +64,43 @@ extern "C" void app_main(void)
 
 #if defined(CONFIG_EXAMPLE_SERIAL_CONFIG_UART)
     esp_modem_dte_config_t dte_config = ESP_MODEM_DTE_DEFAULT_CONFIG();
+    /* setup UART specific configuration based on kconfig options */
+    dte_config.uart_config.tx_io_num = CONFIG_EXAMPLE_MODEM_UART_TX_PIN;
+    dte_config.uart_config.rx_io_num = CONFIG_EXAMPLE_MODEM_UART_RX_PIN;
+    dte_config.uart_config.rts_io_num = CONFIG_EXAMPLE_MODEM_UART_RTS_PIN;
+    dte_config.uart_config.cts_io_num = CONFIG_EXAMPLE_MODEM_UART_CTS_PIN;
+    dte_config.uart_config.flow_control = ESP_MODEM_FLOW_CONTROL_HW;
+    dte_config.uart_config.rx_buffer_size = CONFIG_EXAMPLE_MODEM_UART_RX_BUFFER_SIZE;
+    dte_config.uart_config.tx_buffer_size = CONFIG_EXAMPLE_MODEM_UART_TX_BUFFER_SIZE;
+    dte_config.uart_config.event_queue_size = CONFIG_EXAMPLE_MODEM_UART_EVENT_QUEUE_SIZE;
+    dte_config.task_stack_size = CONFIG_EXAMPLE_MODEM_UART_EVENT_TASK_STACK_SIZE;
+    dte_config.task_priority = CONFIG_EXAMPLE_MODEM_UART_EVENT_TASK_PRIORITY;
+    dte_config.dte_buffer_size = CONFIG_EXAMPLE_MODEM_UART_RX_BUFFER_SIZE / 2;
     auto uart_dte = create_uart_dte(&dte_config);
+
+#if CONFIG_EXAMPLE_MODEM_DEVICE_SHINY == 1
+    ESP_LOGI(TAG, "Initializing esp_modem for the SHINY module...");
     auto dce = create_shiny_dce(&dce_config, uart_dte, esp_netif);
+#elif CONFIG_EXAMPLE_MODEM_DEVICE_BG96 == 1
+    ESP_LOGI(TAG, "Initializing esp_modem for the BG96 module...");
+    auto dce = create_BG96_dce(&dce_config, uart_dte, esp_netif);
+#elif CONFIG_EXAMPLE_MODEM_DEVICE_SIM800 == 1
+    ESP_LOGI(TAG, "Initializing esp_modem for the SIM800 module...");
+    auto dce = create_SIM800_dce(&dce_config, uart_dte, esp_netif);
+#elif CONFIG_EXAMPLE_MODEM_DEVICE_SIM7000 == 1
+    ESP_LOGI(TAG, "Initializing esp_modem for the SIM7000 module...");
+    auto dce = create_SIM7000_dce(&dce_config, uart_dte, esp_netif);
+#elif CONFIG_EXAMPLE_MODEM_DEVICE_SIM7070 == 1
+    ESP_LOGI(TAG, "Initializing esp_modem for the SIM7070 module...");
+    auto dce = create_SIM7070_dce(&dce_config, uart_dte, esp_netif);
+#elif CONFIG_EXAMPLE_MODEM_DEVICE_SIM7600 == 1
+    ESP_LOGI(TAG, "Initializing esp_modem for the SIM7600 module...");
+    auto dce = create_SIM7600_dce(&dce_config, uart_dte, esp_netif);
+#else
+    ESP_LOGI(TAG, "Initializing esp_modem for a generic module...");
+    auto dce = create_generic_dce(&dce_config, uart_dte, esp_netif);
+#endif
+
 
 #elif defined(CONFIG_EXAMPLE_SERIAL_CONFIG_USB)
     struct esp_modem_usb_term_config usb_config = ESP_MODEM_DEFAULT_USB_CONFIG(0x2C7C, 0x0296); // VID and PID of BG96 modem
@@ -82,6 +117,14 @@ extern "C" void app_main(void)
     
     assert(dce != nullptr);
 
+    if(dte_config.uart_config.flow_control == ESP_MODEM_FLOW_CONTROL_HW) {
+        if (command_result::OK != dce->set_flow_control(2, 2)) {
+            ESP_LOGE(TAG, "Failed to set the set_flow_control mode");
+            return;
+        }
+        ESP_LOGI(TAG, "set_flow_control OK");
+    }
+
     // init console REPL environment
     esp_console_repl_config_t repl_config = ESP_CONSOLE_REPL_CONFIG_DEFAULT();
     esp_console_dev_uart_config_t uart_config = ESP_CONSOLE_DEV_UART_CONFIG_DEFAULT();

components/esp_modem/examples/simple_cmux_client/main/Kconfig.projbuild

@@ -61,4 +61,10 @@ menu "Example Configuration"
         help
             HTTPS address of the update binary.
 
+    config EXAMPLE_CLOSE_CMUX_AT_END
+        bool "Close multiplexed mode at the end of the example"
+        default n
+        help
+            Close the multiplexed mode at the end of the example and rollback to command mode.
+
 endmenu

components/esp_modem/examples/simple_cmux_client/main/simple_cmux_client_main.cpp

@@ -164,4 +164,14 @@ extern "C" void app_main(void)
         return;
     }
 #endif // CONFIG_EXAMPLE_PERFORM_OTA
+
+    /* Close multiplexed command/data mode */
+#if CONFIG_EXAMPLE_CLOSE_CMUX_AT_END == 1
+    if (dce->set_mode(esp_modem::modem_mode::COMMAND_MODE)) {
+        std::cout << "Modem has correctly entered command mode" << std::endl;
+    } else {
+        ESP_LOGE(TAG, "Failed to configure desired mode... exiting");
+        return;
+    }
+#endif
 }

components/esp_modem/idf_component.yml

@@ -1,4 +1,4 @@
-version: "0.1.21"
+version: "0.1.23"
 description: esp modem
 url: https://github.com/espressif/esp-protocols/tree/master/components/esp_modem
 dependencies:

components/esp_modem/include/esp_private/c_api_wrapper.hpp

@@ -0,0 +1,45 @@
+// Copyright 2022 Espressif Systems (Shanghai) PTE LTD
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "cxx_include/esp_modem_dce_factory.hpp"
+#include "esp_modem_c_api_types.h"
+
+using namespace esp_modem;
+
+struct esp_modem_dce_wrap { // need to mimic the polymorphic dispatch as CPP uses templated dispatch
+    enum class modem_wrap_dte_type { UART, VFS, USB } dte_type;
+    dce_factory::ModemType modem_type;
+    DCE *dce;
+};
+
+inline dce_factory::ModemType convert_modem_enum(esp_modem_dce_device_t module)
+{
+    switch (module) {
+    case ESP_MODEM_DCE_SIM7600:
+        return esp_modem::dce_factory::ModemType::SIM7600;
+    case ESP_MODEM_DCE_SIM7070:
+        return esp_modem::dce_factory::ModemType::SIM7070;
+    case ESP_MODEM_DCE_SIM7000:
+        return esp_modem::dce_factory::ModemType::SIM7000;
+    case ESP_MODEM_DCE_BG96:
+        return esp_modem::dce_factory::ModemType::BG96;
+    case ESP_MODEM_DCE_SIM800:
+        return esp_modem::dce_factory::ModemType::SIM800;
+    default:
+    case ESP_MODEM_DCE_GENETIC:
+        return esp_modem::dce_factory::ModemType::GenericModule;
+    }
+}

components/esp_modem/include/generate/esp_modem_command_declare.inc

@@ -47,6 +47,7 @@
  * @return OK, FAIL or TIMEOUT
  */ \
 ESP_MODEM_DECLARE_DCE_COMMAND(sync, command_result, 0) \
+    \
 /**
  * @brief Reads the operator name
  * @param[out] operator name
@@ -72,9 +73,10 @@ ESP_MODEM_DECLARE_DCE_COMMAND(set_pin, command_result, 1, STRING_IN(p1, pin)) \
  * @brief Execute the supplied AT command
  * @param[in] at AT command
  * @param[out] out Command output string
+ * @param[in] timeout AT command timeout in milliseconds
  * @return OK, FAIL or TIMEOUT
  */\
-ESP_MODEM_DECLARE_DCE_COMMAND(at, command_result, 2, STRING_IN(p1, cmd), STRING_OUT(p2, out)) \
+ESP_MODEM_DECLARE_DCE_COMMAND(at, command_result, 3, STRING_IN(p1, cmd), STRING_OUT(p2, out), INT_IN(p3, timeout)) \
     \
 /**
  * @brief Checks if the SIM needs a PIN
@@ -112,7 +114,7 @@ ESP_MODEM_DECLARE_DCE_COMMAND(sms_character_set, command_result, 0) \
 ESP_MODEM_DECLARE_DCE_COMMAND(send_sms, command_result, 2, STRING_IN(p1, number), STRING_IN(p2, message)) \
     \
 /**
- * @brief Resumes data mode (Switches back to th data mode, which was temporarily suspended)
+ * @brief Resumes data mode (Switches back to the data mode, which was temporarily suspended)
  * @return OK, FAIL or TIMEOUT
  */ \
 ESP_MODEM_DECLARE_DCE_COMMAND(resume_data_mode, command_result, 0) \

components/esp_modem/src/esp_modem_c_api.cpp

@@ -21,6 +21,7 @@
 #include "esp_modem_c_api_types.h"
 #include "esp_modem_config.h"
 #include "exception_stub.hpp"
+#include "esp_private/c_api_wrapper.hpp"
 #include "cstring"
 
 #ifndef ESP_MODEM_C_API_STR_MAX
@@ -35,12 +36,6 @@ size_t strlcpy(char *dest, const char *src, size_t len);
 // C API definitions
 using namespace esp_modem;
 
-struct esp_modem_dce_wrap { // need to mimic the polymorphic dispatch as CPP uses templated dispatch
-    enum class modem_wrap_dte_type { UART, } dte_type;
-    dce_factory::ModemType modem_type;
-    DCE *dce;
-};
-
 static inline esp_err_t command_response_to_esp_err(command_result res)
 {
     switch (res) {
@@ -54,25 +49,6 @@ static inline esp_err_t command_response_to_esp_err(command_result res)
     return ESP_ERR_INVALID_ARG;
 }
 
-static inline dce_factory::ModemType convert_modem_enum(esp_modem_dce_device_t module)
-{
-    switch (module) {
-    case ESP_MODEM_DCE_SIM7600:
-        return esp_modem::dce_factory::ModemType::SIM7600;
-    case ESP_MODEM_DCE_SIM7070:
-        return esp_modem::dce_factory::ModemType::SIM7070;
-    case ESP_MODEM_DCE_SIM7000:
-        return esp_modem::dce_factory::ModemType::SIM7000;
-    case ESP_MODEM_DCE_BG96:
-        return esp_modem::dce_factory::ModemType::BG96;
-    case ESP_MODEM_DCE_SIM800:
-        return esp_modem::dce_factory::ModemType::SIM800;
-    default:
-    case ESP_MODEM_DCE_GENETIC:
-        return esp_modem::dce_factory::ModemType::GenericModule;
-    }
-}
-
 extern "C" esp_modem_dce_t *esp_modem_new_dev(esp_modem_dce_device_t module, const esp_modem_dte_config_t *dte_config, const esp_modem_dce_config_t *dce_config, esp_netif_t *netif)
 {
     auto dce_wrap = new (std::nothrow) esp_modem_dce_wrap;
@@ -179,14 +155,14 @@ extern "C" esp_err_t esp_modem_set_pin(esp_modem_dce_t *dce_wrap, const char *pi
     return command_response_to_esp_err(dce_wrap->dce->set_pin(pin_str));
 }
 
-extern "C" esp_err_t esp_modem_at(esp_modem_dce_t *dce_wrap, const char *at, char *p_out)
+extern "C" esp_err_t esp_modem_at(esp_modem_dce_t *dce_wrap, const char *at, char *p_out, int timeout)
 {
     if (dce_wrap == nullptr || dce_wrap->dce == nullptr) {
         return ESP_ERR_INVALID_ARG;
     }
     std::string out;
     std::string at_str(at);
-    auto ret = command_response_to_esp_err(dce_wrap->dce->at(at_str, out));
+    auto ret = command_response_to_esp_err(dce_wrap->dce->at(at_str, out, timeout));
     if ((p_out != NULL) && (!out.empty())) {
         strlcpy(p_out, out.c_str(), ESP_MODEM_C_API_STR_MAX);
     }
@@ -243,15 +219,17 @@ extern "C" esp_err_t esp_modem_get_imei(esp_modem_dce_t *dce_wrap, char *p_imei)
     return ret;
 }
 
-extern "C" esp_err_t esp_modem_get_operator_name(esp_modem_dce_t *dce_wrap, char *p_name)
+extern "C" esp_err_t esp_modem_get_operator_name(esp_modem_dce_t *dce_wrap, char *p_name, int *p_act)
 {
-    if (dce_wrap == nullptr || dce_wrap->dce == nullptr) {
+    if (dce_wrap == nullptr || dce_wrap->dce == nullptr || p_name == nullptr || p_act == nullptr) {
         return ESP_ERR_INVALID_ARG;
     }
     std::string name;
-    auto ret = command_response_to_esp_err(dce_wrap->dce->get_operator_name(name));
+    int act;
+    auto ret = command_response_to_esp_err(dce_wrap->dce->get_operator_name(name, act));
     if (ret == ESP_OK && !name.empty()) {
         strlcpy(p_name, name.c_str(), ESP_MODEM_C_API_STR_MAX);
+        *p_act = act;
     }
     return ret;
 }

components/esp_modem/src/esp_modem_command_library.cpp

@@ -273,7 +273,7 @@ command_result set_pdp_context(CommandableIf *t, PdpContext &pdp)
     ESP_LOGV(TAG, "%s", __func__ );
     std::string pdp_command = "AT+CGDCONT=" + std::to_string(pdp.context_id) +
                               ",\"" + pdp.protocol_type + "\",\"" + pdp.apn + "\"\r";
-    return generic_command_common(t, pdp_command);
+    return generic_command_common(t, pdp_command, 150000);
 }
 
 command_result set_data_mode(CommandableIf *t)
@@ -389,11 +389,11 @@ command_result set_pin(CommandableIf *t, const std::string &pin)
     return generic_command_common(t, set_pin_command);
 }
 
-command_result at(CommandableIf *t, const std::string &cmd, std::string &out)
+command_result at(CommandableIf *t, const std::string &cmd, std::string &out, int timeout = 500)
 {
     ESP_LOGV(TAG, "%s", __func__ );
     std::string at_command = cmd + "\r";
-    return generic_get_string(t, at_command, out);
+    return generic_get_string(t, at_command, out, timeout);
 }
 
 command_result get_signal_quality(CommandableIf *t, int &rssi, int &ber)
@@ -458,7 +458,7 @@ command_result get_network_attachment_state(CommandableIf *t, int &state)
 command_result set_radio_state(CommandableIf *t, int state)
 {
     ESP_LOGV(TAG, "%s", __func__ );
-    return generic_command_common(t, "AT+CFUN=" + std::to_string(state) + "\r");
+    return generic_command_common(t, "AT+CFUN=" + std::to_string(state) + "\r", 15000);
 }
 
 command_result get_radio_state(CommandableIf *t, int &state)

components/esp_modem/src/esp_modem_dte.cpp

@@ -36,6 +36,7 @@ command_result DTE::command(const std::string &command, got_line_cb got_line, ui
 {
     Scoped<Lock> l(internal_lock);
     command_result res = command_result::TIMEOUT;
+    signal.clear(GOT_LINE);
     command_term->set_read_cb([&](uint8_t *data, size_t len) {
         if (!data) {
             data = buffer.get();

components/esp_modem/test/host_test/main/LoopbackTerm.cpp

@@ -17,6 +17,7 @@ int LoopbackTerm::write(uint8_t *data, size_t len)
 {
     if (inject_by) {    // injection test: ignore what we write, but respond with injected data
         auto ret = std::async(&LoopbackTerm::batch_read, this);
+        async_results.push_back(std::move(ret));
         return len;
     }
     if (len > 2 && (data[len - 1] == '\r' || data[len - 1] == '+') ) { // Simple AT responder
@@ -99,7 +100,7 @@ LoopbackTerm::LoopbackTerm(bool is_bg96): loopback_data(), data_len(0), pin_ok(f
 
 LoopbackTerm::LoopbackTerm(): loopback_data(), data_len(0), pin_ok(false), is_bg96(false), inject_by(0) {}
 
-int LoopbackTerm::inject(uint8_t *data, size_t len, size_t injected_by)
+int LoopbackTerm::inject(uint8_t *data, size_t len, size_t injected_by, size_t delay_before, size_t delay_after)
 {
     if (data == nullptr) {
         inject_by = 0;
@@ -110,14 +111,20 @@ int LoopbackTerm::inject(uint8_t *data, size_t len, size_t injected_by)
     memcpy(&loopback_data[0], data, len);
     data_len = len;
     inject_by = injected_by;
+    delay_after_inject = delay_after;
+    delay_before_inject = delay_before;
     return len;
 }
 
 void LoopbackTerm::batch_read()
 {
     while (data_len > 0) {
-        on_read(nullptr, std::min(inject_by, data_len));
-        Task::Delay(1);
+        Task::Delay(delay_before_inject);
+        {
+            Scoped<Lock> lock(on_read_guard);
+            on_read(nullptr, std::min(inject_by, data_len));
+        }
+        Task::Delay(delay_after_inject);
     }
 }
 

components/esp_modem/test/host_test/main/LoopbackTerm.h

@@ -17,7 +17,7 @@ public:
      * inject_by defines batch sizes: the read callback is called multiple times
      * with partial data of `inject_by` size
      */
-    int inject(uint8_t *data, size_t len, size_t inject_by);
+    int inject(uint8_t *data, size_t len, size_t inject_by,size_t delay_before=0, size_t delay_after=1);
 
     void start() override;
     void stop() override;
@@ -26,6 +26,12 @@ public:
 
     int read(uint8_t *data, size_t len) override;
 
+    void set_read_cb(std::function<bool(uint8_t *data, size_t len)> f) override
+    {
+        Scoped<Lock> lock(on_read_guard);
+        on_read = std::move(f);
+    }
+
 private:
     enum class status_t {
         STARTED,
@@ -39,4 +45,9 @@ private:
     bool pin_ok;
     bool is_bg96;
     size_t inject_by;
+    size_t delay_before_inject;
+    size_t delay_after_inject;
+    std::vector<std::future<void>> async_results;
+    Lock on_read_guard;
+
 };

components/esp_modem/test/host_test/main/test_modem.cpp

@@ -7,6 +7,28 @@
 
 using namespace esp_modem;
 
+TEST_CASE("DTE command races", "[esp_modem]") {
+    auto term = std::make_unique<LoopbackTerm>(true);
+    auto loopback = term.get();
+    auto dte = std::make_shared<DTE>(std::move(term));
+    CHECK(term == nullptr);
+    esp_modem_dce_config_t dce_config = ESP_MODEM_DCE_DEFAULT_CONFIG("APN");
+    esp_netif_t netif{};
+    auto dce = create_BG96_dce(&dce_config, dte, &netif);
+    CHECK(dce != nullptr);
+    uint8_t resp[] = {'O', 'K', '\n'};
+    // run many commands in succession with the timeout set exactly to the timespan of injected reply
+    // (checks for potential exception, data races, recycled local variables, etc.)
+    for (int i=0; i<1000; ++i) {
+        loopback->inject(&resp[0], sizeof(resp), sizeof(resp), /* 1ms before injecting reply */1, 0);
+        auto ret = dce->command("AT\n", [&](uint8_t *data, size_t len) {
+            return command_result::OK;
+        }, 1);
+        // this command should either timeout or finish successfully
+        CHECK((ret == command_result::TIMEOUT || ret == command_result::OK));
+    }
+}
+
 TEST_CASE("Test polymorphic delete for custom device/dte", "[esp_modem]")
 {
     auto term = std::make_unique<LoopbackTerm>(true);

components/mdns/examples/pytest.ini

@@ -0,0 +1,24 @@
+[pytest]
+# only the files with prefix `pytest_` would be recognized as pytest test scripts.
+python_files = pytest_*.py
+
+addopts =
+  -s
+  --embedded-services esp,idf
+  -W ignore::_pytest.warning_types.PytestExperimentalApiWarning
+  --tb short
+
+# ignore DeprecationWarning
+filterwarnings =
+  ignore:Call to deprecated create function (.*)\(\):DeprecationWarning
+
+# log related
+log_cli = True
+log_cli_level = INFO
+log_cli_format = %(asctime)s %(levelname)s %(message)s
+log_cli_date_format = %Y-%m-%d %H:%M:%S
+
+log_file = test.log
+log_file_level = INFO
+log_file_format = %(asctime)s %(levelname)s %(message)s
+log_file_date_format = %Y-%m-%d %H:%M:%S

components/mdns/examples/pytest_mdns.py

@@ -5,19 +5,19 @@ import socket
 import struct
 import subprocess
 import time
+import pytest
 from threading import Event, Thread
 
 import dpkt
 import dpkt.dns
-import ttfw_idf
-from tiny_test_fw import DUT
-from tiny_test_fw.Utility import console_log
+from pytest_embedded import Dut
+import subprocess
 
 
 def get_dns_query_for_esp(esp_host):
     dns = dpkt.dns.DNS(b'\x00\x00\x01\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x01')
     dns.qd[0].name = esp_host + u'.local'
-    console_log('Created query for esp host: {} '.format(dns.__repr__()))
+    print('Created query for esp host: {} '.format(dns.__repr__()))
     return dns.pack()
 
 
@@ -31,7 +31,7 @@ def get_dns_answer_to_mdns(tester_host):
     arr.name = tester_host
     arr.ip = socket.inet_aton('127.0.0.1')
     dns.an.append(arr)
-    console_log('Created answer to mdns query: {} '.format(dns.__repr__()))
+    print('Created answer to mdns query: {} '.format(dns.__repr__()))
     return dns.pack()
 
 
@@ -80,18 +80,18 @@ def mdns_server(esp_host, events):
             dns = dpkt.dns.DNS(data)
             if len(dns.qd) > 0 and dns.qd[0].type == dpkt.dns.DNS_A:
                 if dns.qd[0].name == TESTER_NAME:
-                    console_log('Received query: {} '.format(dns.__repr__()))
+                    print('Received query: {} '.format(dns.__repr__()))
                     sock.sendto(get_dns_answer_to_mdns(TESTER_NAME), (MCAST_GRP, UDP_PORT))
                 elif dns.qd[0].name == TESTER_NAME_LWIP:
-                    console_log('Received query: {} '.format(dns.__repr__()))
+                    print('Received query: {} '.format(dns.__repr__()))
                     sock.sendto(get_dns_answer_to_mdns_lwip(TESTER_NAME_LWIP, dns.id), addr)
             if len(dns.an) > 0 and dns.an[0].type == dpkt.dns.DNS_A:
-                console_log('Received answer from {}'.format(dns.an[0].name))
+                print('Received answer from {}'.format(dns.an[0].name))
                 if dns.an[0].name == esp_host + u'.local':
-                    console_log('Received answer to esp32-mdns query: {}'.format(dns.__repr__()))
+                    print('Received answer to esp32-mdns query: {}'.format(dns.__repr__()))
                     events['esp_answered'].set()
                 if dns.an[0].name == esp_host + u'-delegated.local':
-                    console_log('Received answer to esp32-mdns-delegate query: {}'.format(dns.__repr__()))
+                    print('Received answer to esp32-mdns-delegate query: {}'.format(dns.__repr__()))
                     events['esp_delegated_answered'].set()
         except socket.timeout:
             break
@@ -99,7 +99,7 @@ def mdns_server(esp_host, events):
             continue
 
 
-def test_examples_protocol_mdns(env, config):
+def test_examples_protocol_mdns(dut):
     """
     steps: |
       1. obtain IP address + init mdns example
@@ -107,60 +107,31 @@ def test_examples_protocol_mdns(env, config):
       3. check the mdns name is accessible
       4. check DUT output if mdns advertized host is resolved
     """
-    dut1 = env.get_dut('mdns-test', 'examples/protocols/mdns', dut_class=ttfw_idf.ESP32DUT, app_config_name=config)
-    # check and log bin size
-    binary_file = os.path.join(dut1.app.binary_path, 'mdns_test.bin')
-    bin_size = os.path.getsize(binary_file)
-    ttfw_idf.log_performance('mdns-test_bin_size', '{}KB'.format(bin_size // 1024))
-    # 1. start mdns application
-    dut1.start_app()
-    # 2. get the dut host name (and IP address)
-    specific_host = dut1.expect(re.compile(r'mdns hostname set to: \[([^\]]+)\]'), timeout=30)[0]
+
+    specific_host = dut.expect(re.compile(b'mdns hostname set to: \[(.*?)\]')).group(1).decode()
 
     mdns_server_events = {'stop': Event(), 'esp_answered': Event(), 'esp_delegated_answered': Event()}
     mdns_responder = Thread(target=mdns_server, args=(str(specific_host), mdns_server_events))
+    ip_address = dut.expect(re.compile(b'IPv4 address:([a-zA-Z0-9]*).*')).group(1).decode()
+    print('Connected to AP with IP: {}'.format(ip_address))
     try:
-        ip_address = dut1.expect(re.compile(r' eth ip: ([^,]+),'), timeout=30)[0]
-        console_log('Connected to AP with IP: {}'.format(ip_address))
-    except DUT.ExpectTimeout:
-        raise ValueError('ENV_TEST_FAILURE: Cannot connect to AP')
-    try:
-        # 3. check the mdns name is accessible
+        # 3. check the mdns name is accessible.
         mdns_responder.start()
         if not mdns_server_events['esp_answered'].wait(timeout=30):
             raise ValueError('Test has failed: did not receive mdns answer within timeout')
         if not mdns_server_events['esp_delegated_answered'].wait(timeout=30):
             raise ValueError('Test has failed: did not receive mdns answer for delegated host within timeout')
         # 4. check DUT output if mdns advertized host is resolved
-        dut1.expect(re.compile(r'mdns-test: Query A: tinytester.local resolved to: 127.0.0.1'), timeout=30)
-        dut1.expect(re.compile(r'mdns-test: gethostbyname: tinytester-lwip.local resolved to: 127.0.0.1'), timeout=30)
-        dut1.expect(re.compile(r'mdns-test: getaddrinfo: tinytester-lwip.local resolved to: 127.0.0.1'), timeout=30)
+        dut.expect(re.compile(b'mdns-test: Query A: tinytester.local resolved to: 127.0.0.1'), timeout=30)
+        dut.expect(re.compile(b'mdns-test: gethostbyname: tinytester-lwip.local resolved to: 127.0.0.1'), timeout=30)
+        dut.expect(re.compile(b'mdns-test: getaddrinfo: tinytester-lwip.local resolved to: 127.0.0.1'), timeout=30)
         # 5. check the DUT answers to `dig` command
         dig_output = subprocess.check_output(['dig', '+short', '-p', '5353', '@224.0.0.251',
                                               '{}.local'.format(specific_host)])
-        console_log('Resolving {} using "dig" succeeded with:\n{}'.format(specific_host, dig_output))
+        print('Resolving {} using "dig" succeeded with:\n{}'.format(specific_host, dig_output))
         if not ip_address.encode('utf-8') in dig_output:
             raise ValueError('Test has failed: Incorrectly resolved DUT hostname using dig'
                              "Output should've contained DUT's IP address:{}".format(ip_address))
     finally:
         mdns_server_events['stop'].set()
         mdns_responder.join()
-
-
-@ttfw_idf.idf_example_test(env_tag='Example_EthKitV1')
-def test_examples_protocol_mdns_default(env, _):
-    test_examples_protocol_mdns(env, 'eth_def')
-
-
-@ttfw_idf.idf_example_test(env_tag='Example_EthKitV1')
-def test_examples_protocol_mdns_socket(env, _):
-    test_examples_protocol_mdns(env, 'eth_socket')
-
-
-@ttfw_idf.idf_example_test(env_tag='Example_EthKitV1')
-def test_examples_protocol_mdns_custom_netif(env, _):
-    test_examples_protocol_mdns(env, 'eth_custom_netif')
-
-
-if __name__ == '__main__':
-    test_examples_protocol_mdns_default()

components/mdns/mdns.c

@@ -3520,7 +3520,7 @@ void mdns_parse_packet(mdns_rx_packet_t * packet)
                         _mdns_remove_parsed_question(parsed_packet, type, service);
                     } else if (service) {
                         //check if TTL is more than half of the full TTL value (4500)
-                        if (ttl > 2250) {
+                        if (ttl > (MDNS_ANSWER_PTR_TTL/2)) {
                             _mdns_remove_scheduled_answer(packet->tcpip_if, packet->ip_protocol, type, service);
                         }
                     }
@@ -3675,7 +3675,7 @@ void mdns_parse_packet(mdns_rx_packet_t * packet)
                     if (col && !_mdns_server->interfaces[packet->tcpip_if].pcbs[packet->ip_protocol].probe_running && service) {
                         do_not_reply = true;
                         _mdns_init_pcb_probe(packet->tcpip_if, packet->ip_protocol, &service, 1, true);
-                    } else if (ttl > 2250 && !col && !parsed_packet->authoritative && !parsed_packet->probe && !parsed_packet->questions && !_mdns_server->interfaces[packet->tcpip_if].pcbs[packet->ip_protocol].probe_running) {
+                    } else if (ttl > (MDNS_ANSWER_TXT_TTL/2) && !col && !parsed_packet->authoritative && !parsed_packet->probe && !parsed_packet->questions && !_mdns_server->interfaces[packet->tcpip_if].pcbs[packet->ip_protocol].probe_running) {
                         _mdns_remove_scheduled_answer(packet->tcpip_if, packet->ip_protocol, type, service);
                     }
                 }