ASIO: Example tests integration

.github/workflows/target-test.yml

@@ -41,7 +41,6 @@ jobs:
         idf_ver: ["latest"] 
         idf_target: ["esp32", "esp32s2", "esp32c3"]
         config: ["eth_custom_netif", "eth_def", "eth_no_ipv6", "eth_socket"]
- 
     runs-on: ubuntu-20.04 
     container: espressif/idf:${{ matrix.idf_ver }} 
     steps: 
@@ -91,7 +90,6 @@ jobs:
         idf_ver: ["latest"]
         idf_target: ["esp32", "esp32s2"]
         example: ["asio_chat", "async_request", "socks4", "ssl_client_server", "tcp_echo_server", "udp_echo_server"]
-
     runs-on: ubuntu-20.04
     container: espressif/idf:${{ matrix.idf_ver }}
     steps:
@@ -99,16 +97,37 @@ jobs:
         uses: actions/checkout@v3
         with:
           submodules: recursive 
-          path: esp-protocols
       - name: Build ${{ matrix.example }} with IDF-${{ matrix.idf_ver }} for ${{ matrix.idf_target }}
+        working-directory: components/asio/examples/${{ matrix.example }}
         env:
           IDF_TARGET: ${{ matrix.idf_target }}
         shell: bash
         run: |
           . ${IDF_PATH}/export.sh
-          cd $GITHUB_WORKSPACE/esp-protocols/components/asio/examples/${{ matrix.example }}
+          test -f sdkconfig.ci && cat sdkconfig.ci >> sdkconfig.defaults || echo "No sdkconfig.ci"
           idf.py set-target ${{ matrix.idf_target }}
           idf.py build
+      - name: Merge binaries with IDF-${{ matrix.idf_ver }} for ${{ matrix.idf_target }} for ${{ matrix.example }}
+        working-directory: components/asio/examples/${{ matrix.example }}/build
+        env:
+          IDF_TARGET: ${{ matrix.idf_target }}
+        shell: bash
+        run: |
+          . ${IDF_PATH}/export.sh
+          esptool.py --chip ${{ matrix.idf_target }} merge_bin --fill-flash-size 4MB -o flash_image.bin @flash_args
+      - uses: actions/upload-artifact@v3
+        with:
+          name: examples_app_bin_${{ matrix.idf_target }}_${{ matrix.idf_ver }}_${{ matrix.example }}
+          path: |
+            components/asio/examples/${{ matrix.example }}/build/bootloader/bootloader.bin
+            components/asio/examples/${{ matrix.example }}/build//partition_table/partition-table.bin
+            components/asio/examples/${{ matrix.example }}/build/*.bin
+            components/asio/examples/${{ matrix.example }}/build/*.elf
+            components/asio/examples/${{ matrix.example }}/build/flasher_args.json
+            components/asio/examples/${{ matrix.example }}/build/config/sdkconfig.h
+            components/asio/examples/${{ matrix.example }}/build/config/sdkconfig.json
+          if-no-files-found: error
+
 
   build_websocket:
     strategy:
@@ -234,3 +253,42 @@ jobs:
         with:
           name: examples_results_${{ matrix.idf_target }}_${{ matrix.idf_ver }}_${{ matrix.config }}
           path: components/mdns/examples/*.xml
+
+  run-target-asio:
+    strategy: 
+      matrix: 
+        idf_ver: ["latest"] 
+        idf_target: ["esp32"]
+        example: ["asio_chat", "tcp_echo_server", "udp_echo_server", "ssl_client_server"]
+    name: Run ASIO Example Test on target
+    needs: build_asio
+    runs-on:
+      - self-hosted
+      - ESP32-ETHERNET-KIT
+    # Skip running on forks since it won't have access to secrets
+    if: github.repository == 'espressif/esp-protocols'
+    steps:
+      - name: Clear repository
+        run: sudo rm -fr $GITHUB_WORKSPACE && mkdir $GITHUB_WORKSPACE
+      - uses: actions/checkout@v3
+      - uses: actions/download-artifact@v2
+        with:
+          name: examples_app_bin_${{ matrix.idf_target }}_${{ matrix.idf_ver }}_${{ matrix.example }}
+          path: components/asio/examples/${{ matrix.example }}/build
+      - name: Install Python packages
+        env:
+          PIP_EXTRA_INDEX_URL: "https://www.piwheels.org/simple"
+        run: |
+            sudo apt-get install -y dnsutils
+      - name: Download Example Test to target ${{ matrix.config }}
+        run: |
+            python -m esptool --chip ${{ matrix.idf_target }} write_flash 0x0 components/asio/examples/${{ matrix.example }}/build/flash_image.bin
+      - name: Run Example Test ${{ matrix.example }} on target
+        working-directory: components/asio/examples/${{ matrix.example }}
+        run: |
+            python -m pytest --log-cli-level DEBUG --junit-xml=./examples_results_${{ matrix.idf_target }}_${{ matrix.idf_ver }}_${{ matrix.config }}.xml --target=${{ matrix.idf_target }}
+      - uses: actions/upload-artifact@v2
+        if: always()
+        with:
+          name: examples_results_${{ matrix.idf_target }}_${{ matrix.idf_ver }}_${{ matrix.example }}
+          path: components/asio/examples/${{ matrix.example }}/*.xml

components/asio/examples/asio_chat/example_test.py

@@ -1,29 +0,0 @@
-# This example code is in the Public Domain (or CC0 licensed, at your option.)
-
-# Unless required by applicable law or agreed to in writing, this
-# software is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
-# CONDITIONS OF ANY KIND, either express or implied.
-
-# -*- coding: utf-8 -*-
-
-from __future__ import print_function, unicode_literals
-
-import re
-
-import ttfw_idf
-
-
-@ttfw_idf.idf_example_test(env_tag='Example_GENERIC')
-def test_examples_asio_chat(env, _):  # type: (ttfw_idf.TinyFW.Env, None) -> None
-    msg = 'asio-chat: received hi'
-    dut = env.get_dut('asio_chat', 'examples/protocols/asio/asio_chat')
-    # start the test and expect the client to receive back it's original data
-    dut.start_app()
-    dut.expect(re.compile(r'{}'.format('Waiting for input')), timeout=30)
-    dut.write(msg)
-    dut.write('exit')
-    dut.expect(re.compile(r'{}'.format(msg)), timeout=30)
-
-
-if __name__ == '__main__':
-    test_examples_asio_chat()

components/asio/examples/asio_chat/pytest.ini

@@ -0,0 +1,24 @@
+[pytest]
+# only the files with prefix `pytest_` would be recognized as pytest test scripts.
+python_files = pytest_*.py
+
+addopts =
+  -s
+  --embedded-services esp,idf
+  -W ignore::_pytest.warning_types.PytestExperimentalApiWarning
+  --tb short
+
+# ignore DeprecationWarning
+filterwarnings =
+  ignore:Call to deprecated create function (.*)\(\):DeprecationWarning
+
+# log related
+log_cli = True
+log_cli_level = INFO
+log_cli_format = %(asctime)s %(levelname)s %(message)s
+log_cli_date_format = %Y-%m-%d %H:%M:%S
+
+log_file = test.log
+log_file_level = INFO
+log_file_format = %(asctime)s %(levelname)s %(message)s
+log_file_date_format = %Y-%m-%d %H:%M:%S

components/asio/examples/asio_chat/pytest_example_test.py

@@ -0,0 +1,21 @@
+# This example code is in the Public Domain (or CC0 licensed, at your option.)
+
+# Unless required by applicable law or agreed to in writing, this
+# software is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied.
+
+# -*- coding: utf-8 -*-
+
+import re
+import pytest
+from pytest_embedded import Dut
+
+
+def test_examples_asio_chat(dut):
+    msg = 'asio-chat: received hi'
+    # start the test and expect the client to receive back it's original data
+    dut.expect(re.compile(b'Waiting for input'), timeout=30)
+    dut.write(msg)
+    dut.write('exit')
+    msg = bytes(msg, encoding='utf8')
+    dut.expect(re.compile(msg), timeout=30)

components/asio/examples/ssl_client_server/example_test.py

@@ -1,16 +0,0 @@
-from __future__ import unicode_literals
-
-import ttfw_idf
-
-
-@ttfw_idf.idf_example_test(env_tag='Example_GENERIC', target=['esp32', 'esp32c3'])
-def test_examples_asio_ssl(env, extra_data):
-
-    dut = env.get_dut('asio_ssl_client_server', 'examples/protocols/asio/ssl_client_server')
-    dut.start_app()
-
-    dut.expect('Reply: GET / HTTP/1.1')
-
-
-if __name__ == '__main__':
-    test_examples_asio_ssl()

components/asio/examples/ssl_client_server/pytest.ini

@@ -0,0 +1,24 @@
+[pytest]
+# only the files with prefix `pytest_` would be recognized as pytest test scripts.
+python_files = pytest_*.py
+
+addopts =
+  -s
+  --embedded-services esp,idf
+  -W ignore::_pytest.warning_types.PytestExperimentalApiWarning
+  --tb short
+
+# ignore DeprecationWarning
+filterwarnings =
+  ignore:Call to deprecated create function (.*)\(\):DeprecationWarning
+
+# log related
+log_cli = True
+log_cli_level = INFO
+log_cli_format = %(asctime)s %(levelname)s %(message)s
+log_cli_date_format = %Y-%m-%d %H:%M:%S
+
+log_file = test.log
+log_file_level = INFO
+log_file_format = %(asctime)s %(levelname)s %(message)s
+log_file_date_format = %Y-%m-%d %H:%M:%S

components/asio/examples/ssl_client_server/pytest_example_test.py

@@ -0,0 +1,8 @@
+from __future__ import unicode_literals
+
+import pytest
+from pytest_embedded import Dut
+
+
+def test_examples_asio_ssl(dut):
+    dut.expect('Reply: GET / HTTP/1.1')

components/asio/examples/tcp_echo_server/pytest.ini

@@ -0,0 +1,24 @@
+[pytest]
+# only the files with prefix `pytest_` would be recognized as pytest test scripts.
+python_files = pytest_*.py
+
+addopts =
+  -s
+  --embedded-services esp,idf
+  -W ignore::_pytest.warning_types.PytestExperimentalApiWarning
+  --tb short
+
+# ignore DeprecationWarning
+filterwarnings =
+  ignore:Call to deprecated create function (.*)\(\):DeprecationWarning
+
+# log related
+log_cli = True
+log_cli_level = INFO
+log_cli_format = %(asctime)s %(levelname)s %(message)s
+log_cli_date_format = %Y-%m-%d %H:%M:%S
+
+log_file = test.log
+log_file_level = INFO
+log_file_format = %(asctime)s %(levelname)s %(message)s
+log_file_date_format = %Y-%m-%d %H:%M:%S

components/asio/examples/tcp_echo_server/pytest_asio_tcp_server_test.py

@@ -1,12 +1,10 @@
 import os
 import re
 import socket
-
-import ttfw_idf
+from pytest_embedded import Dut
 
 
-@ttfw_idf.idf_example_test(env_tag='Example_WIFI_Protocols')
-def test_examples_protocol_asio_tcp_server(env, extra_data):
+def test_examples_protocol_asio_tcp_server(dut):
     """
     steps: |
       1. join AP
@@ -16,20 +14,13 @@ def test_examples_protocol_asio_tcp_server(env, extra_data):
       5. Test evaluates received test message on server stdout
     """
     test_msg = b'echo message from client to server'
-    dut1 = env.get_dut('tcp_echo_server', 'examples/protocols/asio/tcp_echo_server', dut_class=ttfw_idf.ESP32DUT)
-    # check and log bin size
-    binary_file = os.path.join(dut1.app.binary_path, 'asio_tcp_echo_server.bin')
-    bin_size = os.path.getsize(binary_file)
-    ttfw_idf.log_performance('asio_tcp_echo_server_bin_size', '{}KB'.format(bin_size // 1024))
-    # 1. start test
-    dut1.start_app()
     # 2. get the server IP address
-    data = dut1.expect(re.compile(r' IPv4 address: ([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+)'), timeout=30)
+    data = dut.expect(re.compile(b' IPv4 address: ([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+)'), timeout=30).group(1).decode()
     # 3. create tcp client and connect to server
-    dut1.expect('ASIO engine is up and running', timeout=1)
+    dut.expect('ASIO engine is up and running', timeout=1)
     cli = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
     cli.settimeout(30)
-    cli.connect((data[0], 2222))
+    cli.connect((data, 2222))
     cli.send(test_msg)
     data = cli.recv(1024)
     # 4. check the message received back from the server
@@ -40,8 +31,5 @@ def test_examples_protocol_asio_tcp_server(env, extra_data):
         print('Failure!')
         raise ValueError('Wrong data received from asi tcp server: {} (expected:{})'.format(data, test_msg))
     # 5. check the client message appears also on server terminal
-    dut1.expect(test_msg.decode())
+    dut.expect(test_msg.decode())
 
-
-if __name__ == '__main__':
-    test_examples_protocol_asio_tcp_server()

components/asio/examples/tcp_echo_server/sdkconfig.ci

@@ -0,0 +1,2 @@
+CONFIG_EXAMPLE_CONNECT_WIFI=n
+CONFIG_EXAMPLE_CONNECT_ETHERNET=y

components/asio/examples/udp_echo_server/asio_udp_server_test.py

@@ -1,47 +0,0 @@
-import os
-import re
-import socket
-
-import ttfw_idf
-
-
-@ttfw_idf.idf_example_test(env_tag='Example_WIFI_Protocols')
-def test_examples_protocol_asio_udp_server(env, extra_data):
-    """
-    steps: |
-      1. join AP
-      2. Start server
-      3. Test connects to server and sends a test message
-      4. Test evaluates received test message from server
-      5. Test evaluates received test message on server stdout
-    """
-    test_msg = b'echo message from client to server'
-    dut1 = env.get_dut('udp_echo_server', 'examples/protocols/asio/udp_echo_server', dut_class=ttfw_idf.ESP32DUT)
-    # check and log bin size
-    binary_file = os.path.join(dut1.app.binary_path, 'asio_udp_echo_server.bin')
-    bin_size = os.path.getsize(binary_file)
-    ttfw_idf.log_performance('asio_udp_echo_server_bin_size', '{}KB'.format(bin_size // 1024))
-    # 1. start test
-    dut1.start_app()
-    # 2. get the server IP address
-    data = dut1.expect(re.compile(r' IPv4 address: ([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+)'), timeout=30)
-    # 3. create tcp client and connect to server
-    dut1.expect('ASIO engine is up and running', timeout=1)
-    cli = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
-    cli.settimeout(30)
-    cli.connect((data[0], 2222))
-    cli.send(test_msg)
-    data = cli.recv(1024)
-    # 4. check the message received back from the server
-    if (data == test_msg):
-        print('PASS: Received correct message')
-        pass
-    else:
-        print('Failure!')
-        raise ValueError('Wrong data received from asio udp server: {} (expected:{})'.format(data, test_msg))
-    # 5. check the client message appears also on server terminal
-    dut1.expect(test_msg.decode())
-
-
-if __name__ == '__main__':
-    test_examples_protocol_asio_udp_server()

components/asio/examples/udp_echo_server/pytest.ini

@@ -0,0 +1,24 @@
+[pytest]
+# only the files with prefix `pytest_` would be recognized as pytest test scripts.
+python_files = pytest_*.py
+
+addopts =
+  -s
+  --embedded-services esp,idf
+  -W ignore::_pytest.warning_types.PytestExperimentalApiWarning
+  --tb short
+
+# ignore DeprecationWarning
+filterwarnings =
+  ignore:Call to deprecated create function (.*)\(\):DeprecationWarning
+
+# log related
+log_cli = True
+log_cli_level = INFO
+log_cli_format = %(asctime)s %(levelname)s %(message)s
+log_cli_date_format = %Y-%m-%d %H:%M:%S
+
+log_file = test.log
+log_file_level = INFO
+log_file_format = %(asctime)s %(levelname)s %(message)s
+log_file_date_format = %Y-%m-%d %H:%M:%S

components/asio/examples/udp_echo_server/pytest_asio_udp_server_test.py

@@ -0,0 +1,33 @@
+import os
+import re
+import socket
+
+import pytest
+from pytest_embedded import Dut
+
+def test_examples_protocol_asio_udp_server(dut):
+    """
+    steps: |
+      1. join AP
+      2. Start server
+      3. Test connects to server and sends a test message
+      4. Test evaluates received test message from server
+      5. Test evaluates received test message on server stdout
+    """
+    test_msg = b'echo message from client to server'
+    data = dut.expect(re.compile(b' IPv4 address: ([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+)'), timeout=30).group(1).decode()
+    dut.expect('ASIO engine is up and running', timeout=1)
+    cli = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+    cli.settimeout(30)
+    cli.connect((data, 2222))
+    cli.send(test_msg)
+    data = cli.recv(1024)
+    # 4. check the message received back from the server
+    if (data == test_msg):
+        print('PASS: Received correct message')
+        pass
+    else:
+        print('Failure!')
+        raise ValueError('Wrong data received from asio udp server: {} (expected:{})'.format(data, test_msg))
+    # 5. check the client message appears also on server terminal
+    dut.expect(test_msg.decode())

components/asio/examples/udp_echo_server/sdkconfig.ci

@@ -0,0 +1,2 @@
+CONFIG_EXAMPLE_CONNECT_WIFI=n
+CONFIG_EXAMPLE_CONNECT_ETHERNET=y