CI: Added CI example run job

.github/workflows/build-websockets.yml

@@ -1,26 +0,0 @@
-name: Build Websockets
-
-on: [push, pull_request]
-
-jobs:
-  build:
-    strategy:
-      matrix:
-        idf_ver: ["latest"]
-        idf_target: ["esp32"]
-
-    runs-on: ubuntu-20.04
-    container: espressif/idf:${{ matrix.idf_ver }}
-    steps:
-      - name: Checkout esp-protocols
-        uses: actions/checkout@master
-        with:
-          path: esp-protocols
-      - name: Build ${{ matrix.example }} with IDF-${{ matrix.idf_ver }} for ${{ matrix.idf_target }}
-        env:
-          IDF_TARGET: ${{ matrix.idf_target }}
-        shell: bash
-        run: |
-          . ${IDF_PATH}/export.sh
-          cd $GITHUB_WORKSPACE/esp-protocols/components/esp_websocket_client/examples/
-          idf.py build

.github/workflows/build_and_run_example_test.yml

@@ -0,0 +1,76 @@
+name: Build Websockets
+
+on: [push, pull_request]
+
+jobs:
+  build:
+    strategy:
+      matrix:
+        idf_ver: ["latest"]
+        idf_target: ["esp32"]
+
+    runs-on: ubuntu-20.04
+    container: espressif/idf:${{ matrix.idf_ver }}
+    steps:
+      - name: Checkout esp-protocols
+        uses: actions/checkout@v1
+      - name: Build ${{ matrix.example }} with IDF-${{ matrix.idf_ver }} for ${{ matrix.idf_target }}
+        env:
+          IDF_TARGET: ${{ matrix.idf_target }}
+        shell: bash
+        working-directory: components/esp_websocket_client/examples/
+        run: |
+          . ${IDF_PATH}/export.sh
+          cat sdkconfig.ci >> sdkconfig.defaults
+          idf.py build
+      - name: Merge binaries
+        working-directory: components/esp_websocket_client/examples/build
+        env:
+          IDF_TARGET: ${{ matrix.idf_target }}
+        shell: bash
+        run: |
+          . ${IDF_PATH}/export.sh
+          esptool.py --chip ${{ matrix.idf_target }} merge_bin --fill-flash-size 4MB -o flash_image.bin @flash_args
+      - uses: actions/upload-artifact@v2
+        with:
+          name: examples_app_bin_${{ matrix.idf_target }}_${{ matrix.idf_ver }}
+          path: components/esp_websocket_client/examples/build/
+          if-no-files-found: error
+
+  run-target:
+    name: Run Example Test on target
+    needs: build
+    strategy:
+      fail-fast: false
+      matrix:
+        idf_ver: ["latest"]
+        idf_target: ["esp32"]
+    runs-on:
+      - self-hosted
+      - ESP32-ETHERNET-KIT
+    container:
+      image: python:3.7-buster
+      options: --privileged # Privileged mode has access to serial ports
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/download-artifact@v2
+        with:
+          name: examples_app_bin_${{ matrix.idf_target }}_${{ matrix.idf_ver }}
+          path: components/esp_websocket_client/examples/build/
+      - name: Install Python packages
+        env:
+          PIP_EXTRA_INDEX_URL: "https://www.piwheels.org/simple"
+        run: | 
+          pip install -r $GITHUB_WORKSPACE/components/esp_websocket_client/examples/requirements.txt
+      - name: Download Example Test to target
+        run: python -m esptool --chip ${{ matrix.idf_target }} write_flash 0x0 components/esp_websocket_client/examples/build/flash_image.bin
+      - name: Run Example Test on target
+        working-directory: components/esp_websocket_client/examples
+        run: |
+          cp sdkconfig.ci sdkconfig.defaults
+          pytest --log-cli-level DEBUG --junit-xml=./test_app_results_${{ matrix.idf_target }}_${{ matrix.idf_ver }}.xml --target=${{ matrix.idf_target }}
+      - uses: actions/upload-artifact@v2
+        if: always()
+        with:
+          name: examples_results_${{ matrix.idf_target }}_${{ matrix.idf_ver }}
+          path: examples/*.xml

components/esp_websocket_client/examples/example_test.py

@@ -1,146 +0,0 @@
-from __future__ import print_function, unicode_literals
-
-import os
-import random
-import re
-import socket
-import string
-from threading import Event, Thread
-
-import ttfw_idf
-from SimpleWebSocketServer import SimpleWebSocketServer, WebSocket
-from tiny_test_fw import Utility
-
-
-def get_my_ip():
-    s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
-    try:
-        # doesn't even have to be reachable
-        s.connect(('10.255.255.255', 1))
-        IP = s.getsockname()[0]
-    except Exception:
-        IP = '127.0.0.1'
-    finally:
-        s.close()
-    return IP
-
-
-class TestEcho(WebSocket):
-
-    def handleMessage(self):
-        self.sendMessage(self.data)
-        print('Server sent: {}'.format(self.data))
-
-    def handleConnected(self):
-        print('Connection from: {}'.format(self.address))
-
-    def handleClose(self):
-        print('{} closed the connection'.format(self.address))
-
-
-# Simple Websocket server for testing purposes
-class Websocket(object):
-
-    def send_data(self, data):
-        for nr, conn in self.server.connections.items():
-            conn.sendMessage(data)
-
-    def run(self):
-        self.server = SimpleWebSocketServer('', self.port, TestEcho)
-        while not self.exit_event.is_set():
-            self.server.serveonce()
-
-    def __init__(self, port):
-        self.port = port
-        self.exit_event = Event()
-        self.thread = Thread(target=self.run)
-        self.thread.start()
-
-    def __enter__(self):
-        return self
-
-    def __exit__(self, exc_type, exc_value, traceback):
-        self.exit_event.set()
-        self.thread.join(10)
-        if self.thread.is_alive():
-            Utility.console_log('Thread cannot be joined', 'orange')
-
-
-def test_echo(dut):
-    dut.expect('WEBSOCKET_EVENT_CONNECTED')
-    for i in range(0, 5):
-        dut.expect(re.compile(r'Received=hello (\d)'), timeout=30)
-    print('All echos received')
-
-
-def test_close(dut):
-    code = dut.expect(re.compile(r'WEBSOCKET: Received closed message with code=(\d*)'), timeout=60)[0]
-    print('Received close frame with code {}'.format(code))
-
-
-def test_recv_long_msg(dut, websocket, msg_len, repeats):
-    send_msg = ''.join(random.choice(string.ascii_uppercase + string.ascii_lowercase + string.digits) for _ in range(msg_len))
-
-    for _ in range(repeats):
-        websocket.send_data(send_msg)
-
-        recv_msg = ''
-        while len(recv_msg) < msg_len:
-            # Filter out color encoding
-            match = dut.expect(re.compile(r'Received=([a-zA-Z0-9]*).*\n'), timeout=30)[0]
-            recv_msg += match
-
-        if recv_msg == send_msg:
-            print('Sent message and received message are equal')
-        else:
-            raise ValueError('DUT received string do not match sent string, \nexpected: {}\nwith length {}\
-                            \nreceived: {}\nwith length {}'.format(send_msg, len(send_msg), recv_msg, len(recv_msg)))
-
-
-@ttfw_idf.idf_example_test(env_tag='Example_EthKitV1')
-def test_examples_protocol_websocket(env, extra_data):
-    """
-    steps:
-      1. obtain IP address
-      2. connect to uri specified in the config
-      3. send and receive data
-    """
-    dut1 = env.get_dut('websocket', 'examples/protocols/websocket', dut_class=ttfw_idf.ESP32DUT)
-    # check and log bin size
-    binary_file = os.path.join(dut1.app.binary_path, 'websocket_example.bin')
-    bin_size = os.path.getsize(binary_file)
-    ttfw_idf.log_performance('websocket_bin_size', '{}KB'.format(bin_size // 1024))
-
-    try:
-        if 'CONFIG_WEBSOCKET_URI_FROM_STDIN' in dut1.app.get_sdkconfig():
-            uri_from_stdin = True
-        else:
-            uri = dut1.app.get_sdkconfig()['CONFIG_WEBSOCKET_URI'].strip('"')
-            uri_from_stdin = False
-
-    except Exception:
-        print('ENV_TEST_FAILURE: Cannot find uri settings in sdkconfig')
-        raise
-
-    # start test
-    dut1.start_app()
-
-    if uri_from_stdin:
-        server_port = 4455
-        with Websocket(server_port) as ws:
-            uri = 'ws://{}:{}'.format(get_my_ip(), server_port)
-            print('DUT connecting to {}'.format(uri))
-            dut1.expect('Please enter uri of websocket endpoint', timeout=30)
-            dut1.write(uri)
-            test_echo(dut1)
-            # Message length should exceed DUT's buffer size to test fragmentation, default is 1024 byte
-            test_recv_long_msg(dut1, ws, 2000, 3)
-            test_close(dut1)
-
-    else:
-        print('DUT connecting to {}'.format(uri))
-        test_echo(dut1)
-
-
-if __name__ == '__main__':
-    test_examples_protocol_websocket()

components/esp_websocket_client/examples/main/websocket_example.c

@@ -103,7 +103,6 @@ static void websocket_app_start(void)
 
 #else
     websocket_cfg.uri = CONFIG_WEBSOCKET_URI;
-
 #endif /* CONFIG_WEBSOCKET_URI_FROM_STDIN */
 
     ESP_LOGI(TAG, "Connecting to %s...", websocket_cfg.uri);

components/esp_websocket_client/examples/pytest.ini

@@ -0,0 +1,24 @@
+[pytest]
+# only the files with prefix `pytest_` would be recognized as pytest test scripts.
+python_files = pytest_*.py
+
+addopts =
+  -s
+  --embedded-services esp,idf
+  -W ignore::_pytest.warning_types.PytestExperimentalApiWarning
+  --tb short
+
+# ignore DeprecationWarning
+filterwarnings =
+  ignore:Call to deprecated create function (.*)\(\):DeprecationWarning
+
+# log related
+log_cli = True
+log_cli_level = INFO
+log_cli_format = %(asctime)s %(levelname)s %(message)s
+log_cli_date_format = %Y-%m-%d %H:%M:%S
+
+log_file = test.log
+log_file_level = INFO
+log_file_format = %(asctime)s %(levelname)s %(message)s
+log_file_date_format = %Y-%m-%d %H:%M:%S

components/esp_websocket_client/examples/pytest_websocket.py

@@ -0,0 +1,128 @@
+import os
+import random
+import re
+import socket
+import string
+from threading import Event, Thread
+import pytest
+import sys
+
+from SimpleWebSocketServer import SimpleWebSocketServer, WebSocket
+from pytest_embedded import Dut
+
+
+def get_my_ip():
+    s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+    try:
+        # doesn't even have to be reachable
+        s.connect(('8.8.8.8', 1))
+        IP = s.getsockname()[0]
+    except Exception:
+        IP = '127.0.0.1'
+    finally:
+        s.close()
+    return IP
+
+
+class WebsocketTestEcho(WebSocket):
+
+    def handleMessage(self):
+        self.sendMessage(self.data)
+        print('Server sent: {}'.format(self.data))
+
+    def handleConnected(self):
+        print('Connection from: {}'.format(self.address))
+
+    def handleClose(self):
+        print('{} closed the connection'.format(self.address))
+
+
+# Simple Websocket server for testing purposes
+class Websocket(object):
+
+    def send_data(self, data):
+        for nr, conn in self.server.connections.items():
+            conn.sendMessage(data)
+
+    def run(self):
+        self.server = SimpleWebSocketServer('', self.port, WebsocketTestEcho)
+        while not self.exit_event.is_set():
+            self.server.serveonce()
+
+    def __init__(self, port):
+        self.port = port
+        self.exit_event = Event()
+        self.thread = Thread(target=self.run)
+        self.thread.start()
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.exit_event.set()
+        self.thread.join(10)
+        if self.thread.is_alive():
+            Utility.console_log('Thread cannot be joined', 'orange')
+
+
+def test_examples_protocol_websocket(dut):
+    """
+    steps:
+      1. obtain IP address
+      2. connect to uri specified in the config
+      3. send and receive data
+    """
+    def test_echo(dut):
+        dut.expect('WEBSOCKET_EVENT_CONNECTED')
+        for i in range(0, 5):
+            dut.expect(re.compile(b'Received=hello (\\d)'))
+        print('All echos received')
+
+    def test_close(dut):
+        code = dut.expect(re.compile(b'WEBSOCKET: Received closed message with code=(\\d*)'))[0]
+        print('Received close frame with code {}'.format(code))
+ 
+    def test_recv_long_msg(dut, websocket, msg_len, repeats):
+        send_msg = ''.join(random.choice(string.ascii_uppercase + string.ascii_lowercase + string.digits) for _ in range(msg_len))
+
+        for _ in range(repeats):
+            websocket.send_data(send_msg)
+    
+            recv_msg = ''
+            while len(recv_msg) < msg_len:
+                match = dut.expect(re.compile(b'Received=([a-zA-Z0-9]*).*')).group(1).decode()
+                recv_msg += match
+    
+            if recv_msg == send_msg:
+                print('Sent message and received message are equal')
+            else:
+                raise ValueError('DUT received string do not match sent string, \nexpected: {}\nwith length {}\
+                                \nreceived: {}\nwith length {}'.format(send_msg, len(send_msg), recv_msg, len(recv_msg)))
+    
+    # Starting of the test
+    try:
+        if dut.app.sdkconfig.get('WEBSOCKET_URI_FROM_STDIN') is True:
+            uri_from_stdin = True
+        else:
+            uri = dut.app.sdkconfig['WEBSOCKET_URI']
+            uri_from_stdin = False
+
+    except Exception:
+        print('ENV_TEST_FAILURE: Cannot find uri settings in sdkconfig')
+        raise
+
+    if uri_from_stdin:
+        server_port = 8080
+        with Websocket(server_port) as ws:
+            uri = 'ws://{}:{}'.format(get_my_ip(), server_port)
+            print('DUT connecting to {}'.format(uri))
+            dut.expect('Please enter uri of websocket endpoint', timeout=30)
+            dut.write(uri)
+            test_echo(dut)
+            # Message length should exceed DUT's buffer size to test fragmentation, default is 1024 byte
+            test_recv_long_msg(dut, ws, 2000, 3)
+            test_close(dut)
+    else:
+        print('DUT connecting to {}'.format(uri))
+        test_echo(dut)
+

components/esp_websocket_client/examples/requirements.txt

@@ -0,0 +1,4 @@
+pytest-embedded-serial-esp 
+pytest-embedded-idf 
+junit_xml 
+SimpleWebSocketServer

components/esp_websocket_client/examples/sdkconfig.ci

@@ -1,5 +1,6 @@
-CONFIG_WEBSOCKET_URI_FROM_STDIN=y
-CONFIG_WEBSOCKET_URI_FROM_STRING=n
+CONFIG_WEBSOCKET_URI_FROM_STDIN=n
+CONFIG_WEBSOCKET_URI_FROM_STRING=y
+CONFIG_WEBSOCKET_URI="ws://echo.websocket.events"
 CONFIG_EXAMPLE_CONNECT_ETHERNET=y
 CONFIG_EXAMPLE_CONNECT_WIFI=n
 CONFIG_EXAMPLE_USE_INTERNAL_ETHERNET=y