feat(examples): websocket autobahn test suit integration

This commit is contained in:
Suren Gabrielyan
2025-11-03 12:13:43 +04:00
committed by surengab
parent cdf064dc65
commit 0f10215124
24 changed files with 2656 additions and 1 deletions
+254
View File
@@ -0,0 +1,254 @@
name: "websocket-autobahn: build/linux-tests"
on:
push:
branches:
- master
pull_request:
types: [opened, synchronize, reopened, labeled]
jobs:
linux_websocket_autobahn:
runs-on: ubuntu-22.04
if: github.event_name == 'push' || (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'websocket-autobahn'))
env:
TEST_DIR: components/esp_websocket_client/tests/autobahn-testsuite
TESTEE_DIR: components/esp_websocket_client/tests/autobahn-testsuite/testee
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Start Autobahn Fuzzing Server
run: |
mkdir -p ${{ env.TEST_DIR }}/reports/clients
HOST_IP=$(ip route get 8.8.8.8 | grep -oP 'src \K\S+' || hostname -I | awk '{print $1}' || echo "172.17.0.1")
echo "Host IP address: $HOST_IP"
echo "HOST_IP=$HOST_IP" >> $GITHUB_ENV
docker run -d \
--name fuzzing-server \
--network host \
-v ${{ github.workspace }}/${{ env.TEST_DIR }}/config:/config:ro \
-v ${{ github.workspace }}/${{ env.TEST_DIR }}/reports:/reports \
crossbario/autobahn-testsuite:latest \
wstest -m fuzzingserver -s /config/fuzzingserver.json
echo "Waiting for fuzzing server..."
for i in {1..5}; do
if curl -f http://localhost:9001/info >/dev/null 2>&1; then
echo "Fuzzing server ready!"
exit 0
fi
echo "Attempt $i/5 waiting 2s..."
sleep 2
done
echo "Server start failed. Container logs:"
docker logs fuzzing-server
exit 1
- name: Build test (Linux target)
working-directory: ${{ env.TESTEE_DIR }}
run: |
docker run --rm \
-v ${{ github.workspace }}:/work \
-w /work/${{ env.TESTEE_DIR }} \
espressif/idf:latest \
bash -c "
. \$IDF_PATH/export.sh
cp sdkconfig.ci.linux sdkconfig.defaults
echo 'Building...'
idf.py build
"
- name: Verify fuzzing server connectivity
run: |
HOST_IP=${HOST_IP:-$(ip route get 8.8.8.8 | grep -oP 'src \K\S+' || hostname -I | awk '{print $1}' || echo "172.17.0.1")}
echo "Testing connectivity to $HOST_IP:9001"
docker run --rm \
--network host \
espressif/idf:latest \
bash -c "
curl -f http://$HOST_IP:9001/info || {
echo 'ERROR: Cannot connect to fuzzing server at $HOST_IP:9001'
exit 1
}
echo 'Fuzzing server is accessible'
"
- name: Run Autobahn tests
run: |
docker run --rm \
--network host \
-v ${{ github.workspace }}:/work \
-w /work/${{ env.TESTEE_DIR }}/build \
espressif/idf:latest \
bash -c "
apt-get update && apt-get install -y file curl net-tools || true
HOST_IP=\$(ip route get 8.8.8.8 2>/dev/null | grep -oP 'src \\K\\S+' || hostname -I | awk '{print \$1}' || echo '172.17.0.1')
curl -f http://\${HOST_IP}:9001/info || {
echo \"ERROR: Server not reachable at \${HOST_IP}:9001\"
exit 1
}
echo 'Running autobahn_testee.elf'
WS_URI=\"ws://\${HOST_IP}:9001\"
echo \"WebSocket URI: \${WS_URI}\"
(sleep 0.5; printf \"\${WS_URI}\\n\") | timeout 30m ./autobahn_testee.elf || {
EXIT_CODE=\$?
echo 'Test failed'
exit \$EXIT_CODE
}
echo 'All Autobahn tests passed!'
"
- name: Show reports
if: always()
working-directory: ${{ env.TEST_DIR }}
run: |
if [ -d reports/clients ]; then
ls -la reports/clients/
else
echo "No reports"
fi
- name: Generate summary
if: always()
working-directory: ${{ env.TEST_DIR }}
run: python3 scripts/generate_summary.py || true
- name: Upload reports
if: always()
uses: actions/upload-artifact@v4
with:
name: autobahn-reports-linux-${{ github.run_id }}
path: ${{ env.TEST_DIR }}/reports/**
if-no-files-found: warn
linux_websocket_autobahn_perf:
runs-on: ubuntu-22.04
# Run only if the specific label is present
if: contains(github.event.pull_request.labels.*.name, 'websocket-autobahn-perf')
env:
TEST_DIR: components/esp_websocket_client/tests/autobahn-testsuite
TESTEE_DIR: components/esp_websocket_client/tests/autobahn-testsuite/testee
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Start Autobahn Fuzzing Server (Performance)
run: |
mkdir -p ${{ env.TEST_DIR }}/reports/clients
HOST_IP=$(ip route get 8.8.8.8 | grep -oP 'src \K\S+' || hostname -I | awk '{print $1}' || echo "172.17.0.1")
echo "Host IP address: $HOST_IP"
echo "HOST_IP=$HOST_IP" >> $GITHUB_ENV
docker run -d \
--name fuzzing-server-perf \
--network host \
-v ${{ github.workspace }}/${{ env.TEST_DIR }}/config:/config:ro \
-v ${{ github.workspace }}/${{ env.TEST_DIR }}/reports:/reports \
crossbario/autobahn-testsuite:latest \
wstest -m fuzzingserver -s /config/fuzzingserver-perf.json
echo "Waiting for fuzzing server..."
for i in {1..5}; do
if curl -f http://localhost:9001/info >/dev/null 2>&1; then
echo "Fuzzing server ready!"
exit 0
fi
echo "Attempt $i/5 waiting 2s..."
sleep 2
done
echo "Server start failed. Container logs:"
docker logs fuzzing-server-perf
exit 1
- name: Build test (Linux target)
working-directory: ${{ env.TESTEE_DIR }}
run: |
docker run --rm \
-v ${{ github.workspace }}:/work \
-w /work/${{ env.TESTEE_DIR }} \
espressif/idf:latest \
bash -c "
. \$IDF_PATH/export.sh
cp sdkconfig.ci.linux sdkconfig.defaults
echo 'Building...'
idf.py build
"
- name: Verify fuzzing server connectivity
run: |
HOST_IP=${HOST_IP:-$(ip route get 8.8.8.8 | grep -oP 'src \K\S+' || hostname -I | awk '{print $1}' || echo "172.17.0.1")}
echo "Testing connectivity to $HOST_IP:9001"
docker run --rm \
--network host \
espressif/idf:latest \
bash -c "
curl -f http://$HOST_IP:9001/info || {
echo 'ERROR: Cannot connect to fuzzing server at $HOST_IP:9001'
exit 1
}
echo 'Fuzzing server is accessible'
"
- name: Run Autobahn Performance tests
run: |
docker run --rm \
--network host \
-v ${{ github.workspace }}:/work \
-w /work/${{ env.TESTEE_DIR }}/build \
espressif/idf:latest \
bash -c "
apt-get update && apt-get install -y file curl net-tools || true
HOST_IP=\$(ip route get 8.8.8.8 2>/dev/null | grep -oP 'src \\K\\S+' || hostname -I | awk '{print \$1}' || echo '172.17.0.1')
curl -f http://\${HOST_IP}:9001/info || {
echo \"ERROR: Server not reachable at \${HOST_IP}:9001\"
exit 1
}
echo 'Running autobahn_testee.elf'
WS_URI=\"ws://\${HOST_IP}:9001\"
echo \"WebSocket URI: \${WS_URI}\"
(sleep 0.5; printf \"\${WS_URI}\\n\") | timeout 60m ./autobahn_testee.elf || {
EXIT_CODE=\$?
echo 'Test failed'
exit \$EXIT_CODE
}
echo 'All Autobahn performance tests passed!'
"
- name: Show reports
if: always()
working-directory: ${{ env.TEST_DIR }}
run: |
if [ -d reports/clients ]; then
ls -la reports/clients/
else
echo "No reports"
fi
- name: Generate summary
if: always()
working-directory: ${{ env.TEST_DIR }}
run: python3 scripts/generate_summary.py || true
- name: Upload reports
if: always()
uses: actions/upload-artifact@v4
with:
name: autobahn-perf-reports-linux-${{ github.run_id }}
path: ${{ env.TEST_DIR }}/reports/**
if-no-files-found: warn
+226
View File
@@ -0,0 +1,226 @@
name: "websocket-autobahn: build/target-tests"
on:
push:
branches:
- master
pull_request:
types: [opened, synchronize, reopened, labeled]
jobs:
build_websocket_autobahn:
if: contains(github.event.pull_request.labels.*.name, 'websocket-autobahn') || github.event_name == 'push'
name: Build
strategy:
matrix:
idf_ver: ["release-v5.5", "latest"]
idf_target: ["esp32"]
runs-on: ubuntu-22.04
container: espressif/idf:${{ matrix.idf_ver }}
env:
TEST_DIR: components/esp_websocket_client/tests/autobahn-testsuite
TESTEE_DIR: components/esp_websocket_client/tests/autobahn-testsuite/testee
steps:
- name: Checkout esp-protocols
uses: actions/checkout@v4
with:
submodules: recursive
- name: Build autobahn testee with IDF-${{ matrix.idf_ver }} for ${{ matrix.idf_target }}
working-directory: ${{ env.TESTEE_DIR }}
env:
IDF_TARGET: ${{ matrix.idf_target }}
shell: bash
run: |
. ${IDF_PATH}/export.sh
test -f sdkconfig.ci.target.plain_tcp && cat sdkconfig.ci.target.plain_tcp >> sdkconfig.defaults || echo "No sdkconfig.ci.plain_tcp"
idf.py set-target ${{ matrix.idf_target }}
idf.py build
- name: Merge binaries with IDF-${{ matrix.idf_ver }} for ${{ matrix.idf_target }}
working-directory: ${{ env.TESTEE_DIR }}/build
env:
IDF_TARGET: ${{ matrix.idf_target }}
shell: bash
run: |
. ${IDF_PATH}/export.sh
esptool.py --chip ${{ matrix.idf_target }} merge_bin --fill-flash-size 4MB -o flash_image.bin @flash_args
- uses: actions/upload-artifact@v4
with:
name: autobahn_testee_bin_${{ matrix.idf_target }}_${{ matrix.idf_ver }}
path: |
${{ env.TESTEE_DIR }}/build/bootloader/bootloader.bin
${{ env.TESTEE_DIR }}/build/partition_table/partition-table.bin
${{ env.TESTEE_DIR }}/build/*.bin
${{ env.TESTEE_DIR }}/build/*.elf
${{ env.TESTEE_DIR }}/build/flasher_args.json
${{ env.TESTEE_DIR }}/build/config/sdkconfig.h
${{ env.TESTEE_DIR }}/build/config/sdkconfig.json
if-no-files-found: error
# run-target-autobahn:
# # Skip running on forks since it won't have access to secrets
# if: |
# github.repository == 'espressif/esp-protocols' &&
# ( contains(github.event.pull_request.labels.*.name, 'autobahn') || github.event_name == 'push' )
# name: Target test
# needs: build_autobahn
# strategy:
# fail-fast: false
# matrix:
# idf_ver: ["latest"]
# idf_target: ["esp32"]
# runs-on:
# - self-hosted
# - ESP32-ETHERNET-KIT
# env:
# TEST_DIR: components/esp_websocket_client/examples/autobahn-testsuite
# TESTEE_DIR: components/esp_websocket_client/examples/autobahn-testsuite/testee
# steps:
# - uses: actions/checkout@v4
# with:
# submodules: recursive
# - uses: actions/download-artifact@v4
# with:
# name: autobahn_testee_bin_${{ matrix.idf_target }}_${{ matrix.idf_ver }}
# path: ${{ env.TESTEE_DIR }}/build
# - name: Install Docker Compose
# run: |
# sudo apt-get update
# sudo apt-get install -y docker-compose-plugin || sudo apt-get install -y docker-compose
# # Ensure user has permission to use Docker (if not already in docker group)
# sudo usermod -aG docker $USER || true
# # Start Docker service if not running
# sudo systemctl start docker || true
# - name: Start Autobahn Fuzzing Server
# working-directory: ${{ env.TEST_DIR }}
# run: |
# # Get host IP address for ESP32 to connect to
# HOST_IP=$(hostname -I | awk '{print $1}')
# echo "HOST_IP=$HOST_IP" >> $GITHUB_ENV
# echo "Autobahn server will be accessible at ws://$HOST_IP:9001"
#
# # Start the fuzzing server using pre-built image
# # For CI, we may need to specify platform if architecture differs
# echo "Starting Autobahn fuzzing server..."
# # Set platform for CI if needed (uncomment if you get exec format error)
# # export DOCKER_DEFAULT_PLATFORM=linux/amd64
# docker compose up -d || docker-compose up -d
#
# # Wait for server to be ready
# echo "Waiting for fuzzing server to start..."
# sleep 10
#
# # Check if container is running and healthy
# if ! docker ps | grep -q ws-fuzzing-server; then
# echo "Error: Fuzzing server failed to start"
# echo "Container logs:"
# docker compose logs || docker-compose logs
# echo "Checking available Python executables in container:"
# docker compose run --rm fuzzing-server which python python3 || true
# exit 1
# fi
#
# # Verify the server is actually responding
# echo "Checking if server is responding..."
# sleep 5
# if ! curl -s http://localhost:8080 > /dev/null 2>&1; then
# echo "Warning: Server may not be fully ready, but container is running"
# docker compose logs --tail=20 || docker-compose logs --tail=20
# fi
#
# echo "✓ Fuzzing server started successfully"
# - name: Flash ESP32 Testee
# working-directory: ${{ env.TESTEE_DIR }}/build
# env:
# IDF_TARGET: ${{ matrix.idf_target }}
# run: |
# python -m esptool --chip ${{ matrix.idf_target }} write_flash 0x0 flash_image.bin
# - name: Run Autobahn Tests
# working-directory: ${{ env.TESTEE_DIR }}
# env:
# PIP_EXTRA_INDEX_URL: "https://www.piwheels.org/simple"
# run: |
# # Detect ESP32 port if not set in environment
# if [ -z "${ESP_PORT:-}" ]; then
# for port in /dev/ttyUSB* /dev/ttyACM*; do
# if [ -e "$port" ]; then
# export ESP_PORT="$port"
# echo "Detected ESP32 port: $ESP_PORT"
# break
# fi
# done
# fi
#
# # Default to /dev/ttyUSB0 if still not found
# export ESP_PORT="${ESP_PORT:-/dev/ttyUSB0}"
#
# if [ ! -e "$ESP_PORT" ]; then
# echo "Error: ESP32 port not found. Please set ESP_PORT environment variable."
# echo "Available ports:"
# ls -la /dev/tty* || true
# exit 1
# fi
#
# echo "Using ESP32 port: $ESP_PORT"
# export PYENV_ROOT="$HOME/.pyenv"
# export PATH="$PYENV_ROOT/bin:$PATH"
# eval "$(pyenv init --path)"
# eval "$(pyenv init -)"
# if ! pyenv versions --bare | grep -q '^3\.12\.6$'; then
# echo "Installing Python 3.12.6..."
# pyenv install -s 3.12.6
# fi
# if ! pyenv virtualenvs --bare | grep -q '^myenv$'; then
# echo "Creating pyenv virtualenv 'myenv'..."
# pyenv virtualenv 3.12.6 myenv
# fi
# pyenv activate myenv
# python --version
# pip install --prefer-binary pytest-embedded pytest-embedded-serial-esp pytest-embedded-idf pytest-custom_exit_code esptool pyserial
# pip install --extra-index-url https://dl.espressif.com/pypi/ -r $GITHUB_WORKSPACE/ci/requirements.txt
#
# echo "Starting Autobahn test suite on ESP32..."
# echo "Tests may take 15-30 minutes to complete..."
#
# # Send server URI via serial (stdin) and monitor for completion
# # Script is in the parent directory (TEST_DIR) from TESTEE_DIR
# SERVER_URI="ws://$HOST_IP:9001"
# echo "Sending server URI to ESP32: $SERVER_URI"
# python3 ../scripts/monitor_serial.py --port "$ESP_PORT" --uri "$SERVER_URI" --timeout 2400
# - name: Collect Test Reports
# working-directory: ${{ env.TEST_DIR }}
# if: always()
# run: |
# # Stop the fuzzing server
# docker compose down || docker-compose down
#
# # Check if reports were generated
# if [ -d "reports/clients" ]; then
# echo "✓ Test reports found"
# ls -la reports/clients/
# else
# echo "⚠ No test reports found in reports/clients/"
# fi
# - name: Generate Test Summary
# working-directory: ${{ env.TEST_DIR }}
# if: always()
# run: |
# # Generate summary from test results
# # Check for JSON files in both reports/ and reports/clients/
# if [ -d "reports" ] && ( [ -n "$(ls -A reports/*.json 2>/dev/null)" ] || [ -n "$(ls -A reports/clients/*.json 2>/dev/null)" ] ); then
# echo "Generating test summary..."
# python3 scripts/generate_summary.py
# echo ""
# echo "Summary generated successfully!"
# if [ -f "reports/summary.html" ]; then
# echo "HTML summary available at: reports/summary.html"
# fi
# else
# echo "⚠ No JSON test results found, skipping summary generation"
# fi
# - uses: actions/upload-artifact@v4
# if: always()
# with:
# name: autobahn_reports_${{ matrix.idf_target }}_${{ matrix.idf_ver }}
# path: |
# ${{ env.TEST_DIR }}/reports/**
# if-no-files-found: warn
@@ -8,6 +8,7 @@
#include <stdbool.h>
#include <stdint.h>
#include "bsd/string.h"
#include "esp_err.h"
#ifdef __cplusplus
extern "C" {
@@ -0,0 +1,130 @@
# Autobahn WebSocket Testsuite for esp_websocket_client
This directory contains the setup for testing `esp_websocket_client` against the [Autobahn WebSocket Testsuite](https://github.com/crossbario/autobahn-testsuite), which covers frame parsing, control frames, UTF-8 validation, and more.
The Autobahn Testsuite is the de facto standard for testing WebSocket protocol compliance. It runs over 500 test cases covering:
- Frame parsing and generation
- Text and binary messages
- Fragmentation
- Control frames (PING, PONG, CLOSE)
- UTF-8 validation
- Protocol violations
- Edge cases and error handling
## 📋 Prerequisites
1. **Docker** - For running the Autobahn testsuite server.
2. **ESP32 device** with WiFi capability.
3. **ESP-IDF** development environment.
4. **Network** - ESP32 and Docker host on the same network.
## 🚀 Quick Start
### 1. Start the Fuzzing Server
```bash
cd autobahn-testsuite
docker-compose up -d
```
Starts server on port 9001 and web interface on port 8080.
### 2. Configure ESP32 Client
**Required:** Set WiFi credentials in `testee/sdkconfig.defaults` or via `idf.py menuconfig`:
```
CONFIG_EXAMPLE_WIFI_SSID="YourWiFiSSID"
CONFIG_EXAMPLE_WIFI_PASSWORD="YourWiFiPassword"
```
**Optional:** The Server URI is automatically detected and sent by the test script (via `CONFIG_WEBSOCKET_URI_FROM_STDIN`).
To hardcode it instead, disable `CONFIG_WEBSOCKET_URI_FROM_STDIN` and set:
```
CONFIG_AUTOBAHN_SERVER_URI="ws://YOUR_HOST_IP:9001"
```
### 3. Run Tests (Automated)
The `run_tests.sh` script handles building, flashing, and monitoring:
```bash
./run_tests.sh /dev/ttyUSB0
```
*Note: Replace `/dev/ttyUSB0` with your serial port.*
**Using Pytest Manually:**
You can also run the monitoring step directly using pytest:
```bash
pytest pytest_autobahn.py --target esp32 --port /dev/ttyUSB0 --baud 115200 --app-path testee --skip-autoflash y
```
### 4. View Results
- **Web Interface:** `http://localhost:8080` (Click "Client Reports")
- **Summary Report:** Run `python3 scripts/generate_summary.py`
## 📂 Directory Structure
```
autobahn-testsuite/
├── docker-compose.yml # Docker config
├── run_tests.sh # Automated runner script
├── pytest_autobahn.py # Pytest monitoring script
├── config/ # Server config
├── scripts/ # Helper scripts
├── reports/ # Test results
└── testee/ # ESP32 client application
├── sdkconfig.defaults # Default config
├── sdkconfig.ci.* # CI configurations
└── main/ # Source code
```
## ⚙️ Configuration
### Testee
- `sdkconfig.ci.linux`: CI configuration for running the testee on a Linux host (host-based testing), with adjusted stack sizes and lock timeouts.
- `sdkconfig.ci.target.plain_tcp`: CI configuration for ESP32 target using Ethernet (IP101 PHY) and IPv6, without TLS.
### Fuzzing Server
Edit `config/fuzzingserver.json` to include/exclude cases (e.g., exclude performance/compression tests).
## 📊 Test Categories
| Category | Tests | Time | Critical? | Notes |
|----------|-------|------|-----------|-------|
| 1.* Framing | ~64 | 3m | ✅ Yes | Core compliance |
| 2.* Pings/Pongs | ~11 | 1m | ✅ Yes | Keepalive |
| 3.* Reserved Bits | ~7 | 1m | ✅ Yes | Extensions |
| 4.* Opcodes | ~10 | 1m | ✅ Yes | Protocol compliance |
| 5.* Fragmentation | ~20 | 2m | ✅ Yes | Large messages |
| 6.* UTF-8 | ~150 | 10m | ⚠️ Optional | strict validation often skipped |
| 7.* Close | ~35 | 3m | ✅ Yes | Clean disconnect |
| 9.* Performance | ~30 | 60m+ | ⚠️ Optional | Often excluded (resource intensive) |
| 10.* Misc | ~10 | 2m | ✅ Yes | Edge cases |
| 12-13.* Compression | ~200 | 30m | ❌ No | Typically not implemented |
## 💡 Tips
## 🧾 Generate Summary
After running the tests, Autobahn writes per-case JSON results under `reports/` (and/or `reports/clients/`).
You can generate a consolidated console + HTML summary using:
```bash
cd components/esp_websocket_client/tests/autobahn-testsuite
python3 scripts/generate_summary.py
```
This will:
- Print overall pass/fail statistics and per-category breakdown to the console
- Generate `reports/summary.html`
If youre running in CI with GitHub Actions, the script will also write a markdown summary to the path in `GITHUB_STEP_SUMMARY` (when that env var is set).
## 📚 References
- [Autobahn Testsuite Documentation](https://crossbar.io/autobahn/)
- [RFC 6455 - The WebSocket Protocol](https://tools.ietf.org/html/rfc6455)
- [Autobahn Testsuite GitHub](https://github.com/crossbario/autobahn-testsuite)
@@ -0,0 +1,11 @@
{
"url": "ws://0.0.0.0:9001",
"options": {
"failByDrop": false
},
"outdir": "/reports",
"webport": 8080,
"cases": ["9.*"],
"exclude-cases": [],
"exclude-agent-cases": {}
}
@@ -0,0 +1,15 @@
{
"url": "ws://0.0.0.0:9001",
"options": {
"failByDrop": false
},
"outdir": "/reports",
"webport": 8080,
"cases": ["*"],
"exclude-cases": [
"9.*",
"12.*",
"13.*"
],
"exclude-agent-cases": {}
}
@@ -0,0 +1,11 @@
# SPDX-FileCopyrightText: 2025 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Unlicense OR CC0-1.0
def pytest_addoption(parser):
parser.addoption(
"--uri",
action="store",
default=None,
help="Server URI to send via serial (stdin)",
)
@@ -0,0 +1,12 @@
services:
fuzzing-server:
image: crossbario/autobahn-testsuite:latest
container_name: ws-fuzzing-server
platform: linux/amd64 # <— enforce amd64, use QEMU on Apple Silicon
ports:
- "9001:9001"
- "8080:8080"
volumes:
- ./config:/config
- ./reports:/reports
command: wstest -m fuzzingserver -s /config/fuzzingserver.json
@@ -0,0 +1,59 @@
# SPDX-FileCopyrightText: 2025 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Unlicense OR CC0-1.0
import re
import time
import pytest
@pytest.mark.supported_targets
@pytest.mark.generic
def test_autobahn(dut, request):
"""
Monitors ESP32 serial output and detects test completion.
Equivalent to monitor_serial.py but using pytest-embedded.
"""
uri = request.config.getoption("--uri")
# If URI is provided, send it via serial (stdin)
if uri:
# Wait for ESP32 to be ready (booting/connecting)
time.sleep(5)
dut.write(f'{uri}\n')
print(f'Sent URI: {uri}')
# regex patterns
progress_pattern = re.compile(rb'Case (\d+)/(\d+)')
completion_pattern = re.compile(rb'All tests completed\.')
last_case = None
start_time = time.time()
# 40 minutes timeout as in monitor_serial.py
timeout = 2400
print('\n--- Waiting for ESP32 output (tests should start shortly) ---\n')
while (time.time() - start_time) < timeout:
try:
# Expect either progress or completion
# timeout=30 to allow checking total timeout loop regularly
match = dut.expect([progress_pattern, completion_pattern], timeout=30)
except Exception:
# Timeout on single expect call, just continue loop
continue
# Check which pattern matched
# Note: match is a match object from the regex
if match.re == completion_pattern:
print('\n✓ Test suite completed successfully!')
return
if match.re == progress_pattern:
current_case = int(match.group(1))
total_cases = int(match.group(2))
if current_case != last_case:
last_case = current_case
print(f'Progress: Test case {current_case}/{total_cases} ({100*current_case//total_cases}%)')
pytest.fail(f"Timeout of {timeout}s reached without completion")
@@ -0,0 +1,277 @@
#!/bin/bash
# Autobahn Testsuite Runner Script
# This script automates the process of running WebSocket protocol compliance tests
set -e
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
cd "$SCRIPT_DIR"
# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
CYAN='\033[0;36m'
NC='\033[0m' # No Color
echo -e "${BLUE}======================================${NC}"
echo -e "${BLUE}Autobahn WebSocket Testsuite Runner${NC}"
echo -e "${BLUE}======================================${NC}"
echo
# Check if Docker is running
if ! docker info > /dev/null 2>&1; then
echo -e "${RED}Error: Docker is not running${NC}"
echo "Please start Docker and try again"
exit 1
fi
# Function to get host IP
get_host_ip() {
if [[ "$OSTYPE" == "darwin"* ]]; then
# macOS
ifconfig | grep "inet " | grep -v 127.0.0.1 | awk '{print $2}' | head -1
else
# Linux
hostname -I | awk '{print $1}'
fi
}
HOST_IP=$(get_host_ip)
echo -e "${GREEN}Step 1: Starting Autobahn Fuzzing Server${NC}"
echo "Host IP detected: $HOST_IP"
echo
# Start the fuzzing server
docker-compose up -d
# Wait for server to be ready
echo "Waiting for fuzzing server to start..."
sleep 5
# Check if container is running
if ! docker ps | grep -q ws-fuzzing-server; then
echo -e "${RED}Error: Fuzzing server failed to start${NC}"
docker-compose logs
exit 1
fi
echo -e "${GREEN}✓ Fuzzing server started successfully${NC}"
echo " WebSocket endpoint: ws://$HOST_IP:9001"
echo " Web interface: http://$HOST_IP:8080"
echo
echo -e "${YELLOW}Step 2: Running Tests on ESP32${NC}"
echo
# Check for ESP32 port argument
ESP_PORT="${1:-}"
BUILD_ONLY="${2:-}"
if [ -z "$ESP_PORT" ]; then
echo -e "${YELLOW}No serial port specified. Manual mode:${NC}"
echo
echo "To run tests automatically, provide the ESP32 serial port:"
echo " ./run_tests.sh /dev/ttyUSB0"
echo
echo "Or build only:"
echo " ./run_tests.sh /dev/ttyUSB0 build"
echo
echo "Manual steps:"
echo " 1. Update WiFi credentials in testee/sdkconfig.defaults"
echo " 2. Update Autobahn server URI to: ws://$HOST_IP:9001"
echo " 3. Build and flash:"
echo " cd testee"
echo " idf.py build"
echo " idf.py -p PORT flash monitor"
echo
echo -e "${YELLOW}Press Enter when ESP32 testing is complete...${NC}"
read
else
echo "ESP32 serial port: $ESP_PORT"
echo
# Check if ESP-IDF is available
if ! command -v idf.py > /dev/null 2>&1; then
echo -e "${RED}Error: ESP-IDF not found in PATH${NC}"
echo "Please source ESP-IDF environment: . \$HOME/esp/esp-idf/export.sh"
exit 1
fi
# Check if port exists
if [ ! -e "$ESP_PORT" ]; then
echo -e "${RED}Error: Serial port $ESP_PORT does not exist${NC}"
exit 1
fi
cd testee
# Verify we're in the right directory
if [ ! -f "sdkconfig.defaults" ] && [ ! -f "sdkconfig" ]; then
echo -e "${RED}Error: Not in testee directory or sdkconfig files not found${NC}"
echo " Current directory: $(pwd)"
exit 1
fi
# Update server URI in sdkconfig.defaults if needed
if [ -f "sdkconfig.defaults" ]; then
if ! grep -q "CONFIG_AUTOBAHN_SERVER_URI.*$HOST_IP" sdkconfig.defaults 2>/dev/null; then
echo -e "${YELLOW}Updating server URI in sdkconfig.defaults...${NC}"
# Backup original
cp sdkconfig.defaults sdkconfig.defaults.bak 2>/dev/null || true
# Update URI (simple approach - user should verify)
if [[ "$OSTYPE" == "darwin"* ]]; then
# macOS sed requires different syntax
sed -i '' "s|CONFIG_AUTOBAHN_SERVER_URI=.*|CONFIG_AUTOBAHN_SERVER_URI=\"ws://$HOST_IP:9001\"|" sdkconfig.defaults 2>/dev/null || true
else
sed -i.bak "s|CONFIG_AUTOBAHN_SERVER_URI=.*|CONFIG_AUTOBAHN_SERVER_URI=\"ws://$HOST_IP:9001\"|" sdkconfig.defaults 2>/dev/null || true
fi
fi
fi
echo -e "${GREEN}Building testee...${NC}"
idf.py build
if [ "$BUILD_ONLY" = "build" ]; then
echo -e "${GREEN}Build complete. Flash manually with:${NC}"
echo " idf.py -p $ESP_PORT flash monitor"
cd ..
exit 0
fi
echo -e "${GREEN}Flashing ESP32...${NC}"
idf.py -p "$ESP_PORT" flash
echo
echo -e "${YELLOW}Waiting for ESP32 to boot (5 seconds)...${NC}"
sleep 5
# Check if CONFIG_WEBSOCKET_URI_FROM_STDIN is enabled
# We're in testee directory, so check files here
URI_FROM_STDIN=""
if [ -f "sdkconfig" ] && grep -q "CONFIG_WEBSOCKET_URI_FROM_STDIN=y" sdkconfig 2>/dev/null; then
URI_FROM_STDIN="ws://$HOST_IP:9001"
echo -e "${GREEN}✓ CONFIG_WEBSOCKET_URI_FROM_STDIN is enabled (from sdkconfig)${NC}"
echo " URI will be sent via serial"
elif [ -f "sdkconfig.defaults" ] && grep -q "CONFIG_WEBSOCKET_URI_FROM_STDIN=y" sdkconfig.defaults 2>/dev/null; then
URI_FROM_STDIN="ws://$HOST_IP:9001"
echo -e "${GREEN}✓ CONFIG_WEBSOCKET_URI_FROM_STDIN is enabled (from sdkconfig.defaults)${NC}"
echo " URI will be sent via serial"
else
echo -e "${YELLOW}⚠ CONFIG_WEBSOCKET_URI_FROM_STDIN is not enabled${NC}"
echo " ESP32 will use URI from sdkconfig.defaults"
echo " Make sure CONFIG_AUTOBAHN_SERVER_URI is set correctly"
fi
echo
echo -e "${GREEN}Starting test execution...${NC}"
echo
echo -e "${YELLOW}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
echo -e "${BLUE}Serial output (tests in progress):${NC}"
echo -e "${YELLOW}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
echo
echo -e "${YELLOW}Note: Connection errors are expected for some test cases${NC}"
echo " (e.g., error handling tests). The testee will continue automatically."
echo
echo -e "${GREEN}Look for test case progress messages like:${NC}"
echo " 'Starting test case X...'"
echo " 'Test case X completed'"
echo
echo -e "${YELLOW}If no output appears, try:${NC}"
echo " - Press RESET button on ESP32"
echo " - Check WiFi credentials in sdkconfig.defaults"
echo " - Verify serial port: $ESP_PORT"
echo
echo -e "${YELLOW}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
echo
cd ..
# Detect target from sdkconfig if available
TARGET="esp32"
if [ -f "testee/sdkconfig" ]; then
if grep -q "CONFIG_IDF_TARGET=" testee/sdkconfig; then
TARGET=$(grep "CONFIG_IDF_TARGET=" testee/sdkconfig | cut -d'"' -f2)
fi
fi
# Use pytest-embedded to monitor and detect completion
# Only send URI if CONFIG_WEBSOCKET_URI_FROM_STDIN is enabled
# We use --app-path testee so pytest finds the build artifacts
# We use --skip-autoflash y because we already flashed above
PYTEST_ARGS="pytest_autobahn.py --target $TARGET --port $ESP_PORT --baud 115200 --app-path testee --skip-autoflash y"
if [ ! -z "$URI_FROM_STDIN" ]; then
PYTEST_ARGS="$PYTEST_ARGS --uri $URI_FROM_STDIN"
fi
if pytest $PYTEST_ARGS; then
echo
echo -e "${GREEN}✓ Tests completed successfully!${NC}"
echo " Check the web interface for detailed results"
else
echo
echo -e "${YELLOW}⚠ Test execution ended (timeout or error)${NC}"
echo " This may be normal if:"
echo " - Tests are still running (check serial output)"
echo " - Connection errors occurred (some are expected)"
echo " - Timeout was reached"
echo
echo " Check serial output above and web interface for details"
fi
fi
echo
echo -e "${GREEN}Step 3: Viewing Results${NC}"
echo
# Define report paths
REPORT_FILE="$SCRIPT_DIR/reports/clients/index.html"
WEB_INTERFACE="http://localhost:8080"
# Check if reports directory exists and has content
if [ -d "$SCRIPT_DIR/reports/clients" ] && [ "$(ls -A $SCRIPT_DIR/reports/clients/*.json 2>/dev/null)" ]; then
echo -e "${GREEN}✓ Test reports found${NC}"
echo
# Generate and display summary
echo -e "${CYAN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
echo -e "${BLUE}Generating Test Summary...${NC}"
echo -e "${CYAN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
echo
if python3 "$SCRIPT_DIR/scripts/generate_summary.py" 2>/dev/null; then
echo
echo -e "${GREEN}✓ Summary generated successfully${NC}"
else
echo -e "${YELLOW}⚠ Could not generate summary (reports may still be processing)${NC}"
echo " Run manually: cd $SCRIPT_DIR && python3 scripts/generate_summary.py"
fi
else
echo -e "${YELLOW}⚠ No test reports found yet${NC}"
echo " Reports may still be generating, or tests may not have completed"
echo " Check the web interface or wait a few moments and run:"
echo " cd $SCRIPT_DIR && python3 scripts/generate_summary.py"
fi
echo
echo -e "${CYAN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
echo -e "${BLUE}Test Reports Available:${NC}"
echo -e "${CYAN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
if [[ -f "$REPORT_FILE" ]]; then
echo " 📄 Direct file: file://$REPORT_FILE"
fi
echo " 🌐 Web interface: $WEB_INTERFACE (click 'Client Reports' link)"
echo " 📁 Directory: $SCRIPT_DIR/reports/clients/"
if [[ -f "$SCRIPT_DIR/reports/summary.html" ]]; then
echo " 📊 Summary: file://$SCRIPT_DIR/reports/summary.html"
fi
echo
echo -e "${YELLOW}To stop the fuzzing server:${NC}"
echo " docker-compose down"
echo
echo -e "${GREEN}Testing complete!${NC}"
@@ -0,0 +1,764 @@
#!/usr/bin/env python3
# SPDX-FileCopyrightText: 2025 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Unlicense OR CC0-1.0
"""
Autobahn Test Results Summary Generator
This script parses all JSON test results and generates a comprehensive summary.
"""
import glob
import json
import os
from collections import defaultdict
from datetime import datetime
from pathlib import Path
# ANSI color codes for terminal output
class Colors:
GREEN = '\033[92m'
RED = '\033[91m'
YELLOW = '\033[93m'
BLUE = '\033[94m'
CYAN = '\033[96m'
MAGENTA = '\033[95m'
BOLD = '\033[1m'
END = '\033[0m'
# Test category descriptions
CATEGORIES = {
'1': {
'name': 'Framing',
'critical': True,
'description': 'Basic frame structure',
},
'2': {
'name': 'Ping/Pong',
'critical': True,
'description': 'Control frames',
},
'3': {
'name': 'Reserved Bits',
'critical': True,
'description': 'RSV validation',
},
'4': {
'name': 'Opcodes',
'critical': True,
'description': 'Valid/invalid opcodes',
},
'5': {
'name': 'Fragmentation',
'critical': True,
'description': 'Message fragments',
},
'6': {
'name': 'UTF-8',
'critical': False,
'description': 'Text validation',
},
'7': {
'name': 'Close Handshake',
'critical': True,
'description': 'Connection closing',
},
'9': {
'name': 'Performance',
'critical': False,
'description': 'Large messages',
},
'10': {
'name': 'Miscellaneous',
'critical': True,
'description': 'Edge cases',
},
'12': {
'name': 'Compression',
'critical': False,
'description': 'RFC 7692',
},
'13': {
'name': 'Compression',
'critical': False,
'description': 'RFC 7692',
},
}
def get_category_from_test_id(test_id):
"""Extract category number from test ID (e.g., '1.1.1' -> '1')"""
return test_id.split('.')[0]
def parse_test_results(reports_dir):
"""Parse all JSON test results"""
results = []
# Look for JSON files in reports directory and reports/clients subdirectory
json_files = glob.glob(os.path.join(reports_dir, "*.json"))
json_files.extend(glob.glob(os.path.join(reports_dir, "clients", "*.json")))
for json_file in json_files:
try:
with open(json_file, 'r') as f:
data = json.load(f)
results.append({
'id': data.get('id', 'unknown'),
'behavior': data.get('behavior', 'UNKNOWN'),
'description': data.get('description', ''),
'duration': data.get('duration', 0),
'result': data.get('result', ''),
})
except Exception as e:
print(f"Error parsing {json_file}: {e}")
# Sort by test ID with proper numeric sorting
def sort_key(result):
parts = result['id'].split('.')
# Convert to sortable format: pad numbers, keep strings
sortable = []
for p in parts:
if p.isdigit():
sortable.append((0, int(p))) # numbers sort before strings
else:
sortable.append((1, p)) # strings sort after numbers
return tuple(sortable)
return sorted(results, key=sort_key)
def generate_summary(results):
"""Generate comprehensive summary statistics"""
# Overall stats
total = len(results)
by_behavior = defaultdict(int)
by_category = defaultdict(lambda: defaultdict(int))
for result in results:
behavior = result['behavior']
by_behavior[behavior] += 1
category = get_category_from_test_id(result['id'])
by_category[category][behavior] += 1
return {
'total': total,
'by_behavior': dict(by_behavior),
'by_category': dict(by_category),
}
def calculate_grade(pass_rate):
"""Calculate letter grade based on pass rate"""
if pass_rate >= 90:
return 'A', Colors.GREEN
elif pass_rate >= 80:
return 'B', Colors.GREEN
elif pass_rate >= 70:
return 'C', Colors.YELLOW
elif pass_rate >= 60:
return 'D', Colors.YELLOW
else:
return 'F', Colors.RED
def print_banner(text):
"""Print a banner"""
print()
print(f"{Colors.BLUE}{'=' * 80}{Colors.END}")
print(f"{Colors.BLUE}{Colors.BOLD}{text:^80}{Colors.END}")
print(f"{Colors.BLUE}{'=' * 80}{Colors.END}")
print()
def print_summary_table(summary):
"""Print overall summary table"""
print_banner("OVERALL TEST RESULTS")
total = summary['total']
by_behavior = summary['by_behavior']
# Calculate percentages
passed = by_behavior.get('OK', 0)
failed = by_behavior.get('FAILED', 0)
non_strict = by_behavior.get('NON-STRICT', 0)
informational = by_behavior.get('INFORMATIONAL', 0)
unimplemented = by_behavior.get('UNIMPLEMENTED', 0)
pass_rate = (passed / total * 100) if total > 0 else 0
fail_rate = (failed / total * 100) if total > 0 else 0
grade, grade_color = calculate_grade(pass_rate)
print(f'Total Tests: {Colors.BOLD}{total}{Colors.END}')
print()
print(
f'{Colors.GREEN}✅ PASSED: {passed:3d} '
f'({pass_rate:5.1f}%){Colors.END}',
)
print(
f'{Colors.RED}❌ FAILED: {failed:3d} '
f'({fail_rate:5.1f}%){Colors.END}',
)
if non_strict > 0:
print(
f'{Colors.YELLOW}⚠️ NON-STRICT: {non_strict:3d} '
f'({non_strict/total*100:5.1f}%){Colors.END}',
)
if informational > 0:
print(
f'{Colors.CYAN}️ INFORMATIONAL: {informational:3d} '
f'({informational/total*100:5.1f}%){Colors.END}',
)
if unimplemented > 0:
print(
f'{Colors.MAGENTA}🔧 UNIMPLEMENTED: {unimplemented:3d} '
f'({unimplemented/total*100:5.1f}%){Colors.END}',
)
print()
print(
f'Overall Grade: {grade_color}{Colors.BOLD}{grade}{Colors.END}',
)
print()
# Embedded client rating
if pass_rate >= 70:
rating = "Excellent"
rating_color = Colors.GREEN
elif pass_rate >= 55:
rating = "Good"
rating_color = Colors.GREEN
elif pass_rate >= 40:
rating = "Acceptable"
rating_color = Colors.YELLOW
else:
rating = "Needs Improvement"
rating_color = Colors.RED
print(
'Embedded Client Rating: '
f'{rating_color}{Colors.BOLD}{rating}{Colors.END}',
)
print()
def print_category_breakdown(summary):
"""Print detailed category breakdown"""
print_banner("RESULTS BY TEST CATEGORY")
by_category = summary['by_category']
# Header
print(f"{'Category':<25} {'Total':>7} {'Pass':>7} {'Fail':>7} {'Rate':>8} {'Critical':>10} {'Grade':>7}")
print(f"{'-'*25} {'-'*7} {'-'*7} {'-'*7} {'-'*8} {'-'*10} {'-'*7}")
# Sort categories numerically
sorted_categories = sorted(by_category.keys(), key=lambda x: int(x) if x.isdigit() else 999)
for cat_num in sorted_categories:
cat_stats = by_category[cat_num]
cat_info = CATEGORIES.get(cat_num, {'name': f'Category {cat_num}', 'critical': True})
total = sum(cat_stats.values())
passed = cat_stats.get('OK', 0)
failed = cat_stats.get('FAILED', 0)
pass_rate = (passed / total * 100) if total > 0 else 0
grade, grade_color = calculate_grade(pass_rate)
# Format category name
cat_name = f"{cat_num}.* {cat_info['name']}"
critical = "Yes" if cat_info['critical'] else "No"
# Color code the pass rate
if pass_rate >= 70:
rate_color = Colors.GREEN
elif pass_rate >= 50:
rate_color = Colors.YELLOW
else:
rate_color = Colors.RED
print(f"{cat_name:<25} {total:>7} {passed:>7} {failed:>7} "
f"{rate_color}{pass_rate:>7.1f}%{Colors.END} {critical:>10} "
f"{grade_color}{grade:>7}{Colors.END}")
print()
def print_failed_tests(results, limit=20):
"""Print list of failed tests"""
print_banner("FAILED TESTS (First 20)")
failed = [r for r in results if r['behavior'] == 'FAILED']
if not failed:
print(f'{Colors.GREEN}🎉 No failed tests!{Colors.END}\n')
return
print(
'Total Failed: '
f'{Colors.RED}{Colors.BOLD}{len(failed)}{Colors.END}\n',
)
for i, test in enumerate(failed[:limit], 1):
print(f"{i:2d}. {Colors.RED}Test {test['id']:<12}{Colors.END} - {test['description'][:60]}")
if test['result']:
print(f" Reason: {test['result'][:100]}")
if len(failed) > limit:
print(f"\n... and {len(failed) - limit} more failed tests")
print()
def print_recommendations(summary):
"""Print recommendations based on results"""
print_banner("RECOMMENDATIONS")
by_category = summary['by_category']
# Check critical categories
critical_issues = []
for cat_num, cat_info in CATEGORIES.items():
if not cat_info['critical']:
continue
if cat_num not in by_category:
continue
cat_stats = by_category[cat_num]
total = sum(cat_stats.values())
passed = cat_stats.get('OK', 0)
pass_rate = (passed / total * 100) if total > 0 else 0
if pass_rate < 70:
critical_issues.append(
{
'category': f"{cat_num}.* {cat_info['name']}",
'pass_rate': pass_rate,
'description': cat_info['description'],
},
)
if critical_issues:
print(f"{Colors.RED}⚠️ Critical Issues Found:{Colors.END}\n")
for issue in critical_issues:
print(f"{issue['category']}: {issue['pass_rate']:.1f}% pass rate")
print(f" {issue['description']} - This requires attention\n")
else:
print(f"{Colors.GREEN}✅ All critical test categories are performing well!{Colors.END}\n")
# UTF-8 check
if '6' in by_category:
cat_stats = by_category['6']
total = sum(cat_stats.values())
passed = cat_stats.get('OK', 0)
pass_rate = (passed / total * 100) if total > 0 else 0
if pass_rate < 50:
print(
f'{Colors.YELLOW}️ UTF-8 Validation (6.*): '
f'{pass_rate:.1f}% pass rate{Colors.END}',
)
print(
' This is acceptable for embedded clients in '
'trusted environments.',
)
print(
' Consider adding UTF-8 validation if operating in '
'untrusted networks.\n',
)
print(f'{Colors.CYAN}💡 Next Steps:{Colors.END}\n')
print(' 1. Review failed tests in detail (see HTML report)')
print(' 2. Focus on critical categories (1-5, 7, 10)')
print(' 3. UTF-8 failures (category 6) are acceptable for embedded clients')
print(' 4. Compare your results with reference implementations\n')
def generate_markdown_summary(summary, results):
"""Generate markdown summary for GitHub Actions Job Summary"""
total = summary['total']
by_behavior = summary['by_behavior']
by_category = summary['by_category']
passed = by_behavior.get('OK', 0)
failed = by_behavior.get('FAILED', 0)
non_strict = by_behavior.get('NON-STRICT', 0)
informational = by_behavior.get('INFORMATIONAL', 0)
unimplemented = by_behavior.get('UNIMPLEMENTED', 0)
pass_rate = (passed / total * 100) if total > 0 else 0
grade, _ = calculate_grade(pass_rate)
# Determine status emoji and color
if pass_rate >= 90:
status_emoji = "🟢"
elif pass_rate >= 70:
status_emoji = "🟡"
else:
status_emoji = "🔴"
md = f"""# {status_emoji} Autobahn Test Suite Results
## Overall Results
| Metric | Value |
|--------|-------|
| **Total Tests** | {total} |
| **✅ Passed** | {passed} ({pass_rate:.1f}%) |
| **❌ Failed** | {failed} ({(failed/total*100) if total > 0 else 0:.1f}%) |
"""
if non_strict > 0:
md += f"| **⚠️ Non-Strict** | {non_strict} ({non_strict/total*100:.1f}%) |\n"
if informational > 0:
md += f"| **️ Informational** | {informational} ({informational/total*100:.1f}%) |\n"
if unimplemented > 0:
md += f"| **🔧 Unimplemented** | {unimplemented} ({unimplemented/total*100:.1f}%) |\n"
md += f"""
| **Overall Grade** | **{grade}** |
| **Pass Rate** | {pass_rate:.1f}% |
### Progress Bar
<progress value="{passed}" max="{total}"></progress> {pass_rate:.1f}%
---
## Results by Category
| Category | Name | Total | Passed | Failed | Pass Rate | Critical |
|----------|------|-------|--------|--------|-----------|----------|
"""
sorted_categories = sorted(by_category.keys(), key=lambda x: int(x) if x.isdigit() else 999)
for cat_num in sorted_categories:
cat_stats = by_category[cat_num]
cat_info = CATEGORIES.get(cat_num, {'name': f'Category {cat_num}', 'critical': True})
total_cat = sum(cat_stats.values())
passed_cat = cat_stats.get('OK', 0)
failed_cat = cat_stats.get('FAILED', 0)
pass_rate_cat = (passed_cat / total_cat * 100) if total_cat > 0 else 0
critical_mark = "✅ Yes" if cat_info['critical'] else "⚪ No"
pass_mark = "" if pass_rate_cat >= 70 else "⚠️" if pass_rate_cat >= 50 else ""
md += f"| **{cat_num}.** | {cat_info['name']} | {total_cat} | {passed_cat} | {failed_cat} | {pass_mark} {pass_rate_cat:.1f}% | {critical_mark} |\n"
# Failed tests section
failed_tests = [r for r in results if r['behavior'] == 'FAILED']
if failed_tests:
md += f"""
---
## ❌ Failed Tests ({len(failed_tests)} total)
"""
for i, test in enumerate(failed_tests[:20], 1):
md += f"{i}. **Test {test['id']}**: {test['description'][:80]}\n"
if test['result']:
md += f" - Reason: {test['result'][:100]}\n"
if len(failed_tests) > 20:
md += f"\n*... and {len(failed_tests) - 20} more failed tests*\n"
else:
md += "\n---\n\n## 🎉 No Failed Tests!\n\n"
# Recommendations
md += "\n---\n\n## 💡 Recommendations\n\n"
critical_issues = []
for cat_num, cat_info in CATEGORIES.items():
if not cat_info['critical']:
continue
if cat_num not in by_category:
continue
cat_stats = by_category[cat_num]
total_cat = sum(cat_stats.values())
passed_cat = cat_stats.get('OK', 0)
pass_rate_cat = (passed_cat / total_cat * 100) if total_cat > 0 else 0
if pass_rate_cat < 70:
critical_issues.append({
'category': f"{cat_num}.* {cat_info['name']}",
'pass_rate': pass_rate_cat,
})
if critical_issues:
md += "### ⚠️ Critical Issues Found:\n\n"
for issue in critical_issues:
md += f"- **{issue['category']}**: {issue['pass_rate']:.1f}% pass rate - Requires attention\n"
else:
md += "### ✅ All critical test categories are performing well!\n\n"
md += """
### Next Steps:
1. Review failed tests in detail (see uploaded HTML report)
2. Focus on critical categories (1-5, 7, 10)
3. UTF-8 failures (category 6) are acceptable for embedded clients
4. Compare results with reference implementations
---
*Generated by Autobahn Test Suite Summary Generator*
"""
return md
def generate_html_summary(summary, results, output_file):
"""Generate an HTML summary file"""
total = summary['total']
passed = summary['by_behavior'].get('OK', 0)
failed = summary['by_behavior'].get('FAILED', 0)
pass_rate = (passed / total * 100) if total > 0 else 0
fail_rate = (failed / total * 100) if total > 0 else 0
grade, _ = calculate_grade(pass_rate)
html = f"""<!DOCTYPE html>
<html>
<head>
<title>ESP WebSocket Client - Autobahn Test Summary</title>
<style>
body {{ font-family: 'Segoe UI', Arial, sans-serif;
margin: 40px;
background: #f5f5f5; }}
.container {{ max-width: 1200px; margin: 0 auto; background: white;
padding: 30px; border-radius: 8px;
box-shadow: 0 2px 4px rgba(0,0,0,0.1); }}
h1 {{ color: #333; border-bottom: 3px solid #4CAF50; padding-bottom: 10px; }}
h2 {{ color: #555; margin-top: 30px; border-bottom: 2px solid #ddd; padding-bottom: 8px; }}
.stats {{ display: grid; grid-template-columns: repeat(auto-fit, minmax(200px, 1fr)); gap: 20px; margin: 20px 0; }}
.stat-card {{ background: linear-gradient(135deg, #667eea 0%, #764ba2 100%); color: white; padding: 20px; border-radius: 8px; text-align: center; }}
.stat-card.passed {{ background: linear-gradient(135deg, #11998e 0%, #38ef7d 100%); }}
.stat-card.failed {{ background: linear-gradient(135deg, #eb3349 0%, #f45c43 100%); }}
.stat-card h3 {{ margin: 0; font-size: 36px; }}
.stat-card p {{ margin: 5px 0 0 0; opacity: 0.9; }}
table {{ width: 100%; border-collapse: collapse; margin: 20px 0; }}
th {{ background: #667eea; color: white; padding: 12px; text-align: left; }}
td {{ padding: 10px; border-bottom: 1px solid #ddd; }}
tr:hover {{ background: #f5f5f5; }}
.pass {{ color: #11998e; font-weight: bold; }}
.fail {{ color: #eb3349; font-weight: bold; }}
.grade {{ font-size: 24px; font-weight: bold; padding: 10px 20px;
border-radius: 5px; display: inline-block; }}
.grade-a, .grade-b {{ background: #11998e; color: white; }}
.grade-c, .grade-d {{ background: #f39c12; color: white; }}
.grade-f {{ background: #eb3349; color: white; }}
.progress-bar {{ background: #ddd; border-radius: 10px; overflow: hidden;
height: 30px; margin: 10px 0; }}
.progress-fill {{ background: linear-gradient(90deg, #11998e 0%,
#38ef7d 100%);
height: 100%; display: flex; align-items: center;
justify-content: center; color: white;
font-weight: bold; transition: width 0.3s; }}
</style>
</head>
<body>
<div class="container">
<h1>🔬 ESP WebSocket Client - Autobahn Test Suite Results</h1>
<p style="color: #666; font-size: 14px;">Generated: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}</p>
<h2>📊 Overall Results</h2>
<div class="stats">
<div class="stat-card">
<h3>{total}</h3>
<p>Total Tests</p>
</div>
<div class="stat-card passed">
<h3>{passed}</h3>
<p>Passed ({pass_rate:.1f}%)</p>
</div>
<div class="stat-card failed">
<h3>{failed}</h3>
<p>Failed ({fail_rate:.1f}%)</p>
</div>
</div>
<div class="progress-bar">
<div class="progress-fill" style="width: {pass_rate:.1f}%">
{pass_rate:.1f}% Pass Rate
</div>
</div>
<div style="text-align: center; margin: 30px 0;">
<span class="grade grade-{grade.lower()}">
Grade: {grade}
</span>
</div>
<h2>📋 Results by Category</h2>
<table>
<thead>
<tr>
<th>Category</th>
<th>Description</th>
<th>Total</th>
<th>Passed</th>
<th>Failed</th>
<th>Pass Rate</th>
<th>Critical</th>
</tr>
</thead>
<tbody>
"""
# Add category rows
sorted_categories = sorted(summary['by_category'].keys(), key=lambda x: int(x) if x.isdigit() else 999)
for cat_num in sorted_categories:
cat_stats = summary['by_category'][cat_num]
cat_info = CATEGORIES.get(cat_num, {'name': f'Category {cat_num}', 'critical': True, 'description': 'Unknown'})
total = sum(cat_stats.values())
passed = cat_stats.get('OK', 0)
failed = cat_stats.get('FAILED', 0)
pass_rate = (passed / total * 100) if total > 0 else 0
html += f"""
<tr>
<td><strong>{cat_num}.*</strong></td>
<td>{cat_info['name']} - {cat_info['description']}</td>
<td>{total}</td>
<td class="pass">{passed}</td>
<td class="fail">{failed}</td>
<td>
<span class="{'pass' if pass_rate >= 70 else 'fail'}">{pass_rate:.1f}%</span>
</td>
<td>{'✅ Yes' if cat_info['critical'] else '⚪ No'}</td>
</tr>
"""
html += """
</tbody>
</table>
<h2>❌ Failed Tests</h2>
<table>
<thead>
<tr>
<th>Test ID</th>
<th>Description</th>
<th>Result</th>
</tr>
</thead>
<tbody>
"""
failed_tests = [r for r in results if r['behavior'] == 'FAILED']
for test in failed_tests[:50]: # Limit to 50 in HTML
html += f"""
<tr>
<td><strong>{test['id']}</strong></td>
<td>{test['description']}</td>
<td style="font-size: 12px; color: #666;">{test['result'][:100]}</td>
</tr>
"""
if len(failed_tests) > 50:
html += f"""
<tr>
<td colspan="3" style="text-align: center; color: #666;">
... and {len(failed_tests) - 50} more failed tests (see individual reports)
</td>
</tr>
"""
html += """
</tbody>
</table>
<h2>💡 Recommendations</h2>
<div style="background: #e8f5e9; padding: 20px; border-radius: 8px; border-left: 4px solid #4CAF50;">
<p><strong>For Embedded WebSocket Clients:</strong></p>
<ul>
<li>✅ Focus on passing critical categories: 1.* (Framing), 2.* (Ping/Pong), 5.* (Fragmentation), 7.* (Close)</li>
<li>⚠️ UTF-8 validation failures (6.*) are acceptable in trusted environments</li>
<li>🎯 Target >70% overall pass rate for production use</li>
<li>🔍 Review individual test reports for specific implementation issues</li>
</ul>
</div>
<div style="margin-top: 30px; padding: 20px; background: #f5f5f5; border-radius: 8px; text-align: center;">
<p style="color: #666; margin: 0;">
<strong>View Detailed Reports:</strong>
<a href="index.html">Open Full Autobahn Report</a>
</p>
</div>
</div>
</body>
</html>
"""
with open(output_file, 'w') as f:
f.write(html)
print(
f'{Colors.GREEN}✅ HTML summary generated: {output_file}{Colors.END}\n',
)
def main():
"""Main entry point"""
script_dir = Path(__file__).parent
reports_dir = script_dir.parent / 'reports'
# Check if GitHub Actions summary file path is provided
github_summary_file = os.environ.get('GITHUB_STEP_SUMMARY')
if not reports_dir.exists():
print(
f'{Colors.RED}Error: Reports directory not found: '
f'{reports_dir}{Colors.END}',
)
return 1
print(
f'{Colors.CYAN}📖 Parsing test results from: '
f'{reports_dir}{Colors.END}',
)
results = parse_test_results(str(reports_dir))
if not results:
print(f'{Colors.RED}Error: No test results found{Colors.END}')
return 1
print(
f'{Colors.GREEN}✅ Parsed {len(results)} '
f'test results{Colors.END}',
)
summary = generate_summary(results)
# Print console summary
print_summary_table(summary)
print_category_breakdown(summary)
print_failed_tests(results)
print_recommendations(summary)
# Generate HTML summary
html_output = reports_dir / 'summary.html'
generate_html_summary(summary, results, str(html_output))
print(f'{Colors.CYAN}🌐 Open the summary in your browser:{Colors.END}')
print(f' file://{html_output.absolute()}\n')
# Generate markdown summary for GitHub Actions
if github_summary_file:
md_summary = generate_markdown_summary(summary, results)
with open(github_summary_file, 'w') as f:
f.write(md_summary)
print(f'{Colors.GREEN}✅ GitHub Actions summary written to: {github_summary_file}{Colors.END}')
return 0
if __name__ == '__main__':
exit(main())
@@ -0,0 +1,13 @@
cmake_minimum_required(VERSION 3.5)
include($ENV{IDF_PATH}/tools/cmake/project.cmake)
# Add Linux compatibility components if building for Linux target
# COMPONENTS must be set before include() to limit component scanning
if(${IDF_TARGET} STREQUAL "linux")
set(COMPONENTS main)
endif()
# This override applies to this example only and does not touch ESP-IDF itself.
add_compile_options(-Wno-error=implicit-function-declaration)
project(autobahn_testee)
@@ -0,0 +1,11 @@
set(requires_list esp_websocket_client protocol_examples_common esp_event)
if(${IDF_TARGET} STREQUAL "linux")
list(APPEND requires_list esp_netif)
else()
list(APPEND requires_list nvs_flash esp_wifi)
endif()
idf_component_register(SRCS "autobahn_testee.c"
INCLUDE_DIRS "."
REQUIRES ${requires_list})
@@ -0,0 +1,44 @@
menu "Autobahn Testsuite Configuration"
choice WEBSOCKET_URI_SOURCE
prompt "Autobahn Server URI Source"
default WEBSOCKET_URI_FROM_STRING
help
Choose how the Autobahn server URI is provided:
- From string: Use CONFIG_AUTOBAHN_SERVER_URI (compile-time)
- From stdin: Read URI from stdin (serial console at runtime)
config WEBSOCKET_URI_FROM_STRING
bool "From string (CONFIG_AUTOBAHN_SERVER_URI)"
help
Use the URI defined in CONFIG_AUTOBAHN_SERVER_URI.
This is set at compile time.
config WEBSOCKET_URI_FROM_STDIN
bool "From stdin (serial console)"
help
Read the URI from stdin (serial console) at runtime.
Useful for CI/CD where the server IP is only known at runtime.
The application will wait for a line containing the URI.
endchoice
config AUTOBAHN_SERVER_URI
string "Autobahn Fuzzing Server URI"
default "ws://192.168.1.100:9001"
depends on WEBSOCKET_URI_FROM_STRING
help
URI of the Autobahn fuzzing server.
Replace with your Docker host IP address.
Example: ws://192.168.1.100:9001
Only used when WEBSOCKET_URI_FROM_STRING is selected.
if CONFIG_IDF_TARGET = "linux"
config GCOV_ENABLED
bool "Coverage analyzer"
default n
help
Enables coverage analyzing for host tests.
endif
endmenu
@@ -0,0 +1,759 @@
/*
* SPDX-FileCopyrightText: 2025 Espressif Systems (Shanghai) CO LTD
*
* SPDX-License-Identifier: Unlicense OR CC0-1.0
*/
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include "esp_log.h"
#include "esp_timer.h"
#include "esp_system.h"
#include "esp_event.h"
#include "esp_netif.h"
#include "protocol_examples_common.h"
#include "esp_websocket_client.h"
#include "esp_transport_ws.h"
#include "cJSON.h"
#if CONFIG_IDF_TARGET_LINUX
#include <pthread.h>
#include <unistd.h>
#include <semaphore.h>
#include <time.h>
#else
#include "freertos/FreeRTOS.h"
#include "freertos/task.h"
#include "freertos/semphr.h"
#include "esp_heap_caps.h"
#include "nvs_flash.h"
#include "esp_wifi.h"
#endif
#define TAG "autobahn"
#if CONFIG_WEBSOCKET_URI_FROM_STDIN
// URI will be read from stdin at runtime
static char g_autobahn_server_uri[256] = {0};
#define AUTOBAHN_SERVER_URI g_autobahn_server_uri
#else
#define AUTOBAHN_SERVER_URI CONFIG_AUTOBAHN_SERVER_URI
#endif
#if CONFIG_IDF_TARGET_LINUX
#define BUFFER_SIZE 65536
#else
#define BUFFER_SIZE 4096
#endif
// Configure test range here:
// Category 1 (Framing): Tests 1-16
// Category 2 (Ping/Pong): Tests 17-27
// Category 3 (Reserved Bits): Tests 28-34
// Category 4 (Opcodes): Tests 35-44
// Category 5 (Fragmentation): Tests 45-64
// Category 6 (UTF-8): Tests 65-209
// Category 7 (Close Handshake): Tests 210-246
// All tests: Tests 1-300
// Defaults if get_case_count fails
#define DEFAULT_START_CASE 1
#define DEFAULT_END_CASE 300
#if CONFIG_IDF_TARGET_LINUX
static sem_t test_done_sem_storage;
static sem_t *test_done_sem = NULL;
#else
static SemaphoreHandle_t test_done_sem = NULL;
#endif
static bool test_running = false;
// Global to store total case count fetched from server
static int g_total_cases = 0;
#if CONFIG_IDF_TARGET_LINUX
#define MAX_FRAGMENTED_PAYLOAD (16 * 1024 * 1024) // 16MB for Linux performance tests
#else
#define MAX_FRAGMENTED_PAYLOAD 65537 // 64KB+1 for embedded targets (case 1.1.6)
#endif
typedef struct {
uint8_t *buffer;
size_t capacity;
size_t expected_len;
size_t received;
uint8_t opcode;
bool active;
} ws_accumulator_t;
static ws_accumulator_t s_accumulator = {0};
static uint8_t *s_accum_buffer = NULL; // Pre-allocated buffer for fragmented frames
static void ws_accumulator_reset(void)
{
// Reset state but keep buffer allocated for reuse
s_accumulator.buffer = s_accum_buffer;
s_accumulator.expected_len = 0;
s_accumulator.received = 0;
s_accumulator.opcode = 0;
s_accumulator.active = false;
}
static void ws_accumulator_cleanup(void)
{
if (s_accum_buffer) {
free(s_accum_buffer);
s_accum_buffer = NULL;
ESP_LOGD(TAG, "Freed accumulator buffer");
}
ws_accumulator_reset();
s_accumulator.capacity = 0;
}
static esp_err_t ws_accumulator_prepare(size_t total_len, uint8_t opcode)
{
if (total_len == 0) {
return ESP_OK;
}
if (total_len > MAX_FRAGMENTED_PAYLOAD) {
ESP_LOGE(TAG, "Payload too large (%zu > %d)", total_len, MAX_FRAGMENTED_PAYLOAD);
return ESP_ERR_INVALID_SIZE;
}
/* Allocate buffer on-demand when first fragmented frame is detected.
* Allocate only what we need for the current message to avoid permanently
* holding a 64KB buffer on constrained targets like ESP32-S2.
*/
if (!s_accum_buffer) {
size_t free_heap = esp_get_free_heap_size();
#if CONFIG_IDF_TARGET_LINUX
size_t largest_free = free_heap;
#else
size_t largest_free = heap_caps_get_largest_free_block(MALLOC_CAP_DEFAULT);
#endif
ESP_LOGD(TAG, "Attempting accumulator alloc: need=%zu, free=%zu, largest_block=%zu",
total_len, free_heap, largest_free);
s_accum_buffer = (uint8_t *)malloc(total_len);
if (!s_accum_buffer) {
ESP_LOGE(TAG, "Accumulator alloc failed (%zu bytes) - Free heap: %zu, largest block: %zu",
total_len, free_heap, largest_free);
#if !CONFIG_IDF_TARGET_LINUX
ESP_LOGE(TAG, "ESP32-S2 may be low on RAM. Consider reducing BUFFER_SIZE (currently %d) and/or enabling SPIRAM",
BUFFER_SIZE);
#endif
return ESP_ERR_NO_MEM;
}
s_accumulator.capacity = total_len;
ESP_LOGD(TAG, "Allocated accumulator buffer: %zu bytes (Free heap: %lu)",
total_len, esp_get_free_heap_size());
} else if (total_len > s_accumulator.capacity) {
ESP_LOGD(TAG, "Reallocating accumulator buffer: old=%zu new=%zu", s_accumulator.capacity, total_len);
uint8_t *new_ptr = (uint8_t *)realloc(s_accum_buffer, total_len);
if (!new_ptr) {
ESP_LOGE(TAG, "Accumulator realloc failed (%zu bytes)", total_len);
return ESP_ERR_NO_MEM;
}
s_accum_buffer = new_ptr;
s_accumulator.capacity = total_len;
}
s_accumulator.buffer = s_accum_buffer;
// s_accumulator.capacity is managed above
s_accumulator.expected_len = total_len;
s_accumulator.received = 0;
s_accumulator.opcode = opcode;
s_accumulator.active = true;
return ESP_OK;
}
/* ------------------------------------------------------------
* Lowlatency echo handler
* ------------------------------------------------------------ */
static void websocket_event_handler(void *handler_args,
esp_event_base_t base,
int32_t event_id,
void *event_data)
{
esp_websocket_event_data_t *data = (esp_websocket_event_data_t *)event_data;
esp_websocket_client_handle_t client = (esp_websocket_client_handle_t)handler_args;
switch (event_id) {
case WEBSOCKET_EVENT_CONNECTED:
ESP_LOGI(TAG, "Connected");
test_running = true;
break;
case WEBSOCKET_EVENT_DISCONNECTED:
ESP_LOGI(TAG, "Disconnected");
test_running = false;
ws_accumulator_reset();
#if CONFIG_IDF_TARGET_LINUX
if (test_done_sem) {
sem_post(test_done_sem);
}
#else
if (test_done_sem) {
xSemaphoreGive(test_done_sem);
}
#endif
break;
case WEBSOCKET_EVENT_DATA: {
ESP_LOGI(TAG, "WEBSOCKET_EVENT_DATA: opcode=0x%02X len=%d fin=%d payload_len=%d offset=%d",
data->op_code, data->data_len, data->fin, data->payload_len, data->payload_offset);
// Safety check: if not connected, don't process data
if (!test_running || !esp_websocket_client_is_connected(client)) {
ESP_LOGW(TAG, "Received data but not connected, ignoring");
ws_accumulator_reset();
break;
}
/* ---- skip control frames ---- */
if (data->op_code >= 0x08) {
if (data->op_code == 0x09) {
ESP_LOGD(TAG, "PING -> PONG auto-sent");
}
break;
}
/* ---- Determine opcode to echo ---- */
uint8_t send_opcode = 0;
if (data->op_code == 0x1) {
send_opcode = WS_TRANSPORT_OPCODES_TEXT;
} else if (data->op_code == 0x2) {
send_opcode = WS_TRANSPORT_OPCODES_BINARY;
} else if (data->op_code == 0x0) {
send_opcode = WS_TRANSPORT_OPCODES_CONT;
} else {
ESP_LOGW(TAG, "Unsupported opcode 0x%02X - skip", data->op_code);
break;
}
/* Note: send_with_opcode always sets FIN bit, which is correct for these
* simple test cases (all have FIN=1). For fragmented messages, we'd need
* send_with_exact_opcode, but it's not public. */
// Safety check: validate data pointer before processing
if (!data->data_ptr && data->data_len > 0) {
ESP_LOGE(TAG, "NULL data pointer with non-zero length: %d", data->data_len);
break;
}
const uint8_t *payload = (const uint8_t *)data->data_ptr;
size_t len = data->data_len;
bool used_accumulator = false;
// Check if this is a fragmented message (either WebSocket fragmentation or TCP-level fragmentation)
// The WebSocket layer reads large frames in chunks and dispatches multiple events:
// - payload_len = total frame size (set on all chunks)
// - payload_offset = current offset (0, buffer_size, 2*buffer_size, ...)
// - data_len = current chunk size
// - fin = 1 only on the last frame of a fragmented message
size_t total_len = data->payload_len ? data->payload_len : data->data_len;
bool fragmented = (data->payload_len > 0 && data->payload_len > data->data_len) ||
(data->payload_offset > 0) || (data->fin == 0) || s_accumulator.active;
ESP_LOGD(TAG, "Fragmentation check: offset=%d payload_len=%d data_len=%d total_len=%zu fragmented=%d",
data->payload_offset, data->payload_len, data->data_len, total_len, fragmented);
if (fragmented && total_len > 0) {
if (!s_accumulator.active) {
if (send_opcode == WS_TRANSPORT_OPCODES_CONT) {
ESP_LOGW(TAG, "Continuation frame without active accumulator, skipping");
ws_accumulator_reset();
break;
}
size_t initial_len = (data->fin == 0) ? data->data_len : total_len;
if (ws_accumulator_prepare(initial_len, send_opcode) != ESP_OK) {
ESP_LOGE(TAG, "Cannot allocate buffer for fragmented frame len=%zu", initial_len);
break;
}
if (data->fin == 0) {
// Unknown total length for a fragmented message; grow as needed.
s_accumulator.expected_len = 0;
}
}
size_t required_len = s_accumulator.received + data->data_len;
if (required_len > MAX_FRAGMENTED_PAYLOAD) {
ESP_LOGE(TAG, "Payload too large (%zu > %d)", required_len, MAX_FRAGMENTED_PAYLOAD);
ws_accumulator_reset();
break;
}
if (required_len > s_accumulator.capacity) {
uint8_t *new_ptr = (uint8_t *)realloc(s_accum_buffer, required_len);
if (!new_ptr) {
ESP_LOGE(TAG, "Accumulator realloc failed (%zu bytes)", required_len);
ws_accumulator_reset();
break;
}
s_accum_buffer = new_ptr;
s_accumulator.buffer = s_accum_buffer;
s_accumulator.capacity = required_len;
}
if (!s_accumulator.buffer) {
ESP_LOGE(TAG, "Accumulator buffer NULL while processing fragments");
ws_accumulator_reset();
break;
}
if (s_accumulator.expected_len > 0 && required_len > s_accumulator.expected_len) {
ESP_LOGE(TAG, "Data exceeds expected length: received=%zu chunk=%d expected=%zu",
s_accumulator.received, data->data_len, s_accumulator.expected_len);
ws_accumulator_reset();
break;
}
memcpy(s_accumulator.buffer + s_accumulator.received, data->data_ptr, data->data_len);
s_accumulator.received = required_len;
bool end_of_frame = (data->payload_len == 0) ||
((size_t)data->payload_offset + data->data_len >= (size_t)data->payload_len);
if (!(data->fin == 1 && end_of_frame)) {
// wait for more fragments
ESP_LOGD(TAG, "Waiting for more fragments: received=%zu fin=%d end_of_frame=%d",
s_accumulator.received, data->fin, end_of_frame);
break;
}
// Completed full message
payload = s_accumulator.buffer;
len = s_accumulator.received;
send_opcode = s_accumulator.opcode;
s_accumulator.active = false;
used_accumulator = true;
}
// Check connection before attempting to send
if (!esp_websocket_client_is_connected(client)) {
ESP_LOGW(TAG, "Connection lost before echo, skipping");
ws_accumulator_reset();
break;
}
int sent = -1;
int attempt = 0;
const TickType_t backoff[] = {1, 1, 1, 2, 4, 8}; // Shorter backoff for faster retry
int64_t start = esp_timer_get_time();
/* Send echo immediately - use timeout scaled by frame size for large frames */
/* For large messages (>16KB), the send function fragments into multiple chunks */
/* Each chunk needs sufficient timeout, so scale timeout per chunk, not total message */
while (sent < 0 && esp_websocket_client_is_connected(client)) {
/* For zero-length payload, pass NULL pointer (API handles this correctly) */
/* Calculate timeout per chunk: large messages are fragmented, each chunk needs time */
#if CONFIG_IDF_TARGET_LINUX
int send_timeout_ms = 5; // Default 5ms for small frames
if (len > 1024) {
send_timeout_ms = 500; // 500ms per chunk for large messages
} else {
send_timeout_ms = (len / 256) + 10;
if (send_timeout_ms > 100) {
send_timeout_ms = 100;
}
}
TickType_t send_timeout = pdMS_TO_TICKS(send_timeout_ms);
ESP_LOGD(TAG, "Sending echo: opcode=0x%02X len=%zu timeout=%dms",
send_opcode, len, send_timeout_ms);
#else
TickType_t send_timeout = pdMS_TO_TICKS(5); // Default 5ms for small frames
if (len > 1024) {
// For large messages, use a per-chunk timeout that accounts for network delays
// Since messages are fragmented into ~16KB chunks, each chunk needs sufficient time
// Use a fixed generous timeout per chunk for large messages (500ms per chunk)
// For 65535 bytes = 4 chunks, total time could be up to 2 seconds
send_timeout = pdMS_TO_TICKS(500); // 500ms per chunk for large messages
} else {
// Small messages: scale timeout based on size
send_timeout = pdMS_TO_TICKS((len / 256) + 10);
if (send_timeout > pdMS_TO_TICKS(100)) {
send_timeout = pdMS_TO_TICKS(100);
}
}
ESP_LOGD(TAG, "Sending echo: opcode=0x%02X len=%zu timeout=%lums",
send_opcode, len, (unsigned long)(send_timeout * portTICK_PERIOD_MS));
#endif
sent = esp_websocket_client_send_with_opcode(
client, send_opcode,
(len > 0) ? payload : NULL, len,
send_timeout);
if (sent >= 0) {
ESP_LOGD(TAG, "Echo sent successfully: %d bytes", sent);
break;
}
ESP_LOGW(TAG,
"echo send retry: opcode=0x%02X len=%zu fin=%d attempt=%d sent=%d",
send_opcode, len, data->fin, attempt + 1, sent);
if (attempt < (int)(sizeof(backoff) / sizeof(backoff[0]))) {
#if CONFIG_IDF_TARGET_LINUX
usleep(backoff[attempt++] * 1000); // Convert ticks to microseconds
#else
vTaskDelay(backoff[attempt++]);
#endif
} else {
#if CONFIG_IDF_TARGET_LINUX
usleep(32 * 1000);
#else
vTaskDelay(32);
#endif
}
}
int64_t dt = esp_timer_get_time() - start;
if (sent >= 0) {
ESP_LOGI(TAG, "Echo success: opcode=0x%02X len=%d fin=%d in %lldus",
data->op_code, sent, data->fin, (long long)dt);
} else {
ESP_LOGE(TAG, "Echo failed: opcode=0x%02X len=%d fin=%d",
data->op_code, (int)len, data->fin);
}
/* If we allocated an accumulator buffer for this message, free it now to
* avoid holding large buffers across test cases (helps ESP32-S2).
*/
if (used_accumulator) {
ws_accumulator_cleanup();
}
break;
}
case WEBSOCKET_EVENT_ERROR:
ESP_LOGW(TAG, "WebSocket error event");
test_running = false;
ws_accumulator_reset(); // Reset accumulator on error
#if CONFIG_IDF_TARGET_LINUX
if (test_done_sem) {
sem_post(test_done_sem);
}
#else
if (test_done_sem) {
xSemaphoreGive(test_done_sem);
}
#endif
break;
case WEBSOCKET_EVENT_FINISH:
ESP_LOGD(TAG, "WebSocket finish event");
test_running = false;
ws_accumulator_reset(); // Reset accumulator on finish
#if CONFIG_IDF_TARGET_LINUX
if (test_done_sem) {
sem_post(test_done_sem);
}
#else
if (test_done_sem) {
xSemaphoreGive(test_done_sem);
}
#endif
break;
default:
break;
}
}
/* ------------------------------------------------------------
* Handler for get_case_count
* ------------------------------------------------------------ */
static void get_case_count_event_handler(void *handler_args,
esp_event_base_t base,
int32_t event_id,
void *event_data)
{
esp_websocket_event_data_t *data = (esp_websocket_event_data_t *)event_data;
switch (event_id) {
case WEBSOCKET_EVENT_DATA:
if (data->data_len > 0 && data->data_ptr) {
// The server returns a JSON integer (e.g., "518")
// We can parse it directly
char *buf = (char *)malloc(data->data_len + 1);
if (buf) {
memcpy(buf, data->data_ptr, data->data_len);
buf[data->data_len] = '\0';
g_total_cases = atoi(buf);
ESP_LOGI(TAG, "Received total case count: %d", g_total_cases);
free(buf);
}
}
break;
default:
break;
}
}
/* ------------------------------------------------------------ */
static void get_case_count(void)
{
char uri[512];
int ret = snprintf(uri, sizeof(uri), "%s/getCaseCount", AUTOBAHN_SERVER_URI);
if (ret < 0 || ret >= (int)sizeof(uri)) {
ESP_LOGE(TAG, "URI too long");
return;
}
ESP_LOGI(TAG, "Getting case count from: %s", uri);
esp_websocket_client_config_t cfg = {
.uri = uri,
.network_timeout_ms = 10000,
};
esp_websocket_client_handle_t client = esp_websocket_client_init(&cfg);
if (!client) {
ESP_LOGE(TAG, "Failed to init client for getCaseCount");
return;
}
esp_websocket_register_events(client, WEBSOCKET_EVENT_DATA, get_case_count_event_handler, NULL);
if (esp_websocket_client_start(client) == ESP_OK) {
// Wait briefly for response
#if CONFIG_IDF_TARGET_LINUX
usleep(2000 * 1000);
#else
vTaskDelay(pdMS_TO_TICKS(2000));
#endif
esp_websocket_client_stop(client);
}
esp_websocket_client_destroy(client);
}
/* ------------------------------------------------------------ */
static esp_err_t run_test_case(int case_num)
{
char uri[512]; // Increased to accommodate full URI + path
int ret = snprintf(uri, sizeof(uri),
"%s/runCase?case=%d&agent=esp_websocket_client",
AUTOBAHN_SERVER_URI, case_num);
if (ret < 0 || ret >= (int)sizeof(uri)) {
ESP_LOGE(TAG, "URI too long: %s/runCase?case=%d&agent=esp_websocket_client", AUTOBAHN_SERVER_URI, case_num);
return ESP_ERR_INVALID_ARG;
}
ESP_LOGI(TAG, "Running case %d: %s", case_num, uri);
esp_websocket_client_config_t cfg = {
.uri = uri,
.buffer_size = BUFFER_SIZE,
.network_timeout_ms = 10000, // 10s for connection (default), 200ms was too short
.reconnect_timeout_ms = 500,
.task_prio = 10, // High prio → low latency
.task_stack = 8144,
};
esp_websocket_client_handle_t client = esp_websocket_client_init(&cfg);
if (!client) {
return ESP_FAIL;
}
esp_websocket_register_events(client, WEBSOCKET_EVENT_ANY,
websocket_event_handler, (void*)client);
#if CONFIG_IDF_TARGET_LINUX
sem_init(&test_done_sem_storage, 0, 0);
test_done_sem = &test_done_sem_storage;
#else
test_done_sem = xSemaphoreCreateBinary();
#endif
esp_err_t start_ret = esp_websocket_client_start(client);
if (start_ret != ESP_OK) {
ESP_LOGE(TAG, "esp_websocket_client_start() failed: err=0x%x", start_ret);
#if CONFIG_IDF_TARGET_LINUX
if (test_done_sem) {
sem_destroy(test_done_sem);
}
#else
if (test_done_sem) {
vSemaphoreDelete(test_done_sem);
}
#endif
test_done_sem = NULL;
esp_websocket_client_destroy(client);
return start_ret;
}
/* Wait up to 60s so server can close properly */
#if CONFIG_IDF_TARGET_LINUX
{
struct timespec ts;
clock_gettime(CLOCK_REALTIME, &ts);
ts.tv_sec += 60; // absolute timeout (now + 60s)
(void)sem_timedwait(test_done_sem, &ts);
}
#else
xSemaphoreTake(test_done_sem, pdMS_TO_TICKS(60000));
#endif
if (esp_websocket_client_is_connected(client)) {
esp_websocket_client_stop(client);
}
esp_websocket_client_destroy(client);
#if CONFIG_IDF_TARGET_LINUX
if (test_done_sem) {
sem_destroy(test_done_sem);
}
#else
if (test_done_sem) {
vSemaphoreDelete(test_done_sem);
}
#endif
test_done_sem = NULL;
ESP_LOGI(TAG, "Free heap: %lu", esp_get_free_heap_size());
return ESP_OK;
}
/* ------------------------------------------------------------ */
static void update_reports(void)
{
char uri[512]; // Increased to accommodate full URI + path
int ret = snprintf(uri, sizeof(uri),
"%s/updateReports?agent=esp_websocket_client",
AUTOBAHN_SERVER_URI);
if (ret < 0 || ret >= (int)sizeof(uri)) {
ESP_LOGE(TAG, "URI too long: %s/updateReports?agent=esp_websocket_client", AUTOBAHN_SERVER_URI);
return;
}
esp_websocket_client_config_t cfg = { .uri = uri };
esp_websocket_client_handle_t client = esp_websocket_client_init(&cfg);
if (!client) {
ESP_LOGE(TAG, "Failed to initialize WebSocket client for update_reports");
return;
}
esp_err_t start_ret = esp_websocket_client_start(client);
if (start_ret != ESP_OK) {
ESP_LOGE(TAG, "esp_websocket_client_start() failed for update_reports: err=0x%x", start_ret);
esp_websocket_client_destroy(client);
return;
}
#if CONFIG_IDF_TARGET_LINUX
usleep(3000 * 1000); // 3 seconds
#else
vTaskDelay(pdMS_TO_TICKS(3000));
#endif
esp_websocket_client_stop(client);
esp_websocket_client_destroy(client);
ESP_LOGI(TAG, "Reports updated");
}
/* ------------------------------------------------------------ */
static void websocket_app_start(void)
{
ESP_LOGI(TAG, "====================================");
ESP_LOGI(TAG, " Autobahn WebSocket Testsuite Client");
ESP_LOGI(TAG, "====================================");
ESP_LOGI(TAG, "Server: %s", AUTOBAHN_SERVER_URI);
// Accumulator buffer is allocated on-demand only when fragmentation is detected.
// This keeps memory usage low on constrained targets like ESP32-S2.
// Attempt to fetch case count dynamically
get_case_count();
int start_case = DEFAULT_START_CASE;
int end_case = g_total_cases > 0 ? g_total_cases : DEFAULT_END_CASE;
ESP_LOGI(TAG, "Running tests from case %d to %d", start_case, end_case);
for (int i = start_case; i <= end_case; i++) {
ESP_LOGI(TAG, "========== Case %d/%d ==========", i, end_case);
ESP_LOGI(TAG, "Starting test case %d...", i);
esp_err_t ret = run_test_case(i);
if (ret != ESP_OK) {
ESP_LOGW(TAG, "Test case %d failed with error: 0x%x", i, ret);
} else {
ESP_LOGI(TAG, "Test case %d completed", i);
}
#if CONFIG_IDF_TARGET_LINUX
usleep(500 * 1000); // 500ms
#else
vTaskDelay(pdMS_TO_TICKS(500));
#endif
}
update_reports();
// Free accumulator buffer after all tests
ws_accumulator_cleanup();
ESP_LOGI(TAG, "All tests completed.");
}
#if CONFIG_WEBSOCKET_URI_FROM_STDIN
/* ------------------------------------------------------------
* Read URI from stdin (similar to websocket_example.c)
* ------------------------------------------------------------ */
static void get_string(char *line, size_t size)
{
int count = 0;
while (count < size - 1) {
int c = fgetc(stdin);
if (c == '\n' || c == '\r') {
line[count] = '\0';
break;
} else if (c > 0 && c < 127) {
line[count] = c;
++count;
} else if (c == EOF) {
#if CONFIG_IDF_TARGET_LINUX
usleep(10 * 1000); // 10ms
#else
vTaskDelay(pdMS_TO_TICKS(10));
#endif
}
}
line[count] = '\0';
}
#endif /* CONFIG_WEBSOCKET_URI_FROM_STDIN */
/* ------------------------------------------------------------ */
#if CONFIG_IDF_TARGET_LINUX
int main(void)
#else
void app_main(void)
#endif
{
// Disable stdout buffering for immediate output
setvbuf(stdout, NULL, _IONBF, 0);
setvbuf(stderr, NULL, _IONBF, 0);
ESP_LOGI(TAG, "Startup, IDF %s", esp_get_idf_version());
#if !CONFIG_IDF_TARGET_LINUX
ESP_ERROR_CHECK(nvs_flash_init());
#endif
ESP_ERROR_CHECK(esp_netif_init());
ESP_ERROR_CHECK(esp_event_loop_create_default());
// Accumulator buffer is allocated on-demand when needed (fragmented payloads).
// Pre-allocating ~64KB here can prevent esp_websocket_client_init() from
// allocating its own buffers on ESP32-S2.
ESP_ERROR_CHECK(example_connect());
#if !CONFIG_IDF_TARGET_LINUX
/* disable powersave for low latency */
esp_wifi_set_ps(WIFI_PS_NONE);
#endif
#if CONFIG_WEBSOCKET_URI_FROM_STDIN
// Read server URI from stdin
ESP_LOGI(TAG, "Waiting for Autobahn server URI from stdin...");
ESP_LOGI(TAG, "Please send URI in format: ws://<IP>:9001");
// Loop until we get a valid URI
do {
get_string(g_autobahn_server_uri, sizeof(g_autobahn_server_uri));
// Ensure null termination
g_autobahn_server_uri[sizeof(g_autobahn_server_uri) - 1] = '\0';
} while (strlen(g_autobahn_server_uri) == 0);
ESP_LOGI(TAG, "Received server URI: %s", g_autobahn_server_uri);
#endif
websocket_app_start();
#if CONFIG_IDF_TARGET_LINUX
return 0;
#endif
}
@@ -0,0 +1,27 @@
dependencies:
## Required IDF version
idf: ">=5.0"
# WebSocket client component from parent directory
espressif/esp_websocket_client:
version: "^1.0.0"
override_path: "../../../../"
# WiFi connection helper from ESP-IDF examples
espressif/cjson:
version: "^1.7.15"
protocol_examples_common:
path: ${IDF_PATH}/examples/common_components/protocol_examples_common
# Linux compatibility components
espressif/esp_timer:
version: "*"
override_path: "../../../../../../common_components/linux_compat/esp_timer"
rules:
- if: "target == linux"
freertos:
version: "*"
override_path: "../../../../../../common_components/linux_compat/freertos"
rules:
- if: "target == linux"
esp_stubs:
path: ${IDF_PATH}/examples/protocols/linux_stubs/esp_stubs
rules:
- if: "target == linux"
@@ -0,0 +1,26 @@
# Linux target configuration for CI
CONFIG_IDF_TARGET="linux"
CONFIG_IDF_TARGET_LINUX=y
CONFIG_ESP_EVENT_POST_FROM_ISR=n
CONFIG_ESP_EVENT_POST_FROM_IRAM_ISR=n
# Autobahn server URI from stdin (set at runtime)
CONFIG_WEBSOCKET_URI_FROM_STDIN=y
CONFIG_WEBSOCKET_URI_FROM_STRING=n
# WebSocket configuration
CONFIG_WS_TRANSPORT=y
CONFIG_WS_BUFFER_SIZE=8192
# Enable separate TX lock
CONFIG_ESP_WS_CLIENT_SEPARATE_TX_LOCK=y
CONFIG_ESP_WS_CLIENT_TX_LOCK_TIMEOUT_MS=100
# Main task stack
CONFIG_ESP_MAIN_TASK_STACK_SIZE=8192
# Logging
CONFIG_LOG_DEFAULT_LEVEL_INFO=y
# For Linux host builds, allow warnings (do not treat default warnings as errors)
CONFIG_COMPILER_DISABLE_DEFAULT_ERRORS=y
@@ -0,0 +1,15 @@
CONFIG_IDF_TARGET="esp32"
CONFIG_IDF_TARGET_LINUX=n
CONFIG_WEBSOCKET_URI_FROM_STDIN=y
CONFIG_WEBSOCKET_URI_FROM_STRING=n
CONFIG_EXAMPLE_CONNECT_ETHERNET=y
CONFIG_EXAMPLE_CONNECT_WIFI=n
CONFIG_EXAMPLE_USE_INTERNAL_ETHERNET=y
CONFIG_EXAMPLE_ETH_PHY_IP101=y
CONFIG_EXAMPLE_ETH_MDC_GPIO=23
CONFIG_EXAMPLE_ETH_MDIO_GPIO=18
CONFIG_EXAMPLE_ETH_PHY_RST_GPIO=5
CONFIG_EXAMPLE_ETH_PHY_ADDR=1
CONFIG_EXAMPLE_CONNECT_IPV6=y
CONFIG_WS_OVER_TLS_MUTUAL_AUTH=n
CONFIG_WS_OVER_TLS_SERVER_AUTH=n
@@ -9,7 +9,7 @@ else()
set(test_component_dir $ENV{IDF_PATH}/tools/unit-test-app/components)
endif()
set(EXTRA_COMPONENT_DIRS ../../esp_websocket_client
set(EXTRA_COMPONENT_DIRS ../../../esp_websocket_client
${test_component_dir})
project(websocket_unit_test)