diff --git a/.github/workflows/autobahn__linux-test.yml b/.github/workflows/autobahn__linux-test.yml new file mode 100644 index 000000000..eee0e34b6 --- /dev/null +++ b/.github/workflows/autobahn__linux-test.yml @@ -0,0 +1,254 @@ +name: "websocket-autobahn: build/linux-tests" + +on: + push: + branches: + - master + pull_request: + types: [opened, synchronize, reopened, labeled] + +jobs: + linux_websocket_autobahn: + runs-on: ubuntu-22.04 + if: github.event_name == 'push' || (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'websocket-autobahn')) + + env: + TEST_DIR: components/esp_websocket_client/tests/autobahn-testsuite + TESTEE_DIR: components/esp_websocket_client/tests/autobahn-testsuite/testee + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Start Autobahn Fuzzing Server + run: | + mkdir -p ${{ env.TEST_DIR }}/reports/clients + + HOST_IP=$(ip route get 8.8.8.8 | grep -oP 'src \K\S+' || hostname -I | awk '{print $1}' || echo "172.17.0.1") + echo "Host IP address: $HOST_IP" + echo "HOST_IP=$HOST_IP" >> $GITHUB_ENV + + docker run -d \ + --name fuzzing-server \ + --network host \ + -v ${{ github.workspace }}/${{ env.TEST_DIR }}/config:/config:ro \ + -v ${{ github.workspace }}/${{ env.TEST_DIR }}/reports:/reports \ + crossbario/autobahn-testsuite:latest \ + wstest -m fuzzingserver -s /config/fuzzingserver.json + + echo "Waiting for fuzzing server..." + for i in {1..5}; do + if curl -f http://localhost:9001/info >/dev/null 2>&1; then + echo "Fuzzing server ready!" + exit 0 + fi + echo "Attempt $i/5 – waiting 2s..." + sleep 2 + done + + echo "Server start failed. Container logs:" + docker logs fuzzing-server + exit 1 + + - name: Build test (Linux target) + working-directory: ${{ env.TESTEE_DIR }} + run: | + docker run --rm \ + -v ${{ github.workspace }}:/work \ + -w /work/${{ env.TESTEE_DIR }} \ + espressif/idf:latest \ + bash -c " + . \$IDF_PATH/export.sh + cp sdkconfig.ci.linux sdkconfig.defaults + echo 'Building...' + idf.py build + " + + - name: Verify fuzzing server connectivity + run: | + HOST_IP=${HOST_IP:-$(ip route get 8.8.8.8 | grep -oP 'src \K\S+' || hostname -I | awk '{print $1}' || echo "172.17.0.1")} + echo "Testing connectivity to $HOST_IP:9001" + + docker run --rm \ + --network host \ + espressif/idf:latest \ + bash -c " + curl -f http://$HOST_IP:9001/info || { + echo 'ERROR: Cannot connect to fuzzing server at $HOST_IP:9001' + exit 1 + } + echo 'Fuzzing server is accessible' + " + + - name: Run Autobahn tests + run: | + docker run --rm \ + --network host \ + -v ${{ github.workspace }}:/work \ + -w /work/${{ env.TESTEE_DIR }}/build \ + espressif/idf:latest \ + bash -c " + apt-get update && apt-get install -y file curl net-tools || true + + HOST_IP=\$(ip route get 8.8.8.8 2>/dev/null | grep -oP 'src \\K\\S+' || hostname -I | awk '{print \$1}' || echo '172.17.0.1') + curl -f http://\${HOST_IP}:9001/info || { + echo \"ERROR: Server not reachable at \${HOST_IP}:9001\" + exit 1 + } + + echo 'Running autobahn_testee.elf' + WS_URI=\"ws://\${HOST_IP}:9001\" + echo \"WebSocket URI: \${WS_URI}\" + (sleep 0.5; printf \"\${WS_URI}\\n\") | timeout 30m ./autobahn_testee.elf || { + EXIT_CODE=\$? + echo 'Test failed' + exit \$EXIT_CODE + } + echo 'All Autobahn tests passed!' + " + + - name: Show reports + if: always() + working-directory: ${{ env.TEST_DIR }} + run: | + if [ -d reports/clients ]; then + ls -la reports/clients/ + else + echo "No reports" + fi + + - name: Generate summary + if: always() + working-directory: ${{ env.TEST_DIR }} + run: python3 scripts/generate_summary.py || true + + - name: Upload reports + if: always() + uses: actions/upload-artifact@v4 + with: + name: autobahn-reports-linux-${{ github.run_id }} + path: ${{ env.TEST_DIR }}/reports/** + if-no-files-found: warn + + linux_websocket_autobahn_perf: + runs-on: ubuntu-22.04 + # Run only if the specific label is present + if: contains(github.event.pull_request.labels.*.name, 'websocket-autobahn-perf') + + env: + TEST_DIR: components/esp_websocket_client/tests/autobahn-testsuite + TESTEE_DIR: components/esp_websocket_client/tests/autobahn-testsuite/testee + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Start Autobahn Fuzzing Server (Performance) + run: | + mkdir -p ${{ env.TEST_DIR }}/reports/clients + + HOST_IP=$(ip route get 8.8.8.8 | grep -oP 'src \K\S+' || hostname -I | awk '{print $1}' || echo "172.17.0.1") + echo "Host IP address: $HOST_IP" + echo "HOST_IP=$HOST_IP" >> $GITHUB_ENV + + docker run -d \ + --name fuzzing-server-perf \ + --network host \ + -v ${{ github.workspace }}/${{ env.TEST_DIR }}/config:/config:ro \ + -v ${{ github.workspace }}/${{ env.TEST_DIR }}/reports:/reports \ + crossbario/autobahn-testsuite:latest \ + wstest -m fuzzingserver -s /config/fuzzingserver-perf.json + + echo "Waiting for fuzzing server..." + for i in {1..5}; do + if curl -f http://localhost:9001/info >/dev/null 2>&1; then + echo "Fuzzing server ready!" + exit 0 + fi + echo "Attempt $i/5 – waiting 2s..." + sleep 2 + done + + echo "Server start failed. Container logs:" + docker logs fuzzing-server-perf + exit 1 + + - name: Build test (Linux target) + working-directory: ${{ env.TESTEE_DIR }} + run: | + docker run --rm \ + -v ${{ github.workspace }}:/work \ + -w /work/${{ env.TESTEE_DIR }} \ + espressif/idf:latest \ + bash -c " + . \$IDF_PATH/export.sh + cp sdkconfig.ci.linux sdkconfig.defaults + echo 'Building...' + idf.py build + " + + - name: Verify fuzzing server connectivity + run: | + HOST_IP=${HOST_IP:-$(ip route get 8.8.8.8 | grep -oP 'src \K\S+' || hostname -I | awk '{print $1}' || echo "172.17.0.1")} + echo "Testing connectivity to $HOST_IP:9001" + + docker run --rm \ + --network host \ + espressif/idf:latest \ + bash -c " + curl -f http://$HOST_IP:9001/info || { + echo 'ERROR: Cannot connect to fuzzing server at $HOST_IP:9001' + exit 1 + } + echo 'Fuzzing server is accessible' + " + + - name: Run Autobahn Performance tests + run: | + docker run --rm \ + --network host \ + -v ${{ github.workspace }}:/work \ + -w /work/${{ env.TESTEE_DIR }}/build \ + espressif/idf:latest \ + bash -c " + apt-get update && apt-get install -y file curl net-tools || true + + HOST_IP=\$(ip route get 8.8.8.8 2>/dev/null | grep -oP 'src \\K\\S+' || hostname -I | awk '{print \$1}' || echo '172.17.0.1') + curl -f http://\${HOST_IP}:9001/info || { + echo \"ERROR: Server not reachable at \${HOST_IP}:9001\" + exit 1 + } + + echo 'Running autobahn_testee.elf' + WS_URI=\"ws://\${HOST_IP}:9001\" + echo \"WebSocket URI: \${WS_URI}\" + (sleep 0.5; printf \"\${WS_URI}\\n\") | timeout 60m ./autobahn_testee.elf || { + EXIT_CODE=\$? + echo 'Test failed' + exit \$EXIT_CODE + } + echo 'All Autobahn performance tests passed!' + " + + - name: Show reports + if: always() + working-directory: ${{ env.TEST_DIR }} + run: | + if [ -d reports/clients ]; then + ls -la reports/clients/ + else + echo "No reports" + fi + + - name: Generate summary + if: always() + working-directory: ${{ env.TEST_DIR }} + run: python3 scripts/generate_summary.py || true + + - name: Upload reports + if: always() + uses: actions/upload-artifact@v4 + with: + name: autobahn-perf-reports-linux-${{ github.run_id }} + path: ${{ env.TEST_DIR }}/reports/** + if-no-files-found: warn diff --git a/.github/workflows/autobahn__target-test.yml b/.github/workflows/autobahn__target-test.yml new file mode 100644 index 000000000..b67c58752 --- /dev/null +++ b/.github/workflows/autobahn__target-test.yml @@ -0,0 +1,226 @@ +name: "websocket-autobahn: build/target-tests" + +on: + push: + branches: + - master + pull_request: + types: [opened, synchronize, reopened, labeled] + +jobs: + build_websocket_autobahn: + if: contains(github.event.pull_request.labels.*.name, 'websocket-autobahn') || github.event_name == 'push' + name: Build + strategy: + matrix: + idf_ver: ["release-v5.5", "latest"] + idf_target: ["esp32"] + runs-on: ubuntu-22.04 + container: espressif/idf:${{ matrix.idf_ver }} + env: + TEST_DIR: components/esp_websocket_client/tests/autobahn-testsuite + TESTEE_DIR: components/esp_websocket_client/tests/autobahn-testsuite/testee + steps: + - name: Checkout esp-protocols + uses: actions/checkout@v4 + with: + submodules: recursive + - name: Build autobahn testee with IDF-${{ matrix.idf_ver }} for ${{ matrix.idf_target }} + working-directory: ${{ env.TESTEE_DIR }} + env: + IDF_TARGET: ${{ matrix.idf_target }} + shell: bash + run: | + . ${IDF_PATH}/export.sh + test -f sdkconfig.ci.target.plain_tcp && cat sdkconfig.ci.target.plain_tcp >> sdkconfig.defaults || echo "No sdkconfig.ci.plain_tcp" + idf.py set-target ${{ matrix.idf_target }} + idf.py build + - name: Merge binaries with IDF-${{ matrix.idf_ver }} for ${{ matrix.idf_target }} + working-directory: ${{ env.TESTEE_DIR }}/build + env: + IDF_TARGET: ${{ matrix.idf_target }} + shell: bash + run: | + . ${IDF_PATH}/export.sh + esptool.py --chip ${{ matrix.idf_target }} merge_bin --fill-flash-size 4MB -o flash_image.bin @flash_args + - uses: actions/upload-artifact@v4 + with: + name: autobahn_testee_bin_${{ matrix.idf_target }}_${{ matrix.idf_ver }} + path: | + ${{ env.TESTEE_DIR }}/build/bootloader/bootloader.bin + ${{ env.TESTEE_DIR }}/build/partition_table/partition-table.bin + ${{ env.TESTEE_DIR }}/build/*.bin + ${{ env.TESTEE_DIR }}/build/*.elf + ${{ env.TESTEE_DIR }}/build/flasher_args.json + ${{ env.TESTEE_DIR }}/build/config/sdkconfig.h + ${{ env.TESTEE_DIR }}/build/config/sdkconfig.json + if-no-files-found: error + +# run-target-autobahn: +# # Skip running on forks since it won't have access to secrets +# if: | +# github.repository == 'espressif/esp-protocols' && +# ( contains(github.event.pull_request.labels.*.name, 'autobahn') || github.event_name == 'push' ) +# name: Target test +# needs: build_autobahn +# strategy: +# fail-fast: false +# matrix: +# idf_ver: ["latest"] +# idf_target: ["esp32"] +# runs-on: +# - self-hosted +# - ESP32-ETHERNET-KIT +# env: +# TEST_DIR: components/esp_websocket_client/examples/autobahn-testsuite +# TESTEE_DIR: components/esp_websocket_client/examples/autobahn-testsuite/testee +# steps: +# - uses: actions/checkout@v4 +# with: +# submodules: recursive +# - uses: actions/download-artifact@v4 +# with: +# name: autobahn_testee_bin_${{ matrix.idf_target }}_${{ matrix.idf_ver }} +# path: ${{ env.TESTEE_DIR }}/build +# - name: Install Docker Compose +# run: | +# sudo apt-get update +# sudo apt-get install -y docker-compose-plugin || sudo apt-get install -y docker-compose +# # Ensure user has permission to use Docker (if not already in docker group) +# sudo usermod -aG docker $USER || true +# # Start Docker service if not running +# sudo systemctl start docker || true +# - name: Start Autobahn Fuzzing Server +# working-directory: ${{ env.TEST_DIR }} +# run: | +# # Get host IP address for ESP32 to connect to +# HOST_IP=$(hostname -I | awk '{print $1}') +# echo "HOST_IP=$HOST_IP" >> $GITHUB_ENV +# echo "Autobahn server will be accessible at ws://$HOST_IP:9001" +# +# # Start the fuzzing server using pre-built image +# # For CI, we may need to specify platform if architecture differs +# echo "Starting Autobahn fuzzing server..." +# # Set platform for CI if needed (uncomment if you get exec format error) +# # export DOCKER_DEFAULT_PLATFORM=linux/amd64 +# docker compose up -d || docker-compose up -d +# +# # Wait for server to be ready +# echo "Waiting for fuzzing server to start..." +# sleep 10 +# +# # Check if container is running and healthy +# if ! docker ps | grep -q ws-fuzzing-server; then +# echo "Error: Fuzzing server failed to start" +# echo "Container logs:" +# docker compose logs || docker-compose logs +# echo "Checking available Python executables in container:" +# docker compose run --rm fuzzing-server which python python3 || true +# exit 1 +# fi +# +# # Verify the server is actually responding +# echo "Checking if server is responding..." +# sleep 5 +# if ! curl -s http://localhost:8080 > /dev/null 2>&1; then +# echo "Warning: Server may not be fully ready, but container is running" +# docker compose logs --tail=20 || docker-compose logs --tail=20 +# fi +# +# echo "✓ Fuzzing server started successfully" +# - name: Flash ESP32 Testee +# working-directory: ${{ env.TESTEE_DIR }}/build +# env: +# IDF_TARGET: ${{ matrix.idf_target }} +# run: | +# python -m esptool --chip ${{ matrix.idf_target }} write_flash 0x0 flash_image.bin +# - name: Run Autobahn Tests +# working-directory: ${{ env.TESTEE_DIR }} +# env: +# PIP_EXTRA_INDEX_URL: "https://www.piwheels.org/simple" +# run: | +# # Detect ESP32 port if not set in environment +# if [ -z "${ESP_PORT:-}" ]; then +# for port in /dev/ttyUSB* /dev/ttyACM*; do +# if [ -e "$port" ]; then +# export ESP_PORT="$port" +# echo "Detected ESP32 port: $ESP_PORT" +# break +# fi +# done +# fi +# +# # Default to /dev/ttyUSB0 if still not found +# export ESP_PORT="${ESP_PORT:-/dev/ttyUSB0}" +# +# if [ ! -e "$ESP_PORT" ]; then +# echo "Error: ESP32 port not found. Please set ESP_PORT environment variable." +# echo "Available ports:" +# ls -la /dev/tty* || true +# exit 1 +# fi +# +# echo "Using ESP32 port: $ESP_PORT" +# export PYENV_ROOT="$HOME/.pyenv" +# export PATH="$PYENV_ROOT/bin:$PATH" +# eval "$(pyenv init --path)" +# eval "$(pyenv init -)" +# if ! pyenv versions --bare | grep -q '^3\.12\.6$'; then +# echo "Installing Python 3.12.6..." +# pyenv install -s 3.12.6 +# fi +# if ! pyenv virtualenvs --bare | grep -q '^myenv$'; then +# echo "Creating pyenv virtualenv 'myenv'..." +# pyenv virtualenv 3.12.6 myenv +# fi +# pyenv activate myenv +# python --version +# pip install --prefer-binary pytest-embedded pytest-embedded-serial-esp pytest-embedded-idf pytest-custom_exit_code esptool pyserial +# pip install --extra-index-url https://dl.espressif.com/pypi/ -r $GITHUB_WORKSPACE/ci/requirements.txt +# +# echo "Starting Autobahn test suite on ESP32..." +# echo "Tests may take 15-30 minutes to complete..." +# +# # Send server URI via serial (stdin) and monitor for completion +# # Script is in the parent directory (TEST_DIR) from TESTEE_DIR +# SERVER_URI="ws://$HOST_IP:9001" +# echo "Sending server URI to ESP32: $SERVER_URI" +# python3 ../scripts/monitor_serial.py --port "$ESP_PORT" --uri "$SERVER_URI" --timeout 2400 +# - name: Collect Test Reports +# working-directory: ${{ env.TEST_DIR }} +# if: always() +# run: | +# # Stop the fuzzing server +# docker compose down || docker-compose down +# +# # Check if reports were generated +# if [ -d "reports/clients" ]; then +# echo "✓ Test reports found" +# ls -la reports/clients/ +# else +# echo "⚠ No test reports found in reports/clients/" +# fi +# - name: Generate Test Summary +# working-directory: ${{ env.TEST_DIR }} +# if: always() +# run: | +# # Generate summary from test results +# # Check for JSON files in both reports/ and reports/clients/ +# if [ -d "reports" ] && ( [ -n "$(ls -A reports/*.json 2>/dev/null)" ] || [ -n "$(ls -A reports/clients/*.json 2>/dev/null)" ] ); then +# echo "Generating test summary..." +# python3 scripts/generate_summary.py +# echo "" +# echo "Summary generated successfully!" +# if [ -f "reports/summary.html" ]; then +# echo "HTML summary available at: reports/summary.html" +# fi +# else +# echo "⚠ No JSON test results found, skipping summary generation" +# fi +# - uses: actions/upload-artifact@v4 +# if: always() +# with: +# name: autobahn_reports_${{ matrix.idf_target }}_${{ matrix.idf_ver }} +# path: | +# ${{ env.TEST_DIR }}/reports/** +# if-no-files-found: warn diff --git a/common_components/linux_compat/esp_timer/include/esp_timer.h b/common_components/linux_compat/esp_timer/include/esp_timer.h index 8267e403d..b6389dccf 100644 --- a/common_components/linux_compat/esp_timer/include/esp_timer.h +++ b/common_components/linux_compat/esp_timer/include/esp_timer.h @@ -8,6 +8,7 @@ #include #include #include "bsd/string.h" +#include "esp_err.h" #ifdef __cplusplus extern "C" { diff --git a/components/esp_websocket_client/tests/autobahn-testsuite/README.md b/components/esp_websocket_client/tests/autobahn-testsuite/README.md new file mode 100644 index 000000000..6457b7e41 --- /dev/null +++ b/components/esp_websocket_client/tests/autobahn-testsuite/README.md @@ -0,0 +1,130 @@ +# Autobahn WebSocket Testsuite for esp_websocket_client + +This directory contains the setup for testing `esp_websocket_client` against the [Autobahn WebSocket Testsuite](https://github.com/crossbario/autobahn-testsuite), which covers frame parsing, control frames, UTF-8 validation, and more. + +The Autobahn Testsuite is the de facto standard for testing WebSocket protocol compliance. It runs over 500 test cases covering: +- Frame parsing and generation +- Text and binary messages +- Fragmentation +- Control frames (PING, PONG, CLOSE) +- UTF-8 validation +- Protocol violations +- Edge cases and error handling + +## 📋 Prerequisites + +1. **Docker** - For running the Autobahn testsuite server. +2. **ESP32 device** with WiFi capability. +3. **ESP-IDF** development environment. +4. **Network** - ESP32 and Docker host on the same network. + +## 🚀 Quick Start + +### 1. Start the Fuzzing Server + +```bash +cd autobahn-testsuite +docker-compose up -d +``` +Starts server on port 9001 and web interface on port 8080. + +### 2. Configure ESP32 Client + +**Required:** Set WiFi credentials in `testee/sdkconfig.defaults` or via `idf.py menuconfig`: + +``` +CONFIG_EXAMPLE_WIFI_SSID="YourWiFiSSID" +CONFIG_EXAMPLE_WIFI_PASSWORD="YourWiFiPassword" +``` + +**Optional:** The Server URI is automatically detected and sent by the test script (via `CONFIG_WEBSOCKET_URI_FROM_STDIN`). +To hardcode it instead, disable `CONFIG_WEBSOCKET_URI_FROM_STDIN` and set: +``` +CONFIG_AUTOBAHN_SERVER_URI="ws://YOUR_HOST_IP:9001" +``` + +### 3. Run Tests (Automated) + +The `run_tests.sh` script handles building, flashing, and monitoring: + +```bash +./run_tests.sh /dev/ttyUSB0 +``` +*Note: Replace `/dev/ttyUSB0` with your serial port.* + +**Using Pytest Manually:** +You can also run the monitoring step directly using pytest: +```bash +pytest pytest_autobahn.py --target esp32 --port /dev/ttyUSB0 --baud 115200 --app-path testee --skip-autoflash y +``` + +### 4. View Results + +- **Web Interface:** `http://localhost:8080` (Click "Client Reports") +- **Summary Report:** Run `python3 scripts/generate_summary.py` + +## 📂 Directory Structure + +``` +autobahn-testsuite/ +├── docker-compose.yml # Docker config +├── run_tests.sh # Automated runner script +├── pytest_autobahn.py # Pytest monitoring script +├── config/ # Server config +├── scripts/ # Helper scripts +├── reports/ # Test results +└── testee/ # ESP32 client application + ├── sdkconfig.defaults # Default config + ├── sdkconfig.ci.* # CI configurations + └── main/ # Source code +``` + +## ⚙️ Configuration + +### Testee +- `sdkconfig.ci.linux`: CI configuration for running the testee on a Linux host (host-based testing), with adjusted stack sizes and lock timeouts. +- `sdkconfig.ci.target.plain_tcp`: CI configuration for ESP32 target using Ethernet (IP101 PHY) and IPv6, without TLS. + +### Fuzzing Server +Edit `config/fuzzingserver.json` to include/exclude cases (e.g., exclude performance/compression tests). + +## 📊 Test Categories + +| Category | Tests | Time | Critical? | Notes | +|----------|-------|------|-----------|-------| +| 1.* Framing | ~64 | 3m | ✅ Yes | Core compliance | +| 2.* Pings/Pongs | ~11 | 1m | ✅ Yes | Keepalive | +| 3.* Reserved Bits | ~7 | 1m | ✅ Yes | Extensions | +| 4.* Opcodes | ~10 | 1m | ✅ Yes | Protocol compliance | +| 5.* Fragmentation | ~20 | 2m | ✅ Yes | Large messages | +| 6.* UTF-8 | ~150 | 10m | ⚠️ Optional | strict validation often skipped | +| 7.* Close | ~35 | 3m | ✅ Yes | Clean disconnect | +| 9.* Performance | ~30 | 60m+ | ⚠️ Optional | Often excluded (resource intensive) | +| 10.* Misc | ~10 | 2m | ✅ Yes | Edge cases | +| 12-13.* Compression | ~200 | 30m | ❌ No | Typically not implemented | + +## 💡 Tips + + + +## 🧾 Generate Summary + +After running the tests, Autobahn writes per-case JSON results under `reports/` (and/or `reports/clients/`). +You can generate a consolidated console + HTML summary using: + +```bash +cd components/esp_websocket_client/tests/autobahn-testsuite +python3 scripts/generate_summary.py +``` + +This will: +- Print overall pass/fail statistics and per-category breakdown to the console +- Generate `reports/summary.html` + +If you’re running in CI with GitHub Actions, the script will also write a markdown summary to the path in `GITHUB_STEP_SUMMARY` (when that env var is set). + +## 📚 References + +- [Autobahn Testsuite Documentation](https://crossbar.io/autobahn/) +- [RFC 6455 - The WebSocket Protocol](https://tools.ietf.org/html/rfc6455) +- [Autobahn Testsuite GitHub](https://github.com/crossbario/autobahn-testsuite) diff --git a/components/esp_websocket_client/tests/autobahn-testsuite/config/fuzzingserver-perf.json b/components/esp_websocket_client/tests/autobahn-testsuite/config/fuzzingserver-perf.json new file mode 100644 index 000000000..a014edff6 --- /dev/null +++ b/components/esp_websocket_client/tests/autobahn-testsuite/config/fuzzingserver-perf.json @@ -0,0 +1,11 @@ +{ + "url": "ws://0.0.0.0:9001", + "options": { + "failByDrop": false + }, + "outdir": "/reports", + "webport": 8080, + "cases": ["9.*"], + "exclude-cases": [], + "exclude-agent-cases": {} +} diff --git a/components/esp_websocket_client/tests/autobahn-testsuite/config/fuzzingserver.json b/components/esp_websocket_client/tests/autobahn-testsuite/config/fuzzingserver.json new file mode 100644 index 000000000..ea615097b --- /dev/null +++ b/components/esp_websocket_client/tests/autobahn-testsuite/config/fuzzingserver.json @@ -0,0 +1,15 @@ +{ + "url": "ws://0.0.0.0:9001", + "options": { + "failByDrop": false + }, + "outdir": "/reports", + "webport": 8080, + "cases": ["*"], + "exclude-cases": [ + "9.*", + "12.*", + "13.*" + ], + "exclude-agent-cases": {} +} diff --git a/components/esp_websocket_client/tests/autobahn-testsuite/conftest.py b/components/esp_websocket_client/tests/autobahn-testsuite/conftest.py new file mode 100644 index 000000000..7f46723d3 --- /dev/null +++ b/components/esp_websocket_client/tests/autobahn-testsuite/conftest.py @@ -0,0 +1,11 @@ +# SPDX-FileCopyrightText: 2025 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Unlicense OR CC0-1.0 + + +def pytest_addoption(parser): + parser.addoption( + "--uri", + action="store", + default=None, + help="Server URI to send via serial (stdin)", + ) diff --git a/components/esp_websocket_client/tests/autobahn-testsuite/docker-compose.yml b/components/esp_websocket_client/tests/autobahn-testsuite/docker-compose.yml new file mode 100644 index 000000000..d3df30a6c --- /dev/null +++ b/components/esp_websocket_client/tests/autobahn-testsuite/docker-compose.yml @@ -0,0 +1,12 @@ +services: + fuzzing-server: + image: crossbario/autobahn-testsuite:latest + container_name: ws-fuzzing-server + platform: linux/amd64 # <— enforce amd64, use QEMU on Apple Silicon + ports: + - "9001:9001" + - "8080:8080" + volumes: + - ./config:/config + - ./reports:/reports + command: wstest -m fuzzingserver -s /config/fuzzingserver.json diff --git a/components/esp_websocket_client/tests/autobahn-testsuite/pytest_autobahn.py b/components/esp_websocket_client/tests/autobahn-testsuite/pytest_autobahn.py new file mode 100644 index 000000000..c4d14dee8 --- /dev/null +++ b/components/esp_websocket_client/tests/autobahn-testsuite/pytest_autobahn.py @@ -0,0 +1,59 @@ +# SPDX-FileCopyrightText: 2025 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Unlicense OR CC0-1.0 +import re +import time + +import pytest + + +@pytest.mark.supported_targets +@pytest.mark.generic +def test_autobahn(dut, request): + """ + Monitors ESP32 serial output and detects test completion. + Equivalent to monitor_serial.py but using pytest-embedded. + """ + uri = request.config.getoption("--uri") + + # If URI is provided, send it via serial (stdin) + if uri: + # Wait for ESP32 to be ready (booting/connecting) + time.sleep(5) + dut.write(f'{uri}\n') + print(f'Sent URI: {uri}') + + # regex patterns + progress_pattern = re.compile(rb'Case (\d+)/(\d+)') + completion_pattern = re.compile(rb'All tests completed\.') + + last_case = None + start_time = time.time() + # 40 minutes timeout as in monitor_serial.py + timeout = 2400 + + print('\n--- Waiting for ESP32 output (tests should start shortly) ---\n') + + while (time.time() - start_time) < timeout: + try: + # Expect either progress or completion + # timeout=30 to allow checking total timeout loop regularly + match = dut.expect([progress_pattern, completion_pattern], timeout=30) + except Exception: + # Timeout on single expect call, just continue loop + continue + + # Check which pattern matched + # Note: match is a match object from the regex + if match.re == completion_pattern: + print('\n✓ Test suite completed successfully!') + return + + if match.re == progress_pattern: + current_case = int(match.group(1)) + total_cases = int(match.group(2)) + + if current_case != last_case: + last_case = current_case + print(f'Progress: Test case {current_case}/{total_cases} ({100*current_case//total_cases}%)') + + pytest.fail(f"Timeout of {timeout}s reached without completion") diff --git a/components/esp_websocket_client/tests/autobahn-testsuite/run_tests.sh b/components/esp_websocket_client/tests/autobahn-testsuite/run_tests.sh new file mode 100755 index 000000000..3a5e0b793 --- /dev/null +++ b/components/esp_websocket_client/tests/autobahn-testsuite/run_tests.sh @@ -0,0 +1,277 @@ +#!/bin/bash + +# Autobahn Testsuite Runner Script +# This script automates the process of running WebSocket protocol compliance tests + +set -e + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +cd "$SCRIPT_DIR" + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +CYAN='\033[0;36m' +NC='\033[0m' # No Color + +echo -e "${BLUE}======================================${NC}" +echo -e "${BLUE}Autobahn WebSocket Testsuite Runner${NC}" +echo -e "${BLUE}======================================${NC}" +echo + +# Check if Docker is running +if ! docker info > /dev/null 2>&1; then + echo -e "${RED}Error: Docker is not running${NC}" + echo "Please start Docker and try again" + exit 1 +fi + +# Function to get host IP +get_host_ip() { + if [[ "$OSTYPE" == "darwin"* ]]; then + # macOS + ifconfig | grep "inet " | grep -v 127.0.0.1 | awk '{print $2}' | head -1 + else + # Linux + hostname -I | awk '{print $1}' + fi +} + +HOST_IP=$(get_host_ip) + +echo -e "${GREEN}Step 1: Starting Autobahn Fuzzing Server${NC}" +echo "Host IP detected: $HOST_IP" +echo + +# Start the fuzzing server +docker-compose up -d + +# Wait for server to be ready +echo "Waiting for fuzzing server to start..." +sleep 5 + +# Check if container is running +if ! docker ps | grep -q ws-fuzzing-server; then + echo -e "${RED}Error: Fuzzing server failed to start${NC}" + docker-compose logs + exit 1 +fi + +echo -e "${GREEN}✓ Fuzzing server started successfully${NC}" +echo " WebSocket endpoint: ws://$HOST_IP:9001" +echo " Web interface: http://$HOST_IP:8080" +echo + +echo -e "${YELLOW}Step 2: Running Tests on ESP32${NC}" +echo + +# Check for ESP32 port argument +ESP_PORT="${1:-}" +BUILD_ONLY="${2:-}" + +if [ -z "$ESP_PORT" ]; then + echo -e "${YELLOW}No serial port specified. Manual mode:${NC}" + echo + echo "To run tests automatically, provide the ESP32 serial port:" + echo " ./run_tests.sh /dev/ttyUSB0" + echo + echo "Or build only:" + echo " ./run_tests.sh /dev/ttyUSB0 build" + echo + echo "Manual steps:" + echo " 1. Update WiFi credentials in testee/sdkconfig.defaults" + echo " 2. Update Autobahn server URI to: ws://$HOST_IP:9001" + echo " 3. Build and flash:" + echo " cd testee" + echo " idf.py build" + echo " idf.py -p PORT flash monitor" + echo + echo -e "${YELLOW}Press Enter when ESP32 testing is complete...${NC}" + read +else + echo "ESP32 serial port: $ESP_PORT" + echo + + # Check if ESP-IDF is available + if ! command -v idf.py > /dev/null 2>&1; then + echo -e "${RED}Error: ESP-IDF not found in PATH${NC}" + echo "Please source ESP-IDF environment: . \$HOME/esp/esp-idf/export.sh" + exit 1 + fi + + # Check if port exists + if [ ! -e "$ESP_PORT" ]; then + echo -e "${RED}Error: Serial port $ESP_PORT does not exist${NC}" + exit 1 + fi + + cd testee + + # Verify we're in the right directory + if [ ! -f "sdkconfig.defaults" ] && [ ! -f "sdkconfig" ]; then + echo -e "${RED}Error: Not in testee directory or sdkconfig files not found${NC}" + echo " Current directory: $(pwd)" + exit 1 + fi + + # Update server URI in sdkconfig.defaults if needed + if [ -f "sdkconfig.defaults" ]; then + if ! grep -q "CONFIG_AUTOBAHN_SERVER_URI.*$HOST_IP" sdkconfig.defaults 2>/dev/null; then + echo -e "${YELLOW}Updating server URI in sdkconfig.defaults...${NC}" + # Backup original + cp sdkconfig.defaults sdkconfig.defaults.bak 2>/dev/null || true + # Update URI (simple approach - user should verify) + if [[ "$OSTYPE" == "darwin"* ]]; then + # macOS sed requires different syntax + sed -i '' "s|CONFIG_AUTOBAHN_SERVER_URI=.*|CONFIG_AUTOBAHN_SERVER_URI=\"ws://$HOST_IP:9001\"|" sdkconfig.defaults 2>/dev/null || true + else + sed -i.bak "s|CONFIG_AUTOBAHN_SERVER_URI=.*|CONFIG_AUTOBAHN_SERVER_URI=\"ws://$HOST_IP:9001\"|" sdkconfig.defaults 2>/dev/null || true + fi + fi + fi + + echo -e "${GREEN}Building testee...${NC}" + idf.py build + + if [ "$BUILD_ONLY" = "build" ]; then + echo -e "${GREEN}Build complete. Flash manually with:${NC}" + echo " idf.py -p $ESP_PORT flash monitor" + cd .. + exit 0 + fi + + echo -e "${GREEN}Flashing ESP32...${NC}" + idf.py -p "$ESP_PORT" flash + + echo + echo -e "${YELLOW}Waiting for ESP32 to boot (5 seconds)...${NC}" + sleep 5 + + # Check if CONFIG_WEBSOCKET_URI_FROM_STDIN is enabled + # We're in testee directory, so check files here + URI_FROM_STDIN="" + if [ -f "sdkconfig" ] && grep -q "CONFIG_WEBSOCKET_URI_FROM_STDIN=y" sdkconfig 2>/dev/null; then + URI_FROM_STDIN="ws://$HOST_IP:9001" + echo -e "${GREEN}✓ CONFIG_WEBSOCKET_URI_FROM_STDIN is enabled (from sdkconfig)${NC}" + echo " URI will be sent via serial" + elif [ -f "sdkconfig.defaults" ] && grep -q "CONFIG_WEBSOCKET_URI_FROM_STDIN=y" sdkconfig.defaults 2>/dev/null; then + URI_FROM_STDIN="ws://$HOST_IP:9001" + echo -e "${GREEN}✓ CONFIG_WEBSOCKET_URI_FROM_STDIN is enabled (from sdkconfig.defaults)${NC}" + echo " URI will be sent via serial" + else + echo -e "${YELLOW}⚠ CONFIG_WEBSOCKET_URI_FROM_STDIN is not enabled${NC}" + echo " ESP32 will use URI from sdkconfig.defaults" + echo " Make sure CONFIG_AUTOBAHN_SERVER_URI is set correctly" + fi + + echo + echo -e "${GREEN}Starting test execution...${NC}" + echo + echo -e "${YELLOW}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" + echo -e "${BLUE}Serial output (tests in progress):${NC}" + echo -e "${YELLOW}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" + echo + echo -e "${YELLOW}Note: Connection errors are expected for some test cases${NC}" + echo " (e.g., error handling tests). The testee will continue automatically." + echo + echo -e "${GREEN}Look for test case progress messages like:${NC}" + echo " 'Starting test case X...'" + echo " 'Test case X completed'" + echo + echo -e "${YELLOW}If no output appears, try:${NC}" + echo " - Press RESET button on ESP32" + echo " - Check WiFi credentials in sdkconfig.defaults" + echo " - Verify serial port: $ESP_PORT" + echo + echo -e "${YELLOW}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" + echo + + cd .. + + # Detect target from sdkconfig if available + TARGET="esp32" + if [ -f "testee/sdkconfig" ]; then + if grep -q "CONFIG_IDF_TARGET=" testee/sdkconfig; then + TARGET=$(grep "CONFIG_IDF_TARGET=" testee/sdkconfig | cut -d'"' -f2) + fi + fi + + # Use pytest-embedded to monitor and detect completion + # Only send URI if CONFIG_WEBSOCKET_URI_FROM_STDIN is enabled + # We use --app-path testee so pytest finds the build artifacts + # We use --skip-autoflash y because we already flashed above + PYTEST_ARGS="pytest_autobahn.py --target $TARGET --port $ESP_PORT --baud 115200 --app-path testee --skip-autoflash y" + if [ ! -z "$URI_FROM_STDIN" ]; then + PYTEST_ARGS="$PYTEST_ARGS --uri $URI_FROM_STDIN" + fi + + if pytest $PYTEST_ARGS; then + echo + echo -e "${GREEN}✓ Tests completed successfully!${NC}" + echo " Check the web interface for detailed results" + else + echo + echo -e "${YELLOW}⚠ Test execution ended (timeout or error)${NC}" + echo " This may be normal if:" + echo " - Tests are still running (check serial output)" + echo " - Connection errors occurred (some are expected)" + echo " - Timeout was reached" + echo + echo " Check serial output above and web interface for details" + fi +fi + +echo +echo -e "${GREEN}Step 3: Viewing Results${NC}" +echo + +# Define report paths +REPORT_FILE="$SCRIPT_DIR/reports/clients/index.html" +WEB_INTERFACE="http://localhost:8080" + +# Check if reports directory exists and has content +if [ -d "$SCRIPT_DIR/reports/clients" ] && [ "$(ls -A $SCRIPT_DIR/reports/clients/*.json 2>/dev/null)" ]; then + echo -e "${GREEN}✓ Test reports found${NC}" + echo + + # Generate and display summary + echo -e "${CYAN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" + echo -e "${BLUE}Generating Test Summary...${NC}" + echo -e "${CYAN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" + echo + + if python3 "$SCRIPT_DIR/scripts/generate_summary.py" 2>/dev/null; then + echo + echo -e "${GREEN}✓ Summary generated successfully${NC}" + else + echo -e "${YELLOW}⚠ Could not generate summary (reports may still be processing)${NC}" + echo " Run manually: cd $SCRIPT_DIR && python3 scripts/generate_summary.py" + fi +else + echo -e "${YELLOW}⚠ No test reports found yet${NC}" + echo " Reports may still be generating, or tests may not have completed" + echo " Check the web interface or wait a few moments and run:" + echo " cd $SCRIPT_DIR && python3 scripts/generate_summary.py" +fi + +echo +echo -e "${CYAN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" +echo -e "${BLUE}Test Reports Available:${NC}" +echo -e "${CYAN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" +if [[ -f "$REPORT_FILE" ]]; then + echo " 📄 Direct file: file://$REPORT_FILE" +fi +echo " 🌐 Web interface: $WEB_INTERFACE (click 'Client Reports' link)" +echo " 📁 Directory: $SCRIPT_DIR/reports/clients/" +if [[ -f "$SCRIPT_DIR/reports/summary.html" ]]; then + echo " 📊 Summary: file://$SCRIPT_DIR/reports/summary.html" +fi +echo + +echo -e "${YELLOW}To stop the fuzzing server:${NC}" +echo " docker-compose down" +echo + +echo -e "${GREEN}Testing complete!${NC}" diff --git a/components/esp_websocket_client/tests/autobahn-testsuite/scripts/generate_summary.py b/components/esp_websocket_client/tests/autobahn-testsuite/scripts/generate_summary.py new file mode 100755 index 000000000..1310e5811 --- /dev/null +++ b/components/esp_websocket_client/tests/autobahn-testsuite/scripts/generate_summary.py @@ -0,0 +1,764 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2025 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Unlicense OR CC0-1.0 +""" +Autobahn Test Results Summary Generator + +This script parses all JSON test results and generates a comprehensive summary. +""" + +import glob +import json +import os +from collections import defaultdict +from datetime import datetime +from pathlib import Path + + +# ANSI color codes for terminal output +class Colors: + GREEN = '\033[92m' + RED = '\033[91m' + YELLOW = '\033[93m' + BLUE = '\033[94m' + CYAN = '\033[96m' + MAGENTA = '\033[95m' + BOLD = '\033[1m' + END = '\033[0m' + + +# Test category descriptions +CATEGORIES = { + '1': { + 'name': 'Framing', + 'critical': True, + 'description': 'Basic frame structure', + }, + '2': { + 'name': 'Ping/Pong', + 'critical': True, + 'description': 'Control frames', + }, + '3': { + 'name': 'Reserved Bits', + 'critical': True, + 'description': 'RSV validation', + }, + '4': { + 'name': 'Opcodes', + 'critical': True, + 'description': 'Valid/invalid opcodes', + }, + '5': { + 'name': 'Fragmentation', + 'critical': True, + 'description': 'Message fragments', + }, + '6': { + 'name': 'UTF-8', + 'critical': False, + 'description': 'Text validation', + }, + '7': { + 'name': 'Close Handshake', + 'critical': True, + 'description': 'Connection closing', + }, + '9': { + 'name': 'Performance', + 'critical': False, + 'description': 'Large messages', + }, + '10': { + 'name': 'Miscellaneous', + 'critical': True, + 'description': 'Edge cases', + }, + '12': { + 'name': 'Compression', + 'critical': False, + 'description': 'RFC 7692', + }, + '13': { + 'name': 'Compression', + 'critical': False, + 'description': 'RFC 7692', + }, +} + + +def get_category_from_test_id(test_id): + """Extract category number from test ID (e.g., '1.1.1' -> '1')""" + return test_id.split('.')[0] + + +def parse_test_results(reports_dir): + """Parse all JSON test results""" + results = [] + # Look for JSON files in reports directory and reports/clients subdirectory + json_files = glob.glob(os.path.join(reports_dir, "*.json")) + json_files.extend(glob.glob(os.path.join(reports_dir, "clients", "*.json"))) + + for json_file in json_files: + try: + with open(json_file, 'r') as f: + data = json.load(f) + results.append({ + 'id': data.get('id', 'unknown'), + 'behavior': data.get('behavior', 'UNKNOWN'), + 'description': data.get('description', ''), + 'duration': data.get('duration', 0), + 'result': data.get('result', ''), + }) + except Exception as e: + print(f"Error parsing {json_file}: {e}") + + # Sort by test ID with proper numeric sorting + def sort_key(result): + parts = result['id'].split('.') + # Convert to sortable format: pad numbers, keep strings + sortable = [] + for p in parts: + if p.isdigit(): + sortable.append((0, int(p))) # numbers sort before strings + else: + sortable.append((1, p)) # strings sort after numbers + return tuple(sortable) + + return sorted(results, key=sort_key) + + +def generate_summary(results): + """Generate comprehensive summary statistics""" + + # Overall stats + total = len(results) + by_behavior = defaultdict(int) + by_category = defaultdict(lambda: defaultdict(int)) + + for result in results: + behavior = result['behavior'] + by_behavior[behavior] += 1 + + category = get_category_from_test_id(result['id']) + by_category[category][behavior] += 1 + + return { + 'total': total, + 'by_behavior': dict(by_behavior), + 'by_category': dict(by_category), + } + + +def calculate_grade(pass_rate): + """Calculate letter grade based on pass rate""" + if pass_rate >= 90: + return 'A', Colors.GREEN + elif pass_rate >= 80: + return 'B', Colors.GREEN + elif pass_rate >= 70: + return 'C', Colors.YELLOW + elif pass_rate >= 60: + return 'D', Colors.YELLOW + else: + return 'F', Colors.RED + + +def print_banner(text): + """Print a banner""" + print() + print(f"{Colors.BLUE}{'=' * 80}{Colors.END}") + print(f"{Colors.BLUE}{Colors.BOLD}{text:^80}{Colors.END}") + print(f"{Colors.BLUE}{'=' * 80}{Colors.END}") + print() + + +def print_summary_table(summary): + """Print overall summary table""" + print_banner("OVERALL TEST RESULTS") + + total = summary['total'] + by_behavior = summary['by_behavior'] + + # Calculate percentages + passed = by_behavior.get('OK', 0) + failed = by_behavior.get('FAILED', 0) + non_strict = by_behavior.get('NON-STRICT', 0) + informational = by_behavior.get('INFORMATIONAL', 0) + unimplemented = by_behavior.get('UNIMPLEMENTED', 0) + + pass_rate = (passed / total * 100) if total > 0 else 0 + fail_rate = (failed / total * 100) if total > 0 else 0 + + grade, grade_color = calculate_grade(pass_rate) + + print(f'Total Tests: {Colors.BOLD}{total}{Colors.END}') + print() + print( + f'{Colors.GREEN}✅ PASSED: {passed:3d} ' + f'({pass_rate:5.1f}%){Colors.END}', + ) + print( + f'{Colors.RED}❌ FAILED: {failed:3d} ' + f'({fail_rate:5.1f}%){Colors.END}', + ) + if non_strict > 0: + print( + f'{Colors.YELLOW}⚠️ NON-STRICT: {non_strict:3d} ' + f'({non_strict/total*100:5.1f}%){Colors.END}', + ) + if informational > 0: + print( + f'{Colors.CYAN}ℹ️ INFORMATIONAL: {informational:3d} ' + f'({informational/total*100:5.1f}%){Colors.END}', + ) + if unimplemented > 0: + print( + f'{Colors.MAGENTA}🔧 UNIMPLEMENTED: {unimplemented:3d} ' + f'({unimplemented/total*100:5.1f}%){Colors.END}', + ) + + print() + print( + f'Overall Grade: {grade_color}{Colors.BOLD}{grade}{Colors.END}', + ) + print() + + # Embedded client rating + if pass_rate >= 70: + rating = "Excellent" + rating_color = Colors.GREEN + elif pass_rate >= 55: + rating = "Good" + rating_color = Colors.GREEN + elif pass_rate >= 40: + rating = "Acceptable" + rating_color = Colors.YELLOW + else: + rating = "Needs Improvement" + rating_color = Colors.RED + + print( + 'Embedded Client Rating: ' + f'{rating_color}{Colors.BOLD}{rating}{Colors.END}', + ) + print() + + +def print_category_breakdown(summary): + """Print detailed category breakdown""" + print_banner("RESULTS BY TEST CATEGORY") + + by_category = summary['by_category'] + + # Header + print(f"{'Category':<25} {'Total':>7} {'Pass':>7} {'Fail':>7} {'Rate':>8} {'Critical':>10} {'Grade':>7}") + print(f"{'-'*25} {'-'*7} {'-'*7} {'-'*7} {'-'*8} {'-'*10} {'-'*7}") + + # Sort categories numerically + sorted_categories = sorted(by_category.keys(), key=lambda x: int(x) if x.isdigit() else 999) + + for cat_num in sorted_categories: + cat_stats = by_category[cat_num] + cat_info = CATEGORIES.get(cat_num, {'name': f'Category {cat_num}', 'critical': True}) + + total = sum(cat_stats.values()) + passed = cat_stats.get('OK', 0) + failed = cat_stats.get('FAILED', 0) + pass_rate = (passed / total * 100) if total > 0 else 0 + + grade, grade_color = calculate_grade(pass_rate) + + # Format category name + cat_name = f"{cat_num}.* {cat_info['name']}" + critical = "Yes" if cat_info['critical'] else "No" + + # Color code the pass rate + if pass_rate >= 70: + rate_color = Colors.GREEN + elif pass_rate >= 50: + rate_color = Colors.YELLOW + else: + rate_color = Colors.RED + + print(f"{cat_name:<25} {total:>7} {passed:>7} {failed:>7} " + f"{rate_color}{pass_rate:>7.1f}%{Colors.END} {critical:>10} " + f"{grade_color}{grade:>7}{Colors.END}") + + print() + + +def print_failed_tests(results, limit=20): + """Print list of failed tests""" + print_banner("FAILED TESTS (First 20)") + + failed = [r for r in results if r['behavior'] == 'FAILED'] + + if not failed: + print(f'{Colors.GREEN}🎉 No failed tests!{Colors.END}\n') + return + + print( + 'Total Failed: ' + f'{Colors.RED}{Colors.BOLD}{len(failed)}{Colors.END}\n', + ) + + for i, test in enumerate(failed[:limit], 1): + print(f"{i:2d}. {Colors.RED}Test {test['id']:<12}{Colors.END} - {test['description'][:60]}") + if test['result']: + print(f" Reason: {test['result'][:100]}") + + if len(failed) > limit: + print(f"\n... and {len(failed) - limit} more failed tests") + + print() + + +def print_recommendations(summary): + """Print recommendations based on results""" + print_banner("RECOMMENDATIONS") + + by_category = summary['by_category'] + + # Check critical categories + critical_issues = [] + + for cat_num, cat_info in CATEGORIES.items(): + if not cat_info['critical']: + continue + + if cat_num not in by_category: + continue + + cat_stats = by_category[cat_num] + total = sum(cat_stats.values()) + passed = cat_stats.get('OK', 0) + pass_rate = (passed / total * 100) if total > 0 else 0 + + if pass_rate < 70: + critical_issues.append( + { + 'category': f"{cat_num}.* {cat_info['name']}", + 'pass_rate': pass_rate, + 'description': cat_info['description'], + }, + ) + + if critical_issues: + print(f"{Colors.RED}⚠️ Critical Issues Found:{Colors.END}\n") + for issue in critical_issues: + print(f" • {issue['category']}: {issue['pass_rate']:.1f}% pass rate") + print(f" {issue['description']} - This requires attention\n") + else: + print(f"{Colors.GREEN}✅ All critical test categories are performing well!{Colors.END}\n") + + # UTF-8 check + if '6' in by_category: + cat_stats = by_category['6'] + total = sum(cat_stats.values()) + passed = cat_stats.get('OK', 0) + pass_rate = (passed / total * 100) if total > 0 else 0 + + if pass_rate < 50: + print( + f'{Colors.YELLOW}ℹ️ UTF-8 Validation (6.*): ' + f'{pass_rate:.1f}% pass rate{Colors.END}', + ) + print( + ' This is acceptable for embedded clients in ' + 'trusted environments.', + ) + print( + ' Consider adding UTF-8 validation if operating in ' + 'untrusted networks.\n', + ) + + print(f'{Colors.CYAN}💡 Next Steps:{Colors.END}\n') + print(' 1. Review failed tests in detail (see HTML report)') + print(' 2. Focus on critical categories (1-5, 7, 10)') + print(' 3. UTF-8 failures (category 6) are acceptable for embedded clients') + print(' 4. Compare your results with reference implementations\n') + + +def generate_markdown_summary(summary, results): + """Generate markdown summary for GitHub Actions Job Summary""" + total = summary['total'] + by_behavior = summary['by_behavior'] + by_category = summary['by_category'] + + passed = by_behavior.get('OK', 0) + failed = by_behavior.get('FAILED', 0) + non_strict = by_behavior.get('NON-STRICT', 0) + informational = by_behavior.get('INFORMATIONAL', 0) + unimplemented = by_behavior.get('UNIMPLEMENTED', 0) + + pass_rate = (passed / total * 100) if total > 0 else 0 + grade, _ = calculate_grade(pass_rate) + + # Determine status emoji and color + if pass_rate >= 90: + status_emoji = "🟢" + elif pass_rate >= 70: + status_emoji = "🟡" + else: + status_emoji = "🔴" + + md = f"""# {status_emoji} Autobahn Test Suite Results + +## Overall Results + +| Metric | Value | +|--------|-------| +| **Total Tests** | {total} | +| **✅ Passed** | {passed} ({pass_rate:.1f}%) | +| **❌ Failed** | {failed} ({(failed/total*100) if total > 0 else 0:.1f}%) | +""" + + if non_strict > 0: + md += f"| **⚠️ Non-Strict** | {non_strict} ({non_strict/total*100:.1f}%) |\n" + if informational > 0: + md += f"| **ℹ️ Informational** | {informational} ({informational/total*100:.1f}%) |\n" + if unimplemented > 0: + md += f"| **🔧 Unimplemented** | {unimplemented} ({unimplemented/total*100:.1f}%) |\n" + + md += f""" +| **Overall Grade** | **{grade}** | +| **Pass Rate** | {pass_rate:.1f}% | + +### Progress Bar + + {pass_rate:.1f}% + +--- + +## Results by Category + +| Category | Name | Total | Passed | Failed | Pass Rate | Critical | +|----------|------|-------|--------|--------|-----------|----------| +""" + + sorted_categories = sorted(by_category.keys(), key=lambda x: int(x) if x.isdigit() else 999) + for cat_num in sorted_categories: + cat_stats = by_category[cat_num] + cat_info = CATEGORIES.get(cat_num, {'name': f'Category {cat_num}', 'critical': True}) + + total_cat = sum(cat_stats.values()) + passed_cat = cat_stats.get('OK', 0) + failed_cat = cat_stats.get('FAILED', 0) + pass_rate_cat = (passed_cat / total_cat * 100) if total_cat > 0 else 0 + + critical_mark = "✅ Yes" if cat_info['critical'] else "⚪ No" + pass_mark = "✅" if pass_rate_cat >= 70 else "⚠️" if pass_rate_cat >= 50 else "❌" + + md += f"| **{cat_num}.** | {cat_info['name']} | {total_cat} | {passed_cat} | {failed_cat} | {pass_mark} {pass_rate_cat:.1f}% | {critical_mark} |\n" + + # Failed tests section + failed_tests = [r for r in results if r['behavior'] == 'FAILED'] + if failed_tests: + md += f""" +--- + +## ❌ Failed Tests ({len(failed_tests)} total) + +""" + for i, test in enumerate(failed_tests[:20], 1): + md += f"{i}. **Test {test['id']}**: {test['description'][:80]}\n" + if test['result']: + md += f" - Reason: {test['result'][:100]}\n" + + if len(failed_tests) > 20: + md += f"\n*... and {len(failed_tests) - 20} more failed tests*\n" + else: + md += "\n---\n\n## 🎉 No Failed Tests!\n\n" + + # Recommendations + md += "\n---\n\n## 💡 Recommendations\n\n" + + critical_issues = [] + for cat_num, cat_info in CATEGORIES.items(): + if not cat_info['critical']: + continue + if cat_num not in by_category: + continue + + cat_stats = by_category[cat_num] + total_cat = sum(cat_stats.values()) + passed_cat = cat_stats.get('OK', 0) + pass_rate_cat = (passed_cat / total_cat * 100) if total_cat > 0 else 0 + + if pass_rate_cat < 70: + critical_issues.append({ + 'category': f"{cat_num}.* {cat_info['name']}", + 'pass_rate': pass_rate_cat, + }) + + if critical_issues: + md += "### ⚠️ Critical Issues Found:\n\n" + for issue in critical_issues: + md += f"- **{issue['category']}**: {issue['pass_rate']:.1f}% pass rate - Requires attention\n" + else: + md += "### ✅ All critical test categories are performing well!\n\n" + + md += """ +### Next Steps: +1. Review failed tests in detail (see uploaded HTML report) +2. Focus on critical categories (1-5, 7, 10) +3. UTF-8 failures (category 6) are acceptable for embedded clients +4. Compare results with reference implementations + +--- +*Generated by Autobahn Test Suite Summary Generator* +""" + + return md + + +def generate_html_summary(summary, results, output_file): + """Generate an HTML summary file""" + total = summary['total'] + passed = summary['by_behavior'].get('OK', 0) + failed = summary['by_behavior'].get('FAILED', 0) + + pass_rate = (passed / total * 100) if total > 0 else 0 + fail_rate = (failed / total * 100) if total > 0 else 0 + grade, _ = calculate_grade(pass_rate) + + html = f""" + + + ESP WebSocket Client - Autobahn Test Summary + + + +
+

🔬 ESP WebSocket Client - Autobahn Test Suite Results

+

Generated: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}

+ +

📊 Overall Results

+
+
+

{total}

+

Total Tests

+
+
+

{passed}

+

Passed ({pass_rate:.1f}%)

+
+
+

{failed}

+

Failed ({fail_rate:.1f}%)

+
+
+ +
+
+ {pass_rate:.1f}% Pass Rate +
+
+ +
+ + Grade: {grade} + +
+ +

📋 Results by Category

+ + + + + + + + + + + + + +""" + + # Add category rows + sorted_categories = sorted(summary['by_category'].keys(), key=lambda x: int(x) if x.isdigit() else 999) + for cat_num in sorted_categories: + cat_stats = summary['by_category'][cat_num] + cat_info = CATEGORIES.get(cat_num, {'name': f'Category {cat_num}', 'critical': True, 'description': 'Unknown'}) + + total = sum(cat_stats.values()) + passed = cat_stats.get('OK', 0) + failed = cat_stats.get('FAILED', 0) + pass_rate = (passed / total * 100) if total > 0 else 0 + + html += f""" + + + + + + + + + +""" + + html += """ + +
CategoryDescriptionTotalPassedFailedPass RateCritical
{cat_num}.*{cat_info['name']} - {cat_info['description']}{total}{passed}{failed} + {pass_rate:.1f}% + {'✅ Yes' if cat_info['critical'] else '⚪ No'}
+ +

❌ Failed Tests

+ + + + + + + + + +""" + + failed_tests = [r for r in results if r['behavior'] == 'FAILED'] + for test in failed_tests[:50]: # Limit to 50 in HTML + html += f""" + + + + + +""" + + if len(failed_tests) > 50: + html += f""" + + + +""" + + html += """ + +
Test IDDescriptionResult
{test['id']}{test['description']}{test['result'][:100]}
+ ... and {len(failed_tests) - 50} more failed tests (see individual reports) +
+ +

💡 Recommendations

+
+

For Embedded WebSocket Clients:

+
    +
  • ✅ Focus on passing critical categories: 1.* (Framing), 2.* (Ping/Pong), 5.* (Fragmentation), 7.* (Close)
    • +
  • ⚠️ UTF-8 validation failures (6.*) are acceptable in trusted environments
    • +
  • 🎯 Target >70% overall pass rate for production use
    • +
  • 🔍 Review individual test reports for specific implementation issues
    • +
+
+ +
+

+ View Detailed Reports: + Open Full Autobahn Report +

+
+
+ + +""" + + with open(output_file, 'w') as f: + f.write(html) + + print( + f'{Colors.GREEN}✅ HTML summary generated: {output_file}{Colors.END}\n', + ) + + +def main(): + """Main entry point""" + script_dir = Path(__file__).parent + reports_dir = script_dir.parent / 'reports' + + # Check if GitHub Actions summary file path is provided + github_summary_file = os.environ.get('GITHUB_STEP_SUMMARY') + + if not reports_dir.exists(): + print( + f'{Colors.RED}Error: Reports directory not found: ' + f'{reports_dir}{Colors.END}', + ) + return 1 + + print( + f'{Colors.CYAN}📖 Parsing test results from: ' + f'{reports_dir}{Colors.END}', + ) + results = parse_test_results(str(reports_dir)) + + if not results: + print(f'{Colors.RED}Error: No test results found{Colors.END}') + return 1 + + print( + f'{Colors.GREEN}✅ Parsed {len(results)} ' + f'test results{Colors.END}', + ) + + summary = generate_summary(results) + + # Print console summary + print_summary_table(summary) + print_category_breakdown(summary) + print_failed_tests(results) + print_recommendations(summary) + + # Generate HTML summary + html_output = reports_dir / 'summary.html' + generate_html_summary(summary, results, str(html_output)) + + print(f'{Colors.CYAN}🌐 Open the summary in your browser:{Colors.END}') + print(f' file://{html_output.absolute()}\n') + + # Generate markdown summary for GitHub Actions + if github_summary_file: + md_summary = generate_markdown_summary(summary, results) + with open(github_summary_file, 'w') as f: + f.write(md_summary) + print(f'{Colors.GREEN}✅ GitHub Actions summary written to: {github_summary_file}{Colors.END}') + + return 0 + + +if __name__ == '__main__': + exit(main()) diff --git a/components/esp_websocket_client/tests/autobahn-testsuite/testee/CMakeLists.txt b/components/esp_websocket_client/tests/autobahn-testsuite/testee/CMakeLists.txt new file mode 100644 index 000000000..a8c220c60 --- /dev/null +++ b/components/esp_websocket_client/tests/autobahn-testsuite/testee/CMakeLists.txt @@ -0,0 +1,13 @@ +cmake_minimum_required(VERSION 3.5) +include($ENV{IDF_PATH}/tools/cmake/project.cmake) + +# Add Linux compatibility components if building for Linux target +# COMPONENTS must be set before include() to limit component scanning +if(${IDF_TARGET} STREQUAL "linux") + set(COMPONENTS main) +endif() + +# This override applies to this example only and does not touch ESP-IDF itself. +add_compile_options(-Wno-error=implicit-function-declaration) + +project(autobahn_testee) diff --git a/components/esp_websocket_client/tests/autobahn-testsuite/testee/main/CMakeLists.txt b/components/esp_websocket_client/tests/autobahn-testsuite/testee/main/CMakeLists.txt new file mode 100644 index 000000000..7160506e4 --- /dev/null +++ b/components/esp_websocket_client/tests/autobahn-testsuite/testee/main/CMakeLists.txt @@ -0,0 +1,11 @@ +set(requires_list esp_websocket_client protocol_examples_common esp_event) + +if(${IDF_TARGET} STREQUAL "linux") + list(APPEND requires_list esp_netif) +else() + list(APPEND requires_list nvs_flash esp_wifi) +endif() + +idf_component_register(SRCS "autobahn_testee.c" + INCLUDE_DIRS "." + REQUIRES ${requires_list}) diff --git a/components/esp_websocket_client/tests/autobahn-testsuite/testee/main/Kconfig.projbuild b/components/esp_websocket_client/tests/autobahn-testsuite/testee/main/Kconfig.projbuild new file mode 100644 index 000000000..b1a29c874 --- /dev/null +++ b/components/esp_websocket_client/tests/autobahn-testsuite/testee/main/Kconfig.projbuild @@ -0,0 +1,44 @@ +menu "Autobahn Testsuite Configuration" + + choice WEBSOCKET_URI_SOURCE + prompt "Autobahn Server URI Source" + default WEBSOCKET_URI_FROM_STRING + help + Choose how the Autobahn server URI is provided: + - From string: Use CONFIG_AUTOBAHN_SERVER_URI (compile-time) + - From stdin: Read URI from stdin (serial console at runtime) + + config WEBSOCKET_URI_FROM_STRING + bool "From string (CONFIG_AUTOBAHN_SERVER_URI)" + help + Use the URI defined in CONFIG_AUTOBAHN_SERVER_URI. + This is set at compile time. + + config WEBSOCKET_URI_FROM_STDIN + bool "From stdin (serial console)" + help + Read the URI from stdin (serial console) at runtime. + Useful for CI/CD where the server IP is only known at runtime. + The application will wait for a line containing the URI. + + endchoice + + config AUTOBAHN_SERVER_URI + string "Autobahn Fuzzing Server URI" + default "ws://192.168.1.100:9001" + depends on WEBSOCKET_URI_FROM_STRING + help + URI of the Autobahn fuzzing server. + Replace with your Docker host IP address. + Example: ws://192.168.1.100:9001 + Only used when WEBSOCKET_URI_FROM_STRING is selected. + + if CONFIG_IDF_TARGET = "linux" + config GCOV_ENABLED + bool "Coverage analyzer" + default n + help + Enables coverage analyzing for host tests. + endif + +endmenu diff --git a/components/esp_websocket_client/tests/autobahn-testsuite/testee/main/autobahn_testee.c b/components/esp_websocket_client/tests/autobahn-testsuite/testee/main/autobahn_testee.c new file mode 100644 index 000000000..421beea2b --- /dev/null +++ b/components/esp_websocket_client/tests/autobahn-testsuite/testee/main/autobahn_testee.c @@ -0,0 +1,759 @@ +/* + * SPDX-FileCopyrightText: 2025 Espressif Systems (Shanghai) CO LTD + * + * SPDX-License-Identifier: Unlicense OR CC0-1.0 + */ +#include +#include +#include +#include "esp_log.h" +#include "esp_timer.h" +#include "esp_system.h" +#include "esp_event.h" +#include "esp_netif.h" +#include "protocol_examples_common.h" +#include "esp_websocket_client.h" +#include "esp_transport_ws.h" +#include "cJSON.h" + +#if CONFIG_IDF_TARGET_LINUX +#include +#include +#include +#include +#else +#include "freertos/FreeRTOS.h" +#include "freertos/task.h" +#include "freertos/semphr.h" +#include "esp_heap_caps.h" +#include "nvs_flash.h" +#include "esp_wifi.h" +#endif + +#define TAG "autobahn" +#if CONFIG_WEBSOCKET_URI_FROM_STDIN +// URI will be read from stdin at runtime +static char g_autobahn_server_uri[256] = {0}; +#define AUTOBAHN_SERVER_URI g_autobahn_server_uri +#else +#define AUTOBAHN_SERVER_URI CONFIG_AUTOBAHN_SERVER_URI +#endif +#if CONFIG_IDF_TARGET_LINUX +#define BUFFER_SIZE 65536 +#else +#define BUFFER_SIZE 4096 +#endif +// Configure test range here: +// Category 1 (Framing): Tests 1-16 +// Category 2 (Ping/Pong): Tests 17-27 +// Category 3 (Reserved Bits): Tests 28-34 +// Category 4 (Opcodes): Tests 35-44 +// Category 5 (Fragmentation): Tests 45-64 +// Category 6 (UTF-8): Tests 65-209 +// Category 7 (Close Handshake): Tests 210-246 +// All tests: Tests 1-300 +// Defaults if get_case_count fails +#define DEFAULT_START_CASE 1 +#define DEFAULT_END_CASE 300 + +#if CONFIG_IDF_TARGET_LINUX +static sem_t test_done_sem_storage; +static sem_t *test_done_sem = NULL; +#else +static SemaphoreHandle_t test_done_sem = NULL; +#endif +static bool test_running = false; + +// Global to store total case count fetched from server +static int g_total_cases = 0; + +#if CONFIG_IDF_TARGET_LINUX +#define MAX_FRAGMENTED_PAYLOAD (16 * 1024 * 1024) // 16MB for Linux performance tests +#else +#define MAX_FRAGMENTED_PAYLOAD 65537 // 64KB+1 for embedded targets (case 1.1.6) +#endif + +typedef struct { + uint8_t *buffer; + size_t capacity; + size_t expected_len; + size_t received; + uint8_t opcode; + bool active; +} ws_accumulator_t; + +static ws_accumulator_t s_accumulator = {0}; +static uint8_t *s_accum_buffer = NULL; // Pre-allocated buffer for fragmented frames + +static void ws_accumulator_reset(void) +{ + // Reset state but keep buffer allocated for reuse + s_accumulator.buffer = s_accum_buffer; + s_accumulator.expected_len = 0; + s_accumulator.received = 0; + s_accumulator.opcode = 0; + s_accumulator.active = false; +} + +static void ws_accumulator_cleanup(void) +{ + if (s_accum_buffer) { + free(s_accum_buffer); + s_accum_buffer = NULL; + ESP_LOGD(TAG, "Freed accumulator buffer"); + } + ws_accumulator_reset(); + s_accumulator.capacity = 0; +} + +static esp_err_t ws_accumulator_prepare(size_t total_len, uint8_t opcode) +{ + if (total_len == 0) { + return ESP_OK; + } + + if (total_len > MAX_FRAGMENTED_PAYLOAD) { + ESP_LOGE(TAG, "Payload too large (%zu > %d)", total_len, MAX_FRAGMENTED_PAYLOAD); + return ESP_ERR_INVALID_SIZE; + } + + /* Allocate buffer on-demand when first fragmented frame is detected. + * Allocate only what we need for the current message to avoid permanently + * holding a 64KB buffer on constrained targets like ESP32-S2. + */ + if (!s_accum_buffer) { + size_t free_heap = esp_get_free_heap_size(); +#if CONFIG_IDF_TARGET_LINUX + size_t largest_free = free_heap; +#else + size_t largest_free = heap_caps_get_largest_free_block(MALLOC_CAP_DEFAULT); +#endif + ESP_LOGD(TAG, "Attempting accumulator alloc: need=%zu, free=%zu, largest_block=%zu", + total_len, free_heap, largest_free); + + s_accum_buffer = (uint8_t *)malloc(total_len); + if (!s_accum_buffer) { + ESP_LOGE(TAG, "Accumulator alloc failed (%zu bytes) - Free heap: %zu, largest block: %zu", + total_len, free_heap, largest_free); +#if !CONFIG_IDF_TARGET_LINUX + ESP_LOGE(TAG, "ESP32-S2 may be low on RAM. Consider reducing BUFFER_SIZE (currently %d) and/or enabling SPIRAM", + BUFFER_SIZE); +#endif + return ESP_ERR_NO_MEM; + } + s_accumulator.capacity = total_len; + ESP_LOGD(TAG, "Allocated accumulator buffer: %zu bytes (Free heap: %lu)", + total_len, esp_get_free_heap_size()); + } else if (total_len > s_accumulator.capacity) { + ESP_LOGD(TAG, "Reallocating accumulator buffer: old=%zu new=%zu", s_accumulator.capacity, total_len); + uint8_t *new_ptr = (uint8_t *)realloc(s_accum_buffer, total_len); + if (!new_ptr) { + ESP_LOGE(TAG, "Accumulator realloc failed (%zu bytes)", total_len); + return ESP_ERR_NO_MEM; + } + s_accum_buffer = new_ptr; + s_accumulator.capacity = total_len; + } + + s_accumulator.buffer = s_accum_buffer; + // s_accumulator.capacity is managed above + s_accumulator.expected_len = total_len; + s_accumulator.received = 0; + s_accumulator.opcode = opcode; + s_accumulator.active = true; + return ESP_OK; +} + +/* ------------------------------------------------------------ + * Low‑latency echo handler + * ------------------------------------------------------------ */ +static void websocket_event_handler(void *handler_args, + esp_event_base_t base, + int32_t event_id, + void *event_data) +{ + esp_websocket_event_data_t *data = (esp_websocket_event_data_t *)event_data; + esp_websocket_client_handle_t client = (esp_websocket_client_handle_t)handler_args; + + switch (event_id) { + case WEBSOCKET_EVENT_CONNECTED: + ESP_LOGI(TAG, "Connected"); + test_running = true; + break; + + case WEBSOCKET_EVENT_DISCONNECTED: + ESP_LOGI(TAG, "Disconnected"); + test_running = false; + ws_accumulator_reset(); +#if CONFIG_IDF_TARGET_LINUX + if (test_done_sem) { + sem_post(test_done_sem); + } +#else + if (test_done_sem) { + xSemaphoreGive(test_done_sem); + } +#endif + break; + + case WEBSOCKET_EVENT_DATA: { + ESP_LOGI(TAG, "WEBSOCKET_EVENT_DATA: opcode=0x%02X len=%d fin=%d payload_len=%d offset=%d", + data->op_code, data->data_len, data->fin, data->payload_len, data->payload_offset); + + // Safety check: if not connected, don't process data + if (!test_running || !esp_websocket_client_is_connected(client)) { + ESP_LOGW(TAG, "Received data but not connected, ignoring"); + ws_accumulator_reset(); + break; + } + + /* ---- skip control frames ---- */ + if (data->op_code >= 0x08) { + if (data->op_code == 0x09) { + ESP_LOGD(TAG, "PING -> PONG auto-sent"); + } + break; + } + + /* ---- Determine opcode to echo ---- */ + uint8_t send_opcode = 0; + if (data->op_code == 0x1) { + send_opcode = WS_TRANSPORT_OPCODES_TEXT; + } else if (data->op_code == 0x2) { + send_opcode = WS_TRANSPORT_OPCODES_BINARY; + } else if (data->op_code == 0x0) { + send_opcode = WS_TRANSPORT_OPCODES_CONT; + } else { + ESP_LOGW(TAG, "Unsupported opcode 0x%02X - skip", data->op_code); + break; + } + + /* Note: send_with_opcode always sets FIN bit, which is correct for these + * simple test cases (all have FIN=1). For fragmented messages, we'd need + * send_with_exact_opcode, but it's not public. */ + + // Safety check: validate data pointer before processing + if (!data->data_ptr && data->data_len > 0) { + ESP_LOGE(TAG, "NULL data pointer with non-zero length: %d", data->data_len); + break; + } + + const uint8_t *payload = (const uint8_t *)data->data_ptr; + size_t len = data->data_len; + bool used_accumulator = false; + + // Check if this is a fragmented message (either WebSocket fragmentation or TCP-level fragmentation) + // The WebSocket layer reads large frames in chunks and dispatches multiple events: + // - payload_len = total frame size (set on all chunks) + // - payload_offset = current offset (0, buffer_size, 2*buffer_size, ...) + // - data_len = current chunk size + // - fin = 1 only on the last frame of a fragmented message + size_t total_len = data->payload_len ? data->payload_len : data->data_len; + bool fragmented = (data->payload_len > 0 && data->payload_len > data->data_len) || + (data->payload_offset > 0) || (data->fin == 0) || s_accumulator.active; + + ESP_LOGD(TAG, "Fragmentation check: offset=%d payload_len=%d data_len=%d total_len=%zu fragmented=%d", + data->payload_offset, data->payload_len, data->data_len, total_len, fragmented); + + if (fragmented && total_len > 0) { + if (!s_accumulator.active) { + if (send_opcode == WS_TRANSPORT_OPCODES_CONT) { + ESP_LOGW(TAG, "Continuation frame without active accumulator, skipping"); + ws_accumulator_reset(); + break; + } + size_t initial_len = (data->fin == 0) ? data->data_len : total_len; + if (ws_accumulator_prepare(initial_len, send_opcode) != ESP_OK) { + ESP_LOGE(TAG, "Cannot allocate buffer for fragmented frame len=%zu", initial_len); + break; + } + if (data->fin == 0) { + // Unknown total length for a fragmented message; grow as needed. + s_accumulator.expected_len = 0; + } + } + + size_t required_len = s_accumulator.received + data->data_len; + if (required_len > MAX_FRAGMENTED_PAYLOAD) { + ESP_LOGE(TAG, "Payload too large (%zu > %d)", required_len, MAX_FRAGMENTED_PAYLOAD); + ws_accumulator_reset(); + break; + } + if (required_len > s_accumulator.capacity) { + uint8_t *new_ptr = (uint8_t *)realloc(s_accum_buffer, required_len); + if (!new_ptr) { + ESP_LOGE(TAG, "Accumulator realloc failed (%zu bytes)", required_len); + ws_accumulator_reset(); + break; + } + s_accum_buffer = new_ptr; + s_accumulator.buffer = s_accum_buffer; + s_accumulator.capacity = required_len; + } + + if (!s_accumulator.buffer) { + ESP_LOGE(TAG, "Accumulator buffer NULL while processing fragments"); + ws_accumulator_reset(); + break; + } + + if (s_accumulator.expected_len > 0 && required_len > s_accumulator.expected_len) { + ESP_LOGE(TAG, "Data exceeds expected length: received=%zu chunk=%d expected=%zu", + s_accumulator.received, data->data_len, s_accumulator.expected_len); + ws_accumulator_reset(); + break; + } + + memcpy(s_accumulator.buffer + s_accumulator.received, data->data_ptr, data->data_len); + s_accumulator.received = required_len; + + bool end_of_frame = (data->payload_len == 0) || + ((size_t)data->payload_offset + data->data_len >= (size_t)data->payload_len); + if (!(data->fin == 1 && end_of_frame)) { + // wait for more fragments + ESP_LOGD(TAG, "Waiting for more fragments: received=%zu fin=%d end_of_frame=%d", + s_accumulator.received, data->fin, end_of_frame); + break; + } + + // Completed full message + payload = s_accumulator.buffer; + len = s_accumulator.received; + send_opcode = s_accumulator.opcode; + s_accumulator.active = false; + used_accumulator = true; + } + + // Check connection before attempting to send + if (!esp_websocket_client_is_connected(client)) { + ESP_LOGW(TAG, "Connection lost before echo, skipping"); + ws_accumulator_reset(); + break; + } + + int sent = -1; + int attempt = 0; + const TickType_t backoff[] = {1, 1, 1, 2, 4, 8}; // Shorter backoff for faster retry + int64_t start = esp_timer_get_time(); + + /* Send echo immediately - use timeout scaled by frame size for large frames */ + /* For large messages (>16KB), the send function fragments into multiple chunks */ + /* Each chunk needs sufficient timeout, so scale timeout per chunk, not total message */ + while (sent < 0 && esp_websocket_client_is_connected(client)) { + /* For zero-length payload, pass NULL pointer (API handles this correctly) */ + /* Calculate timeout per chunk: large messages are fragmented, each chunk needs time */ +#if CONFIG_IDF_TARGET_LINUX + int send_timeout_ms = 5; // Default 5ms for small frames + if (len > 1024) { + send_timeout_ms = 500; // 500ms per chunk for large messages + } else { + send_timeout_ms = (len / 256) + 10; + if (send_timeout_ms > 100) { + send_timeout_ms = 100; + } + } + TickType_t send_timeout = pdMS_TO_TICKS(send_timeout_ms); + ESP_LOGD(TAG, "Sending echo: opcode=0x%02X len=%zu timeout=%dms", + send_opcode, len, send_timeout_ms); +#else + TickType_t send_timeout = pdMS_TO_TICKS(5); // Default 5ms for small frames + if (len > 1024) { + // For large messages, use a per-chunk timeout that accounts for network delays + // Since messages are fragmented into ~16KB chunks, each chunk needs sufficient time + // Use a fixed generous timeout per chunk for large messages (500ms per chunk) + // For 65535 bytes = 4 chunks, total time could be up to 2 seconds + send_timeout = pdMS_TO_TICKS(500); // 500ms per chunk for large messages + } else { + // Small messages: scale timeout based on size + send_timeout = pdMS_TO_TICKS((len / 256) + 10); + if (send_timeout > pdMS_TO_TICKS(100)) { + send_timeout = pdMS_TO_TICKS(100); + } + } + + ESP_LOGD(TAG, "Sending echo: opcode=0x%02X len=%zu timeout=%lums", + send_opcode, len, (unsigned long)(send_timeout * portTICK_PERIOD_MS)); +#endif + + sent = esp_websocket_client_send_with_opcode( + client, send_opcode, + (len > 0) ? payload : NULL, len, + send_timeout); + + if (sent >= 0) { + ESP_LOGD(TAG, "Echo sent successfully: %d bytes", sent); + break; + } + ESP_LOGW(TAG, + "echo send retry: opcode=0x%02X len=%zu fin=%d attempt=%d sent=%d", + send_opcode, len, data->fin, attempt + 1, sent); + if (attempt < (int)(sizeof(backoff) / sizeof(backoff[0]))) { +#if CONFIG_IDF_TARGET_LINUX + usleep(backoff[attempt++] * 1000); // Convert ticks to microseconds +#else + vTaskDelay(backoff[attempt++]); +#endif + } else { +#if CONFIG_IDF_TARGET_LINUX + usleep(32 * 1000); +#else + vTaskDelay(32); +#endif + } + } + + int64_t dt = esp_timer_get_time() - start; + if (sent >= 0) { + ESP_LOGI(TAG, "Echo success: opcode=0x%02X len=%d fin=%d in %lldus", + data->op_code, sent, data->fin, (long long)dt); + } else { + ESP_LOGE(TAG, "Echo failed: opcode=0x%02X len=%d fin=%d", + data->op_code, (int)len, data->fin); + } + + /* If we allocated an accumulator buffer for this message, free it now to + * avoid holding large buffers across test cases (helps ESP32-S2). + */ + if (used_accumulator) { + ws_accumulator_cleanup(); + } + break; + } + + case WEBSOCKET_EVENT_ERROR: + ESP_LOGW(TAG, "WebSocket error event"); + test_running = false; + ws_accumulator_reset(); // Reset accumulator on error +#if CONFIG_IDF_TARGET_LINUX + if (test_done_sem) { + sem_post(test_done_sem); + } +#else + if (test_done_sem) { + xSemaphoreGive(test_done_sem); + } +#endif + break; + + case WEBSOCKET_EVENT_FINISH: + ESP_LOGD(TAG, "WebSocket finish event"); + test_running = false; + ws_accumulator_reset(); // Reset accumulator on finish +#if CONFIG_IDF_TARGET_LINUX + if (test_done_sem) { + sem_post(test_done_sem); + } +#else + if (test_done_sem) { + xSemaphoreGive(test_done_sem); + } +#endif + break; + + default: + break; + } +} + +/* ------------------------------------------------------------ + * Handler for get_case_count + * ------------------------------------------------------------ */ +static void get_case_count_event_handler(void *handler_args, + esp_event_base_t base, + int32_t event_id, + void *event_data) +{ + esp_websocket_event_data_t *data = (esp_websocket_event_data_t *)event_data; + + switch (event_id) { + case WEBSOCKET_EVENT_DATA: + if (data->data_len > 0 && data->data_ptr) { + // The server returns a JSON integer (e.g., "518") + // We can parse it directly + char *buf = (char *)malloc(data->data_len + 1); + if (buf) { + memcpy(buf, data->data_ptr, data->data_len); + buf[data->data_len] = '\0'; + g_total_cases = atoi(buf); + ESP_LOGI(TAG, "Received total case count: %d", g_total_cases); + free(buf); + } + } + break; + default: + break; + } +} + +/* ------------------------------------------------------------ */ +static void get_case_count(void) +{ + char uri[512]; + int ret = snprintf(uri, sizeof(uri), "%s/getCaseCount", AUTOBAHN_SERVER_URI); + if (ret < 0 || ret >= (int)sizeof(uri)) { + ESP_LOGE(TAG, "URI too long"); + return; + } + + ESP_LOGI(TAG, "Getting case count from: %s", uri); + + esp_websocket_client_config_t cfg = { + .uri = uri, + .network_timeout_ms = 10000, + }; + + esp_websocket_client_handle_t client = esp_websocket_client_init(&cfg); + if (!client) { + ESP_LOGE(TAG, "Failed to init client for getCaseCount"); + return; + } + + esp_websocket_register_events(client, WEBSOCKET_EVENT_DATA, get_case_count_event_handler, NULL); + + if (esp_websocket_client_start(client) == ESP_OK) { + // Wait briefly for response +#if CONFIG_IDF_TARGET_LINUX + usleep(2000 * 1000); +#else + vTaskDelay(pdMS_TO_TICKS(2000)); +#endif + esp_websocket_client_stop(client); + } + esp_websocket_client_destroy(client); +} + +/* ------------------------------------------------------------ */ +static esp_err_t run_test_case(int case_num) +{ + char uri[512]; // Increased to accommodate full URI + path + int ret = snprintf(uri, sizeof(uri), + "%s/runCase?case=%d&agent=esp_websocket_client", + AUTOBAHN_SERVER_URI, case_num); + if (ret < 0 || ret >= (int)sizeof(uri)) { + ESP_LOGE(TAG, "URI too long: %s/runCase?case=%d&agent=esp_websocket_client", AUTOBAHN_SERVER_URI, case_num); + return ESP_ERR_INVALID_ARG; + } + ESP_LOGI(TAG, "Running case %d: %s", case_num, uri); + + esp_websocket_client_config_t cfg = { + .uri = uri, + .buffer_size = BUFFER_SIZE, + .network_timeout_ms = 10000, // 10s for connection (default), 200ms was too short + .reconnect_timeout_ms = 500, + .task_prio = 10, // High prio → low latency + .task_stack = 8144, + }; + + esp_websocket_client_handle_t client = esp_websocket_client_init(&cfg); + if (!client) { + return ESP_FAIL; + } + + esp_websocket_register_events(client, WEBSOCKET_EVENT_ANY, + websocket_event_handler, (void*)client); + +#if CONFIG_IDF_TARGET_LINUX + sem_init(&test_done_sem_storage, 0, 0); + test_done_sem = &test_done_sem_storage; +#else + test_done_sem = xSemaphoreCreateBinary(); +#endif + + esp_err_t start_ret = esp_websocket_client_start(client); + if (start_ret != ESP_OK) { + ESP_LOGE(TAG, "esp_websocket_client_start() failed: err=0x%x", start_ret); +#if CONFIG_IDF_TARGET_LINUX + if (test_done_sem) { + sem_destroy(test_done_sem); + } +#else + if (test_done_sem) { + vSemaphoreDelete(test_done_sem); + } +#endif + test_done_sem = NULL; + esp_websocket_client_destroy(client); + return start_ret; + } + + /* Wait up to 60 s so server can close properly */ +#if CONFIG_IDF_TARGET_LINUX + { + struct timespec ts; + clock_gettime(CLOCK_REALTIME, &ts); + ts.tv_sec += 60; // absolute timeout (now + 60s) + (void)sem_timedwait(test_done_sem, &ts); + } +#else + xSemaphoreTake(test_done_sem, pdMS_TO_TICKS(60000)); +#endif + + if (esp_websocket_client_is_connected(client)) { + esp_websocket_client_stop(client); + } + + esp_websocket_client_destroy(client); +#if CONFIG_IDF_TARGET_LINUX + if (test_done_sem) { + sem_destroy(test_done_sem); + } +#else + if (test_done_sem) { + vSemaphoreDelete(test_done_sem); + } +#endif + test_done_sem = NULL; + ESP_LOGI(TAG, "Free heap: %lu", esp_get_free_heap_size()); + return ESP_OK; +} + +/* ------------------------------------------------------------ */ +static void update_reports(void) +{ + char uri[512]; // Increased to accommodate full URI + path + int ret = snprintf(uri, sizeof(uri), + "%s/updateReports?agent=esp_websocket_client", + AUTOBAHN_SERVER_URI); + if (ret < 0 || ret >= (int)sizeof(uri)) { + ESP_LOGE(TAG, "URI too long: %s/updateReports?agent=esp_websocket_client", AUTOBAHN_SERVER_URI); + return; + } + esp_websocket_client_config_t cfg = { .uri = uri }; + esp_websocket_client_handle_t client = esp_websocket_client_init(&cfg); + if (!client) { + ESP_LOGE(TAG, "Failed to initialize WebSocket client for update_reports"); + return; + } + esp_err_t start_ret = esp_websocket_client_start(client); + if (start_ret != ESP_OK) { + ESP_LOGE(TAG, "esp_websocket_client_start() failed for update_reports: err=0x%x", start_ret); + esp_websocket_client_destroy(client); + return; + } +#if CONFIG_IDF_TARGET_LINUX + usleep(3000 * 1000); // 3 seconds +#else + vTaskDelay(pdMS_TO_TICKS(3000)); +#endif + esp_websocket_client_stop(client); + esp_websocket_client_destroy(client); + ESP_LOGI(TAG, "Reports updated"); +} + +/* ------------------------------------------------------------ */ +static void websocket_app_start(void) +{ + ESP_LOGI(TAG, "===================================="); + ESP_LOGI(TAG, " Autobahn WebSocket Testsuite Client"); + ESP_LOGI(TAG, "===================================="); + + ESP_LOGI(TAG, "Server: %s", AUTOBAHN_SERVER_URI); + + // Accumulator buffer is allocated on-demand only when fragmentation is detected. + // This keeps memory usage low on constrained targets like ESP32-S2. + + // Attempt to fetch case count dynamically + get_case_count(); + + int start_case = DEFAULT_START_CASE; + int end_case = g_total_cases > 0 ? g_total_cases : DEFAULT_END_CASE; + + ESP_LOGI(TAG, "Running tests from case %d to %d", start_case, end_case); + + for (int i = start_case; i <= end_case; i++) { + ESP_LOGI(TAG, "========== Case %d/%d ==========", i, end_case); + ESP_LOGI(TAG, "Starting test case %d...", i); + esp_err_t ret = run_test_case(i); + if (ret != ESP_OK) { + ESP_LOGW(TAG, "Test case %d failed with error: 0x%x", i, ret); + } else { + ESP_LOGI(TAG, "Test case %d completed", i); + } +#if CONFIG_IDF_TARGET_LINUX + usleep(500 * 1000); // 500ms +#else + vTaskDelay(pdMS_TO_TICKS(500)); +#endif + } + update_reports(); + + // Free accumulator buffer after all tests + ws_accumulator_cleanup(); + ESP_LOGI(TAG, "All tests completed."); +} + +#if CONFIG_WEBSOCKET_URI_FROM_STDIN +/* ------------------------------------------------------------ + * Read URI from stdin (similar to websocket_example.c) + * ------------------------------------------------------------ */ +static void get_string(char *line, size_t size) +{ + int count = 0; + while (count < size - 1) { + int c = fgetc(stdin); + if (c == '\n' || c == '\r') { + line[count] = '\0'; + break; + } else if (c > 0 && c < 127) { + line[count] = c; + ++count; + } else if (c == EOF) { +#if CONFIG_IDF_TARGET_LINUX + usleep(10 * 1000); // 10ms +#else + vTaskDelay(pdMS_TO_TICKS(10)); +#endif + } + } + line[count] = '\0'; +} +#endif /* CONFIG_WEBSOCKET_URI_FROM_STDIN */ + +/* ------------------------------------------------------------ */ +#if CONFIG_IDF_TARGET_LINUX +int main(void) +#else +void app_main(void) +#endif +{ + // Disable stdout buffering for immediate output + setvbuf(stdout, NULL, _IONBF, 0); + setvbuf(stderr, NULL, _IONBF, 0); + + ESP_LOGI(TAG, "Startup, IDF %s", esp_get_idf_version()); +#if !CONFIG_IDF_TARGET_LINUX + ESP_ERROR_CHECK(nvs_flash_init()); +#endif + ESP_ERROR_CHECK(esp_netif_init()); + ESP_ERROR_CHECK(esp_event_loop_create_default()); + + // Accumulator buffer is allocated on-demand when needed (fragmented payloads). + // Pre-allocating ~64KB here can prevent esp_websocket_client_init() from + // allocating its own buffers on ESP32-S2. + + ESP_ERROR_CHECK(example_connect()); + +#if !CONFIG_IDF_TARGET_LINUX + /* disable power‑save for low latency */ + esp_wifi_set_ps(WIFI_PS_NONE); +#endif + +#if CONFIG_WEBSOCKET_URI_FROM_STDIN + // Read server URI from stdin + ESP_LOGI(TAG, "Waiting for Autobahn server URI from stdin..."); + ESP_LOGI(TAG, "Please send URI in format: ws://:9001"); + // Loop until we get a valid URI + do { + get_string(g_autobahn_server_uri, sizeof(g_autobahn_server_uri)); + // Ensure null termination + g_autobahn_server_uri[sizeof(g_autobahn_server_uri) - 1] = '\0'; + } while (strlen(g_autobahn_server_uri) == 0); + + ESP_LOGI(TAG, "Received server URI: %s", g_autobahn_server_uri); +#endif + + websocket_app_start(); +#if CONFIG_IDF_TARGET_LINUX + return 0; +#endif +} diff --git a/components/esp_websocket_client/tests/autobahn-testsuite/testee/main/idf_component.yml b/components/esp_websocket_client/tests/autobahn-testsuite/testee/main/idf_component.yml new file mode 100644 index 000000000..1f26cc20f --- /dev/null +++ b/components/esp_websocket_client/tests/autobahn-testsuite/testee/main/idf_component.yml @@ -0,0 +1,27 @@ +dependencies: + ## Required IDF version + idf: ">=5.0" + # WebSocket client component from parent directory + espressif/esp_websocket_client: + version: "^1.0.0" + override_path: "../../../../" + # WiFi connection helper from ESP-IDF examples + espressif/cjson: + version: "^1.7.15" + protocol_examples_common: + path: ${IDF_PATH}/examples/common_components/protocol_examples_common + # Linux compatibility components + espressif/esp_timer: + version: "*" + override_path: "../../../../../../common_components/linux_compat/esp_timer" + rules: + - if: "target == linux" + freertos: + version: "*" + override_path: "../../../../../../common_components/linux_compat/freertos" + rules: + - if: "target == linux" + esp_stubs: + path: ${IDF_PATH}/examples/protocols/linux_stubs/esp_stubs + rules: + - if: "target == linux" diff --git a/components/esp_websocket_client/tests/autobahn-testsuite/testee/sdkconfig.ci.linux b/components/esp_websocket_client/tests/autobahn-testsuite/testee/sdkconfig.ci.linux new file mode 100644 index 000000000..02baf7376 --- /dev/null +++ b/components/esp_websocket_client/tests/autobahn-testsuite/testee/sdkconfig.ci.linux @@ -0,0 +1,26 @@ +# Linux target configuration for CI +CONFIG_IDF_TARGET="linux" +CONFIG_IDF_TARGET_LINUX=y +CONFIG_ESP_EVENT_POST_FROM_ISR=n +CONFIG_ESP_EVENT_POST_FROM_IRAM_ISR=n + +# Autobahn server URI from stdin (set at runtime) +CONFIG_WEBSOCKET_URI_FROM_STDIN=y +CONFIG_WEBSOCKET_URI_FROM_STRING=n + +# WebSocket configuration +CONFIG_WS_TRANSPORT=y +CONFIG_WS_BUFFER_SIZE=8192 + +# Enable separate TX lock +CONFIG_ESP_WS_CLIENT_SEPARATE_TX_LOCK=y +CONFIG_ESP_WS_CLIENT_TX_LOCK_TIMEOUT_MS=100 + +# Main task stack +CONFIG_ESP_MAIN_TASK_STACK_SIZE=8192 + +# Logging +CONFIG_LOG_DEFAULT_LEVEL_INFO=y + +# For Linux host builds, allow warnings (do not treat default warnings as errors) +CONFIG_COMPILER_DISABLE_DEFAULT_ERRORS=y diff --git a/components/esp_websocket_client/tests/autobahn-testsuite/testee/sdkconfig.ci.target.plain_tcp b/components/esp_websocket_client/tests/autobahn-testsuite/testee/sdkconfig.ci.target.plain_tcp new file mode 100644 index 000000000..6efc58013 --- /dev/null +++ b/components/esp_websocket_client/tests/autobahn-testsuite/testee/sdkconfig.ci.target.plain_tcp @@ -0,0 +1,15 @@ +CONFIG_IDF_TARGET="esp32" +CONFIG_IDF_TARGET_LINUX=n +CONFIG_WEBSOCKET_URI_FROM_STDIN=y +CONFIG_WEBSOCKET_URI_FROM_STRING=n +CONFIG_EXAMPLE_CONNECT_ETHERNET=y +CONFIG_EXAMPLE_CONNECT_WIFI=n +CONFIG_EXAMPLE_USE_INTERNAL_ETHERNET=y +CONFIG_EXAMPLE_ETH_PHY_IP101=y +CONFIG_EXAMPLE_ETH_MDC_GPIO=23 +CONFIG_EXAMPLE_ETH_MDIO_GPIO=18 +CONFIG_EXAMPLE_ETH_PHY_RST_GPIO=5 +CONFIG_EXAMPLE_ETH_PHY_ADDR=1 +CONFIG_EXAMPLE_CONNECT_IPV6=y +CONFIG_WS_OVER_TLS_MUTUAL_AUTH=n +CONFIG_WS_OVER_TLS_SERVER_AUTH=n diff --git a/components/esp_websocket_client/test/CMakeLists.txt b/components/esp_websocket_client/tests/unit/CMakeLists.txt similarity index 89% rename from components/esp_websocket_client/test/CMakeLists.txt rename to components/esp_websocket_client/tests/unit/CMakeLists.txt index 2052fc087..e619a97d2 100644 --- a/components/esp_websocket_client/test/CMakeLists.txt +++ b/components/esp_websocket_client/tests/unit/CMakeLists.txt @@ -9,7 +9,7 @@ else() set(test_component_dir $ENV{IDF_PATH}/tools/unit-test-app/components) endif() -set(EXTRA_COMPONENT_DIRS ../../esp_websocket_client +set(EXTRA_COMPONENT_DIRS ../../../esp_websocket_client ${test_component_dir}) project(websocket_unit_test) diff --git a/components/esp_websocket_client/test/main/CMakeLists.txt b/components/esp_websocket_client/tests/unit/main/CMakeLists.txt similarity index 100% rename from components/esp_websocket_client/test/main/CMakeLists.txt rename to components/esp_websocket_client/tests/unit/main/CMakeLists.txt diff --git a/components/esp_websocket_client/test/main/test_websocket_client.c b/components/esp_websocket_client/tests/unit/main/test_websocket_client.c similarity index 100% rename from components/esp_websocket_client/test/main/test_websocket_client.c rename to components/esp_websocket_client/tests/unit/main/test_websocket_client.c diff --git a/components/esp_websocket_client/test/pytest_websocket.py b/components/esp_websocket_client/tests/unit/pytest_websocket.py similarity index 100% rename from components/esp_websocket_client/test/pytest_websocket.py rename to components/esp_websocket_client/tests/unit/pytest_websocket.py diff --git a/components/esp_websocket_client/test/sdkconfig.ci b/components/esp_websocket_client/tests/unit/sdkconfig.ci similarity index 100% rename from components/esp_websocket_client/test/sdkconfig.ci rename to components/esp_websocket_client/tests/unit/sdkconfig.ci diff --git a/components/esp_websocket_client/test/sdkconfig.defaults b/components/esp_websocket_client/tests/unit/sdkconfig.defaults similarity index 100% rename from components/esp_websocket_client/test/sdkconfig.defaults rename to components/esp_websocket_client/tests/unit/sdkconfig.defaults