Add initial hardware testing support (#6313)

- Added workflow triggered by cron or label "hil_test"
- Added examples with both pytest and unity

.github/scripts/on-push.sh

@@ -58,7 +58,7 @@ fi
 
 SCRIPTS_DIR="./.github/scripts"
 if [ "$BUILD_PIO" -eq 0 ]; then
-    source ./.github/scripts/install-arduino-ide.sh
+    source ${SCRIPTS_DIR}/install-arduino-ide.sh
     source ${SCRIPTS_DIR}/install-arduino-core-esp32.sh
 
     FQBN_ESP32="espressif:esp32:esp32:PSRAM=enabled,PartitionScheme=huge_app"
@@ -80,7 +80,7 @@ if [ "$BUILD_PIO" -eq 0 ]; then
     build "esp32s2" $FQBN_ESP32S2 $CHUNK_INDEX $CHUNKS_CNT $SKETCHES_ESP32XX
     build "esp32c3" $FQBN_ESP32C3 $CHUNK_INDEX $CHUNKS_CNT $SKETCHES_ESP32XX
 else
-    source ./${SCRIPTS_DIR}/install-platformio-esp32.sh
+    source ${SCRIPTS_DIR}/install-platformio-esp32.sh
     # PlatformIO ESP32 Test
     BOARD="esp32dev"
     OPTIONS="board_build.partitions = huge_app.csv"

.github/scripts/sketch_utils.sh

@@ -32,13 +32,13 @@ function build_sketch(){ # build_sketch <ide_path> <user_path> <fqbn> <path-to-i
         $win_opts $xtra_opts "$sketch"
 }
 
-function count_sketches(){ # count_sketches <path> <target>
+function count_sketches(){ # count_sketches <path> [target]
     local path=$1
     local target=$2
 
-    if [ $# -lt 2 ]; then
+    if [ $# -lt 1 ]; then
       echo "ERROR: Illegal number of parameters"
-      echo "USAGE: ${0} count <path> <target>"
+      echo "USAGE: ${0} count <path> [target]"
     fi
 
     rm -rf sketches.txt
@@ -47,7 +47,7 @@ function count_sketches(){ # count_sketches <path> <target>
         return 0
     fi
 
-    local sketches=$(find $path -name *.ino)
+    local sketches=$(find $path -name *.ino | sort)
     local sketchnum=0
     for sketch in $sketches; do
         local sketchdir=$(dirname $sketch)
@@ -55,7 +55,7 @@ function count_sketches(){ # count_sketches <path> <target>
         local sketchname=$(basename $sketch)
         if [[ "$sketchdirname.ino" != "$sketchname" ]]; then
             continue
-        elif [[ -f "$sketchdir/.skip.$target" ]]; then
+        elif [[ -n $target ]] && [[ -f "$sketchdir/.skip.$target" ]]; then
             continue
         else
             echo $sketch >> sketches.txt

.github/scripts/tests_build.sh

@@ -0,0 +1,56 @@
+#!/bin/bash
+
+SCRIPTS_DIR="./.github/scripts"
+BUILD_CMD=""
+
+if [ $# -eq 3 ]; then
+    chunk_build=1
+elif [ $# -eq 2 ]; then
+    chunk_build=0
+else
+  echo "ERROR: Illegal number of parameters"
+  echo "USAGE:
+        ${0} <target> <sketch_dir>
+        ${0} <target> <chunk> <total_chunks>
+        "
+  exit 0
+fi
+
+target=$1
+
+case "$target" in
+    "esp32") fqbn="espressif:esp32:esp32:PSRAM=enabled,PartitionScheme=huge_app"
+    ;;
+    "esp32s2") fqbn="espressif:esp32:esp32s2:PSRAM=enabled,PartitionScheme=huge_app"
+    ;;
+    "esp32c3") fqbn="espressif:esp32:esp32c3:PartitionScheme=huge_app"
+    ;;
+esac
+
+if [ -z $fqbn ]; then
+  echo "Unvalid chip $1"
+  exit 0
+fi
+
+source ${SCRIPTS_DIR}/install-arduino-ide.sh
+source ${SCRIPTS_DIR}/install-arduino-core-esp32.sh
+
+args="$ARDUINO_IDE_PATH $ARDUINO_USR_PATH \"$fqbn\""
+
+if [ $chunk_build -eq 1 ]; then
+    chunk_index=$2
+    chunk_max=$3
+
+    if [ "$chunk_index" -gt "$chunk_max" ] &&  [ "$chunk_max" -ge 2 ]; then
+        chunk_index=$chunk_max
+    fi
+    BUILD_CMD="${SCRIPTS_DIR}/sketch_utils.sh chunk_build"
+    args+=" $target $PWD/tests $chunk_index $chunk_max"
+else
+    sketchdir=$2
+    BUILD_CMD="${SCRIPTS_DIR}/sketch_utils.sh build"
+    args+=" $PWD/tests/$sketchdir/$sketchdir.ino"
+fi
+
+${BUILD_CMD} ${args}
+

.github/scripts/tests_run.sh

@@ -0,0 +1,71 @@
+#!/bin/bash
+
+target=$1
+chunk_idex=$2
+chunks_num=$3
+
+SCRIPTS_DIR="./.github/scripts"
+COUNT_SKETCHES="${SCRIPTS_DIR}/sketch_utils.sh count"
+
+source ${SCRIPTS_DIR}/install-arduino-ide.sh
+
+if [ "$chunks_num" -le 0 ]; then
+    echo "ERROR: Chunks count must be positive number"
+    return 1
+fi
+if [ "$chunk_idex" -ge "$chunks_num" ] && [ "$chunks_num" -ge 2 ]; then
+    echo "ERROR: Chunk index must be less than chunks count"
+    return 1
+fi
+
+set +e
+${COUNT_SKETCHES} $PWD/tests $target
+sketchcount=$?
+set -e
+sketches=$(cat sketches.txt)
+rm -rf sketches.txt
+
+chunk_size=$(( $sketchcount / $chunks_num ))
+all_chunks=$(( $chunks_num * $chunk_size ))
+if [ "$all_chunks" -lt "$sketchcount" ]; then
+    chunk_size=$(( $chunk_size + 1 ))
+fi
+
+start_index=0
+end_index=0
+if [ "$chunk_idex" -ge "$chunks_num" ]; then
+    start_index=$chunk_idex
+    end_index=$sketchcount
+else
+    start_index=$(( $chunk_idex * $chunk_size ))
+    if [ "$sketchcount" -le "$start_index" ]; then
+        echo "Skipping job"
+        return 0
+    fi
+
+    end_index=$(( $(( $chunk_idex + 1 )) * $chunk_size ))
+    if [ "$end_index" -gt "$sketchcount" ]; then
+        end_index=$sketchcount
+    fi
+fi
+
+start_num=$(( $start_index + 1 ))
+sketchnum=0
+
+for sketch in $sketches; do
+    sketchdir=$(dirname $sketch)
+    sketchdirname=$(basename $sketchdir)
+    sketchname=$(basename $sketch)
+    sketchnum=$(($sketchnum + 1))
+    if [ "$sketchnum" -le "$start_index" ] \
+    || [ "$sketchnum" -gt "$end_index" ]; then
+        continue
+    fi
+    echo ""
+    echo "Test for Sketch Index $(($sketchnum - 1)) - $sketchdirname"
+    pytest tests -k test_$sketchdirname --junit-xml=tests/$sketchdirname/$sketchdirname.xml
+    result=$?
+    if [ $result -ne 0 ]; then
+        return $result
+    fi
+done

.github/workflows/hil.yml

@@ -0,0 +1,120 @@
+name: Run tests in hardware
+
+on:
+  pull_request:
+    types: [opened, reopened, synchronize, labeled]
+
+  schedule:
+    - cron: '0 2 * * *'
+
+env:
+  MAX_CHUNKS: 15
+
+concurrency:
+  group: hil-${{github.event.pull_request.number || github.ref}}
+  cancel-in-progress: true
+
+jobs:
+  gen_chunks:
+    if: |
+      contains(github.event.pull_request.labels.*.name, 'hil_test') ||
+      github.event_name == 'schedule'
+    name: Generate Chunks matrix
+    runs-on: ubuntu-latest
+    outputs:
+      chunks: ${{ steps.gen-chunks.outputs.chunks }}
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v2
+
+      - name: Generate Chunks matrix
+        id: gen-chunks
+        run: |
+          set +e
+          bash .github/scripts/sketch_utils.sh count tests
+          sketches=$((? - 1))
+          if [[ $sketches -gt ${{env.MAX_CHUNKS}} ]]; then
+            $sketches=${{env.MAX_CHUNKS}}
+          fi
+          set -e
+          rm sketches.txt
+          CHUNKS=$(jq -c -n '$ARGS.positional' --args `seq 0 1 $sketches`)
+          echo "::set-output name=chunks::${CHUNKS}"
+
+  Build:
+    needs: gen_chunks
+    name: ${{matrix.chip}}-Build#${{matrix.chunks}}
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        chip: ['esp32', 'esp32s2', 'esp32c3']
+        chunks: ${{fromJson(needs.gen_chunks.outputs.chunks)}}
+
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v2
+
+      - name: Build sketches
+        run: |
+          bash .github/scripts/tests_build.sh ${{matrix.chip}} ${{matrix.chunks}} ${{env.MAX_CHUNKS}}
+      - name: Upload ${{matrix.chip}}-${{matrix.chunks}} artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          name: ${{matrix.chip}}-${{matrix.chunks}}.artifacts
+          path: |
+             tests/*/build/*.bin
+             tests/*/build/*.json
+  Test:
+    needs: [gen_chunks, Build]
+    name: ${{matrix.chip}}-Test#${{matrix.chunks}}
+    runs-on: ESP32
+    strategy:
+      fail-fast: false
+      matrix:
+        chip: ['esp32', 'esp32s2', 'esp32c3']
+        chunks: ${{fromJson(needs.gen_chunks.outputs.chunks)}}
+    container:
+      image: python:3.10.1-bullseye
+      options: --privileged
+
+    steps:
+       - name: Checkout repository
+         uses: actions/checkout@v2
+
+       - name: Download ${{matrix.chip}}-${{matrix.chunks}} artifacts
+         uses: actions/download-artifact@v2
+         with:
+           name: ${{matrix.chip}}-${{matrix.chunks}}.artifacts
+           path: tests/
+
+       - name: Check Artifacts
+         run: |
+           ls -R tests
+           cat tests/*/build/build.options.json
+
+       - name: Install dependencies
+         run: |
+           pip install -U pip
+           pip install -r tests/requirements.txt
+
+       - name: Run Tests
+         run: |
+           bash .github/scripts/tests_run.sh ${{matrix.chip}} ${{matrix.chunks}} ${{env.MAX_CHUNKS}}
+
+       - name: Upload test result artifacts
+         uses: actions/upload-artifact@v2
+         if: always()
+         with:
+           name: test_results-${{matrix.chip}}-${{matrix.chunks}}
+           path: tests/*/*.xml
+
+  event_file:
+    name: "Event File"
+    needs: Test
+    runs-on: ubuntu-latest
+    steps:
+       - name: Upload
+         uses: actions/upload-artifact@v2
+         with:
+           name: Event File
+           path: ${{github.event_path}}

.github/workflows/publish.yml

@@ -0,0 +1,38 @@
+name: Unit Test Results
+
+on:
+  workflow_run:
+    workflows: [Run tests in hardware]
+
+    types:
+      - completed
+
+jobs:
+  debug:
+    name: Debug
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Debug Action
+        uses: hmarr/debug-action@v2.0.0
+
+  unit-test-results:
+    name: Unit Test Results
+    runs-on: ubuntu-latest
+    steps:
+      - name: Download and Extract Artifacts
+        env:
+          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
+        run: |
+          mkdir -p artifacts && cd artifacts
+          artifacts_url=${{ github.event.workflow_run.artifacts_url }}
+          gh api "$artifacts_url" -q '.artifacts[] | [.name, .archive_download_url] | @tsv' | while read artifact
+          do
+            IFS=$'\t' read name url <<< "$artifact"
+            gh api $url > "$name.zip"
+            unzip -d "$name" "$name.zip"
+          done
+      - name: Publish Unit Test Results
+        uses: EnricoMi/publish-unit-test-result-action@v1
+        with:
+          commit: ${{ github.event.workflow

.gitignore

@@ -22,3 +22,6 @@ boards.sloeber.txt
 # Ignore docs build (Sphinx)
 docs/build
 docs/source/_build
+
+# Test log files
+*.log

tests/.gitignore

@@ -0,0 +1,2 @@
+build/
+__pycache__/

tests/hello_world/hello_world.ino

@@ -0,0 +1,12 @@
+void setup(){
+  // Open serial communications and wait for port to open:
+  Serial.begin(115200);
+  while (!Serial) {
+    ;
+  }
+
+  Serial.println("Hello Arduino!");
+}
+
+void loop(){
+}

tests/hello_world/test_hello_world.py

@@ -0,0 +1,2 @@
+def test_hello_world(dut):
+    dut.expect('Hello Arduino!')

tests/pytest.ini

@@ -0,0 +1,13 @@
+[pytest]
+addopts = --embedded-services esp,arduino
+
+# log related
+log_cli = True
+log_cli_level = INFO
+log_cli_format = %(asctime)s %(levelname)s %(message)s
+log_cli_date_format = %Y-%m-%d %H:%M:%S
+
+log_file = test.log
+log_file_level = INFO
+log_file_format = %(asctime)s %(levelname)s %(message)s
+log_file_date_format = %Y-%m-%d %H:%M:%S

tests/requirements.txt

@@ -0,0 +1,13 @@
+pyserial>=3.0
+esptool>=3.1
+pytest-cov
+cryptography<3.4; platform_machine == "armv7l"
+
+pytest>=6.2.0
+pexpect>=4.4
+
+pytest-embedded>=0.5.1
+pytest-embedded-serial>=0.5.1
+pytest-embedded-serial-esp>=0.5.1
+pytest-embedded-arduino>=0.5.1
+

tests/unity/test_unity.py

@@ -0,0 +1,2 @@
+def test_unity(dut):
+    dut.expect_unity_test_output(timeout=240)

tests/unity/unity.ino

@@ -0,0 +1,33 @@
+#include <unity.h>
+
+
+/* These functions are intended to be called before and after each test. */
+void setUp(void) {
+}
+
+void tearDown(void){
+}
+
+
+void test_pass(void){
+  TEST_ASSERT_EQUAL(1, 1);
+}
+
+void test_fail(void){
+  TEST_ASSERT_EQUAL(1, 1);
+}
+
+void setup() {
+  Serial.begin(115200);
+  while (!Serial) {
+    ;
+  }
+  
+  UNITY_BEGIN();
+  RUN_TEST(test_pass);
+  RUN_TEST(test_fail);
+  UNITY_END();
+}
+
+void loop() {
+}