From b37f08f9617f304c32210a84f2bcf6c6211916a9 Mon Sep 17 00:00:00 2001 From: He Yin Ling Date: Sat, 28 Apr 2018 17:17:05 +0800 Subject: [PATCH 01/14] test: add mesh basic function test cases --- .gitlab-ci.yml | 6 + .../integration_test/INIT_COND_SSC.yml | 217 ++++++++++++++++++ .../integration_test/TC_IT_MESH_COMM.yml | 178 ++++++++++++++ .../integration_test/TC_IT_MESH_EST.yml | 217 ++++++++++++++++++ .../integration_test/TEST_ENV_SSC.yml | 8 + 5 files changed, 626 insertions(+) create mode 100644 components/idf_test/integration_test/TC_IT_MESH_COMM.yml create mode 100644 components/idf_test/integration_test/TC_IT_MESH_EST.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d484f9f38b..d7b4cd3db2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1250,6 +1250,12 @@ IT_010_01: - ESP32_IDF - SSC_T5_1 +IT_011_01: + <<: *test_template + tags: + - ESP32_IDF + - SSC_T50_1 + IT_501_01: <<: *test_template tags: diff --git a/components/idf_test/integration_test/INIT_COND_SSC.yml b/components/idf_test/integration_test/INIT_COND_SSC.yml index 797e4445c6..0676015330 100644 --- a/components/idf_test/integration_test/INIT_COND_SSC.yml +++ b/components/idf_test/integration_test/INIT_COND_SSC.yml @@ -7,6 +7,17 @@ - - SSC SSC1 ram - - 'R SSC1 C +FREEHEAP:' +.MESH_INIT_COND: &MESH_INIT_COND + test script: InitCondBase + restore post cmd set: + - '' + - - SSC SSC[1-] mesh -Q -o 1 + - - R SSC[1-] C MESH_NETWORK + - - SSC SSC[1-] mesh -Q -o 3 + - - R SSC[1-] C +MESH_CONFIG:ALL + - - SSC SSC[1-] ram + - - R SSC[1-] A :(\d+) + initial condition: - tag: APM1 <<: *SSC_INIT_COND @@ -2038,3 +2049,209 @@ initial condition: - - R SSC[1-2] C +BLEADV:OK - - SSC SSC1 ram - - R SSC1 A :(\d+) +- tag: ENABLED_1 + <<: *MESH_INIT_COND + initial condition detail: if mesh tree not exist, start one node first, then start others, after mesh network + established, root connect server + check cmd set: + - '' + - - SSC SSC[1-] mesh -Q -o 2 + - - R SSC[1-] T + - - MESHTREE + - - R PC_COM RE "MESHTREE:%%s%20nodes"%%() + - - SOC SOC1 LISTEN + - - R SOC_COM L OK + - - SSC MNODE(0) mesh -S -o 0 -i -t + - - '' + - - SOC SOC1 MACCEPT GSOC1 + - - P MNODE(0) C +CONNECT,OK + - R SOC_COM L OK + - - SSC SSC[1-] mesh -F -o 4 -a 1 + - - P SSC[1-] C +MESHFLAG:OK + - - SSC SSC[1-] mesh -F -o 5 -a 1 + - - P SSC[1-] C +MESHFLAG:OK + restore cmd set: + - '' + - - SSC SSC[1-] reboot + - - P SSC[1-] C !!!ready!!! + - - SSC SSC[1-] mesh -I + - - P SSC[1-] C +MESH:INITED + - - SSC SSC[1-] mesh -A -o 12 -t -c -l -f -b + - - P SSC[1-] C +MESH_SET_PARENT_SWITCH:OK + - - SSC SSC[1-] mesh -A -o 9 -t -s + - - P SSC[1-] C +MESH_SET_AP_AUTH_MODE:OK C +MESH_SET_AP_AUTH_PWD:OK + - - SSC SSC[1-] mesh -P -g -s -p -n + -m -y + - - P SSC[1-] C +MESH:CONFIG,OK + - - SSC SSC1 mesh -T + - - P SSC1 C +MESH:START,OK + - - DELAY 10 + - - '' + - - SSC SSC[2-] mesh -T + - - P SSC[2-] C +MESH:START,OK + - - DELAY + - - '' + - - SSC SSC[1-] mesh -Q -o 2 + - - R SSC[1-] T + - - MESHTREE + - - R PC_COM RE "MESHTREE:%%s%20nodes"%%() + - - SSC MNODE(0) mesh -Q -o 1 -t + - - R MNODE(0) C NETWORK_TIME:PASS + - - SOC SOC1 LISTEN + - - R SOC_COM L OK + - - SSC MNODE(0) mesh -S -o 0 -i -t + - - '' + - - SOC SOC1 MACCEPT GSOC1 + - - P MNODE(0) C +CONNECT,OK + - R SOC_COM L OK + - - SSC SSC[1-] mesh -F -o 4 -a 1 + - - P SSC[1-] C +MESHFLAG:OK + - - SSC SSC[1-] mesh -F -o 5 -a 1 + - - P SSC[1-] C +MESHFLAG:OK + force restore cmd set: + - '' + - - SSC SSC[1-] restore + - - P SSC[1-] C !!!ready!!! + - - SSC SSC[1-] mesh -I + - - P SSC[1-] C +MESH:INITED + - - SSC SSC[1-] mesh -A -o 12 -t -c -l -f -b + - - P SSC[1-] C +MESH_SET_PARENT_SWITCH:OK + - - SSC SSC[1-] mesh -A -o 9 -t -s + - - P SSC[1-] C +MESH_SET_AP_AUTH_MODE:OK C +MESH_SET_AP_AUTH_PWD:OK + - - SSC SSC[1-] mesh -P -g -s -p -n + -m -y + - - P SSC[1-] C +MESH:CONFIG,OK + - - SSC SSC1 mesh -T + - - P SSC1 C +MESH:START,OK + - - DELAY 10 + - - '' + - - SSC SSC[2-] mesh -T + - - P SSC[2-] C +MESH:START,OK + - - DELAY + - - '' + - - SSC SSC[1-] mesh -Q -o 2 + - - R SSC[1-] T + - - MESHTREE + - - R PC_COM RE "MESHTREE:%%s%20nodes"%%() + - - SSC MNODE(0) mesh -Q -o 1 -t + - - R MNODE(0) C NETWORK_TIME:PASS + - - SOC SOC1 LISTEN + - - R SOC_COM L OK + - - SSC MNODE(0) mesh -S -o 0 -i -t + - - '' + - - SOC SOC1 MACCEPT GSOC1 + - - P MNODE(0) C +CONNECT,OK + - R SOC_COM L OK + - - SSC SSC[1-] mesh -F -o 4 -a 1 + - - P SSC[1-] C +MESHFLAG:OK + - - SSC SSC[1-] mesh -F -o 5 -a 1 + - - P SSC[1-] C +MESHFLAG:OK +- tag: ENABLED_2 + <<: *MESH_INIT_COND + initial condition detail: if mesh tree not exist, start all nodes together + check cmd set: + - '' + - - SSC SSC[1-] mesh -Q -o 2 + - - R SSC[1-] T + - - MESHTREE + - - R PC_COM RE "MESHTREE:%%s%20nodes"%%() + restore cmd set: + - '' + - - SSC SSC[1-] reboot + - - P SSC[1-] C !!!ready!!! + - - SSC SSC[1-] mesh -I + - - P SSC[1-] C +MESH:INITED + - - SSC SSC[1-] mesh -A -o 12 -t -c -l -f -b + - - P SSC[1-] C +MESH_SET_PARENT_SWITCH:OK + - - SSC SSC[1-] mesh -A -o 9 -t -s + - - P SSC[1-] C +MESH_SET_AP_AUTH_MODE:OK C +MESH_SET_AP_AUTH_PWD:OK + - - SSC SSC[1-] mesh -P -g -s -p -n + -m -y + - - P SSC[1-] C +MESH:CONFIG,OK + - - SSC SSC[1-] mesh -T + - - P SSC[1-] C +MESH:START,OK + - - DELAY + - - '' + - - SSC SSC[1-] mesh -Q -o 2 + - - R SSC[1-] T + - - MESHTREE + - - R PC_COM RE "MESHTREE:%%s%20nodes"%%() + - - SSC MNODE(0) mesh -Q -o 1 -t + - - R MNODE(0) C NETWORK_TIME:PASS + force restore cmd set: + - '' + - - SSC SSC[1-] restore + - - P SSC[1-] C !!!ready!!! + - - SSC SSC[1-] mesh -I + - - P SSC[1-] C +MESH:INITED + - - SSC SSC[1-] mesh -A -o 12 -t -c -l -f -b + - - P SSC[1-] C +MESH_SET_PARENT_SWITCH:OK + - - SSC SSC[1-] mesh -A -o 9 -t -s + - - P SSC[1-] C +MESH_SET_AP_AUTH_MODE:OK C +MESH_SET_AP_AUTH_PWD:OK + - - SSC SSC[1-] mesh -P -g -s -p -n + -m -y + - - P SSC[1-] C +MESH:CONFIG,OK + - - SSC SSC[1-] mesh -T + - - P SSC[1-] C +MESH:START,OK + - - DELAY + - - '' + - - SSC SSC[1-] mesh -Q -o 2 + - - R SSC[1-] T + - - MESHTREE + - - R PC_COM RE "MESHTREE:%%s%20nodes"%%() + - - SSC MNODE(0) mesh -Q -o 1 -t + - - R MNODE(0) C NETWORK_TIME:PASS +- tag: ENABLED_3 + <<: *MESH_INIT_COND + initial condition detail: all mesh nodes in softap+sta mode, mesh configed but not started + check cmd set: + - '' + restore cmd set: + - '' + - - SSC SSC[1-] reboot + - - P SSC[1-] C !!!ready!!! + - - SSC SSC[1-] mesh -I + - - P SSC[1-] C +MESH:INITED + - - SSC SSC[1-] mesh -A -o 12 -t -c -l -f -b + - - P SSC[1-] C +MESH_SET_PARENT_SWITCH:OK + - - SSC SSC[1-] mesh -A -o 9 -t -s + - - P SSC[1-] C +MESH_SET_AP_AUTH_MODE:OK C +MESH_SET_AP_AUTH_PWD:OK + - - SSC SSC[1-] mesh -P -g -s -p -n + -m -y + - - P SSC[1-] C +MESH:CONFIG,OK + force restore cmd set: + - '' + - - SSC SSC[1-] restore + - - P SSC[1-] C !!!ready!!! + - - SSC SSC[1-] mesh -I + - - P SSC[1-] C +MESH:INITED + - - SSC SSC[1-] mesh -A -o 12 -t -c -l -f -b + - - P SSC[1-] C +MESH_SET_PARENT_SWITCH:OK + - - SSC SSC[1-] mesh -A -o 9 -t -s + - - P SSC[1-] C +MESH_SET_AP_AUTH_MODE:OK C +MESH_SET_AP_AUTH_PWD:OK + - - SSC SSC[1-] mesh -P -g -s -p -n + -m -y + - - P SSC[1-] C +MESH:CONFIG,OK +- tag: DISABLED_1 + <<: *MESH_INIT_COND + initial condition detail: all mesh node in softap+sta mode, disable all mesh node + check cmd set: + - '' + - - ASSERT + - - '' + restore cmd set: + - '' + - - SSC SSC[1-] reboot + - - P SSC[1-] C !!!ready!!! + - - SSC SSC[1-] op -Q + - - P SSC[1-] C +CURMODE:3 + - - SSC SSC[1-] sta -D + - - P SSC[1-] C +QAP:OK + force restore cmd set: + - '' + - - SSC SSC[1-] restore + - - P SSC[1-] C !!!ready!!! + - - SSC SSC[1-] op -S -o 3 + - - P SSC[1-] C +MODE:OK + - - SSC SSC[1-] sta -D + - - P SSC[1-] C +QAP:OK \ No newline at end of file diff --git a/components/idf_test/integration_test/TC_IT_MESH_COMM.yml b/components/idf_test/integration_test/TC_IT_MESH_COMM.yml new file mode 100644 index 0000000000..30e07fc46b --- /dev/null +++ b/components/idf_test/integration_test/TC_IT_MESH_COMM.yml @@ -0,0 +1,178 @@ +test cases: +- CI ready: 'Yes' + ID: MESH_COMM_0101 + SDK: ESP32_IDF + Test App: SSC_MESH + auto test: 'Yes' + category: Function + cmd set: + - '' + - - SSC MNODE(0) meshsend -S -d -l 1000 -c 20 -b 20 -f 2 + - - P MNODE(0) C +MESHSEND,OK + - P MNODE(0,-1) C +MESHRXPKT:OK + execution time: 0.0 + expected result: |- + 1. succeed + 2. succeed + 3. succeed + 4. succeed + initial condition: ENABLED_2 + level: Integration + module: Mesh + steps: |- + 1. mesh init + 2. mesh config + 3. mesh start + 4. root send unicast to leaf + sub module: Communication + summary: root send unicast to leaf + test environment: SSC_T50_1 + test point 1: basic function + test point 2: unicast test + version: v1 (2017-7-20) +- CI ready: 'Yes' + ID: MESH_COMM_0106 + SDK: ESP32_IDF + Test App: SSC_MESH + auto test: 'Yes' + category: Function + cmd set: + - '' + - - SSC MNODE(0,-1) meshsend -S -d -l 1000 -c 20 -b 20 -f 2 + - - P MNODE(0,-1) C +MESHSEND,OK + - P MNODE(0) C +MESHRXPKT:OK + execution time: 0.0 + expected result: |- + 1. succeed + 2. succeed + 3. succeed + 4. succeed + initial condition: ENABLED_1 + level: Integration + module: Mesh + steps: |- + 1. mesh init + 2. mesh config + 3. mesh start + 4. leaf send unicast to root + sub module: Communication + summary: leaf send unicast to root + test environment: SSC_T50_1 + test point 1: basic function + test point 2: unicast test + version: v1 (2017-7-20) +- CI ready: 'Yes' + ID: MESH_COMM_0116 + SDK: ESP32_IDF + Test App: SSC + auto test: 'Yes' + category: Function + cmd set: + - '' + - - SSC MNODE(0,-1) meshsend -S -d -l 1000 -c 100 -f 2 + - - P MNODE(0,-1) C +MESHTXPKT:OK C +MESHSEND,OK + - P MNODE(0,1) C +MESHRXPKT:OK + - - DELAY 5 + - - '' + - - SSC MNODE(0,-1) meshsend -S -d -l 1000 -c 10 -f 2 + - - P MNODE(0,-1) C +MESHTXPKT:OK C +MESHSEND,OK + - P MNODE(0,1) C +MESHRXPKT:OK + expected result: |- + 1. succeed + 2. succeed + 3. succeed + 4. succeed + initial condition: ENABLED_2 + level: Integration + module: Mesh + steps: |- + 1. mesh init + 2. mesh config + 3. mesh start + 4. root send unicast to others + sub module: Communication + summary: fluid control test when node send packets upward + test environment: SSC_T50_1 + test point 1: basic function + test point 2: unicast test + version: v1 (2017-7-20) +- CI ready: 'Yes' + ID: MESH_COMM_0210 + SDK: ESP32_IDF + Test App: SSC_MESH + auto test: 'Yes' + category: Function + cmd set: + - '' + - - SSC MNODE(0) meshsend -S -d -l 1460 -c 20 -b 20 -f 4 + - - P MNODE(0) C +MESHTXPKT:OK C +MESHSEND,OK + - P MNODE(0,0) C +MESHRXPKT:OK + execution time: 0.0 + expected result: |- + 1. succeed + 2. succeed + 3. succeed + 4. succeed + initial condition: ENABLED_1 + level: Integration + module: Mesh + steps: |- + 1. mesh init + 2. mesh config + 3. mesh start + 4. root send unicast + sub module: Communication + summary: root send unicast downward with flag=FROMDS + test environment: SSC_T50_1 + test point 1: basic function + test point 2: meshsend parameter check + version: v1 (2017-7-20) +- CI ready: 'Yes' + ID: MESH_COMM_0902 + SDK: ESP32_IDF + Test App: SSC_MESH + auto test: 'Yes' + category: Function + cmd set: + - '' + - - SSC MNODE(0,0,0) mesh -D -o 0 -g + - - P MNODE(0,0,0) C +MESH_ADDR_LIST_ADD + - - SSC MNODE(0,0,0) mesh -D -o 0 -g + - - P MNODE(0,0,0) C +MESH_ADDR_LIST_ADD + - - SSC MNODE(0,0,0) meshsend -P -p 0 -t 7 -l 12 + - - P MNODE(0,0,0) C +MESHSEND:OPTION,1 + - - SSC MNODE(0,0,0) meshsend -S -w 1 -d 01:00:5E:00:00:00 -c 10 -b 20 -f + 2 + - - P MNODE(0,0,0) C +MESHTXPKT:ERROR C +MESHSEND,OK + - P MNODE(0,0) NC +MESHRXPKT:OK + - - SSC MNODE(0,0,0) meshsend -S -w 1 -d 01:00:5E:00:00:00 -l 0 -c 10 -b 20 -f + 2 + - - P MNODE(0,0,0) C +MESHTXPKT:ERROR C +MESHSEND,OK + - P MNODE(0,0) NC +MESHRXPKT:OK + - - SSC MNODE(0,0,0) meshsend -S -w 1 -d 01:00:5E:00:00:00 -l 1000 -c 10 -b 20 -f + 2 + - - P MNODE(0,0,0) C +MESHTXPKT:OK C +MESHSEND,OK + - P MNODE(0) C +MESHRXPKT:OK + - - SSC MNODE(0,0,0) meshsend -S -w 1 -d 01:00:5E:00:00:00 -l 1460 -c 10 -b 20 -f + 2 + - - P MNODE(0,0,0) C +MESHTXPKT:ERROR C +MESHSEND,OK + execution time: 0.0 + expected result: |- + 1. succeed + 2. succeed + 3. succeed + 4. succeed + initial condition: ENABLED_1 + level: Integration + module: Mesh + steps: |- + 1. mesh init + 2. mesh config + 3. mesh start + 4. node send multicast with different length + sub module: Communication + summary: node send multicast with different length + test environment: SSC_T50_1 + test point 1: basic function + test point 2: meshsend parameter check + version: v1 (2017-7-20) diff --git a/components/idf_test/integration_test/TC_IT_MESH_EST.yml b/components/idf_test/integration_test/TC_IT_MESH_EST.yml new file mode 100644 index 0000000000..9d5ac0b766 --- /dev/null +++ b/components/idf_test/integration_test/TC_IT_MESH_EST.yml @@ -0,0 +1,217 @@ +test cases: +- CI ready: 'Yes' + ID: MESH_EST_0106 + SDK: ESP32_IDF + Test App: SSC_MESH + auto test: 'Yes' + category: Function + cmd set: + - '' + - - SSC SSC[1-] mesh -I + - - P SSC[1-] C +MESH:INITED + - - SSC SSC[1-] mesh -A -o 7 -t 25 + - - P SSC[1-] C +MESH_SET_TX_POWER:OK + - - SSC SSC[1-] mesh -P -g -s -p -n -m -y + - - P SSC[1-] C +MESH:CONFIG,OK + - - SSC SSC[1-] mesh -T + - - P SSC[1-] C +MESH:START + - - DELAY + - - '' + - - SSC SSC[1-] mesh -Q -o 2 + - - R SSC[1-] T + - - MESHTREE + - - R PC_COM RE "MESHTREE:%%s%20nodes"%%() + execution time: 0.0 + expected result: |- + 1. succeed + 2. succeed + 3. succeed + initial condition: DISABLED_1 + level: Integration + module: Mesh + steps: |- + 1. mesh init + 2. mesh config + 3. mesh start + sub module: Network Establish + summary: set node with different tx power, then establish network + test environment: SSC_T50_1 + test point 1: basic function + test point 2: mesh config + version: v1 (2017-7-20) +- CI ready: 'Yes' + ID: MESH_EST_0313 + SDK: ESP32_IDF + Test App: SSC_MESH + auto test: 'Yes' + category: Function + cmd set: + - '' + - - SSC SSC[1-] mesh -I + - - P SSC[1-] C +MESH:INITED + - - SSC SSC[1-] mesh -P -g -s -p -n + -m -y + - - P SSC[1-] C +MESH:CONFIG,OK + - - SSC SSC[1-2] mesh -T + - - P SSC[1-2] C +MESH:START + - - DELAY 30 + - - '' + - - SSC SSC[1-2] mesh -Q -o 2 + - - R SSC[1-2] T 2 + - - MESHTREE + - - R PC_COM C MESHTREE:2%20nodes + - - SSC MNODE(0) reboot + - - P MNODE(0) C !!!ready!!! + - P MNODE(0,0) C MESH_EVENT_DISCONNECTED + - - DELAY 10 + - - P MNODE(0,0) C MESH_EVENT_CONNECTED,1 + execution time: 0.0 + expected result: |- + 1. succeed + 2. succeed + 3. succeed + 4. succeed + initial condition: DISABLED_1 + level: Integration + module: Mesh + steps: |- + 1. switch to sta+softap mode + 2. mesh init + 3. mesh config + 4. mesh start + sub module: Network Establish + summary: 2 nodes do mesh network establish,then reboot root + test environment: SSC_T50_1 + test point 1: basic function + test point 2: mesh network establish + version: v1 (2017-7-20) +- CI ready: 'Yes' + ID: MESH_EST_0317 + SDK: ESP32_IDF + Test App: SSC_MESH + auto test: 'Yes' + category: Function + cmd set: + - '' + - - SSC SSC[1-] mesh -I + - - P SSC[1-] C +MESH:INITED + - - SSC SSC[1-] mesh -P -g -s -p -n + -m 1 -y 15 + - - P SSC[1-] C +MESH:CONFIG,OK + - - SSC SSC[1-15] mesh -T + - - P SSC[1-15] C +MESH:START + - - DELAY + - - '' + - - SSC SSC[1-15] mesh -Q -o 2 + - - R SSC[1-15] T 15 + - - MESHTREE + - - R PC_COM C MESHTREE:15%20nodes + execution time: 0.0 + expected result: |- + 1. succeed + 2. succeed + 3. succeed + 4. succeed + 5. succeed + initial condition: DISABLED_1 + level: Integration + module: Mesh + steps: |- + 1. switch to sta+softap mode + 2. mesh init + 3. mesh config: set mesh layer=15 + 4. start 15 nodes + 5. check mesh tree + sub module: Network Establish + summary: set special mesh tree which layer=15 + test environment: SSC_T50_1 + test point 1: basic function + test point 2: mesh network establish + version: v1 (2017-7-20) +- CI ready: 'Yes' + ID: MESH_EST_0404 + SDK: ESP32_IDF + Test App: SSC_MESH + auto test: 'Yes' + category: Function + cmd set: + - '' + - - SSC MNODE(0) reboot + - - P MNODE(0) C !!!ready!!! + - - DELAY 2 + - - '' + - - SSC MNODE(0) mesh -I + - - P MNODE(0) C +MESH:INITED + - - SSC MNODE(0) mesh -A -o 9 -t -s + - - P MNODE(0) C +MESH_SET_AP_AUTH_MODE:OK C +MESH_SET_AP_AUTH_PWD:OK + - - SSC MNODE(0) mesh -P -g -s -p -n + -m -y + - - P MNODE(0) C +MESH:CONFIG,OK + - - SSC MNODE(0) mesh -T + - - P MNODE(0) C +MESH:START + - - DELAY + - - '' + - - SSC SSC[1-] mesh -Q -o 2 + - - R SSC[1-] T + - - MESHTREE + - - R PC_COM RE "MESHTREE:%%s%20nodes"%%() + execution time: 0.0 + expected result: |- + 1. succeed + 2. succeed + 3. succeed + 4. succeed + 5. succeed + initial condition: ENABLED_2 + level: Integration + module: Mesh + steps: |- + 1. mesh init + 2. mesh config + 3. mesh start + 4. save value + 5. all nodes reboot + sub module: Network Establish + summary: after network is establish, disable root,wait some time, start the root node again + test environment: SSC_T50_1 + test point 1: basic function + test point 2: mesh network re-establish + version: v1 (2017-7-20) +- CI ready: 'Yes' + ID: MESH_EST_0405 + SDK: ESP32_IDF + Test App: SSC_MESH + auto test: 'Yes' + category: Function + cmd set: + - '' + - - SSC MNODE(0) reboot + - - P MNODE(0) C !!!ready!!! + - - DELAY + - - '' + - - SSC SSC[1-] mesh -Q -o 2 + - - R SSC[1-] T + - - MESHTREE + - - R PC_COM RE "MESHTREE:%%s%20nodes"%%() + execution time: 0.0 + expected result: |- + 1. succeed + 2. succeed + 3. succeed + 4. succeed + 5. succeed + initial condition: ENABLED_2 + level: Integration + module: Mesh + steps: |- + 1. mesh init + 2. mesh config + 3. mesh start + 4. save value + 5. all nodes reboot + sub module: Network Establish + summary: after network is establish, disable root, check mesh network + test environment: SSC_T50_1 + test point 1: basic function + test point 2: mesh network re-establish + version: v1 (2017-7-20) diff --git a/components/idf_test/integration_test/TEST_ENV_SSC.yml b/components/idf_test/integration_test/TEST_ENV_SSC.yml index e9675ec477..08c652f971 100644 --- a/components/idf_test/integration_test/TEST_ENV_SSC.yml +++ b/components/idf_test/integration_test/TEST_ENV_SSC.yml @@ -238,3 +238,11 @@ test environment: PC has 1 wired NIC connected to AP. PC has 1 WiFi NIC. 6 SSC target connect with PC by UART. +- tag: SSC_T50_1 + <<: *TEST_ENV + Special: Y + Target Count: 50 + test environment detail: |- + PC has 1 wired NIC connected to AP. + PC has 1 WiFi NIC. + 50 SSC target connect with PC by UART. From 0fc888cf7881ba9b1ab632f6c10f3235ec264bb1 Mon Sep 17 00:00:00 2001 From: He Yin Ling Date: Sun, 24 Jun 2018 13:58:27 +0800 Subject: [PATCH 02/14] CI: remove test report stage: Test report is not used as expected: 1. we rarely download and use test report 2. current test report method doesn't handle large amount data well, need to be replaced by other methods Test report also make test jobs allow to fail. It breaks the original flow of Gitlab CI and make user confused. --- .gitlab-ci.yml | 57 -------------------------------------------------- 1 file changed, 57 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d7b4cd3db2..cd6c340ebc 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,7 +3,6 @@ stages: - assign_test - unit_test - test - - test_report - deploy variables: @@ -343,61 +342,6 @@ test_build_system: - cd test_build_system - ${IDF_PATH}/tools/ci/test_build_system.sh -test_report: - stage: test_report - image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG - tags: - - report - only: - - master - - triggers - - schedules - - /^release\/v/ - - /^v\d+\.\d+(\.\d+)?($|-)/ - variables: - LOG_PATH: "$CI_PROJECT_DIR/$CI_COMMIT_SHA" - TEST_CASE_FILE_PATH: "$CI_PROJECT_DIR/components/idf_test" - REPORT_PATH: "$CI_PROJECT_DIR/CI_Test_Report" - MODULE_UPDATE_FILE: "$CI_PROJECT_DIR/tools/unit-test-app/tools/ModuleDefinition.yml" - #dependencies: - #We need all UT* and IT* artifacts except for only a few other - artifacts: - when: always - paths: - - $REPORT_PATH - - $LOG_PATH - expire_in: 12 mos - script: - # calc log path - - VER_NUM=`git rev-list HEAD | wc -l | awk '{print $1}'` - - SHA_ID=`echo $CI_COMMIT_SHA | cut -c 1-7` - - REVISION="${VER_NUM}_${SHA_ID}" - # replace / to _ in branch name - - ESCAPED_BRANCH_NAME=`echo $CI_COMMIT_REF_NAME | sed 's/\//___/g'` - # result path and artifacts path - - RESULT_PATH="$CI_PROJECT_NAME/$ESCAPED_BRANCH_NAME/$REVISION" - - ARTIFACTS_PATH="$GITLAB_HTTP_SERVER/idf/esp-idf/builds/$CI_JOB_ID/artifacts/browse/$CI_COMMIT_SHA" - # clone test bench - - git clone $GITLAB_SSH_SERVER/yinling/auto_test_script.git - - cd auto_test_script - - python $CHECKOUT_REF_SCRIPT auto_test_script - # generate report - - TEST_RESULT=Pass - - python CITestReport.py -l $LOG_PATH -t $TEST_CASE_FILE_PATH -p $REPORT_PATH -r $RESULT_PATH -a $ARTIFACTS_PATH -m $MODULE_UPDATE_FILE || TEST_RESULT=Fail - # commit to CI-test-result project - - git clone $GITLAB_SSH_SERVER/qa/CI-test-result.git - - rm -rf "CI-test-result/RawData/$RESULT_PATH" - - cp -R $CI_PROJECT_NAME CI-test-result/RawData - - cd CI-test-result - # config git user - - git config --global user.email "ci-test-result@espressif.com" - - git config --global user.name "ci-test-result" - # commit test result - - git add . - - git commit . -m "update test result for $CI_PROJECT_NAME/$CI_COMMIT_REF_NAME/$CI_COMMIT_SHA, pipeline ID $CI_PIPELINE_ID" || exit 0 - - git push origin master - - test "${TEST_RESULT}" = "Pass" || exit 1 - test_esp_err_to_name_on_host: stage: test image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG @@ -632,7 +576,6 @@ assign_test: - /^v\d+\.\d+(\.\d+)?($|-)/ - triggers - schedules - allow_failure: true dependencies: - assign_test - build_ssc_00 From 5637424affeb4e8347d2320bd4f591c52188d3f6 Mon Sep 17 00:00:00 2001 From: He Yin Ling Date: Sun, 24 Jun 2018 14:11:57 +0800 Subject: [PATCH 03/14] CI: minor optimize of CI config file: 1. set shorter expire time for artifacts 2. set dependency for example test jobs, to limit the artifacts it downloads --- .gitlab-ci.yml | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index cd6c340ebc..2b5b2b9634 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -150,6 +150,8 @@ build_ssc_01: build_ssc_02: <<: *build_ssc_template +# If you want to add new build ssc jobs, please add it into dependencies of `assign_test` and `.test_template` + build_esp_idf_tests: <<: *build_template artifacts: @@ -157,7 +159,7 @@ build_esp_idf_tests: - tools/unit-test-app/output - components/idf_test/unit_test/TestCaseAll.yml - components/idf_test/unit_test/CIConfigs/*.yml - expire_in: 6 mos + expire_in: 1 mos script: - cd tools/unit-test-app - MAKEFLAGS= make help # make sure kconfig tools are built in single process @@ -217,6 +219,7 @@ build_examples_06: build_examples_07: <<: *build_examples_template +# If you want to add new build example jobs, please add it into dependencies of `.example_test_template` build_docs: stage: build @@ -529,14 +532,21 @@ assign_test: - /^v\d+\.\d+(\.\d+)?($|-)/ - triggers - schedules - # gitlab ci do not support match job with RegEx or wildcard now in dependencies. - # we have a lot build example jobs and the binaries them exceed the limitation of artifacts. - # we can't artifact them in one job. For example test jobs, download all artifacts from previous stages. + dependencies: + - assign_test + - build_examples_00 + - build_examples_01 + - build_examples_02 + - build_examples_03 + - build_examples_04 + - build_examples_05 + - build_examples_06 + - build_examples_07 artifacts: when: always paths: - $LOG_PATH - expire_in: 6 mos + expire_in: 1 mos variables: TEST_FW_PATH: "$CI_PROJECT_DIR/tools/tiny-test-fw" TEST_CASE_PATH: "$CI_PROJECT_DIR/examples" @@ -585,7 +595,7 @@ assign_test: when: always paths: - $LOG_PATH - expire_in: 6 mos + expire_in: 1 mos variables: LOCAL_ENV_CONFIG_PATH: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/ESP32_IDF" LOG_PATH: "$CI_PROJECT_DIR/$CI_COMMIT_SHA" @@ -614,7 +624,7 @@ nvs_compatible_test: paths: - $LOG_PATH - nvs_wifi.bin - expire_in: 6 mos + expire_in: 1 mos tags: - ESP32_IDF - NVS_Compatible From 35a3ef68927fa36787338f8fa03be9b1f0c89bbd Mon Sep 17 00:00:00 2001 From: He Yin Ling Date: Mon, 25 Jun 2018 10:33:59 +0800 Subject: [PATCH 04/14] CI: add stage `host_test`: 1. Add `host_test` stage for test jobs running on host. 2. Rename stage `test` to `integration_test`. --- .gitlab-ci.yml | 59 ++++++++++++++++---------------------------------- 1 file changed, 19 insertions(+), 40 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2b5b2b9634..0dadca0f32 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,8 +1,9 @@ stages: - build - assign_test + - host_test - unit_test - - test + - integration_test - deploy variables: @@ -252,22 +253,21 @@ build_docs: - make html - ../check_doc_warnings.sh -test_nvs_on_host: - stage: test +.host_test_template: &host_test_template + stage: host_test image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG tags: - host_test dependencies: [] + +test_nvs_on_host: + <<: *host_test_template script: - cd components/nvs_flash/test_nvs_host - make test test_nvs_coverage: - stage: test - image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG - tags: - - host_test - dependencies: [] + <<: *host_test_template artifacts: paths: - components/nvs_flash/test_nvs_host/coverage_report @@ -281,63 +281,46 @@ test_nvs_coverage: - make coverage_report test_partition_table_on_host: - stage: test - image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG + <<: *host_test_template tags: - build - dependencies: [] script: - cd components/partition_table/test_gen_esp32part_host - ./gen_esp32part_tests.py test_wl_on_host: - stage: test - image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG - tags: - - host_test + <<: *host_test_template artifacts: paths: - components/wear_levelling/test_wl_host/coverage_report.zip - dependencies: [] script: - cd components/wear_levelling/test_wl_host - make test test_fatfs_on_host: - stage: test - image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG + <<: *host_test_template tags: - wl_host_test - dependencies: [] script: - cd components/fatfs/test_fatfs_host/ - make test test_spiffs_on_host: - stage: test - image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG + <<: *host_test_template tags: - wl_host_test - dependencies: [] script: - cd components/spiffs/test_spiffs_host/ - make test test_multi_heap_on_host: - stage: test - image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG - tags: - - host_test + <<: *host_test_template script: - cd components/heap/test_multi_heap_host - ./test_all_configs.sh test_build_system: - stage: test - image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG - tags: - - host_test - dependencies: [] + <<: *host_test_template script: - ${IDF_PATH}/tools/ci/test_configure_ci_environment.sh - rm -rf test_build_system @@ -346,11 +329,7 @@ test_build_system: - ${IDF_PATH}/tools/ci/test_build_system.sh test_esp_err_to_name_on_host: - stage: test - image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG - tags: - - host_test - dependencies: [] + <<: *host_test_template script: - cd tools/ - ./gen_esp_err_to_name.py @@ -387,7 +366,7 @@ push_master_to_github: deploy_docs: - stage: assign_test + stage: host_test image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG tags: - deploy @@ -422,7 +401,7 @@ deploy_docs: - echo "[document preview][zh_CN] $CI_DOCKER_REGISTRY/docs/esp-idf/zh_CN/${GIT_VER}/index.html" check_doc_links: - stage: test + stage: host_test image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG tags: - check_doc_links @@ -524,7 +503,7 @@ assign_test: - python CIAssignTestCases.py -t $IDF_PATH/components/idf_test/integration_test -c $IDF_PATH/.gitlab-ci.yml -b $IDF_PATH/SSC/ssc_bin .example_test_template: &example_test_template - stage: test + stage: integration_test when: on_success only: - master @@ -578,7 +557,7 @@ assign_test: ENV_FILE: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/EnvConfig.yml" .test_template: &test_template - stage: test + stage: integration_test when: on_success only: - master From 831e8d653f2c0ffecb2902d46c8fca80be0dabc9 Mon Sep 17 00:00:00 2001 From: He Yin Ling Date: Fri, 15 Jun 2018 17:33:39 +0800 Subject: [PATCH 05/14] tiny-test-fw: print expect failure when test fails --- tools/tiny-test-fw/DUT.py | 40 ++++++++++++++++++++++++++++-------- tools/tiny-test-fw/Env.py | 5 ++++- tools/tiny-test-fw/TinyFW.py | 6 +++--- 3 files changed, 39 insertions(+), 12 deletions(-) diff --git a/tools/tiny-test-fw/DUT.py b/tools/tiny-test-fw/DUT.py index 882afa1f99..652a6247d2 100644 --- a/tools/tiny-test-fw/DUT.py +++ b/tools/tiny-test-fw/DUT.py @@ -117,7 +117,7 @@ class _DataCache(_queue.Queue): break return ret - def get_data(self, timeout=0): + def get_data(self, timeout=0.0): """ get a copy of data from cache. @@ -214,6 +214,7 @@ class BaseDUT(object): """ DEFAULT_EXPECT_TIMEOUT = 5 + MAX_EXPECT_FAILURES_TO_SAVED = 10 def __init__(self, name, port, log_file, app, **kwargs): @@ -224,12 +225,18 @@ class BaseDUT(object): self.app = app self.data_cache = _DataCache() self.receive_thread = None + self.expect_failures = [] # open and start during init self.open() def __str__(self): return "DUT({}: {})".format(self.name, str(self.port)) + def _save_expect_failure(self, pattern, data, start_time): + self.expect_failures.insert(0, {"pattern": pattern, "data": data, + "start": start_time, "end": time.time()}) + self.expect_failures = self.expect_failures[:self.MAX_EXPECT_FAILURES_TO_SAVED] + # define for methods need to be overwritten by Port @classmethod def list_available_ports(cls): @@ -444,7 +451,9 @@ class BaseDUT(object): data = self.data_cache.get_data(time.time() + timeout - start_time) if ret is None: - raise ExpectTimeout(self.name + ": " + _pattern_to_string(pattern)) + pattern = _pattern_to_string(pattern) + self._save_expect_failure(pattern, data, start_time) + raise ExpectTimeout(self.name + ": " + pattern) return ret def _expect_multi(self, expect_all, expect_item_list, timeout): @@ -508,7 +517,9 @@ class BaseDUT(object): # flush already matched data self.data_cache.flush(slice_index) else: - raise ExpectTimeout(self.name + ": " + str([_pattern_to_string(x) for x in expect_items])) + pattern = str([_pattern_to_string(x["pattern"]) for x in expect_items]) + self._save_expect_failure(pattern, data, start_time) + raise ExpectTimeout(self.name + ": " + pattern) @_expect_lock def expect_any(self, *expect_items, **timeout): @@ -554,6 +565,22 @@ class BaseDUT(object): timeout["timeout"] = self.DEFAULT_EXPECT_TIMEOUT return self._expect_multi(True, expect_items, **timeout) + @staticmethod + def _format_ts(ts): + return "{}:{}".format(time.strftime("%m-%d %H:%M:%S", time.localtime(ts)), str(ts % 1)[2:5]) + + def print_debug_info(self): + """ + Print debug info of current DUT. Currently we will print debug info for expect failures. + """ + Utility.console_log("DUT debug info for DUT: {}:".format(self.name), color="orange") + + for failure in self.expect_failures: + Utility.console_log("\t[pattern]: {}\r\n\t[data]: {}\r\n\t[time]: {} - {}\r\n" + .format(failure["pattern"], failure["data"], + self._format_ts(failure["start"]), self._format_ts(failure["end"])), + color="orange") + class SerialDUT(BaseDUT): """ serial with logging received data feature """ @@ -574,17 +601,14 @@ class SerialDUT(BaseDUT): self.serial_configs.update(kwargs) super(SerialDUT, self).__init__(name, port, log_file, app, **kwargs) - @staticmethod - def _format_data(data): + def _format_data(self, data): """ format data for logging. do decode and add timestamp. :param data: raw data from read :return: formatted data (str) """ - timestamp = time.time() - timestamp = "[{}:{}]".format(time.strftime("%m-%d %H:%M:%S", time.localtime(timestamp)), - str(timestamp % 1)[2:5]) + timestamp = "[{}]".format(self._format_ts(time.time())) formatted_data = timestamp.encode() + b"\r\n" + data + b"\r\n" return formatted_data diff --git a/tools/tiny-test-fw/Env.py b/tools/tiny-test-fw/Env.py index b623847edb..b18df22737 100644 --- a/tools/tiny-test-fw/Env.py +++ b/tools/tiny-test-fw/Env.py @@ -162,14 +162,17 @@ class Env(object): return if_addr[self.PROTO_MAP[proto]][0] @_synced - def close(self): + def close(self, dut_debug=False): """ close() close all DUTs of the Env. + :param dut_debug: if dut_debug is True, then print all dut expect failures before close it :return: None """ for dut_name in self.allocated_duts: dut = self.allocated_duts[dut_name]["dut"] + if dut_debug: + dut.print_debug_info() dut.close() self.allocated_duts = dict() diff --git a/tools/tiny-test-fw/TinyFW.py b/tools/tiny-test-fw/TinyFW.py index 34463b2857..78783c72a6 100644 --- a/tools/tiny-test-fw/TinyFW.py +++ b/tools/tiny-test-fw/TinyFW.py @@ -154,6 +154,7 @@ def test_method(**kwargs): xunit_file = os.path.join(env_inst.app_cls.get_log_folder(env_config["test_suite_name"]), XUNIT_FILE_NAME) XUNIT_RECEIVER.begin_case(test_func.__name__, time.time(), test_func_file_name) + result = False try: Utility.console_log("starting running test: " + test_func.__name__, color="green") # execute test function @@ -163,12 +164,11 @@ def test_method(**kwargs): except Exception as e: # handle all the exceptions here traceback.print_exc() - result = False # log failure XUNIT_RECEIVER.failure(str(e), test_func_file_name) finally: - # do close all DUTs - env_inst.close() + # do close all DUTs, if result is False then print DUT debug info + env_inst.close(dut_debug=(not result)) # end case and output result XUNIT_RECEIVER.end_case(test_func.__name__, time.time()) with open(xunit_file, "ab+") as f: From 3956fe30c04a231b8e53f9f5d94437b04fe0cc94 Mon Sep 17 00:00:00 2001 From: He Yin Ling Date: Fri, 15 Jun 2018 17:45:24 +0800 Subject: [PATCH 06/14] tiny-test-fw: filter examples with test level --- tools/tiny-test-fw/IDF/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tools/tiny-test-fw/IDF/__init__.py b/tools/tiny-test-fw/IDF/__init__.py index 5e1a4d6fc9..fad52f41be 100644 --- a/tools/tiny-test-fw/IDF/__init__.py +++ b/tools/tiny-test-fw/IDF/__init__.py @@ -21,7 +21,7 @@ from IDF.IDFDUT import IDFDUT def idf_example_test(app=Example, dut=IDFDUT, chip="ESP32", - module="examples", execution_time=1, + module="examples", execution_time=1, level="example", **kwargs): """ decorator for testing idf examples (with default values for some keyword args). @@ -31,12 +31,13 @@ def idf_example_test(app=Example, dut=IDFDUT, chip="ESP32", :param chip: chip supported, string or tuple :param module: module, string :param execution_time: execution time in minutes, int + :param level: test level, could be used to filter test cases, string :param kwargs: other keyword args :return: test method """ # not use partial function as define as function support auto generating document return TinyFW.test_method(app=app, dut=dut, chip=chip, module=module, - execution_time=execution_time, **kwargs) + execution_time=execution_time, level=level, **kwargs) def log_performance(item, value): From 609e39a620baa22b9fd3ea39d60c0da7de2ce04b Mon Sep 17 00:00:00 2001 From: He Yin Ling Date: Thu, 21 Jun 2018 16:38:46 +0800 Subject: [PATCH 07/14] tiny-test-fw: fix incorrect wait timeout in expect --- tools/tiny-test-fw/DUT.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tools/tiny-test-fw/DUT.py b/tools/tiny-test-fw/DUT.py index 652a6247d2..32210f4090 100644 --- a/tools/tiny-test-fw/DUT.py +++ b/tools/tiny-test-fw/DUT.py @@ -444,11 +444,14 @@ class BaseDUT(object): start_time = time.time() while True: ret, index = method(data, pattern) - if ret is not None or time.time() - start_time > timeout: + if ret is not None: self.data_cache.flush(index) break + time_remaining = start_time + timeout - time.time() + if time_remaining < 0: + break # wait for new data from cache - data = self.data_cache.get_data(time.time() + timeout - start_time) + data = self.data_cache.get_data(time_remaining) if ret is None: pattern = _pattern_to_string(pattern) @@ -501,10 +504,11 @@ class BaseDUT(object): else: match_succeed = True if matched_expect_items else False - if time.time() - start_time > timeout or match_succeed: + time_remaining = start_time + timeout - time.time() + if time_remaining < 0 or match_succeed: break else: - data = self.data_cache.get_data(time.time() + timeout - start_time) + data = self.data_cache.get_data(time_remaining) if match_succeed: # do callback and flush matched data cache From fa9c836f02d1721858b72a0128b56e2fdf8c3eba Mon Sep 17 00:00:00 2001 From: He Yin Ling Date: Thu, 21 Jun 2018 17:11:32 +0800 Subject: [PATCH 08/14] tiny-test-fw: save DUT log in different thread: We found some SD card on Raspberry Pi could have very bad performance. It could take seconds to save small amount of data. If the DUT receives data and save it as log, then it stops receiving data until log is saved. This could lead to expect timeout. As an workaround to this issue, ``BaseDUT`` class will create a thread to save logs. Then data will be passed to ``expect`` as soon as received. --- tools/tiny-test-fw/DUT.py | 68 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 66 insertions(+), 2 deletions(-) diff --git a/tools/tiny-test-fw/DUT.py b/tools/tiny-test-fw/DUT.py index 32210f4090..6380c43d61 100644 --- a/tools/tiny-test-fw/DUT.py +++ b/tools/tiny-test-fw/DUT.py @@ -154,6 +154,52 @@ class _DataCache(_queue.Queue): self.data_cache = self.data_cache[index:] +class _LogThread(threading.Thread, _queue.Queue): + """ + We found some SD card on Raspberry Pi could have very bad performance. + It could take seconds to save small amount of data. + If the DUT receives data and save it as log, then it stops receiving data until log is saved. + This could lead to expect timeout. + As an workaround to this issue, ``BaseDUT`` class will create a thread to save logs. + Then data will be passed to ``expect`` as soon as received. + """ + def __init__(self): + threading.Thread.__init__(self, name="LogThread") + _queue.Queue.__init__(self, maxsize=0) + self.setDaemon(True) + self.flush_lock = threading.Lock() + + def save_log(self, filename, data): + """ + :param filename: log file name + :param data: log data. Must be ``bytes``. + """ + self.put({"filename": filename, "data": data}) + + def flush_data(self): + with self.flush_lock: + data_cache = dict() + while True: + # move all data from queue to data cache + try: + log = self.get_nowait() + try: + data_cache[log["filename"]] += log["data"] + except KeyError: + data_cache[log["filename"]] = log["data"] + except _queue.Empty: + break + # flush data + for filename in data_cache: + with open(filename, "ab+") as f: + f.write(data_cache[filename]) + + def run(self): + while True: + time.sleep(1) + self.flush_data() + + class _RecvThread(threading.Thread): PERFORMANCE_PATTERN = re.compile(r"\[Performance]\[(\w+)]: ([^\r\n]+)\r?\n") @@ -216,6 +262,9 @@ class BaseDUT(object): DEFAULT_EXPECT_TIMEOUT = 5 MAX_EXPECT_FAILURES_TO_SAVED = 10 + LOG_THREAD = _LogThread() + LOG_THREAD.start() + def __init__(self, name, port, log_file, app, **kwargs): self.expect_lock = threading.Lock() @@ -233,10 +282,25 @@ class BaseDUT(object): return "DUT({}: {})".format(self.name, str(self.port)) def _save_expect_failure(self, pattern, data, start_time): + """ + Save expect failure. If the test fails, then it will print the expect failures. + In some cases, user will handle expect exceptions. + The expect failures could be false alarm, and test case might generate a lot of such failures. + Therefore, we don't print the failure immediately and limit the max size of failure list. + """ self.expect_failures.insert(0, {"pattern": pattern, "data": data, "start": start_time, "end": time.time()}) self.expect_failures = self.expect_failures[:self.MAX_EXPECT_FAILURES_TO_SAVED] + def _save_dut_log(self, data): + """ + Save DUT log into file using another thread. + This is a workaround for some devices takes long time for file system operations. + + See descriptions in ``_LogThread`` for details. + """ + self.LOG_THREAD.save_log(self.log_file, data) + # define for methods need to be overwritten by Port @classmethod def list_available_ports(cls): @@ -336,6 +400,7 @@ class BaseDUT(object): if self.receive_thread: self.receive_thread.exit() self._port_close() + self.LOG_THREAD.flush_data() def write(self, data, eol="\r\n", flush=True): """ @@ -625,8 +690,7 @@ class SerialDUT(BaseDUT): def _port_read(self, size=1): data = self.port_inst.read(size) if data: - with open(self.log_file, "ab+") as _log_file: - _log_file.write(self._format_data(data)) + self._save_dut_log(self._format_data(data)) return data def _port_write(self, data): From 146a9597067d426f3b0ff52a21158976c3e3c769 Mon Sep 17 00:00:00 2001 From: He Yin Ling Date: Thu, 21 Jun 2018 19:58:24 +0800 Subject: [PATCH 09/14] esp32/test: set 60s timeout for one deep sleep case --- components/esp32/test/test_sleep.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/esp32/test/test_sleep.c b/components/esp32/test/test_sleep.c index fafff84718..70fe47f3ab 100644 --- a/components/esp32/test/test_sleep.c +++ b/components/esp32/test/test_sleep.c @@ -371,4 +371,4 @@ static void check_time_deepsleep(void) TEST_ASSERT_MESSAGE(dt_ms > 0, "Time in deep sleep is negative"); } -TEST_CASE_MULTIPLE_STAGES("check a time after wakeup from deep sleep", "[deepsleep][reset=DEEPSLEEP_RESET]", trigger_deepsleep, check_time_deepsleep); +TEST_CASE_MULTIPLE_STAGES("check a time after wakeup from deep sleep", "[deepsleep][reset=DEEPSLEEP_RESET][timeout=60]", trigger_deepsleep, check_time_deepsleep); From bf72ed92a998a1f5e37032f740211c00ed98dc9c Mon Sep 17 00:00:00 2001 From: He Yin Ling Date: Thu, 26 Jul 2018 15:07:32 +0800 Subject: [PATCH 10/14] CI: erase nvs partition before test: Latest NVS partition bin can't be parsed by old IDF revision. Need to erase before test. --- tools/tiny-test-fw/IDF/IDFDUT.py | 54 ++++++++++++++++++++++++------ tools/tiny-test-fw/IDF/__init__.py | 38 ++++++++++++++++++--- tools/unit-test-app/unit_test.py | 9 ++--- 3 files changed, 80 insertions(+), 21 deletions(-) diff --git a/tools/tiny-test-fw/IDF/IDFDUT.py b/tools/tiny-test-fw/IDF/IDFDUT.py index 4c722e06ea..984eca5446 100644 --- a/tools/tiny-test-fw/IDF/IDFDUT.py +++ b/tools/tiny-test-fw/IDF/IDFDUT.py @@ -17,6 +17,10 @@ import os import re import subprocess import functools +import random +import tempfile + +from serial.tools import list_ports import DUT @@ -40,6 +44,8 @@ class IDFDUT(DUT.SerialDUT): """ IDF DUT, extends serial with ESPTool methods """ CHIP_TYPE_PATTERN = re.compile(r"Detecting chip type[.:\s]+(.+)") + # if need to erase NVS partition in start app + ERASE_NVS = True def __init__(self, name, port, log_file, app, **kwargs): self.download_config, self.partition_table = app.process_app_info() @@ -68,24 +74,39 @@ class IDFDUT(DUT.SerialDUT): return cls.get_chip(app, port) is not None @_tool_method - def start_app(self): + def start_app(self, erase_nvs=ERASE_NVS): """ download and start app. + :param: erase_nvs: whether erase NVS partition during flash :return: None """ + if erase_nvs: + address = self.partition_table["nvs"]["offset"] + size = self.partition_table["nvs"]["size"] + nvs_file = tempfile.NamedTemporaryFile() + nvs_file.write(chr(0xFF) * size) + nvs_file.flush() + download_config = self.download_config + [address, nvs_file.name] + else: + download_config = self.download_config + retry_baud_rates = ["921600", "115200"] error = IDFToolError() - for baud_rate in retry_baud_rates: - try: - subprocess.check_output(["python", self.app.esptool, - "--port", self.port, "--baud", baud_rate] - + self.download_config) - break - except subprocess.CalledProcessError as error: - continue - else: - raise error + try: + for baud_rate in retry_baud_rates: + try: + subprocess.check_output(["python", self.app.esptool, + "--port", self.port, "--baud", baud_rate] + + download_config) + break + except subprocess.CalledProcessError as error: + continue + else: + raise error + finally: + if erase_nvs: + nvs_file.close() @_tool_method def reset(self): @@ -96,6 +117,17 @@ class IDFDUT(DUT.SerialDUT): """ subprocess.check_output(["python", self.app.esptool, "--port", self.port, "run"]) + @_tool_method + def erase_partition(self, partition): + """ + :param partition: partition name to erase + :return: None + """ + address = self.partition_table[partition]["offset"] + size = self.partition_table[partition]["size"] + with open(".erase_partition.tmp", "wb") as f: + f.write(chr(0xFF) * size) + @_tool_method def dump_flush(self, output_file, **kwargs): """ diff --git a/tools/tiny-test-fw/IDF/__init__.py b/tools/tiny-test-fw/IDF/__init__.py index fad52f41be..c7480c43f6 100644 --- a/tools/tiny-test-fw/IDF/__init__.py +++ b/tools/tiny-test-fw/IDF/__init__.py @@ -20,9 +20,8 @@ from IDF.IDFApp import IDFApp, Example, UT from IDF.IDFDUT import IDFDUT -def idf_example_test(app=Example, dut=IDFDUT, chip="ESP32", - module="examples", execution_time=1, level="example", - **kwargs): +def idf_example_test(app=Example, dut=IDFDUT, chip="ESP32", module="examples", execution_time=1, + level="example", erase_nvs=True, **kwargs): """ decorator for testing idf examples (with default values for some keyword args). @@ -32,10 +31,41 @@ def idf_example_test(app=Example, dut=IDFDUT, chip="ESP32", :param module: module, string :param execution_time: execution time in minutes, int :param level: test level, could be used to filter test cases, string + :param erase_nvs: if need to erase_nvs in DUT.start_app() :param kwargs: other keyword args :return: test method """ - # not use partial function as define as function support auto generating document + try: + # try to config the default behavior of erase nvs + dut.ERASE_NVS = erase_nvs + except AttributeError: + pass + + return TinyFW.test_method(app=app, dut=dut, chip=chip, module=module, + execution_time=execution_time, level=level, **kwargs) + + +def idf_unit_test(app=UT, dut=IDFDUT, chip="ESP32", module="unit-test", execution_time=1, + level="unit", erase_nvs=True, **kwargs): + """ + decorator for testing idf unit tests (with default values for some keyword args). + + :param app: test application class + :param dut: dut class + :param chip: chip supported, string or tuple + :param module: module, string + :param execution_time: execution time in minutes, int + :param level: test level, could be used to filter test cases, string + :param erase_nvs: if need to erase_nvs in DUT.start_app() + :param kwargs: other keyword args + :return: test method + """ + try: + # try to config the default behavior of erase nvs + dut.ERASE_NVS = erase_nvs + except AttributeError: + pass + return TinyFW.test_method(app=app, dut=dut, chip=chip, module=module, execution_time=execution_time, level=level, **kwargs) diff --git a/tools/unit-test-app/unit_test.py b/tools/unit-test-app/unit_test.py index b7453722e5..bf913c2784 100644 --- a/tools/unit-test-app/unit_test.py +++ b/tools/unit-test-app/unit_test.py @@ -105,8 +105,7 @@ def format_test_case_config(test_case_data): return case_config -@TinyFW.test_method(app=UT, dut=IDF.IDFDUT, chip="ESP32", module="unit_test", - execution_time=1, env_tag="UT_T1_1") +@IDF.idf_unit_test(env_tag="UT_T1_1") def run_unit_test_cases(env, extra_data): """ extra_data can be three types of value @@ -339,8 +338,7 @@ def case_run(duts, ut_config, env, one_case, failed_cases): Utility.console_log("Failed: " + one_case["name"], color="red") -@TinyFW.test_method(app=UT, dut=IDF.IDFDUT, chip="ESP32", module="master_slave_test_case", execution_time=1, - env_tag="UT_T2_1") +@IDF.idf_unit_test(env_tag="UT_T2_1") def run_multiple_devices_cases(env, extra_data): """ extra_data can be two types of value @@ -377,8 +375,7 @@ def run_multiple_devices_cases(env, extra_data): raise AssertionError("Unit Test Failed") -@TinyFW.test_method(app=UT, dut=IDF.IDFDUT, chip="ESP32", module="unit_test", - execution_time=1, env_tag="UT_T1_1") +@IDF.idf_unit_test(env_tag="UT_T1_1") def run_multiple_stage_cases(env, extra_data): """ extra_data can be 2 types of value From 715dfac07e8bc60e3c3322db383e26b85e736734 Mon Sep 17 00:00:00 2001 From: He Yin Ling Date: Thu, 26 Jul 2018 21:17:06 +0800 Subject: [PATCH 11/14] tiny-test-fw: fix exception during print debug info: DUT log is unicode, might not be able to encode --- tools/tiny-test-fw/DUT.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/tiny-test-fw/DUT.py b/tools/tiny-test-fw/DUT.py index 6380c43d61..0193cb6c70 100644 --- a/tools/tiny-test-fw/DUT.py +++ b/tools/tiny-test-fw/DUT.py @@ -645,7 +645,7 @@ class BaseDUT(object): Utility.console_log("DUT debug info for DUT: {}:".format(self.name), color="orange") for failure in self.expect_failures: - Utility.console_log("\t[pattern]: {}\r\n\t[data]: {}\r\n\t[time]: {} - {}\r\n" + Utility.console_log(u"\t[pattern]: {}\r\n\t[data]: {}\r\n\t[time]: {} - {}\r\n" .format(failure["pattern"], failure["data"], self._format_ts(failure["start"]), self._format_ts(failure["end"])), color="orange") From 2d31597fa939020efcc0a934655b578e898d4aa9 Mon Sep 17 00:00:00 2001 From: He Yin Ling Date: Thu, 26 Jul 2018 15:18:52 +0800 Subject: [PATCH 12/14] tiny-test-fw: support ignore test cases in CI assign test stage --- tools/tiny-test-fw/TinyFW.py | 1 + tools/tiny-test-fw/Utility/CIAssignTest.py | 1 + 2 files changed, 2 insertions(+) diff --git a/tools/tiny-test-fw/TinyFW.py b/tools/tiny-test-fw/TinyFW.py index 78783c72a6..b05150b730 100644 --- a/tools/tiny-test-fw/TinyFW.py +++ b/tools/tiny-test-fw/TinyFW.py @@ -107,6 +107,7 @@ MANDATORY_INFO = { "execution_time": 1, "env_tag": "default", "category": "function", + "ignore": False, } diff --git a/tools/tiny-test-fw/Utility/CIAssignTest.py b/tools/tiny-test-fw/Utility/CIAssignTest.py index 8d29dc4ec5..3dbea9eff1 100644 --- a/tools/tiny-test-fw/Utility/CIAssignTest.py +++ b/tools/tiny-test-fw/Utility/CIAssignTest.py @@ -125,6 +125,7 @@ class AssignTest(object): # by default we only run function in CI, as other tests could take long time DEFAULT_FILTER = { "category": "function", + "ignore": False, } def __init__(self, test_case_path, ci_config_file, case_group=Group): From b86c8b32605ba8c0c26d81825a4ae2a4c86f3ff6 Mon Sep 17 00:00:00 2001 From: He Yin Ling Date: Fri, 27 Jul 2018 19:45:12 +0800 Subject: [PATCH 13/14] ci: temp disable http client tests: connection is not stable. need to replace with local test servers. --- examples/protocols/esp_http_client/esp_http_client_test.py | 2 +- examples/protocols/https_request/example_test.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/protocols/esp_http_client/esp_http_client_test.py b/examples/protocols/esp_http_client/esp_http_client_test.py index 49194dd8c9..9ca1d6f091 100644 --- a/examples/protocols/esp_http_client/esp_http_client_test.py +++ b/examples/protocols/esp_http_client/esp_http_client_test.py @@ -14,7 +14,7 @@ import TinyFW import IDF -@IDF.idf_example_test(env_tag="Example_WIFI") +@IDF.idf_example_test(env_tag="Example_WIFI", ignore=True) def test_examples_protocol_esp_http_client(env, extra_data): """ steps: | diff --git a/examples/protocols/https_request/example_test.py b/examples/protocols/https_request/example_test.py index 169ae63302..e362a25121 100644 --- a/examples/protocols/https_request/example_test.py +++ b/examples/protocols/https_request/example_test.py @@ -14,7 +14,7 @@ import TinyFW import IDF -@IDF.idf_example_test(env_tag="Example_WIFI") +@IDF.idf_example_test(env_tag="Example_WIFI", ignore=True) def test_examples_protocol_https_request(env, extra_data): """ steps: | From 1884151ca8d08eacdc7149e14e2530634ad5a5ff Mon Sep 17 00:00:00 2001 From: He Yin Ling Date: Tue, 3 Jul 2018 22:00:09 +0800 Subject: [PATCH 14/14] CI: support test one case multiple times by @bot --- tools/tiny-test-fw/CIAssignUnitTest.py | 12 +++++++++--- tools/tiny-test-fw/TinyFW.py | 2 +- tools/tiny-test-fw/Utility/CIAssignTest.py | 11 +++++++++++ tools/tiny-test-fw/Utility/CaseConfig.py | 8 ++++++-- tools/unit-test-app/tools/UnitTestParser.py | 19 +------------------ 5 files changed, 28 insertions(+), 24 deletions(-) diff --git a/tools/tiny-test-fw/CIAssignUnitTest.py b/tools/tiny-test-fw/CIAssignUnitTest.py index 9885d4fd7d..a87df781f1 100644 --- a/tools/tiny-test-fw/CIAssignUnitTest.py +++ b/tools/tiny-test-fw/CIAssignUnitTest.py @@ -109,16 +109,22 @@ class UnitTestAssignTest(CIAssignTest.AssignTest): with open(test_case_path, "r") as f: raw_data = yaml.load(f) test_cases = raw_data["test cases"] + # filter keys are lower case. Do map lower case keys with original keys. + try: + key_mapping = {x.lower(): x for x in test_cases[0].keys()} + except IndexError: + key_mapping = dict() if case_filter: for key in case_filter: filtered_cases = [] for case in test_cases: try: + mapped_key = key_mapping[key] # bot converts string to lower case - if isinstance(case[key], str): - _value = case[key].lower() + if isinstance(case[mapped_key], str): + _value = case[mapped_key].lower() else: - _value = case[key] + _value = case[mapped_key] if _value in case_filter[key]: filtered_cases.append(case) except KeyError: diff --git a/tools/tiny-test-fw/TinyFW.py b/tools/tiny-test-fw/TinyFW.py index b05150b730..c475f3824d 100644 --- a/tools/tiny-test-fw/TinyFW.py +++ b/tools/tiny-test-fw/TinyFW.py @@ -131,7 +131,7 @@ def test_method(**kwargs): test_func_file_name = frame[1][1] case_info = MANDATORY_INFO.copy() - case_info["name"] = test_func.__name__ + case_info["name"] = case_info["ID"] = test_func.__name__ case_info.update(kwargs) @functools.wraps(test_func) diff --git a/tools/tiny-test-fw/Utility/CIAssignTest.py b/tools/tiny-test-fw/Utility/CIAssignTest.py index 3dbea9eff1..2df66fe81d 100644 --- a/tools/tiny-test-fw/Utility/CIAssignTest.py +++ b/tools/tiny-test-fw/Utility/CIAssignTest.py @@ -189,6 +189,16 @@ class AssignTest(object): bot_filter = dict() return bot_filter + def _apply_bot_test_count(self): + """ + Bot could also pass test count. + If filtered cases need to be tested for several times, then we do duplicate them here. + """ + test_count = os.getenv("BOT_TEST_COUNT") + if test_count: + test_count = int(test_count) + self.test_cases *= test_count + def assign_cases(self): """ separate test cases to groups and assign test cases to CI jobs. @@ -199,6 +209,7 @@ class AssignTest(object): failed_to_assign = [] case_filter = self._apply_bot_filter() self.test_cases = self._search_cases(self.test_case_path, case_filter) + self._apply_bot_test_count() test_groups = self._group_cases() for group in test_groups: for job in self.jobs: diff --git a/tools/tiny-test-fw/Utility/CaseConfig.py b/tools/tiny-test-fw/Utility/CaseConfig.py index af013ec282..3260c9b6ee 100644 --- a/tools/tiny-test-fw/Utility/CaseConfig.py +++ b/tools/tiny-test-fw/Utility/CaseConfig.py @@ -68,11 +68,15 @@ def _convert_to_lower_case(item): def _filter_one_case(test_method, case_filter): """ Apply filter for one case (the filter logic is the same as described in ``filter_test_cases``) """ filter_result = True - for key in case_filter: + # filter keys are lower case. Do map lower case keys with original keys. + key_mapping = {x.lower(): x for x in test_method.case_info.keys()} + + for orig_key in case_filter: + key = key_mapping[orig_key] if key in test_method.case_info: # the filter key is both in case and filter # we need to check if they match - filter_item = _convert_to_lower_case(case_filter[key]) + filter_item = _convert_to_lower_case(case_filter[orig_key]) accepted_item = _convert_to_lower_case(test_method.case_info[key]) if isinstance(filter_item, (tuple, list)) \ diff --git a/tools/unit-test-app/tools/UnitTestParser.py b/tools/unit-test-app/tools/UnitTestParser.py index 1898bf6598..aa488bc550 100644 --- a/tools/unit-test-app/tools/UnitTestParser.py +++ b/tools/unit-test-app/tools/UnitTestParser.py @@ -179,28 +179,11 @@ class Parser(object): """ prop = self.parse_case_properities(description) - idf_path = os.getenv("IDF_PATH") - - # use relative file path to IDF_PATH, to make sure file path is consist - relative_file_path = os.path.relpath(file_name, idf_path) - - file_name_hash = int(hashlib.sha256(relative_file_path).hexdigest(), base=16) % 1000 - - if file_name_hash in self.file_name_cache: - self.file_name_cache[file_name_hash] += 1 - else: - self.file_name_cache[file_name_hash] = 1 - - tc_id = "UT_%s_%s_%03d%02d" % (self.module_map[prop["module"]]['module abbr'], - self.module_map[prop["module"]]['sub module abbr'], - file_name_hash, - self.file_name_cache[file_name_hash]) - test_case = deepcopy(TEST_CASE_PATTERN) test_case.update({"config": config_name, "module": self.module_map[prop["module"]]['module'], "CI ready": "No" if prop["ignore"] == "Yes" else "Yes", - "ID": tc_id, + "ID": name, "test point 2": prop["module"], "steps": name, "test environment": prop["test_env"],