From c47bc43afdea856c130a99a8ae77ba0ff99f739f Mon Sep 17 00:00:00 2001 From: Yinling Date: Mon, 2 Jan 2017 13:12:30 +0800 Subject: [PATCH 1/4] CI: fix bug that restore initial condition fail: in STA+AP mode, when STA is reconnecting to external AP, its AP interface is not connectable. do disconnect on STA interface first --- .../integration_test/InitialConditionAll.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/components/idf_test/integration_test/InitialConditionAll.yml b/components/idf_test/integration_test/InitialConditionAll.yml index 3821894552..5d4ce7ab3f 100644 --- a/components/idf_test/integration_test/InitialConditionAll.yml +++ b/components/idf_test/integration_test/InitialConditionAll.yml @@ -260,6 +260,8 @@ initial condition: - ['R SSC1 C +MAC:AP,OK'] - - SSC SSC1 dhcp -S -o 2 - [R SSC1 C +DHCP] + - - SSC SSC1 sta -D + - ['R SSC1 C +QAP'] - - SSC SSC1 ap -S -s -p -t - ['R SSC1 C +SAP:OK'] initial condition detail: testing ap on sta + ap mode (autogen by APM1) @@ -271,6 +273,8 @@ initial condition: - ['R SSC1 C +MAC:AP,OK'] - - SSC SSC1 dhcp -S -o 2 - [R SSC1 C +DHCP] + - - SSC SSC1 sta -D + - ['R SSC1 C +QAP'] - - SSC SSC1 ap -S -s -p -t - ['R SSC1 C +SAP:OK'] restore post cmd set: @@ -305,6 +309,8 @@ initial condition: - ['R SSC1 C +MAC:AP,OK'] - - SSC SSC1 dhcp -S -o 2 - [R SSC1 C +DHCP] + - - SSC SSC1 sta -D + - ['R SSC1 C +QAP'] - - SSC SSC1 ap -S -s -p -t - ['R SSC1 C +SAP:OK'] - - WIFI CONN @@ -319,6 +325,8 @@ initial condition: - ['R SSC1 C +MAC:AP,OK'] - - SSC SSC1 dhcp -S -o 2 - [R SSC1 C +DHCP] + - - SSC SSC1 sta -D + - ['R SSC1 C +QAP'] - - SSC SSC1 ap -S -s -p -t - ['R SSC1 C +SAP:OK'] - - WIFI CONN @@ -2854,6 +2862,8 @@ initial condition: - ['R SSC1 C +MODE:OK'] - - SSC SSC2 op -S -o 3 - ['R SSC2 C +MODE:OK'] + - - SSC SSC1 sta -D + - ['R SSC1 C +QAP'] - - SSC SSC2 sta -D - ['R SSC2 C +QAP:'] - - SSC SSC2 soc -T @@ -2873,6 +2883,8 @@ initial condition: - ['R SSC1 C +MODE:OK'] - - SSC SSC2 op -S -o 3 - ['R SSC2 C +MODE:OK'] + - - SSC SSC1 sta -D + - ['R SSC1 C +QAP'] - - SSC SSC2 sta -D - ['R SSC2 C +QAP:'] - - SSC SSC2 soc -T From b9a6d276a2cda8a7aa0ec32fdff2ca71fd8b64fc Mon Sep 17 00:00:00 2001 From: Yinling Date: Mon, 2 Jan 2017 13:15:10 +0800 Subject: [PATCH 2/4] CI: update known issues --- .../idf_test/integration_test/KnownIssues | 109 +++++++++++++----- 1 file changed, 81 insertions(+), 28 deletions(-) diff --git a/components/idf_test/integration_test/KnownIssues b/components/idf_test/integration_test/KnownIssues index e0991f39b0..c6cb7b77b9 100644 --- a/components/idf_test/integration_test/KnownIssues +++ b/components/idf_test/integration_test/KnownIssues @@ -5,24 +5,6 @@ TCPIP_ICMP_0101 ^TCPIP_ICMP_0101 -# IGMP cases are not supported for now -TCPIP_IGMP_0101 -TCPIP_IGMP_0102 -TCPIP_IGMP_0103 -TCPIP_IGMP_0104 -TCPIP_IGMP_0201 -TCPIP_IGMP_0202 -TCPIP_IGMP_0203 -TCPIP_IGMP_0204 -^TCPIP_IGMP_0101 -^TCPIP_IGMP_0102 -^TCPIP_IGMP_0103 -^TCPIP_IGMP_0104 -^TCPIP_IGMP_0201 -^TCPIP_IGMP_0202 -^TCPIP_IGMP_0203 -^TCPIP_IGMP_0204 - # don't support PHY mode command WIFI_SCAN_0201 WIFI_SCAN_0302 @@ -48,29 +30,62 @@ WIFI_CONN_0801 ^WIFI_CONN_0801 # disconnect reason -WIFI_CONN_0904 -^WIFI_CONN_0904 WIFI_CONN_0901 ^WIFI_CONN_0901 +WIFI_CONN_0904 +^WIFI_CONN_0904 +^WIFI_CONN_0902 +WIFI_CONN_0902 # Wifi connect issue -WIFI_CONN_0104 -^WIFI_CONN_0104 +^WIFI_CONN_0101 +WIFI_CONN_0101 +WIFI_CONN_0103 +^WIFI_CONN_0401 +^WIFI_CONN_0103 +WIFI_CONN_0102 +WIFI_CONN_0601 ^WIFI_CONN_0601 +WIFI_ADDR_0102 +^WIFI_ADDR_0102 +WIFI_CONN_0502 +WIFI_CONN_0501 # Wifi scan issue -WIFI_SCAN_0303 +^WIFI_SCAN_0101 +^WIFI_SCAN_0102 +WIFI_SCAN_0103 ^WIFI_SCAN_0103 +WIFI_SCAN_0104 +^WIFI_SCAN_0104 ^WIFI_SCAN_0105 +WIFI_SCAN_0303 +WIFI_SCAN_0304 +^WIFI_SCAN_0104 -# set mac address may lead to exception -WIFI_ADDR_0101 -^WIFI_ADDR_0101 +# IGMP cases are supported but as UDP is not stable, exclude them first +TCPIP_IGMP_0101 +TCPIP_IGMP_0102 +TCPIP_IGMP_0103 +TCPIP_IGMP_0104 +TCPIP_IGMP_0201 +TCPIP_IGMP_0202 +TCPIP_IGMP_0203 +TCPIP_IGMP_0204 +^TCPIP_IGMP_0101 +^TCPIP_IGMP_0102 +^TCPIP_IGMP_0103 +^TCPIP_IGMP_0104 +^TCPIP_IGMP_0201 +^TCPIP_IGMP_0202 +^TCPIP_IGMP_0203 +^TCPIP_IGMP_0204 # DHCP issues ^TCPIP_DHCP_0301 TCPIP_DHCP_0301 TCPIP_DHCP_0101 +^TCPIP_DHCP_0101 TCPIP_DHCP_0207 ^TCPIP_DHCP_0207 TCPIP_DHCP_0208 @@ -79,6 +94,14 @@ TCPIP_DHCP_0205 ^TCPIP_DHCP_0205 TCPIP_DHCP_0209 ^TCPIP_DHCP_0209 +^TCPIP_DHCP_0204 +TCPIP_DHCP_0204 +^TCPIP_DHCP_0210 +TCPIP_DHCP_0210 +^TCPIP_DHCP_0211 +TCPIP_DHCP_0211 +^TCPIP_DHCP_0206 +^TCPIP_DHCP_0302 # TCP issue TCPIP_TCP_0402 @@ -90,17 +113,47 @@ TCPIP_TCP_0103 ^TCPIP_TCP_0103 TCPIP_TCP_0112 ^TCPIP_TCP_0112 - +TCPIP_TCP_0106 +TCPIP_TCP_0107 +TCPIP_TCP_0105 +^TCPIP_TCP_0407 +^TCPIP_TCP_0404 +^TCPIP_TCP_0102 +^TCPIP_TCP_0105 +^TCPIP_TCP_0107 +^TCPIP_TCP_0206 +TCPIP_TCP_0102 +^TCPIP_TCP_0403 +^TCPIP_TCP_0402 +^TCPIP_TCP_0101 +^TCPIP_TCP_0203 +^TCPIP_TCP_0106 +^TCPIP_TCP_0201 +^TCPIP_TCP_0412 +TCPIP_TCP_0101 # UDP issue TCPIP_UDP_0103 ^TCPIP_UDP_0103 +^TCPIP_UDP_0108 TCPIP_UDP_0110 ^TCPIP_UDP_0110 +^TCPIP_UDP_0112 TCPIP_UDP_0305 ^TCPIP_UDP_0305 ^TCPIP_UDP_0304 TCPIP_UDP_0104 +^TCPIP_UDP_0104 +^TCPIP_UDP_0301 +TCPIP_UDP_0301 +TCPIP_UDP_0302 +TCPIP_UDP_0303 +^TCPIP_UDP_0303 +^TCPIP_UDP_0307 +^TCPIP_UDP_0302 +^TCPIP_UDP_0306 - +#DNS +^TCPIP_DNS_0103 +^TCPIP_DNS_0102 From 16de41941e7fde28a7b83a2b71893879645c10dc Mon Sep 17 00:00:00 2001 From: Yinling Date: Mon, 2 Jan 2017 16:25:26 +0800 Subject: [PATCH 3/4] CI: fix bug that test report job failed --- .gitlab-ci.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 75a1fb59ff..8cb5c8bcac 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -180,7 +180,8 @@ test_report: - git clone $GITLAB_SSH_SERVER/yinling/auto_test_script.git - cd auto_test_script # generate report - - python CITestReport.py -l $LOG_PATH -t $TEST_CASE_FILE_PATH -p $REPORT_PATH -r $RESULT_PATH -a $ARTIFACTS_PATH || FAIL=True + - TEST_RESULT=Pass + - python CITestReport.py -l $LOG_PATH -t $TEST_CASE_FILE_PATH -p $REPORT_PATH -r $RESULT_PATH -a $ARTIFACTS_PATH || TEST_RESULT=Fail # commit to CI-test-result project - git clone $GITLAB_SSH_SERVER/qa/CI-test-result.git - rm -rf CI-test-result/RawData/$RESULT_PATH @@ -193,7 +194,7 @@ test_report: - git add . - git commit . -m "update test result for $CI_PROJECT_NAME/$CI_BUILD_REF_NAME/$CI_BUILD_REF, pipeline ID $CI_PIPELINE_ID" || exit 0 - git push origin master - - test "${FAIL}" = "True" && exit 1 + - test "${TEST_RESULT}" = "Pass" || exit 1 push_master_to_github: before_script: From 6c41080fd91fb8d20dd63aabe1afeb3268fab027 Mon Sep 17 00:00:00 2001 From: Yinling Date: Mon, 2 Jan 2017 19:28:01 +0800 Subject: [PATCH 4/4] CI: update known issues --- .../idf_test/integration_test/KnownIssues | 139 ++++++++++++------ 1 file changed, 95 insertions(+), 44 deletions(-) diff --git a/components/idf_test/integration_test/KnownIssues b/components/idf_test/integration_test/KnownIssues index c6cb7b77b9..884bd2f590 100644 --- a/components/idf_test/integration_test/KnownIssues +++ b/components/idf_test/integration_test/KnownIssues @@ -38,30 +38,45 @@ WIFI_CONN_0904 WIFI_CONN_0902 # Wifi connect issue -^WIFI_CONN_0101 WIFI_CONN_0101 -WIFI_CONN_0103 -^WIFI_CONN_0401 -^WIFI_CONN_0103 +^WIFI_CONN_0101 WIFI_CONN_0102 +^WIFI_CONN_0102 +WIFI_CONN_0103 +^WIFI_CONN_0103 +WIFI_CONN_0104 +^WIFI_CONN_0104 +WIFI_CONN_0201 +^WIFI_CONN_0201 +WIFI_CONN_0401 +^WIFI_CONN_0401 WIFI_CONN_0601 ^WIFI_CONN_0601 WIFI_ADDR_0102 ^WIFI_ADDR_0102 WIFI_CONN_0502 +^WIFI_CONN_0502 WIFI_CONN_0501 +^WIFI_CONN_0501 # Wifi scan issue +WIFI_SCAN_0101 ^WIFI_SCAN_0101 +WIFI_SCAN_0102 ^WIFI_SCAN_0102 WIFI_SCAN_0103 ^WIFI_SCAN_0103 WIFI_SCAN_0104 ^WIFI_SCAN_0104 +WIFI_SCAN_0105 ^WIFI_SCAN_0105 WIFI_SCAN_0303 +^WIFI_SCAN_0303 WIFI_SCAN_0304 -^WIFI_SCAN_0104 +^WIFI_SCAN_0304 +WIFI_MODE_0101 +WIFI_MODE_0102 +WIFI_MODE_0103 # IGMP cases are supported but as UDP is not stable, exclude them first TCPIP_IGMP_0101 @@ -82,78 +97,114 @@ TCPIP_IGMP_0204 ^TCPIP_IGMP_0204 # DHCP issues -^TCPIP_DHCP_0301 -TCPIP_DHCP_0301 TCPIP_DHCP_0101 ^TCPIP_DHCP_0101 +TCPIP_DHCP_0202 +^TCPIP_DHCP_0202 +TCPIP_DHCP_0204 +^TCPIP_DHCP_0204 +TCPIP_DHCP_0205 +^TCPIP_DHCP_0205 +TCPIP_DHCP_0206 +^TCPIP_DHCP_0206 TCPIP_DHCP_0207 ^TCPIP_DHCP_0207 TCPIP_DHCP_0208 ^TCPIP_DHCP_0208 -TCPIP_DHCP_0205 -^TCPIP_DHCP_0205 TCPIP_DHCP_0209 ^TCPIP_DHCP_0209 -^TCPIP_DHCP_0204 -TCPIP_DHCP_0204 -^TCPIP_DHCP_0210 TCPIP_DHCP_0210 -^TCPIP_DHCP_0211 +^TCPIP_DHCP_0210 TCPIP_DHCP_0211 -^TCPIP_DHCP_0206 +^TCPIP_DHCP_0211 +TCPIP_DHCP_0301 +^TCPIP_DHCP_0301 +TCPIP_DHCP_0302 ^TCPIP_DHCP_0302 # TCP issue -TCPIP_TCP_0402 -^TCPIP_TCP_0406 -^TCPIP_TCP_0401 -TCPIP_TCP_0210 -^TCPIP_TCP_0210 +TCPIP_TCP_0101 +^TCPIP_TCP_0101 +TCPIP_TCP_0102 +^TCPIP_TCP_0102 TCPIP_TCP_0103 ^TCPIP_TCP_0103 +TCPIP_TCP_0104 +^TCPIP_TCP_0104 +TCPIP_TCP_0105 +^TCPIP_TCP_0105 +TCPIP_TCP_0106 +^TCPIP_TCP_0106 +TCPIP_TCP_0107 +^TCPIP_TCP_0107 TCPIP_TCP_0112 ^TCPIP_TCP_0112 -TCPIP_TCP_0106 -TCPIP_TCP_0107 -TCPIP_TCP_0105 -^TCPIP_TCP_0407 -^TCPIP_TCP_0404 -^TCPIP_TCP_0102 -^TCPIP_TCP_0105 -^TCPIP_TCP_0107 -^TCPIP_TCP_0206 -TCPIP_TCP_0102 -^TCPIP_TCP_0403 -^TCPIP_TCP_0402 -^TCPIP_TCP_0101 -^TCPIP_TCP_0203 -^TCPIP_TCP_0106 +TCPIP_TCP_0201 ^TCPIP_TCP_0201 +TCPIP_TCP_0202 +^TCPIP_TCP_0202 +TCPIP_TCP_0203 +^TCPIP_TCP_0203 +TCPIP_TCP_0204 +^TCPIP_TCP_0204 +TCPIP_TCP_0206 +^TCPIP_TCP_0206 +TCPIP_TCP_0208 +^TCPIP_TCP_0208 +TCPIP_TCP_0210 +^TCPIP_TCP_0210 +TCPIP_TCP_0401 +^TCPIP_TCP_0401 +TCPIP_TCP_0402 +^TCPIP_TCP_0402 +TCPIP_TCP_0403 +^TCPIP_TCP_0403 +TCPIP_TCP_0404 +^TCPIP_TCP_0404 +TCPIP_TCP_0406 +^TCPIP_TCP_0406 +TCPIP_TCP_0407 +^TCPIP_TCP_0407 +TCPIP_TCP_0408 +^TCPIP_TCP_0408 +TCPIP_TCP_0412 ^TCPIP_TCP_0412 -TCPIP_TCP_0101 +TCPIP_TCP_0411 +^TCPIP_TCP_0411 # UDP issue +TCPIP_UDP_0102 +^TCPIP_UDP_0102 TCPIP_UDP_0103 ^TCPIP_UDP_0103 +TCPIP_UDP_0104 +^TCPIP_UDP_0104 +TCPIP_UDP_0108 ^TCPIP_UDP_0108 TCPIP_UDP_0110 ^TCPIP_UDP_0110 +TCPIP_UDP_0112 ^TCPIP_UDP_0112 -TCPIP_UDP_0305 -^TCPIP_UDP_0305 -^TCPIP_UDP_0304 -TCPIP_UDP_0104 -^TCPIP_UDP_0104 -^TCPIP_UDP_0301 TCPIP_UDP_0301 +^TCPIP_UDP_0301 TCPIP_UDP_0302 +^TCPIP_UDP_0302 TCPIP_UDP_0303 ^TCPIP_UDP_0303 -^TCPIP_UDP_0307 -^TCPIP_UDP_0302 +TCPIP_UDP_0304 +^TCPIP_UDP_0304 +TCPIP_UDP_0305 +^TCPIP_UDP_0305 +TCPIP_UDP_0306 ^TCPIP_UDP_0306 +TCPIP_UDP_0307 +^TCPIP_UDP_0307 #DNS -^TCPIP_DNS_0103 +TCPIP_DNS_0101 +^TCPIP_DNS_0101 +TCPIP_DNS_0102 ^TCPIP_DNS_0102 +TCPIP_DNS_0103 +^TCPIP_DNS_0103