From 381464d1cf29521ecf9baa65c764f2b1330f5554 Mon Sep 17 00:00:00 2001 From: He Yin Ling Date: Wed, 12 May 2021 10:16:20 +0800 Subject: [PATCH 1/2] test: support multiple targets for iperf example test --- examples/wifi/iperf/iperf_test.py | 30 +++++++++++++---------------- examples/wifi/iperf/sdkconfig.ci.99 | 30 ----------------------------- 2 files changed, 13 insertions(+), 47 deletions(-) diff --git a/examples/wifi/iperf/iperf_test.py b/examples/wifi/iperf/iperf_test.py index 9ac638dc73..2aad6ebd7b 100644 --- a/examples/wifi/iperf/iperf_test.py +++ b/examples/wifi/iperf/iperf_test.py @@ -524,7 +524,7 @@ class IperfTestUtilitySoftap(IperfTestUtility): return server_raw_data, rssi, heap_size -@ttfw_idf.idf_example_test(env_tag='Example_ShieldBox_Basic', category='stress') +@ttfw_idf.idf_example_test(env_tag='Example_ShieldBox_Basic', target=['ESP32', 'ESP32S2', 'ESP32C3'], category='stress') def test_wifi_throughput_with_different_configs(env, extra_data): """ steps: | @@ -552,8 +552,7 @@ def test_wifi_throughput_with_different_configs(env, extra_data): 'sdkconfig.ci.{}'.format(config_name)) # 2. get DUT and download - dut = env.get_dut('iperf', 'examples/wifi/iperf', dut_class=ttfw_idf.ESP32DUT, - app_config_name=config_name) + dut = env.get_dut('iperf', 'examples/wifi/iperf', app_config_name=config_name) dut.start_app() dut.expect_any('iperf>', 'esp32>') @@ -585,7 +584,7 @@ def test_wifi_throughput_with_different_configs(env, extra_data): report.generate_report() -@ttfw_idf.idf_example_test(env_tag='Example_ShieldBox', category='stress') +@ttfw_idf.idf_example_test(env_tag='Example_ShieldBox', target=['ESP32', 'ESP32S2', 'ESP32C3'], category='stress') def test_wifi_throughput_vs_rssi(env, extra_data): """ steps: | @@ -608,8 +607,7 @@ def test_wifi_throughput_vs_rssi(env, extra_data): } # 1. get DUT and download - dut = env.get_dut('iperf', 'examples/wifi/iperf', dut_class=ttfw_idf.ESP32DUT, - app_config_name=BEST_PERFORMANCE_CONFIG) + dut = env.get_dut('iperf', 'examples/wifi/iperf', app_config_name=BEST_PERFORMANCE_CONFIG) dut.start_app() dut.expect_any('iperf>', 'esp32>') @@ -640,7 +638,8 @@ def test_wifi_throughput_vs_rssi(env, extra_data): report.generate_report() -@ttfw_idf.idf_example_test(env_tag='Example_ShieldBox_Basic') +@ttfw_idf.idf_example_test(env_tag='Example_ShieldBox_Basic', + target=['ESP32', 'ESP32S2', 'ESP32C3'], ci_target=['ESP32']) def test_wifi_throughput_basic(env, extra_data): """ steps: | @@ -655,8 +654,7 @@ def test_wifi_throughput_basic(env, extra_data): } # 1. get DUT - dut = env.get_dut('iperf', 'examples/wifi/iperf', dut_class=ttfw_idf.ESP32DUT, - app_config_name=BEST_PERFORMANCE_CONFIG) + dut = env.get_dut('iperf', 'examples/wifi/iperf', app_config_name=BEST_PERFORMANCE_CONFIG) dut.start_app() dut.expect_any('iperf>', 'esp32>') @@ -693,7 +691,7 @@ def test_wifi_throughput_basic(env, extra_data): env.close_dut('iperf') -@ttfw_idf.idf_example_test(env_tag='Example_ShieldBox2', category='stress') +@ttfw_idf.idf_example_test(env_tag='Example_ShieldBox2', target=['ESP32', 'ESP32S2', 'ESP32C3'], category='stress') def test_softap_throughput_vs_rssi(env, extra_data): """ steps: | @@ -712,13 +710,11 @@ def test_softap_throughput_vs_rssi(env, extra_data): } # 1. get DUT and download - softap_dut = env.get_dut('softap_iperf', 'examples/wifi/iperf', dut_class=ttfw_idf.ESP32DUT, - app_config_name=BEST_PERFORMANCE_CONFIG) + softap_dut = env.get_dut('softap_iperf', 'examples/wifi/iperf') softap_dut.start_app() softap_dut.expect_any('iperf>', 'esp32>') - sta_dut = env.get_dut('sta_iperf', 'examples/wifi/iperf', dut_class=ttfw_idf.ESP32DUT, - app_config_name=BEST_PERFORMANCE_CONFIG) + sta_dut = env.get_dut('sta_iperf', 'examples/wifi/iperf', app_config_name=BEST_PERFORMANCE_CONFIG) sta_dut.start_app() sta_dut.expect_any('iperf>', 'esp32>') @@ -741,7 +737,7 @@ def test_softap_throughput_vs_rssi(env, extra_data): if __name__ == '__main__': - test_wifi_throughput_basic(env_config_file='EnvConfig.yml') - test_wifi_throughput_with_different_configs(env_config_file='EnvConfig.yml') - test_wifi_throughput_vs_rssi(env_config_file='EnvConfig.yml') + # test_wifi_throughput_basic(env_config_file='EnvConfig.yml') + # test_wifi_throughput_with_different_configs(env_config_file='EnvConfig.yml') + test_wifi_throughput_vs_rssi(env_config_file='EnvConfig.yml', target='ESP32C3') test_softap_throughput_vs_rssi(env_config_file='EnvConfig.yml') diff --git a/examples/wifi/iperf/sdkconfig.ci.99 b/examples/wifi/iperf/sdkconfig.ci.99 index 52c69eb3da..e69de29bb2 100644 --- a/examples/wifi/iperf/sdkconfig.ci.99 +++ b/examples/wifi/iperf/sdkconfig.ci.99 @@ -1,30 +0,0 @@ -CONFIG_ESP32_DEFAULT_CPU_FREQ_240=y -CONFIG_ESP32_DEFAULT_CPU_FREQ_MHZ=240 -CONFIG_MEMMAP_SMP=y - -CONFIG_ESP_SYSTEM_EVENT_TASK_STACK_SIZE=4096 - -CONFIG_ESP32_WIFI_STATIC_RX_BUFFER_NUM=16 -CONFIG_ESP32_WIFI_DYNAMIC_RX_BUFFER_NUM=64 -CONFIG_ESP32_WIFI_DYNAMIC_TX_BUFFER_NUM=64 -CONFIG_ESP32_WIFI_AMPDU_TX_ENABLED=y -CONFIG_ESP32_WIFI_TX_BA_WIN=32 -CONFIG_ESP32_WIFI_AMPDU_RX_ENABLED=y -CONFIG_ESP32_WIFI_RX_BA_WIN=32 - -CONFIG_FREERTOS_UNICORE=n -CONFIG_FREERTOS_HZ=1000 - -CONFIG_ESP_INT_WDT=n -CONFIG_ESP_TASK_WDT=n - -CONFIG_LWIP_TCP_SND_BUF_DEFAULT=65534 -CONFIG_LWIP_TCP_WND_DEFAULT=65534 -CONFIG_LWIP_TCP_RECVMBOX_SIZE=64 -CONFIG_LWIP_UDP_RECVMBOX_SIZE=64 -CONFIG_LWIP_TCPIP_RECVMBOX_SIZE=64 -CONFIG_LWIP_ETHARP_TRUST_IP_MAC=n - -CONFIG_ESPTOOLPY_FLASHMODE_QIO=y -CONFIG_ESPTOOLPY_FLASHFREQ_80M=y -CONFIG_LWIP_IRAM_OPTIMIZATION=y From 6fe2da9f898030909acd7dbefb9f2fbf4cf61aa2 Mon Sep 17 00:00:00 2001 From: He Yin Ling Date: Thu, 13 May 2021 16:26:08 +0800 Subject: [PATCH 2/2] ci: save built binaries could be tested locally: we have some test cases not executed in CI. we need to save those binaries as artifacts so we can test locally. --- .../python_packages/ttfw_idf/CIScanTests.py | 33 ++++++++++++++----- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/tools/ci/python_packages/ttfw_idf/CIScanTests.py b/tools/ci/python_packages/ttfw_idf/CIScanTests.py index ed66fc88ad..c1b38f89bb 100644 --- a/tools/ci/python_packages/ttfw_idf/CIScanTests.py +++ b/tools/ci/python_packages/ttfw_idf/CIScanTests.py @@ -6,6 +6,11 @@ import os from collections import defaultdict from copy import deepcopy +try: + from typing import Any +except ImportError: + # Only used for type annotations + pass from find_apps import find_apps from find_build_apps import BUILD_SYSTEM_CMAKE, BUILD_SYSTEMS from idf_py_actions.constants import PREVIEW_TARGETS, SUPPORTED_TARGETS @@ -28,14 +33,14 @@ BUILD_ALL_LABELS = [ ] -def _has_build_all_label(): +def _has_build_all_label(): # type: () -> bool for label in BUILD_ALL_LABELS: if os.getenv(label): return True return False -def _judge_build_or_not(action, build_all): # type: (str, bool) -> (bool, bool) +def _judge_build_or_not(action, build_all): # type: (str, bool) -> tuple[bool, bool] """ :return: (build_or_not_for_test_related_apps, build_or_not_for_non_related_apps) """ @@ -45,7 +50,7 @@ def _judge_build_or_not(action, build_all): # type: (str, bool) -> (bool, bool) labels = TEST_LABELS[action] if not isinstance(labels, list): - labels = [labels] + labels = [labels] # type: ignore for label in labels: if os.getenv(label): @@ -55,13 +60,23 @@ def _judge_build_or_not(action, build_all): # type: (str, bool) -> (bool, bool) return False, False -def output_json(apps_dict_list, target, build_system, output_dir): +def output_json(apps_dict_list, target, build_system, output_dir): # type: (list, str, str, str) -> None output_path = os.path.join(output_dir, 'scan_{}_{}.json'.format(target.lower(), build_system)) with open(output_path, 'w') as fw: fw.writelines([json.dumps(app) + '\n' for app in apps_dict_list]) -def main(): +# we might need artifacts to run test cases locally. +# So we need to save artifacts which have test case not executed by CI. +class _ExampleAssignTest(ExampleAssignTest): + DEFAULT_FILTER = {} # type: dict[str, Any] + + +class _TestAppsAssignTest(TestAppsAssignTest): + DEFAULT_FILTER = {} # type: dict[str, Any] + + +def main(): # type: () -> None parser = argparse.ArgumentParser(description='Scan the required build tests') parser.add_argument('test_type', choices=TEST_LABELS.keys(), @@ -101,14 +116,14 @@ def main(): output_json([], target, args.build_system, args.output_path) SystemExit(0) - paths = set([os.path.join(os.getenv('IDF_PATH'), path) if not os.path.isabs(path) else path for path in args.paths]) + paths = set([os.path.join(str(os.getenv('IDF_PATH')), path) if not os.path.isabs(path) else path for path in args.paths]) test_cases = [] for path in paths: if args.test_type == 'example_test': - assign = ExampleAssignTest(path, args.ci_config_file) + assign = _ExampleAssignTest(path, args.ci_config_file) elif args.test_type in ['test_apps', 'component_ut']: - assign = TestAppsAssignTest(path, args.ci_config_file) + assign = _TestAppsAssignTest(path, args.ci_config_file) else: raise SystemExit(1) # which is impossible @@ -123,7 +138,7 @@ def main(): ... } ''' - scan_info_dict = defaultdict(dict) + scan_info_dict = defaultdict(dict) # type: dict[str, dict] # store the test cases dir, exclude these folders when scan for standalone apps default_exclude = args.exclude if args.exclude else []