espressif/esp-idf
1
0
Fork 1
mirror of https://github.com/espressif/esp-idf.git synced 2025-08-05 05:34:32 +02:00
Code Issues Packages Projects Releases Wiki Activity

test(ci): increase timeout for iperf expect

Browse Source
This commit is contained in:
Alexey Lapshin
2025-05-20 11:25:22 +07:00
parent 0f2e45c1ad
commit e89241586d
1 changed files with 131 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files

169
tools/ci/python_packages/idf_iperf_test_util/IperfUtility.py
View File

@@ -7,11 +7,13 @@ import subprocess
import time import time
import pexpect import pexpect
from idf_iperf_test_util import LineChart
from pytest_embedded import Dut from pytest_embedded import Dut
from idf_iperf_test_util import LineChart
try: try:
from typing import Any, Tuple from typing import Any
from typing import Tuple
except ImportError: except ImportError:
# Only used for type annotations # Only used for type annotations
pass pass
@@ -114,21 +116,27 @@ class TestResult(object):
continue continue
throughput_list.append(throughput) throughput_list.append(throughput)
max_throughput = max(max_throughput, throughput) max_throughput = max(max_throughput, throughput)
if throughput == 0 and rssi > self.ZERO_POINT_THRESHOLD \ if throughput == 0 and rssi > self.ZERO_POINT_THRESHOLD and fall_to_0_recorded < 1:
and fall_to_0_recorded < 1:
# throughput fall to 0 error. we only record 1 records for one test # throughput fall to 0 error. we only record 1 records for one test
self.error_list.append('[Error][fall to 0][{}][att: {}][rssi: {}]: 0 throughput interval: {}-{}' self.error_list.append(
.format(ap_ssid, att, rssi, result[0], result[1])) '[Error][fall to 0][{}][att: {}][rssi: {}]: 0 throughput interval: {}-{}'.format(
ap_ssid, att, rssi, result[0], result[1]
)
)
fall_to_0_recorded += 1 fall_to_0_recorded += 1
if len(throughput_list) < self.THROUGHPUT_QUALIFY_COUNT: if len(throughput_list) < self.THROUGHPUT_QUALIFY_COUNT:
self.error_list.append('[Error][Fatal][{}][att: {}][rssi: {}]: Only {} throughput values found, expected at least {}' self.error_list.append(
.format(ap_ssid, att, rssi, len(throughput_list), self.THROUGHPUT_QUALIFY_COUNT)) '[Error][Fatal][{}][att: {}][rssi: {}]: Only {} throughput values found, expected at least {}'.format(
ap_ssid, att, rssi, len(throughput_list), self.THROUGHPUT_QUALIFY_COUNT
)
)
max_throughput = 0.0 max_throughput = 0.0
if max_throughput == 0 and rssi > self.ZERO_THROUGHPUT_THRESHOLD: if max_throughput == 0 and rssi > self.ZERO_THROUGHPUT_THRESHOLD:
self.error_list.append('[Error][Fatal][{}][att: {}][rssi: {}]: No throughput data found' self.error_list.append(
.format(ap_ssid, att, rssi)) '[Error][Fatal][{}][att: {}][rssi: {}]: No throughput data found'.format(ap_ssid, att, rssi)
)
self._save_result(max_throughput, ap_ssid, att, rssi, heap_size) self._save_result(max_throughput, ap_ssid, att, rssi, heap_size)
@@ -141,6 +149,7 @@ class TestResult(object):
1. throughput value 30% worse than the next point with lower RSSI 1. throughput value 30% worse than the next point with lower RSSI
2. throughput value 30% worse than the next point with larger attenuate 2. throughput value 30% worse than the next point with larger attenuate
""" """
def analysis_bad_point(data: dict, index_type: str) -> None: def analysis_bad_point(data: dict, index_type: str) -> None:
for ap_ssid in data: for ap_ssid in data:
result_dict = data[ap_ssid] result_dict = data[ap_ssid]
@@ -150,14 +159,18 @@ class TestResult(object):
index_list.reverse() index_list.reverse()
for i, index_value in enumerate(index_list[1:]): for i, index_value in enumerate(index_list[1:]):
if index_value < self.BAD_POINT_RSSI_THRESHOLD or \ if (
result_dict[index_list[i]] < self.BAD_POINT_MIN_THRESHOLD: index_value < self.BAD_POINT_RSSI_THRESHOLD
or result_dict[index_list[i]] < self.BAD_POINT_MIN_THRESHOLD
):
continue continue
_percentage = result_dict[index_value] / result_dict[index_list[i]] _percentage = result_dict[index_value] / result_dict[index_list[i]]
if _percentage < 1 - self.BAD_POINT_PERCENTAGE_THRESHOLD: if _percentage < 1 - self.BAD_POINT_PERCENTAGE_THRESHOLD:
self.error_list.append('[Error][Bad point][{}][{}: {}]: drop {:.02f}%' self.error_list.append(
.format(ap_ssid, index_type, index_value, '[Error][Bad point][{}][{}: {}]: drop {:.02f}%'.format(
(1 - _percentage) * 100)) ap_ssid, index_type, index_value, (1 - _percentage) * 100
)
)
analysis_bad_point(self.throughput_by_rssi, 'rssi') analysis_bad_point(self.throughput_by_rssi, 'rssi')
analysis_bad_point(self.throughput_by_att, 'att') analysis_bad_point(self.throughput_by_att, 'att')
@@ -184,11 +197,14 @@ class TestResult(object):
else: else:
file_name = 'ThroughputVs{}_{}_{}.html'.format(type_name, self.proto, self.direction) file_name = 'ThroughputVs{}_{}_{}.html'.format(type_name, self.proto, self.direction)
LineChart.draw_line_chart(os.path.join(path, file_name), LineChart.draw_line_chart(
os.path.join(path, file_name),
'Throughput Vs {} ({} {})'.format(type_name, self.proto, self.direction), 'Throughput Vs {} ({} {})'.format(type_name, self.proto, self.direction),
'{} (dbm)'.format(type_name), '{} (dbm)'.format(type_name),
'Throughput (Mbps)', 'Throughput (Mbps)',
data, range_list) data,
range_list,
)
return file_name return file_name
def draw_rssi_vs_att_figure(self, path: str, ap_ssid: str) -> str: def draw_rssi_vs_att_figure(self, path: str, ap_ssid: str) -> str:
@@ -201,18 +217,14 @@ class TestResult(object):
file_name = 'AttVsRSSI.html' file_name = 'AttVsRSSI.html'
else: else:
file_name = 'AttVsRSSI.html' file_name = 'AttVsRSSI.html'
LineChart.draw_line_chart(os.path.join(path, file_name), LineChart.draw_line_chart(
'Att Vs RSSI', os.path.join(path, file_name), 'Att Vs RSSI', 'Att (dbm)', 'RSSI (dbm)', self.att_rssi_map, self.ATT_RANGE
'Att (dbm)', )
'RSSI (dbm)',
self.att_rssi_map,
self.ATT_RANGE)
return file_name return file_name
def get_best_throughput(self) -> Any: def get_best_throughput(self) -> Any:
"""get the best throughput during test""" """get the best throughput during test"""
best_for_aps = [max(self.throughput_by_att[ap_ssid].values()) best_for_aps = [max(self.throughput_by_att[ap_ssid].values()) for ap_ssid in self.throughput_by_att]
for ap_ssid in self.throughput_by_att]
return max(best_for_aps) return max(best_for_aps)
def __str__(self) -> str: def __str__(self) -> str:
@@ -224,8 +236,9 @@ class TestResult(object):
3. min free heap size during test 3. min free heap size during test
""" """
if self.throughput_by_att: if self.throughput_by_att:
ret = '[{}_{}][{}]: {}\r\n\r\n'.format(self.proto, self.direction, self.config_name, ret = '[{}_{}][{}]: {}\r\n\r\n'.format(
'Fail' if self.error_list else 'Success') self.proto, self.direction, self.config_name, 'Fail' if self.error_list else 'Success'
)
ret += 'Performance for each AP:\r\n' ret += 'Performance for each AP:\r\n'
for ap_ssid in self.throughput_by_att: for ap_ssid in self.throughput_by_att:
ret += '[{}]: {:.02f} Mbps\r\n'.format(ap_ssid, max(self.throughput_by_att[ap_ssid].values())) ret += '[{}]: {:.02f} Mbps\r\n'.format(ap_ssid, max(self.throughput_by_att[ap_ssid].values()))
@@ -239,8 +252,16 @@ class TestResult(object):
class IperfTestUtility(object): class IperfTestUtility(object):
"""iperf test implementation""" """iperf test implementation"""
def __init__(self, dut:Dut, config_name:str, ap_ssid:str, ap_password:str, def __init__(
pc_nic_ip:str, pc_iperf_log_file:str, test_result:Any=None) -> None: self,
dut: Dut,
config_name: str,
ap_ssid: str,
ap_password: str,
pc_nic_ip: str,
pc_iperf_log_file: str,
test_result: Any = None,
) -> None:
self.config_name = config_name self.config_name = config_name
self.dut = dut self.dut = dut
@@ -281,8 +302,7 @@ class IperfTestUtility(object):
self.dut.write('sta_scan {}'.format(self.ap_ssid)) self.dut.write('sta_scan {}'.format(self.ap_ssid))
for _ in range(SCAN_RETRY_COUNT): for _ in range(SCAN_RETRY_COUNT):
try: try:
rssi = int(self.dut.expect(r'\[{}]\[rssi=(-\d+)]'.format(self.ap_ssid), rssi = int(self.dut.expect(r'\[{}]\[rssi=(-\d+)]'.format(self.ap_ssid), timeout=SCAN_TIMEOUT).group(1))
timeout=SCAN_TIMEOUT).group(1))
break break
except pexpect.TIMEOUT: except pexpect.TIMEOUT:
continue continue
@@ -307,17 +327,21 @@ class IperfTestUtility(object):
if direction == 'tx': if direction == 'tx':
with open(PC_IPERF_TEMP_LOG_FILE, 'w') as f: with open(PC_IPERF_TEMP_LOG_FILE, 'w') as f:
if proto == 'tcp': if proto == 'tcp':
process = subprocess.Popen(['iperf', '-s', '-B', self.pc_nic_ip, process = subprocess.Popen(
'-t', str(TEST_TIME), '-i', '1', '-f', 'm'], ['iperf', '-s', '-B', self.pc_nic_ip, '-t', str(TEST_TIME), '-i', '1', '-f', 'm'],
stdout=f, stderr=f) stdout=f,
stderr=f,
)
if bw_limit > 0: if bw_limit > 0:
self.dut.write('iperf -c {} -i 1 -t {} -b {}'.format(self.pc_nic_ip, TEST_TIME, bw_limit)) self.dut.write('iperf -c {} -i 1 -t {} -b {}'.format(self.pc_nic_ip, TEST_TIME, bw_limit))
else: else:
self.dut.write('iperf -c {} -i 1 -t {}'.format(self.pc_nic_ip, TEST_TIME)) self.dut.write('iperf -c {} -i 1 -t {}'.format(self.pc_nic_ip, TEST_TIME))
else: else:
process = subprocess.Popen(['iperf', '-s', '-u', '-B', self.pc_nic_ip, process = subprocess.Popen(
'-t', str(TEST_TIME), '-i', '1', '-f', 'm'], ['iperf', '-s', '-u', '-B', self.pc_nic_ip, '-t', str(TEST_TIME), '-i', '1', '-f', 'm'],
stdout=f, stderr=f) stdout=f,
stderr=f,
)
if bw_limit > 0: if bw_limit > 0:
self.dut.write('iperf -c {} -u -i 1 -t {} -b {}'.format(self.pc_nic_ip, TEST_TIME, bw_limit)) self.dut.write('iperf -c {} -u -i 1 -t {} -b {}'.format(self.pc_nic_ip, TEST_TIME, bw_limit))
else: else:
@@ -343,11 +367,15 @@ class IperfTestUtility(object):
# compatible with old iperf example binary # compatible with old iperf example binary
logging.info('create iperf tcp server fail') logging.info('create iperf tcp server fail')
if bw_limit > 0: if bw_limit > 0:
process = subprocess.Popen(['iperf', '-c', dut_ip, '-b', str(bw_limit) + 'm', process = subprocess.Popen(
'-t', str(TEST_TIME), '-f', 'm'], stdout=f, stderr=f) ['iperf', '-c', dut_ip, '-b', str(bw_limit) + 'm', '-t', str(TEST_TIME), '-f', 'm'],
stdout=f,
stderr=f,
)
else: else:
process = subprocess.Popen(['iperf', '-c', dut_ip, process = subprocess.Popen(
'-t', str(TEST_TIME), '-f', 'm'], stdout=f, stderr=f) ['iperf', '-c', dut_ip, '-t', str(TEST_TIME), '-f', 'm'], stdout=f, stderr=f
)
for _ in range(TEST_TIMEOUT): for _ in range(TEST_TIMEOUT):
if process.poll() is not None: if process.poll() is not None:
break break
@@ -363,8 +391,11 @@ class IperfTestUtility(object):
except pexpect.TIMEOUT: except pexpect.TIMEOUT:
# compatible with old iperf example binary # compatible with old iperf example binary
logging.info('create iperf udp server fail') logging.info('create iperf udp server fail')
process = subprocess.Popen(['iperf', '-c', dut_ip, '-u', '-b', str(bw_limit) + 'm', process = subprocess.Popen(
'-t', str(TEST_TIME), '-f', 'm'], stdout=f, stderr=f) ['iperf', '-c', dut_ip, '-u', '-b', str(bw_limit) + 'm', '-t', str(TEST_TIME), '-f', 'm'],
stdout=f,
stderr=f,
)
for _ in range(TEST_TIMEOUT): for _ in range(TEST_TIMEOUT):
if process.poll() is not None: if process.poll() is not None:
break break
@@ -376,7 +407,9 @@ class IperfTestUtility(object):
stop_bw = 100 stop_bw = 100
n = 10 n = 10
step = int((stop_bw - start_bw) / n) step = int((stop_bw - start_bw) / n)
self.dut.write('iperf -s -u -i 1 -t {}'.format(TEST_TIME + 4 * (n + 1))) # 4 sec for each bw step instance start/stop self.dut.write(
'iperf -s -u -i 1 -t {}'.format(TEST_TIME + 4 * (n + 1))
) # 4 sec for each bw step instance start/stop
# wait until DUT TCP server created # wait until DUT TCP server created
try: try:
self.dut.expect('Socket bound', timeout=5) self.dut.expect('Socket bound', timeout=5)
@@ -384,8 +417,22 @@ class IperfTestUtility(object):
# compatible with old iperf example binary # compatible with old iperf example binary
logging.info('create iperf udp server fail') logging.info('create iperf udp server fail')
for bandwidth in range(start_bw, stop_bw, step): for bandwidth in range(start_bw, stop_bw, step):
process = subprocess.Popen(['iperf', '-c', dut_ip, '-u', '-b', str(bandwidth) + 'm', process = subprocess.Popen(
'-t', str(TEST_TIME / (n + 1)), '-f', 'm'], stdout=f, stderr=f) [
'iperf',
'-c',
dut_ip,
'-u',
'-b',
str(bandwidth) + 'm',
'-t',
str(TEST_TIME / (n + 1)),
'-f',
'm',
],
stdout=f,
stderr=f,
)
for _ in range(TEST_TIMEOUT): for _ in range(TEST_TIMEOUT):
if process.poll() is not None: if process.poll() is not None:
break break
@@ -402,13 +449,16 @@ class IperfTestUtility(object):
# save PC iperf logs to console # save PC iperf logs to console
with open(self.pc_iperf_log_file, 'a+') as f: with open(self.pc_iperf_log_file, 'a+') as f:
f.write('## [{}] `{}`\r\n##### {}' f.write(
.format(self.config_name, '## [{}] `{}`\r\n##### {}'.format(
self.config_name,
'{}_{}'.format(proto, direction), '{}_{}'.format(proto, direction),
time.strftime('%m-%d %H:%M:%S', time.localtime(time.time())))) time.strftime('%m-%d %H:%M:%S', time.localtime(time.time())),
)
)
f.write('\r\n```\r\n\r\n' + pc_raw_data + '\r\n```\r\n') f.write('\r\n```\r\n\r\n' + pc_raw_data + '\r\n```\r\n')
self.dut.write('heap') self.dut.write('heap')
heap_size = self.dut.expect(r'min heap size: (\d+)\D').group(1) heap_size = self.dut.expect(r'min heap size: (\d+)\D', timeout=120).group(1)
# return server raw data (for parsing test results) and RSSI # return server raw data (for parsing test results) and RSSI
return server_raw_data, rssi, heap_size return server_raw_data, rssi, heap_size
@@ -426,11 +476,14 @@ class IperfTestUtility(object):
heap_size = INVALID_HEAP_SIZE heap_size = INVALID_HEAP_SIZE
try: try:
server_raw_data, rssi, heap_size = self._test_once(proto, direction, bw_limit) server_raw_data, rssi, heap_size = self._test_once(proto, direction, bw_limit)
throughput = self._save_test_result('{}_{}'.format(proto, direction), throughput = self._save_test_result(
server_raw_data, atten_val, '{}_{}'.format(proto, direction), server_raw_data, atten_val, rssi, heap_size
rssi, heap_size) )
logging.info('[{}][{}_{}][{}][{}]: {:.02f}' logging.info(
.format(self.config_name, proto, direction, rssi, self.ap_ssid, throughput)) '[{}][{}_{}][{}][{}]: {:.02f}'.format(
self.config_name, proto, direction, rssi, self.ap_ssid, throughput
)
)
self.lowest_rssi_scanned = min(self.lowest_rssi_scanned, rssi) self.lowest_rssi_scanned = min(self.lowest_rssi_scanned, rssi)
except (ValueError, IndexError): except (ValueError, IndexError):
self._save_test_result('{}_{}'.format(proto, direction), '', atten_val, rssi, heap_size) self._save_test_result('{}_{}'.format(proto, direction), '', atten_val, rssi, heap_size)
@@ -452,7 +505,8 @@ class IperfTestUtility(object):
self.run_test('udp', 'rx', atten_val, bw_limit) self.run_test('udp', 'rx', atten_val, bw_limit)
if self.fail_to_scan > 10: if self.fail_to_scan > 10:
logging.info( logging.info(
'Fail to scan AP for more than 10 times. Lowest RSSI scanned is {}'.format(self.lowest_rssi_scanned)) 'Fail to scan AP for more than 10 times. Lowest RSSI scanned is {}'.format(self.lowest_rssi_scanned)
)
raise AssertionError raise AssertionError
def wait_ap_power_on(self) -> bool: def wait_ap_power_on(self) -> bool:
@@ -467,8 +521,7 @@ class IperfTestUtility(object):
for _ in range(WAIT_AP_POWER_ON_TIMEOUT // SCAN_TIMEOUT): for _ in range(WAIT_AP_POWER_ON_TIMEOUT // SCAN_TIMEOUT):
try: try:
self.dut.write('scan {}'.format(self.ap_ssid)) self.dut.write('scan {}'.format(self.ap_ssid))
self.dut.expect(r'\[{}]\[rssi=(-\d+)]'.format(self.ap_ssid), self.dut.expect(r'\[{}]\[rssi=(-\d+)]'.format(self.ap_ssid), timeout=SCAN_TIMEOUT)
timeout=SCAN_TIMEOUT)
ret = True ret = True
break break
except pexpect.TIMEOUT: except pexpect.TIMEOUT: