Merge branch 'bugfix/update_iperf_example_test_case' into 'master'

test: update iperf example test

Closes IDFCI-355

See merge request espressif/esp-idf!11245

examples/wifi/iperf/components/iperf/iperf.c

@@ -149,6 +149,7 @@ static esp_err_t IRAM_ATTR iperf_run_tcp_server(void)
         return ESP_FAIL;
     }
 
+    printf("iperf tcp server create successfully\n");
     buffer = s_iperf_ctrl.buffer;
     want_recv = s_iperf_ctrl.buffer_len;
     while (!s_iperf_ctrl.finish) {

examples/wifi/iperf/iperf_test.py

@@ -63,13 +63,16 @@ class TestResult(object):
 
     ZERO_POINT_THRESHOLD = -88  # RSSI, dbm
     ZERO_THROUGHPUT_THRESHOLD = -92  # RSSI, dbm
-    BAD_POINT_RSSI_THRESHOLD = -85  # RSSI, dbm
-    BAD_POINT_MIN_THRESHOLD = 3  # Mbps
+    BAD_POINT_RSSI_THRESHOLD = -75  # RSSI, dbm
+    BAD_POINT_MIN_THRESHOLD = 10  # Mbps
     BAD_POINT_PERCENTAGE_THRESHOLD = 0.3
 
     # we need at least 1/2 valid points to qualify the test result
     THROUGHPUT_QUALIFY_COUNT = TEST_TIME // 2
 
+    RSSI_RANGE = [-x for x in range(10, 100)]
+    ATT_RANGE = [x for x in range(0, 64)]
+
     def __init__(self, proto, direction, config_name):
         self.proto = proto
         self.direction = direction
@@ -180,16 +183,6 @@ class TestResult(object):
         analysis_bad_point(self.throughput_by_rssi, "rssi")
         analysis_bad_point(self.throughput_by_att, "att")
 
-    @staticmethod
-    def _convert_to_draw_format(data, label):
-        keys = data.keys()
-        keys.sort()
-        return {
-            "x-axis": keys,
-            "y-axis": [data[x] for x in keys],
-            "label": label,
-        }
-
     def draw_throughput_figure(self, path, ap_ssid, draw_type):
         """
         :param path: folder to save figure. make sure the folder is already created.
@@ -200,25 +193,24 @@ class TestResult(object):
         if draw_type == "rssi":
             type_name = "RSSI"
             data = self.throughput_by_rssi
+            range_list = self.RSSI_RANGE
         elif draw_type == "att":
             type_name = "Att"
             data = self.throughput_by_att
+            range_list = self.ATT_RANGE
         else:
             raise AssertionError("draw type not supported")
         if isinstance(ap_ssid, list):
-            file_name = "ThroughputVs{}_{}_{}_{}.png".format(type_name, self.proto, self.direction,
-                                                             hash(ap_ssid)[:6])
-            data_list = [self._convert_to_draw_format(data[_ap_ssid], _ap_ssid)
-                         for _ap_ssid in ap_ssid]
+            file_name = "ThroughputVs{}_{}_{}_{}.html".format(type_name, self.proto, self.direction,
+                                                              hash(ap_ssid)[:6])
         else:
-            file_name = "ThroughputVs{}_{}_{}_{}.png".format(type_name, self.proto, self.direction, ap_ssid)
-            data_list = [self._convert_to_draw_format(data[ap_ssid], ap_ssid)]
+            file_name = "ThroughputVs{}_{}_{}_{}.html".format(type_name, self.proto, self.direction, ap_ssid)
 
         LineChart.draw_line_chart(os.path.join(path, file_name),
                                   "Throughput Vs {} ({} {})".format(type_name, self.proto, self.direction),
-                                  "Throughput (Mbps)",
                                   "{} (dbm)".format(type_name),
-                                  data_list)
+                                  "Throughput (Mbps)",
+                                  data, range_list)
         return file_name
 
     def draw_rssi_vs_att_figure(self, path, ap_ssid):
@@ -228,17 +220,15 @@ class TestResult(object):
         :return: file_name
         """
         if isinstance(ap_ssid, list):
-            file_name = "AttVsRSSI_{}.png".format(hash(ap_ssid)[:6])
-            data_list = [self._convert_to_draw_format(self.att_rssi_map[_ap_ssid], _ap_ssid)
-                         for _ap_ssid in ap_ssid]
+            file_name = "AttVsRSSI_{}.html".format(hash(ap_ssid)[:6])
         else:
-            file_name = "AttVsRSSI_{}.png".format(ap_ssid)
-            data_list = [self._convert_to_draw_format(self.att_rssi_map[ap_ssid], ap_ssid)]
+            file_name = "AttVsRSSI_{}.html".format(ap_ssid)
         LineChart.draw_line_chart(os.path.join(path, file_name),
                                   "Att Vs RSSI",
                                   "Att (dbm)",
                                   "RSSI (dbm)",
-                                  data_list)
+                                  self.att_rssi_map,
+                                  self.ATT_RANGE)
         return file_name
 
     def get_best_throughput(self):
@@ -305,7 +295,7 @@ class IperfTestUtility(object):
         except subprocess.CalledProcessError:
             pass
         self.dut.write("restart")
-        self.dut.expect("iperf>")
+        self.dut.expect_any("iperf>", "esp32>")
         self.dut.write("scan {}".format(self.ap_ssid))
         for _ in range(SCAN_RETRY_COUNT):
             try:
@@ -358,6 +348,12 @@ class IperfTestUtility(object):
             with open(PC_IPERF_TEMP_LOG_FILE, "w") as f:
                 if proto == "tcp":
                     self.dut.write("iperf -s -i 1 -t {}".format(TEST_TIME))
+                    # wait until DUT TCP server created
+                    try:
+                        self.dut.expect("iperf tcp server create successfully", timeout=1)
+                    except DUT.ExpectTimeout:
+                        # compatible with old iperf example binary
+                        pass
                     process = subprocess.Popen(["iperf", "-c", dut_ip,
                                                 "-t", str(TEST_TIME), "-f", "m"],
                                                stdout=f, stderr=f)
@@ -431,7 +427,7 @@ class IperfTestUtility(object):
         :return: True or False
         """
         self.dut.write("restart")
-        self.dut.expect("iperf>")
+        self.dut.expect_any("iperf>", "esp32>")
         for _ in range(WAIT_AP_POWER_ON_TIMEOUT // SCAN_TIMEOUT):
             try:
                 self.dut.write("scan {}".format(self.ap_ssid))
@@ -446,6 +442,91 @@ class IperfTestUtility(object):
         return ret
 
 
+class IperfTestUtilitySoftap(IperfTestUtility):
+    """ iperf test implementation """
+    def __init__(self, dut, softap_dut, config_name, test_result=None):
+        super(IperfTestUtility, self).__init__(dut, config_name, 'softap', '1234567890', None, None, test_result=None)
+        self.softap_dut = softap_dut
+        self.softap_ip = '192.168.4.1'
+
+    def setup(self):
+        """
+        setup iperf test:
+
+        1. kill current iperf process
+        2. reboot DUT (currently iperf is not very robust, need to reboot DUT)
+        3. scan to get AP RSSI
+        4. connect to AP
+        """
+        self.softap_dut.write("restart")
+        self.softap_dut.expect_any("iperf>", "esp32>", timeout=30)
+        self.softap_dut.write("ap {} {}".format(self.ap_ssid, self.ap_password))
+        self.dut.write("restart")
+        self.dut.expect_any("iperf>", "esp32>", timeout=30)
+        self.dut.write("scan {}".format(self.ap_ssid))
+        for _ in range(SCAN_RETRY_COUNT):
+            try:
+                rssi = int(self.dut.expect(re.compile(r"\[{}]\[rssi=(-\d+)]".format(self.ap_ssid)),
+                                           timeout=SCAN_TIMEOUT)[0])
+                break
+            except DUT.ExpectTimeout:
+                continue
+        else:
+            raise AssertionError("Failed to scan AP")
+        self.dut.write("sta {} {}".format(self.ap_ssid, self.ap_password))
+        dut_ip = self.dut.expect(re.compile(r"sta ip: ([\d.]+), mask: ([\d.]+), gw: ([\d.]+)"))[0]
+        return dut_ip, rssi
+
+    def _test_once(self, proto, direction):
+        """ do measure once for one type """
+        # connect and scan to get RSSI
+        dut_ip, rssi = self.setup()
+
+        assert direction in ["rx", "tx"]
+        assert proto in ["tcp", "udp"]
+
+        # run iperf test
+        if direction == "tx":
+            if proto == "tcp":
+                self.softap_dut.write("iperf -s -i 1 -t {}".format(TEST_TIME))
+                # wait until DUT TCP server created
+                try:
+                    self.softap_dut.expect("iperf tcp server create successfully", timeout=1)
+                except DUT.ExpectTimeout:
+                    # compatible with old iperf example binary
+                    pass
+                self.dut.write("iperf -c {} -i 1 -t {}".format(self.softap_ip, TEST_TIME))
+            else:
+                self.softap_dut.write("iperf -s -u -i 1 -t {}".format(TEST_TIME))
+                self.dut.write("iperf -c {} -u -i 1 -t {}".format(self.softap_ip, TEST_TIME))
+        else:
+            if proto == "tcp":
+                self.dut.write("iperf -s -i 1 -t {}".format(TEST_TIME))
+                # wait until DUT TCP server created
+                try:
+                    self.dut.expect("iperf tcp server create successfully", timeout=1)
+                except DUT.ExpectTimeout:
+                    # compatible with old iperf example binary
+                    pass
+                self.softap_dut.write("iperf -c {} -i 1 -t {}".format(dut_ip, TEST_TIME))
+            else:
+                self.dut.write("iperf -s -u -i 1 -t {}".format(TEST_TIME))
+                self.softap_dut.write("iperf -c {} -u -i 1 -t {}".format(dut_ip, TEST_TIME))
+        time.sleep(60)
+
+        if direction == "tx":
+            server_raw_data = self.dut.read()
+        else:
+            server_raw_data = self.softap_dut.read()
+        self.dut.write("iperf -a")
+        self.softap_dut.write("iperf -a")
+        self.dut.write("heap")
+        heap_size = self.dut.expect(re.compile(r"min heap size: (\d+)\D"))[0]
+
+        # return server raw data (for parsing test results) and RSSI
+        return server_raw_data, rssi, heap_size
+
+
 @ttfw_idf.idf_example_test(env_tag="Example_ShieldBox_Basic", category="stress")
 def test_wifi_throughput_with_different_configs(env, extra_data):
     """
@@ -477,7 +558,7 @@ def test_wifi_throughput_with_different_configs(env, extra_data):
         dut = env.get_dut("iperf", "examples/wifi/iperf", dut_class=ttfw_idf.ESP32DUT,
                           app_config_name=config_name)
         dut.start_app()
-        dut.expect("iperf>")
+        dut.expect_any("iperf>", "esp32>")
 
         # 3. run test for each required att value
         test_result[config_name] = {
@@ -533,7 +614,7 @@ def test_wifi_throughput_vs_rssi(env, extra_data):
     dut = env.get_dut("iperf", "examples/wifi/iperf", dut_class=ttfw_idf.ESP32DUT,
                       app_config_name=BEST_PERFORMANCE_CONFIG)
     dut.start_app()
-    dut.expect("iperf>")
+    dut.expect_any("iperf>", "esp32>")
 
     # 2. run test for each required att value
     for ap_info in ap_list:
@@ -557,7 +638,7 @@ def test_wifi_throughput_vs_rssi(env, extra_data):
     env.close_dut("iperf")
 
     # 4. generate report
-    report = TestReport.ThroughputVsRssiReport(os.path.join(env.log_path, "ThroughputVsRssiReport"),
+    report = TestReport.ThroughputVsRssiReport(os.path.join(env.log_path, "STAThroughputVsRssiReport"),
                                                test_result)
     report.generate_report()
 
@@ -580,7 +661,7 @@ def test_wifi_throughput_basic(env, extra_data):
     dut = env.get_dut("iperf", "examples/wifi/iperf", dut_class=ttfw_idf.ESP32DUT,
                       app_config_name=BEST_PERFORMANCE_CONFIG)
     dut.start_app()
-    dut.expect("iperf>")
+    dut.expect_any("iperf>", "esp32>")
 
     # 2. preparing
     test_result = {
@@ -615,7 +696,55 @@ def test_wifi_throughput_basic(env, extra_data):
     env.close_dut("iperf")
 
 
+@ttfw_idf.idf_example_test(env_tag="Example_ShieldBox2", category="stress")
+def test_softap_throughput_vs_rssi(env, extra_data):
+    """
+    steps: |
+      1. build with best performance config
+      2. switch on one router
+      3. set attenuator value from 0-60 for each router
+      4. test TCP tx rx and UDP tx rx throughput
+    """
+    att_port = env.get_variable("attenuator_port")
+
+    test_result = {
+        "tcp_tx": TestResult("tcp", "tx", BEST_PERFORMANCE_CONFIG),
+        "tcp_rx": TestResult("tcp", "rx", BEST_PERFORMANCE_CONFIG),
+        "udp_tx": TestResult("udp", "tx", BEST_PERFORMANCE_CONFIG),
+        "udp_rx": TestResult("udp", "rx", BEST_PERFORMANCE_CONFIG),
+    }
+
+    # 1. get DUT and download
+    softap_dut = env.get_dut("softap_iperf", "examples/wifi/iperf", dut_class=ttfw_idf.ESP32DUT,
+                             app_config_name=BEST_PERFORMANCE_CONFIG)
+    softap_dut.start_app()
+    softap_dut.expect_any("iperf>", "esp32>")
+
+    sta_dut = env.get_dut("sta_iperf", "examples/wifi/iperf", dut_class=ttfw_idf.ESP32DUT,
+                          app_config_name=BEST_PERFORMANCE_CONFIG)
+    sta_dut.start_app()
+    sta_dut.expect_any("iperf>", "esp32>")
+
+    # 2. run test for each required att value
+    test_utility = IperfTestUtilitySoftap(sta_dut, softap_dut, BEST_PERFORMANCE_CONFIG, test_result)
+
+    Attenuator.set_att(att_port, 0)
+
+    for atten_val in ATTEN_VALUE_LIST:
+        assert Attenuator.set_att(att_port, atten_val) is True
+        test_utility.run_all_cases(atten_val)
+
+    env.close_dut("softap_iperf")
+    env.close_dut("sta_iperf")
+
+    # 3. generate report
+    report = TestReport.ThroughputVsRssiReport(os.path.join(env.log_path, "SoftAPThroughputVsRssiReport"),
+                                               test_result)
+    report.generate_report()
+
+
 if __name__ == '__main__':
     test_wifi_throughput_basic(env_config_file="EnvConfig.yml")
     test_wifi_throughput_with_different_configs(env_config_file="EnvConfig.yml")
     test_wifi_throughput_vs_rssi(env_config_file="EnvConfig.yml")
+    test_softap_throughput_vs_rssi(env_config_file="EnvConfig.yml")

tools/ci/python_packages/idf_iperf_test_util/LineChart.py

@@ -12,18 +12,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import matplotlib
-# fix can't draw figure with docker
-matplotlib.use('Agg')
-import matplotlib.pyplot as plt  # noqa: E402 - matplotlib.use('Agg') need to be before this
+from collections import OrderedDict
+
+import pyecharts.options as opts
+from pyecharts.charts import Line
 
 
-# candidate colors
-LINE_STYLE_CANDIDATE = ['b-o', 'r-o', 'k-o', 'm-o', 'c-o', 'g-o', 'y-o',
-                        'b-s', 'r-s', 'k-s', 'm-s', 'c-s', 'g-s', 'y-s']
-
-
-def draw_line_chart(file_name, title, x_label, y_label, data_list):
+def draw_line_chart(file_name, title, x_label, y_label, data_series, range_list):
     """
     draw line chart and save to file.
 
@@ -31,20 +26,33 @@ def draw_line_chart(file_name, title, x_label, y_label, data_list):
     :param title: chart title
     :param x_label: x-axis label
     :param y_label: y-axis label
-    :param data_list: a list of line data.
-                      each line is a dict of ("x-axis": list, "y-axis": list, "label": string)
+    :param data_series: a dict {"name": data}. data is a dict.
+    :param range_list: a list of x-axis range
     """
 
-    plt.figure(figsize=(12, 6))
-    plt.grid(True)
-
-    for i, data in enumerate(data_list):
-        plt.plot(data["x-axis"], data["y-axis"], LINE_STYLE_CANDIDATE[i], label=data["label"])
-
-    plt.xlabel(x_label)
-    plt.ylabel(y_label)
-    plt.legend(fontsize=12)
-    plt.title(title)
-    plt.tight_layout(pad=3, w_pad=3, h_pad=3)
-    plt.savefig(file_name)
-    plt.close()
+    line = Line()
+    # echarts do not support minus number for x axis, convert to string
+    _range_list = [str(x) for x in range_list]
+    line.add_xaxis(_range_list)
+    for item in data_series:
+        _data = OrderedDict.fromkeys(_range_list, None)
+        for key in data_series[item]:
+            _data[str(key)] = data_series[item][key]
+        _data = list(_data.values())
+        try:
+            legend = item + " (max: {:.02f})".format(max([x for x in _data if x]))
+        except TypeError:
+            legend = item
+        line.add_yaxis(legend, _data, is_smooth=True, is_connect_nones=True,
+                       label_opts=opts.LabelOpts(is_show=False))
+    line.set_global_opts(
+        datazoom_opts=opts.DataZoomOpts(range_start=0, range_end=100),
+        title_opts=opts.TitleOpts(title=title, pos_left="center"),
+        legend_opts=opts.LegendOpts(pos_top="10%", pos_left="right", orient="vertical"),
+        tooltip_opts=opts.TooltipOpts(trigger="axis"),
+        xaxis_opts=opts.AxisOpts(type_="category", name=x_label, splitline_opts=opts.SplitLineOpts(is_show=True)),
+        yaxis_opts=opts.AxisOpts(type_="value", name=y_label,
+                                 axistick_opts=opts.AxisTickOpts(is_show=True),
+                                 splitline_opts=opts.SplitLineOpts(is_show=True)),
+    )
+    line.render(file_name)

tools/ci/python_packages/idf_iperf_test_util/TestReport.py

@@ -36,7 +36,7 @@ class ThroughputForConfigsReport(object):
             self.sdkconfigs[config_name] = self._parse_config_file(sdkconfig_files[config_name])
         if not os.path.exists(output_path):
             os.makedirs(output_path)
-        self.sort_order = self.sdkconfigs.keys()
+        self.sort_order = list(self.sdkconfigs.keys())
         self.sort_order.sort()
 
     @staticmethod
@@ -162,7 +162,7 @@ class ThroughputVsRssiReport(object):
         self.output_path = output_path
         self.raw_data_path = os.path.join(output_path, "raw_data")
         self.results = throughput_results
-        self.throughput_types = self.results.keys()
+        self.throughput_types = list(self.results.keys())
         self.throughput_types.sort()
         if not os.path.exists(self.raw_data_path):
             os.makedirs(self.raw_data_path)
@@ -229,7 +229,7 @@ class ThroughputVsRssiReport(object):
             result.draw_throughput_figure(self.raw_data_path, ap_ssid, "att")
             result.draw_rssi_vs_att_figure(self.raw_data_path, ap_ssid)
 
-            ret += "\r\n![throughput Vs RSSI]({})\r\n".format(os.path.join("raw_data", file_name))
+            ret += "\r\n[throughput Vs RSSI]({})\r\n".format(os.path.join("raw_data", file_name))
 
         return ret
 

tools/ci/python_packages/tiny_test_fw/Env.py

@@ -44,6 +44,7 @@ class Env(object):
     :keyword env_config_file: test env config file path
     :keyword test_name: test suite name, used when generate log folder name
     """
+    CURRENT_LOG_FOLDER = ""
 
     def __init__(self,
                  app=None,
@@ -59,6 +60,8 @@ class Env(object):
         if not os.path.exists(self.log_path):
             os.makedirs(self.log_path)
 
+        Env.CURRENT_LOG_FOLDER = self.log_path
+
         self.allocated_duts = dict()
         self.lock = threading.RLock()
 

tools/ci/python_packages/tiny_test_fw/Utility/__init__.py

@@ -1,7 +1,9 @@
 from __future__ import print_function
 import os.path
 import sys
+import time
 
+from .. import Env
 
 _COLOR_CODES = {
     "white": u'\033[0m',
@@ -19,6 +21,19 @@ _COLOR_CODES = {
 }
 
 
+def _get_log_file_name():
+    if Env.Env.CURRENT_LOG_FOLDER:
+        file_name = os.path.join(Env.Env.CURRENT_LOG_FOLDER, "console.log")
+    else:
+        raise OSError("env log folder does not exist, will not save to log file")
+    return file_name
+
+
+def format_timestamp():
+    ts = time.time()
+    return "{}:{}".format(time.strftime("%m-%d %H:%M:%S", time.localtime(ts)), str(ts % 1)[2:5])
+
+
 def console_log(data, color="white", end="\n"):
     """
     log data to console.
@@ -37,6 +52,13 @@ def console_log(data, color="white", end="\n"):
         # reset color to white for later logs
         print(_COLOR_CODES["white"] + u"\r")
     sys.stdout.flush()
+    log_data = "[{}] ".format(format_timestamp()) + data
+    try:
+        log_file = _get_log_file_name()
+        with open(log_file, "a+") as f:
+            f.write(log_data + end)
+    except OSError:
+        pass
 
 
 __LOADED_MODULES = dict()