From 4a7315b1b01941ff8f63a435665b969d9a42c6a9 Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Sun, 24 Apr 2022 17:09:56 +0800 Subject: [PATCH 1/5] ci: improve import path --- .../provisioning/wifi_prov_mgr/pytest_wifi_prov_mgr.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/examples/provisioning/wifi_prov_mgr/pytest_wifi_prov_mgr.py b/examples/provisioning/wifi_prov_mgr/pytest_wifi_prov_mgr.py index 71f63436d8..4d5ea37a9c 100644 --- a/examples/provisioning/wifi_prov_mgr/pytest_wifi_prov_mgr.py +++ b/examples/provisioning/wifi_prov_mgr/pytest_wifi_prov_mgr.py @@ -6,8 +6,15 @@ from __future__ import print_function import logging +import os +import sys + +try: + import esp_prov +except ImportError: + sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', '..', 'tools')) + import esp_prov -import esp_prov import pytest from pytest_embedded import Dut From 5c997bd5dd52ecc7e57098805ac36119777c9be9 Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Mon, 25 Apr 2022 17:26:29 +0800 Subject: [PATCH 2/5] ci(pytest): upgrade to 0.7.0 --- .gitlab-ci.yml | 2 +- conftest.py | 8 ++++---- tools/test_apps/system/panic/conftest.py | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 69136e530c..4401e10dd8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -73,7 +73,7 @@ variables: TEST_ENV_CONFIG_REPO: "https://gitlab-ci-token:${BOT_TOKEN}@${CI_SERVER_HOST}:${CI_SERVER_PORT}/qa/ci-test-runner-configs.git" CI_AUTO_TEST_SCRIPT_REPO_URL: "https://gitlab-ci-token:${BOT_TOKEN}@${CI_SERVER_HOST}:${CI_SERVER_PORT}/qa/auto_test_script.git" CI_AUTO_TEST_SCRIPT_REPO_BRANCH: "ci/v4.1" - PYTEST_EMBEDDED_VERSION: "0.6.0" + PYTEST_EMBEDDED_VERSION: "0.7.0" # cache python dependencies PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip" diff --git a/conftest.py b/conftest.py index 374f676b48..b2730058de 100644 --- a/conftest.py +++ b/conftest.py @@ -29,7 +29,7 @@ from _pytest.python import Function from _pytest.reports import TestReport from _pytest.runner import CallInfo from _pytest.terminal import TerminalReporter -from pytest_embedded.plugin import apply_count, parse_configuration +from pytest_embedded.plugin import multi_dut_argument, multi_dut_fixture from pytest_embedded.utils import find_by_suffix SUPPORTED_TARGETS = ['esp32', 'esp32s2', 'esp32c3', 'esp32s3', 'esp32c2'] @@ -75,7 +75,7 @@ def session_tempdir() -> str: @pytest.fixture -@parse_configuration +@multi_dut_argument def config(request: FixtureRequest) -> str: return getattr(request, 'param', None) or DEFAULT_SDKCONFIG @@ -91,7 +91,7 @@ def test_case_name(request: FixtureRequest, target: str, config: str) -> str: @pytest.fixture -@apply_count +@multi_dut_fixture def build_dir(app_path: str, target: Optional[str], config: Optional[str]) -> str: """ Check local build dir with the following priority: @@ -137,7 +137,7 @@ def build_dir(app_path: str, target: Optional[str], config: Optional[str]) -> st @pytest.fixture(autouse=True) -@apply_count +@multi_dut_fixture def junit_properties( test_case_name: str, record_xml_attribute: Callable[[str, object], None] ) -> None: diff --git a/tools/test_apps/system/panic/conftest.py b/tools/test_apps/system/panic/conftest.py index f1c0c6f223..fc5b0445ae 100644 --- a/tools/test_apps/system/panic/conftest.py +++ b/tools/test_apps/system/panic/conftest.py @@ -138,7 +138,7 @@ class PanicTestDut(IdfDut): """Extract the core dump from flash, run espcoredump on it""" coredump_file_name = os.path.join(self.logdir, 'coredump_data.bin') logging.info('Writing flash binary core dump to %s', coredump_file_name) - self.serial.dump_flash(coredump_file_name, partition='coredump') + self.serial.dump_flash(partition='coredump', output=coredump_file_name) output_file_name = os.path.join(self.logdir, 'coredump_flash_result.txt') self._call_espcoredump( From 511ccdcb708f7f49097956865e29ca8a3fd2750a Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Fri, 29 Apr 2022 12:19:32 +0800 Subject: [PATCH 3/5] ci(pytest): support multi-dut different app --- tools/ci/build_pytest_apps.py | 9 +-- tools/ci/idf_ci_utils.py | 113 ++++++++++++++++++++++++---------- 2 files changed, 87 insertions(+), 35 deletions(-) diff --git a/tools/ci/build_pytest_apps.py b/tools/ci/build_pytest_apps.py index 85e43a26bd..c8ed196f7d 100644 --- a/tools/ci/build_pytest_apps.py +++ b/tools/ci/build_pytest_apps.py @@ -13,7 +13,7 @@ import sys from collections import defaultdict from typing import List -from idf_ci_utils import IDF_PATH, get_pytest_cases +from idf_ci_utils import IDF_PATH, PytestCase, get_pytest_cases try: from build_apps import build_apps @@ -28,15 +28,16 @@ except ImportError: def main(args: argparse.Namespace) -> None: - pytest_cases = [] + pytest_cases: List[PytestCase] = [] for path in args.paths: pytest_cases += get_pytest_cases(path, args.target) paths = set() app_configs = defaultdict(set) for case in pytest_cases: - paths.add(case.app_path) - app_configs[case.app_path].add(case.config) + for app in case.apps: + paths.add(app.path) + app_configs[app.path].add(app.config) app_dirs = list(paths) if not app_dirs: diff --git a/tools/ci/idf_ci_utils.py b/tools/ci/idf_ci_utils.py index 9866a5c23d..916a27843e 100644 --- a/tools/ci/idf_ci_utils.py +++ b/tools/ci/idf_ci_utils.py @@ -1,19 +1,22 @@ # internal use only for CI # some CI related util functions # -# SPDX-FileCopyrightText: 2020-2021 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2020-2022 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 # + import io import logging import os import subprocess import sys from contextlib import redirect_stdout -from typing import TYPE_CHECKING, List +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, List, Set if TYPE_CHECKING: - from _pytest.nodes import Function + from _pytest.python import Function + IDF_PATH = os.path.abspath( os.getenv('IDF_PATH', os.path.join(os.path.dirname(__file__), '..', '..')) @@ -113,39 +116,82 @@ def is_in_directory(file_path: str, folder: str) -> bool: return os.path.realpath(file_path).startswith(os.path.realpath(folder) + os.sep) -class PytestCase: - def __init__(self, test_path: str, target: str, config: str, case: str): - self.app_path = os.path.dirname(test_path) - self.test_path = test_path - self.target = target - self.config = config - self.case = case +def to_list(s: Any) -> List[Any]: + if isinstance(s, set) or isinstance(s, tuple): + return list(s) + elif isinstance(s, list): + return s + else: + return [s] - def __repr__(self) -> str: - return f'{self.test_path}: {self.target}.{self.config}.{self.case}' + +@dataclass +class PytestApp: + path: str + target: str + config: str + + def __hash__(self) -> int: + return hash((self.path, self.target, self.config)) + + +@dataclass +class PytestCase: + path: str + name: str + apps: Set[PytestApp] + + def __hash__(self) -> int: + return hash((self.path, self.name, self.apps)) class PytestCollectPlugin: def __init__(self, target: str) -> None: self.target = target - self.nodes: List[PytestCase] = [] + self.cases: List[PytestCase] = [] + + @staticmethod + def get_param(item: 'Function', key: str, default: Any = None) -> Any: + if not hasattr(item, 'callspec'): + raise ValueError(f'Function {item} does not have params') + + return item.callspec.params.get(key, default) or default def pytest_collection_modifyitems(self, items: List['Function']) -> None: + from pytest_embedded.plugin import parse_multi_dut_args + for item in items: - try: - file_path = str(item.path) - except AttributeError: - # pytest 6.x - file_path = item.fspath - - target = self.target - if hasattr(item, 'callspec'): - config = item.callspec.params.get('config', 'default') - else: - config = 'default' + count = 1 + case_path = str(item.path) case_name = item.originalname + target = self.target + # funcargs is not calculated while collection + if hasattr(item, 'callspec'): + count = item.callspec.params.get('count', 1) + app_paths = to_list( + parse_multi_dut_args( + count, + self.get_param(item, 'app_path', os.path.dirname(case_path)), + ) + ) + configs = to_list( + parse_multi_dut_args( + count, self.get_param(item, 'config', 'default') + ) + ) + targets = to_list( + parse_multi_dut_args(count, self.get_param(item, 'target', target)) + ) + else: + app_paths = [os.path.dirname(case_path)] + configs = ['default'] + targets = [target] - self.nodes.append(PytestCase(file_path, target, config, case_name)) + case_apps = set() + for i in range(count): + case_apps.add(PytestApp(app_paths[i], targets[i], configs[i])) + + self.cases.append(PytestCase(case_path, case_name, case_apps)) def get_pytest_cases(folder: str, target: str) -> List[PytestCase]: @@ -156,18 +202,23 @@ def get_pytest_cases(folder: str, target: str) -> List[PytestCase]: with io.StringIO() as buf: with redirect_stdout(buf): - res = pytest.main(['--collect-only', folder, '-q', '--target', target], plugins=[collector]) + res = pytest.main( + ['--collect-only', folder, '-q', '--target', target], + plugins=[collector], + ) if res.value != ExitCode.OK: if res.value == ExitCode.NO_TESTS_COLLECTED: - print(f'WARNING: no pytest app found for target {target} under folder {folder}') + print( + f'WARNING: no pytest app found for target {target} under folder {folder}' + ) else: print(buf.getvalue()) raise RuntimeError('pytest collection failed') - return collector.nodes + return collector.cases -def get_pytest_app_paths(folder: str, target: str) -> List[str]: - nodes = get_pytest_cases(folder, target) +def get_pytest_app_paths(folder: str, target: str) -> Set[str]: + cases = get_pytest_cases(folder, target) - return list({node.app_path for node in nodes}) + return set({app.path for case in cases for app in case.apps}) From 52b5a8348ef44787e4aaa88ce0eb5470f07dae6b Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Sat, 7 May 2022 12:18:56 +0800 Subject: [PATCH 4/5] test: add pytest_wifi_getting_started script --- .gitlab/ci/target-test.yml | 10 +++++ .../pytest_wifi_getting_started.py | 42 +++++++++++++++++++ pytest.ini | 3 ++ tools/ci/idf_ci_utils.py | 9 ++-- 4 files changed, 60 insertions(+), 4 deletions(-) create mode 100644 examples/wifi/getting_started/pytest_wifi_getting_started.py diff --git a/.gitlab/ci/target-test.yml b/.gitlab/ci/target-test.yml index ae09296711..dbc0b43183 100644 --- a/.gitlab/ci/target-test.yml +++ b/.gitlab/ci/target-test.yml @@ -111,6 +111,16 @@ example_test_pytest_esp32_flash_encryption: TARGET: ESP32 ENV_MARKER: flash_encryption +example_test_pytest_esp32_multi_dut_generic: + extends: + - .pytest_examples_dir_template + - .rules:test:example_test-esp32 + needs: + - build_pytest_examples_esp32 + variables: + TARGET: ESP32 + ENV_MARKER: multi_dut_generic + example_test_pytest_esp32c3_flash_encryption: extends: - .pytest_examples_dir_template diff --git a/examples/wifi/getting_started/pytest_wifi_getting_started.py b/examples/wifi/getting_started/pytest_wifi_getting_started.py new file mode 100644 index 0000000000..a85e0a9aa8 --- /dev/null +++ b/examples/wifi/getting_started/pytest_wifi_getting_started.py @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: CC0-1.0 + +import os.path +from typing import Tuple + +import pytest +from pytest_embedded_idf.dut import IdfDut + +# @pytest.mark.supported_targets +# This test should support all targets, even between different target types +# For now our CI only support multi dut with esp32 +# If you want to enable different target type, please use the following param +# @pytest.mark.parametrize( +# 'count, app_path, target', [ +# (2, +# f'{os.path.join(os.path.dirname(__file__), "softAP")}|{os.path.join(os.path.dirname(__file__), "station")}', +# 'esp32|esp32s2'), +# ], +# indirect=True, +# ) + + +@pytest.mark.esp32 +@pytest.mark.multi_dut_generic +@pytest.mark.parametrize( + 'count, app_path', [ + (2, + f'{os.path.join(os.path.dirname(__file__), "softAP")}|{os.path.join(os.path.dirname(__file__), "station")}'), + ], indirect=True +) +def test_wifi_getting_started(dut: Tuple[IdfDut, IdfDut]) -> None: + softap = dut[0] + station = dut[1] + + ssid = 'myssid' + password = 'mypassword' + tag = 'wifi station' + + station.expect(f'{tag}: got ip:', timeout=60) + station.expect(f'{tag}: connected to ap SSID:{ssid} password:{password}', timeout=60) + softap.expect('station .+ join, AID=', timeout=60) diff --git a/pytest.ini b/pytest.ini index a386445b25..50c9554f53 100644 --- a/pytest.ini +++ b/pytest.ini @@ -35,6 +35,9 @@ markers = flash_encryption: Flash Encryption runners ir_transceiver: runners with a pair of IR transmitter and receiver + ## multi-dut markers + multi_dut_generic: tests should be run on generic runners, at least have two duts connected. + # log related log_cli = True log_cli_level = INFO diff --git a/tools/ci/idf_ci_utils.py b/tools/ci/idf_ci_utils.py index 916a27843e..460cf640a0 100644 --- a/tools/ci/idf_ci_utils.py +++ b/tools/ci/idf_ci_utils.py @@ -117,12 +117,13 @@ def is_in_directory(file_path: str, folder: str) -> bool: def to_list(s: Any) -> List[Any]: - if isinstance(s, set) or isinstance(s, tuple): + if isinstance(s, (set, tuple)): return list(s) - elif isinstance(s, list): + + if isinstance(s, list): return s - else: - return [s] + + return [s] @dataclass From 3697cd044060b4a0031cdb191fc3e7819a1c4daa Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Sat, 7 May 2022 12:54:02 +0800 Subject: [PATCH 5/5] docs: add multi dut chapter --- .../contribute/esp-idf-tests-with-pytest.rst | 95 +++++++++++++++++-- 1 file changed, 87 insertions(+), 8 deletions(-) diff --git a/docs/en/contribute/esp-idf-tests-with-pytest.rst b/docs/en/contribute/esp-idf-tests-with-pytest.rst index 92eb879202..621f4d878c 100644 --- a/docs/en/contribute/esp-idf-tests-with-pytest.rst +++ b/docs/en/contribute/esp-idf-tests-with-pytest.rst @@ -19,6 +19,11 @@ In ESP-IDF, we use the following plugins by default: All the introduced concepts and usages are based on the default behavior in ESP-IDF. Not all of them are available in vanilla pytest. +Installation +------------ + +``$ pip install -U pytest-embedded-serial-esp~=0.7.0 pytest-embedded-idf~=0.7.0`` + Basic Concepts -------------- @@ -95,12 +100,12 @@ Pytest Execution Process 1. Get all the python files with the prefix ``pytest_`` 2. Get all the test functions with the prefix ``test_`` - 3. Apply the `params `__, duplicate the test functions. - 4. Filter the test cases with CLI options. Introduced detail usages `here <#filter-the-test-cases>`__ + 3. Apply the `params `__, and duplicate the test functions. + 4. Filter the test cases with CLI options. Introduced detailed usages `here <#filter-the-test-cases>`__ 3. Test Running Phase - 1. Construct the `fixtures `__. In ESP-IDF, the common fixtures are initialized with this order: + 1. Construct the `fixtures `__. In ESP-IDF, the common fixtures are initialized in this order: 1. ``pexpect_proc``: `pexpect `__ instance @@ -116,7 +121,7 @@ Pytest Execution Process 2. Run the real test function - 3. Deconstruct the fixtures with this order: + 3. Deconstruct the fixtures in this order: 1. ``dut`` @@ -160,7 +165,7 @@ This code example is taken from :idf_file:`pytest_console_basic.py `__. @@ -198,7 +203,7 @@ You can use ``pytest.mark.parametrize`` with “config” to apply the same test 'nohistory', # <-- run with app built by sdkconfig.ci.nohistory ], indirect=True) # <-- `indirect=True` is required -Overall, this test case would be duplicated to 4 test functions: +Overall, this test function would be replicated to 4 test cases: - esp32.history.test_console_advanced - esp32.nohistory.test_console_advanced @@ -208,6 +213,80 @@ Overall, this test case would be duplicated to 4 test functions: Advanced Examples ~~~~~~~~~~~~~~~~~ +Multi Dut Tests with the Same App +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +This code example is taken from :idf_file:`pytest_usb_host.py `. + +.. code:: python + + @pytest.mark.esp32s2 + @pytest.mark.esp32s3 + @pytest.mark.usb_host + @pytest.mark.parametrize('count', [ + 2, + ], indirect=True) + def test_usb_host(dut: Tuple[IdfDut, IdfDut]) -> None: + device = dut[0] # <-- assume the first dut is the device + host = dut[1] # <-- and the second dut is the host + ... + +After setting the param ``count`` to 2, all these fixtures are changed into tuples. + +Multi Dut Tests with Different Apps +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +This code example is taken from :idf_file:`pytest_wifi_getting_started.py `. + +.. code:: python + + @pytest.mark.esp32 + @pytest.mark.multi_dut_generic + @pytest.mark.parametrize( + 'count, app_path', [ + (2, + f'{os.path.join(os.path.dirname(__file__), "softAP")}|{os.path.join(os.path.dirname(__file__), "station")}'), + ], indirect=True + ) + def test_wifi_getting_started(dut: Tuple[IdfDut, IdfDut]) -> None: + softap = dut[0] + station = dut[1] + ... + +Here the first dut was flashed with the app :idf_file:`softap `, and the second dut was flashed with the app :idf_file:`station `. + +.. note:: + + Here the ``app_path`` should be set with absolute path. the ``__file__`` macro in python would return the absolute path of the test script itself. + +Multi Dut Tests with Different Apps, and Targets +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +This code example is taken from :idf_file:`pytest_wifi_getting_started.py `. As the comment says, for now it's not running in the ESP-IDF CI. + +.. code:: python + + @pytest.mark.parametrize( + 'count, app_path, target', [ + (2, + f'{os.path.join(os.path.dirname(__file__), "softAP")}|{os.path.join(os.path.dirname(__file__), "station")}', + 'esp32|esp32s2'), + (2, + f'{os.path.join(os.path.dirname(__file__), "softAP")}|{os.path.join(os.path.dirname(__file__), "station")}', + 'esp32s2|esp32'), + ], + indirect=True, + ) + def test_wifi_getting_started(dut: Tuple[IdfDut, IdfDut]) -> None: + softap = dut[0] + station = dut[1] + ... + +Overall, this test function would be replicated to 2 test cases: + +- softap with esp32 target, and station with esp32s2 target +- softap with esp32s2 target, and station with esp32 target + Support different targets with different sdkconfig files ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -278,7 +357,7 @@ Mark Known Failure Cases Sometimes a test couldn't pass for the following reasons: - Has a bug -- Success ratio too low because of environment issue, such as network issue. Retry couldn't help +- The success ratio is too low because of environment issue, such as network issue. Retry couldn't help Now you may mark this test case with marker `xfail `__ with a user-friendly readable reason. @@ -397,7 +476,7 @@ You can call pytest with ``--junitxml `` to generate the JUnit report. Skip Auto Flash Binary ~~~~~~~~~~~~~~~~~~~~~~ -Skipping auto-flash binary everytime would be useful when you're debugging your test script. +Skipping auto-flash binary every time would be useful when you're debugging your test script. You can call pytest with ``--skip-autoflash y`` to achieve it.