Merge branch 'example/pytest_custom_bootloader' into 'master'

Example: pytest for examples/custom_bootloader

See merge request espressif/esp-idf!16206

.gitlab/ci/build.yml

@@ -39,6 +39,13 @@ build_examples_pytest_esp32:
   script:
     - python tools/ci/build_pytest_apps.py --all-pytest-apps --under-dir examples --target esp32 --size-info $SIZE_INFO_LOCATION -vv
 
+build_examples_pytest_esp32s2:
+  extends:
+    - .build_pytest_template
+    - .rules:build:example_test-esp32s2
+  script:
+    - python tools/ci/build_pytest_apps.py --all-pytest-apps --under-dir examples --target esp32s2 --size-info $SIZE_INFO_LOCATION -vv
+
 build_examples_pytest_esp32c3:
   extends:
     - .build_pytest_template

.gitlab/ci/target-test.yml

@@ -8,7 +8,7 @@
     reports:
       junit: XUNIT_RESULT.xml
   script:
-    - pytest $TEST_DIR -m $TARGET_MARKER -m $ENV_MARKER --junitxml=XUNIT_RESULT.xml
+    - pytest $TEST_DIR --target $TARGET -m $ENV_MARKER --junitxml=XUNIT_RESULT.xml
 
 .pytest_examples_dir_template:
   extends: .pytest_template
@@ -22,12 +22,25 @@ example_test_pytest_esp32_generic:
   needs:
     - build_examples_pytest_esp32
   variables:
-    TARGET_MARKER: esp32
+    TARGET: esp32
     ENV_MARKER: generic
   tags: # in gitlab 14.1 or later, we can use `parallel: matrix` with the `tags` keyword. https://docs.gitlab.com/ee/ci/jobs/job_control.html#run-a-matrix-of-parallel-trigger-jobs
     - ESP32
     - Example_GENERIC
 
+example_test_pytest_esp32s2_generic:
+  extends:
+    - .pytest_examples_dir_template
+    - .rules:test:example_test-esp32s2
+  needs:
+    - build_examples_pytest_esp32s2
+  variables:
+    TARGET: esp32s2
+    ENV_MARKER: generic
+  tags:
+    - ESP32S2
+    - Example_GENERIC
+
 example_test_pytest_esp32c3_generic:
   extends:
     - .pytest_examples_dir_template
@@ -35,7 +48,7 @@ example_test_pytest_esp32c3_generic:
   needs:
     - build_examples_pytest_esp32c3
   variables:
-    TARGET_MARKER: esp32c3
+    TARGET: esp32c3
     ENV_MARKER: generic
   tags:
     - ESP32C3

conftest.py

@@ -1,7 +1,7 @@
 # SPDX-FileCopyrightText: 2021 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
 
-# pylint: disable=W0621
+# pylint: disable=W0621  # redefined-outer-name
 
 # This file is a pytest root configuration file and provide the following functionalities:
 # 1. Defines a few fixtures that could be used under the whole project.
@@ -16,15 +16,20 @@
 import logging
 import os
 import sys
-from typing import List, Optional
+from typing import Callable, List, Optional
 
 import pytest
 from _pytest.config import Config
 from _pytest.fixtures import FixtureRequest
+from _pytest.nodes import Item
 from pytest_embedded.plugin import parse_configuration
+from pytest_embedded_idf.app import IdfApp
 
 
-def _is_target_marker(marker: str) -> bool:
+##################
+# Help Functions #
+##################
+def is_target_marker(marker: str) -> bool:
     if marker.startswith('esp32'):
         return True
 
@@ -34,12 +39,19 @@ def _is_target_marker(marker: str) -> bool:
     return False
 
 
+def format_case_id(target: str, config: str, case: str) -> str:
+    return f'{target}.{config}.{case}'
+
+
+############
+# Fixtures #
+############
 @pytest.fixture(scope='session')
 def target_markers(pytestconfig: Config) -> List[str]:
     res = []
     for item in pytestconfig.getini('markers'):
         marker = item.split(':')[0]
-        if _is_target_marker(marker):
+        if is_target_marker(marker):
             res.append(marker)
     return res
 
@@ -54,34 +66,9 @@ def env_markers(pytestconfig: Config) -> List[str]:
     return res
 
 
-@pytest.fixture(scope='session')
-def param_markers(pytestconfig: Config) -> List[str]:
-    res: List[str] = []
-    offset = -1
-    while True:
-        try:
-            offset = pytestconfig.invocation_params.args.index('-m', offset + 1)
-        except ValueError:
-            return res
-        res.append(pytestconfig.invocation_params.args[offset + 1])  # we want the marker after '-m'
-
-
 @pytest.fixture
-def target(request: FixtureRequest, target_markers: List[str], param_markers: List[str]) -> Optional[str]:
-    param_target_markers = [marker for marker in param_markers if marker in target_markers]
-    if len(param_target_markers) > 1:
-        raise ValueError('Please only specify one target marker at the same time')
-    elif len(param_target_markers) == 0:
-        target = None
-    else:
-        target = param_target_markers[0]
-
-    return getattr(request, 'param', None) or target
-
-
-@pytest.fixture
-def config(request: FixtureRequest) -> Optional[str]:
-    return getattr(request, 'param', None) or request.config.option.__dict__.get('config') or None
+def config(request: FixtureRequest) -> str:
+    return getattr(request, 'param', None) or request.config.getoption('config', 'default')  # type: ignore
 
 
 @pytest.fixture
@@ -129,3 +116,25 @@ def build_dir(request: FixtureRequest, app_path: str, target: Optional[str], con
     logging.error(
         f'no build dir valid. Please build the binary via "idf.py -B {recommend_place} build" and run pytest again')
     sys.exit(1)
+
+
+@pytest.fixture(autouse=True)
+def junit_properties(app: IdfApp, config: str, test_case_name: str,
+                     record_xml_attribute: Callable[[str, object], None]) -> None:
+    """
+    This fixture is autoused and will modify the junit report test case name to <target>.<config>.<case_name>
+    """
+    record_xml_attribute('name', format_case_id(app.target, config, test_case_name))
+
+
+##################
+# Hook functions #
+##################
+@pytest.hookimpl(trylast=True)
+def pytest_collection_modifyitems(config: Config, items: List[Item]) -> None:
+    target = config.getoption('target', None)
+    if not target:
+        return
+
+    # filter all the test cases with "--target"
+    items[:] = [item for item in items if target in [marker.name for marker in item.iter_markers()]]

examples/custom_bootloader/bootloader_hooks/example_test.py

@@ -1,18 +0,0 @@
-from __future__ import print_function
-
-import ttfw_idf
-
-
-@ttfw_idf.idf_example_test(env_tag='Example_GENERIC', target=['esp32', 'esp32s2', 'esp32c3'])
-def test_custom_bootloader_hooks_example(env, _):  # type: ignore
-    # Test with default build configurations
-    dut = env.get_dut('main', 'examples/custom_bootloader/bootloader_hooks')
-    dut.start_app()
-
-    # Expect to read both hooks messages
-    dut.expect('This hook is called BEFORE bootloader initialization')
-    dut.expect('This hook is called AFTER bootloader initialization')
-
-
-if __name__ == '__main__':
-    test_custom_bootloader_hooks_example()

examples/custom_bootloader/bootloader_hooks/pytest_custom_bootloader_hooks.py

@@ -0,0 +1,15 @@
+# SPDX-FileCopyrightText: 2021 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: CC0-1.0
+
+import pytest
+from pytest_embedded.dut import Dut
+
+
+@pytest.mark.esp32
+@pytest.mark.esp32s2
+@pytest.mark.esp32c3
+@pytest.mark.generic
+def test_custom_bootloader_hooks_example(dut: Dut) -> None:
+    # Expect to read both hooks messages
+    dut.expect_exact('This hook is called BEFORE bootloader initialization')
+    dut.expect_exact('This hook is called AFTER bootloader initialization')

examples/custom_bootloader/bootloader_override/example_test.py

@@ -1,23 +0,0 @@
-from __future__ import print_function
-
-import ttfw_idf
-
-
-@ttfw_idf.idf_example_test(env_tag='Example_GENERIC', target=['esp32', 'esp32s2', 'esp32c3'])
-def test_custom_bootloader_impl_example(env, _):  # type: ignore
-    # Test with default build configurations
-    dut = env.get_dut('main', 'examples/custom_bootloader/bootloader_override')
-    dut.start_app()
-
-    # Expect to read a message from the custom bootloader
-    # This message is defined in the Kconfig file, retrieve it while deleting
-    # leading and trailing quotes (")
-    welcome_message = dut.app.get_sdkconfig()['CONFIG_EXAMPLE_BOOTLOADER_WELCOME_MESSAGE'].strip("\"")
-    dut.expect(welcome_message)
-
-    # Expect to read a message from the user application
-    dut.expect('Application started!')
-
-
-if __name__ == '__main__':
-    test_custom_bootloader_impl_example()

examples/custom_bootloader/bootloader_override/pytest_custom_bootloader_override.py

@@ -0,0 +1,21 @@
+# SPDX-FileCopyrightText: 2021 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: CC0-1.0
+
+import pytest
+from pytest_embedded.dut import Dut
+from pytest_embedded_idf.app import IdfApp
+
+
+@pytest.mark.esp32
+@pytest.mark.esp32s2
+@pytest.mark.esp32c3
+@pytest.mark.generic
+def test_custom_bootloader_impl_example(app: IdfApp, dut: Dut) -> None:
+    # Expect to read a message from the custom bootloader
+    # This message is defined in the Kconfig file, retrieve it while deleting
+    # leading and trailing quotes (")
+    welcome_message = app.sdkconfig['EXAMPLE_BOOTLOADER_WELCOME_MESSAGE']
+    dut.expect_exact(welcome_message)
+
+    # Expect to read a message from the user application
+    dut.expect_exact('Application started!')

pytest.ini

@@ -2,10 +2,15 @@
 # only the files with prefix `pytest_` would be recognized as pytest test scripts.
 python_files = pytest_*.py
 
-addopts = --embedded-services esp,idf
+# ignore PytestExperimentalApiWarning for record_xml_attribute
+addopts =
+  --embedded-services esp,idf
+  -W ignore::_pytest.warning_types.PytestExperimentalApiWarning
 
 markers =
   esp32: support esp32 target
+  esp32s2: support esp32s2 target
+  esp32s3: support esp32s3 target
   esp32c3: support esp32c3 target
   generic: tests should be run on generic runners
 
@@ -15,3 +20,9 @@ log_cli = True
 log_cli_level = INFO
 log_cli_format = %(asctime)s %(levelname)s %(message)s
 log_cli_date_format = %Y-%m-%d %H:%M:%S
+
+# junit related
+junit_family = xunit1
+## log all to `system-out` when case fail
+junit_logging = log
+junit_log_passing_tests = False