ci: add qemu example

.gitlab-ci.yml

@@ -62,6 +62,7 @@ variables:
   AFL_FUZZER_TEST_IMAGE: "$CI_DOCKER_REGISTRY/afl-fuzzer-test-v5.0:2-1"
   CLANG_STATIC_ANALYSIS_IMAGE: "${CI_DOCKER_REGISTRY}/clang-static-analysis-v5.0:2-1"
   TARGET_TEST_ENV_IMAGE: "$CI_DOCKER_REGISTRY/target-test-env-v5.0:2"
+  QEMU_IMAGE: "${CI_DOCKER_REGISTRY}/qemu-v5.0:2-20210826"
 
   SONARQUBE_SCANNER_IMAGE: "${CI_DOCKER_REGISTRY}/sonarqube-scanner:3"
   LINUX_SHELL_IMAGE: "${CI_DOCKER_REGISTRY}/linux-shells-v5.0:2"
@@ -209,13 +210,15 @@ before_script:
     - fetch_submodules
     - *download_test_python_contraint_file
     - $IDF_PATH/tools/idf_tools.py install-python-env
+    # TODO: remove this, IDFCI-1207
+    - pip install esptool -c ~/.espressif/${CI_PYTHON_CONSTRAINT_FILE}
     - pip install
       "pytest-embedded-serial-esp~=$PYTEST_EMBEDDED_VERSION"
       "pytest-embedded-idf~=$PYTEST_EMBEDDED_VERSION"
+      "pytest-embedded-qemu~=$PYTEST_EMBEDDED_VERSION"
       pytest-rerunfailures
       scapy
       google-api-python-client
-    - cd $IDF_PATH
     - export EXTRA_CFLAGS=${PEDANTIC_CFLAGS}
     - export EXTRA_CXXFLAGS=${PEDANTIC_CXXFLAGS}
 

.gitlab/ci/host-test.yml

@@ -434,3 +434,12 @@ test_gen_soc_caps_kconfig:
   script:
     - cd ${IDF_PATH}/tools/gen_soc_caps_kconfig/
     - ./test/test_gen_soc_caps_kconfig.py
+
+test_pytest_qemu:
+  extends:
+    - .host_test_template
+    - .before_script_pytest
+  image: $QEMU_IMAGE
+  script:
+    - run_cmd python tools/ci/build_pytest_apps.py . --target esp32 -m qemu -vv
+    - pytest --target esp32 -m qemu --embedded-services idf,qemu

conftest.py

@@ -31,6 +31,7 @@ from _pytest.runner import CallInfo
 from _pytest.terminal import TerminalReporter
 from pytest_embedded.plugin import multi_dut_argument, multi_dut_fixture
 from pytest_embedded.utils import find_by_suffix
+from pytest_embedded_idf.dut import IdfDut
 
 SUPPORTED_TARGETS = ['esp32', 'esp32s2', 'esp32c3', 'esp32s3', 'esp32c2']
 PREVIEW_TARGETS = ['linux', 'esp32h2']
@@ -74,6 +75,23 @@ def session_tempdir() -> str:
     return _TEST_SESSION_TMPDIR
 
 
+@pytest.fixture()
+def log_minimum_free_heap_size(dut: IdfDut, config: str) -> Callable[..., None]:
+    def real_func() -> None:
+        res = dut.expect(r'Minimum free heap size: (\d+) bytes')
+        logging.info('\n------ heap size info ------\n'
+                     '[app_name] {}\n'
+                     '[config_name] {}\n'
+                     '[target] {}\n'
+                     '[minimum_free_heap_size] {} Bytes\n'
+                     '------ heap size end ------'.format(os.path.basename(dut.app.app_path),
+                                                          config,
+                                                          dut.target,
+                                                          res.group(1).decode('utf8')))
+
+    return real_func
+
+
 @pytest.fixture
 @multi_dut_argument
 def config(request: FixtureRequest) -> str:

examples/get-started/hello_world/README.md

@@ -24,12 +24,10 @@ Below is short explanation of remaining files in the project folder.
 
 ```
 ├── CMakeLists.txt
-├── example_test.py            Python script used for automated example testing
+├── pytest_hello_world.py      Python script used for automated testing
 ├── main
-│   ├── CMakeLists.txt
+│   ├── CMakeLists.txt
-│   ├── component.mk           Component make file
+│   └── hello_world_main.c
-│   └── hello_world_main.c
-├── Makefile                   Makefile used by legacy GNU Make
 └── README.md                  This is the file you are currently reading
 ```
 

examples/get-started/hello_world/example_test.py

@@ -1,20 +0,0 @@
-#!/usr/bin/env python
-
-from __future__ import division, print_function, unicode_literals
-
-import ttfw_idf
-
-
-@ttfw_idf.idf_example_test(env_tag='Example_GENERIC', target=['esp32', 'esp32s2', 'esp32c3'], ci_target=['esp32'])
-def test_examples_hello_world(env, extra_data):
-    app_name = 'hello_world'
-    dut = env.get_dut(app_name, 'examples/get-started/hello_world')
-    dut.start_app()
-    res = dut.expect(ttfw_idf.MINIMUM_FREE_HEAP_SIZE_RE)
-    if not res:
-        raise ValueError('Maximum heap size info not found')
-    ttfw_idf.print_heap_size(app_name, dut.app.config_name, dut.TARGET, res[0])
-
-
-if __name__ == '__main__':
-    test_examples_hello_world()

examples/get-started/hello_world/pytest_hello_world.py

@@ -0,0 +1,22 @@
+# SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: CC0-1.0
+
+from typing import Callable
+
+import pytest
+from pytest_embedded_idf.dut import IdfDut
+from pytest_embedded_qemu.dut import QemuDut
+
+
+@pytest.mark.supported_targets
+@pytest.mark.generic
+def test_hello_world(dut: IdfDut, log_minimum_free_heap_size: Callable[..., None]) -> None:
+    dut.expect('Hello world!')
+    log_minimum_free_heap_size()
+
+
+@pytest.mark.esp32  # we only support qemu on esp32 for now
+@pytest.mark.host_test
+@pytest.mark.qemu
+def test_hello_world_host(dut: QemuDut) -> None:
+    dut.expect('Hello world!')

pytest.ini

@@ -7,14 +7,16 @@ python_files = pytest_*.py
 addopts =
   -s
   --embedded-services esp,idf
-  -W ignore::_pytest.warning_types.PytestExperimentalApiWarning
   --tb short
 
 # ignore DeprecationWarning
 filterwarnings =
-  ignore:Call to deprecated create function (.*)\(\):DeprecationWarning
+  ignore::DeprecationWarning:matplotlib.*:
+  ignore::DeprecationWarning:google.protobuf.*:
+  ignore::_pytest.warning_types.PytestExperimentalApiWarning
 
 markers =
+  # target markers
   esp32: support esp32 target
   esp32s2: support esp32s2 target
   esp32s3: support esp32s3 target
@@ -36,9 +38,13 @@ markers =
   ir_transceiver: runners with a pair of IR transmitter and receiver
   wifi: wifi runner
 
-  ## multi-dut markers
+  # multi-dut markers
   multi_dut_generic: tests should be run on generic runners, at least have two duts connected.
 
+  # host_test markers
+  host_test: tests which shouldn't be built at the build stage, and instead built in host_test stage.
+  qemu: build and test using qemu-system-xtensa, not real target.
+
 # log related
 log_cli = True
 log_cli_level = INFO

tools/ci/build_pytest_apps.py

@@ -30,7 +30,7 @@ except ImportError:
 def main(args: argparse.Namespace) -> None:
     pytest_cases: List[PytestCase] = []
     for path in args.paths:
-        pytest_cases += get_pytest_cases(path, args.target)
+        pytest_cases += get_pytest_cases(path, args.target, args.marker_expr)
 
     paths = set()
     app_configs = defaultdict(set)
@@ -94,7 +94,15 @@ if __name__ == '__main__':
     parser = argparse.ArgumentParser(
         description='Build all the pytest apps under specified paths. Will auto remove those non-test apps binaries'
     )
-    parser.add_argument('--target', required=True, help='Build apps for given target.')
+    parser.add_argument(
+        '-t', '--target', required=True, help='Build apps for given target.'
+    )
+    parser.add_argument(
+        '-m',
+        '--marker-expr',
+        default='not host_test',  # host_test apps would be built and tested under the same job
+        help='only build tests matching given mark expression. For example: -m "host_test and generic".',
+    )
     parser.add_argument(
         '--config',
         default=['sdkconfig.ci=default', 'sdkconfig.ci.*=', '=default'],

tools/ci/idf_ci_utils.py

@@ -12,12 +12,11 @@ import subprocess
 import sys
 from contextlib import redirect_stdout
 from dataclasses import dataclass
-from typing import TYPE_CHECKING, Any, List, Set
+from typing import TYPE_CHECKING, Any, List, Optional, Set
 
 if TYPE_CHECKING:
     from _pytest.python import Function
 
-
 IDF_PATH = os.path.abspath(
     os.getenv('IDF_PATH', os.path.join(os.path.dirname(__file__), '..', '..'))
 )
@@ -158,7 +157,7 @@ class PytestCollectPlugin:
 
         return item.callspec.params.get(key, default) or default
 
-    def pytest_collection_modifyitems(self, items: List['Function']) -> None:
+    def pytest_report_collectionfinish(self, items: List['Function']) -> None:
         from pytest_embedded.plugin import parse_multi_dut_args
 
         for item in items:
@@ -195,17 +194,22 @@ class PytestCollectPlugin:
             self.cases.append(PytestCase(case_path, case_name, case_apps))
 
 
-def get_pytest_cases(folder: str, target: str) -> List[PytestCase]:
+def get_pytest_cases(
+    folder: str, target: str, marker_expr: Optional[str] = None
+) -> List[PytestCase]:
     import pytest
     from _pytest.config import ExitCode
 
     collector = PytestCollectPlugin(target)
+    if marker_expr:
+        marker_expr = f'{target} and ({marker_expr})'
+    else:
+        marker_expr = target  # target is also a marker
 
     with io.StringIO() as buf:
         with redirect_stdout(buf):
             res = pytest.main(
-                ['--collect-only', folder, '-q', '--target', target],
+                ['--collect-only', folder, '-q', '-m', marker_expr], plugins=[collector]
-                plugins=[collector],
             )
         if res.value != ExitCode.OK:
             if res.value == ExitCode.NO_TESTS_COLLECTED:
@@ -219,7 +223,9 @@ def get_pytest_cases(folder: str, target: str) -> List[PytestCase]:
     return collector.cases
 
 
-def get_pytest_app_paths(folder: str, target: str) -> Set[str]:
+def get_pytest_app_paths(
-    cases = get_pytest_cases(folder, target)
+    folder: str, target: str, marker_expr: Optional[str] = None
+) -> Set[str]:
+    cases = get_pytest_cases(folder, target, marker_expr)
 
     return set({app.path for case in cases for app in case.apps})