Merge branch 'ci/fix_search_multi_dut_with_markers' into 'master'

ci: fix collect multi-dut test case with markers issue

Closes IDFCI-1945

See merge request espressif/esp-idf!28448

conftest.py

@@ -1,14 +1,13 @@
 # SPDX-FileCopyrightText: 2021-2024 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
 # pylint: disable=W0621  # redefined-outer-name
-
+#
 # IDF is using [pytest](https://github.com/pytest-dev/pytest) and
 # [pytest-embedded plugin](https://github.com/espressif/pytest-embedded) as its test framework.
-
+#
 # if you found any bug or have any question,
 # please report to https://github.com/espressif/pytest-embedded/issues
 # or discuss at https://github.com/espressif/pytest-embedded/discussions
-
 import os
 import sys
 
@@ -80,6 +79,17 @@ def config(request: FixtureRequest) -> str:
     return getattr(request, 'param', None) or DEFAULT_SDKCONFIG  # type: ignore
 
 
+@pytest.fixture
+@multi_dut_fixture
+def target(request: FixtureRequest, dut_total: int,  dut_index: int) -> str:
+    plugin = request.config.stash[IDF_PYTEST_EMBEDDED_KEY]
+
+    if dut_total == 1:
+        return plugin.target[0]  # type: ignore
+
+    return plugin.target[dut_index]  # type: ignore
+
+
 @pytest.fixture
 def test_func_name(request: FixtureRequest) -> str:
     return request.node.function.__name__  # type: ignore
@@ -432,11 +442,12 @@ def pytest_configure(config: Config) -> None:
         for f in glob.glob(os.path.join(IDF_PATH, app_info_filepattern)):
             apps.extend(import_apps_from_txt(f))
 
-    config.stash[IDF_PYTEST_EMBEDDED_KEY] = IdfPytestEmbedded(
+    if '--collect-only' not in config.invocation_params.args:
-        target=target,
+        config.stash[IDF_PYTEST_EMBEDDED_KEY] = IdfPytestEmbedded(
-        apps=apps,
+            target=target,
-    )
+            apps=apps,
-    config.pluginmanager.register(config.stash[IDF_PYTEST_EMBEDDED_KEY])
+        )
+        config.pluginmanager.register(config.stash[IDF_PYTEST_EMBEDDED_KEY])
 
 
 def pytest_unconfigure(config: Config) -> None:

pytest.ini

@@ -13,6 +13,7 @@ addopts =
   --logfile-extension ".txt"
   --check-duplicates y
   --ignore-glob */managed_components/*
+  --ignore pytest_embedded_log
 
 # ignore DeprecationWarning
 filterwarnings =

tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml

@@ -83,6 +83,5 @@
       --parallel-count ${CI_NODE_TOTAL:-1}
       --parallel-index ${CI_NODE_INDEX:-1}
       ${PYTEST_EXTRA_FLAGS}
-      --app-info-filepattern "list_job_*.txt"
   after_script:
     - python tools/ci/artifacts_handler.py upload --type logs junit_reports

tools/ci/idf_pytest/plugin.py

@@ -1,6 +1,5 @@
 # SPDX-FileCopyrightText: 2023-2024 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
-
 import os
 import typing as t
 from collections import defaultdict
@@ -16,11 +15,20 @@ from idf_build_apps import App
 from idf_build_apps.constants import BuildStatus
 from pytest_embedded import Dut
 from pytest_embedded.plugin import parse_multi_dut_args
-from pytest_embedded.utils import find_by_suffix, to_list
+from pytest_embedded.utils import find_by_suffix
-from pytest_ignore_test_results.ignore_results import ChildCase, ChildCasesStashKey
+from pytest_embedded.utils import to_list
+from pytest_ignore_test_results.ignore_results import ChildCase
+from pytest_ignore_test_results.ignore_results import ChildCasesStashKey
 
-from .constants import DEFAULT_SDKCONFIG, PREVIEW_TARGETS, SUPPORTED_TARGETS, CollectMode, PytestApp, PytestCase
+from .constants import CollectMode
-from .utils import comma_sep_str_to_list, format_case_id, merge_junit_files
+from .constants import DEFAULT_SDKCONFIG
+from .constants import PREVIEW_TARGETS
+from .constants import PytestApp
+from .constants import PytestCase
+from .constants import SUPPORTED_TARGETS
+from .utils import comma_sep_str_to_list
+from .utils import format_case_id
+from .utils import merge_junit_files
 
 IDF_PYTEST_EMBEDDED_KEY = pytest.StashKey['IdfPytestEmbedded']()
 ITEM_FAILED_CASES_KEY = pytest.StashKey[list]()
@@ -43,9 +51,10 @@ class IdfPytestEmbedded:
         apps: t.Optional[t.List[App]] = None,
     ):
         if isinstance(target, str):
-            self.target = sorted(comma_sep_str_to_list(target))
+            # sequence also matters
+            self.target = comma_sep_str_to_list(target)
         else:
-            self.target = sorted(target)
+            self.target = target
 
         if not self.target:
             raise ValueError('`target` should not be empty')

tools/ci/idf_pytest/tests/conftest.py

@@ -1,10 +1,12 @@
-# SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
+# SPDX-FileCopyrightText: 2023-2024 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
-
 import os
 import sys
+import tempfile
 from pathlib import Path
 
+import pytest
+
 tools_ci_dir = os.path.join(os.path.dirname(__file__), '..', '..')
 if tools_ci_dir not in sys.path:
     sys.path.append(tools_ci_dir)
@@ -13,6 +15,8 @@ tools_dir = os.path.join(os.path.dirname(__file__), '..', '..', '..')
 if tools_dir not in sys.path:
     sys.path.append(tools_dir)
 
+from idf_ci_utils import IDF_PATH  # noqa: E402
+
 
 def create_project(name: str, folder: Path) -> Path:
     p = folder / name
@@ -46,3 +50,10 @@ void app_main(void) {}
         )
 
     return p
+
+
+@pytest.fixture
+def tmp_path() -> Path:
+    os.makedirs(os.path.join(IDF_PATH, 'pytest_embedded_log'), exist_ok=True)
+
+    return Path(tempfile.mkdtemp(prefix=os.path.join(IDF_PATH, 'pytest_embedded_log') + os.sep))

tools/ci/idf_pytest/tests/test_get_all_apps.py

@@ -1,9 +1,9 @@
 # SPDX-FileCopyrightText: 2023-2024 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
-
 from pathlib import Path
 
-from idf_pytest.script import SUPPORTED_TARGETS, get_all_apps
+from idf_pytest.script import get_all_apps
+from idf_pytest.script import SUPPORTED_TARGETS
 
 from conftest import create_project
 
@@ -38,6 +38,26 @@ def test_foo(dut):
     assert len(non_test_related_apps) == 2 * len(SUPPORTED_TARGETS) - 2
 
 
+def test_get_all_apps_multi_dut_with_markers_test_script(tmp_path: Path) -> None:
+    create_project('foo', tmp_path)
+
+    (tmp_path / 'foo' / 'pytest_get_all_apps_multi_dut_with_markers_test_script.py').write_text(
+        """import pytest
+
+@pytest.mark.esp32
+@pytest.mark.parametrize('count', [2, 3], indirect=True)
+def test_foo(dut):
+    pass
+""",
+        encoding='utf-8',
+    )
+
+    test_related_apps, non_test_related_apps = get_all_apps([str(tmp_path)], target='all')
+
+    assert len(test_related_apps) == 1
+    assert len(non_test_related_apps) == len(SUPPORTED_TARGETS) - 1
+
+
 def test_get_all_apps_multi_dut_test_script(tmp_path: Path) -> None:
     create_project('foo', tmp_path)
     with open(tmp_path / 'foo' / 'pytest_get_all_apps_multi_dut_test_script.py', 'w') as fw:

tools/ci/idf_pytest/tests/test_get_pytest_cases.py

@@ -1,6 +1,5 @@
-# SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
+# SPDX-FileCopyrightText: 2023-2024 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
-
 from pathlib import Path
 
 from idf_pytest.constants import CollectMode
@@ -45,11 +44,14 @@ def test_get_pytest_cases_single_specific(tmp_path: Path) -> None:
 def test_get_pytest_cases_multi_specific(tmp_path: Path) -> None:
     script = tmp_path / 'pytest_get_pytest_cases_multi_specific.py'
     script.write_text(TEMPLATE_SCRIPT)
-    cases = get_pytest_cases([str(tmp_path)], 'esp32s3,esp32s2, esp32s2')
+    cases = get_pytest_cases([str(tmp_path)], 'esp32s2,esp32s2, esp32s3')
 
     assert len(cases) == 1
     assert cases[0].targets == ['esp32s2', 'esp32s2', 'esp32s3']
 
+    cases = get_pytest_cases([str(tmp_path)], 'esp32s3,esp32s2,esp32s2')  # order matters
+    assert len(cases) == 0
+
 
 def test_get_pytest_cases_multi_all(tmp_path: Path) -> None:
     script = tmp_path / 'pytest_get_pytest_cases_multi_all.py'