Merge branch 'ci/refactor_test_apps_fuhanxi' into 'master'

ci: enable test_apps s3 c3 tests

Closes IDFCI-1124

See merge request espressif/esp-idf!17397

.gitlab-ci.yml

@@ -73,7 +73,7 @@ variables:
   TEST_ENV_CONFIG_REPO: "https://gitlab-ci-token:${BOT_TOKEN}@${CI_SERVER_HOST}:${CI_SERVER_PORT}/qa/ci-test-runner-configs.git"
   CI_AUTO_TEST_SCRIPT_REPO_URL: "https://gitlab-ci-token:${BOT_TOKEN}@${CI_SERVER_HOST}:${CI_SERVER_PORT}/qa/auto_test_script.git"
   CI_AUTO_TEST_SCRIPT_REPO_BRANCH: "ci/v4.1"
-  PYTEST_EMBEDDED_TAG: "v0.6.0rc0"
+  PYTEST_EMBEDDED_VERSION: "0.6.0"
 
   # cache python dependencies
   PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
@@ -169,11 +169,11 @@ before_script:
     - fetch_submodules
     - *download_test_python_contraint_file
     - $IDF_PATH/tools/idf_tools.py install-python-env
-    - cd /tmp
+    - pip install
-    - retry_failed git clone --depth 1 --branch $PYTEST_EMBEDDED_TAG https://gitlab-ci-token:${BOT_TOKEN}@${CI_SERVER_HOST}:${CI_SERVER_PORT}/idf/pytest-embedded.git
+      "pytest-embedded-serial-esp~=$PYTEST_EMBEDDED_VERSION"
-    - cd pytest-embedded && bash foreach.sh install
+      "pytest-embedded-idf~=$PYTEST_EMBEDDED_VERSION"
-    - pip install pytest-rerunfailures
+      pytest-rerunfailures
-    - pip install scapy
+      scapy
     - cd $IDF_PATH
 
 default:

.gitlab/ci/build.yml

@@ -93,6 +93,16 @@ build_pytest_components_esp32c3:
   script:
     - run_cmd python tools/ci/build_pytest_apps.py components --target esp32c3 --size-info $SIZE_INFO_LOCATION -vv
 
+build_non_test_components_apps:
+  extends:
+    - .build_template
+    - .build_test_apps_template
+    - .rules:build:component_ut
+  variables:
+    IDF_TARGET: all
+    TEST_PREFIX: component_ut
+    TEST_RELATIVE_DIR: component_ut
+
 build_pytest_test_apps_esp32:
   extends:
     - .build_pytest_template
@@ -107,15 +117,19 @@ build_pytest_test_apps_esp32s2:
   script:
     - run_cmd python tools/ci/build_pytest_apps.py tools/test_apps --target esp32s2 --size-info $SIZE_INFO_LOCATION -vv
 
-build_non_test_components_apps:
+build_pytest_test_apps_esp32s3:
   extends:
-    - .build_template
+    - .build_pytest_template
-    - .build_test_apps_template
+    - .rules:build:custom_test-esp32s3
-    - .rules:build:component_ut
+  script:
-  variables:
+    - run_cmd python tools/ci/build_pytest_apps.py tools/test_apps --target esp32s3 --size-info $SIZE_INFO_LOCATION -vv
-    IDF_TARGET: all
+
-    TEST_PREFIX: component_ut
+build_pytest_test_apps_esp32c3:
-    TEST_RELATIVE_DIR: component_ut
+  extends:
+    - .build_pytest_template
+    - .rules:build:custom_test-esp32c3
+  script:
+    - run_cmd python tools/ci/build_pytest_apps.py tools/test_apps --target esp32c3 --size-info $SIZE_INFO_LOCATION -vv
 
 .build_template_app_template:
   extends: .build_template

.gitlab/ci/target-test.yml

@@ -3,6 +3,7 @@
   stage: target_test
   timeout: 1 hour
   extends: .before_script_pytest
+  tags: [$TARGET, $ENV_MARKER]
   artifacts:
     when: always
     paths:
@@ -26,11 +27,8 @@ example_test_pytest_esp32_generic:
   needs:
     - build_pytest_examples_esp32
   variables:
-    TARGET: esp32
+    TARGET: ESP32
     ENV_MARKER: generic
-  tags: # in gitlab 14.1 or later, we can use `parallel: matrix` with the `tags` keyword. https://docs.gitlab.com/ee/ci/jobs/job_control.html#run-a-matrix-of-parallel-trigger-jobs
-    - ESP32
-    - Example_GENERIC
 
 example_test_pytest_esp32s2_generic:
   extends:
@@ -39,11 +37,8 @@ example_test_pytest_esp32s2_generic:
   needs:
     - build_pytest_examples_esp32s2
   variables:
-    TARGET: esp32s2
+    TARGET: ESP32S2
     ENV_MARKER: generic
-  tags:
-    - ESP32S2
-    - Example_GENERIC
 
 example_test_pytest_esp32s3_generic:
   extends:
@@ -52,11 +47,8 @@ example_test_pytest_esp32s3_generic:
   needs:
     - build_pytest_examples_esp32s3
   variables:
-    TARGET: esp32s3
+    TARGET: ESP32S3
     ENV_MARKER: generic
-  tags:
-    - ESP32S3
-    - Example_GENERIC
 
 example_test_pytest_esp32c3_generic:
   extends:
@@ -65,11 +57,8 @@ example_test_pytest_esp32c3_generic:
   needs:
     - build_pytest_examples_esp32c3
   variables:
-    TARGET: esp32c3
+    TARGET: ESP32C3
     ENV_MARKER: generic
-  tags:
-    - ESP32C3
-    - Example_GENERIC
 
 example_test_pytest_esp32c3_flash_suspend:
   extends:
@@ -78,11 +67,8 @@ example_test_pytest_esp32c3_flash_suspend:
   needs:
     - build_pytest_examples_esp32c3
   variables:
-    TARGET: esp32c3
+    TARGET: ESP32C3
     ENV_MARKER: flash_suspend
-  tags:
-    - ESP32C3_IDF
-    - UT_T1_Flash_Suspend
 
 .pytest_components_dir_template:
   extends: .pytest_template
@@ -96,11 +82,8 @@ component_ut_pytest_esp32_generic:
   needs:
     - build_pytest_components_esp32
   variables:
-    TARGET: esp32
+    TARGET: ESP32
     ENV_MARKER: generic
-  tags:
-    - ESP32
-    - COMPONENT_UT_GENERIC
 
 component_ut_pytest_esp32_ip101:
   extends:
@@ -109,11 +92,8 @@ component_ut_pytest_esp32_ip101:
   needs:
     - build_pytest_components_esp32
   variables:
-    TARGET: esp32
+    TARGET: ESP32
     ENV_MARKER: ip101
-  tags:
-    - ESP32
-    - COMPONENT_UT_IP101
 
 component_ut_pytest_esp32_lan8720:
   extends:
@@ -122,11 +102,8 @@ component_ut_pytest_esp32_lan8720:
   needs:
     - build_pytest_components_esp32
   variables:
-    TARGET: esp32
+    TARGET: ESP32
     ENV_MARKER: lan8720
-  tags:
-    - ESP32
-    - COMPONENT_UT_LAN8720
 
 component_ut_pytest_esp32s2_generic:
   extends:
@@ -135,11 +112,8 @@ component_ut_pytest_esp32s2_generic:
   needs:
     - build_pytest_components_esp32s2
   variables:
-    TARGET: esp32s2
+    TARGET: ESP32S2
     ENV_MARKER: generic
-  tags:
-    - ESP32S2
-    - COMPONENT_UT_GENERIC
 
 component_ut_pytest_esp32s3_generic:
   extends:
@@ -148,11 +122,8 @@ component_ut_pytest_esp32s3_generic:
   needs:
     - build_pytest_components_esp32s3
   variables:
-    TARGET: esp32s3
+    TARGET: ESP32S3
     ENV_MARKER: generic
-  tags:
-    - ESP32S3
-    - COMPONENT_UT_GENERIC
 
 component_ut_pytest_esp32s3_octal_psram:
   extends:
@@ -161,11 +132,8 @@ component_ut_pytest_esp32s3_octal_psram:
   needs:
     - build_pytest_components_esp32s3
   variables:
-    TARGET: esp32s3
+    TARGET: ESP32S3
     ENV_MARKER: octal_psram
-  tags:
-    - ESP32S3
-    - MSPI_F8R8
 
 component_ut_pytest_esp32c3_generic:
   extends:
@@ -174,11 +142,8 @@ component_ut_pytest_esp32c3_generic:
   needs:
     - build_pytest_components_esp32c3
   variables:
-    TARGET: esp32c3
+    TARGET: ESP32C3
     ENV_MARKER: generic
-  tags:
-    - ESP32C3
-    - COMPONENT_UT_GENERIC
 
 .pytest_test_apps_dir_template:
   extends: .pytest_template
@@ -192,12 +157,9 @@ test_app_test_pytest_esp32_generic:
   needs:
     - build_pytest_test_apps_esp32
   variables:
-    TARGET: esp32
+    TARGET: ESP32
     ENV_MARKER: generic
     SETUP_TOOLS: "1"  # need gdb
-  tags:
-    - ESP32
-    - Example_GENERIC
 
 test_app_test_pytest_esp32s2_generic:
   extends:
@@ -206,12 +168,29 @@ test_app_test_pytest_esp32s2_generic:
   needs:
     - build_pytest_test_apps_esp32s2
   variables:
-    TARGET: esp32s2
+    TARGET: ESP32S2
     ENV_MARKER: generic
     SETUP_TOOLS: "1"  # need gdb
-  tags:
+
-    - ESP32S2
+test_app_test_pytest_esp32s3_generic:
-    - Example_GENERIC
+  extends:
+    - .pytest_test_apps_dir_template
+    - .rules:test:custom_test-esp32s3
+  needs:
+    - build_pytest_test_apps_esp32s3
+  variables:
+    TARGET: ESP32S3
+    ENV_MARKER: generic
+
+test_app_test_pytest_esp32c3_generic:
+  extends:
+    - .pytest_test_apps_dir_template
+    - .rules:test:custom_test-esp32c3
+  needs:
+    - build_pytest_test_apps_esp32c3
+  variables:
+    TARGET: ESP32C3
+    ENV_MARKER: generic
 
 # for parallel jobs, CI_JOB_NAME will be "job_name index/total" (for example, "IT_001 1/2")
 # we need to convert to pattern "job_name_index.yml"

conftest.py

@@ -22,6 +22,7 @@ from typing import Callable, List, Optional
 import pytest
 from _pytest.config import Config
 from _pytest.fixtures import FixtureRequest
+from _pytest.main import Session
 from _pytest.nodes import Item
 from _pytest.python import Function
 from pytest_embedded.plugin import parse_configuration
@@ -95,7 +96,7 @@ def build_dir(
     """
     param_or_cli: str = getattr(
         request, 'param', None
-    ) or request.config.option.__dict__.get('build_dir')
+    ) or request.config.getoption('build_dir')
     if param_or_cli is not None:  # respect the param and the cli
         return param_or_cli
 
@@ -146,6 +147,12 @@ def pytest_addoption(parser: pytest.Parser) -> None:
     )
 
 
+@pytest.hookimpl(tryfirst=True)
+def pytest_sessionstart(session: Session) -> None:
+    if session.config.option.target:
+        session.config.option.target = session.config.getoption('target').lower()
+
+
 @pytest.hookimpl(tryfirst=True)
 def pytest_collection_modifyitems(config: Config, items: List[Function]) -> None:
     target = config.getoption('target', None)  # use the `build` dir

tools/ci/check_copyright_ignore.txt

@@ -2502,7 +2502,6 @@ tools/ci/python_packages/tiny_test_fw/DUT.py
 tools/ci/python_packages/tiny_test_fw/Env.py
 tools/ci/python_packages/tiny_test_fw/EnvConfig.py
 tools/ci/python_packages/tiny_test_fw/TinyFW.py
-tools/ci/python_packages/tiny_test_fw/Utility/CIAssignTest.py
 tools/ci/python_packages/tiny_test_fw/Utility/CaseConfig.py
 tools/ci/python_packages/tiny_test_fw/Utility/GitlabCIJob.py
 tools/ci/python_packages/tiny_test_fw/Utility/TestCase.py

tools/ci/python_packages/tiny_test_fw/Utility/CIAssignTest.py

@@ -1,16 +1,5 @@
-# Copyright 2015-2017 Espressif Systems (Shanghai) PTE LTD
+# SPDX-FileCopyrightText: 2015-2022 Espressif Systems (Shanghai) CO LTD
-#
+# SPDX-License-Identifier: Apache-2.0
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http:#www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
 
 """
 Common logic to assign test cases to CI jobs.
@@ -187,8 +176,11 @@ class AssignTest(object):
 
         job_list = list()
         for job_name in ci_config:
+            if 'pytest' in job_name:
+                continue
             if self.CI_TEST_JOB_PATTERN.search(job_name) is not None:
                 job_list.extend(self._handle_parallel_attribute(job_name, ci_config[job_name]))
+
         job_list.sort(key=lambda x: x['name'])
         return job_list
 

tools/test_apps/peripherals/i2c_wifi/app_test.py

@@ -1,18 +0,0 @@
-#!/usr/bin/env python
-import tiny_test_fw
-import ttfw_idf
-
-
-@ttfw_idf.idf_custom_test(env_tag='Example_GENERIC')
-def test_startup(env, _):  # type: (tiny_test_fw.Env.Env, None) -> None
-    # Only test for master usage.
-    dut = env.get_dut('i2c_wifi', 'tools/test_apps/peripherals/i2c_wifi')
-    dut.start_app()
-
-    dut.expect('I2C-WIFI test success')
-
-    env.close_dut(dut.name)
-
-
-if __name__ == '__main__':
-    test_startup()

tools/test_apps/peripherals/i2c_wifi/pytest_i2c_wifi.py

@@ -0,0 +1,11 @@
+# SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: CC0-1.0
+
+import pytest
+from pytest_embedded_idf.dut import IdfDut
+
+
+@pytest.mark.supported_targets
+@pytest.mark.generic
+def test_startup(dut: IdfDut) -> None:
+    dut.expect_exact('I2C-WIFI test success')

tools/test_apps/phy/phy_multi_init_data_test/app_test.py

@@ -1,33 +0,0 @@
-#!/usr/bin/env python
-# SPDX-FileCopyrightText: 2021 Espressif Systems (Shanghai) CO LTD
-# SPDX-License-Identifier: Apache-2.0
-import glob
-import os
-
-import tiny_test_fw
-import ttfw_idf
-from tiny_test_fw import Utility
-
-
-@ttfw_idf.idf_custom_test(env_tag='Example_GENERIC', group='test-apps')
-def test_phy_multi_init_data_bin(env, _):
-    # type: (tiny_test_fw.Env.Env, None) -> None
-    config_files = glob.glob(os.path.join(os.path.dirname(__file__), 'sdkconfig.ci.*'))
-    config_names = [os.path.basename(s).replace('sdkconfig.ci.', '') for s in config_files]
-    for name in config_names:
-        dut = env.get_dut('phy_multi_init_data_test', 'tools/test_apps/phy/phy_multi_init_data_test',app_config_name=name)
-        dut.start_app()
-
-        if 'CONFIG_ESP_PHY_MULTIPLE_INIT_DATA_BIN_EMBED' in dut.app.get_sdkconfig().keys():
-            Utility.console_log('multi init data bin embed test')
-            dut.expect('load embedded multi phy init data')
-        else:
-            Utility.console_log('multi init data bin test')
-            dut.expect('Support multiple PHY init data bins')
-
-        dut.expect('wifi_init finished')
-        Utility.console_log('Test Success')
-
-
-if __name__ == '__main__':
-    test_phy_multi_init_data_bin()

tools/test_apps/phy/phy_multi_init_data_test/pytest_phy_multi_init_data.py

@@ -0,0 +1,19 @@
+# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: CC0-1.0
+
+import pytest
+from pytest_embedded_idf.dut import IdfDut
+
+
+@pytest.mark.supported_targets
+@pytest.mark.generic
+@pytest.mark.parametrize('config', [
+    'phy_multiple_init_data',
+    'phy_multiple_init_data_embed',
+], indirect=True)
+def test_phy_multi_init_data_bin(dut: IdfDut, config: str) -> None:
+    if config == 'phy_multiple_init_data':
+        dut.expect_exact('Support multiple PHY init data bins')
+    else:
+        dut.expect_exact('load embedded multi phy init data')
+    dut.expect_exact('wifi_init finished')