diff --git a/.gitlab/ci/dependencies/dependencies.yml b/.gitlab/ci/dependencies/dependencies.yml index 3e7a1e890d..30ac2e5a12 100644 --- a/.gitlab/ci/dependencies/dependencies.yml +++ b/.gitlab/ci/dependencies/dependencies.yml @@ -156,6 +156,40 @@ build:integration_test: - "build:component_ut-{0}" - build:target_test +# To reduce the runner usage. +# Do not create these jobs by default labels on development branches +# Only triggered by `target_test` or related_changes changes +"test:example_test-{0}-{1}-related_changes": + matrix: + - *all_targets + - - ota + - bt + - wifi + patterns: + - "example_test-related_changes-{1}" + labels: + - target_test + included_in: + - "build:example_test-{0}" + - "build:example_test" + - build:target_test + +# Include nightly_run cases for ota +"test:example_test-{0}-ota-related_changes-include_nightly_run": + matrix: + - - esp32 + - esp32c3 + specific_rules: + - "if-include_nightly_run-rule" + - "if-example_test-related_changes-ota-rule" + labels: + - target_test + included_in: + - "build:example_test-{0}" + - "build:example_test" + - build:target_test + + # due to the lack of runners, c2 tests will only be triggered by label "test:{0}-esp32c2": matrix: diff --git a/.gitlab/ci/dependencies/generate_rules.py b/.gitlab/ci/dependencies/generate_rules.py index a6c3f9248d..2d8ef88018 100755 --- a/.gitlab/ci/dependencies/generate_rules.py +++ b/.gitlab/ci/dependencies/generate_rules.py @@ -82,6 +82,7 @@ class RulesWriter: RULE_LABEL_TEMPLATE = ' - <<: *if-label-{0}' RULE_PATTERN_TEMPLATE = ' - <<: *if-dev-push\n' \ ' changes: *patterns-{0}' + SPECIFIC_RULE_TEMPLATE = ' - <<: *{0}' RULES_TEMPLATE = inspect.cleandoc(r""" .rules:{0}: rules: @@ -156,6 +157,8 @@ class RulesWriter: continue if 'included_in' in v: for item in _list(v['included_in']): + if 'specific_rules' in v: + res[item]['specific_rules'].update(_list(v['specific_rules'])) if 'labels' in v: res[item]['labels'].update(_list(v['labels'])) if 'patterns' in v: @@ -210,6 +213,12 @@ class RulesWriter: _rules.append(self.RULE_PROTECTED) if name.startswith('test:'): _rules.append(self.RULE_BUILD_ONLY) + + for specific_rule in cfg['specific_rules']: + if f'.{specific_rule}' in self.rules_cfg: + _rules.append(self.SPECIFIC_RULE_TEMPLATE.format(specific_rule)) + else: + print('WARNING: specific_rule {} not exists'.format(specific_rule)) for label in cfg['labels']: _rules.append(self.RULE_LABEL_TEMPLATE.format(label)) for pattern in cfg['patterns']: diff --git a/.gitlab/ci/rules.yml b/.gitlab/ci/rules.yml index 5ac167cae1..618084bd1a 100644 --- a/.gitlab/ci/rules.yml +++ b/.gitlab/ci/rules.yml @@ -196,6 +196,20 @@ - "examples/build_system/cmake/import_lib/main/lib/tinyxml2" - "examples/peripherals/secure_element/atecc608_ecdsa/components/esp-cryptoauthlib" +.patterns-example_test-related_changes-ota: &patterns-example_test-related_changes-ota + - "examples/system/ota/**/*" + - "examples/common_components/protocol_examples_common/**/*" + - "components/app_update/**/*" + +.patterns-example_test-related_changes-bt: &patterns-example_test-related_changes-bt + - "components/bt/host/nimble/nimble" + - "examples/bluetooth/nimble/**/*" + +.patterns-example_test-related_changes-wifi: &patterns-example_test-related_changes-wifi + - "components/esp_wifi/lib" + - "examples/protocols/**/*" + - "examples/wifi/**/*" + ############## # if anchors # ############## @@ -226,6 +240,21 @@ .if-revert-branch: &if-revert-branch if: '$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME =~ /^revert-/' +##################### +# Specific if rules # +##################### + +.if-include_nightly_run-rule: &if-include_nightly_run-rule + if: '$INCLUDE_NIGHTLY_RUN || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*include_nightly_run(?:,[^,\n\r]+)*$/i || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*target_test(?:,[^,\n\r]+)*$/i' + variables: + INCLUDE_NIGHTLY_RUN: "1" + +.if-example_test-related_changes-ota-rule: &if-example_test-related_changes-ota-rule + <<: *if-dev-push + changes: *patterns-example_test-related_changes-ota + variables: + INCLUDE_NIGHTLY_RUN: "1" + ######### # Rules # ######### @@ -728,6 +757,8 @@ - <<: *if-revert-branch when: never - <<: *if-protected + - <<: *if-example_test-related_changes-ota-rule + - <<: *if-include_nightly_run-rule - <<: *if-label-build - <<: *if-label-example_test - <<: *if-label-example_test_esp32 @@ -748,12 +779,20 @@ changes: *patterns-build_target_test - <<: *if-dev-push changes: *patterns-example_test + - <<: *if-dev-push + changes: *patterns-example_test-related_changes-bt + - <<: *if-dev-push + changes: *patterns-example_test-related_changes-ota + - <<: *if-dev-push + changes: *patterns-example_test-related_changes-wifi .rules:build:example_test-esp32: rules: - <<: *if-revert-branch when: never - <<: *if-protected + - <<: *if-example_test-related_changes-ota-rule + - <<: *if-include_nightly_run-rule - <<: *if-label-build - <<: *if-label-example_test - <<: *if-label-example_test_esp32 @@ -769,6 +808,12 @@ changes: *patterns-build_target_test - <<: *if-dev-push changes: *patterns-example_test + - <<: *if-dev-push + changes: *patterns-example_test-related_changes-bt + - <<: *if-dev-push + changes: *patterns-example_test-related_changes-ota + - <<: *if-dev-push + changes: *patterns-example_test-related_changes-wifi .rules:build:example_test-esp32c2: rules: @@ -777,18 +822,27 @@ - <<: *if-protected - <<: *if-label-build - <<: *if-label-example_test_esp32c2 + - <<: *if-label-target_test - <<: *if-dev-push changes: *patterns-build_components - <<: *if-dev-push changes: *patterns-build_system - <<: *if-dev-push changes: *patterns-build_target_test + - <<: *if-dev-push + changes: *patterns-example_test-related_changes-bt + - <<: *if-dev-push + changes: *patterns-example_test-related_changes-ota + - <<: *if-dev-push + changes: *patterns-example_test-related_changes-wifi .rules:build:example_test-esp32c3: rules: - <<: *if-revert-branch when: never - <<: *if-protected + - <<: *if-example_test-related_changes-ota-rule + - <<: *if-include_nightly_run-rule - <<: *if-label-build - <<: *if-label-example_test - <<: *if-label-example_test_esp32c3 @@ -803,6 +857,12 @@ changes: *patterns-build_target_test - <<: *if-dev-push changes: *patterns-example_test + - <<: *if-dev-push + changes: *patterns-example_test-related_changes-bt + - <<: *if-dev-push + changes: *patterns-example_test-related_changes-ota + - <<: *if-dev-push + changes: *patterns-example_test-related_changes-wifi .rules:build:example_test-esp32h2: rules: @@ -823,6 +883,12 @@ changes: *patterns-build_target_test - <<: *if-dev-push changes: *patterns-example_test + - <<: *if-dev-push + changes: *patterns-example_test-related_changes-bt + - <<: *if-dev-push + changes: *patterns-example_test-related_changes-ota + - <<: *if-dev-push + changes: *patterns-example_test-related_changes-wifi .rules:build:example_test-esp32s2: rules: @@ -843,6 +909,12 @@ changes: *patterns-build_target_test - <<: *if-dev-push changes: *patterns-example_test + - <<: *if-dev-push + changes: *patterns-example_test-related_changes-bt + - <<: *if-dev-push + changes: *patterns-example_test-related_changes-ota + - <<: *if-dev-push + changes: *patterns-example_test-related_changes-wifi .rules:build:example_test-esp32s3: rules: @@ -863,6 +935,12 @@ changes: *patterns-build_target_test - <<: *if-dev-push changes: *patterns-example_test + - <<: *if-dev-push + changes: *patterns-example_test-related_changes-bt + - <<: *if-dev-push + changes: *patterns-example_test-related_changes-ota + - <<: *if-dev-push + changes: *patterns-example_test-related_changes-wifi .rules:build:integration_test: rules: @@ -895,6 +973,8 @@ - <<: *if-revert-branch when: never - <<: *if-protected + - <<: *if-example_test-related_changes-ota-rule + - <<: *if-include_nightly_run-rule - <<: *if-label-build - <<: *if-label-component_ut - <<: *if-label-component_ut_esp32 @@ -943,6 +1023,12 @@ changes: *patterns-custom_test - <<: *if-dev-push changes: *patterns-example_test + - <<: *if-dev-push + changes: *patterns-example_test-related_changes-bt + - <<: *if-dev-push + changes: *patterns-example_test-related_changes-ota + - <<: *if-dev-push + changes: *patterns-example_test-related_changes-wifi - <<: *if-dev-push changes: *patterns-integration_test - <<: *if-dev-push @@ -1286,6 +1372,50 @@ - <<: *if-dev-push changes: *patterns-example_test +.rules:test:example_test-esp32-bt-related_changes: + rules: + - <<: *if-revert-branch + when: never + - <<: *if-protected + - <<: *if-label-build-only + when: never + - <<: *if-label-target_test + - <<: *if-dev-push + changes: *patterns-example_test-related_changes-bt + +.rules:test:example_test-esp32-ota-related_changes: + rules: + - <<: *if-revert-branch + when: never + - <<: *if-protected + - <<: *if-label-build-only + when: never + - <<: *if-label-target_test + - <<: *if-dev-push + changes: *patterns-example_test-related_changes-ota + +.rules:test:example_test-esp32-ota-related_changes-include_nightly_run: + rules: + - <<: *if-revert-branch + when: never + - <<: *if-protected + - <<: *if-label-build-only + when: never + - <<: *if-example_test-related_changes-ota-rule + - <<: *if-include_nightly_run-rule + - <<: *if-label-target_test + +.rules:test:example_test-esp32-wifi-related_changes: + rules: + - <<: *if-revert-branch + when: never + - <<: *if-protected + - <<: *if-label-build-only + when: never + - <<: *if-label-target_test + - <<: *if-dev-push + changes: *patterns-example_test-related_changes-wifi + .rules:test:example_test-esp32c2: rules: - <<: *if-revert-branch @@ -1295,6 +1425,39 @@ when: never - <<: *if-label-example_test_esp32c2 +.rules:test:example_test-esp32c2-bt-related_changes: + rules: + - <<: *if-revert-branch + when: never + - <<: *if-protected + - <<: *if-label-build-only + when: never + - <<: *if-label-target_test + - <<: *if-dev-push + changes: *patterns-example_test-related_changes-bt + +.rules:test:example_test-esp32c2-ota-related_changes: + rules: + - <<: *if-revert-branch + when: never + - <<: *if-protected + - <<: *if-label-build-only + when: never + - <<: *if-label-target_test + - <<: *if-dev-push + changes: *patterns-example_test-related_changes-ota + +.rules:test:example_test-esp32c2-wifi-related_changes: + rules: + - <<: *if-revert-branch + when: never + - <<: *if-protected + - <<: *if-label-build-only + when: never + - <<: *if-label-target_test + - <<: *if-dev-push + changes: *patterns-example_test-related_changes-wifi + .rules:test:example_test-esp32c3: rules: - <<: *if-revert-branch @@ -1310,6 +1473,50 @@ - <<: *if-dev-push changes: *patterns-example_test +.rules:test:example_test-esp32c3-bt-related_changes: + rules: + - <<: *if-revert-branch + when: never + - <<: *if-protected + - <<: *if-label-build-only + when: never + - <<: *if-label-target_test + - <<: *if-dev-push + changes: *patterns-example_test-related_changes-bt + +.rules:test:example_test-esp32c3-ota-related_changes: + rules: + - <<: *if-revert-branch + when: never + - <<: *if-protected + - <<: *if-label-build-only + when: never + - <<: *if-label-target_test + - <<: *if-dev-push + changes: *patterns-example_test-related_changes-ota + +.rules:test:example_test-esp32c3-ota-related_changes-include_nightly_run: + rules: + - <<: *if-revert-branch + when: never + - <<: *if-protected + - <<: *if-label-build-only + when: never + - <<: *if-example_test-related_changes-ota-rule + - <<: *if-include_nightly_run-rule + - <<: *if-label-target_test + +.rules:test:example_test-esp32c3-wifi-related_changes: + rules: + - <<: *if-revert-branch + when: never + - <<: *if-protected + - <<: *if-label-build-only + when: never + - <<: *if-label-target_test + - <<: *if-dev-push + changes: *patterns-example_test-related_changes-wifi + .rules:test:example_test-esp32h2: rules: - <<: *if-revert-branch @@ -1325,6 +1532,39 @@ - <<: *if-dev-push changes: *patterns-example_test +.rules:test:example_test-esp32h2-bt-related_changes: + rules: + - <<: *if-revert-branch + when: never + - <<: *if-protected + - <<: *if-label-build-only + when: never + - <<: *if-label-target_test + - <<: *if-dev-push + changes: *patterns-example_test-related_changes-bt + +.rules:test:example_test-esp32h2-ota-related_changes: + rules: + - <<: *if-revert-branch + when: never + - <<: *if-protected + - <<: *if-label-build-only + when: never + - <<: *if-label-target_test + - <<: *if-dev-push + changes: *patterns-example_test-related_changes-ota + +.rules:test:example_test-esp32h2-wifi-related_changes: + rules: + - <<: *if-revert-branch + when: never + - <<: *if-protected + - <<: *if-label-build-only + when: never + - <<: *if-label-target_test + - <<: *if-dev-push + changes: *patterns-example_test-related_changes-wifi + .rules:test:example_test-esp32s2: rules: - <<: *if-revert-branch @@ -1340,6 +1580,39 @@ - <<: *if-dev-push changes: *patterns-example_test +.rules:test:example_test-esp32s2-bt-related_changes: + rules: + - <<: *if-revert-branch + when: never + - <<: *if-protected + - <<: *if-label-build-only + when: never + - <<: *if-label-target_test + - <<: *if-dev-push + changes: *patterns-example_test-related_changes-bt + +.rules:test:example_test-esp32s2-ota-related_changes: + rules: + - <<: *if-revert-branch + when: never + - <<: *if-protected + - <<: *if-label-build-only + when: never + - <<: *if-label-target_test + - <<: *if-dev-push + changes: *patterns-example_test-related_changes-ota + +.rules:test:example_test-esp32s2-wifi-related_changes: + rules: + - <<: *if-revert-branch + when: never + - <<: *if-protected + - <<: *if-label-build-only + when: never + - <<: *if-label-target_test + - <<: *if-dev-push + changes: *patterns-example_test-related_changes-wifi + .rules:test:example_test-esp32s3: rules: - <<: *if-revert-branch @@ -1349,6 +1622,39 @@ when: never - <<: *if-label-example_test_esp32s3 +.rules:test:example_test-esp32s3-bt-related_changes: + rules: + - <<: *if-revert-branch + when: never + - <<: *if-protected + - <<: *if-label-build-only + when: never + - <<: *if-label-target_test + - <<: *if-dev-push + changes: *patterns-example_test-related_changes-bt + +.rules:test:example_test-esp32s3-ota-related_changes: + rules: + - <<: *if-revert-branch + when: never + - <<: *if-protected + - <<: *if-label-build-only + when: never + - <<: *if-label-target_test + - <<: *if-dev-push + changes: *patterns-example_test-related_changes-ota + +.rules:test:example_test-esp32s3-wifi-related_changes: + rules: + - <<: *if-revert-branch + when: never + - <<: *if-protected + - <<: *if-label-build-only + when: never + - <<: *if-label-target_test + - <<: *if-dev-push + changes: *patterns-example_test-related_changes-wifi + .rules:test:host_test: rules: - <<: *if-revert-branch diff --git a/.gitlab/ci/target-test.yml b/.gitlab/ci/target-test.yml index 68b8e37c4f..928ab88261 100644 --- a/.gitlab/ci/target-test.yml +++ b/.gitlab/ci/target-test.yml @@ -83,7 +83,7 @@ example_test_pytest_esp32c3_flash_suspend: example_test_pytest_esp32_ethernet_ota: extends: - .pytest_examples_dir_template - - .rules:test:example_test-esp32 + - .rules:test:example_test-esp32-ota-related_changes needs: - build_pytest_examples_esp32 tags: [ esp32, ethernet_ota ] @@ -91,7 +91,7 @@ example_test_pytest_esp32_ethernet_ota: example_test_pytest_esp32_wifi_ota: extends: - .pytest_examples_dir_template - - .rules:test:example_test-esp32 + - .rules:test:example_test-esp32-ota-related_changes-include_nightly_run needs: - build_pytest_examples_esp32 tags: [ esp32, wifi_ota ] @@ -99,7 +99,7 @@ example_test_pytest_esp32_wifi_ota: example_test_pytest_esp32_flash_encryption_ota: extends: - .pytest_examples_dir_template - - .rules:test:example_test-esp32 + - .rules:test:example_test-esp32-ota-related_changes needs: - build_pytest_examples_esp32 tags: [ esp32, flash_encryption_ota ] @@ -107,7 +107,7 @@ example_test_pytest_esp32_flash_encryption_ota: example_test_pytest_esp32c3_flash_encryption_wifi_ota: extends: - .pytest_examples_dir_template - - .rules:test:example_test-esp32c3 + - .rules:test:example_test-esp32c3-ota-related_changes-include_nightly_run needs: - build_pytest_examples_esp32c3 tags: [ esp32c3, flash_encryption_wifi_ota ] @@ -131,7 +131,7 @@ example_test_pytest_esp32_8mb_flash: example_test_pytest_esp32_wifi: extends: - .pytest_examples_dir_template - - .rules:test:example_test-esp32 + - .rules:test:example_test-esp32-wifi-related_changes needs: - build_pytest_examples_esp32 tags: [ esp32, wifi ] @@ -467,7 +467,9 @@ example_test_004B: - Example_TWAI2 example_test_005: - extends: .example_test_esp32_template + extends: + - .example_test_esp32_template + - .rules:test:example_test-esp32-bt-related_changes tags: - ESP32 - Example_WIFI_BT diff --git a/conftest.py b/conftest.py index 5925c531fe..37f2908ddb 100644 --- a/conftest.py +++ b/conftest.py @@ -316,7 +316,10 @@ class IdfPytestEmbedded: item.add_marker(_target) # filter all the test cases with "nightly_run" marker - if os.getenv('NIGHTLY_RUN') == '1': + if os.getenv('INCLUDE_NIGHTLY_RUN') == '1': + # Do not filter nightly_run cases + pass + elif os.getenv('NIGHTLY_RUN') == '1': items[:] = [ item for item in items if 'nightly_run' in item_marker_names(item) ] diff --git a/examples/system/ota/advanced_https_ota/pytest_advanced_ota.py b/examples/system/ota/advanced_https_ota/pytest_advanced_ota.py index 7fb817f829..8fdd2c8d00 100644 --- a/examples/system/ota/advanced_https_ota/pytest_advanced_ota.py +++ b/examples/system/ota/advanced_https_ota/pytest_advanced_ota.py @@ -577,8 +577,8 @@ def test_examples_protocol_advanced_https_ota_example_partial_request(dut: Dut) @pytest.mark.esp32s2 @pytest.mark.esp32s3 @pytest.mark.wifi_ota +@pytest.mark.nightly_run @pytest.mark.parametrize('config', ['nimble',], indirect=True) -@pytest.mark.xfail(run=False) def test_examples_protocol_advanced_https_ota_example_nimble_gatts(dut: Dut) -> None: """ Run an OTA image update while a BLE GATT Server is running in background. This GATT server will be using NimBLE Host stack. @@ -621,8 +621,8 @@ def test_examples_protocol_advanced_https_ota_example_nimble_gatts(dut: Dut) -> @pytest.mark.esp32s2 @pytest.mark.esp32s3 @pytest.mark.wifi_ota +@pytest.mark.nightly_run @pytest.mark.parametrize('config', ['bluedroid',], indirect=True) -@pytest.mark.xfail(run=False) def test_examples_protocol_advanced_https_ota_example_bluedroid_gatts(dut: Dut) -> None: """ Run an OTA image update while a BLE GATT Server is running in background. This GATT server will be using Bluedroid Host stack. diff --git a/examples/system/ota/otatool/pytest_otatool.py b/examples/system/ota/otatool/pytest_otatool.py index b60471085b..185ec52af5 100644 --- a/examples/system/ota/otatool/pytest_otatool.py +++ b/examples/system/ota/otatool/pytest_otatool.py @@ -11,7 +11,7 @@ from pytest_embedded import Dut @pytest.mark.supported_targets -@pytest.mark.wifi +@pytest.mark.generic def test_otatool_example(dut: Dut) -> None: # Verify factory firmware dut.expect('OTA Tool Example') diff --git a/examples/system/ota/simple_ota_example/pytest_simple_ota.py b/examples/system/ota/simple_ota_example/pytest_simple_ota.py index 05d4e67b5c..b8c04f1415 100644 --- a/examples/system/ota/simple_ota_example/pytest_simple_ota.py +++ b/examples/system/ota/simple_ota_example/pytest_simple_ota.py @@ -119,7 +119,7 @@ def calc_all_sha256(dut: Dut) -> Tuple[str, str]: @pytest.mark.esp32s2 @pytest.mark.esp32s3 @pytest.mark.wifi_ota -@pytest.mark.xfail(run=False) +@pytest.mark.nightly_run def test_examples_protocol_simple_ota_example(dut: Dut) -> None: """ steps: | @@ -233,9 +233,9 @@ def test_examples_protocol_simple_ota_example_with_flash_encryption(dut: Dut) -> @pytest.mark.esp32c3 @pytest.mark.flash_encryption_wifi_ota +@pytest.mark.nightly_run @pytest.mark.parametrize('config', ['flash_enc_wifi',], indirect=True) @pytest.mark.parametrize('skip_autoflash', ['y'], indirect=True) -@pytest.mark.flaky(reruns=3, reruns_delay=5) def test_examples_protocol_simple_ota_example_with_flash_encryption_wifi(dut: Dut) -> None: """ steps: |