CI: add specific job rules

.gitlab/ci/dependencies/dependencies.yml

@@ -156,6 +156,40 @@ build:integration_test:
     - "build:component_ut-{0}"
     - build:target_test
 
+# To reduce the runner usage.
+# Do not create these jobs by default labels on development branches
+# Only triggered by `target_test` or related_changes changes
+"test:example_test-{0}-{1}-related_changes":
+  matrix:
+    - *all_targets
+    - - ota
+      - bt
+      - wifi
+  patterns:
+    - "example_test-related_changes-{1}"
+  labels:
+    - target_test
+  included_in:
+    - "build:example_test-{0}"
+    - "build:example_test"
+    - build:target_test
+
+# Include nightly_run cases for ota
+"test:example_test-{0}-ota-related_changes-include_nightly_run":
+  matrix:
+    - - esp32
+      - esp32c3
+  specific_rules:
+    - "if-include_nightly_run-rule"
+    - "if-example_test-related_changes-ota-rule"
+  labels:
+    - target_test
+  included_in:
+    - "build:example_test-{0}"
+    - "build:example_test"
+    - build:target_test
+
+
 # due to the lack of runners, c2 tests will only be triggered by label
 "test:{0}-esp32c2":
   matrix:

.gitlab/ci/dependencies/generate_rules.py

@@ -82,6 +82,7 @@ class RulesWriter:
     RULE_LABEL_TEMPLATE = '    - <<: *if-label-{0}'
     RULE_PATTERN_TEMPLATE = '    - <<: *if-dev-push\n' \
                             '      changes: *patterns-{0}'
+    SPECIFIC_RULE_TEMPLATE = '    - <<: *{0}'
     RULES_TEMPLATE = inspect.cleandoc(r"""
     .rules:{0}:
       rules:
@@ -156,6 +157,8 @@ class RulesWriter:
                 continue
             if 'included_in' in v:
                 for item in _list(v['included_in']):
+                    if 'specific_rules' in v:
+                        res[item]['specific_rules'].update(_list(v['specific_rules']))
                     if 'labels' in v:
                         res[item]['labels'].update(_list(v['labels']))
                     if 'patterns' in v:
@@ -210,6 +213,12 @@ class RulesWriter:
                 _rules.append(self.RULE_PROTECTED)
             if name.startswith('test:'):
                 _rules.append(self.RULE_BUILD_ONLY)
+
+            for specific_rule in cfg['specific_rules']:
+                if f'.{specific_rule}' in self.rules_cfg:
+                    _rules.append(self.SPECIFIC_RULE_TEMPLATE.format(specific_rule))
+                else:
+                    print('WARNING: specific_rule {} not exists'.format(specific_rule))
             for label in cfg['labels']:
                 _rules.append(self.RULE_LABEL_TEMPLATE.format(label))
             for pattern in cfg['patterns']:

.gitlab/ci/rules.yml

@@ -196,6 +196,20 @@
   - "examples/build_system/cmake/import_lib/main/lib/tinyxml2"
   - "examples/peripherals/secure_element/atecc608_ecdsa/components/esp-cryptoauthlib"
 
+.patterns-example_test-related_changes-ota: &patterns-example_test-related_changes-ota
+  - "examples/system/ota/**/*"
+  - "examples/common_components/protocol_examples_common/**/*"
+  - "components/app_update/**/*"
+
+.patterns-example_test-related_changes-bt: &patterns-example_test-related_changes-bt
+  - "components/bt/host/nimble/nimble"
+  - "examples/bluetooth/nimble/**/*"
+
+.patterns-example_test-related_changes-wifi: &patterns-example_test-related_changes-wifi
+  - "components/esp_wifi/lib"
+  - "examples/protocols/**/*"
+  - "examples/wifi/**/*"
+
 ##############
 # if anchors #
 ##############
@@ -226,6 +240,21 @@
 .if-revert-branch: &if-revert-branch
   if: '$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME =~ /^revert-/'
 
+#####################
+# Specific if rules #
+#####################
+
+.if-include_nightly_run-rule: &if-include_nightly_run-rule
+  if: '$INCLUDE_NIGHTLY_RUN || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*include_nightly_run(?:,[^,\n\r]+)*$/i || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*target_test(?:,[^,\n\r]+)*$/i'
+  variables:
+    INCLUDE_NIGHTLY_RUN: "1"
+
+.if-example_test-related_changes-ota-rule: &if-example_test-related_changes-ota-rule
+  <<: *if-dev-push
+  changes: *patterns-example_test-related_changes-ota
+  variables:
+    INCLUDE_NIGHTLY_RUN: "1"
+
 #########
 # Rules #
 #########
@@ -728,6 +757,8 @@
     - <<: *if-revert-branch
       when: never
     - <<: *if-protected
+    - <<: *if-example_test-related_changes-ota-rule
+    - <<: *if-include_nightly_run-rule
     - <<: *if-label-build
     - <<: *if-label-example_test
     - <<: *if-label-example_test_esp32
@@ -748,12 +779,20 @@
       changes: *patterns-build_target_test
     - <<: *if-dev-push
       changes: *patterns-example_test
+    - <<: *if-dev-push
+      changes: *patterns-example_test-related_changes-bt
+    - <<: *if-dev-push
+      changes: *patterns-example_test-related_changes-ota
+    - <<: *if-dev-push
+      changes: *patterns-example_test-related_changes-wifi
 
 .rules:build:example_test-esp32:
   rules:
     - <<: *if-revert-branch
       when: never
     - <<: *if-protected
+    - <<: *if-example_test-related_changes-ota-rule
+    - <<: *if-include_nightly_run-rule
     - <<: *if-label-build
     - <<: *if-label-example_test
     - <<: *if-label-example_test_esp32
@@ -769,6 +808,12 @@
       changes: *patterns-build_target_test
     - <<: *if-dev-push
       changes: *patterns-example_test
+    - <<: *if-dev-push
+      changes: *patterns-example_test-related_changes-bt
+    - <<: *if-dev-push
+      changes: *patterns-example_test-related_changes-ota
+    - <<: *if-dev-push
+      changes: *patterns-example_test-related_changes-wifi
 
 .rules:build:example_test-esp32c2:
   rules:
@@ -777,18 +822,27 @@
     - <<: *if-protected
     - <<: *if-label-build
     - <<: *if-label-example_test_esp32c2
+    - <<: *if-label-target_test
     - <<: *if-dev-push
       changes: *patterns-build_components
     - <<: *if-dev-push
       changes: *patterns-build_system
     - <<: *if-dev-push
       changes: *patterns-build_target_test
+    - <<: *if-dev-push
+      changes: *patterns-example_test-related_changes-bt
+    - <<: *if-dev-push
+      changes: *patterns-example_test-related_changes-ota
+    - <<: *if-dev-push
+      changes: *patterns-example_test-related_changes-wifi
 
 .rules:build:example_test-esp32c3:
   rules:
     - <<: *if-revert-branch
       when: never
     - <<: *if-protected
+    - <<: *if-example_test-related_changes-ota-rule
+    - <<: *if-include_nightly_run-rule
     - <<: *if-label-build
     - <<: *if-label-example_test
     - <<: *if-label-example_test_esp32c3
@@ -803,6 +857,12 @@
       changes: *patterns-build_target_test
     - <<: *if-dev-push
       changes: *patterns-example_test
+    - <<: *if-dev-push
+      changes: *patterns-example_test-related_changes-bt
+    - <<: *if-dev-push
+      changes: *patterns-example_test-related_changes-ota
+    - <<: *if-dev-push
+      changes: *patterns-example_test-related_changes-wifi
 
 .rules:build:example_test-esp32h2:
   rules:
@@ -823,6 +883,12 @@
       changes: *patterns-build_target_test
     - <<: *if-dev-push
       changes: *patterns-example_test
+    - <<: *if-dev-push
+      changes: *patterns-example_test-related_changes-bt
+    - <<: *if-dev-push
+      changes: *patterns-example_test-related_changes-ota
+    - <<: *if-dev-push
+      changes: *patterns-example_test-related_changes-wifi
 
 .rules:build:example_test-esp32s2:
   rules:
@@ -843,6 +909,12 @@
       changes: *patterns-build_target_test
     - <<: *if-dev-push
       changes: *patterns-example_test
+    - <<: *if-dev-push
+      changes: *patterns-example_test-related_changes-bt
+    - <<: *if-dev-push
+      changes: *patterns-example_test-related_changes-ota
+    - <<: *if-dev-push
+      changes: *patterns-example_test-related_changes-wifi
 
 .rules:build:example_test-esp32s3:
   rules:
@@ -863,6 +935,12 @@
       changes: *patterns-build_target_test
     - <<: *if-dev-push
       changes: *patterns-example_test
+    - <<: *if-dev-push
+      changes: *patterns-example_test-related_changes-bt
+    - <<: *if-dev-push
+      changes: *patterns-example_test-related_changes-ota
+    - <<: *if-dev-push
+      changes: *patterns-example_test-related_changes-wifi
 
 .rules:build:integration_test:
   rules:
@@ -895,6 +973,8 @@
     - <<: *if-revert-branch
       when: never
     - <<: *if-protected
+    - <<: *if-example_test-related_changes-ota-rule
+    - <<: *if-include_nightly_run-rule
     - <<: *if-label-build
     - <<: *if-label-component_ut
     - <<: *if-label-component_ut_esp32
@@ -943,6 +1023,12 @@
       changes: *patterns-custom_test
     - <<: *if-dev-push
       changes: *patterns-example_test
+    - <<: *if-dev-push
+      changes: *patterns-example_test-related_changes-bt
+    - <<: *if-dev-push
+      changes: *patterns-example_test-related_changes-ota
+    - <<: *if-dev-push
+      changes: *patterns-example_test-related_changes-wifi
     - <<: *if-dev-push
       changes: *patterns-integration_test
     - <<: *if-dev-push
@@ -1286,6 +1372,50 @@
     - <<: *if-dev-push
       changes: *patterns-example_test
 
+.rules:test:example_test-esp32-bt-related_changes:
+  rules:
+    - <<: *if-revert-branch
+      when: never
+    - <<: *if-protected
+    - <<: *if-label-build-only
+      when: never
+    - <<: *if-label-target_test
+    - <<: *if-dev-push
+      changes: *patterns-example_test-related_changes-bt
+
+.rules:test:example_test-esp32-ota-related_changes:
+  rules:
+    - <<: *if-revert-branch
+      when: never
+    - <<: *if-protected
+    - <<: *if-label-build-only
+      when: never
+    - <<: *if-label-target_test
+    - <<: *if-dev-push
+      changes: *patterns-example_test-related_changes-ota
+
+.rules:test:example_test-esp32-ota-related_changes-include_nightly_run:
+  rules:
+    - <<: *if-revert-branch
+      when: never
+    - <<: *if-protected
+    - <<: *if-label-build-only
+      when: never
+    - <<: *if-example_test-related_changes-ota-rule
+    - <<: *if-include_nightly_run-rule
+    - <<: *if-label-target_test
+
+.rules:test:example_test-esp32-wifi-related_changes:
+  rules:
+    - <<: *if-revert-branch
+      when: never
+    - <<: *if-protected
+    - <<: *if-label-build-only
+      when: never
+    - <<: *if-label-target_test
+    - <<: *if-dev-push
+      changes: *patterns-example_test-related_changes-wifi
+
 .rules:test:example_test-esp32c2:
   rules:
     - <<: *if-revert-branch
@@ -1295,6 +1425,39 @@
       when: never
     - <<: *if-label-example_test_esp32c2
 
+.rules:test:example_test-esp32c2-bt-related_changes:
+  rules:
+    - <<: *if-revert-branch
+      when: never
+    - <<: *if-protected
+    - <<: *if-label-build-only
+      when: never
+    - <<: *if-label-target_test
+    - <<: *if-dev-push
+      changes: *patterns-example_test-related_changes-bt
+
+.rules:test:example_test-esp32c2-ota-related_changes:
+  rules:
+    - <<: *if-revert-branch
+      when: never
+    - <<: *if-protected
+    - <<: *if-label-build-only
+      when: never
+    - <<: *if-label-target_test
+    - <<: *if-dev-push
+      changes: *patterns-example_test-related_changes-ota
+
+.rules:test:example_test-esp32c2-wifi-related_changes:
+  rules:
+    - <<: *if-revert-branch
+      when: never
+    - <<: *if-protected
+    - <<: *if-label-build-only
+      when: never
+    - <<: *if-label-target_test
+    - <<: *if-dev-push
+      changes: *patterns-example_test-related_changes-wifi
+
 .rules:test:example_test-esp32c3:
   rules:
     - <<: *if-revert-branch
@@ -1310,6 +1473,50 @@
     - <<: *if-dev-push
       changes: *patterns-example_test
 
+.rules:test:example_test-esp32c3-bt-related_changes:
+  rules:
+    - <<: *if-revert-branch
+      when: never
+    - <<: *if-protected
+    - <<: *if-label-build-only
+      when: never
+    - <<: *if-label-target_test
+    - <<: *if-dev-push
+      changes: *patterns-example_test-related_changes-bt
+
+.rules:test:example_test-esp32c3-ota-related_changes:
+  rules:
+    - <<: *if-revert-branch
+      when: never
+    - <<: *if-protected
+    - <<: *if-label-build-only
+      when: never
+    - <<: *if-label-target_test
+    - <<: *if-dev-push
+      changes: *patterns-example_test-related_changes-ota
+
+.rules:test:example_test-esp32c3-ota-related_changes-include_nightly_run:
+  rules:
+    - <<: *if-revert-branch
+      when: never
+    - <<: *if-protected
+    - <<: *if-label-build-only
+      when: never
+    - <<: *if-example_test-related_changes-ota-rule
+    - <<: *if-include_nightly_run-rule
+    - <<: *if-label-target_test
+
+.rules:test:example_test-esp32c3-wifi-related_changes:
+  rules:
+    - <<: *if-revert-branch
+      when: never
+    - <<: *if-protected
+    - <<: *if-label-build-only
+      when: never
+    - <<: *if-label-target_test
+    - <<: *if-dev-push
+      changes: *patterns-example_test-related_changes-wifi
+
 .rules:test:example_test-esp32h2:
   rules:
     - <<: *if-revert-branch
@@ -1325,6 +1532,39 @@
     - <<: *if-dev-push
       changes: *patterns-example_test
 
+.rules:test:example_test-esp32h2-bt-related_changes:
+  rules:
+    - <<: *if-revert-branch
+      when: never
+    - <<: *if-protected
+    - <<: *if-label-build-only
+      when: never
+    - <<: *if-label-target_test
+    - <<: *if-dev-push
+      changes: *patterns-example_test-related_changes-bt
+
+.rules:test:example_test-esp32h2-ota-related_changes:
+  rules:
+    - <<: *if-revert-branch
+      when: never
+    - <<: *if-protected
+    - <<: *if-label-build-only
+      when: never
+    - <<: *if-label-target_test
+    - <<: *if-dev-push
+      changes: *patterns-example_test-related_changes-ota
+
+.rules:test:example_test-esp32h2-wifi-related_changes:
+  rules:
+    - <<: *if-revert-branch
+      when: never
+    - <<: *if-protected
+    - <<: *if-label-build-only
+      when: never
+    - <<: *if-label-target_test
+    - <<: *if-dev-push
+      changes: *patterns-example_test-related_changes-wifi
+
 .rules:test:example_test-esp32s2:
   rules:
     - <<: *if-revert-branch
@@ -1340,6 +1580,39 @@
     - <<: *if-dev-push
       changes: *patterns-example_test
 
+.rules:test:example_test-esp32s2-bt-related_changes:
+  rules:
+    - <<: *if-revert-branch
+      when: never
+    - <<: *if-protected
+    - <<: *if-label-build-only
+      when: never
+    - <<: *if-label-target_test
+    - <<: *if-dev-push
+      changes: *patterns-example_test-related_changes-bt
+
+.rules:test:example_test-esp32s2-ota-related_changes:
+  rules:
+    - <<: *if-revert-branch
+      when: never
+    - <<: *if-protected
+    - <<: *if-label-build-only
+      when: never
+    - <<: *if-label-target_test
+    - <<: *if-dev-push
+      changes: *patterns-example_test-related_changes-ota
+
+.rules:test:example_test-esp32s2-wifi-related_changes:
+  rules:
+    - <<: *if-revert-branch
+      when: never
+    - <<: *if-protected
+    - <<: *if-label-build-only
+      when: never
+    - <<: *if-label-target_test
+    - <<: *if-dev-push
+      changes: *patterns-example_test-related_changes-wifi
+
 .rules:test:example_test-esp32s3:
   rules:
     - <<: *if-revert-branch
@@ -1349,6 +1622,39 @@
       when: never
     - <<: *if-label-example_test_esp32s3
 
+.rules:test:example_test-esp32s3-bt-related_changes:
+  rules:
+    - <<: *if-revert-branch
+      when: never
+    - <<: *if-protected
+    - <<: *if-label-build-only
+      when: never
+    - <<: *if-label-target_test
+    - <<: *if-dev-push
+      changes: *patterns-example_test-related_changes-bt
+
+.rules:test:example_test-esp32s3-ota-related_changes:
+  rules:
+    - <<: *if-revert-branch
+      when: never
+    - <<: *if-protected
+    - <<: *if-label-build-only
+      when: never
+    - <<: *if-label-target_test
+    - <<: *if-dev-push
+      changes: *patterns-example_test-related_changes-ota
+
+.rules:test:example_test-esp32s3-wifi-related_changes:
+  rules:
+    - <<: *if-revert-branch
+      when: never
+    - <<: *if-protected
+    - <<: *if-label-build-only
+      when: never
+    - <<: *if-label-target_test
+    - <<: *if-dev-push
+      changes: *patterns-example_test-related_changes-wifi
+
 .rules:test:host_test:
   rules:
     - <<: *if-revert-branch

.gitlab/ci/target-test.yml

@@ -83,7 +83,7 @@ example_test_pytest_esp32c3_flash_suspend:
 example_test_pytest_esp32_ethernet_ota:
   extends:
     - .pytest_examples_dir_template
-    - .rules:test:example_test-esp32
+    - .rules:test:example_test-esp32-ota-related_changes
   needs:
     - build_pytest_examples_esp32
   tags: [ esp32, ethernet_ota ]
@@ -91,7 +91,7 @@ example_test_pytest_esp32_ethernet_ota:
 example_test_pytest_esp32_wifi_ota:
   extends:
     - .pytest_examples_dir_template
-    - .rules:test:example_test-esp32
+    - .rules:test:example_test-esp32-ota-related_changes-include_nightly_run
   needs:
     - build_pytest_examples_esp32
   tags: [ esp32, wifi_ota ]
@@ -99,7 +99,7 @@ example_test_pytest_esp32_wifi_ota:
 example_test_pytest_esp32_flash_encryption_ota:
   extends:
     - .pytest_examples_dir_template
-    - .rules:test:example_test-esp32
+    - .rules:test:example_test-esp32-ota-related_changes
   needs:
     - build_pytest_examples_esp32
   tags: [ esp32, flash_encryption_ota ]
@@ -107,7 +107,7 @@ example_test_pytest_esp32_flash_encryption_ota:
 example_test_pytest_esp32c3_flash_encryption_wifi_ota:
   extends:
     - .pytest_examples_dir_template
-    - .rules:test:example_test-esp32c3
+    - .rules:test:example_test-esp32c3-ota-related_changes-include_nightly_run
   needs:
     - build_pytest_examples_esp32c3
   tags: [ esp32c3, flash_encryption_wifi_ota ]
@@ -131,7 +131,7 @@ example_test_pytest_esp32_8mb_flash:
 example_test_pytest_esp32_wifi:
   extends:
     - .pytest_examples_dir_template
-    - .rules:test:example_test-esp32
+    - .rules:test:example_test-esp32-wifi-related_changes
   needs:
     - build_pytest_examples_esp32
   tags: [ esp32, wifi ]
@@ -467,7 +467,9 @@ example_test_004B:
     - Example_TWAI2
 
 example_test_005:
-  extends: .example_test_esp32_template
+  extends:
+    - .example_test_esp32_template
+    - .rules:test:example_test-esp32-bt-related_changes
   tags:
     - ESP32
     - Example_WIFI_BT

conftest.py

@@ -316,7 +316,10 @@ class IdfPytestEmbedded:
                     item.add_marker(_target)
 
         # filter all the test cases with "nightly_run" marker
-        if os.getenv('NIGHTLY_RUN') == '1':
+        if os.getenv('INCLUDE_NIGHTLY_RUN') == '1':
+            # Do not filter nightly_run cases
+            pass
+        elif os.getenv('NIGHTLY_RUN') == '1':
             items[:] = [
                 item for item in items if 'nightly_run' in item_marker_names(item)
             ]

examples/system/ota/advanced_https_ota/pytest_advanced_ota.py

@@ -577,8 +577,8 @@ def test_examples_protocol_advanced_https_ota_example_partial_request(dut: Dut)
 @pytest.mark.esp32s2
 @pytest.mark.esp32s3
 @pytest.mark.wifi_ota
+@pytest.mark.nightly_run
 @pytest.mark.parametrize('config', ['nimble',], indirect=True)
-@pytest.mark.xfail(run=False)
 def test_examples_protocol_advanced_https_ota_example_nimble_gatts(dut: Dut) -> None:
     """
     Run an OTA image update while a BLE GATT Server is running in background. This GATT server will be using NimBLE Host stack.
@@ -621,8 +621,8 @@ def test_examples_protocol_advanced_https_ota_example_nimble_gatts(dut: Dut) ->
 @pytest.mark.esp32s2
 @pytest.mark.esp32s3
 @pytest.mark.wifi_ota
+@pytest.mark.nightly_run
 @pytest.mark.parametrize('config', ['bluedroid',], indirect=True)
-@pytest.mark.xfail(run=False)
 def test_examples_protocol_advanced_https_ota_example_bluedroid_gatts(dut: Dut) -> None:
     """
     Run an OTA image update while a BLE GATT Server is running in background. This GATT server will be using Bluedroid Host stack.

examples/system/ota/otatool/pytest_otatool.py

@@ -11,7 +11,7 @@ from pytest_embedded import Dut
 
 
 @pytest.mark.supported_targets
-@pytest.mark.wifi
+@pytest.mark.generic
 def test_otatool_example(dut: Dut) -> None:
     # Verify factory firmware
     dut.expect('OTA Tool Example')

examples/system/ota/simple_ota_example/pytest_simple_ota.py

@@ -119,7 +119,7 @@ def calc_all_sha256(dut: Dut) -> Tuple[str, str]:
 @pytest.mark.esp32s2
 @pytest.mark.esp32s3
 @pytest.mark.wifi_ota
-@pytest.mark.xfail(run=False)
+@pytest.mark.nightly_run
 def test_examples_protocol_simple_ota_example(dut: Dut) -> None:
     """
     steps: |
@@ -233,9 +233,9 @@ def test_examples_protocol_simple_ota_example_with_flash_encryption(dut: Dut) ->
 
 @pytest.mark.esp32c3
 @pytest.mark.flash_encryption_wifi_ota
+@pytest.mark.nightly_run
 @pytest.mark.parametrize('config', ['flash_enc_wifi',], indirect=True)
 @pytest.mark.parametrize('skip_autoflash', ['y'], indirect=True)
-@pytest.mark.flaky(reruns=3, reruns_delay=5)
 def test_examples_protocol_simple_ota_example_with_flash_encryption_wifi(dut: Dut) -> None:
     """
     steps: |