diff --git a/.gitlab/ci/rules.yml b/.gitlab/ci/rules.yml
index d7207e9dea..9f39debe8b 100644
--- a/.gitlab/ci/rules.yml
+++ b/.gitlab/ci/rules.yml
@@ -24,6 +24,7 @@
   - "tools/ci/sonar_exclude_list.txt"
 
 .patterns-example_test: &patterns-example_test
+  - "tools/ci/idf_pytest/**/*"
   - "tools/ci/python_packages/gitlab_api.py"
   - "tools/ci/python_packages/idf_http_server_test/**/*"
   - "tools/ci/python_packages/idf_iperf_test_util/**/*"
@@ -70,6 +71,7 @@
   - "tools/test_sbom/*"
 
 .patterns-custom_test: &patterns-custom_test
+  - "tools/ci/idf_pytest/**/*"
   - "tools/ci/python_packages/gitlab_api.py"
   - "tools/ci/python_packages/common_test_methods.py"
 
@@ -77,6 +79,7 @@
   - "tools/ldgen/**/*"
 
 .patterns-component_ut: &patterns-component_ut
+  - "tools/ci/idf_pytest/**/*"
   - "tools/ci/python_packages/gitlab_api.py"
   - "tools/ci/python_packages/common_test_methods.py"
   - "tools/test_apps/configs/sdkconfig.debug_helpers"
diff --git a/tools/ci/exclude_check_tools_files.txt b/tools/ci/exclude_check_tools_files.txt
index 80c982710f..01f0f5a95c 100644
--- a/tools/ci/exclude_check_tools_files.txt
+++ b/tools/ci/exclude_check_tools_files.txt
@@ -39,6 +39,5 @@ tools/templates/sample_component/CMakeLists.txt
 tools/templates/sample_component/include/main.h
 tools/templates/sample_component/main.c
 tools/ci/cleanup_ignore_lists.py
-tools/ci/idf_pytest/**/*
 tools/ci/artifacts_handler.py
 tools/unit-test-app/**/*
diff --git a/tools/ci/idf_pytest/plugin.py b/tools/ci/idf_pytest/plugin.py
index b9dd1ed9a8..c5c5576a05 100644
--- a/tools/ci/idf_pytest/plugin.py
+++ b/tools/ci/idf_pytest/plugin.py
@@ -49,7 +49,7 @@ class IdfPytestEmbedded:
         # implement like this since this is a limitation of pytest, couldn't get fixture values while collecting
         # https://github.com/pytest-dev/pytest/discussions/9689
         if not hasattr(item, 'callspec'):
-            raise ValueError(f'Function {item} does not have params')
+            return default
 
         return item.callspec.params.get(key, default) or default
 
@@ -177,7 +177,11 @@ class IdfPytestEmbedded:
             # finally!
             filtered_items.append(item)
 
-        items[:] = filtered_items[:]
+        # sort the test cases with (app folder, config)
+        items[:] = sorted(
+            filtered_items,
+            key=lambda x: (os.path.dirname(x.path), self.get_param(x, 'config', DEFAULT_SDKCONFIG))
+        )
 
     def pytest_report_collectionfinish(self, items: t.List[Function]) -> None:
         for item in items: