New Custom Testing Framework

HISTORY.rst

@@ -44,9 +44,10 @@ Please check `Migration guide from 5.x to 6.0 <https://docs.platformio.org/en/la
 
 * **Unit Testing**
 
-  - New `Unit Testing <https://docs.platformio.org/en/latest/advanced/unit-testing/index.html>`_ engine and documentation
+  - New `Unit Testing <https://docs.platformio.org/en/latest/advanced/unit-testing/index.html>`_ solution and documentation
-  - Added support for `Test Hierarchies <https://docs.platformio.org/en/latest/advanced/unit-testing/structure.html>`_ (`issue #4135 <https://github.com/platformio/platformio-core/issues/4135>`_)
+  - New `Test Hierarchies <https://docs.platformio.org/en/latest/advanced/unit-testing/structure.html>`_ (`issue #4135 <https://github.com/platformio/platformio-core/issues/4135>`_)
-  - Added a new "test" `build configuration <https://docs.platformio.org/en/latest/projectconf/build_configurations.html>`__
+  - New `Custom Testing Framework <https://docs.platformio.org/en/latest/advanced/unit-testing/frameworks/custom/index.html>`_
+  - New "test" `build configuration <https://docs.platformio.org/en/latest/projectconf/build_configurations.html>`__
   - Generate reports in JUnit and JSON formats using the `pio test --output-format <https://docs.platformio.org/en/latest/core/userguide/cmd_test.html#cmdoption-pio-test-output-format>`__ option  (`issue #2891 <https://github.com/platformio/platformio-core/issues/2891>`_)
   - Provide more information when the native program crashed on a host (errored with a negative return code) (`issue #3429 <https://github.com/platformio/platformio-core/issues/3429>`_)
   - Fixed an issue when command line parameters ("--ignore", "--filter") do not override values defined in the |PIOCONF| (`issue #3845 <https://github.com/platformio/platformio-core/issues/3845>`_)

platformio/commands/run/processor.py

@@ -68,7 +68,10 @@ class EnvironmentProcessor(object):
         if "clean" not in build_targets:
             install_project_env_dependencies(
                 self.name,
-                {"project_targets": build_targets},
+                {
+                    "project_targets": build_targets,
+                    "piotest_running_name": build_vars.get("piotest_running_name"),
+                },
             )
 
         result = PlatformFactory.new(self.options["platform"], autoinstall=True).run(

platformio/package/commands/install.py

@@ -216,7 +216,9 @@ def _install_project_env_libraries(project_env, options):
 
     lib_deps = config.get(f"env:{project_env}", "lib_deps")
     if "__test" in options.get("project_targets", []):
-        test_runner = TestRunnerFactory.new(TestSuite(project_env, "*"), config)
+        test_runner = TestRunnerFactory.new(
+            TestSuite(project_env, options.get("piotest_running_name", "*")), config
+        )
         lib_deps.extend(test_runner.EXTRA_LIB_DEPS or [])
 
     for library in lib_deps:

platformio/test/runners/factory.py

@@ -41,14 +41,24 @@ class TestRunnerFactory(object):
         module_name = f"platformio.test.runners.{test_framework}"
         runner_cls = None
         if test_framework == "custom":
-            custom_runner_path = os.path.join(
+            test_dir = project_config.get("platformio", "test_dir")
-                project_config.get("platformio", "test_dir"), "custom_runner.py"
+            custom_runner_path = os.path.join(test_dir, "custom_test_runner.py")
-            )
+            test_name = test_suite.test_name if test_suite.test_name != "*" else None
+            while test_name:
+                if os.path.isfile(
+                    os.path.join(test_dir, test_name, "custom_test_runner.py")
+                ):
+                    custom_runner_path = os.path.join(
+                        test_dir, test_name, "custom_test_runner.py"
+                    )
+                    break
+                test_name = os.path.dirname(test_name)  # parent dir
+
             try:
                 mod = load_python_module(module_name, custom_runner_path)
-            except ImportError:
+            except (FileNotFoundError, ImportError):
                 raise UserSideException(
-                    "Could not find custom unit testing runner "
+                    "Could not find custom test runner "
                     f"by this path -> {custom_runner_path}"
                 )
         else:

tests/commands/test_test.py

@@ -62,13 +62,14 @@ def test_calculator_example(tmp_path: Path):
     assert junit_failed_testcase.find("failure").get("message") == "Expected 32 Was 33"
 
 
-def test_nested_suites(clirunner, validate_cliresult, tmp_path: Path):
+def test_group_and_custom_runner(clirunner, validate_cliresult, tmp_path: Path):
     project_dir = tmp_path / "project"
     project_dir.mkdir()
     (project_dir / "platformio.ini").write_text(
         """
 [env:native]
 platform = native
+test_framework = custom
 """
     )
     test_dir = project_dir / "test"
@@ -108,8 +109,23 @@ void my_extra_fun(void) {
 """
     )
 
+    # test group
+    test_group = test_dir / "group"
+    test_group.mkdir(parents=True)
+    (test_group / "custom_test_runner.py").write_text(
+        """
+import click
+
+from platformio.test.runners.unity import UnityTestRunner
+
+class CustomTestRunner(UnityTestRunner):
+    def teardown(self):
+        click.echo("CustomTestRunner::TearDown called")
+"""
+    )
+
     # test suite
-    test_suite_dir = test_dir / "set" / "test_nested"
+    test_suite_dir = test_group / "test_nested"
     test_include_dir = test_suite_dir / "include"
     test_include_dir.mkdir(parents=True)
     (test_include_dir / "my_nested.h").write_text(
@@ -148,6 +164,7 @@ int main() {
     )
     validate_cliresult(result)
     assert "Called from my_extra_fun" in result.output
+    assert "CustomTestRunner::TearDown called" in result.output
     assert "Disabled test suite" not in result.output