List available project tests with a new "pio test --list-tests" option

HISTORY.rst

@@ -9,12 +9,12 @@ Release Notes
 PlatformIO Core 6
 -----------------
 
-**A professional collaborative platform for safety-critical and declarative embedded development**
+**A professional collaborative platform for declarative, safety-critical, and test-driven embedded development.**
 
 6.0.0 (2022-??-??)
 ~~~~~~~~~~~~~~~~~~
 
-Please check `Migration guide from 5.x to 6.0 <https://docs.platformio.org/en/latest/core/migration.html>`__.
+Please check the `Migration guide from 5.x to 6.0 <https://docs.platformio.org/en/latest/core/migration.html>`__.
 
 * **Package Management**
 
@@ -56,9 +56,10 @@ Please check `Migration guide from 5.x to 6.0 <https://docs.platformio.org/en/la
   - Added support for a custom `testing command <https://docs.platformio.org/en/latest/projectconf/section_env_test.html#test-testing-command>`__
   - Added support for a `custom Unity library <https://docs.platformio.org/en/latest/advanced/unit-testing/frameworks/custom/examples/custom_unity_library.html>`__ (`issue #3980 <https://github.com/platformio/platformio-core/issues/3980>`_)
   - Added support for the ``socket://`` and ``rfc2217://`` protocols using `test_port <https://docs.platformio.org/en/latest/projectconf/section_env_test.html#test-port>`__ option (`issue #4229 <https://github.com/platformio/platformio-core/issues/4229>`_)
+  - List available project tests with a new `pio test --list-tests <https://docs.platformio.org/en/latest/core/userguide/cmd_test.html#cmdoption-pio-test-list-tests>`__ option
   - Pass extra arguments to the testing program with a new `pio test --program-arg <https://docs.platformio.org/en/latest/core/userguide/cmd_test.html#cmdoption-pio-test-a>`__ option (`issue #3132 <https://github.com/platformio/platformio-core/issues/3132>`_)
   - Generate reports in JUnit and JSON formats using the `pio test <https://docs.platformio.org/en/latest/core/userguide/cmd_test.html>`__ command (`issue #2891 <https://github.com/platformio/platformio-core/issues/2891>`_)
-  - Provide more information when the native program crashed on a host (errored with a negative return code) (`issue #3429 <https://github.com/platformio/platformio-core/issues/3429>`_)
+  - Provide more information when the native program crashed on a host (errored with a non-zero return code) (`issue #3429 <https://github.com/platformio/platformio-core/issues/3429>`_)
   - Fixed an issue when command line parameters (``--ignore``, ``--filter``) do not override values defined in the |PIOCONF| (`issue #3845 <https://github.com/platformio/platformio-core/issues/3845>`_)
   - Renamed the "test_build_project_src" project configuration option to the `test_build_src <https://docs.platformio.org/en/latest//projectconf/section_env_test.html#test-build-src>`__
   - Removed the "test_transport" option in favor of the `Custom "unity_config.h" <https://docs.platformio.org/en/latest/advanced/unit-testing/frameworks/unity.html>`_
@@ -85,7 +86,7 @@ Please check `Migration guide from 5.x to 6.0 <https://docs.platformio.org/en/la
 * **Integration**
 
   - Added a new build variable (``COMPILATIONDB_INCLUDE_TOOLCHAIN``) to include toolchain paths in the compilation database (`issue #3735 <https://github.com/platformio/platformio-core/issues/3735>`_)
-  - Changed default path for compilation database `compile_commands.json <https://docs.platformio.org/en/latest/integration/compile_commands.html>`__ to the project root
+  - Changed a default path for compilation database `compile_commands.json <https://docs.platformio.org/en/latest/integration/compile_commands.html>`__ to the project root
 
 * **Project Configuration**
 

platformio/debug/helpers.py

@@ -26,7 +26,7 @@ from platformio.commands.run.command import cli as cmd_run
 from platformio.commands.run.command import print_processing_header
 from platformio.compat import IS_WINDOWS, is_bytes
 from platformio.debug.exception import DebugInvalidOptionsError
-from platformio.test.command import get_test_names
+from platformio.test.helpers import list_test_names
 from platformio.test.result import TestSuite
 from platformio.test.runners.base import TestRunnerOptions
 from platformio.test.runners.factory import TestRunnerFactory
@@ -82,7 +82,7 @@ def predebug_project(
 ):  # pylint: disable=too-many-arguments
     debug_testname = project_config.get("env:" + env_name, "debug_test")
     if debug_testname:
-        test_names = get_test_names(project_config)
+        test_names = list_test_names(project_config)
         if debug_testname not in test_names:
             raise DebugInvalidOptionsError(
                 "Unknown test name `%s`. Valid names are `%s`"

platformio/test/command.py

@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import fnmatch
 import os
 import shutil
 
@@ -20,9 +19,9 @@ import click
 
 from platformio import app, exception, fs, util
 from platformio.project.config import ProjectConfig
-from platformio.test.exception import TestDirNotExistsError
+from platformio.test.helpers import list_test_suites
 from platformio.test.reports.base import TestReportFactory
-from platformio.test.result import TestResult, TestStatus, TestSuite
+from platformio.test.result import TestResult, TestStatus
 from platformio.test.runners.base import TestRunnerOptions
 from platformio.test.runners.factory import TestRunnerFactory
 
@@ -83,6 +82,7 @@ from platformio.test.runners.factory import TestRunnerFactory
     multiple=True,
     help="A program argument (multiple are allowed)",
 )
+@click.option("--list-tests", is_flag=True)
 @click.option("--json-output", type=click.Path(resolve_path=True))
 @click.option("--junit-output", type=click.Path(resolve_path=True))
 @click.option("--verbose", "-v", is_flag=True)
@@ -103,6 +103,7 @@ def test_cmd(  # pylint: disable=too-many-arguments,too-many-locals,redefined-bu
     monitor_rts,
     monitor_dtr,
     program_args,
+    list_tests,
     json_output,
     junit_output,
     verbose,
@@ -110,9 +111,14 @@ def test_cmd(  # pylint: disable=too-many-arguments,too-many-locals,redefined-bu
     app.set_session_var("custom_project_conf", project_conf)
 
     with fs.cd(project_dir):
-        config = ProjectConfig.get_instance(project_conf)
+        project_config = ProjectConfig.get_instance(project_conf)
-        config.validate(envs=environment)
+        project_config.validate(envs=environment)
-        test_names = get_test_names(config)
+
+        test_result = TestResult(project_dir)
+        test_suites = list_test_suites(
+            project_config, environments=environment, filters=filter, ignores=ignore
+        )
+        test_names = sorted(set(s.test_name for s in test_suites))
 
         if not verbose:
             click.echo("Verbose mode can be enabled via `-v, --verbose` option")
@@ -120,62 +126,36 @@ def test_cmd(  # pylint: disable=too-many-arguments,too-many-locals,redefined-bu
         if verbose:
             click.echo(" (%s)" % ", ".join(test_names))
 
-        test_result = TestResult(project_dir)
+        for test_suite in test_suites:
-        default_envs = config.default_envs()
+            test_result.add_suite(test_suite)
-        for env_name in config.envs():
+            if list_tests or test_suite.is_finished():  # skipped by user
-            for test_name in test_names:
+                continue
-                test_suite = TestSuite(env_name, test_name)
+            runner = TestRunnerFactory.new(
-                test_result.add_suite(test_suite)
+                test_suite,
-
+                project_config,
-                # filter and ignore patterns
+                TestRunnerOptions(
-                patterns = dict(filter=list(filter), ignore=list(ignore))
+                    verbose=verbose,
-                for key in patterns:
+                    without_building=without_building,
-                    if patterns[key]:  # overriden from CLI
+                    without_uploading=without_uploading,
-                        continue
+                    without_testing=without_testing,
-                    patterns[key].extend(
+                    upload_port=upload_port,
-                        config.get(f"env:{env_name}", f"test_{key}", [])
+                    test_port=test_port,
-                    )
+                    no_reset=no_reset,
-
+                    monitor_rts=monitor_rts,
-                skip_conditions = [
+                    monitor_dtr=monitor_dtr,
-                    environment and env_name not in environment,
+                    program_args=program_args,
-                    not environment and default_envs and env_name not in default_envs,
+                ),
-                    test_name != "*"
+            )
-                    and patterns["filter"]
+            click.echo()
-                    and not any(
+            print_suite_header(test_suite)
-                        fnmatch.fnmatch(test_name, p) for p in patterns["filter"]
+            runner.start(ctx)
-                    ),
+            print_suite_footer(test_suite)
-                    test_name != "*"
-                    and any(fnmatch.fnmatch(test_name, p) for p in patterns["ignore"]),
-                ]
-                if any(skip_conditions):
-                    continue
-
-                runner = TestRunnerFactory.new(
-                    test_suite,
-                    config,
-                    TestRunnerOptions(
-                        verbose=verbose,
-                        without_building=without_building,
-                        without_uploading=without_uploading,
-                        without_testing=without_testing,
-                        upload_port=upload_port,
-                        test_port=test_port,
-                        no_reset=no_reset,
-                        monitor_rts=monitor_rts,
-                        monitor_dtr=monitor_dtr,
-                        program_args=program_args,
-                    ),
-                )
-                click.echo()
-                print_suite_header(test_suite)
-                runner.start(ctx)
-                print_suite_footer(test_suite)
 
     # Reset custom project config
     app.set_session_var("custom_project_conf", None)
 
     stdout_report = TestReportFactory.new("stdout", test_result)
-    stdout_report.generate(verbose=verbose)
+    stdout_report.generate(verbose=verbose or list_tests)
 
     for output_format, output_path in [("json", json_output), ("junit", junit_output)]:
         if not output_path:
@@ -187,20 +167,6 @@ def test_cmd(  # pylint: disable=too-many-arguments,too-many-locals,redefined-bu
         raise exception.ReturnErrorCode(1)
 
 
-def get_test_names(config):
-    test_dir = config.get("platformio", "test_dir")
-    if not os.path.isdir(test_dir):
-        raise TestDirNotExistsError(test_dir)
-    names = []
-    for root, _, __ in os.walk(test_dir):
-        if not os.path.basename(root).startswith("test_"):
-            continue
-        names.append(os.path.relpath(root, test_dir))
-    if not names:
-        names = ["*"]
-    return names
-
-
 def print_suite_header(test_suite):
     click.echo(
         "Processing %s in %s environment"

platformio/test/helpers.py

@@ -0,0 +1,62 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+from fnmatch import fnmatch
+
+from platformio.test.exception import TestDirNotExistsError
+from platformio.test.result import TestSuite
+
+
+def list_test_names(project_config):
+    test_dir = project_config.get("platformio", "test_dir")
+    if not os.path.isdir(test_dir):
+        raise TestDirNotExistsError(test_dir)
+    names = []
+    for root, _, __ in os.walk(test_dir):
+        if not os.path.basename(root).startswith("test_"):
+            continue
+        names.append(os.path.relpath(root, test_dir).replace("\\", "/"))
+    if not names:
+        names = ["*"]
+    return names
+
+
+def list_test_suites(project_config, environments, filters, ignores):
+    result = []
+    default_envs = project_config.default_envs()
+    test_names = list_test_names(project_config)
+    for env_name in project_config.envs():
+        for test_name in test_names:
+
+            # filter and ignore patterns
+            patterns = dict(filter=list(filters), ignore=list(ignores))
+            for key in patterns:
+                if patterns[key]:  # overriden from CLI
+                    continue
+                patterns[key].extend(
+                    project_config.get(f"env:{env_name}", f"test_{key}", [])
+                )
+
+            skip_conditions = [
+                environments and env_name not in environments,
+                not environments and default_envs and env_name not in default_envs,
+                test_name != "*"
+                and patterns["filter"]
+                and not any(fnmatch(test_name, p) for p in patterns["filter"]),
+                test_name != "*"
+                and any(fnmatch(test_name, p) for p in patterns["ignore"]),
+            ]
+            result.append(TestSuite(env_name, test_name, finished=any(skip_conditions)))
+    return result

platformio/test/result.py

@@ -93,13 +93,13 @@ class TestCase:
 
 
 class TestSuite:
-    def __init__(self, env_name, test_name):
+    def __init__(self, env_name, test_name, finished=False):
         self.env_name = env_name
         self.test_name = test_name
         self.timestamp = 0
         self.duration = 0
         self._cases = []
-        self._finished = False
+        self._finished = finished
 
     @property
     def cases(self):

tests/commands/test_test.py

@@ -65,6 +65,27 @@ def test_calculator_example(tmp_path: Path):
     assert junit_failed_testcase.find("failure").get("message") == "Expected 32 Was 33"
 
 
+def test_list_tests(clirunner, validate_cliresult, tmp_path: Path):
+    json_output_path = tmp_path / "report.json"
+    result = clirunner.invoke(
+        pio_test_cmd,
+        [
+            "-d",
+            os.path.join("examples", "unit-testing", "calculator"),
+            "--list-tests",
+            "--json-output",
+            str(json_output_path),
+        ],
+    )
+    validate_cliresult(result)
+    # test JSON
+    json_report = load_json(str(json_output_path))
+    assert json_report["testcase_nums"] == 0
+    assert json_report["failure_nums"] == 0
+    assert json_report["skipped_nums"] == 0
+    assert len(json_report["test_suites"]) == 6
+
+
 def test_group_and_custom_runner(clirunner, validate_cliresult, tmp_path: Path):
     project_dir = tmp_path / "project"
     project_dir.mkdir()