Pass extra arguments to the testing program with a new "pio test --program-arg" option // Resolve # 3132

HISTORY.rst

@@ -47,13 +47,15 @@ Please check `Migration guide from 5.x to 6.0 <https://docs.platformio.org/en/la
 
   - Refactored from scratch `Unit Testing <https://docs.platformio.org/en/latest/advanced/unit-testing/index.html>`_ solution and its documentation
   - New: `Test Hierarchies <https://docs.platformio.org/en/latest/advanced/unit-testing/structure.html>`_ (`issue #4135 <https://github.com/platformio/platformio-core/issues/4135>`_)
-  - New: `Custom Testing Framework <https://docs.platformio.org/en/latest/advanced/unit-testing/frameworks/custom/index.html>`_
-  - New: Using hardware `Simulators <https://docs.platformio.org/en/latest/advanced/unit-testing/simulators/index.html>`__ for Unit Testing (QEMU, Renode, SimAVR, and custom solutions)
-  - New: `Semihosting <https://docs.platformio.org/en/latest/advanced/unit-testing/semihosting.html>`__ (`issue #3516 <https://github.com/platformio/platformio-core/issues/3516>`_)
   - New: `doctest <https://docs.platformio.org/en/latest/advanced/unit-testing/frameworks/doctest.html>`__ testing framework (`issue #4240 <https://github.com/platformio/platformio-core/issues/4240>`_)
-  - Added a new "test" `build configuration <https://docs.platformio.org/en/latest/projectconf/build_configurations.html>`__
+  - New: `Semihosting <https://docs.platformio.org/en/latest/advanced/unit-testing/semihosting.html>`__ (`issue #3516 <https://github.com/platformio/platformio-core/issues/3516>`_)
+  - New: Hardware `Simulators <https://docs.platformio.org/en/latest/advanced/unit-testing/simulators/index.html>`__ for Unit Testing (QEMU, Renode, SimAVR, and custom solutions)
+  - New: ``test`` `build configuration <https://docs.platformio.org/en/latest/projectconf/build_configurations.html>`__
+  - Added support for a `custom testing framework <https://docs.platformio.org/en/latest/advanced/unit-testing/frameworks/custom/index.html>`_
+  - Added support for a custom `testing command <https://docs.platformio.org/en/latest/projectconf/section_env_test.html#test-testing-command>`__
+  - Added support for a `custom Unity library <https://docs.platformio.org/en/latest/advanced/unit-testing/frameworks/custom/examples/custom_unity_library.html>`__ (`issue #3980 <https://github.com/platformio/platformio-core/issues/3980>`_)
   - Added support for the ``socket://`` and ``rfc2217://`` protocols using `test_port <https://docs.platformio.org/en/latest/projectconf/section_env_test.html#test-port>`__ option (`issue #4229 <https://github.com/platformio/platformio-core/issues/4229>`_)
-  - Added support for a `Custom Unity Library <https://docs.platformio.org/en/latest/advanced/unit-testing/frameworks/custom/examples/custom_unity_library.html>`__ (`issue #3980 <https://github.com/platformio/platformio-core/issues/3980>`_)
+  - Pass extra arguments to the testing program with a new `pio test --program-arg <https://docs.platformio.org/en/latest/core/userguide/cmd_test.html#cmdoption-pio-test-a>`__ option (`issue #3132 <https://github.com/platformio/platformio-core/issues/3132>`_)
   - Generate reports in JUnit and JSON formats using the `pio test --output-format <https://docs.platformio.org/en/latest/core/userguide/cmd_test.html#cmdoption-pio-test-output-format>`__ option  (`issue #2891 <https://github.com/platformio/platformio-core/issues/2891>`_)
   - Provide more information when the native program crashed on a host (errored with a negative return code) (`issue #3429 <https://github.com/platformio/platformio-core/issues/3429>`_)
   - Fixed an issue when command line parameters (``--ignore``, ``--filter``) do not override values defined in the |PIOCONF| (`issue #3845 <https://github.com/platformio/platformio-core/issues/3845>`_)

platformio/test/command.py

@@ -28,19 +28,19 @@ from platformio.test.runners.factory import TestRunnerFactory
 
 
 @click.command("test", short_help="Unit Testing")
-@click.option("--environment", "-e", multiple=True, metavar="<environment>")
+@click.option("--environment", "-e", multiple=True)
 @click.option(
     "--filter",
     "-f",
     multiple=True,
-    metavar="<pattern>",
+    metavar="PATTERN",
     help="Filter tests by a pattern",
 )
 @click.option(
     "--ignore",
     "-i",
     multiple=True,
-    metavar="<pattern>",
+    metavar="PATTERN",
     help="Ignore tests by a pattern",
 )
 @click.option("--upload-port")
@@ -76,6 +76,13 @@ from platformio.test.runners.factory import TestRunnerFactory
     type=click.IntRange(0, 1),
     help="Set initial DTR line state for Serial Monitor",
 )
+@click.option(
+    "-a",
+    "--program-arg",
+    "program_args",
+    multiple=True,
+    help="A program argument (multiple are allowed)",
+)
 @click.option("--output-format", type=click.Choice(["json", "junit"]))
 @click.option(
     "--output-path",
@@ -99,6 +106,7 @@ def test_cmd(  # pylint: disable=too-many-arguments,too-many-locals,redefined-bu
     no_reset,
     monitor_rts,
     monitor_dtr,
+    program_args,
     output_format,
     output_path,
     verbose,
@@ -116,7 +124,7 @@ def test_cmd(  # pylint: disable=too-many-arguments,too-many-locals,redefined-bu
         if verbose:
             click.echo(" (%s)" % ", ".join(test_names))
 
-        test_result = TestResult(os.path.basename(project_dir))
+        test_result = TestResult(project_dir)
         default_envs = config.default_envs()
         for env_name in config.envs():
             for test_name in test_names:
@@ -159,6 +167,7 @@ def test_cmd(  # pylint: disable=too-many-arguments,too-many-locals,redefined-bu
                         no_reset=no_reset,
                         monitor_rts=monitor_rts,
                         monitor_dtr=monitor_dtr,
+                        program_args=program_args,
                     ),
                 )
                 click.echo()

platformio/test/runners/base.py

@@ -38,6 +38,7 @@ class TestRunnerOptions:  # pylint: disable=too-many-instance-attributes
         no_reset=False,
         monitor_rts=None,
         monitor_dtr=None,
+        program_args=None,
     ):
         self.verbose = verbose
         self.without_building = without_building
@@ -49,6 +50,7 @@ class TestRunnerOptions:  # pylint: disable=too-many-instance-attributes
         self.no_reset = no_reset
         self.monitor_rts = monitor_rts
         self.monitor_dtr = monitor_dtr
+        self.program_args = program_args
 
 
 class TestRunnerBase:

platformio/test/runners/readers/program.py

@@ -62,9 +62,10 @@ class ProgramTestOutputReader:
         if custom_testing_command:
             return custom_testing_command
         build_dir = self.test_runner.project_config.get("platformio", "build_dir")
-        return [
-            os.path.join(build_dir, self.test_runner.test_suite.env_name, "program")
-        ]
+        cmd = [os.path.join(build_dir, self.test_runner.test_suite.env_name, "program")]
+        if self.test_runner.options.program_args:
+            cmd.extend(self.test_runner.options.program_args)
+        return cmd
 
     async def gather_results(self):
         exit_future = asyncio.Future(loop=self.aio_loop)

tests/commands/test_test.py

@@ -20,6 +20,7 @@ from pathlib import Path
 import pytest
 
 from platformio import proc
+from platformio.fs import load_json
 from platformio.test.command import test_cmd as pio_test_cmd
 
 
@@ -528,7 +529,6 @@ int main(int argc, char **argv)
         ],
     )
     assert result.exit_code != 0
-
     # test JUnit output
     junit_testsuites = ET.parse(junit_output_path).getroot()
     assert int(junit_testsuites.get("tests")) == 8
@@ -536,8 +536,29 @@ int main(int argc, char **argv)
     assert int(junit_testsuites.get("failures")) == 3
     assert len(junit_testsuites.findall("testsuite")) == 1
     junit_failed_testcase = junit_testsuites.find(
-        ".//testcase[@name='scoped test suite -> part of scoped']"
+        ".//testcase[@name='scoped test suite/part of scoped']"
     )
     assert junit_failed_testcase.get("status") == "FAILED"
     assert junit_failed_testcase.find("failure").get("message") == "Error message"
     assert "TEST SUITE: scoped test suite" in junit_failed_testcase.find("failure").text
+
+    # test program arguments
+    json_output_path = tmp_path / "report.json"
+    result = clirunner.invoke(
+        pio_test_cmd,
+        [
+            "-d",
+            str(project_dir),
+            "--output-format=json",
+            "--output-path",
+            str(json_output_path),
+            "-a",
+            "-aa=1",  # fail after the 1 error
+        ],
+    )
+    assert result.exit_code != 0
+    assert "1 test cases" in result.output
+    # test JSON
+    json_report = load_json(str(json_output_path))
+    assert json_report["testcase_nums"] == 1
+    assert json_report["failure_nums"] == 1