From daa34818622f09b2c9fa7743289f3d54bc46edf0 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 7 May 2022 13:31:19 +0300 Subject: [PATCH] Pass extra arguments to the testing program with a new "pio test --program-arg" option // Resolve # 3132 --- HISTORY.rst | 12 ++++++----- docs | 2 +- platformio/test/command.py | 17 +++++++++++---- platformio/test/runners/base.py | 2 ++ platformio/test/runners/readers/program.py | 7 +++--- tests/commands/test_test.py | 25 ++++++++++++++++++++-- 6 files changed, 50 insertions(+), 15 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 141408ee..8930b255 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -47,13 +47,15 @@ Please check `Migration guide from 5.x to 6.0 `_ solution and its documentation - New: `Test Hierarchies `_ (`issue #4135 `_) - - New: `Custom Testing Framework `_ - - New: Using hardware `Simulators `__ for Unit Testing (QEMU, Renode, SimAVR, and custom solutions) - - New: `Semihosting `__ (`issue #3516 `_) - New: `doctest `__ testing framework (`issue #4240 `_) - - Added a new "test" `build configuration `__ + - New: `Semihosting `__ (`issue #3516 `_) + - New: Hardware `Simulators `__ for Unit Testing (QEMU, Renode, SimAVR, and custom solutions) + - New: ``test`` `build configuration `__ + - Added support for a `custom testing framework `_ + - Added support for a custom `testing command `__ + - Added support for a `custom Unity library `__ (`issue #3980 `_) - Added support for the ``socket://`` and ``rfc2217://`` protocols using `test_port `__ option (`issue #4229 `_) - - Added support for a `Custom Unity Library `__ (`issue #3980 `_) + - Pass extra arguments to the testing program with a new `pio test --program-arg `__ option (`issue #3132 `_) - Generate reports in JUnit and JSON formats using the `pio test --output-format `__ option (`issue #2891 `_) - Provide more information when the native program crashed on a host (errored with a negative return code) (`issue #3429 `_) - Fixed an issue when command line parameters (``--ignore``, ``--filter``) do not override values defined in the |PIOCONF| (`issue #3845 `_) diff --git a/docs b/docs index f1012c0c..bd05ccee 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit f1012c0c989d1baf28c0c10b4be6cc429a919eee +Subproject commit bd05ccee932129856ccd4c46a5abe67594402b72 diff --git a/platformio/test/command.py b/platformio/test/command.py index 32663936..b9171399 100644 --- a/platformio/test/command.py +++ b/platformio/test/command.py @@ -28,19 +28,19 @@ from platformio.test.runners.factory import TestRunnerFactory @click.command("test", short_help="Unit Testing") -@click.option("--environment", "-e", multiple=True, metavar="") +@click.option("--environment", "-e", multiple=True) @click.option( "--filter", "-f", multiple=True, - metavar="", + metavar="PATTERN", help="Filter tests by a pattern", ) @click.option( "--ignore", "-i", multiple=True, - metavar="", + metavar="PATTERN", help="Ignore tests by a pattern", ) @click.option("--upload-port") @@ -76,6 +76,13 @@ from platformio.test.runners.factory import TestRunnerFactory type=click.IntRange(0, 1), help="Set initial DTR line state for Serial Monitor", ) +@click.option( + "-a", + "--program-arg", + "program_args", + multiple=True, + help="A program argument (multiple are allowed)", +) @click.option("--output-format", type=click.Choice(["json", "junit"])) @click.option( "--output-path", @@ -99,6 +106,7 @@ def test_cmd( # pylint: disable=too-many-arguments,too-many-locals,redefined-bu no_reset, monitor_rts, monitor_dtr, + program_args, output_format, output_path, verbose, @@ -116,7 +124,7 @@ def test_cmd( # pylint: disable=too-many-arguments,too-many-locals,redefined-bu if verbose: click.echo(" (%s)" % ", ".join(test_names)) - test_result = TestResult(os.path.basename(project_dir)) + test_result = TestResult(project_dir) default_envs = config.default_envs() for env_name in config.envs(): for test_name in test_names: @@ -159,6 +167,7 @@ def test_cmd( # pylint: disable=too-many-arguments,too-many-locals,redefined-bu no_reset=no_reset, monitor_rts=monitor_rts, monitor_dtr=monitor_dtr, + program_args=program_args, ), ) click.echo() diff --git a/platformio/test/runners/base.py b/platformio/test/runners/base.py index eb13ca2a..f5dfd83f 100644 --- a/platformio/test/runners/base.py +++ b/platformio/test/runners/base.py @@ -38,6 +38,7 @@ class TestRunnerOptions: # pylint: disable=too-many-instance-attributes no_reset=False, monitor_rts=None, monitor_dtr=None, + program_args=None, ): self.verbose = verbose self.without_building = without_building @@ -49,6 +50,7 @@ class TestRunnerOptions: # pylint: disable=too-many-instance-attributes self.no_reset = no_reset self.monitor_rts = monitor_rts self.monitor_dtr = monitor_dtr + self.program_args = program_args class TestRunnerBase: diff --git a/platformio/test/runners/readers/program.py b/platformio/test/runners/readers/program.py index 4ceb5c0d..391c87af 100644 --- a/platformio/test/runners/readers/program.py +++ b/platformio/test/runners/readers/program.py @@ -62,9 +62,10 @@ class ProgramTestOutputReader: if custom_testing_command: return custom_testing_command build_dir = self.test_runner.project_config.get("platformio", "build_dir") - return [ - os.path.join(build_dir, self.test_runner.test_suite.env_name, "program") - ] + cmd = [os.path.join(build_dir, self.test_runner.test_suite.env_name, "program")] + if self.test_runner.options.program_args: + cmd.extend(self.test_runner.options.program_args) + return cmd async def gather_results(self): exit_future = asyncio.Future(loop=self.aio_loop) diff --git a/tests/commands/test_test.py b/tests/commands/test_test.py index 77376a2c..362ae5a3 100644 --- a/tests/commands/test_test.py +++ b/tests/commands/test_test.py @@ -20,6 +20,7 @@ from pathlib import Path import pytest from platformio import proc +from platformio.fs import load_json from platformio.test.command import test_cmd as pio_test_cmd @@ -528,7 +529,6 @@ int main(int argc, char **argv) ], ) assert result.exit_code != 0 - # test JUnit output junit_testsuites = ET.parse(junit_output_path).getroot() assert int(junit_testsuites.get("tests")) == 8 @@ -536,8 +536,29 @@ int main(int argc, char **argv) assert int(junit_testsuites.get("failures")) == 3 assert len(junit_testsuites.findall("testsuite")) == 1 junit_failed_testcase = junit_testsuites.find( - ".//testcase[@name='scoped test suite -> part of scoped']" + ".//testcase[@name='scoped test suite/part of scoped']" ) assert junit_failed_testcase.get("status") == "FAILED" assert junit_failed_testcase.find("failure").get("message") == "Error message" assert "TEST SUITE: scoped test suite" in junit_failed_testcase.find("failure").text + + # test program arguments + json_output_path = tmp_path / "report.json" + result = clirunner.invoke( + pio_test_cmd, + [ + "-d", + str(project_dir), + "--output-format=json", + "--output-path", + str(json_output_path), + "-a", + "-aa=1", # fail after the 1 error + ], + ) + assert result.exit_code != 0 + assert "1 test cases" in result.output + # test JSON + json_report = load_json(str(json_output_path)) + assert json_report["testcase_nums"] == 1 + assert json_report["failure_nums"] == 1