From 82778473fecdb3b23dea52c9046b06d5006497e1 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 6 May 2022 20:00:23 +0300 Subject: [PATCH] New: "doctest" testing framework // Resolve #4240 --- HISTORY.rst | 1 + docs | 2 +- platformio/project/options.py | 2 +- platformio/test/runners/doctest.py | 129 +++++++++++++++++++++++++++++ tests/commands/test_test.py | 108 ++++++++++++++++++++++++ 5 files changed, 240 insertions(+), 2 deletions(-) create mode 100644 platformio/test/runners/doctest.py diff --git a/HISTORY.rst b/HISTORY.rst index 220a2576..141408ee 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -50,6 +50,7 @@ Please check `Migration guide from 5.x to 6.0 `_ - New: Using hardware `Simulators `__ for Unit Testing (QEMU, Renode, SimAVR, and custom solutions) - New: `Semihosting `__ (`issue #3516 `_) + - New: `doctest `__ testing framework (`issue #4240 `_) - Added a new "test" `build configuration `__ - Added support for the ``socket://`` and ``rfc2217://`` protocols using `test_port `__ option (`issue #4229 `_) - Added support for a `Custom Unity Library `__ (`issue #3980 `_) diff --git a/docs b/docs index f9b22185..f1012c0c 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit f9b221850985ffe88020d8bb2f36230b47542050 +Subproject commit f1012c0c989d1baf28c0c10b4be6cc429a919eee diff --git a/platformio/project/options.py b/platformio/project/options.py index 0981e8c7..a573ad81 100644 --- a/platformio/project/options.py +++ b/platformio/project/options.py @@ -652,7 +652,7 @@ ProjectOptions = OrderedDict( group="test", name="test_framework", description="A unit testing framework", - type=click.Choice(["unity", "custom"]), + type=click.Choice(["doctest", "unity", "custom"]), default="unity", ), ConfigEnvOption( diff --git a/platformio/test/runners/doctest.py b/platformio/test/runners/doctest.py new file mode 100644 index 00000000..f409fa74 --- /dev/null +++ b/platformio/test/runners/doctest.py @@ -0,0 +1,129 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import click + +from platformio.test.result import TestCase, TestCaseSource, TestStatus +from platformio.test.runners.base import TestRunnerBase + + +class DoctestTestCaseParser: + def __init__(self): + self._tmp_tc = None + self._name_tokens = [] + + def parse(self, line): + if line.strip().startswith("[doctest]"): + return None + if self.is_divider(line): + return self._on_divider() + + if not self._tmp_tc: + self._tmp_tc = TestCase("", TestStatus.PASSED, stdout="") + self._name_tokens = [] + + self._tmp_tc.stdout += line + line = line.strip() + + # source + if not self._tmp_tc.source and line: + self._tmp_tc.source = self.parse_source(line) + return None + + # name + if not self._tmp_tc.name: + if line: + self._name_tokens.append(line) + return None + self._tmp_tc.name = self.parse_name(self._name_tokens) + return None + + if self._tmp_tc.status != TestStatus.FAILED: + self._parse_assert(line) + + return None + + @staticmethod + def is_divider(line): + line = line.strip() + return line.startswith("===") and line.endswith("===") + + def _on_divider(self): + # if the first unprocessed test case + if not self._tmp_tc: + return None + test_case = TestCase( + name=self._tmp_tc.name, + status=self._tmp_tc.status, + message=self._tmp_tc.message, + source=self._tmp_tc.source, + stdout=self._tmp_tc.stdout, + ) + self._tmp_tc = None + return test_case + + @staticmethod + def parse_source(line): + assert line.endswith(":"), line + file_, line = line[:-1].rsplit(":", 1) + return TestCaseSource(file_, int(line)) + + @staticmethod + def parse_name(tokens): + cleaned_tokens = [] + for token in tokens: + if token.startswith("TEST ") and ":" in token: + token = token[token.index(":") + 1 :] + cleaned_tokens.append(token.strip()) + return " -> ".join(cleaned_tokens) + + def _parse_assert(self, line): + status_tokens = [ + (TestStatus.FAILED, "ERROR"), + (TestStatus.FAILED, "FATAL ERROR"), + (TestStatus.WARNED, "WARNING"), + ] + for status, token in status_tokens: + index = line.find(": %s:" % token) + if index == -1: + continue + self._tmp_tc.status = status + self._tmp_tc.message = line[index + len(token) + 3 :].strip() or None + + +class DoctestTestRunner(TestRunnerBase): + + EXTRA_LIB_DEPS = ["doctest/doctest@^2.4.8"] + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._tc_parser = DoctestTestCaseParser() + + def configure_build_env(self, env): + if "-std=" not in env.subst("$CXXFLAGS"): + env.Append(CXXFLAGS=["-std=c++11"]) + env.Append(CPPDEFINES=["DOCTEST_CONFIG_COLORS_NONE"]) + + def on_testing_line_output(self, line): + if self.options.verbose: + click.echo(line, nl=False) + + test_case = self._tc_parser.parse(line) + if test_case: + self._tc_parser = DoctestTestCaseParser() + click.echo(test_case.humanize()) + self.test_suite.add_case(test_case) + + if "[doctest] Status:" in line: + self.test_suite.on_finish() diff --git a/tests/commands/test_test.py b/tests/commands/test_test.py index a21eaed6..258994d4 100644 --- a/tests/commands/test_test.py +++ b/tests/commands/test_test.py @@ -433,3 +433,111 @@ void unittest_uart_end(){} ], ) validate_cliresult(result) + + +def test_doctest_framework(clirunner, tmp_path: Path): + project_dir = tmp_path / "project" + project_dir.mkdir() + (project_dir / "platformio.ini").write_text( + """ +[env:native] +platform = native +test_framework = doctest +""" + ) + test_dir = project_dir / "test" / "test_dummy" + test_dir.mkdir(parents=True) + (test_dir / "test_main.cpp").write_text( + """ +#define DOCTEST_CONFIG_IMPLEMENT +#include + +TEST_CASE("[math] basic stuff") +{ + CHECK(6 > 5); + CHECK(6 > 7); +} + +TEST_CASE("should be skipped " * doctest::skip()) +{ + CHECK(2 > 5); +} + +TEST_CASE("vectors can be sized and resized") +{ + std::vector v(5); + + REQUIRE(v.size() == 5); + REQUIRE(v.capacity() >= 5); + + SUBCASE("adding to the vector increases it's size") + { + v.push_back(1); + + CHECK(v.size() == 6); + CHECK(v.capacity() >= 6); + } + SUBCASE("reserving increases just the capacity") + { + v.reserve(6); + + CHECK(v.size() == 5); + CHECK(v.capacity() >= 6); + } +} + +TEST_CASE("WARN level of asserts don't fail the test case") +{ + WARN(0); + WARN_FALSE(1); + WARN_EQ(1, 0); +} + +TEST_SUITE("scoped test suite") +{ + TEST_CASE("part of scoped") + { + FAIL("Error message"); + } + + TEST_CASE("part of scoped 2") + { + FAIL(""); + } +} + +int main(int argc, char **argv) +{ + doctest::Context context; + context.setOption("success", true); + context.setOption("no-exitcode", true); + context.applyCommandLine(argc, argv); + return context.run(); +} +""" + ) + junit_output_path = tmp_path / "junit.xml" + result = clirunner.invoke( + pio_test_cmd, + [ + "-d", + str(project_dir), + "--output-format=junit", + "--output-path", + str(junit_output_path), + ], + ) + assert result.exit_code != 0 + + # test JUnit output + junit_testsuites = ET.parse(junit_output_path).getroot() + assert int(junit_testsuites.get("tests")) == 8 + assert int(junit_testsuites.get("errors")) == 0 + assert int(junit_testsuites.get("failures")) == 3 + assert len(junit_testsuites.findall("testsuite")) == 1 + junit_failed_testcase = junit_testsuites.find( + ".//testcase[@name='scoped test suite -> part of scoped']" + ) + assert junit_failed_testcase.get("status") == "FAILED" + assert junit_failed_testcase.find("failure").get("message") == "Error message" + assert "TEST SUITE: scoped test suite" in junit_failed_testcase.find("failure").text