New: "doctest" testing framework // Resolve #4240

HISTORY.rst

@@ -50,6 +50,7 @@ Please check `Migration guide from 5.x to 6.0 <https://docs.platformio.org/en/la
   - New: `Custom Testing Framework <https://docs.platformio.org/en/latest/advanced/unit-testing/frameworks/custom/index.html>`_
   - New: Using hardware `Simulators <https://docs.platformio.org/en/latest/advanced/unit-testing/simulators/index.html>`__ for Unit Testing (QEMU, Renode, SimAVR, and custom solutions)
   - New: `Semihosting <https://docs.platformio.org/en/latest/advanced/unit-testing/semihosting.html>`__ (`issue #3516 <https://github.com/platformio/platformio-core/issues/3516>`_)
+  - New: `doctest <https://docs.platformio.org/en/latest/advanced/unit-testing/frameworks/doctest.html>`__ testing framework (`issue #4240 <https://github.com/platformio/platformio-core/issues/4240>`_)
   - Added a new "test" `build configuration <https://docs.platformio.org/en/latest/projectconf/build_configurations.html>`__
   - Added support for the ``socket://`` and ``rfc2217://`` protocols using `test_port <https://docs.platformio.org/en/latest/projectconf/section_env_test.html#test-port>`__ option (`issue #4229 <https://github.com/platformio/platformio-core/issues/4229>`_)
   - Added support for a `Custom Unity Library <https://docs.platformio.org/en/latest/advanced/unit-testing/frameworks/custom/examples/custom_unity_library.html>`__ (`issue #3980 <https://github.com/platformio/platformio-core/issues/3980>`_)

platformio/project/options.py

@@ -652,7 +652,7 @@ ProjectOptions = OrderedDict(
                 group="test",
                 name="test_framework",
                 description="A unit testing framework",
-                type=click.Choice(["unity", "custom"]),
+                type=click.Choice(["doctest", "unity", "custom"]),
                 default="unity",
             ),
             ConfigEnvOption(

platformio/test/runners/doctest.py

@@ -0,0 +1,129 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+
+from platformio.test.result import TestCase, TestCaseSource, TestStatus
+from platformio.test.runners.base import TestRunnerBase
+
+
+class DoctestTestCaseParser:
+    def __init__(self):
+        self._tmp_tc = None
+        self._name_tokens = []
+
+    def parse(self, line):
+        if line.strip().startswith("[doctest]"):
+            return None
+        if self.is_divider(line):
+            return self._on_divider()
+
+        if not self._tmp_tc:
+            self._tmp_tc = TestCase("", TestStatus.PASSED, stdout="")
+            self._name_tokens = []
+
+        self._tmp_tc.stdout += line
+        line = line.strip()
+
+        # source
+        if not self._tmp_tc.source and line:
+            self._tmp_tc.source = self.parse_source(line)
+            return None
+
+        # name
+        if not self._tmp_tc.name:
+            if line:
+                self._name_tokens.append(line)
+                return None
+            self._tmp_tc.name = self.parse_name(self._name_tokens)
+            return None
+
+        if self._tmp_tc.status != TestStatus.FAILED:
+            self._parse_assert(line)
+
+        return None
+
+    @staticmethod
+    def is_divider(line):
+        line = line.strip()
+        return line.startswith("===") and line.endswith("===")
+
+    def _on_divider(self):
+        # if the first unprocessed test case
+        if not self._tmp_tc:
+            return None
+        test_case = TestCase(
+            name=self._tmp_tc.name,
+            status=self._tmp_tc.status,
+            message=self._tmp_tc.message,
+            source=self._tmp_tc.source,
+            stdout=self._tmp_tc.stdout,
+        )
+        self._tmp_tc = None
+        return test_case
+
+    @staticmethod
+    def parse_source(line):
+        assert line.endswith(":"), line
+        file_, line = line[:-1].rsplit(":", 1)
+        return TestCaseSource(file_, int(line))
+
+    @staticmethod
+    def parse_name(tokens):
+        cleaned_tokens = []
+        for token in tokens:
+            if token.startswith("TEST ") and ":" in token:
+                token = token[token.index(":") + 1 :]
+            cleaned_tokens.append(token.strip())
+        return " -> ".join(cleaned_tokens)
+
+    def _parse_assert(self, line):
+        status_tokens = [
+            (TestStatus.FAILED, "ERROR"),
+            (TestStatus.FAILED, "FATAL ERROR"),
+            (TestStatus.WARNED, "WARNING"),
+        ]
+        for status, token in status_tokens:
+            index = line.find(": %s:" % token)
+            if index == -1:
+                continue
+            self._tmp_tc.status = status
+            self._tmp_tc.message = line[index + len(token) + 3 :].strip() or None
+
+
+class DoctestTestRunner(TestRunnerBase):
+
+    EXTRA_LIB_DEPS = ["doctest/doctest@^2.4.8"]
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._tc_parser = DoctestTestCaseParser()
+
+    def configure_build_env(self, env):
+        if "-std=" not in env.subst("$CXXFLAGS"):
+            env.Append(CXXFLAGS=["-std=c++11"])
+        env.Append(CPPDEFINES=["DOCTEST_CONFIG_COLORS_NONE"])
+
+    def on_testing_line_output(self, line):
+        if self.options.verbose:
+            click.echo(line, nl=False)
+
+        test_case = self._tc_parser.parse(line)
+        if test_case:
+            self._tc_parser = DoctestTestCaseParser()
+            click.echo(test_case.humanize())
+            self.test_suite.add_case(test_case)
+
+        if "[doctest] Status:" in line:
+            self.test_suite.on_finish()

tests/commands/test_test.py

@@ -433,3 +433,111 @@ void unittest_uart_end(){}
         ],
     )
     validate_cliresult(result)
+
+
+def test_doctest_framework(clirunner, tmp_path: Path):
+    project_dir = tmp_path / "project"
+    project_dir.mkdir()
+    (project_dir / "platformio.ini").write_text(
+        """
+[env:native]
+platform = native
+test_framework = doctest
+"""
+    )
+    test_dir = project_dir / "test" / "test_dummy"
+    test_dir.mkdir(parents=True)
+    (test_dir / "test_main.cpp").write_text(
+        """
+#define DOCTEST_CONFIG_IMPLEMENT
+#include <doctest.h>
+
+TEST_CASE("[math] basic stuff")
+{
+	CHECK(6 > 5);
+	CHECK(6 > 7);
+}
+
+TEST_CASE("should be skipped " * doctest::skip())
+{
+	CHECK(2 > 5);
+}
+
+TEST_CASE("vectors can be sized and resized")
+{
+	std::vector<int> v(5);
+
+	REQUIRE(v.size() == 5);
+	REQUIRE(v.capacity() >= 5);
+
+	SUBCASE("adding to the vector increases it's size")
+	{
+		v.push_back(1);
+
+		CHECK(v.size() == 6);
+		CHECK(v.capacity() >= 6);
+	}
+	SUBCASE("reserving increases just the capacity")
+	{
+		v.reserve(6);
+
+		CHECK(v.size() == 5);
+		CHECK(v.capacity() >= 6);
+	}
+}
+
+TEST_CASE("WARN level of asserts don't fail the test case")
+{
+	WARN(0);
+	WARN_FALSE(1);
+	WARN_EQ(1, 0);
+}
+
+TEST_SUITE("scoped test suite")
+{
+	TEST_CASE("part of scoped")
+	{
+		FAIL("Error message");
+	}
+
+	TEST_CASE("part of scoped 2")
+	{
+		FAIL("");
+	}
+}
+
+int main(int argc, char **argv)
+{
+	doctest::Context context;
+	context.setOption("success", true);
+	context.setOption("no-exitcode", true);
+	context.applyCommandLine(argc, argv);
+	return context.run();
+}
+"""
+    )
+    junit_output_path = tmp_path / "junit.xml"
+    result = clirunner.invoke(
+        pio_test_cmd,
+        [
+            "-d",
+            str(project_dir),
+            "--output-format=junit",
+            "--output-path",
+            str(junit_output_path),
+        ],
+    )
+    assert result.exit_code != 0
+
+    # test JUnit output
+    junit_testsuites = ET.parse(junit_output_path).getroot()
+    assert int(junit_testsuites.get("tests")) == 8
+    assert int(junit_testsuites.get("errors")) == 0
+    assert int(junit_testsuites.get("failures")) == 3
+    assert len(junit_testsuites.findall("testsuite")) == 1
+    junit_failed_testcase = junit_testsuites.find(
+        ".//testcase[@name='scoped test suite -> part of scoped']"
+    )
+    assert junit_failed_testcase.get("status") == "FAILED"
+    assert junit_failed_testcase.find("failure").get("message") == "Error message"
+    assert "TEST SUITE: scoped test suite" in junit_failed_testcase.find("failure").text