Rename unit testing module to "test"

platformio/test/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

platformio/test/command.py

@@ -0,0 +1,233 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import fnmatch
+import os
+import shutil
+
+import click
+
+from platformio import app, exception, fs, util
+from platformio.project.config import ProjectConfig
+from platformio.test.exception import TestDirNotExistsError
+from platformio.test.reports.base import TestReportFactory
+from platformio.test.result import TestResult, TestStatus, TestSuite
+from platformio.test.runners.base import TestRunnerOptions
+from platformio.test.runners.factory import TestRunnerFactory
+
+
+@click.command("test", short_help="Unit Testing")
+@click.option("--environment", "-e", multiple=True, metavar="<environment>")
+@click.option(
+    "--filter",
+    "-f",
+    multiple=True,
+    metavar="<pattern>",
+    help="Filter tests by a pattern",
+)
+@click.option(
+    "--ignore",
+    "-i",
+    multiple=True,
+    metavar="<pattern>",
+    help="Ignore tests by a pattern",
+)
+@click.option("--upload-port")
+@click.option("--test-port")
+@click.option(
+    "-d",
+    "--project-dir",
+    default=os.getcwd,
+    type=click.Path(
+        exists=True, file_okay=False, dir_okay=True, writable=True, resolve_path=True
+    ),
+)
+@click.option(
+    "-c",
+    "--project-conf",
+    type=click.Path(
+        exists=True, file_okay=True, dir_okay=False, readable=True, resolve_path=True
+    ),
+)
+@click.option("--without-building", is_flag=True)
+@click.option("--without-uploading", is_flag=True)
+@click.option("--without-testing", is_flag=True)
+@click.option("--no-reset", is_flag=True)
+@click.option(
+    "--monitor-rts",
+    default=None,
+    type=click.IntRange(0, 1),
+    help="Set initial RTS line state for Serial Monitor",
+)
+@click.option(
+    "--monitor-dtr",
+    default=None,
+    type=click.IntRange(0, 1),
+    help="Set initial DTR line state for Serial Monitor",
+)
+@click.option("--output-format", type=click.Choice(["json", "junit"]))
+@click.option(
+    "--output-path",
+    default=os.getcwd,
+    type=click.Path(dir_okay=True, resolve_path=True),
+)
+@click.option("--verbose", "-v", is_flag=True)
+@click.pass_context
+def test_cmd(  # pylint: disable=too-many-arguments,too-many-locals,redefined-builtin
+    ctx,
+    environment,
+    ignore,
+    filter,
+    upload_port,
+    test_port,
+    project_dir,
+    project_conf,
+    without_building,
+    without_uploading,
+    without_testing,
+    no_reset,
+    monitor_rts,
+    monitor_dtr,
+    output_format,
+    output_path,
+    verbose,
+):
+    app.set_session_var("custom_project_conf", project_conf)
+
+    with fs.cd(project_dir):
+        config = ProjectConfig.get_instance(project_conf)
+        config.validate(envs=environment)
+        test_names = get_test_names(config)
+
+        if not verbose:
+            click.echo("Verbose mode can be enabled via `-v, --verbose` option")
+        click.secho("Collected %d tests" % len(test_names), bold=True, nl=not verbose)
+        if verbose:
+            click.echo(" (%s)" % ", ".join(test_names))
+
+        test_result = TestResult(os.path.basename(project_dir))
+        default_envs = config.default_envs()
+        for env_name in config.envs():
+            for test_name in test_names:
+                test_suite = TestSuite(env_name, test_name)
+                test_result.add_suite(test_suite)
+
+                # filter and ignore patterns
+                patterns = dict(filter=list(filter), ignore=list(ignore))
+                for key in patterns:
+                    if patterns[key]:  # overriden from CLI
+                        continue
+                    patterns[key].extend(
+                        config.get(f"env:{env_name}", f"test_{key}", [])
+                    )
+
+                skip_conditions = [
+                    environment and env_name not in environment,
+                    not environment and default_envs and env_name not in default_envs,
+                    test_name != "*"
+                    and patterns["filter"]
+                    and not any(
+                        fnmatch.fnmatch(test_name, p) for p in patterns["filter"]
+                    ),
+                    test_name != "*"
+                    and any(fnmatch.fnmatch(test_name, p) for p in patterns["ignore"]),
+                ]
+                if any(skip_conditions):
+                    continue
+
+                runner = TestRunnerFactory.new(
+                    test_suite,
+                    config,
+                    TestRunnerOptions(
+                        verbose=verbose,
+                        without_building=without_building,
+                        without_uploading=without_uploading,
+                        without_testing=without_testing,
+                        upload_port=upload_port,
+                        test_port=test_port,
+                        no_reset=no_reset,
+                        monitor_rts=monitor_rts,
+                        monitor_dtr=monitor_dtr,
+                    ),
+                )
+                click.echo()
+                print_suite_header(test_suite)
+                runner.start(ctx)
+                print_suite_footer(test_suite)
+
+        # automatically generate JSON report for PIO IDE
+        TestReportFactory.new("json", test_result).generate(
+            os.path.join(
+                config.get("platformio", "build_dir"), "pio-test-report-latest.json"
+            )
+        )
+
+    # Reset custom project config
+    app.set_session_var("custom_project_conf", None)
+
+    stdout_report = TestReportFactory.new("stdout", test_result)
+    stdout_report.generate(verbose=verbose)
+
+    if output_format:
+        custom_report = TestReportFactory.new(output_format, test_result)
+        custom_report.generate(output_path=output_path, verbose=True)
+
+    if test_result.is_errored or test_result.get_status_nums(TestStatus.FAILED):
+        raise exception.ReturnErrorCode(1)
+
+
+def get_test_names(config):
+    test_dir = config.get("platformio", "test_dir")
+    if not os.path.isdir(test_dir):
+        raise TestDirNotExistsError(test_dir)
+    names = []
+    for root, _, __ in os.walk(test_dir):
+        if not os.path.basename(root).startswith("test_"):
+            continue
+        names.append(os.path.relpath(root, test_dir))
+    if not names:
+        names = ["*"]
+    return names
+
+
+def print_suite_header(test_suite):
+    click.echo(
+        "Processing %s in %s environment"
+        % (
+            click.style(test_suite.test_name, fg="yellow", bold=True),
+            click.style(test_suite.env_name, fg="cyan", bold=True),
+        )
+    )
+    terminal_width, _ = shutil.get_terminal_size()
+    click.secho("-" * terminal_width, bold=True)
+
+
+def print_suite_footer(test_suite):
+    is_error = test_suite.status in (TestStatus.FAILED, TestStatus.ERRORED)
+    util.print_labeled_bar(
+        "%s [%s] Took %.2f seconds"
+        % (
+            click.style(
+                "%s:%s" % (test_suite.env_name, test_suite.test_name), bold=True
+            ),
+            (
+                click.style(test_suite.status.name, fg="red", bold=True)
+                if is_error
+                else click.style("PASSED", fg="green", bold=True)
+            ),
+            test_suite.duration,
+        ),
+        is_error=is_error,
+        sep="-",
+    )

platformio/test/exception.py

@@ -0,0 +1,34 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from platformio.exception import PlatformioException, UserSideException
+
+
+class UnitTestError(PlatformioException):
+    pass
+
+
+class TestDirNotExistsError(UnitTestError, UserSideException):
+
+    MESSAGE = (
+        "A test folder '{0}' does not exist.\nPlease create 'test' "
+        "directory in the project root and put a test set.\n"
+        "More details about Unit "
+        "Testing: https://docs.platformio.org/page/plus/"
+        "unit-testing.html"
+    )
+
+
+class UnitTestSuiteError(UnitTestError):
+    pass

platformio/test/reports/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

platformio/test/reports/base.py

@@ -0,0 +1,34 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import importlib
+
+from platformio.test.result import TestResult
+
+
+class TestReportBase:
+    def __init__(self, test_result):
+        self.test_result = test_result
+
+    def generate(self, output_path, verbose):
+        raise NotImplementedError()
+
+
+class TestReportFactory:
+    @staticmethod
+    def new(format, test_result) -> TestReportBase:  # pylint: disable=redefined-builtin
+        assert isinstance(test_result, TestResult)
+        mod = importlib.import_module(f"platformio.test.reports.{format}")
+        report_cls = getattr(mod, "%sTestReport" % format.lower().capitalize())
+        return report_cls(test_result)

platformio/test/reports/json.py

@@ -0,0 +1,99 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import datetime
+import json
+import os
+
+import click
+
+from platformio.test.reports.base import TestReportBase
+from platformio.test.result import TestStatus
+
+
+class JsonTestReport(TestReportBase):
+    def generate(self, output_path, verbose=False):
+        if os.path.isdir(output_path):
+            output_path = os.path.join(
+                output_path,
+                "pio-test-report-%s-%s.json"
+                % (
+                    self.test_result.name,
+                    datetime.datetime.now().strftime("%Y%m%d%H%M%S"),
+                ),
+            )
+
+        with open(output_path, mode="w", encoding="utf8") as fp:
+            json.dump(self.to_json(), fp)
+
+        if verbose:
+            click.secho(f"Saved JSON report to the {output_path}", fg="green")
+
+    def to_json(self):
+        result = dict(
+            version="1.0",
+            name=self.test_result.name,
+            duration=self.test_result.duration,
+            testcase_nums=self.test_result.case_nums,
+            error_nums=self.test_result.get_status_nums(TestStatus.ERRORED),
+            failure_nums=self.test_result.get_status_nums(TestStatus.FAILED),
+            skipped_nums=self.test_result.get_status_nums(TestStatus.SKIPPED),
+            test_suites=[],
+        )
+        for test_suite in self.test_result.suites:
+            result["test_suites"].append(self.test_suite_to_json(test_suite))
+        return result
+
+    def test_suite_to_json(self, test_suite):
+        result = dict(
+            env_name=test_suite.env_name,
+            test_name=test_suite.test_name,
+            status=test_suite.status.name,
+            duration=test_suite.duration,
+            timestamp=datetime.datetime.fromtimestamp(test_suite.timestamp).strftime(
+                "%Y-%m-%dT%H:%M:%S"
+            )
+            if test_suite.timestamp
+            else None,
+            testcase_nums=len(test_suite.cases),
+            error_nums=test_suite.get_status_nums(TestStatus.ERRORED),
+            failure_nums=test_suite.get_status_nums(TestStatus.FAILED),
+            skipped_nums=test_suite.get_status_nums(TestStatus.SKIPPED),
+            test_cases=[],
+        )
+        for test_case in test_suite.cases:
+            result["test_cases"].append(self.test_case_to_json(test_case))
+        return result
+
+    @staticmethod
+    def test_case_to_json(test_case):
+        result = dict(
+            name=test_case.name,
+            status=test_case.status.name,
+            message=test_case.message,
+            stdout=test_case.stdout,
+            duration=test_case.duration,
+            exception=None,
+            source=None,
+        )
+        if test_case.exception:
+            result["exception"] = "%s: %s" % (
+                test_case.exception.__class__.__name__,
+                test_case.exception,
+            )
+        if test_case.source:
+            result["source"] = dict(
+                file=test_case.source.file, line=test_case.source.line
+            )
+        return result

platformio/test/reports/junit.py

@@ -0,0 +1,102 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import datetime
+import os
+import xml.etree.ElementTree as ET
+
+import click
+
+from platformio.test.reports.base import TestReportBase
+from platformio.test.result import TestStatus
+
+
+class JunitTestReport(TestReportBase):
+    def generate(self, output_path, verbose=False):
+        if os.path.isdir(output_path):
+            output_path = os.path.join(
+                output_path,
+                "pio-test-report-%s-%s-junit.xml"
+                % (
+                    self.test_result.name,
+                    datetime.datetime.now().strftime("%Y%m%d%H%M%S"),
+                ),
+            )
+
+        with open(output_path, mode="wb") as fp:
+            self.build_xml_tree().write(fp, encoding="utf8")
+
+        if verbose:
+            click.secho(f"Saved JUnit report to the {output_path}", fg="green")
+
+    def build_xml_tree(self):
+        root = ET.Element("testsuites")
+        root.set("name", self.test_result.name)
+        root.set("tests", str(self.test_result.case_nums))
+        root.set("errors", str(self.test_result.get_status_nums(TestStatus.ERRORED)))
+        root.set("failures", str(self.test_result.get_status_nums(TestStatus.FAILED)))
+        root.set("time", str(self.test_result.duration))
+        for suite in self.test_result.suites:
+            root.append(self.build_testsuite_node(suite))
+        return ET.ElementTree(root)
+
+    def build_testsuite_node(self, test_suite):
+        element = ET.Element("testsuite")
+        element.set("name", f"{test_suite.env_name}:{test_suite.test_name}")
+        element.set("tests", str(len(test_suite.cases)))
+        element.set("errors", str(test_suite.get_status_nums(TestStatus.ERRORED)))
+        element.set("failures", str(test_suite.get_status_nums(TestStatus.FAILED)))
+        element.set("skipped", str(test_suite.get_status_nums(TestStatus.SKIPPED)))
+        element.set("time", str(test_suite.duration))
+        if test_suite.timestamp:
+            element.set(
+                "timestamp",
+                datetime.datetime.fromtimestamp(test_suite.timestamp).strftime(
+                    "%Y-%m-%dT%H:%M:%S"
+                ),
+            )
+        for test_case in test_suite.cases:
+            element.append(self.build_testcase_node(test_case))
+        return element
+
+    def build_testcase_node(self, test_case):
+        element = ET.Element("testcase")
+        element.set("name", str(test_case.name))
+        element.set("time", str(test_case.duration))
+        element.set("status", str(test_case.status.name))
+        if test_case.status == TestStatus.SKIPPED:
+            element.append(ET.Element("skipped"))
+        elif test_case.status == TestStatus.ERRORED:
+            element.append(self.build_testcase_error_node(test_case))
+        elif test_case.status == TestStatus.FAILED:
+            element.append(self.build_testcase_failure_node(test_case))
+        return element
+
+    @staticmethod
+    def build_testcase_error_node(test_case):
+        element = ET.Element("error")
+        element.set("type", test_case.exception.__class__.__name__)
+        element.set("message", str(test_case.exception))
+        if test_case.stdout:
+            element.text = test_case.stdout
+        return element
+
+    @staticmethod
+    def build_testcase_failure_node(test_case):
+        element = ET.Element("failure")
+        if test_case.message:
+            element.set("message", test_case.message)
+        if test_case.stdout:
+            element.text = test_case.stdout
+        return element

platformio/test/reports/stdout.py

@@ -0,0 +1,73 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+from tabulate import tabulate
+
+from platformio import util
+from platformio.test.reports.base import TestReportBase
+from platformio.test.result import TestStatus
+
+
+class StdoutTestReport(TestReportBase):
+    def generate(self, verbose=False):  # pylint: disable=arguments-differ
+        click.echo()
+
+        tabular_data = []
+        failed_nums = self.test_result.get_status_nums(TestStatus.FAILED)
+        skipped_nums = self.test_result.get_status_nums(TestStatus.SKIPPED)
+        is_error = failed_nums > 0 or self.test_result.is_errored
+
+        for test_suite in self.test_result.suites:
+            if not verbose and test_suite.status == TestStatus.SKIPPED:
+                continue
+            status_str = test_suite.status.name
+            if test_suite.status in (TestStatus.FAILED, TestStatus.ERRORED):
+                status_str = click.style(status_str, fg="red")
+            elif test_suite.status == TestStatus.PASSED:
+                status_str = click.style(status_str, fg="green")
+
+            tabular_data.append(
+                (
+                    click.style(test_suite.env_name, fg="cyan"),
+                    test_suite.test_name,
+                    status_str,
+                    util.humanize_duration_time(test_suite.duration or None),
+                )
+            )
+
+        if tabular_data:
+            click.echo(
+                tabulate(
+                    tabular_data,
+                    headers=[
+                        click.style(s, bold=True)
+                        for s in ("Environment", "Test", "Status", "Duration")
+                    ],
+                ),
+                err=is_error,
+            )
+
+        util.print_labeled_bar(
+            "%d test cases: %s%s%d succeeded in %s"
+            % (
+                self.test_result.case_nums,
+                ("%d failed, " % failed_nums) if failed_nums else "",
+                ("%d skipped, " % skipped_nums) if skipped_nums else "",
+                self.test_result.get_status_nums(TestStatus.PASSED),
+                util.humanize_duration_time(self.test_result.duration),
+            ),
+            is_error=is_error,
+            fg="red" if is_error else "green",
+        )

platformio/test/result.py

@@ -0,0 +1,138 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import enum
+import functools
+import operator
+import time
+
+
+class TestStatus(enum.Enum):
+    PASSED = enum.auto()
+    FAILED = enum.auto()
+    SKIPPED = enum.auto()
+    ERRORED = enum.auto()
+
+    @classmethod
+    def from_string(cls, value: str):
+        value = value.lower()
+        if value.startswith("pass"):
+            return cls.PASSED
+        if value.startswith(("ignore", "skip")):
+            return cls.SKIPPED
+        if value.startswith("fail"):
+            return cls.FAILED
+        raise ValueError(f"Unknown test status `{value}`")
+
+
+class TestCaseSource:
+    def __init__(self, file, line=None):
+        self.file = file
+        self.line = line
+
+
+class TestCase:
+    def __init__(  # pylint: disable=too-many-arguments
+        self,
+        name,
+        status,
+        message=None,
+        stdout=None,
+        source=None,
+        duration=0,
+        exception=None,
+    ):
+        assert isinstance(status, TestStatus)
+        if status == TestStatus.ERRORED:
+            assert isinstance(exception, Exception)
+        self.name = name.strip()
+        self.status = status
+        self.message = message
+        self.stdout = stdout
+        self.source = source
+        self.duration = duration
+        self.exception = exception
+
+
+class TestSuite:
+    def __init__(self, env_name, test_name):
+        self.env_name = env_name
+        self.test_name = test_name
+        self.timestamp = 0
+        self.duration = 0
+        self._cases = []
+        self._finished = False
+
+    @property
+    def cases(self):
+        return self._cases
+
+    @property
+    def status(self):
+        for s in (TestStatus.ERRORED, TestStatus.FAILED):
+            if self.get_status_nums(s):
+                return s
+        if self._cases and any(c.status == TestStatus.PASSED for c in self._cases):
+            return TestStatus.PASSED
+        return TestStatus.SKIPPED
+
+    def get_status_nums(self, status):
+        return len([True for c in self._cases if c.status == status])
+
+    def add_case(self, case: TestCase):
+        assert isinstance(case, TestCase)
+        self._cases.append(case)
+
+    def is_finished(self):
+        return self._finished
+
+    def on_start(self):
+        self.timestamp = time.time()
+
+    def on_finish(self):
+        if self.is_finished():
+            return
+        self._finished = True
+        self.duration = time.time() - self.timestamp
+
+
+class TestResult:
+    def __init__(self, name):
+        self.name = name
+        self._suites = []
+
+    @property
+    def suites(self):
+        return self._suites
+
+    def add_suite(self, suite):
+        assert isinstance(suite, TestSuite)
+        self._suites.append(suite)
+
+    @property
+    def duration(self):
+        return functools.reduce(operator.add, [s.duration for s in self._suites])
+
+    @property
+    def case_nums(self):
+        return functools.reduce(operator.add, [len(s.cases) for s in self._suites])
+
+    @property
+    def is_errored(self):
+        return any(s.status == TestStatus.ERRORED for s in self._suites)
+
+    def get_status_nums(self, status):
+        return functools.reduce(
+            operator.add, [s.get_status_nums(status) for s in self._suites]
+        )

platformio/test/runners/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

platformio/test/runners/base.py

@@ -0,0 +1,200 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+
+from platformio.exception import ReturnErrorCode
+from platformio.platform.factory import PlatformFactory
+from platformio.test.exception import UnitTestSuiteError
+from platformio.test.result import TestCase, TestCaseSource, TestStatus
+from platformio.test.runners.mixins.embedded import TestRunnerEmbeddedMixin
+from platformio.test.runners.mixins.native import TestRunnerNativeMixin
+
+CTX_META_TEST_IS_RUNNING = __name__ + ".test_running"
+CTX_META_TEST_RUNNING_NAME = __name__ + ".test_running_name"
+
+
+class TestRunnerOptions:  # pylint: disable=too-many-instance-attributes
+    def __init__(  # pylint: disable=too-many-arguments
+        self,
+        verbose=False,
+        without_building=False,
+        without_uploading=False,
+        without_testing=False,
+        without_debugging=True,
+        upload_port=None,
+        test_port=None,
+        no_reset=False,
+        monitor_rts=None,
+        monitor_dtr=None,
+    ):
+        self.verbose = verbose
+        self.without_building = without_building
+        self.without_uploading = without_uploading
+        self.without_testing = without_testing
+        self.without_debugging = without_debugging
+        self.upload_port = upload_port
+        self.test_port = test_port
+        self.no_reset = no_reset
+        self.monitor_rts = monitor_rts
+        self.monitor_dtr = monitor_dtr
+
+
+class TestRunnerBase(TestRunnerNativeMixin, TestRunnerEmbeddedMixin):
+
+    NAME = None
+    EXTRA_LIB_DEPS = None
+    TESTCASE_PARSE_RE = None
+
+    def __init__(self, test_suite, project_config, options=None):
+        self.test_suite = test_suite
+        self.options = options
+        self.project_config = project_config
+        self.platform = PlatformFactory.new(
+            self.project_config.get(f"env:{self.test_suite.env_name}", "platform"),
+            autoinstall=True,
+        )
+        self.cmd_ctx = None
+
+    @property
+    def name(self):
+        return self.__class__.__name__.replace("TestRunner", "").lower()
+
+    def get_test_speed(self):
+        return int(
+            self.project_config.get(f"env:{self.test_suite.env_name}", "test_speed")
+        )
+
+    def start(self, cmd_ctx):
+        # setup command context
+        self.cmd_ctx = cmd_ctx
+        self.cmd_ctx.meta[CTX_META_TEST_IS_RUNNING] = True
+        if self.test_suite.test_name != "*":
+            self.cmd_ctx.meta[CTX_META_TEST_RUNNING_NAME] = self.test_suite.test_name
+
+        self.test_suite.on_start()
+        try:
+            self.setup()
+            for stage in ("building", "uploading", "testing"):
+                getattr(self, f"stage_{stage}")()
+        except Exception as exc:  # pylint: disable=broad-except
+            click.secho(str(exc), fg="red", err=True)
+            self.test_suite.add_case(
+                TestCase(
+                    name=f"{self.test_suite.env_name}:{self.test_suite.test_name}",
+                    status=TestStatus.ERRORED,
+                    exception=exc,
+                )
+            )
+        finally:
+            self.test_suite.on_finish()
+            self.teardown()
+
+    def setup(self):
+        pass
+
+    def stage_building(self):
+        if self.options.without_building:
+            return None
+        click.secho("Building...", bold=self.options.verbose)
+        targets = ["__test"]
+        if not self.options.without_debugging:
+            targets.append("__debug")
+        if self.platform.is_embedded():
+            targets.append("checkprogsize")
+        try:
+            return self.run_project_targets(targets)
+        except ReturnErrorCode:
+            raise UnitTestSuiteError("Building stage has failed, see errors above.")
+
+    def stage_uploading(self):
+        if self.options.without_uploading or not self.platform.is_embedded():
+            return None
+        click.secho("Uploading...", bold=self.options.verbose)
+        targets = ["upload"]
+        if self.options.without_building:
+            targets.append("nobuild")
+        else:
+            targets.append("__test")
+        if not self.options.without_debugging:
+            targets.append("__debug")
+        try:
+            return self.run_project_targets(targets)
+        except ReturnErrorCode:
+            raise UnitTestSuiteError("Uploading stage has failed, see errors above.")
+
+    def stage_testing(self):
+        if self.options.without_testing:
+            return None
+        click.secho("Testing...", bold=self.options.verbose)
+        if self.platform.is_embedded():
+            return self.stage_testing_on_target()
+        return self.stage_testing_on_host()
+
+    def teardown(self):
+        pass
+
+    def run_project_targets(self, targets):
+        # pylint: disable=import-outside-toplevel
+        from platformio.commands.run.command import cli as run_cmd
+
+        assert self.cmd_ctx
+        return self.cmd_ctx.invoke(
+            run_cmd,
+            project_conf=self.project_config.path,
+            upload_port=self.options.upload_port,
+            verbose=self.options.verbose,
+            silent=not self.options.verbose,
+            environment=[self.test_suite.env_name],
+            disable_auto_clean="nobuild" in targets,
+            target=targets,
+        )
+
+    def configure_build_env(self, env):  # pylint: disable=no-self-use
+        """
+        Configure SCons build environment
+        Called in "builder/tools/piotest" tool
+        """
+        return env
+
+    def on_test_output(self, data):
+        click.echo(data, nl=False)
+        self.parse_testcases(data)
+
+    def parse_testcases(self, data):
+        if not self.TESTCASE_PARSE_RE:
+            raise NotImplementedError()
+
+        for line in data.split("\n"):
+            line = line.strip()
+            if not line:
+                continue
+            match = self.TESTCASE_PARSE_RE.search(line)
+            if not match:
+                continue
+            data = match.groupdict()
+            source = None
+            if "source_file" in data:
+                source = TestCaseSource(
+                    file=data["source_file"], line=int(data.get("source_line"))
+                )
+            self.test_suite.add_case(
+                TestCase(
+                    name=data.get("name"),
+                    status=TestStatus.from_string(data.get("status")),
+                    message=data.get("message"),
+                    stdout=line,
+                    source=source,
+                )
+            )

platformio/test/runners/factory.py

@@ -0,0 +1,57 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import importlib
+import os
+import re
+
+from platformio.compat import load_python_module
+from platformio.exception import UserSideException
+from platformio.project.config import ProjectConfig
+from platformio.test.result import TestSuite
+from platformio.test.runners.base import TestRunnerBase, TestRunnerOptions
+
+
+class TestRunnerFactory(object):
+    @staticmethod
+    def get_clsname(name):
+        name = re.sub(r"[^\da-z\_\-]+", "", name, flags=re.I)
+        return "%sTestRunner" % name.lower().capitalize()
+
+    @classmethod
+    def new(cls, test_suite, project_config, options=None) -> TestRunnerBase:
+        assert isinstance(test_suite, TestSuite)
+        assert isinstance(project_config, ProjectConfig)
+        if options:
+            assert isinstance(options, TestRunnerOptions)
+        test_framework = project_config.get(
+            f"env:{test_suite.env_name}", "test_framework"
+        )
+        module_name = f"platformio.test.runners.{test_framework}"
+        runner_cls = None
+        if test_framework == "custom":
+            custom_runner_path = os.path.join(
+                project_config.get("platformio", "test_dir"), "custom_runner.py"
+            )
+            try:
+                mod = load_python_module(module_name, custom_runner_path)
+            except ImportError:
+                raise UserSideException(
+                    "Could not find custom unit testing runner "
+                    f"by this path -> {custom_runner_path}"
+                )
+        else:
+            mod = importlib.import_module(module_name)
+        runner_cls = getattr(mod, cls.get_clsname(test_framework))
+        return runner_cls(test_suite, project_config, options)

platformio/test/runners/mixins/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

platformio/test/runners/mixins/embedded.py

@@ -0,0 +1,121 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from time import sleep
+
+import click
+import serial
+
+from platformio import util
+from platformio.exception import UserSideException
+
+
+class TestRunnerEmbeddedMixin:
+
+    SERIAL_TIMEOUT = 600
+
+    def stage_testing_on_target(self):
+        click.echo(
+            "If you don't see any output for the first 10 secs, "
+            "please reset board (press reset button)"
+        )
+        click.echo()
+
+        try:
+            ser = serial.Serial(
+                baudrate=self.get_test_speed(), timeout=self.SERIAL_TIMEOUT
+            )
+            ser.port = self.get_test_port()
+            ser.rts = self.options.monitor_rts
+            ser.dtr = self.options.monitor_dtr
+            ser.open()
+        except serial.SerialException as e:
+            click.secho(str(e), fg="red", err=True)
+            return None
+
+        if not self.options.no_reset:
+            ser.flushInput()
+            ser.setDTR(False)
+            ser.setRTS(False)
+            sleep(0.1)
+            ser.setDTR(True)
+            ser.setRTS(True)
+            sleep(0.1)
+
+        while not self.test_suite.is_finished():
+            line = ser.readline().strip()
+
+            # fix non-ascii output from device
+            for i, c in enumerate(line[::-1]):
+                if not isinstance(c, int):
+                    c = ord(c)
+                if c > 127:
+                    line = line[-i:]
+                    break
+
+            if not line:
+                continue
+            if isinstance(line, bytes):
+                line = line.decode("utf8", "ignore")
+            self.on_test_output(line)
+        ser.close()
+
+    def get_test_port(self):
+        # if test port is specified manually or in config
+        port = self.options.test_port or self.project_config.get(
+            f"env:{self.test_suite.env_name}", "test_port"
+        )
+        if port:
+            return port
+
+        board = self.project_config.get(f"env:{self.test_suite.env_name}", "board")
+        board_hwids = self.platform.board_config(board).get("build.hwids", [])
+        port = None
+        elapsed = 0
+        while elapsed < 5 and not port:
+            for item in util.get_serialports():
+                port = item["port"]
+                for hwid in board_hwids:
+                    hwid_str = ("%s:%s" % (hwid[0], hwid[1])).replace("0x", "")
+                    if hwid_str in item["hwid"] and self.is_serial_port_ready(port):
+                        return port
+
+            if port and not self.is_serial_port_ready(port):
+                port = None
+
+            if not port:
+                sleep(0.25)
+                elapsed += 0.25
+
+        if not port:
+            raise UserSideException(
+                "Please specify `test_port` for environment or use "
+                "global `--test-port` option."
+            )
+        return port
+
+    @staticmethod
+    def is_serial_port_ready(port, timeout=3):
+        if not port:
+            return False
+        elapsed = 0
+        while elapsed < timeout:
+            try:
+                serial.Serial(port, timeout=1).close()
+                return True
+            except:  # pylint: disable=bare-except
+                pass
+            sleep(1)
+            elapsed += 1
+        return False

platformio/test/runners/mixins/native.py

@@ -0,0 +1,42 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import signal
+
+from platformio import proc
+from platformio.test.exception import UnitTestError
+
+
+class TestRunnerNativeMixin:
+    def stage_testing_on_host(self):
+        build_dir = self.project_config.get("platformio", "build_dir")
+        result = proc.exec_command(
+            [os.path.join(build_dir, self.test_suite.env_name, "program")],
+            stdout=proc.LineBufferedAsyncPipe(self.on_test_output),
+            stderr=proc.LineBufferedAsyncPipe(self.on_test_output),
+        )
+        if result["returncode"] == 0:
+            return True
+        try:
+            sig = signal.Signals(abs(result["returncode"]))
+            try:
+                signal_description = signal.strsignal(sig)
+            except AttributeError:
+                signal_description = ""
+            raise UnitTestError(
+                f"Program received signal {sig.name} ({signal_description})"
+            )
+        except ValueError:
+            raise UnitTestError("Program errored with %d code" % result["returncode"])

platformio/test/runners/unity.py

@@ -0,0 +1,271 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import re
+import string
+from pathlib import Path
+
+import click
+
+from platformio.test.exception import UnitTestSuiteError
+from platformio.test.runners.base import TestRunnerBase
+
+
+class UnityTestRunner(TestRunnerBase):
+
+    EXTRA_LIB_DEPS = ["throwtheswitch/Unity@^2.5.2"]
+
+    # example
+    # test/test_foo.cpp:44:test_function_foo:FAIL: Expected 32 Was 33
+    TESTCASE_PARSE_RE = re.compile(
+        r"(?P<source_file>[^:]+):(?P<source_line>\d+):(?P<name>[^:]+):"
+        r"(?P<status>PASS|IGNORE|FAIL)(:\s*(?P<message>.+)$)?"
+    )
+
+    UNITY_CONFIG_H = """
+#ifndef UNITY_CONFIG_H
+#define UNITY_CONFIG_H
+
+#ifndef NULL
+#ifndef __cplusplus
+#define NULL (void*)0
+#else
+#define NULL 0
+#endif
+#endif
+
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+
+void unityOutputStart(unsigned long);
+void unityOutputChar(unsigned int);
+void unityOutputFlush();
+void unityOutputComplete();
+
+#define UNITY_OUTPUT_START()    unityOutputStart((unsigned long) $baudrate)
+#define UNITY_OUTPUT_CHAR(c)    unityOutputChar(c)
+#define UNITY_OUTPUT_FLUSH()    unityOutputFlush()
+#define UNITY_OUTPUT_COMPLETE() unityOutputComplete()
+
+#ifdef __cplusplus
+}
+#endif /* extern "C" */
+
+#endif /* UNITY_CONFIG_H */
+
+"""
+
+    UNITY_CONFIG_C = """
+#include <unity_config.h>
+
+#if !defined(UNITY_WEAK_ATTRIBUTE) && !defined(UNITY_WEAK_PRAGMA)
+#   if defined(__GNUC__) || defined(__ghs__) /* __GNUC__ includes clang */
+#       if !(defined(__WIN32__) && defined(__clang__)) && !defined(__TMS470__)
+#           define UNITY_WEAK_ATTRIBUTE __attribute__((weak))
+#       endif
+#   endif
+#endif
+
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+
+#ifdef UNITY_WEAK_ATTRIBUTE
+    UNITY_WEAK_ATTRIBUTE void setUp(void) { }
+    UNITY_WEAK_ATTRIBUTE void tearDown(void) { }
+    UNITY_WEAK_ATTRIBUTE void suiteSetUp(void) { }
+    UNITY_WEAK_ATTRIBUTE int suiteTearDown(int num_failures) { return num_failures; }
+#elif defined(UNITY_WEAK_PRAGMA)
+    #pragma weak setUp
+    void setUp(void) { }
+    #pragma weak tearDown
+    void tearDown(void) { }
+    #pragma weak suiteSetUp
+    void suiteSetUp(void) { }
+    #pragma weak suiteTearDown
+    int suiteTearDown(int num_failures) { return num_failures; }
+#endif
+
+#ifdef __cplusplus
+}
+#endif /* extern "C" */
+
+$framework_config_code
+    """
+
+    UNITY_FRAMEWORK_CONFIG = dict(
+        native=dict(
+            code="""
+#include <stdio.h>
+void unityOutputStart(unsigned long baudrate) { }
+void unityOutputChar(unsigned int c) { putchar(c); }
+void unityOutputFlush(void) { fflush(stdout); }
+void unityOutputComplete(void) { }
+        """,
+            language="c",
+        ),
+        arduino=dict(
+            code="""
+#include <Arduino.h>
+void unityOutputStart(unsigned long baudrate) { Serial.begin(baudrate); }
+void unityOutputChar(unsigned int c) { Serial.write(c); }
+void unityOutputFlush(void) { Serial.flush(); }
+void unityOutputComplete(void) { Serial.end(); }
+        """,
+            language="cpp",
+        ),
+        mbed=dict(
+            code="""
+#include <mbed.h>
+#if MBED_MAJOR_VERSION == 6
+UnbufferedSerial pc(USBTX, USBRX);
+#else
+RawSerial pc(USBTX, USBRX);
+#endif
+void unityOutputStart(unsigned long baudrate) { pc.baud(baudrate); }
+void unityOutputChar(unsigned int c) {
+#if MBED_MAJOR_VERSION == 6
+    pc.write(&c, 1);
+#else
+    pc.putc(c);
+#endif
+}
+void unityOutputFlush(void) { }
+void unityOutputComplete(void) { }
+        """,
+            language="cpp",
+        ),
+        espidf=dict(
+            code="""
+#include <stdio.h>
+void unityOutputStart(unsigned long baudrate) { }
+void unityOutputChar(unsigned int c) { putchar(c); }
+void unityOutputFlush(void) { fflush(stdout); }
+void unityOutputComplete(void) { }
+        """,
+            language="c",
+        ),
+        zephyr=dict(
+            code="""
+#include <sys/printk.h>
+void unityOutputStart(unsigned long baudrate) { }
+void unityOutputChar(unsigned int c) { printk("%c", c); }
+void unityOutputFlush(void) { }
+void unityOutputComplete(void) { }
+        """,
+            language="c",
+        ),
+        legacy_custom_transport=dict(
+            code="""
+#include <unittest_transport.h>
+void unityOutputStart(unsigned long baudrate) { unittest_uart_begin(); }
+void unityOutputChar(unsigned int c) { unittest_uart_putchar(c); }
+void unityOutputFlush(void) { unittest_uart_flush(); }
+void unityOutputComplete(void) { unittest_uart_end(); }
+        """,
+            language="cpp",
+        ),
+    )
+
+    def get_unity_framework_config(self):
+        if not self.platform.is_embedded():
+            return self.UNITY_FRAMEWORK_CONFIG["native"]
+        if (
+            self.project_config.get(
+                f"env:{self.test_suite.env_name}", "test_transport", None
+            )
+            == "custom"
+        ):
+            framework = "legacy_custom_transport"
+        else:
+            framework = (
+                self.project_config.get(f"env:{self.test_suite.env_name}", "framework")
+                or [None]
+            )[0]
+        if framework and framework in self.UNITY_FRAMEWORK_CONFIG:
+            return self.UNITY_FRAMEWORK_CONFIG[framework]
+        raise UnitTestSuiteError(
+            f"Could not find Unity configuration for the `{framework}` framework.\n"
+            "Learn how to create a custom Unity configuration at"
+            "https://docs.platformio.org/page/plus/unit-testing.html"
+        )
+
+    def configure_build_env(self, env):
+        env.Append(CPPDEFINES=["UNITY_INCLUDE_CONFIG_H"])
+        if self.custom_unity_config_exists(
+            [env.subst(item) for item in (env.get("CPPPATH") or [])]
+        ):
+            return env
+        env.Replace(
+            UNITY_CONFIG_DIR=os.path.join("$BUILD_DIR", "unity_config"),
+            BUILD_UNITY_CONFIG_DIR=os.path.join("$BUILD_DIR", "unity_config_build"),
+        )
+        env.Append(CPPPATH=["$UNITY_CONFIG_DIR"])
+        self.generate_unity_extras(env.subst("$UNITY_CONFIG_DIR"))
+        env.BuildSources("$BUILD_UNITY_CONFIG_DIR", "$UNITY_CONFIG_DIR")
+        return env
+
+    @staticmethod
+    def custom_unity_config_exists(include_dirs):
+        return any(
+            os.path.isfile(os.path.join(d, "unity_config.h")) for d in include_dirs
+        )
+
+    def generate_unity_extras(self, dst_dir):
+        dst_dir = Path(dst_dir)
+        dst_dir.mkdir(parents=True, exist_ok=True)
+        unity_h = dst_dir / "unity_config.h"
+        if not unity_h.is_file():
+            unity_h.write_text(
+                string.Template(self.UNITY_CONFIG_H).substitute(
+                    baudrate=self.get_test_speed()
+                ),
+                encoding="utf8",
+            )
+        framework_config = self.get_unity_framework_config()
+        unity_c = dst_dir / ("unity_config.%s" % framework_config.get("language", "c"))
+        if not unity_c.is_file():
+            unity_c.write_text(
+                string.Template(self.UNITY_CONFIG_C).substitute(
+                    framework_config_code=framework_config["code"]
+                ),
+                encoding="utf8",
+            )
+
+    def on_test_output(self, data):
+        if not data.strip():
+            return click.echo(data, nl=False)
+
+        if all(s in data for s in ("Tests", "Failures", "Ignored")):
+            self.test_suite.on_finish()
+
+        # beautify output
+        for line in data.strip().split("\n"):
+            line = line.strip()
+            if line.endswith(":PASS"):
+                click.echo("%s\t[%s]" % (line[:-5], click.style("PASSED", fg="green")))
+            elif line.endswith(":IGNORE"):
+                click.echo(
+                    "%s\t[%s]" % (line[:-7], click.style("IGNORED", fg="yellow"))
+                )
+            elif ":FAIL" in line:
+                click.echo("%s\t[%s]" % (line, click.style("FAILED", fg="red")))
+            else:
+                click.echo(line)
+
+        return self.parse_testcases(data)