PIO Check (#2982)

platformio/commands/check/__init__.py

@@ -0,0 +1,15 @@
+# Copyright (c) 2019-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from platformio.commands.check.command import cli

platformio/commands/check/command.py

@@ -0,0 +1,247 @@
+# Copyright (c) 2019-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# pylint: disable=too-many-arguments,too-many-locals,too-many-branches
+# pylint: disable=redefined-builtin,too-many-statements
+
+import os
+from collections import Counter
+from os.path import basename, dirname, isfile, join
+from time import time
+
+import click
+from tabulate import tabulate
+
+from platformio import exception, fs, util
+from platformio.commands.check.tools import CheckToolFactory, DefectItem
+from platformio.compat import dump_json_to_unicode
+from platformio.project.config import ProjectConfig
+from platformio.project.helpers import (find_project_dir_above,
+                                        get_project_dir,
+                                        get_project_include_dir,
+                                        get_project_src_dir)
+
+
+@click.command("check", short_help="Run a static analysis tool on code")
+@click.option("-e", "--environment", multiple=True)
+@click.option("-d",
+              "--project-dir",
+              default=os.getcwd,
+              type=click.Path(exists=True,
+                              file_okay=True,
+                              dir_okay=True,
+                              writable=True,
+                              resolve_path=True))
+@click.option("-c",
+              "--project-conf",
+              type=click.Path(exists=True,
+                              file_okay=True,
+                              dir_okay=False,
+                              readable=True,
+                              resolve_path=True))
+@click.option("--filter", multiple=True, help="Pattern: +<include> -<exclude>")
+@click.option("--flags", multiple=True)
+@click.option("--severity",
+              type=click.Choice(DefectItem.SEVERITY_LABELS.values()),
+              default=DefectItem.SEVERITY_LABELS[DefectItem.SEVERITY_LOW])
+@click.option("-s", "--silent", is_flag=True)
+@click.option("-v", "--verbose", is_flag=True)
+@click.option("--json-output", is_flag=True)
+def cli(environment, project_dir, project_conf, filter, flags, severity,
+        silent, verbose, json_output):
+    # find project directory on upper level
+    if isfile(project_dir):
+        project_dir = find_project_dir_above(project_dir)
+
+    results = []
+    with fs.cd(project_dir):
+        config = ProjectConfig.get_instance(
+            project_conf or join(project_dir, "platformio.ini"))
+        config.validate(environment)
+
+        default_envs = config.default_envs()
+        for envname in config.envs():
+            skipenv = any([
+                environment and envname not in environment, not environment
+                and default_envs and envname not in default_envs
+            ])
+
+            env_options = config.items(env=envname, as_dict=True)
+            env_dump = []
+            for k, v in env_options.items():
+                if k not in ("platform", "framework", "board"):
+                    continue
+                env_dump.append(
+                    "%s: %s" % (k, ", ".join(v) if isinstance(v, list) else v))
+
+            default_filter = [
+                "+<%s/>" % basename(d)
+                for d in (get_project_src_dir(), get_project_include_dir())
+            ]
+
+            tool_options = dict(
+                verbose=verbose,
+                silent=silent,
+                filter=filter
+                or env_options.get("check_filter", default_filter),
+                flags=flags or env_options.get("check_flags", ()),
+                severity=severity if not silent else "high")
+
+            for tool in env_options.get("check_tool", ["cppcheck"]):
+                if skipenv:
+                    results.append({"env": envname, "tool": tool})
+                    continue
+                if not silent and not json_output:
+                    print_processing_header(tool, envname, env_dump)
+
+                ct = CheckToolFactory.new(tool, project_dir, config, envname,
+                                          tool_options)
+
+                result = {"env": envname, "tool": tool, "duration": time()}
+                rc = ct.check(on_defect_callback=None if (
+                    json_output or verbose
+                ) else lambda defect: click.echo(repr(defect)))
+
+                result['defects'] = ct.get_defects()
+                result['duration'] = time() - result['duration']
+                result['succeeded'] = (
+                    rc == 0 and not any(d.severity == DefectItem.SEVERITY_HIGH
+                                        for d in result['defects']))
+                results.append(result)
+
+                if verbose:
+                    click.echo("\n".join(repr(d) for d in result['defects']))
+
+                if not json_output and not silent:
+                    print_processing_footer(result)
+
+    if json_output:
+        click.echo(dump_json_to_unicode(results_to_json(results)))
+    elif not silent:
+        print_check_summary(results)
+
+    command_failed = any(r.get("succeeded") is False for r in results)
+    if command_failed:
+        raise exception.ReturnErrorCode(1)
+
+
+def results_to_json(raw):
+    results = []
+    for item in raw:
+        item.update({
+            "ignored": item.get("succeeded") is None,
+            "succeeded": bool(item.get("succeeded")),
+            "defects": [d.to_json() for d in item.get("defects", [])]
+        })
+        results.append(item)
+
+    return results
+
+
+def print_processing_header(tool, envname, envdump):
+    click.echo(
+        "Checking %s > %s (%s)" %
+        (click.style(envname, fg="cyan", bold=True), tool, "; ".join(envdump)))
+    terminal_width, _ = click.get_terminal_size()
+    click.secho("-" * terminal_width, bold=True)
+
+
+def print_processing_footer(result):
+    is_failed = not result.get("succeeded")
+    util.print_labeled_bar(
+        "[%s] Took %.2f seconds" %
+        ((click.style("FAILED", fg="red", bold=True) if is_failed else
+          click.style("PASSED", fg="green", bold=True)), result['duration']),
+        is_error=is_failed)
+
+
+def print_defects_stats(results):
+    components = dict()
+
+    def _append_defect(component, defect):
+        if not components.get(component):
+            components[component] = Counter()
+        components[component].update(
+            {DefectItem.SEVERITY_LABELS[defect.severity]: 1})
+
+    for result in results:
+        for defect in result.get("defects", []):
+            component = dirname(defect.file) or defect.file
+            _append_defect(component, defect)
+
+            if component.startswith(get_project_dir()):
+                while os.sep in component:
+                    component = dirname(component)
+                    _append_defect(component, defect)
+
+    if not components:
+        return
+
+    severity_labels = list(DefectItem.SEVERITY_LABELS.values())
+    severity_labels.reverse()
+    tabular_data = list()
+    for k, v in components.items():
+        tool_defect = [v.get(s, 0) for s in severity_labels]
+        tabular_data.append([k] + tool_defect)
+
+    total = ["Total"] + [sum(d) for d in list(zip(*tabular_data))[1:]]
+    tabular_data.sort()
+    tabular_data.append([])  # Empty line as delimeter
+    tabular_data.append(total)
+
+    headers = ["Component"]
+    headers.extend([l.upper() for l in severity_labels])
+    headers = [click.style(h, bold=True) for h in headers]
+    click.echo(tabulate(tabular_data, headers=headers, numalign="center"))
+    click.echo()
+
+
+def print_check_summary(results):
+    click.echo()
+
+    tabular_data = []
+    succeeded_nums = 0
+    failed_nums = 0
+    duration = 0
+
+    print_defects_stats(results)
+    for result in results:
+        duration += result.get("duration", 0)
+        if result.get("succeeded") is False:
+            failed_nums += 1
+            status_str = click.style("FAILED", fg="red")
+        elif result.get("succeeded") is None:
+            status_str = "IGNORED"
+        else:
+            succeeded_nums += 1
+            status_str = click.style("PASSED", fg="green")
+
+        tabular_data.append(
+            (click.style(result['env'], fg="cyan"), result['tool'], status_str,
+             util.humanize_duration_time(result.get("duration"))))
+
+    click.echo(tabulate(tabular_data,
+                        headers=[
+                            click.style(s, bold=True)
+                            for s in ("Environment", "Tool", "Status",
+                                      "Duration")
+                        ]),
+               err=failed_nums)
+
+    util.print_labeled_bar(
+        "%s%d succeeded in %s" %
+        ("%d failed, " % failed_nums if failed_nums else "", succeeded_nums,
+         util.humanize_duration_time(duration)),
+        is_error=failed_nums,
+        fg="red" if failed_nums else "green")

platformio/commands/check/tools.py

@@ -0,0 +1,385 @@
+# Copyright (c) 2019-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# pylint: disable=too-many-arguments,too-many-instance-attributes
+# pylint: disable=redefined-builtin
+
+import re
+import sys
+from os import remove
+from os.path import isfile, join, relpath
+from tempfile import NamedTemporaryFile
+
+import click
+
+from platformio import exception, fs, proc
+from platformio.managers.core import get_core_package_dir
+from platformio.project.helpers import (get_project_core_dir, get_project_dir,
+                                        load_project_ide_data)
+
+
+class DefectItem(object):
+
+    SEVERITY_HIGH = 1
+    SEVERITY_MEDIUM = 2
+    SEVERITY_LOW = 4
+    SEVERITY_LABELS = {4: "low", 2: "medium", 1: "high"}
+
+    def __init__(self,
+                 severity,
+                 category,
+                 message,
+                 file="unknown",
+                 line=0,
+                 column=0,
+                 id=None,
+                 callstack=None,
+                 cwe=None):
+        assert severity in (self.SEVERITY_HIGH, self.SEVERITY_MEDIUM,
+                            self.SEVERITY_LOW)
+        self.severity = severity
+        self.category = category
+        self.message = message
+        self.line = line
+        self.column = column
+        self.callstack = callstack
+        self.cwe = cwe
+        self.id = id
+        self.file = file
+        if file.startswith(get_project_dir()):
+            self.file = relpath(file, get_project_dir())
+
+    def __repr__(self):
+        defect_color = None
+        if self.severity == self.SEVERITY_HIGH:
+            defect_color = "red"
+        elif self.severity == self.SEVERITY_MEDIUM:
+            defect_color = "yellow"
+
+        format_str = "{file}:{line}: [{severity}:{category}] {message} {id}"
+        return format_str.format(severity=click.style(
+            self.SEVERITY_LABELS[self.severity], fg=defect_color),
+                                 category=click.style(self.category.lower(),
+                                                      fg=defect_color),
+                                 file=click.style(self.file, bold=True),
+                                 message=self.message,
+                                 line=self.line,
+                                 id="%s" % "[%s]" % self.id if self.id else "")
+
+    def __or__(self, defect):
+        return self.severity | defect.severity
+
+    @staticmethod
+    def severity_to_int(label):
+        for key, value in DefectItem.SEVERITY_LABELS.items():
+            if label == value:
+                return key
+        raise Exception("Unknown severity label -> %s" % label)
+
+    def to_json(self):
+        return {
+            "severity": self.SEVERITY_LABELS[self.severity],
+            "category": self.category,
+            "message": self.message,
+            "file": self.file,
+            "line": self.line,
+            "column": self.column,
+            "callstack": self.callstack,
+            "id": self.id,
+            "cwe": self.cwe
+        }
+
+
+class CheckToolFactory(object):
+
+    @staticmethod
+    def new(tool, project_dir, config, envname, options):
+        clsname = "%sCheckTool" % tool.title()
+        try:
+            obj = getattr(sys.modules[__name__], clsname)(project_dir, config,
+                                                          envname, options)
+        except AttributeError:
+            raise exception.PlatformioException("Unknown check tool `%s`" %
+                                                tool)
+        assert isinstance(obj, CheckToolBase)
+        return obj
+
+
+class CheckToolBase(object):
+
+    def __init__(self, project_dir, config, envname, options):
+        self.config = config
+        self.envname = envname
+        self.options = options
+        self.cpp_defines = []
+        self.cpp_includes = []
+        self._bad_input = False
+        self._load_cpp_data(project_dir, envname)
+
+        self._defects = []
+        self._on_defect_callback = None
+
+    def _load_cpp_data(self, project_dir, envname):
+        data = load_project_ide_data(project_dir, envname)
+        if not data:
+            return
+        self.cpp_includes = data.get("includes", [])
+        self.cpp_defines = data.get("defines", [])
+        self.cpp_defines.extend(
+            self._get_toolchain_defines(data.get("cc_path")))
+
+    def get_flags(self, tool):
+        result = []
+        flags = self.options.get("flags", [])
+        for flag in flags:
+            if ":" not in flag:
+                result.extend([f for f in flag.split(" ") if f])
+            elif flag.startswith("%s:" % tool):
+                result.extend(
+                    [f for f in flag.split(":", 1)[1].split(" ") if f])
+
+        return result
+
+    @staticmethod
+    def _get_toolchain_defines(cc_path):
+        defines = []
+        result = proc.exec_command(
+            "echo | %s -dM -E -x c++ -" % cc_path, shell=True)
+
+        for line in result['out'].split("\n"):
+            tokens = line.strip().split(" ", 2)
+            if not tokens or tokens[0] != "#define":
+                continue
+            if len(tokens) > 2:
+                defines.append("%s=%s" % (tokens[1], tokens[2]))
+            else:
+                defines.append(tokens[1])
+
+        return defines
+
+    @staticmethod
+    def is_flag_set(flag, flags):
+        return any(flag in f for f in flags)
+
+    def get_defects(self):
+        return self._defects
+
+    def configure_command(self):
+        raise NotImplementedError
+
+    def on_tool_output(self, line):
+        line = self.tool_output_filter(line)
+        if not line:
+            return
+
+        defect = self.parse_defect(line)
+        if isinstance(defect, DefectItem):
+            self._defects.append(defect)
+            if self._on_defect_callback:
+                self._on_defect_callback(defect)
+        elif self.options.get("verbose"):
+            click.echo(line)
+
+    @staticmethod
+    def tool_output_filter(line):
+        return line
+
+    @staticmethod
+    def parse_defect(raw_line):
+        return raw_line
+
+    def clean_up(self):
+        pass
+
+    def exceeds_severity_threshold(self, severity):
+        return severity <= DefectItem.severity_to_int(
+            self.options.get("severity"))
+
+    def get_project_src_files(self):
+        file_extensions = ["h", "hpp", "c", "cc", "cpp", "ino"]
+        return fs.match_src_files(get_project_dir(),
+                                  self.options.get("filter"), file_extensions)
+
+    def check(self, on_defect_callback=None):
+        self._on_defect_callback = on_defect_callback
+        cmd = self.configure_command()
+        if self.options.get("verbose"):
+            click.echo(" ".join(cmd))
+
+        proc.exec_command(
+            cmd,
+            stdout=proc.LineBufferedAsyncPipe(self.on_tool_output),
+            stderr=proc.LineBufferedAsyncPipe(self.on_tool_output))
+
+        self.clean_up()
+
+        return self._bad_input
+
+
+class CppcheckCheckTool(CheckToolBase):
+
+    def __init__(self, *args, **kwargs):
+        self._tmp_files = []
+        self.defect_fields = [
+            "severity", "message", "file", "line", "column", "callstack",
+            "cwe", "id"
+        ]
+        super(CppcheckCheckTool, self).__init__(*args, **kwargs)
+
+    def tool_output_filter(self, line):
+        if not self.options.get(
+                "verbose") and "--suppress=unmatchedSuppression:" in line:
+            return ""
+
+        if any(msg in line for msg in ("No C or C++ source files found",
+                                       "unrecognized command line option")):
+            self._bad_input = True
+
+        return line
+
+    def parse_defect(self, raw_line):
+        if "<&PIO&>" not in raw_line or any(f not in raw_line
+                                            for f in self.defect_fields):
+            return None
+
+        args = dict()
+        for field in raw_line.split("<&PIO&>"):
+            field = field.strip().replace('"', "")
+            name, value = field.split("=", 1)
+            args[name] = value
+
+        args['category'] = args['severity']
+        if args['severity'] == "error":
+            args['severity'] = DefectItem.SEVERITY_HIGH
+        elif args['severity'] == "warning":
+            args['severity'] = DefectItem.SEVERITY_MEDIUM
+        else:
+            args['severity'] = DefectItem.SEVERITY_LOW
+
+        if self.exceeds_severity_threshold(args['severity']):
+            return DefectItem(**args)
+
+        return None
+
+    def configure_command(self):
+        tool_path = join(get_core_package_dir("tool-cppcheck"), "cppcheck")
+
+        cmd = [
+            tool_path, "--error-exitcode=1",
+            "--verbose" if self.options.get("verbose") else "--quiet"
+        ]
+
+        cmd.append('--template="%s"' % "<&PIO&>".join(
+            ["{0}={{{0}}}".format(f) for f in self.defect_fields]))
+
+        flags = self.get_flags("cppcheck")
+        if not self.is_flag_set("--platform", flags):
+            cmd.append("--platform=unspecified")
+        if not self.is_flag_set("--enable", flags):
+            enabled_checks = [
+                "warning", "style", "performance", "portability",
+                "unusedFunction"
+            ]
+            cmd.append("--enable=%s" % ",".join(enabled_checks))
+
+        cmd.extend(["-D%s" % d for d in self.cpp_defines])
+        cmd.extend(flags)
+
+        cmd.append("--file-list=%s" % self._generate_src_file())
+        cmd.append("--includes-file=%s" % self._generate_inc_file())
+
+        core_dir = get_project_core_dir()
+        cmd.append("--suppress=*:%s*" % core_dir)
+        cmd.append("--suppress=unmatchedSuppression:%s*" % core_dir)
+
+        return cmd
+
+    def _create_tmp_file(self, data):
+        with NamedTemporaryFile("w", delete=False) as fp:
+            fp.write(data)
+            self._tmp_files.append(fp.name)
+            return fp.name
+
+    def _generate_src_file(self):
+        return self._create_tmp_file("\n".join(self.get_project_src_files()))
+
+    def _generate_inc_file(self):
+        return self._create_tmp_file("\n".join(self.cpp_includes))
+
+    def clean_up(self):
+        for f in self._tmp_files:
+            if isfile(f):
+                remove(f)
+
+        # delete temporary dump files generated by addons
+        if not self.is_flag_set("--addon", self.get_flags("cppcheck")):
+            return
+        for f in self.get_project_src_files():
+            dump_file = f + ".dump"
+            if isfile(dump_file):
+                remove(dump_file)
+
+
+class ClangtidyCheckTool(CheckToolBase):
+
+    def tool_output_filter(self, line):
+        if not self.options.get(
+                "verbose") and "[clang-diagnostic-error]" in line:
+            return ""
+
+        if "[CommonOptionsParser]" in line:
+            self._bad_input = True
+            return line
+
+        if any(d in line for d in ("note: ", "error: ", "warning: ")):
+            return line
+
+        return ""
+
+    def parse_defect(self, raw_line):
+        match = re.match(r"^(.*):(\d+):(\d+):\s+([^:]+):\s(.+)\[([^]]+)\]$",
+                         raw_line)
+        if not match:
+            return raw_line
+
+        file, line, column, category, message, defect_id = match.groups()
+
+        severity = DefectItem.SEVERITY_LOW
+        if category == "error":
+            severity = DefectItem.SEVERITY_HIGH
+        elif category == "warning":
+            severity = DefectItem.SEVERITY_MEDIUM
+
+        if self.exceeds_severity_threshold(severity):
+            return DefectItem(severity, category, message, file, line, column,
+                              defect_id)
+
+        return None
+
+    def configure_command(self):
+        tool_path = join(get_core_package_dir("tool-clangtidy"), "clang-tidy")
+
+        cmd = [tool_path, "--quiet"]
+        flags = self.get_flags("clangtidy")
+        if not self.is_flag_set("--checks", flags):
+            cmd.append("--checks=*")
+
+        cmd.extend(flags)
+        cmd.extend(self.get_project_src_files())
+        cmd.append("--")
+
+        cmd.extend(["-D%s" % d for d in self.cpp_defines])
+        cmd.extend(["-I%s" % inc for inc in self.cpp_includes])
+
+        return cmd

platformio/managers/core.py

@@ -29,7 +29,9 @@ CORE_PACKAGES = {
     "~2.%d%d.190418" % (sys.version_info[0], sys.version_info[1]),
     "tool-pioplus": "^2.5.2",
     "tool-unity": "~1.20403.0",
-    "tool-scons": "~2.20501.7" if PY2 else "~3.30101.0"
+    "tool-scons": "~2.20501.7" if PY2 else "~3.30101.0",
+    "tool-cppcheck": "~1.189.0",
+    "tool-clangtidy": "^1.80000.0"
 }
 
 PIOPLUS_AUTO_UPDATES_MAX = 100

platformio/project/options.py

@@ -195,6 +195,11 @@ ProjectOptions = OrderedDict([
                         type=click.Path(
                             exists=True, file_okay=True, dir_okay=False)),
 
+        # Check
+        ConfigEnvOption(name="check_tool", multiple=True),
+        ConfigEnvOption(name="check_filter", multiple=True),
+        ConfigEnvOption(name="check_flags", multiple=True),
+
         # Other
         ConfigEnvOption(name="extra_scripts",
                         oldnames=["extra_script"],

tests/commands/test_check.py

@@ -0,0 +1,294 @@
+# Copyright (c) 2019-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import json
+from os.path import isfile, join
+
+from platformio.commands.check import cli as cmd_check
+
+DEFAULT_CONFIG = """
+[env:native]
+platform = native
+"""
+
+TEST_CODE = """
+#include <stdlib.h>
+
+void run_defects() {
+    /* Freeing a pointer twice */
+    int* doubleFreePi = (int*)malloc(sizeof(int));
+    *doubleFreePi=2;
+    free(doubleFreePi);
+    free(doubleFreePi);
+
+    /* Reading uninitialized memory */
+    int* uninitializedPi = (int*)malloc(sizeof(int));
+    *uninitializedPi++;
+    free(uninitializedPi);
+
+    /* Delete instead of delete [] */
+    int* wrongDeletePi = new int[10];
+    wrongDeletePi++;
+    delete wrongDeletePi;
+
+    /* Index out of bounds */
+    int arr[10];
+    for(int i=0; i < 11; i++) {
+        arr[i] = 0;
+    }
+}
+
+void unusedFuntion(){
+}
+
+int main() {
+    run_defects();
+}
+"""
+
+EXPECTED_ERRORS = 4
+EXPECTED_WARNINGS = 1
+EXPECTED_STYLE = 1
+EXPECTED_DEFECTS = EXPECTED_ERRORS + EXPECTED_WARNINGS + EXPECTED_STYLE
+
+
+def count_defects(output):
+    error, warning, style = 0, 0, 0
+    for l in output.split("\n"):
+        if "[high:error]" in l:
+            error += 1
+        elif "[medium:warning]" in l:
+            warning += 1
+        elif "[low:style]" in l:
+            style += 1
+    return error, warning, style
+
+
+def prepare_project(tmpdir):
+    tmpdir.join("platformio.ini").write(DEFAULT_CONFIG)
+    tmpdir.mkdir("src").join("main.cpp").write(TEST_CODE)
+
+
+def test_check_cli_output(clirunner, tmpdir):
+    prepare_project(tmpdir)
+
+    result = clirunner.invoke(
+        cmd_check, ["--project-dir", str(tmpdir)])
+
+    errors, warnings, style = count_defects(result.output)
+
+    assert (result.exit_code != 0)
+    assert (errors + warnings + style == EXPECTED_DEFECTS)
+
+
+def test_check_json_output(clirunner, tmpdir):
+    prepare_project(tmpdir)
+
+    result = clirunner.invoke(
+        cmd_check, ["--project-dir", str(tmpdir), "--json-output"])
+    output = json.loads(result.stdout.strip())
+
+    assert isinstance(output, list)
+    assert (len(output[0].get("defects", [])) == EXPECTED_DEFECTS)
+
+
+def test_check_tool_defines_passed(clirunner, tmpdir):
+    prepare_project(tmpdir)
+
+    result = clirunner.invoke(
+        cmd_check, ["--project-dir", str(tmpdir), "--verbose"])
+    output = result.output
+
+    assert ("PLATFORMIO=" in output)
+    assert ("__GNUC__" in output)
+
+
+def test_check_severity_threshold(clirunner, tmpdir):
+    prepare_project(tmpdir)
+
+    result = clirunner.invoke(
+        cmd_check, ["--project-dir", str(tmpdir), "--severity=high"])
+
+    errors, warnings, style = count_defects(result.output)
+
+    assert (result.exit_code != 0)
+    assert (errors == EXPECTED_ERRORS)
+    assert (warnings == 0)
+    assert (style == 0)
+
+
+def test_check_includes_passed(clirunner, tmpdir):
+    prepare_project(tmpdir)
+
+    result = clirunner.invoke(
+        cmd_check, ["--project-dir", str(tmpdir), "--verbose"])
+    output = result.output
+
+    inc_count = 0
+    for l in output.split("\n"):
+        if l.startswith("Includes:"):
+            inc_count = l.count("-I")
+
+    # at least 1 include path for default mode
+    assert (inc_count > 1)
+
+
+def test_check_silent_mode(clirunner, tmpdir):
+    prepare_project(tmpdir)
+
+    result = clirunner.invoke(
+        cmd_check, ["--project-dir", str(tmpdir), "--silent"])
+
+    errors, warnings, style = count_defects(result.output)
+
+    assert result.exit_code != 0
+    assert errors == EXPECTED_ERRORS
+    assert warnings == 0
+    assert style == 0
+
+
+def test_check_filter_sources(clirunner, tmpdir):
+    prepare_project(tmpdir)
+
+    tmpdir.mkdir(join("src", "app")).join("additional.cpp").write(TEST_CODE)
+
+    result = clirunner.invoke(
+        cmd_check,
+        ["--project-dir",
+         str(tmpdir), "--filter=-<*> +<src/app/>"])
+
+    errors, warnings, style = count_defects(result.output)
+
+    assert result.exit_code != 0
+    assert errors == EXPECTED_ERRORS
+    assert warnings == EXPECTED_WARNINGS
+    assert style == EXPECTED_STYLE
+
+
+def test_check_failed_if_no_source_files(clirunner, tmpdir):
+    tmpdir.join("platformio.ini").write(DEFAULT_CONFIG)
+    tmpdir.mkdir("src")
+
+    result = clirunner.invoke(
+        cmd_check, ["--project-dir", str(tmpdir)])
+
+    errors, warnings, style = count_defects(result.output)
+
+    assert result.exit_code != 0
+    assert errors == 0
+    assert warnings == 0
+    assert style == 0
+
+
+def test_check_failed_if_bad_flag_passed(clirunner, tmpdir):
+    prepare_project(tmpdir)
+
+    result = clirunner.invoke(
+        cmd_check, ["--project-dir", str(tmpdir), '"--flags=--UNKNOWN"'])
+
+    errors, warnings, style = count_defects(result.output)
+
+    assert result.exit_code != 0
+    assert errors == 0
+    assert warnings == 0
+    assert style == 0
+
+
+def test_check_success_if_no_errors(clirunner, tmpdir):
+    tmpdir.join("platformio.ini").write(DEFAULT_CONFIG)
+    tmpdir.mkdir("src").join("main.c").write("""
+#include <stdlib.h>
+
+void unused_functin(){
+    int unusedVar = 0;
+    int* iP = &unusedVar;
+    *iP++;
+}
+
+int main() {
+}
+""")
+
+    result = clirunner.invoke(
+        cmd_check, ["--project-dir", str(tmpdir)])
+
+    errors, warnings, style = count_defects(result.output)
+
+    assert "[PASSED]" in result.output
+    assert result.exit_code == 0
+    assert errors == 0
+    assert warnings == 1
+    assert style == 1
+
+
+def test_check_individual_flags_passed(clirunner, tmpdir):
+    config = DEFAULT_CONFIG + "\ncheck_tool = cppcheck, clangtidy"
+    config += "\ncheck_flags = cppcheck: --std=c++11 \n\tclangtidy: --fix-errors"
+    tmpdir.join("platformio.ini").write(config)
+    tmpdir.mkdir("src").join("main.cpp").write(TEST_CODE)
+    result = clirunner.invoke(
+        cmd_check, ["--project-dir", str(tmpdir), "-v"])
+
+    clang_flags_found = cppcheck_flags_found = False
+    for l in result.output.split("\n"):
+        if "--fix" in l and "clang-tidy" in l and "--std=c++11" not in l:
+            clang_flags_found = True
+        elif "--std=c++11" in l and "cppcheck" in l and "--fix" not in l:
+            cppcheck_flags_found = True
+
+    assert clang_flags_found
+    assert cppcheck_flags_found
+
+
+def test_check_cppcheck_misra_addon(clirunner, tmpdir):
+    prepare_project(tmpdir)
+
+    tmpdir.join("misra.json").write("""
+{
+    "script": "addons/misra.py",
+    "args": ["--rule-texts=rules.txt"]
+}
+""")
+
+    tmpdir.join("rules.txt").write("""
+Appendix A Summary of guidelines
+Rule 3.1 Required
+R3.1 text.
+Rule 4.1 Required
+R4.1 text.
+Rule 10.4 Mandatory
+R10.4 text.
+Rule 11.5 Advisory
+R11.5 text.
+Rule 15.5 Advisory
+R15.5 text.
+Rule 15.6 Required
+R15.6 text.
+Rule 17.7 Required
+R17.7 text.
+Rule 20.1 Advisory
+R20.1 text.
+Rule 21.3 Required
+R21.3 Found MISRA defect
+Rule 21.4
+R21.4 text.
+""")
+
+    result = clirunner.invoke(cmd_check, [
+        "--project-dir", str(tmpdir), "--flags=--addon=misra.json"])
+
+    assert result.exit_code != 0
+    assert "R21.3 Found MISRA defect" in result.output
+    assert not isfile(join(str(tmpdir), "src", "main.cpp.dump"))