From c720933d34553f71caac427b8ddc912a3825131c Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sun, 8 Sep 2019 23:33:25 +0300 Subject: [PATCH] Refactor PIO Check --- docs | 2 +- examples | 2 +- platformio/commands/check/command.py | 17 +- platformio/commands/check/defect.py | 94 +++++ platformio/commands/check/tools.py | 385 ------------------- platformio/commands/check/tools/__init__.py | 32 ++ platformio/commands/check/tools/base.py | 144 +++++++ platformio/commands/check/tools/clangtidy.py | 72 ++++ platformio/commands/check/tools/cppcheck.py | 124 ++++++ platformio/project/options.py | 3 + tests/commands/test_check.py | 98 +++-- 11 files changed, 527 insertions(+), 446 deletions(-) create mode 100644 platformio/commands/check/defect.py delete mode 100644 platformio/commands/check/tools.py create mode 100644 platformio/commands/check/tools/__init__.py create mode 100644 platformio/commands/check/tools/base.py create mode 100644 platformio/commands/check/tools/clangtidy.py create mode 100644 platformio/commands/check/tools/cppcheck.py diff --git a/docs b/docs index 29f80d45..98017a5f 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 29f80d45f2d7fe14918b507d84ec8badc54fe087 +Subproject commit 98017a5fffec8603af4358bd988dce5e98cad52a diff --git a/examples b/examples index a71564ab..6859117a 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit a71564ab46d27c387f17814056b659f826b7db24 +Subproject commit 6859117a8c0b5d293a00e29b339250d0587f31de diff --git a/platformio/commands/check/command.py b/platformio/commands/check/command.py index 00b6c274..52b7c339 100644 --- a/platformio/commands/check/command.py +++ b/platformio/commands/check/command.py @@ -24,7 +24,8 @@ import click from tabulate import tabulate from platformio import exception, fs, util -from platformio.commands.check.tools import CheckToolFactory, DefectItem +from platformio.commands.check.tools import CheckToolFactory +from platformio.commands.check.defect import DefectItem from platformio.compat import dump_json_to_unicode from platformio.project.config import ProjectConfig from platformio.project.helpers import (find_project_dir_above, @@ -53,8 +54,8 @@ from platformio.project.helpers import (find_project_dir_above, @click.option("--filter", multiple=True, help="Pattern: + -") @click.option("--flags", multiple=True) @click.option("--severity", - type=click.Choice(DefectItem.SEVERITY_LABELS.values()), - default=DefectItem.SEVERITY_LABELS[DefectItem.SEVERITY_LOW]) + multiple=True, + type=click.Choice(DefectItem.SEVERITY_LABELS.values())) @click.option("-s", "--silent", is_flag=True) @click.option("-v", "--verbose", is_flag=True) @click.option("--json-output", is_flag=True) @@ -95,8 +96,11 @@ def cli(environment, project_dir, project_conf, filter, flags, severity, silent=silent, filter=filter or env_options.get("check_filter", default_filter), - flags=flags or env_options.get("check_flags", ()), - severity=severity if not silent else "high") + flags=flags or env_options.get("check_flags"), + severity=[ + DefectItem.SEVERITY_LABELS[DefectItem.SEVERITY_HIGH] + ] if silent else + (severity or env_options.get("check_severity"))) for tool in env_options.get("check_tool", ["cppcheck"]): if skipenv: @@ -124,6 +128,8 @@ def cli(environment, project_dir, project_conf, filter, flags, severity, click.echo("\n".join(repr(d) for d in result['defects'])) if not json_output and not silent: + if not result['defects']: + click.echo("No defects found") print_processing_footer(result) if json_output: @@ -216,6 +222,7 @@ def print_check_summary(results): duration = 0 print_defects_stats(results) + for result in results: duration += result.get("duration", 0) if result.get("succeeded") is False: diff --git a/platformio/commands/check/defect.py b/platformio/commands/check/defect.py new file mode 100644 index 00000000..7c2fbcff --- /dev/null +++ b/platformio/commands/check/defect.py @@ -0,0 +1,94 @@ +# Copyright (c) 2019-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from os.path import relpath + +import click + +from platformio.project.helpers import get_project_dir + +# pylint: disable=too-many-instance-attributes, redefined-builtin +# pylint: disable=too-many-arguments + + +class DefectItem(object): + + SEVERITY_HIGH = 1 + SEVERITY_MEDIUM = 2 + SEVERITY_LOW = 4 + SEVERITY_LABELS = {4: "low", 2: "medium", 1: "high"} + + def __init__(self, + severity, + category, + message, + file="unknown", + line=0, + column=0, + id=None, + callstack=None, + cwe=None): + assert severity in (self.SEVERITY_HIGH, self.SEVERITY_MEDIUM, + self.SEVERITY_LOW) + self.severity = severity + self.category = category + self.message = message + self.line = line + self.column = column + self.callstack = callstack + self.cwe = cwe + self.id = id + self.file = file + if file.startswith(get_project_dir()): + self.file = relpath(file, get_project_dir()) + + def __repr__(self): + defect_color = None + if self.severity == self.SEVERITY_HIGH: + defect_color = "red" + elif self.severity == self.SEVERITY_MEDIUM: + defect_color = "yellow" + + format_str = "{file}:{line}: [{severity}:{category}] {message} {id}" + return format_str.format(severity=click.style( + self.SEVERITY_LABELS[self.severity], fg=defect_color), + category=click.style(self.category.lower(), + fg=defect_color), + file=click.style(self.file, bold=True), + message=self.message, + line=self.line, + id="%s" % "[%s]" % self.id if self.id else "") + + def __or__(self, defect): + return self.severity | defect.severity + + @staticmethod + def severity_to_int(label): + for key, value in DefectItem.SEVERITY_LABELS.items(): + if label == value: + return key + raise Exception("Unknown severity label -> %s" % label) + + def to_json(self): + return { + "severity": self.SEVERITY_LABELS[self.severity], + "category": self.category, + "message": self.message, + "file": self.file, + "line": self.line, + "column": self.column, + "callstack": self.callstack, + "id": self.id, + "cwe": self.cwe + } diff --git a/platformio/commands/check/tools.py b/platformio/commands/check/tools.py deleted file mode 100644 index c2b682db..00000000 --- a/platformio/commands/check/tools.py +++ /dev/null @@ -1,385 +0,0 @@ -# Copyright (c) 2019-present PlatformIO -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# pylint: disable=too-many-arguments,too-many-instance-attributes -# pylint: disable=redefined-builtin - -import re -import sys -from os import remove -from os.path import isfile, join, relpath -from tempfile import NamedTemporaryFile - -import click - -from platformio import exception, fs, proc -from platformio.managers.core import get_core_package_dir -from platformio.project.helpers import (get_project_core_dir, get_project_dir, - load_project_ide_data) - - -class DefectItem(object): - - SEVERITY_HIGH = 1 - SEVERITY_MEDIUM = 2 - SEVERITY_LOW = 4 - SEVERITY_LABELS = {4: "low", 2: "medium", 1: "high"} - - def __init__(self, - severity, - category, - message, - file="unknown", - line=0, - column=0, - id=None, - callstack=None, - cwe=None): - assert severity in (self.SEVERITY_HIGH, self.SEVERITY_MEDIUM, - self.SEVERITY_LOW) - self.severity = severity - self.category = category - self.message = message - self.line = line - self.column = column - self.callstack = callstack - self.cwe = cwe - self.id = id - self.file = file - if file.startswith(get_project_dir()): - self.file = relpath(file, get_project_dir()) - - def __repr__(self): - defect_color = None - if self.severity == self.SEVERITY_HIGH: - defect_color = "red" - elif self.severity == self.SEVERITY_MEDIUM: - defect_color = "yellow" - - format_str = "{file}:{line}: [{severity}:{category}] {message} {id}" - return format_str.format(severity=click.style( - self.SEVERITY_LABELS[self.severity], fg=defect_color), - category=click.style(self.category.lower(), - fg=defect_color), - file=click.style(self.file, bold=True), - message=self.message, - line=self.line, - id="%s" % "[%s]" % self.id if self.id else "") - - def __or__(self, defect): - return self.severity | defect.severity - - @staticmethod - def severity_to_int(label): - for key, value in DefectItem.SEVERITY_LABELS.items(): - if label == value: - return key - raise Exception("Unknown severity label -> %s" % label) - - def to_json(self): - return { - "severity": self.SEVERITY_LABELS[self.severity], - "category": self.category, - "message": self.message, - "file": self.file, - "line": self.line, - "column": self.column, - "callstack": self.callstack, - "id": self.id, - "cwe": self.cwe - } - - -class CheckToolFactory(object): - - @staticmethod - def new(tool, project_dir, config, envname, options): - clsname = "%sCheckTool" % tool.title() - try: - obj = getattr(sys.modules[__name__], clsname)(project_dir, config, - envname, options) - except AttributeError: - raise exception.PlatformioException("Unknown check tool `%s`" % - tool) - assert isinstance(obj, CheckToolBase) - return obj - - -class CheckToolBase(object): - - def __init__(self, project_dir, config, envname, options): - self.config = config - self.envname = envname - self.options = options - self.cpp_defines = [] - self.cpp_includes = [] - self._bad_input = False - self._load_cpp_data(project_dir, envname) - - self._defects = [] - self._on_defect_callback = None - - def _load_cpp_data(self, project_dir, envname): - data = load_project_ide_data(project_dir, envname) - if not data: - return - self.cpp_includes = data.get("includes", []) - self.cpp_defines = data.get("defines", []) - self.cpp_defines.extend( - self._get_toolchain_defines(data.get("cc_path"))) - - def get_flags(self, tool): - result = [] - flags = self.options.get("flags", []) - for flag in flags: - if ":" not in flag: - result.extend([f for f in flag.split(" ") if f]) - elif flag.startswith("%s:" % tool): - result.extend( - [f for f in flag.split(":", 1)[1].split(" ") if f]) - - return result - - @staticmethod - def _get_toolchain_defines(cc_path): - defines = [] - result = proc.exec_command( - "echo | %s -dM -E -x c++ -" % cc_path, shell=True) - - for line in result['out'].split("\n"): - tokens = line.strip().split(" ", 2) - if not tokens or tokens[0] != "#define": - continue - if len(tokens) > 2: - defines.append("%s=%s" % (tokens[1], tokens[2])) - else: - defines.append(tokens[1]) - - return defines - - @staticmethod - def is_flag_set(flag, flags): - return any(flag in f for f in flags) - - def get_defects(self): - return self._defects - - def configure_command(self): - raise NotImplementedError - - def on_tool_output(self, line): - line = self.tool_output_filter(line) - if not line: - return - - defect = self.parse_defect(line) - if isinstance(defect, DefectItem): - self._defects.append(defect) - if self._on_defect_callback: - self._on_defect_callback(defect) - elif self.options.get("verbose"): - click.echo(line) - - @staticmethod - def tool_output_filter(line): - return line - - @staticmethod - def parse_defect(raw_line): - return raw_line - - def clean_up(self): - pass - - def exceeds_severity_threshold(self, severity): - return severity <= DefectItem.severity_to_int( - self.options.get("severity")) - - def get_project_src_files(self): - file_extensions = ["h", "hpp", "c", "cc", "cpp", "ino"] - return fs.match_src_files(get_project_dir(), - self.options.get("filter"), file_extensions) - - def check(self, on_defect_callback=None): - self._on_defect_callback = on_defect_callback - cmd = self.configure_command() - if self.options.get("verbose"): - click.echo(" ".join(cmd)) - - proc.exec_command( - cmd, - stdout=proc.LineBufferedAsyncPipe(self.on_tool_output), - stderr=proc.LineBufferedAsyncPipe(self.on_tool_output)) - - self.clean_up() - - return self._bad_input - - -class CppcheckCheckTool(CheckToolBase): - - def __init__(self, *args, **kwargs): - self._tmp_files = [] - self.defect_fields = [ - "severity", "message", "file", "line", "column", "callstack", - "cwe", "id" - ] - super(CppcheckCheckTool, self).__init__(*args, **kwargs) - - def tool_output_filter(self, line): - if not self.options.get( - "verbose") and "--suppress=unmatchedSuppression:" in line: - return "" - - if any(msg in line for msg in ("No C or C++ source files found", - "unrecognized command line option")): - self._bad_input = True - - return line - - def parse_defect(self, raw_line): - if "<&PIO&>" not in raw_line or any(f not in raw_line - for f in self.defect_fields): - return None - - args = dict() - for field in raw_line.split("<&PIO&>"): - field = field.strip().replace('"', "") - name, value = field.split("=", 1) - args[name] = value - - args['category'] = args['severity'] - if args['severity'] == "error": - args['severity'] = DefectItem.SEVERITY_HIGH - elif args['severity'] == "warning": - args['severity'] = DefectItem.SEVERITY_MEDIUM - else: - args['severity'] = DefectItem.SEVERITY_LOW - - if self.exceeds_severity_threshold(args['severity']): - return DefectItem(**args) - - return None - - def configure_command(self): - tool_path = join(get_core_package_dir("tool-cppcheck"), "cppcheck") - - cmd = [ - tool_path, "--error-exitcode=1", - "--verbose" if self.options.get("verbose") else "--quiet" - ] - - cmd.append('--template="%s"' % "<&PIO&>".join( - ["{0}={{{0}}}".format(f) for f in self.defect_fields])) - - flags = self.get_flags("cppcheck") - if not self.is_flag_set("--platform", flags): - cmd.append("--platform=unspecified") - if not self.is_flag_set("--enable", flags): - enabled_checks = [ - "warning", "style", "performance", "portability", - "unusedFunction" - ] - cmd.append("--enable=%s" % ",".join(enabled_checks)) - - cmd.extend(["-D%s" % d for d in self.cpp_defines]) - cmd.extend(flags) - - cmd.append("--file-list=%s" % self._generate_src_file()) - cmd.append("--includes-file=%s" % self._generate_inc_file()) - - core_dir = get_project_core_dir() - cmd.append("--suppress=*:%s*" % core_dir) - cmd.append("--suppress=unmatchedSuppression:%s*" % core_dir) - - return cmd - - def _create_tmp_file(self, data): - with NamedTemporaryFile("w", delete=False) as fp: - fp.write(data) - self._tmp_files.append(fp.name) - return fp.name - - def _generate_src_file(self): - return self._create_tmp_file("\n".join(self.get_project_src_files())) - - def _generate_inc_file(self): - return self._create_tmp_file("\n".join(self.cpp_includes)) - - def clean_up(self): - for f in self._tmp_files: - if isfile(f): - remove(f) - - # delete temporary dump files generated by addons - if not self.is_flag_set("--addon", self.get_flags("cppcheck")): - return - for f in self.get_project_src_files(): - dump_file = f + ".dump" - if isfile(dump_file): - remove(dump_file) - - -class ClangtidyCheckTool(CheckToolBase): - - def tool_output_filter(self, line): - if not self.options.get( - "verbose") and "[clang-diagnostic-error]" in line: - return "" - - if "[CommonOptionsParser]" in line: - self._bad_input = True - return line - - if any(d in line for d in ("note: ", "error: ", "warning: ")): - return line - - return "" - - def parse_defect(self, raw_line): - match = re.match(r"^(.*):(\d+):(\d+):\s+([^:]+):\s(.+)\[([^]]+)\]$", - raw_line) - if not match: - return raw_line - - file, line, column, category, message, defect_id = match.groups() - - severity = DefectItem.SEVERITY_LOW - if category == "error": - severity = DefectItem.SEVERITY_HIGH - elif category == "warning": - severity = DefectItem.SEVERITY_MEDIUM - - if self.exceeds_severity_threshold(severity): - return DefectItem(severity, category, message, file, line, column, - defect_id) - - return None - - def configure_command(self): - tool_path = join(get_core_package_dir("tool-clangtidy"), "clang-tidy") - - cmd = [tool_path, "--quiet"] - flags = self.get_flags("clangtidy") - if not self.is_flag_set("--checks", flags): - cmd.append("--checks=*") - - cmd.extend(flags) - cmd.extend(self.get_project_src_files()) - cmd.append("--") - - cmd.extend(["-D%s" % d for d in self.cpp_defines]) - cmd.extend(["-I%s" % inc for inc in self.cpp_includes]) - - return cmd diff --git a/platformio/commands/check/tools/__init__.py b/platformio/commands/check/tools/__init__.py new file mode 100644 index 00000000..9853b595 --- /dev/null +++ b/platformio/commands/check/tools/__init__.py @@ -0,0 +1,32 @@ +# Copyright (c) 2019-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from platformio import exception +from platformio.commands.check.tools.cppcheck import CppcheckCheckTool +from platformio.commands.check.tools.clangtidy import ClangtidyCheckTool + + +class CheckToolFactory(object): + + @staticmethod + def new(tool, project_dir, config, envname, options): + cls = None + if tool == "cppcheck": + cls = CppcheckCheckTool + elif tool == "clangtidy": + cls = ClangtidyCheckTool + else: + raise exception.PlatformioException("Unknown check tool `%s`" % + tool) + return cls(project_dir, config, envname, options) diff --git a/platformio/commands/check/tools/base.py b/platformio/commands/check/tools/base.py new file mode 100644 index 00000000..e2a9ccb9 --- /dev/null +++ b/platformio/commands/check/tools/base.py @@ -0,0 +1,144 @@ +# Copyright (c) 2019-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import click + +from platformio import fs, proc +from platformio.commands.check.defect import DefectItem +from platformio.project.helpers import (get_project_dir, load_project_ide_data) + + +class CheckToolBase(object): # pylint: disable=too-many-instance-attributes + + def __init__(self, project_dir, config, envname, options): + self.config = config + self.envname = envname + self.options = options + self.cpp_defines = [] + self.cpp_includes = [] + + self._defects = [] + self._on_defect_callback = None + self._bad_input = False + self._load_cpp_data(project_dir, envname) + + # detect all defects by default + if not self.options.get("severity"): + self.options['severity'] = [ + DefectItem.SEVERITY_LOW, DefectItem.SEVERITY_MEDIUM, + DefectItem.SEVERITY_HIGH + ] + # cast to severity by ids + self.options['severity'] = [ + s if isinstance(s, int) else DefectItem.severity_to_int(s) + for s in self.options['severity'] + ] + + def _load_cpp_data(self, project_dir, envname): + data = load_project_ide_data(project_dir, envname) + if not data: + return + self.cpp_includes = data.get("includes", []) + self.cpp_defines = data.get("defines", []) + self.cpp_defines.extend( + self._get_toolchain_defines(data.get("cc_path"))) + + def get_flags(self, tool): + result = [] + flags = self.options.get("flags") or [] + for flag in flags: + if ":" not in flag: + result.extend([f for f in flag.split(" ") if f]) + elif flag.startswith("%s:" % tool): + result.extend( + [f for f in flag.split(":", 1)[1].split(" ") if f]) + + return result + + @staticmethod + def _get_toolchain_defines(cc_path): + defines = [] + result = proc.exec_command("echo | %s -dM -E -x c++ -" % cc_path, + shell=True) + + for line in result['out'].split("\n"): + tokens = line.strip().split(" ", 2) + if not tokens or tokens[0] != "#define": + continue + if len(tokens) > 2: + defines.append("%s=%s" % (tokens[1], tokens[2])) + else: + defines.append(tokens[1]) + + return defines + + @staticmethod + def is_flag_set(flag, flags): + return any(flag in f for f in flags) + + def get_defects(self): + return self._defects + + def configure_command(self): + raise NotImplementedError + + def on_tool_output(self, line): + line = self.tool_output_filter(line) + if not line: + return + + defect = self.parse_defect(line) + + if not isinstance(defect, DefectItem): + if self.options.get("verbose"): + click.echo(line) + return + + if defect.severity not in self.options['severity']: + return + + self._defects.append(defect) + if self._on_defect_callback: + self._on_defect_callback(defect) + + @staticmethod + def tool_output_filter(line): + return line + + @staticmethod + def parse_defect(raw_line): + return raw_line + + def clean_up(self): + pass + + def get_project_src_files(self): + file_extensions = ["h", "hpp", "c", "cc", "cpp", "ino"] + return fs.match_src_files(get_project_dir(), + self.options.get("filter"), file_extensions) + + def check(self, on_defect_callback=None): + self._on_defect_callback = on_defect_callback + cmd = self.configure_command() + if self.options.get("verbose"): + click.echo(" ".join(cmd)) + + proc.exec_command( + cmd, + stdout=proc.LineBufferedAsyncPipe(self.on_tool_output), + stderr=proc.LineBufferedAsyncPipe(self.on_tool_output)) + + self.clean_up() + + return self._bad_input diff --git a/platformio/commands/check/tools/clangtidy.py b/platformio/commands/check/tools/clangtidy.py new file mode 100644 index 00000000..83eec7a2 --- /dev/null +++ b/platformio/commands/check/tools/clangtidy.py @@ -0,0 +1,72 @@ +# Copyright (c) 2019-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +from os.path import join + + +from platformio.commands.check.tools.base import CheckToolBase +from platformio.commands.check.defect import DefectItem +from platformio.managers.core import get_core_package_dir + + +class ClangtidyCheckTool(CheckToolBase): + + def tool_output_filter(self, line): + if not self.options.get( + "verbose") and "[clang-diagnostic-error]" in line: + return "" + + if "[CommonOptionsParser]" in line: + self._bad_input = True + return line + + if any(d in line for d in ("note: ", "error: ", "warning: ")): + return line + + return "" + + def parse_defect(self, raw_line): + match = re.match(r"^(.*):(\d+):(\d+):\s+([^:]+):\s(.+)\[([^]]+)\]$", + raw_line) + if not match: + return raw_line + + file, line, column, category, message, defect_id = match.groups() + + severity = DefectItem.SEVERITY_LOW + if category == "error": + severity = DefectItem.SEVERITY_HIGH + elif category == "warning": + severity = DefectItem.SEVERITY_MEDIUM + + return DefectItem(severity, category, message, file, line, column, + defect_id) + + def configure_command(self): + tool_path = join(get_core_package_dir("tool-clangtidy"), "clang-tidy") + + cmd = [tool_path, "--quiet"] + flags = self.get_flags("clangtidy") + if not self.is_flag_set("--checks", flags): + cmd.append("--checks=*") + + cmd.extend(flags) + cmd.extend(self.get_project_src_files()) + cmd.append("--") + + cmd.extend(["-D%s" % d for d in self.cpp_defines]) + cmd.extend(["-I%s" % inc for inc in self.cpp_includes]) + + return cmd diff --git a/platformio/commands/check/tools/cppcheck.py b/platformio/commands/check/tools/cppcheck.py new file mode 100644 index 00000000..b7e3a3e0 --- /dev/null +++ b/platformio/commands/check/tools/cppcheck.py @@ -0,0 +1,124 @@ +# Copyright (c) 2019-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from os import remove +from os.path import isfile, join +from tempfile import NamedTemporaryFile + + +from platformio.commands.check.tools.base import CheckToolBase +from platformio.commands.check.defect import DefectItem +from platformio.managers.core import get_core_package_dir +from platformio.project.helpers import get_project_core_dir + + +class CppcheckCheckTool(CheckToolBase): + + def __init__(self, *args, **kwargs): + self._tmp_files = [] + self.defect_fields = [ + "severity", "message", "file", "line", "column", "callstack", + "cwe", "id" + ] + super(CppcheckCheckTool, self).__init__(*args, **kwargs) + + def tool_output_filter(self, line): + if not self.options.get( + "verbose") and "--suppress=unmatchedSuppression:" in line: + return "" + + if any(msg in line for msg in ("No C or C++ source files found", + "unrecognized command line option")): + self._bad_input = True + + return line + + def parse_defect(self, raw_line): + if "<&PIO&>" not in raw_line or any(f not in raw_line + for f in self.defect_fields): + return None + + args = dict() + for field in raw_line.split("<&PIO&>"): + field = field.strip().replace('"', "") + name, value = field.split("=", 1) + args[name] = value + + args['category'] = args['severity'] + if args['severity'] == "error": + args['severity'] = DefectItem.SEVERITY_HIGH + elif args['severity'] == "warning": + args['severity'] = DefectItem.SEVERITY_MEDIUM + else: + args['severity'] = DefectItem.SEVERITY_LOW + + return DefectItem(**args) + + def configure_command(self): + tool_path = join(get_core_package_dir("tool-cppcheck"), "cppcheck") + + cmd = [ + tool_path, "--error-exitcode=1", + "--verbose" if self.options.get("verbose") else "--quiet" + ] + + cmd.append('--template="%s"' % "<&PIO&>".join( + ["{0}={{{0}}}".format(f) for f in self.defect_fields])) + + flags = self.get_flags("cppcheck") + if not self.is_flag_set("--platform", flags): + cmd.append("--platform=unspecified") + if not self.is_flag_set("--enable", flags): + enabled_checks = [ + "warning", "style", "performance", "portability", + "unusedFunction" + ] + cmd.append("--enable=%s" % ",".join(enabled_checks)) + + cmd.extend(["-D%s" % d for d in self.cpp_defines]) + cmd.extend(flags) + + cmd.append("--file-list=%s" % self._generate_src_file()) + cmd.append("--includes-file=%s" % self._generate_inc_file()) + + core_dir = get_project_core_dir() + cmd.append("--suppress=*:%s*" % core_dir) + cmd.append("--suppress=unmatchedSuppression:%s*" % core_dir) + + return cmd + + def _create_tmp_file(self, data): + with NamedTemporaryFile("w", delete=False) as fp: + fp.write(data) + self._tmp_files.append(fp.name) + return fp.name + + def _generate_src_file(self): + return self._create_tmp_file("\n".join(self.get_project_src_files())) + + def _generate_inc_file(self): + return self._create_tmp_file("\n".join(self.cpp_includes)) + + def clean_up(self): + for f in self._tmp_files: + if isfile(f): + remove(f) + + # delete temporary dump files generated by addons + if not self.is_flag_set("--addon", self.get_flags("cppcheck")): + return + for f in self.get_project_src_files(): + dump_file = f + ".dump" + if isfile(dump_file): + remove(dump_file) diff --git a/platformio/project/options.py b/platformio/project/options.py index 96733c2a..59a11081 100644 --- a/platformio/project/options.py +++ b/platformio/project/options.py @@ -199,6 +199,9 @@ ProjectOptions = OrderedDict([ ConfigEnvOption(name="check_tool", multiple=True), ConfigEnvOption(name="check_filter", multiple=True), ConfigEnvOption(name="check_flags", multiple=True), + ConfigEnvOption(name="check_severity", + multiple=True, + type=click.Choice(["low", "medium", "high"])), # Other ConfigEnvOption(name="extra_scripts", diff --git a/tests/commands/test_check.py b/tests/commands/test_check.py index 72352429..29a377ab 100644 --- a/tests/commands/test_check.py +++ b/tests/commands/test_check.py @@ -12,10 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. - import json from os.path import isfile, join +import pytest + from platformio.commands.check import cli as cmd_check DEFAULT_CONFIG = """ @@ -64,6 +65,14 @@ EXPECTED_STYLE = 1 EXPECTED_DEFECTS = EXPECTED_ERRORS + EXPECTED_WARNINGS + EXPECTED_STYLE +@pytest.fixture(scope="module") +def check_dir(tmpdir_factory): + tmpdir = tmpdir_factory.mktemp("project") + tmpdir.join("platformio.ini").write(DEFAULT_CONFIG) + tmpdir.mkdir("src").join("main.cpp").write(TEST_CODE) + return tmpdir + + def count_defects(output): error, warning, style = 0, 0, 0 for l in output.split("\n"): @@ -76,16 +85,8 @@ def count_defects(output): return error, warning, style -def prepare_project(tmpdir): - tmpdir.join("platformio.ini").write(DEFAULT_CONFIG) - tmpdir.mkdir("src").join("main.cpp").write(TEST_CODE) - - -def test_check_cli_output(clirunner, tmpdir): - prepare_project(tmpdir) - - result = clirunner.invoke( - cmd_check, ["--project-dir", str(tmpdir)]) +def test_check_cli_output(clirunner, check_dir): + result = clirunner.invoke(cmd_check, ["--project-dir", str(check_dir)]) errors, warnings, style = count_defects(result.output) @@ -93,33 +94,30 @@ def test_check_cli_output(clirunner, tmpdir): assert (errors + warnings + style == EXPECTED_DEFECTS) -def test_check_json_output(clirunner, tmpdir): - prepare_project(tmpdir) - +def test_check_json_output(clirunner, check_dir): result = clirunner.invoke( - cmd_check, ["--project-dir", str(tmpdir), "--json-output"]) + cmd_check, + ["--project-dir", str(check_dir), "--json-output"]) output = json.loads(result.stdout.strip()) assert isinstance(output, list) assert (len(output[0].get("defects", [])) == EXPECTED_DEFECTS) -def test_check_tool_defines_passed(clirunner, tmpdir): - prepare_project(tmpdir) - +def test_check_tool_defines_passed(clirunner, check_dir): result = clirunner.invoke( - cmd_check, ["--project-dir", str(tmpdir), "--verbose"]) + cmd_check, + ["--project-dir", str(check_dir), "--verbose"]) output = result.output assert ("PLATFORMIO=" in output) assert ("__GNUC__" in output) -def test_check_severity_threshold(clirunner, tmpdir): - prepare_project(tmpdir) - +def test_check_severity_threshold(clirunner, check_dir): result = clirunner.invoke( - cmd_check, ["--project-dir", str(tmpdir), "--severity=high"]) + cmd_check, + ["--project-dir", str(check_dir), "--severity=high"]) errors, warnings, style = count_defects(result.output) @@ -129,11 +127,10 @@ def test_check_severity_threshold(clirunner, tmpdir): assert (style == 0) -def test_check_includes_passed(clirunner, tmpdir): - prepare_project(tmpdir) - +def test_check_includes_passed(clirunner, check_dir): result = clirunner.invoke( - cmd_check, ["--project-dir", str(tmpdir), "--verbose"]) + cmd_check, + ["--project-dir", str(check_dir), "--verbose"]) output = result.output inc_count = 0 @@ -145,11 +142,10 @@ def test_check_includes_passed(clirunner, tmpdir): assert (inc_count > 1) -def test_check_silent_mode(clirunner, tmpdir): - prepare_project(tmpdir) - +def test_check_silent_mode(clirunner, check_dir): result = clirunner.invoke( - cmd_check, ["--project-dir", str(tmpdir), "--silent"]) + cmd_check, + ["--project-dir", str(check_dir), "--silent"]) errors, warnings, style = count_defects(result.output) @@ -159,15 +155,13 @@ def test_check_silent_mode(clirunner, tmpdir): assert style == 0 -def test_check_filter_sources(clirunner, tmpdir): - prepare_project(tmpdir) - - tmpdir.mkdir(join("src", "app")).join("additional.cpp").write(TEST_CODE) +def test_check_filter_sources(clirunner, check_dir): + check_dir.mkdir(join("src", "app")).join("additional.cpp").write(TEST_CODE) result = clirunner.invoke( cmd_check, ["--project-dir", - str(tmpdir), "--filter=-<*> +"]) + str(check_dir), "--filter=-<*> +"]) errors, warnings, style = count_defects(result.output) @@ -181,8 +175,7 @@ def test_check_failed_if_no_source_files(clirunner, tmpdir): tmpdir.join("platformio.ini").write(DEFAULT_CONFIG) tmpdir.mkdir("src") - result = clirunner.invoke( - cmd_check, ["--project-dir", str(tmpdir)]) + result = clirunner.invoke(cmd_check, ["--project-dir", str(tmpdir)]) errors, warnings, style = count_defects(result.output) @@ -192,11 +185,10 @@ def test_check_failed_if_no_source_files(clirunner, tmpdir): assert style == 0 -def test_check_failed_if_bad_flag_passed(clirunner, tmpdir): - prepare_project(tmpdir) - +def test_check_failed_if_bad_flag_passed(clirunner, check_dir): result = clirunner.invoke( - cmd_check, ["--project-dir", str(tmpdir), '"--flags=--UNKNOWN"']) + cmd_check, ["--project-dir", + str(check_dir), '"--flags=--UNKNOWN"']) errors, warnings, style = count_defects(result.output) @@ -221,8 +213,7 @@ int main() { } """) - result = clirunner.invoke( - cmd_check, ["--project-dir", str(tmpdir)]) + result = clirunner.invoke(cmd_check, ["--project-dir", str(tmpdir)]) errors, warnings, style = count_defects(result.output) @@ -238,8 +229,7 @@ def test_check_individual_flags_passed(clirunner, tmpdir): config += "\ncheck_flags = cppcheck: --std=c++11 \n\tclangtidy: --fix-errors" tmpdir.join("platformio.ini").write(config) tmpdir.mkdir("src").join("main.cpp").write(TEST_CODE) - result = clirunner.invoke( - cmd_check, ["--project-dir", str(tmpdir), "-v"]) + result = clirunner.invoke(cmd_check, ["--project-dir", str(tmpdir), "-v"]) clang_flags_found = cppcheck_flags_found = False for l in result.output.split("\n"): @@ -252,17 +242,15 @@ def test_check_individual_flags_passed(clirunner, tmpdir): assert cppcheck_flags_found -def test_check_cppcheck_misra_addon(clirunner, tmpdir): - prepare_project(tmpdir) - - tmpdir.join("misra.json").write(""" +def test_check_cppcheck_misra_addon(clirunner, check_dir): + check_dir.join("misra.json").write(""" { "script": "addons/misra.py", "args": ["--rule-texts=rules.txt"] } """) - tmpdir.join("rules.txt").write(""" + check_dir.join("rules.txt").write(""" Appendix A Summary of guidelines Rule 3.1 Required R3.1 text. @@ -286,9 +274,11 @@ Rule 21.4 R21.4 text. """) - result = clirunner.invoke(cmd_check, [ - "--project-dir", str(tmpdir), "--flags=--addon=misra.json"]) + result = clirunner.invoke( + cmd_check, + ["--project-dir", + str(check_dir), "--flags=--addon=misra.json"]) assert result.exit_code != 0 assert "R21.3 Found MISRA defect" in result.output - assert not isfile(join(str(tmpdir), "src", "main.cpp.dump")) + assert not isfile(join(str(check_dir), "src", "main.cpp.dump"))