diff --git a/platformio/check/__init__.py b/platformio/check/__init__.py new file mode 100644 index 00000000..895dc9b9 --- /dev/null +++ b/platformio/check/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2019-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/check/defect.py b/platformio/check/defect.py new file mode 100644 index 00000000..0b25084c --- /dev/null +++ b/platformio/check/defect.py @@ -0,0 +1,95 @@ +# Copyright (c) 2019-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from os.path import relpath + +import click + +from platformio.project.helpers import get_project_dir + +# pylint: disable=too-many-instance-attributes, redefined-builtin +# pylint: disable=too-many-arguments + + +class DefectItem(object): + + SEVERITY_HIGH = 1 + SEVERITY_MEDIUM = 2 + SEVERITY_LOW = 4 + SEVERITY_LABELS = {4: "low", 2: "medium", 1: "high"} + + def __init__( + self, + severity, + category, + message, + file="unknown", + line=0, + column=0, + id=None, + callstack=None, + cwe=None, + ): + assert severity in (self.SEVERITY_HIGH, self.SEVERITY_MEDIUM, self.SEVERITY_LOW) + self.severity = severity + self.category = category + self.message = message + self.line = line + self.column = column + self.callstack = callstack + self.cwe = cwe + self.id = id + self.file = file + if file.startswith(get_project_dir()): + self.file = relpath(file, get_project_dir()) + + def __repr__(self): + defect_color = None + if self.severity == self.SEVERITY_HIGH: + defect_color = "red" + elif self.severity == self.SEVERITY_MEDIUM: + defect_color = "yellow" + + format_str = "{file}:{line}: [{severity}:{category}] {message} {id}" + return format_str.format( + severity=click.style(self.SEVERITY_LABELS[self.severity], fg=defect_color), + category=click.style(self.category.lower(), fg=defect_color), + file=click.style(self.file, bold=True), + message=self.message, + line=self.line, + id="%s" % "[%s]" % self.id if self.id else "", + ) + + def __or__(self, defect): + return self.severity | defect.severity + + @staticmethod + def severity_to_int(label): + for key, value in DefectItem.SEVERITY_LABELS.items(): + if label == value: + return key + raise Exception("Unknown severity label -> %s" % label) + + def to_json(self): + return { + "severity": self.SEVERITY_LABELS[self.severity], + "category": self.category, + "message": self.message, + "file": self.file, + "line": self.line, + "column": self.column, + "callstack": self.callstack, + "id": self.id, + "cwe": self.cwe, + } diff --git a/platformio/check/tools/__init__.py b/platformio/check/tools/__init__.py new file mode 100644 index 00000000..a6161fb2 --- /dev/null +++ b/platformio/check/tools/__init__.py @@ -0,0 +1,30 @@ +# Copyright (c) 2019-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from platformio import exception +from platformio.check.tools.clangtidy import ClangtidyCheckTool +from platformio.check.tools.cppcheck import CppcheckCheckTool + + +class CheckToolFactory(object): + @staticmethod + def new(tool, project_dir, config, envname, options): + cls = None + if tool == "cppcheck": + cls = CppcheckCheckTool + elif tool == "clangtidy": + cls = ClangtidyCheckTool + else: + raise exception.PlatformioException("Unknown check tool `%s`" % tool) + return cls(project_dir, config, envname, options) diff --git a/platformio/check/tools/base.py b/platformio/check/tools/base.py new file mode 100644 index 00000000..578b72c0 --- /dev/null +++ b/platformio/check/tools/base.py @@ -0,0 +1,143 @@ +# Copyright (c) 2019-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import click + +from platformio import fs, proc +from platformio.check.defect import DefectItem +from platformio.project.helpers import get_project_dir, load_project_ide_data + + +class CheckToolBase(object): # pylint: disable=too-many-instance-attributes + def __init__(self, project_dir, config, envname, options): + self.config = config + self.envname = envname + self.options = options + self.cpp_defines = [] + self.cpp_includes = [] + + self._defects = [] + self._on_defect_callback = None + self._bad_input = False + self._load_cpp_data(project_dir, envname) + + # detect all defects by default + if not self.options.get("severity"): + self.options["severity"] = [ + DefectItem.SEVERITY_LOW, + DefectItem.SEVERITY_MEDIUM, + DefectItem.SEVERITY_HIGH, + ] + # cast to severity by ids + self.options["severity"] = [ + s if isinstance(s, int) else DefectItem.severity_to_int(s) + for s in self.options["severity"] + ] + + def _load_cpp_data(self, project_dir, envname): + data = load_project_ide_data(project_dir, envname) + if not data: + return + self.cpp_includes = data.get("includes", []) + self.cpp_defines = data.get("defines", []) + self.cpp_defines.extend(self._get_toolchain_defines(data.get("cc_path"))) + + def get_flags(self, tool): + result = [] + flags = self.options.get("flags") or [] + for flag in flags: + if ":" not in flag: + result.extend([f for f in flag.split(" ") if f]) + elif flag.startswith("%s:" % tool): + result.extend([f for f in flag.split(":", 1)[1].split(" ") if f]) + + return result + + @staticmethod + def _get_toolchain_defines(cc_path): + defines = [] + result = proc.exec_command("echo | %s -dM -E -x c++ -" % cc_path, shell=True) + + for line in result["out"].split("\n"): + tokens = line.strip().split(" ", 2) + if not tokens or tokens[0] != "#define": + continue + if len(tokens) > 2: + defines.append("%s=%s" % (tokens[1], tokens[2])) + else: + defines.append(tokens[1]) + + return defines + + @staticmethod + def is_flag_set(flag, flags): + return any(flag in f for f in flags) + + def get_defects(self): + return self._defects + + def configure_command(self): + raise NotImplementedError + + def on_tool_output(self, line): + line = self.tool_output_filter(line) + if not line: + return + + defect = self.parse_defect(line) + + if not isinstance(defect, DefectItem): + if self.options.get("verbose"): + click.echo(line) + return + + if defect.severity not in self.options["severity"]: + return + + self._defects.append(defect) + if self._on_defect_callback: + self._on_defect_callback(defect) + + @staticmethod + def tool_output_filter(line): + return line + + @staticmethod + def parse_defect(raw_line): + return raw_line + + def clean_up(self): + pass + + def get_project_src_files(self): + file_extensions = ["h", "hpp", "c", "cc", "cpp", "ino"] + return fs.match_src_files( + get_project_dir(), self.options.get("filter"), file_extensions + ) + + def check(self, on_defect_callback=None): + self._on_defect_callback = on_defect_callback + cmd = self.configure_command() + if self.options.get("verbose"): + click.echo(" ".join(cmd)) + + proc.exec_command( + cmd, + stdout=proc.LineBufferedAsyncPipe(self.on_tool_output), + stderr=proc.LineBufferedAsyncPipe(self.on_tool_output), + ) + + self.clean_up() + + return self._bad_input diff --git a/platformio/check/tools/clangtidy.py b/platformio/check/tools/clangtidy.py new file mode 100644 index 00000000..407375c5 --- /dev/null +++ b/platformio/check/tools/clangtidy.py @@ -0,0 +1,67 @@ +# Copyright (c) 2019-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +from os.path import join + +from platformio.check.defect import DefectItem +from platformio.check.tools.base import CheckToolBase +from platformio.managers.core import get_core_package_dir + + +class ClangtidyCheckTool(CheckToolBase): + def tool_output_filter(self, line): + if not self.options.get("verbose") and "[clang-diagnostic-error]" in line: + return "" + + if "[CommonOptionsParser]" in line: + self._bad_input = True + return line + + if any(d in line for d in ("note: ", "error: ", "warning: ")): + return line + + return "" + + def parse_defect(self, raw_line): + match = re.match(r"^(.*):(\d+):(\d+):\s+([^:]+):\s(.+)\[([^]]+)\]$", raw_line) + if not match: + return raw_line + + file_, line, column, category, message, defect_id = match.groups() + + severity = DefectItem.SEVERITY_LOW + if category == "error": + severity = DefectItem.SEVERITY_HIGH + elif category == "warning": + severity = DefectItem.SEVERITY_MEDIUM + + return DefectItem(severity, category, message, file_, line, column, defect_id) + + def configure_command(self): + tool_path = join(get_core_package_dir("tool-clangtidy"), "clang-tidy") + + cmd = [tool_path, "--quiet"] + flags = self.get_flags("clangtidy") + if not self.is_flag_set("--checks", flags): + cmd.append("--checks=*") + + cmd.extend(flags) + cmd.extend(self.get_project_src_files()) + cmd.append("--") + + cmd.extend(["-D%s" % d for d in self.cpp_defines]) + cmd.extend(["-I%s" % inc for inc in self.cpp_includes]) + + return cmd diff --git a/platformio/check/tools/cppcheck.py b/platformio/check/tools/cppcheck.py new file mode 100644 index 00000000..f26e7824 --- /dev/null +++ b/platformio/check/tools/cppcheck.py @@ -0,0 +1,142 @@ +# Copyright (c) 2019-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from os import remove +from os.path import isfile, join +from tempfile import NamedTemporaryFile + +from platformio.check.defect import DefectItem +from platformio.check.tools.base import CheckToolBase +from platformio.managers.core import get_core_package_dir +from platformio.project.helpers import get_project_core_dir + + +class CppcheckCheckTool(CheckToolBase): + def __init__(self, *args, **kwargs): + self._tmp_files = [] + self.defect_fields = [ + "severity", + "message", + "file", + "line", + "column", + "callstack", + "cwe", + "id", + ] + super(CppcheckCheckTool, self).__init__(*args, **kwargs) + + def tool_output_filter(self, line): + if ( + not self.options.get("verbose") + and "--suppress=unmatchedSuppression:" in line + ): + return "" + + if any( + msg in line + for msg in ( + "No C or C++ source files found", + "unrecognized command line option", + ) + ): + self._bad_input = True + + return line + + def parse_defect(self, raw_line): + if "<&PIO&>" not in raw_line or any( + f not in raw_line for f in self.defect_fields + ): + return None + + args = dict() + for field in raw_line.split("<&PIO&>"): + field = field.strip().replace('"', "") + name, value = field.split("=", 1) + args[name] = value + + args["category"] = args["severity"] + if args["severity"] == "error": + args["severity"] = DefectItem.SEVERITY_HIGH + elif args["severity"] == "warning": + args["severity"] = DefectItem.SEVERITY_MEDIUM + else: + args["severity"] = DefectItem.SEVERITY_LOW + + return DefectItem(**args) + + def configure_command(self): + tool_path = join(get_core_package_dir("tool-cppcheck"), "cppcheck") + + cmd = [ + tool_path, + "--error-exitcode=1", + "--verbose" if self.options.get("verbose") else "--quiet", + ] + + cmd.append( + '--template="%s"' + % "<&PIO&>".join(["{0}={{{0}}}".format(f) for f in self.defect_fields]) + ) + + flags = self.get_flags("cppcheck") + if not self.is_flag_set("--platform", flags): + cmd.append("--platform=unspecified") + if not self.is_flag_set("--enable", flags): + enabled_checks = [ + "warning", + "style", + "performance", + "portability", + "unusedFunction", + ] + cmd.append("--enable=%s" % ",".join(enabled_checks)) + + cmd.extend(["-D%s" % d for d in self.cpp_defines]) + cmd.extend(flags) + + cmd.append("--file-list=%s" % self._generate_src_file()) + cmd.append("--includes-file=%s" % self._generate_inc_file()) + + core_dir = get_project_core_dir() + cmd.append("--suppress=*:%s*" % core_dir) + cmd.append("--suppress=unmatchedSuppression:%s*" % core_dir) + + return cmd + + def _create_tmp_file(self, data): + with NamedTemporaryFile("w", delete=False) as fp: + fp.write(data) + self._tmp_files.append(fp.name) + return fp.name + + def _generate_src_file(self): + return self._create_tmp_file("\n".join(self.get_project_src_files())) + + def _generate_inc_file(self): + return self._create_tmp_file("\n".join(self.cpp_includes)) + + def clean_up(self): + for f in self._tmp_files: + if isfile(f): + remove(f) + + # delete temporary dump files generated by addons + if not self.is_flag_set("--addon", self.get_flags("cppcheck")): + return + for f in self.get_project_src_files(): + dump_file = f + ".dump" + if isfile(dump_file): + remove(dump_file) diff --git a/platformio/commands/check.py b/platformio/commands/check.py new file mode 100644 index 00000000..1c46640f --- /dev/null +++ b/platformio/commands/check.py @@ -0,0 +1,292 @@ +# Copyright (c) 2019-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=too-many-arguments,too-many-locals,too-many-branches +# pylint: disable=redefined-builtin,too-many-statements + +import os +from collections import Counter +from os.path import basename, dirname, isfile, join +from time import time + +import click +from tabulate import tabulate + +from platformio import exception, fs, util +from platformio.check.defect import DefectItem +from platformio.check.tools import CheckToolFactory +from platformio.compat import dump_json_to_unicode +from platformio.project.config import ProjectConfig +from platformio.project.helpers import ( + find_project_dir_above, + get_project_dir, + get_project_include_dir, + get_project_src_dir, +) + + +@click.command("check", short_help="Run a static analysis tool on code") +@click.option("-e", "--environment", multiple=True) +@click.option( + "-d", + "--project-dir", + default=os.getcwd, + type=click.Path( + exists=True, file_okay=True, dir_okay=True, writable=True, resolve_path=True + ), +) +@click.option( + "-c", + "--project-conf", + type=click.Path( + exists=True, file_okay=True, dir_okay=False, readable=True, resolve_path=True + ), +) +@click.option("--filter", multiple=True, help="Pattern: + -") +@click.option("--flags", multiple=True) +@click.option( + "--severity", multiple=True, type=click.Choice(DefectItem.SEVERITY_LABELS.values()) +) +@click.option("-s", "--silent", is_flag=True) +@click.option("-v", "--verbose", is_flag=True) +@click.option("--json-output", is_flag=True) +def cli( + environment, + project_dir, + project_conf, + filter, + flags, + severity, + silent, + verbose, + json_output, +): + # find project directory on upper level + if isfile(project_dir): + project_dir = find_project_dir_above(project_dir) + + results = [] + with fs.cd(project_dir): + config = ProjectConfig.get_instance( + project_conf or join(project_dir, "platformio.ini") + ) + config.validate(environment) + + default_envs = config.default_envs() + for envname in config.envs(): + skipenv = any( + [ + environment and envname not in environment, + not environment and default_envs and envname not in default_envs, + ] + ) + + env_options = config.items(env=envname, as_dict=True) + env_dump = [] + for k, v in env_options.items(): + if k not in ("platform", "framework", "board"): + continue + env_dump.append( + "%s: %s" % (k, ", ".join(v) if isinstance(v, list) else v) + ) + + default_filter = [ + "+<%s/>" % basename(d) + for d in (get_project_src_dir(), get_project_include_dir()) + ] + + tool_options = dict( + verbose=verbose, + silent=silent, + filter=filter or env_options.get("check_filter", default_filter), + flags=flags or env_options.get("check_flags"), + severity=[DefectItem.SEVERITY_LABELS[DefectItem.SEVERITY_HIGH]] + if silent + else (severity or env_options.get("check_severity")), + ) + + for tool in env_options.get("check_tool", ["cppcheck"]): + if skipenv: + results.append({"env": envname, "tool": tool}) + continue + if not silent and not json_output: + print_processing_header(tool, envname, env_dump) + + ct = CheckToolFactory.new( + tool, project_dir, config, envname, tool_options + ) + + result = {"env": envname, "tool": tool, "duration": time()} + rc = ct.check( + on_defect_callback=None + if (json_output or verbose) + else lambda defect: click.echo(repr(defect)) + ) + + result["defects"] = ct.get_defects() + result["duration"] = time() - result["duration"] + result["succeeded"] = rc == 0 and not any( + d.severity == DefectItem.SEVERITY_HIGH for d in result["defects"] + ) + results.append(result) + + if verbose: + click.echo("\n".join(repr(d) for d in result["defects"])) + + if not json_output and not silent: + if not result["defects"]: + click.echo("No defects found") + print_processing_footer(result) + + if json_output: + click.echo(dump_json_to_unicode(results_to_json(results))) + elif not silent: + print_check_summary(results) + + command_failed = any(r.get("succeeded") is False for r in results) + if command_failed: + raise exception.ReturnErrorCode(1) + + +def results_to_json(raw): + results = [] + for item in raw: + item.update( + { + "ignored": item.get("succeeded") is None, + "succeeded": bool(item.get("succeeded")), + "defects": [d.to_json() for d in item.get("defects", [])], + } + ) + results.append(item) + + return results + + +def print_processing_header(tool, envname, envdump): + click.echo( + "Checking %s > %s (%s)" + % (click.style(envname, fg="cyan", bold=True), tool, "; ".join(envdump)) + ) + terminal_width, _ = click.get_terminal_size() + click.secho("-" * terminal_width, bold=True) + + +def print_processing_footer(result): + is_failed = not result.get("succeeded") + util.print_labeled_bar( + "[%s] Took %.2f seconds" + % ( + ( + click.style("FAILED", fg="red", bold=True) + if is_failed + else click.style("PASSED", fg="green", bold=True) + ), + result["duration"], + ), + is_error=is_failed, + ) + + +def print_defects_stats(results): + components = dict() + + def _append_defect(component, defect): + if not components.get(component): + components[component] = Counter() + components[component].update({DefectItem.SEVERITY_LABELS[defect.severity]: 1}) + + for result in results: + for defect in result.get("defects", []): + component = dirname(defect.file) or defect.file + _append_defect(component, defect) + + if component.startswith(get_project_dir()): + while os.sep in component: + component = dirname(component) + _append_defect(component, defect) + + if not components: + return + + severity_labels = list(DefectItem.SEVERITY_LABELS.values()) + severity_labels.reverse() + tabular_data = list() + for k, v in components.items(): + tool_defect = [v.get(s, 0) for s in severity_labels] + tabular_data.append([k] + tool_defect) + + total = ["Total"] + [sum(d) for d in list(zip(*tabular_data))[1:]] + tabular_data.sort() + tabular_data.append([]) # Empty line as delimeter + tabular_data.append(total) + + headers = ["Component"] + headers.extend([l.upper() for l in severity_labels]) + headers = [click.style(h, bold=True) for h in headers] + click.echo(tabulate(tabular_data, headers=headers, numalign="center")) + click.echo() + + +def print_check_summary(results): + click.echo() + + tabular_data = [] + succeeded_nums = 0 + failed_nums = 0 + duration = 0 + + print_defects_stats(results) + + for result in results: + duration += result.get("duration", 0) + if result.get("succeeded") is False: + failed_nums += 1 + status_str = click.style("FAILED", fg="red") + elif result.get("succeeded") is None: + status_str = "IGNORED" + else: + succeeded_nums += 1 + status_str = click.style("PASSED", fg="green") + + tabular_data.append( + ( + click.style(result["env"], fg="cyan"), + result["tool"], + status_str, + util.humanize_duration_time(result.get("duration")), + ) + ) + + click.echo( + tabulate( + tabular_data, + headers=[ + click.style(s, bold=True) + for s in ("Environment", "Tool", "Status", "Duration") + ], + ), + err=failed_nums, + ) + + util.print_labeled_bar( + "%s%d succeeded in %s" + % ( + "%d failed, " % failed_nums if failed_nums else "", + succeeded_nums, + util.humanize_duration_time(duration), + ), + is_error=failed_nums, + fg="red" if failed_nums else "green", + )