This commit is contained in:
Ivan Kravets authored and GitHub committed 2019-09-08 18:04:41 +03:00
1 parent b7bc4401eb
commit f61d03ec8f
8 files changed
+951 -3

No files matched your search

+15
View File
@@ -0,0 +1,15 @@
# Copyright (c) 2019-present PlatformIO <contact@platformio.org>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from platformio.commands.check.command import cli
+247
View File
@@ -0,0 +1,247 @@
# Copyright (c) 2019-present PlatformIO <contact@platformio.org>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# pylint: disable=too-many-arguments,too-many-locals,too-many-branches
# pylint: disable=redefined-builtin,too-many-statements
import os
from collections import Counter
from os.path import basename, dirname, isfile, join
from time import time
import click
from tabulate import tabulate
from platformio import exception, fs, util
from platformio.commands.check.tools import CheckToolFactory, DefectItem
from platformio.compat import dump_json_to_unicode
from platformio.project.config import ProjectConfig
from platformio.project.helpers import (find_project_dir_above,
get_project_dir,
get_project_include_dir,
get_project_src_dir)
@click.command("check", short_help="Run a static analysis tool on code")
@click.option("-e", "--environment", multiple=True)
@click.option("-d",
"--project-dir",
default=os.getcwd,
type=click.Path(exists=True,
file_okay=True,
dir_okay=True,
writable=True,
resolve_path=True))
@click.option("-c",
"--project-conf",
type=click.Path(exists=True,
file_okay=True,
dir_okay=False,
readable=True,
resolve_path=True))
@click.option("--filter", multiple=True, help="Pattern: +<include> -<exclude>")
@click.option("--flags", multiple=True)
@click.option("--severity",
type=click.Choice(DefectItem.SEVERITY_LABELS.values()),
default=DefectItem.SEVERITY_LABELS[DefectItem.SEVERITY_LOW])
@click.option("-s", "--silent", is_flag=True)
@click.option("-v", "--verbose", is_flag=True)
@click.option("--json-output", is_flag=True)
def cli(environment, project_dir, project_conf, filter, flags, severity,
silent, verbose, json_output):
# find project directory on upper level
if isfile(project_dir):
project_dir = find_project_dir_above(project_dir)
results = []
with fs.cd(project_dir):
config = ProjectConfig.get_instance(
project_conf or join(project_dir, "platformio.ini"))
config.validate(environment)
default_envs = config.default_envs()
for envname in config.envs():
skipenv = any([
environment and envname not in environment, not environment
and default_envs and envname not in default_envs
])
env_options = config.items(env=envname, as_dict=True)
env_dump = []
for k, v in env_options.items():
if k not in ("platform", "framework", "board"):
continue
env_dump.append(
"%s: %s" % (k, ", ".join(v) if isinstance(v, list) else v))
default_filter = [
"+<%s/>" % basename(d)
for d in (get_project_src_dir(), get_project_include_dir())
]
tool_options = dict(
verbose=verbose,
silent=silent,
filter=filter
or env_options.get("check_filter", default_filter),
flags=flags or env_options.get("check_flags", ()),
severity=severity if not silent else "high")
for tool in env_options.get("check_tool", ["cppcheck"]):
if skipenv:
results.append({"env": envname, "tool": tool})
continue
if not silent and not json_output:
print_processing_header(tool, envname, env_dump)
ct = CheckToolFactory.new(tool, project_dir, config, envname,
tool_options)
result = {"env": envname, "tool": tool, "duration": time()}
rc = ct.check(on_defect_callback=None if (
json_output or verbose
) else lambda defect: click.echo(repr(defect)))
result['defects'] = ct.get_defects()
result['duration'] = time() - result['duration']
result['succeeded'] = (
rc == 0 and not any(d.severity == DefectItem.SEVERITY_HIGH
for d in result['defects']))
results.append(result)
if verbose:
click.echo("\n".join(repr(d) for d in result['defects']))
if not json_output and not silent:
print_processing_footer(result)
if json_output:
click.echo(dump_json_to_unicode(results_to_json(results)))
elif not silent:
print_check_summary(results)
command_failed = any(r.get("succeeded") is False for r in results)
if command_failed:
raise exception.ReturnErrorCode(1)
def results_to_json(raw):
results = []
for item in raw:
item.update({
"ignored": item.get("succeeded") is None,
"succeeded": bool(item.get("succeeded")),
"defects": [d.to_json() for d in item.get("defects", [])]
})
results.append(item)
return results
def print_processing_header(tool, envname, envdump):
click.echo(
"Checking %s > %s (%s)" %
(click.style(envname, fg="cyan", bold=True), tool, "; ".join(envdump)))
terminal_width, _ = click.get_terminal_size()
click.secho("-" * terminal_width, bold=True)
def print_processing_footer(result):
is_failed = not result.get("succeeded")
util.print_labeled_bar(
"[%s] Took %.2f seconds" %
((click.style("FAILED", fg="red", bold=True) if is_failed else
click.style("PASSED", fg="green", bold=True)), result['duration']),
is_error=is_failed)
def print_defects_stats(results):
components = dict()
def _append_defect(component, defect):
if not components.get(component):
components[component] = Counter()
components[component].update(
{DefectItem.SEVERITY_LABELS[defect.severity]: 1})
for result in results:
for defect in result.get("defects", []):
component = dirname(defect.file) or defect.file
_append_defect(component, defect)
if component.startswith(get_project_dir()):
while os.sep in component:
component = dirname(component)
_append_defect(component, defect)
if not components:
return
severity_labels = list(DefectItem.SEVERITY_LABELS.values())
severity_labels.reverse()
tabular_data = list()
for k, v in components.items():
tool_defect = [v.get(s, 0) for s in severity_labels]
tabular_data.append([k] + tool_defect)
total = ["Total"] + [sum(d) for d in list(zip(*tabular_data))[1:]]
tabular_data.sort()
tabular_data.append([]) # Empty line as delimeter
tabular_data.append(total)
headers = ["Component"]
headers.extend([l.upper() for l in severity_labels])
headers = [click.style(h, bold=True) for h in headers]
click.echo(tabulate(tabular_data, headers=headers, numalign="center"))
click.echo()
def print_check_summary(results):
click.echo()
tabular_data = []
succeeded_nums = 0
failed_nums = 0
duration = 0
print_defects_stats(results)
for result in results:
duration += result.get("duration", 0)
if result.get("succeeded") is False:
failed_nums += 1
status_str = click.style("FAILED", fg="red")
elif result.get("succeeded") is None:
status_str = "IGNORED"
else:
succeeded_nums += 1
status_str = click.style("PASSED", fg="green")
tabular_data.append(
(click.style(result['env'], fg="cyan"), result['tool'], status_str,
util.humanize_duration_time(result.get("duration"))))
click.echo(tabulate(tabular_data,
headers=[
click.style(s, bold=True)
for s in ("Environment", "Tool", "Status",
"Duration")
]),
err=failed_nums)
util.print_labeled_bar(
"%s%d succeeded in %s" %
("%d failed, " % failed_nums if failed_nums else "", succeeded_nums,
util.humanize_duration_time(duration)),
is_error=failed_nums,
fg="red" if failed_nums else "green")
+385
View File
@@ -0,0 +1,385 @@
# Copyright (c) 2019-present PlatformIO <contact@platformio.org>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# pylint: disable=too-many-arguments,too-many-instance-attributes
# pylint: disable=redefined-builtin
import re
import sys
from os import remove
from os.path import isfile, join, relpath
from tempfile import NamedTemporaryFile
import click
from platformio import exception, fs, proc
from platformio.managers.core import get_core_package_dir
from platformio.project.helpers import (get_project_core_dir, get_project_dir,
load_project_ide_data)
class DefectItem(object):
SEVERITY_HIGH = 1
SEVERITY_MEDIUM = 2
SEVERITY_LOW = 4
SEVERITY_LABELS = {4: "low", 2: "medium", 1: "high"}
def __init__(self,
severity,
category,
message,
file="unknown",
line=0,
column=0,
id=None,
callstack=None,
cwe=None):
assert severity in (self.SEVERITY_HIGH, self.SEVERITY_MEDIUM,
self.SEVERITY_LOW)
self.severity = severity
self.category = category
self.message = message
self.line = line
self.column = column
self.callstack = callstack
self.cwe = cwe
self.id = id
self.file = file
if file.startswith(get_project_dir()):
self.file = relpath(file, get_project_dir())
def __repr__(self):
defect_color = None
if self.severity == self.SEVERITY_HIGH:
defect_color = "red"
elif self.severity == self.SEVERITY_MEDIUM:
defect_color = "yellow"
format_str = "{file}:{line}: [{severity}:{category}] {message} {id}"
return format_str.format(severity=click.style(
self.SEVERITY_LABELS[self.severity], fg=defect_color),
category=click.style(self.category.lower(),
fg=defect_color),
file=click.style(self.file, bold=True),
message=self.message,
line=self.line,
id="%s" % "[%s]" % self.id if self.id else "")
def __or__(self, defect):
return self.severity | defect.severity
@staticmethod
def severity_to_int(label):
for key, value in DefectItem.SEVERITY_LABELS.items():
if label == value:
return key
raise Exception("Unknown severity label -> %s" % label)
def to_json(self):
return {
"severity": self.SEVERITY_LABELS[self.severity],
"category": self.category,
"message": self.message,
"file": self.file,
"line": self.line,
"column": self.column,
"callstack": self.callstack,
"id": self.id,
"cwe": self.cwe
}
class CheckToolFactory(object):
@staticmethod
def new(tool, project_dir, config, envname, options):
clsname = "%sCheckTool" % tool.title()
try:
obj = getattr(sys.modules[__name__], clsname)(project_dir, config,
envname, options)
except AttributeError:
raise exception.PlatformioException("Unknown check tool `%s`" %
tool)
assert isinstance(obj, CheckToolBase)
return obj
class CheckToolBase(object):
def __init__(self, project_dir, config, envname, options):
self.config = config
self.envname = envname
self.options = options
self.cpp_defines = []
self.cpp_includes = []
self._bad_input = False
self._load_cpp_data(project_dir, envname)
self._defects = []
self._on_defect_callback = None
def _load_cpp_data(self, project_dir, envname):
data = load_project_ide_data(project_dir, envname)
if not data:
return
self.cpp_includes = data.get("includes", [])
self.cpp_defines = data.get("defines", [])
self.cpp_defines.extend(
self._get_toolchain_defines(data.get("cc_path")))
def get_flags(self, tool):
result = []
flags = self.options.get("flags", [])
for flag in flags:
if ":" not in flag:
result.extend([f for f in flag.split(" ") if f])
elif flag.startswith("%s:" % tool):
result.extend(
[f for f in flag.split(":", 1)[1].split(" ") if f])
return result
@staticmethod
def _get_toolchain_defines(cc_path):
defines = []
result = proc.exec_command(
"echo | %s -dM -E -x c++ -" % cc_path, shell=True)
for line in result['out'].split("\n"):
tokens = line.strip().split(" ", 2)
if not tokens or tokens[0] != "#define":
continue
if len(tokens) > 2:
defines.append("%s=%s" % (tokens[1], tokens[2]))
else:
defines.append(tokens[1])
return defines
@staticmethod
def is_flag_set(flag, flags):
return any(flag in f for f in flags)
def get_defects(self):
return self._defects
def configure_command(self):
raise NotImplementedError
def on_tool_output(self, line):
line = self.tool_output_filter(line)
if not line:
return
defect = self.parse_defect(line)
if isinstance(defect, DefectItem):
self._defects.append(defect)
if self._on_defect_callback:
self._on_defect_callback(defect)
elif self.options.get("verbose"):
click.echo(line)
@staticmethod
def tool_output_filter(line):
return line
@staticmethod
def parse_defect(raw_line):
return raw_line
def clean_up(self):
pass
def exceeds_severity_threshold(self, severity):
return severity <= DefectItem.severity_to_int(
self.options.get("severity"))
def get_project_src_files(self):
file_extensions = ["h", "hpp", "c", "cc", "cpp", "ino"]
return fs.match_src_files(get_project_dir(),
self.options.get("filter"), file_extensions)
def check(self, on_defect_callback=None):
self._on_defect_callback = on_defect_callback
cmd = self.configure_command()
if self.options.get("verbose"):
click.echo(" ".join(cmd))
proc.exec_command(
cmd,
stdout=proc.LineBufferedAsyncPipe(self.on_tool_output),
stderr=proc.LineBufferedAsyncPipe(self.on_tool_output))
self.clean_up()
return self._bad_input
class CppcheckCheckTool(CheckToolBase):
def __init__(self, *args, **kwargs):
self._tmp_files = []
self.defect_fields = [
"severity", "message", "file", "line", "column", "callstack",
"cwe", "id"
]
super(CppcheckCheckTool, self).__init__(*args, **kwargs)
def tool_output_filter(self, line):
if not self.options.get(
"verbose") and "--suppress=unmatchedSuppression:" in line:
return ""
if any(msg in line for msg in ("No C or C++ source files found",
"unrecognized command line option")):
self._bad_input = True
return line
def parse_defect(self, raw_line):
if "<&PIO&>" not in raw_line or any(f not in raw_line
for f in self.defect_fields):
return None
args = dict()
for field in raw_line.split("<&PIO&>"):
field = field.strip().replace('"', "")
name, value = field.split("=", 1)
args[name] = value
args['category'] = args['severity']
if args['severity'] == "error":
args['severity'] = DefectItem.SEVERITY_HIGH
elif args['severity'] == "warning":
args['severity'] = DefectItem.SEVERITY_MEDIUM
else:
args['severity'] = DefectItem.SEVERITY_LOW
if self.exceeds_severity_threshold(args['severity']):
return DefectItem(**args)
return None
def configure_command(self):
tool_path = join(get_core_package_dir("tool-cppcheck"), "cppcheck")
cmd = [
tool_path, "--error-exitcode=1",
"--verbose" if self.options.get("verbose") else "--quiet"
]
cmd.append('--template="%s"' % "<&PIO&>".join(
["{0}={{{0}}}".format(f) for f in self.defect_fields]))
flags = self.get_flags("cppcheck")
if not self.is_flag_set("--platform", flags):
cmd.append("--platform=unspecified")
if not self.is_flag_set("--enable", flags):
enabled_checks = [
"warning", "style", "performance", "portability",
"unusedFunction"
]
cmd.append("--enable=%s" % ",".join(enabled_checks))
cmd.extend(["-D%s" % d for d in self.cpp_defines])
cmd.extend(flags)
cmd.append("--file-list=%s" % self._generate_src_file())
cmd.append("--includes-file=%s" % self._generate_inc_file())
core_dir = get_project_core_dir()
cmd.append("--suppress=*:%s*" % core_dir)
cmd.append("--suppress=unmatchedSuppression:%s*" % core_dir)
return cmd
def _create_tmp_file(self, data):
with NamedTemporaryFile("w", delete=False) as fp:
fp.write(data)
self._tmp_files.append(fp.name)
return fp.name
def _generate_src_file(self):
return self._create_tmp_file("\n".join(self.get_project_src_files()))
def _generate_inc_file(self):
return self._create_tmp_file("\n".join(self.cpp_includes))
def clean_up(self):
for f in self._tmp_files:
if isfile(f):
remove(f)
# delete temporary dump files generated by addons
if not self.is_flag_set("--addon", self.get_flags("cppcheck")):
return
for f in self.get_project_src_files():
dump_file = f + ".dump"
if isfile(dump_file):
remove(dump_file)
class ClangtidyCheckTool(CheckToolBase):
def tool_output_filter(self, line):
if not self.options.get(
"verbose") and "[clang-diagnostic-error]" in line:
return ""
if "[CommonOptionsParser]" in line:
self._bad_input = True
return line
if any(d in line for d in ("note: ", "error: ", "warning: ")):
return line
return ""
def parse_defect(self, raw_line):
match = re.match(r"^(.*):(\d+):(\d+):\s+([^:]+):\s(.+)\[([^]]+)\]$",
raw_line)
if not match:
return raw_line
file, line, column, category, message, defect_id = match.groups()
severity = DefectItem.SEVERITY_LOW
if category == "error":
severity = DefectItem.SEVERITY_HIGH
elif category == "warning":
severity = DefectItem.SEVERITY_MEDIUM
if self.exceeds_severity_threshold(severity):
return DefectItem(severity, category, message, file, line, column,
defect_id)
return None
def configure_command(self):
tool_path = join(get_core_package_dir("tool-clangtidy"), "clang-tidy")
cmd = [tool_path, "--quiet"]
flags = self.get_flags("clangtidy")
if not self.is_flag_set("--checks", flags):
cmd.append("--checks=*")
cmd.extend(flags)
cmd.extend(self.get_project_src_files())
cmd.append("--")
cmd.extend(["-D%s" % d for d in self.cpp_defines])
cmd.extend(["-I%s" % inc for inc in self.cpp_includes])
return cmd
+3 -1
View File
@@ -29,7 +29,9 @@ CORE_PACKAGES = {
"~2.%d%d.190418" % (sys.version_info[0], sys.version_info[1]),
"tool-pioplus": "^2.5.2",
"tool-unity": "~1.20403.0",
"tool-scons": "~2.20501.7" if PY2 else "~3.30101.0"
"tool-scons": "~2.20501.7" if PY2 else "~3.30101.0",
"tool-cppcheck": "~1.189.0",
"tool-clangtidy": "^1.80000.0"
}
PIOPLUS_AUTO_UPDATES_MAX = 100
+5
View File
@@ -195,6 +195,11 @@ ProjectOptions = OrderedDict([
type=click.Path(
exists=True, file_okay=True, dir_okay=False)),
# Check
ConfigEnvOption(name="check_tool", multiple=True),
ConfigEnvOption(name="check_filter", multiple=True),
ConfigEnvOption(name="check_flags", multiple=True),
# Other
ConfigEnvOption(name="extra_scripts",
oldnames=["extra_script"],