From 33d87367e7363b9e0b7fce9a8527da0a662f5496 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 29 Nov 2014 22:48:15 +0200 Subject: [PATCH] Add Telemetry Service --- HISTORY.rst | 50 +++++---- platformio/commands/run.py | 3 + platformio/libmanager.py | 11 ++ platformio/pkgmanager.py | 10 ++ platformio/telemetry.py | 202 +++++++++++++++++++++++++++++++++++++ 5 files changed, 257 insertions(+), 19 deletions(-) create mode 100644 platformio/telemetry.py diff --git a/HISTORY.rst b/HISTORY.rst index 1e018976..e3bcb7de 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -4,6 +4,9 @@ Release History 0.9.0 (?) --------- +* Added *Telemetry Service* which should help us make *PlatformIO* better +* Implemented *PlatformIO AppState Manager* which allow to have multiple + ``.platformio`` states. * Refactored *Package Manager* * Download Manager: fixed SHA1 verification within *Cygwin Environment* (`issue #26 `_) @@ -11,15 +14,19 @@ Release History 0.8.0 (2014-10-19) ------------------ -* Avoided trademark issues in ``library.json`` with new fields: - ``frameworks``, ``platforms`` and ``dependencies`` (`issue #17 `_) +* Avoided trademark issues in `library.json `_ + with the new fields: `frameworks `_, + `platforms `_ + and `dependencies `_ + (`issue #17 `_) * Switched logic from "Library Name" to "Library Registry ID" for all - ``platformio lib`` commands (install, uninstall, update and etc.) -* Renamed ``author`` field to ``authors`` and allowed to setup multiple authors - per library in ``library.json`` -* Added option to specify "maintainer" status in ``authors`` field -* New filters/options for ``platformio lib search`` command: ``--framework`` - and ``--platform`` + `platformio lib `_ + commands (install, uninstall, update and etc.) +* Renamed ``author`` field to `authors `_ + and allowed to setup multiple authors per library in `library.json `_ +* Added option to specify "maintainer" status in `authors `_ field +* New filters/options for `platformio lib search `_ + command: ``--framework`` and ``--platform`` 0.7.1 (2014-10-06) ------------------ @@ -32,14 +39,15 @@ Release History 0.7.0 (2014-09-24) ------------------ -* Implemented new ``[platformio]`` section for Configuration File with ``home_dir`` +* Implemented new `[platformio] `_ + section for Configuration File with `home_dir `_ option (`issue #14 `_) * Implemented *Library Manager* (`issue #6 `_) 0.6.0 (2014-08-09) ------------------ -* Implemented ``serialports monitor`` (`issue #10 `_) +* Implemented `platformio serialports monitor `_ (`issue #10 `_) * Fixed an issue ``ImportError: No module named platformio.util`` (`issue #9 `_) * Fixed bug with auto-conversation from Arduino \*.ino to \*.cpp @@ -52,7 +60,8 @@ Release History frameworks (`issue #7 `_) * Added `Arduino example `_ with external library (Adafruit CC3000) -* Implemented ``platformio upgrade`` command and "auto-check" for the latest +* Implemented `platformio upgrade `_ + command and "auto-check" for the latest version (`issue #8 `_) * Fixed an issue with "auto-reset" for Raspduino board * Fixed a bug with nested libs building @@ -60,18 +69,21 @@ Release History 0.4.0 (2014-07-31) ------------------ -* Implemented ``serialports`` command +* Implemented `platformio serialports `_ command * Allowed to put special build flags only for ``src`` files via - ``srcbuild_flags`` environment option + `srcbuild_flags `_ + environment option * Allowed to override some of settings via system environment variables such as: ``$PIOSRCBUILD_FLAGS`` and ``$PIOENVS_DIR`` -* Added ``--upload-port`` option for ``platformio run`` command +* Added ``--upload-port`` option for `platformio run `_ command * Implemented (especially for `SmartAnthill `_) - ``platformio run -t uploadlazy`` target (no dependencies to framework libs, - ELF and etc.) -* Allowed to skip default packages via ``platformio install --skip-default-package`` flag -* Added tools for Raspberry Pi platform -* Added support for Microduino and Raspduino boards in ``atmelavr`` platform + `platformio run -t uploadlazy `_ + target (no dependencies to framework libs, ELF and etc.) +* Allowed to skip default packages via `platformio install --skip-default-package `_ + option +* Added tools for *Raspberry Pi* platform +* Added support for *Microduino* and *Raspduino* boards in + `atmelavr `_ platform 0.3.1 (2014-06-21) diff --git a/platformio/commands/run.py b/platformio/commands/run.py index 4ed95bef..adfd3be3 100644 --- a/platformio/commands/run.py +++ b/platformio/commands/run.py @@ -3,6 +3,7 @@ from click import command, echo, option, secho, style +from platformio import telemetry from platformio.exception import (InvalidEnvName, ProjectEnvsNotAvaialable, UndefinedEnvPlatform, UnknownEnvNames) from platformio.platforms.base import PlatformFactory @@ -38,6 +39,8 @@ def cli(environment, target, upload_port): echo("Processing %s environment:" % style(envname, fg="cyan")) + telemetry.on_run_environment(envname, config.items(section)) + variables = ["PIOENV=" + envname] if upload_port: variables.append("UPLOAD_PORT=%s" % upload_port) diff --git a/platformio/libmanager.py b/platformio/libmanager.py index 72657bf7..f85520c3 100644 --- a/platformio/libmanager.py +++ b/platformio/libmanager.py @@ -8,6 +8,7 @@ from os.path import isdir, isfile, join from shutil import rmtree from tempfile import gettempdir +from platformio import telemetry from platformio.downloader import FileDownloader from platformio.exception import LibAlreadyInstalledError, LibNotInstalledError from platformio.unpacker import FileUnpacker @@ -76,11 +77,21 @@ class LibraryManager(object): info = self.get_info(id_) rename(tmplib_dir, join(self.lib_dir, "%s_ID%d" % ( re.sub(r"[^\da-z]+", "_", info['name'], flags=re.I), id_))) + + telemetry.on_event( + category="LibraryManager", action="Install", + label="#%d %s" % (id_, info['name']) + ) + return True def uninstall(self, id_): for libdir, item in self.get_installed().iteritems(): if "id" in item and item['id'] == id_: rmtree(join(self.lib_dir, libdir)) + telemetry.on_event( + category="LibraryManager", action="Uninstall", + label="#%d %s" % (id_, item['name']) + ) return True raise LibNotInstalledError(id_) diff --git a/platformio/pkgmanager.py b/platformio/pkgmanager.py index b5df5bdf..bc8be51e 100644 --- a/platformio/pkgmanager.py +++ b/platformio/pkgmanager.py @@ -99,6 +99,9 @@ class PackageManager(object): # remove archive remove(dlpath) + telemetry.on_event( + category="PackageManager", action="Install", label=name) + def uninstall(self, name): echo("Uninstalling %s package: \t" % style(name, fg="cyan"), nl=False) @@ -111,6 +114,10 @@ class PackageManager(object): self._unregister(name) echo("[%s]" % style("OK", fg="green")) + # report usage + telemetry.on_event( + category="PackageManager", action="Uninstall", label=name) + def update(self, name): echo("Updating %s package:" % style(name, fg="yellow")) @@ -130,6 +137,9 @@ class PackageManager(object): self.uninstall(name) self.install(name) + telemetry.on_event( + category="PackageManager", action="Update", label=name) + def _register(self, name, version): data = self.get_installed() data[name] = { diff --git a/platformio/telemetry.py b/platformio/telemetry.py new file mode 100644 index 00000000..78a6b328 --- /dev/null +++ b/platformio/telemetry.py @@ -0,0 +1,202 @@ +# Copyright (C) Ivan Kravets +# See LICENSE for details. + +import platform +import re +import uuid +from sys import argv as sys_argv +from time import time + +import click +import requests + +from platformio import __version__, app +from platformio.util import exec_command, get_systype + + +class TelemetryBase(object): + + MACHINE_ID = str(uuid.uuid5(uuid.NAMESPACE_OID, str(uuid.getnode()))) + + def __init__(self): + self._params = {} + + def __getitem__(self, name): + return self._params.get(name, None) + + def __setitem__(self, name, value): + self._params[name] = value + + def __delitem__(self, name): + if name in self._params: + del self._params[name] + + def send(self, hittype): + raise NotImplementedError() + + +class MeasurementProtocol(TelemetryBase): + + TRACKING_ID = "UA-1768265-9" + PARAMS_MAP = { + "screen_name": "cd", + "event_category": "ec", + "event_action": "ea", + "event_label": "el", + "event_value": "ev" + } + + def __init__(self): + TelemetryBase.__init__(self) + self['v'] = 1 + self['tid'] = self.TRACKING_ID + self['cid'] = self.MACHINE_ID + + self['sr'] = "%dx%d" % click.get_terminal_size() + self._prefill_screen_name() + self._prefill_appinfo() + self._prefill_custom_data() + + @classmethod + def session_instance(cls): + try: + return cls._session_instance + except AttributeError: + cls._session_instance = requests.Session() + return cls._session_instance + + def __getitem__(self, name): + if name in self.PARAMS_MAP: + name = self.PARAMS_MAP[name] + return TelemetryBase.__getitem__(self, name) + + def __setitem__(self, name, value): + if name in self.PARAMS_MAP: + name = self.PARAMS_MAP[name] + TelemetryBase.__setitem__(self, name, value) + + def _prefill_appinfo(self): + self['av'] = __version__ + + # gather dependent packages + dpdata = [] + dpdata.append("Click/%s" % click.__version__) + # dpdata.append("Requests/%s" % requests.__version__) + try: + result = exec_command(["scons", "--version"]) + match = re.search(r"engine: v([\d\.]+)", result['out']) + if match: + dpdata.append("SCons/%s" % match.group(1)) + except: # pylint: disable=W0702 + pass + self['an'] = " ".join(dpdata) + + def _prefill_custom_data(self): + self['cd1'] = get_systype() + self['cd2'] = "Python/%s %s" % (platform.python_version(), + platform.platform()) + + def _prefill_screen_name(self): + args = [str(s).lower() for s in sys_argv[1:]] + if not args: + return + + if args[0] in ("lib", "settings"): + cmd_path = args[:2] + else: + cmd_path = args[:1] + + self['screen_name'] = " ".join([p.title() for p in cmd_path]) + self['cd3'] = " ".join(args) + + def send(self, hittype): + self['t'] = hittype + + # correct queue time + if "qt" in self._params and isinstance(self['qt'], float): + self['qt'] = int((time() - self['qt']) * 1000) + + try: + r = self.session_instance().post( + "https://ssl.google-analytics.com/collect", + data=self._params + ) + r.raise_for_status() + except: # pylint: disable=W0702 + backup_report(self._params) + return False + return True + + +def on_command(ctx): # pylint: disable=W0613 + mp = MeasurementProtocol() + if mp.send("screenview"): + resend_backuped_reports() + + +def on_run_environment(name, options): # pylint: disable=W0613 + # on_event("RunEnv", "Name", name) + for opt, value in options: + on_event("RunEnv", opt.title(), value) + + +def on_event(category, action, label=None, value=None, screen_name=None): + mp = MeasurementProtocol() + mp['event_category'] = category[:150] + mp['event_action'] = action[:500] + if label: + mp['event_label'] = label[:500] + if value: + mp['event_value'] = int(value) + if screen_name: + mp['screen_name'] = screen_name[:2048] + return mp.send("event") + + +def on_exception(e): + mp = MeasurementProtocol() + mp['exd'] = "%s: %s" % (type(e).__name__, e) + mp['exf'] = 1 + return mp.send("exception") + + +def backup_report(params): + KEEP_MAX_REPORTS = 1000 + tm = app.get_state_item("telemetry", {}) + if "backup" not in tm: + tm['backup'] = [] + + # skip static options + for key in params.keys(): + if key in ("v", "tid", "cid", "cd1", "cd2", "sr", "an"): + del params[key] + + # store time in UNIX format + if "qt" not in params: + params['qt'] = time() + elif not isinstance(params['qt'], float): + params['qt'] = time() - (params['qt'] / 1000) + + tm['backup'].append(params) + tm['backup'] = tm['backup'][KEEP_MAX_REPORTS*-1:] + app.set_state_item("telemetry", tm) + + +def resend_backuped_reports(): + MAX_RESEND_REPORTS = 10 + + resent_nums = 0 + while resent_nums < MAX_RESEND_REPORTS: + tm = app.get_state_item("telemetry", {}) + if "backup" not in tm or not tm['backup']: + break + + report = tm['backup'].pop() + app.set_state_item("telemetry", tm) + resent_nums += 1 + + mp = MeasurementProtocol() + for key, value in report.items(): + mp[key] = value + if not mp.send(report['t']): + break