From 777a47fd99d6d83e21ebcb6cc4f1ecd2a758aa20 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 25 Apr 2020 13:14:54 +0300 Subject: [PATCH] Minor improvements --- platformio/builder/main.py | 2 +- platformio/builder/tools/pioplatform.py | 27 +++++-------- platformio/commands/debug/process/client.py | 11 ++++-- platformio/commands/run/processor.py | 4 +- platformio/managers/platform.py | 44 ++++++++++++++++++++- platformio/telemetry.py | 24 +++++------ 6 files changed, 74 insertions(+), 38 deletions(-) diff --git a/platformio/builder/main.py b/platformio/builder/main.py index 86adf234..c895f3f3 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -147,7 +147,7 @@ env.LoadPioPlatform() env.SConscriptChdir(0) env.SConsignFile( - join("$BUILD_DIR", ".sconsign.py%d%d" % (sys.version_info[0], sys.version_info[1])) + join("$BUILD_DIR", ".sconsign%d%d.db" % (sys.version_info[0], sys.version_info[1])) ) for item in env.GetExtraScripts("pre"): diff --git a/platformio/builder/tools/pioplatform.py b/platformio/builder/tools/pioplatform.py index be5e1fda..e280372b 100644 --- a/platformio/builder/tools/pioplatform.py +++ b/platformio/builder/tools/pioplatform.py @@ -140,13 +140,10 @@ def PrintConfiguration(env): # pylint: disable=too-many-statements def _get_plaform_data(): data = ["PLATFORM: %s %s" % (platform.title, platform.version)] - src_manifest_path = platform.pm.get_src_manifest_path(platform.get_dir()) - if src_manifest_path: - src_manifest = fs.load_json(src_manifest_path) - if "version" in src_manifest: - data.append("#" + src_manifest["version"]) - if int(ARGUMENTS.get("PIOVERBOSE", 0)): - data.append("(%s)" % src_manifest["url"]) + if platform.src_version: + data.append("#" + platform.src_version) + if int(ARGUMENTS.get("PIOVERBOSE", 0)) and platform.src_url: + data.append("(%s)" % platform.src_url) if board_config: data.extend([">", board_config.get("name")]) return data @@ -196,20 +193,14 @@ def PrintConfiguration(env): # pylint: disable=too-many-statements def _get_packages_data(): data = [] - for name, options in platform.packages.items(): - if options.get("optional"): - continue - pkg_dir = platform.get_package_dir(name) - if not pkg_dir: - continue - manifest = platform.pm.load_manifest(pkg_dir) - original_version = util.get_original_version(manifest["version"]) - info = "%s %s" % (manifest["name"], manifest["version"]) + for item in platform.dump_used_packages(): + original_version = util.get_original_version(item["version"]) + info = "%s %s" % (item["name"], item["version"]) extra = [] if original_version: extra.append(original_version) - if "__src_url" in manifest and int(ARGUMENTS.get("PIOVERBOSE", 0)): - extra.append(manifest["__src_url"]) + if "src_url" in item and int(ARGUMENTS.get("PIOVERBOSE", 0)): + extra.append(item["src_url"]) if extra: info += " (%s)" % ", ".join(extra) data.append(info) diff --git a/platformio/commands/debug/process/client.py b/platformio/commands/debug/process/client.py index 7ae5363e..a58438b7 100644 --- a/platformio/commands/debug/process/client.py +++ b/platformio/commands/debug/process/client.py @@ -194,7 +194,7 @@ class GDBClient(BaseProcess): # pylint: disable=too-many-instance-attributes # go to init break automatically if self.INIT_COMPLETED_BANNER.encode() in data: telemetry.send_event( - "Debug", "Started", telemetry.encode_run_environment(self.env_options) + "Debug", "Started", telemetry.dump_run_environment(self.env_options) ) self._auto_continue_timer = task.LoopingCall(self._auto_exec_continue) self._auto_continue_timer.start(0.1) @@ -231,8 +231,11 @@ class GDBClient(BaseProcess): # pylint: disable=too-many-instance-attributes self._target_is_run = True def _handle_error(self, data): - self._errors_buffer += data - if self.PIO_SRC_NAME.encode() not in data or b"Error in sourced" not in data: + self._errors_buffer = (self._errors_buffer + data)[-8192:] # keep last 8 KBytes + if not ( + self.PIO_SRC_NAME.encode() in self._errors_buffer + and b"Error in sourced" in self._errors_buffer + ): return last_erros = self._errors_buffer.decode() @@ -240,7 +243,7 @@ class GDBClient(BaseProcess): # pylint: disable=too-many-instance-attributes last_erros = re.sub(r'((~|&)"|\\n\"|\\t)', " ", last_erros, flags=re.M) err = "%s -> %s" % ( - telemetry.encode_run_environment(self.env_options), + telemetry.dump_run_environment(self.env_options), last_erros, ) telemetry.send_exception("DebugInitError: %s" % err) diff --git a/platformio/commands/run/processor.py b/platformio/commands/run/processor.py index 75b09b40..23ccc333 100644 --- a/platformio/commands/run/processor.py +++ b/platformio/commands/run/processor.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from platformio import exception, telemetry +from platformio import exception from platformio.commands.platform import platform_install as cmd_platform_install from platformio.commands.test.processor import CTX_META_TEST_RUNNING_NAME from platformio.managers.platform import PlatformFactory @@ -62,8 +62,6 @@ class EnvironmentProcessor(object): build_vars = self.get_build_variables() build_targets = list(self.get_build_targets()) - telemetry.send_run_environment(self.options, build_targets) - # skip monitor target, we call it above if "monitor" in build_targets: build_targets.remove("monitor") diff --git a/platformio/managers/platform.py b/platformio/managers/platform.py index 8e107c1f..c667a956 100644 --- a/platformio/managers/platform.py +++ b/platformio/managers/platform.py @@ -370,6 +370,21 @@ class PlatformPackagesMixin(object): return None return self.pm.load_manifest(pkg_dir).get("version") + def dump_used_packages(self): + result = [] + for name, options in self.packages.items(): + if options.get("optional"): + continue + pkg_dir = self.get_package_dir(name) + if not pkg_dir: + continue + manifest = self.pm.load_manifest(pkg_dir) + item = {"name": manifest["name"], "version": manifest["version"]} + if manifest.get("__src_url"): + item["src_url"] = manifest.get("__src_url") + result.append(item) + return result + class PlatformRunMixin(object): @@ -398,6 +413,8 @@ class PlatformRunMixin(object): self.configure_default_packages(options, targets) self.install_packages(silent=True) + self._report_non_sensitive_data(options, targets) + self.silent = silent self.verbose = verbose or app.get_setting("force_verbose") @@ -416,6 +433,17 @@ class PlatformRunMixin(object): return result + def _report_non_sensitive_data(self, options, targets): + topts = options.copy() + topts["platform_packages"] = [ + dict(name=item["name"], version=item["version"]) + for item in self.dump_used_packages() + ] + topts["platform"] = {"name": self.name, "version": self.version} + if self.src_version: + topts["platform"]["src_version"] = self.src_version + telemetry.send_run_environment(topts, targets) + def _run_scons(self, variables, targets, jobs): args = [ proc.get_pythonexe_path(), @@ -531,14 +559,20 @@ class PlatformBase(PlatformPackagesMixin, PlatformRunMixin): self.silent = False self.verbose = False - self._BOARDS_CACHE = {} self._manifest = fs.load_json(manifest_path) + self._BOARDS_CACHE = {} self._custom_packages = None self.config = ProjectConfig.get_instance() self.pm = PackageManager( self.config.get_optional_dir("packages"), self.package_repositories ) + + self._src_manifest = None + src_manifest_path = self.pm.get_src_manifest_path(self.get_dir()) + if src_manifest_path: + self._src_manifest = fs.load_json(src_manifest_path) + # if self.engines and "platformio" in self.engines: # if self.PIO_VERSION not in semantic_version.SimpleSpec( # self.engines['platformio']): @@ -561,6 +595,14 @@ class PlatformBase(PlatformPackagesMixin, PlatformRunMixin): def version(self): return self._manifest["version"] + @property + def src_version(self): + return self._src_manifest.get("version") if self._src_manifest else None + + @property + def src_url(self): + return self._src_manifest.get("url") if self._src_manifest else None + @property def homepage(self): return self._manifest.get("homepage") diff --git a/platformio/telemetry.py b/platformio/telemetry.py index 1e12584d..9b438694 100644 --- a/platformio/telemetry.py +++ b/platformio/telemetry.py @@ -14,6 +14,7 @@ import atexit import hashlib +import json import os import re import sys @@ -99,10 +100,12 @@ class MeasurementProtocol(TelemetryBase): def _prefill_sysargs(self): args = [] for arg in sys.argv[1:]: - arg = str(arg).lower() - if "@" in arg or os.path.exists(arg): + arg = str(arg) + if arg == "account": # ignore account cmd which can contain username + return + if any(("@" in arg, "/" in arg, "\\" in arg)): arg = "***" - args.append(arg) + args.append(arg.lower()) self["cd3"] = " ".join(args) def _prefill_custom_data(self): @@ -257,6 +260,7 @@ class MPDataPusher(object): def _send_data(self, data): if self._http_offline: return False + print("MP:_send_data", data) try: r = self._http_session.post( "https://ssl.google-analytics.com/collect", @@ -320,8 +324,8 @@ def measure_ci(): send_event(**event) -def encode_run_environment(options): - non_sensative_keys = [ +def dump_run_environment(options): + non_sensitive_data = [ "platform", "platform_packages", "framework", @@ -331,20 +335,18 @@ def encode_run_environment(options): "debug_tool", "monitor_filters", ] - safe_options = [ - "%s=%s" % (k, v) for k, v in sorted(options.items()) if k in non_sensative_keys - ] + safe_options = {k: v for k, v in options.items() if k in non_sensitive_data} if is_platformio_project(os.getcwd()): phash = hashlib.sha1(hashlib_encode_data(app.get_cid())) - safe_options.append("pid=%s" % phash.hexdigest()) - return "&".join(safe_options) + safe_options["pid"] = phash.hexdigest() + return json.dumps(safe_options, sort_keys=True, ensure_ascii=False) def send_run_environment(options, targets): send_event( "Env", " ".join([t.title() for t in targets or ["run"]]), - encode_run_environment(options), + dump_run_environment(options), )