Minor improvements

platformio/builder/main.py

@@ -147,7 +147,7 @@ env.LoadPioPlatform()
 
 env.SConscriptChdir(0)
 env.SConsignFile(
-    join("$BUILD_DIR", ".sconsign.py%d%d" % (sys.version_info[0], sys.version_info[1]))
+    join("$BUILD_DIR", ".sconsign%d%d.db" % (sys.version_info[0], sys.version_info[1]))
 )
 
 for item in env.GetExtraScripts("pre"):

platformio/builder/tools/pioplatform.py

@@ -140,13 +140,10 @@ def PrintConfiguration(env):  # pylint: disable=too-many-statements
 
     def _get_plaform_data():
         data = ["PLATFORM: %s %s" % (platform.title, platform.version)]
-        src_manifest_path = platform.pm.get_src_manifest_path(platform.get_dir())
+        if platform.src_version:
-        if src_manifest_path:
+            data.append("#" + platform.src_version)
-            src_manifest = fs.load_json(src_manifest_path)
+        if int(ARGUMENTS.get("PIOVERBOSE", 0)) and platform.src_url:
-            if "version" in src_manifest:
+            data.append("(%s)" % platform.src_url)
-                data.append("#" + src_manifest["version"])
-            if int(ARGUMENTS.get("PIOVERBOSE", 0)):
-                data.append("(%s)" % src_manifest["url"])
         if board_config:
             data.extend([">", board_config.get("name")])
         return data
@@ -196,20 +193,14 @@ def PrintConfiguration(env):  # pylint: disable=too-many-statements
 
     def _get_packages_data():
         data = []
-        for name, options in platform.packages.items():
+        for item in platform.dump_used_packages():
-            if options.get("optional"):
+            original_version = util.get_original_version(item["version"])
-                continue
+            info = "%s %s" % (item["name"], item["version"])
-            pkg_dir = platform.get_package_dir(name)
-            if not pkg_dir:
-                continue
-            manifest = platform.pm.load_manifest(pkg_dir)
-            original_version = util.get_original_version(manifest["version"])
-            info = "%s %s" % (manifest["name"], manifest["version"])
             extra = []
             if original_version:
                 extra.append(original_version)
-            if "__src_url" in manifest and int(ARGUMENTS.get("PIOVERBOSE", 0)):
+            if "src_url" in item and int(ARGUMENTS.get("PIOVERBOSE", 0)):
-                extra.append(manifest["__src_url"])
+                extra.append(item["src_url"])
             if extra:
                 info += " (%s)" % ", ".join(extra)
             data.append(info)

platformio/commands/debug/process/client.py

@@ -194,7 +194,7 @@ class GDBClient(BaseProcess):  # pylint: disable=too-many-instance-attributes
         # go to init break automatically
         if self.INIT_COMPLETED_BANNER.encode() in data:
             telemetry.send_event(
-                "Debug", "Started", telemetry.encode_run_environment(self.env_options)
+                "Debug", "Started", telemetry.dump_run_environment(self.env_options)
             )
             self._auto_continue_timer = task.LoopingCall(self._auto_exec_continue)
             self._auto_continue_timer.start(0.1)
@@ -231,8 +231,11 @@ class GDBClient(BaseProcess):  # pylint: disable=too-many-instance-attributes
         self._target_is_run = True
 
     def _handle_error(self, data):
-        self._errors_buffer += data
+        self._errors_buffer = (self._errors_buffer + data)[-8192:]  # keep last 8 KBytes
-        if self.PIO_SRC_NAME.encode() not in data or b"Error in sourced" not in data:
+        if not (
+            self.PIO_SRC_NAME.encode() in self._errors_buffer
+            and b"Error in sourced" in self._errors_buffer
+        ):
             return
 
         last_erros = self._errors_buffer.decode()
@@ -240,7 +243,7 @@ class GDBClient(BaseProcess):  # pylint: disable=too-many-instance-attributes
         last_erros = re.sub(r'((~|&)"|\\n\"|\\t)', " ", last_erros, flags=re.M)
 
         err = "%s -> %s" % (
-            telemetry.encode_run_environment(self.env_options),
+            telemetry.dump_run_environment(self.env_options),
             last_erros,
         )
         telemetry.send_exception("DebugInitError: %s" % err)

platformio/commands/run/processor.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from platformio import exception, telemetry
+from platformio import exception
 from platformio.commands.platform import platform_install as cmd_platform_install
 from platformio.commands.test.processor import CTX_META_TEST_RUNNING_NAME
 from platformio.managers.platform import PlatformFactory
@@ -62,8 +62,6 @@ class EnvironmentProcessor(object):
         build_vars = self.get_build_variables()
         build_targets = list(self.get_build_targets())
 
-        telemetry.send_run_environment(self.options, build_targets)
-
         # skip monitor target, we call it above
         if "monitor" in build_targets:
             build_targets.remove("monitor")

platformio/managers/platform.py

@@ -370,6 +370,21 @@ class PlatformPackagesMixin(object):
             return None
         return self.pm.load_manifest(pkg_dir).get("version")
 
+    def dump_used_packages(self):
+        result = []
+        for name, options in self.packages.items():
+            if options.get("optional"):
+                continue
+            pkg_dir = self.get_package_dir(name)
+            if not pkg_dir:
+                continue
+            manifest = self.pm.load_manifest(pkg_dir)
+            item = {"name": manifest["name"], "version": manifest["version"]}
+            if manifest.get("__src_url"):
+                item["src_url"] = manifest.get("__src_url")
+            result.append(item)
+        return result
+
 
 class PlatformRunMixin(object):
 
@@ -398,6 +413,8 @@ class PlatformRunMixin(object):
         self.configure_default_packages(options, targets)
         self.install_packages(silent=True)
 
+        self._report_non_sensitive_data(options, targets)
+
         self.silent = silent
         self.verbose = verbose or app.get_setting("force_verbose")
 
@@ -416,6 +433,17 @@ class PlatformRunMixin(object):
 
         return result
 
+    def _report_non_sensitive_data(self, options, targets):
+        topts = options.copy()
+        topts["platform_packages"] = [
+            dict(name=item["name"], version=item["version"])
+            for item in self.dump_used_packages()
+        ]
+        topts["platform"] = {"name": self.name, "version": self.version}
+        if self.src_version:
+            topts["platform"]["src_version"] = self.src_version
+        telemetry.send_run_environment(topts, targets)
+
     def _run_scons(self, variables, targets, jobs):
         args = [
             proc.get_pythonexe_path(),
@@ -531,14 +559,20 @@ class PlatformBase(PlatformPackagesMixin, PlatformRunMixin):
         self.silent = False
         self.verbose = False
 
-        self._BOARDS_CACHE = {}
         self._manifest = fs.load_json(manifest_path)
+        self._BOARDS_CACHE = {}
         self._custom_packages = None
 
         self.config = ProjectConfig.get_instance()
         self.pm = PackageManager(
             self.config.get_optional_dir("packages"), self.package_repositories
         )
+
+        self._src_manifest = None
+        src_manifest_path = self.pm.get_src_manifest_path(self.get_dir())
+        if src_manifest_path:
+            self._src_manifest = fs.load_json(src_manifest_path)
+
         # if self.engines and "platformio" in self.engines:
         #     if self.PIO_VERSION not in semantic_version.SimpleSpec(
         #             self.engines['platformio']):
@@ -561,6 +595,14 @@ class PlatformBase(PlatformPackagesMixin, PlatformRunMixin):
     def version(self):
         return self._manifest["version"]
 
+    @property
+    def src_version(self):
+        return self._src_manifest.get("version") if self._src_manifest else None
+
+    @property
+    def src_url(self):
+        return self._src_manifest.get("url") if self._src_manifest else None
+
     @property
     def homepage(self):
         return self._manifest.get("homepage")

platformio/telemetry.py

@@ -14,6 +14,7 @@
 
 import atexit
 import hashlib
+import json
 import os
 import re
 import sys
@@ -99,10 +100,12 @@ class MeasurementProtocol(TelemetryBase):
     def _prefill_sysargs(self):
         args = []
         for arg in sys.argv[1:]:
-            arg = str(arg).lower()
+            arg = str(arg)
-            if "@" in arg or os.path.exists(arg):
+            if arg == "account":  # ignore account cmd which can contain username
+                return
+            if any(("@" in arg, "/" in arg, "\\" in arg)):
                 arg = "***"
-            args.append(arg)
+            args.append(arg.lower())
         self["cd3"] = " ".join(args)
 
     def _prefill_custom_data(self):
@@ -257,6 +260,7 @@ class MPDataPusher(object):
     def _send_data(self, data):
         if self._http_offline:
             return False
+        print("MP:_send_data", data)
         try:
             r = self._http_session.post(
                 "https://ssl.google-analytics.com/collect",
@@ -320,8 +324,8 @@ def measure_ci():
     send_event(**event)
 
 
-def encode_run_environment(options):
+def dump_run_environment(options):
-    non_sensative_keys = [
+    non_sensitive_data = [
         "platform",
         "platform_packages",
         "framework",
@@ -331,20 +335,18 @@ def encode_run_environment(options):
         "debug_tool",
         "monitor_filters",
     ]
-    safe_options = [
+    safe_options = {k: v for k, v in options.items() if k in non_sensitive_data}
-        "%s=%s" % (k, v) for k, v in sorted(options.items()) if k in non_sensative_keys
-    ]
     if is_platformio_project(os.getcwd()):
         phash = hashlib.sha1(hashlib_encode_data(app.get_cid()))
-        safe_options.append("pid=%s" % phash.hexdigest())
+        safe_options["pid"] = phash.hexdigest()
-    return "&".join(safe_options)
+    return json.dumps(safe_options, sort_keys=True, ensure_ascii=False)
 
 
 def send_run_environment(options, targets):
     send_event(
         "Env",
         " ".join([t.title() for t in targets or ["run"]]),
-        encode_run_environment(options),
+        dump_run_environment(options),
     )