Merge branch 'release/v6.1.8'

.github/ISSUE_TEMPLATE.md

@@ -6,9 +6,8 @@ What kind of issue is this?
       use [Community Forums](https://community.platformio.org) or [Premium Support](https://platformio.org/support)
 
 - [ ] **PlatformIO IDE**.
-      All issues related to PlatformIO IDE should be reported to appropriate repository:
-      [PlatformIO IDE for Atom](https://github.com/platformio/platformio-atom-ide/issues) or
-      [PlatformIO IDE for VSCode](https://github.com/platformio/platformio-vscode-ide/issues)
+      All issues related to PlatformIO IDE should be reported to the
+      [PlatformIO IDE for VSCode](https://github.com/platformio/platformio-vscode-ide/issues) repository
 
 - [ ] **Development Platform or Board**.
       All issues (building, uploading, adding new boards, etc.) related to PlatformIO development platforms

.github/workflows/deployment.yml

@@ -35,7 +35,7 @@ jobs:
           tox -e testcore
 
       - name: Build Python source tarball
-        run:  python setup.py sdist
+        run:  python setup.py sdist bdist_wheel
 
       - name: Publish package to PyPI
         if: ${{ github.ref == 'refs/heads/master' }}

.github/workflows/projects.yml

@@ -13,11 +13,11 @@ jobs:
             folder: "Marlin"
             config_dir: "Marlin"
             env_name: "mega2560"
-          # - esphome:
-          #   repository: "esphome/esphome"
-          #   folder: "esphome"
-          #   config_dir: "esphome"
-          #   env_name: "esp32-arduino"
+          - esphome:
+            repository: "esphome/esphome"
+            folder: "esphome"
+            config_dir: "esphome"
+            env_name: "esp32-arduino"
           - smartknob:
             repository: "scottbez1/smartknob"
             folder: "smartknob"

CONTRIBUTING.md

@@ -6,12 +6,13 @@ To get started, <a href="https://cla-assistant.io/platformio/platformio-core">si
 1. Fork the repository on GitHub
 2. Clone repository `git clone --recursive https://github.com/YourGithubUsername/platformio-core.git`
 3. Run `pip install tox`
-4. Go to the root of project where is located `tox.ini` and run `tox -e py37`
+4. Go to the root of the PlatformIO Core project where `tox.ini` is located (``cd platformio-core``) and run `tox -e py39`.
+   You can replace `py39` with your own Python version. For example, `py311` means Python 3.11.
 5. Activate current development environment:
 
-   * Windows: `.tox\py37\Scripts\activate`
-   * Bash/ZSH: `source .tox/py37/bin/activate`
-   * Fish: `source .tox/py37/bin/activate.fish`
+   * Windows: `.tox\py39\Scripts\activate`
+   * Bash/ZSH: `source .tox/py39/bin/activate`
+   * Fish: `source .tox/py39/bin/activate.fish`
 
 6. Make changes to code, documentation, etc.
 7. Lint source code `make before-commit`

HISTORY.rst

@@ -15,6 +15,19 @@ PlatformIO Core 6
 
 **A professional collaborative platform for declarative, safety-critical, and test-driven embedded development.**
 
+6.1.8 (2023-07-05)
+~~~~~~~~~~~~~~~~~~
+
+* Added a new ``--lint`` option to the `pio project config <https://docs.platformio.org/en/latest/core/userguide/project/cmd_config.html>`__ command, enabling users to efficiently perform linting on the |PIOCONF|
+* Enhanced the parsing of the |PIOCONF| to provide comprehensive diagnostic information
+* Expanded the functionality of the |LIBRARYJSON| manifest by allowing the use of the underscore symbol in the `keywords <https://docs.platformio.org/en/latest/manifests/library-json/fields/keywords.html>`__ field
+* Optimized project integration templates to address the issue of long paths on Windows (`issue #4652 <https://github.com/platformio/platformio-core/issues/4652>`_)
+* Refactored |UNITTESTING| engine to resolve compiler warnings with "-Wpedantic" option (`pull #4671 <https://github.com/platformio/platformio-core/pull/4671>`_)
+* Eliminated erroneous warning regarding the use of obsolete PlatformIO Core when downgrading to the stable version (`issue #4664 <https://github.com/platformio/platformio-core/issues/4664>`_)
+* Updated the `pio project metadata <https://docs.platformio.org/en/latest/core/userguide/project/cmd_metadata.html>`__ command to return C/C++ flags as parsed Unix shell arguments when dumping project build metadata
+* Resolved a critical issue related to the usage of the ``-include`` flag within the `build_flags <https://docs.platformio.org/en/latest/projectconf/sections/env/options/build/build_flags.html>`__ option, specifically when employing dynamic variables (`issue #4682 <https://github.com/platformio/platformio-core/issues/4682>`_)
+* Removed PlatformIO IDE for Atom from the documentation as `Atom has been deprecated <https://github.blog/2022-06-08-sunsetting-atom/>`__
+
 6.1.7 (2023-05-08)
 ~~~~~~~~~~~~~~~~~~
 

platformio/__init__.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-VERSION = (6, 1, 7)
+VERSION = (6, 1, 8)
 __version__ = ".".join([str(s) for s in VERSION])
 
 __title__ = "platformio"

platformio/__main__.py

@@ -14,7 +14,7 @@
 
 import os
 import sys
-from traceback import format_exc
+import traceback
 
 import click
 
@@ -53,13 +53,13 @@ def cli(ctx, force, caller, no_ansi):  # pylint: disable=unused-argument
     except:  # pylint: disable=bare-except
         pass
 
-    maintenance.on_platformio_start(ctx, caller)
+    maintenance.on_cmd_start(ctx, caller)
 
 
 @cli.result_callback()
 @click.pass_context
-def process_result(ctx, result, *_, **__):
-    maintenance.on_platformio_end(ctx, result)
+def process_result(*_, **__):
+    maintenance.on_cmd_end()
 
 
 def configure():
@@ -96,6 +96,7 @@ def main(argv=None):
     if argv:
         assert isinstance(argv, list)
         sys.argv = argv
+
     try:
         ensure_python3(raise_exception=True)
         configure()
@@ -106,18 +107,18 @@ def main(argv=None):
     except Exception as exc:  # pylint: disable=broad-except
         if not isinstance(exc, exception.ReturnErrorCode):
             maintenance.on_platformio_exception(exc)
-            error_str = "Error: "
+            error_str = f"{exc.__class__.__name__}: "
             if isinstance(exc, exception.PlatformioException):
                 error_str += str(exc)
             else:
-                error_str += format_exc()
+                error_str += traceback.format_exc()
                 error_str += """
 ============================================================
 
 An unexpected error occurred. Further steps:
 
 * Verify that you have the latest version of PlatformIO using
-  `pip install -U platformio` command
+  `python -m pip install -U platformio` command
 
 * Try to find answer in FAQ Troubleshooting section
   https://docs.platformio.org/page/faq/index.html
@@ -129,6 +130,8 @@ An unexpected error occurred. Further steps:
 """
             click.secho(error_str, fg="red", err=True)
         exit_code = int(str(exc)) if str(exc).isdigit() else 1
+
+    maintenance.on_platformio_exit()
     sys.argv = prev_sys_argv
     return exit_code
 

platformio/account/client.py

@@ -16,7 +16,7 @@ import os
 import time
 
 from platformio import __accounts_api__, app
-from platformio.exception import PlatformioException
+from platformio.exception import PlatformioException, UserSideException
 from platformio.http import HTTPClient, HTTPClientError
 
 
@@ -24,11 +24,11 @@ class AccountError(PlatformioException):
     MESSAGE = "{0}"
 
 
-class AccountNotAuthorized(AccountError):
+class AccountNotAuthorized(AccountError, UserSideException):
     MESSAGE = "You are not authorized! Please log in to PlatformIO Account."
 
 
-class AccountAlreadyAuthorized(AccountError):
+class AccountAlreadyAuthorized(AccountError, UserSideException):
     MESSAGE = "You are already authorized with {0} account."
 
 

platformio/app.py

@@ -18,6 +18,7 @@ import json
 import os
 import platform
 import socket
+import time
 import uuid
 
 from platformio import __version__, exception, fs, proc
@@ -68,18 +69,23 @@ SESSION_VARS = {
     "command_ctx": None,
     "caller_id": None,
     "custom_project_conf": None,
+    "pause_telemetry": False,
 }
 
 
+def resolve_state_path(conf_option_dir, file_name, ensure_dir_exists=True):
+    state_dir = ProjectConfig.get_instance().get("platformio", conf_option_dir)
+    if ensure_dir_exists and not os.path.isdir(state_dir):
+        os.makedirs(state_dir)
+    return os.path.join(state_dir, file_name)
+
+
 class State:
     def __init__(self, path=None, lock=False):
         self.path = path
         self.lock = lock
         if not self.path:
-            core_dir = ProjectConfig.get_instance().get("platformio", "core_dir")
-            if not os.path.isdir(core_dir):
-                os.makedirs(core_dir)
-            self.path = os.path.join(core_dir, "appstate.json")
+            self.path = resolve_state_path("core_dir", "appstate.json")
         self._storage = {}
         self._lockfile = None
         self.modified = False
@@ -248,6 +254,7 @@ def get_cid():
     cid = str(cid)
     if IS_WINDOWS or os.getuid() > 0:  # pylint: disable=no-member
         set_state_item("cid", cid)
+        set_state_item("created_at", int(time.time()))
     return cid
 
 

platformio/assets/templates/ide-projects/atom/.clang_complete.tpl

@@ -1,6 +0,0 @@
-% for include in filter_includes(includes):
--I{{include}}
-% end
-% for define in defines:
--D{{!define}}
-% end

platformio/assets/templates/ide-projects/atom/.gcc-flags.json.tpl

@@ -1,9 +0,0 @@
-% _defines = " ".join(["-D%s" % d.replace(" ", "\\\\ ") for d in defines])
-{
-  "execPath": "{{ cxx_path }}",
-  "gccDefaultCFlags": "-fsyntax-only {{! to_unix_path(cc_flags).replace(' -MMD ', ' ').replace('"', '\\"') }} {{ !_defines.replace('"', '\\"') }}",
-  "gccDefaultCppFlags": "-fsyntax-only {{! to_unix_path(cxx_flags).replace(' -MMD ', ' ').replace('"', '\\"') }} {{ !_defines.replace('"', '\\"') }}",
-  "gccErrorLimit": 15,
-  "gccIncludePaths": "{{ ','.join(filter_includes(includes)) }}",
-  "gccSuppressWarnings": false
-}

platformio/assets/templates/ide-projects/atom/.gitignore.tpl

@@ -1,3 +0,0 @@
-.pio
-.clang_complete
-.gcc-flags.json

platformio/assets/templates/ide-projects/qtcreator/platformio.cflags.tpl

@@ -1 +0,0 @@
-{{cc_flags.replace('-mlongcalls', '-mlong-calls')}}

platformio/assets/templates/ide-projects/qtcreator/platformio.cxxflags.tpl

@@ -1 +0,0 @@
-{{cxx_flags.replace('-mlongcalls', '-mlong-calls')}}

platformio/builder/main.py

@@ -72,7 +72,8 @@ DEFAULT_ENV_OPTIONS = dict(
     variables=clivars,
     # Propagating External Environment
     ENV=os.environ,
-    UNIX_TIME=int(time()),
+    TIMESTAMP=int(time()),
+    UNIX_TIME="$TIMESTAMP",  # deprecated
     BUILD_DIR=os.path.join("$PROJECT_BUILD_DIR", "$PIOENV"),
     BUILD_SRC_DIR=os.path.join("$BUILD_DIR", "src"),
     BUILD_TEST_DIR=os.path.join("$BUILD_DIR", "test"),

platformio/builder/tools/piobuild.py

@@ -200,13 +200,16 @@ def ParseFlagsExtended(env, flags):  # pylint: disable=too-many-branches
     # fix relative CPPPATH & LIBPATH
     for k in ("CPPPATH", "LIBPATH"):
         for i, p in enumerate(result.get(k, [])):
+            p = env.subst(p)
             if os.path.isdir(p):
                 result[k][i] = os.path.abspath(p)
 
     # fix relative path for "-include"
     for i, f in enumerate(result.get("CCFLAGS", [])):
         if isinstance(f, tuple) and f[0] == "-include":
-            result["CCFLAGS"][i] = (f[0], env.File(os.path.abspath(f[1].get_path())))
+            p = env.subst(f[1].get_path())
+            if os.path.exists(p):
+                result["CCFLAGS"][i] = (f[0], os.path.abspath(p))
 
     return result
 

platformio/builder/tools/piointegration.py

@@ -16,6 +16,7 @@
 import glob
 import os
 
+import click
 import SCons.Defaults  # pylint: disable=import-error
 import SCons.Subst  # pylint: disable=import-error
 from SCons.Script import COMMAND_LINE_TARGETS  # pylint: disable=import-error
@@ -154,8 +155,12 @@ def DumpIntegrationData(*args):
         ],
         "defines": dump_defines(projenv),
         "includes": projenv.DumpIntegrationIncludes(),
-        "cc_flags": _subst_cmd(projenv, "$CFLAGS $CCFLAGS $CPPFLAGS"),
-        "cxx_flags": _subst_cmd(projenv, "$CXXFLAGS $CCFLAGS $CPPFLAGS"),
+        "cc_flags": click.parser.split_arg_string(
+            _subst_cmd(projenv, "$CFLAGS $CCFLAGS $CPPFLAGS")
+        ),
+        "cxx_flags": click.parser.split_arg_string(
+            _subst_cmd(projenv, "$CXXFLAGS $CCFLAGS $CPPFLAGS")
+        ),
         "cc_path": where_is_program(
             globalenv.subst("$CC"), globalenv.subst("${ENV['PATH']}")
         ),

platformio/check/tools/base.py

@@ -60,8 +60,8 @@ class CheckToolBase:  # pylint: disable=too-many-instance-attributes
         data = load_build_metadata(self.project_dir, self.envname)
         if not data:
             return
-        self.cc_flags = click.parser.split_arg_string(data.get("cc_flags", ""))
-        self.cxx_flags = click.parser.split_arg_string(data.get("cxx_flags", ""))
+        self.cc_flags = data.get("cc_flags", [])
+        self.cxx_flags = data.get("cxx_flags", [])
         self.cpp_includes = self._dump_includes(data.get("includes", {}))
         self.cpp_defines = data.get("defines", [])
         self.cc_path = data.get("cc_path")

platformio/cli.py

@@ -63,6 +63,21 @@ class PlatformioCLI(click.MultiCommand):
             ]
         )
 
+    @classmethod
+    def reveal_cmd_path_args(cls, ctx):
+        result = []
+        group = ctx.command
+        args = cls.leftover_args[::]
+        while args:
+            cmd_name = args.pop(0)
+            next_group = group.get_command(ctx, cmd_name)
+            if next_group:
+                group = next_group
+                result.append(cmd_name)
+            if not hasattr(group, "get_command"):
+                break
+        return result
+
     def invoke(self, ctx):
         PlatformioCLI.leftover_args = ctx.args
         if hasattr(ctx, "protected_args"):

platformio/commands/upgrade.py

@@ -53,16 +53,15 @@ def cli(dev, verbose):
         subprocess.run(
             [python_exe, "-m", "pip", "install", "--upgrade", pkg_spec],
             check=True,
-            capture_output=not verbose,
+            stdout=subprocess.PIPE if not verbose else None,
         )
-        r = subprocess.run(
+        output = subprocess.run(
             [python_exe, "-m", "platformio", "--version"],
             check=True,
-            capture_output=True,
-            text=True,
-        )
-        assert "version" in r.stdout
-        actual_version = r.stdout.split("version", 1)[1].strip()
+            stdout=subprocess.PIPE,
+        ).stdout.decode()
+        assert "version" in output
+        actual_version = output.split("version", 1)[1].strip()
         click.secho(
             "PlatformIO has been successfully upgraded to %s" % actual_version,
             fg="green",

platformio/compat.py

@@ -17,6 +17,7 @@
 import importlib.util
 import inspect
 import locale
+import shlex
 import sys
 
 from platformio.exception import UserSideException
@@ -29,6 +30,20 @@ else:
     from asyncio import get_event_loop as aio_get_running_loop
 
 
+if sys.version_info >= (3, 8):
+    from shlex import join as shlex_join
+else:
+
+    def shlex_join(split_command):
+        return " ".join(shlex.quote(arg) for arg in split_command)
+
+
+if sys.version_info >= (3, 9):
+    from asyncio import to_thread as aio_to_thread
+else:
+    from starlette.concurrency import run_in_threadpool as aio_to_thread
+
+
 PY2 = sys.version_info[0] == 2  # DO NOT REMOVE IT. ESP8266/ESP32 depend on it
 IS_CYGWIN = sys.platform.startswith("cygwin")
 IS_WINDOWS = WINDOWS = sys.platform.startswith("win")

platformio/debug/exception.py

@@ -30,3 +30,7 @@ class DebugSupportError(DebugError, UserSideException):
 
 class DebugInvalidOptionsError(DebugError, UserSideException):
     pass
+
+
+class DebugInitError(DebugError, UserSideException):
+    pass

platformio/debug/process/gdb.py

@@ -13,13 +13,13 @@
 # limitations under the License.
 
 import os
-import re
 import signal
 import time
 
 from platformio import telemetry
 from platformio.compat import aio_get_running_loop, is_bytes
 from platformio.debug import helpers
+from platformio.debug.exception import DebugInitError
 from platformio.debug.process.client import DebugClientProcess
 
 
@@ -130,11 +130,7 @@ class GDBClientProcess(DebugClientProcess):
         self._handle_error(data)
         # go to init break automatically
         if self.INIT_COMPLETED_BANNER.encode() in data:
-            telemetry.send_event(
-                "Debug",
-                "Started",
-                telemetry.dump_run_environment(self.debug_config.env_options),
-            )
+            telemetry.log_debug_started(self.debug_config)
             self._auto_exec_continue()
 
     def console_log(self, msg):
@@ -179,14 +175,7 @@ class GDBClientProcess(DebugClientProcess):
             and b"Error in sourced" in self._errors_buffer
         ):
             return
-
-        last_erros = self._errors_buffer.decode()
-        last_erros = " ".join(reversed(last_erros.split("\n")))
-        last_erros = re.sub(r'((~|&)"|\\n\"|\\t)', " ", last_erros, flags=re.M)
-
-        err = "%s -> %s" % (
-            telemetry.dump_run_environment(self.debug_config.env_options),
-            last_erros,
+        telemetry.log_debug_exception(
+            DebugInitError(self._errors_buffer.decode()), self.debug_config
         )
-        telemetry.send_exception("DebugInitError: %s" % err)
         self.transport.close()

platformio/device/finder.py

@@ -163,7 +163,7 @@ class SerialPortFinder:
         for item in list_serial_ports(as_objects=True):
             if item.vid == device.vid and item.pid == device.pid:
                 candidates.append(item)
-        if len(candidates) == 1:
+        if len(candidates) <= 1:
             return device.device
         for item in candidates:
             if ("GDB" if self.prefer_gdb_port else "UART") in item.description:

platformio/exception.py

@@ -81,7 +81,7 @@ class InvalidSettingValue(UserSideException):
     MESSAGE = "Invalid value '{0}' for the setting '{1}'"
 
 
-class InvalidJSONFile(PlatformioException):
+class InvalidJSONFile(ValueError, UserSideException):
     MESSAGE = "Could not load broken JSON: {0}"
 
 

platformio/home/rpc/handlers/app.py

@@ -12,12 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import os
 from pathlib import Path
 
 from platformio import __version__, app, fs, util
 from platformio.home.rpc.handlers.base import BaseRPCHandler
-from platformio.project.config import ProjectConfig
 from platformio.project.helpers import is_platformio_project
 
 
@@ -32,16 +30,11 @@ class AppRPC(BaseRPCHandler):
         "projectsDir",
     ]
 
-    @staticmethod
-    def get_state_path():
-        core_dir = ProjectConfig.get_instance().get("platformio", "core_dir")
-        if not os.path.isdir(core_dir):
-            os.makedirs(core_dir)
-        return os.path.join(core_dir, "homestate.json")
-
     @staticmethod
     def load_state():
-        with app.State(AppRPC.get_state_path(), lock=True) as state:
+        with app.State(
+            app.resolve_state_path("core_dir", "homestate.json"), lock=True
+        ) as state:
             storage = state.get("storage", {})
 
             # base data
@@ -81,7 +74,9 @@ class AppRPC(BaseRPCHandler):
 
     @staticmethod
     def save_state(state):
-        with app.State(AppRPC.get_state_path(), lock=True) as s:
+        with app.State(
+            app.resolve_state_path("core_dir", "homestate.json"), lock=True
+        ) as s:
             s.clear()
             s.update(state)
             storage = s.get("storage", {})

platformio/home/rpc/handlers/os.py

@@ -19,10 +19,10 @@ import shutil
 from functools import cmp_to_key
 
 import click
-from starlette.concurrency import run_in_threadpool
 
 from platformio import fs
 from platformio.cache import ContentCache
+from platformio.compat import aio_to_thread
 from platformio.device.list.util import list_logical_devices
 from platformio.home.rpc.handlers.base import BaseRPCHandler
 from platformio.http import HTTPSession, ensure_internet_on
@@ -33,12 +33,14 @@ class HTTPAsyncSession(HTTPSession):
         self, *args, **kwargs
     ):
         func = super().request
-        return await run_in_threadpool(func, *args, **kwargs)
+        return await aio_to_thread(func, *args, **kwargs)
 
 
 class OSRPC(BaseRPCHandler):
-    @staticmethod
-    async def fetch_content(url, data=None, headers=None, cache_valid=None):
+    _http_session = None
+
+    @classmethod
+    async def fetch_content(cls, url, data=None, headers=None, cache_valid=None):
         if not headers:
             headers = {
                 "User-Agent": (
@@ -57,11 +59,13 @@ class OSRPC(BaseRPCHandler):
         # check internet before and resolve issue with 60 seconds timeout
         ensure_internet_on(raise_exception=True)
 
-        session = HTTPAsyncSession()
+        if not cls._http_session:
+            cls._http_session = HTTPAsyncSession()
+
         if data:
-            r = await session.post(url, data=data, headers=headers)
+            r = await cls._http_session.post(url, data=data, headers=headers)
         else:
-            r = await session.get(url, headers=headers)
+            r = await cls._http_session.get(url, headers=headers)
 
         r.raise_for_status()
         result = r.text
@@ -73,9 +77,9 @@ class OSRPC(BaseRPCHandler):
     async def request_content(self, uri, data=None, headers=None, cache_valid=None):
         if uri.startswith("http"):
             return await self.fetch_content(uri, data, headers, cache_valid)
-        if os.path.isfile(uri):
-            with io.open(uri, encoding="utf-8") as fp:
-                return fp.read()
+        local_path = uri[7:] if uri.startswith("file://") else uri
+        with io.open(local_path, encoding="utf-8") as fp:
+            return fp.read()
         return None
 
     @staticmethod

platformio/home/rpc/handlers/piocore.py

@@ -22,13 +22,13 @@ import threading
 
 import click
 from ajsonrpc.core import JSONRPC20DispatchException
-from starlette.concurrency import run_in_threadpool
 
 from platformio import __main__, __version__, app, fs, proc, util
 from platformio.compat import (
     IS_WINDOWS,
     aio_create_task,
     aio_get_running_loop,
+    aio_to_thread,
     get_locale_encoding,
     is_bytes,
 )
@@ -177,7 +177,7 @@ class PIOCoreRPC(BaseRPCHandler):
 
     @staticmethod
     async def _call_subprocess(args, options):
-        result = await run_in_threadpool(
+        result = await aio_to_thread(
             proc.exec_command,
             [get_core_fullpath()] + args,
             cwd=options.get("cwd") or os.getcwd(),
@@ -197,7 +197,7 @@ class PIOCoreRPC(BaseRPCHandler):
                 exit_code,
             )
 
-        return await run_in_threadpool(
+        return await aio_to_thread(
             _thread_safe_call, args=args, cwd=options.get("cwd") or os.getcwd()
         )
 

platformio/home/rpc/handlers/platform.py

@@ -12,21 +12,55 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import os.path
+
+from platformio.compat import aio_to_thread
 from platformio.home.rpc.handlers.base import BaseRPCHandler
 from platformio.package.manager.platform import PlatformPackageManager
+from platformio.package.manifest.parser import ManifestParserFactory
+from platformio.package.meta import PackageSpec
 from platformio.platform.factory import PlatformFactory
 
 
 class PlatformRPC(BaseRPCHandler):
+    async def fetch_platforms(self, search_query=None, page=0, force_installed=False):
+        if force_installed:
+            return {
+                "items": await aio_to_thread(
+                    self._load_installed_platforms, search_query
+                )
+            }
+
+        search_result = await self.factory.manager.dispatcher["registry.call_client"](
+            method="list_packages",
+            query=search_query,
+            qualifiers={
+                "types": ["platform"],
+            },
+            page=page,
+        )
+        return {
+            "page": search_result["page"],
+            "limit": search_result["limit"],
+            "total": search_result["total"],
+            "items": [
+                {
+                    "id": item["id"],
+                    "ownername": item["owner"]["username"],
+                    "name": item["name"],
+                    "version": item["version"]["name"],
+                    "description": item["description"],
+                    "tier": item["tier"],
+                }
+                for item in search_result["items"]
+            ],
+        }
+
     @staticmethod
-    def list_installed(options=None):
-        result = []
-        options = options or {}
+    def _load_installed_platforms(search_query=None):
+        search_query = (search_query or "").strip()
 
         def _matchSearchQuery(p):
-            searchQuery = options.get("searchQuery")
-            if not searchQuery:
-                return True
             content_blocks = [p.name, p.title, p.description]
             if p.frameworks:
                 content_blocks.append(" ".join(p.frameworks.keys()))
@@ -34,28 +68,73 @@ class PlatformRPC(BaseRPCHandler):
                 board_data = board.get_brief_data()
                 for key in ("id", "mcu", "vendor"):
                     content_blocks.append(board_data.get(key))
-            return searchQuery.strip() in " ".join(content_blocks)
+            return search_query in " ".join(content_blocks)
 
+        items = []
         pm = PlatformPackageManager()
         for pkg in pm.get_installed():
             p = PlatformFactory.new(pkg)
-            if not _matchSearchQuery(p):
+            if search_query and not _matchSearchQuery(p):
                 continue
-            result.append(
-                dict(
-                    __pkg_path=pkg.path,
-                    __pkg_meta=pkg.metadata.as_dict(),
-                    name=p.name,
-                    title=p.title,
-                    description=p.description,
-                )
+            items.append(
+                {
+                    "__pkg_path": pkg.path,
+                    "ownername": pkg.metadata.spec.owner if pkg.metadata.spec else None,
+                    "name": p.name,
+                    "version": str(pkg.metadata.version),
+                    "title": p.title,
+                    "description": p.description,
+                }
             )
-        return result
+        return items
+
+    async def fetch_boards(self, platform_spec):
+        spec = PackageSpec(platform_spec)
+        if spec.owner:
+            return await self.factory.manager.dispatcher["registry.call_client"](
+                method="get_package",
+                typex="platform",
+                owner=spec.owner,
+                name=spec.name,
+                extra_path="/boards",
+            )
+        return await aio_to_thread(self._load_installed_boards, spec)
 
     @staticmethod
-    def get_boards(spec):
-        p = PlatformFactory.new(spec)
+    def _load_installed_boards(platform_spec):
+        p = PlatformFactory.new(platform_spec)
         return sorted(
             [b.get_brief_data() for b in p.get_boards().values()],
             key=lambda item: item["name"],
         )
+
+    async def fetch_examples(self, platform_spec):
+        spec = PackageSpec(platform_spec)
+        if spec.owner:
+            return await self.factory.manager.dispatcher["registry.call_client"](
+                method="get_package",
+                typex="platform",
+                owner=spec.owner,
+                name=spec.name,
+                extra_path="/examples",
+            )
+        return await aio_to_thread(self._load_installed_examples, spec)
+
+    @staticmethod
+    def _load_installed_examples(platform_spec):
+        platform = PlatformFactory.new(platform_spec)
+        platform_dir = platform.get_dir()
+        parser = ManifestParserFactory.new_from_dir(platform_dir)
+        result = parser.as_dict().get("examples") or []
+        for example in result:
+            example["files"] = [
+                {
+                    "path": item,
+                    "url": (
+                        "file://%s"
+                        + os.path.join(platform_dir, "examples", example["name"], item)
+                    ),
+                }
+                for item in example["files"]
+            ]
+        return result

platformio/home/rpc/handlers/project.py

@@ -15,6 +15,7 @@
 import os
 import shutil
 import time
+from pathlib import Path
 
 import semantic_version
 from ajsonrpc.core import JSONRPC20DispatchException
@@ -267,15 +268,39 @@ class ProjectRPC(BaseRPCHandler):
         )
         return new_project_dir
 
-    async def create_empty(self, configuration, options=None):
+    async def init_v2(self, configuration, options=None):
         project_dir = os.path.join(configuration["location"], configuration["name"])
         if not os.path.isdir(project_dir):
             os.makedirs(project_dir)
 
+        envclone = os.environ.copy()
+        envclone["PLATFORMIO_FORCE_ANSI"] = "true"
+        options = options or {}
+        options["spawn"] = {"env": envclone, "cwd": project_dir}
+
+        args = ["project", "init"]
+        ide = app.get_session_var("caller_id")
+        if ide in ProjectGenerator.get_supported_ides():
+            args.extend(["--ide", ide])
+
+        if configuration.get("example"):
+            await self.factory.notify_clients(
+                method=options.get("stdoutNotificationMethod"),
+                params=["Copying example files...\n"],
+                actor="frontend",
+            )
+            await self._pre_init_example(configuration, project_dir)
+        else:
+            args.extend(self._pre_init_empty(configuration))
+
+        return await self.factory.manager.dispatcher["core.exec"](args, options=options)
+
+    @staticmethod
+    def _pre_init_empty(configuration):
         project_options = []
         platform = configuration["platform"]
-        board = configuration.get("board", {}).get("id")
-        env_name = board or platform["name"]
+        board_id = configuration.get("board", {}).get("id")
+        env_name = board_id or platform["name"]
         if configuration.get("description"):
             project_options.append(("description", configuration.get("description")))
         try:
@@ -288,20 +313,25 @@ class ProjectRPC(BaseRPCHandler):
             project_options.append(
                 ("platform", "{name} @ {version}".format(**platform))
             )
-        if board:
-            project_options.append(("board", board))
+        if board_id:
+            project_options.append(("board", board_id))
         if configuration.get("framework"):
             project_options.append(("framework", configuration["framework"]["name"]))
 
-        args = ["project", "init", "-e", env_name, "--sample-code"]
-        ide = app.get_session_var("caller_id")
-        if ide in ProjectGenerator.get_supported_ides():
-            args.extend(["--ide", ide])
+        args = ["-e", env_name, "--sample-code"]
         for name, value in project_options:
             args.extend(["-O", f"{name}={value}"])
+        return args
 
-        envclone = os.environ.copy()
-        envclone["PLATFORMIO_FORCE_ANSI"] = "true"
-        options = options or {}
-        options["spawn"] = {"env": envclone, "cwd": project_dir}
-        return await self.factory.manager.dispatcher["core.exec"](args, options=options)
+    async def _pre_init_example(self, configuration, project_dir):
+        for item in configuration["example"]["files"]:
+            p = Path(project_dir).joinpath(item["path"])
+            if not p.parent.is_dir():
+                p.parent.mkdir(parents=True)
+            p.write_text(
+                await self.factory.manager.dispatcher["os.request_content"](
+                    item["url"]
+                ),
+                encoding="utf-8",
+            )
+        return []

platformio/home/rpc/handlers/registry.py

@@ -13,8 +13,8 @@
 # limitations under the License.
 
 from ajsonrpc.core import JSONRPC20DispatchException
-from starlette.concurrency import run_in_threadpool
 
+from platformio.compat import aio_to_thread
 from platformio.home.rpc.handlers.base import BaseRPCHandler
 from platformio.registry.client import RegistryClient
 
@@ -24,7 +24,7 @@ class RegistryRPC(BaseRPCHandler):
     async def call_client(method, *args, **kwargs):
         try:
             client = RegistryClient()
-            return await run_in_threadpool(getattr(client, method), *args, **kwargs)
+            return await aio_to_thread(getattr(client, method), *args, **kwargs)
         except Exception as exc:  # pylint: disable=bare-except
             raise JSONRPC20DispatchException(
                 code=5000, message="Registry Call Error", data=str(exc)

platformio/home/rpc/server.py

@@ -14,6 +14,7 @@
 
 from urllib.parse import parse_qs
 
+import ajsonrpc.utils
 import click
 from ajsonrpc.core import JSONRPC20Error, JSONRPC20Request
 from ajsonrpc.dispatcher import Dispatcher
@@ -24,6 +25,10 @@ from platformio.compat import aio_create_task, aio_get_running_loop
 from platformio.http import InternetConnectionError
 from platformio.proc import force_exit
 
+# Remove this line when PR is merged
+# https://github.com/pavlov99/ajsonrpc/pull/22
+ajsonrpc.utils.is_invalid_params = lambda: False
+
 
 class JSONRPCServerFactoryBase:
     connection_nums = 0

platformio/http.py

@@ -18,7 +18,7 @@ import socket
 from urllib.parse import urljoin
 
 import requests.adapters
-from requests.packages.urllib3.util.retry import Retry  # pylint:disable=import-error
+from urllib3.util.retry import Retry
 
 from platformio import __check_internet_hosts__, app, util
 from platformio.cache import ContentCache, cleanup_content_cache
@@ -27,7 +27,7 @@ from platformio.exception import PlatformioException, UserSideException
 __default_requests_timeout__ = (10, None)  # (connect, read)
 
 
-class HTTPClientError(PlatformioException):
+class HTTPClientError(UserSideException):
     def __init__(self, message, response=None):
         super().__init__()
         self.message = message
@@ -50,7 +50,10 @@ class HTTPSession(requests.Session):
         self._x_base_url = kwargs.pop("x_base_url") if "x_base_url" in kwargs else None
         super().__init__(*args, **kwargs)
         self.headers.update({"User-Agent": app.get_user_agent()})
-        self.verify = app.get_setting("enable_proxy_strict_ssl")
+        try:
+            self.verify = app.get_setting("enable_proxy_strict_ssl")
+        except PlatformioException:
+            self.verify = True
 
     def request(  # pylint: disable=signature-differs,arguments-differ
         self, method, url, *args, **kwargs
@@ -154,7 +157,10 @@ class HTTPClient:
         with ContentCache("http") as cc:
             result = cc.get(cache_key)
             if result is not None:
-                return json.loads(result)
+                try:
+                    return json.loads(result)
+                except json.JSONDecodeError:
+                    pass
             response = self.send_request(method, path, **kwargs)
             data = self._parse_json_response(response)
             cc.set(cache_key, response.text, cache_valid)

platformio/maintenance.py

@@ -25,23 +25,20 @@ from platformio.cli import PlatformioCLI
 from platformio.commands.upgrade import get_latest_version
 from platformio.http import HTTPClientError, InternetConnectionError, ensure_internet_on
 from platformio.package.manager.core import update_core_packages
-from platformio.package.manager.tool import ToolPackageManager
-from platformio.package.meta import PackageSpec
 from platformio.package.version import pepver_to_semver
 from platformio.system.prune import calculate_unnecessary_system_data
 
 
-def on_platformio_start(ctx, caller):
+def on_cmd_start(ctx, caller):
     app.set_session_var("command_ctx", ctx)
     set_caller(caller)
-    telemetry.on_command()
-
+    telemetry.on_cmd_start(ctx)
     if PlatformioCLI.in_silence():
         return
     after_upgrade(ctx)
 
 
-def on_platformio_end(ctx, result):  # pylint: disable=unused-argument
+def on_cmd_end():
     if PlatformioCLI.in_silence():
         return
 
@@ -60,8 +57,12 @@ def on_platformio_end(ctx, result):  # pylint: disable=unused-argument
         )
 
 
-def on_platformio_exception(e):
-    telemetry.on_exception(e)
+def on_platformio_exception(exc):
+    telemetry.log_exception(exc)
+
+
+def on_platformio_exit():
+    telemetry.on_exit()
 
 
 def set_caller(caller=None):
@@ -79,11 +80,10 @@ def set_caller(caller=None):
 
 class Upgrader:
     def __init__(self, from_version, to_version):
-        self.from_version = pepver_to_semver(from_version)
-        self.to_version = pepver_to_semver(to_version)
-
+        self.from_version = from_version
+        self.to_version = to_version
         self._upgraders = [
-            (semantic_version.Version("4.4.0-a.8"), self._update_pkg_metadata),
+            (semantic_version.Version("6.1.8-a.1"), self._appstate_migration),
         ]
 
     def run(self, ctx):
@@ -99,37 +99,43 @@ class Upgrader:
         return all(result)
 
     @staticmethod
-    def _update_pkg_metadata(_):
-        pm = ToolPackageManager()
-        for pkg in pm.get_installed():
-            if not pkg.metadata or pkg.metadata.spec.external or pkg.metadata.spec.id:
-                continue
-            result = pm.search_registry_packages(PackageSpec(name=pkg.metadata.name))
-            if len(result) != 1:
-                continue
-            result = result[0]
-            pkg.metadata.spec = PackageSpec(
-                id=result["id"],
-                owner=result["owner"]["username"],
-                name=result["name"],
+    def _appstate_migration(_):
+        state_path = app.resolve_state_path("core_dir", "appstate.json")
+        if not os.path.isfile(state_path):
+            return True
+        app.delete_state_item("telemetry")
+        created_at = app.get_state_item("created_at", None)
+        if not created_at:
+            state_stat = os.stat(state_path)
+            app.set_state_item(
+                "created_at",
+                int(
+                    state_stat.st_birthtime
+                    if hasattr(state_stat, "st_birthtime")
+                    else state_stat.st_ctime
+                ),
             )
-            pkg.dump_meta()
         return True
 
 
 def after_upgrade(ctx):
     terminal_width = shutil.get_terminal_size().columns
-    last_version = app.get_state_item("last_version", "0.0.0")
-    if last_version == __version__:
-        return
+    last_version_str = app.get_state_item("last_version", "0.0.0")
+    if last_version_str == __version__:
+        return None
 
-    if last_version == "0.0.0":
+    if last_version_str == "0.0.0":
         app.set_state_item("last_version", __version__)
-    elif pepver_to_semver(last_version) > pepver_to_semver(__version__):
+        return print_welcome_banner()
+
+    last_version = pepver_to_semver(last_version_str)
+    current_version = pepver_to_semver(__version__)
+
+    if last_version > current_version and not last_version.prerelease:
         click.secho("*" * terminal_width, fg="yellow")
         click.secho(
             "Obsolete PIO Core v%s is used (previous was %s)"
-            % (__version__, last_version),
+            % (__version__, last_version_str),
             fg="yellow",
         )
         click.secho("Please remove multiple PIO Cores from a system:", fg="yellow")
@@ -139,43 +145,50 @@ def after_upgrade(ctx):
             fg="cyan",
         )
         click.secho("*" * terminal_width, fg="yellow")
-        return
-    else:
-        click.secho("Please wait while upgrading PlatformIO...", fg="yellow")
+        return None
 
-        # Update PlatformIO's Core packages
-        cleanup_content_cache("http")
-        update_core_packages()
+    click.secho("Please wait while upgrading PlatformIO...", fg="yellow")
 
-        u = Upgrader(last_version, __version__)
-        if u.run(ctx):
-            app.set_state_item("last_version", __version__)
-            click.secho(
-                "PlatformIO has been successfully upgraded to %s!\n" % __version__,
-                fg="green",
-            )
-            telemetry.send_event(
-                category="Auto",
-                action="Upgrade",
-                label="%s > %s" % (last_version, __version__),
-            )
+    # Update PlatformIO's Core packages
+    cleanup_content_cache("http")
+    update_core_packages()
 
-    # PlatformIO banner
+    u = Upgrader(last_version, current_version)
+    if u.run(ctx):
+        app.set_state_item("last_version", __version__)
+        click.secho(
+            "PlatformIO has been successfully upgraded to %s!\n" % __version__,
+            fg="green",
+        )
+        telemetry.log_event(
+            "pio_upgrade_core",
+            {
+                "label": "%s > %s" % (last_version_str, __version__),
+                "from_version": last_version_str,
+                "to_version": __version__,
+            },
+        )
+
+    return print_welcome_banner()
+
+
+def print_welcome_banner():
+    terminal_width = shutil.get_terminal_size().columns
     click.echo("*" * terminal_width)
     click.echo("If you like %s, please:" % (click.style("PlatformIO", fg="cyan")))
-    click.echo(
-        "- %s us on Twitter to stay up-to-date "
-        "on the latest project news > %s"
-        % (
-            click.style("follow", fg="cyan"),
-            click.style("https://twitter.com/PlatformIO_Org", fg="cyan"),
-        )
-    )
     click.echo(
         "- %s it on GitHub > %s"
         % (
             click.style("star", fg="cyan"),
-            click.style("https://github.com/platformio/platformio", fg="cyan"),
+            click.style("https://github.com/platformio/platformio-core", fg="cyan"),
+        )
+    )
+    click.echo(
+        "- %s us on LinkedIn to stay up-to-date "
+        "on the latest project news > %s"
+        % (
+            click.style("follow", fg="cyan"),
+            click.style("https://www.linkedin.com/company/platformio/", fg="cyan"),
         )
     )
     if not os.getenv("PLATFORMIO_IDE"):
@@ -228,7 +241,7 @@ def check_platformio_upgrade():
     else:
         click.secho("platformio upgrade", fg="cyan", nl=False)
         click.secho("` or `", fg="yellow", nl=False)
-        click.secho("pip install -U platformio", fg="cyan", nl=False)
+        click.secho("python -m pip install -U platformio", fg="cyan", nl=False)
         click.secho("` command.", fg="yellow")
     click.secho("Changes: ", fg="yellow", nl=False)
     click.secho("https://docs.platformio.org/en/latest/history.html", fg="cyan")

platformio/package/commands/list.py

@@ -59,7 +59,8 @@ def humanize_package(pkg, spec=None, verbose=False):
     if spec and not isinstance(spec, PackageSpec):
         spec = PackageSpec(spec)
     data = [
-        click.style("{name} @ {version}".format(**pkg.metadata.as_dict()), fg="cyan")
+        click.style(pkg.metadata.name, fg="cyan"),
+        click.style(f"@ {str(pkg.metadata.version)}", bold=True),
     ]
     extra_data = ["required: %s" % (spec.humanize() if spec else "Any")]
     if verbose:
@@ -135,20 +136,20 @@ def list_global_packages(options):
         ("libraries", LibraryPackageManager(options.get("storage_dir"))),
     ]
     only_packages = any(
-        options.get(type_) or options.get(f"only_{type_}") for (type_, _) in data
+        options.get(typex) or options.get(f"only_{typex}") for (typex, _) in data
     )
-    for type_, pm in data:
+    for typex, pm in data:
         skip_conds = [
             only_packages
-            and not options.get(type_)
-            and not options.get(f"only_{type_}"),
+            and not options.get(typex)
+            and not options.get(f"only_{typex}"),
             not pm.get_installed(),
         ]
         if any(skip_conds):
             continue
-        click.secho(type_.capitalize(), bold=True)
+        click.secho(typex.capitalize(), bold=True)
         print_dependency_tree(
-            pm, filter_specs=options.get(type_), verbose=options.get("verbose")
+            pm, filter_specs=options.get(typex), verbose=options.get("verbose")
         )
         click.echo()
 
@@ -156,12 +157,12 @@ def list_global_packages(options):
 def list_project_packages(options):
     environments = options["environments"]
     only_packages = any(
-        options.get(type_) or options.get(f"only_{type_}")
-        for type_ in ("platforms", "tools", "libraries")
+        options.get(typex) or options.get(f"only_{typex}")
+        for typex in ("platforms", "tools", "libraries")
     )
     only_platform_packages = any(
-        options.get(type_) or options.get(f"only_{type_}")
-        for type_ in ("platforms", "tools")
+        options.get(typex) or options.get(f"only_{typex}")
+        for typex in ("platforms", "tools")
     )
     only_library_packages = options.get("libraries") or options.get("only_libraries")
 

platformio/package/commands/publish.py

@@ -56,7 +56,7 @@ def validate_datetime(ctx, param, value):  # pylint: disable=unused-argument
 )
 @click.option(
     "--type",
-    "type_",
+    "typex",
     type=click.Choice(list(PackageType.items().values())),
     help="Custom package type",
 )
@@ -83,7 +83,7 @@ def validate_datetime(ctx, param, value):  # pylint: disable=unused-argument
     hidden=True,
 )
 def package_publish_cmd(  # pylint: disable=too-many-arguments, too-many-locals
-    package, owner, type_, released_at, private, notify, no_interactive, non_interactive
+    package, owner, typex, released_at, private, notify, no_interactive, non_interactive
 ):
     click.secho("Preparing a package...", fg="cyan")
     no_interactive = no_interactive or non_interactive
@@ -103,14 +103,14 @@ def package_publish_cmd(  # pylint: disable=too-many-arguments, too-many-locals
                 p = PackagePacker(package)
                 archive_path = p.pack()
 
-        type_ = type_ or PackageType.from_archive(archive_path)
+        typex = typex or PackageType.from_archive(archive_path)
         manifest = ManifestSchema().load_manifest(
             ManifestParserFactory.new_from_archive(archive_path).as_dict()
         )
         name = manifest.get("name")
         version = manifest.get("version")
         data = [
-            ("Type:", type_),
+            ("Type:", typex),
             ("Owner:", owner),
             ("Name:", name),
             ("Version:", version),
@@ -124,13 +124,13 @@ def package_publish_cmd(  # pylint: disable=too-many-arguments, too-many-locals
         check_archive_file_names(archive_path)
 
         # look for duplicates
-        check_package_duplicates(owner, type_, name, version, manifest.get("system"))
+        check_package_duplicates(owner, typex, name, version, manifest.get("system"))
 
         if not no_interactive:
             click.confirm(
                 "Are you sure you want to publish the %s %s to the registry?\n"
                 % (
-                    type_,
+                    typex,
                     click.style(
                         "%s/%s@%s" % (owner, name, version),
                         fg="cyan",
@@ -146,7 +146,7 @@ def package_publish_cmd(  # pylint: disable=too-many-arguments, too-many-locals
         )
         click.echo("Publishing...")
         response = RegistryClient().publish_package(
-            owner, type_, archive_path, released_at, private, notify
+            owner, typex, archive_path, released_at, private, notify
         )
         if not do_not_pack:
             os.remove(archive_path)

platformio/package/exception.py

@@ -13,10 +13,10 @@
 # limitations under the License.
 
 from platformio import util
-from platformio.exception import PlatformioException, UserSideException
+from platformio.exception import UserSideException
 
 
-class PackageException(PlatformioException):
+class PackageException(UserSideException):
     pass
 
 
@@ -51,14 +51,14 @@ class MissingPackageManifestError(ManifestException):
     MESSAGE = "Could not find one of '{0}' manifest files in the package"
 
 
-class UnknownPackageError(UserSideException):
+class UnknownPackageError(PackageException):
     MESSAGE = (
         "Could not find the package with '{0}' requirements for your system '%s'"
         % util.get_systype()
     )
 
 
-class NotGlobalLibDir(UserSideException):
+class NotGlobalLibDir(PackageException):
     MESSAGE = (
         "The `{0}` is not a PlatformIO project.\n\n"
         "To manage libraries in global storage `{1}`,\n"

platformio/package/lockfile.py

@@ -15,7 +15,7 @@
 import os
 from time import sleep, time
 
-from platformio.exception import PlatformioException
+from platformio.exception import UserSideException
 
 LOCKFILE_TIMEOUT = 3600  # in seconds, 1 hour
 LOCKFILE_DELAY = 0.2
@@ -36,11 +36,11 @@ except ImportError:
         LOCKFILE_CURRENT_INTERFACE = None
 
 
-class LockFileExists(PlatformioException):
+class LockFileExists(UserSideException):
     pass
 
 
-class LockFileTimeoutError(PlatformioException):
+class LockFileTimeoutError(UserSideException):
     pass
 
 

platformio/package/manager/_registry.py

@@ -25,12 +25,13 @@ from platformio.registry.mirror import RegistryFileMirrorIterator
 
 class PackageManagerRegistryMixin:
     def install_from_registry(self, spec, search_qualifiers=None):
+        package = version = None
         if spec.owner and spec.name and not search_qualifiers:
             package = self.fetch_registry_package(spec)
             if not package:
                 raise UnknownPackageError(spec.humanize())
             version = self.pick_best_registry_version(package["versions"], spec)
-        else:
+        elif spec.id or spec.name:
             packages = self.search_registry_packages(spec, search_qualifiers)
             if not packages:
                 raise UnknownPackageError(spec.humanize())

platformio/package/manifest/schema.py

@@ -183,7 +183,7 @@ class ManifestSchema(BaseSchema):
             validate=[
                 validate.Length(min=1, max=50),
                 validate.Regexp(
-                    r"^[a-z\d\-\+\. ]+$", error="Only [a-z0-9-+. ] chars are allowed"
+                    r"^[a-z\d\-_\+\. ]+$", error="Only [a-z0-9+_-. ] chars are allowed"
                 ),
             ]
         )
@@ -276,9 +276,9 @@ class ManifestSchema(BaseSchema):
     @staticmethod
     @memoized(expire="1h")
     def load_spdx_licenses():
-        version = "3.20"
+        version = "3.21"
         spdx_data_url = (
             "https://raw.githubusercontent.com/spdx/license-list-data/"
-            "v%s/json/licenses.json" % version
+            f"v{version}/json/licenses.json"
         )
         return json.loads(fetch_remote_content(spdx_data_url))

platformio/package/meta.py

@@ -24,7 +24,7 @@ import semantic_version
 from platformio import fs
 from platformio.compat import get_object_members, hashlib_encode_data, string_types
 from platformio.package.manifest.parser import ManifestFileType
-from platformio.package.version import cast_version_to_semver
+from platformio.package.version import SemanticVersionError, cast_version_to_semver
 from platformio.util import items_in_list
 
 
@@ -175,7 +175,7 @@ class PackageSpec:  # pylint: disable=too-many-instance-attributes
         if requirements:
             try:
                 self.requirements = requirements
-            except ValueError as exc:
+            except SemanticVersionError as exc:
                 if not self.name or self.uri or self.raw:
                     raise exc
                 self.raw = "%s=%s" % (self.name, requirements)
@@ -224,11 +224,14 @@ class PackageSpec:  # pylint: disable=too-many-instance-attributes
         if not value:
             self._requirements = None
             return
-        self._requirements = (
-            value
-            if isinstance(value, semantic_version.SimpleSpec)
-            else semantic_version.SimpleSpec(str(value))
-        )
+        try:
+            self._requirements = (
+                value
+                if isinstance(value, semantic_version.SimpleSpec)
+                else semantic_version.SimpleSpec(str(value))
+            )
+        except ValueError as exc:
+            raise SemanticVersionError(exc) from exc
 
     def humanize(self):
         result = ""

platformio/package/vcsclient.py

@@ -18,14 +18,10 @@ import subprocess
 from urllib.parse import urlparse
 
 from platformio import proc
-from platformio.package.exception import (
-    PackageException,
-    PlatformioException,
-    UserSideException,
-)
+from platformio.exception import UserSideException
 
 
-class VCSBaseException(PackageException):
+class VCSBaseException(UserSideException):
     pass
 
 
@@ -74,8 +70,8 @@ class VCSClientBase:
                 self.get_cmd_output(["--version"])
             else:
                 assert self.run_cmd(["--version"])
-        except (AssertionError, OSError, PlatformioException) as exc:
-            raise UserSideException(
+        except (AssertionError, OSError) as exc:
+            raise VCSBaseException(
                 "VCS: `%s` client is not installed in your system" % self.command
             ) from exc
         return True

platformio/package/version.py

@@ -16,6 +16,12 @@ import re
 
 import semantic_version
 
+from platformio.exception import UserSideException
+
+
+class SemanticVersionError(UserSideException):
+    pass
+
 
 def cast_version_to_semver(value, force=True, raise_exception=False):
     assert value
@@ -29,7 +35,7 @@ def cast_version_to_semver(value, force=True, raise_exception=False):
         except ValueError:
             pass
     if raise_exception:
-        raise ValueError("Invalid SemVer version %s" % value)
+        raise SemanticVersionError("Invalid SemVer version %s" % value)
     # parse commit hash
     if re.match(r"^[\da-f]+$", value, flags=re.I):
         return semantic_version.Version("0.0.0+sha." + value)

platformio/platform/_run.py

@@ -52,7 +52,6 @@ class PlatformRunMixin:
 
         self.ensure_engine_compatible()
         self.configure_project_packages(variables["pioenv"], targets)
-        self._report_non_sensitive_data(variables["pioenv"], targets)
 
         self.silent = silent
         self.verbose = verbose or app.get_setting("force_verbose")
@@ -64,20 +63,13 @@ class PlatformRunMixin:
         if not os.path.isfile(variables["build_script"]):
             raise BuildScriptNotFound(variables["build_script"])
 
+        telemetry.log_platform_run(self, self.config, variables["pioenv"], targets)
         result = self._run_scons(variables, targets, jobs)
+
         assert "returncode" in result
 
         return result
 
-    def _report_non_sensitive_data(self, env, targets):
-        options = self.config.items(env=env, as_dict=True)
-        options["platform_packages"] = [
-            dict(name=item["name"], version=item["version"])
-            for item in self.dump_used_packages()
-        ]
-        options["platform"] = {"name": self.name, "version": self.version}
-        telemetry.send_run_environment(options, targets)
-
     def _run_scons(self, variables, targets, jobs):
         scons_dir = get_core_package_dir("tool-scons")
         args = [

platformio/platform/board.py

@@ -14,10 +14,10 @@
 
 import os
 
-from platformio import fs, telemetry, util
+from platformio import fs, util
 from platformio.compat import MISSING
 from platformio.debug.exception import DebugInvalidOptionsError, DebugSupportError
-from platformio.exception import UserSideException
+from platformio.exception import InvalidJSONFile, UserSideException
 from platformio.platform.exception import InvalidBoardManifest
 
 
@@ -28,7 +28,7 @@ class PlatformBoardConfig:
         self.manifest_path = manifest_path
         try:
             self._manifest = fs.load_json(manifest_path)
-        except ValueError as exc:
+        except InvalidJSONFile as exc:
             raise InvalidBoardManifest(manifest_path) from exc
         if not set(["name", "url", "vendor"]) <= set(self._manifest):
             raise UserSideException(
@@ -119,7 +119,6 @@ class PlatformBoardConfig:
         if tool_name == "custom":
             return tool_name
         if not debug_tools:
-            telemetry.send_event("Debug", "Request", self.id)
             raise DebugSupportError(self._manifest["name"])
         if tool_name:
             if tool_name in debug_tools:

platformio/platform/exception.py

@@ -12,10 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from platformio.exception import PlatformioException
+from platformio.exception import UserSideException
 
 
-class PlatformException(PlatformioException):
+class PlatformException(UserSideException):
     pass
 
 

platformio/project/commands/config.py

@@ -30,16 +30,23 @@ from platformio.project.helpers import is_platformio_project
     default=os.getcwd,
     type=click.Path(exists=True, file_okay=False, dir_okay=True),
 )
+@click.option("--lint", is_flag=True)
 @click.option("--json-output", is_flag=True)
-def project_config_cmd(project_dir, json_output):
+def project_config_cmd(project_dir, lint, json_output):
     if not is_platformio_project(project_dir):
         raise NotPlatformIOProjectError(project_dir)
     with fs.cd(project_dir):
-        config = ProjectConfig.get_instance()
+        if lint:
+            return lint_configuration(json_output)
+        return print_configuration(json_output)
+
+
+def print_configuration(json_output=False):
+    config = ProjectConfig.get_instance()
     if json_output:
         return click.echo(config.to_json())
     click.echo(
-        "Computed project configuration for %s" % click.style(project_dir, fg="cyan")
+        "Computed project configuration for %s" % click.style(os.getcwd(), fg="cyan")
     )
     for section, options in config.as_tuple():
         click.secho(section, fg="cyan")
@@ -55,3 +62,43 @@ def project_config_cmd(project_dir, json_output):
         )
         click.echo()
     return None
+
+
+def lint_configuration(json_output=False):
+    result = ProjectConfig.lint()
+    errors = result["errors"]
+    warnings = result["warnings"]
+    if json_output:
+        return click.echo(result)
+    if not errors and not warnings:
+        return click.secho(
+            'The "platformio.ini" configuration file is free from linting errors.',
+            fg="green",
+        )
+    if errors:
+        click.echo(
+            tabulate(
+                [
+                    (
+                        click.style(error["type"], fg="red"),
+                        error["message"],
+                        error.get("source", "") + (f":{error.get('lineno')}")
+                        if "lineno" in error
+                        else "",
+                    )
+                    for error in errors
+                ],
+                tablefmt="plain",
+            )
+        )
+    if warnings:
+        click.echo(
+            tabulate(
+                [
+                    (click.style("Warning", fg="yellow"), warning)
+                    for warning in warnings
+                ],
+                tablefmt="plain",
+            )
+        )
+    return None

platformio/project/commands/metadata.py

@@ -37,6 +37,7 @@ from platformio.project.helpers import load_build_metadata
 @click.option("--json-output", is_flag=True)
 @click.option("--json-output-path", type=click.Path())
 def project_metadata_cmd(project_dir, environments, json_output, json_output_path):
+    project_dir = os.path.abspath(project_dir)
     with fs.cd(project_dir):
         config = ProjectConfig.get_instance()
         config.validate(environments)

platformio/project/config.py

@@ -86,7 +86,7 @@ class ProjectConfigBase:
         if path and os.path.isfile(path):
             self.read(path, parse_extra)
 
-        self._maintain_renaimed_options()
+        self._maintain_renamed_options()
 
     def __getattr__(self, name):
         return getattr(self._parser, name)
@@ -98,7 +98,7 @@ class ProjectConfigBase:
         try:
             self._parser.read(path, "utf-8")
         except configparser.Error as exc:
-            raise exception.InvalidProjectConfError(path, str(exc))
+            raise exception.InvalidProjectConfError(path, str(exc)) from exc
 
         if not parse_extra:
             return
@@ -110,7 +110,7 @@ class ProjectConfigBase:
             for item in glob.glob(pattern, recursive=True):
                 self.read(item)
 
-    def _maintain_renaimed_options(self):
+    def _maintain_renamed_options(self):
         renamed_options = {}
         for option in ProjectOptions.values():
             if option.oldnames:
@@ -324,6 +324,7 @@ class ProjectConfigBase:
                         f"`${{this.__env__}}` is called from the `{parent_section}` "
                         "section that is not valid PlatformIO environment, see",
                         option,
+                        " ",
                         section,
                     )
                 return parent_section[4:]
@@ -332,7 +333,10 @@ class ProjectConfigBase:
             value = self.get(section, option)
         except RecursionError as exc:
             raise exception.ProjectOptionValueError(
-                "Infinite recursion has been detected", option, section
+                "Infinite recursion has been detected",
+                option,
+                " ",
+                section,
             ) from exc
         if isinstance(value, list):
             return "\n".join(value)
@@ -359,7 +363,10 @@ class ProjectConfigBase:
             if not self.expand_interpolations:
                 return value
             raise exception.ProjectOptionValueError(
-                exc.format_message(), option, section
+                exc.format_message(),
+                option,
+                " (%s) " % option_meta.description,
+                section,
             )
 
     @staticmethod
@@ -424,16 +431,51 @@ class ProjectConfigBase:
         return True
 
 
+class ProjectConfigLintMixin:
+    @classmethod
+    def lint(cls, path=None):
+        errors = []
+        warnings = []
+        try:
+            config = cls.get_instance(path)
+            config.validate(silent=True)
+            warnings = config.warnings
+            config.as_tuple()
+        except Exception as exc:  # pylint: disable=broad-exception-caught
+            if exc.__cause__ is not None:
+                exc = exc.__cause__
+
+            item = {"type": exc.__class__.__name__, "message": str(exc)}
+            for attr in ("lineno", "source"):
+                if hasattr(exc, attr):
+                    item[attr] = getattr(exc, attr)
+
+            if item["type"] == "ParsingError" and hasattr(exc, "errors"):
+                for lineno, line in getattr(exc, "errors"):
+                    errors.append(
+                        {
+                            "type": item["type"],
+                            "message": f"Parsing error: {line}",
+                            "lineno": lineno,
+                            "source": item["source"],
+                        }
+                    )
+            else:
+                errors.append(item)
+        return {"errors": errors, "warnings": warnings}
+
+
 class ProjectConfigDirsMixin:
     def get_optional_dir(self, name):
         """
         Deprecated, used by platformio-node-helpers.project.observer.fetchLibDirs
         PlatformIO IDE for Atom depends on platformio-node-helpers@~7.2.0
+        PIO Home 3.0 Project Inspection depends on it
         """
         return self.get("platformio", f"{name}_dir")
 
 
-class ProjectConfig(ProjectConfigBase, ProjectConfigDirsMixin):
+class ProjectConfig(ProjectConfigBase, ProjectConfigLintMixin, ProjectConfigDirsMixin):
     _instances = {}
 
     @staticmethod

platformio/project/exception.py

@@ -51,4 +51,4 @@ class InvalidEnvNameError(ProjectError, UserSideException):
 
 
 class ProjectOptionValueError(ProjectError, UserSideException):
-    MESSAGE = "{0} for option `{1}` in section [{2}]"
+    MESSAGE = "{0} for option `{1}`{2}in section [{3}]"

platformio/project/helpers.py

@@ -164,6 +164,7 @@ load_project_ide_data = load_build_metadata
 
 def _load_build_metadata(project_dir, env_names, debug=False):
     # pylint: disable=import-outside-toplevel
+    from platformio import app
     from platformio.run.cli import cli as cmd_run
 
     args = ["--project-dir", project_dir, "--target", "__idedata"]
@@ -171,13 +172,15 @@ def _load_build_metadata(project_dir, env_names, debug=False):
         args.extend(["--target", "__debug"])
     for name in env_names:
         args.extend(["-e", name])
+    app.set_session_var("pause_telemetry", True)
     result = CliRunner().invoke(cmd_run, args)
+    app.set_session_var("pause_telemetry", False)
     if result.exit_code != 0 and not isinstance(
         result.exception, exception.ReturnErrorCode
     ):
         raise result.exception
     if '"includes":' not in result.output:
-        raise exception.PlatformioException(result.output)
+        raise exception.UserSideException(result.output)
     return _get_cached_build_metadata(project_dir, env_names)
 
 

platformio/project/integration/generator.py

@@ -52,7 +52,7 @@ class ProjectGenerator:
 
     @staticmethod
     def get_ide_tpls_dir():
-        return os.path.join(fs.get_assets_dir(), "templates", "ide-projects")
+        return os.path.join(os.path.dirname(__file__), "tpls")
 
     @classmethod
     def get_supported_ides(cls):

platformio/project/integration/tpls/clion/CMakeListsPrivate.txt.tpl

@@ -8,6 +8,7 @@
 % import os
 % import re
 %
+% from platformio.compat import shlex_join
 % from platformio.project.helpers import load_build_metadata
 %
 % def _normalize_path(path):
@@ -64,17 +65,16 @@ set(CLION_SVD_FILE_PATH "{{ _normalize_path(svd_path) }}" CACHE FILEPATH "Periph
 
 SET(CMAKE_C_COMPILER "{{ _normalize_path(cc_path) }}")
 SET(CMAKE_CXX_COMPILER "{{ _normalize_path(cxx_path) }}")
-SET(CMAKE_CXX_FLAGS "{{ _normalize_path(to_unix_path(cxx_flags)) }}")
-SET(CMAKE_C_FLAGS "{{ _normalize_path(to_unix_path(cc_flags)) }}")
+SET(CMAKE_CXX_FLAGS {{ _normalize_path(to_unix_path(shlex_join(cxx_flags))) }})
+SET(CMAKE_C_FLAGS {{ _normalize_path(to_unix_path(shlex_join(cc_flags))) }})
 
-% STD_RE = re.compile(r"\-std=[a-z\+]+(\w+)")
-% cc_stds = STD_RE.findall(cc_flags)
-% cxx_stds = STD_RE.findall(cxx_flags)
+% cc_stds = [arg for arg in cc_flags if arg.startswith("-std=")]
+% cxx_stds = [arg for arg in cxx_flags if arg.startswith("-std=")]
 % if cc_stds:
-SET(CMAKE_C_STANDARD {{ cc_stds[-1] }})
+SET(CMAKE_C_STANDARD {{ cc_stds[-1][-2:] }})
 % end
 % if cxx_stds:
-set(CMAKE_CXX_STANDARD {{ cxx_stds[-1] }})
+set(CMAKE_CXX_STANDARD {{ cxx_stds[-1][-2:] }})
 % end
 
 if (CMAKE_BUILD_TYPE MATCHES "{{ env_name }}")

platformio/project/integration/tpls/eclipse/.settings/language.settings.xml.tpl

@@ -1,6 +1,4 @@
-% import re
-% STD_RE = re.compile(r"(\-std=[a-z\+]+\w+)")
-% cxx_stds = STD_RE.findall(cxx_flags)
+% cxx_stds = [arg for arg in cxx_flags if arg.startswith("-std=")]
 % cxx_std = cxx_stds[-1] if cxx_stds else ""
 %
 % if cxx_path.startswith(user_home_dir):

platformio/project/integration/tpls/emacs/.ccls.tpl

@@ -1,7 +1,9 @@
+% from platformio.compat import shlex_join
+%
 clang
 
-{{"%c"}} {{ !cc_flags }}
-{{"%cpp"}} {{ !cxx_flags }}
+{{"%c"}} {{ shlex_join(cc_flags) }}
+{{"%cpp"}} {{ shlex_join(cxx_flags) }}
 
 % for include in filter_includes(includes):
 -I{{ !include }}

platformio/project/integration/tpls/qtcreator/platformio.cflags.tpl

@@ -0,0 +1,3 @@
+% from platformio.compat import shlex_join
+%
+{{shlex_join(cc_flags).replace('-mlongcalls', '-mlong-calls')}}

platformio/project/integration/tpls/qtcreator/platformio.cxxflags.tpl

@@ -0,0 +1,3 @@
+% from platformio.compat import shlex_join
+%
+{{shlex_join(cxx_flags).replace('-mlongcalls', '-mlong-calls')}}

platformio/project/integration/tpls/sublimetext/.ccls.tpl

@@ -1,7 +1,9 @@
+% from platformio.compat import shlex_join
+%
 clang
 
-{{"%c"}} {{ !cc_flags }}
-{{"%cpp"}} {{ !cxx_flags }}
+{{"%c"}} {{ shlex_join(cc_flags) }}
+{{"%cpp"}} {{ shlex_join(cxx_flags) }}
 
 % for include in filter_includes(includes):
 -I{{ !include }}

platformio/project/integration/tpls/vim/.ccls.tpl

@@ -1,7 +1,9 @@
+% from platformio.compat import shlex_join
+%
 clang
 
-{{"%c"}} {{ !cc_flags }}
-{{"%cpp"}} {{ !cxx_flags }}
+{{"%c"}} {{ shlex_join(cc_flags) }}
+{{"%cpp"}} {{ shlex_join(cxx_flags) }}
 
 % for include in filter_includes(includes):
 -I{{ !include }}

platformio/project/integration/tpls/vscode/.vscode/c_cpp_properties.json.tpl

@@ -1,27 +1,25 @@
 % import os
 % import platform
-% import re
-%
-% import click
 %
 % systype = platform.system().lower()
 %
 % cpp_standards_remap = {
-%   "0x": "11",
-%   "1y": "14",
-%   "1z": "17",
-%   "2a": "20",
-%   "2b": "23"
+%   "c++0x": "c++11",
+%   "c++1y": "c++14",
+%   "c++1z": "c++17",
+%   "c++2a": "c++20",
+%   "c++2b": "c++23",
+%   "gnu++0x": "gnu++11",
+%   "gnu++1y": "gnu++14",
+%   "gnu++1z": "gnu++17",
+%   "gnu++2a": "gnu++20",
+%   "gnu++2b": "gnu++23"
 % }
 %
 % def _escape(text):
 %   return to_unix_path(text).replace('"', '\\"')
 % end
 %
-% def split_args(args_string):
-%   return click.parser.split_arg_string(to_unix_path(args_string))
-% end
-%
 % def filter_args(args, allowed, ignore=None):
 %   if not allowed:
 %     return []
@@ -31,18 +29,18 @@
 %   result = []
 %   i = 0
 %   length = len(args)
-%     while(i < length):
-%      if any(args[i].startswith(f) for f in allowed) and not any(
-%        args[i].startswith(f) for f in ignore):
-%        result.append(args[i])
-%        if i + 1 < length and not args[i + 1].startswith("-"):
-%          i += 1
-%          result.append(args[i])
-%        end
+%   while(i < length):
+%     if any(args[i].startswith(f) for f in allowed) and not any(
+%           args[i].startswith(f) for f in ignore):
+%       result.append(args[i])
+%       if i + 1 < length and not args[i + 1].startswith("-"):
+%         i += 1
+%         result.append(args[i])
 %       end
-%      i += 1
-%    end
-%    return result
+%     end
+%     i += 1
+%   end
+%   return result
 % end
 %
 % def _find_abs_path(inc, inc_paths):
@@ -76,12 +74,10 @@
 %
 % cleaned_includes = filter_includes(includes, ["toolchain"])
 %
-% STD_RE = re.compile(r"\-std=[a-z\+]+(\w+)")
-% cc_stds = STD_RE.findall(cc_flags)
-% cxx_stds = STD_RE.findall(cxx_flags)
-% cc_m_flags = split_args(cc_flags)
+% cc_stds = [arg[5:] for arg in cc_flags if arg.startswith("-std=")]
+% cxx_stds = [arg[5:] for arg in cxx_flags if arg.startswith("-std=")]
 % forced_includes = _find_forced_includes(
-%   filter_args(cc_m_flags, ["-include", "-imacros"]), cleaned_includes)
+%   filter_args(cc_flags, ["-include", "-imacros"]), cleaned_includes)
 %
 //
 // !!! WARNING !!! AUTO-GENERATED FILE!
@@ -114,10 +110,10 @@
                 ""
             ],
 % if cc_stds:
-            "cStandard": "c{{ cc_stds[-1] }}",
+            "cStandard": "{{ cc_stds[-1] }}",
 % end
 % if cxx_stds:
-            "cppStandard": "c++{{ cpp_standards_remap.get(cxx_stds[-1], cxx_stds[-1]) }}",
+            "cppStandard": "{{ cpp_standards_remap.get(cxx_stds[-1], cxx_stds[-1]) }}",
 % end
 % if forced_includes:
             "forcedInclude": [
@@ -130,7 +126,7 @@
             "compilerPath": "{{ cc_path }}",
             "compilerArgs": [
 % for flag in [
-%     f for f in filter_args(cc_m_flags, ["-m", "-i", "@"], ["-include", "-imacros"])
+%     f for f in filter_args(cc_flags, ["-m", "-i", "@"], ["-include", "-imacros"])
 % ]:
                 "{{ flag }}",
 % end

platformio/registry/client.py

@@ -142,12 +142,12 @@ class RegistryClient(HTTPClient):
             x_with_authorization=self.allowed_private_packages(),
         )
 
-    def get_package(self, type_, owner, name, version=None, extra_path=None):
+    def get_package(self, typex, owner, name, version=None, extra_path=None):
         try:
             return self.fetch_json_data(
                 "get",
                 "/v3/packages/{owner}/{type}/{name}{extra_path}".format(
-                    type=type_,
+                    type=typex,
                     owner=owner.lower(),
                     name=name.lower(),
                     extra_path=extra_path or "",

platformio/system/commands/info.py

@@ -77,7 +77,6 @@ def system_info_cmd(json_output):
             ).get_installed()
         ),
     }
-
     click.echo(
         json.dumps(data)
         if json_output

platformio/system/completion.py

@@ -29,8 +29,10 @@ class ShellType(Enum):
 
 
 def get_bash_version():
-    result = subprocess.run(["bash", "--version"], capture_output=True, check=True)
-    match = re.search(r"version\s+(\d+)\.(\d+)", result.stdout.decode(), re.IGNORECASE)
+    output = subprocess.run(
+        ["bash", "--version"], check=True, stdout=subprocess.PIPE
+    ).stdout.decode()
+    match = re.search(r"version\s+(\d+)\.(\d+)", output, re.IGNORECASE)
     if match:
         return (int(match.group(1)), int(match.group(2)))
     return (0, 0)

platformio/telemetry.py

@@ -14,299 +14,181 @@
 
 import atexit
 import hashlib
-import json
 import os
 import queue
 import re
-import shutil
 import sys
 import threading
+import time
+import traceback
 from collections import deque
-from time import sleep, time
-from traceback import format_exc
 
 import requests
 
-from platformio import __version__, app, exception, util
+from platformio import __title__, __version__, app, exception, fs, util
 from platformio.cli import PlatformioCLI
-from platformio.compat import hashlib_encode_data, string_types
-from platformio.http import HTTPSession
-from platformio.proc import is_ci, is_container
-from platformio.project.helpers import is_platformio_project
+from platformio.compat import hashlib_encode_data
+from platformio.debug.config.base import DebugConfigBase
+from platformio.http import HTTPSession, ensure_internet_on
+from platformio.proc import is_ci
+
+KEEP_MAX_REPORTS = 100
+SEND_MAX_EVENTS = 25
 
 
-class TelemetryBase:
-    def __init__(self):
-        self._params = {}
+class MeasurementProtocol:
+    def __init__(self, events=None):
+        self.client_id = app.get_cid()
+        self._events = events or []
+        self._user_properties = {}
 
-    def __getitem__(self, name):
-        return self._params.get(name, None)
+        self.set_user_property("systype", util.get_systype())
+        created_at = app.get_state_item("created_at", None)
+        if created_at:
+            self.set_user_property("created_at", int(created_at))
 
-    def __setitem__(self, name, value):
-        self._params[name] = value
+    @staticmethod
+    def event_to_dict(name, params, timestamp=None):
+        event = {"name": name, "params": params}
+        if timestamp is not None:
+            event["timestamp"] = timestamp
+        return event
 
-    def __delitem__(self, name):
-        if name in self._params:
-            del self._params[name]
+    def set_user_property(self, name, value):
+        self._user_properties[name] = value
 
-    def send(self, hittype):
-        raise NotImplementedError()
+    def add_event(self, name, params):
+        self._events.append(self.event_to_dict(name, params))
 
-
-class MeasurementProtocol(TelemetryBase):
-    TID = "UA-1768265-9"
-    PARAMS_MAP = {
-        "screen_name": "cd",
-        "event_category": "ec",
-        "event_action": "ea",
-        "event_label": "el",
-        "event_value": "ev",
-    }
-
-    def __init__(self):
-        super().__init__()
-        self["v"] = 1
-        self["tid"] = self.TID
-        self["cid"] = app.get_cid()
-
-        try:
-            self["sr"] = "%dx%d" % shutil.get_terminal_size()
-        except ValueError:
-            pass
-
-        self._prefill_screen_name()
-        self._prefill_appinfo()
-        self._prefill_sysargs()
-        self._prefill_custom_data()
-
-    def __getitem__(self, name):
-        if name in self.PARAMS_MAP:
-            name = self.PARAMS_MAP[name]
-        return super().__getitem__(name)
-
-    def __setitem__(self, name, value):
-        if name in self.PARAMS_MAP:
-            name = self.PARAMS_MAP[name]
-        super().__setitem__(name, value)
-
-    def _prefill_appinfo(self):
-        self["av"] = __version__
-        self["an"] = app.get_user_agent()
-
-    def _prefill_sysargs(self):
-        args = []
-        for arg in sys.argv[1:]:
-            arg = str(arg)
-            if arg == "account":  # ignore account cmd which can contain username
-                return
-            if any(("@" in arg, "/" in arg, "\\" in arg)):
-                arg = "***"
-            args.append(arg.lower())
-        self["cd3"] = " ".join(args)
-
-    def _prefill_custom_data(self):
-        caller_id = str(app.get_session_var("caller_id"))
-        self["cd1"] = util.get_systype()
-        self["cd4"] = 1 if (not is_ci() and (caller_id or not is_container())) else 0
-        if caller_id:
-            self["cd5"] = caller_id.lower()
-
-    def _prefill_screen_name(self):
-        def _first_arg_from_list(args_, list_):
-            for _arg in args_:
-                if _arg in list_:
-                    return _arg
-            return None
-
-        args = []
-        for arg in PlatformioCLI.leftover_args:
-            if not isinstance(arg, string_types):
-                arg = str(arg)
-            if not arg.startswith("-"):
-                args.append(arg.lower())
-        if not args:
-            return
-
-        cmd_path = args[:1]
-        if args[0] in (
-            "access",
-            "account",
-            "device",
-            "org",
-            "package",
-            "pkg",
-            "platform",
-            "project",
-            "settings",
-            "system",
-            "team",
-        ):
-            cmd_path = args[:2]
-        if args[0] == "lib" and len(args) > 1:
-            lib_subcmds = (
-                "builtin",
-                "install",
-                "list",
-                "register",
-                "search",
-                "show",
-                "stats",
-                "uninstall",
-                "update",
-            )
-            sub_cmd = _first_arg_from_list(args[1:], lib_subcmds)
-            if sub_cmd:
-                cmd_path.append(sub_cmd)
-        elif args[0] == "remote" and len(args) > 1:
-            remote_subcmds = ("agent", "device", "run", "test")
-            sub_cmd = _first_arg_from_list(args[1:], remote_subcmds)
-            if sub_cmd:
-                cmd_path.append(sub_cmd)
-                if len(args) > 2 and sub_cmd in ("agent", "device"):
-                    remote2_subcmds = ("list", "start", "monitor")
-                    sub_cmd = _first_arg_from_list(args[2:], remote2_subcmds)
-                    if sub_cmd:
-                        cmd_path.append(sub_cmd)
-        self["screen_name"] = " ".join([p.title() for p in cmd_path])
-
-    def _ignore_hit(self):
-        if not app.get_setting("enable_telemetry"):
-            return True
-        if self["ea"] in ("Idedata", "__Idedata"):
-            return True
-        return False
-
-    def send(self, hittype):
-        if self._ignore_hit():
-            return
-        self["t"] = hittype
-        # correct queue time
-        if "qt" in self._params and isinstance(self["qt"], float):
-            self["qt"] = int((time() - self["qt"]) * 1000)
-        MPDataPusher().push(self._params)
+    def to_payload(self):
+        return {
+            "client_id": self.client_id,
+            "user_properties": self._user_properties,
+            "events": self._events,
+        }
 
 
 @util.singleton
-class MPDataPusher:
-    MAX_WORKERS = 5
-
+class TelemetryLogger:
     def __init__(self):
-        self._queue = queue.LifoQueue()
-        self._failedque = deque()
+        self._events = deque()
+
+        self._sender_thread = None
+        self._sender_queue = queue.Queue()
+        self._sender_terminated = False
+
         self._http_session = HTTPSession()
         self._http_offline = False
-        self._workers = []
 
-    def push(self, item):
-        # if network is off-line
-        if self._http_offline:
-            if "qt" not in item:
-                item["qt"] = time()
-            self._failedque.append(item)
+    def close(self):
+        self._http_session.close()
+
+    def log_event(self, name, params, timestamp=None, instant_sending=False):
+        if not app.get_setting("enable_telemetry") or app.get_session_var(
+            "pause_telemetry"
+        ):
+            return None
+        timestamp = timestamp or int(time.time())
+        self._events.append(
+            MeasurementProtocol.event_to_dict(name, params, timestamp=timestamp)
+        )
+        if self._http_offline:  # if network is off-line
+            return False
+        if instant_sending:
+            self.send()
+        return True
+
+    def send(self):
+        if not self._events or self._sender_terminated:
             return
-
-        self._queue.put(item)
-        self._tune_workers()
-
-    def in_wait(self):
-        return self._queue.unfinished_tasks
-
-    def get_items(self):
-        items = list(self._failedque)
-        try:
-            while True:
-                items.append(self._queue.get_nowait())
-        except queue.Empty:
-            pass
-        return items
-
-    def _tune_workers(self):
-        for i, w in enumerate(self._workers):
-            if not w.is_alive():
-                del self._workers[i]
-
-        need_nums = min(self._queue.qsize(), self.MAX_WORKERS)
-        active_nums = len(self._workers)
-        if need_nums <= active_nums:
-            return
-
-        for i in range(need_nums - active_nums):
-            t = threading.Thread(target=self._worker)
-            t.daemon = True
-            t.start()
-            self._workers.append(t)
-
-    def _worker(self):
-        while True:
+        if not self._sender_thread:
+            self._sender_thread = threading.Thread(
+                target=self._sender_worker, daemon=True
+            )
+            self._sender_thread.start()
+        while self._events:
+            events = []
             try:
-                item = self._queue.get()
-                _item = item.copy()
-                if "qt" not in _item:
-                    _item["qt"] = time()
-                self._failedque.append(_item)
-                if self._send_data(item):
-                    self._failedque.remove(_item)
-                self._queue.task_done()
-            except:  # pylint: disable=W0702
+                while len(events) < SEND_MAX_EVENTS:
+                    events.append(self._events.popleft())
+            except IndexError:
+                pass
+            self._sender_queue.put(events)
+
+    def _sender_worker(self):
+        while True:
+            if self._sender_terminated:
+                return
+            try:
+                events = self._sender_queue.get()
+                if not self._commit_events(events):
+                    self._events.extend(events)
+                self._sender_queue.task_done()
+            except (queue.Empty, ValueError):
                 pass
 
-    def _send_data(self, data):
+    def _commit_events(self, events):
         if self._http_offline:
             return False
+        mp = MeasurementProtocol(events)
+        payload = mp.to_payload()
+        # print("_commit_payload", payload)
         try:
             r = self._http_session.post(
-                "https://ssl.google-analytics.com/collect",
-                data=data,
-                timeout=1,
+                "https://telemetry.platformio.org/collect",
+                json=payload,
+                timeout=(2, 5),  # connect, read
             )
             r.raise_for_status()
             return True
         except requests.exceptions.HTTPError as exc:
             # skip Bad Request
-            if 400 >= exc.response.status_code < 500:
+            if exc.response.status_code >= 400 and exc.response.status_code < 500:
                 return True
         except:  # pylint: disable=bare-except
             pass
         self._http_offline = True
         return False
 
+    def terminate_sender(self):
+        self._sender_terminated = True
 
-def on_command():
-    resend_backuped_reports()
+    def is_sending(self):
+        return self._sender_queue.unfinished_tasks
 
-    mp = MeasurementProtocol()
-    mp.send("screenview")
+    def get_unsent_events(self):
+        result = list(self._events)
+        try:
+            while True:
+                result.extend(self._sender_queue.get_nowait())
+        except queue.Empty:
+            pass
+        return result
 
+
+def log_event(name, params, instant_sending=False):
+    TelemetryLogger().log_event(name, params, instant_sending=instant_sending)
+
+
+def on_cmd_start(cmd_ctx):
+    process_postponed_logs()
+    log_command(cmd_ctx)
+
+
+def on_exit():
+    TelemetryLogger().send()
+
+
+def log_command(ctx):
+    params = {
+        "path_args": PlatformioCLI.reveal_cmd_path_args(ctx),
+    }
     if is_ci():
-        measure_ci()
+        params["ci_actor"] = resolve_ci_actor() or "Unknown"
+    log_event("cmd_run", params)
 
 
-def on_exception(e):
-    skip_conditions = [
-        isinstance(e, cls)
-        for cls in (
-            IOError,
-            exception.ReturnErrorCode,
-            exception.UserSideException,
-        )
-    ]
-    if any(skip_conditions):
-        return
-    is_fatal = any(
-        [
-            not isinstance(e, exception.PlatformioException),
-            "Error" in e.__class__.__name__,
-        ]
-    )
-    description = "%s: %s" % (
-        type(e).__name__,
-        " ".join(reversed(format_exc().split("\n"))) if is_fatal else str(e),
-    )
-    send_exception(description, is_fatal)
-
-
-def measure_ci():
-    event = {"category": "CI", "action": "NoName", "label": None}
+def resolve_ci_actor():
     known_cis = (
         "GITHUB_ACTIONS",
         "TRAVIS",
@@ -318,123 +200,184 @@ def measure_ci():
     )
     for name in known_cis:
         if os.getenv(name, "false").lower() == "true":
-            event["action"] = name
-            break
-    send_event(**event)
+            return name
+    return None
 
 
-def dump_run_environment(options):
+def dump_project_env_params(config, env, platform):
     non_sensitive_data = [
         "platform",
-        "platform_packages",
         "framework",
         "board",
         "upload_protocol",
         "check_tool",
         "debug_tool",
-        "monitor_filters",
         "test_framework",
     ]
-    safe_options = {k: v for k, v in options.items() if k in non_sensitive_data}
-    if is_platformio_project(os.getcwd()):
-        phash = hashlib.sha1(hashlib_encode_data(app.get_cid()))
-        safe_options["pid"] = phash.hexdigest()
-    return json.dumps(safe_options, sort_keys=True, ensure_ascii=False)
+    section = f"env:{env}"
+    params = {
+        option: config.get(section, option)
+        for option in non_sensitive_data
+        if config.has_option(section, option)
+    }
+    params["pid"] = hashlib.sha1(hashlib_encode_data(config.path)).hexdigest()
+    params["platform_name"] = platform.name
+    params["platform_version"] = platform.version
+    return params
 
 
-def send_run_environment(options, targets):
-    send_event(
-        "Env",
-        " ".join([t.title() for t in targets or ["run"]]),
-        dump_run_environment(options),
+def log_platform_run(platform, project_config, project_env, targets=None):
+    params = dump_project_env_params(project_config, project_env, platform)
+    if targets:
+        params["targets"] = targets
+    log_event("platform_run", params, instant_sending=True)
+
+
+def log_exception(exc):
+    skip_conditions = [
+        isinstance(exc, cls)
+        for cls in (
+            IOError,
+            exception.ReturnErrorCode,
+            exception.UserSideException,
+        )
+    ]
+    skip_conditions.append(not isinstance(exc, Exception))
+    if any(skip_conditions):
+        return
+    is_fatal = any(
+        [
+            not isinstance(exc, exception.PlatformioException),
+            "Error" in exc.__class__.__name__,
+        ]
+    )
+
+    def _strip_module_path(match):
+        module_path = match.group(1).replace(fs.get_source_dir() + os.sep, "")
+        sp_folder_name = "site-packages"
+        sp_pos = module_path.find(sp_folder_name)
+        if sp_pos != -1:
+            module_path = module_path[sp_pos + len(sp_folder_name) + 1 :]
+        module_path = fs.to_unix_path(module_path)
+        return f'File "{module_path}",'
+
+    trace = re.sub(
+        r'File "([^"]+)",',
+        _strip_module_path,
+        traceback.format_exc(),
+        flags=re.MULTILINE,
+    )
+
+    params = {
+        "name": exc.__class__.__name__,
+        "description": str(exc),
+        "traceback": trace,
+        "cmd_args": sys.argv[1:],
+        "is_fatal": is_fatal,
+    }
+    log_event("exception", params)
+
+
+def log_debug_started(debug_config: DebugConfigBase):
+    log_event(
+        "debug_started",
+        dump_project_env_params(
+            debug_config.project_config, debug_config.env_name, debug_config.platform
+        ),
     )
 
 
-def send_event(category, action, label=None, value=None, screen_name=None):
-    mp = MeasurementProtocol()
-    mp["event_category"] = category[:150]
-    mp["event_action"] = action[:500]
-    if label:
-        mp["event_label"] = label[:500]
-    if value:
-        mp["event_value"] = int(value)
-    if screen_name:
-        mp["screen_name"] = screen_name[:2048]
-    mp.send("event")
-
-
-def send_exception(description, is_fatal=False):
+def log_debug_exception(exc, debug_config: DebugConfigBase):
     # cleanup sensitive information, such as paths
-    description = description.replace("Traceback (most recent call last):", "")
-    description = description.replace("\\", "/")
+    description = fs.to_unix_path(str(exc))
     description = re.sub(
         r'(^|\s+|")(?:[a-z]\:)?((/[^"/]+)+)(\s+|"|$)',
         lambda m: " %s " % os.path.join(*m.group(2).split("/")[-2:]),
         description,
         re.I | re.M,
     )
-    description = re.sub(r"\s+", " ", description, flags=re.M)
-
-    mp = MeasurementProtocol()
-    mp["exd"] = description[:8192].strip()
-    mp["exf"] = 1 if is_fatal else 0
-    mp.send("exception")
+    params = {
+        "name": exc.__class__.__name__,
+        "description": description.strip(),
+    }
+    params.update(
+        dump_project_env_params(
+            debug_config.project_config, debug_config.env_name, debug_config.platform
+        )
+    )
+    log_event("debug_exception", params)
 
 
 @atexit.register
 def _finalize():
     timeout = 1000  # msec
     elapsed = 0
+    telemetry = TelemetryLogger()
+    telemetry.terminate_sender()
     try:
         while elapsed < timeout:
-            if not MPDataPusher().in_wait():
+            if not telemetry.is_sending():
                 break
-            sleep(0.2)
+            time.sleep(0.2)
             elapsed += 200
-        backup_reports(MPDataPusher().get_items())
     except KeyboardInterrupt:
         pass
+    postpone_events(telemetry.get_unsent_events())
+    telemetry.close()
 
 
-def backup_reports(items):
-    if not items:
-        return
-
-    KEEP_MAX_REPORTS = 100
-    tm = app.get_state_item("telemetry", {})
-    if "backup" not in tm:
-        tm["backup"] = []
-
-    for params in items:
-        # skip static options
-        for key in list(params.keys()):
-            if key in ("v", "tid", "cid", "cd1", "cd2", "sr", "an"):
-                del params[key]
-
-        # store time in UNIX format
-        if "qt" not in params:
-            params["qt"] = time()
-        elif not isinstance(params["qt"], float):
-            params["qt"] = time() - (params["qt"] / 1000)
-
-        tm["backup"].append(params)
-
-    tm["backup"] = tm["backup"][KEEP_MAX_REPORTS * -1 :]
-    app.set_state_item("telemetry", tm)
+def load_postponed_events():
+    state_path = app.resolve_state_path(
+        "cache_dir", "telemetry.json", ensure_dir_exists=False
+    )
+    if not os.path.isfile(state_path):
+        return []
+    with app.State(state_path) as state:
+        return state.get("events", [])
 
 
-def resend_backuped_reports():
-    tm = app.get_state_item("telemetry", {})
-    if "backup" not in tm or not tm["backup"]:
-        return False
-
-    for report in tm["backup"]:
-        mp = MeasurementProtocol()
-        for key, value in report.items():
-            mp[key] = value
-        mp.send(report["t"])
-
-    # clean
-    tm["backup"] = []
-    app.set_state_item("telemetry", tm)
+def save_postponed_events(events):
+    state_path = app.resolve_state_path("cache_dir", "telemetry.json")
+    if not events:
+        try:
+            if os.path.isfile(state_path):
+                os.remove(state_path)
+        except:  # pylint: disable=bare-except
+            pass
+        return None
+    with app.State(state_path, lock=True) as state:
+        state["events"] = events
+        state.modified = True
+    return True
+
+
+def postpone_events(events):
+    if not events:
+        return None
+    postponed_events = load_postponed_events() or []
+    timestamp = int(time.time())
+    for event in events:
+        if "timestamp" not in event:
+            event["timestamp"] = timestamp
+        postponed_events.append(event)
+    save_postponed_events(postponed_events[KEEP_MAX_REPORTS * -1 :])
+    return True
+
+
+def process_postponed_logs():
+    if not ensure_internet_on():
+        return None
+    events = load_postponed_events()
+    if not events:
+        return None
+    save_postponed_events([])  # clean
+    telemetry = TelemetryLogger()
+    for event in events:
+        if set(["name", "params", "timestamp"]) <= set(event.keys()):
+            telemetry.log_event(
+                event["name"],
+                event["params"],
+                timestamp=event["timestamp"],
+                instant_sending=False,
+            )
     return True

platformio/test/runners/unity.py

@@ -55,8 +55,8 @@ extern "C"
 
 void unityOutputStart(unsigned long);
 void unityOutputChar(unsigned int);
-void unityOutputFlush();
-void unityOutputComplete();
+void unityOutputFlush(void);
+void unityOutputComplete(void);
 
 #define UNITY_OUTPUT_START()    unityOutputStart((unsigned long) $baudrate)
 #define UNITY_OUTPUT_CHAR(c)    unityOutputChar(c)
@@ -246,18 +246,20 @@ void unityOutputComplete(void) { unittest_uart_end(); }
         unity_h = dst_dir / "unity_config.h"
         if not unity_h.is_file():
             unity_h.write_text(
-                string.Template(self.UNITY_CONFIG_H).substitute(
-                    baudrate=self.get_test_speed()
-                ),
+                string.Template(self.UNITY_CONFIG_H)
+                .substitute(baudrate=self.get_test_speed())
+                .strip()
+                + "\n",
                 encoding="utf8",
             )
         framework_config = self.get_unity_framework_config()
         unity_c = dst_dir / ("unity_config.%s" % framework_config.get("language", "c"))
         if not unity_c.is_file():
             unity_c.write_text(
-                string.Template(self.UNITY_CONFIG_C).substitute(
-                    framework_config_code=framework_config["code"]
-                ),
+                string.Template(self.UNITY_CONFIG_C)
+                .substitute(framework_config_code=framework_config["code"])
+                .strip()
+                + "\n",
                 encoding="utf8",
             )
 

platformio/util.py

@@ -12,13 +12,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import datetime
 import functools
 import math
 import platform
 import re
 import shutil
 import time
-from datetime import datetime
 
 import click
 
@@ -169,8 +169,8 @@ def items_in_list(needle, haystack):
 
 def parse_datetime(datestr):
     if "T" in datestr and "Z" in datestr:
-        return datetime.strptime(datestr, "%Y-%m-%dT%H:%M:%SZ")
-    return datetime.strptime(datestr)
+        return datetime.datetime.strptime(datestr, "%Y-%m-%dT%H:%M:%SZ")
+    return datetime.datetime.strptime(datestr)
 
 
 def merge_dicts(d1, d2, path=None):

setup.py

@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import sys
 from setuptools import find_packages, setup
 
 from platformio import (
@@ -25,31 +24,48 @@ from platformio import (
     __version__,
 )
 
-PY36 = sys.version_info < (3, 7)
-
+env_marker_below_37 = "python_version < '3.7'"
+env_marker_gte_37 = "python_version >= '3.7'"
 
 minimal_requirements = [
     "bottle==0.12.*",
-    "click%s" % ("==8.0.4" if PY36 else ">=8.0.4,<9"),
+    "click==8.0.4; " + env_marker_below_37,
+    "click==8.1.*; " + env_marker_gte_37,
     "colorama",
-    "marshmallow==%s" % ("3.14.1" if PY36 else "3.*"),
-    "pyelftools>=0.27,<1",
+    "marshmallow==3.14.1; " + env_marker_below_37,
+    "marshmallow==3.19.*; " + env_marker_gte_37,
+    "pyelftools==0.29",
     "pyserial==3.5.*",  # keep in sync "device/monitor/terminal.py"
     "requests==2.*",
-    "urllib3<2", # issue 4614: urllib3 v2.0 only supports OpenSSL 1.1.1+
-    "requests==%s" % ("2.27.1" if PY36 else "2.*"),
     "semantic_version==2.10.*",
-    "tabulate==%s" % ("0.8.10" if PY36 else "0.9.*"),
+    "tabulate==0.*",
 ]
 
 home_requirements = [
-    "aiofiles==%s" % ("0.8.0" if PY36 else "23.1.*"),
-    "ajsonrpc==1.*",
-    "starlette==%s" % ("0.19.1" if PY36 else "0.26.*"),
-    "uvicorn==%s" % ("0.16.0" if PY36 else "0.22.*"),
-    "wsproto==%s" % ("1.0.0" if PY36 else "1.2.*"),
+    "aiofiles>=0.8.0",
+    "ajsonrpc==1.2.*",
+    "starlette==0.19.1; " + env_marker_below_37,
+    "starlette==0.28.*; " + env_marker_gte_37,
+    "uvicorn==0.16.0; " + env_marker_below_37,
+    "uvicorn==0.22.*; " + env_marker_gte_37,
+    "wsproto==1.0.0; " + env_marker_below_37,
+    "wsproto==1.2.*; " + env_marker_gte_37,
 ]
 
+# issue 4614: urllib3 v2.0 only supports OpenSSL 1.1.1+
+try:
+    import ssl
+
+    if ssl.OPENSSL_VERSION.startswith("OpenSSL ") and ssl.OPENSSL_VERSION_INFO < (
+        1,
+        1,
+        1,
+    ):
+        minimal_requirements.append("urllib3<2")
+except ImportError:
+    pass
+
+
 setup(
     name=__title__,
     version=__version__,
@@ -65,11 +81,11 @@ setup(
     package_data={
         "platformio": [
             "assets/system/99-platformio-udev.rules",
-            "assets/templates/ide-projects/*/*.tpl",
-            "assets/templates/ide-projects/*/.*.tpl",  # include hidden files
-            "assets/templates/ide-projects/*/.*/*.tpl",  # include hidden folders
-            "assets/templates/ide-projects/*/*/*.tpl",  # NetBeans
-            "assets/templates/ide-projects/*/*/*/*.tpl", # NetBeans
+            "project/integration/tpls/*/*.tpl",
+            "project/integration/tpls/*/.*.tpl",  # include hidden files
+            "project/integration/tpls/*/.*/*.tpl",  # include hidden folders
+            "project/integration/tpls/*/*/*.tpl",  # NetBeans
+            "project/integration/tpls/*/*/*/*.tpl",  # NetBeans
         ]
     },
     entry_points={

tests/commands/pkg/test_install.py

@@ -469,7 +469,7 @@ def test_custom_project_tools(
     project_dir = tmp_path / "project"
     project_dir.mkdir()
     (project_dir / "platformio.ini").write_text(PROJECT_CONFIG_TPL)
-    spec = "platformio/tool-openocd"
+    spec = "platformio/tool-openocd @ ^2"
     result = clirunner.invoke(
         package_install_cmd,
         ["-d", str(project_dir), "-e", "devkit", "-t", spec],
@@ -503,7 +503,7 @@ def test_custom_project_tools(
 
         # check saved deps
         assert config.get("env:devkit", "platform_packages") == [
-            "platformio/tool-openocd@^2.1100.211028",
+            "platformio/tool-openocd@^2",
         ]
 
         # install tool without saving to config
@@ -518,7 +518,7 @@ def test_custom_project_tools(
             PackageSpec("tool-openocd@2.1100.211028"),
         ]
         assert config.get("env:devkit", "platform_packages") == [
-            "platformio/tool-openocd@^2.1100.211028",
+            "platformio/tool-openocd@^2",
         ]
 
         # unknown tool

tests/commands/pkg/test_uninstall.py

@@ -313,7 +313,7 @@ def test_custom_project_tools(
     project_dir = tmp_path / "project"
     project_dir.mkdir()
     (project_dir / "platformio.ini").write_text(PROJECT_CONFIG_TPL)
-    spec = "platformio/tool-openocd"
+    spec = "platformio/tool-openocd@^2"
     result = clirunner.invoke(
         package_install_cmd,
         ["-d", str(project_dir), "-e", "devkit", "-t", spec],
@@ -329,7 +329,7 @@ def test_custom_project_tools(
         assert not os.path.exists(config.get("platformio", "platforms_dir"))
         # check saved deps
         assert config.get("env:devkit", "platform_packages") == [
-            "platformio/tool-openocd@^2.1100.211028",
+            "platformio/tool-openocd@^2",
         ]
         # uninstall
         result = clirunner.invoke(

tests/commands/test_init.py

@@ -62,7 +62,7 @@ def test_init_duplicated_boards(clirunner, validate_cliresult, tmpdir):
 
 def test_init_ide_without_board(clirunner, tmpdir):
     with tmpdir.as_cwd():
-        result = clirunner.invoke(project_init_cmd, ["--ide", "atom"])
+        result = clirunner.invoke(project_init_cmd, ["--ide", "vscode"])
         assert result.exit_code != 0
         assert isinstance(result.exception, ProjectEnvsNotAvailableError)
 

tests/commands/test_run.py

@@ -23,6 +23,10 @@ def test_generic_build(clirunner, validate_cliresult, tmpdir):
         ("-DTEST_SINGLE_MACRO", "-DTEST_SINGLE_MACRO"),
         ('-DTEST_STR_SPACE="Andrew Smith"', '"-DTEST_STR_SPACE=Andrew Smith"'),
         ("-Iextra_inc", "-Iextra_inc"),
+        (
+            "-include $PROJECT_DIR/lib/component/component-forced-include.h",
+            "component-forced-include.h",
+        ),
     ]
 
     tmpdir.join("platformio.ini").write(
@@ -95,6 +99,10 @@ projenv.Append(CPPDEFINES="POST_SCRIPT_MACRO")
 #error "POST_SCRIPT_MACRO"
 #endif
 
+#ifndef I_AM_FORCED_COMPONENT_INCLUDE
+#error "I_AM_FORCED_COMPONENT_INCLUDE"
+#endif
+
 #ifdef COMMENTED_MACRO
 #error "COMMENTED_MACRO"
 #endif
@@ -124,6 +132,11 @@ void dummy(void);
 void dummy(void ) {};
     """
     )
+    component_dir.join("component-forced-include.h").write(
+        """
+#define I_AM_FORCED_COMPONENT_INCLUDE
+    """
+    )
 
     result = clirunner.invoke(cmd_run, ["--project-dir", str(tmpdir), "--verbose"])
     validate_cliresult(result)

tests/misc/test_misc.py

@@ -36,13 +36,13 @@ def test_ping_internet_ips():
 def test_api_internet_offline(without_internet, isolated_pio_core):
     regclient = RegistryClient()
     with pytest.raises(http.InternetConnectionError):
-        regclient.fetch_json_data("get", "/v2/stats")
+        regclient.fetch_json_data("get", "/v3/search")
 
 
 def test_api_cache(monkeypatch, isolated_pio_core):
     regclient = RegistryClient()
-    api_kwargs = {"method": "get", "path": "/v2/stats", "x_cache_valid": "10s"}
+    api_kwargs = {"method": "get", "path": "/v3/search", "x_cache_valid": "10s"}
     result = regclient.fetch_json_data(**api_kwargs)
-    assert result and "boards" in result
+    assert result and "total" in result
     monkeypatch.setattr(http, "_internet_on", lambda: False)
     assert regclient.fetch_json_data(**api_kwargs) == result