Added --json-output support for pkg list command

Fix typo

.github/workflows/docs.yml

@@ -49,12 +49,12 @@ jobs:
     name: Deploy Docs
     needs: build
     runs-on: ubuntu-latest
+    if: ${{ github.event_name == 'push' && (github.ref == 'refs/heads/develop' || github.ref == 'refs/heads/master') }}
     env:
       DOCS_REPO: platformio/platformio-docs
       DOCS_DIR: platformio-docs
       LATEST_DOCS_DIR: latest-docs
       RELEASE_BUILD: ${{ startsWith(github.ref, 'refs/tags/v') }}
-    if: ${{ github.event_name == 'push' }}
     steps:
       - name: Download artifact
         uses: actions/download-artifact@v3

HISTORY.rst

@@ -7,6 +7,7 @@ Release Notes
 .. |INTERPOLATION| replace:: `Interpolation of Values <https://docs.platformio.org/en/latest/projectconf/interpolation.html>`__
 .. |UNITTESTING| replace:: `Unit Testing <https://docs.platformio.org/en/latest/advanced/unit-testing/index.html>`__
 .. |DEBUGGING| replace:: `Debugging <https://docs.platformio.org/en/latest/plus/debugging.html>`__
+.. |STATICCODEANALYSIS| replace:: `Static Code Analysis <https://docs.platformio.org/en/latest/advanced/static-code-analysis/index.html>`__
 
 .. _release_notes_6:
 
@@ -17,6 +18,29 @@ Unlock the true potential of embedded software development with
 PlatformIO's collaborative ecosystem, embracing declarative principles,
 test-driven methodologies, and modern toolchains for unrivaled success.
 
+6.1.15 (2024-??-??)
+~~~~~~~~~~~~~~~~~~~
+
+* Resolved an issue where the |LDF| couldn't locate a library dependency declared via version control system repository (`issue #4885 <https://github.com/platformio/platformio-core/issues/4885>`_)
+
+6.1.14 (2024-03-21)
+~~~~~~~~~~~~~~~~~~~
+
+* Introduced the ``--json-output`` option to the `pio test <https://docs.platformio.org/en/latest/core/userguide/cmd_test.html>`__ command, enabling users to generate test results in the JSON format
+* Upgraded the build engine to the latest version of SCons (4.7.0) to improve build performance, reliability, and compatibility with other tools and systems (`release notes <https://github.com/SCons/scons/releases/tag/4.7.0>`__)
+* Broadened version support for the ``pyelftools`` dependency, enabling compatibility with lower versions and facilitating integration with a wider range of third-party tools (`issue #4834 <https://github.com/platformio/platformio-core/issues/4834>`_)
+* Addressed an issue where passing a relative path (``--project-dir``) to the `pio project init <https://docs.platformio.org/en/latest/core/userguide/project/cmd_init.html>`__ command resulted in an error (`issue #4847 <https://github.com/platformio/platformio-core/issues/4847>`_)
+* Enhanced |STATICCODEANALYSIS| to accommodate scenarios where custom ``src_dir`` or ``include_dir`` are located outside the project folder (`pull #4874 <https://github.com/platformio/platformio-core/pull/4874>`_)
+* Corrected the validation of ``symlink://`` `package specifications <https://docs.platformio.org/en/latest/core/userguide/pkg/cmd_install.html#local-folder>`__ , resolving an issue that caused the package manager to repeatedly reinstall dependencies (`pull #4870 <https://github.com/platformio/platformio-core/pull/4870>`_)
+* Resolved an issue related to the relative package path in the `pio pkg publish <https://docs.platformio.org/en/latest/core/userguide/pkg/cmd_publish.html>`__ command
+* Resolved an issue where the |LDF| selected an incorrect library version (`issue #4860 <https://github.com/platformio/platformio-core/issues/4860>`_)
+* Resolved an issue with the ``hexlify`` filter in the `device monitor <https://docs.platformio.org/en/latest/core/userguide/device/cmd_monitor.html>`__ command, ensuring proper representation of characters with Unicode code points higher than 127 (`issue #4732 <https://github.com/platformio/platformio-core/issues/4732>`_)
+
+6.1.13 (2024-01-12)
+~~~~~~~~~~~~~~~~~~~
+
+* Expanded support for SCons variables declared in the legacy format ``${SCONS_VARNAME}`` (`issue #4828 <https://github.com/platformio/platformio-core/issues/4828>`_)
+
 6.1.12 (2024-01-10)
 ~~~~~~~~~~~~~~~~~~~
 

platformio/__init__.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-VERSION = (6, 1, 12)
+VERSION = (6, 1, "15a1")
 __version__ = ".".join([str(s) for s in VERSION])
 
 __title__ = "platformio"
@@ -38,15 +38,6 @@ __registry_mirror_hosts__ = [
 ]
 __pioremote_endpoint__ = "ssl:host=remote.platformio.org:port=4413"
 
-__core_packages__ = {
-    "contrib-piohome": "~3.4.2",
-    "contrib-pioremote": "~1.0.0",
-    "tool-scons": "~4.40600.0",
-    "tool-cppcheck": "~1.21100.0",
-    "tool-clangtidy": "~1.150005.0",
-    "tool-pvs-studio": "~7.18.0",
-}
-
 __check_internet_hosts__ = [
     "185.199.110.153",  # Github.com
     "88.198.170.159",  # platformio.org

platformio/__main__.py

@@ -66,15 +66,6 @@ def configure():
     if IS_CYGWIN:
         raise exception.CygwinEnvDetected()
 
-    # https://urllib3.readthedocs.org
-    # /en/latest/security.html#insecureplatformwarning
-    try:
-        import urllib3  # pylint: disable=import-outside-toplevel
-
-        urllib3.disable_warnings()
-    except (AttributeError, ImportError):
-        pass
-
     # Handle IOError issue with VSCode's Terminal (Windows)
     click_echo_origin = [click.echo, click.secho]
 

platformio/account/client.py

@@ -17,7 +17,7 @@ import time
 
 from platformio import __accounts_api__, app
 from platformio.exception import PlatformioException, UserSideException
-from platformio.http import HTTPClient, HTTPClientError
+from platformio.http import HttpApiClient, HttpClientApiError
 
 
 class AccountError(PlatformioException):
@@ -32,7 +32,7 @@ class AccountAlreadyAuthorized(AccountError, UserSideException):
     MESSAGE = "You are already authorized with {0} account."
 
 
-class AccountClient(HTTPClient):  # pylint:disable=too-many-public-methods
+class AccountClient(HttpApiClient):  # pylint:disable=too-many-public-methods
     SUMMARY_CACHE_TTL = 60 * 60 * 24 * 7
 
     def __init__(self):
@@ -60,7 +60,7 @@ class AccountClient(HTTPClient):  # pylint:disable=too-many-public-methods
     def fetch_json_data(self, *args, **kwargs):
         try:
             return super().fetch_json_data(*args, **kwargs)
-        except HTTPClientError as exc:
+        except HttpClientApiError as exc:
             raise AccountError(exc) from exc
 
     def fetch_authentication_token(self):

platformio/account/commands/destroy.py

@@ -19,18 +19,18 @@ from platformio.account.client import AccountClient, AccountNotAuthorized
 
 @click.command("destroy", short_help="Destroy account")
 def account_destroy_cmd():
-    client = AccountClient()
-    click.confirm(
-        "Are you sure you want to delete the %s user account?\n"
-        "Warning! All linked data will be permanently removed and can not be restored."
-        % client.get_logged_username(),
-        abort=True,
-    )
-    client.destroy_account()
-    try:
-        client.logout()
-    except AccountNotAuthorized:
-        pass
+    with AccountClient() as client:
+        click.confirm(
+            "Are you sure you want to delete the %s user account?\n"
+            "Warning! All linked data will be permanently removed and can not be restored."
+            % client.get_logged_username(),
+            abort=True,
+        )
+        client.destroy_account()
+        try:
+            client.logout()
+        except AccountNotAuthorized:
+            pass
     click.secho(
         "User account has been destroyed.",
         fg="green",

platformio/account/commands/forgot.py

@@ -20,8 +20,8 @@ from platformio.account.client import AccountClient
 @click.command("forgot", short_help="Forgot password")
 @click.option("--username", prompt="Username or email")
 def account_forgot_cmd(username):
-    client = AccountClient()
-    client.forgot_password(username)
+    with AccountClient() as client:
+        client.forgot_password(username)
     click.secho(
         "If this account is registered, we will send the "
         "further instructions to your email.",

platformio/account/commands/login.py

@@ -21,6 +21,6 @@ from platformio.account.client import AccountClient
 @click.option("-u", "--username", prompt="Username or email")
 @click.option("-p", "--password", prompt=True, hide_input=True)
 def account_login_cmd(username, password):
-    client = AccountClient()
-    client.login(username, password)
+    with AccountClient() as client:
+        client.login(username, password)
     click.secho("Successfully logged in!", fg="green")

platformio/account/commands/logout.py

@@ -19,6 +19,6 @@ from platformio.account.client import AccountClient
 
 @click.command("logout", short_help="Log out of PlatformIO Account")
 def account_logout_cmd():
-    client = AccountClient()
-    client.logout()
+    with AccountClient() as client:
+        client.logout()
     click.secho("Successfully logged out!", fg="green")

platformio/account/commands/password.py

@@ -21,6 +21,6 @@ from platformio.account.client import AccountClient
 @click.option("--old-password", prompt=True, hide_input=True)
 @click.option("--new-password", prompt=True, hide_input=True, confirmation_prompt=True)
 def account_password_cmd(old_password, new_password):
-    client = AccountClient()
-    client.change_password(old_password, new_password)
+    with AccountClient() as client:
+        client.change_password(old_password, new_password)
     click.secho("Password successfully changed!", fg="green")

platformio/account/commands/register.py

@@ -43,8 +43,8 @@ from platformio.account.validate import (
 @click.option("--firstname", prompt=True)
 @click.option("--lastname", prompt=True)
 def account_register_cmd(username, email, password, firstname, lastname):
-    client = AccountClient()
-    client.registration(username, email, password, firstname, lastname)
+    with AccountClient() as client:
+        client.registration(username, email, password, firstname, lastname)
     click.secho(
         "An account has been successfully created. "
         "Please check your mail to activate your account and verify your email address.",

platformio/account/commands/show.py

@@ -25,8 +25,8 @@ from platformio.account.client import AccountClient
 @click.option("--offline", is_flag=True)
 @click.option("--json-output", is_flag=True)
 def account_show_cmd(offline, json_output):
-    client = AccountClient()
-    info = client.get_account_info(offline)
+    with AccountClient() as client:
+        info = client.get_account_info(offline)
     if json_output:
         click.echo(json.dumps(info))
         return

platformio/account/commands/token.py

@@ -24,8 +24,8 @@ from platformio.account.client import AccountClient
 @click.option("--regenerate", is_flag=True)
 @click.option("--json-output", is_flag=True)
 def account_token_cmd(password, regenerate, json_output):
-    client = AccountClient()
-    auth_token = client.auth_token(password, regenerate)
+    with AccountClient() as client:
+        auth_token = client.auth_token(password, regenerate)
     if json_output:
         click.echo(json.dumps({"status": "success", "result": auth_token}))
         return

platformio/account/commands/update.py

@@ -25,8 +25,8 @@ from platformio.account.validate import validate_email, validate_username
 @click.option("--firstname")
 @click.option("--lastname")
 def account_update_cmd(current_password, **kwargs):
-    client = AccountClient()
-    profile = client.get_profile()
+    with AccountClient() as client:
+        profile = client.get_profile()
     new_profile = profile.copy()
     if not any(kwargs.values()):
         for field in profile:

platformio/account/org/commands/add.py

@@ -25,8 +25,8 @@ from platformio.account.client import AccountClient
     "username",
 )
 def org_add_cmd(orgname, username):
-    client = AccountClient()
-    client.add_org_owner(orgname, username)
+    with AccountClient() as client:
+        client.add_org_owner(orgname, username)
     return click.secho(
         "The new owner `%s` has been successfully added to the `%s` organization."
         % (username, orgname),

platformio/account/org/commands/create.py

@@ -30,8 +30,8 @@ from platformio.account.validate import validate_email, validate_orgname
     "--displayname",
 )
 def org_create_cmd(orgname, email, displayname):
-    client = AccountClient()
-    client.create_org(orgname, email, displayname)
+    with AccountClient() as client:
+        client.create_org(orgname, email, displayname)
     return click.secho(
         "The organization `%s` has been successfully created." % orgname,
         fg="green",

platformio/account/org/commands/destroy.py

@@ -20,14 +20,14 @@ from platformio.account.client import AccountClient
 @click.command("destroy", short_help="Destroy organization")
 @click.argument("orgname")
 def org_destroy_cmd(orgname):
-    client = AccountClient()
-    click.confirm(
-        "Are you sure you want to delete the `%s` organization account?\n"
-        "Warning! All linked data will be permanently removed and can not be restored."
-        % orgname,
-        abort=True,
-    )
-    client.destroy_org(orgname)
+    with AccountClient() as client:
+        click.confirm(
+            "Are you sure you want to delete the `%s` organization account?\n"
+            "Warning! All linked data will be permanently removed and can not be restored."
+            % orgname,
+            abort=True,
+        )
+        client.destroy_org(orgname)
     return click.secho(
         "Organization `%s` has been destroyed." % orgname,
         fg="green",

platformio/account/org/commands/list.py

@@ -23,8 +23,8 @@ from platformio.account.client import AccountClient
 @click.command("list", short_help="List organizations and their members")
 @click.option("--json-output", is_flag=True)
 def org_list_cmd(json_output):
-    client = AccountClient()
-    orgs = client.list_orgs()
+    with AccountClient() as client:
+        orgs = client.list_orgs()
     if json_output:
         return click.echo(json.dumps(orgs))
     if not orgs:

platformio/account/org/commands/remove.py

@@ -25,8 +25,8 @@ from platformio.account.client import AccountClient
     "username",
 )
 def org_remove_cmd(orgname, username):
-    client = AccountClient()
-    client.remove_org_owner(orgname, username)
+    with AccountClient() as client:
+        client.remove_org_owner(orgname, username)
     return click.secho(
         "The `%s` owner has been successfully removed from the `%s` organization."
         % (username, orgname),

platformio/account/org/commands/update.py

@@ -31,8 +31,8 @@ from platformio.account.validate import validate_email, validate_orgname
 )
 @click.option("--displayname")
 def org_update_cmd(cur_orgname, **kwargs):
-    client = AccountClient()
-    org = client.get_org(cur_orgname)
+    with AccountClient() as client:
+        org = client.get_org(cur_orgname)
     new_org = {
         key: value if value is not None else org[key] for key, value in kwargs.items()
     }

platformio/account/team/commands/add.py

@@ -29,8 +29,8 @@ from platformio.account.validate import validate_orgname_teamname
 )
 def team_add_cmd(orgname_teamname, username):
     orgname, teamname = orgname_teamname.split(":", 1)
-    client = AccountClient()
-    client.add_team_member(orgname, teamname, username)
+    with AccountClient() as client:
+        client.add_team_member(orgname, teamname, username)
     return click.secho(
         "The new member %s has been successfully added to the %s team."
         % (username, teamname),

platformio/account/team/commands/create.py

@@ -29,8 +29,8 @@ from platformio.account.validate import validate_orgname_teamname
 )
 def team_create_cmd(orgname_teamname, description):
     orgname, teamname = orgname_teamname.split(":", 1)
-    client = AccountClient()
-    client.create_team(orgname, teamname, description)
+    with AccountClient() as client:
+        client.create_team(orgname, teamname, description)
     return click.secho(
         "The team %s has been successfully created." % teamname,
         fg="green",

platformio/account/team/commands/destroy.py

@@ -32,8 +32,8 @@ def team_destroy_cmd(orgname_teamname):
         ),
         abort=True,
     )
-    client = AccountClient()
-    client.destroy_team(orgname, teamname)
+    with AccountClient() as client:
+        client.destroy_team(orgname, teamname)
     return click.secho(
         "The team %s has been successfully destroyed." % teamname,
         fg="green",

platformio/account/team/commands/list.py

@@ -24,19 +24,22 @@ from platformio.account.client import AccountClient
 @click.argument("orgname", required=False)
 @click.option("--json-output", is_flag=True)
 def team_list_cmd(orgname, json_output):
-    client = AccountClient()
-    data = {}
-    if not orgname:
-        for item in client.list_orgs():
-            teams = client.list_teams(item.get("orgname"))
-            data[item.get("orgname")] = teams
-    else:
-        teams = client.list_teams(orgname)
-        data[orgname] = teams
+    with AccountClient() as client:
+        data = {}
+        if not orgname:
+            for item in client.list_orgs():
+                teams = client.list_teams(item.get("orgname"))
+                data[item.get("orgname")] = teams
+        else:
+            teams = client.list_teams(orgname)
+            data[orgname] = teams
+
     if json_output:
         return click.echo(json.dumps(data[orgname] if orgname else data))
+
     if not any(data.values()):
         return click.secho("You do not have any teams.", fg="yellow")
+
     for org_name, teams in data.items():
         for team in teams:
             click.echo()
@@ -48,11 +51,13 @@ def team_list_cmd(orgname, json_output):
             table_data.append(
                 (
                     "Members:",
-                    ", ".join(
-                        (member.get("username") for member in team.get("members"))
-                    )
-                    if team.get("members")
-                    else "-",
+                    (
+                        ", ".join(
+                            (member.get("username") for member in team.get("members"))
+                        )
+                        if team.get("members")
+                        else "-"
+                    ),
                 )
             )
             click.echo(tabulate(table_data, tablefmt="plain"))

platformio/account/team/commands/remove.py

@@ -27,8 +27,8 @@ from platformio.account.validate import validate_orgname_teamname
 @click.argument("username")
 def team_remove_cmd(orgname_teamname, username):
     orgname, teamname = orgname_teamname.split(":", 1)
-    client = AccountClient()
-    client.remove_team_member(orgname, teamname, username)
+    with AccountClient() as client:
+        client.remove_team_member(orgname, teamname, username)
     return click.secho(
         "The %s member has been successfully removed from the %s team."
         % (username, teamname),

platformio/account/team/commands/update.py

@@ -34,8 +34,8 @@ from platformio.account.validate import validate_orgname_teamname, validate_team
 )
 def team_update_cmd(orgname_teamname, **kwargs):
     orgname, teamname = orgname_teamname.split(":", 1)
-    client = AccountClient()
-    team = client.get_team(orgname, teamname)
+    with AccountClient() as client:
+        team = client.get_team(orgname, teamname)
     new_team = {
         key: value if value is not None else team[key] for key, value in kwargs.items()
     }

platformio/app.py

@@ -258,10 +258,6 @@ def get_cid():
     return cid
 
 
-def get_project_id(project_dir):
-    return hashlib.sha1(hashlib_encode_data(project_dir)).hexdigest()
-
-
 def get_user_agent():
     data = [
         "PlatformIO/%s" % __version__,

platformio/assets/system/99-platformio-udev.rules

@@ -36,6 +36,8 @@ ATTRS{idVendor}=="067b", ATTRS{idProduct}=="2303", MODE:="0666", ENV{ID_MM_DEVIC
 
 # QinHeng Electronics HL-340 USB-Serial adapter
 ATTRS{idVendor}=="1a86", ATTRS{idProduct}=="7523", MODE:="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
+# QinHeng Electronics CH343 USB-Serial adapter
+ATTRS{idVendor}=="1a86", ATTRS{idProduct}=="55d3", MODE:="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
 # QinHeng Electronics CH9102 USB-Serial adapter
 ATTRS{idVendor}=="1a86", ATTRS{idProduct}=="55d4", MODE:="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
 
@@ -85,6 +87,8 @@ ATTRS{idVendor}=="2e8a", ATTRS{idProduct}=="[01]*", MODE:="0666", ENV{ID_MM_DEVI
 # AIR32F103
 ATTRS{idVendor}=="0d28", ATTRS{idProduct}=="0204", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
 
+# STM32 virtual COM port
+ATTRS{idVendor}=="0483", ATTRS{idProduct}=="5740", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
 
 #
 # Debuggers
@@ -173,4 +177,4 @@ ATTRS{product}=="*CMSIS-DAP*", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID
 ATTRS{idVendor}=="03eb", ATTRS{idProduct}=="2107", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
 
 # Espressif USB JTAG/serial debug unit
-ATTRS{idVendor}=="303a", ATTR{idProduct}=="1001", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
+ATTRS{idVendor}=="303a", ATTRS{idProduct}=="1001", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"

platformio/builder/main.py

@@ -15,7 +15,7 @@
 import json
 import os
 import sys
-from time import time
+import time
 
 import click
 from SCons.Script import ARGUMENTS  # pylint: disable=import-error
@@ -31,7 +31,7 @@ from SCons.Script import Variables  # pylint: disable=import-error
 from platformio import app, fs
 from platformio.platform.base import PlatformBase
 from platformio.proc import get_pythonexe_path
-from platformio.project.helpers import get_project_dir
+from platformio.project.helpers import get_build_type, get_project_dir
 
 AllowSubstExceptions(NameError)
 
@@ -61,7 +61,7 @@ DEFAULT_ENV_OPTIONS = dict(
         "piotarget",
         "piolib",
         "pioupload",
-        "piosize",
+        "piomemusage",
         "pioino",
         "piomisc",
         "piointegration",
@@ -71,15 +71,7 @@ DEFAULT_ENV_OPTIONS = dict(
     variables=clivars,
     # Propagating External Environment
     ENV=os.environ,
-    UNIX_TIME=int(time()),
-    BUILD_DIR=os.path.join("$PROJECT_BUILD_DIR", "$PIOENV"),
-    BUILD_SRC_DIR=os.path.join("$BUILD_DIR", "src"),
-    BUILD_TEST_DIR=os.path.join("$BUILD_DIR", "test"),
-    COMPILATIONDB_PATH=os.path.join("$PROJECT_DIR", "compile_commands.json"),
-    LIBPATH=["$BUILD_DIR"],
-    PROGNAME="program",
-    PROGPATH=os.path.join("$BUILD_DIR", "$PROGNAME$PROGSUFFIX"),
-    PROG_PATH="$PROGPATH",  # deprecated
+    UNIX_TIME=int(time.time()),
     PYTHONEXE=get_pythonexe_path(),
 )
 
@@ -126,13 +118,21 @@ env.Replace(
     PROJECT_DATA_DIR=config.get("platformio", "data_dir"),
     PROJECTDATA_DIR="$PROJECT_DATA_DIR",  # legacy for dev/platform
     PROJECT_BUILD_DIR=config.get("platformio", "build_dir"),
-    BUILD_TYPE=env.GetBuildType(),
+    BUILD_TYPE=get_build_type(config, env["PIOENV"], COMMAND_LINE_TARGETS),
+    BUILD_DIR=os.path.join("$PROJECT_BUILD_DIR", "$PIOENV", "$BUILD_TYPE"),
+    BUILD_SRC_DIR=os.path.join("$BUILD_DIR", "src"),
+    BUILD_TEST_DIR=os.path.join("$BUILD_DIR", "test"),
     BUILD_CACHE_DIR=config.get("platformio", "build_cache_dir"),
+    LIBPATH=["$BUILD_DIR"],
     LIBSOURCE_DIRS=[
         config.get("platformio", "lib_dir"),
         os.path.join("$PROJECT_LIBDEPS_DIR", "$PIOENV"),
         config.get("platformio", "globallib_dir"),
     ],
+    COMPILATIONDB_PATH=os.path.join("$PROJECT_DIR", "compile_commands.json"),
+    PROGNAME="program",
+    PROGPATH=os.path.join("$BUILD_DIR", "$PROGNAME$PROGSUFFIX"),
+    PROG_PATH="$PROGPATH",  # deprecated
 )
 
 if int(ARGUMENTS.get("ISATTY", 0)):
@@ -183,7 +183,7 @@ env.SConscript(env.GetExtraScripts("post"), exports="env")
 
 # Checking program size
 if env.get("SIZETOOL") and not (
-    set(["nobuild", "sizedata"]) & set(COMMAND_LINE_TARGETS)
+    set(["nobuild", "__memusage"]) & set(COMMAND_LINE_TARGETS)
 ):
     env.Depends("upload", "checkprogsize")
     # Replace platform's "size" target with our
@@ -224,24 +224,27 @@ if env.IsIntegrationDump():
     data = projenv.DumpIntegrationData(env)
     # dump to file for the further reading by project.helpers.load_build_metadata
     with open(
-        projenv.subst(os.path.join("$BUILD_DIR", "idedata.json")),
+        projenv.subst(os.path.join("$BUILD_DIR", "metadata.json")),
         mode="w",
         encoding="utf8",
     ) as fp:
         json.dump(data, fp)
-    click.echo("\n%s\n" % json.dumps(data))  # pylint: disable=undefined-variable
+    click.echo(
+        "Metadata has been saved to the following location: %s"
+        % projenv.subst(os.path.join("$BUILD_DIR", "metadata.json"))
+    )
     env.Exit(0)
 
-if "sizedata" in COMMAND_LINE_TARGETS:
+if "__memusage" in COMMAND_LINE_TARGETS:
     AlwaysBuild(
         env.Alias(
-            "sizedata",
+            "__memusage",
             DEFAULT_TARGETS,
-            env.VerboseAction(env.DumpSizeData, "Generating memory usage report..."),
+            env.VerboseAction(env.DumpMemoryUsage, "Generating memory usage report..."),
         )
     )
 
-    Default("sizedata")
+    Default("__memusage")
 
 # issue #4604: process targets sequentially
 for index, target in enumerate(

platformio/builder/tools/piobuild.py

@@ -54,7 +54,7 @@ def GetBuildType(env):
         modes.append("debug")
     if "__test" in COMMAND_LINE_TARGETS or env.GetProjectOption("build_type") == "test":
         modes.append("test")
-    return "+".join(modes or ["release"])
+    return ", ".join(modes or ["release"])
 
 
 def BuildProgram(env):

platformio/builder/tools/piointegration.py

@@ -23,7 +23,7 @@ from platformio.proc import exec_command, where_is_program
 
 
 def IsIntegrationDump(_):
-    return set(["__idedata", "idedata"]) & set(COMMAND_LINE_TARGETS)
+    return set(["__idedata", "__metadata"]) & set(COMMAND_LINE_TARGETS)
 
 
 def DumpIntegrationIncludes(env):
@@ -141,7 +141,7 @@ def _split_flags_string(env, s):
 def DumpIntegrationData(*args):
     projenv, globalenv = args[0:2]  # pylint: disable=unbalanced-tuple-unpacking
     data = {
-        "build_type": globalenv.GetBuildType(),
+        "build_type": globalenv["BUILD_TYPE"],
         "env_name": globalenv["PIOENV"],
         "libsource_dirs": [
             globalenv.subst(item) for item in globalenv.GetLibSourceDirs()

platformio/builder/tools/piolib.py

@@ -29,7 +29,7 @@ from SCons.Script import DefaultEnvironment  # pylint: disable=import-error
 from platformio import exception, fs
 from platformio.builder.tools import piobuild
 from platformio.compat import IS_WINDOWS, hashlib_encode_data, string_types
-from platformio.http import HTTPClientError, InternetConnectionError
+from platformio.http import HttpClientApiError, InternetConnectionError
 from platformio.package.exception import (
     MissingPackageManifestError,
     UnknownPackageError,
@@ -39,7 +39,7 @@ from platformio.package.manifest.parser import (
     ManifestParserError,
     ManifestParserFactory,
 )
-from platformio.package.meta import PackageCompatibility, PackageItem
+from platformio.package.meta import PackageCompatibility, PackageItem, PackageSpec
 from platformio.project.options import ProjectOptions
 
 
@@ -309,10 +309,10 @@ class LibBuilderBase:
         if not self.dependencies or self._deps_are_processed:
             return
         self._deps_are_processed = True
-        for item in self.dependencies:
+        for dependency in self.dependencies:
             found = False
             for lb in self.env.GetLibBuilders():
-                if item["name"] != lb.name:
+                if not lb.is_dependency_compatible(dependency):
                     continue
                 found = True
                 if lb not in self.depbuilders:
@@ -322,9 +322,28 @@ class LibBuilderBase:
             if not found and self.verbose:
                 sys.stderr.write(
                     "Warning: Ignored `%s` dependency for `%s` "
-                    "library\n" % (item["name"], self.name)
+                    "library\n" % (dependency["name"], self.name)
                 )
 
+    def is_dependency_compatible(self, dependency):
+        pkg = PackageItem(self.path)
+        qualifiers = {"name": self.name, "version": self.version}
+        if pkg.metadata:
+            qualifiers = {"name": pkg.metadata.name, "version": pkg.metadata.version}
+            if pkg.metadata.spec and pkg.metadata.spec.owner:
+                qualifiers["owner"] = pkg.metadata.spec.owner
+        dep_qualifiers = {
+            k: v for k, v in dependency.items() if k in ("owner", "name", "version")
+        }
+        if (
+            "version" in dep_qualifiers
+            and not PackageSpec(dep_qualifiers["version"]).requirements
+        ):
+            del dep_qualifiers["version"]
+        return PackageCompatibility.from_dependency(dep_qualifiers).is_compatible(
+            PackageCompatibility(**qualifiers)
+        )
+
     def get_search_files(self):
         return [
             os.path.join(self.src_dir, item)
@@ -986,7 +1005,7 @@ class ProjectAsLibBuilder(LibBuilderBase):
                 lm.install(spec)
                 did_install = True
             except (
-                HTTPClientError,
+                HttpClientApiError,
                 UnknownPackageError,
                 InternetConnectionError,
             ) as exc:

platformio/builder/tools/piomaxlen.py

@@ -23,10 +23,10 @@ from SCons.Subst import quote_spaces  # pylint: disable=import-error
 from platformio.compat import IS_WINDOWS, hashlib_encode_data
 
 # There are the next limits depending on a platform:
-# - Windows = 8192
+# - Windows = 8191
 # - Unix    = 131072
 # We need ~512 characters for compiler and temporary file paths
-MAX_LINE_LENGTH = (8192 if IS_WINDOWS else 131072) - 512
+MAX_LINE_LENGTH = (8191 if IS_WINDOWS else 131072) - 512
 
 WINPATHSEP_RE = re.compile(r"\\([^\"'\\]|$)")
 

platformio/builder/tools/piomemusage.py

@@ -14,33 +14,33 @@
 
 # pylint: disable=too-many-locals
 
-import json
+import os
 import sys
-from os import environ, makedirs, remove
-from os.path import isdir, join, splitdrive
+import time
 
 from elftools.elf.descriptions import describe_sh_flags
 from elftools.elf.elffile import ELFFile
 
 from platformio.compat import IS_WINDOWS
 from platformio.proc import exec_command
+from platformio.project.memusage import save_report
 
 
 def _run_tool(cmd, env, tool_args):
-    sysenv = environ.copy()
+    sysenv = os.environ.copy()
     sysenv["PATH"] = str(env["ENV"]["PATH"])
 
     build_dir = env.subst("$BUILD_DIR")
-    if not isdir(build_dir):
-        makedirs(build_dir)
-    tmp_file = join(build_dir, "size-data-longcmd.txt")
+    if not os.path.isdir(build_dir):
+        os.makedirs(build_dir)
+    tmp_file = os.path.join(build_dir, "size-data-longcmd.txt")
 
     with open(tmp_file, mode="w", encoding="utf8") as fp:
         fp.write("\n".join(tool_args))
 
     cmd.append("@" + tmp_file)
     result = exec_command(cmd, env=sysenv)
-    remove(tmp_file)
+    os.remove(tmp_file)
 
     return result
 
@@ -92,8 +92,8 @@ def _collect_sections_info(env, elffile):
         }
 
         sections[section.name] = section_data
-        sections[section.name]["in_flash"] = env.pioSizeIsFlashSection(section_data)
-        sections[section.name]["in_ram"] = env.pioSizeIsRamSection(section_data)
+        sections[section.name]["in_flash"] = env.memusageIsFlashSection(section_data)
+        sections[section.name]["in_ram"] = env.memusageIsRamSection(section_data)
 
     return sections
 
@@ -106,7 +106,7 @@ def _collect_symbols_info(env, elffile, elf_path, sections):
         sys.stderr.write("Couldn't find symbol table. Is ELF file stripped?")
         env.Exit(1)
 
-    sysenv = environ.copy()
+    sysenv = os.environ.copy()
     sysenv["PATH"] = str(env["ENV"]["PATH"])
 
     symbol_addrs = []
@@ -117,7 +117,7 @@ def _collect_symbols_info(env, elffile, elf_path, sections):
         symbol_size = s["st_size"]
         symbol_type = symbol_info["type"]
 
-        if not env.pioSizeIsValidSymbol(s.name, symbol_type, symbol_addr):
+        if not env.memusageIsValidSymbol(s.name, symbol_type, symbol_addr):
             continue
 
         symbol = {
@@ -126,7 +126,7 @@ def _collect_symbols_info(env, elffile, elf_path, sections):
             "name": s.name,
             "type": symbol_type,
             "size": symbol_size,
-            "section": env.pioSizeDetermineSection(sections, symbol_addr),
+            "section": env.memusageDetermineSection(sections, symbol_addr),
         }
 
         if s.name.startswith("_Z"):
@@ -144,8 +144,8 @@ def _collect_symbols_info(env, elffile, elf_path, sections):
         if not location or "?" in location:
             continue
         if IS_WINDOWS:
-            drive, tail = splitdrive(location)
-            location = join(drive.upper(), tail)
+            drive, tail = os.path.splitdrive(location)
+            location = os.path.join(drive.upper(), tail)
         symbol["file"] = location
         symbol["line"] = 0
         if ":" in location:
@@ -156,7 +156,7 @@ def _collect_symbols_info(env, elffile, elf_path, sections):
     return symbols
 
 
-def pioSizeDetermineSection(_, sections, symbol_addr):
+def memusageDetermineSection(_, sections, symbol_addr):
     for section, info in sections.items():
         if not info.get("in_flash", False) and not info.get("in_ram", False):
             continue
@@ -165,22 +165,22 @@ def pioSizeDetermineSection(_, sections, symbol_addr):
     return "unknown"
 
 
-def pioSizeIsValidSymbol(_, symbol_name, symbol_type, symbol_address):
+def memusageIsValidSymbol(_, symbol_name, symbol_type, symbol_address):
     return symbol_name and symbol_address != 0 and symbol_type != "STT_NOTYPE"
 
 
-def pioSizeIsRamSection(_, section):
+def memusageIsRamSection(_, section):
     return (
         section.get("type", "") in ("SHT_NOBITS", "SHT_PROGBITS")
         and section.get("flags", "") == "WA"
     )
 
 
-def pioSizeIsFlashSection(_, section):
+def memusageIsFlashSection(_, section):
     return section.get("type", "") == "SHT_PROGBITS" and "A" in section.get("flags", "")
 
 
-def pioSizeCalculateFirmwareSize(_, sections):
+def memusageCalculateFirmwareSize(_, sections):
     flash_size = ram_size = 0
     for section_info in sections.values():
         if section_info.get("in_flash", False):
@@ -191,20 +191,22 @@ def pioSizeCalculateFirmwareSize(_, sections):
     return ram_size, flash_size
 
 
-def DumpSizeData(_, target, source, env):  # pylint: disable=unused-argument
-    data = {"device": {}, "memory": {}, "version": 1}
+def DumpMemoryUsage(_, target, source, env):  # pylint: disable=unused-argument
+    result = {"version": 1, "timestamp": int(time.time()), "device": {}, "memory": {}}
 
     board = env.BoardConfig()
     if board:
-        data["device"] = {
+        result["device"] = {
             "mcu": board.get("build.mcu", ""),
             "cpu": board.get("build.cpu", ""),
             "frequency": board.get("build.f_cpu"),
             "flash": int(board.get("upload.maximum_size", 0)),
             "ram": int(board.get("upload.maximum_ram_size", 0)),
         }
-        if data["device"]["frequency"] and data["device"]["frequency"].endswith("L"):
-            data["device"]["frequency"] = int(data["device"]["frequency"][0:-1])
+        if result["device"]["frequency"] and result["device"]["frequency"].endswith(
+            "L"
+        ):
+            result["device"]["frequency"] = int(result["device"]["frequency"][0:-1])
 
     elf_path = env.subst("$PIOMAINPROG")
 
@@ -216,16 +218,16 @@ def DumpSizeData(_, target, source, env):  # pylint: disable=unused-argument
             env.Exit(1)
 
         sections = _collect_sections_info(env, elffile)
-        firmware_ram, firmware_flash = env.pioSizeCalculateFirmwareSize(sections)
-        data["memory"]["total"] = {
+        firmware_ram, firmware_flash = env.memusageCalculateFirmwareSize(sections)
+        result["memory"]["total"] = {
             "ram_size": firmware_ram,
             "flash_size": firmware_flash,
-            "sections": sections,
         }
+        result["memory"]["sections"] = sections
 
         files = {}
         for symbol in _collect_symbols_info(env, elffile, elf_path, sections):
-            file_path = symbol.get("file") or "unknown"
+            file_path = symbol.pop("file", "unknown")
             if not files.get(file_path, {}):
                 files[file_path] = {"symbols": [], "ram_size": 0, "flash_size": 0}
 
@@ -240,16 +242,16 @@ def DumpSizeData(_, target, source, env):  # pylint: disable=unused-argument
 
             files[file_path]["symbols"].append(symbol)
 
-        data["memory"]["files"] = []
+        result["memory"]["files"] = []
         for k, v in files.items():
             file_data = {"path": k}
             file_data.update(v)
-            data["memory"]["files"].append(file_data)
+            result["memory"]["files"].append(file_data)
 
-    with open(
-        join(env.subst("$BUILD_DIR"), "sizedata.json"), mode="w", encoding="utf8"
-    ) as fp:
-        fp.write(json.dumps(data))
+    print(
+        "Memory usage report has been saved to the following location: "
+        f"\"{save_report(os.getcwd(), env['PIOENV'], result)}\""
+    )
 
 
 def exists(_):
@@ -257,10 +259,10 @@ def exists(_):
 
 
 def generate(env):
-    env.AddMethod(pioSizeIsRamSection)
-    env.AddMethod(pioSizeIsFlashSection)
-    env.AddMethod(pioSizeCalculateFirmwareSize)
-    env.AddMethod(pioSizeDetermineSection)
-    env.AddMethod(pioSizeIsValidSymbol)
-    env.AddMethod(DumpSizeData)
+    env.AddMethod(memusageIsRamSection)
+    env.AddMethod(memusageIsFlashSection)
+    env.AddMethod(memusageCalculateFirmwareSize)
+    env.AddMethod(memusageDetermineSection)
+    env.AddMethod(memusageIsValidSymbol)
+    env.AddMethod(DumpMemoryUsage)
     return env

platformio/builder/tools/pioplatform.py

@@ -75,9 +75,11 @@ def LoadPioPlatform(env):
             continue
         env.PrependENVPath(
             "PATH",
-            os.path.join(pkg.path, "bin")
-            if os.path.isdir(os.path.join(pkg.path, "bin"))
-            else pkg.path,
+            (
+                os.path.join(pkg.path, "bin")
+                if os.path.isdir(os.path.join(pkg.path, "bin"))
+                else pkg.path
+            ),
         )
         if (
             not IS_WINDOWS

platformio/check/cli.py

@@ -103,10 +103,21 @@ def cli(
                     "%s: %s" % (k, ", ".join(v) if isinstance(v, list) else v)
                 )
 
-            default_src_filters = [
-                "+<%s>" % os.path.basename(config.get("platformio", "src_dir")),
-                "+<%s>" % os.path.basename(config.get("platformio", "include_dir")),
-            ]
+            default_src_filters = []
+            for d in (
+                config.get("platformio", "src_dir"),
+                config.get("platformio", "include_dir"),
+            ):
+                try:
+                    default_src_filters.append("+<%s>" % os.path.relpath(d))
+                except ValueError as exc:
+                    # On Windows if sources are located on a different logical drive
+                    if not json_output and not silent:
+                        click.echo(
+                            "Error: Project cannot be analyzed! The project folder `%s`"
+                            " is located on a different logical drive\n" % d
+                        )
+                    raise exception.ReturnErrorCode(1) from exc
 
             env_src_filters = (
                 src_filters
@@ -122,9 +133,11 @@ def cli(
                 silent=silent,
                 src_filters=env_src_filters,
                 flags=flags or env_options.get("check_flags"),
-                severity=[DefectItem.SEVERITY_LABELS[DefectItem.SEVERITY_HIGH]]
-                if silent
-                else severity or config.get("env:" + envname, "check_severity"),
+                severity=(
+                    [DefectItem.SEVERITY_LABELS[DefectItem.SEVERITY_HIGH]]
+                    if silent
+                    else severity or config.get("env:" + envname, "check_severity")
+                ),
                 skip_packages=skip_packages or env_options.get("check_skip_packages"),
                 platform_packages=env_options.get("platform_packages"),
             )
@@ -142,9 +155,11 @@ def cli(
 
                 result = {"env": envname, "tool": tool, "duration": time()}
                 rc = ct.check(
-                    on_defect_callback=None
-                    if (json_output or verbose)
-                    else lambda defect: click.echo(repr(defect))
+                    on_defect_callback=(
+                        None
+                        if (json_output or verbose)
+                        else lambda defect: click.echo(repr(defect))
+                    )
                 )
 
                 result["defects"] = ct.get_defects()

platformio/commands/upgrade.py

@@ -19,9 +19,9 @@ import subprocess
 import click
 
 from platformio import VERSION, __version__, app, exception
-from platformio.http import fetch_remote_content
+from platformio.dependencies import get_pip_dependencies
+from platformio.http import fetch_http_content
 from platformio.package.manager.core import update_core_packages
-from platformio.pipdeps import get_pip_dependencies
 from platformio.proc import get_pythonexe_path
 
 PYPI_JSON_URL = "https://pypi.org/pypi/platformio/json"
@@ -133,7 +133,7 @@ def get_latest_version():
 
 def get_develop_latest_version():
     version = None
-    content = fetch_remote_content(DEVELOP_INIT_SCRIPT_URL)
+    content = fetch_http_content(DEVELOP_INIT_SCRIPT_URL)
     for line in content.split("\n"):
         line = line.strip()
         if not line.startswith("VERSION"):
@@ -150,5 +150,5 @@ def get_develop_latest_version():
 
 
 def get_pypi_latest_version():
-    content = fetch_remote_content(PYPI_JSON_URL)
+    content = fetch_http_content(PYPI_JSON_URL)
     return json.loads(content)["info"]["version"]

platformio/compat.py

@@ -17,6 +17,7 @@
 import importlib.util
 import inspect
 import locale
+import os
 import shlex
 import sys
 
@@ -41,10 +42,14 @@ else:
 if sys.version_info >= (3, 9):
     from asyncio import to_thread as aio_to_thread
 else:
-    from starlette.concurrency import run_in_threadpool as aio_to_thread
+    try:
+        from starlette.concurrency import run_in_threadpool as aio_to_thread
+    except ImportError:
+        pass
 
 
 PY2 = sys.version_info[0] == 2  # DO NOT REMOVE IT. ESP8266/ESP32 depend on it
+PY36 = sys.version_info[0:2] == (3, 6)
 IS_CYGWIN = sys.platform.startswith("cygwin")
 IS_WINDOWS = WINDOWS = sys.platform.startswith("win")
 IS_MACOS = sys.platform.startswith("darwin")
@@ -132,3 +137,12 @@ def path_to_unicode(path):
     and custom device monitor filters
     """
     return path
+
+
+def is_proxy_set(socks=False):
+    for var in ("HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY"):
+        value = os.getenv(var, os.getenv(var.lower()))
+        if not value or (socks and not value.startswith("socks5://")):
+            continue
+        return True
+    return False

platformio/debug/config/base.py

@@ -148,7 +148,9 @@ class DebugConfigBase:  # pylint: disable=too-many-instance-attributes
         )
 
     def _load_build_data(self):
-        data = load_build_metadata(os.getcwd(), self.env_name, cache=True, debug=True)
+        data = load_build_metadata(
+            os.getcwd(), self.env_name, cache=True, force_targets=["__debug"]
+        )
         if not data:
             raise DebugInvalidOptionsError("Could not load a build configuration")
         return data
@@ -194,9 +196,11 @@ class DebugConfigBase:  # pylint: disable=too-many-instance-attributes
                     cwd=server_package_dir if server_package else None,
                     executable=result.get("executable"),
                     arguments=[
-                        a.replace("$PACKAGE_DIR", server_package_dir)
-                        if server_package_dir
-                        else a
+                        (
+                            a.replace("$PACKAGE_DIR", server_package_dir)
+                            if server_package_dir
+                            else a
+                        )
                         for a in result.get("arguments", [])
                     ],
                 )

platformio/dependencies.py

@@ -12,11 +12,20 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import os
 import platform
-import sys
 
-PY36 = sys.version_info[0:2] == (3, 6)
+from platformio.compat import PY36, is_proxy_set
+
+
+def get_core_dependencies():
+    return {
+        "contrib-piohome": "~3.4.2",
+        "contrib-pioremote": "~1.0.0",
+        "tool-scons": "~4.40700.0",
+        "tool-cppcheck": "~1.21100.0",
+        "tool-clangtidy": "~1.150005.0",
+        "tool-pvs-studio": "~7.18.0",
+    }
 
 
 def get_pip_dependencies():
@@ -24,10 +33,10 @@ def get_pip_dependencies():
         "bottle == 0.12.*",
         "click >=8.0.4, <9",
         "colorama",
+        "httpx%s >=0.22.0, <0.28" % ("[socks]" if is_proxy_set(socks=True) else ""),
         "marshmallow == 3.*",
-        "pyelftools == 0.30",
+        "pyelftools >=0.27, <1",
         "pyserial == 3.5.*",  # keep in sync "device/monitor/terminal.py"
-        "requests%s == 2.*" % ("[socks]" if is_proxy_set(socks=True) else ""),
         "semantic_version == 2.10.*",
         "tabulate == 0.*",
     ]
@@ -35,8 +44,8 @@ def get_pip_dependencies():
     home = [
         # PIO Home requirements
         "ajsonrpc == 1.2.*",
-        "starlette >=0.19, <0.35",
-        "uvicorn %s" % ("== 0.16.0" if PY36 else ">=0.16, <0.26"),
+        "starlette >=0.19, <0.38",
+        "uvicorn %s" % ("== 0.16.0" if PY36 else ">=0.16, <0.30"),
         "wsproto == 1.*",
     ]
 
@@ -60,12 +69,3 @@ def get_pip_dependencies():
         pass
 
     return core + home + extra
-
-
-def is_proxy_set(socks=False):
-    for var in ("HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY"):
-        value = os.getenv(var, os.getenv(var.lower()))
-        if not value or (socks and not value.startswith("socks5://")):
-            continue
-        return True
-    return False

platformio/device/list/util.py

@@ -144,9 +144,9 @@ def list_mdns_services():
             if service.properties:
                 try:
                     properties = {
-                        k.decode("utf8"): v.decode("utf8")
-                        if isinstance(v, bytes)
-                        else v
+                        k.decode("utf8"): (
+                            v.decode("utf8") if isinstance(v, bytes) else v
+                        )
                         for k, v in service.properties.items()
                     }
                     json.dumps(properties)

platformio/device/monitor/command.py

@@ -58,7 +58,7 @@ from platformio.project.options import ProjectOptions
     "--encoding",
     help=(
         "Set the encoding for the serial port "
-        "(e.g. hexlify, Latin1, UTF-8) [default=%s]"
+        "(e.g. hexlify, Latin-1, UTF-8) [default=%s]"
         % ProjectOptions["env.monitor_encoding"].default
     ),
 )
@@ -125,9 +125,11 @@ def device_monitor_cmd(**options):
         options = apply_project_monitor_options(options, project_options)
         register_filters(platform=platform, options=options)
         options["port"] = SerialPortFinder(
-            board_config=platform.board_config(project_options.get("board"))
-            if platform and project_options.get("board")
-            else None,
+            board_config=(
+                platform.board_config(project_options.get("board"))
+                if platform and project_options.get("board")
+                else None
+            ),
             upload_protocol=project_options.get("upload_protocol"),
             ensure_ready=True,
         ).find(initial_port=options["port"])

platformio/device/monitor/filters/base.py

@@ -25,11 +25,12 @@ from platformio.project.config import ProjectConfig
 class DeviceMonitorFilterBase(miniterm.Transform):
     def __init__(self, options=None):
         """Called by PlatformIO to pass context"""
-        miniterm.Transform.__init__(self)
+        super().__init__()
 
         self.options = options or {}
         self.project_dir = self.options.get("project_dir")
         self.environment = self.options.get("environment")
+        self._running_terminal = None
 
         self.config = ProjectConfig.get_instance()
         if not self.environment:
@@ -47,6 +48,12 @@ class DeviceMonitorFilterBase(miniterm.Transform):
     def NAME(self):
         raise NotImplementedError("Please declare NAME attribute for the filter class")
 
+    def set_running_terminal(self, terminal):
+        self._running_terminal = terminal
+
+    def get_running_terminal(self):
+        return self._running_terminal
+
 
 def register_filters(platform=None, options=None):
     # project filters

platformio/device/monitor/filters/hexlify.py

@@ -24,12 +24,18 @@ class Hexlify(DeviceMonitorFilterBase):
         super().__init__(*args, **kwargs)
         self._counter = 0
 
+    def set_running_terminal(self, terminal):
+        # force to Latin-1, issue #4732
+        if terminal.input_encoding == "UTF-8":
+            terminal.set_rx_encoding("Latin-1")
+        super().set_running_terminal(terminal)
+
     def rx(self, text):
         result = ""
-        for b in serial.iterbytes(text):
+        for c in serial.iterbytes(text):
             if (self._counter % 16) == 0:
                 result += "\n{:04X} | ".format(self._counter)
-            asciicode = ord(b)
+            asciicode = ord(c)
             if asciicode <= 255:
                 result += "{:02X} ".format(asciicode)
             else:

platformio/device/monitor/terminal.py

@@ -110,6 +110,12 @@ def new_terminal(options):
     term.raw = options["raw"]
     term.set_rx_encoding(options["encoding"])
     term.set_tx_encoding(options["encoding"])
+    for ts in (term.tx_transformations, term.rx_transformations):
+        for t in ts:
+            try:
+                t.set_running_terminal(term)
+            except AttributeError:
+                pass
     return term
 
 

platformio/fs.py

@@ -29,15 +29,16 @@ from platformio.compat import IS_WINDOWS
 
 
 class cd:
-    def __init__(self, new_path):
-        self.new_path = new_path
-        self.prev_path = os.getcwd()
+    def __init__(self, path):
+        self.path = path
+        self._old_cwd = []
 
     def __enter__(self):
-        os.chdir(self.new_path)
+        self._old_cwd.append(os.getcwd())
+        os.chdir(self.path)
 
-    def __exit__(self, etype, value, traceback):
-        os.chdir(self.prev_path)
+    def __exit__(self, *excinfo):
+        os.chdir(self._old_cwd.pop())
 
 
 def get_source_dir():

platformio/home/rpc/handlers/account.py

@@ -12,19 +12,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from ajsonrpc.core import JSONRPC20DispatchException
-
 from platformio.account.client import AccountClient
 from platformio.home.rpc.handlers.base import BaseRPCHandler
 
 
 class AccountRPC(BaseRPCHandler):
+    NAMESPACE = "account"
+
     @staticmethod
     def call_client(method, *args, **kwargs):
-        try:
-            client = AccountClient()
+        with AccountClient() as client:
             return getattr(client, method)(*args, **kwargs)
-        except Exception as exc:  # pylint: disable=bare-except
-            raise JSONRPC20DispatchException(
-                code=5000, message="PIO Account Call Error", data=str(exc)
-            ) from exc

platformio/home/rpc/handlers/app.py

@@ -20,6 +20,7 @@ from platformio.project.helpers import is_platformio_project
 
 
 class AppRPC(BaseRPCHandler):
+    NAMESPACE = "app"
     IGNORE_STORAGE_KEYS = [
         "cid",
         "coreVersion",

platformio/home/rpc/handlers/base.py

@@ -14,4 +14,6 @@
 
 
 class BaseRPCHandler:
+    NAMESPACE = None
+
     factory = None

platformio/home/rpc/handlers/core.py

@@ -0,0 +1,123 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import asyncio
+import functools
+import os
+
+from platformio import __main__, __version__, app, proc, util
+from platformio.compat import (
+    IS_WINDOWS,
+    aio_create_task,
+    aio_get_running_loop,
+    get_locale_encoding,
+    shlex_join,
+)
+from platformio.exception import UserSideException
+from platformio.home.rpc.handlers.base import BaseRPCHandler
+
+
+class PIOCoreCallError(UserSideException):
+    MESSAGE = 'An error occured while executing PIO Core command: "{0}"\n\n{1}'
+
+
+class PIOCoreProtocol(asyncio.SubprocessProtocol):
+    def __init__(self, exit_future, on_data_callback=None):
+        self.exit_future = exit_future
+        self.on_data_callback = on_data_callback
+        self.stdout = ""
+        self.stderr = ""
+        self._is_exited = False
+        self._encoding = get_locale_encoding()
+
+    def pipe_data_received(self, fd, data):
+        data = data.decode(self._encoding, "replace")
+        pipe = ["stdin", "stdout", "stderr"][fd]
+        if pipe == "stdout":
+            self.stdout += data
+        if pipe == "stderr":
+            self.stderr += data
+        if self.on_data_callback:
+            self.on_data_callback(pipe=pipe, data=data)
+
+    def connection_lost(self, exc):
+        self.process_exited()
+
+    def process_exited(self):
+        if self._is_exited:
+            return
+        self.exit_future.set_result(True)
+        self._is_exited = True
+
+
+@util.memoized(expire="60s")
+def get_core_fullpath():
+    return proc.where_is_program("platformio" + (".exe" if IS_WINDOWS else ""))
+
+
+class CoreRPC(BaseRPCHandler):
+    NAMESPACE = "core"
+
+    @staticmethod
+    def version():
+        return __version__
+
+    async def exec(self, args, options=None, raise_exception=True):
+        options = options or {}
+        loop = aio_get_running_loop()
+        exit_future = loop.create_future()
+        data_callback = functools.partial(
+            self._on_exec_data_received, exec_options=options
+        )
+
+        if args[0] != "--caller" and app.get_session_var("caller_id"):
+            args = ["--caller", app.get_session_var("caller_id")] + args
+        kwargs = options.get("spawn", {})
+
+        if "force_ansi" in options:
+            environ = kwargs.get("env", os.environ.copy())
+            environ["PLATFORMIO_FORCE_ANSI"] = "true"
+            kwargs["env"] = environ
+
+        transport, protocol = await loop.subprocess_exec(
+            lambda: PIOCoreProtocol(exit_future, data_callback),
+            get_core_fullpath(),
+            *args,
+            stdin=None,
+            **kwargs,
+        )
+        await exit_future
+        transport.close()
+        return_code = transport.get_returncode()
+        if return_code != 0 and raise_exception:
+            raise PIOCoreCallError(
+                shlex_join(["pio"] + args), f"{protocol.stdout}\n{protocol.stderr}"
+            )
+        return {
+            "stdout": protocol.stdout,
+            "stderr": protocol.stderr,
+            "returncode": return_code,
+        }
+
+    def _on_exec_data_received(self, exec_options, pipe, data):
+        notification_method = exec_options.get(f"{pipe}NotificationMethod")
+        if not notification_method:
+            return
+        aio_create_task(
+            self.factory.notify_clients(
+                method=notification_method,
+                params=[data],
+                actor="frontend",
+            )
+        )

platformio/home/rpc/handlers/ide.py

@@ -22,6 +22,7 @@ from platformio.home.rpc.handlers.base import BaseRPCHandler
 
 
 class IDERPC(BaseRPCHandler):
+    NAMESPACE = "ide"
     COMMAND_TIMEOUT = 1.5  # in seconds
 
     def __init__(self):

platformio/home/rpc/handlers/memusage.py

@@ -0,0 +1,127 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import functools
+import os
+
+from platformio.home.rpc.handlers.base import BaseRPCHandler
+from platformio.project import memusage
+
+
+class MemUsageRPC(BaseRPCHandler):
+    NAMESPACE = "memusage"
+
+    async def profile(self, project_dir, env, options=None):
+        options = options or {}
+        report_dir = memusage.get_report_dir(project_dir, env)
+        if options.get("lazy"):
+            existing_reports = memusage.list_reports(report_dir)
+            if existing_reports:
+                return existing_reports[-1]
+        await self.factory.manager.dispatcher["core.exec"](
+            ["run", "-d", project_dir, "-e", env, "-t", "__memusage"],
+            options=options.get("exec"),
+        )
+        return memusage.list_reports(report_dir)[-1]
+
+    @staticmethod
+    def load_report(path):
+        return memusage.read_report(path)
+
+    def summary(self, report_path):
+        max_top_items = 10
+        report_dir = os.path.dirname(report_path)
+        existing_reports = memusage.list_reports(report_dir)
+        current_report = memusage.read_report(report_path)
+        previous_report = None
+        try:
+            current_index = existing_reports.index(report_path)
+            if current_index > 0:
+                previous_report = memusage.read_report(
+                    existing_reports[current_index - 1]
+                )
+        except ValueError:
+            pass
+
+        return dict(
+            timestamp=dict(
+                current=current_report["timestamp"],
+                previous=previous_report["timestamp"] if previous_report else None,
+            ),
+            device=current_report["device"],
+            trend=dict(
+                current=current_report["memory"]["total"],
+                previous=(
+                    previous_report["memory"]["total"] if previous_report else None
+                ),
+            ),
+            top=dict(
+                files=self._calculate_top_files(current_report["memory"]["files"])[
+                    0:max_top_items
+                ],
+                symbols=self._calculate_top_symbols(current_report["memory"]["files"])[
+                    0:max_top_items
+                ],
+                sections=sorted(
+                    current_report["memory"]["sections"].values(),
+                    key=lambda item: item["size"],
+                    reverse=True,
+                )[0:max_top_items],
+            ),
+        )
+
+    @staticmethod
+    def _calculate_top_files(items):
+        return [
+            {"path": item["path"], "ram": item["ram_size"], "flash": item["flash_size"]}
+            for item in sorted(
+                items,
+                key=lambda item: item["ram_size"] + item["flash_size"],
+                reverse=True,
+            )
+        ]
+
+    @staticmethod
+    def _calculate_top_symbols(files):
+        symbols = functools.reduce(
+            lambda result, filex: result
+            + [
+                {
+                    "name": s["name"],
+                    "type": s["type"],
+                    "size": s["size"],
+                    "file": filex["path"],
+                    "line": s.get("line"),
+                }
+                for s in filex["symbols"]
+            ],
+            files,
+            [],
+        )
+        return sorted(symbols, key=lambda item: item["size"], reverse=True)
+
+    async def history(self, project_dir, env, nums=10):
+        result = []
+        report_dir = memusage.get_report_dir(project_dir, env)
+        reports = memusage.list_reports(report_dir)[nums * -1 :]
+        for path in reports:
+            data = memusage.read_report(path)
+            result.append(
+                {
+                    "timestamp": data["timestamp"],
+                    "ram": data["memory"]["total"]["ram_size"],
+                    "flash": data["memory"]["total"]["flash_size"],
+                }
+            )
+        return result

platformio/home/rpc/handlers/misc.py

@@ -22,6 +22,8 @@ from platformio.home.rpc.handlers.os import OSRPC
 
 
 class MiscRPC(BaseRPCHandler):
+    NAMESPACE = "misc"
+
     async def load_latest_tweets(self, data_url):
         cache_key = ContentCache.key_from_args(data_url, "tweets")
         cache_valid = "180d"

platformio/home/rpc/handlers/os.py

@@ -15,32 +15,22 @@
 import glob
 import io
 import os
-import shutil
 from functools import cmp_to_key
 
 import click
 
 from platformio import fs
 from platformio.cache import ContentCache
-from platformio.compat import aio_to_thread
 from platformio.device.list.util import list_logical_devices
 from platformio.home.rpc.handlers.base import BaseRPCHandler
 from platformio.http import HTTPSession, ensure_internet_on
 
 
-class HTTPAsyncSession(HTTPSession):
-    async def request(  # pylint: disable=signature-differs,invalid-overridden-method
-        self, *args, **kwargs
-    ):
-        func = super().request
-        return await aio_to_thread(func, *args, **kwargs)
-
-
 class OSRPC(BaseRPCHandler):
-    _http_session = None
+    NAMESPACE = "os"
 
     @classmethod
-    async def fetch_content(cls, url, data=None, headers=None, cache_valid=None):
+    def fetch_content(cls, url, data=None, headers=None, cache_valid=None):
         if not headers:
             headers = {
                 "User-Agent": (
@@ -52,35 +42,33 @@ class OSRPC(BaseRPCHandler):
         cache_key = ContentCache.key_from_args(url, data) if cache_valid else None
         with ContentCache() as cc:
             if cache_key:
-                result = cc.get(cache_key)
-                if result is not None:
-                    return result
+                content = cc.get(cache_key)
+                if content is not None:
+                    return content
 
         # check internet before and resolve issue with 60 seconds timeout
         ensure_internet_on(raise_exception=True)
 
-        if not cls._http_session:
-            cls._http_session = HTTPAsyncSession()
+        with HTTPSession() as session:
+            if data:
+                response = session.post(url, data=data, headers=headers)
+            else:
+                response = session.get(url, headers=headers)
 
-        if data:
-            r = await cls._http_session.post(url, data=data, headers=headers)
-        else:
-            r = await cls._http_session.get(url, headers=headers)
+            response.raise_for_status()
+            content = response.text
+            if cache_valid:
+                with ContentCache() as cc:
+                    cc.set(cache_key, content, cache_valid)
+            return content
 
-        r.raise_for_status()
-        result = r.text
-        if cache_valid:
-            with ContentCache() as cc:
-                cc.set(cache_key, result, cache_valid)
-        return result
-
-    async def request_content(self, uri, data=None, headers=None, cache_valid=None):
+    @classmethod
+    def request_content(cls, uri, data=None, headers=None, cache_valid=None):
         if uri.startswith("http"):
-            return await self.fetch_content(uri, data, headers, cache_valid)
+            return cls.fetch_content(uri, data, headers, cache_valid)
         local_path = uri[7:] if uri.startswith("file://") else uri
         with io.open(local_path, encoding="utf-8") as fp:
             return fp.read()
-        return None
 
     @staticmethod
     def open_url(url):
@@ -110,22 +98,10 @@ class OSRPC(BaseRPCHandler):
     def is_dir(path):
         return os.path.isdir(path)
 
-    @staticmethod
-    def make_dirs(path):
-        return os.makedirs(path)
-
     @staticmethod
     def get_file_mtime(path):
         return os.path.getmtime(path)
 
-    @staticmethod
-    def rename(src, dst):
-        return os.rename(src, dst)
-
-    @staticmethod
-    def copy(src, dst):
-        return shutil.copytree(src, dst, symlinks=True)
-
     @staticmethod
     def glob(pathnames, root=None):
         if not isinstance(pathnames, list):

platformio/home/rpc/handlers/piocore.py

@@ -1,229 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import asyncio
-import functools
-import io
-import json
-import os
-import sys
-import threading
-
-import click
-from ajsonrpc.core import JSONRPC20DispatchException
-
-from platformio import __main__, __version__, app, fs, proc, util
-from platformio.compat import (
-    IS_WINDOWS,
-    aio_create_task,
-    aio_get_running_loop,
-    aio_to_thread,
-    get_locale_encoding,
-    is_bytes,
-)
-from platformio.exception import PlatformioException
-from platformio.home.rpc.handlers.base import BaseRPCHandler
-
-
-class PIOCoreProtocol(asyncio.SubprocessProtocol):
-    def __init__(self, exit_future, on_data_callback=None):
-        self.exit_future = exit_future
-        self.on_data_callback = on_data_callback
-        self.stdout = ""
-        self.stderr = ""
-        self._is_exited = False
-        self._encoding = get_locale_encoding()
-
-    def pipe_data_received(self, fd, data):
-        data = data.decode(self._encoding, "replace")
-        pipe = ["stdin", "stdout", "stderr"][fd]
-        if pipe == "stdout":
-            self.stdout += data
-        if pipe == "stderr":
-            self.stderr += data
-        if self.on_data_callback:
-            self.on_data_callback(pipe=pipe, data=data)
-
-    def connection_lost(self, exc):
-        self.process_exited()
-
-    def process_exited(self):
-        if self._is_exited:
-            return
-        self.exit_future.set_result(True)
-        self._is_exited = True
-
-
-class MultiThreadingStdStream:
-    def __init__(self, parent_stream):
-        self._buffers = {threading.get_ident(): parent_stream}
-
-    def __getattr__(self, name):
-        thread_id = threading.get_ident()
-        self._ensure_thread_buffer(thread_id)
-        return getattr(self._buffers[thread_id], name)
-
-    def _ensure_thread_buffer(self, thread_id):
-        if thread_id not in self._buffers:
-            self._buffers[thread_id] = io.StringIO()
-
-    def write(self, value):
-        thread_id = threading.get_ident()
-        self._ensure_thread_buffer(thread_id)
-        return self._buffers[thread_id].write(
-            value.decode() if is_bytes(value) else value
-        )
-
-    def get_value_and_reset(self):
-        result = ""
-        try:
-            result = self.getvalue()
-            self.seek(0)
-            self.truncate(0)
-        except AttributeError:
-            pass
-        return result
-
-
-@util.memoized(expire="60s")
-def get_core_fullpath():
-    return proc.where_is_program("platformio" + (".exe" if IS_WINDOWS else ""))
-
-
-class PIOCoreRPC(BaseRPCHandler):
-    @staticmethod
-    def version():
-        return __version__
-
-    async def exec(self, args, options=None):
-        loop = aio_get_running_loop()
-        exit_future = loop.create_future()
-        data_callback = functools.partial(
-            self._on_exec_data_received, exec_options=options
-        )
-        if args[0] != "--caller" and app.get_session_var("caller_id"):
-            args = ["--caller", app.get_session_var("caller_id")] + args
-        transport, protocol = await loop.subprocess_exec(
-            lambda: PIOCoreProtocol(exit_future, data_callback),
-            get_core_fullpath(),
-            *args,
-            stdin=None,
-            **options.get("spawn", {}),
-        )
-        await exit_future
-        transport.close()
-        return {
-            "stdout": protocol.stdout,
-            "stderr": protocol.stderr,
-            "returncode": transport.get_returncode(),
-        }
-
-    def _on_exec_data_received(self, exec_options, pipe, data):
-        notification_method = exec_options.get(f"{pipe}NotificationMethod")
-        if not notification_method:
-            return
-        aio_create_task(
-            self.factory.notify_clients(
-                method=notification_method,
-                params=[data],
-                actor="frontend",
-            )
-        )
-
-    @staticmethod
-    def setup_multithreading_std_streams():
-        if isinstance(sys.stdout, MultiThreadingStdStream):
-            return
-        PIOCoreRPC.thread_stdout = MultiThreadingStdStream(sys.stdout)
-        PIOCoreRPC.thread_stderr = MultiThreadingStdStream(sys.stderr)
-        sys.stdout = PIOCoreRPC.thread_stdout
-        sys.stderr = PIOCoreRPC.thread_stderr
-
-    @staticmethod
-    async def call(args, options=None):
-        for i, arg in enumerate(args):
-            if not isinstance(arg, str):
-                args[i] = str(arg)
-
-        options = options or {}
-        to_json = "--json-output" in args
-
-        try:
-            if options.get("force_subprocess"):
-                result = await PIOCoreRPC._call_subprocess(args, options)
-                return PIOCoreRPC._process_result(result, to_json)
-            result = await PIOCoreRPC._call_inline(args, options)
-            try:
-                return PIOCoreRPC._process_result(result, to_json)
-            except ValueError:
-                # fall-back to subprocess method
-                result = await PIOCoreRPC._call_subprocess(args, options)
-                return PIOCoreRPC._process_result(result, to_json)
-        except Exception as exc:  # pylint: disable=bare-except
-            raise JSONRPC20DispatchException(
-                code=5000, message="PIO Core Call Error", data=str(exc)
-            ) from exc
-
-    @staticmethod
-    async def _call_subprocess(args, options):
-        result = await aio_to_thread(
-            proc.exec_command,
-            [get_core_fullpath()] + args,
-            cwd=options.get("cwd") or os.getcwd(),
-        )
-        return (result["out"], result["err"], result["returncode"])
-
-    @staticmethod
-    async def _call_inline(args, options):
-        PIOCoreRPC.setup_multithreading_std_streams()
-
-        def _thread_safe_call(args, cwd):
-            with fs.cd(cwd):
-                exit_code = __main__.main(["-c"] + args)
-            return (
-                PIOCoreRPC.thread_stdout.get_value_and_reset(),
-                PIOCoreRPC.thread_stderr.get_value_and_reset(),
-                exit_code,
-            )
-
-        return await aio_to_thread(
-            _thread_safe_call, args=args, cwd=options.get("cwd") or os.getcwd()
-        )
-
-    @staticmethod
-    def _process_result(result, to_json=False):
-        out, err, code = result
-        if out and is_bytes(out):
-            out = out.decode(get_locale_encoding())
-        if err and is_bytes(err):
-            err = err.decode(get_locale_encoding())
-        text = ("%s\n\n%s" % (out, err)).strip()
-        if code != 0:
-            raise PlatformioException(text)
-        if not to_json:
-            return text
-        try:
-            return json.loads(out)
-        except ValueError as exc:
-            click.secho("%s => `%s`" % (exc, out), fg="red", err=True)
-            # if PIO Core prints unhandled warnings
-            for line in out.split("\n"):
-                line = line.strip()
-                if not line:
-                    continue
-                try:
-                    return json.loads(line)
-                except ValueError:
-                    pass
-            raise exc

platformio/home/rpc/handlers/platform.py

@@ -14,8 +14,8 @@
 
 import os.path
 
-from platformio.compat import aio_to_thread
 from platformio.home.rpc.handlers.base import BaseRPCHandler
+from platformio.home.rpc.handlers.registry import RegistryRPC
 from platformio.package.manager.platform import PlatformPackageManager
 from platformio.package.manifest.parser import ManifestParserFactory
 from platformio.package.meta import PackageSpec
@@ -23,15 +23,13 @@ from platformio.platform.factory import PlatformFactory
 
 
 class PlatformRPC(BaseRPCHandler):
-    async def fetch_platforms(self, search_query=None, page=0, force_installed=False):
-        if force_installed:
-            return {
-                "items": await aio_to_thread(
-                    self._load_installed_platforms, search_query
-                )
-            }
+    NAMESPACE = "platform"
 
-        search_result = await self.factory.manager.dispatcher["registry.call_client"](
+    def fetch_platforms(self, search_query=None, page=0, force_installed=False):
+        if force_installed:
+            return {"items": self._load_installed_platforms(search_query)}
+
+        search_result = RegistryRPC.call_client(
             method="list_packages",
             query=search_query,
             qualifiers={
@@ -88,17 +86,17 @@ class PlatformRPC(BaseRPCHandler):
             )
         return items
 
-    async def fetch_boards(self, platform_spec):
+    def fetch_boards(self, platform_spec):
         spec = PackageSpec(platform_spec)
         if spec.owner:
-            return await self.factory.manager.dispatcher["registry.call_client"](
+            return RegistryRPC.call_client(
                 method="get_package",
                 typex="platform",
                 owner=spec.owner,
                 name=spec.name,
                 extra_path="/boards",
             )
-        return await aio_to_thread(self._load_installed_boards, spec)
+        return self._load_installed_boards(spec)
 
     @staticmethod
     def _load_installed_boards(platform_spec):
@@ -108,17 +106,17 @@ class PlatformRPC(BaseRPCHandler):
             key=lambda item: item["name"],
         )
 
-    async def fetch_examples(self, platform_spec):
+    def fetch_examples(self, platform_spec):
         spec = PackageSpec(platform_spec)
         if spec.owner:
-            return await self.factory.manager.dispatcher["registry.call_client"](
+            return RegistryRPC.call_client(
                 method="get_package",
                 typex="platform",
                 owner=spec.owner,
                 name=spec.name,
                 extra_path="/examples",
             )
-        return await aio_to_thread(self._load_installed_examples, spec)
+        return self._load_installed_examples(spec)
 
     @staticmethod
     def _load_installed_examples(platform_spec):

platformio/home/rpc/handlers/project.py

@@ -13,29 +13,24 @@
 # limitations under the License.
 
 import os
-import shutil
-import time
 from pathlib import Path
 
 import semantic_version
-from ajsonrpc.core import JSONRPC20DispatchException
 
-from platformio import app, exception, fs
-from platformio.home.rpc.handlers.app import AppRPC
+from platformio import app, fs
 from platformio.home.rpc.handlers.base import BaseRPCHandler
-from platformio.home.rpc.handlers.piocore import PIOCoreRPC
 from platformio.package.manager.platform import PlatformPackageManager
 from platformio.platform.factory import PlatformFactory
 from platformio.project.config import ProjectConfig
-from platformio.project.exception import ProjectError
-from platformio.project.helpers import get_project_dir, is_platformio_project
+from platformio.project.helpers import get_project_dir
 from platformio.project.integration.generator import ProjectGenerator
-from platformio.project.options import get_config_options_schema
 
 
 class ProjectRPC(BaseRPCHandler):
+    NAMESPACE = "project"
+
     @staticmethod
-    def config_call(init_kwargs, method, *args):
+    async def config_call(init_kwargs, method, *args):
         assert isinstance(init_kwargs, dict)
         assert "path" in init_kwargs
         if os.path.isdir(init_kwargs["path"]):
@@ -48,249 +43,20 @@ class ProjectRPC(BaseRPCHandler):
         with fs.cd(project_dir):
             return getattr(ProjectConfig(**init_kwargs), method)(*args)
 
-    @staticmethod
-    def config_load(path):
-        return ProjectConfig(
-            path, parse_extra=False, expand_interpolations=False
-        ).as_tuple()
-
-    @staticmethod
-    def config_dump(path, data):
-        config = ProjectConfig(path, parse_extra=False, expand_interpolations=False)
-        config.update(data, clear=True)
-        return config.save()
-
-    @staticmethod
-    def config_update_description(path, text):
-        config = ProjectConfig(path, parse_extra=False, expand_interpolations=False)
-        if not config.has_section("platformio"):
-            config.add_section("platformio")
-        if text:
-            config.set("platformio", "description", text)
-        else:
-            if config.has_option("platformio", "description"):
-                config.remove_option("platformio", "description")
-            if not config.options("platformio"):
-                config.remove_section("platformio")
-        return config.save()
-
-    @staticmethod
-    def get_config_schema():
-        return get_config_options_schema()
-
-    @staticmethod
-    def get_projects():
-        def _get_project_data():
-            data = {"boards": [], "envLibdepsDirs": [], "libExtraDirs": []}
-            config = ProjectConfig()
-            data["envs"] = config.envs()
-            data["description"] = config.get("platformio", "description")
-            data["libExtraDirs"].extend(config.get("platformio", "lib_extra_dirs", []))
-
-            libdeps_dir = config.get("platformio", "libdeps_dir")
-            for section in config.sections():
-                if not section.startswith("env:"):
-                    continue
-                data["envLibdepsDirs"].append(os.path.join(libdeps_dir, section[4:]))
-                if config.has_option(section, "board"):
-                    data["boards"].append(config.get(section, "board"))
-                data["libExtraDirs"].extend(config.get(section, "lib_extra_dirs", []))
-
-            # skip non existing folders and resolve full path
-            for key in ("envLibdepsDirs", "libExtraDirs"):
-                data[key] = [
-                    fs.expanduser(d) if d.startswith("~") else os.path.abspath(d)
-                    for d in data[key]
-                    if os.path.isdir(d)
-                ]
-
-            return data
-
-        def _path_to_name(path):
-            return (os.path.sep).join(path.split(os.path.sep)[-2:])
-
-        result = []
-        pm = PlatformPackageManager()
-        for project_dir in AppRPC.load_state()["storage"]["recentProjects"]:
-            if not os.path.isdir(project_dir):
-                continue
-            data = {}
-            boards = []
-            try:
-                with fs.cd(project_dir):
-                    data = _get_project_data()
-            except ProjectError:
-                continue
-
-            for board_id in data.get("boards", []):
-                name = board_id
-                try:
-                    name = pm.board_config(board_id)["name"]
-                except exception.PlatformioException:
-                    pass
-                boards.append({"id": board_id, "name": name})
-
-            result.append(
-                {
-                    "path": project_dir,
-                    "name": _path_to_name(project_dir),
-                    "modified": int(os.path.getmtime(project_dir)),
-                    "boards": boards,
-                    "description": data.get("description"),
-                    "envs": data.get("envs", []),
-                    "envLibStorages": [
-                        {"name": os.path.basename(d), "path": d}
-                        for d in data.get("envLibdepsDirs", [])
-                    ],
-                    "extraLibStorages": [
-                        {"name": _path_to_name(d), "path": d}
-                        for d in data.get("libExtraDirs", [])
-                    ],
-                }
-            )
-        return result
-
-    @staticmethod
-    def get_project_examples():
-        result = []
-        pm = PlatformPackageManager()
-        for pkg in pm.get_installed():
-            examples_dir = os.path.join(pkg.path, "examples")
-            if not os.path.isdir(examples_dir):
-                continue
-            items = []
-            for project_dir, _, __ in os.walk(examples_dir):
-                project_description = None
-                try:
-                    config = ProjectConfig(os.path.join(project_dir, "platformio.ini"))
-                    config.validate(silent=True)
-                    project_description = config.get("platformio", "description")
-                except ProjectError:
-                    continue
-
-                path_tokens = project_dir.split(os.path.sep)
-                items.append(
-                    {
-                        "name": "/".join(
-                            path_tokens[path_tokens.index("examples") + 1 :]
-                        ),
-                        "path": project_dir,
-                        "description": project_description,
-                    }
-                )
-            manifest = pm.load_manifest(pkg)
-            result.append(
-                {
-                    "platform": {
-                        "title": manifest["title"],
-                        "version": manifest["version"],
-                    },
-                    "items": sorted(items, key=lambda item: item["name"]),
-                }
-            )
-        return sorted(result, key=lambda data: data["platform"]["title"])
-
-    async def init(self, board, framework, project_dir):
-        assert project_dir
-        if not os.path.isdir(project_dir):
-            os.makedirs(project_dir)
-        args = ["init", "--board", board, "--sample-code"]
-        if framework:
-            args.extend(["--project-option", "framework = %s" % framework])
-        ide = app.get_session_var("caller_id")
-        if ide in ProjectGenerator.get_supported_ides():
-            args.extend(["--ide", ide])
-        await PIOCoreRPC.call(
-            args, options={"cwd": project_dir, "force_subprocess": True}
-        )
-        return project_dir
-
-    @staticmethod
-    async def import_arduino(board, use_arduino_libs, arduino_project_dir):
-        board = str(board)
-        # don't import PIO Project
-        if is_platformio_project(arduino_project_dir):
-            return arduino_project_dir
-
-        is_arduino_project = any(
-            os.path.isfile(
-                os.path.join(
-                    arduino_project_dir,
-                    "%s.%s" % (os.path.basename(arduino_project_dir), ext),
-                )
-            )
-            for ext in ("ino", "pde")
-        )
-        if not is_arduino_project:
-            raise JSONRPC20DispatchException(
-                code=4000, message="Not an Arduino project: %s" % arduino_project_dir
-            )
-
-        state = AppRPC.load_state()
-        project_dir = os.path.join(
-            state["storage"]["projectsDir"], time.strftime("%y%m%d-%H%M%S-") + board
-        )
-        if not os.path.isdir(project_dir):
-            os.makedirs(project_dir)
-        args = ["init", "--board", board]
-        args.extend(["--project-option", "framework = arduino"])
-        if use_arduino_libs:
-            args.extend(
-                ["--project-option", "lib_extra_dirs = ~/Documents/Arduino/libraries"]
-            )
-        ide = app.get_session_var("caller_id")
-        if ide in ProjectGenerator.get_supported_ides():
-            args.extend(["--ide", ide])
-        await PIOCoreRPC.call(
-            args, options={"cwd": project_dir, "force_subprocess": True}
-        )
-        with fs.cd(project_dir):
-            config = ProjectConfig()
-            src_dir = config.get("platformio", "src_dir")
-            if os.path.isdir(src_dir):
-                fs.rmtree(src_dir)
-            shutil.copytree(arduino_project_dir, src_dir, symlinks=True)
-        return project_dir
-
-    @staticmethod
-    async def import_pio(project_dir):
-        if not project_dir or not is_platformio_project(project_dir):
-            raise JSONRPC20DispatchException(
-                code=4001, message="Not an PlatformIO project: %s" % project_dir
-            )
-        new_project_dir = os.path.join(
-            AppRPC.load_state()["storage"]["projectsDir"],
-            time.strftime("%y%m%d-%H%M%S-") + os.path.basename(project_dir),
-        )
-        shutil.copytree(project_dir, new_project_dir, symlinks=True)
-
-        args = ["init"]
-        ide = app.get_session_var("caller_id")
-        if ide in ProjectGenerator.get_supported_ides():
-            args.extend(["--ide", ide])
-        await PIOCoreRPC.call(
-            args, options={"cwd": new_project_dir, "force_subprocess": True}
-        )
-        return new_project_dir
-
-    async def init_v2(self, configuration, options=None):
+    async def init(self, configuration, options=None):
         project_dir = os.path.join(configuration["location"], configuration["name"])
         if not os.path.isdir(project_dir):
             os.makedirs(project_dir)
 
-        envclone = os.environ.copy()
-        envclone["PLATFORMIO_FORCE_ANSI"] = "true"
-        options = options or {}
-        options["spawn"] = {"env": envclone, "cwd": project_dir}
-
-        args = ["project", "init"]
+        args = ["project", "init", "-d", project_dir]
         ide = app.get_session_var("caller_id")
         if ide in ProjectGenerator.get_supported_ides():
             args.extend(["--ide", ide])
 
+        exec_options = options.get("exec", {})
         if configuration.get("example"):
             await self.factory.notify_clients(
-                method=options.get("stdoutNotificationMethod"),
+                method=exec_options.get("stdoutNotificationMethod"),
                 params=["Copying example files...\n"],
                 actor="frontend",
             )
@@ -298,7 +64,9 @@ class ProjectRPC(BaseRPCHandler):
         else:
             args.extend(self._pre_init_empty(configuration))
 
-        return await self.factory.manager.dispatcher["core.exec"](args, options=options)
+        return await self.factory.manager.dispatcher["core.exec"](
+            args, options=exec_options, raise_exception=False
+        )
 
     @staticmethod
     def _pre_init_empty(configuration):
@@ -342,10 +110,10 @@ class ProjectRPC(BaseRPCHandler):
         return []
 
     @staticmethod
-    def configuration(project_dir, env):
-        assert is_platformio_project(project_dir)
+    async def configuration(project_dir, env):
         with fs.cd(project_dir):
-            config = ProjectConfig(os.path.join(project_dir, "platformio.ini"))
+            config = ProjectConfig.get_instance()
+            config.validate(envs=[env])
             platform = PlatformFactory.from_env(env, autoinstall=True)
             platform_pkg = PlatformPackageManager().get_package(platform.get_dir())
             board_id = config.get(f"env:{env}", "board", None)
@@ -372,15 +140,19 @@ class ProjectRPC(BaseRPCHandler):
 
             return dict(
                 platform=dict(
-                    ownername=platform_pkg.metadata.spec.owner
-                    if platform_pkg.metadata.spec
-                    else None,
+                    ownername=(
+                        platform_pkg.metadata.spec.owner
+                        if platform_pkg.metadata.spec
+                        else None
+                    ),
                     name=platform.name,
                     title=platform.title,
                     version=str(platform_pkg.metadata.version),
                 ),
-                board=platform.board_config(board_id).get_brief_data()
-                if board_id
-                else None,
+                board=(
+                    platform.board_config(board_id).get_brief_data()
+                    if board_id
+                    else None
+                ),
                 frameworks=frameworks or None,
             )

platformio/home/rpc/handlers/registry.py

@@ -12,20 +12,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from ajsonrpc.core import JSONRPC20DispatchException
-
-from platformio.compat import aio_to_thread
 from platformio.home.rpc.handlers.base import BaseRPCHandler
 from platformio.registry.client import RegistryClient
 
 
 class RegistryRPC(BaseRPCHandler):
+    NAMESPACE = "registry"
+
     @staticmethod
-    async def call_client(method, *args, **kwargs):
-        try:
-            client = RegistryClient()
-            return await aio_to_thread(getattr(client, method), *args, **kwargs)
-        except Exception as exc:  # pylint: disable=bare-except
-            raise JSONRPC20DispatchException(
-                code=5000, message="Registry Call Error", data=str(exc)
-            ) from exc
+    def call_client(method, *args, **kwargs):
+        with RegistryClient() as client:
+            return getattr(client, method)(*args, **kwargs)

platformio/home/rpc/server.py

@@ -12,22 +12,24 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import functools
+import inspect
 from urllib.parse import parse_qs
 
-import ajsonrpc.utils
+import ajsonrpc.manager
 import click
 from ajsonrpc.core import JSONRPC20Error, JSONRPC20Request
 from ajsonrpc.dispatcher import Dispatcher
 from ajsonrpc.manager import AsyncJSONRPCResponseManager, JSONRPC20Response
 from starlette.endpoints import WebSocketEndpoint
 
-from platformio.compat import aio_create_task, aio_get_running_loop
+from platformio.compat import aio_create_task, aio_get_running_loop, aio_to_thread
 from platformio.http import InternetConnectionError
 from platformio.proc import force_exit
 
 # Remove this line when PR is merged
 # https://github.com/pavlov99/ajsonrpc/pull/22
-ajsonrpc.utils.is_invalid_params = lambda: False
+ajsonrpc.manager.is_invalid_params = lambda *args, **kwargs: False
 
 
 class JSONRPCServerFactoryBase:
@@ -44,9 +46,18 @@ class JSONRPCServerFactoryBase:
     def __call__(self, *args, **kwargs):
         raise NotImplementedError
 
-    def add_object_handler(self, handler, namespace):
-        handler.factory = self
-        self.manager.dispatcher.add_object(handler, prefix="%s." % namespace)
+    def add_object_handler(self, obj):
+        obj.factory = self
+        namespace = obj.NAMESPACE or obj.__class__.__name__
+        for name in dir(obj):
+            method = getattr(obj, name)
+            if name.startswith("_") or not (
+                inspect.ismethod(method) or inspect.isfunction(method)
+            ):
+                continue
+            if not inspect.iscoroutinefunction(method):
+                method = functools.partial(aio_to_thread, method)
+            self.manager.dispatcher.add_function(method, name=f"{namespace}.{name}")
 
     def on_client_connect(self, connection, actor=None):
         self._clients[connection] = {"actor": actor}

platformio/home/run.py

@@ -28,10 +28,11 @@ from platformio.compat import aio_get_running_loop
 from platformio.exception import PlatformioException
 from platformio.home.rpc.handlers.account import AccountRPC
 from platformio.home.rpc.handlers.app import AppRPC
+from platformio.home.rpc.handlers.core import CoreRPC
 from platformio.home.rpc.handlers.ide import IDERPC
+from platformio.home.rpc.handlers.memusage import MemUsageRPC
 from platformio.home.rpc.handlers.misc import MiscRPC
 from platformio.home.rpc.handlers.os import OSRPC
-from platformio.home.rpc.handlers.piocore import PIOCoreRPC
 from platformio.home.rpc.handlers.platform import PlatformRPC
 from platformio.home.rpc.handlers.project import ProjectRPC
 from platformio.home.rpc.handlers.registry import RegistryRPC
@@ -67,15 +68,16 @@ def run_server(host, port, no_open, shutdown_timeout, home_url):
         raise PlatformioException("Invalid path to PIO Home Contrib")
 
     ws_rpc_factory = WebSocketJSONRPCServerFactory(shutdown_timeout)
-    ws_rpc_factory.add_object_handler(AccountRPC(), namespace="account")
-    ws_rpc_factory.add_object_handler(AppRPC(), namespace="app")
-    ws_rpc_factory.add_object_handler(IDERPC(), namespace="ide")
-    ws_rpc_factory.add_object_handler(MiscRPC(), namespace="misc")
-    ws_rpc_factory.add_object_handler(OSRPC(), namespace="os")
-    ws_rpc_factory.add_object_handler(PIOCoreRPC(), namespace="core")
-    ws_rpc_factory.add_object_handler(ProjectRPC(), namespace="project")
-    ws_rpc_factory.add_object_handler(PlatformRPC(), namespace="platform")
-    ws_rpc_factory.add_object_handler(RegistryRPC(), namespace="registry")
+    ws_rpc_factory.add_object_handler(AccountRPC())
+    ws_rpc_factory.add_object_handler(AppRPC())
+    ws_rpc_factory.add_object_handler(IDERPC())
+    ws_rpc_factory.add_object_handler(MemUsageRPC())
+    ws_rpc_factory.add_object_handler(MiscRPC())
+    ws_rpc_factory.add_object_handler(OSRPC())
+    ws_rpc_factory.add_object_handler(CoreRPC())
+    ws_rpc_factory.add_object_handler(ProjectRPC())
+    ws_rpc_factory.add_object_handler(PlatformRPC())
+    ws_rpc_factory.add_object_handler(RegistryRPC())
 
     path = urlparse(home_url).path
     routes = [

platformio/http.py

@@ -12,22 +12,25 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import contextlib
+import itertools
 import json
 import socket
-from urllib.parse import urljoin
+import time
 
-import requests.adapters
-from urllib3.util.retry import Retry
+import httpx
 
 from platformio import __check_internet_hosts__, app, util
 from platformio.cache import ContentCache, cleanup_content_cache
+from platformio.compat import is_proxy_set
 from platformio.exception import PlatformioException, UserSideException
-from platformio.pipdeps import is_proxy_set
 
-__default_requests_timeout__ = (10, None)  # (connect, read)
+RETRIES_BACKOFF_FACTOR = 2  # 0s, 2s, 4s, 8s, etc.
+RETRIES_METHOD_WHITELIST = ["GET"]
+RETRIES_STATUS_FORCELIST = [429, 500, 502, 503, 504]
 
 
-class HTTPClientError(UserSideException):
+class HttpClientApiError(UserSideException):
     def __init__(self, message, response=None):
         super().__init__()
         self.message = message
@@ -40,84 +43,138 @@ class HTTPClientError(UserSideException):
 class InternetConnectionError(UserSideException):
     MESSAGE = (
         "You are not connected to the Internet.\n"
-        "PlatformIO needs the Internet connection to"
-        " download dependent packages or to work with PlatformIO Account."
+        "PlatformIO needs the Internet connection to "
+        "download dependent packages or to work with PlatformIO Account."
     )
 
 
-class HTTPSession(requests.Session):
-    def __init__(self, *args, **kwargs):
-        self._x_base_url = kwargs.pop("x_base_url") if "x_base_url" in kwargs else None
-        super().__init__(*args, **kwargs)
-        self.headers.update({"User-Agent": app.get_user_agent()})
-        try:
-            self.verify = app.get_setting("enable_proxy_strict_ssl")
-        except PlatformioException:
-            self.verify = True
+def exponential_backoff(factor):
+    yield 0
+    for n in itertools.count(2):
+        yield factor * (2 ** (n - 2))
 
-    def request(  # pylint: disable=signature-differs,arguments-differ
-        self, method, url, *args, **kwargs
+
+def apply_default_kwargs(kwargs=None):
+    kwargs = kwargs or {}
+    # enable redirects by default
+    kwargs["follow_redirects"] = kwargs.get("follow_redirects", True)
+
+    try:
+        kwargs["verify"] = kwargs.get(
+            "verify", app.get_setting("enable_proxy_strict_ssl")
+        )
+    except PlatformioException:
+        kwargs["verify"] = True
+
+    headers = kwargs.pop("headers", {})
+    if "User-Agent" not in headers:
+        headers.update({"User-Agent": app.get_user_agent()})
+        kwargs["headers"] = headers
+
+    retry = kwargs.pop("retry", None)
+    if retry:
+        kwargs["transport"] = HTTPRetryTransport(verify=kwargs["verify"], **retry)
+
+    return kwargs
+
+
+class HTTPRetryTransport(httpx.HTTPTransport):
+    def __init__(  # pylint: disable=too-many-arguments
+        self,
+        verify=True,
+        retries=1,
+        backoff_factor=None,
+        status_forcelist=None,
+        method_whitelist=None,
     ):
-        # print("HTTPSession::request", self._x_base_url, method, url, args, kwargs)
-        if "timeout" not in kwargs:
-            kwargs["timeout"] = __default_requests_timeout__
-        return super().request(
-            method,
-            url
-            if url.startswith("http") or not self._x_base_url
-            else urljoin(self._x_base_url, url),
-            *args,
-            **kwargs
+        super().__init__(verify=verify)
+        self._retries = retries
+        self._backoff_factor = backoff_factor or RETRIES_BACKOFF_FACTOR
+        self._status_forcelist = status_forcelist or RETRIES_STATUS_FORCELIST
+        self._method_whitelist = method_whitelist or RETRIES_METHOD_WHITELIST
+
+    def handle_request(self, request):
+        retries_left = self._retries
+        delays = exponential_backoff(factor=RETRIES_BACKOFF_FACTOR)
+        while retries_left > 0:
+            retries_left -= 1
+            try:
+                response = super().handle_request(request)
+                if response.status_code in RETRIES_STATUS_FORCELIST:
+                    if request.method.upper() not in self._method_whitelist:
+                        return response
+                    raise httpx.HTTPStatusError(
+                        f"Server error '{response.status_code} {response.reason_phrase}' "
+                        f"for url '{request.url}'\n",
+                        request=request,
+                        response=response,
+                    )
+                return response
+            except httpx.HTTPError:
+                if retries_left == 0:
+                    raise
+                time.sleep(next(delays) or 1)
+
+        raise httpx.RequestError(
+            f"Could not process '{request.url}' request", request=request
         )
 
 
-class HTTPSessionIterator:
-    def __init__(self, endpoints):
+class HTTPSession(httpx.Client):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **apply_default_kwargs(kwargs))
+
+
+class HttpEndpointPool:
+    def __init__(self, endpoints, session_retry=None):
         if not isinstance(endpoints, list):
             endpoints = [endpoints]
         self.endpoints = endpoints
-        self.endpoints_iter = iter(endpoints)
-        # https://urllib3.readthedocs.io/en/stable/reference/urllib3.util.html
-        self.retry = Retry(
-            total=5,
-            backoff_factor=1,  # [0, 2, 4, 8, 16] secs
-            # method_whitelist=list(Retry.DEFAULT_METHOD_WHITELIST) + ["POST"],
-            status_forcelist=[413, 429, 500, 502, 503, 504],
-        )
+        self.session_retry = session_retry
 
-    def __iter__(self):  # pylint: disable=non-iterator-returned
-        return self
-
-    def __next__(self):
-        base_url = next(self.endpoints_iter)
-        session = HTTPSession(x_base_url=base_url)
-        adapter = requests.adapters.HTTPAdapter(max_retries=self.retry)
-        session.mount(base_url, adapter)
-        return session
-
-
-class HTTPClient:
-    def __init__(self, endpoints):
-        self._session_iter = HTTPSessionIterator(endpoints)
-        self._session = None
-        self._next_session()
-
-    def __del__(self):
-        if not self._session:
-            return
-        try:
-            self._session.close()
-        except:  # pylint: disable=bare-except
-            pass
+        self._endpoints_iter = iter(endpoints)
         self._session = None
 
-    def _next_session(self):
+        self.next()
+
+    def close(self):
         if self._session:
             self._session.close()
-        self._session = next(self._session_iter)
+
+    def next(self):
+        if self._session:
+            self._session.close()
+        self._session = HTTPSession(
+            base_url=next(self._endpoints_iter), retry=self.session_retry
+        )
+
+    def request(self, method, *args, **kwargs):
+        while True:
+            try:
+                return self._session.request(method, *args, **kwargs)
+            except httpx.HTTPError as exc:
+                try:
+                    self.next()
+                except StopIteration as exc2:
+                    raise exc from exc2
+
+
+class HttpApiClient(contextlib.AbstractContextManager):
+    def __init__(self, endpoints):
+        self._endpoint = HttpEndpointPool(endpoints, session_retry=dict(retries=5))
+
+    def __exit__(self, *excinfo):
+        self.close()
+
+    def __del__(self):
+        self.close()
+
+    def close(self):
+        if getattr(self, "_endpoint"):
+            self._endpoint.close()
 
     @util.throttle(500)
-    def send_request(self, method, path, **kwargs):
+    def send_request(self, method, *args, **kwargs):
         # check Internet before and resolve issue with 60 seconds timeout
         ensure_internet_on(raise_exception=True)
 
@@ -131,23 +188,28 @@ class HTTPClient:
             # pylint: disable=import-outside-toplevel
             from platformio.account.client import AccountClient
 
-            headers["Authorization"] = (
-                "Bearer %s" % AccountClient().fetch_authentication_token()
-            )
+            with AccountClient() as client:
+                headers["Authorization"] = (
+                    "Bearer %s" % client.fetch_authentication_token()
+                )
         kwargs["headers"] = headers
 
-        while True:
-            try:
-                return getattr(self._session, method)(path, **kwargs)
-            except requests.exceptions.RequestException as exc:
-                try:
-                    self._next_session()
-                except Exception as exc2:
-                    raise HTTPClientError(str(exc2)) from exc
+        try:
+            return self._endpoint.request(method, *args, **kwargs)
+        except httpx.HTTPError as exc:
+            raise HttpClientApiError(str(exc)) from exc
 
     def fetch_json_data(self, method, path, **kwargs):
         if method not in ("get", "head", "options"):
             cleanup_content_cache("http")
+        # remove empty params
+        if kwargs.get("params"):
+            kwargs["params"] = {
+                key: value
+                for key, value in kwargs.get("params").items()
+                if value is not None
+            }
+
         cache_valid = kwargs.pop("x_cache_valid") if "x_cache_valid" in kwargs else None
         if not cache_valid:
             return self._parse_json_response(self.send_request(method, path, **kwargs))
@@ -177,7 +239,7 @@ class HTTPClient:
             message = response.json()["message"]
         except (KeyError, ValueError):
             message = response.text
-        raise HTTPClientError(message, response)
+        raise HttpClientApiError(message, response)
 
 
 #
@@ -192,7 +254,7 @@ def _internet_on():
     for host in __check_internet_hosts__:
         try:
             if is_proxy_set():
-                requests.get("http://%s" % host, allow_redirects=False, timeout=timeout)
+                httpx.get("http://%s" % host, follow_redirects=False, timeout=timeout)
                 return True
             # try to resolve `host` for both AF_INET and AF_INET6, and then try to connect
             # to all possible addresses (IPv4 and IPv6) in turn until a connection succeeds:
@@ -211,9 +273,8 @@ def ensure_internet_on(raise_exception=False):
     return result
 
 
-def fetch_remote_content(*args, **kwargs):
-    with HTTPSession() as s:
-        r = s.get(*args, **kwargs)
-        r.raise_for_status()
-        r.close()
-        return r.text
+def fetch_http_content(*args, **kwargs):
+    with HTTPSession() as session:
+        response = session.get(*args, **kwargs)
+        response.raise_for_status()
+        return response.text

platformio/maintenance.py

@@ -23,7 +23,11 @@ from platformio import __version__, app, exception, fs, telemetry
 from platformio.cache import cleanup_content_cache
 from platformio.cli import PlatformioCLI
 from platformio.commands.upgrade import get_latest_version
-from platformio.http import HTTPClientError, InternetConnectionError, ensure_internet_on
+from platformio.http import (
+    HttpClientApiError,
+    InternetConnectionError,
+    ensure_internet_on,
+)
 from platformio.package.manager.core import update_core_packages
 from platformio.package.version import pepver_to_semver
 from platformio.system.prune import calculate_unnecessary_system_data
@@ -46,7 +50,7 @@ def on_cmd_end():
         check_platformio_upgrade()
         check_prune_system()
     except (
-        HTTPClientError,
+        HttpClientApiError,
         InternetConnectionError,
         exception.GetLatestVersionError,
     ):

platformio/package/commands/install.py

@@ -222,9 +222,11 @@ def _install_project_env_libraries(project_env, options):
 
     env_lm = LibraryPackageManager(
         os.path.join(config.get("platformio", "libdeps_dir"), project_env),
-        compatibility=PackageCompatibility(**compatibility_qualifiers)
-        if compatibility_qualifiers
-        else None,
+        compatibility=(
+            PackageCompatibility(**compatibility_qualifiers)
+            if compatibility_qualifiers
+            else None
+        ),
     )
     private_lm = LibraryPackageManager(
         os.path.join(config.get("platformio", "lib_dir"))

platformio/package/commands/list.py

@@ -12,8 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import json
 import os
-from typing import List
 
 import click
 
@@ -21,13 +21,13 @@ from platformio import fs
 from platformio.package.manager.library import LibraryPackageManager
 from platformio.package.manager.platform import PlatformPackageManager
 from platformio.package.manager.tool import ToolPackageManager
-from platformio.package.meta import PackageItem, PackageSpec
+from platformio.package.meta import PackageInfo, PackageItem, PackageSpec
 from platformio.platform.exception import UnknownPlatform
 from platformio.platform.factory import PlatformFactory
 from platformio.project.config import ProjectConfig
 
 
-@click.command("list", short_help="List installed packages")
+@click.command("list", short_help="List project packages")
 @click.option(
     "-d",
     "--project-dir",
@@ -48,79 +48,116 @@ from platformio.project.config import ProjectConfig
 @click.option("--only-platforms", is_flag=True, help="List only platform packages")
 @click.option("--only-tools", is_flag=True, help="List only tool packages")
 @click.option("--only-libraries", is_flag=True, help="List only library packages")
+@click.option("--json-output", is_flag=True)
 @click.option("-v", "--verbose", is_flag=True)
 def package_list_cmd(**options):
-    if options.get("global"):
+    data = (
         list_global_packages(options)
+        if options.get("global")
+        else list_project_packages(options)
+    )
+
+    if options.get("json_output"):
+        return click.echo(_dump_to_json(data, options))
+
+    def _print_items(typex, items):
+        click.secho(typex.capitalize(), bold=True)
+        print_dependency_tree(items, verbose=options.get("verbose"))
+        click.echo()
+
+    if options.get("global"):
+        for typex, items in data.items():
+            _print_items(typex, items)
     else:
-        list_project_packages(options)
+        for env, env_data in data.items():
+            click.echo("Resolving %s dependencies..." % click.style(env, fg="cyan"))
+            for typex, items in env_data.items():
+                _print_items(typex, items)
+
+    return None
 
 
-def humanize_package(pkg, spec=None, verbose=False):
-    if spec and not isinstance(spec, PackageSpec):
-        spec = PackageSpec(spec)
-    data = [
-        click.style(pkg.metadata.name, fg="cyan"),
-        click.style(f"@ {str(pkg.metadata.version)}", bold=True),
-    ]
-    extra_data = ["required: %s" % (spec.humanize() if spec else "Any")]
-    if verbose:
-        extra_data.append(pkg.path)
-    data.append("(%s)" % ", ".join(extra_data))
-    return " ".join(data)
+def _dump_to_json(data, options):
+    result = {}
+
+    if options.get("global"):
+        for typex, items in data.items():
+            result[typex] = [info.as_dict(with_manifest=True) for info in items]
+    else:
+        for env, env_data in data.items():
+            result[env] = {}
+            for typex, items in env_data.items():
+                result[env][typex] = [
+                    info.as_dict(with_manifest=True) for info in items
+                ]
+    return json.dumps(result)
 
 
-def print_dependency_tree(pm, specs=None, filter_specs=None, level=0, verbose=False):
+def build_package_info(pm, specs=None, filter_specs=None, resolve_dependencies=True):
     filtered_pkgs = [
-        pm.get_package(spec) for spec in filter_specs or [] if pm.get_package(spec)
+        pm.get_package(spec) for spec in filter_specs if pm.get_package(spec)
     ]
-    candidates = {}
+    candidates = []
     if specs:
         for spec in specs:
-            pkg = pm.get_package(spec)
-            if not pkg:
-                continue
-            candidates[pkg.path] = (pkg, spec)
+            candidates.append(
+                PackageInfo(
+                    spec if isinstance(spec, PackageSpec) else PackageSpec(spec),
+                    pm.get_package(spec),
+                )
+            )
     else:
-        candidates = {pkg.path: (pkg, pkg.metadata.spec) for pkg in pm.get_installed()}
+        candidates = [PackageInfo(pkg.metadata.spec, pkg) for pkg in pm.get_installed()]
     if not candidates:
-        return
-    candidates = sorted(candidates.values(), key=lambda item: item[0].metadata.name)
+        return []
 
-    for index, (pkg, spec) in enumerate(candidates):
-        if filtered_pkgs and not _pkg_tree_contains(pm, pkg, filtered_pkgs):
-            continue
-        printed_pkgs = pm.memcache_get("__printed_pkgs", [])
-        if printed_pkgs and pkg.path in printed_pkgs:
-            continue
-        printed_pkgs.append(pkg.path)
-        pm.memcache_set("__printed_pkgs", printed_pkgs)
+    candidates = sorted(
+        candidates,
+        key=lambda info: info.item.metadata.name if info.item else info.spec.humanize(),
+    )
 
-        click.echo(
-            "%s%s %s"
-            % (
-                "│   " * level,
-                "├──" if index < len(candidates) - 1 else "└──",
-                humanize_package(
-                    pkg,
-                    spec=spec,
-                    verbose=verbose,
+    result = []
+    for info in candidates:
+        if filter_specs and (
+            not info.item or not _pkg_tree_contains(pm, info.item, filtered_pkgs)
+        ):
+            continue
+        if not info.item:
+            if not info.spec.external and not info.spec.owner:  # built-in library?
+                continue
+            result.append(info)
+            continue
+
+        visited_pkgs = pm.memcache_get("__visited_pkgs", [])
+        if visited_pkgs and info.item.path in visited_pkgs:
+            continue
+        visited_pkgs.append(info.item.path)
+        pm.memcache_set("__visited_pkgs", visited_pkgs)
+
+        result.append(
+            PackageInfo(
+                info.spec,
+                info.item,
+                (
+                    build_package_info(
+                        pm,
+                        specs=[
+                            pm.dependency_to_spec(item)
+                            for item in pm.get_pkg_dependencies(info.item)
+                        ],
+                        filter_specs=filter_specs,
+                        resolve_dependencies=True,
+                    )
+                    if resolve_dependencies and pm.get_pkg_dependencies(info.item)
+                    else []
                 ),
             )
         )
 
-        dependencies = pm.get_pkg_dependencies(pkg)
-        if dependencies:
-            print_dependency_tree(
-                pm,
-                specs=[pm.dependency_to_spec(item) for item in dependencies],
-                filter_specs=filter_specs,
-                level=level + 1,
-                verbose=verbose,
-            )
+    return result
 
 
-def _pkg_tree_contains(pm, root: PackageItem, children: List[PackageItem]):
+def _pkg_tree_contains(pm, root: PackageItem, children: list[PackageItem]):
     if root in children:
         return True
     for dependency in pm.get_pkg_dependencies(root) or []:
@@ -139,6 +176,7 @@ def list_global_packages(options):
     only_packages = any(
         options.get(typex) or options.get(f"only_{typex}") for (typex, _) in data
     )
+    result = {}
     for typex, pm in data:
         skip_conds = [
             only_packages
@@ -148,82 +186,115 @@ def list_global_packages(options):
         ]
         if any(skip_conds):
             continue
-        click.secho(typex.capitalize(), bold=True)
-        print_dependency_tree(
-            pm, filter_specs=options.get(typex), verbose=options.get("verbose")
-        )
-        click.echo()
+        result[typex] = build_package_info(pm, filter_specs=options.get(typex))
+
+    return result
 
 
 def list_project_packages(options):
     environments = options["environments"]
-    only_packages = any(
+    only_filtered_packages = any(
         options.get(typex) or options.get(f"only_{typex}")
         for typex in ("platforms", "tools", "libraries")
     )
-    only_platform_packages = any(
-        options.get(typex) or options.get(f"only_{typex}")
-        for typex in ("platforms", "tools")
-    )
+    only_platform_package = options.get("platforms") or options.get("only_platforms")
+    only_tool_packages = options.get("tools") or options.get("only_tools")
     only_library_packages = options.get("libraries") or options.get("only_libraries")
 
+    result = {}
     with fs.cd(options["project_dir"]):
         config = ProjectConfig.get_instance()
         config.validate(environments)
         for env in config.envs():
             if environments and env not in environments:
                 continue
-            click.echo("Resolving %s dependencies..." % click.style(env, fg="cyan"))
-            found = False
-            if not only_packages or only_platform_packages:
-                _found = print_project_env_platform_packages(env, options)
-                found = found or _found
-            if not only_packages or only_library_packages:
-                _found = print_project_env_library_packages(env, options)
-                found = found or _found
-            if not found:
-                click.echo("No packages")
-            if (not environments and len(config.envs()) > 1) or len(environments) > 1:
-                click.echo()
+            result[env] = {}
+            if not only_filtered_packages or only_platform_package:
+                result[env]["platforms"] = list_project_env_platform_package(
+                    env, options
+                )
+            if not only_filtered_packages or only_tool_packages:
+                result[env]["tools"] = list_project_env_tool_packages(env, options)
+            if not only_filtered_packages or only_library_packages:
+                result[env]["libraries"] = list_project_env_library_packages(
+                    env, options
+                )
+
+    return result
 
 
-def print_project_env_platform_packages(project_env, options):
-    try:
-        p = PlatformFactory.from_env(project_env)
-    except UnknownPlatform:
-        return None
-    click.echo(
-        "Platform %s"
-        % (
-            humanize_package(
-                PlatformPackageManager().get_package(p.get_dir()),
-                p.config.get(f"env:{project_env}", "platform"),
-                verbose=options.get("verbose"),
-            )
-        )
+def list_project_env_platform_package(project_env, options):
+    pm = PlatformPackageManager()
+    return build_package_info(
+        pm,
+        specs=[PackageSpec(pm.config.get(f"env:{project_env}", "platform"))],
+        filter_specs=options.get("platforms"),
+        resolve_dependencies=False,
     )
-    print_dependency_tree(
+
+
+def list_project_env_tool_packages(project_env, options):
+    try:
+        p = PlatformFactory.from_env(project_env, targets=["upload"])
+    except UnknownPlatform:
+        return []
+
+    return build_package_info(
         p.pm,
-        specs=[p.get_package_spec(name) for name in p.packages],
+        specs=[
+            p.get_package_spec(name)
+            for name, options in p.packages.items()
+            if not options.get("optional")
+        ],
         filter_specs=options.get("tools"),
     )
-    click.echo()
-    return True
 
 
-def print_project_env_library_packages(project_env, options):
+def list_project_env_library_packages(project_env, options):
     config = ProjectConfig.get_instance()
     lib_deps = config.get(f"env:{project_env}", "lib_deps")
     lm = LibraryPackageManager(
         os.path.join(config.get("platformio", "libdeps_dir"), project_env)
     )
-    if not lib_deps or not lm.get_installed():
-        return None
-    click.echo("Libraries")
-    print_dependency_tree(
+    return build_package_info(
         lm,
         lib_deps,
         filter_specs=options.get("libraries"),
-        verbose=options.get("verbose"),
     )
-    return True
+
+
+def humanize_package(info, verbose=False):
+    data = (
+        [
+            click.style(info.item.metadata.name, fg="cyan"),
+            click.style(f"@ {str(info.item.metadata.version)}", bold=True),
+        ]
+        if info.item
+        else ["Not installed"]
+    )
+    extra_data = ["required: %s" % (info.spec.humanize() if info.spec else "Any")]
+    if verbose and info.item:
+        extra_data.append(info.item.path)
+    data.append("(%s)" % ", ".join(extra_data))
+    return " ".join(data)
+
+
+def print_dependency_tree(items, verbose=False, level=0):
+    for index, info in enumerate(items):
+        click.echo(
+            "%s%s %s"
+            % (
+                "│   " * level,
+                "├──" if index < len(items) - 1 else "└──",
+                humanize_package(
+                    info,
+                    verbose=verbose,
+                ),
+            )
+        )
+        if info.dependencies:
+            print_dependency_tree(
+                info.dependencies,
+                verbose=verbose,
+                level=level + 1,
+            )

platformio/package/commands/publish.py

@@ -86,8 +86,10 @@ def package_publish_cmd(  # pylint: disable=too-many-arguments, too-many-locals
     package, owner, typex, released_at, private, notify, no_interactive, non_interactive
 ):
     click.secho("Preparing a package...", fg="cyan")
+    package = os.path.abspath(package)
     no_interactive = no_interactive or non_interactive
-    owner = owner or AccountClient().get_logged_username()
+    with AccountClient() as client:
+        owner = owner or client.get_logged_username()
     do_not_pack = (
         not os.path.isdir(package)
         and isinstance(FileUnpacker.new_archiver(package), TARArchiver)
@@ -145,9 +147,10 @@ def package_publish_cmd(  # pylint: disable=too-many-arguments, too-many-locals
             fg="yellow",
         )
         click.echo("Publishing...")
-        response = RegistryClient().publish_package(
-            owner, typex, archive_path, released_at, private, notify
-        )
+        with RegistryClient() as client:
+            response = client.publish_package(
+                owner, typex, archive_path, released_at, private, notify
+            )
         if not do_not_pack:
             os.remove(archive_path)
         click.secho(response.get("message"), fg="green")

platformio/package/commands/search.py

@@ -29,8 +29,8 @@ from platformio.registry.client import RegistryClient
     type=click.Choice(["relevance", "popularity", "trending", "added", "updated"]),
 )
 def package_search_cmd(query, page, sort):
-    client = RegistryClient()
-    result = client.list_packages(query, page=page, sort=sort)
+    with RegistryClient() as client:
+        result = client.list_packages(query, page=page, sort=sort)
     if not result["total"]:
         click.secho("Nothing has been found by your request", fg="yellow")
         click.echo(
@@ -65,10 +65,12 @@ def print_search_item(item):
     click.echo(
         "%s • %s • Published on %s"
         % (
-            item["type"].capitalize()
-            if item["tier"] == "community"
-            else click.style(
-                ("%s %s" % (item["tier"], item["type"])).title(), bold=True
+            (
+                item["type"].capitalize()
+                if item["tier"] == "community"
+                else click.style(
+                    ("%s %s" % (item["tier"], item["type"])).title(), bold=True
+                )
             ),
             item["version"]["name"],
             util.parse_datetime(item["version"]["released_at"]).strftime("%c"),

platformio/package/commands/show.py

@@ -124,31 +124,31 @@ def package_show_cmd(spec, pkg_type):
 
 def fetch_package_data(spec, pkg_type=None):
     assert isinstance(spec, PackageSpec)
-    client = RegistryClient()
-    if pkg_type and spec.owner and spec.name:
+    with RegistryClient() as client:
+        if pkg_type and spec.owner and spec.name:
+            return client.get_package(
+                pkg_type, spec.owner, spec.name, version=spec.requirements
+            )
+        qualifiers = {}
+        if spec.id:
+            qualifiers["ids"] = str(spec.id)
+        if spec.name:
+            qualifiers["names"] = spec.name.lower()
+        if pkg_type:
+            qualifiers["types"] = pkg_type
+        if spec.owner:
+            qualifiers["owners"] = spec.owner.lower()
+        packages = client.list_packages(qualifiers=qualifiers)["items"]
+        if not packages:
+            return None
+        if len(packages) > 1:
+            PackageManagerRegistryMixin.print_multi_package_issue(
+                click.echo, packages, spec
+            )
+            return None
         return client.get_package(
-            pkg_type, spec.owner, spec.name, version=spec.requirements
+            packages[0]["type"],
+            packages[0]["owner"]["username"],
+            packages[0]["name"],
+            version=spec.requirements,
         )
-    qualifiers = {}
-    if spec.id:
-        qualifiers["ids"] = str(spec.id)
-    if spec.name:
-        qualifiers["names"] = spec.name.lower()
-    if pkg_type:
-        qualifiers["types"] = pkg_type
-    if spec.owner:
-        qualifiers["owners"] = spec.owner.lower()
-    packages = client.list_packages(qualifiers=qualifiers)["items"]
-    if not packages:
-        return None
-    if len(packages) > 1:
-        PackageManagerRegistryMixin.print_multi_package_issue(
-            click.echo, packages, spec
-        )
-        return None
-    return client.get_package(
-        packages[0]["type"],
-        packages[0]["owner"]["username"],
-        packages[0]["name"],
-        version=spec.requirements,
-    )

platformio/package/commands/unpublish.py

@@ -36,11 +36,14 @@ from platformio.registry.client import RegistryClient
 )
 def package_unpublish_cmd(package, type, undo):  # pylint: disable=redefined-builtin
     spec = PackageSpec(package)
-    response = RegistryClient().unpublish_package(
-        owner=spec.owner or AccountClient().get_logged_username(),
-        type=type,
-        name=spec.name,
-        version=str(spec.requirements),
-        undo=undo,
-    )
-    click.secho(response.get("message"), fg="green")
+    with AccountClient() as client:
+        owner = spec.owner or client.get_logged_username()
+    with RegistryClient() as client:
+        response = client.unpublish_package(
+            owner=owner,
+            type=type,
+            name=spec.name,
+            version=str(spec.requirements),
+            undo=undo,
+        )
+        click.secho(response.get("message"), fg="green")

platformio/package/download.py

@@ -12,48 +12,28 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import io
+import os
+import tempfile
+import time
 from email.utils import parsedate
-from os.path import getsize, join
-from time import mktime
+from urllib.parse import urlparse
 
 import click
+import httpx
 
 from platformio import fs
 from platformio.compat import is_terminal
-from platformio.http import HTTPSession
+from platformio.http import apply_default_kwargs
 from platformio.package.exception import PackageException
 
 
 class FileDownloader:
-    def __init__(self, url, dest_dir=None):
-        self._http_session = HTTPSession()
-        self._http_response = None
-        # make connection
-        self._http_response = self._http_session.get(
-            url,
-            stream=True,
-        )
-        if self._http_response.status_code != 200:
-            raise PackageException(
-                "Got the unrecognized status code '{0}' when downloaded {1}".format(
-                    self._http_response.status_code, url
-                )
-            )
+    def __init__(self, url, dst_dir=None):
+        self.url = url
+        self.dst_dir = dst_dir
 
-        disposition = self._http_response.headers.get("content-disposition")
-        if disposition and "filename=" in disposition:
-            self._fname = (
-                disposition[disposition.index("filename=") + 9 :]
-                .replace('"', "")
-                .replace("'", "")
-            )
-        else:
-            self._fname = [p for p in url.split("/") if p][-1]
-        self._fname = str(self._fname)
-        self._destination = self._fname
-        if dest_dir:
-            self.set_destination(join(dest_dir, self._fname))
+        self._destination = None
+        self._http_response = None
 
     def set_destination(self, destination):
         self._destination = destination
@@ -69,18 +49,34 @@ class FileDownloader:
             return -1
         return int(self._http_response.headers["content-length"])
 
+    def get_disposition_filname(self):
+        disposition = self._http_response.headers.get("content-disposition")
+        if disposition and "filename=" in disposition:
+            return (
+                disposition[disposition.index("filename=") + 9 :]
+                .replace('"', "")
+                .replace("'", "")
+            )
+        return [p for p in urlparse(self.url).path.split("/") if p][-1]
+
     def start(self, with_progress=True, silent=False):
         label = "Downloading"
-        file_size = self.get_size()
-        itercontent = self._http_response.iter_content(
-            chunk_size=io.DEFAULT_BUFFER_SIZE
-        )
-        try:
+        with httpx.stream("GET", self.url, **apply_default_kwargs()) as response:
+            if response.status_code != 200:
+                raise PackageException(
+                    f"Got the unrecognized status code '{response.status_code}' "
+                    "when downloading '{self.url}'"
+                )
+            self._http_response = response
+            total_size = self.get_size()
+            if not self._destination:
+                assert self.dst_dir
+
             with open(self._destination, "wb") as fp:
-                if file_size == -1 or not with_progress or silent:
+                if total_size == -1 or not with_progress or silent:
                     if not silent:
                         click.echo(f"{label}...")
-                    for chunk in itercontent:
+                    for chunk in response.iter_bytes():
                         fp.write(chunk)
 
                 elif not is_terminal():
@@ -88,10 +84,10 @@ class FileDownloader:
                     print_percent_step = 10
                     printed_percents = 0
                     downloaded_size = 0
-                    for chunk in itercontent:
+                    for chunk in response.iter_bytes():
                         fp.write(chunk)
                         downloaded_size += len(chunk)
-                        if (downloaded_size / file_size * 100) >= (
+                        if (downloaded_size / total_size * 100) >= (
                             printed_percents + print_percent_step
                         ):
                             printed_percents += print_percent_step
@@ -100,33 +96,39 @@ class FileDownloader:
 
                 else:
                     with click.progressbar(
-                        length=file_size,
-                        iterable=itercontent,
+                        length=total_size,
+                        iterable=response.iter_bytes(),
                         label=label,
                         update_min_steps=min(
-                            256 * 1024, file_size / 100
+                            256 * 1024, total_size / 100
                         ),  # every 256Kb or less
                     ) as pb:
                         for chunk in pb:
                             pb.update(len(chunk))
                             fp.write(chunk)
-        finally:
-            self._http_response.close()
-            self._http_session.close()
 
-        if self.get_lmtime():
-            self._preserve_filemtime(self.get_lmtime())
+        last_modified = self.get_lmtime()
+        if last_modified:
+            self._preserve_filemtime(last_modified)
 
         return True
 
+    def _set_tmp_destination(self):
+        dst_dir = self.dst_dir or tempfile.mkdtemp()
+        self.set_destination(os.path.join(dst_dir, self.get_disposition_filname()))
+
+    def _preserve_filemtime(self, lmdate):
+        lmtime = time.mktime(parsedate(lmdate))
+        fs.change_filemtime(self._destination, lmtime)
+
     def verify(self, checksum=None):
-        _dlsize = getsize(self._destination)
-        if self.get_size() != -1 and _dlsize != self.get_size():
+        remote_size = self.get_size()
+        downloaded_size = os.path.getsize(self._destination)
+        if remote_size not in (-1, downloaded_size):
             raise PackageException(
-                (
-                    "The size ({0:d} bytes) of downloaded file '{1}' "
-                    "is not equal to remote size ({2:d} bytes)"
-                ).format(_dlsize, self._fname, self.get_size())
+                f"The size ({downloaded_size} bytes) of downloaded file "
+                f"'{self._destination}' is not equal to remote size "
+                f"({remote_size} bytes)"
             )
         if not checksum:
             return True
@@ -142,7 +144,7 @@ class FileDownloader:
 
         if not hash_algo:
             raise PackageException(
-                "Could not determine checksum algorithm by %s" % checksum
+                f"Could not determine checksum algorithm by {checksum}"
             )
 
         dl_checksum = fs.calculate_file_hashsum(hash_algo, self._destination)
@@ -150,16 +152,7 @@ class FileDownloader:
             raise PackageException(
                 "The checksum '{0}' of the downloaded file '{1}' "
                 "does not match to the remote '{2}'".format(
-                    dl_checksum, self._fname, checksum
+                    dl_checksum, self._destination, checksum
                 )
             )
         return True
-
-    def _preserve_filemtime(self, lmdate):
-        lmtime = mktime(parsedate(lmdate))
-        fs.change_filemtime(self._destination, lmtime)
-
-    def __del__(self):
-        self._http_session.close()
-        if self._http_response:
-            self._http_response.close()

platformio/package/manager/_install.py

@@ -98,9 +98,13 @@ class PackageManagerInstallMixin:
         else:
             pkg = self.install_from_registry(
                 spec,
-                search_qualifiers=compatibility.to_search_qualifiers()
-                if compatibility
-                else None,
+                search_qualifiers=(
+                    compatibility.to_search_qualifiers(
+                        ["platforms", "frameworks", "authors"]
+                    )
+                    if compatibility
+                    else None
+                ),
             )
 
         if not pkg or not pkg.metadata:

platformio/package/manager/_registry.py

@@ -15,6 +15,7 @@
 import time
 
 import click
+import httpx
 
 from platformio.package.exception import UnknownPackageError
 from platformio.package.meta import PackageSpec
@@ -57,7 +58,7 @@ class PackageManagerRegistryMixin:
                     ),
                     checksum or pkgfile["checksum"]["sha256"],
                 )
-            except Exception as exc:  # pylint: disable=broad-except
+            except httpx.HTTPError as exc:
                 self.log.warning(
                     click.style("Warning! Package Mirror: %s" % exc, fg="yellow")
                 )

platformio/package/manager/base.py

@@ -280,11 +280,15 @@ class BasePackageManager(  # pylint: disable=too-many-public-methods,too-many-in
 
         # external "URL" mismatch
         if spec.external:
-            # local folder mismatch
-            if os.path.abspath(spec.uri) == os.path.abspath(pkg.path) or (
+            # local/symlinked folder mismatch
+            check_conds = [
+                os.path.abspath(spec.uri) == os.path.abspath(pkg.path),
                 spec.uri.startswith("file://")
-                and os.path.abspath(pkg.path) == os.path.abspath(spec.uri[7:])
-            ):
+                and os.path.abspath(pkg.path) == os.path.abspath(spec.uri[7:]),
+                spec.uri.startswith("symlink://")
+                and os.path.abspath(pkg.path) == os.path.abspath(spec.uri[10:]),
+            ]
+            if any(check_conds):
                 return True
             if spec.uri != pkg.metadata.spec.uri:
                 return False

platformio/package/manager/core.py

@@ -14,7 +14,8 @@
 
 import os
 
-from platformio import __core_packages__, exception
+from platformio import exception
+from platformio.dependencies import get_core_dependencies
 from platformio.package.exception import UnknownPackageError
 from platformio.package.manager.tool import ToolPackageManager
 from platformio.package.meta import PackageSpec
@@ -23,7 +24,7 @@ from platformio.package.meta import PackageSpec
 def get_installed_core_packages():
     result = []
     pm = ToolPackageManager()
-    for name, requirements in __core_packages__.items():
+    for name, requirements in get_core_dependencies().items():
         spec = PackageSpec(owner="platformio", name=name, requirements=requirements)
         pkg = pm.get_package(spec)
         if pkg:
@@ -32,11 +33,11 @@ def get_installed_core_packages():
 
 
 def get_core_package_dir(name, spec=None, auto_install=True):
-    if name not in __core_packages__:
+    if name not in get_core_dependencies():
         raise exception.PlatformioException("Please upgrade PlatformIO Core")
     pm = ToolPackageManager()
     spec = spec or PackageSpec(
-        owner="platformio", name=name, requirements=__core_packages__[name]
+        owner="platformio", name=name, requirements=get_core_dependencies()[name]
     )
     pkg = pm.get_package(spec)
     if pkg:
@@ -50,7 +51,7 @@ def get_core_package_dir(name, spec=None, auto_install=True):
 
 def update_core_packages():
     pm = ToolPackageManager()
-    for name, requirements in __core_packages__.items():
+    for name, requirements in get_core_dependencies().items():
         spec = PackageSpec(owner="platformio", name=name, requirements=requirements)
         try:
             pm.update(spec, spec)
@@ -65,7 +66,7 @@ def remove_unnecessary_core_packages(dry_run=False):
     pm = ToolPackageManager()
     best_pkg_versions = {}
 
-    for name, requirements in __core_packages__.items():
+    for name, requirements in get_core_dependencies().items():
         spec = PackageSpec(owner="platformio", name=name, requirements=requirements)
         pkg = pm.get_package(spec)
         if not pkg:

platformio/package/manager/platform.py

@@ -15,7 +15,7 @@
 import os
 
 from platformio import util
-from platformio.http import HTTPClientError, InternetConnectionError
+from platformio.http import HttpClientApiError, InternetConnectionError
 from platformio.package.exception import UnknownPackageError
 from platformio.package.manager.base import BasePackageManager
 from platformio.package.manager.core import get_installed_core_packages
@@ -128,7 +128,7 @@ class PlatformPackageManager(BasePackageManager):  # pylint: disable=too-many-an
                 key = "%s:%s" % (board["platform"], board["id"])
                 if key not in know_boards:
                     boards.append(board)
-        except (HTTPClientError, InternetConnectionError):
+        except (HttpClientApiError, InternetConnectionError):
             pass
         return sorted(boards, key=lambda b: b["name"])
 

platformio/package/manifest/parser.py

@@ -22,7 +22,7 @@ from urllib.parse import urlparse
 
 from platformio import util
 from platformio.compat import get_object_members, string_types
-from platformio.http import fetch_remote_content
+from platformio.http import fetch_http_content
 from platformio.package.exception import ManifestParserError, UnknownManifestError
 from platformio.project.helpers import is_platformio_project
 
@@ -103,7 +103,7 @@ class ManifestParserFactory:
 
     @staticmethod
     def new_from_url(remote_url):
-        content = fetch_remote_content(remote_url)
+        content = fetch_http_content(remote_url)
         return ManifestParserFactory.new(
             content,
             ManifestFileType.from_uri(remote_url) or ManifestFileType.LIBRARY_JSON,
@@ -294,9 +294,11 @@ class BaseManifestParser:
             if not matched_files:
                 continue
             result[root] = dict(
-                name="Examples"
-                if root == examples_dir
-                else os.path.relpath(root, examples_dir),
+                name=(
+                    "Examples"
+                    if root == examples_dir
+                    else os.path.relpath(root, examples_dir)
+                ),
                 base=os.path.relpath(root, package_dir),
                 files=matched_files,
             )

platformio/package/manifest/schema.py

@@ -17,12 +17,12 @@
 import json
 import re
 
+import httpx
 import marshmallow
-import requests
 import semantic_version
 from marshmallow import Schema, ValidationError, fields, validate, validates
 
-from platformio.http import fetch_remote_content
+from platformio.http import fetch_http_content
 from platformio.package.exception import ManifestValidationError
 from platformio.util import memoized
 
@@ -252,7 +252,7 @@ class ManifestSchema(BaseSchema):
     def validate_license(self, value):
         try:
             spdx = self.load_spdx_licenses()
-        except requests.exceptions.RequestException as exc:
+        except httpx.HTTPError as exc:
             raise ValidationError(
                 "Could not load SPDX licenses for validation"
             ) from exc
@@ -276,9 +276,9 @@ class ManifestSchema(BaseSchema):
     @staticmethod
     @memoized(expire="1h")
     def load_spdx_licenses():
-        version = "3.22"
+        version = "3.23"
         spdx_data_url = (
             "https://raw.githubusercontent.com/spdx/license-list-data/"
             f"v{version}/json/licenses.json"
         )
-        return json.loads(fetch_remote_content(spdx_data_url))
+        return json.loads(fetch_http_content(spdx_data_url))

platformio/package/meta.py

@@ -23,7 +23,7 @@ import semantic_version
 
 from platformio import fs
 from platformio.compat import get_object_members, hashlib_encode_data, string_types
-from platformio.package.manifest.parser import ManifestFileType
+from platformio.package.manifest.parser import ManifestFileType, ManifestParserFactory
 from platformio.package.version import SemanticVersionError, cast_version_to_semver
 from platformio.util import items_in_list
 
@@ -65,7 +65,14 @@ class PackageType:
 
 
 class PackageCompatibility:
-    KNOWN_QUALIFIERS = ("platforms", "frameworks", "authors")
+    KNOWN_QUALIFIERS = (
+        "owner",
+        "name",
+        "version",
+        "platforms",
+        "frameworks",
+        "authors",
+    )
 
     @classmethod
     def from_dependency(cls, dependency):
@@ -89,19 +96,45 @@ class PackageCompatibility:
     def __repr__(self):
         return "PackageCompatibility <%s>" % self.qualifiers
 
-    def to_search_qualifiers(self):
-        return self.qualifiers
+    def to_search_qualifiers(self, fields=None):
+        result = {}
+        for name, value in self.qualifiers.items():
+            if not fields or name in fields:
+                result[name] = value
+        return result
 
     def is_compatible(self, other):
         assert isinstance(other, PackageCompatibility)
-        for key, value in self.qualifiers.items():
+        for key, current_value in self.qualifiers.items():
             other_value = other.qualifiers.get(key)
-            if not value or not other_value:
+            if not current_value or not other_value:
                 continue
-            if not items_in_list(value, other_value):
+            if any(isinstance(v, list) for v in (current_value, other_value)):
+                if not items_in_list(current_value, other_value):
+                    return False
+                continue
+            if key == "version":
+                if not self._compare_versions(current_value, other_value):
+                    return False
+                continue
+            if current_value != other_value:
                 return False
         return True
 
+    def _compare_versions(self, current, other):
+        if current == other:
+            return True
+        try:
+            version = (
+                other
+                if isinstance(other, semantic_version.Version)
+                else cast_version_to_semver(other)
+            )
+            return version in semantic_version.SimpleSpec(current)
+        except ValueError:
+            pass
+        return False
+
 
 class PackageOutdatedResult:
     UPDATE_INCREMENT_MAJOR = "major"
@@ -485,9 +518,11 @@ class PackageItem:
 
     def __eq__(self, other):
         conds = [
-            os.path.realpath(self.path) == os.path.realpath(other.path)
-            if self.path and other.path
-            else self.path == other.path,
+            (
+                os.path.realpath(self.path) == os.path.realpath(other.path)
+                if self.path and other.path
+                else self.path == other.path
+            ),
             self.metadata == other.metadata,
         ]
         return all(conds)
@@ -526,3 +561,29 @@ class PackageItem:
                 break
         assert location
         return self.metadata.dump(os.path.join(location, self.METAFILE_NAME))
+
+    def as_dict(self):
+        return {"path": self.path, "metadata": self.metadata.as_dict()}
+
+
+class PackageInfo:
+
+    def __init__(self, spec: PackageSpec, item: PackageItem = None, dependencies=None):
+        assert isinstance(spec, PackageSpec)
+        self.spec = spec
+        self.item = item
+        self.dependencies = dependencies or []
+
+    def as_dict(self, with_manifest=False):
+        result = {
+            "spec": self.spec.as_dict(),
+            "item": self.item.as_dict() if self.item else None,
+            "dependencies": [d.as_dict() for d in self.dependencies],
+        }
+        if with_manifest:
+            result["manifest"] = (
+                ManifestParserFactory.new_from_dir(self.item.path).as_dict()
+                if self.item
+                else None
+            )
+        return result

platformio/package/unpack.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 import os
+import sys
 from tarfile import open as tarfile_open
 from time import mktime
 from zipfile import ZipFile
@@ -82,19 +83,23 @@ class TARArchiver(BaseArchiver):
         ).startswith(base)
 
     def extract_item(self, item, dest_dir):
+        if sys.version_info >= (3, 12):
+            self._afo.extract(item, dest_dir, filter="data")
+            return self.after_extract(item, dest_dir)
+
+        # apply custom security logic
         dest_dir = self.resolve_path(dest_dir)
         bad_conds = [
             self.is_bad_path(item.name, dest_dir),
             self.is_link(item) and self.is_bad_link(item, dest_dir),
         ]
-        if not any(bad_conds):
-            super().extract_item(item, dest_dir)
-        else:
-            click.secho(
+        if any(bad_conds):
+            return click.secho(
                 "Blocked insecure item `%s` from TAR archive" % item.name,
                 fg="red",
                 err=True,
             )
+        return super().extract_item(item, dest_dir)
 
 
 class ZIPArchiver(BaseArchiver):

platformio/platform/_run.py

@@ -116,9 +116,9 @@ class PlatformRunMixin:
                 args,
                 stdout=proc.BuildAsyncPipe(
                     line_callback=self._on_stdout_line,
-                    data_callback=lambda data: None
-                    if self.silent
-                    else _write_and_flush(sys.stdout, data),
+                    data_callback=lambda data: (
+                        None if self.silent else _write_and_flush(sys.stdout, data)
+                    ),
                 ),
                 stderr=proc.BuildAsyncPipe(
                     line_callback=self._on_stderr_line,

platformio/platform/factory.py

@@ -16,6 +16,8 @@ import os
 import re
 import sys
 
+import httpx
+
 from platformio import fs
 from platformio.compat import load_python_module
 from platformio.package.meta import PackageItem
@@ -31,13 +33,16 @@ class PlatformFactory:
         name = re.sub(r"[^\da-z\_]+", "", name, flags=re.I)
         return "%sPlatform" % name.lower().capitalize()
 
-    @staticmethod
-    def load_platform_module(name, path):
+    @classmethod
+    def load_platform_module(cls, name, path):
         # backward compatibiility with the legacy dev-platforms
         sys.modules["platformio.managers.platform"] = base
         try:
             return load_python_module("platformio.platform.%s" % name, path)
         except ImportError as exc:
+            if exc.name == "requests" and not sys.modules.get("requests"):
+                sys.modules["requests"] = httpx
+                return cls.load_platform_module(name, path)
             raise UnknownPlatform(name) from exc
 
     @classmethod

platformio/project/commands/config.py

@@ -82,9 +82,11 @@ def lint_configuration(json_output=False):
                     (
                         click.style(error["type"], fg="red"),
                         error["message"],
-                        error.get("source", "") + (f":{error.get('lineno')}")
-                        if "lineno" in error
-                        else "",
+                        (
+                            error.get("source", "") + (f":{error.get('lineno')}")
+                            if "lineno" in error
+                            else ""
+                        ),
                     )
                     for error in errors
                 ],

platformio/project/commands/init.py

@@ -79,6 +79,7 @@ def project_init_cmd(
     env_prefix,
     silent,
 ):
+    project_dir = os.path.abspath(project_dir)
     is_new_project = not is_platformio_project(project_dir)
     if is_new_project:
         if not silent:
@@ -223,7 +224,7 @@ def init_lib_readme(lib_dir):
 This directory is intended for project specific (private) libraries.
 PlatformIO will compile them to static libraries and link into executable file.
 
-The source code of each library should be placed in a an own separate directory
+The source code of each library should be placed in an own separate directory
 ("lib/your_library_name/[here are source files]").
 
 For example, see a structure of the following two libraries `Foo` and `Bar`:

platformio/project/config.py

@@ -164,6 +164,7 @@ class ProjectConfigBase:
 
     @staticmethod
     def get_section_scope(section):
+        assert section
         return section.split(":", 1)[0] if ":" in section else section
 
     def walk_options(self, root_section):
@@ -343,8 +344,11 @@ class ProjectConfigBase:
         section, option = match.group(1), match.group(2)
 
         # handle built-in variables
-        if section is None and option in self.BUILTIN_VARS:
-            return self.BUILTIN_VARS[option]()
+        if section is None:
+            if option in self.BUILTIN_VARS:
+                return self.BUILTIN_VARS[option]()
+            # SCons varaibles
+            return f"${{{option}}}"
 
         # handle system environment variables
         if section == "sysenv":

platformio/project/helpers.py

@@ -12,22 +12,29 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import hashlib
 import os
 import re
 import subprocess
-from hashlib import sha1
 
 from click.testing import CliRunner
 
 from platformio import __version__, exception, fs
 from platformio.compat import IS_MACOS, IS_WINDOWS, hashlib_encode_data
 from platformio.project.config import ProjectConfig
+from platformio.project.options import ProjectOptions
 
 
 def get_project_dir():
     return os.getcwd()
 
 
+def get_project_id(project_dir=None):
+    return hashlib.sha1(
+        hashlib_encode_data(project_dir or get_project_dir())
+    ).hexdigest()
+
+
 def is_platformio_project(project_dir=None):
     if not project_dir:
         project_dir = get_project_dir()
@@ -92,7 +99,7 @@ def get_default_projects_dir():
 
 def compute_project_checksum(config):
     # rebuild when PIO Core version changes
-    checksum = sha1(hashlib_encode_data(__version__))
+    checksum = hashlib.sha1(hashlib_encode_data(__version__))
 
     # configuration file state
     config_data = config.to_json()
@@ -131,27 +138,27 @@ def compute_project_checksum(config):
     return checksum.hexdigest()
 
 
-def load_build_metadata(project_dir, env_or_envs, cache=False, debug=False):
-    assert env_or_envs
-    env_names = env_or_envs
-    if not isinstance(env_names, list):
-        env_names = [env_names]
+def get_build_type(config, env, run_targets=None):
+    types = []
+    run_targets = run_targets or []
+    env_build_type = config.get(f"env:{env}", "build_type")
+    if set(["__debug", "__memusage"]) & set(run_targets) or env_build_type == "debug":
+        types.append("debug")
+    if "__test" in run_targets or env_build_type == "test":
+        types.append("test")
+    return ", ".join(types or [ProjectOptions["env.build_type"].default])
 
-    with fs.cd(project_dir):
-        result = _get_cached_build_metadata(project_dir, env_names) if cache else {}
-        # incompatible build-type data
-        for name in list(result.keys()):
-            build_type = result[name].get("build_type", "")
-            outdated_conds = [
-                not build_type,
-                debug and "debug" not in build_type,
-                not debug and "debug" in build_type,
-            ]
-            if any(outdated_conds):
-                del result[name]
-        missed_env_names = set(env_names) - set(result.keys())
-        if missed_env_names:
-            result.update(_load_build_metadata(project_dir, missed_env_names, debug))
+
+def load_build_metadata(project_dir, env_or_envs, cache=False, force_targets=None):
+    assert env_or_envs
+    envs = env_or_envs
+    if not isinstance(envs, list):
+        envs = [envs]
+    with fs.cd(project_dir or os.getcwd()):
+        result = _get_cached_build_metadata(envs, force_targets) if cache else {}
+        missed_envs = set(envs) - set(result.keys())
+        if missed_envs:
+            result.update(_load_build_metadata(missed_envs, force_targets))
 
     if not isinstance(env_or_envs, list) and env_or_envs in result:
         return result[env_or_envs]
@@ -162,16 +169,28 @@ def load_build_metadata(project_dir, env_or_envs, cache=False, debug=False):
 load_project_ide_data = load_build_metadata
 
 
-def _load_build_metadata(project_dir, env_names, debug=False):
+def _get_cached_build_metadata(envs, force_targets=None):
+    config = ProjectConfig.get_instance(os.path.join(os.getcwd(), "platformio.ini"))
+    build_dir = config.get("platformio", "build_dir")
+    result = {}
+    for env in envs:
+        build_type = get_build_type(config, env, force_targets)
+        json_path = os.path.join(build_dir, env, build_type, "metadata.json")
+        if os.path.isfile(json_path):
+            result[env] = fs.load_json(json_path)
+    return result
+
+
+def _load_build_metadata(envs, force_targets=None):
     # pylint: disable=import-outside-toplevel
     from platformio import app
     from platformio.run.cli import cli as cmd_run
 
-    args = ["--project-dir", project_dir, "--target", "__idedata"]
-    if debug:
-        args.extend(["--target", "__debug"])
-    for name in env_names:
-        args.extend(["-e", name])
+    args = ["--target", "__metadata"]
+    for target in force_targets or []:
+        args.extend(["--target", target])
+    for env in envs:
+        args.extend(["-e", env])
     app.set_session_var("pause_telemetry", True)
     result = CliRunner().invoke(cmd_run, args)
     app.set_session_var("pause_telemetry", False)
@@ -179,18 +198,6 @@ def _load_build_metadata(project_dir, env_names, debug=False):
         result.exception, exception.ReturnErrorCode
     ):
         raise result.exception
-    if '"includes":' not in result.output:
+    if "Metadata has been saved to the following location" not in result.output:
         raise exception.UserSideException(result.output)
-    return _get_cached_build_metadata(project_dir, env_names)
-
-
-def _get_cached_build_metadata(project_dir, env_names):
-    build_dir = ProjectConfig.get_instance(
-        os.path.join(project_dir, "platformio.ini")
-    ).get("platformio", "build_dir")
-    result = {}
-    for name in env_names:
-        if not os.path.isfile(os.path.join(build_dir, name, "idedata.json")):
-            continue
-        result[name] = fs.load_json(os.path.join(build_dir, name, "idedata.json"))
-    return result
+    return _get_cached_build_metadata(envs, force_targets)

platformio/project/integration/generator.py

@@ -91,9 +91,11 @@ class ProjectGenerator:
             "default_debug_env_name": get_default_debug_env(self.config),
             "env_name": self.env_name,
             "user_home_dir": os.path.abspath(fs.expanduser("~")),
-            "platformio_path": sys.argv[0]
-            if os.path.isfile(sys.argv[0])
-            else where_is_program("platformio"),
+            "platformio_path": (
+                sys.argv[0]
+                if os.path.isfile(sys.argv[0])
+                else where_is_program("platformio")
+            ),
             "env_path": os.getenv("PATH"),
             "env_pathsep": os.pathsep,
         }

platformio/project/memusage.py

@@ -0,0 +1,58 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import gzip
+import json
+import os
+import time
+
+from platformio import fs
+from platformio.project.config import ProjectConfig
+
+
+def get_report_dir(project_dir, env):
+    with fs.cd(project_dir):
+        return os.path.join(
+            ProjectConfig.get_instance().get("platformio", "memusage_dir"), env
+        )
+
+
+def list_reports(report_dir):
+    if not os.path.isdir(report_dir):
+        return []
+    return [os.path.join(report_dir, item) for item in sorted(os.listdir(report_dir))]
+
+
+def read_report(path):
+    with gzip.open(path, mode="rt", encoding="utf8") as fp:
+        return json.load(fp)
+
+
+def save_report(project_dir, env, data):
+    report_dir = get_report_dir(project_dir, env)
+    if not os.path.isdir(report_dir):
+        os.makedirs(report_dir)
+    report_path = os.path.join(report_dir, f"{int(time.time())}.json.gz")
+    with gzip.open(report_path, mode="wt", encoding="utf8") as fp:
+        json.dump(data, fp)
+    rotate_reports(report_dir)
+    return report_path
+
+
+def rotate_reports(report_dir, max_reports=100):
+    reports = os.listdir(report_dir)
+    if len(reports) < max_reports:
+        return
+    for fname in sorted(reports)[0 : len(reports) - max_reports]:
+        os.remove(os.path.join(report_dir, fname))

platformio/project/options.py

@@ -240,6 +240,17 @@ ProjectOptions = OrderedDict(
                 default=os.path.join("${platformio.workspace_dir}", "libdeps"),
                 validate=validate_dir,
             ),
+            ConfigPlatformioOption(
+                group="directory",
+                name="memusage_dir",
+                description=(
+                    "A location where PlatformIO Core will store "
+                    "project memory usage reports"
+                ),
+                sysenvvar="PLATFORMIO_MEMUSAGE_DIR",
+                default=os.path.join("${platformio.workspace_dir}", "memusage"),
+                validate=validate_dir,
+            ),
             ConfigPlatformioOption(
                 group="directory",
                 name="include_dir",
@@ -549,7 +560,7 @@ ProjectOptions = OrderedDict(
             ConfigEnvOption(
                 group="monitor",
                 name="monitor_encoding",
-                description="Custom encoding (e.g. hexlify, Latin1, UTF-8)",
+                description="Custom encoding (e.g. hexlify, Latin-1, UTF-8)",
                 default="UTF-8",
             ),
             # Library

platformio/public.py

@@ -17,6 +17,7 @@
 from platformio.device.list.util import list_logical_devices, list_serial_ports
 from platformio.device.monitor.filters.base import DeviceMonitorFilterBase
 from platformio.fs import to_unix_path
+from platformio.http import fetch_http_content
 from platformio.platform.base import PlatformBase
 from platformio.project.config import ProjectConfig
 from platformio.project.helpers import get_project_watch_lib_dirs, load_build_metadata

platformio/registry/access/commands/grant.py

@@ -31,8 +31,8 @@ from platformio.registry.client import RegistryClient
 )
 @click.option("--urn-type", type=click.Choice(["prn:reg:pkg"]), default="prn:reg:pkg")
 def access_grant_cmd(level, client, urn, urn_type):  # pylint: disable=unused-argument
-    reg_client = RegistryClient()
-    reg_client.grant_access_for_resource(urn=urn, client=client, level=level)
+    with RegistryClient() as reg_client:
+        reg_client.grant_access_for_resource(urn=urn, client=client, level=level)
     return click.secho(
         "Access for resource %s has been granted for %s" % (urn, client),
         fg="green",

platformio/registry/access/commands/list.py

@@ -25,8 +25,8 @@ from platformio.registry.client import RegistryClient
 @click.option("--urn-type", type=click.Choice(["prn:reg:pkg"]), default="prn:reg:pkg")
 @click.option("--json-output", is_flag=True)
 def access_list_cmd(owner, urn_type, json_output):  # pylint: disable=unused-argument
-    reg_client = RegistryClient()
-    resources = reg_client.list_resources(owner=owner)
+    with RegistryClient() as client:
+        resources = client.list_resources(owner=owner)
     if json_output:
         return click.echo(json.dumps(resources))
     if not resources:
@@ -41,9 +41,11 @@ def access_list_cmd(owner, urn_type, json_output):  # pylint: disable=unused-arg
         table_data.append(
             (
                 "Access:",
-                click.style("Private", fg="red")
-                if resource.get("private", False)
-                else "Public",
+                (
+                    click.style("Private", fg="red")
+                    if resource.get("private", False)
+                    else "Public"
+                ),
             )
         )
         table_data.append(

platformio/registry/access/commands/private.py

@@ -25,8 +25,8 @@ from platformio.registry.client import RegistryClient
 )
 @click.option("--urn-type", type=click.Choice(["prn:reg:pkg"]), default="prn:reg:pkg")
 def access_private_cmd(urn, urn_type):  # pylint: disable=unused-argument
-    client = RegistryClient()
-    client.update_resource(urn=urn, private=1)
+    with RegistryClient() as client:
+        client.update_resource(urn=urn, private=1)
     return click.secho(
         "The resource %s has been successfully updated." % urn,
         fg="green",

platformio/registry/access/commands/public.py

@@ -25,8 +25,8 @@ from platformio.registry.client import RegistryClient
 )
 @click.option("--urn-type", type=click.Choice(["prn:reg:pkg"]), default="prn:reg:pkg")
 def access_public_cmd(urn, urn_type):  # pylint: disable=unused-argument
-    client = RegistryClient()
-    client.update_resource(urn=urn, private=0)
+    with RegistryClient() as client:
+        client.update_resource(urn=urn, private=0)
     return click.secho(
         "The resource %s has been successfully updated." % urn,
         fg="green",

platformio/registry/access/commands/revoke.py

@@ -30,8 +30,8 @@ from platformio.registry.client import RegistryClient
 )
 @click.option("--urn-type", type=click.Choice(["prn:reg:pkg"]), default="prn:reg:pkg")
 def access_revoke_cmd(client, urn, urn_type):  # pylint: disable=unused-argument
-    reg_client = RegistryClient()
-    reg_client.revoke_access_from_resource(urn=urn, client=client)
+    with RegistryClient() as reg_client:
+        reg_client.revoke_access_from_resource(urn=urn, client=client)
     return click.secho(
         "Access for resource %s has been revoked for %s" % (urn, client),
         fg="green",

platformio/registry/client.py

@@ -16,13 +16,14 @@
 
 from platformio import __registry_mirror_hosts__, fs
 from platformio.account.client import AccountClient, AccountError
-from platformio.http import HTTPClient, HTTPClientError
+from platformio.http import HttpApiClient, HttpClientApiError
 
 
-class RegistryClient(HTTPClient):
-    def __init__(self):
-        endpoints = [f"https://api.{host}" for host in __registry_mirror_hosts__]
-        super().__init__(endpoints)
+class RegistryClient(HttpApiClient):
+    def __init__(self, endpoints=None):
+        super().__init__(
+            endpoints or [f"https://api.{host}" for host in __registry_mirror_hosts__]
+        )
 
     @staticmethod
     def allowed_private_packages():
@@ -34,7 +35,8 @@ class RegistryClient(HTTPClient):
             ]
         )
         try:
-            info = AccountClient().get_account_info() or {}
+            with AccountClient() as client:
+                info = client.get_account_info() or {}
             for item in info.get("packages", []):
                 if set(item.keys()) & private_permissions:
                     return True
@@ -156,7 +158,7 @@ class RegistryClient(HTTPClient):
                 x_cache_valid="1h",
                 x_with_authorization=self.allowed_private_packages(),
             )
-        except HTTPClientError as exc:
+        except HttpClientApiError as exc:
             if exc.response is not None and exc.response.status_code == 404:
                 return None
             raise exc

platformio/registry/mirror.py

@@ -17,7 +17,6 @@ from urllib.parse import urlparse
 
 from platformio import __registry_mirror_hosts__
 from platformio.cache import ContentCache
-from platformio.http import HTTPClient
 from platformio.registry.client import RegistryClient
 
 
@@ -49,15 +48,17 @@ class RegistryFileMirrorIterator:
                 except (ValueError, KeyError):
                     pass
 
-            http = self.get_http_client()
-            response = http.send_request(
+            registry = self.get_api_client()
+            response = registry.send_request(
                 "head",
                 self._url_parts.path,
-                allow_redirects=False,
-                params=dict(bypass=",".join(self._visited_mirrors))
-                if self._visited_mirrors
-                else None,
-                x_with_authorization=RegistryClient.allowed_private_packages(),
+                follow_redirects=False,
+                params=(
+                    dict(bypass=",".join(self._visited_mirrors))
+                    if self._visited_mirrors
+                    else None
+                ),
+                x_with_authorization=registry.allowed_private_packages(),
             )
             stop_conditions = [
                 response.status_code not in (302, 307),
@@ -85,14 +86,14 @@ class RegistryFileMirrorIterator:
                 response.headers.get("X-PIO-Content-SHA256"),
             )
 
-    def get_http_client(self):
+    def get_api_client(self):
         if self._mirror not in RegistryFileMirrorIterator.HTTP_CLIENT_INSTANCES:
             endpoints = [self._mirror]
             for host in __registry_mirror_hosts__:
                 endpoint = f"https://dl.{host}"
                 if endpoint not in endpoints:
                     endpoints.append(endpoint)
-            RegistryFileMirrorIterator.HTTP_CLIENT_INSTANCES[self._mirror] = HTTPClient(
-                endpoints
+            RegistryFileMirrorIterator.HTTP_CLIENT_INSTANCES[self._mirror] = (
+                RegistryClient(endpoints)
             )
         return RegistryFileMirrorIterator.HTTP_CLIENT_INSTANCES[self._mirror]

platformio/remote/client/device_monitor.py

@@ -123,9 +123,11 @@ class DeviceMonitorClient(  # pylint: disable=too-many-instance-attributes
                         index=i + 1,
                         host=device[0] + ":" if len(result) > 1 else "",
                         port=device[1]["port"],
-                        description=device[1]["description"]
-                        if device[1]["description"] != "n/a"
-                        else "",
+                        description=(
+                            device[1]["description"]
+                            if device[1]["description"] != "n/a"
+                            else ""
+                        ),
                     )
                 )
             device_index = click.prompt(