Added --json-output support for pkg list command

# Conflicts:
#	platformio/commands/upgrade.py

.github/workflows/docs.yml

@@ -49,12 +49,12 @@ jobs:
     name: Deploy Docs
     needs: build
     runs-on: ubuntu-latest
+    if: ${{ github.event_name == 'push' && (github.ref == 'refs/heads/develop' || github.ref == 'refs/heads/master') }}
     env:
       DOCS_REPO: platformio/platformio-docs
       DOCS_DIR: platformio-docs
       LATEST_DOCS_DIR: latest-docs
       RELEASE_BUILD: ${{ startsWith(github.ref, 'refs/tags/v') }}
-    if: ${{ github.event_name == 'push' }}
     steps:
       - name: Download artifact
         uses: actions/download-artifact@v3

platformio/__main__.py

@@ -66,15 +66,6 @@ def configure():
     if IS_CYGWIN:
         raise exception.CygwinEnvDetected()
 
-    # https://urllib3.readthedocs.org
-    # /en/latest/security.html#insecureplatformwarning
-    try:
-        import urllib3  # pylint: disable=import-outside-toplevel
-
-        urllib3.disable_warnings()
-    except (AttributeError, ImportError):
-        pass
-
     # Handle IOError issue with VSCode's Terminal (Windows)
     click_echo_origin = [click.echo, click.secho]
 

platformio/account/client.py

@@ -17,7 +17,7 @@ import time
 
 from platformio import __accounts_api__, app
 from platformio.exception import PlatformioException, UserSideException
-from platformio.http import HTTPClient, HTTPClientError
+from platformio.http import HttpApiClient, HttpClientApiError
 
 
 class AccountError(PlatformioException):
@@ -32,7 +32,7 @@ class AccountAlreadyAuthorized(AccountError, UserSideException):
     MESSAGE = "You are already authorized with {0} account."
 
 
-class AccountClient(HTTPClient):  # pylint:disable=too-many-public-methods
+class AccountClient(HttpApiClient):  # pylint:disable=too-many-public-methods
     SUMMARY_CACHE_TTL = 60 * 60 * 24 * 7
 
     def __init__(self):
@@ -60,7 +60,7 @@ class AccountClient(HTTPClient):  # pylint:disable=too-many-public-methods
     def fetch_json_data(self, *args, **kwargs):
         try:
             return super().fetch_json_data(*args, **kwargs)
-        except HTTPClientError as exc:
+        except HttpClientApiError as exc:
             raise AccountError(exc) from exc
 
     def fetch_authentication_token(self):

platformio/account/commands/destroy.py

@@ -19,18 +19,18 @@ from platformio.account.client import AccountClient, AccountNotAuthorized
 
 @click.command("destroy", short_help="Destroy account")
 def account_destroy_cmd():
-    client = AccountClient()
-    click.confirm(
-        "Are you sure you want to delete the %s user account?\n"
-        "Warning! All linked data will be permanently removed and can not be restored."
-        % client.get_logged_username(),
-        abort=True,
-    )
-    client.destroy_account()
-    try:
-        client.logout()
-    except AccountNotAuthorized:
-        pass
+    with AccountClient() as client:
+        click.confirm(
+            "Are you sure you want to delete the %s user account?\n"
+            "Warning! All linked data will be permanently removed and can not be restored."
+            % client.get_logged_username(),
+            abort=True,
+        )
+        client.destroy_account()
+        try:
+            client.logout()
+        except AccountNotAuthorized:
+            pass
     click.secho(
         "User account has been destroyed.",
         fg="green",

platformio/account/commands/forgot.py

@@ -20,8 +20,8 @@ from platformio.account.client import AccountClient
 @click.command("forgot", short_help="Forgot password")
 @click.option("--username", prompt="Username or email")
 def account_forgot_cmd(username):
-    client = AccountClient()
-    client.forgot_password(username)
+    with AccountClient() as client:
+        client.forgot_password(username)
     click.secho(
         "If this account is registered, we will send the "
         "further instructions to your email.",

platformio/account/commands/login.py

@@ -21,6 +21,6 @@ from platformio.account.client import AccountClient
 @click.option("-u", "--username", prompt="Username or email")
 @click.option("-p", "--password", prompt=True, hide_input=True)
 def account_login_cmd(username, password):
-    client = AccountClient()
-    client.login(username, password)
+    with AccountClient() as client:
+        client.login(username, password)
     click.secho("Successfully logged in!", fg="green")

platformio/account/commands/logout.py

@@ -19,6 +19,6 @@ from platformio.account.client import AccountClient
 
 @click.command("logout", short_help="Log out of PlatformIO Account")
 def account_logout_cmd():
-    client = AccountClient()
-    client.logout()
+    with AccountClient() as client:
+        client.logout()
     click.secho("Successfully logged out!", fg="green")

platformio/account/commands/password.py

@@ -21,6 +21,6 @@ from platformio.account.client import AccountClient
 @click.option("--old-password", prompt=True, hide_input=True)
 @click.option("--new-password", prompt=True, hide_input=True, confirmation_prompt=True)
 def account_password_cmd(old_password, new_password):
-    client = AccountClient()
-    client.change_password(old_password, new_password)
+    with AccountClient() as client:
+        client.change_password(old_password, new_password)
     click.secho("Password successfully changed!", fg="green")

platformio/account/commands/register.py

@@ -43,8 +43,8 @@ from platformio.account.validate import (
 @click.option("--firstname", prompt=True)
 @click.option("--lastname", prompt=True)
 def account_register_cmd(username, email, password, firstname, lastname):
-    client = AccountClient()
-    client.registration(username, email, password, firstname, lastname)
+    with AccountClient() as client:
+        client.registration(username, email, password, firstname, lastname)
     click.secho(
         "An account has been successfully created. "
         "Please check your mail to activate your account and verify your email address.",

platformio/account/commands/show.py

@@ -25,8 +25,8 @@ from platformio.account.client import AccountClient
 @click.option("--offline", is_flag=True)
 @click.option("--json-output", is_flag=True)
 def account_show_cmd(offline, json_output):
-    client = AccountClient()
-    info = client.get_account_info(offline)
+    with AccountClient() as client:
+        info = client.get_account_info(offline)
     if json_output:
         click.echo(json.dumps(info))
         return

platformio/account/commands/token.py

@@ -24,8 +24,8 @@ from platformio.account.client import AccountClient
 @click.option("--regenerate", is_flag=True)
 @click.option("--json-output", is_flag=True)
 def account_token_cmd(password, regenerate, json_output):
-    client = AccountClient()
-    auth_token = client.auth_token(password, regenerate)
+    with AccountClient() as client:
+        auth_token = client.auth_token(password, regenerate)
     if json_output:
         click.echo(json.dumps({"status": "success", "result": auth_token}))
         return

platformio/account/commands/update.py

@@ -25,8 +25,8 @@ from platformio.account.validate import validate_email, validate_username
 @click.option("--firstname")
 @click.option("--lastname")
 def account_update_cmd(current_password, **kwargs):
-    client = AccountClient()
-    profile = client.get_profile()
+    with AccountClient() as client:
+        profile = client.get_profile()
     new_profile = profile.copy()
     if not any(kwargs.values()):
         for field in profile:

platformio/account/org/commands/add.py

@@ -25,8 +25,8 @@ from platformio.account.client import AccountClient
     "username",
 )
 def org_add_cmd(orgname, username):
-    client = AccountClient()
-    client.add_org_owner(orgname, username)
+    with AccountClient() as client:
+        client.add_org_owner(orgname, username)
     return click.secho(
         "The new owner `%s` has been successfully added to the `%s` organization."
         % (username, orgname),

platformio/account/org/commands/create.py

@@ -30,8 +30,8 @@ from platformio.account.validate import validate_email, validate_orgname
     "--displayname",
 )
 def org_create_cmd(orgname, email, displayname):
-    client = AccountClient()
-    client.create_org(orgname, email, displayname)
+    with AccountClient() as client:
+        client.create_org(orgname, email, displayname)
     return click.secho(
         "The organization `%s` has been successfully created." % orgname,
         fg="green",

platformio/account/org/commands/destroy.py

@@ -20,14 +20,14 @@ from platformio.account.client import AccountClient
 @click.command("destroy", short_help="Destroy organization")
 @click.argument("orgname")
 def org_destroy_cmd(orgname):
-    client = AccountClient()
-    click.confirm(
-        "Are you sure you want to delete the `%s` organization account?\n"
-        "Warning! All linked data will be permanently removed and can not be restored."
-        % orgname,
-        abort=True,
-    )
-    client.destroy_org(orgname)
+    with AccountClient() as client:
+        click.confirm(
+            "Are you sure you want to delete the `%s` organization account?\n"
+            "Warning! All linked data will be permanently removed and can not be restored."
+            % orgname,
+            abort=True,
+        )
+        client.destroy_org(orgname)
     return click.secho(
         "Organization `%s` has been destroyed." % orgname,
         fg="green",

platformio/account/org/commands/list.py

@@ -23,8 +23,8 @@ from platformio.account.client import AccountClient
 @click.command("list", short_help="List organizations and their members")
 @click.option("--json-output", is_flag=True)
 def org_list_cmd(json_output):
-    client = AccountClient()
-    orgs = client.list_orgs()
+    with AccountClient() as client:
+        orgs = client.list_orgs()
     if json_output:
         return click.echo(json.dumps(orgs))
     if not orgs:

platformio/account/org/commands/remove.py

@@ -25,8 +25,8 @@ from platformio.account.client import AccountClient
     "username",
 )
 def org_remove_cmd(orgname, username):
-    client = AccountClient()
-    client.remove_org_owner(orgname, username)
+    with AccountClient() as client:
+        client.remove_org_owner(orgname, username)
     return click.secho(
         "The `%s` owner has been successfully removed from the `%s` organization."
         % (username, orgname),

platformio/account/org/commands/update.py

@@ -31,8 +31,8 @@ from platformio.account.validate import validate_email, validate_orgname
 )
 @click.option("--displayname")
 def org_update_cmd(cur_orgname, **kwargs):
-    client = AccountClient()
-    org = client.get_org(cur_orgname)
+    with AccountClient() as client:
+        org = client.get_org(cur_orgname)
     new_org = {
         key: value if value is not None else org[key] for key, value in kwargs.items()
     }

platformio/account/team/commands/add.py

@@ -29,8 +29,8 @@ from platformio.account.validate import validate_orgname_teamname
 )
 def team_add_cmd(orgname_teamname, username):
     orgname, teamname = orgname_teamname.split(":", 1)
-    client = AccountClient()
-    client.add_team_member(orgname, teamname, username)
+    with AccountClient() as client:
+        client.add_team_member(orgname, teamname, username)
     return click.secho(
         "The new member %s has been successfully added to the %s team."
         % (username, teamname),

platformio/account/team/commands/create.py

@@ -29,8 +29,8 @@ from platformio.account.validate import validate_orgname_teamname
 )
 def team_create_cmd(orgname_teamname, description):
     orgname, teamname = orgname_teamname.split(":", 1)
-    client = AccountClient()
-    client.create_team(orgname, teamname, description)
+    with AccountClient() as client:
+        client.create_team(orgname, teamname, description)
     return click.secho(
         "The team %s has been successfully created." % teamname,
         fg="green",

platformio/account/team/commands/destroy.py

@@ -32,8 +32,8 @@ def team_destroy_cmd(orgname_teamname):
         ),
         abort=True,
     )
-    client = AccountClient()
-    client.destroy_team(orgname, teamname)
+    with AccountClient() as client:
+        client.destroy_team(orgname, teamname)
     return click.secho(
         "The team %s has been successfully destroyed." % teamname,
         fg="green",

platformio/account/team/commands/list.py

@@ -24,19 +24,22 @@ from platformio.account.client import AccountClient
 @click.argument("orgname", required=False)
 @click.option("--json-output", is_flag=True)
 def team_list_cmd(orgname, json_output):
-    client = AccountClient()
-    data = {}
-    if not orgname:
-        for item in client.list_orgs():
-            teams = client.list_teams(item.get("orgname"))
-            data[item.get("orgname")] = teams
-    else:
-        teams = client.list_teams(orgname)
-        data[orgname] = teams
+    with AccountClient() as client:
+        data = {}
+        if not orgname:
+            for item in client.list_orgs():
+                teams = client.list_teams(item.get("orgname"))
+                data[item.get("orgname")] = teams
+        else:
+            teams = client.list_teams(orgname)
+            data[orgname] = teams
+
     if json_output:
         return click.echo(json.dumps(data[orgname] if orgname else data))
+
     if not any(data.values()):
         return click.secho("You do not have any teams.", fg="yellow")
+
     for org_name, teams in data.items():
         for team in teams:
             click.echo()

platformio/account/team/commands/remove.py

@@ -27,8 +27,8 @@ from platformio.account.validate import validate_orgname_teamname
 @click.argument("username")
 def team_remove_cmd(orgname_teamname, username):
     orgname, teamname = orgname_teamname.split(":", 1)
-    client = AccountClient()
-    client.remove_team_member(orgname, teamname, username)
+    with AccountClient() as client:
+        client.remove_team_member(orgname, teamname, username)
     return click.secho(
         "The %s member has been successfully removed from the %s team."
         % (username, teamname),

platformio/account/team/commands/update.py

@@ -34,8 +34,8 @@ from platformio.account.validate import validate_orgname_teamname, validate_team
 )
 def team_update_cmd(orgname_teamname, **kwargs):
     orgname, teamname = orgname_teamname.split(":", 1)
-    client = AccountClient()
-    team = client.get_team(orgname, teamname)
+    with AccountClient() as client:
+        team = client.get_team(orgname, teamname)
     new_team = {
         key: value if value is not None else team[key] for key, value in kwargs.items()
     }

platformio/app.py

@@ -258,10 +258,6 @@ def get_cid():
     return cid
 
 
-def get_project_id(project_dir):
-    return hashlib.sha1(hashlib_encode_data(project_dir)).hexdigest()
-
-
 def get_user_agent():
     data = [
         "PlatformIO/%s" % __version__,

platformio/builder/main.py

@@ -15,7 +15,7 @@
 import json
 import os
 import sys
-from time import time
+import time
 
 import click
 from SCons.Script import ARGUMENTS  # pylint: disable=import-error
@@ -31,7 +31,7 @@ from SCons.Script import Variables  # pylint: disable=import-error
 from platformio import app, fs
 from platformio.platform.base import PlatformBase
 from platformio.proc import get_pythonexe_path
-from platformio.project.helpers import get_project_dir
+from platformio.project.helpers import get_build_type, get_project_dir
 
 AllowSubstExceptions(NameError)
 
@@ -61,7 +61,7 @@ DEFAULT_ENV_OPTIONS = dict(
         "piotarget",
         "piolib",
         "pioupload",
-        "piosize",
+        "piomemusage",
         "pioino",
         "piomisc",
         "piointegration",
@@ -71,15 +71,7 @@ DEFAULT_ENV_OPTIONS = dict(
     variables=clivars,
     # Propagating External Environment
     ENV=os.environ,
-    UNIX_TIME=int(time()),
-    BUILD_DIR=os.path.join("$PROJECT_BUILD_DIR", "$PIOENV"),
-    BUILD_SRC_DIR=os.path.join("$BUILD_DIR", "src"),
-    BUILD_TEST_DIR=os.path.join("$BUILD_DIR", "test"),
-    COMPILATIONDB_PATH=os.path.join("$PROJECT_DIR", "compile_commands.json"),
-    LIBPATH=["$BUILD_DIR"],
-    PROGNAME="program",
-    PROGPATH=os.path.join("$BUILD_DIR", "$PROGNAME$PROGSUFFIX"),
-    PROG_PATH="$PROGPATH",  # deprecated
+    UNIX_TIME=int(time.time()),
     PYTHONEXE=get_pythonexe_path(),
 )
 
@@ -126,13 +118,21 @@ env.Replace(
     PROJECT_DATA_DIR=config.get("platformio", "data_dir"),
     PROJECTDATA_DIR="$PROJECT_DATA_DIR",  # legacy for dev/platform
     PROJECT_BUILD_DIR=config.get("platformio", "build_dir"),
-    BUILD_TYPE=env.GetBuildType(),
+    BUILD_TYPE=get_build_type(config, env["PIOENV"], COMMAND_LINE_TARGETS),
+    BUILD_DIR=os.path.join("$PROJECT_BUILD_DIR", "$PIOENV", "$BUILD_TYPE"),
+    BUILD_SRC_DIR=os.path.join("$BUILD_DIR", "src"),
+    BUILD_TEST_DIR=os.path.join("$BUILD_DIR", "test"),
     BUILD_CACHE_DIR=config.get("platformio", "build_cache_dir"),
+    LIBPATH=["$BUILD_DIR"],
     LIBSOURCE_DIRS=[
         config.get("platformio", "lib_dir"),
         os.path.join("$PROJECT_LIBDEPS_DIR", "$PIOENV"),
         config.get("platformio", "globallib_dir"),
     ],
+    COMPILATIONDB_PATH=os.path.join("$PROJECT_DIR", "compile_commands.json"),
+    PROGNAME="program",
+    PROGPATH=os.path.join("$BUILD_DIR", "$PROGNAME$PROGSUFFIX"),
+    PROG_PATH="$PROGPATH",  # deprecated
 )
 
 if int(ARGUMENTS.get("ISATTY", 0)):
@@ -183,7 +183,7 @@ env.SConscript(env.GetExtraScripts("post"), exports="env")
 
 # Checking program size
 if env.get("SIZETOOL") and not (
-    set(["nobuild", "sizedata"]) & set(COMMAND_LINE_TARGETS)
+    set(["nobuild", "__memusage"]) & set(COMMAND_LINE_TARGETS)
 ):
     env.Depends("upload", "checkprogsize")
     # Replace platform's "size" target with our
@@ -224,24 +224,27 @@ if env.IsIntegrationDump():
     data = projenv.DumpIntegrationData(env)
     # dump to file for the further reading by project.helpers.load_build_metadata
     with open(
-        projenv.subst(os.path.join("$BUILD_DIR", "idedata.json")),
+        projenv.subst(os.path.join("$BUILD_DIR", "metadata.json")),
         mode="w",
         encoding="utf8",
     ) as fp:
         json.dump(data, fp)
-    click.echo("\n%s\n" % json.dumps(data))  # pylint: disable=undefined-variable
+    click.echo(
+        "Metadata has been saved to the following location: %s"
+        % projenv.subst(os.path.join("$BUILD_DIR", "metadata.json"))
+    )
     env.Exit(0)
 
-if "sizedata" in COMMAND_LINE_TARGETS:
+if "__memusage" in COMMAND_LINE_TARGETS:
     AlwaysBuild(
         env.Alias(
-            "sizedata",
+            "__memusage",
             DEFAULT_TARGETS,
-            env.VerboseAction(env.DumpSizeData, "Generating memory usage report..."),
+            env.VerboseAction(env.DumpMemoryUsage, "Generating memory usage report..."),
         )
     )
 
-    Default("sizedata")
+    Default("__memusage")
 
 # issue #4604: process targets sequentially
 for index, target in enumerate(

platformio/builder/tools/piointegration.py

@@ -23,7 +23,7 @@ from platformio.proc import exec_command, where_is_program
 
 
 def IsIntegrationDump(_):
-    return set(["__idedata", "idedata"]) & set(COMMAND_LINE_TARGETS)
+    return set(["__idedata", "__metadata"]) & set(COMMAND_LINE_TARGETS)
 
 
 def DumpIntegrationIncludes(env):
@@ -141,7 +141,7 @@ def _split_flags_string(env, s):
 def DumpIntegrationData(*args):
     projenv, globalenv = args[0:2]  # pylint: disable=unbalanced-tuple-unpacking
     data = {
-        "build_type": globalenv.GetBuildType(),
+        "build_type": globalenv["BUILD_TYPE"],
         "env_name": globalenv["PIOENV"],
         "libsource_dirs": [
             globalenv.subst(item) for item in globalenv.GetLibSourceDirs()

platformio/builder/tools/piolib.py

@@ -29,7 +29,7 @@ from SCons.Script import DefaultEnvironment  # pylint: disable=import-error
 from platformio import exception, fs
 from platformio.builder.tools import piobuild
 from platformio.compat import IS_WINDOWS, hashlib_encode_data, string_types
-from platformio.http import HTTPClientError, InternetConnectionError
+from platformio.http import HttpClientApiError, InternetConnectionError
 from platformio.package.exception import (
     MissingPackageManifestError,
     UnknownPackageError,
@@ -1005,7 +1005,7 @@ class ProjectAsLibBuilder(LibBuilderBase):
                 lm.install(spec)
                 did_install = True
             except (
-                HTTPClientError,
+                HttpClientApiError,
                 UnknownPackageError,
                 InternetConnectionError,
             ) as exc:

platformio/builder/tools/piomemusage.py

@@ -14,33 +14,33 @@
 
 # pylint: disable=too-many-locals
 
-import json
+import os
 import sys
-from os import environ, makedirs, remove
-from os.path import isdir, join, splitdrive
+import time
 
 from elftools.elf.descriptions import describe_sh_flags
 from elftools.elf.elffile import ELFFile
 
 from platformio.compat import IS_WINDOWS
 from platformio.proc import exec_command
+from platformio.project.memusage import save_report
 
 
 def _run_tool(cmd, env, tool_args):
-    sysenv = environ.copy()
+    sysenv = os.environ.copy()
     sysenv["PATH"] = str(env["ENV"]["PATH"])
 
     build_dir = env.subst("$BUILD_DIR")
-    if not isdir(build_dir):
-        makedirs(build_dir)
-    tmp_file = join(build_dir, "size-data-longcmd.txt")
+    if not os.path.isdir(build_dir):
+        os.makedirs(build_dir)
+    tmp_file = os.path.join(build_dir, "size-data-longcmd.txt")
 
     with open(tmp_file, mode="w", encoding="utf8") as fp:
         fp.write("\n".join(tool_args))
 
     cmd.append("@" + tmp_file)
     result = exec_command(cmd, env=sysenv)
-    remove(tmp_file)
+    os.remove(tmp_file)
 
     return result
 
@@ -92,8 +92,8 @@ def _collect_sections_info(env, elffile):
         }
 
         sections[section.name] = section_data
-        sections[section.name]["in_flash"] = env.pioSizeIsFlashSection(section_data)
-        sections[section.name]["in_ram"] = env.pioSizeIsRamSection(section_data)
+        sections[section.name]["in_flash"] = env.memusageIsFlashSection(section_data)
+        sections[section.name]["in_ram"] = env.memusageIsRamSection(section_data)
 
     return sections
 
@@ -106,7 +106,7 @@ def _collect_symbols_info(env, elffile, elf_path, sections):
         sys.stderr.write("Couldn't find symbol table. Is ELF file stripped?")
         env.Exit(1)
 
-    sysenv = environ.copy()
+    sysenv = os.environ.copy()
     sysenv["PATH"] = str(env["ENV"]["PATH"])
 
     symbol_addrs = []
@@ -117,7 +117,7 @@ def _collect_symbols_info(env, elffile, elf_path, sections):
         symbol_size = s["st_size"]
         symbol_type = symbol_info["type"]
 
-        if not env.pioSizeIsValidSymbol(s.name, symbol_type, symbol_addr):
+        if not env.memusageIsValidSymbol(s.name, symbol_type, symbol_addr):
             continue
 
         symbol = {
@@ -126,7 +126,7 @@ def _collect_symbols_info(env, elffile, elf_path, sections):
             "name": s.name,
             "type": symbol_type,
             "size": symbol_size,
-            "section": env.pioSizeDetermineSection(sections, symbol_addr),
+            "section": env.memusageDetermineSection(sections, symbol_addr),
         }
 
         if s.name.startswith("_Z"):
@@ -144,8 +144,8 @@ def _collect_symbols_info(env, elffile, elf_path, sections):
         if not location or "?" in location:
             continue
         if IS_WINDOWS:
-            drive, tail = splitdrive(location)
-            location = join(drive.upper(), tail)
+            drive, tail = os.path.splitdrive(location)
+            location = os.path.join(drive.upper(), tail)
         symbol["file"] = location
         symbol["line"] = 0
         if ":" in location:
@@ -156,7 +156,7 @@ def _collect_symbols_info(env, elffile, elf_path, sections):
     return symbols
 
 
-def pioSizeDetermineSection(_, sections, symbol_addr):
+def memusageDetermineSection(_, sections, symbol_addr):
     for section, info in sections.items():
         if not info.get("in_flash", False) and not info.get("in_ram", False):
             continue
@@ -165,22 +165,22 @@ def pioSizeDetermineSection(_, sections, symbol_addr):
     return "unknown"
 
 
-def pioSizeIsValidSymbol(_, symbol_name, symbol_type, symbol_address):
+def memusageIsValidSymbol(_, symbol_name, symbol_type, symbol_address):
     return symbol_name and symbol_address != 0 and symbol_type != "STT_NOTYPE"
 
 
-def pioSizeIsRamSection(_, section):
+def memusageIsRamSection(_, section):
     return (
         section.get("type", "") in ("SHT_NOBITS", "SHT_PROGBITS")
         and section.get("flags", "") == "WA"
     )
 
 
-def pioSizeIsFlashSection(_, section):
+def memusageIsFlashSection(_, section):
     return section.get("type", "") == "SHT_PROGBITS" and "A" in section.get("flags", "")
 
 
-def pioSizeCalculateFirmwareSize(_, sections):
+def memusageCalculateFirmwareSize(_, sections):
     flash_size = ram_size = 0
     for section_info in sections.values():
         if section_info.get("in_flash", False):
@@ -191,20 +191,22 @@ def pioSizeCalculateFirmwareSize(_, sections):
     return ram_size, flash_size
 
 
-def DumpSizeData(_, target, source, env):  # pylint: disable=unused-argument
-    data = {"device": {}, "memory": {}, "version": 1}
+def DumpMemoryUsage(_, target, source, env):  # pylint: disable=unused-argument
+    result = {"version": 1, "timestamp": int(time.time()), "device": {}, "memory": {}}
 
     board = env.BoardConfig()
     if board:
-        data["device"] = {
+        result["device"] = {
             "mcu": board.get("build.mcu", ""),
             "cpu": board.get("build.cpu", ""),
             "frequency": board.get("build.f_cpu"),
             "flash": int(board.get("upload.maximum_size", 0)),
             "ram": int(board.get("upload.maximum_ram_size", 0)),
         }
-        if data["device"]["frequency"] and data["device"]["frequency"].endswith("L"):
-            data["device"]["frequency"] = int(data["device"]["frequency"][0:-1])
+        if result["device"]["frequency"] and result["device"]["frequency"].endswith(
+            "L"
+        ):
+            result["device"]["frequency"] = int(result["device"]["frequency"][0:-1])
 
     elf_path = env.subst("$PIOMAINPROG")
 
@@ -216,16 +218,16 @@ def DumpSizeData(_, target, source, env):  # pylint: disable=unused-argument
             env.Exit(1)
 
         sections = _collect_sections_info(env, elffile)
-        firmware_ram, firmware_flash = env.pioSizeCalculateFirmwareSize(sections)
-        data["memory"]["total"] = {
+        firmware_ram, firmware_flash = env.memusageCalculateFirmwareSize(sections)
+        result["memory"]["total"] = {
             "ram_size": firmware_ram,
             "flash_size": firmware_flash,
-            "sections": sections,
         }
+        result["memory"]["sections"] = sections
 
         files = {}
         for symbol in _collect_symbols_info(env, elffile, elf_path, sections):
-            file_path = symbol.get("file") or "unknown"
+            file_path = symbol.pop("file", "unknown")
             if not files.get(file_path, {}):
                 files[file_path] = {"symbols": [], "ram_size": 0, "flash_size": 0}
 
@@ -240,16 +242,16 @@ def DumpSizeData(_, target, source, env):  # pylint: disable=unused-argument
 
             files[file_path]["symbols"].append(symbol)
 
-        data["memory"]["files"] = []
+        result["memory"]["files"] = []
         for k, v in files.items():
             file_data = {"path": k}
             file_data.update(v)
-            data["memory"]["files"].append(file_data)
+            result["memory"]["files"].append(file_data)
 
-    with open(
-        join(env.subst("$BUILD_DIR"), "sizedata.json"), mode="w", encoding="utf8"
-    ) as fp:
-        fp.write(json.dumps(data))
+    print(
+        "Memory usage report has been saved to the following location: "
+        f"\"{save_report(os.getcwd(), env['PIOENV'], result)}\""
+    )
 
 
 def exists(_):
@@ -257,10 +259,10 @@ def exists(_):
 
 
 def generate(env):
-    env.AddMethod(pioSizeIsRamSection)
-    env.AddMethod(pioSizeIsFlashSection)
-    env.AddMethod(pioSizeCalculateFirmwareSize)
-    env.AddMethod(pioSizeDetermineSection)
-    env.AddMethod(pioSizeIsValidSymbol)
-    env.AddMethod(DumpSizeData)
+    env.AddMethod(memusageIsRamSection)
+    env.AddMethod(memusageIsFlashSection)
+    env.AddMethod(memusageCalculateFirmwareSize)
+    env.AddMethod(memusageDetermineSection)
+    env.AddMethod(memusageIsValidSymbol)
+    env.AddMethod(DumpMemoryUsage)
     return env

platformio/commands/upgrade.py

@@ -20,7 +20,7 @@ import click
 
 from platformio import VERSION, __version__, app, exception
 from platformio.dependencies import get_pip_dependencies
-from platformio.http import fetch_remote_content
+from platformio.http import fetch_http_content
 from platformio.package.manager.core import update_core_packages
 from platformio.proc import get_pythonexe_path
 
@@ -133,7 +133,7 @@ def get_latest_version():
 
 def get_develop_latest_version():
     version = None
-    content = fetch_remote_content(DEVELOP_INIT_SCRIPT_URL)
+    content = fetch_http_content(DEVELOP_INIT_SCRIPT_URL)
     for line in content.split("\n"):
         line = line.strip()
         if not line.startswith("VERSION"):
@@ -150,5 +150,5 @@ def get_develop_latest_version():
 
 
 def get_pypi_latest_version():
-    content = fetch_remote_content(PYPI_JSON_URL)
+    content = fetch_http_content(PYPI_JSON_URL)
     return json.loads(content)["info"]["version"]

platformio/debug/config/base.py

@@ -149,7 +149,7 @@ class DebugConfigBase:  # pylint: disable=too-many-instance-attributes
 
     def _load_build_data(self):
         data = load_build_metadata(
-            os.getcwd(), self.env_name, cache=True, build_type="debug"
+            os.getcwd(), self.env_name, cache=True, force_targets=["__debug"]
         )
         if not data:
             raise DebugInvalidOptionsError("Could not load a build configuration")

platformio/dependencies.py

@@ -33,10 +33,10 @@ def get_pip_dependencies():
         "bottle == 0.12.*",
         "click >=8.0.4, <9",
         "colorama",
+        "httpx%s >=0.22.0, <0.28" % ("[socks]" if is_proxy_set(socks=True) else ""),
         "marshmallow == 3.*",
         "pyelftools >=0.27, <1",
         "pyserial == 3.5.*",  # keep in sync "device/monitor/terminal.py"
-        "requests%s == 2.*" % ("[socks]" if is_proxy_set(socks=True) else ""),
         "semantic_version == 2.10.*",
         "tabulate == 0.*",
     ]

platformio/fs.py

@@ -29,15 +29,16 @@ from platformio.compat import IS_WINDOWS
 
 
 class cd:
-    def __init__(self, new_path):
-        self.new_path = new_path
-        self.prev_path = os.getcwd()
+    def __init__(self, path):
+        self.path = path
+        self._old_cwd = []
 
     def __enter__(self):
-        os.chdir(self.new_path)
+        self._old_cwd.append(os.getcwd())
+        os.chdir(self.path)
 
-    def __exit__(self, etype, value, traceback):
-        os.chdir(self.prev_path)
+    def __exit__(self, *excinfo):
+        os.chdir(self._old_cwd.pop())
 
 
 def get_source_dir():

platformio/home/rpc/handlers/account.py

@@ -12,19 +12,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from ajsonrpc.core import JSONRPC20DispatchException
-
 from platformio.account.client import AccountClient
 from platformio.home.rpc.handlers.base import BaseRPCHandler
 
 
 class AccountRPC(BaseRPCHandler):
+    NAMESPACE = "account"
+
     @staticmethod
     def call_client(method, *args, **kwargs):
-        try:
-            client = AccountClient()
+        with AccountClient() as client:
             return getattr(client, method)(*args, **kwargs)
-        except Exception as exc:  # pylint: disable=bare-except
-            raise JSONRPC20DispatchException(
-                code=5000, message="PIO Account Call Error", data=str(exc)
-            ) from exc

platformio/home/rpc/handlers/app.py

@@ -20,6 +20,7 @@ from platformio.project.helpers import is_platformio_project
 
 
 class AppRPC(BaseRPCHandler):
+    NAMESPACE = "app"
     IGNORE_STORAGE_KEYS = [
         "cid",
         "coreVersion",

platformio/home/rpc/handlers/base.py

@@ -14,4 +14,6 @@
 
 
 class BaseRPCHandler:
+    NAMESPACE = None
+
     factory = None

platformio/home/rpc/handlers/core.py

@@ -0,0 +1,123 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import asyncio
+import functools
+import os
+
+from platformio import __main__, __version__, app, proc, util
+from platformio.compat import (
+    IS_WINDOWS,
+    aio_create_task,
+    aio_get_running_loop,
+    get_locale_encoding,
+    shlex_join,
+)
+from platformio.exception import UserSideException
+from platformio.home.rpc.handlers.base import BaseRPCHandler
+
+
+class PIOCoreCallError(UserSideException):
+    MESSAGE = 'An error occured while executing PIO Core command: "{0}"\n\n{1}'
+
+
+class PIOCoreProtocol(asyncio.SubprocessProtocol):
+    def __init__(self, exit_future, on_data_callback=None):
+        self.exit_future = exit_future
+        self.on_data_callback = on_data_callback
+        self.stdout = ""
+        self.stderr = ""
+        self._is_exited = False
+        self._encoding = get_locale_encoding()
+
+    def pipe_data_received(self, fd, data):
+        data = data.decode(self._encoding, "replace")
+        pipe = ["stdin", "stdout", "stderr"][fd]
+        if pipe == "stdout":
+            self.stdout += data
+        if pipe == "stderr":
+            self.stderr += data
+        if self.on_data_callback:
+            self.on_data_callback(pipe=pipe, data=data)
+
+    def connection_lost(self, exc):
+        self.process_exited()
+
+    def process_exited(self):
+        if self._is_exited:
+            return
+        self.exit_future.set_result(True)
+        self._is_exited = True
+
+
+@util.memoized(expire="60s")
+def get_core_fullpath():
+    return proc.where_is_program("platformio" + (".exe" if IS_WINDOWS else ""))
+
+
+class CoreRPC(BaseRPCHandler):
+    NAMESPACE = "core"
+
+    @staticmethod
+    def version():
+        return __version__
+
+    async def exec(self, args, options=None, raise_exception=True):
+        options = options or {}
+        loop = aio_get_running_loop()
+        exit_future = loop.create_future()
+        data_callback = functools.partial(
+            self._on_exec_data_received, exec_options=options
+        )
+
+        if args[0] != "--caller" and app.get_session_var("caller_id"):
+            args = ["--caller", app.get_session_var("caller_id")] + args
+        kwargs = options.get("spawn", {})
+
+        if "force_ansi" in options:
+            environ = kwargs.get("env", os.environ.copy())
+            environ["PLATFORMIO_FORCE_ANSI"] = "true"
+            kwargs["env"] = environ
+
+        transport, protocol = await loop.subprocess_exec(
+            lambda: PIOCoreProtocol(exit_future, data_callback),
+            get_core_fullpath(),
+            *args,
+            stdin=None,
+            **kwargs,
+        )
+        await exit_future
+        transport.close()
+        return_code = transport.get_returncode()
+        if return_code != 0 and raise_exception:
+            raise PIOCoreCallError(
+                shlex_join(["pio"] + args), f"{protocol.stdout}\n{protocol.stderr}"
+            )
+        return {
+            "stdout": protocol.stdout,
+            "stderr": protocol.stderr,
+            "returncode": return_code,
+        }
+
+    def _on_exec_data_received(self, exec_options, pipe, data):
+        notification_method = exec_options.get(f"{pipe}NotificationMethod")
+        if not notification_method:
+            return
+        aio_create_task(
+            self.factory.notify_clients(
+                method=notification_method,
+                params=[data],
+                actor="frontend",
+            )
+        )

platformio/home/rpc/handlers/ide.py

@@ -22,6 +22,7 @@ from platformio.home.rpc.handlers.base import BaseRPCHandler
 
 
 class IDERPC(BaseRPCHandler):
+    NAMESPACE = "ide"
     COMMAND_TIMEOUT = 1.5  # in seconds
 
     def __init__(self):

platformio/home/rpc/handlers/memusage.py

@@ -0,0 +1,127 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import functools
+import os
+
+from platformio.home.rpc.handlers.base import BaseRPCHandler
+from platformio.project import memusage
+
+
+class MemUsageRPC(BaseRPCHandler):
+    NAMESPACE = "memusage"
+
+    async def profile(self, project_dir, env, options=None):
+        options = options or {}
+        report_dir = memusage.get_report_dir(project_dir, env)
+        if options.get("lazy"):
+            existing_reports = memusage.list_reports(report_dir)
+            if existing_reports:
+                return existing_reports[-1]
+        await self.factory.manager.dispatcher["core.exec"](
+            ["run", "-d", project_dir, "-e", env, "-t", "__memusage"],
+            options=options.get("exec"),
+        )
+        return memusage.list_reports(report_dir)[-1]
+
+    @staticmethod
+    def load_report(path):
+        return memusage.read_report(path)
+
+    def summary(self, report_path):
+        max_top_items = 10
+        report_dir = os.path.dirname(report_path)
+        existing_reports = memusage.list_reports(report_dir)
+        current_report = memusage.read_report(report_path)
+        previous_report = None
+        try:
+            current_index = existing_reports.index(report_path)
+            if current_index > 0:
+                previous_report = memusage.read_report(
+                    existing_reports[current_index - 1]
+                )
+        except ValueError:
+            pass
+
+        return dict(
+            timestamp=dict(
+                current=current_report["timestamp"],
+                previous=previous_report["timestamp"] if previous_report else None,
+            ),
+            device=current_report["device"],
+            trend=dict(
+                current=current_report["memory"]["total"],
+                previous=(
+                    previous_report["memory"]["total"] if previous_report else None
+                ),
+            ),
+            top=dict(
+                files=self._calculate_top_files(current_report["memory"]["files"])[
+                    0:max_top_items
+                ],
+                symbols=self._calculate_top_symbols(current_report["memory"]["files"])[
+                    0:max_top_items
+                ],
+                sections=sorted(
+                    current_report["memory"]["sections"].values(),
+                    key=lambda item: item["size"],
+                    reverse=True,
+                )[0:max_top_items],
+            ),
+        )
+
+    @staticmethod
+    def _calculate_top_files(items):
+        return [
+            {"path": item["path"], "ram": item["ram_size"], "flash": item["flash_size"]}
+            for item in sorted(
+                items,
+                key=lambda item: item["ram_size"] + item["flash_size"],
+                reverse=True,
+            )
+        ]
+
+    @staticmethod
+    def _calculate_top_symbols(files):
+        symbols = functools.reduce(
+            lambda result, filex: result
+            + [
+                {
+                    "name": s["name"],
+                    "type": s["type"],
+                    "size": s["size"],
+                    "file": filex["path"],
+                    "line": s.get("line"),
+                }
+                for s in filex["symbols"]
+            ],
+            files,
+            [],
+        )
+        return sorted(symbols, key=lambda item: item["size"], reverse=True)
+
+    async def history(self, project_dir, env, nums=10):
+        result = []
+        report_dir = memusage.get_report_dir(project_dir, env)
+        reports = memusage.list_reports(report_dir)[nums * -1 :]
+        for path in reports:
+            data = memusage.read_report(path)
+            result.append(
+                {
+                    "timestamp": data["timestamp"],
+                    "ram": data["memory"]["total"]["ram_size"],
+                    "flash": data["memory"]["total"]["flash_size"],
+                }
+            )
+        return result

platformio/home/rpc/handlers/misc.py

@@ -22,6 +22,8 @@ from platformio.home.rpc.handlers.os import OSRPC
 
 
 class MiscRPC(BaseRPCHandler):
+    NAMESPACE = "misc"
+
     async def load_latest_tweets(self, data_url):
         cache_key = ContentCache.key_from_args(data_url, "tweets")
         cache_valid = "180d"

platformio/home/rpc/handlers/os.py

@@ -15,32 +15,22 @@
 import glob
 import io
 import os
-import shutil
 from functools import cmp_to_key
 
 import click
 
 from platformio import fs
 from platformio.cache import ContentCache
-from platformio.compat import aio_to_thread
 from platformio.device.list.util import list_logical_devices
 from platformio.home.rpc.handlers.base import BaseRPCHandler
 from platformio.http import HTTPSession, ensure_internet_on
 
 
-class HTTPAsyncSession(HTTPSession):
-    async def request(  # pylint: disable=signature-differs,invalid-overridden-method
-        self, *args, **kwargs
-    ):
-        func = super().request
-        return await aio_to_thread(func, *args, **kwargs)
-
-
 class OSRPC(BaseRPCHandler):
-    _http_session = None
+    NAMESPACE = "os"
 
     @classmethod
-    async def fetch_content(cls, url, data=None, headers=None, cache_valid=None):
+    def fetch_content(cls, url, data=None, headers=None, cache_valid=None):
         if not headers:
             headers = {
                 "User-Agent": (
@@ -52,35 +42,33 @@ class OSRPC(BaseRPCHandler):
         cache_key = ContentCache.key_from_args(url, data) if cache_valid else None
         with ContentCache() as cc:
             if cache_key:
-                result = cc.get(cache_key)
-                if result is not None:
-                    return result
+                content = cc.get(cache_key)
+                if content is not None:
+                    return content
 
         # check internet before and resolve issue with 60 seconds timeout
         ensure_internet_on(raise_exception=True)
 
-        if not cls._http_session:
-            cls._http_session = HTTPAsyncSession()
+        with HTTPSession() as session:
+            if data:
+                response = session.post(url, data=data, headers=headers)
+            else:
+                response = session.get(url, headers=headers)
 
-        if data:
-            r = await cls._http_session.post(url, data=data, headers=headers)
-        else:
-            r = await cls._http_session.get(url, headers=headers)
+            response.raise_for_status()
+            content = response.text
+            if cache_valid:
+                with ContentCache() as cc:
+                    cc.set(cache_key, content, cache_valid)
+            return content
 
-        r.raise_for_status()
-        result = r.text
-        if cache_valid:
-            with ContentCache() as cc:
-                cc.set(cache_key, result, cache_valid)
-        return result
-
-    async def request_content(self, uri, data=None, headers=None, cache_valid=None):
+    @classmethod
+    def request_content(cls, uri, data=None, headers=None, cache_valid=None):
         if uri.startswith("http"):
-            return await self.fetch_content(uri, data, headers, cache_valid)
+            return cls.fetch_content(uri, data, headers, cache_valid)
         local_path = uri[7:] if uri.startswith("file://") else uri
         with io.open(local_path, encoding="utf-8") as fp:
             return fp.read()
-        return None
 
     @staticmethod
     def open_url(url):
@@ -110,22 +98,10 @@ class OSRPC(BaseRPCHandler):
     def is_dir(path):
         return os.path.isdir(path)
 
-    @staticmethod
-    def make_dirs(path):
-        return os.makedirs(path)
-
     @staticmethod
     def get_file_mtime(path):
         return os.path.getmtime(path)
 
-    @staticmethod
-    def rename(src, dst):
-        return os.rename(src, dst)
-
-    @staticmethod
-    def copy(src, dst):
-        return shutil.copytree(src, dst, symlinks=True)
-
     @staticmethod
     def glob(pathnames, root=None):
         if not isinstance(pathnames, list):

platformio/home/rpc/handlers/piocore.py

@@ -1,229 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import asyncio
-import functools
-import io
-import json
-import os
-import sys
-import threading
-
-import click
-from ajsonrpc.core import JSONRPC20DispatchException
-
-from platformio import __main__, __version__, app, fs, proc, util
-from platformio.compat import (
-    IS_WINDOWS,
-    aio_create_task,
-    aio_get_running_loop,
-    aio_to_thread,
-    get_locale_encoding,
-    is_bytes,
-)
-from platformio.exception import PlatformioException
-from platformio.home.rpc.handlers.base import BaseRPCHandler
-
-
-class PIOCoreProtocol(asyncio.SubprocessProtocol):
-    def __init__(self, exit_future, on_data_callback=None):
-        self.exit_future = exit_future
-        self.on_data_callback = on_data_callback
-        self.stdout = ""
-        self.stderr = ""
-        self._is_exited = False
-        self._encoding = get_locale_encoding()
-
-    def pipe_data_received(self, fd, data):
-        data = data.decode(self._encoding, "replace")
-        pipe = ["stdin", "stdout", "stderr"][fd]
-        if pipe == "stdout":
-            self.stdout += data
-        if pipe == "stderr":
-            self.stderr += data
-        if self.on_data_callback:
-            self.on_data_callback(pipe=pipe, data=data)
-
-    def connection_lost(self, exc):
-        self.process_exited()
-
-    def process_exited(self):
-        if self._is_exited:
-            return
-        self.exit_future.set_result(True)
-        self._is_exited = True
-
-
-class MultiThreadingStdStream:
-    def __init__(self, parent_stream):
-        self._buffers = {threading.get_ident(): parent_stream}
-
-    def __getattr__(self, name):
-        thread_id = threading.get_ident()
-        self._ensure_thread_buffer(thread_id)
-        return getattr(self._buffers[thread_id], name)
-
-    def _ensure_thread_buffer(self, thread_id):
-        if thread_id not in self._buffers:
-            self._buffers[thread_id] = io.StringIO()
-
-    def write(self, value):
-        thread_id = threading.get_ident()
-        self._ensure_thread_buffer(thread_id)
-        return self._buffers[thread_id].write(
-            value.decode() if is_bytes(value) else value
-        )
-
-    def get_value_and_reset(self):
-        result = ""
-        try:
-            result = self.getvalue()
-            self.seek(0)
-            self.truncate(0)
-        except AttributeError:
-            pass
-        return result
-
-
-@util.memoized(expire="60s")
-def get_core_fullpath():
-    return proc.where_is_program("platformio" + (".exe" if IS_WINDOWS else ""))
-
-
-class PIOCoreRPC(BaseRPCHandler):
-    @staticmethod
-    def version():
-        return __version__
-
-    async def exec(self, args, options=None):
-        loop = aio_get_running_loop()
-        exit_future = loop.create_future()
-        data_callback = functools.partial(
-            self._on_exec_data_received, exec_options=options
-        )
-        if args[0] != "--caller" and app.get_session_var("caller_id"):
-            args = ["--caller", app.get_session_var("caller_id")] + args
-        transport, protocol = await loop.subprocess_exec(
-            lambda: PIOCoreProtocol(exit_future, data_callback),
-            get_core_fullpath(),
-            *args,
-            stdin=None,
-            **options.get("spawn", {}),
-        )
-        await exit_future
-        transport.close()
-        return {
-            "stdout": protocol.stdout,
-            "stderr": protocol.stderr,
-            "returncode": transport.get_returncode(),
-        }
-
-    def _on_exec_data_received(self, exec_options, pipe, data):
-        notification_method = exec_options.get(f"{pipe}NotificationMethod")
-        if not notification_method:
-            return
-        aio_create_task(
-            self.factory.notify_clients(
-                method=notification_method,
-                params=[data],
-                actor="frontend",
-            )
-        )
-
-    @staticmethod
-    def setup_multithreading_std_streams():
-        if isinstance(sys.stdout, MultiThreadingStdStream):
-            return
-        PIOCoreRPC.thread_stdout = MultiThreadingStdStream(sys.stdout)
-        PIOCoreRPC.thread_stderr = MultiThreadingStdStream(sys.stderr)
-        sys.stdout = PIOCoreRPC.thread_stdout
-        sys.stderr = PIOCoreRPC.thread_stderr
-
-    @staticmethod
-    async def call(args, options=None):
-        for i, arg in enumerate(args):
-            if not isinstance(arg, str):
-                args[i] = str(arg)
-
-        options = options or {}
-        to_json = "--json-output" in args
-
-        try:
-            if options.get("force_subprocess"):
-                result = await PIOCoreRPC._call_subprocess(args, options)
-                return PIOCoreRPC._process_result(result, to_json)
-            result = await PIOCoreRPC._call_inline(args, options)
-            try:
-                return PIOCoreRPC._process_result(result, to_json)
-            except ValueError:
-                # fall-back to subprocess method
-                result = await PIOCoreRPC._call_subprocess(args, options)
-                return PIOCoreRPC._process_result(result, to_json)
-        except Exception as exc:  # pylint: disable=bare-except
-            raise JSONRPC20DispatchException(
-                code=5000, message="PIO Core Call Error", data=str(exc)
-            ) from exc
-
-    @staticmethod
-    async def _call_subprocess(args, options):
-        result = await aio_to_thread(
-            proc.exec_command,
-            [get_core_fullpath()] + args,
-            cwd=options.get("cwd") or os.getcwd(),
-        )
-        return (result["out"], result["err"], result["returncode"])
-
-    @staticmethod
-    async def _call_inline(args, options):
-        PIOCoreRPC.setup_multithreading_std_streams()
-
-        def _thread_safe_call(args, cwd):
-            with fs.cd(cwd):
-                exit_code = __main__.main(["-c"] + args)
-            return (
-                PIOCoreRPC.thread_stdout.get_value_and_reset(),
-                PIOCoreRPC.thread_stderr.get_value_and_reset(),
-                exit_code,
-            )
-
-        return await aio_to_thread(
-            _thread_safe_call, args=args, cwd=options.get("cwd") or os.getcwd()
-        )
-
-    @staticmethod
-    def _process_result(result, to_json=False):
-        out, err, code = result
-        if out and is_bytes(out):
-            out = out.decode(get_locale_encoding())
-        if err and is_bytes(err):
-            err = err.decode(get_locale_encoding())
-        text = ("%s\n\n%s" % (out, err)).strip()
-        if code != 0:
-            raise PlatformioException(text)
-        if not to_json:
-            return text
-        try:
-            return json.loads(out)
-        except ValueError as exc:
-            click.secho("%s => `%s`" % (exc, out), fg="red", err=True)
-            # if PIO Core prints unhandled warnings
-            for line in out.split("\n"):
-                line = line.strip()
-                if not line:
-                    continue
-                try:
-                    return json.loads(line)
-                except ValueError:
-                    pass
-            raise exc

platformio/home/rpc/handlers/platform.py

@@ -14,8 +14,8 @@
 
 import os.path
 
-from platformio.compat import aio_to_thread
 from platformio.home.rpc.handlers.base import BaseRPCHandler
+from platformio.home.rpc.handlers.registry import RegistryRPC
 from platformio.package.manager.platform import PlatformPackageManager
 from platformio.package.manifest.parser import ManifestParserFactory
 from platformio.package.meta import PackageSpec
@@ -23,15 +23,13 @@ from platformio.platform.factory import PlatformFactory
 
 
 class PlatformRPC(BaseRPCHandler):
-    async def fetch_platforms(self, search_query=None, page=0, force_installed=False):
-        if force_installed:
-            return {
-                "items": await aio_to_thread(
-                    self._load_installed_platforms, search_query
-                )
-            }
+    NAMESPACE = "platform"
 
-        search_result = await self.factory.manager.dispatcher["registry.call_client"](
+    def fetch_platforms(self, search_query=None, page=0, force_installed=False):
+        if force_installed:
+            return {"items": self._load_installed_platforms(search_query)}
+
+        search_result = RegistryRPC.call_client(
             method="list_packages",
             query=search_query,
             qualifiers={
@@ -88,17 +86,17 @@ class PlatformRPC(BaseRPCHandler):
             )
         return items
 
-    async def fetch_boards(self, platform_spec):
+    def fetch_boards(self, platform_spec):
         spec = PackageSpec(platform_spec)
         if spec.owner:
-            return await self.factory.manager.dispatcher["registry.call_client"](
+            return RegistryRPC.call_client(
                 method="get_package",
                 typex="platform",
                 owner=spec.owner,
                 name=spec.name,
                 extra_path="/boards",
             )
-        return await aio_to_thread(self._load_installed_boards, spec)
+        return self._load_installed_boards(spec)
 
     @staticmethod
     def _load_installed_boards(platform_spec):
@@ -108,17 +106,17 @@ class PlatformRPC(BaseRPCHandler):
             key=lambda item: item["name"],
         )
 
-    async def fetch_examples(self, platform_spec):
+    def fetch_examples(self, platform_spec):
         spec = PackageSpec(platform_spec)
         if spec.owner:
-            return await self.factory.manager.dispatcher["registry.call_client"](
+            return RegistryRPC.call_client(
                 method="get_package",
                 typex="platform",
                 owner=spec.owner,
                 name=spec.name,
                 extra_path="/examples",
             )
-        return await aio_to_thread(self._load_installed_examples, spec)
+        return self._load_installed_examples(spec)
 
     @staticmethod
     def _load_installed_examples(platform_spec):

platformio/home/rpc/handlers/project.py

@@ -13,29 +13,24 @@
 # limitations under the License.
 
 import os
-import shutil
-import time
 from pathlib import Path
 
 import semantic_version
-from ajsonrpc.core import JSONRPC20DispatchException
 
-from platformio import app, exception, fs
-from platformio.home.rpc.handlers.app import AppRPC
+from platformio import app, fs
 from platformio.home.rpc.handlers.base import BaseRPCHandler
-from platformio.home.rpc.handlers.piocore import PIOCoreRPC
 from platformio.package.manager.platform import PlatformPackageManager
 from platformio.platform.factory import PlatformFactory
 from platformio.project.config import ProjectConfig
-from platformio.project.exception import ProjectError
-from platformio.project.helpers import get_project_dir, is_platformio_project
+from platformio.project.helpers import get_project_dir
 from platformio.project.integration.generator import ProjectGenerator
-from platformio.project.options import get_config_options_schema
 
 
 class ProjectRPC(BaseRPCHandler):
+    NAMESPACE = "project"
+
     @staticmethod
-    def config_call(init_kwargs, method, *args):
+    async def config_call(init_kwargs, method, *args):
         assert isinstance(init_kwargs, dict)
         assert "path" in init_kwargs
         if os.path.isdir(init_kwargs["path"]):
@@ -48,249 +43,20 @@ class ProjectRPC(BaseRPCHandler):
         with fs.cd(project_dir):
             return getattr(ProjectConfig(**init_kwargs), method)(*args)
 
-    @staticmethod
-    def config_load(path):
-        return ProjectConfig(
-            path, parse_extra=False, expand_interpolations=False
-        ).as_tuple()
-
-    @staticmethod
-    def config_dump(path, data):
-        config = ProjectConfig(path, parse_extra=False, expand_interpolations=False)
-        config.update(data, clear=True)
-        return config.save()
-
-    @staticmethod
-    def config_update_description(path, text):
-        config = ProjectConfig(path, parse_extra=False, expand_interpolations=False)
-        if not config.has_section("platformio"):
-            config.add_section("platformio")
-        if text:
-            config.set("platformio", "description", text)
-        else:
-            if config.has_option("platformio", "description"):
-                config.remove_option("platformio", "description")
-            if not config.options("platformio"):
-                config.remove_section("platformio")
-        return config.save()
-
-    @staticmethod
-    def get_config_schema():
-        return get_config_options_schema()
-
-    @staticmethod
-    def get_projects():
-        def _get_project_data():
-            data = {"boards": [], "envLibdepsDirs": [], "libExtraDirs": []}
-            config = ProjectConfig()
-            data["envs"] = config.envs()
-            data["description"] = config.get("platformio", "description")
-            data["libExtraDirs"].extend(config.get("platformio", "lib_extra_dirs", []))
-
-            libdeps_dir = config.get("platformio", "libdeps_dir")
-            for section in config.sections():
-                if not section.startswith("env:"):
-                    continue
-                data["envLibdepsDirs"].append(os.path.join(libdeps_dir, section[4:]))
-                if config.has_option(section, "board"):
-                    data["boards"].append(config.get(section, "board"))
-                data["libExtraDirs"].extend(config.get(section, "lib_extra_dirs", []))
-
-            # skip non existing folders and resolve full path
-            for key in ("envLibdepsDirs", "libExtraDirs"):
-                data[key] = [
-                    fs.expanduser(d) if d.startswith("~") else os.path.abspath(d)
-                    for d in data[key]
-                    if os.path.isdir(d)
-                ]
-
-            return data
-
-        def _path_to_name(path):
-            return (os.path.sep).join(path.split(os.path.sep)[-2:])
-
-        result = []
-        pm = PlatformPackageManager()
-        for project_dir in AppRPC.load_state()["storage"]["recentProjects"]:
-            if not os.path.isdir(project_dir):
-                continue
-            data = {}
-            boards = []
-            try:
-                with fs.cd(project_dir):
-                    data = _get_project_data()
-            except ProjectError:
-                continue
-
-            for board_id in data.get("boards", []):
-                name = board_id
-                try:
-                    name = pm.board_config(board_id)["name"]
-                except exception.PlatformioException:
-                    pass
-                boards.append({"id": board_id, "name": name})
-
-            result.append(
-                {
-                    "path": project_dir,
-                    "name": _path_to_name(project_dir),
-                    "modified": int(os.path.getmtime(project_dir)),
-                    "boards": boards,
-                    "description": data.get("description"),
-                    "envs": data.get("envs", []),
-                    "envLibStorages": [
-                        {"name": os.path.basename(d), "path": d}
-                        for d in data.get("envLibdepsDirs", [])
-                    ],
-                    "extraLibStorages": [
-                        {"name": _path_to_name(d), "path": d}
-                        for d in data.get("libExtraDirs", [])
-                    ],
-                }
-            )
-        return result
-
-    @staticmethod
-    def get_project_examples():
-        result = []
-        pm = PlatformPackageManager()
-        for pkg in pm.get_installed():
-            examples_dir = os.path.join(pkg.path, "examples")
-            if not os.path.isdir(examples_dir):
-                continue
-            items = []
-            for project_dir, _, __ in os.walk(examples_dir):
-                project_description = None
-                try:
-                    config = ProjectConfig(os.path.join(project_dir, "platformio.ini"))
-                    config.validate(silent=True)
-                    project_description = config.get("platformio", "description")
-                except ProjectError:
-                    continue
-
-                path_tokens = project_dir.split(os.path.sep)
-                items.append(
-                    {
-                        "name": "/".join(
-                            path_tokens[path_tokens.index("examples") + 1 :]
-                        ),
-                        "path": project_dir,
-                        "description": project_description,
-                    }
-                )
-            manifest = pm.load_manifest(pkg)
-            result.append(
-                {
-                    "platform": {
-                        "title": manifest["title"],
-                        "version": manifest["version"],
-                    },
-                    "items": sorted(items, key=lambda item: item["name"]),
-                }
-            )
-        return sorted(result, key=lambda data: data["platform"]["title"])
-
-    async def init(self, board, framework, project_dir):
-        assert project_dir
-        if not os.path.isdir(project_dir):
-            os.makedirs(project_dir)
-        args = ["init", "--board", board, "--sample-code"]
-        if framework:
-            args.extend(["--project-option", "framework = %s" % framework])
-        ide = app.get_session_var("caller_id")
-        if ide in ProjectGenerator.get_supported_ides():
-            args.extend(["--ide", ide])
-        await PIOCoreRPC.call(
-            args, options={"cwd": project_dir, "force_subprocess": True}
-        )
-        return project_dir
-
-    @staticmethod
-    async def import_arduino(board, use_arduino_libs, arduino_project_dir):
-        board = str(board)
-        # don't import PIO Project
-        if is_platformio_project(arduino_project_dir):
-            return arduino_project_dir
-
-        is_arduino_project = any(
-            os.path.isfile(
-                os.path.join(
-                    arduino_project_dir,
-                    "%s.%s" % (os.path.basename(arduino_project_dir), ext),
-                )
-            )
-            for ext in ("ino", "pde")
-        )
-        if not is_arduino_project:
-            raise JSONRPC20DispatchException(
-                code=4000, message="Not an Arduino project: %s" % arduino_project_dir
-            )
-
-        state = AppRPC.load_state()
-        project_dir = os.path.join(
-            state["storage"]["projectsDir"], time.strftime("%y%m%d-%H%M%S-") + board
-        )
-        if not os.path.isdir(project_dir):
-            os.makedirs(project_dir)
-        args = ["init", "--board", board]
-        args.extend(["--project-option", "framework = arduino"])
-        if use_arduino_libs:
-            args.extend(
-                ["--project-option", "lib_extra_dirs = ~/Documents/Arduino/libraries"]
-            )
-        ide = app.get_session_var("caller_id")
-        if ide in ProjectGenerator.get_supported_ides():
-            args.extend(["--ide", ide])
-        await PIOCoreRPC.call(
-            args, options={"cwd": project_dir, "force_subprocess": True}
-        )
-        with fs.cd(project_dir):
-            config = ProjectConfig()
-            src_dir = config.get("platformio", "src_dir")
-            if os.path.isdir(src_dir):
-                fs.rmtree(src_dir)
-            shutil.copytree(arduino_project_dir, src_dir, symlinks=True)
-        return project_dir
-
-    @staticmethod
-    async def import_pio(project_dir):
-        if not project_dir or not is_platformio_project(project_dir):
-            raise JSONRPC20DispatchException(
-                code=4001, message="Not an PlatformIO project: %s" % project_dir
-            )
-        new_project_dir = os.path.join(
-            AppRPC.load_state()["storage"]["projectsDir"],
-            time.strftime("%y%m%d-%H%M%S-") + os.path.basename(project_dir),
-        )
-        shutil.copytree(project_dir, new_project_dir, symlinks=True)
-
-        args = ["init"]
-        ide = app.get_session_var("caller_id")
-        if ide in ProjectGenerator.get_supported_ides():
-            args.extend(["--ide", ide])
-        await PIOCoreRPC.call(
-            args, options={"cwd": new_project_dir, "force_subprocess": True}
-        )
-        return new_project_dir
-
-    async def init_v2(self, configuration, options=None):
+    async def init(self, configuration, options=None):
         project_dir = os.path.join(configuration["location"], configuration["name"])
         if not os.path.isdir(project_dir):
             os.makedirs(project_dir)
 
-        envclone = os.environ.copy()
-        envclone["PLATFORMIO_FORCE_ANSI"] = "true"
-        options = options or {}
-        options["spawn"] = {"env": envclone, "cwd": project_dir}
-
-        args = ["project", "init"]
+        args = ["project", "init", "-d", project_dir]
         ide = app.get_session_var("caller_id")
         if ide in ProjectGenerator.get_supported_ides():
             args.extend(["--ide", ide])
 
+        exec_options = options.get("exec", {})
         if configuration.get("example"):
             await self.factory.notify_clients(
-                method=options.get("stdoutNotificationMethod"),
+                method=exec_options.get("stdoutNotificationMethod"),
                 params=["Copying example files...\n"],
                 actor="frontend",
             )
@@ -298,7 +64,9 @@ class ProjectRPC(BaseRPCHandler):
         else:
             args.extend(self._pre_init_empty(configuration))
 
-        return await self.factory.manager.dispatcher["core.exec"](args, options=options)
+        return await self.factory.manager.dispatcher["core.exec"](
+            args, options=exec_options, raise_exception=False
+        )
 
     @staticmethod
     def _pre_init_empty(configuration):
@@ -342,10 +110,10 @@ class ProjectRPC(BaseRPCHandler):
         return []
 
     @staticmethod
-    def configuration(project_dir, env):
-        assert is_platformio_project(project_dir)
+    async def configuration(project_dir, env):
         with fs.cd(project_dir):
-            config = ProjectConfig(os.path.join(project_dir, "platformio.ini"))
+            config = ProjectConfig.get_instance()
+            config.validate(envs=[env])
             platform = PlatformFactory.from_env(env, autoinstall=True)
             platform_pkg = PlatformPackageManager().get_package(platform.get_dir())
             board_id = config.get(f"env:{env}", "board", None)

platformio/home/rpc/handlers/registry.py

@@ -12,20 +12,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from ajsonrpc.core import JSONRPC20DispatchException
-
-from platformio.compat import aio_to_thread
 from platformio.home.rpc.handlers.base import BaseRPCHandler
 from platformio.registry.client import RegistryClient
 
 
 class RegistryRPC(BaseRPCHandler):
+    NAMESPACE = "registry"
+
     @staticmethod
-    async def call_client(method, *args, **kwargs):
-        try:
-            client = RegistryClient()
-            return await aio_to_thread(getattr(client, method), *args, **kwargs)
-        except Exception as exc:  # pylint: disable=bare-except
-            raise JSONRPC20DispatchException(
-                code=5000, message="Registry Call Error", data=str(exc)
-            ) from exc
+    def call_client(method, *args, **kwargs):
+        with RegistryClient() as client:
+            return getattr(client, method)(*args, **kwargs)

platformio/home/rpc/server.py

@@ -12,22 +12,24 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import functools
+import inspect
 from urllib.parse import parse_qs
 
-import ajsonrpc.utils
+import ajsonrpc.manager
 import click
 from ajsonrpc.core import JSONRPC20Error, JSONRPC20Request
 from ajsonrpc.dispatcher import Dispatcher
 from ajsonrpc.manager import AsyncJSONRPCResponseManager, JSONRPC20Response
 from starlette.endpoints import WebSocketEndpoint
 
-from platformio.compat import aio_create_task, aio_get_running_loop
+from platformio.compat import aio_create_task, aio_get_running_loop, aio_to_thread
 from platformio.http import InternetConnectionError
 from platformio.proc import force_exit
 
 # Remove this line when PR is merged
 # https://github.com/pavlov99/ajsonrpc/pull/22
-ajsonrpc.utils.is_invalid_params = lambda: False
+ajsonrpc.manager.is_invalid_params = lambda *args, **kwargs: False
 
 
 class JSONRPCServerFactoryBase:
@@ -44,9 +46,18 @@ class JSONRPCServerFactoryBase:
     def __call__(self, *args, **kwargs):
         raise NotImplementedError
 
-    def add_object_handler(self, handler, namespace):
-        handler.factory = self
-        self.manager.dispatcher.add_object(handler, prefix="%s." % namespace)
+    def add_object_handler(self, obj):
+        obj.factory = self
+        namespace = obj.NAMESPACE or obj.__class__.__name__
+        for name in dir(obj):
+            method = getattr(obj, name)
+            if name.startswith("_") or not (
+                inspect.ismethod(method) or inspect.isfunction(method)
+            ):
+                continue
+            if not inspect.iscoroutinefunction(method):
+                method = functools.partial(aio_to_thread, method)
+            self.manager.dispatcher.add_function(method, name=f"{namespace}.{name}")
 
     def on_client_connect(self, connection, actor=None):
         self._clients[connection] = {"actor": actor}

platformio/home/run.py

@@ -28,10 +28,11 @@ from platformio.compat import aio_get_running_loop
 from platformio.exception import PlatformioException
 from platformio.home.rpc.handlers.account import AccountRPC
 from platformio.home.rpc.handlers.app import AppRPC
+from platformio.home.rpc.handlers.core import CoreRPC
 from platformio.home.rpc.handlers.ide import IDERPC
+from platformio.home.rpc.handlers.memusage import MemUsageRPC
 from platformio.home.rpc.handlers.misc import MiscRPC
 from platformio.home.rpc.handlers.os import OSRPC
-from platformio.home.rpc.handlers.piocore import PIOCoreRPC
 from platformio.home.rpc.handlers.platform import PlatformRPC
 from platformio.home.rpc.handlers.project import ProjectRPC
 from platformio.home.rpc.handlers.registry import RegistryRPC
@@ -67,15 +68,16 @@ def run_server(host, port, no_open, shutdown_timeout, home_url):
         raise PlatformioException("Invalid path to PIO Home Contrib")
 
     ws_rpc_factory = WebSocketJSONRPCServerFactory(shutdown_timeout)
-    ws_rpc_factory.add_object_handler(AccountRPC(), namespace="account")
-    ws_rpc_factory.add_object_handler(AppRPC(), namespace="app")
-    ws_rpc_factory.add_object_handler(IDERPC(), namespace="ide")
-    ws_rpc_factory.add_object_handler(MiscRPC(), namespace="misc")
-    ws_rpc_factory.add_object_handler(OSRPC(), namespace="os")
-    ws_rpc_factory.add_object_handler(PIOCoreRPC(), namespace="core")
-    ws_rpc_factory.add_object_handler(ProjectRPC(), namespace="project")
-    ws_rpc_factory.add_object_handler(PlatformRPC(), namespace="platform")
-    ws_rpc_factory.add_object_handler(RegistryRPC(), namespace="registry")
+    ws_rpc_factory.add_object_handler(AccountRPC())
+    ws_rpc_factory.add_object_handler(AppRPC())
+    ws_rpc_factory.add_object_handler(IDERPC())
+    ws_rpc_factory.add_object_handler(MemUsageRPC())
+    ws_rpc_factory.add_object_handler(MiscRPC())
+    ws_rpc_factory.add_object_handler(OSRPC())
+    ws_rpc_factory.add_object_handler(CoreRPC())
+    ws_rpc_factory.add_object_handler(ProjectRPC())
+    ws_rpc_factory.add_object_handler(PlatformRPC())
+    ws_rpc_factory.add_object_handler(RegistryRPC())
 
     path = urlparse(home_url).path
     routes = [

platformio/http.py

@@ -12,22 +12,25 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import contextlib
+import itertools
 import json
 import socket
-from urllib.parse import urljoin
+import time
 
-import requests.adapters
-from urllib3.util.retry import Retry
+import httpx
 
 from platformio import __check_internet_hosts__, app, util
 from platformio.cache import ContentCache, cleanup_content_cache
 from platformio.compat import is_proxy_set
 from platformio.exception import PlatformioException, UserSideException
 
-__default_requests_timeout__ = (10, None)  # (connect, read)
+RETRIES_BACKOFF_FACTOR = 2  # 0s, 2s, 4s, 8s, etc.
+RETRIES_METHOD_WHITELIST = ["GET"]
+RETRIES_STATUS_FORCELIST = [429, 500, 502, 503, 504]
 
 
-class HTTPClientError(UserSideException):
+class HttpClientApiError(UserSideException):
     def __init__(self, message, response=None):
         super().__init__()
         self.message = message
@@ -40,86 +43,138 @@ class HTTPClientError(UserSideException):
 class InternetConnectionError(UserSideException):
     MESSAGE = (
         "You are not connected to the Internet.\n"
-        "PlatformIO needs the Internet connection to"
-        " download dependent packages or to work with PlatformIO Account."
+        "PlatformIO needs the Internet connection to "
+        "download dependent packages or to work with PlatformIO Account."
     )
 
 
-class HTTPSession(requests.Session):
-    def __init__(self, *args, **kwargs):
-        self._x_base_url = kwargs.pop("x_base_url") if "x_base_url" in kwargs else None
-        super().__init__(*args, **kwargs)
-        self.headers.update({"User-Agent": app.get_user_agent()})
-        try:
-            self.verify = app.get_setting("enable_proxy_strict_ssl")
-        except PlatformioException:
-            self.verify = True
+def exponential_backoff(factor):
+    yield 0
+    for n in itertools.count(2):
+        yield factor * (2 ** (n - 2))
 
-    def request(  # pylint: disable=signature-differs,arguments-differ
-        self, method, url, *args, **kwargs
+
+def apply_default_kwargs(kwargs=None):
+    kwargs = kwargs or {}
+    # enable redirects by default
+    kwargs["follow_redirects"] = kwargs.get("follow_redirects", True)
+
+    try:
+        kwargs["verify"] = kwargs.get(
+            "verify", app.get_setting("enable_proxy_strict_ssl")
+        )
+    except PlatformioException:
+        kwargs["verify"] = True
+
+    headers = kwargs.pop("headers", {})
+    if "User-Agent" not in headers:
+        headers.update({"User-Agent": app.get_user_agent()})
+        kwargs["headers"] = headers
+
+    retry = kwargs.pop("retry", None)
+    if retry:
+        kwargs["transport"] = HTTPRetryTransport(verify=kwargs["verify"], **retry)
+
+    return kwargs
+
+
+class HTTPRetryTransport(httpx.HTTPTransport):
+    def __init__(  # pylint: disable=too-many-arguments
+        self,
+        verify=True,
+        retries=1,
+        backoff_factor=None,
+        status_forcelist=None,
+        method_whitelist=None,
     ):
-        # print("HTTPSession::request", self._x_base_url, method, url, args, kwargs)
-        if "timeout" not in kwargs:
-            kwargs["timeout"] = __default_requests_timeout__
-        return super().request(
-            method,
-            (
-                url
-                if url.startswith("http") or not self._x_base_url
-                else urljoin(self._x_base_url, url)
-            ),
-            *args,
-            **kwargs
+        super().__init__(verify=verify)
+        self._retries = retries
+        self._backoff_factor = backoff_factor or RETRIES_BACKOFF_FACTOR
+        self._status_forcelist = status_forcelist or RETRIES_STATUS_FORCELIST
+        self._method_whitelist = method_whitelist or RETRIES_METHOD_WHITELIST
+
+    def handle_request(self, request):
+        retries_left = self._retries
+        delays = exponential_backoff(factor=RETRIES_BACKOFF_FACTOR)
+        while retries_left > 0:
+            retries_left -= 1
+            try:
+                response = super().handle_request(request)
+                if response.status_code in RETRIES_STATUS_FORCELIST:
+                    if request.method.upper() not in self._method_whitelist:
+                        return response
+                    raise httpx.HTTPStatusError(
+                        f"Server error '{response.status_code} {response.reason_phrase}' "
+                        f"for url '{request.url}'\n",
+                        request=request,
+                        response=response,
+                    )
+                return response
+            except httpx.HTTPError:
+                if retries_left == 0:
+                    raise
+                time.sleep(next(delays) or 1)
+
+        raise httpx.RequestError(
+            f"Could not process '{request.url}' request", request=request
         )
 
 
-class HTTPSessionIterator:
-    def __init__(self, endpoints):
+class HTTPSession(httpx.Client):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **apply_default_kwargs(kwargs))
+
+
+class HttpEndpointPool:
+    def __init__(self, endpoints, session_retry=None):
         if not isinstance(endpoints, list):
             endpoints = [endpoints]
         self.endpoints = endpoints
-        self.endpoints_iter = iter(endpoints)
-        # https://urllib3.readthedocs.io/en/stable/reference/urllib3.util.html
-        self.retry = Retry(
-            total=5,
-            backoff_factor=1,  # [0, 2, 4, 8, 16] secs
-            # method_whitelist=list(Retry.DEFAULT_METHOD_WHITELIST) + ["POST"],
-            status_forcelist=[413, 429, 500, 502, 503, 504],
-        )
+        self.session_retry = session_retry
 
-    def __iter__(self):  # pylint: disable=non-iterator-returned
-        return self
-
-    def __next__(self):
-        base_url = next(self.endpoints_iter)
-        session = HTTPSession(x_base_url=base_url)
-        adapter = requests.adapters.HTTPAdapter(max_retries=self.retry)
-        session.mount(base_url, adapter)
-        return session
-
-
-class HTTPClient:
-    def __init__(self, endpoints):
-        self._session_iter = HTTPSessionIterator(endpoints)
-        self._session = None
-        self._next_session()
-
-    def __del__(self):
-        if not self._session:
-            return
-        try:
-            self._session.close()
-        except:  # pylint: disable=bare-except
-            pass
+        self._endpoints_iter = iter(endpoints)
         self._session = None
 
-    def _next_session(self):
+        self.next()
+
+    def close(self):
         if self._session:
             self._session.close()
-        self._session = next(self._session_iter)
+
+    def next(self):
+        if self._session:
+            self._session.close()
+        self._session = HTTPSession(
+            base_url=next(self._endpoints_iter), retry=self.session_retry
+        )
+
+    def request(self, method, *args, **kwargs):
+        while True:
+            try:
+                return self._session.request(method, *args, **kwargs)
+            except httpx.HTTPError as exc:
+                try:
+                    self.next()
+                except StopIteration as exc2:
+                    raise exc from exc2
+
+
+class HttpApiClient(contextlib.AbstractContextManager):
+    def __init__(self, endpoints):
+        self._endpoint = HttpEndpointPool(endpoints, session_retry=dict(retries=5))
+
+    def __exit__(self, *excinfo):
+        self.close()
+
+    def __del__(self):
+        self.close()
+
+    def close(self):
+        if getattr(self, "_endpoint"):
+            self._endpoint.close()
 
     @util.throttle(500)
-    def send_request(self, method, path, **kwargs):
+    def send_request(self, method, *args, **kwargs):
         # check Internet before and resolve issue with 60 seconds timeout
         ensure_internet_on(raise_exception=True)
 
@@ -133,23 +188,28 @@ class HTTPClient:
             # pylint: disable=import-outside-toplevel
             from platformio.account.client import AccountClient
 
-            headers["Authorization"] = (
-                "Bearer %s" % AccountClient().fetch_authentication_token()
-            )
+            with AccountClient() as client:
+                headers["Authorization"] = (
+                    "Bearer %s" % client.fetch_authentication_token()
+                )
         kwargs["headers"] = headers
 
-        while True:
-            try:
-                return getattr(self._session, method)(path, **kwargs)
-            except requests.exceptions.RequestException as exc:
-                try:
-                    self._next_session()
-                except Exception as exc2:
-                    raise HTTPClientError(str(exc2)) from exc
+        try:
+            return self._endpoint.request(method, *args, **kwargs)
+        except httpx.HTTPError as exc:
+            raise HttpClientApiError(str(exc)) from exc
 
     def fetch_json_data(self, method, path, **kwargs):
         if method not in ("get", "head", "options"):
             cleanup_content_cache("http")
+        # remove empty params
+        if kwargs.get("params"):
+            kwargs["params"] = {
+                key: value
+                for key, value in kwargs.get("params").items()
+                if value is not None
+            }
+
         cache_valid = kwargs.pop("x_cache_valid") if "x_cache_valid" in kwargs else None
         if not cache_valid:
             return self._parse_json_response(self.send_request(method, path, **kwargs))
@@ -179,7 +239,7 @@ class HTTPClient:
             message = response.json()["message"]
         except (KeyError, ValueError):
             message = response.text
-        raise HTTPClientError(message, response)
+        raise HttpClientApiError(message, response)
 
 
 #
@@ -194,7 +254,7 @@ def _internet_on():
     for host in __check_internet_hosts__:
         try:
             if is_proxy_set():
-                requests.get("http://%s" % host, allow_redirects=False, timeout=timeout)
+                httpx.get("http://%s" % host, follow_redirects=False, timeout=timeout)
                 return True
             # try to resolve `host` for both AF_INET and AF_INET6, and then try to connect
             # to all possible addresses (IPv4 and IPv6) in turn until a connection succeeds:
@@ -213,9 +273,8 @@ def ensure_internet_on(raise_exception=False):
     return result
 
 
-def fetch_remote_content(*args, **kwargs):
-    with HTTPSession() as s:
-        r = s.get(*args, **kwargs)
-        r.raise_for_status()
-        r.close()
-        return r.text
+def fetch_http_content(*args, **kwargs):
+    with HTTPSession() as session:
+        response = session.get(*args, **kwargs)
+        response.raise_for_status()
+        return response.text

platformio/maintenance.py

@@ -23,7 +23,11 @@ from platformio import __version__, app, exception, fs, telemetry
 from platformio.cache import cleanup_content_cache
 from platformio.cli import PlatformioCLI
 from platformio.commands.upgrade import get_latest_version
-from platformio.http import HTTPClientError, InternetConnectionError, ensure_internet_on
+from platformio.http import (
+    HttpClientApiError,
+    InternetConnectionError,
+    ensure_internet_on,
+)
 from platformio.package.manager.core import update_core_packages
 from platformio.package.version import pepver_to_semver
 from platformio.system.prune import calculate_unnecessary_system_data
@@ -46,7 +50,7 @@ def on_cmd_end():
         check_platformio_upgrade()
         check_prune_system()
     except (
-        HTTPClientError,
+        HttpClientApiError,
         InternetConnectionError,
         exception.GetLatestVersionError,
     ):

platformio/package/commands/list.py

@@ -12,8 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import json
 import os
-from typing import List
 
 import click
 
@@ -21,13 +21,13 @@ from platformio import fs
 from platformio.package.manager.library import LibraryPackageManager
 from platformio.package.manager.platform import PlatformPackageManager
 from platformio.package.manager.tool import ToolPackageManager
-from platformio.package.meta import PackageItem, PackageSpec
+from platformio.package.meta import PackageInfo, PackageItem, PackageSpec
 from platformio.platform.exception import UnknownPlatform
 from platformio.platform.factory import PlatformFactory
 from platformio.project.config import ProjectConfig
 
 
-@click.command("list", short_help="List installed packages")
+@click.command("list", short_help="List project packages")
 @click.option(
     "-d",
     "--project-dir",
@@ -48,79 +48,116 @@ from platformio.project.config import ProjectConfig
 @click.option("--only-platforms", is_flag=True, help="List only platform packages")
 @click.option("--only-tools", is_flag=True, help="List only tool packages")
 @click.option("--only-libraries", is_flag=True, help="List only library packages")
+@click.option("--json-output", is_flag=True)
 @click.option("-v", "--verbose", is_flag=True)
 def package_list_cmd(**options):
-    if options.get("global"):
+    data = (
         list_global_packages(options)
+        if options.get("global")
+        else list_project_packages(options)
+    )
+
+    if options.get("json_output"):
+        return click.echo(_dump_to_json(data, options))
+
+    def _print_items(typex, items):
+        click.secho(typex.capitalize(), bold=True)
+        print_dependency_tree(items, verbose=options.get("verbose"))
+        click.echo()
+
+    if options.get("global"):
+        for typex, items in data.items():
+            _print_items(typex, items)
     else:
-        list_project_packages(options)
+        for env, env_data in data.items():
+            click.echo("Resolving %s dependencies..." % click.style(env, fg="cyan"))
+            for typex, items in env_data.items():
+                _print_items(typex, items)
+
+    return None
 
 
-def humanize_package(pkg, spec=None, verbose=False):
-    if spec and not isinstance(spec, PackageSpec):
-        spec = PackageSpec(spec)
-    data = [
-        click.style(pkg.metadata.name, fg="cyan"),
-        click.style(f"@ {str(pkg.metadata.version)}", bold=True),
-    ]
-    extra_data = ["required: %s" % (spec.humanize() if spec else "Any")]
-    if verbose:
-        extra_data.append(pkg.path)
-    data.append("(%s)" % ", ".join(extra_data))
-    return " ".join(data)
+def _dump_to_json(data, options):
+    result = {}
+
+    if options.get("global"):
+        for typex, items in data.items():
+            result[typex] = [info.as_dict(with_manifest=True) for info in items]
+    else:
+        for env, env_data in data.items():
+            result[env] = {}
+            for typex, items in env_data.items():
+                result[env][typex] = [
+                    info.as_dict(with_manifest=True) for info in items
+                ]
+    return json.dumps(result)
 
 
-def print_dependency_tree(pm, specs=None, filter_specs=None, level=0, verbose=False):
+def build_package_info(pm, specs=None, filter_specs=None, resolve_dependencies=True):
     filtered_pkgs = [
-        pm.get_package(spec) for spec in filter_specs or [] if pm.get_package(spec)
+        pm.get_package(spec) for spec in filter_specs if pm.get_package(spec)
     ]
-    candidates = {}
+    candidates = []
     if specs:
         for spec in specs:
-            pkg = pm.get_package(spec)
-            if not pkg:
-                continue
-            candidates[pkg.path] = (pkg, spec)
+            candidates.append(
+                PackageInfo(
+                    spec if isinstance(spec, PackageSpec) else PackageSpec(spec),
+                    pm.get_package(spec),
+                )
+            )
     else:
-        candidates = {pkg.path: (pkg, pkg.metadata.spec) for pkg in pm.get_installed()}
+        candidates = [PackageInfo(pkg.metadata.spec, pkg) for pkg in pm.get_installed()]
     if not candidates:
-        return
-    candidates = sorted(candidates.values(), key=lambda item: item[0].metadata.name)
+        return []
 
-    for index, (pkg, spec) in enumerate(candidates):
-        if filtered_pkgs and not _pkg_tree_contains(pm, pkg, filtered_pkgs):
-            continue
-        printed_pkgs = pm.memcache_get("__printed_pkgs", [])
-        if printed_pkgs and pkg.path in printed_pkgs:
-            continue
-        printed_pkgs.append(pkg.path)
-        pm.memcache_set("__printed_pkgs", printed_pkgs)
+    candidates = sorted(
+        candidates,
+        key=lambda info: info.item.metadata.name if info.item else info.spec.humanize(),
+    )
 
-        click.echo(
-            "%s%s %s"
-            % (
-                "│   " * level,
-                "├──" if index < len(candidates) - 1 else "└──",
-                humanize_package(
-                    pkg,
-                    spec=spec,
-                    verbose=verbose,
+    result = []
+    for info in candidates:
+        if filter_specs and (
+            not info.item or not _pkg_tree_contains(pm, info.item, filtered_pkgs)
+        ):
+            continue
+        if not info.item:
+            if not info.spec.external and not info.spec.owner:  # built-in library?
+                continue
+            result.append(info)
+            continue
+
+        visited_pkgs = pm.memcache_get("__visited_pkgs", [])
+        if visited_pkgs and info.item.path in visited_pkgs:
+            continue
+        visited_pkgs.append(info.item.path)
+        pm.memcache_set("__visited_pkgs", visited_pkgs)
+
+        result.append(
+            PackageInfo(
+                info.spec,
+                info.item,
+                (
+                    build_package_info(
+                        pm,
+                        specs=[
+                            pm.dependency_to_spec(item)
+                            for item in pm.get_pkg_dependencies(info.item)
+                        ],
+                        filter_specs=filter_specs,
+                        resolve_dependencies=True,
+                    )
+                    if resolve_dependencies and pm.get_pkg_dependencies(info.item)
+                    else []
                 ),
             )
         )
 
-        dependencies = pm.get_pkg_dependencies(pkg)
-        if dependencies:
-            print_dependency_tree(
-                pm,
-                specs=[pm.dependency_to_spec(item) for item in dependencies],
-                filter_specs=filter_specs,
-                level=level + 1,
-                verbose=verbose,
-            )
+    return result
 
 
-def _pkg_tree_contains(pm, root: PackageItem, children: List[PackageItem]):
+def _pkg_tree_contains(pm, root: PackageItem, children: list[PackageItem]):
     if root in children:
         return True
     for dependency in pm.get_pkg_dependencies(root) or []:
@@ -139,6 +176,7 @@ def list_global_packages(options):
     only_packages = any(
         options.get(typex) or options.get(f"only_{typex}") for (typex, _) in data
     )
+    result = {}
     for typex, pm in data:
         skip_conds = [
             only_packages
@@ -148,82 +186,115 @@ def list_global_packages(options):
         ]
         if any(skip_conds):
             continue
-        click.secho(typex.capitalize(), bold=True)
-        print_dependency_tree(
-            pm, filter_specs=options.get(typex), verbose=options.get("verbose")
-        )
-        click.echo()
+        result[typex] = build_package_info(pm, filter_specs=options.get(typex))
+
+    return result
 
 
 def list_project_packages(options):
     environments = options["environments"]
-    only_packages = any(
+    only_filtered_packages = any(
         options.get(typex) or options.get(f"only_{typex}")
         for typex in ("platforms", "tools", "libraries")
     )
-    only_platform_packages = any(
-        options.get(typex) or options.get(f"only_{typex}")
-        for typex in ("platforms", "tools")
-    )
+    only_platform_package = options.get("platforms") or options.get("only_platforms")
+    only_tool_packages = options.get("tools") or options.get("only_tools")
     only_library_packages = options.get("libraries") or options.get("only_libraries")
 
+    result = {}
     with fs.cd(options["project_dir"]):
         config = ProjectConfig.get_instance()
         config.validate(environments)
         for env in config.envs():
             if environments and env not in environments:
                 continue
-            click.echo("Resolving %s dependencies..." % click.style(env, fg="cyan"))
-            found = False
-            if not only_packages or only_platform_packages:
-                _found = print_project_env_platform_packages(env, options)
-                found = found or _found
-            if not only_packages or only_library_packages:
-                _found = print_project_env_library_packages(env, options)
-                found = found or _found
-            if not found:
-                click.echo("No packages")
-            if (not environments and len(config.envs()) > 1) or len(environments) > 1:
-                click.echo()
+            result[env] = {}
+            if not only_filtered_packages or only_platform_package:
+                result[env]["platforms"] = list_project_env_platform_package(
+                    env, options
+                )
+            if not only_filtered_packages or only_tool_packages:
+                result[env]["tools"] = list_project_env_tool_packages(env, options)
+            if not only_filtered_packages or only_library_packages:
+                result[env]["libraries"] = list_project_env_library_packages(
+                    env, options
+                )
+
+    return result
 
 
-def print_project_env_platform_packages(project_env, options):
-    try:
-        p = PlatformFactory.from_env(project_env)
-    except UnknownPlatform:
-        return None
-    click.echo(
-        "Platform %s"
-        % (
-            humanize_package(
-                PlatformPackageManager().get_package(p.get_dir()),
-                p.config.get(f"env:{project_env}", "platform"),
-                verbose=options.get("verbose"),
-            )
-        )
+def list_project_env_platform_package(project_env, options):
+    pm = PlatformPackageManager()
+    return build_package_info(
+        pm,
+        specs=[PackageSpec(pm.config.get(f"env:{project_env}", "platform"))],
+        filter_specs=options.get("platforms"),
+        resolve_dependencies=False,
     )
-    print_dependency_tree(
+
+
+def list_project_env_tool_packages(project_env, options):
+    try:
+        p = PlatformFactory.from_env(project_env, targets=["upload"])
+    except UnknownPlatform:
+        return []
+
+    return build_package_info(
         p.pm,
-        specs=[p.get_package_spec(name) for name in p.packages],
+        specs=[
+            p.get_package_spec(name)
+            for name, options in p.packages.items()
+            if not options.get("optional")
+        ],
         filter_specs=options.get("tools"),
     )
-    click.echo()
-    return True
 
 
-def print_project_env_library_packages(project_env, options):
+def list_project_env_library_packages(project_env, options):
     config = ProjectConfig.get_instance()
     lib_deps = config.get(f"env:{project_env}", "lib_deps")
     lm = LibraryPackageManager(
         os.path.join(config.get("platformio", "libdeps_dir"), project_env)
     )
-    if not lib_deps or not lm.get_installed():
-        return None
-    click.echo("Libraries")
-    print_dependency_tree(
+    return build_package_info(
         lm,
         lib_deps,
         filter_specs=options.get("libraries"),
-        verbose=options.get("verbose"),
     )
-    return True
+
+
+def humanize_package(info, verbose=False):
+    data = (
+        [
+            click.style(info.item.metadata.name, fg="cyan"),
+            click.style(f"@ {str(info.item.metadata.version)}", bold=True),
+        ]
+        if info.item
+        else ["Not installed"]
+    )
+    extra_data = ["required: %s" % (info.spec.humanize() if info.spec else "Any")]
+    if verbose and info.item:
+        extra_data.append(info.item.path)
+    data.append("(%s)" % ", ".join(extra_data))
+    return " ".join(data)
+
+
+def print_dependency_tree(items, verbose=False, level=0):
+    for index, info in enumerate(items):
+        click.echo(
+            "%s%s %s"
+            % (
+                "│   " * level,
+                "├──" if index < len(items) - 1 else "└──",
+                humanize_package(
+                    info,
+                    verbose=verbose,
+                ),
+            )
+        )
+        if info.dependencies:
+            print_dependency_tree(
+                info.dependencies,
+                verbose=verbose,
+                level=level + 1,
+            )

platformio/package/commands/publish.py

@@ -88,7 +88,8 @@ def package_publish_cmd(  # pylint: disable=too-many-arguments, too-many-locals
     click.secho("Preparing a package...", fg="cyan")
     package = os.path.abspath(package)
     no_interactive = no_interactive or non_interactive
-    owner = owner or AccountClient().get_logged_username()
+    with AccountClient() as client:
+        owner = owner or client.get_logged_username()
     do_not_pack = (
         not os.path.isdir(package)
         and isinstance(FileUnpacker.new_archiver(package), TARArchiver)
@@ -146,9 +147,10 @@ def package_publish_cmd(  # pylint: disable=too-many-arguments, too-many-locals
             fg="yellow",
         )
         click.echo("Publishing...")
-        response = RegistryClient().publish_package(
-            owner, typex, archive_path, released_at, private, notify
-        )
+        with RegistryClient() as client:
+            response = client.publish_package(
+                owner, typex, archive_path, released_at, private, notify
+            )
         if not do_not_pack:
             os.remove(archive_path)
         click.secho(response.get("message"), fg="green")

platformio/package/commands/search.py

@@ -29,8 +29,8 @@ from platformio.registry.client import RegistryClient
     type=click.Choice(["relevance", "popularity", "trending", "added", "updated"]),
 )
 def package_search_cmd(query, page, sort):
-    client = RegistryClient()
-    result = client.list_packages(query, page=page, sort=sort)
+    with RegistryClient() as client:
+        result = client.list_packages(query, page=page, sort=sort)
     if not result["total"]:
         click.secho("Nothing has been found by your request", fg="yellow")
         click.echo(

platformio/package/commands/show.py

@@ -124,31 +124,31 @@ def package_show_cmd(spec, pkg_type):
 
 def fetch_package_data(spec, pkg_type=None):
     assert isinstance(spec, PackageSpec)
-    client = RegistryClient()
-    if pkg_type and spec.owner and spec.name:
+    with RegistryClient() as client:
+        if pkg_type and spec.owner and spec.name:
+            return client.get_package(
+                pkg_type, spec.owner, spec.name, version=spec.requirements
+            )
+        qualifiers = {}
+        if spec.id:
+            qualifiers["ids"] = str(spec.id)
+        if spec.name:
+            qualifiers["names"] = spec.name.lower()
+        if pkg_type:
+            qualifiers["types"] = pkg_type
+        if spec.owner:
+            qualifiers["owners"] = spec.owner.lower()
+        packages = client.list_packages(qualifiers=qualifiers)["items"]
+        if not packages:
+            return None
+        if len(packages) > 1:
+            PackageManagerRegistryMixin.print_multi_package_issue(
+                click.echo, packages, spec
+            )
+            return None
         return client.get_package(
-            pkg_type, spec.owner, spec.name, version=spec.requirements
+            packages[0]["type"],
+            packages[0]["owner"]["username"],
+            packages[0]["name"],
+            version=spec.requirements,
         )
-    qualifiers = {}
-    if spec.id:
-        qualifiers["ids"] = str(spec.id)
-    if spec.name:
-        qualifiers["names"] = spec.name.lower()
-    if pkg_type:
-        qualifiers["types"] = pkg_type
-    if spec.owner:
-        qualifiers["owners"] = spec.owner.lower()
-    packages = client.list_packages(qualifiers=qualifiers)["items"]
-    if not packages:
-        return None
-    if len(packages) > 1:
-        PackageManagerRegistryMixin.print_multi_package_issue(
-            click.echo, packages, spec
-        )
-        return None
-    return client.get_package(
-        packages[0]["type"],
-        packages[0]["owner"]["username"],
-        packages[0]["name"],
-        version=spec.requirements,
-    )

platformio/package/commands/unpublish.py

@@ -36,11 +36,14 @@ from platformio.registry.client import RegistryClient
 )
 def package_unpublish_cmd(package, type, undo):  # pylint: disable=redefined-builtin
     spec = PackageSpec(package)
-    response = RegistryClient().unpublish_package(
-        owner=spec.owner or AccountClient().get_logged_username(),
-        type=type,
-        name=spec.name,
-        version=str(spec.requirements),
-        undo=undo,
-    )
-    click.secho(response.get("message"), fg="green")
+    with AccountClient() as client:
+        owner = spec.owner or client.get_logged_username()
+    with RegistryClient() as client:
+        response = client.unpublish_package(
+            owner=owner,
+            type=type,
+            name=spec.name,
+            version=str(spec.requirements),
+            undo=undo,
+        )
+        click.secho(response.get("message"), fg="green")

platformio/package/download.py

@@ -12,48 +12,28 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import io
+import os
+import tempfile
+import time
 from email.utils import parsedate
-from os.path import getsize, join
-from time import mktime
+from urllib.parse import urlparse
 
 import click
+import httpx
 
 from platformio import fs
 from platformio.compat import is_terminal
-from platformio.http import HTTPSession
+from platformio.http import apply_default_kwargs
 from platformio.package.exception import PackageException
 
 
 class FileDownloader:
-    def __init__(self, url, dest_dir=None):
-        self._http_session = HTTPSession()
-        self._http_response = None
-        # make connection
-        self._http_response = self._http_session.get(
-            url,
-            stream=True,
-        )
-        if self._http_response.status_code != 200:
-            raise PackageException(
-                "Got the unrecognized status code '{0}' when downloaded {1}".format(
-                    self._http_response.status_code, url
-                )
-            )
+    def __init__(self, url, dst_dir=None):
+        self.url = url
+        self.dst_dir = dst_dir
 
-        disposition = self._http_response.headers.get("content-disposition")
-        if disposition and "filename=" in disposition:
-            self._fname = (
-                disposition[disposition.index("filename=") + 9 :]
-                .replace('"', "")
-                .replace("'", "")
-            )
-        else:
-            self._fname = [p for p in url.split("/") if p][-1]
-        self._fname = str(self._fname)
-        self._destination = self._fname
-        if dest_dir:
-            self.set_destination(join(dest_dir, self._fname))
+        self._destination = None
+        self._http_response = None
 
     def set_destination(self, destination):
         self._destination = destination
@@ -69,18 +49,34 @@ class FileDownloader:
             return -1
         return int(self._http_response.headers["content-length"])
 
+    def get_disposition_filname(self):
+        disposition = self._http_response.headers.get("content-disposition")
+        if disposition and "filename=" in disposition:
+            return (
+                disposition[disposition.index("filename=") + 9 :]
+                .replace('"', "")
+                .replace("'", "")
+            )
+        return [p for p in urlparse(self.url).path.split("/") if p][-1]
+
     def start(self, with_progress=True, silent=False):
         label = "Downloading"
-        file_size = self.get_size()
-        itercontent = self._http_response.iter_content(
-            chunk_size=io.DEFAULT_BUFFER_SIZE
-        )
-        try:
+        with httpx.stream("GET", self.url, **apply_default_kwargs()) as response:
+            if response.status_code != 200:
+                raise PackageException(
+                    f"Got the unrecognized status code '{response.status_code}' "
+                    "when downloading '{self.url}'"
+                )
+            self._http_response = response
+            total_size = self.get_size()
+            if not self._destination:
+                assert self.dst_dir
+
             with open(self._destination, "wb") as fp:
-                if file_size == -1 or not with_progress or silent:
+                if total_size == -1 or not with_progress or silent:
                     if not silent:
                         click.echo(f"{label}...")
-                    for chunk in itercontent:
+                    for chunk in response.iter_bytes():
                         fp.write(chunk)
 
                 elif not is_terminal():
@@ -88,10 +84,10 @@ class FileDownloader:
                     print_percent_step = 10
                     printed_percents = 0
                     downloaded_size = 0
-                    for chunk in itercontent:
+                    for chunk in response.iter_bytes():
                         fp.write(chunk)
                         downloaded_size += len(chunk)
-                        if (downloaded_size / file_size * 100) >= (
+                        if (downloaded_size / total_size * 100) >= (
                             printed_percents + print_percent_step
                         ):
                             printed_percents += print_percent_step
@@ -100,33 +96,39 @@ class FileDownloader:
 
                 else:
                     with click.progressbar(
-                        length=file_size,
-                        iterable=itercontent,
+                        length=total_size,
+                        iterable=response.iter_bytes(),
                         label=label,
                         update_min_steps=min(
-                            256 * 1024, file_size / 100
+                            256 * 1024, total_size / 100
                         ),  # every 256Kb or less
                     ) as pb:
                         for chunk in pb:
                             pb.update(len(chunk))
                             fp.write(chunk)
-        finally:
-            self._http_response.close()
-            self._http_session.close()
 
-        if self.get_lmtime():
-            self._preserve_filemtime(self.get_lmtime())
+        last_modified = self.get_lmtime()
+        if last_modified:
+            self._preserve_filemtime(last_modified)
 
         return True
 
+    def _set_tmp_destination(self):
+        dst_dir = self.dst_dir or tempfile.mkdtemp()
+        self.set_destination(os.path.join(dst_dir, self.get_disposition_filname()))
+
+    def _preserve_filemtime(self, lmdate):
+        lmtime = time.mktime(parsedate(lmdate))
+        fs.change_filemtime(self._destination, lmtime)
+
     def verify(self, checksum=None):
-        _dlsize = getsize(self._destination)
-        if self.get_size() != -1 and _dlsize != self.get_size():
+        remote_size = self.get_size()
+        downloaded_size = os.path.getsize(self._destination)
+        if remote_size not in (-1, downloaded_size):
             raise PackageException(
-                (
-                    "The size ({0:d} bytes) of downloaded file '{1}' "
-                    "is not equal to remote size ({2:d} bytes)"
-                ).format(_dlsize, self._fname, self.get_size())
+                f"The size ({downloaded_size} bytes) of downloaded file "
+                f"'{self._destination}' is not equal to remote size "
+                f"({remote_size} bytes)"
             )
         if not checksum:
             return True
@@ -142,7 +144,7 @@ class FileDownloader:
 
         if not hash_algo:
             raise PackageException(
-                "Could not determine checksum algorithm by %s" % checksum
+                f"Could not determine checksum algorithm by {checksum}"
             )
 
         dl_checksum = fs.calculate_file_hashsum(hash_algo, self._destination)
@@ -150,16 +152,7 @@ class FileDownloader:
             raise PackageException(
                 "The checksum '{0}' of the downloaded file '{1}' "
                 "does not match to the remote '{2}'".format(
-                    dl_checksum, self._fname, checksum
+                    dl_checksum, self._destination, checksum
                 )
             )
         return True
-
-    def _preserve_filemtime(self, lmdate):
-        lmtime = mktime(parsedate(lmdate))
-        fs.change_filemtime(self._destination, lmtime)
-
-    def __del__(self):
-        self._http_session.close()
-        if self._http_response:
-            self._http_response.close()

platformio/package/manager/_registry.py

@@ -15,6 +15,7 @@
 import time
 
 import click
+import httpx
 
 from platformio.package.exception import UnknownPackageError
 from platformio.package.meta import PackageSpec
@@ -57,7 +58,7 @@ class PackageManagerRegistryMixin:
                     ),
                     checksum or pkgfile["checksum"]["sha256"],
                 )
-            except Exception as exc:  # pylint: disable=broad-except
+            except httpx.HTTPError as exc:
                 self.log.warning(
                     click.style("Warning! Package Mirror: %s" % exc, fg="yellow")
                 )

platformio/package/manager/platform.py

@@ -15,7 +15,7 @@
 import os
 
 from platformio import util
-from platformio.http import HTTPClientError, InternetConnectionError
+from platformio.http import HttpClientApiError, InternetConnectionError
 from platformio.package.exception import UnknownPackageError
 from platformio.package.manager.base import BasePackageManager
 from platformio.package.manager.core import get_installed_core_packages
@@ -128,7 +128,7 @@ class PlatformPackageManager(BasePackageManager):  # pylint: disable=too-many-an
                 key = "%s:%s" % (board["platform"], board["id"])
                 if key not in know_boards:
                     boards.append(board)
-        except (HTTPClientError, InternetConnectionError):
+        except (HttpClientApiError, InternetConnectionError):
             pass
         return sorted(boards, key=lambda b: b["name"])
 

platformio/package/manifest/parser.py

@@ -22,7 +22,7 @@ from urllib.parse import urlparse
 
 from platformio import util
 from platformio.compat import get_object_members, string_types
-from platformio.http import fetch_remote_content
+from platformio.http import fetch_http_content
 from platformio.package.exception import ManifestParserError, UnknownManifestError
 from platformio.project.helpers import is_platformio_project
 
@@ -103,7 +103,7 @@ class ManifestParserFactory:
 
     @staticmethod
     def new_from_url(remote_url):
-        content = fetch_remote_content(remote_url)
+        content = fetch_http_content(remote_url)
         return ManifestParserFactory.new(
             content,
             ManifestFileType.from_uri(remote_url) or ManifestFileType.LIBRARY_JSON,

platformio/package/manifest/schema.py

@@ -17,12 +17,12 @@
 import json
 import re
 
+import httpx
 import marshmallow
-import requests
 import semantic_version
 from marshmallow import Schema, ValidationError, fields, validate, validates
 
-from platformio.http import fetch_remote_content
+from platformio.http import fetch_http_content
 from platformio.package.exception import ManifestValidationError
 from platformio.util import memoized
 
@@ -252,7 +252,7 @@ class ManifestSchema(BaseSchema):
     def validate_license(self, value):
         try:
             spdx = self.load_spdx_licenses()
-        except requests.exceptions.RequestException as exc:
+        except httpx.HTTPError as exc:
             raise ValidationError(
                 "Could not load SPDX licenses for validation"
             ) from exc
@@ -281,4 +281,4 @@ class ManifestSchema(BaseSchema):
             "https://raw.githubusercontent.com/spdx/license-list-data/"
             f"v{version}/json/licenses.json"
         )
-        return json.loads(fetch_remote_content(spdx_data_url))
+        return json.loads(fetch_http_content(spdx_data_url))

platformio/package/meta.py

@@ -23,7 +23,7 @@ import semantic_version
 
 from platformio import fs
 from platformio.compat import get_object_members, hashlib_encode_data, string_types
-from platformio.package.manifest.parser import ManifestFileType
+from platformio.package.manifest.parser import ManifestFileType, ManifestParserFactory
 from platformio.package.version import SemanticVersionError, cast_version_to_semver
 from platformio.util import items_in_list
 
@@ -561,3 +561,29 @@ class PackageItem:
                 break
         assert location
         return self.metadata.dump(os.path.join(location, self.METAFILE_NAME))
+
+    def as_dict(self):
+        return {"path": self.path, "metadata": self.metadata.as_dict()}
+
+
+class PackageInfo:
+
+    def __init__(self, spec: PackageSpec, item: PackageItem = None, dependencies=None):
+        assert isinstance(spec, PackageSpec)
+        self.spec = spec
+        self.item = item
+        self.dependencies = dependencies or []
+
+    def as_dict(self, with_manifest=False):
+        result = {
+            "spec": self.spec.as_dict(),
+            "item": self.item.as_dict() if self.item else None,
+            "dependencies": [d.as_dict() for d in self.dependencies],
+        }
+        if with_manifest:
+            result["manifest"] = (
+                ManifestParserFactory.new_from_dir(self.item.path).as_dict()
+                if self.item
+                else None
+            )
+        return result

platformio/platform/factory.py

@@ -16,6 +16,8 @@ import os
 import re
 import sys
 
+import httpx
+
 from platformio import fs
 from platformio.compat import load_python_module
 from platformio.package.meta import PackageItem
@@ -31,13 +33,16 @@ class PlatformFactory:
         name = re.sub(r"[^\da-z\_]+", "", name, flags=re.I)
         return "%sPlatform" % name.lower().capitalize()
 
-    @staticmethod
-    def load_platform_module(name, path):
+    @classmethod
+    def load_platform_module(cls, name, path):
         # backward compatibiility with the legacy dev-platforms
         sys.modules["platformio.managers.platform"] = base
         try:
             return load_python_module("platformio.platform.%s" % name, path)
         except ImportError as exc:
+            if exc.name == "requests" and not sys.modules.get("requests"):
+                sys.modules["requests"] = httpx
+                return cls.load_platform_module(name, path)
             raise UnknownPlatform(name) from exc
 
     @classmethod

platformio/project/helpers.py

@@ -12,22 +12,29 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import hashlib
 import os
 import re
 import subprocess
-from hashlib import sha1
 
 from click.testing import CliRunner
 
 from platformio import __version__, exception, fs
 from platformio.compat import IS_MACOS, IS_WINDOWS, hashlib_encode_data
 from platformio.project.config import ProjectConfig
+from platformio.project.options import ProjectOptions
 
 
 def get_project_dir():
     return os.getcwd()
 
 
+def get_project_id(project_dir=None):
+    return hashlib.sha1(
+        hashlib_encode_data(project_dir or get_project_dir())
+    ).hexdigest()
+
+
 def is_platformio_project(project_dir=None):
     if not project_dir:
         project_dir = get_project_dir()
@@ -92,7 +99,7 @@ def get_default_projects_dir():
 
 def compute_project_checksum(config):
     # rebuild when PIO Core version changes
-    checksum = sha1(hashlib_encode_data(__version__))
+    checksum = hashlib.sha1(hashlib_encode_data(__version__))
 
     # configuration file state
     config_data = config.to_json()
@@ -131,27 +138,27 @@ def compute_project_checksum(config):
     return checksum.hexdigest()
 
 
-def load_build_metadata(project_dir, env_or_envs, cache=False, build_type=None):
-    assert env_or_envs
-    env_names = env_or_envs
-    if not isinstance(env_names, list):
-        env_names = [env_names]
+def get_build_type(config, env, run_targets=None):
+    types = []
+    run_targets = run_targets or []
+    env_build_type = config.get(f"env:{env}", "build_type")
+    if set(["__debug", "__memusage"]) & set(run_targets) or env_build_type == "debug":
+        types.append("debug")
+    if "__test" in run_targets or env_build_type == "test":
+        types.append("test")
+    return ", ".join(types or [ProjectOptions["env.build_type"].default])
 
-    with fs.cd(project_dir):
-        result = _get_cached_build_metadata(env_names) if cache else {}
-        # incompatible build-type data
-        for env_name in list(result.keys()):
-            if build_type is None:
-                build_type = ProjectConfig.get_instance().get(
-                    f"env:{env_name}", "build_type"
-                )
-            if result[env_name].get("build_type", "") != build_type:
-                del result[env_name]
-        missed_env_names = set(env_names) - set(result.keys())
-        if missed_env_names:
-            result.update(
-                _load_build_metadata(project_dir, missed_env_names, build_type)
-            )
+
+def load_build_metadata(project_dir, env_or_envs, cache=False, force_targets=None):
+    assert env_or_envs
+    envs = env_or_envs
+    if not isinstance(envs, list):
+        envs = [envs]
+    with fs.cd(project_dir or os.getcwd()):
+        result = _get_cached_build_metadata(envs, force_targets) if cache else {}
+        missed_envs = set(envs) - set(result.keys())
+        if missed_envs:
+            result.update(_load_build_metadata(missed_envs, force_targets))
 
     if not isinstance(env_or_envs, list) and env_or_envs in result:
         return result[env_or_envs]
@@ -162,18 +169,28 @@ def load_build_metadata(project_dir, env_or_envs, cache=False, build_type=None):
 load_project_ide_data = load_build_metadata
 
 
-def _load_build_metadata(project_dir, env_names, build_type=None):
+def _get_cached_build_metadata(envs, force_targets=None):
+    config = ProjectConfig.get_instance(os.path.join(os.getcwd(), "platformio.ini"))
+    build_dir = config.get("platformio", "build_dir")
+    result = {}
+    for env in envs:
+        build_type = get_build_type(config, env, force_targets)
+        json_path = os.path.join(build_dir, env, build_type, "metadata.json")
+        if os.path.isfile(json_path):
+            result[env] = fs.load_json(json_path)
+    return result
+
+
+def _load_build_metadata(envs, force_targets=None):
     # pylint: disable=import-outside-toplevel
     from platformio import app
     from platformio.run.cli import cli as cmd_run
 
-    args = ["--project-dir", project_dir, "--target", "__idedata"]
-    if build_type == "debug":
-        args.extend(["--target", "__debug"])
-    # if build_type == "test":
-    #     args.extend(["--target", "__test"])
-    for name in env_names:
-        args.extend(["-e", name])
+    args = ["--target", "__metadata"]
+    for target in force_targets or []:
+        args.extend(["--target", target])
+    for env in envs:
+        args.extend(["-e", env])
     app.set_session_var("pause_telemetry", True)
     result = CliRunner().invoke(cmd_run, args)
     app.set_session_var("pause_telemetry", False)
@@ -181,18 +198,6 @@ def _load_build_metadata(project_dir, env_names, build_type=None):
         result.exception, exception.ReturnErrorCode
     ):
         raise result.exception
-    if '"includes":' not in result.output:
+    if "Metadata has been saved to the following location" not in result.output:
         raise exception.UserSideException(result.output)
-    return _get_cached_build_metadata(env_names)
-
-
-def _get_cached_build_metadata(env_names):
-    build_dir = ProjectConfig.get_instance().get("platformio", "build_dir")
-    result = {}
-    for env_name in env_names:
-        if not os.path.isfile(os.path.join(build_dir, env_name, "idedata.json")):
-            continue
-        result[env_name] = fs.load_json(
-            os.path.join(build_dir, env_name, "idedata.json")
-        )
-    return result
+    return _get_cached_build_metadata(envs, force_targets)

platformio/project/memusage.py

@@ -0,0 +1,58 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import gzip
+import json
+import os
+import time
+
+from platformio import fs
+from platformio.project.config import ProjectConfig
+
+
+def get_report_dir(project_dir, env):
+    with fs.cd(project_dir):
+        return os.path.join(
+            ProjectConfig.get_instance().get("platformio", "memusage_dir"), env
+        )
+
+
+def list_reports(report_dir):
+    if not os.path.isdir(report_dir):
+        return []
+    return [os.path.join(report_dir, item) for item in sorted(os.listdir(report_dir))]
+
+
+def read_report(path):
+    with gzip.open(path, mode="rt", encoding="utf8") as fp:
+        return json.load(fp)
+
+
+def save_report(project_dir, env, data):
+    report_dir = get_report_dir(project_dir, env)
+    if not os.path.isdir(report_dir):
+        os.makedirs(report_dir)
+    report_path = os.path.join(report_dir, f"{int(time.time())}.json.gz")
+    with gzip.open(report_path, mode="wt", encoding="utf8") as fp:
+        json.dump(data, fp)
+    rotate_reports(report_dir)
+    return report_path
+
+
+def rotate_reports(report_dir, max_reports=100):
+    reports = os.listdir(report_dir)
+    if len(reports) < max_reports:
+        return
+    for fname in sorted(reports)[0 : len(reports) - max_reports]:
+        os.remove(os.path.join(report_dir, fname))

platformio/project/options.py

@@ -240,6 +240,17 @@ ProjectOptions = OrderedDict(
                 default=os.path.join("${platformio.workspace_dir}", "libdeps"),
                 validate=validate_dir,
             ),
+            ConfigPlatformioOption(
+                group="directory",
+                name="memusage_dir",
+                description=(
+                    "A location where PlatformIO Core will store "
+                    "project memory usage reports"
+                ),
+                sysenvvar="PLATFORMIO_MEMUSAGE_DIR",
+                default=os.path.join("${platformio.workspace_dir}", "memusage"),
+                validate=validate_dir,
+            ),
             ConfigPlatformioOption(
                 group="directory",
                 name="include_dir",

platformio/public.py

@@ -17,6 +17,7 @@
 from platformio.device.list.util import list_logical_devices, list_serial_ports
 from platformio.device.monitor.filters.base import DeviceMonitorFilterBase
 from platformio.fs import to_unix_path
+from platformio.http import fetch_http_content
 from platformio.platform.base import PlatformBase
 from platformio.project.config import ProjectConfig
 from platformio.project.helpers import get_project_watch_lib_dirs, load_build_metadata

platformio/registry/access/commands/grant.py

@@ -31,8 +31,8 @@ from platformio.registry.client import RegistryClient
 )
 @click.option("--urn-type", type=click.Choice(["prn:reg:pkg"]), default="prn:reg:pkg")
 def access_grant_cmd(level, client, urn, urn_type):  # pylint: disable=unused-argument
-    reg_client = RegistryClient()
-    reg_client.grant_access_for_resource(urn=urn, client=client, level=level)
+    with RegistryClient() as reg_client:
+        reg_client.grant_access_for_resource(urn=urn, client=client, level=level)
     return click.secho(
         "Access for resource %s has been granted for %s" % (urn, client),
         fg="green",

platformio/registry/access/commands/list.py

@@ -25,8 +25,8 @@ from platformio.registry.client import RegistryClient
 @click.option("--urn-type", type=click.Choice(["prn:reg:pkg"]), default="prn:reg:pkg")
 @click.option("--json-output", is_flag=True)
 def access_list_cmd(owner, urn_type, json_output):  # pylint: disable=unused-argument
-    reg_client = RegistryClient()
-    resources = reg_client.list_resources(owner=owner)
+    with RegistryClient() as client:
+        resources = client.list_resources(owner=owner)
     if json_output:
         return click.echo(json.dumps(resources))
     if not resources:

platformio/registry/access/commands/private.py

@@ -25,8 +25,8 @@ from platformio.registry.client import RegistryClient
 )
 @click.option("--urn-type", type=click.Choice(["prn:reg:pkg"]), default="prn:reg:pkg")
 def access_private_cmd(urn, urn_type):  # pylint: disable=unused-argument
-    client = RegistryClient()
-    client.update_resource(urn=urn, private=1)
+    with RegistryClient() as client:
+        client.update_resource(urn=urn, private=1)
     return click.secho(
         "The resource %s has been successfully updated." % urn,
         fg="green",

platformio/registry/access/commands/public.py

@@ -25,8 +25,8 @@ from platformio.registry.client import RegistryClient
 )
 @click.option("--urn-type", type=click.Choice(["prn:reg:pkg"]), default="prn:reg:pkg")
 def access_public_cmd(urn, urn_type):  # pylint: disable=unused-argument
-    client = RegistryClient()
-    client.update_resource(urn=urn, private=0)
+    with RegistryClient() as client:
+        client.update_resource(urn=urn, private=0)
     return click.secho(
         "The resource %s has been successfully updated." % urn,
         fg="green",

platformio/registry/access/commands/revoke.py

@@ -30,8 +30,8 @@ from platformio.registry.client import RegistryClient
 )
 @click.option("--urn-type", type=click.Choice(["prn:reg:pkg"]), default="prn:reg:pkg")
 def access_revoke_cmd(client, urn, urn_type):  # pylint: disable=unused-argument
-    reg_client = RegistryClient()
-    reg_client.revoke_access_from_resource(urn=urn, client=client)
+    with RegistryClient() as reg_client:
+        reg_client.revoke_access_from_resource(urn=urn, client=client)
     return click.secho(
         "Access for resource %s has been revoked for %s" % (urn, client),
         fg="green",

platformio/registry/client.py

@@ -16,13 +16,14 @@
 
 from platformio import __registry_mirror_hosts__, fs
 from platformio.account.client import AccountClient, AccountError
-from platformio.http import HTTPClient, HTTPClientError
+from platformio.http import HttpApiClient, HttpClientApiError
 
 
-class RegistryClient(HTTPClient):
-    def __init__(self):
-        endpoints = [f"https://api.{host}" for host in __registry_mirror_hosts__]
-        super().__init__(endpoints)
+class RegistryClient(HttpApiClient):
+    def __init__(self, endpoints=None):
+        super().__init__(
+            endpoints or [f"https://api.{host}" for host in __registry_mirror_hosts__]
+        )
 
     @staticmethod
     def allowed_private_packages():
@@ -34,7 +35,8 @@ class RegistryClient(HTTPClient):
             ]
         )
         try:
-            info = AccountClient().get_account_info() or {}
+            with AccountClient() as client:
+                info = client.get_account_info() or {}
             for item in info.get("packages", []):
                 if set(item.keys()) & private_permissions:
                     return True
@@ -156,7 +158,7 @@ class RegistryClient(HTTPClient):
                 x_cache_valid="1h",
                 x_with_authorization=self.allowed_private_packages(),
             )
-        except HTTPClientError as exc:
+        except HttpClientApiError as exc:
             if exc.response is not None and exc.response.status_code == 404:
                 return None
             raise exc

platformio/registry/mirror.py

@@ -17,7 +17,6 @@ from urllib.parse import urlparse
 
 from platformio import __registry_mirror_hosts__
 from platformio.cache import ContentCache
-from platformio.http import HTTPClient
 from platformio.registry.client import RegistryClient
 
 
@@ -49,17 +48,17 @@ class RegistryFileMirrorIterator:
                 except (ValueError, KeyError):
                     pass
 
-            http = self.get_http_client()
-            response = http.send_request(
+            registry = self.get_api_client()
+            response = registry.send_request(
                 "head",
                 self._url_parts.path,
-                allow_redirects=False,
+                follow_redirects=False,
                 params=(
                     dict(bypass=",".join(self._visited_mirrors))
                     if self._visited_mirrors
                     else None
                 ),
-                x_with_authorization=RegistryClient.allowed_private_packages(),
+                x_with_authorization=registry.allowed_private_packages(),
             )
             stop_conditions = [
                 response.status_code not in (302, 307),
@@ -87,14 +86,14 @@ class RegistryFileMirrorIterator:
                 response.headers.get("X-PIO-Content-SHA256"),
             )
 
-    def get_http_client(self):
+    def get_api_client(self):
         if self._mirror not in RegistryFileMirrorIterator.HTTP_CLIENT_INSTANCES:
             endpoints = [self._mirror]
             for host in __registry_mirror_hosts__:
                 endpoint = f"https://dl.{host}"
                 if endpoint not in endpoints:
                     endpoints.append(endpoint)
-            RegistryFileMirrorIterator.HTTP_CLIENT_INSTANCES[self._mirror] = HTTPClient(
-                endpoints
+            RegistryFileMirrorIterator.HTTP_CLIENT_INSTANCES[self._mirror] = (
+                RegistryClient(endpoints)
             )
         return RegistryFileMirrorIterator.HTTP_CLIENT_INSTANCES[self._mirror]

platformio/remote/factory/client.py

@@ -37,7 +37,8 @@ class RemoteClientFactory(pb.PBClientFactory, protocol.ReconnectingClientFactory
 
         auth_token = None
         try:
-            auth_token = AccountClient().fetch_authentication_token()
+            with AccountClient() as client:
+                auth_token = client.fetch_authentication_token()
         except Exception as exc:  # pylint:disable=broad-except
             d = defer.Deferred()
             d.addErrback(self.clientAuthorizationFailed)

platformio/run/cli.py

@@ -308,7 +308,7 @@ def print_processing_summary(results, verbose=False):
 
 def print_target_list(envs):
     tabular_data = []
-    for env, data in load_build_metadata(os.getcwd(), envs).items():
+    for env, data in load_build_metadata(None, envs, cache=True).items():
         tabular_data.extend(
             sorted(
                 [

platformio/telemetry.py

@@ -22,13 +22,14 @@ import time
 import traceback
 from collections import deque
 
-import requests
+import httpx
 
 from platformio import __title__, __version__, app, exception, fs, util
 from platformio.cli import PlatformioCLI
 from platformio.debug.config.base import DebugConfigBase
 from platformio.http import HTTPSession
 from platformio.proc import is_ci
+from platformio.project.helpers import get_project_id
 
 KEEP_MAX_REPORTS = 100
 SEND_MAX_EVENTS = 25
@@ -133,13 +134,11 @@ class TelemetryLogger:
         # print("_commit_payload", payload)
         try:
             r = self._http_session.post(
-                "https://collector.platformio.org/collect",
-                json=payload,
-                timeout=(2, 5),  # connect, read
+                "https://collector.platformio.org/collect", json=payload, timeout=2
             )
             r.raise_for_status()
             return True
-        except requests.exceptions.HTTPError as exc:
+        except httpx.HTTPStatusError as exc:
             # skip Bad Request
             if exc.response.status_code >= 400 and exc.response.status_code < 500:
                 return True
@@ -218,7 +217,7 @@ def dump_project_env_params(config, env, platform):
         for option in non_sensitive_data
         if config.has_option(section, option)
     }
-    params["pid"] = app.get_project_id(os.path.dirname(config.path))
+    params["pid"] = get_project_id(os.path.dirname(config.path))
     params["platform_name"] = platform.name
     params["platform_version"] = platform.version
     return params

platformio/test/runners/readers/native.py

@@ -76,13 +76,22 @@ class NativeTestOutputReader:
             os.path.join(
                 build_dir,
                 self.test_runner.test_suite.env_name,
+                "test",
                 "program.exe" if IS_WINDOWS else "program",
             )
         ]
         # if user changed PROGNAME
         if not os.path.exists(cmd[0]):
             build_data = load_build_metadata(
-                os.getcwd(), self.test_runner.test_suite.env_name, cache=True
+                os.getcwd(),
+                self.test_runner.test_suite.env_name,
+                cache=True,
+                force_targets=["__test"]
+                + (
+                    ["__debug"]
+                    if not self.test_runner.options.without_debugging
+                    else []
+                ),
             )
             if build_data:
                 cmd[0] = build_data["prog_path"]

tests/commands/test_account_org_team.py

@@ -19,11 +19,11 @@ import os
 import random
 
 import pytest
-import requests
 
 from platformio.account.cli import cli as cmd_account
 from platformio.account.org.cli import cli as cmd_org
 from platformio.account.team.cli import cli as cmd_team
+from platformio.http import HTTPSession
 
 pytestmark = pytest.mark.skipif(
     not all(
@@ -60,12 +60,11 @@ def verify_account(email_contents):
         .split("This link will expire within 12 hours.")[0]
         .strip()
     )
-    with requests.Session() as session:
+    with HTTPSession() as session:
         result = session.get(link).text
         link = result.split('<a href="')[1].split('"', 1)[0]
         link = link.replace("&amp;", "&")
         session.get(link)
-        session.close()
 
 
 def test_account_register(

tests/commands/test_test.py

@@ -268,7 +268,7 @@ test_testing_command =
     atmega328p
     -f
     16000000L
-    ${platformio.build_dir}/${this.__env__}/firmware.elf
+    ${platformio.build_dir}/${this.__env__}/test/firmware.elf
 """
     )
     test_dir = project_dir / "test" / "test_dummy"

tests/conftest.py

@@ -143,8 +143,8 @@ def get_pkg_latest_version():
         if not isinstance(spec, PackageSpec):
             spec = PackageSpec(spec)
         pkg_type = pkg_type or PackageType.LIBRARY
-        client = RegistryClient()
-        pkg = client.get_package(pkg_type, spec.owner, spec.name)
+        with RegistryClient() as client:
+            pkg = client.get_package(pkg_type, spec.owner, spec.name)
         return pkg["version"]["name"]
 
     return wrap

tests/misc/test_misc.py

@@ -15,7 +15,6 @@
 # pylint: disable=unused-argument
 
 import pytest
-import requests
 
 from platformio import __check_internet_hosts__, http, proc
 from platformio.registry.client import RegistryClient
@@ -30,19 +29,20 @@ def test_platformio_cli():
 
 def test_ping_internet_ips():
     for host in __check_internet_hosts__:
-        requests.get("http://%s" % host, allow_redirects=False, timeout=2)
+        with http.HTTPSession(follow_redirects=False, timeout=2) as session:
+            session.get("http://%s" % host)
 
 
 def test_api_internet_offline(without_internet, isolated_pio_core):
-    regclient = RegistryClient()
-    with pytest.raises(http.InternetConnectionError):
-        regclient.fetch_json_data("get", "/v3/search")
+    with RegistryClient() as client:
+        with pytest.raises(http.InternetConnectionError):
+            client.fetch_json_data("get", "/v3/search")
 
 
 def test_api_cache(monkeypatch, isolated_pio_core):
-    regclient = RegistryClient()
-    api_kwargs = {"method": "get", "path": "/v3/search", "x_cache_valid": "10s"}
-    result = regclient.fetch_json_data(**api_kwargs)
-    assert result and "total" in result
-    monkeypatch.setattr(http, "_internet_on", lambda: False)
-    assert regclient.fetch_json_data(**api_kwargs) == result
+    with RegistryClient() as client:
+        api_kwargs = {"method": "get", "path": "/v3/search", "x_cache_valid": "10s"}
+        result = client.fetch_json_data(**api_kwargs)
+        assert result and "total" in result
+        monkeypatch.setattr(http, "_internet_on", lambda: False)
+        assert client.fetch_json_data(**api_kwargs) == result