Merge branch 'release/v6.1.7'

.github/workflows/docs.yml

@@ -65,7 +65,7 @@ jobs:
           mkdir ./${{ env.LATEST_DOCS_DIR }}
           tar -xzf ./docs.tar.gz -C ./${{ env.LATEST_DOCS_DIR }}
       - name: Delete Artifact
-        uses: geekyeggo/delete-artifact@v1
+        uses: geekyeggo/delete-artifact@v2
         with:
           name: docs
       - name: Select Docs type

.pylintrc

@@ -8,4 +8,5 @@ disable=
     invalid-name,
     too-few-public-methods,
     consider-using-f-string,
-    cyclic-import
+    cyclic-import,
+    use-dict-literal

CODE_OF_CONDUCT.md

@@ -0,0 +1,3 @@
+# Code of Conduct
+
+See https://piolabs.com/legal/code-of-conduct.html

HISTORY.rst

@@ -6,6 +6,7 @@ Release Notes
 .. |LDF| replace:: `LDF <https://docs.platformio.org/en/latest/librarymanager/ldf.html>`__
 .. |INTERPOLATION| replace:: `Interpolation of Values <https://docs.platformio.org/en/latest/projectconf/interpolation.html>`__
 .. |UNITTESTING| replace:: `Unit Testing <https://docs.platformio.org/en/latest/advanced/unit-testing/index.html>`__
+.. |DEBUGGING| replace:: `Debugging <https://docs.platformio.org/en/latest/plus/debugging.html>`__
 
 .. _release_notes_6:
 
@@ -14,6 +15,28 @@ PlatformIO Core 6
 
 **A professional collaborative platform for declarative, safety-critical, and test-driven embedded development.**
 
+6.1.7 (2023-05-08)
+~~~~~~~~~~~~~~~~~~
+
+* Introduced a new ``--sample-code`` option to the `pio project init <https://docs.platformio.org/en/latest/core/userguide/project/cmd_init.html>`__ command, which allows users to include sample code in the newly created project
+* Added validation for `project working environment names <https://docs.platformio.org/en/latest/projectconf/sections/env/index.html#working-env-name>`__ to ensure that they only contain lowercase letters ``a-z``, numbers ``0-9``, and special characters ``_`` (underscore) and ``-`` (hyphen)
+* Added the ability to show a detailed library dependency tree only in `verbose mode <https://docs.platformio.org/en/latest/core/userguide/cmd_run.html#cmdoption-pio-run-v>`__, which can help you understand the relationship between libraries and troubleshoot issues more effectively (`issue #4517 <https://github.com/platformio/platformio-core/issues/4517>`_)
+* Added the ability to run only the `device monitor <https://docs.platformio.org/en/latest/core/userguide/device/cmd_monitor.html>`__ when using the `pio run -t monitor <https://docs.platformio.org/en/latest/core/userguide/cmd_run.html>`__ command, saving you time and resources by skipping the build process
+* Implemented a new feature to store device monitor logs in the project's ``logs`` folder, making it easier to access and review device monitor logs for your projects (`issue #4596 <https://github.com/platformio/platformio-core/issues/4596>`_)
+* Improved support for projects located on Windows network drives, including Network Shared Folder, Dropbox, OneDrive, Google Drive, and other similar services (`issue #3417 <https://github.com/platformio/platformio-core/issues/3417>`_)
+* Improved source file filtering functionality for the `Static Code Analysis <https://docs.platformio.org/en/latest/advanced/static-code-analysis/index.html>`__ feature, making it easier to analyze only the code you need to
+* Upgraded the build engine to the latest version of SCons (4.5.2) to improve build performance, reliability, and compatibility with other tools and systems (`release notes <https://github.com/SCons/scons/releases/tag/4.5.2>`__)
+* Implemented a fix for shell injection vulnerabilities when converting INO files to CPP, ensuring your code is safe and secure (`issue #4532 <https://github.com/platformio/platformio-core/issues/4532>`_)
+* Restored the project generator for the `NetBeans IDE <https://docs.platformio.org/en/latest/integration/ide/netbeans.html>`__, providing you with more flexibility and options for your development workflow
+* Resolved installation issues with PIO Remote on Raspberry Pi and other small form-factor PCs (`issue #4425 <https://github.com/platformio/platformio-core/issues/4425>`_, `issue #4493 <https://github.com/platformio/platformio-core/issues/4493>`_, `issue #4607 <https://github.com/platformio/platformio-core/issues/4607>`_)
+* Resolved an issue where the `build_cache_dir <https://docs.platformio.org/en/latest/projectconf/sections/platformio/options/directory/build_cache_dir.html>`__ setting was not being recognized consistently across multiple environments (`issue #4574 <https://github.com/platformio/platformio-core/issues/4574>`_)
+* Resolved an issue where organization details could not be updated using the `pio org update <https://docs.platformio.org/en/latest/core/userguide/org/cmd_update.html>`__ command
+* Resolved an issue where the incorrect debugging environment was generated for VSCode in "Auto" mode (`issue #4597 <https://github.com/platformio/platformio-core/issues/4597>`_)
+* Resolved an issue where native tests would fail if a custom program name was specified (`issue #4546 <https://github.com/platformio/platformio-core/issues/4546>`_)
+* Resolved an issue where the PlatformIO |DEBUGGING| solution was not escaping the tool installation process into MI2 correctly (`issue #4565 <https://github.com/platformio/platformio-core/issues/4565>`_)
+* Resolved an issue where multiple targets were not executed sequentially (`issue #4604 <https://github.com/platformio/platformio-core/issues/4604>`_)
+* Resolved an issue where upgrading PlatformIO Core fails on Windows with Python 3.11 (`issue #4540 <https://github.com/platformio/platformio-core/issues/4540>`_)
+
 6.1.6 (2023-01-23)
 ~~~~~~~~~~~~~~~~~~
 

SECURITY.md

@@ -0,0 +1,34 @@
+# Security Policy
+
+## Supported Versions
+
+We are committed to ensuring the security and protection of PlatformIO Core. 
+To this end, we support only the following versions:
+
+| Version | Supported          |
+| ------- | ------------------ |
+| 6.1.x   | :white_check_mark: |
+| < 6.1   | :x:                |
+
+Unsupported versions of the PlatformIO Core may have known vulnerabilities or security issues that could compromise the security of our organization's systems and data. 
+Therefore, it is important that all developers use only supported versions of the PlatformIO Core.
+
+## Reporting a Vulnerability
+
+We take the security of our systems and data very seriously. We encourage responsible disclosure of any vulnerabilities or security issues that you may find in our systems or applications. If you believe you have discovered a vulnerability, please report it to us immediately.
+
+To report a vulnerability, please send an email to our security team at contact@piolabs.com. Please include as much information as possible, including:
+
+- A description of the vulnerability and how it can be exploited
+- Steps to reproduce the vulnerability
+- Any additional information that can help us understand and reproduce the vulnerability
+
+Once we receive your report, our security team will acknowledge receipt within 24 hours and will work to validate the reported vulnerability. We will provide periodic updates on the progress of the vulnerability assessment, and will notify you once a fix has been deployed.
+
+If the vulnerability is accepted, we will work to remediate the issue as quickly as possible. We may also provide credit or recognition to the individual who reported the vulnerability, at our discretion.
+
+If the vulnerability is declined, we will provide a justification for our decision and may offer guidance on how to improve the report or how to test the system more effectively.
+
+Please note that we will not take any legal action against individuals who report vulnerabilities in good faith and in accordance with this policy.
+
+Thank you for helping us keep our systems and data secure.

platformio/__init__.py

@@ -12,9 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import sys
+VERSION = (6, 1, 7)
-
-VERSION = (6, 1, 6)
 __version__ = ".".join([str(s) for s in VERSION])
 
 __title__ = "platformio"
@@ -46,8 +44,8 @@ __pioremote_endpoint__ = "ssl:host=remote.platformio.org:port=4413"
 
 __core_packages__ = {
     "contrib-piohome": "~3.4.2",
-    "contrib-pysite": "~2.%d%d.0" % (sys.version_info.major, sys.version_info.minor),
+    "contrib-pioremote": "~1.0.0",
-    "tool-scons": "~4.40400.0",
+    "tool-scons": "~4.40502.0",
     "tool-cppcheck": "~1.270.0",
     "tool-clangtidy": "~1.150005.0",
     "tool-pvs-studio": "~7.18.0",

platformio/account/client.py

@@ -21,22 +21,18 @@ from platformio.http import HTTPClient, HTTPClientError
 
 
 class AccountError(PlatformioException):
-
     MESSAGE = "{0}"
 
 
 class AccountNotAuthorized(AccountError):
-
     MESSAGE = "You are not authorized! Please log in to PlatformIO Account."
 
 
 class AccountAlreadyAuthorized(AccountError):
-
     MESSAGE = "You are already authorized with {0} account."
 
 
 class AccountClient(HTTPClient):  # pylint:disable=too-many-public-methods
-
     SUMMARY_CACHE_TTL = 60 * 60 * 24 * 7
 
     def __init__(self):
@@ -298,7 +294,7 @@ class AccountClient(HTTPClient):  # pylint:disable=too-many-public-methods
         return self.fetch_json_data(
             "delete",
             "/v1/orgs/%s/owners" % orgname,
-            data={"username": username},
+            params={"username": username},
             x_with_authorization=True,
         )
 
@@ -351,6 +347,6 @@ class AccountClient(HTTPClient):  # pylint:disable=too-many-public-methods
         return self.fetch_json_data(
             "delete",
             "/v1/orgs/%s/teams/%s/members" % (orgname, teamname),
-            data={"username": username},
+            params={"username": username},
             x_with_authorization=True,
         )

platformio/account/org/commands/update.py

@@ -22,29 +22,27 @@ from platformio.account.validate import validate_email, validate_orgname
 @click.argument("cur_orgname")
 @click.option(
     "--orgname",
-    callback=lambda _, __, value: validate_orgname(value),
+    callback=lambda _, __, value: validate_orgname(value) if value else value,
     help="A new orgname",
 )
-@click.option("--email")
+@click.option(
+    "--email",
+    callback=lambda _, __, value: validate_email(value) if value else value,
+)
 @click.option("--displayname")
 def org_update_cmd(cur_orgname, **kwargs):
     client = AccountClient()
     org = client.get_org(cur_orgname)
-    del org["owners"]
+    new_org = {
-    new_org = org.copy()
+        key: value if value is not None else org[key] for key, value in kwargs.items()
+    }
     if not any(kwargs.values()):
-        for field in org:
+        for key in kwargs:
-            new_org[field] = click.prompt(
+            new_org[key] = click.prompt(key.capitalize(), default=org[key])
-                field.replace("_", " ").capitalize(), default=org[field]
+            if key == "email":
-            )
+                validate_email(new_org[key])
-            if field == "email":
+            if key == "orgname":
-                validate_email(new_org[field])
+                validate_orgname(new_org[key])
-            if field == "orgname":
-                validate_orgname(new_org[field])
-    else:
-        new_org.update(
-            {key.replace("new_", ""): value for key, value in kwargs.items() if value}
-        )
     client.update_org(cur_orgname, new_org)
     return click.secho(
         "The organization `%s` has been successfully updated." % cur_orgname,

platformio/account/team/commands/create.py

@@ -22,9 +22,7 @@ from platformio.account.validate import validate_orgname_teamname
 @click.argument(
     "orgname_teamname",
     metavar="ORGNAME:TEAMNAME",
-    callback=lambda _, __, value: validate_orgname_teamname(
+    callback=lambda _, __, value: validate_orgname_teamname(value),
-        value, teamname_validate=True
-    ),
 )
 @click.option(
     "--description",

platformio/account/team/commands/update.py

@@ -26,7 +26,7 @@ from platformio.account.validate import validate_orgname_teamname, validate_team
 )
 @click.option(
     "--name",
-    callback=lambda _, __, value: validate_teamname(value),
+    callback=lambda _, __, value: validate_teamname(value) if value else value,
     help="A new team name",
 )
 @click.option(
@@ -36,18 +36,14 @@ def team_update_cmd(orgname_teamname, **kwargs):
     orgname, teamname = orgname_teamname.split(":", 1)
     client = AccountClient()
     team = client.get_team(orgname, teamname)
-    del team["id"]
+    new_team = {
-    del team["members"]
+        key: value if value is not None else team[key] for key, value in kwargs.items()
-    new_team = team.copy()
+    }
     if not any(kwargs.values()):
-        for field in team:
+        for key in kwargs:
-            new_team[field] = click.prompt(
+            new_team[key] = click.prompt(key.capitalize(), default=team[key])
-                field.replace("_", " ").capitalize(), default=team[field]
+            if key == "name":
-            )
+                validate_teamname(new_team[key])
-            if field == "name":
-                validate_teamname(new_team[field])
-    else:
-        new_team.update({key: value for key, value in kwargs.items() if value})
     client.update_team(orgname, teamname, new_team)
     return click.secho(
         "The team %s has been successfully updated." % teamname,

platformio/account/validate.py

@@ -18,8 +18,10 @@ import click
 
 
 def validate_username(value, field="username"):
-    value = str(value).strip()
+    value = str(value).strip() if value else None
-    if not re.match(r"^[a-z\d](?:[a-z\d]|-(?=[a-z\d])){0,37}$", value, flags=re.I):
+    if not value or not re.match(
+        r"^[a-z\d](?:[a-z\d]|-(?=[a-z\d])){0,37}$", value, flags=re.I
+    ):
         raise click.BadParameter(
             "Invalid %s format. "
             "%s must contain only alphanumeric characters "
@@ -30,16 +32,22 @@ def validate_username(value, field="username"):
     return value
 
 
+def validate_orgname(value):
+    return validate_username(value, "Organization name")
+
+
 def validate_email(value):
-    value = str(value).strip()
+    value = str(value).strip() if value else None
-    if not re.match(r"^[a-z\d_\.\+\-]+@[a-z\d\-]+\.[a-z\d\-\.]+$", value, flags=re.I):
+    if not value or not re.match(
+        r"^[a-z\d_\.\+\-]+@[a-z\d\-]+\.[a-z\d\-\.]+$", value, flags=re.I
+    ):
         raise click.BadParameter("Invalid email address")
     return value
 
 
 def validate_password(value):
-    value = str(value).strip()
+    value = str(value).strip() if value else None
-    if not re.match(r"^(?=.*[a-z])(?=.*\d).{8,}$", value):
+    if not value or not re.match(r"^(?=.*[a-z])(?=.*\d).{8,}$", value):
         raise click.BadParameter(
             "Invalid password format. "
             "Password must contain at least 8 characters"
@@ -48,27 +56,11 @@ def validate_password(value):
     return value
 
 
-def validate_orgname(value):
-    return validate_username(value, "Organization name")
-
-
-def validate_orgname_teamname(value, teamname_validate=False):
-    if ":" not in value:
-        raise click.BadParameter(
-            "Please specify organization and team name in the next"
-            " format - orgname:teamname. For example, mycompany:DreamTeam"
-        )
-    teamname = str(value.strip().split(":", 1)[1])
-    if teamname_validate:
-        validate_teamname(teamname)
-    return value
-
-
 def validate_teamname(value):
-    if not value:
+    value = str(value).strip() if value else None
-        return value
+    if not value or not re.match(
-    value = str(value).strip()
+        r"^[a-z\d](?:[a-z\d]|[\-_ ](?=[a-z\d])){0,19}$", value, flags=re.I
-    if not re.match(r"^[a-z\d](?:[a-z\d]|[\-_ ](?=[a-z\d])){0,19}$", value, flags=re.I):
+    ):
         raise click.BadParameter(
             "Invalid team name format. "
             "Team name must only contain alphanumeric characters, "
@@ -77,3 +69,16 @@ def validate_teamname(value):
             " not be longer than 20 characters."
         )
     return value
+
+
+def validate_orgname_teamname(value):
+    value = str(value).strip() if value else None
+    if not value or ":" not in value:
+        raise click.BadParameter(
+            "Please specify organization and team name using the following"
+            " format - orgname:teamname. For example, mycompany:DreamTeam"
+        )
+    orgname, teamname = value.split(":", 1)
+    validate_orgname(orgname)
+    validate_teamname(teamname)
+    return value

platformio/assets/templates/ide-projects/vscode/.vscode/launch.json.tpl

@@ -16,7 +16,7 @@
 %    "request": "launch",
 %    "name": "PIO Debug (skip Pre-Debug)",
 %    "executable": _escape_path(prog_path),
-%    "projectEnvName": env_name,
+%    "projectEnvName": env_name if forced_env_name else default_debug_env_name,
 %    "toolchainBinDir": _escape_path(os.path.dirname(gdb_path)),
 %    "internalConsoleOptions": "openOnSessionStart",
 %  }
@@ -28,7 +28,7 @@
 %  debug["name"] = "PIO Debug"
 %  debug["preLaunchTask"] = {
 %    "type": "PlatformIO",
-%    "task": ("Pre-Debug (%s)" % env_name) if len(config.envs()) > 1 and original_env_name else "Pre-Debug",
+%    "task": ("Pre-Debug (%s)" % env_name) if len(config.envs()) > 1 and forced_env_name else "Pre-Debug",
 %  }
 %  noloading = predebug.copy()
 %  noloading["name"] = "PIO Debug (without uploading)"

platformio/builder/main.py

@@ -28,9 +28,9 @@ from SCons.Script import DefaultEnvironment  # pylint: disable=import-error
 from SCons.Script import Import  # pylint: disable=import-error
 from SCons.Script import Variables  # pylint: disable=import-error
 
-from platformio import app, compat, fs
+from platformio import app, fs
 from platformio.platform.base import PlatformBase
-from platformio.proc import get_pythonexe_path
+from platformio.proc import get_pythonexe_path, where_is_program
 from platformio.project.helpers import get_project_dir
 
 AllowSubstExceptions(NameError)
@@ -99,6 +99,7 @@ if not int(ARGUMENTS.get("PIOVERBOSE", 0)):
         DEFAULT_ENV_OPTIONS["%sSTR" % name] = "%s $TARGET" % (value)
 
 env = DefaultEnvironment(**DEFAULT_ENV_OPTIONS)
+env.SConscriptChdir(False)
 
 # Load variables from CLI
 env.Replace(
@@ -139,19 +140,6 @@ if int(ARGUMENTS.get("ISATTY", 0)):
     # pylint: disable=protected-access
     click._compat.isatty = lambda stream: True
 
-if compat.IS_WINDOWS and sys.version_info >= (3, 8) and os.getcwd().startswith("\\\\"):
-    click.secho("!!! WARNING !!!\t\t" * 3, fg="red")
-    click.secho(
-        "Your project is located on a mapped network drive but the "
-        "current command-line shell does not support the UNC paths.",
-        fg="yellow",
-    )
-    click.secho(
-        "Please move your project to a physical drive or check this workaround: "
-        "https://bit.ly/3kuU5mP\n",
-        fg="yellow",
-    )
-
 if env.subst("$BUILD_CACHE_DIR"):
     if not os.path.isdir(env.subst("$BUILD_CACHE_DIR")):
         os.makedirs(env.subst("$BUILD_CACHE_DIR"))
@@ -170,18 +158,17 @@ if not os.path.isdir(env.subst("$BUILD_DIR")):
 env.LoadProjectOptions()
 env.LoadPioPlatform()
 
-env.SConscriptChdir(0)
 env.SConsignFile(
     os.path.join(
-        "$BUILD_DIR", ".sconsign%d%d" % (sys.version_info[0], sys.version_info[1])
+        "$BUILD_CACHE_DIR" if env.subst("$BUILD_CACHE_DIR") else "$BUILD_DIR",
+        ".sconsign%d%d" % (sys.version_info[0], sys.version_info[1]),
     )
 )
 
-for item in env.GetExtraScripts("pre"):
+env.SConscript(env.GetExtraScripts("pre"), exports="env")
-    env.SConscript(item, exports="env")
 
 if env.IsCleanTarget():
-    env.CleanProject("cleanall" in COMMAND_LINE_TARGETS)
+    env.CleanProject(fullclean=int(ARGUMENTS.get("FULLCLEAN", 0)))
     env.Exit(0)
 
 env.SConscript("$BUILD_SCRIPT")
@@ -191,8 +178,7 @@ if "UPLOAD_FLAGS" in env:
 if env.GetProjectOption("upload_command"):
     env.Replace(UPLOADCMD=env.GetProjectOption("upload_command"))
 
-for item in env.GetExtraScripts("post"):
+env.SConscript(env.GetExtraScripts("post"), exports="env")
-    env.SConscript(item, exports="env")
 
 ##############################################################################
 
@@ -208,6 +194,13 @@ if env.get("SIZETOOL") and not (
     Default("checkprogsize")
 
 if "compiledb" in COMMAND_LINE_TARGETS:
+    # Resolve absolute path of toolchain
+    for cmd in ("CC", "CXX", "AS"):
+        if cmd not in env:
+            continue
+        if os.path.isabs(env[cmd]):
+            continue
+        env[cmd] = where_is_program(env.subst("$%s" % cmd), env.subst("${ENV['PATH']}"))
     env.Alias("compiledb", env.CompilationDatabase("$COMPILATIONDB_PATH"))
 
 # Print configured protocols
@@ -257,3 +250,9 @@ if "sizedata" in COMMAND_LINE_TARGETS:
     )
 
     Default("sizedata")
+
+# issue #4604: process targets sequentially
+for index, target in enumerate(
+    [t for t in COMMAND_LINE_TARGETS if not t.startswith("__")][1:]
+):
+    env.Depends(target, COMMAND_LINE_TARGETS[index])

platformio/builder/tools/compilation_db.py

@@ -1,224 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-# Copyright 2020 MongoDB Inc.
-#
-# Permission is hereby granted, free of charge, to any person obtaining
-# a copy of this software and associated documentation files (the
-# "Software"), to deal in the Software without restriction, including
-# without limitation the rights to use, copy, modify, merge, publish,
-# distribute, sublicense, and/or sell copies of the Software, and to
-# permit persons to whom the Software is furnished to do so, subject to
-# the following conditions:
-#
-# The above copyright notice and this permission notice shall be included
-# in all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
-# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
-# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
-# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-
-# pylint: disable=unused-argument, protected-access, unused-variable, import-error
-# Original: https://github.com/mongodb/mongo/blob/master/site_scons/site_tools/compilation_db.py
-
-import itertools
-import json
-import os
-
-import SCons
-
-from platformio.builder.tools.piobuild import SRC_ASM_EXT, SRC_C_EXT, SRC_CXX_EXT
-from platformio.proc import where_is_program
-
-# Implements the ability for SCons to emit a compilation database for the MongoDB project. See
-# http://clang.llvm.org/docs/JSONCompilationDatabase.html for details on what a compilation
-# database is, and why you might want one. The only user visible entry point here is
-# 'env.CompilationDatabase'. This method takes an optional 'target' to name the file that
-# should hold the compilation database, otherwise, the file defaults to compile_commands.json,
-# which is the name that most clang tools search for by default.
-
-# Is there a better way to do this than this global? Right now this exists so that the
-# emitter we add can record all of the things it emits, so that the scanner for the top level
-# compilation database can access the complete list, and also so that the writer has easy
-# access to write all of the files. But it seems clunky. How can the emitter and the scanner
-# communicate more gracefully?
-__COMPILATION_DB_ENTRIES = []
-
-
-# We make no effort to avoid rebuilding the entries. Someday, perhaps we could and even
-# integrate with the cache, but there doesn't seem to be much call for it.
-class __CompilationDbNode(SCons.Node.Python.Value):
-    def __init__(self, value):
-        SCons.Node.Python.Value.__init__(self, value)
-        self.Decider(changed_since_last_build_node)
-
-
-def changed_since_last_build_node(*args, **kwargs):
-    """Dummy decider to force always building"""
-    return True
-
-
-def makeEmitCompilationDbEntry(comstr):
-    """
-    Effectively this creates a lambda function to capture:
-    * command line
-    * source
-    * target
-    :param comstr: unevaluated command line
-    :return: an emitter which has captured the above
-    """
-    user_action = SCons.Action.Action(comstr)
-
-    def EmitCompilationDbEntry(target, source, env):
-        """
-        This emitter will be added to each c/c++ object build to capture the info needed
-        for clang tools
-        :param target: target node(s)
-        :param source: source node(s)
-        :param env: Environment for use building this node
-        :return: target(s), source(s)
-        """
-
-        # Resolve absolute path of toolchain
-        for cmd in ("CC", "CXX", "AS"):
-            if cmd not in env:
-                continue
-            if os.path.isabs(env[cmd]):
-                continue
-            env[cmd] = where_is_program(
-                env.subst("$%s" % cmd), env.subst("${ENV['PATH']}")
-            )
-
-        dbtarget = __CompilationDbNode(source)
-
-        entry = env.__COMPILATIONDB_Entry(
-            target=dbtarget,
-            source=[],
-            __COMPILATIONDB_UTARGET=target,
-            __COMPILATIONDB_USOURCE=source,
-            __COMPILATIONDB_UACTION=user_action,
-            __COMPILATIONDB_ENV=env,
-        )
-
-        # Technically, these next two lines should not be required: it should be fine to
-        # cache the entries. However, they don't seem to update properly. Since they are quick
-        # to re-generate disable caching and sidestep this problem.
-        env.AlwaysBuild(entry)
-        env.NoCache(entry)
-
-        __COMPILATION_DB_ENTRIES.append(dbtarget)
-
-        return target, source
-
-    return EmitCompilationDbEntry
-
-
-def CompilationDbEntryAction(target, source, env, **kw):
-    """
-    Create a dictionary with evaluated command line, target, source
-    and store that info as an attribute on the target
-    (Which has been stored in __COMPILATION_DB_ENTRIES array
-    :param target: target node(s)
-    :param source: source node(s)
-    :param env: Environment for use building this node
-    :param kw:
-    :return: None
-    """
-
-    command = env["__COMPILATIONDB_UACTION"].strfunction(
-        target=env["__COMPILATIONDB_UTARGET"],
-        source=env["__COMPILATIONDB_USOURCE"],
-        env=env["__COMPILATIONDB_ENV"],
-    )
-
-    entry = {
-        "directory": env.Dir("#").abspath,
-        "command": command,
-        "file": str(env["__COMPILATIONDB_USOURCE"][0]),
-    }
-
-    target[0].write(entry)
-
-
-def WriteCompilationDb(target, source, env):
-    entries = []
-
-    for s in __COMPILATION_DB_ENTRIES:
-        item = s.read()
-        item["file"] = os.path.abspath(item["file"])
-        entries.append(item)
-
-    with open(str(target[0]), mode="w", encoding="utf8") as target_file:
-        json.dump(
-            entries, target_file, sort_keys=True, indent=4, separators=(",", ": ")
-        )
-
-
-def ScanCompilationDb(node, env, path):
-    return __COMPILATION_DB_ENTRIES
-
-
-def generate(env, **kwargs):
-    static_obj, shared_obj = SCons.Tool.createObjBuilders(env)
-
-    env["COMPILATIONDB_COMSTR"] = kwargs.get(
-        "COMPILATIONDB_COMSTR", "Building compilation database $TARGET"
-    )
-
-    components_by_suffix = itertools.chain(
-        itertools.product(
-            [".%s" % ext for ext in SRC_C_EXT],
-            [
-                (static_obj, SCons.Defaults.StaticObjectEmitter, "$CCCOM"),
-                (shared_obj, SCons.Defaults.SharedObjectEmitter, "$SHCCCOM"),
-            ],
-        ),
-        itertools.product(
-            [".%s" % ext for ext in SRC_CXX_EXT],
-            [
-                (static_obj, SCons.Defaults.StaticObjectEmitter, "$CXXCOM"),
-                (shared_obj, SCons.Defaults.SharedObjectEmitter, "$SHCXXCOM"),
-            ],
-        ),
-        itertools.product(
-            [".%s" % ext for ext in SRC_ASM_EXT],
-            [(static_obj, SCons.Defaults.StaticObjectEmitter, "$ASCOM")],
-        ),
-    )
-
-    for entry in components_by_suffix:
-        suffix = entry[0]
-        builder, base_emitter, command = entry[1]
-
-        # Assumes a dictionary emitter
-        emitter = builder.emitter[suffix]
-        builder.emitter[suffix] = SCons.Builder.ListEmitter(
-            [emitter, makeEmitCompilationDbEntry(command)]
-        )
-
-    env["BUILDERS"]["__COMPILATIONDB_Entry"] = SCons.Builder.Builder(
-        action=SCons.Action.Action(CompilationDbEntryAction, None),
-    )
-
-    env["BUILDERS"]["__COMPILATIONDB_Database"] = SCons.Builder.Builder(
-        action=SCons.Action.Action(WriteCompilationDb, "$COMPILATIONDB_COMSTR"),
-        target_scanner=SCons.Scanner.Scanner(
-            function=ScanCompilationDb, node_class=None
-        ),
-    )
-
-    def CompilationDatabase(env, target):
-        result = env.__COMPILATIONDB_Database(target=target, source=[])
-
-        env.AlwaysBuild(result)
-        env.NoCache(result)
-
-        return result
-
-    env.AddMethod(CompilationDatabase, "CompilationDatabase")
-
-
-def exists(env):
-    return True

platformio/builder/tools/piobuild.py

@@ -239,7 +239,7 @@ def ProcessUnFlags(env, flags):
     for scope in unflag_scopes:
         for unflags in parsed.values():
             for unflag in unflags:
-                for current in env.get(scope, []):
+                for current in list(env.get(scope, [])):
                     conditions = [
                         unflag == current,
                         not isinstance(unflag, (tuple, list))

platformio/builder/tools/pioino.py

@@ -25,7 +25,6 @@ from platformio.compat import get_filesystem_encoding, get_locale_encoding
 
 
 class InoToCPPConverter:
-
     PROTOTYPE_RE = re.compile(
         r"""^(
         (?:template\<.*\>\s*)?      # template
@@ -103,7 +102,7 @@ class InoToCPPConverter:
         return "\n".join(["#include <Arduino.h>"] + lines) if lines else None
 
     def process(self, contents):
-        out_file = self._main_ino + ".cpp"
+        out_file = re.sub(r"[\"\'\;]+", "", self._main_ino) + ".cpp"
         assert self._gcc_preprocess(contents, out_file)
         contents = self.read_safe_contents(out_file)
         contents = self._join_multiline_strings(contents)

platformio/builder/tools/piolib.py

@@ -29,7 +29,7 @@ from SCons.Script import DefaultEnvironment  # pylint: disable=import-error
 from platformio import exception, fs
 from platformio.builder.tools import piobuild
 from platformio.compat import IS_WINDOWS, hashlib_encode_data, string_types
-from platformio.http import HTTPClientError, InternetIsOffline
+from platformio.http import HTTPClientError, InternetConnectionError
 from platformio.package.exception import (
     MissingPackageManifestError,
     UnknownPackageError,
@@ -109,7 +109,6 @@ class LibBuilderFactory:
 
 
 class LibBuilderBase:
-
     CLASSIC_SCANNER = SCons.Scanner.C.CScanner()
     CCONDITIONAL_SCANNER = SCons.Scanner.C.CConditionalScanner()
     # Max depth of nested includes:
@@ -298,11 +297,12 @@ class LibBuilderBase:
         with fs.cd(self.path):
             self.env.ProcessFlags(self.build_flags)
             if self.extra_script:
-                self.env.SConscriptChdir(1)
+                self.env.SConscriptChdir(True)
                 self.env.SConscript(
                     os.path.abspath(self.extra_script),
                     exports={"env": self.env, "pio_lib_builder": self},
                 )
+                self.env.SConscriptChdir(False)
             self.env.ProcessUnFlags(self.build_unflags)
 
     def process_dependencies(self):
@@ -982,7 +982,11 @@ class ProjectAsLibBuilder(LibBuilderBase):
             try:
                 lm.install(spec)
                 did_install = True
-            except (HTTPClientError, UnknownPackageError, InternetIsOffline) as exc:
+            except (
+                HTTPClientError,
+                UnknownPackageError,
+                InternetConnectionError,
+            ) as exc:
                 click.secho("Warning! %s" % exc, fg="yellow")
 
         # reset cache
@@ -1157,7 +1161,7 @@ def ConfigureProjectLibBuilder(env):
                 click.echo("Path: %s" % lb.path, nl=False)
                 click.echo(")", nl=False)
             click.echo("")
-            if lb.depbuilders:
+            if lb.verbose and lb.depbuilders:
                 _print_deps_tree(lb, level + 1)
 
     project = ProjectAsLibBuilder(env, "$PROJECT_DIR")

platformio/builder/tools/piotarget.py

@@ -16,7 +16,6 @@ import os
 
 from SCons.Action import Action  # pylint: disable=import-error
 from SCons.Script import ARGUMENTS  # pylint: disable=import-error
-from SCons.Script import COMMAND_LINE_TARGETS  # pylint: disable=import-error
 from SCons.Script import AlwaysBuild  # pylint: disable=import-error
 
 from platformio import compat, fs
@@ -29,10 +28,10 @@ def VerboseAction(_, act, actstr):
 
 
 def IsCleanTarget(env):
-    return env.GetOption("clean") or ("cleanall" in COMMAND_LINE_TARGETS)
+    return env.GetOption("clean")
 
 
-def CleanProject(env, clean_all=False):
+def CleanProject(env, fullclean=False):
     def _relpath(path):
         if compat.IS_WINDOWS:
             prefix = os.getcwd()[:2].lower()
@@ -56,7 +55,7 @@ def CleanProject(env, clean_all=False):
     else:
         print("Build environment is clean")
 
-    if clean_all and os.path.isdir(libdeps_dir):
+    if fullclean and os.path.isdir(libdeps_dir):
         _clean_dir(libdeps_dir)
 
     print("Done cleaning")

platformio/check/cli.py

@@ -38,18 +38,15 @@ from platformio.project.helpers import find_project_dir_above, get_project_dir
     "-d",
     "--project-dir",
     default=os.getcwd,
-    type=click.Path(
+    type=click.Path(exists=True, file_okay=True, dir_okay=True, writable=True),
-        exists=True, file_okay=True, dir_okay=True, writable=True, resolve_path=True
-    ),
 )
 @click.option(
     "-c",
     "--project-conf",
-    type=click.Path(
+    type=click.Path(exists=True, file_okay=True, dir_okay=False, readable=True),
-        exists=True, file_okay=True, dir_okay=False, readable=True, resolve_path=True
-    ),
 )
-@click.option("--pattern", multiple=True)
+@click.option("--pattern", multiple=True, hidden=True)
+@click.option("-f", "--src-filters", multiple=True)
 @click.option("--flags", multiple=True)
 @click.option(
     "--severity", multiple=True, type=click.Choice(DefectItem.SEVERITY_LABELS.values())
@@ -67,6 +64,7 @@ def cli(
     environment,
     project_dir,
     project_conf,
+    src_filters,
     pattern,
     flags,
     severity,
@@ -105,14 +103,24 @@ def cli(
                     "%s: %s" % (k, ", ".join(v) if isinstance(v, list) else v)
                 )
 
-            default_patterns = [
+            default_src_filters = [
-                config.get("platformio", "src_dir"),
+                "+<%s>" % os.path.basename(config.get("platformio", "src_dir")),
-                config.get("platformio", "include_dir"),
+                "+<%s>" % os.path.basename(config.get("platformio", "include_dir")),
             ]
+
+            src_filters = (
+                src_filters
+                or pattern
+                or env_options.get(
+                    "check_src_filters",
+                    env_options.get("check_patterns", default_src_filters),
+                )
+            )
+
             tool_options = dict(
                 verbose=verbose,
                 silent=silent,
-                patterns=pattern or env_options.get("check_patterns", default_patterns),
+                src_filters=src_filters,
                 flags=flags or env_options.get("check_flags"),
                 severity=[DefectItem.SEVERITY_LABELS[DefectItem.SEVERITY_HIGH]]
                 if silent
@@ -265,7 +273,7 @@ def print_defects_stats(results):
     tabular_data.append(total)
 
     headers = ["Component"]
-    headers.extend([l.upper() for l in severity_labels])
+    headers.extend([label.upper() for label in severity_labels])
     headers = [click.style(h, bold=True) for h in headers]
     click.echo(tabulate(tabular_data, headers=headers, numalign="center"))
     click.echo()

platformio/check/defect.py

@@ -16,6 +16,7 @@ import os
 
 import click
 
+from platformio.exception import PlatformioException
 from platformio.project.helpers import get_project_dir
 
 # pylint: disable=too-many-instance-attributes, redefined-builtin
@@ -23,7 +24,6 @@ from platformio.project.helpers import get_project_dir
 
 
 class DefectItem:
-
     SEVERITY_HIGH = 1
     SEVERITY_MEDIUM = 2
     SEVERITY_LOW = 4
@@ -79,7 +79,7 @@ class DefectItem:
         for key, value in DefectItem.SEVERITY_LABELS.items():
             if label == value:
                 return key
-        raise Exception("Unknown severity label -> %s" % label)
+        raise PlatformioException("Unknown severity label -> %s" % label)
 
     def as_dict(self):
         return {

platformio/check/tools/base.py

@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import glob
 import os
 import tempfile
 
@@ -30,6 +29,7 @@ class CheckToolBase:  # pylint: disable=too-many-instance-attributes
         self.config = config
         self.envname = envname
         self.options = options
+        self.project_dir = project_dir
         self.cc_flags = []
         self.cxx_flags = []
         self.cpp_includes = []
@@ -41,7 +41,7 @@ class CheckToolBase:  # pylint: disable=too-many-instance-attributes
         self._defects = []
         self._on_defect_callback = None
         self._bad_input = False
-        self._load_cpp_data(project_dir)
+        self._load_cpp_data()
 
         # detect all defects by default
         if not self.options.get("severity"):
@@ -56,8 +56,8 @@ class CheckToolBase:  # pylint: disable=too-many-instance-attributes
             for s in self.options["severity"]
         ]
 
-    def _load_cpp_data(self, project_dir):
+    def _load_cpp_data(self):
-        data = load_build_metadata(project_dir, self.envname)
+        data = load_build_metadata(self.project_dir, self.envname)
         if not data:
             return
         self.cc_flags = click.parser.split_arg_string(data.get("cc_flags", ""))
@@ -99,6 +99,13 @@ class CheckToolBase:  # pylint: disable=too-many-instance-attributes
                 includes_file,
             )
             result = proc.exec_command(cmd, shell=True)
+
+            if result["returncode"] != 0:
+                click.echo("Warning: Failed to extract toolchain defines!")
+                if self.options.get("verbose"):
+                    click.echo(result["out"])
+                    click.echo(result["err"])
+
             for line in result["out"].split("\n"):
                 tokens = line.strip().split(" ", 2)
                 if not tokens or tokens[0] != "#define":
@@ -201,7 +208,7 @@ class CheckToolBase:  # pylint: disable=too-many-instance-attributes
         return result
 
     @staticmethod
-    def get_project_target_files(patterns):
+    def get_project_target_files(project_dir, src_filters):
         c_extension = (".c",)
         cpp_extensions = (".cc", ".cpp", ".cxx", ".ino")
         header_extensions = (".h", ".hh", ".hpp", ".hxx")
@@ -216,13 +223,9 @@ class CheckToolBase:  # pylint: disable=too-many-instance-attributes
             elif path.endswith(cpp_extensions):
                 result["c++"].append(os.path.abspath(path))
 
-        for pattern in patterns:
+        src_filters = normalize_src_filters(src_filters)
-            for item in glob.glob(pattern, recursive=True):
+        for f in fs.match_src_files(project_dir, src_filters):
-                if not os.path.isdir(item):
+            _add_file(f)
-                    _add_file(item)
-                for root, _, files in os.walk(item, followlinks=True):
-                    for f in files:
-                        _add_file(os.path.join(root, f))
 
         return result
 
@@ -243,3 +246,22 @@ class CheckToolBase:  # pylint: disable=too-many-instance-attributes
         self.clean_up()
 
         return self._bad_input
+
+
+#
+# Helpers
+#
+
+
+def normalize_src_filters(src_filters):
+    def _normalize(src_filters):
+        return (
+            src_filters
+            if src_filters.startswith(("+<", "-<"))
+            else "+<%s>" % src_filters
+        )
+
+    if isinstance(src_filters, (list, tuple)):
+        return " ".join([_normalize(f) for f in src_filters])
+
+    return _normalize(src_filters)

platformio/check/tools/clangtidy.py

@@ -64,7 +64,9 @@ class ClangtidyCheckTool(CheckToolBase):
         ):
             cmd.append("--checks=*")
 
-        project_files = self.get_project_target_files(self.options["patterns"])
+        project_files = self.get_project_target_files(
+            self.project_dir, self.options["src_filters"]
+        )
 
         src_files = []
         for items in project_files.values():

platformio/check/tools/cppcheck.py

@@ -96,7 +96,7 @@ class CppcheckCheckTool(CheckToolBase):
                     )
                     click.echo()
                 self._bad_input = True
-                self._buffer = ""
+            self._buffer = ""
             return None
 
         self._buffer = ""
@@ -214,7 +214,9 @@ class CppcheckCheckTool(CheckToolBase):
         if not self.is_flag_set("--addon", self.get_flags("cppcheck")):
             return
 
-        for files in self.get_project_target_files(self.options["patterns"]).values():
+        for files in self.get_project_target_files(
+            self.project_dir, self.options["src_filters"]
+        ).values():
             for f in files:
                 dump_file = f + ".dump"
                 if os.path.isfile(dump_file):
@@ -243,7 +245,9 @@ class CppcheckCheckTool(CheckToolBase):
     def check(self, on_defect_callback=None):
         self._on_defect_callback = on_defect_callback
 
-        project_files = self.get_project_target_files(self.options["patterns"])
+        project_files = self.get_project_target_files(
+            self.project_dir, self.options["src_filters"]
+        )
         src_files_scope = ("c", "c++")
         if not any(project_files[t] for t in src_files_scope):
             click.echo("Error: Nothing to check.")

platformio/check/tools/pvsstudio.py

@@ -227,7 +227,7 @@ class PvsStudioCheckTool(CheckToolBase):  # pylint: disable=too-many-instance-at
     def check(self, on_defect_callback=None):
         self._on_defect_callback = on_defect_callback
         for scope, files in self.get_project_target_files(
-            self.options["patterns"]
+            self.project_dir, self.options["src_filters"]
         ).items():
             if scope not in ("c", "c++"):
                 continue

platformio/cli.py

@@ -19,7 +19,6 @@ import click
 
 
 class PlatformioCLI(click.MultiCommand):
-
     leftover_args = []
 
     def __init__(self, *args, **kwargs):

platformio/commands/boards.py

@@ -42,7 +42,7 @@ def cli(query, installed, json_output):  # pylint: disable=R0912
         grpboards[board["platform"]].append(board)
 
     terminal_width = shutil.get_terminal_size().columns
-    for (platform, boards) in sorted(grpboards.items()):
+    for platform, boards in sorted(grpboards.items()):
         click.echo("")
         click.echo("Platform: ", nl=False)
         click.secho(platform, bold=True)

platformio/commands/ci.py

@@ -51,15 +51,13 @@ def validate_path(ctx, param, value):  # pylint: disable=unused-argument
 @click.option(
     "--build-dir",
     default=tempfile.mkdtemp,
-    type=click.Path(file_okay=False, dir_okay=True, writable=True, resolve_path=True),
+    type=click.Path(file_okay=False, dir_okay=True, writable=True),
 )
 @click.option("--keep-build-dir", is_flag=True)
 @click.option(
     "-c",
     "--project-conf",
-    type=click.Path(
+    type=click.Path(exists=True, file_okay=True, dir_okay=False, readable=True),
-        exists=True, file_okay=True, dir_okay=False, readable=True, resolve_path=True
-    ),
 )
 @click.option("-O", "--project-option", multiple=True)
 @click.option("-e", "--environment", "environments", multiple=True)
@@ -109,8 +107,8 @@ def cli(  # pylint: disable=too-many-arguments, too-many-branches
         ctx.invoke(
             project_init_cmd,
             project_dir=build_dir,
-            board=board,
+            boards=board,
-            project_option=project_option,
+            project_options=project_option,
         )
 
         # process project

platformio/commands/lib.py

@@ -65,9 +65,7 @@ def invoke_command(ctx, cmd, **kwargs):
     "--storage-dir",
     multiple=True,
     default=None,
-    type=click.Path(
+    type=click.Path(exists=True, file_okay=False, dir_okay=True, writable=True),
-        exists=True, file_okay=False, dir_okay=True, writable=True, resolve_path=True
-    ),
     help="Manage custom library storage",
 )
 @click.option(

platformio/commands/upgrade.py

@@ -13,23 +13,28 @@
 # limitations under the License.
 
 import json
-import os
 import re
-from zipfile import ZipFile
+import subprocess
 
 import click
 
 from platformio import VERSION, __version__, app, exception
-from platformio.compat import IS_WINDOWS
 from platformio.http import fetch_remote_content
 from platformio.package.manager.core import update_core_packages
-from platformio.proc import exec_command, get_pythonexe_path
+from platformio.proc import get_pythonexe_path
-from platformio.project.helpers import get_project_cache_dir
+
+PYPI_JSON_URL = "https://pypi.org/pypi/platformio/json"
+DEVELOP_ZIP_URL = "https://github.com/platformio/platformio-core/archive/develop.zip"
+DEVELOP_INIT_SCRIPT_URL = (
+    "https://raw.githubusercontent.com/platformio/platformio-core"
+    "/develop/platformio/__init__.py"
+)
 
 
 @click.command("upgrade", short_help="Upgrade PlatformIO Core to the latest version")
 @click.option("--dev", is_flag=True, help="Use development branch")
-def cli(dev):
+@click.option("--verbose", "-v", is_flag=True)
+def cli(dev, verbose):
     update_core_packages()
     if not dev and __version__ == get_latest_version():
         return click.secho(
@@ -38,29 +43,26 @@ def cli(dev):
             fg="green",
         )
 
-    click.secho("Please wait while upgrading PlatformIO ...", fg="yellow")
+    click.secho("Please wait while upgrading PlatformIO Core ...", fg="yellow")
 
+    python_exe = get_pythonexe_path()
     to_develop = dev or not all(c.isdigit() for c in __version__ if c != ".")
-    cmds = (
+    pkg_spec = DEVELOP_ZIP_URL if to_develop else "platformio"
-        ["pip", "install", "--upgrade", download_dist_package(to_develop)],
-        ["platformio", "--version"],
-    )
 
-    cmd = None
-    r = {}
     try:
-        for cmd in cmds:
+        subprocess.run(
-            cmd = [get_pythonexe_path(), "-m"] + cmd
+            [python_exe, "-m", "pip", "install", "--upgrade", pkg_spec],
-            r = exec_command(cmd)
+            check=True,
-
+            capture_output=not verbose,
-            # try pip with disabled cache
+        )
-            if r["returncode"] != 0 and cmd[2] == "pip":
+        r = subprocess.run(
-                cmd.insert(3, "--no-cache-dir")
+            [python_exe, "-m", "platformio", "--version"],
-                r = exec_command(cmd)
+            check=True,
-
+            capture_output=True,
-            assert r["returncode"] == 0
+            text=True,
-        assert "version" in r["out"]
+        )
-        actual_version = r["out"].strip().split("version", 1)[1].strip()
+        assert "version" in r.stdout
+        actual_version = r.stdout.split("version", 1)[1].strip()
         click.secho(
             "PlatformIO has been successfully upgraded to %s" % actual_version,
             fg="green",
@@ -71,52 +73,24 @@ def cli(dev):
             click.secho(
                 "Warning! Please restart IDE to affect PIO Home changes", fg="yellow"
             )
-    except Exception as exc:
+    except (AssertionError, subprocess.CalledProcessError) as exc:
-        if not r:
+        click.secho(
-            raise exception.UpgradeError("\n".join([str(cmd), str(exc)])) from exc
+            "\nWarning!!! Could not automatically upgrade the PlatformIO Core.",
-        permission_errors = ("permission denied", "not permitted")
+            fg="red",
-        if any(m in r["err"].lower() for m in permission_errors) and not IS_WINDOWS:
+        )
-            click.secho(
+        click.secho(
-                """
+            "Please upgrade it manually using the following command:\n",
------------------
+            fg="red",
-Permission denied
+        )
------------------
+        click.secho(f'"{python_exe}" -m pip install -U {pkg_spec}\n', fg="cyan")
-You need the `sudo` permission to install Python packages. Try
+        raise exception.ReturnErrorCode(1) from exc
-
-> sudo pip install -U platformio
-
-WARNING! Don't use `sudo` for the rest PlatformIO commands.
-""",
-                fg="yellow",
-                err=True,
-            )
-            raise exception.ReturnErrorCode(1)
-        raise exception.UpgradeError("\n".join([str(cmd), r["out"], r["err"]]))
 
     return True
 
 
-def download_dist_package(to_develop):
+def get_pkg_spec(to_develop):
-    if not to_develop:
+    if to_develop:
-        return "platformio"
+        return
-    dl_url = "https://github.com/platformio/platformio-core/archive/develop.zip"
-    cache_dir = get_project_cache_dir()
-    if not os.path.isdir(cache_dir):
-        os.makedirs(cache_dir)
-    pkg_name = os.path.join(cache_dir, "piocoredevelop.zip")
-    try:
-        with open(pkg_name, "wb") as fp:
-            r = exec_command(
-                ["curl", "-fsSL", dl_url], stdout=fp, universal_newlines=True
-            )
-            assert r["returncode"] == 0
-        # check ZIP structure
-        with ZipFile(pkg_name) as zp:
-            assert zp.testzip() is None
-        return pkg_name
-    except:  # pylint: disable=bare-except
-        pass
-    return dl_url
 
 
 def get_latest_version():
@@ -133,10 +107,7 @@ def get_latest_version():
 
 def get_develop_latest_version():
     version = None
-    content = fetch_remote_content(
+    content = fetch_remote_content(DEVELOP_INIT_SCRIPT_URL)
-        "https://raw.githubusercontent.com/platformio/platformio"
-        "/develop/platformio/__init__.py"
-    )
     for line in content.split("\n"):
         line = line.strip()
         if not line.startswith("VERSION"):
@@ -153,5 +124,5 @@ def get_develop_latest_version():
 
 
 def get_pypi_latest_version():
-    content = fetch_remote_content("https://pypi.org/pypi/platformio/json")
+    content = fetch_remote_content(PYPI_JSON_URL)
     return json.loads(content)["info"]["version"]

platformio/debug/cli.py

@@ -28,9 +28,9 @@ from platformio.debug import helpers
 from platformio.debug.config.factory import DebugConfigFactory
 from platformio.debug.exception import DebugInvalidOptionsError
 from platformio.debug.process.gdb import GDBClientProcess
+from platformio.exception import ReturnErrorCode
 from platformio.platform.factory import PlatformFactory
 from platformio.project.config import ProjectConfig
-from platformio.project.exception import ProjectEnvsNotAvailableError
 from platformio.project.helpers import is_platformio_project
 from platformio.project.options import ProjectOptions
 
@@ -44,16 +44,12 @@ from platformio.project.options import ProjectOptions
     "-d",
     "--project-dir",
     default=os.getcwd,
-    type=click.Path(
+    type=click.Path(exists=True, file_okay=False, dir_okay=True, writable=True),
-        exists=True, file_okay=False, dir_okay=True, writable=True, resolve_path=True
-    ),
 )
 @click.option(
     "-c",
     "--project-conf",
-    type=click.Path(
+    type=click.Path(exists=True, file_okay=True, dir_okay=False, readable=True),
-        exists=True, file_okay=True, dir_okay=False, readable=True, resolve_path=True
-    ),
 )
 @click.option("--environment", "-e", metavar="<environment>")
 @click.option("--load-mode", type=ProjectOptions["env.debug_load_mode"].type)
@@ -81,61 +77,57 @@ def cli(
             project_dir = os.getenv(name)
 
     with fs.cd(project_dir):
-        return _debug_in_project_dir(
+        project_config = ProjectConfig.get_instance(project_conf)
+        project_config.validate(envs=[environment] if environment else None)
+        env_name = environment or helpers.get_default_debug_env(project_config)
+
+        if not interface:
+            return helpers.predebug_project(
+                ctx, project_dir, project_config, env_name, False, verbose
+            )
+
+        configure_args = (
             ctx,
-            project_dir,
+            project_config,
-            project_conf,
+            env_name,
-            environment,
             load_mode,
             verbose,
-            interface,
             __unprocessed,
         )
+        if helpers.is_gdbmi_mode():
+            os.environ["PLATFORMIO_DISABLE_PROGRESSBAR"] = "true"
+            stream = helpers.GDBMIConsoleStream()
+            with proc.capture_std_streams(stream):
+                debug_config = _configure(*configure_args)
+            stream.close()
+        else:
+            debug_config = _configure(*configure_args)
+
+        _run(project_dir, debug_config, __unprocessed)
+
+    return None
 
 
-def _debug_in_project_dir(
+def _configure(ctx, project_config, env_name, load_mode, verbose, __unprocessed):
-    ctx,
+    platform = PlatformFactory.new(
-    project_dir,
+        project_config.get(f"env:{env_name}", "platform"), autoinstall=True
-    project_conf,
+    )
-    environment,
-    load_mode,
-    verbose,
-    interface,
-    __unprocessed,
-):
-    project_config = ProjectConfig.get_instance(project_conf)
-    project_config.validate(envs=[environment] if environment else None)
-    env_name = environment or helpers.get_default_debug_env(project_config)
-
-    if not interface:
-        return helpers.predebug_project(
-            ctx, project_dir, project_config, env_name, False, verbose
-        )
-
-    env_options = project_config.items(env=env_name, as_dict=True)
-    if "platform" not in env_options:
-        raise ProjectEnvsNotAvailableError()
-
     debug_config = DebugConfigFactory.new(
-        PlatformFactory.new(env_options["platform"], autoinstall=True),
+        platform,
         project_config,
         env_name,
     )
-
     if "--version" in __unprocessed:
-        return subprocess.run(
+        raise ReturnErrorCode(
-            [debug_config.client_executable_path, "--version"], check=True
+            subprocess.run(
+                [debug_config.client_executable_path, "--version"], check=True
+            ).returncode
         )
 
     try:
         fs.ensure_udev_rules()
     except exception.InvalidUdevRules as exc:
-        click.echo(
+        click.echo(str(exc))
-            helpers.escape_gdbmi_stream("~", str(exc) + "\n")
-            if helpers.is_gdbmi_mode()
-            else str(exc) + "\n",
-            nl=False,
-        )
 
     rebuild_prog = False
     preload = debug_config.load_cmds == ["preload"]
@@ -157,25 +149,10 @@ def _debug_in_project_dir(
         debug_config.load_cmds = []
 
     if rebuild_prog:
-        if helpers.is_gdbmi_mode():
+        click.echo("Preparing firmware for debugging...")
-            click.echo(
+        helpers.predebug_project(
-                helpers.escape_gdbmi_stream(
+            ctx, os.getcwd(), project_config, env_name, preload, verbose
-                    "~", "Preparing firmware for debugging...\n"
+        )
-                ),
-                nl=False,
-            )
-            stream = helpers.GDBMIConsoleStream()
-            with proc.capture_std_streams(stream):
-                helpers.predebug_project(
-                    ctx, project_dir, project_config, env_name, preload, verbose
-                )
-            stream.close()
-        else:
-            click.echo("Preparing firmware for debugging...")
-            helpers.predebug_project(
-                ctx, project_dir, project_config, env_name, preload, verbose
-            )
-
         # save SHA sum of newly created prog
         if load_mode == "modified":
             helpers.is_prog_obsolete(debug_config.program_path)
@@ -183,6 +160,10 @@ def _debug_in_project_dir(
     if not os.path.isfile(debug_config.program_path):
         raise DebugInvalidOptionsError("Program/firmware is missed")
 
+    return debug_config
+
+
+def _run(project_dir, debug_config, __unprocessed):
     loop = asyncio.ProactorEventLoop() if IS_WINDOWS else asyncio.get_event_loop()
     asyncio.set_event_loop(loop)
 
@@ -199,5 +180,3 @@ def _debug_in_project_dir(
     finally:
         client.close()
         loop.close()
-
-    return True

platformio/debug/config/base.py

@@ -146,9 +146,9 @@ class DebugConfigBase:  # pylint: disable=too-many-instance-attributes
 
     def _load_build_data(self):
         data = load_build_metadata(os.getcwd(), self.env_name, cache=True, debug=True)
-        if data:
+        if not data:
-            return data
+            raise DebugInvalidOptionsError("Could not load a build configuration")
-        raise DebugInvalidOptionsError("Could not load a build configuration")
+        return data
 
     def _configure_server(self):
         # user disabled server in platformio.ini

platformio/debug/config/blackmagic.py

@@ -18,7 +18,6 @@ from platformio.device.finder import SerialPortFinder, is_pattern_port
 
 
 class BlackmagicDebugConfig(DebugConfigBase):
-
     GDB_INIT_SCRIPT = """
 define pio_reset_halt_target
     set language c

platformio/debug/config/generic.py

@@ -16,7 +16,6 @@ from platformio.debug.config.base import DebugConfigBase
 
 
 class GenericDebugConfig(DebugConfigBase):
-
     GDB_INIT_SCRIPT = """
 define pio_reset_halt_target
     monitor reset halt

platformio/debug/config/jlink.py

@@ -16,7 +16,6 @@ from platformio.debug.config.base import DebugConfigBase
 
 
 class JlinkDebugConfig(DebugConfigBase):
-
     GDB_INIT_SCRIPT = """
 define pio_reset_halt_target
     monitor reset

platformio/debug/config/mspdebug.py

@@ -16,7 +16,6 @@ from platformio.debug.config.base import DebugConfigBase
 
 
 class MspdebugDebugConfig(DebugConfigBase):
-
     GDB_INIT_SCRIPT = """
 define pio_reset_halt_target
 end

platformio/debug/config/native.py

@@ -17,7 +17,6 @@ from platformio.debug.config.base import DebugConfigBase
 
 
 class NativeDebugConfig(DebugConfigBase):
-
     GDB_INIT_SCRIPT = """
 define pio_reset_halt_target
 end

platformio/debug/config/qemu.py

@@ -16,7 +16,6 @@ from platformio.debug.config.base import DebugConfigBase
 
 
 class QemuDebugConfig(DebugConfigBase):
-
     GDB_INIT_SCRIPT = """
 define pio_reset_halt_target
     monitor system_reset

platformio/debug/config/renode.py

@@ -16,7 +16,6 @@ from platformio.debug.config.base import DebugConfigBase
 
 
 class RenodeDebugConfig(DebugConfigBase):
-
     GDB_INIT_SCRIPT = """
 define pio_reset_halt_target
     monitor machine Reset

platformio/debug/exception.py

@@ -20,7 +20,6 @@ class DebugError(PlatformioException):
 
 
 class DebugSupportError(DebugError, UserSideException):
-
     MESSAGE = (
         "Currently, PlatformIO does not support debugging for `{0}`.\n"
         "Please request support at https://github.com/platformio/"

platformio/debug/helpers.py

@@ -31,7 +31,6 @@ from platformio.test.runners.factory import TestRunnerFactory
 
 
 class GDBMIConsoleStream(BytesIO):  # pylint: disable=too-few-public-methods
-
     STDOUT = sys.stdout
 
     def write(self, text):
@@ -91,7 +90,7 @@ def predebug_project(
             TestSuite(env_name, debug_testname),
             project_config,
             TestRunnerOptions(
-                verbose=verbose,
+                verbose=3 if verbose else 0,
                 without_building=False,
                 without_debugging=False,
                 without_uploading=not preload,

platformio/debug/process/base.py

@@ -53,7 +53,6 @@ class DebugSubprocessProtocol(asyncio.SubprocessProtocol):
 
 
 class DebugBaseProcess:
-
     STDOUT_CHUNK_SIZE = 2048
     LOG_FILE = None
 

platformio/debug/process/gdb.py

@@ -24,7 +24,6 @@ from platformio.debug.process.client import DebugClientProcess
 
 
 class GDBClientProcess(DebugClientProcess):
-
     PIO_SRC_NAME = ".pioinit"
     INIT_COMPLETED_BANNER = "PlatformIO: Initialization completed"
 

platformio/debug/process/server.py

@@ -26,7 +26,6 @@ from platformio.proc import where_is_program
 
 
 class DebugServerProcess(DebugBaseProcess):
-
     STD_BUFFER_SIZE = 1024
 
     def __init__(self, debug_config):

platformio/device/monitor/command.py

@@ -104,7 +104,7 @@ from platformio.project.options import ProjectOptions
     "-d",
     "--project-dir",
     default=os.getcwd,
-    type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True),
+    type=click.Path(exists=True, file_okay=False, dir_okay=True),
 )
 @click.option(
     "-e",
@@ -132,24 +132,24 @@ def device_monitor_cmd(**options):
             ensure_ready=True,
         ).find(initial_port=options["port"])
 
-    if options["menu_char"] == options["exit_char"]:
+        if options["menu_char"] == options["exit_char"]:
-        raise exception.UserSideException(
+            raise exception.UserSideException(
-            "--exit-char can not be the same as --menu-char"
+                "--exit-char can not be the same as --menu-char"
-        )
-
-    # check for unknown filters
-    if options["filters"]:
-        known_filters = set(get_available_filters())
-        unknown_filters = set(options["filters"]) - known_filters
-        if unknown_filters:
-            options["filters"] = list(known_filters & set(options["filters"]))
-            click.secho(
-                ("Warning! Skipping unknown filters `%s`. Known filters are `%s`")
-                % (", ".join(unknown_filters), ", ".join(sorted(known_filters))),
-                fg="yellow",
             )
 
-    start_terminal(options)
+        # check for unknown filters
+        if options["filters"]:
+            known_filters = set(get_available_filters())
+            unknown_filters = set(options["filters"]) - known_filters
+            if unknown_filters:
+                options["filters"] = list(known_filters & set(options["filters"]))
+                click.secho(
+                    ("Warning! Skipping unknown filters `%s`. Known filters are `%s`")
+                    % (", ".join(unknown_filters), ", ".join(sorted(known_filters))),
+                    fg="yellow",
+                )
+
+        start_terminal(options)
 
 
 def get_project_options(environment=None):

platformio/device/monitor/filters/log2file.py

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 import io
-import os.path
+import os
 from datetime import datetime
 
 from platformio.device.monitor.filters.base import DeviceMonitorFilterBase
@@ -27,8 +27,10 @@ class LogToFile(DeviceMonitorFilterBase):
         self._log_fp = None
 
     def __call__(self):
-        log_file_name = "platformio-device-monitor-%s.log" % datetime.now().strftime(
+        if not os.path.isdir("logs"):
-            "%y%m%d-%H%M%S"
+            os.makedirs("logs")
+        log_file_name = os.path.join(
+            "logs", "device-monitor-%s.log" % datetime.now().strftime("%y%m%d-%H%M%S")
         )
         print("--- Logging an output to %s" % os.path.abspath(log_file_name))
         # pylint: disable=consider-using-with

platformio/device/monitor/terminal.py

@@ -144,9 +144,8 @@ def new_serial_instance(options):  # pylint: disable=too-many-branches
             except KeyboardInterrupt as exc:
                 click.echo("", err=True)
                 raise UserSideException("User aborted and port is not given") from exc
-            else:
+            if not port:
-                if not port:
+                raise UserSideException("Port is not given")
-                    raise UserSideException("Port is not given")
         try:
             serial_instance = serial.serial_for_url(
                 port,

platformio/exception.py

@@ -14,7 +14,6 @@
 
 
 class PlatformioException(Exception):
-
     MESSAGE = None
 
     def __str__(self):  # pragma: no cover
@@ -26,7 +25,6 @@ class PlatformioException(Exception):
 
 
 class ReturnErrorCode(PlatformioException):
-
     MESSAGE = "{0}"
 
 
@@ -35,7 +33,6 @@ class UserSideException(PlatformioException):
 
 
 class AbortedByUser(UserSideException):
-
     MESSAGE = "Aborted by user"
 
 
@@ -49,7 +46,6 @@ class InvalidUdevRules(UserSideException):
 
 
 class MissedUdevRules(InvalidUdevRules):
-
     MESSAGE = (
         "Warning! Please install `99-platformio-udev.rules`. \nMore details: "
         "https://docs.platformio.org/en/latest/core/installation/udev-rules.html"
@@ -57,7 +53,6 @@ class MissedUdevRules(InvalidUdevRules):
 
 
 class OutdatedUdevRules(InvalidUdevRules):
-
     MESSAGE = (
         "Warning! Your `{0}` are outdated. Please update or reinstall them."
         "\nMore details: "
@@ -71,32 +66,26 @@ class OutdatedUdevRules(InvalidUdevRules):
 
 
 class GetSerialPortsError(PlatformioException):
-
     MESSAGE = "No implementation for your platform ('{0}') available"
 
 
 class GetLatestVersionError(PlatformioException):
-
     MESSAGE = "Can not retrieve the latest PlatformIO version"
 
 
 class InvalidSettingName(UserSideException):
-
     MESSAGE = "Invalid setting with the name '{0}'"
 
 
 class InvalidSettingValue(UserSideException):
-
     MESSAGE = "Invalid value '{0}' for the setting '{1}'"
 
 
 class InvalidJSONFile(PlatformioException):
-
     MESSAGE = "Could not load broken JSON: {0}"
 
 
 class CIBuildEnvsEmpty(UserSideException):
-
     MESSAGE = (
         "Can't find PlatformIO build environments.\n"
         "Please specify `--board` or path to `platformio.ini` with "
@@ -104,18 +93,7 @@ class CIBuildEnvsEmpty(UserSideException):
     )
 
 
-class UpgradeError(PlatformioException):
-
-    MESSAGE = """{0}
-
-* Upgrade using `pip install -U platformio`
-* Try different installation/upgrading steps:
-  https://docs.platformio.org/page/installation.html
-"""
-
-
 class HomeDirPermissionsError(UserSideException):
-
     MESSAGE = (
         "The directory `{0}` or its parent directory is not owned by the "
         "current user and PlatformIO can not store configuration data.\n"
@@ -126,7 +104,6 @@ class HomeDirPermissionsError(UserSideException):
 
 
 class CygwinEnvDetected(PlatformioException):
-
     MESSAGE = (
         "PlatformIO does not work within Cygwin environment. "
         "Use native Terminal instead."

platformio/fs.py

@@ -24,7 +24,7 @@ import sys
 
 import click
 
-from platformio import exception, proc
+from platformio import exception
 from platformio.compat import IS_WINDOWS
 
 
@@ -181,7 +181,7 @@ def match_src_files(src_dir, src_filter=None, src_exts=None, followlinks=True):
     result = set()
     # correct fs directory separator
     src_filter = src_filter.replace("/", os.sep).replace("\\", os.sep)
-    for (action, pattern) in re.findall(r"(\+|\-)<([^>]+)>", src_filter):
+    for action, pattern in re.findall(r"(\+|\-)<([^>]+)>", src_filter):
         candidates = _find_candidates(pattern)
         if action == "+":
             result |= candidates
@@ -193,26 +193,7 @@ def match_src_files(src_dir, src_filter=None, src_exts=None, followlinks=True):
 def to_unix_path(path):
     if not IS_WINDOWS or not path:
         return path
-    return re.sub(r"[\\]+", "/", path)
+    return path.replace("\\", "/")
-
-
-def normalize_path(path):
-    path = os.path.abspath(path)
-    if not IS_WINDOWS or not path.startswith("\\\\"):
-        return path
-    try:
-        result = proc.exec_command(["net", "use"])
-        if result["returncode"] != 0:
-            return path
-        share_re = re.compile(r"\s([A-Z]\:)\s+(\\\\[^\s]+)")
-        for line in result["out"].split("\n"):
-            share = share_re.search(line)
-            if not share:
-                continue
-            path = path.replace(share.group(2), share.group(1))
-    except OSError:
-        pass
-    return path
 
 
 def expanduser(path):

platformio/home/cli.py

@@ -13,10 +13,11 @@
 # limitations under the License.
 
 import mimetypes
+import socket
 
 import click
 
-from platformio.home.helpers import is_port_used
+from platformio.compat import IS_WINDOWS
 from platformio.home.run import run_server
 from platformio.package.manager.core import get_core_package_dir
 
@@ -95,3 +96,23 @@ def cli(port, host, no_open, shutdown_timeout, session_id):
         shutdown_timeout=shutdown_timeout,
         home_url=home_url,
     )
+
+
+def is_port_used(host, port):
+    socket.setdefaulttimeout(1)
+    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    if IS_WINDOWS:
+        try:
+            s.bind((host, port))
+            s.close()
+            return False
+        except (OSError, socket.error):
+            pass
+    else:
+        try:
+            s.connect((host, port))
+            s.close()
+        except socket.error:
+            return False
+
+    return True

platformio/home/helpers.py

@@ -1,44 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import socket
-
-from platformio import util
-from platformio.compat import IS_WINDOWS
-from platformio.proc import where_is_program
-
-
-@util.memoized(expire="60s")
-def get_core_fullpath():
-    return where_is_program("platformio" + (".exe" if IS_WINDOWS else ""))
-
-
-def is_port_used(host, port):
-    socket.setdefaulttimeout(1)
-    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-    if IS_WINDOWS:
-        try:
-            s.bind((host, port))
-            s.close()
-            return False
-        except (OSError, socket.error):
-            pass
-    else:
-        try:
-            s.connect((host, port))
-            s.close()
-        except socket.error:
-            return False
-
-    return True

platformio/home/rpc/handlers/account.py

@@ -15,9 +15,10 @@
 from ajsonrpc.core import JSONRPC20DispatchException
 
 from platformio.account.client import AccountClient
+from platformio.home.rpc.handlers.base import BaseRPCHandler
 
 
-class AccountRPC:
+class AccountRPC(BaseRPCHandler):
     @staticmethod
     def call_client(method, *args, **kwargs):
         try:
@@ -25,5 +26,5 @@ class AccountRPC:
             return getattr(client, method)(*args, **kwargs)
         except Exception as exc:  # pylint: disable=bare-except
             raise JSONRPC20DispatchException(
-                code=4003, message="PIO Account Call Error", data=str(exc)
+                code=5000, message="PIO Account Call Error", data=str(exc)
             ) from exc

platformio/home/rpc/handlers/app.py

@@ -16,12 +16,12 @@ import os
 from pathlib import Path
 
 from platformio import __version__, app, fs, util
+from platformio.home.rpc.handlers.base import BaseRPCHandler
 from platformio.project.config import ProjectConfig
 from platformio.project.helpers import is_platformio_project
 
 
-class AppRPC:
+class AppRPC(BaseRPCHandler):
-
     IGNORE_STORAGE_KEYS = [
         "cid",
         "coreVersion",

platformio/home/rpc/handlers/base.py

@@ -0,0 +1,17 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+class BaseRPCHandler:
+    factory = None

platformio/home/rpc/handlers/ide.py

@@ -18,10 +18,10 @@ from pathlib import Path
 from ajsonrpc.core import JSONRPC20DispatchException
 
 from platformio.compat import aio_get_running_loop
+from platformio.home.rpc.handlers.base import BaseRPCHandler
 
 
-class IDERPC:
+class IDERPC(BaseRPCHandler):
-
     COMMAND_TIMEOUT = 1.5  # in seconds
 
     def __init__(self):
@@ -51,11 +51,12 @@ class IDERPC:
 
     def on_command_result(self, cmd_id, value):
         if cmd_id not in self._cmd_queue:
-            return
+            return False
         if self._cmd_queue[cmd_id]["method"] == "get_pio_project_dirs":
             value = [str(Path(p).resolve()) for p in value]
         self._cmd_queue[cmd_id]["future"].set_result(value)
         del self._cmd_queue[cmd_id]
+        return True
 
     def _process_commands(self):
         for cmd_id in list(self._cmd_queue):

platformio/home/rpc/handlers/misc.py

@@ -17,10 +17,11 @@ import time
 
 from platformio.cache import ContentCache
 from platformio.compat import aio_create_task
+from platformio.home.rpc.handlers.base import BaseRPCHandler
 from platformio.home.rpc.handlers.os import OSRPC
 
 
-class MiscRPC:
+class MiscRPC(BaseRPCHandler):
     async def load_latest_tweets(self, data_url):
         cache_key = ContentCache.key_from_args(data_url, "tweets")
         cache_valid = "180d"

platformio/home/rpc/handlers/os.py

@@ -24,6 +24,7 @@ from starlette.concurrency import run_in_threadpool
 from platformio import fs
 from platformio.cache import ContentCache
 from platformio.device.list.util import list_logical_devices
+from platformio.home.rpc.handlers.base import BaseRPCHandler
 from platformio.http import HTTPSession, ensure_internet_on
 
 
@@ -35,7 +36,7 @@ class HTTPAsyncSession(HTTPSession):
         return await run_in_threadpool(func, *args, **kwargs)
 
 
-class OSRPC:
+class OSRPC(BaseRPCHandler):
     @staticmethod
     async def fetch_content(url, data=None, headers=None, cache_valid=None):
         if not headers:
@@ -89,6 +90,14 @@ class OSRPC:
     def open_file(path):
         return click.launch(path)
 
+    @staticmethod
+    def call_path_module_func(name, args, **kwargs):
+        return getattr(os.path, name)(*args, **kwargs)
+
+    @staticmethod
+    def get_path_separator():
+        return os.sep
+
     @staticmethod
     def is_file(path):
         return os.path.isfile(path)
@@ -156,9 +165,4 @@ class OSRPC:
 
     @staticmethod
     def get_logical_devices():
-        items = []
+        return list_logical_devices()
-        for item in list_logical_devices():
-            if item["name"]:
-                item["name"] = item["name"]
-            items.append(item)
-        return items

platformio/home/rpc/handlers/piocore.py

@@ -12,6 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import asyncio
+import functools
 import io
 import json
 import os
@@ -22,9 +24,45 @@ import click
 from ajsonrpc.core import JSONRPC20DispatchException
 from starlette.concurrency import run_in_threadpool
 
-from platformio import __main__, __version__, fs, proc
+from platformio import __main__, __version__, app, fs, proc, util
-from platformio.compat import get_locale_encoding, is_bytes
+from platformio.compat import (
-from platformio.home import helpers
+    IS_WINDOWS,
+    aio_create_task,
+    aio_get_running_loop,
+    get_locale_encoding,
+    is_bytes,
+)
+from platformio.exception import PlatformioException
+from platformio.home.rpc.handlers.base import BaseRPCHandler
+
+
+class PIOCoreProtocol(asyncio.SubprocessProtocol):
+    def __init__(self, exit_future, on_data_callback=None):
+        self.exit_future = exit_future
+        self.on_data_callback = on_data_callback
+        self.stdout = ""
+        self.stderr = ""
+        self._is_exited = False
+        self._encoding = get_locale_encoding()
+
+    def pipe_data_received(self, fd, data):
+        data = data.decode(self._encoding, "replace")
+        pipe = ["stdin", "stdout", "stderr"][fd]
+        if pipe == "stdout":
+            self.stdout += data
+        if pipe == "stderr":
+            self.stderr += data
+        if self.on_data_callback:
+            self.on_data_callback(pipe=pipe, data=data)
+
+    def connection_lost(self, exc):
+        self.process_exited()
+
+    def process_exited(self):
+        if self._is_exited:
+            return
+        self.exit_future.set_result(True)
+        self._is_exited = True
 
 
 class MultiThreadingStdStream:
@@ -58,11 +96,51 @@ class MultiThreadingStdStream:
         return result
 
 
-class PIOCoreRPC:
+@util.memoized(expire="60s")
+def get_core_fullpath():
+    return proc.where_is_program("platformio" + (".exe" if IS_WINDOWS else ""))
+
+
+class PIOCoreRPC(BaseRPCHandler):
     @staticmethod
     def version():
         return __version__
 
+    async def exec(self, args, options=None):
+        loop = aio_get_running_loop()
+        exit_future = loop.create_future()
+        data_callback = functools.partial(
+            self._on_exec_data_received, exec_options=options
+        )
+        if args[0] != "--caller" and app.get_session_var("caller_id"):
+            args = ["--caller", app.get_session_var("caller_id")] + args
+        transport, protocol = await loop.subprocess_exec(
+            lambda: PIOCoreProtocol(exit_future, data_callback),
+            get_core_fullpath(),
+            *args,
+            stdin=None,
+            **options.get("spawn", {}),
+        )
+        await exit_future
+        transport.close()
+        return {
+            "stdout": protocol.stdout,
+            "stderr": protocol.stderr,
+            "returncode": transport.get_returncode(),
+        }
+
+    def _on_exec_data_received(self, exec_options, pipe, data):
+        notification_method = exec_options.get(f"{pipe}NotificationMethod")
+        if not notification_method:
+            return
+        aio_create_task(
+            self.factory.notify_clients(
+                method=notification_method,
+                params=[data],
+                actor="frontend",
+            )
+        )
+
     @staticmethod
     def setup_multithreading_std_streams():
         if isinstance(sys.stdout, MultiThreadingStdStream):
@@ -94,14 +172,14 @@ class PIOCoreRPC:
                 return PIOCoreRPC._process_result(result, to_json)
         except Exception as exc:  # pylint: disable=bare-except
             raise JSONRPC20DispatchException(
-                code=4003, message="PIO Core Call Error", data=str(exc)
+                code=5000, message="PIO Core Call Error", data=str(exc)
             ) from exc
 
     @staticmethod
     async def _call_subprocess(args, options):
         result = await run_in_threadpool(
             proc.exec_command,
-            [helpers.get_core_fullpath()] + args,
+            [get_core_fullpath()] + args,
             cwd=options.get("cwd") or os.getcwd(),
         )
         return (result["out"], result["err"], result["returncode"])
@@ -132,7 +210,7 @@ class PIOCoreRPC:
             err = err.decode(get_locale_encoding())
         text = ("%s\n\n%s" % (out, err)).strip()
         if code != 0:
-            raise Exception(text)
+            raise PlatformioException(text)
         if not to_json:
             return text
         try:

platformio/home/rpc/handlers/platform.py

@@ -0,0 +1,61 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from platformio.home.rpc.handlers.base import BaseRPCHandler
+from platformio.package.manager.platform import PlatformPackageManager
+from platformio.platform.factory import PlatformFactory
+
+
+class PlatformRPC(BaseRPCHandler):
+    @staticmethod
+    def list_installed(options=None):
+        result = []
+        options = options or {}
+
+        def _matchSearchQuery(p):
+            searchQuery = options.get("searchQuery")
+            if not searchQuery:
+                return True
+            content_blocks = [p.name, p.title, p.description]
+            if p.frameworks:
+                content_blocks.append(" ".join(p.frameworks.keys()))
+            for board in p.get_boards().values():
+                board_data = board.get_brief_data()
+                for key in ("id", "mcu", "vendor"):
+                    content_blocks.append(board_data.get(key))
+            return searchQuery.strip() in " ".join(content_blocks)
+
+        pm = PlatformPackageManager()
+        for pkg in pm.get_installed():
+            p = PlatformFactory.new(pkg)
+            if not _matchSearchQuery(p):
+                continue
+            result.append(
+                dict(
+                    __pkg_path=pkg.path,
+                    __pkg_meta=pkg.metadata.as_dict(),
+                    name=p.name,
+                    title=p.title,
+                    description=p.description,
+                )
+            )
+        return result
+
+    @staticmethod
+    def get_boards(spec):
+        p = PlatformFactory.new(spec)
+        return sorted(
+            [b.get_brief_data() for b in p.get_boards().values()],
+            key=lambda item: item["name"],
+        )

platformio/home/rpc/handlers/project.py

@@ -16,10 +16,12 @@ import os
 import shutil
 import time
 
+import semantic_version
 from ajsonrpc.core import JSONRPC20DispatchException
 
-from platformio import exception, fs
+from platformio import app, exception, fs
 from platformio.home.rpc.handlers.app import AppRPC
+from platformio.home.rpc.handlers.base import BaseRPCHandler
 from platformio.home.rpc.handlers.piocore import PIOCoreRPC
 from platformio.package.manager.platform import PlatformPackageManager
 from platformio.project.config import ProjectConfig
@@ -29,7 +31,7 @@ from platformio.project.integration.generator import ProjectGenerator
 from platformio.project.options import get_config_options_schema
 
 
-class ProjectRPC:
+class ProjectRPC(BaseRPCHandler):
     @staticmethod
     def config_call(init_kwargs, method, *args):
         assert isinstance(init_kwargs, dict)
@@ -184,83 +186,17 @@ class ProjectRPC:
 
     async def init(self, board, framework, project_dir):
         assert project_dir
-        state = AppRPC.load_state()
         if not os.path.isdir(project_dir):
             os.makedirs(project_dir)
-        args = ["init", "--board", board]
+        args = ["init", "--board", board, "--sample-code"]
         if framework:
             args.extend(["--project-option", "framework = %s" % framework])
-        if (
+        ide = app.get_session_var("caller_id")
-            state["storage"]["coreCaller"]
+        if ide in ProjectGenerator.get_supported_ides():
-            and state["storage"]["coreCaller"] in ProjectGenerator.get_supported_ides()
+            args.extend(["--ide", ide])
-        ):
-            args.extend(["--ide", state["storage"]["coreCaller"]])
         await PIOCoreRPC.call(
             args, options={"cwd": project_dir, "force_subprocess": True}
         )
-        return self._generate_project_main(project_dir, board, framework)
-
-    @staticmethod
-    def _generate_project_main(project_dir, board, framework):
-        main_content = None
-        if framework == "arduino":
-            main_content = "\n".join(
-                [
-                    "#include <Arduino.h>",
-                    "",
-                    "void setup() {",
-                    "  // put your setup code here, to run once:",
-                    "}",
-                    "",
-                    "void loop() {",
-                    "  // put your main code here, to run repeatedly:",
-                    "}",
-                    "",
-                ]
-            )
-        elif framework == "mbed":
-            main_content = "\n".join(
-                [
-                    "#include <mbed.h>",
-                    "",
-                    "int main() {",
-                    "",
-                    "  // put your setup code here, to run once:",
-                    "",
-                    "  while(1) {",
-                    "    // put your main code here, to run repeatedly:",
-                    "  }",
-                    "}",
-                    "",
-                ]
-            )
-        if not main_content:
-            return project_dir
-
-        is_cpp_project = True
-        pm = PlatformPackageManager()
-        try:
-            board = pm.board_config(board)
-            platforms = board.get("platforms", board.get("platform"))
-            if not isinstance(platforms, list):
-                platforms = [platforms]
-            c_based_platforms = ["intel_mcs51", "ststm8"]
-            is_cpp_project = not set(platforms) & set(c_based_platforms)
-        except exception.PlatformioException:
-            pass
-
-        with fs.cd(project_dir):
-            config = ProjectConfig()
-            src_dir = config.get("platformio", "src_dir")
-            main_path = os.path.join(
-                src_dir, "main.%s" % ("cpp" if is_cpp_project else "c")
-            )
-            if os.path.isfile(main_path):
-                return project_dir
-            if not os.path.isdir(src_dir):
-                os.makedirs(src_dir)
-            with open(main_path, mode="w", encoding="utf8") as fp:
-                fp.write(main_content.strip())
         return project_dir
 
     @staticmethod
@@ -296,11 +232,9 @@ class ProjectRPC:
             args.extend(
                 ["--project-option", "lib_extra_dirs = ~/Documents/Arduino/libraries"]
             )
-        if (
+        ide = app.get_session_var("caller_id")
-            state["storage"]["coreCaller"]
+        if ide in ProjectGenerator.get_supported_ides():
-            and state["storage"]["coreCaller"] in ProjectGenerator.get_supported_ides()
+            args.extend(["--ide", ide])
-        ):
-            args.extend(["--ide", state["storage"]["coreCaller"]])
         await PIOCoreRPC.call(
             args, options={"cwd": project_dir, "force_subprocess": True}
         )
@@ -324,14 +258,50 @@ class ProjectRPC:
         )
         shutil.copytree(project_dir, new_project_dir, symlinks=True)
 
-        state = AppRPC.load_state()
         args = ["init"]
-        if (
+        ide = app.get_session_var("caller_id")
-            state["storage"]["coreCaller"]
+        if ide in ProjectGenerator.get_supported_ides():
-            and state["storage"]["coreCaller"] in ProjectGenerator.get_supported_ides()
+            args.extend(["--ide", ide])
-        ):
-            args.extend(["--ide", state["storage"]["coreCaller"]])
         await PIOCoreRPC.call(
             args, options={"cwd": new_project_dir, "force_subprocess": True}
         )
         return new_project_dir
+
+    async def create_empty(self, configuration, options=None):
+        project_dir = os.path.join(configuration["location"], configuration["name"])
+        if not os.path.isdir(project_dir):
+            os.makedirs(project_dir)
+
+        project_options = []
+        platform = configuration["platform"]
+        board = configuration.get("board", {}).get("id")
+        env_name = board or platform["name"]
+        if configuration.get("description"):
+            project_options.append(("description", configuration.get("description")))
+        try:
+            v = semantic_version.Version(platform.get("version"))
+            assert not v.prerelease
+            project_options.append(
+                ("platform", "{name} @ ^{version}".format(**platform))
+            )
+        except (AssertionError, ValueError):
+            project_options.append(
+                ("platform", "{name} @ {version}".format(**platform))
+            )
+        if board:
+            project_options.append(("board", board))
+        if configuration.get("framework"):
+            project_options.append(("framework", configuration["framework"]["name"]))
+
+        args = ["project", "init", "-e", env_name, "--sample-code"]
+        ide = app.get_session_var("caller_id")
+        if ide in ProjectGenerator.get_supported_ides():
+            args.extend(["--ide", ide])
+        for name, value in project_options:
+            args.extend(["-O", f"{name}={value}"])
+
+        envclone = os.environ.copy()
+        envclone["PLATFORMIO_FORCE_ANSI"] = "true"
+        options = options or {}
+        options["spawn"] = {"env": envclone, "cwd": project_dir}
+        return await self.factory.manager.dispatcher["core.exec"](args, options=options)

platformio/home/rpc/handlers/registry.py

@@ -13,17 +13,19 @@
 # limitations under the License.
 
 from ajsonrpc.core import JSONRPC20DispatchException
+from starlette.concurrency import run_in_threadpool
 
+from platformio.home.rpc.handlers.base import BaseRPCHandler
 from platformio.registry.client import RegistryClient
 
 
-class RegistryRPC:
+class RegistryRPC(BaseRPCHandler):
     @staticmethod
-    def call_client(method, *args, **kwargs):
+    async def call_client(method, *args, **kwargs):
         try:
             client = RegistryClient()
-            return getattr(client, method)(*args, **kwargs)
+            return await run_in_threadpool(getattr(client, method), *args, **kwargs)
         except Exception as exc:  # pylint: disable=bare-except
             raise JSONRPC20DispatchException(
-                code=4003, message="Registry Call Error", data=str(exc)
+                code=5000, message="Registry Call Error", data=str(exc)
             ) from exc

platformio/home/rpc/server.py

@@ -12,17 +12,20 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from urllib.parse import parse_qs
+
 import click
+from ajsonrpc.core import JSONRPC20Error, JSONRPC20Request
 from ajsonrpc.dispatcher import Dispatcher
-from ajsonrpc.manager import AsyncJSONRPCResponseManager
+from ajsonrpc.manager import AsyncJSONRPCResponseManager, JSONRPC20Response
 from starlette.endpoints import WebSocketEndpoint
 
 from platformio.compat import aio_create_task, aio_get_running_loop
+from platformio.http import InternetConnectionError
 from platformio.proc import force_exit
 
 
 class JSONRPCServerFactoryBase:
-
     connection_nums = 0
     shutdown_timer = None
 
@@ -31,20 +34,25 @@ class JSONRPCServerFactoryBase:
         self.manager = AsyncJSONRPCResponseManager(
             Dispatcher(), is_server_error_verbose=True
         )
+        self._clients = {}
 
     def __call__(self, *args, **kwargs):
         raise NotImplementedError
 
     def add_object_handler(self, handler, namespace):
+        handler.factory = self
         self.manager.dispatcher.add_object(handler, prefix="%s." % namespace)
 
-    def on_client_connect(self):
+    def on_client_connect(self, connection, actor=None):
+        self._clients[connection] = {"actor": actor}
         self.connection_nums += 1
         if self.shutdown_timer:
             self.shutdown_timer.cancel()
             self.shutdown_timer = None
 
-    def on_client_disconnect(self):
+    def on_client_disconnect(self, connection):
+        if connection in self._clients:
+            del self._clients[connection]
         self.connection_nums -= 1
         if self.connection_nums < 1:
             self.connection_nums = 0
@@ -67,6 +75,14 @@ class JSONRPCServerFactoryBase:
             self.shutdown_timeout, _auto_shutdown_server
         )
 
+    async def notify_clients(self, method, params=None, actor=None):
+        for client, options in self._clients.items():
+            if actor and options["actor"] != actor:
+                continue
+            request = JSONRPC20Request(method, params, is_notification=True)
+            await client.send_text(self.manager.serialize(request.body))
+        return True
+
 
 class WebSocketJSONRPCServerFactory(JSONRPCServerFactoryBase):
     def __call__(self, *args, **kwargs):
@@ -81,17 +97,30 @@ class WebSocketJSONRPCServer(WebSocketEndpoint):
 
     async def on_connect(self, websocket):
         await websocket.accept()
-        self.factory.on_client_connect()  # pylint: disable=no-member
+        qs = parse_qs(self.scope.get("query_string", b""))
+        actors = qs.get(b"actor")
+        self.factory.on_client_connect(  # pylint: disable=no-member
+            websocket, actor=actors[0].decode() if actors else None
+        )
 
     async def on_receive(self, websocket, data):
         aio_create_task(self._handle_rpc(websocket, data))
 
     async def on_disconnect(self, websocket, close_code):
-        self.factory.on_client_disconnect()  # pylint: disable=no-member
+        self.factory.on_client_disconnect(websocket)  # pylint: disable=no-member
 
     async def _handle_rpc(self, websocket, data):
         # pylint: disable=no-member
         response = await self.factory.manager.get_response_for_payload(data)
         if response.error and response.error.data:
             click.secho("Error: %s" % response.error.data, fg="red", err=True)
+            if InternetConnectionError.MESSAGE in response.error.data:
+                response = JSONRPC20Response(
+                    id=response.id,
+                    error=JSONRPC20Error(
+                        code=4008,
+                        message="No Internet Connection",
+                        data=response.error.data,
+                    ),
+                )
         await websocket.send_text(self.factory.manager.serialize(response.body))

platformio/home/run.py

@@ -32,6 +32,7 @@ from platformio.home.rpc.handlers.ide import IDERPC
 from platformio.home.rpc.handlers.misc import MiscRPC
 from platformio.home.rpc.handlers.os import OSRPC
 from platformio.home.rpc.handlers.piocore import PIOCoreRPC
+from platformio.home.rpc.handlers.platform import PlatformRPC
 from platformio.home.rpc.handlers.project import ProjectRPC
 from platformio.home.rpc.handlers.registry import RegistryRPC
 from platformio.home.rpc.server import WebSocketJSONRPCServerFactory
@@ -44,7 +45,7 @@ class ShutdownMiddleware:
         self.app = app
 
     async def __call__(self, scope, receive, send):
-        if scope["type"] == "http" and b"__shutdown__" in scope.get("query_string", {}):
+        if scope["type"] == "http" and b"__shutdown__" in scope.get("query_string", ""):
             await shutdown_server()
         await self.app(scope, receive, send)
 
@@ -73,6 +74,7 @@ def run_server(host, port, no_open, shutdown_timeout, home_url):
     ws_rpc_factory.add_object_handler(OSRPC(), namespace="os")
     ws_rpc_factory.add_object_handler(PIOCoreRPC(), namespace="core")
     ws_rpc_factory.add_object_handler(ProjectRPC(), namespace="project")
+    ws_rpc_factory.add_object_handler(PlatformRPC(), namespace="platform")
     ws_rpc_factory.add_object_handler(RegistryRPC(), namespace="registry")
 
     path = urlparse(home_url).path

platformio/http.py

@@ -37,8 +37,7 @@ class HTTPClientError(PlatformioException):
         return self.message
 
 
-class InternetIsOffline(UserSideException):
+class InternetConnectionError(UserSideException):
-
     MESSAGE = (
         "You are not connected to the Internet.\n"
         "PlatformIO needs the Internet connection to"
@@ -204,7 +203,7 @@ def _internet_on():
 def ensure_internet_on(raise_exception=False):
     result = _internet_on()
     if raise_exception and not result:
-        raise InternetIsOffline()
+        raise InternetConnectionError()
     return result
 
 

platformio/maintenance.py

@@ -22,9 +22,8 @@ import semantic_version
 from platformio import __version__, app, exception, fs, telemetry
 from platformio.cache import cleanup_content_cache
 from platformio.cli import PlatformioCLI
-from platformio.commands.platform import platform_update as cmd_platform_update
 from platformio.commands.upgrade import get_latest_version
-from platformio.http import HTTPClientError, InternetIsOffline, ensure_internet_on
+from platformio.http import HTTPClientError, InternetConnectionError, ensure_internet_on
 from platformio.package.manager.core import update_core_packages
 from platformio.package.manager.tool import ToolPackageManager
 from platformio.package.meta import PackageSpec
@@ -51,7 +50,7 @@ def on_platformio_end(ctx, result):  # pylint: disable=unused-argument
         check_prune_system()
     except (
         HTTPClientError,
-        InternetIsOffline,
+        InternetConnectionError,
         exception.GetLatestVersionError,
     ):
         click.secho(
@@ -67,15 +66,15 @@ def on_platformio_exception(e):
 
 def set_caller(caller=None):
     caller = caller or os.getenv("PLATFORMIO_CALLER")
+    if not caller:
+        if os.getenv("CODESPACES"):
+            caller = "codespaces"
+        elif os.getenv("VSCODE_PID") or os.getenv("VSCODE_NLS_CONFIG"):
+            caller = "vscode"
+        elif os.getenv("GITPOD_WORKSPACE_ID") or os.getenv("GITPOD_WORKSPACE_URL"):
+            caller = "gitpod"
     if caller:
-        return app.set_session_var("caller_id", caller)
+        app.set_session_var("caller_id", caller)
-    if os.getenv("CODESPACES"):
-        caller = "codespaces"
-    elif os.getenv("VSCODE_PID") or os.getenv("VSCODE_NLS_CONFIG"):
-        caller = "vscode"
-    elif os.getenv("GITPOD_WORKSPACE_ID") or os.getenv("GITPOD_WORKSPACE_URL"):
-        caller = "gitpod"
-    return app.set_session_var("caller_id", caller)
 
 
 class Upgrader:
@@ -84,7 +83,6 @@ class Upgrader:
         self.to_version = pepver_to_semver(to_version)
 
         self._upgraders = [
-            (semantic_version.Version("3.5.0-a.2"), self._update_dev_platforms),
             (semantic_version.Version("4.4.0-a.8"), self._update_pkg_metadata),
         ]
 
@@ -100,11 +98,6 @@ class Upgrader:
 
         return all(result)
 
-    @staticmethod
-    def _update_dev_platforms(ctx):
-        ctx.invoke(cmd_platform_update)
-        return True
-
     @staticmethod
     def _update_pkg_metadata(_):
         pm = ToolPackageManager()
@@ -166,8 +159,6 @@ def after_upgrade(ctx):
                 action="Upgrade",
                 label="%s > %s" % (last_version, __version__),
             )
-        else:
-            raise exception.UpgradeError("Auto upgrading...")
 
     # PlatformIO banner
     click.echo("*" * terminal_width)

platformio/package/commands/install.py

@@ -39,7 +39,7 @@ from platformio.test.runners.factory import TestRunnerFactory
     "-d",
     "--project-dir",
     default=os.getcwd,
-    type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True),
+    type=click.Path(exists=True, file_okay=False, dir_okay=True),
 )
 @click.option("-e", "--environment", "environments", multiple=True)
 @click.option("-p", "--platform", "platforms", metavar="SPECIFICATION", multiple=True)
@@ -55,7 +55,7 @@ from platformio.test.runners.factory import TestRunnerFactory
 @click.option(
     "--storage-dir",
     default=None,
-    type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True),
+    type=click.Path(exists=True, file_okay=False, dir_okay=True),
     help="Custom Package Manager storage for global packages",
 )
 @click.option("-f", "--force", is_flag=True, help="Reinstall package if it exists")

platformio/package/commands/list.py

@@ -31,7 +31,7 @@ from platformio.project.config import ProjectConfig
     "-d",
     "--project-dir",
     default=os.getcwd,
-    type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True),
+    type=click.Path(exists=True, file_okay=False, dir_okay=True),
 )
 @click.option("-e", "--environment", "environments", multiple=True)
 @click.option("-p", "--platform", "platforms", metavar="SPECIFICATION", multiple=True)
@@ -41,7 +41,7 @@ from platformio.project.config import ProjectConfig
 @click.option(
     "--storage-dir",
     default=None,
-    type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True),
+    type=click.Path(exists=True, file_okay=False, dir_okay=True),
     help="Custom Package Manager storage for global packages",
 )
 @click.option("--only-platforms", is_flag=True, help="List only platform packages")
@@ -137,7 +137,7 @@ def list_global_packages(options):
     only_packages = any(
         options.get(type_) or options.get(f"only_{type_}") for (type_, _) in data
     )
-    for (type_, pm) in data:
+    for type_, pm in data:
         skip_conds = [
             only_packages
             and not options.get(type_)

platformio/package/commands/outdated.py

@@ -58,7 +58,7 @@ class OutdatedCandidate:
     "-d",
     "--project-dir",
     default=os.getcwd,
-    type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True),
+    type=click.Path(exists=True, file_okay=False, dir_okay=True),
 )
 @click.option("-e", "--environment", "environments", multiple=True)
 def package_outdated_cmd(project_dir, environments):

platformio/package/commands/pack.py

@@ -26,7 +26,7 @@ from platformio.package.pack import PackagePacker
     "package",
     default=os.getcwd,
     metavar="<source directory, tar.gz or zip>",
-    type=click.Path(exists=True, file_okay=True, dir_okay=True, resolve_path=True),
+    type=click.Path(exists=True, file_okay=True, dir_okay=True),
 )
 @click.option(
     "-o", "--output", help="A destination path (folder or a full path to file)"

platformio/package/commands/publish.py

@@ -47,7 +47,7 @@ def validate_datetime(ctx, param, value):  # pylint: disable=unused-argument
     "package",
     default=os.getcwd,
     metavar="<source directory, tar.gz or zip>",
-    type=click.Path(exists=True, file_okay=True, dir_okay=True, resolve_path=True),
+    type=click.Path(exists=True, file_okay=True, dir_okay=True),
 )
 @click.option(
     "--owner",

platformio/package/commands/uninstall.py

@@ -33,7 +33,7 @@ from platformio.project.savedeps import pkg_to_save_spec, save_project_dependenc
     "-d",
     "--project-dir",
     default=os.getcwd,
-    type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True),
+    type=click.Path(exists=True, file_okay=False, dir_okay=True),
 )
 @click.option("-e", "--environment", "environments", multiple=True)
 @click.option("-p", "--platform", "platforms", metavar="SPECIFICATION", multiple=True)
@@ -49,7 +49,7 @@ from platformio.project.savedeps import pkg_to_save_spec, save_project_dependenc
 @click.option(
     "--storage-dir",
     default=None,
-    type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True),
+    type=click.Path(exists=True, file_okay=False, dir_okay=True),
     help="Custom Package Manager storage for global packages",
 )
 @click.option("-s", "--silent", is_flag=True, help="Suppress progress reporting")

platformio/package/commands/update.py

@@ -33,7 +33,7 @@ from platformio.project.savedeps import pkg_to_save_spec, save_project_dependenc
     "-d",
     "--project-dir",
     default=os.getcwd,
-    type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True),
+    type=click.Path(exists=True, file_okay=False, dir_okay=True),
 )
 @click.option("-e", "--environment", "environments", multiple=True)
 @click.option("-p", "--platform", "platforms", metavar="SPECIFICATION", multiple=True)
@@ -49,7 +49,7 @@ from platformio.project.savedeps import pkg_to_save_spec, save_project_dependenc
 @click.option(
     "--storage-dir",
     default=None,
-    type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True),
+    type=click.Path(exists=True, file_okay=False, dir_okay=True),
     help="Custom Package Manager storage for global packages",
 )
 @click.option("-s", "--silent", is_flag=True, help="Suppress progress reporting")

platformio/package/download.py

@@ -105,7 +105,7 @@ class FileDownloader:
                         label=label,
                         update_min_steps=min(
                             256 * 1024, file_size / 100
-                        ),  # every 256Kb or less,
+                        ),  # every 256Kb or less
                     ) as pb:
                         for chunk in pb:
                             pb.update(len(chunk))

platformio/package/exception.py

@@ -48,12 +48,10 @@ class ManifestValidationError(ManifestException):
 
 
 class MissingPackageManifestError(ManifestException):
-
     MESSAGE = "Could not find one of '{0}' manifest files in the package"
 
 
 class UnknownPackageError(UserSideException):
-
     MESSAGE = (
         "Could not find the package with '{0}' requirements for your system '%s'"
         % util.get_systype()
@@ -61,7 +59,6 @@ class UnknownPackageError(UserSideException):
 
 
 class NotGlobalLibDir(UserSideException):
-
     MESSAGE = (
         "The `{0}` is not a PlatformIO project.\n\n"
         "To manage libraries in global storage `{1}`,\n"

platformio/package/manager/_download.py

@@ -26,7 +26,6 @@ from platformio.package.lockfile import LockFile
 
 
 class PackageManagerDownloadMixin:
-
     DOWNLOAD_CACHE_EXPIRE = 86400 * 30  # keep package in a local cache for 1 month
 
     def compute_download_path(self, *args):
@@ -62,7 +61,7 @@ class PackageManagerDownloadMixin:
             self.set_download_utime(dl_path)
             return dl_path
 
-        with_progress = not silent and not app.is_disabled_progressbar()
+        with_progress = not app.is_disabled_progressbar()
         tmp_fd, tmp_path = tempfile.mkstemp(dir=self.get_download_dir())
         try:
             with LockFile(dl_path):
@@ -71,7 +70,7 @@ class PackageManagerDownloadMixin:
                     fd.set_destination(tmp_path)
                     fd.start(with_progress=with_progress, silent=silent)
                 except IOError as exc:
-                    raise_error = not with_progress
+                    raise_error = not silent
                     if with_progress:
                         try:
                             fd = FileDownloader(url)

platformio/package/manager/_install.py

@@ -27,7 +27,6 @@ from platformio.package.vcsclient import VCSClientFactory
 
 
 class PackageManagerInstallMixin:
-
     _INSTALL_HISTORY = None  # avoid circle dependencies
 
     @staticmethod

platformio/package/manager/core.py

@@ -12,19 +12,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import json
 import os
-import shutil
-import subprocess
-import sys
-from datetime import date
 
-from platformio import __core_packages__, exception, fs, util
+from platformio import __core_packages__, exception
-from platformio.exception import UserSideException
 from platformio.package.exception import UnknownPackageError
 from platformio.package.manager.tool import ToolPackageManager
-from platformio.package.meta import PackageItem, PackageSpec
+from platformio.package.meta import PackageSpec
-from platformio.proc import get_pythonexe_path
 
 
 def get_installed_core_packages():
@@ -98,131 +91,3 @@ def remove_unnecessary_core_packages(dry_run=False):
         pm.uninstall(pkg)
 
     return candidates
-
-
-def inject_contrib_pysite():
-    # pylint: disable=import-outside-toplevel
-    from site import addsitedir
-
-    try:
-        contrib_pysite_dir = get_core_package_dir("contrib-pysite")
-    except UnknownPackageError:
-        pm = ToolPackageManager()
-        contrib_pysite_dir = build_contrib_pysite_package(
-            os.path.join(pm.package_dir, "contrib-pysite")
-        )
-
-    if contrib_pysite_dir in sys.path:
-        return True
-
-    addsitedir(contrib_pysite_dir)
-    sys.path.insert(0, contrib_pysite_dir)
-
-    try:
-        # pylint: disable=import-error,unused-import,unused-variable
-        from OpenSSL import SSL
-
-    except:  # pylint: disable=bare-except
-        build_contrib_pysite_package(contrib_pysite_dir)
-
-    return True
-
-
-def build_contrib_pysite_package(target_dir, with_metadata=True):
-    systype = util.get_systype()
-    if os.path.isdir(target_dir):
-        fs.rmtree(target_dir)
-    os.makedirs(target_dir)
-
-    # issue 3865: There is no "rustup" in "Raspbian GNU/Linux 10 (buster)"
-    os.environ["CRYPTOGRAPHY_DONT_BUILD_RUST"] = "1"
-
-    # build dependencies
-    args = [
-        get_pythonexe_path(),
-        "-m",
-        "pip",
-        "install",
-        "--no-compile",
-        "-t",
-        target_dir,
-    ]
-    if "linux" in systype:
-        args.extend(["--no-binary", ":all:"])
-    try:
-        subprocess.run(args + get_contrib_pysite_deps(), check=True, env=os.environ)
-    except subprocess.CalledProcessError as exc:
-        if "linux" in systype:
-            raise UserSideException(
-                "\n\nPlease ensure that the next packages are installed:\n\n"
-                "sudo apt install python3-dev libffi-dev libssl-dev\n"
-            ) from exc
-        raise exc
-
-    # build manifests
-    with open(
-        os.path.join(target_dir, "package.json"), mode="w", encoding="utf8"
-    ) as fp:
-        json.dump(
-            dict(
-                name="contrib-pysite",
-                version="2.%d%d.%s"
-                % (
-                    sys.version_info.major,
-                    sys.version_info.minor,
-                    date.today().strftime("%y%m%d"),
-                ),
-                system=list(
-                    set([systype, "linux_armv6l", "linux_armv7l", "linux_armv8l"])
-                )
-                if systype.startswith("linux_arm")
-                else systype,
-                description="Extra Python package for PlatformIO Core",
-                keywords=["platformio", "platformio-core"],
-                homepage="https://docs.platformio.org/page/core/index.html",
-                repository={
-                    "type": "git",
-                    "url": "https://github.com/platformio/platformio-core",
-                },
-            ),
-            fp,
-        )
-
-    # generate package metadata
-    if with_metadata:
-        pm = ToolPackageManager()
-        pkg = PackageItem(target_dir)
-        pkg.metadata = pm.build_metadata(
-            target_dir, PackageSpec(owner="platformio", name="contrib-pysite")
-        )
-        pkg.dump_meta()
-
-    # remove unused files
-    for root, dirs, files in os.walk(target_dir):
-        for t in ("_test", "test", "tests"):
-            if t in dirs:
-                shutil.rmtree(os.path.join(root, t))
-        for name in files:
-            if name.endswith((".chm", ".pyc")):
-                os.remove(os.path.join(root, name))
-
-    return target_dir
-
-
-def get_contrib_pysite_deps():
-    systype = util.get_systype()
-    twisted_version = "22.1.0"
-    if "linux_arm" in systype:
-        result = [
-            # twisted[tls], see setup.py for %twisted_version%
-            "twisted == %s" % twisted_version,
-            # pyopenssl depends on it, use RUST-less version
-            "cryptography >= 3.3, < 35.0.0",
-            "pyopenssl >= 16.0.0, <= 21.0.0",
-            "service_identity >= 18.1.0, <= 21.1.0",
-        ]
-    else:
-        result = ["twisted[tls] == %s" % twisted_version]
-    if "windows" in systype:
-        result.append("pywin32 != 226")
-    return result

platformio/package/manager/platform.py

@@ -15,7 +15,7 @@
 import os
 
 from platformio import util
-from platformio.http import HTTPClientError, InternetIsOffline
+from platformio.http import HTTPClientError, InternetConnectionError
 from platformio.package.exception import UnknownPackageError
 from platformio.package.manager.base import BasePackageManager
 from platformio.package.manager.core import get_installed_core_packages
@@ -128,7 +128,7 @@ class PlatformPackageManager(BasePackageManager):  # pylint: disable=too-many-an
                 key = "%s:%s" % (board["platform"], board["id"])
                 if key not in know_boards:
                     boards.append(board)
-        except (HTTPClientError, InternetIsOffline):
+        except (HTTPClientError, InternetConnectionError):
             pass
         return sorted(boards, key=lambda b: b["name"])
 

platformio/package/manifest/schema.py

@@ -276,7 +276,7 @@ class ManifestSchema(BaseSchema):
     @staticmethod
     @memoized(expire="1h")
     def load_spdx_licenses():
-        version = "3.19"
+        version = "3.20"
         spdx_data_url = (
             "https://raw.githubusercontent.com/spdx/license-list-data/"
             "v%s/json/licenses.json" % version

platformio/package/meta.py

@@ -65,7 +65,6 @@ class PackageType:
 
 
 class PackageCompatibility:
-
     KNOWN_QUALIFIERS = ("platforms", "frameworks", "authors")
 
     @classmethod
@@ -468,7 +467,6 @@ class PackageMetaData:
 
 
 class PackageItem:
-
     METAFILE_NAME = ".piopm"
 
     def __init__(self, path, metadata=None):

platformio/package/unpack.py

@@ -20,11 +20,11 @@ from zipfile import ZipFile
 import click
 
 from platformio import fs
+from platformio.compat import is_terminal
 from platformio.package.exception import PackageException
 
 
 class ExtractArchiveItemError(PackageException):
-
     MESSAGE = (
         "Could not extract `{0}` to `{1}`. Try to disable antivirus "
         "tool or check this solution -> https://bit.ly/faq-package-manager"
@@ -159,18 +159,38 @@ class FileUnpacker:
 
     def unpack(
         self, dest_dir=None, with_progress=True, check_unpacked=True, silent=False
-    ):
+    ):  # pylint: disable=too-many-branches
         assert self._archiver
+        label = "Unpacking"
+        items = self._archiver.get_items()
         if not dest_dir:
             dest_dir = os.getcwd()
+
         if not with_progress or silent:
             if not silent:
-                click.echo("Unpacking...")
+                click.echo(f"{label}...")
-            for item in self._archiver.get_items():
+            for item in items:
                 self._archiver.extract_item(item, dest_dir)
+        elif not is_terminal():
+            click.echo(f"{label} 0%", nl=False)
+            print_percent_step = 10
+            printed_percents = 0
+            unpacked_nums = 0
+            for item in items:
+                self._archiver.extract_item(item, dest_dir)
+                unpacked_nums += 1
+                if (unpacked_nums / len(items) * 100) >= (
+                    printed_percents + print_percent_step
+                ):
+                    printed_percents += print_percent_step
+                    click.echo(f" {printed_percents}%", nl=False)
+            click.echo("")
         else:
-            items = self._archiver.get_items()
+            with click.progressbar(
-            with click.progressbar(items, label="Unpacking") as pb:
+                items,
+                label=label,
+                update_min_steps=min(50, len(items) / 100),  # every 50 files or less
+            ) as pb:
                 for item in pb:
                     self._archiver.extract_item(item, dest_dir)
 

platformio/package/vcsclient.py

@@ -58,7 +58,6 @@ class VCSClientFactory:
 
 
 class VCSClientBase:
-
     command = None
 
     def __init__(self, src_dir, remote_url=None, tag=None, silent=False):
@@ -128,7 +127,6 @@ class VCSClientBase:
 
 
 class GitClient(VCSClientBase):
-
     command = "git"
     _configured = False
 
@@ -232,7 +230,6 @@ class GitClient(VCSClientBase):
 
 
 class HgClient(VCSClientBase):
-
     command = "hg"
 
     def export(self):
@@ -256,7 +253,6 @@ class HgClient(VCSClientBase):
 
 
 class SvnClient(VCSClientBase):
-
     command = "svn"
 
     def export(self):

platformio/platform/_run.py

@@ -25,10 +25,10 @@ from platformio import app, fs, proc, telemetry
 from platformio.compat import hashlib_encode_data
 from platformio.package.manager.core import get_core_package_dir
 from platformio.platform.exception import BuildScriptNotFound
+from platformio.run.helpers import KNOWN_CLEAN_TARGETS, KNOWN_FULLCLEAN_TARGETS
 
 
 class PlatformRunMixin:
-
     LINE_ERROR_RE = re.compile(r"(^|\s+)error:?\s+", re.I)
 
     @staticmethod
@@ -57,9 +57,6 @@ class PlatformRunMixin:
         self.silent = silent
         self.verbose = verbose or app.get_setting("force_verbose")
 
-        if "clean" in targets:
-            targets = ["-c", "."]
-
         variables["platform_manifest"] = self.manifest_path
 
         if "build_script" not in variables:
@@ -93,16 +90,22 @@ class PlatformRunMixin:
             "--sconstruct",
             os.path.join(fs.get_source_dir(), "builder", "main.py"),
         ]
-        args.append("PIOVERBOSE=%d" % (1 if self.verbose else 0))
+        args.append("PIOVERBOSE=%d" % int(self.verbose))
         # pylint: disable=protected-access
-        args.append("ISATTY=%d" % (1 if click._compat.isatty(sys.stdout) else 0))
+        args.append("ISATTY=%d" % int(click._compat.isatty(sys.stdout)))
-        args += targets
-
         # encode and append variables
         for key, value in variables.items():
             args.append("%s=%s" % (key.upper(), self.encode_scons_arg(value)))
 
-        proc.copy_pythonpath_to_osenv()
+        if set(KNOWN_CLEAN_TARGETS + KNOWN_FULLCLEAN_TARGETS) & set(targets):
+            args.append("--clean")
+            args.append(
+                "FULLCLEAN=%d"
+                % (1 if set(KNOWN_FULLCLEAN_TARGETS) & set(targets) else 0)
+            )
+        elif targets:
+            args.extend(targets)
+
         # force SCons output to Unicode
         os.environ["PYTHONIOENCODING"] = "utf-8"
 

platformio/platform/base.py

@@ -29,7 +29,6 @@ from platformio.project.config import ProjectConfig
 class PlatformBase(  # pylint: disable=too-many-instance-attributes,too-many-public-methods
     PlatformPackagesMixin, PlatformRunMixin
 ):
-
     CORE_SEMVER = pepver_to_semver(__version__)
     _BOARDS_CACHE = {}
 
@@ -208,6 +207,15 @@ class PlatformBase(  # pylint: disable=too-many-instance-attributes,too-many-pub
     def configure_debug_session(self, debug_config):
         raise NotImplementedError
 
+    def generate_sample_code(self, project_config, environment):
+        raise NotImplementedError
+
+    def on_installed(self):
+        pass
+
+    def on_uninstalled(self):
+        pass
+
     def get_lib_storages(self):
         storages = {}
         for opts in (self.frameworks or {}).values():
@@ -228,9 +236,3 @@ class PlatformBase(  # pylint: disable=too-many-instance-attributes,too-many-pub
                 storages[libcore_dir] = "%s-core-%s" % (opts["package"], item)
 
         return [dict(name=name, path=path) for path, name in storages.items()]
-
-    def on_installed(self):
-        pass
-
-    def on_uninstalled(self):
-        pass

platformio/platform/exception.py

@@ -20,12 +20,10 @@ class PlatformException(PlatformioException):
 
 
 class UnknownPlatform(PlatformException):
-
     MESSAGE = "Unknown development platform '{0}'"
 
 
 class IncompatiblePlatform(PlatformException):
-
     MESSAGE = (
         "Development platform '{0}' is not compatible with PlatformIO Core v{1} and "
         "depends on PlatformIO Core {2}.\n"
@@ -33,20 +31,16 @@ class IncompatiblePlatform(PlatformException):
 
 
 class UnknownBoard(PlatformException):
-
     MESSAGE = "Unknown board ID '{0}'"
 
 
 class InvalidBoardManifest(PlatformException):
-
     MESSAGE = "Invalid board JSON manifest '{0}'"
 
 
 class UnknownFramework(PlatformException):
-
     MESSAGE = "Unknown framework '{0}'"
 
 
 class BuildScriptNotFound(PlatformException):
-
     MESSAGE = "Invalid path '{0}' to build script"

platformio/project/commands/config.py

@@ -28,7 +28,7 @@ from platformio.project.helpers import is_platformio_project
     "-d",
     "--project-dir",
     default=os.getcwd,
-    type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True),
+    type=click.Path(exists=True, file_okay=False, dir_okay=True),
 )
 @click.option("--json-output", is_flag=True)
 def project_config_cmd(project_dir, json_output):

platformio/project/commands/init.py

@@ -24,12 +24,14 @@ from platformio import fs
 from platformio.package.commands.install import install_project_dependencies
 from platformio.package.manager.platform import PlatformPackageManager
 from platformio.platform.exception import UnknownBoard
+from platformio.platform.factory import PlatformFactory
 from platformio.project.config import ProjectConfig
 from platformio.project.helpers import is_platformio_project
 from platformio.project.integration.generator import ProjectGenerator
+from platformio.project.options import ProjectOptions
 
 
-def validate_boards(ctx, param, value):  # pylint: disable=W0613
+def validate_boards(ctx, param, value):  # pylint: disable=unused-argument
     pm = PlatformPackageManager()
     for id_ in value:
         try:
@@ -47,25 +49,33 @@ def validate_boards(ctx, param, value):  # pylint: disable=W0613
     "--project-dir",
     "-d",
     default=os.getcwd,
-    type=click.Path(
+    type=click.Path(exists=True, file_okay=False, dir_okay=True, writable=True),
-        exists=True, file_okay=False, dir_okay=True, writable=True, resolve_path=True
+)
-    ),
+@click.option(
+    "-b", "--board", "boards", multiple=True, metavar="ID", callback=validate_boards
 )
-@click.option("-b", "--board", multiple=True, metavar="ID", callback=validate_boards)
 @click.option("--ide", type=click.Choice(ProjectGenerator.get_supported_ides()))
 @click.option("-e", "--environment", help="Update existing environment")
-@click.option("-O", "--project-option", multiple=True)
+@click.option(
-@click.option("--env-prefix", default="")
+    "-O",
+    "--project-option",
+    "project_options",
+    multiple=True,
+    help="A `name=value` pair",
+)
+@click.option("--sample-code", is_flag=True)
 @click.option("--no-install-dependencies", is_flag=True)
+@click.option("--env-prefix", default="")
 @click.option("-s", "--silent", is_flag=True)
 def project_init_cmd(
     project_dir,
-    board,
+    boards,
     ide,
     environment,
-    project_option,
+    project_options,
-    env_prefix,
+    sample_code,
     no_install_dependencies,
+    env_prefix,
     silent,
 ):
     is_new_project = not is_platformio_project(project_dir)
@@ -74,23 +84,23 @@ def project_init_cmd(
             print_header(project_dir)
         init_base_project(project_dir)
 
-    if environment:
-        update_project_env(project_dir, environment, project_option)
-    elif board:
-        update_board_envs(project_dir, board, project_option, env_prefix)
-
     with fs.cd(project_dir):
+        if environment:
+            update_project_env(environment, project_options)
+        elif boards:
+            update_board_envs(project_dir, boards, project_options, env_prefix)
+
         generator = None
         config = ProjectConfig.get_instance(os.path.join(project_dir, "platformio.ini"))
         if ide:
             config.validate()
             # init generator and pick the best env if user didn't specify
-            generator = ProjectGenerator(config, environment, ide, board)
+            generator = ProjectGenerator(config, environment, ide, boards)
             if not environment:
                 environment = generator.env_name
 
         # resolve project dependencies
-        if not no_install_dependencies and (environment or board):
+        if not no_install_dependencies and (environment or boards):
             install_project_dependencies(
                 options=dict(
                     project_dir=project_dir,
@@ -99,6 +109,9 @@ def project_init_cmd(
                 )
             )
 
+        if environment and sample_code:
+            init_sample_code(config, environment)
+
         if generator:
             if not silent:
                 click.echo(
@@ -106,31 +119,22 @@ def project_init_cmd(
                 )
             generator.generate()
 
-    if is_new_project:
+        if is_new_project:
-        init_cvs_ignore(project_dir)
+            init_cvs_ignore()
 
     if not silent:
         print_footer(is_new_project)
 
 
 def print_header(project_dir):
-    if project_dir == os.getcwd():
+    click.echo("The following files/directories have been created in ", nl=False)
-        click.secho("\nThe current working directory ", fg="yellow", nl=False)
-        try:
-            click.secho(project_dir, fg="cyan", nl=False)
-        except UnicodeEncodeError:
-            click.secho(json.dumps(project_dir), fg="cyan", nl=False)
-        click.secho(" will be used for the project.", fg="yellow")
-        click.echo("")
-
-    click.echo("The next files/directories have been created in ", nl=False)
     try:
         click.secho(project_dir, fg="cyan")
     except UnicodeEncodeError:
         click.secho(json.dumps(project_dir), fg="cyan")
     click.echo("%s - Put project header files here" % click.style("include", fg="cyan"))
     click.echo(
-        "%s - Put here project specific (private) libraries"
+        "%s - Put project specific (private) libraries here"
         % click.style("lib", fg="cyan")
     )
     click.echo("%s - Put project source files here" % click.style("src", fg="cyan"))
@@ -140,18 +144,9 @@ def print_header(project_dir):
 
 
 def print_footer(is_new_project):
-    if is_new_project:
+    action = "initialized" if is_new_project else "updated"
-        return click.secho(
-            "\nProject has been successfully initialized! Useful commands:\n"
-            "`pio run` - process/build project from the current directory\n"
-            "`pio run --target upload` or `pio run -t upload` "
-            "- upload firmware to a target\n"
-            "`pio run --target clean` - clean project (remove compiled files)"
-            "\n`pio run --help` - additional information",
-            fg="green",
-        )
     return click.secho(
-        "Project has been successfully updated!",
+        f"Project has been successfully {action}!",
         fg="green",
     )
 
@@ -166,7 +161,7 @@ def init_base_project(project_dir):
             (config.get("platformio", "lib_dir"), init_lib_readme),
             (config.get("platformio", "test_dir"), init_test_readme),
         ]
-        for (path, cb) in dir_to_readme:
+        for path, cb in dir_to_readme:
             if os.path.isdir(path):
                 continue
             os.makedirs(path)
@@ -291,15 +286,15 @@ More information about PlatformIO Unit Testing:
         )
 
 
-def init_cvs_ignore(project_dir):
+def init_cvs_ignore():
-    conf_path = os.path.join(project_dir, ".gitignore")
+    conf_path = ".gitignore"
     if os.path.isfile(conf_path):
         return
     with open(conf_path, mode="w", encoding="utf8") as fp:
         fp.write(".pio\n")
 
 
-def update_board_envs(project_dir, board_ids, project_option, env_prefix):
+def update_board_envs(project_dir, boards, extra_project_options, env_prefix):
     config = ProjectConfig(
         os.path.join(project_dir, "platformio.ini"), parse_extra=False
     )
@@ -311,7 +306,7 @@ def update_board_envs(project_dir, board_ids, project_option, env_prefix):
 
     pm = PlatformPackageManager()
     modified = False
-    for id_ in board_ids:
+    for id_ in boards:
         board_config = pm.board_config(id_)
         if id_ in used_boards:
             continue
@@ -324,7 +319,7 @@ def update_board_envs(project_dir, board_ids, project_option, env_prefix):
         if frameworks:
             envopts["framework"] = frameworks[0]
 
-        for item in project_option:
+        for item in extra_project_options:
             if "=" not in item:
                 continue
             _name, _value = item.split("=", 1)
@@ -340,21 +335,76 @@ def update_board_envs(project_dir, board_ids, project_option, env_prefix):
         config.save()
 
 
-def update_project_env(project_dir, environment, project_option):
+def update_project_env(environment, extra_project_options=None):
-    if not project_option:
+    if not extra_project_options:
         return
+    env_section = "env:%s" % environment
+    option_to_sections = {"platformio": [], env_section: []}
+    for item in extra_project_options:
+        assert "=" in item
+        name, value = item.split("=", 1)
+        name = name.strip()
+        destination = env_section
+        for option in ProjectOptions.values():
+            if option.scope in option_to_sections and option.name == name:
+                destination = option.scope
+                break
+        option_to_sections[destination].append((name, value.strip()))
+
     config = ProjectConfig(
-        os.path.join(project_dir, "platformio.ini"), parse_extra=False
+        "platformio.ini", parse_extra=False, expand_interpolations=False
     )
-
+    for section, options in option_to_sections.items():
-    section = "env:%s" % environment
+        if not options:
-    if not config.has_section(section):
-        config.add_section(section)
-
-    for item in project_option:
-        if "=" not in item:
             continue
-        _name, _value = item.split("=", 1)
+        if not config.has_section(section):
-        config.set(section, _name.strip(), _value.strip())
+            config.add_section(section)
+        for name, value in options:
+            config.set(section, name, value)
 
     config.save()
+
+
+def init_sample_code(config, environment):
+    platform_spec = config.get(f"env:{environment}", "platform", None)
+    if not platform_spec:
+        return None
+    p = PlatformFactory.new(platform_spec)
+    try:
+        return p.generate_sample_code(config, environment)
+    except NotImplementedError:
+        pass
+
+    framework = config.get(f"env:{environment}", "framework", None)
+    if framework != ["arduino"]:
+        return None
+    main_content = """
+#include <Arduino.h>
+
+// put function declarations here:
+int myFunction(int, int);
+
+void setup() {
+  // put your setup code here, to run once:
+  int result = myFunction(2, 3);
+}
+
+void loop() {
+  // put your main code here, to run repeatedly:
+}
+
+// put function definitions here:
+int myFunction(int x, int y) {
+  return x + y;
+}
+"""
+    is_cpp_project = p.name not in ["intel_mcs51", "ststm8"]
+    src_dir = config.get("platformio", "src_dir")
+    main_path = os.path.join(src_dir, "main.%s" % ("cpp" if is_cpp_project else "c"))
+    if os.path.isfile(main_path):
+        return None
+    if not os.path.isdir(src_dir):
+        os.makedirs(src_dir)
+    with open(main_path, mode="w", encoding="utf8") as fp:
+        fp.write(main_content.strip())
+    return True

platformio/project/commands/metadata.py

@@ -31,11 +31,11 @@ from platformio.project.helpers import load_build_metadata
     "-d",
     "--project-dir",
     default=os.getcwd,
-    type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True),
+    type=click.Path(exists=True, file_okay=False, dir_okay=True),
 )
 @click.option("-e", "--environment", "environments", multiple=True)
 @click.option("--json-output", is_flag=True)
-@click.option("--json-output-path", type=click.Path(resolve_path=True))
+@click.option("--json-output-path", type=click.Path())
 def project_metadata_cmd(project_dir, environments, json_output, json_output_path):
     with fs.cd(project_dir):
         config = ProjectConfig.get_instance()

platformio/project/config.py

@@ -39,7 +39,7 @@ CONFIG_HEADER = """
 
 
 class ProjectConfigBase:
-
+    ENVNAME_RE = re.compile(r"^[a-z\d\_\-]+$", flags=re.I)
     INLINE_COMMENT_RE = re.compile(r"\s+;.*$")
     VARTPL_RE = re.compile(r"\$\{([^\.\}\()]+)\.([^\}]+)\}")
 
@@ -389,16 +389,38 @@ class ProjectConfigBase:
     def validate(self, envs=None, silent=False):
         if not os.path.isfile(self.path):
             raise exception.NotPlatformIOProjectError(os.path.dirname(self.path))
+
+        known_envs = set(self.envs())
+
         # check envs
-        known = set(self.envs())
+        if not known_envs:
-        if not known:
             raise exception.ProjectEnvsNotAvailableError()
-        unknown = set(list(envs or []) + self.default_envs()) - known
+        unknown_envs = set(list(envs or []) + self.default_envs()) - known_envs
-        if unknown:
+        if unknown_envs:
-            raise exception.UnknownEnvNamesError(", ".join(unknown), ", ".join(known))
+            raise exception.UnknownEnvNamesError(
+                ", ".join(unknown_envs), ", ".join(known_envs)
+            )
+
+        for env in known_envs:
+            # check envs names
+            if not self.ENVNAME_RE.match(env):
+                raise exception.InvalidEnvNameError(env)
+
+            # check simultaneous use of `monitor_raw` and `monitor_filters`
+            if self.get(f"env:{env}", "monitor_raw", False) and self.get(
+                f"env:{env}", "monitor_filters", None
+            ):
+                self.warnings.append(
+                    "The `monitor_raw` and `monitor_filters` options cannot be "
+                    f"used simultaneously for the `{env}` environment in the "
+                    "`platformio.ini` file. The `monitor_filters` option will "
+                    "be disabled to avoid conflicts."
+                )
+
         if not silent:
             for warning in self.warnings:
                 click.secho("Warning! %s" % warning, fg="yellow")
+
         return True
 
 
@@ -412,7 +434,6 @@ class ProjectConfigDirsMixin:
 
 
 class ProjectConfig(ProjectConfigBase, ProjectConfigDirsMixin):
-
     _instances = {}
 
     @staticmethod

platformio/project/exception.py

@@ -20,7 +20,6 @@ class ProjectError(PlatformioException):
 
 
 class NotPlatformIOProjectError(ProjectError, UserSideException):
-
     MESSAGE = (
         "Not a PlatformIO project. `platformio.ini` file has not been "
         "found in current working directory ({0}). To initialize new project "
@@ -29,25 +28,27 @@ class NotPlatformIOProjectError(ProjectError, UserSideException):
 
 
 class InvalidProjectConfError(ProjectError, UserSideException):
-
     MESSAGE = "Invalid '{0}' (project configuration file): '{1}'"
 
 
 class UndefinedEnvPlatformError(ProjectError, UserSideException):
-
     MESSAGE = "Please specify platform for '{0}' environment"
 
 
 class ProjectEnvsNotAvailableError(ProjectError, UserSideException):
-
     MESSAGE = "Please setup environments in `platformio.ini` file"
 
 
 class UnknownEnvNamesError(ProjectError, UserSideException):
-
     MESSAGE = "Unknown environment names '{0}'. Valid names are '{1}'"
 
 
-class ProjectOptionValueError(ProjectError, UserSideException):
+class InvalidEnvNameError(ProjectError, UserSideException):
+    MESSAGE = (
+        "Invalid environment name '{0}'. The name can contain "
+        "alphanumeric, underscore, and hyphen characters (a-z, 0-9, -, _)"
+    )
 
+
+class ProjectOptionValueError(ProjectError, UserSideException):
     MESSAGE = "{0} for option `{1}` in section [{2}]"

platformio/project/helpers.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 import os
+import re
 import subprocess
 from hashlib import sha1
 
@@ -24,7 +25,7 @@ from platformio.project.config import ProjectConfig
 
 
 def get_project_dir():
-    return fs.normalize_path(os.getcwd())
+    return os.getcwd()
 
 
 def is_platformio_project(project_dir=None):
@@ -94,7 +95,16 @@ def compute_project_checksum(config):
     checksum = sha1(hashlib_encode_data(__version__))
 
     # configuration file state
-    checksum.update(hashlib_encode_data(config.to_json()))
+    config_data = config.to_json()
+    if IS_WINDOWS:
+        # issue #4600: fix drive letter
+        config_data = re.sub(
+            r"([A-Z]):\\",
+            lambda match: "%s:\\" % match.group(1).lower(),
+            config_data,
+            flags=re.I,
+        )
+    checksum.update(hashlib_encode_data(config_data))
 
     # project file structure
     check_suffixes = (".c", ".cc", ".cpp", ".h", ".hpp", ".s", ".S")

platformio/project/integration/generator.py

@@ -19,33 +19,34 @@ import sys
 import bottle
 
 from platformio import fs, util
+from platformio.debug.helpers import get_default_debug_env
 from platformio.proc import where_is_program
 from platformio.project.helpers import load_build_metadata
 
 
 class ProjectGenerator:
-    def __init__(self, config, env_name, ide, board_ids=None):
+    def __init__(self, config, env_name, ide, boards=None):
         self.config = config
         self.project_dir = os.path.dirname(config.path)
-        self.original_env_name = env_name
+        self.forced_env_name = env_name
-        self.env_name = str(env_name or self.get_best_envname(board_ids))
+        self.env_name = str(env_name or self.get_best_envname(boards))
         self.ide = str(ide)
 
-    def get_best_envname(self, board_ids=None):
+    def get_best_envname(self, boards=None):
         envname = None
         default_envs = self.config.default_envs()
         if default_envs:
             envname = default_envs[0]
-            if not board_ids:
+            if not boards:
                 return envname
 
         for env in self.config.envs():
-            if not board_ids:
+            if not boards:
                 return env
             if not envname:
                 envname = env
             items = self.config.items(env=env, as_dict=True)
-            if "board" in items and items.get("board") in board_ids:
+            if "board" in items and items.get("board") in boards:
                 return env
         return envname
 
@@ -86,7 +87,8 @@ class ProjectGenerator:
                 "platformio", "name", os.path.basename(self.project_dir)
             ),
             "project_dir": self.project_dir,
-            "original_env_name": self.original_env_name,
+            "forced_env_name": self.forced_env_name,
+            "default_debug_env_name": get_default_debug_env(self.config),
             "env_name": self.env_name,
             "user_home_dir": os.path.abspath(fs.expanduser("~")),
             "platformio_path": sys.argv[0]
@@ -132,7 +134,7 @@ class ProjectGenerator:
             for root, _, files in os.walk(self.config.get("platformio", "src_dir")):
                 for f in files:
                     result.append(
-                        os.path.relpath(os.path.join(os.path.realpath(root), f))
+                        os.path.relpath(os.path.join(os.path.abspath(root), f))
                     )
         return result
 

platformio/project/options.py

@@ -114,7 +114,7 @@ def validate_dir(path):
         path = fs.expanduser(path)
     if "$" in path:
         path = expand_dir_templates(path)
-    return fs.normalize_path(path)
+    return os.path.abspath(path)
 
 
 def get_default_core_dir():
@@ -649,10 +649,10 @@ ProjectOptions = OrderedDict(
             ),
             ConfigEnvOption(
                 group="check",
-                name="check_patterns",
+                name="check_src_filters",
+                oldnames=["check_patterns"],
                 description=(
-                    "Configure a list of target files or directories for checking "
+                    "Configure a list of target files or directories for checking"
-                    "(Unix shell-style wildcards)"
                 ),
                 multiple=True,
             ),

platformio/registry/client.py

@@ -142,12 +142,15 @@ class RegistryClient(HTTPClient):
             x_with_authorization=self.allowed_private_packages(),
         )
 
-    def get_package(self, type_, owner, name, version=None):
+    def get_package(self, type_, owner, name, version=None, extra_path=None):
         try:
             return self.fetch_json_data(
                 "get",
-                "/v3/packages/{owner}/{type}/{name}".format(
+                "/v3/packages/{owner}/{type}/{name}{extra_path}".format(
-                    type=type_, owner=owner.lower(), name=name.lower()
+                    type=type_,
+                    owner=owner.lower(),
+                    name=name.lower(),
+                    extra_path=extra_path or "",
                 ),
                 params=dict(version=version) if version else None,
                 x_cache_valid="1h",

platformio/registry/mirror.py

@@ -22,7 +22,6 @@ from platformio.registry.client import RegistryClient
 
 
 class RegistryFileMirrorIterator:
-
     HTTP_CLIENT_INSTANCES = {}
 
     def __init__(self, download_url):

platformio/remote/ac/base.py

@@ -17,7 +17,6 @@ from twisted.spread import pb  # pylint: disable=import-error
 
 
 class AsyncCommandBase:
-
     MAX_BUFFER_SIZE = 1024 * 1024  # 1Mb
 
     def __init__(self, options=None, on_end_callback=None):

platformio/remote/cli.py

@@ -17,7 +17,9 @@
 
 import os
 import subprocess
+import sys
 import threading
+from site import addsitedir
 from tempfile import mkdtemp
 from time import sleep
 
@@ -29,7 +31,7 @@ from platformio.device.monitor.command import (
     device_monitor_cmd,
     get_project_options,
 )
-from platformio.package.manager.core import inject_contrib_pysite
+from platformio.package.manager.core import get_core_package_dir
 from platformio.project.exception import NotPlatformIOProjectError
 from platformio.project.options import ProjectOptions
 from platformio.run.cli import cli as cmd_run
@@ -41,7 +43,11 @@ from platformio.test.cli import cli as test_cmd
 @click.pass_context
 def cli(ctx, agent):
     ctx.obj = agent
-    inject_contrib_pysite()
+    # inject twisted dependencies
+    contrib_dir = get_core_package_dir("contrib-pioremote")
+    if contrib_dir not in sys.path:
+        addsitedir(contrib_dir)
+        sys.path.insert(0, contrib_dir)
 
 
 @cli.group("agent", short_help="Start a new agent or list active")
@@ -56,7 +62,7 @@ def remote_agent():
     "-d",
     "--working-dir",
     envvar="PLATFORMIO_REMOTE_AGENT_DIR",
-    type=click.Path(file_okay=False, dir_okay=True, writable=True, resolve_path=True),
+    type=click.Path(file_okay=False, dir_okay=True, writable=True),
 )
 def remote_agent_start(name, share, working_dir):
     from platformio.remote.client.agent_service import RemoteAgentService
@@ -96,9 +102,7 @@ def remote_update(agents, only_check, dry_run):
     "-d",
     "--project-dir",
     default=os.getcwd,
-    type=click.Path(
+    type=click.Path(exists=True, file_okay=True, dir_okay=True, writable=True),
-        exists=True, file_okay=True, dir_okay=True, writable=True, resolve_path=True
-    ),
 )
 @click.option("--disable-auto-clean", is_flag=True)
 @click.option("-r", "--force-remote", is_flag=True)
@@ -118,7 +122,6 @@ def remote_run(
     silent,
     verbose,
 ):
-
     from platformio.remote.client.run_or_test import RunOrTestClient
 
     cr = RunOrTestClient(
@@ -187,9 +190,7 @@ def remote_run(
     "-d",
     "--project-dir",
     default=os.getcwd,
-    type=click.Path(
+    type=click.Path(exists=True, file_okay=False, dir_okay=True, writable=True),
-        exists=True, file_okay=False, dir_okay=True, writable=True, resolve_path=True
-    ),
 )
 @click.option("-r", "--force-remote", is_flag=True)
 @click.option("--without-building", is_flag=True)
@@ -211,7 +212,6 @@ def remote_test(  # pylint: disable=redefined-builtin
     without_uploading,
     verbose,
 ):
-
     from platformio.remote.client.run_or_test import RunOrTestClient
 
     cr = RunOrTestClient(
@@ -336,7 +336,7 @@ def device_list(agents, json_output):
     "-d",
     "--project-dir",
     default=os.getcwd,
-    type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True),
+    type=click.Path(exists=True, file_okay=False, dir_okay=True),
 )
 @click.option(
     "-e",
@@ -345,9 +345,7 @@ def device_list(agents, json_output):
 )
 @click.option(
     "--sock",
-    type=click.Path(
+    type=click.Path(exists=True, file_okay=False, dir_okay=True, writable=True),
-        exists=True, file_okay=False, dir_okay=True, writable=True, resolve_path=True
-    ),
 )
 @click.pass_obj
 @click.pass_context

platformio/remote/client/async_base.py

@@ -34,7 +34,7 @@ class AsyncClientBase(RemoteClientBase):
     def cb_async_result(self, result):
         if self._acs_total == 0:
             self._acs_total = len(result)
-        for (success, value) in result:
+        for success, value in result:
             if not success:
                 raise pb.Error(value)
             self.acread_data(*value)

platformio/remote/client/base.py

@@ -33,7 +33,6 @@ from platformio.remote.factory.ssl import SSLContextFactory
 class RemoteClientBase(  # pylint: disable=too-many-instance-attributes
     pb.Referenceable
 ):
-
     PING_DELAY = 60
     PING_MAX_FAILURES = 3
     DEBUG = False