Merge branch 'release/v4.3.4'

.github/workflows/core.yml

@@ -12,13 +12,14 @@ jobs:
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v2
+        with:
+          submodules: "recursive"
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v1
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install dependencies
         run: |
-          git submodule update --init --recursive
           python -m pip install --upgrade pip
           pip install tox
 

.github/workflows/docs.yml

@@ -7,13 +7,14 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
+        with:
+          submodules: "recursive"
       - name: Set up Python
         uses: actions/setup-python@v1
         with:
           python-version: 3.7
       - name: Install dependencies
         run: |
-          git submodule update --init --recursive
           python -m pip install --upgrade pip
           pip install tox
 

.github/workflows/examples.yml

@@ -12,13 +12,14 @@ jobs:
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v2
+        with:
+          submodules: "recursive"
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v1
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install dependencies
         run: |
-          git submodule update --init --recursive
           python -m pip install --upgrade pip
           pip install tox
 

CONTRIBUTING.md

@@ -1,20 +1,20 @@
 Contributing
 ------------
 
-To get started, <a href="https://www.clahub.com/agreements/platformio/platformio-core">sign the Contributor License Agreement</a>.
+To get started, <a href="https://cla-assistant.io/platformio/platformio-core">sign the Contributor License Agreement</a>.
 
 1. Fork the repository on GitHub.
 2. Clone repository `git clone --recursive https://github.com/YourGithubUsername/platformio-core.git`
 3. Run `pip install tox`
-4. Go to the root of project where is located `tox.ini` and run `tox -e py27`
+4. Go to the root of project where is located `tox.ini` and run `tox -e py37`
 5. Activate current development environment:
 
-   * Windows: `.tox\py27\Scripts\activate`
+   * Windows: `.tox\py37\Scripts\activate`
-   * Bash/ZSH: `source .tox/py27/bin/activate`
+   * Bash/ZSH: `source .tox/py37/bin/activate`
-   * Fish: `source .tox/py27/bin/activate.fish`
+   * Fish: `source .tox/py37/bin/activate.fish`
 
 6. Make changes to code, documentation, etc.
-7. Lint source code `make lint`
+7. Lint source code `make before-commit`
 8. Run the tests `make test`
 9. Build documentation `tox -e docs` (creates a directory _build under docs where you can find the html)
 10. Commit changes to your forked repository

HISTORY.rst

@@ -6,6 +6,13 @@ Release Notes
 PlatformIO Core 4
 -----------------
 
+4.3.4 (2020-05-23)
+~~~~~~~~~~~~~~~~~~
+
+* Added `PlatformIO CLI Shell Completion <https://docs.platformio.org/page/core/userguide/system/completion/index.html>`__ for Fish, Zsh, Bash, and PowerShell (`issue #3435 <https://github.com/platformio/platformio-core/issues/3435>`_)
+* Automatically build ``contrib-pysite`` package on a target machine when pre-built package is not compatible (`issue #3482 <https://github.com/platformio/platformio-core/issues/3482>`_)
+* Fixed an issue on Windows when installing a library dependency from Git repository (`issue #2844 <https://github.com/platformio/platformio-core/issues/2844>`_, `issue #3328 <https://github.com/platformio/platformio-core/issues/3328>`_)
+
 4.3.3 (2020-04-28)
 ~~~~~~~~~~~~~~~~~~
 
@@ -17,7 +24,7 @@ PlatformIO Core 4
 * New `Account Management System <https://docs.platformio.org/page/plus/pio-account.html>`__ (preview)
 * Open source `PIO Remote <http://docs.platformio.org/page/plus/pio-remote.html>`__ client
 * Improved `PIO Check <http://docs.platformio.org/page/plus/pio-check.html>`__ with more accurate project processing
-* Echo what is typed when ``send_on_enter`` device monitor filter <https://docs.platformio.org/page/projectconf/section_env_monitor.html#monitor-filters>`__ is used (`issue #3452 <https://github.com/platformio/platformio-core/issues/3452>`_)
+* Echo what is typed when ``send_on_enter`` `device monitor filter <https://docs.platformio.org/page/projectconf/section_env_monitor.html#monitor-filters>`__ is used (`issue #3452 <https://github.com/platformio/platformio-core/issues/3452>`_)
 * Fixed PIO Unit Testing for Zephyr RTOS
 * Fixed UnicodeDecodeError on Windows when network drive (NAS) is used (`issue #3417 <https://github.com/platformio/platformio-core/issues/3417>`_)
 * Fixed an issue when saving libraries in new project results in error "No option 'lib_deps' in section" (`issue #3442 <https://github.com/platformio/platformio-core/issues/3442>`_)

platformio/__init__.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-VERSION = (4, 3, 3)
+VERSION = (4, 3, 4)
 __version__ = ".".join([str(s) for s in VERSION])
 
 __title__ = "platformio"

platformio/__main__.py

@@ -22,6 +22,13 @@ from platformio import __version__, exception, maintenance, util
 from platformio.commands import PlatformioCLI
 from platformio.compat import CYGWIN
 
+try:
+    import click_completion  # pylint: disable=import-error
+
+    click_completion.init()
+except:  # pylint: disable=bare-except
+    pass
+
 
 @click.command(
     cls=PlatformioCLI, context_settings=dict(help_option_names=["-h", "--help"])

platformio/builder/main.py

@@ -28,7 +28,7 @@ from SCons.Script import DefaultEnvironment  # pylint: disable=import-error
 from SCons.Script import Import  # pylint: disable=import-error
 from SCons.Script import Variables  # pylint: disable=import-error
 
-from platformio import fs
+from platformio import compat, fs
 from platformio.compat import dump_json_to_unicode
 from platformio.managers.platform import PlatformBase
 from platformio.proc import get_pythonexe_path
@@ -120,6 +120,18 @@ env.Replace(
     ],
 )
 
+if (
+    compat.WINDOWS
+    and sys.version_info >= (3, 8)
+    and env["PROJECT_DIR"].startswith("\\\\")
+):
+    click.secho(
+        "There is a known issue with Python 3.8+ and mapped network drives on "
+        "Windows.\nPlease downgrade Python to the latest 3.7. More details at:\n"
+        "https://github.com/platformio/platformio-core/issues/3417",
+        fg="yellow",
+    )
+
 if env.subst("$BUILD_CACHE_DIR"):
     if not isdir(env.subst("$BUILD_CACHE_DIR")):
         makedirs(env.subst("$BUILD_CACHE_DIR"))

platformio/builder/tools/compilation_db.py

@@ -1,17 +1,24 @@
 # Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-# Copyright 2015 MongoDB Inc.
+# Copyright 2020 MongoDB Inc.
 #
-# Licensed under the Apache License, Version 2.0 (the "License");
+# Permission is hereby granted, free of charge, to any person obtaining
-# you may not use this file except in compliance with the License.
+# a copy of this software and associated documentation files (the
-# You may obtain a copy of the License at
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
 #
-# http://www.apache.org/licenses/LICENSE-2.0
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
 #
-# Unless required by applicable law or agreed to in writing, software
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
-# distributed under the License is distributed on an "AS IS" BASIS,
+# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-# See the License for the specific language governing permissions and
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
-# limitations under the License.
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 # pylint: disable=unused-argument, protected-access, unused-variable, import-error
 # Original: https://github.com/mongodb/mongo/blob/master/site_scons/site_tools/compilation_db.py

platformio/commands/account/client.py

@@ -22,9 +22,13 @@ from requests.packages.urllib3.util.retry import Retry  # pylint:disable=import-
 
 from platformio import __pioaccount_api__, app
 from platformio.commands.account import exception
+from platformio.exception import InternetIsOffline
 
 
 class AccountClient(object):
+
+    SUMMARY_CACHE_TTL = 60 * 60 * 24 * 7
+
     def __init__(
         self, api_base_url=__pioaccount_api__, retries=3,
     ):
@@ -43,21 +47,40 @@ class AccountClient(object):
         adapter = requests.adapters.HTTPAdapter(max_retries=retry)
         self._session.mount(api_base_url, adapter)
 
+    @staticmethod
+    def get_refresh_token():
+        try:
+            return app.get_state_item("account").get("auth").get("refresh_token")
+        except:  # pylint:disable=bare-except
+            raise exception.AccountNotAuthorized()
+
+    @staticmethod
+    def delete_local_session():
+        app.delete_state_item("account")
+
+    @staticmethod
+    def delete_local_state(key):
+        account = app.get_state_item("account")
+        if not account or key not in account:
+            return
+        del account[key]
+        app.set_state_item("account", account)
+
     def login(self, username, password):
         try:
             self.fetch_authentication_token()
         except:  # pylint:disable=bare-except
             pass
         else:
-            raise exception.AccountAlreadyAuthenticated(
+            raise exception.AccountAlreadyAuthorized(
                 app.get_state_item("account", {}).get("email", "")
             )
 
-        response = self._session.post(
+        result = self.send_request(
+            "post",
             self.api_base_url + "/v1/login",
             data={"username": username, "password": password},
         )
-        result = self.raise_error_from_response(response)
         app.set_state_item("account", result)
         return result
 
@@ -67,44 +90,39 @@ class AccountClient(object):
         except:  # pylint:disable=bare-except
             pass
         else:
-            raise exception.AccountAlreadyAuthenticated(
+            raise exception.AccountAlreadyAuthorized(
                 app.get_state_item("account", {}).get("email", "")
             )
 
-        response = self._session.post(
+        result = self.send_request(
+            "post",
             self.api_base_url + "/v1/login/code",
             data={"client_id": client_id, "code": code, "redirect_uri": redirect_uri},
         )
-        result = self.raise_error_from_response(response)
         app.set_state_item("account", result)
         return result
 
     def logout(self):
+        refresh_token = self.get_refresh_token()
+        self.delete_local_session()
         try:
-            refresh_token = self.get_refresh_token()
+            self.send_request(
-        except:  # pylint:disable=bare-except
+                "post",
-            raise exception.AccountNotAuthenticated()
+                self.api_base_url + "/v1/logout",
-        response = requests.post(
+                data={"refresh_token": refresh_token},
-            self.api_base_url + "/v1/logout", data={"refresh_token": refresh_token},
+            )
-        )
-        try:
-            self.raise_error_from_response(response)
         except exception.AccountError:
             pass
-        app.delete_state_item("account")
         return True
 
     def change_password(self, old_password, new_password):
-        try:
+        token = self.fetch_authentication_token()
-            token = self.fetch_authentication_token()
+        self.send_request(
-        except:  # pylint:disable=bare-except
+            "post",
-            raise exception.AccountNotAuthenticated()
-        response = self._session.post(
             self.api_base_url + "/v1/password",
             headers={"Authorization": "Bearer %s" % token},
             data={"old_password": old_password, "new_password": new_password},
         )
-        self.raise_error_from_response(response)
         return True
 
     def registration(
@@ -115,11 +133,12 @@ class AccountClient(object):
         except:  # pylint:disable=bare-except
             pass
         else:
-            raise exception.AccountAlreadyAuthenticated(
+            raise exception.AccountAlreadyAuthorized(
                 app.get_state_item("account", {}).get("email", "")
             )
 
-        response = self._session.post(
+        return self.send_request(
+            "post",
             self.api_base_url + "/v1/registration",
             data={
                 "username": username,
@@ -129,70 +148,73 @@ class AccountClient(object):
                 "lastname": lastname,
             },
         )
-        return self.raise_error_from_response(response)
 
     def auth_token(self, password, regenerate):
-        try:
+        token = self.fetch_authentication_token()
-            token = self.fetch_authentication_token()
+        result = self.send_request(
-        except:  # pylint:disable=bare-except
+            "post",
-            raise exception.AccountNotAuthenticated()
-        response = self._session.post(
             self.api_base_url + "/v1/token",
             headers={"Authorization": "Bearer %s" % token},
             data={"password": password, "regenerate": 1 if regenerate else 0},
         )
-        return self.raise_error_from_response(response).get("auth_token")
+        return result.get("auth_token")
 
     def forgot_password(self, username):
-        response = self._session.post(
+        return self.send_request(
-            self.api_base_url + "/v1/forgot", data={"username": username},
+            "post", self.api_base_url + "/v1/forgot", data={"username": username},
         )
-        return self.raise_error_from_response(response).get("auth_token")
 
     def get_profile(self):
-        try:
+        token = self.fetch_authentication_token()
-            token = self.fetch_authentication_token()
+        return self.send_request(
-        except:  # pylint:disable=bare-except
+            "get",
-            raise exception.AccountNotAuthenticated()
-        response = self._session.get(
             self.api_base_url + "/v1/profile",
             headers={"Authorization": "Bearer %s" % token},
         )
-        return self.raise_error_from_response(response)
 
     def update_profile(self, profile, current_password):
-        try:
+        token = self.fetch_authentication_token()
-            token = self.fetch_authentication_token()
-        except:  # pylint:disable=bare-except
-            raise exception.AccountNotAuthenticated()
         profile["current_password"] = current_password
-        response = self._session.put(
+        self.delete_local_state("summary")
+        response = self.send_request(
+            "put",
             self.api_base_url + "/v1/profile",
             headers={"Authorization": "Bearer %s" % token},
             data=profile,
         )
-        return self.raise_error_from_response(response)
+        return response
 
     def get_account_info(self, offline):
+        account = app.get_state_item("account")
+        if not account:
+            raise exception.AccountNotAuthorized()
+        if (
+            account.get("summary")
+            and account["summary"].get("expire_at", 0) > time.time()
+        ):
+            return account["summary"]
         if offline:
-            account = app.get_state_item("account")
-            if not account:
-                raise exception.AccountNotAuthenticated()
             return {
                 "profile": {
                     "email": account.get("email"),
                     "username": account.get("username"),
                 }
             }
-        try:
+        token = self.fetch_authentication_token()
-            token = self.fetch_authentication_token()
+        result = self.send_request(
-        except:  # pylint:disable=bare-except
+            "get",
-            raise exception.AccountNotAuthenticated()
-        response = self._session.get(
             self.api_base_url + "/v1/summary",
             headers={"Authorization": "Bearer %s" % token},
         )
-        return self.raise_error_from_response(response)
+        account["summary"] = dict(
+            profile=result.get("profile"),
+            packages=result.get("packages"),
+            subscriptions=result.get("subscriptions"),
+            user_id=result.get("user_id"),
+            expire_at=int(time.time()) + self.SUMMARY_CACHE_TTL,
+        )
+        app.set_state_item("account", account)
+        return result
 
     def fetch_authentication_token(self):
         if "PLATFORMIO_AUTH_TOKEN" in os.environ:
@@ -202,25 +224,30 @@ class AccountClient(object):
             if auth.get("access_token_expire") > time.time():
                 return auth.get("access_token")
             if auth.get("refresh_token"):
-                response = self._session.post(
+                try:
-                    self.api_base_url + "/v1/login",
+                    result = self.send_request(
-                    headers={"Authorization": "Bearer %s" % auth.get("refresh_token")},
+                        "post",
-                )
+                        self.api_base_url + "/v1/login",
-                result = self.raise_error_from_response(response)
+                        headers={
-                app.set_state_item("account", result)
+                            "Authorization": "Bearer %s" % auth.get("refresh_token")
-                return result.get("auth").get("access_token")
+                        },
-        raise exception.AccountNotAuthenticated()
+                    )
+                    app.set_state_item("account", result)
+                    return result.get("auth").get("access_token")
+                except exception.AccountError:
+                    self.delete_local_session()
+        raise exception.AccountNotAuthorized()
 
-    @staticmethod
+    def send_request(self, method, url, headers=None, data=None):
-    def get_refresh_token():
         try:
-            auth = app.get_state_item("account").get("auth").get("refresh_token")
+            response = getattr(self._session, method)(
-            return auth
+                url, headers=headers or {}, data=data or {}
-        except:  # pylint:disable=bare-except
+            )
-            raise exception.AccountNotAuthenticated()
+        except requests.exceptions.ConnectionError:
+            raise InternetIsOffline()
+        return self.raise_error_from_response(response)
 
-    @staticmethod
+    def raise_error_from_response(self, response, expected_codes=(200, 201, 202)):
-    def raise_error_from_response(response, expected_codes=(200, 201, 202)):
         if response.status_code in expected_codes:
             try:
                 return response.json()
@@ -231,5 +258,5 @@ class AccountClient(object):
         except (KeyError, ValueError):
             message = response.text
         if "Authorization session has been expired" in message:
-            app.delete_state_item("account")
+            self.delete_local_session()
         raise exception.AccountError(message)

platformio/commands/account/command.py

@@ -167,7 +167,7 @@ def account_update(current_password, **kwargs):
         return None
     try:
         client.logout()
-    except exception.AccountNotAuthenticated:
+    except exception.AccountNotAuthorized:
         pass
     if email_changed:
         return click.secho(

platformio/commands/account/exception.py

@@ -20,11 +20,11 @@ class AccountError(PlatformioException):
     MESSAGE = "{0}"
 
 
-class AccountNotAuthenticated(AccountError):
+class AccountNotAuthorized(AccountError):
 
-    MESSAGE = "You are not authenticated! Please login to PIO Account."
+    MESSAGE = "You are not authorized! Please log in to PIO Account."
 
 
-class AccountAlreadyAuthenticated(AccountError):
+class AccountAlreadyAuthorized(AccountError):
 
-    MESSAGE = "You are already authenticated with {0} account."
+    MESSAGE = "You are already authorized with {0} account."

platformio/commands/home/command.py

@@ -22,11 +22,7 @@ import click
 
 from platformio import exception
 from platformio.compat import WINDOWS
-from platformio.managers.core import (
+from platformio.managers.core import get_core_package_dir, inject_contrib_pysite
-    build_contrib_pysite_deps,
-    get_core_package_dir,
-    inject_contrib_pysite,
-)
 
 
 @click.command("home", short_help="PIO Home")
@@ -55,12 +51,7 @@ def cli(port, host, no_open, shutdown_timeout):
     # import contrib modules
     inject_contrib_pysite()
 
-    try:
+    from autobahn.twisted.resource import WebSocketResource
-        from autobahn.twisted.resource import WebSocketResource
-    except:  # pylint: disable=bare-except
-        build_contrib_pysite_deps(get_core_package_dir("contrib-pysite"))
-        from autobahn.twisted.resource import WebSocketResource
-
     from twisted.internet import reactor
     from twisted.web import server
     from twisted.internet.error import CannotListenError

platformio/commands/home/rpc/handlers/os.py

@@ -107,7 +107,7 @@ class OSRPC(object):
 
     @staticmethod
     def copy(src, dst):
-        return shutil.copytree(src, dst)
+        return shutil.copytree(src, dst, symlinks=True)
 
     @staticmethod
     def glob(pathnames, root=None):

platformio/commands/home/rpc/handlers/piocore.py

@@ -51,6 +51,8 @@ class MultiThreadingStdStream(object):
     def write(self, value):
         thread_id = thread_get_ident()
         self._ensure_thread_buffer(thread_id)
+        if PY2 and isinstance(value, unicode):  # pylint: disable=undefined-variable
+            value = value.encode()
         return self._buffers[thread_id].write(
             value.decode() if is_bytes(value) else value
         )
@@ -59,8 +61,8 @@ class MultiThreadingStdStream(object):
         result = ""
         try:
             result = self.getvalue()
-            self.truncate(0)
             self.seek(0)
+            self.truncate(0)
         except AttributeError:
             pass
         return result

platformio/commands/home/rpc/handlers/project.py

@@ -300,7 +300,7 @@ class ProjectRPC(object):
             src_dir = config.get_optional_dir("src")
             if os.path.isdir(src_dir):
                 fs.rmtree(src_dir)
-            shutil.copytree(arduino_project_dir, src_dir)
+            shutil.copytree(arduino_project_dir, src_dir, symlinks=True)
         return project_dir
 
     @staticmethod
@@ -313,7 +313,7 @@ class ProjectRPC(object):
             AppRPC.load_state()["storage"]["projectsDir"],
             time.strftime("%y%m%d-%H%M%S-") + os.path.basename(project_dir),
         )
-        shutil.copytree(project_dir, new_project_dir)
+        shutil.copytree(project_dir, new_project_dir, symlinks=True)
 
         state = AppRPC.load_state()
         args = ["init"]

platformio/commands/remote/command.py

@@ -44,7 +44,7 @@ def cli(ctx, agent):
             "https://docs.platformio.org/page/core/installation.html"
         )
     ctx.obj = agent
-    inject_contrib_pysite()
+    inject_contrib_pysite(verify_openssl=True)
 
 
 @cli.group("agent", short_help="Start a new agent or list active")

platformio/commands/remote/factory/client.py

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 from twisted.cred import credentials  # pylint: disable=import-error
-from twisted.internet import protocol, reactor  # pylint: disable=import-error
+from twisted.internet import defer, protocol, reactor  # pylint: disable=import-error
 from twisted.spread import pb  # pylint: disable=import-error
 
 from platformio.app import get_host_id
@@ -35,17 +35,27 @@ class RemoteClientFactory(pb.PBClientFactory, protocol.ReconnectingClientFactory
         self.remote_client.log.info("Successfully connected")
         self.remote_client.log.info("Authenticating")
 
+        auth_token = None
+        try:
+            auth_token = AccountClient().fetch_authentication_token()
+        except Exception as e:  # pylint:disable=broad-except
+            d = defer.Deferred()
+            d.addErrback(self.clientAuthorizationFailed)
+            d.errback(pb.Error(e))
+            return d
+
         d = self.login(
-            credentials.UsernamePassword(
+            credentials.UsernamePassword(auth_token.encode(), get_host_id().encode(),),
-                AccountClient().fetch_authentication_token().encode(),
-                get_host_id().encode(),
-            ),
             client=self.remote_client,
         )
         d.addCallback(self.remote_client.cb_client_authorization_made)
-        d.addErrback(self.remote_client.cb_client_authorization_failed)
+        d.addErrback(self.clientAuthorizationFailed)
         return d
 
+    def clientAuthorizationFailed(self, err):
+        AccountClient.delete_local_session()
+        self.remote_client.cb_client_authorization_failed(err)
+
     def clientConnectionFailed(self, connector, reason):
         self.remote_client.log.warn(
             "Could not connect to PIO Remote Cloud. Reconnecting..."

platformio/commands/system/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

platformio/commands/system/command.py

@@ -0,0 +1,96 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import subprocess
+
+import click
+
+from platformio import proc
+from platformio.commands.system.completion import (
+    get_completion_install_path,
+    install_completion_code,
+    uninstall_completion_code,
+)
+
+
+@click.group("system", short_help="Miscellaneous system commands")
+def cli():
+    pass
+
+
+@cli.group("completion", short_help="Shell completion support")
+def completion():
+    # pylint: disable=import-error,import-outside-toplevel
+    try:
+        import click_completion  # pylint: disable=unused-import,unused-variable
+    except ImportError:
+        click.echo("Installing dependent packages...")
+        subprocess.check_call(
+            [proc.get_pythonexe_path(), "-m", "pip", "install", "click-completion"],
+        )
+
+
+@completion.command("install", short_help="Install shell completion files/code")
+@click.option(
+    "--shell",
+    default=None,
+    type=click.Choice(["fish", "bash", "zsh", "powershell", "auto"]),
+    help="The shell type, default=auto",
+)
+@click.option(
+    "--path",
+    type=click.Path(file_okay=True, dir_okay=False, readable=True, resolve_path=True),
+    help="Custom installation path of the code to be evaluated by the shell. "
+    "The standard installation path is used by default.",
+)
+def completion_install(shell, path):
+
+    import click_completion  # pylint: disable=import-outside-toplevel,import-error
+
+    shell = shell or click_completion.get_auto_shell()
+    path = path or get_completion_install_path(shell)
+    install_completion_code(shell, path)
+    click.echo(
+        "PlatformIO CLI completion has been installed for %s shell to %s \n"
+        "Please restart a current shell session."
+        % (click.style(shell, fg="cyan"), click.style(path, fg="blue"))
+    )
+
+
+@completion.command("uninstall", short_help="Uninstall shell completion files/code")
+@click.option(
+    "--shell",
+    default=None,
+    type=click.Choice(["fish", "bash", "zsh", "powershell", "auto"]),
+    help="The shell type, default=auto",
+)
+@click.option(
+    "--path",
+    type=click.Path(file_okay=True, dir_okay=False, readable=True, resolve_path=True),
+    help="Custom installation path of the code to be evaluated by the shell. "
+    "The standard installation path is used by default.",
+)
+def completion_uninstall(shell, path):
+
+    import click_completion  # pylint: disable=import-outside-toplevel,import-error
+
+    shell = shell or click_completion.get_auto_shell()
+    path = path or get_completion_install_path(shell)
+    uninstall_completion_code(shell, path)
+    click.echo(
+        "PlatformIO CLI completion has been uninstalled for %s shell from %s \n"
+        "Please restart a current shell session."
+        % (click.style(shell, fg="cyan"), click.style(path, fg="blue"))
+    )

platformio/commands/system/completion.py

@@ -0,0 +1,73 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import subprocess
+
+import click
+
+
+def get_completion_install_path(shell):
+    home_dir = os.path.expanduser("~")
+    prog_name = click.get_current_context().find_root().info_name
+    if shell == "fish":
+        return os.path.join(
+            home_dir, ".config", "fish", "completions", "%s.fish" % prog_name
+        )
+    if shell == "bash":
+        return os.path.join(home_dir, ".bash_completion")
+    if shell == "zsh":
+        return os.path.join(home_dir, ".zshrc")
+    if shell == "powershell":
+        return subprocess.check_output(
+            ["powershell", "-NoProfile", "echo $profile"]
+        ).strip()
+    raise click.ClickException("%s is not supported." % shell)
+
+
+def is_completion_code_installed(shell, path):
+    if shell == "fish" or not os.path.exists(path):
+        return False
+
+    import click_completion  # pylint: disable=import-error,import-outside-toplevel
+
+    with open(path) as fp:
+        return click_completion.get_code(shell=shell) in fp.read()
+
+
+def install_completion_code(shell, path):
+    import click_completion  # pylint: disable=import-error,import-outside-toplevel
+
+    if is_completion_code_installed(shell, path):
+        return None
+
+    return click_completion.install(shell=shell, path=path, append=shell != "fish")
+
+
+def uninstall_completion_code(shell, path):
+    if not os.path.exists(path):
+        return True
+    if shell == "fish":
+        os.remove(path)
+        return True
+
+    import click_completion  # pylint: disable=import-error,import-outside-toplevel
+
+    with open(path, "r+") as fp:
+        contents = fp.read()
+        fp.seek(0)
+        fp.truncate()
+        fp.write(contents.replace(click_completion.get_code(shell=shell), ""))
+
+    return True

platformio/exception.py

@@ -232,8 +232,8 @@ class InternetIsOffline(UserSideException):
 
     MESSAGE = (
         "You are not connected to the Internet.\n"
-        "If you build a project first time, we need Internet connection "
+        "PlatformIO needs the Internet connection to"
-        "to install all dependencies and toolchains."
+        " download dependent packages or to work with PIO Account."
     )
 
 

platformio/managers/core.py

@@ -24,7 +24,7 @@ from platformio.proc import get_pythonexe_path
 from platformio.project.config import ProjectConfig
 
 CORE_PACKAGES = {
-    "contrib-piohome": "~3.2.0",
+    "contrib-piohome": "~3.2.1",
     "contrib-pysite": "~2.%d%d.0" % (sys.version_info.major, sys.version_info.minor),
     "tool-unity": "~1.20500.0",
     "tool-scons": "~2.20501.7" if PY2 else "~3.30102.0",
@@ -100,15 +100,27 @@ def update_core_packages(only_check=False, silent=False):
     return True
 
 
-def inject_contrib_pysite():
+def inject_contrib_pysite(verify_openssl=False):
-    from site import addsitedir  # pylint: disable=import-outside-toplevel
+    # pylint: disable=import-outside-toplevel
+    from site import addsitedir
 
     contrib_pysite_dir = get_core_package_dir("contrib-pysite")
     if contrib_pysite_dir in sys.path:
-        return
+        return True
     addsitedir(contrib_pysite_dir)
     sys.path.insert(0, contrib_pysite_dir)
 
+    if not verify_openssl:
+        return True
+
+    try:
+        # pylint: disable=import-error,unused-import,unused-variable
+        from OpenSSL import SSL
+    except:  # pylint: disable=bare-except
+        build_contrib_pysite_deps(get_core_package_dir("contrib-pysite"))
+
+    return True
+
 
 def build_contrib_pysite_deps(target_dir):
     if os.path.isdir(target_dir):
@@ -126,13 +138,13 @@ def build_contrib_pysite_deps(target_dir):
 
     pythonexe = get_pythonexe_path()
     for dep in get_contrib_pysite_deps():
-        subprocess.call(
+        subprocess.check_call(
             [
                 pythonexe,
                 "-m",
                 "pip",
                 "install",
-                "--no-cache-dir",
+                # "--no-cache-dir",
                 "--no-compile",
                 "-t",
                 target_dir,
@@ -146,11 +158,11 @@ def get_contrib_pysite_deps():
     sys_type = util.get_systype()
     py_version = "%d%d" % (sys.version_info.major, sys.version_info.minor)
 
-    twisted_version = "19.7.0"
+    twisted_version = "19.10.0" if PY2 else "20.3.0"
     result = [
         "twisted == %s" % twisted_version,
-        "autobahn == 19.10.1",
+        "autobahn == 20.4.3",
-        "json-rpc == 1.12.1",
+        "json-rpc == 1.13.0",
     ]
 
     # twisted[tls], see setup.py for %twisted_version%
@@ -159,12 +171,12 @@ def get_contrib_pysite_deps():
     )
 
     # zeroconf
-    if sys.version_info.major < 3:
+    if PY2:
         result.append(
             "https://github.com/ivankravets/python-zeroconf/" "archive/pio-py27.zip"
         )
     else:
-        result.append("zeroconf == 0.23.0")
+        result.append("zeroconf == 0.26.0")
 
     if "windows" in sys_type:
         result.append("pypiwin32 == 223")

platformio/managers/package.py

@@ -475,7 +475,7 @@ class PkgInstallerMixin(object):
                     self.unpack(_url, tmp_dir)
                 else:
                     fs.rmtree(tmp_dir)
-                    shutil.copytree(_url, tmp_dir)
+                    shutil.copytree(_url, tmp_dir, symlinks=True)
             elif url.startswith(("http://", "https://")):
                 dlpath = self.download(url, tmp_dir, sha1)
                 assert isfile(dlpath)
@@ -582,7 +582,11 @@ class PkgInstallerMixin(object):
         # remove previous/not-satisfied package
         if isdir(pkg_dir):
             fs.rmtree(pkg_dir)
-        shutil.move(tmp_dir, pkg_dir)
+        shutil.copytree(tmp_dir, pkg_dir, symlinks=True)
+        try:
+            shutil.rmtree(tmp_dir)
+        except:  # pylint: disable=bare-except
+            pass
         assert isdir(pkg_dir)
         self.cache_reset()
         return pkg_dir

platformio/package/manifest/schema.py

@@ -242,7 +242,7 @@ class ManifestSchema(BaseSchema):
     def load_spdx_licenses():
         r = requests.get(
             "https://raw.githubusercontent.com/spdx/license-list-data"
-            "/v3.8/json/licenses.json"
+            "/v3.9/json/licenses.json"
         )
         r.raise_for_status()
         return r.json()

tests/commands/test_account.py

@@ -20,6 +20,14 @@ import pytest
 
 from platformio.commands.account.command import cli as cmd_account
 
+pytestmark = pytest.mark.skipif(
+    not (
+        os.environ.get("PLATFORMIO_TEST_ACCOUNT_LOGIN")
+        and os.environ.get("PLATFORMIO_TEST_ACCOUNT_PASSWORD")
+    ),
+    reason="requires PLATFORMIO_TEST_ACCOUNT_LOGIN, PLATFORMIO_TEST_ACCOUNT_PASSWORD environ variables",
+)
+
 
 @pytest.fixture(scope="session")
 def credentials():
@@ -93,7 +101,7 @@ def test_account_login(clirunner, credentials, validate_cliresult, isolated_pio_
         )
         assert result.exit_code > 0
         assert result.exception
-        assert "You are already authenticated with" in str(result.exception)
+        assert "You are already authorized with" in str(result.exception)
     finally:
         clirunner.invoke(cmd_account, ["logout"])
 
@@ -113,7 +121,7 @@ def test_account_logout(clirunner, credentials, validate_cliresult, isolated_pio
         result = clirunner.invoke(cmd_account, ["logout"])
         assert result.exit_code > 0
         assert result.exception
-        assert "You are not authenticated! Please login to PIO Account" in str(
+        assert "You are not authorized! Please log in to PIO Account" in str(
             result.exception
         )
     finally:
@@ -192,7 +200,7 @@ def test_account_password_change(
         )
         assert result.exit_code > 0
         assert result.exception
-        assert "You are not authenticated! Please login to PIO Account" in str(
+        assert "You are not authorized! Please log in to PIO Account" in str(
             result.exception
         )
 
@@ -250,7 +258,7 @@ def test_account_token_with_invalid_password(
         )
         assert result.exit_code > 0
         assert result.exception
-        assert "You are not authenticated! Please login to PIO Account" in str(
+        assert "You are not authorized! Please log in to PIO Account" in str(
             result.exception
         )
 
@@ -302,7 +310,7 @@ def test_account_token(clirunner, credentials, validate_cliresult, isolated_pio_
         )
         assert result.exit_code > 0
         assert result.exception
-        assert "You are not authenticated! Please login to PIO Account" in str(
+        assert "You are not authorized! Please log in to PIO Account" in str(
             result.exception
         )
 
@@ -371,7 +379,7 @@ def test_account_summary(clirunner, credentials, validate_cliresult, isolated_pi
         result = clirunner.invoke(cmd_account, ["show"],)
         assert result.exit_code > 0
         assert result.exception
-        assert "You are not authenticated! Please login to PIO Account" in str(
+        assert "You are not authorized! Please log in to PIO Account" in str(
             result.exception
         )
 
@@ -381,6 +389,16 @@ def test_account_summary(clirunner, credentials, validate_cliresult, isolated_pi
         )
         validate_cliresult(result)
 
+        result = clirunner.invoke(cmd_account, ["show", "--json-output", "--offline"])
+        validate_cliresult(result)
+        json_result = json.loads(result.output.strip())
+        assert not json_result.get("user_id")
+        assert json_result.get("profile")
+        assert json_result.get("profile").get("username")
+        assert json_result.get("profile").get("email")
+        assert not json_result.get("packages")
+        assert not json_result.get("subscriptions")
+
         result = clirunner.invoke(cmd_account, ["show"])
         validate_cliresult(result)
         assert credentials["login"] in result.output
@@ -407,12 +425,19 @@ def test_account_summary(clirunner, credentials, validate_cliresult, isolated_pi
         result = clirunner.invoke(cmd_account, ["show", "--json-output", "--offline"])
         validate_cliresult(result)
         json_result = json.loads(result.output.strip())
-        assert not json_result.get("user_id")
+        assert json_result.get("user_id")
         assert json_result.get("profile")
         assert json_result.get("profile").get("username")
         assert json_result.get("profile").get("email")
-        assert not json_result.get("packages")
+        assert credentials["login"] == json_result.get("profile").get(
-        assert not json_result.get("subscriptions")
+            "username"
+        ) or credentials["login"] == json_result.get("profile").get("email")
+        assert json_result.get("profile").get("firstname")
+        assert json_result.get("profile").get("lastname")
+        assert json_result.get("packages")
+        assert json_result.get("packages")[0].get("name")
+        assert json_result.get("packages")[0].get("path")
+        assert json_result.get("subscriptions") is not None
     finally:
         clirunner.invoke(cmd_account, ["logout"])
 
@@ -427,7 +452,7 @@ def test_account_profile_update_with_invalid_password(
         )
         assert result.exit_code > 0
         assert result.exception
-        assert "You are not authenticated! Please login to PIO Account" in str(
+        assert "You are not authorized! Please log in to PIO Account" in str(
             result.exception
         )
 
@@ -460,7 +485,7 @@ def test_account_profile_update_only_firstname_and_lastname(
         )
         assert result.exit_code > 0
         assert result.exception
-        assert "You are not authenticated! Please login to PIO Account" in str(
+        assert "You are not authorized! Please log in to PIO Account" in str(
             result.exception
         )
 
@@ -508,7 +533,7 @@ def test_account_profile_update(
         )
         assert result.exit_code > 0
         assert result.exception
-        assert "You are not authenticated! Please login to PIO Account" in str(
+        assert "You are not authorized! Please log in to PIO Account" in str(
             result.exception
         )
 
@@ -549,7 +574,7 @@ def test_account_profile_update(
         result = clirunner.invoke(cmd_account, ["show"],)
         assert result.exit_code > 0
         assert result.exception
-        assert "You are not authenticated! Please login to PIO Account" in str(
+        assert "You are not authorized! Please log in to PIO Account" in str(
             result.exception
         )
 

tox.ini

@@ -13,13 +13,13 @@
 # limitations under the License.
 
 [tox]
-envlist = py27,py37
+envlist = py27,py37,py38
 
 [testenv]
 passenv = *
 usedevelop = True
 deps =
-    py36,py37: black
+    py36,py37,py38: black
     isort
     pylint
     pytest