From 5f79ab34f5475ce074efb09353a54af784496749 Mon Sep 17 00:00:00 2001
From: Ivan Kravets <me@ikravets.com>
Date: Wed, 29 Apr 2020 12:40:04 +0300
Subject: [PATCH 01/42] Automatically build ``contrib-pysite`` package on a
 target machine when pre-built package is not compatible // Resolve #3482

---
 HISTORY.rst                           |  5 +++++
 platformio/commands/home/command.py   | 13 ++-----------
 platformio/commands/remote/command.py |  2 +-
 platformio/managers/core.py           | 27 +++++++++++++++++++--------
 4 files changed, 27 insertions(+), 20 deletions(-)

diff --git a/HISTORY.rst b/HISTORY.rst
index a2bf6ad8..7b124df6 100644
--- a/HISTORY.rst
+++ b/HISTORY.rst
@@ -6,6 +6,11 @@ Release Notes
 PlatformIO Core 4
 -----------------
 
+4.3.4 (2020-??-??)
+~~~~~~~~~~~~~~~~~~
+
+* Automatically build ``contrib-pysite`` package on a target machine when pre-built package is not compatible (`issue #3482 <https://github.com/platformio/platformio-core/issues/3482>`_)
+
 4.3.3 (2020-04-28)
 ~~~~~~~~~~~~~~~~~~
 
diff --git a/platformio/commands/home/command.py b/platformio/commands/home/command.py
index 1eaad046..32d28063 100644
--- a/platformio/commands/home/command.py
+++ b/platformio/commands/home/command.py
@@ -22,11 +22,7 @@ import click
 
 from platformio import exception
 from platformio.compat import WINDOWS
-from platformio.managers.core import (
-    build_contrib_pysite_deps,
-    get_core_package_dir,
-    inject_contrib_pysite,
-)
+from platformio.managers.core import get_core_package_dir, inject_contrib_pysite
 
 
 @click.command("home", short_help="PIO Home")
@@ -55,12 +51,7 @@ def cli(port, host, no_open, shutdown_timeout):
     # import contrib modules
     inject_contrib_pysite()
 
-    try:
-        from autobahn.twisted.resource import WebSocketResource
-    except:  # pylint: disable=bare-except
-        build_contrib_pysite_deps(get_core_package_dir("contrib-pysite"))
-        from autobahn.twisted.resource import WebSocketResource
-
+    from autobahn.twisted.resource import WebSocketResource
     from twisted.internet import reactor
     from twisted.web import server
     from twisted.internet.error import CannotListenError
diff --git a/platformio/commands/remote/command.py b/platformio/commands/remote/command.py
index d4a1cc34..f9e24c29 100644
--- a/platformio/commands/remote/command.py
+++ b/platformio/commands/remote/command.py
@@ -44,7 +44,7 @@ def cli(ctx, agent):
             "https://docs.platformio.org/page/core/installation.html"
         )
     ctx.obj = agent
-    inject_contrib_pysite()
+    inject_contrib_pysite(verify_openssl=True)
 
 
 @cli.group("agent", short_help="Start a new agent or list active")
diff --git a/platformio/managers/core.py b/platformio/managers/core.py
index cb6ad31c..64b4a8b6 100644
--- a/platformio/managers/core.py
+++ b/platformio/managers/core.py
@@ -100,15 +100,26 @@ def update_core_packages(only_check=False, silent=False):
     return True
 
 
-def inject_contrib_pysite():
-    from site import addsitedir  # pylint: disable=import-outside-toplevel
+def inject_contrib_pysite(verify_openssl=False):
+    # pylint: disable=import-outside-toplevel
+    from site import addsitedir
 
     contrib_pysite_dir = get_core_package_dir("contrib-pysite")
     if contrib_pysite_dir in sys.path:
-        return
+        return True
     addsitedir(contrib_pysite_dir)
     sys.path.insert(0, contrib_pysite_dir)
 
+    if not verify_openssl:
+        return True
+
+    try:
+        from OpenSSL import SSL  # pylint: disable=import-error,unused-import
+    except:  # pylint: disable=bare-except
+        build_contrib_pysite_deps(get_core_package_dir("contrib-pysite"))
+
+    return True
+
 
 def build_contrib_pysite_deps(target_dir):
     if os.path.isdir(target_dir):
@@ -146,11 +157,11 @@ def get_contrib_pysite_deps():
     sys_type = util.get_systype()
     py_version = "%d%d" % (sys.version_info.major, sys.version_info.minor)
 
-    twisted_version = "19.7.0"
+    twisted_version = "19.10.0" if PY2 else "20.3.0"
     result = [
         "twisted == %s" % twisted_version,
-        "autobahn == 19.10.1",
-        "json-rpc == 1.12.1",
+        "autobahn == 20.4.3",
+        "json-rpc == 1.13.0",
     ]
 
     # twisted[tls], see setup.py for %twisted_version%
@@ -159,12 +170,12 @@ def get_contrib_pysite_deps():
     )
 
     # zeroconf
-    if sys.version_info.major < 3:
+    if PY2:
         result.append(
             "https://github.com/ivankravets/python-zeroconf/" "archive/pio-py27.zip"
         )
     else:
-        result.append("zeroconf == 0.23.0")
+        result.append("zeroconf == 0.26.0")
 
     if "windows" in sys_type:
         result.append("pypiwin32 == 223")

From 2c2309acac4117eb155179c30a7b6c396119dde1 Mon Sep 17 00:00:00 2001
From: Ivan Kravets <me@ikravets.com>
Date: Wed, 29 Apr 2020 12:41:56 +0300
Subject: [PATCH 02/42] Bump version to 4.3.4a1

---
 platformio/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/platformio/__init__.py b/platformio/__init__.py
index 78bda530..b4381155 100644
--- a/platformio/__init__.py
+++ b/platformio/__init__.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-VERSION = (4, 3, 3)
+VERSION = (4, 3, "4a1")
 __version__ = ".".join([str(s) for s in VERSION])
 
 __title__ = "platformio"

From f7995ce49a19bc77a56862af0d6e6f9bde92ad63 Mon Sep 17 00:00:00 2001
From: Ivan Kravets <me@ikravets.com>
Date: Wed, 29 Apr 2020 12:56:54 +0300
Subject: [PATCH 03/42] PyLint fix

---
 platformio/managers/core.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/platformio/managers/core.py b/platformio/managers/core.py
index 64b4a8b6..a1bfef08 100644
--- a/platformio/managers/core.py
+++ b/platformio/managers/core.py
@@ -114,7 +114,8 @@ def inject_contrib_pysite(verify_openssl=False):
         return True
 
     try:
-        from OpenSSL import SSL  # pylint: disable=import-error,unused-import
+        # pylint: disable=import-error,unused-import,unused-variable
+        from OpenSSL import SSL
     except:  # pylint: disable=bare-except
         build_contrib_pysite_deps(get_core_package_dir("contrib-pysite"))
 

From 75abe8a0af7e4bd7b9de4c7a1f704961caef6412 Mon Sep 17 00:00:00 2001
From: Ivan Kravets <me@ikravets.com>
Date: Fri, 1 May 2020 23:36:29 +0300
Subject: [PATCH 04/42] Sync docs

---
 docs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs b/docs
index 790be9c1..3b956734 160000
--- a/docs
+++ b/docs
@@ -1 +1 @@
-Subproject commit 790be9c1994750272d1312b0b4f4b50ba1973071
+Subproject commit 3b95673463c80a89ac86de5144d30f3de120583d

From fd04f31c5f8c7d93d758144008b6442df93fe8e5 Mon Sep 17 00:00:00 2001
From: Ivan Kravets <me@ikravets.com>
Date: Fri, 1 May 2020 23:37:38 +0300
Subject: [PATCH 05/42] Update link to CLA provider

---
 CONTRIBUTING.md | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index d9764348..ec53bebb 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -1,20 +1,20 @@
 Contributing
 ------------
 
-To get started, <a href="https://www.clahub.com/agreements/platformio/platformio-core">sign the Contributor License Agreement</a>.
+To get started, <a href="https://cla-assistant.io/platformio/platformio-core">sign the Contributor License Agreement</a>.
 
 1. Fork the repository on GitHub.
 2. Clone repository `git clone --recursive https://github.com/YourGithubUsername/platformio-core.git`
 3. Run `pip install tox`
-4. Go to the root of project where is located `tox.ini` and run `tox -e py27`
+4. Go to the root of project where is located `tox.ini` and run `tox -e py37`
 5. Activate current development environment:
 
-   * Windows: `.tox\py27\Scripts\activate`
-   * Bash/ZSH: `source .tox/py27/bin/activate`
-   * Fish: `source .tox/py27/bin/activate.fish`
+   * Windows: `.tox\py37\Scripts\activate`
+   * Bash/ZSH: `source .tox/py37/bin/activate`
+   * Fish: `source .tox/py37/bin/activate.fish`
 
 6. Make changes to code, documentation, etc.
-7. Lint source code `make lint`
+7. Lint source code `make before-commit`
 8. Run the tests `make test`
 9. Build documentation `tox -e docs` (creates a directory _build under docs where you can find the html)
 10. Commit changes to your forked repository

From b82eaca45ef409d81828901a1dec871653411ecf Mon Sep 17 00:00:00 2001
From: Ivan Kravets <me@ikravets.com>
Date: Sat, 2 May 2020 15:29:24 +0300
Subject: [PATCH 06/42] Enable caching for PIP when building contrib-pysite

---
 platformio/managers/core.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/platformio/managers/core.py b/platformio/managers/core.py
index a1bfef08..cd511e1a 100644
--- a/platformio/managers/core.py
+++ b/platformio/managers/core.py
@@ -144,7 +144,7 @@ def build_contrib_pysite_deps(target_dir):
                 "-m",
                 "pip",
                 "install",
-                "--no-cache-dir",
+                # "--no-cache-dir",
                 "--no-compile",
                 "-t",
                 target_dir,

From 7181b7632bfd830683df2182dd9a9805a33d65f3 Mon Sep 17 00:00:00 2001
From: Ivan Kravets <me@ikravets.com>
Date: Sun, 3 May 2020 11:06:26 +0300
Subject: [PATCH 07/42] Docs: Increase content width

---
 docs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs b/docs
index 3b956734..2e44951f 160000
--- a/docs
+++ b/docs
@@ -1 +1 @@
-Subproject commit 3b95673463c80a89ac86de5144d30f3de120583d
+Subproject commit 2e44951feb6458976cb2ddfe6d73dbe6ea903fc7

From 2ab47b79684a835b129340f2cec5238f43b48648 Mon Sep 17 00:00:00 2001
From: ShahRustam <shahrustam98@gmail.com>
Date: Mon, 4 May 2020 13:14:52 +0300
Subject: [PATCH 08/42] Remove account state item if refreshing token failed
 (#3487)

---
 platformio/commands/account/client.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/platformio/commands/account/client.py b/platformio/commands/account/client.py
index 5381dab9..c2bc3c5f 100644
--- a/platformio/commands/account/client.py
+++ b/platformio/commands/account/client.py
@@ -206,9 +206,12 @@ class AccountClient(object):
                     self.api_base_url + "/v1/login",
                     headers={"Authorization": "Bearer %s" % auth.get("refresh_token")},
                 )
-                result = self.raise_error_from_response(response)
-                app.set_state_item("account", result)
-                return result.get("auth").get("access_token")
+                try:
+                    result = self.raise_error_from_response(response)
+                    app.set_state_item("account", result)
+                    return result.get("auth").get("access_token")
+                except exception.AccountError:
+                    app.delete_state_item("account")
         raise exception.AccountNotAuthenticated()
 
     @staticmethod

From 44ee7d6a6be8ed3f9e56a06b1549a1b426339c6b Mon Sep 17 00:00:00 2001
From: Ivan Kravets <me@ikravets.com>
Date: Mon, 4 May 2020 15:50:11 +0300
Subject: [PATCH 09/42] Docs: Sync ESP8266 dev-platform

---
 docs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs b/docs
index 2e44951f..d910093e 160000
--- a/docs
+++ b/docs
@@ -1 +1 @@
-Subproject commit 2e44951feb6458976cb2ddfe6d73dbe6ea903fc7
+Subproject commit d910093eded0f01cdf0421488bf8b108e84f2f1e

From 8480ebde893f20805e97c77fb412ce4e1b7e4e41 Mon Sep 17 00:00:00 2001
From: Ivan Kravets <me@ikravets.com>
Date: Wed, 6 May 2020 12:27:29 +0300
Subject: [PATCH 10/42] Rename "authenticating" to "authorizing"  wording for
 PIO Account

---
 platformio/commands/account/client.py    | 50 ++++++++++++------------
 platformio/commands/account/command.py   |  2 +-
 platformio/commands/account/exception.py |  8 ++--
 platformio/managers/core.py              |  2 +-
 4 files changed, 32 insertions(+), 30 deletions(-)

diff --git a/platformio/commands/account/client.py b/platformio/commands/account/client.py
index c2bc3c5f..312b9f36 100644
--- a/platformio/commands/account/client.py
+++ b/platformio/commands/account/client.py
@@ -43,13 +43,24 @@ class AccountClient(object):
         adapter = requests.adapters.HTTPAdapter(max_retries=retry)
         self._session.mount(api_base_url, adapter)
 
+    @staticmethod
+    def get_refresh_token():
+        try:
+            return app.get_state_item("account").get("auth").get("refresh_token")
+        except:  # pylint:disable=bare-except
+            raise exception.AccountNotAuthorized()
+
+    @staticmethod
+    def delete_local_session():
+        app.delete_state_item("account")
+
     def login(self, username, password):
         try:
             self.fetch_authentication_token()
         except:  # pylint:disable=bare-except
             pass
         else:
-            raise exception.AccountAlreadyAuthenticated(
+            raise exception.AccountAlreadyAuthorized(
                 app.get_state_item("account", {}).get("email", "")
             )
 
@@ -67,7 +78,7 @@ class AccountClient(object):
         except:  # pylint:disable=bare-except
             pass
         else:
-            raise exception.AccountAlreadyAuthenticated(
+            raise exception.AccountAlreadyAuthorized(
                 app.get_state_item("account", {}).get("email", "")
             )
 
@@ -83,7 +94,8 @@ class AccountClient(object):
         try:
             refresh_token = self.get_refresh_token()
         except:  # pylint:disable=bare-except
-            raise exception.AccountNotAuthenticated()
+            raise exception.AccountNotAuthorized()
+        self.delete_local_session()
         response = requests.post(
             self.api_base_url + "/v1/logout", data={"refresh_token": refresh_token},
         )
@@ -91,14 +103,13 @@ class AccountClient(object):
             self.raise_error_from_response(response)
         except exception.AccountError:
             pass
-        app.delete_state_item("account")
         return True
 
     def change_password(self, old_password, new_password):
         try:
             token = self.fetch_authentication_token()
         except:  # pylint:disable=bare-except
-            raise exception.AccountNotAuthenticated()
+            raise exception.AccountNotAuthorized()
         response = self._session.post(
             self.api_base_url + "/v1/password",
             headers={"Authorization": "Bearer %s" % token},
@@ -115,7 +126,7 @@ class AccountClient(object):
         except:  # pylint:disable=bare-except
             pass
         else:
-            raise exception.AccountAlreadyAuthenticated(
+            raise exception.AccountAlreadyAuthorized(
                 app.get_state_item("account", {}).get("email", "")
             )
 
@@ -135,7 +146,7 @@ class AccountClient(object):
         try:
             token = self.fetch_authentication_token()
         except:  # pylint:disable=bare-except
-            raise exception.AccountNotAuthenticated()
+            raise exception.AccountNotAuthorized()
         response = self._session.post(
             self.api_base_url + "/v1/token",
             headers={"Authorization": "Bearer %s" % token},
@@ -153,7 +164,7 @@ class AccountClient(object):
         try:
             token = self.fetch_authentication_token()
         except:  # pylint:disable=bare-except
-            raise exception.AccountNotAuthenticated()
+            raise exception.AccountNotAuthorized()
         response = self._session.get(
             self.api_base_url + "/v1/profile",
             headers={"Authorization": "Bearer %s" % token},
@@ -164,7 +175,7 @@ class AccountClient(object):
         try:
             token = self.fetch_authentication_token()
         except:  # pylint:disable=bare-except
-            raise exception.AccountNotAuthenticated()
+            raise exception.AccountNotAuthorized()
         profile["current_password"] = current_password
         response = self._session.put(
             self.api_base_url + "/v1/profile",
@@ -177,7 +188,7 @@ class AccountClient(object):
         if offline:
             account = app.get_state_item("account")
             if not account:
-                raise exception.AccountNotAuthenticated()
+                raise exception.AccountNotAuthorized()
             return {
                 "profile": {
                     "email": account.get("email"),
@@ -187,7 +198,7 @@ class AccountClient(object):
         try:
             token = self.fetch_authentication_token()
         except:  # pylint:disable=bare-except
-            raise exception.AccountNotAuthenticated()
+            raise exception.AccountNotAuthorized()
         response = self._session.get(
             self.api_base_url + "/v1/summary",
             headers={"Authorization": "Bearer %s" % token},
@@ -211,19 +222,10 @@ class AccountClient(object):
                     app.set_state_item("account", result)
                     return result.get("auth").get("access_token")
                 except exception.AccountError:
-                    app.delete_state_item("account")
-        raise exception.AccountNotAuthenticated()
+                    self.delete_local_session()
+        raise exception.AccountNotAuthorized()
 
-    @staticmethod
-    def get_refresh_token():
-        try:
-            auth = app.get_state_item("account").get("auth").get("refresh_token")
-            return auth
-        except:  # pylint:disable=bare-except
-            raise exception.AccountNotAuthenticated()
-
-    @staticmethod
-    def raise_error_from_response(response, expected_codes=(200, 201, 202)):
+    def raise_error_from_response(self, response, expected_codes=(200, 201, 202)):
         if response.status_code in expected_codes:
             try:
                 return response.json()
@@ -234,5 +236,5 @@ class AccountClient(object):
         except (KeyError, ValueError):
             message = response.text
         if "Authorization session has been expired" in message:
-            app.delete_state_item("account")
+            self.delete_local_session()
         raise exception.AccountError(message)
diff --git a/platformio/commands/account/command.py b/platformio/commands/account/command.py
index 539f1a5b..0177d00a 100644
--- a/platformio/commands/account/command.py
+++ b/platformio/commands/account/command.py
@@ -167,7 +167,7 @@ def account_update(current_password, **kwargs):
         return None
     try:
         client.logout()
-    except exception.AccountNotAuthenticated:
+    except exception.AccountNotAuthorized:
         pass
     if email_changed:
         return click.secho(
diff --git a/platformio/commands/account/exception.py b/platformio/commands/account/exception.py
index 213be0e1..a1a0059e 100644
--- a/platformio/commands/account/exception.py
+++ b/platformio/commands/account/exception.py
@@ -20,11 +20,11 @@ class AccountError(PlatformioException):
     MESSAGE = "{0}"
 
 
-class AccountNotAuthenticated(AccountError):
+class AccountNotAuthorized(AccountError):
 
-    MESSAGE = "You are not authenticated! Please login to PIO Account."
+    MESSAGE = "You are not authorized! Please log in to PIO Account."
 
 
-class AccountAlreadyAuthenticated(AccountError):
+class AccountAlreadyAuthorized(AccountError):
 
-    MESSAGE = "You are already authenticated with {0} account."
+    MESSAGE = "You are already authorized with {0} account."
diff --git a/platformio/managers/core.py b/platformio/managers/core.py
index cd511e1a..677a1c81 100644
--- a/platformio/managers/core.py
+++ b/platformio/managers/core.py
@@ -24,7 +24,7 @@ from platformio.proc import get_pythonexe_path
 from platformio.project.config import ProjectConfig
 
 CORE_PACKAGES = {
-    "contrib-piohome": "~3.2.0",
+    "contrib-piohome": "~3.2.1",
     "contrib-pysite": "~2.%d%d.0" % (sys.version_info.major, sys.version_info.minor),
     "tool-unity": "~1.20500.0",
     "tool-scons": "~2.20501.7" if PY2 else "~3.30102.0",

From f78ffaded03bfd875264124623612c8ffa7669ce Mon Sep 17 00:00:00 2001
From: Ivan Kravets <me@ikravets.com>
Date: Wed, 6 May 2020 12:29:01 +0300
Subject: [PATCH 11/42] Remove local PIO Account session from PIO Remote when
 token is broken

---
 platformio/commands/remote/factory/client.py | 22 ++++++++++++++------
 1 file changed, 16 insertions(+), 6 deletions(-)

diff --git a/platformio/commands/remote/factory/client.py b/platformio/commands/remote/factory/client.py
index 202c7da6..26abe080 100644
--- a/platformio/commands/remote/factory/client.py
+++ b/platformio/commands/remote/factory/client.py
@@ -13,7 +13,7 @@
 # limitations under the License.
 
 from twisted.cred import credentials  # pylint: disable=import-error
-from twisted.internet import protocol, reactor  # pylint: disable=import-error
+from twisted.internet import defer, protocol, reactor  # pylint: disable=import-error
 from twisted.spread import pb  # pylint: disable=import-error
 
 from platformio.app import get_host_id
@@ -35,17 +35,27 @@ class RemoteClientFactory(pb.PBClientFactory, protocol.ReconnectingClientFactory
         self.remote_client.log.info("Successfully connected")
         self.remote_client.log.info("Authenticating")
 
+        auth_token = None
+        try:
+            auth_token = AccountClient().fetch_authentication_token()
+        except Exception as e:  # pylint:disable=broad-except
+            d = defer.Deferred()
+            d.addErrback(self.clientAuthorizationFailed)
+            d.errback(pb.Error(e))
+            return d
+
         d = self.login(
-            credentials.UsernamePassword(
-                AccountClient().fetch_authentication_token().encode(),
-                get_host_id().encode(),
-            ),
+            credentials.UsernamePassword(auth_token.encode(), get_host_id().encode(),),
             client=self.remote_client,
         )
         d.addCallback(self.remote_client.cb_client_authorization_made)
-        d.addErrback(self.remote_client.cb_client_authorization_failed)
+        d.addErrback(self.clientAuthorizationFailed)
         return d
 
+    def clientAuthorizationFailed(self, err):
+        AccountClient.delete_local_session()
+        self.remote_client.cb_client_authorization_failed(err)
+
     def clientConnectionFailed(self, connector, reason):
         self.remote_client.log.warn(
             "Could not connect to PIO Remote Cloud. Reconnecting..."

From f32dbeeb6d505820a95953aa1dd4724265cd97a6 Mon Sep 17 00:00:00 2001
From: Ivan Kravets <me@ikravets.com>
Date: Wed, 6 May 2020 12:30:41 +0300
Subject: [PATCH 12/42] Bump version to 4.3.4a2

---
 docs                   | 2 +-
 platformio/__init__.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs b/docs
index d910093e..2fd23581 160000
--- a/docs
+++ b/docs
@@ -1 +1 @@
-Subproject commit d910093eded0f01cdf0421488bf8b108e84f2f1e
+Subproject commit 2fd2358162db56ddbc344f0a874953ac96ea441f
diff --git a/platformio/__init__.py b/platformio/__init__.py
index b4381155..6fe0066d 100644
--- a/platformio/__init__.py
+++ b/platformio/__init__.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-VERSION = (4, 3, "4a1")
+VERSION = (4, 3, "4a2")
 __version__ = ".".join([str(s) for s in VERSION])
 
 __title__ = "platformio"

From 7e7a6d7807d3e58e19d5d6e320555e16f7d07444 Mon Sep 17 00:00:00 2001
From: ShahRustam <shahrustam98@gmail.com>
Date: Wed, 6 May 2020 19:25:24 +0300
Subject: [PATCH 13/42] Skip account tests if env variables not presented
 (#3494)

* added skip if env variables not presented. fix exception texts

* fix texts

* fix texts
---
 tests/commands/test_account.py | 28 ++++++++++++++++++----------
 1 file changed, 18 insertions(+), 10 deletions(-)

diff --git a/tests/commands/test_account.py b/tests/commands/test_account.py
index 20b02734..6355f561 100644
--- a/tests/commands/test_account.py
+++ b/tests/commands/test_account.py
@@ -20,6 +20,14 @@ import pytest
 
 from platformio.commands.account.command import cli as cmd_account
 
+pytestmark = pytest.mark.skipif(
+    not (
+        os.environ.get("PLATFORMIO_TEST_ACCOUNT_LOGIN")
+        and os.environ.get("PLATFORMIO_TEST_ACCOUNT_PASSWORD")
+    ),
+    reason="requires PLATFORMIO_TEST_ACCOUNT_LOGIN, PLATFORMIO_TEST_ACCOUNT_PASSWORD environ variables",
+)
+
 
 @pytest.fixture(scope="session")
 def credentials():
@@ -93,7 +101,7 @@ def test_account_login(clirunner, credentials, validate_cliresult, isolated_pio_
         )
         assert result.exit_code > 0
         assert result.exception
-        assert "You are already authenticated with" in str(result.exception)
+        assert "You are already authorized with" in str(result.exception)
     finally:
         clirunner.invoke(cmd_account, ["logout"])
 
@@ -113,7 +121,7 @@ def test_account_logout(clirunner, credentials, validate_cliresult, isolated_pio
         result = clirunner.invoke(cmd_account, ["logout"])
         assert result.exit_code > 0
         assert result.exception
-        assert "You are not authenticated! Please login to PIO Account" in str(
+        assert "You are not authorized! Please log in to PIO Account" in str(
             result.exception
         )
     finally:
@@ -192,7 +200,7 @@ def test_account_password_change(
         )
         assert result.exit_code > 0
         assert result.exception
-        assert "You are not authenticated! Please login to PIO Account" in str(
+        assert "You are not authorized! Please log in to PIO Account" in str(
             result.exception
         )
 
@@ -250,7 +258,7 @@ def test_account_token_with_invalid_password(
         )
         assert result.exit_code > 0
         assert result.exception
-        assert "You are not authenticated! Please login to PIO Account" in str(
+        assert "You are not authorized! Please log in to PIO Account" in str(
             result.exception
         )
 
@@ -302,7 +310,7 @@ def test_account_token(clirunner, credentials, validate_cliresult, isolated_pio_
         )
         assert result.exit_code > 0
         assert result.exception
-        assert "You are not authenticated! Please login to PIO Account" in str(
+        assert "You are not authorized! Please log in to PIO Account" in str(
             result.exception
         )
 
@@ -371,7 +379,7 @@ def test_account_summary(clirunner, credentials, validate_cliresult, isolated_pi
         result = clirunner.invoke(cmd_account, ["show"],)
         assert result.exit_code > 0
         assert result.exception
-        assert "You are not authenticated! Please login to PIO Account" in str(
+        assert "You are not authorized! Please log in to PIO Account" in str(
             result.exception
         )
 
@@ -427,7 +435,7 @@ def test_account_profile_update_with_invalid_password(
         )
         assert result.exit_code > 0
         assert result.exception
-        assert "You are not authenticated! Please login to PIO Account" in str(
+        assert "You are not authorized! Please log in to PIO Account" in str(
             result.exception
         )
 
@@ -460,7 +468,7 @@ def test_account_profile_update_only_firstname_and_lastname(
         )
         assert result.exit_code > 0
         assert result.exception
-        assert "You are not authenticated! Please login to PIO Account" in str(
+        assert "You are not authorized! Please log in to PIO Account" in str(
             result.exception
         )
 
@@ -508,7 +516,7 @@ def test_account_profile_update(
         )
         assert result.exit_code > 0
         assert result.exception
-        assert "You are not authenticated! Please login to PIO Account" in str(
+        assert "You are not authorized! Please log in to PIO Account" in str(
             result.exception
         )
 
@@ -549,7 +557,7 @@ def test_account_profile_update(
         result = clirunner.invoke(cmd_account, ["show"],)
         assert result.exit_code > 0
         assert result.exception
-        assert "You are not authenticated! Please login to PIO Account" in str(
+        assert "You are not authorized! Please log in to PIO Account" in str(
             result.exception
         )
 

From 16966a49573eb05d14506c3e2e629ef722a6a697 Mon Sep 17 00:00:00 2001
From: Valerii Koval <valeros08@gmail.com>
Date: Fri, 8 May 2020 01:14:15 +0300
Subject: [PATCH 14/42] Add initialization config for new simavr tool

---
 platformio/commands/debug/initcfgs.py | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/platformio/commands/debug/initcfgs.py b/platformio/commands/debug/initcfgs.py
index 55e7c34f..bb319270 100644
--- a/platformio/commands/debug/initcfgs.py
+++ b/platformio/commands/debug/initcfgs.py
@@ -141,6 +141,17 @@ $INIT_BREAK
 monitor start
 """
 
+GDB_SIMAVR_INIT_CONFIG = """
+define pio_reset_halt_target
+end
+
+define pio_reset_run_target
+end
+
+target extended-remote $DEBUG_PORT
+$INIT_BREAK
+$LOAD_CMDS
+"""
 
 TOOL_TO_CONFIG = {
     "jlink": GDB_JLINK_INIT_CONFIG,
@@ -148,6 +159,7 @@ TOOL_TO_CONFIG = {
     "qemu": GDB_QEMU_INIT_CONFIG,
     "blackmagic": GDB_BLACKMAGIC_INIT_CONFIG,
     "renode": GDB_RENODE_INIT_CONFIG,
+    "simavr": GDB_SIMAVR_INIT_CONFIG,
 }
 
 

From 7b43444d812f531d5e5ceb2da44488e36ae51676 Mon Sep 17 00:00:00 2001
From: Valerii Koval <valeros08@gmail.com>
Date: Fri, 8 May 2020 12:31:48 +0300
Subject: [PATCH 15/42] Add special debug port for simavr tool

---
 platformio/commands/debug/process/server.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/platformio/commands/debug/process/server.py b/platformio/commands/debug/process/server.py
index 7bd5d485..0c555009 100644
--- a/platformio/commands/debug/process/server.py
+++ b/platformio/commands/debug/process/server.py
@@ -117,6 +117,8 @@ class DebugServer(BaseProcess):
             self._debug_port = ":2331"
         elif "qemu" in server_executable.lower():
             self._debug_port = ":1234"
+        elif "simavr" in server_executable.lower():
+            self._debug_port = ":1234"
 
         yield self._wait_until_ready()
 

From a1ff5e1a4f8c03e423510733bf890fca09b11e8c Mon Sep 17 00:00:00 2001
From: ShahRustam <shahrustam98@gmail.com>
Date: Fri, 8 May 2020 21:34:52 +0300
Subject: [PATCH 16/42] Save summary data to local session. (#3497)

* Save summary data to local session.

* naming

* fix account summary test

* add ttl for summary cache

* refactoring get_account_info

* fix
---
 platformio/commands/account/client.py | 30 +++++++++++++++++++++++----
 tests/commands/test_account.py        | 23 +++++++++++++++++---
 2 files changed, 46 insertions(+), 7 deletions(-)

diff --git a/platformio/commands/account/client.py b/platformio/commands/account/client.py
index 312b9f36..b25365c0 100644
--- a/platformio/commands/account/client.py
+++ b/platformio/commands/account/client.py
@@ -25,6 +25,9 @@ from platformio.commands.account import exception
 
 
 class AccountClient(object):
+
+    SUMMARY_CACHE_TTL = 60 * 60 * 24 * 7
+
     def __init__(
         self, api_base_url=__pioaccount_api__, retries=3,
     ):
@@ -54,6 +57,14 @@ class AccountClient(object):
     def delete_local_session():
         app.delete_state_item("account")
 
+    @staticmethod
+    def delete_local_state(key):
+        account = app.get_state_item("account")
+        if not account or key not in account:
+            return
+        del account[key]
+        app.set_state_item("account", account)
+
     def login(self, username, password):
         try:
             self.fetch_authentication_token()
@@ -182,13 +193,19 @@ class AccountClient(object):
             headers={"Authorization": "Bearer %s" % token},
             data=profile,
         )
+        self.delete_local_state("summary")
         return self.raise_error_from_response(response)
 
     def get_account_info(self, offline):
+        account = app.get_state_item("account")
+        if not account:
+            raise exception.AccountNotAuthorized()
+        if (
+            account.get("summary")
+            and account["summary"].get("expire_at", 0) > time.time()
+        ):
+            return account["summary"]
         if offline:
-            account = app.get_state_item("account")
-            if not account:
-                raise exception.AccountNotAuthorized()
             return {
                 "profile": {
                     "email": account.get("email"),
@@ -203,7 +220,12 @@ class AccountClient(object):
             self.api_base_url + "/v1/summary",
             headers={"Authorization": "Bearer %s" % token},
         )
-        return self.raise_error_from_response(response)
+        result = self.raise_error_from_response(response)
+        account["summary"] = dict(
+            **result, expire_at=int(time.time()) + self.SUMMARY_CACHE_TTL
+        )
+        app.set_state_item("account", account)
+        return result
 
     def fetch_authentication_token(self):
         if "PLATFORMIO_AUTH_TOKEN" in os.environ:
diff --git a/tests/commands/test_account.py b/tests/commands/test_account.py
index 6355f561..ef7ffbad 100644
--- a/tests/commands/test_account.py
+++ b/tests/commands/test_account.py
@@ -389,6 +389,16 @@ def test_account_summary(clirunner, credentials, validate_cliresult, isolated_pi
         )
         validate_cliresult(result)
 
+        result = clirunner.invoke(cmd_account, ["show", "--json-output", "--offline"])
+        validate_cliresult(result)
+        json_result = json.loads(result.output.strip())
+        assert not json_result.get("user_id")
+        assert json_result.get("profile")
+        assert json_result.get("profile").get("username")
+        assert json_result.get("profile").get("email")
+        assert not json_result.get("packages")
+        assert not json_result.get("subscriptions")
+
         result = clirunner.invoke(cmd_account, ["show"])
         validate_cliresult(result)
         assert credentials["login"] in result.output
@@ -415,12 +425,19 @@ def test_account_summary(clirunner, credentials, validate_cliresult, isolated_pi
         result = clirunner.invoke(cmd_account, ["show", "--json-output", "--offline"])
         validate_cliresult(result)
         json_result = json.loads(result.output.strip())
-        assert not json_result.get("user_id")
+        assert json_result.get("user_id")
         assert json_result.get("profile")
         assert json_result.get("profile").get("username")
         assert json_result.get("profile").get("email")
-        assert not json_result.get("packages")
-        assert not json_result.get("subscriptions")
+        assert credentials["login"] == json_result.get("profile").get(
+            "username"
+        ) or credentials["login"] == json_result.get("profile").get("email")
+        assert json_result.get("profile").get("firstname")
+        assert json_result.get("profile").get("lastname")
+        assert json_result.get("packages")
+        assert json_result.get("packages")[0].get("name")
+        assert json_result.get("packages")[0].get("path")
+        assert json_result.get("subscriptions") is not None
     finally:
         clirunner.invoke(cmd_account, ["logout"])
 

From ac510c1553713884f1d81f98d29ed5fdb0e79073 Mon Sep 17 00:00:00 2001
From: ShahRustam <shahrustam98@gmail.com>
Date: Sat, 9 May 2020 16:35:21 +0300
Subject: [PATCH 17/42] fix summary caching (#3500)

---
 platformio/commands/account/client.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/platformio/commands/account/client.py b/platformio/commands/account/client.py
index b25365c0..825c57d4 100644
--- a/platformio/commands/account/client.py
+++ b/platformio/commands/account/client.py
@@ -222,7 +222,11 @@ class AccountClient(object):
         )
         result = self.raise_error_from_response(response)
         account["summary"] = dict(
-            **result, expire_at=int(time.time()) + self.SUMMARY_CACHE_TTL
+            profile=result.get("profile"),
+            packages=result.get("packages"),
+            subscriptions=result.get("subscriptions"),
+            user_id=result.get("user_id"),
+            expire_at=int(time.time()) + self.SUMMARY_CACHE_TTL,
         )
         app.set_state_item("account", account)
         return result

From 03228c528e3c67edd0129a53ba8af525a0d67de7 Mon Sep 17 00:00:00 2001
From: Ivan Kravets <me@ikravets.com>
Date: Sat, 9 May 2020 16:36:05 +0300
Subject: [PATCH 18/42] Bump version to 4.3.4a3

---
 platformio/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/platformio/__init__.py b/platformio/__init__.py
index 6fe0066d..904c62d4 100644
--- a/platformio/__init__.py
+++ b/platformio/__init__.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-VERSION = (4, 3, "4a2")
+VERSION = (4, 3, "4a3")
 __version__ = ".".join([str(s) for s in VERSION])
 
 __title__ = "platformio"

From 01a1981ca141081b83927eab8c6d8fb3ada4dae1 Mon Sep 17 00:00:00 2001
From: Ivan Kravets <me@ikravets.com>
Date: Sun, 10 May 2020 18:35:50 +0300
Subject: [PATCH 19/42] Added PlatformIO CLI Shell Completion for Fish, Zsh,
 Bash, and PowerShell // Resolve #3435

---
 HISTORY.rst                                   |  1 +
 docs                                          |  2 +-
 platformio/__main__.py                        |  7 ++
 .../commands/home/rpc/handlers/piocore.py     |  2 +-
 platformio/commands/misc/__init__.py          | 13 +++
 platformio/commands/misc/command.py           | 96 +++++++++++++++++++
 platformio/commands/misc/completion.py        | 73 ++++++++++++++
 platformio/managers/core.py                   |  2 +-
 8 files changed, 193 insertions(+), 3 deletions(-)
 create mode 100644 platformio/commands/misc/__init__.py
 create mode 100644 platformio/commands/misc/command.py
 create mode 100644 platformio/commands/misc/completion.py

diff --git a/HISTORY.rst b/HISTORY.rst
index 7b124df6..82091206 100644
--- a/HISTORY.rst
+++ b/HISTORY.rst
@@ -9,6 +9,7 @@ PlatformIO Core 4
 4.3.4 (2020-??-??)
 ~~~~~~~~~~~~~~~~~~
 
+* Added `PlatformIO CLI Shell Completion <https://docs.platformio.org/page/core/userguide/misc/completion/index.html>`__  for Fish, Zsh, Bash, and PowerShell (`issue #3435 <https://github.com/platformio/platformio-core/issues/3435>`_)
 * Automatically build ``contrib-pysite`` package on a target machine when pre-built package is not compatible (`issue #3482 <https://github.com/platformio/platformio-core/issues/3482>`_)
 
 4.3.3 (2020-04-28)
diff --git a/docs b/docs
index 2fd23581..695be976 160000
--- a/docs
+++ b/docs
@@ -1 +1 @@
-Subproject commit 2fd2358162db56ddbc344f0a874953ac96ea441f
+Subproject commit 695be97646509b2ee01616b3e18303bec35567d2
diff --git a/platformio/__main__.py b/platformio/__main__.py
index 6679d52e..8420544b 100644
--- a/platformio/__main__.py
+++ b/platformio/__main__.py
@@ -22,6 +22,13 @@ from platformio import __version__, exception, maintenance, util
 from platformio.commands import PlatformioCLI
 from platformio.compat import CYGWIN
 
+try:
+    import click_completion  # pylint: disable=import-error
+
+    click_completion.init()
+except:  # pylint: disable=bare-except
+    pass
+
 
 @click.command(
     cls=PlatformioCLI, context_settings=dict(help_option_names=["-h", "--help"])
diff --git a/platformio/commands/home/rpc/handlers/piocore.py b/platformio/commands/home/rpc/handlers/piocore.py
index 9bcef9e5..cea7cb5b 100644
--- a/platformio/commands/home/rpc/handlers/piocore.py
+++ b/platformio/commands/home/rpc/handlers/piocore.py
@@ -59,8 +59,8 @@ class MultiThreadingStdStream(object):
         result = ""
         try:
             result = self.getvalue()
-            self.truncate(0)
             self.seek(0)
+            self.truncate(0)
         except AttributeError:
             pass
         return result
diff --git a/platformio/commands/misc/__init__.py b/platformio/commands/misc/__init__.py
new file mode 100644
index 00000000..b0514903
--- /dev/null
+++ b/platformio/commands/misc/__init__.py
@@ -0,0 +1,13 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/platformio/commands/misc/command.py b/platformio/commands/misc/command.py
new file mode 100644
index 00000000..46fc0c4e
--- /dev/null
+++ b/platformio/commands/misc/command.py
@@ -0,0 +1,96 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import subprocess
+
+import click
+
+from platformio import proc
+from platformio.commands.misc.completion import (
+    get_completion_install_path,
+    install_completion_code,
+    uninstall_completion_code,
+)
+
+
+@click.group("misc", short_help="Miscellaneous commands")
+def cli():
+    pass
+
+
+@cli.group("completion", short_help="Shell completion support")
+def completion():
+    # pylint: disable=import-outside-toplevel
+    try:
+        import click_completion  # pylint: disable=import-error,unused-import
+    except ImportError:
+        click.echo("Installing dependent packages...")
+        subprocess.check_call(
+            [proc.get_pythonexe_path(), "-m", "pip", "install", "click-completion"],
+        )
+
+
+@completion.command("install", short_help="Install shell completion files/code")
+@click.option(
+    "--shell",
+    default=None,
+    type=click.Choice(["fish", "bash", "zsh", "powershell", "auto"]),
+    help="The shell type, default=auto",
+)
+@click.option(
+    "--path",
+    type=click.Path(file_okay=True, dir_okay=False, readable=True, resolve_path=True),
+    help="Custom installation path of the code to be evaluated by the shell. "
+    "The standard installation path is used by default.",
+)
+def completion_install(shell, path):
+
+    import click_completion  # pylint: disable=import-outside-toplevel,import-error
+
+    shell = shell or click_completion.get_auto_shell()
+    path = path or get_completion_install_path(shell)
+    install_completion_code(shell, path)
+    click.echo(
+        "PlatformIO CLI completion has been installed for %s shell to %s \n"
+        "Please restart a current shell session."
+        % (click.style(shell, fg="cyan"), click.style(path, fg="blue"))
+    )
+
+
+@completion.command("uninstall", short_help="Uninstall shell completion files/code")
+@click.option(
+    "--shell",
+    default=None,
+    type=click.Choice(["fish", "bash", "zsh", "powershell", "auto"]),
+    help="The shell type, default=auto",
+)
+@click.option(
+    "--path",
+    type=click.Path(file_okay=True, dir_okay=False, readable=True, resolve_path=True),
+    help="Custom installation path of the code to be evaluated by the shell. "
+    "The standard installation path is used by default.",
+)
+def completion_uninstall(shell, path):
+
+    import click_completion  # pylint: disable=import-outside-toplevel,import-error
+
+    shell = shell or click_completion.get_auto_shell()
+    path = path or get_completion_install_path(shell)
+    uninstall_completion_code(shell, path)
+    click.echo(
+        "PlatformIO CLI completion has been uninstalled for %s shell from %s \n"
+        "Please restart a current shell session."
+        % (click.style(shell, fg="cyan"), click.style(path, fg="blue"))
+    )
diff --git a/platformio/commands/misc/completion.py b/platformio/commands/misc/completion.py
new file mode 100644
index 00000000..032225c4
--- /dev/null
+++ b/platformio/commands/misc/completion.py
@@ -0,0 +1,73 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import subprocess
+
+import click
+
+
+def get_completion_install_path(shell):
+    home_dir = os.path.expanduser("~")
+    prog_name = click.get_current_context().find_root().info_name
+    if shell == "fish":
+        return os.path.join(
+            home_dir, ".config", "fish", "completions", "%s.fish" % prog_name
+        )
+    if shell == "bash":
+        return os.path.join(home_dir, ".bash_completion")
+    if shell == "zsh":
+        return os.path.join(home_dir, ".zshrc")
+    if shell == "powershell":
+        return subprocess.check_output(
+            ["powershell", "-NoProfile", "echo $profile"]
+        ).strip()
+    raise click.ClickException("%s is not supported." % shell)
+
+
+def is_completion_code_installed(shell, path):
+    if shell == "fish" or not os.path.exists(path):
+        return False
+
+    import click_completion  # pylint: disable=import-error,import-outside-toplevel
+
+    with open(path) as fp:
+        return click_completion.get_code(shell=shell) in fp.read()
+
+
+def install_completion_code(shell, path):
+    import click_completion  # pylint: disable=import-error,import-outside-toplevel
+
+    if is_completion_code_installed(shell, path):
+        return None
+
+    return click_completion.install(shell=shell, path=path)
+
+
+def uninstall_completion_code(shell, path):
+    if not os.path.exists(path):
+        return True
+    if shell == "fish":
+        os.remove(path)
+        return True
+
+    import click_completion  # pylint: disable=import-error,import-outside-toplevel
+
+    with open(path, "r+") as fp:
+        contents = fp.read()
+        fp.seek(0)
+        fp.truncate()
+        fp.write(contents.replace(click_completion.get_code(shell=shell), ""))
+
+    return True
diff --git a/platformio/managers/core.py b/platformio/managers/core.py
index 677a1c81..53f435fd 100644
--- a/platformio/managers/core.py
+++ b/platformio/managers/core.py
@@ -138,7 +138,7 @@ def build_contrib_pysite_deps(target_dir):
 
     pythonexe = get_pythonexe_path()
     for dep in get_contrib_pysite_deps():
-        subprocess.call(
+        subprocess.check_call(
             [
                 pythonexe,
                 "-m",

From b2ed027bc34c29278c0b659d5b62712286b31850 Mon Sep 17 00:00:00 2001
From: Ivan Kravets <me@ikravets.com>
Date: Sun, 10 May 2020 18:36:16 +0300
Subject: [PATCH 20/42] Bump version to 4.3.4a4

---
 platformio/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/platformio/__init__.py b/platformio/__init__.py
index 904c62d4..f4f6529a 100644
--- a/platformio/__init__.py
+++ b/platformio/__init__.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-VERSION = (4, 3, "4a3")
+VERSION = (4, 3, "4a4")
 __version__ = ".".join([str(s) for s in VERSION])
 
 __title__ = "platformio"

From c76940f7ce068b6419f03c8562d0d70a7f1cceaf Mon Sep 17 00:00:00 2001
From: Ivan Kravets <me@ikravets.com>
Date: Sun, 10 May 2020 19:11:06 +0300
Subject: [PATCH 21/42] PyLint fix

---
 docs                                | 2 +-
 platformio/commands/misc/command.py | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs b/docs
index 695be976..3dcfc88a 160000
--- a/docs
+++ b/docs
@@ -1 +1 @@
-Subproject commit 695be97646509b2ee01616b3e18303bec35567d2
+Subproject commit 3dcfc88af4e20c36b44123bab12952c757a74e93
diff --git a/platformio/commands/misc/command.py b/platformio/commands/misc/command.py
index 46fc0c4e..757206b3 100644
--- a/platformio/commands/misc/command.py
+++ b/platformio/commands/misc/command.py
@@ -32,9 +32,9 @@ def cli():
 
 @cli.group("completion", short_help="Shell completion support")
 def completion():
-    # pylint: disable=import-outside-toplevel
+    # pylint: disable=import-error,import-outside-toplevel
     try:
-        import click_completion  # pylint: disable=import-error,unused-import
+        import click_completion  # pylint: disable=unused-import,unused-variable
     except ImportError:
         click.echo("Installing dependent packages...")
         subprocess.check_call(

From 7555d667483594c60e191760cfd43fb8e8365576 Mon Sep 17 00:00:00 2001
From: Valerii Koval <valeros08@gmail.com>
Date: Mon, 11 May 2020 18:21:26 +0300
Subject: [PATCH 22/42] Revert "Add special debug port for simavr tool"

This reverts commit 7b43444d812f531d5e5ceb2da44488e36ae51676.
---
 platformio/commands/debug/process/server.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/platformio/commands/debug/process/server.py b/platformio/commands/debug/process/server.py
index 0c555009..7bd5d485 100644
--- a/platformio/commands/debug/process/server.py
+++ b/platformio/commands/debug/process/server.py
@@ -117,8 +117,6 @@ class DebugServer(BaseProcess):
             self._debug_port = ":2331"
         elif "qemu" in server_executable.lower():
             self._debug_port = ":1234"
-        elif "simavr" in server_executable.lower():
-            self._debug_port = ":1234"
 
         yield self._wait_until_ready()
 

From 2a0a1247e37ebe93dcb7967388d6b55f6aa67f43 Mon Sep 17 00:00:00 2001
From: Valerii Koval <valeros08@gmail.com>
Date: Mon, 11 May 2020 18:21:48 +0300
Subject: [PATCH 23/42] Revert "Add initialization config for new simavr tool"

This reverts commit 16966a49573eb05d14506c3e2e629ef722a6a697.
---
 platformio/commands/debug/initcfgs.py | 12 ------------
 1 file changed, 12 deletions(-)

diff --git a/platformio/commands/debug/initcfgs.py b/platformio/commands/debug/initcfgs.py
index bb319270..55e7c34f 100644
--- a/platformio/commands/debug/initcfgs.py
+++ b/platformio/commands/debug/initcfgs.py
@@ -141,17 +141,6 @@ $INIT_BREAK
 monitor start
 """
 
-GDB_SIMAVR_INIT_CONFIG = """
-define pio_reset_halt_target
-end
-
-define pio_reset_run_target
-end
-
-target extended-remote $DEBUG_PORT
-$INIT_BREAK
-$LOAD_CMDS
-"""
 
 TOOL_TO_CONFIG = {
     "jlink": GDB_JLINK_INIT_CONFIG,
@@ -159,7 +148,6 @@ TOOL_TO_CONFIG = {
     "qemu": GDB_QEMU_INIT_CONFIG,
     "blackmagic": GDB_BLACKMAGIC_INIT_CONFIG,
     "renode": GDB_RENODE_INIT_CONFIG,
-    "simavr": GDB_SIMAVR_INIT_CONFIG,
 }
 
 

From 96a68c6b14614b0b02a98d86bb7d882402a39a54 Mon Sep 17 00:00:00 2001
From: Ivan Kravets <me@ikravets.com>
Date: Mon, 11 May 2020 22:10:32 +0300
Subject: [PATCH 24/42] Docs: Sync Atmel AVR dev-platform

---
 docs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs b/docs
index 3dcfc88a..c04945bf 160000
--- a/docs
+++ b/docs
@@ -1 +1 @@
-Subproject commit 3dcfc88af4e20c36b44123bab12952c757a74e93
+Subproject commit c04945bf3a0ae1524bc720743ea770fcc1d2fba9

From 099e3c719899cb3139c5bb3422acf85240d2fc47 Mon Sep 17 00:00:00 2001
From: Ivan Kravets <me@ikravets.com>
Date: Wed, 13 May 2020 00:48:11 +0300
Subject: [PATCH 25/42] Use original MongoDB license for "compilation_db.py"

---
 platformio/builder/tools/compilation_db.py | 27 ++++++++++++++--------
 1 file changed, 17 insertions(+), 10 deletions(-)

diff --git a/platformio/builder/tools/compilation_db.py b/platformio/builder/tools/compilation_db.py
index 52463e83..150a832e 100644
--- a/platformio/builder/tools/compilation_db.py
+++ b/platformio/builder/tools/compilation_db.py
@@ -1,17 +1,24 @@
 # Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-# Copyright 2015 MongoDB Inc.
+# Copyright 2020 MongoDB Inc.
 #
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
 #
-# http://www.apache.org/licenses/LICENSE-2.0
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
 #
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
+# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 # pylint: disable=unused-argument, protected-access, unused-variable, import-error
 # Original: https://github.com/mongodb/mongo/blob/master/site_scons/site_tools/compilation_db.py

From 54d73e834b844104a00f674b76b78f8419e8d2c8 Mon Sep 17 00:00:00 2001
From: Ivan Kravets <me@ikravets.com>
Date: Thu, 14 May 2020 18:08:51 +0300
Subject: [PATCH 26/42] Github Actions: Checkout submodules recursive

---
 .github/workflows/core.yml     | 3 ++-
 .github/workflows/docs.yml     | 3 ++-
 .github/workflows/examples.yml | 3 ++-
 docs                           | 2 +-
 4 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml
index 9d8a57fc..c84a97d7 100644
--- a/.github/workflows/core.yml
+++ b/.github/workflows/core.yml
@@ -12,13 +12,14 @@ jobs:
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v2
+        with:
+          submodules: "recursive"
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v1
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install dependencies
         run: |
-          git submodule update --init --recursive
           python -m pip install --upgrade pip
           pip install tox
 
diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
index 491728a5..bfe2c116 100644
--- a/.github/workflows/docs.yml
+++ b/.github/workflows/docs.yml
@@ -7,13 +7,14 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
+        with:
+          submodules: "recursive"
       - name: Set up Python
         uses: actions/setup-python@v1
         with:
           python-version: 3.7
       - name: Install dependencies
         run: |
-          git submodule update --init --recursive
           python -m pip install --upgrade pip
           pip install tox
 
diff --git a/.github/workflows/examples.yml b/.github/workflows/examples.yml
index 3fa5b26c..b5452909 100644
--- a/.github/workflows/examples.yml
+++ b/.github/workflows/examples.yml
@@ -12,13 +12,14 @@ jobs:
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v2
+        with:
+          submodules: "recursive"
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v1
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install dependencies
         run: |
-          git submodule update --init --recursive
           python -m pip install --upgrade pip
           pip install tox
 
diff --git a/docs b/docs
index c04945bf..826897f3 160000
--- a/docs
+++ b/docs
@@ -1 +1 @@
-Subproject commit c04945bf3a0ae1524bc720743ea770fcc1d2fba9
+Subproject commit 826897f3abf814074d11361deeed0de9ce6abf06

From eac6c1c552bbb41cbc7c76b6456c7646f5b05eaf Mon Sep 17 00:00:00 2001
From: ShahRustam <shahrustam98@gmail.com>
Date: Sun, 17 May 2020 22:44:27 +0300
Subject: [PATCH 27/42] Handle error when internet is offline. Resolve # 3503
 (#3505)

* Handle error when internet is offline.

* Fix

* minor fix
---
 platformio/commands/account/client.py | 102 +++++++++++++-------------
 platformio/exception.py               |   4 +-
 2 files changed, 51 insertions(+), 55 deletions(-)

diff --git a/platformio/commands/account/client.py b/platformio/commands/account/client.py
index 825c57d4..6c1ee42e 100644
--- a/platformio/commands/account/client.py
+++ b/platformio/commands/account/client.py
@@ -21,6 +21,7 @@ import requests.adapters
 from requests.packages.urllib3.util.retry import Retry  # pylint:disable=import-error
 
 from platformio import __pioaccount_api__, app
+from platformio.exception import InternetIsOffline
 from platformio.commands.account import exception
 
 
@@ -75,11 +76,11 @@ class AccountClient(object):
                 app.get_state_item("account", {}).get("email", "")
             )
 
-        response = self._session.post(
+        result = self.send_request(
+            "post",
             self.api_base_url + "/v1/login",
             data={"username": username, "password": password},
         )
-        result = self.raise_error_from_response(response)
         app.set_state_item("account", result)
         return result
 
@@ -93,40 +94,35 @@ class AccountClient(object):
                 app.get_state_item("account", {}).get("email", "")
             )
 
-        response = self._session.post(
+        result = self.send_request(
+            "post",
             self.api_base_url + "/v1/login/code",
             data={"client_id": client_id, "code": code, "redirect_uri": redirect_uri},
         )
-        result = self.raise_error_from_response(response)
         app.set_state_item("account", result)
         return result
 
     def logout(self):
-        try:
-            refresh_token = self.get_refresh_token()
-        except:  # pylint:disable=bare-except
-            raise exception.AccountNotAuthorized()
+        refresh_token = self.get_refresh_token()
         self.delete_local_session()
-        response = requests.post(
-            self.api_base_url + "/v1/logout", data={"refresh_token": refresh_token},
-        )
         try:
-            self.raise_error_from_response(response)
+            self.send_request(
+                "post",
+                self.api_base_url + "/v1/logout",
+                data={"refresh_token": refresh_token},
+            )
         except exception.AccountError:
             pass
         return True
 
     def change_password(self, old_password, new_password):
-        try:
-            token = self.fetch_authentication_token()
-        except:  # pylint:disable=bare-except
-            raise exception.AccountNotAuthorized()
-        response = self._session.post(
+        token = self.fetch_authentication_token()
+        self.send_request(
+            "post",
             self.api_base_url + "/v1/password",
             headers={"Authorization": "Bearer %s" % token},
             data={"old_password": old_password, "new_password": new_password},
         )
-        self.raise_error_from_response(response)
         return True
 
     def registration(
@@ -141,7 +137,8 @@ class AccountClient(object):
                 app.get_state_item("account", {}).get("email", "")
             )
 
-        response = self._session.post(
+        return self.send_request(
+            "post",
             self.api_base_url + "/v1/registration",
             data={
                 "username": username,
@@ -151,50 +148,41 @@ class AccountClient(object):
                 "lastname": lastname,
             },
         )
-        return self.raise_error_from_response(response)
 
     def auth_token(self, password, regenerate):
-        try:
-            token = self.fetch_authentication_token()
-        except:  # pylint:disable=bare-except
-            raise exception.AccountNotAuthorized()
-        response = self._session.post(
+        token = self.fetch_authentication_token()
+        result = self.send_request(
+            "post",
             self.api_base_url + "/v1/token",
             headers={"Authorization": "Bearer %s" % token},
             data={"password": password, "regenerate": 1 if regenerate else 0},
         )
-        return self.raise_error_from_response(response).get("auth_token")
+        return result.get("auth_token")
 
     def forgot_password(self, username):
-        response = self._session.post(
-            self.api_base_url + "/v1/forgot", data={"username": username},
+        return self.send_request(
+            "post", self.api_base_url + "/v1/forgot", data={"username": username},
         )
-        return self.raise_error_from_response(response).get("auth_token")
 
     def get_profile(self):
-        try:
-            token = self.fetch_authentication_token()
-        except:  # pylint:disable=bare-except
-            raise exception.AccountNotAuthorized()
-        response = self._session.get(
+        token = self.fetch_authentication_token()
+        return self.send_request(
+            "get",
             self.api_base_url + "/v1/profile",
             headers={"Authorization": "Bearer %s" % token},
         )
-        return self.raise_error_from_response(response)
 
     def update_profile(self, profile, current_password):
-        try:
-            token = self.fetch_authentication_token()
-        except:  # pylint:disable=bare-except
-            raise exception.AccountNotAuthorized()
+        token = self.fetch_authentication_token()
         profile["current_password"] = current_password
-        response = self._session.put(
+        self.delete_local_state("summary")
+        response = self.send_request(
+            "put",
             self.api_base_url + "/v1/profile",
             headers={"Authorization": "Bearer %s" % token},
             data=profile,
         )
-        self.delete_local_state("summary")
-        return self.raise_error_from_response(response)
+        return response
 
     def get_account_info(self, offline):
         account = app.get_state_item("account")
@@ -212,15 +200,12 @@ class AccountClient(object):
                     "username": account.get("username"),
                 }
             }
-        try:
-            token = self.fetch_authentication_token()
-        except:  # pylint:disable=bare-except
-            raise exception.AccountNotAuthorized()
-        response = self._session.get(
+        token = self.fetch_authentication_token()
+        result = self.send_request(
+            "get",
             self.api_base_url + "/v1/summary",
             headers={"Authorization": "Bearer %s" % token},
         )
-        result = self.raise_error_from_response(response)
         account["summary"] = dict(
             profile=result.get("profile"),
             packages=result.get("packages"),
@@ -239,18 +224,29 @@ class AccountClient(object):
             if auth.get("access_token_expire") > time.time():
                 return auth.get("access_token")
             if auth.get("refresh_token"):
-                response = self._session.post(
-                    self.api_base_url + "/v1/login",
-                    headers={"Authorization": "Bearer %s" % auth.get("refresh_token")},
-                )
                 try:
-                    result = self.raise_error_from_response(response)
+                    result = self.send_request(
+                        "post",
+                        self.api_base_url + "/v1/login",
+                        headers={
+                            "Authorization": "Bearer %s" % auth.get("refresh_token")
+                        },
+                    )
                     app.set_state_item("account", result)
                     return result.get("auth").get("access_token")
                 except exception.AccountError:
                     self.delete_local_session()
         raise exception.AccountNotAuthorized()
 
+    def send_request(self, method, url, headers=None, data=None):
+        try:
+            response = getattr(self._session, method)(
+                url, headers=headers or {}, data=data or {}
+            )
+        except requests.exceptions.ConnectionError:
+            raise InternetIsOffline()
+        return self.raise_error_from_response(response)
+
     def raise_error_from_response(self, response, expected_codes=(200, 201, 202)):
         if response.status_code in expected_codes:
             try:
diff --git a/platformio/exception.py b/platformio/exception.py
index 905c5ab1..d291ad7f 100644
--- a/platformio/exception.py
+++ b/platformio/exception.py
@@ -232,8 +232,8 @@ class InternetIsOffline(UserSideException):
 
     MESSAGE = (
         "You are not connected to the Internet.\n"
-        "If you build a project first time, we need Internet connection "
-        "to install all dependencies and toolchains."
+        "PlatformIO needs the Internet connection to"
+        " download dependent packages or to work with PIO Account."
     )
 
 

From 9724660dda04b55e7f7475f8f28f293fc9e9b3f5 Mon Sep 17 00:00:00 2001
From: Ivan Kravets <me@ikravets.com>
Date: Tue, 19 May 2020 13:27:44 +0300
Subject: [PATCH 28/42] Update SPDX licenses to 3.9

---
 platformio/package/manifest/schema.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/platformio/package/manifest/schema.py b/platformio/package/manifest/schema.py
index be49b3ee..11d3f902 100644
--- a/platformio/package/manifest/schema.py
+++ b/platformio/package/manifest/schema.py
@@ -242,7 +242,7 @@ class ManifestSchema(BaseSchema):
     def load_spdx_licenses():
         r = requests.get(
             "https://raw.githubusercontent.com/spdx/license-list-data"
-            "/v3.8/json/licenses.json"
+            "/v3.9/json/licenses.json"
         )
         r.raise_for_status()
         return r.json()

From 457a2187230aaf64ed6c767791b6dae812357dc1 Mon Sep 17 00:00:00 2001
From: Ivan Kravets <me@ikravets.com>
Date: Tue, 19 May 2020 13:27:54 +0300
Subject: [PATCH 29/42] Sync docs

---
 HISTORY.rst | 2 +-
 docs        | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/HISTORY.rst b/HISTORY.rst
index 82091206..148537a9 100644
--- a/HISTORY.rst
+++ b/HISTORY.rst
@@ -23,7 +23,7 @@ PlatformIO Core 4
 * New `Account Management System <https://docs.platformio.org/page/plus/pio-account.html>`__ (preview)
 * Open source `PIO Remote <http://docs.platformio.org/page/plus/pio-remote.html>`__ client
 * Improved `PIO Check <http://docs.platformio.org/page/plus/pio-check.html>`__ with more accurate project processing
-* Echo what is typed when ``send_on_enter`` device monitor filter <https://docs.platformio.org/page/projectconf/section_env_monitor.html#monitor-filters>`__ is used (`issue #3452 <https://github.com/platformio/platformio-core/issues/3452>`_)
+* Echo what is typed when ``send_on_enter`` `device monitor filter <https://docs.platformio.org/page/projectconf/section_env_monitor.html#monitor-filters>`__ is used (`issue #3452 <https://github.com/platformio/platformio-core/issues/3452>`_)
 * Fixed PIO Unit Testing for Zephyr RTOS
 * Fixed UnicodeDecodeError on Windows when network drive (NAS) is used (`issue #3417 <https://github.com/platformio/platformio-core/issues/3417>`_)
 * Fixed an issue when saving libraries in new project results in error "No option 'lib_deps' in section" (`issue #3442 <https://github.com/platformio/platformio-core/issues/3442>`_)
diff --git a/docs b/docs
index 826897f3..d6d6a519 160000
--- a/docs
+++ b/docs
@@ -1 +1 @@
-Subproject commit 826897f3abf814074d11361deeed0de9ce6abf06
+Subproject commit d6d6a519d96bef5e1849dc8e329986772cc9314c

From e31591a35e5bd630337ad8b8a9402f647d3d175f Mon Sep 17 00:00:00 2001
From: Valerii Koval <valeros08@gmail.com>
Date: Tue, 19 May 2020 22:37:05 +0300
Subject: [PATCH 30/42] Print warning about an issue with mapped network drives
 on Windows // Issue #3417

Starting with Python 3.8 paths to mapped network drives are resolved
to their real path in the system, e.g.: "Z:\path" becomes "\\path" which
causes weird errors in the default terminal with a message that UNC
paths are not supported
---
 platformio/builder/main.py | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/platformio/builder/main.py b/platformio/builder/main.py
index c895f3f3..c35f51c5 100644
--- a/platformio/builder/main.py
+++ b/platformio/builder/main.py
@@ -28,7 +28,7 @@ from SCons.Script import DefaultEnvironment  # pylint: disable=import-error
 from SCons.Script import Import  # pylint: disable=import-error
 from SCons.Script import Variables  # pylint: disable=import-error
 
-from platformio import fs
+from platformio import compat, fs
 from platformio.compat import dump_json_to_unicode
 from platformio.managers.platform import PlatformBase
 from platformio.proc import get_pythonexe_path
@@ -120,6 +120,16 @@ env.Replace(
     ],
 )
 
+if (
+    compat.WINDOWS
+    and sys.version_info >= (3, 8)
+    and env["PROJECT_DIR"].startswith("\\\\")
+):
+    click.secho(
+        "There is a known issue with Python 3.8+ and mapped network drives on "
+        "Windows.\nPlease downgrade Python to the latest 3.7. More details at:\n"
+        "https://github.com/platformio/platformio-core/issues/3417", fg="yellow")
+
 if env.subst("$BUILD_CACHE_DIR"):
     if not isdir(env.subst("$BUILD_CACHE_DIR")):
         makedirs(env.subst("$BUILD_CACHE_DIR"))

From 8840b289688660998b12973ccef58dc19406c1d9 Mon Sep 17 00:00:00 2001
From: Valerii Koval <valeros08@gmail.com>
Date: Wed, 20 May 2020 17:04:50 +0300
Subject: [PATCH 31/42] Handle possible issue on Python 2.x when writing to
 thread buffer

The problem happens when value has type "unicode" that shouldn't be decoded
---
 platformio/commands/home/rpc/handlers/piocore.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/platformio/commands/home/rpc/handlers/piocore.py b/platformio/commands/home/rpc/handlers/piocore.py
index cea7cb5b..44cb6983 100644
--- a/platformio/commands/home/rpc/handlers/piocore.py
+++ b/platformio/commands/home/rpc/handlers/piocore.py
@@ -52,7 +52,7 @@ class MultiThreadingStdStream(object):
         thread_id = thread_get_ident()
         self._ensure_thread_buffer(thread_id)
         return self._buffers[thread_id].write(
-            value.decode() if is_bytes(value) else value
+            value.decode() if not PY2 and is_bytes(value) else value
         )
 
     def get_value_and_reset(self):

From bdd57bf356f2111f8034315b5eec85a1846277a4 Mon Sep 17 00:00:00 2001
From: Ivan Kravets <me@ikravets.com>
Date: Wed, 20 May 2020 20:57:55 +0300
Subject: [PATCH 32/42] Ensure that copytree preserves symlinks

---
 docs                                             | 2 +-
 platformio/builder/main.py                       | 4 +++-
 platformio/commands/account/client.py            | 2 +-
 platformio/commands/home/rpc/handlers/os.py      | 2 +-
 platformio/commands/home/rpc/handlers/project.py | 4 ++--
 platformio/managers/package.py                   | 2 +-
 6 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/docs b/docs
index d6d6a519..d8c82a5d 160000
--- a/docs
+++ b/docs
@@ -1 +1 @@
-Subproject commit d6d6a519d96bef5e1849dc8e329986772cc9314c
+Subproject commit d8c82a5d442a78b476e7405db12041ac45a3f72f
diff --git a/platformio/builder/main.py b/platformio/builder/main.py
index c35f51c5..7184da7c 100644
--- a/platformio/builder/main.py
+++ b/platformio/builder/main.py
@@ -128,7 +128,9 @@ if (
     click.secho(
         "There is a known issue with Python 3.8+ and mapped network drives on "
         "Windows.\nPlease downgrade Python to the latest 3.7. More details at:\n"
-        "https://github.com/platformio/platformio-core/issues/3417", fg="yellow")
+        "https://github.com/platformio/platformio-core/issues/3417",
+        fg="yellow",
+    )
 
 if env.subst("$BUILD_CACHE_DIR"):
     if not isdir(env.subst("$BUILD_CACHE_DIR")):
diff --git a/platformio/commands/account/client.py b/platformio/commands/account/client.py
index 6c1ee42e..fb679dc0 100644
--- a/platformio/commands/account/client.py
+++ b/platformio/commands/account/client.py
@@ -21,8 +21,8 @@ import requests.adapters
 from requests.packages.urllib3.util.retry import Retry  # pylint:disable=import-error
 
 from platformio import __pioaccount_api__, app
-from platformio.exception import InternetIsOffline
 from platformio.commands.account import exception
+from platformio.exception import InternetIsOffline
 
 
 class AccountClient(object):
diff --git a/platformio/commands/home/rpc/handlers/os.py b/platformio/commands/home/rpc/handlers/os.py
index 745ae817..2997e8aa 100644
--- a/platformio/commands/home/rpc/handlers/os.py
+++ b/platformio/commands/home/rpc/handlers/os.py
@@ -107,7 +107,7 @@ class OSRPC(object):
 
     @staticmethod
     def copy(src, dst):
-        return shutil.copytree(src, dst)
+        return shutil.copytree(src, dst, symlinks=True)
 
     @staticmethod
     def glob(pathnames, root=None):
diff --git a/platformio/commands/home/rpc/handlers/project.py b/platformio/commands/home/rpc/handlers/project.py
index 77a04646..3f4cdc88 100644
--- a/platformio/commands/home/rpc/handlers/project.py
+++ b/platformio/commands/home/rpc/handlers/project.py
@@ -300,7 +300,7 @@ class ProjectRPC(object):
             src_dir = config.get_optional_dir("src")
             if os.path.isdir(src_dir):
                 fs.rmtree(src_dir)
-            shutil.copytree(arduino_project_dir, src_dir)
+            shutil.copytree(arduino_project_dir, src_dir, symlinks=True)
         return project_dir
 
     @staticmethod
@@ -313,7 +313,7 @@ class ProjectRPC(object):
             AppRPC.load_state()["storage"]["projectsDir"],
             time.strftime("%y%m%d-%H%M%S-") + os.path.basename(project_dir),
         )
-        shutil.copytree(project_dir, new_project_dir)
+        shutil.copytree(project_dir, new_project_dir, symlinks=True)
 
         state = AppRPC.load_state()
         args = ["init"]
diff --git a/platformio/managers/package.py b/platformio/managers/package.py
index 07f2a23f..fd227073 100644
--- a/platformio/managers/package.py
+++ b/platformio/managers/package.py
@@ -475,7 +475,7 @@ class PkgInstallerMixin(object):
                     self.unpack(_url, tmp_dir)
                 else:
                     fs.rmtree(tmp_dir)
-                    shutil.copytree(_url, tmp_dir)
+                    shutil.copytree(_url, tmp_dir, symlinks=True)
             elif url.startswith(("http://", "https://")):
                 dlpath = self.download(url, tmp_dir, sha1)
                 assert isfile(dlpath)

From 735435306dbe2175bd2ccb8cb8cdc32e2b0ae974 Mon Sep 17 00:00:00 2001
From: Valerii Koval <valeros08@gmail.com>
Date: Wed, 20 May 2020 21:32:55 +0300
Subject: [PATCH 33/42] Copy and remove cloned package instead of moving //
 Resolve #2844, Resolve #3328
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

On Windows, it’s not possible to move a file which is used by another
process (e.g. Git extension in VSCode)
---
 platformio/managers/package.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/platformio/managers/package.py b/platformio/managers/package.py
index fd227073..92ba4515 100644
--- a/platformio/managers/package.py
+++ b/platformio/managers/package.py
@@ -582,7 +582,11 @@ class PkgInstallerMixin(object):
         # remove previous/not-satisfied package
         if isdir(pkg_dir):
             fs.rmtree(pkg_dir)
-        shutil.move(tmp_dir, pkg_dir)
+        shutil.copytree(tmp_dir, pkg_dir, symlinks=True)
+        try:
+            shutil.rmtree(tmp_dir)
+        except:  # pylint: disable=bare-except
+            pass
         assert isdir(pkg_dir)
         self.cache_reset()
         return pkg_dir

From 09a5952248ea49b69b78092a1736f85e84207267 Mon Sep 17 00:00:00 2001
From: Valerii Koval <valeros08@gmail.com>
Date: Wed, 20 May 2020 21:51:13 +0300
Subject: [PATCH 34/42] Add new record to history log

Mention issues about permission error on Windows when cloning
package from Git repository
---
 HISTORY.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/HISTORY.rst b/HISTORY.rst
index 148537a9..75202dfc 100644
--- a/HISTORY.rst
+++ b/HISTORY.rst
@@ -11,6 +11,7 @@ PlatformIO Core 4
 
 * Added `PlatformIO CLI Shell Completion <https://docs.platformio.org/page/core/userguide/misc/completion/index.html>`__  for Fish, Zsh, Bash, and PowerShell (`issue #3435 <https://github.com/platformio/platformio-core/issues/3435>`_)
 * Automatically build ``contrib-pysite`` package on a target machine when pre-built package is not compatible (`issue #3482 <https://github.com/platformio/platformio-core/issues/3482>`_)
+* Fixed an issue on Windows when installing a dependency from Git repository raised a permission error (`issue #2844 <https://github.com/platformio/platformio-core/issues/2844>`_, `issue #3328 <https://github.com/platformio/platformio-core/issues/3328>`_)
 
 4.3.3 (2020-04-28)
 ~~~~~~~~~~~~~~~~~~

From 56795940b950fc76b1ea7f44ca8202bbd67e61ae Mon Sep 17 00:00:00 2001
From: Ivan Kravets <me@ikravets.com>
Date: Thu, 21 May 2020 15:39:24 +0300
Subject: [PATCH 35/42] Sync teensy dev-platform

---
 docs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs b/docs
index d8c82a5d..bc9d81b1 160000
--- a/docs
+++ b/docs
@@ -1 +1 @@
-Subproject commit d8c82a5d442a78b476e7405db12041ac45a3f72f
+Subproject commit bc9d81b1d34ecc70b60c83d4062c03e0b30e95db

From ec34a65cfffe0afafc104210ec2fc69fa9bf4ac5 Mon Sep 17 00:00:00 2001
From: Ivan Kravets <me@ikravets.com>
Date: Thu, 21 May 2020 15:40:38 +0300
Subject: [PATCH 36/42] Bump version to 4.3.4a5

---
 platformio/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/platformio/__init__.py b/platformio/__init__.py
index f4f6529a..af520b21 100644
--- a/platformio/__init__.py
+++ b/platformio/__init__.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-VERSION = (4, 3, "4a4")
+VERSION = (4, 3, "4a5")
 __version__ = ".".join([str(s) for s in VERSION])
 
 __title__ = "platformio"

From e2c5a3c49870d5fab860825342d06bcb4a385ead Mon Sep 17 00:00:00 2001
From: Ivan Kravets <me@ikravets.com>
Date: Fri, 22 May 2020 14:12:27 +0300
Subject: [PATCH 37/42] Add Python 3.8 for Tox

---
 tox.ini | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tox.ini b/tox.ini
index 48792b86..e8e8bdb2 100644
--- a/tox.ini
+++ b/tox.ini
@@ -13,13 +13,13 @@
 # limitations under the License.
 
 [tox]
-envlist = py27,py37
+envlist = py27,py37,py38
 
 [testenv]
 passenv = *
 usedevelop = True
 deps =
-    py36,py37: black
+    py36,py3,py38: black
     isort
     pylint
     pytest

From 32cb0d6e4deec18dc650afcf08bd5988eb8069e9 Mon Sep 17 00:00:00 2001
From: Valerii Koval <valeros08@gmail.com>
Date: Fri, 22 May 2020 14:17:17 +0300
Subject: [PATCH 38/42] Handle possible issue on Python 2.x when writing to
 thread buffer

The problem happens when value has type "unicode" that shouldn't be decoded
---
 platformio/commands/home/rpc/handlers/piocore.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/platformio/commands/home/rpc/handlers/piocore.py b/platformio/commands/home/rpc/handlers/piocore.py
index 44cb6983..940b1dd2 100644
--- a/platformio/commands/home/rpc/handlers/piocore.py
+++ b/platformio/commands/home/rpc/handlers/piocore.py
@@ -51,8 +51,10 @@ class MultiThreadingStdStream(object):
     def write(self, value):
         thread_id = thread_get_ident()
         self._ensure_thread_buffer(thread_id)
+        if PY2 and isinstance(value, unicode):
+            value = value.encode()
         return self._buffers[thread_id].write(
-            value.decode() if not PY2 and is_bytes(value) else value
+            value.decode() if is_bytes(value) else value
         )
 
     def get_value_and_reset(self):

From 4921bf8b6a175597a27f0b73e82beccbdef4fc15 Mon Sep 17 00:00:00 2001
From: Valerii Koval <valeros08@gmail.com>
Date: Fri, 22 May 2020 14:22:41 +0300
Subject: [PATCH 39/42] PyLint fix

---
 platformio/commands/home/rpc/handlers/piocore.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/platformio/commands/home/rpc/handlers/piocore.py b/platformio/commands/home/rpc/handlers/piocore.py
index 940b1dd2..00adf5b6 100644
--- a/platformio/commands/home/rpc/handlers/piocore.py
+++ b/platformio/commands/home/rpc/handlers/piocore.py
@@ -51,7 +51,7 @@ class MultiThreadingStdStream(object):
     def write(self, value):
         thread_id = thread_get_ident()
         self._ensure_thread_buffer(thread_id)
-        if PY2 and isinstance(value, unicode):
+        if PY2 and isinstance(value, unicode):  # pylint: disable=undefined-variable
             value = value.encode()
         return self._buffers[thread_id].write(
             value.decode() if is_bytes(value) else value

From 7dce494ad62792927be171a302dd2d5ddd514be0 Mon Sep 17 00:00:00 2001
From: Ivan Kravets <me@ikravets.com>
Date: Sat, 23 May 2020 20:00:56 +0300
Subject: [PATCH 40/42] Rename "misc" command to "system", do not append
 completion code for Fish shell // Resolve 3435

---
 docs                                               | 2 +-
 platformio/commands/{misc => system}/__init__.py   | 0
 platformio/commands/{misc => system}/command.py    | 4 ++--
 platformio/commands/{misc => system}/completion.py | 2 +-
 tox.ini                                            | 2 +-
 5 files changed, 5 insertions(+), 5 deletions(-)
 rename platformio/commands/{misc => system}/__init__.py (100%)
 rename platformio/commands/{misc => system}/command.py (96%)
 rename platformio/commands/{misc => system}/completion.py (96%)

diff --git a/docs b/docs
index bc9d81b1..2cd0dce6 160000
--- a/docs
+++ b/docs
@@ -1 +1 @@
-Subproject commit bc9d81b1d34ecc70b60c83d4062c03e0b30e95db
+Subproject commit 2cd0dce683f40f5606bf7476d47e7bb052cfebba
diff --git a/platformio/commands/misc/__init__.py b/platformio/commands/system/__init__.py
similarity index 100%
rename from platformio/commands/misc/__init__.py
rename to platformio/commands/system/__init__.py
diff --git a/platformio/commands/misc/command.py b/platformio/commands/system/command.py
similarity index 96%
rename from platformio/commands/misc/command.py
rename to platformio/commands/system/command.py
index 757206b3..48336bfd 100644
--- a/platformio/commands/misc/command.py
+++ b/platformio/commands/system/command.py
@@ -18,14 +18,14 @@ import subprocess
 import click
 
 from platformio import proc
-from platformio.commands.misc.completion import (
+from platformio.commands.system.completion import (
     get_completion_install_path,
     install_completion_code,
     uninstall_completion_code,
 )
 
 
-@click.group("misc", short_help="Miscellaneous commands")
+@click.group("system", short_help="Miscellaneous system commands")
 def cli():
     pass
 
diff --git a/platformio/commands/misc/completion.py b/platformio/commands/system/completion.py
similarity index 96%
rename from platformio/commands/misc/completion.py
rename to platformio/commands/system/completion.py
index 032225c4..1a969203 100644
--- a/platformio/commands/misc/completion.py
+++ b/platformio/commands/system/completion.py
@@ -52,7 +52,7 @@ def install_completion_code(shell, path):
     if is_completion_code_installed(shell, path):
         return None
 
-    return click_completion.install(shell=shell, path=path)
+    return click_completion.install(shell=shell, path=path, append=shell != "fish")
 
 
 def uninstall_completion_code(shell, path):
diff --git a/tox.ini b/tox.ini
index e8e8bdb2..3db3a8ef 100644
--- a/tox.ini
+++ b/tox.ini
@@ -19,7 +19,7 @@ envlist = py27,py37,py38
 passenv = *
 usedevelop = True
 deps =
-    py36,py3,py38: black
+    py36,py37,py38: black
     isort
     pylint
     pytest

From b68953b73396e1461f9751f7570ea608c09c9bff Mon Sep 17 00:00:00 2001
From: Ivan Kravets <me@ikravets.com>
Date: Sat, 23 May 2020 20:01:25 +0300
Subject: [PATCH 41/42] Bump version to 4.3.4b1

---
 platformio/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/platformio/__init__.py b/platformio/__init__.py
index af520b21..34f79414 100644
--- a/platformio/__init__.py
+++ b/platformio/__init__.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-VERSION = (4, 3, "4a5")
+VERSION = (4, 3, "4b1")
 __version__ = ".".join([str(s) for s in VERSION])
 
 __title__ = "platformio"

From 81843087550123a66a77e3d0d0563ad759676921 Mon Sep 17 00:00:00 2001
From: Ivan Kravets <me@ikravets.com>
Date: Sat, 23 May 2020 20:33:13 +0300
Subject: [PATCH 42/42] Bump version to 4.3.4

---
 HISTORY.rst            | 6 +++---
 docs                   | 2 +-
 platformio/__init__.py | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/HISTORY.rst b/HISTORY.rst
index 75202dfc..ab9119fe 100644
--- a/HISTORY.rst
+++ b/HISTORY.rst
@@ -6,12 +6,12 @@ Release Notes
 PlatformIO Core 4
 -----------------
 
-4.3.4 (2020-??-??)
+4.3.4 (2020-05-23)
 ~~~~~~~~~~~~~~~~~~
 
-* Added `PlatformIO CLI Shell Completion <https://docs.platformio.org/page/core/userguide/misc/completion/index.html>`__  for Fish, Zsh, Bash, and PowerShell (`issue #3435 <https://github.com/platformio/platformio-core/issues/3435>`_)
+* Added `PlatformIO CLI Shell Completion <https://docs.platformio.org/page/core/userguide/system/completion/index.html>`__ for Fish, Zsh, Bash, and PowerShell (`issue #3435 <https://github.com/platformio/platformio-core/issues/3435>`_)
 * Automatically build ``contrib-pysite`` package on a target machine when pre-built package is not compatible (`issue #3482 <https://github.com/platformio/platformio-core/issues/3482>`_)
-* Fixed an issue on Windows when installing a dependency from Git repository raised a permission error (`issue #2844 <https://github.com/platformio/platformio-core/issues/2844>`_, `issue #3328 <https://github.com/platformio/platformio-core/issues/3328>`_)
+* Fixed an issue on Windows when installing a library dependency from Git repository (`issue #2844 <https://github.com/platformio/platformio-core/issues/2844>`_, `issue #3328 <https://github.com/platformio/platformio-core/issues/3328>`_)
 
 4.3.3 (2020-04-28)
 ~~~~~~~~~~~~~~~~~~
diff --git a/docs b/docs
index 2cd0dce6..68341524 160000
--- a/docs
+++ b/docs
@@ -1 +1 @@
-Subproject commit 2cd0dce683f40f5606bf7476d47e7bb052cfebba
+Subproject commit 683415246be491a91c2f8e63fa46b0e6ab55f91b
diff --git a/platformio/__init__.py b/platformio/__init__.py
index 34f79414..1e2e3fd1 100644
--- a/platformio/__init__.py
+++ b/platformio/__init__.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-VERSION = (4, 3, "4b1")
+VERSION = (4, 3, 4)
 __version__ = ".".join([str(s) for s in VERSION])
 
 __title__ = "platformio"