diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml index 9d8a57fc..c84a97d7 100644 --- a/.github/workflows/core.yml +++ b/.github/workflows/core.yml @@ -12,13 +12,14 @@ jobs: runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v2 + with: + submodules: "recursive" - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | - git submodule update --init --recursive python -m pip install --upgrade pip pip install tox diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 491728a5..bfe2c116 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -7,13 +7,14 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 + with: + submodules: "recursive" - name: Set up Python uses: actions/setup-python@v1 with: python-version: 3.7 - name: Install dependencies run: | - git submodule update --init --recursive python -m pip install --upgrade pip pip install tox diff --git a/.github/workflows/examples.yml b/.github/workflows/examples.yml index 3fa5b26c..b5452909 100644 --- a/.github/workflows/examples.yml +++ b/.github/workflows/examples.yml @@ -12,13 +12,14 @@ jobs: runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v2 + with: + submodules: "recursive" - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | - git submodule update --init --recursive python -m pip install --upgrade pip pip install tox diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d9764348..ec53bebb 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,20 +1,20 @@ Contributing ------------ -To get started, sign the Contributor License Agreement. +To get started, sign the Contributor License Agreement. 1. Fork the repository on GitHub. 2. Clone repository `git clone --recursive https://github.com/YourGithubUsername/platformio-core.git` 3. Run `pip install tox` -4. Go to the root of project where is located `tox.ini` and run `tox -e py27` +4. Go to the root of project where is located `tox.ini` and run `tox -e py37` 5. Activate current development environment: - * Windows: `.tox\py27\Scripts\activate` - * Bash/ZSH: `source .tox/py27/bin/activate` - * Fish: `source .tox/py27/bin/activate.fish` + * Windows: `.tox\py37\Scripts\activate` + * Bash/ZSH: `source .tox/py37/bin/activate` + * Fish: `source .tox/py37/bin/activate.fish` 6. Make changes to code, documentation, etc. -7. Lint source code `make lint` +7. Lint source code `make before-commit` 8. Run the tests `make test` 9. Build documentation `tox -e docs` (creates a directory _build under docs where you can find the html) 10. Commit changes to your forked repository diff --git a/HISTORY.rst b/HISTORY.rst index a2bf6ad8..ab9119fe 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -6,6 +6,13 @@ Release Notes PlatformIO Core 4 ----------------- +4.3.4 (2020-05-23) +~~~~~~~~~~~~~~~~~~ + +* Added `PlatformIO CLI Shell Completion `__ for Fish, Zsh, Bash, and PowerShell (`issue #3435 `_) +* Automatically build ``contrib-pysite`` package on a target machine when pre-built package is not compatible (`issue #3482 `_) +* Fixed an issue on Windows when installing a library dependency from Git repository (`issue #2844 `_, `issue #3328 `_) + 4.3.3 (2020-04-28) ~~~~~~~~~~~~~~~~~~ @@ -17,7 +24,7 @@ PlatformIO Core 4 * New `Account Management System `__ (preview) * Open source `PIO Remote `__ client * Improved `PIO Check `__ with more accurate project processing -* Echo what is typed when ``send_on_enter`` device monitor filter `__ is used (`issue #3452 `_) +* Echo what is typed when ``send_on_enter`` `device monitor filter `__ is used (`issue #3452 `_) * Fixed PIO Unit Testing for Zephyr RTOS * Fixed UnicodeDecodeError on Windows when network drive (NAS) is used (`issue #3417 `_) * Fixed an issue when saving libraries in new project results in error "No option 'lib_deps' in section" (`issue #3442 `_) diff --git a/docs b/docs index 790be9c1..68341524 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 790be9c1994750272d1312b0b4f4b50ba1973071 +Subproject commit 683415246be491a91c2f8e63fa46b0e6ab55f91b diff --git a/platformio/__init__.py b/platformio/__init__.py index 78bda530..1e2e3fd1 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 3, 3) +VERSION = (4, 3, 4) __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" diff --git a/platformio/__main__.py b/platformio/__main__.py index 6679d52e..8420544b 100644 --- a/platformio/__main__.py +++ b/platformio/__main__.py @@ -22,6 +22,13 @@ from platformio import __version__, exception, maintenance, util from platformio.commands import PlatformioCLI from platformio.compat import CYGWIN +try: + import click_completion # pylint: disable=import-error + + click_completion.init() +except: # pylint: disable=bare-except + pass + @click.command( cls=PlatformioCLI, context_settings=dict(help_option_names=["-h", "--help"]) diff --git a/platformio/builder/main.py b/platformio/builder/main.py index c895f3f3..7184da7c 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -28,7 +28,7 @@ from SCons.Script import DefaultEnvironment # pylint: disable=import-error from SCons.Script import Import # pylint: disable=import-error from SCons.Script import Variables # pylint: disable=import-error -from platformio import fs +from platformio import compat, fs from platformio.compat import dump_json_to_unicode from platformio.managers.platform import PlatformBase from platformio.proc import get_pythonexe_path @@ -120,6 +120,18 @@ env.Replace( ], ) +if ( + compat.WINDOWS + and sys.version_info >= (3, 8) + and env["PROJECT_DIR"].startswith("\\\\") +): + click.secho( + "There is a known issue with Python 3.8+ and mapped network drives on " + "Windows.\nPlease downgrade Python to the latest 3.7. More details at:\n" + "https://github.com/platformio/platformio-core/issues/3417", + fg="yellow", + ) + if env.subst("$BUILD_CACHE_DIR"): if not isdir(env.subst("$BUILD_CACHE_DIR")): makedirs(env.subst("$BUILD_CACHE_DIR")) diff --git a/platformio/builder/tools/compilation_db.py b/platformio/builder/tools/compilation_db.py index 52463e83..150a832e 100644 --- a/platformio/builder/tools/compilation_db.py +++ b/platformio/builder/tools/compilation_db.py @@ -1,17 +1,24 @@ # Copyright (c) 2014-present PlatformIO -# Copyright 2015 MongoDB Inc. +# Copyright 2020 MongoDB Inc. # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: # -# http://www.apache.org/licenses/LICENSE-2.0 +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # pylint: disable=unused-argument, protected-access, unused-variable, import-error # Original: https://github.com/mongodb/mongo/blob/master/site_scons/site_tools/compilation_db.py diff --git a/platformio/commands/account/client.py b/platformio/commands/account/client.py index 5381dab9..fb679dc0 100644 --- a/platformio/commands/account/client.py +++ b/platformio/commands/account/client.py @@ -22,9 +22,13 @@ from requests.packages.urllib3.util.retry import Retry # pylint:disable=import- from platformio import __pioaccount_api__, app from platformio.commands.account import exception +from platformio.exception import InternetIsOffline class AccountClient(object): + + SUMMARY_CACHE_TTL = 60 * 60 * 24 * 7 + def __init__( self, api_base_url=__pioaccount_api__, retries=3, ): @@ -43,21 +47,40 @@ class AccountClient(object): adapter = requests.adapters.HTTPAdapter(max_retries=retry) self._session.mount(api_base_url, adapter) + @staticmethod + def get_refresh_token(): + try: + return app.get_state_item("account").get("auth").get("refresh_token") + except: # pylint:disable=bare-except + raise exception.AccountNotAuthorized() + + @staticmethod + def delete_local_session(): + app.delete_state_item("account") + + @staticmethod + def delete_local_state(key): + account = app.get_state_item("account") + if not account or key not in account: + return + del account[key] + app.set_state_item("account", account) + def login(self, username, password): try: self.fetch_authentication_token() except: # pylint:disable=bare-except pass else: - raise exception.AccountAlreadyAuthenticated( + raise exception.AccountAlreadyAuthorized( app.get_state_item("account", {}).get("email", "") ) - response = self._session.post( + result = self.send_request( + "post", self.api_base_url + "/v1/login", data={"username": username, "password": password}, ) - result = self.raise_error_from_response(response) app.set_state_item("account", result) return result @@ -67,44 +90,39 @@ class AccountClient(object): except: # pylint:disable=bare-except pass else: - raise exception.AccountAlreadyAuthenticated( + raise exception.AccountAlreadyAuthorized( app.get_state_item("account", {}).get("email", "") ) - response = self._session.post( + result = self.send_request( + "post", self.api_base_url + "/v1/login/code", data={"client_id": client_id, "code": code, "redirect_uri": redirect_uri}, ) - result = self.raise_error_from_response(response) app.set_state_item("account", result) return result def logout(self): + refresh_token = self.get_refresh_token() + self.delete_local_session() try: - refresh_token = self.get_refresh_token() - except: # pylint:disable=bare-except - raise exception.AccountNotAuthenticated() - response = requests.post( - self.api_base_url + "/v1/logout", data={"refresh_token": refresh_token}, - ) - try: - self.raise_error_from_response(response) + self.send_request( + "post", + self.api_base_url + "/v1/logout", + data={"refresh_token": refresh_token}, + ) except exception.AccountError: pass - app.delete_state_item("account") return True def change_password(self, old_password, new_password): - try: - token = self.fetch_authentication_token() - except: # pylint:disable=bare-except - raise exception.AccountNotAuthenticated() - response = self._session.post( + token = self.fetch_authentication_token() + self.send_request( + "post", self.api_base_url + "/v1/password", headers={"Authorization": "Bearer %s" % token}, data={"old_password": old_password, "new_password": new_password}, ) - self.raise_error_from_response(response) return True def registration( @@ -115,11 +133,12 @@ class AccountClient(object): except: # pylint:disable=bare-except pass else: - raise exception.AccountAlreadyAuthenticated( + raise exception.AccountAlreadyAuthorized( app.get_state_item("account", {}).get("email", "") ) - response = self._session.post( + return self.send_request( + "post", self.api_base_url + "/v1/registration", data={ "username": username, @@ -129,70 +148,73 @@ class AccountClient(object): "lastname": lastname, }, ) - return self.raise_error_from_response(response) def auth_token(self, password, regenerate): - try: - token = self.fetch_authentication_token() - except: # pylint:disable=bare-except - raise exception.AccountNotAuthenticated() - response = self._session.post( + token = self.fetch_authentication_token() + result = self.send_request( + "post", self.api_base_url + "/v1/token", headers={"Authorization": "Bearer %s" % token}, data={"password": password, "regenerate": 1 if regenerate else 0}, ) - return self.raise_error_from_response(response).get("auth_token") + return result.get("auth_token") def forgot_password(self, username): - response = self._session.post( - self.api_base_url + "/v1/forgot", data={"username": username}, + return self.send_request( + "post", self.api_base_url + "/v1/forgot", data={"username": username}, ) - return self.raise_error_from_response(response).get("auth_token") def get_profile(self): - try: - token = self.fetch_authentication_token() - except: # pylint:disable=bare-except - raise exception.AccountNotAuthenticated() - response = self._session.get( + token = self.fetch_authentication_token() + return self.send_request( + "get", self.api_base_url + "/v1/profile", headers={"Authorization": "Bearer %s" % token}, ) - return self.raise_error_from_response(response) def update_profile(self, profile, current_password): - try: - token = self.fetch_authentication_token() - except: # pylint:disable=bare-except - raise exception.AccountNotAuthenticated() + token = self.fetch_authentication_token() profile["current_password"] = current_password - response = self._session.put( + self.delete_local_state("summary") + response = self.send_request( + "put", self.api_base_url + "/v1/profile", headers={"Authorization": "Bearer %s" % token}, data=profile, ) - return self.raise_error_from_response(response) + return response def get_account_info(self, offline): + account = app.get_state_item("account") + if not account: + raise exception.AccountNotAuthorized() + if ( + account.get("summary") + and account["summary"].get("expire_at", 0) > time.time() + ): + return account["summary"] if offline: - account = app.get_state_item("account") - if not account: - raise exception.AccountNotAuthenticated() return { "profile": { "email": account.get("email"), "username": account.get("username"), } } - try: - token = self.fetch_authentication_token() - except: # pylint:disable=bare-except - raise exception.AccountNotAuthenticated() - response = self._session.get( + token = self.fetch_authentication_token() + result = self.send_request( + "get", self.api_base_url + "/v1/summary", headers={"Authorization": "Bearer %s" % token}, ) - return self.raise_error_from_response(response) + account["summary"] = dict( + profile=result.get("profile"), + packages=result.get("packages"), + subscriptions=result.get("subscriptions"), + user_id=result.get("user_id"), + expire_at=int(time.time()) + self.SUMMARY_CACHE_TTL, + ) + app.set_state_item("account", account) + return result def fetch_authentication_token(self): if "PLATFORMIO_AUTH_TOKEN" in os.environ: @@ -202,25 +224,30 @@ class AccountClient(object): if auth.get("access_token_expire") > time.time(): return auth.get("access_token") if auth.get("refresh_token"): - response = self._session.post( - self.api_base_url + "/v1/login", - headers={"Authorization": "Bearer %s" % auth.get("refresh_token")}, - ) - result = self.raise_error_from_response(response) - app.set_state_item("account", result) - return result.get("auth").get("access_token") - raise exception.AccountNotAuthenticated() + try: + result = self.send_request( + "post", + self.api_base_url + "/v1/login", + headers={ + "Authorization": "Bearer %s" % auth.get("refresh_token") + }, + ) + app.set_state_item("account", result) + return result.get("auth").get("access_token") + except exception.AccountError: + self.delete_local_session() + raise exception.AccountNotAuthorized() - @staticmethod - def get_refresh_token(): + def send_request(self, method, url, headers=None, data=None): try: - auth = app.get_state_item("account").get("auth").get("refresh_token") - return auth - except: # pylint:disable=bare-except - raise exception.AccountNotAuthenticated() + response = getattr(self._session, method)( + url, headers=headers or {}, data=data or {} + ) + except requests.exceptions.ConnectionError: + raise InternetIsOffline() + return self.raise_error_from_response(response) - @staticmethod - def raise_error_from_response(response, expected_codes=(200, 201, 202)): + def raise_error_from_response(self, response, expected_codes=(200, 201, 202)): if response.status_code in expected_codes: try: return response.json() @@ -231,5 +258,5 @@ class AccountClient(object): except (KeyError, ValueError): message = response.text if "Authorization session has been expired" in message: - app.delete_state_item("account") + self.delete_local_session() raise exception.AccountError(message) diff --git a/platformio/commands/account/command.py b/platformio/commands/account/command.py index 539f1a5b..0177d00a 100644 --- a/platformio/commands/account/command.py +++ b/platformio/commands/account/command.py @@ -167,7 +167,7 @@ def account_update(current_password, **kwargs): return None try: client.logout() - except exception.AccountNotAuthenticated: + except exception.AccountNotAuthorized: pass if email_changed: return click.secho( diff --git a/platformio/commands/account/exception.py b/platformio/commands/account/exception.py index 213be0e1..a1a0059e 100644 --- a/platformio/commands/account/exception.py +++ b/platformio/commands/account/exception.py @@ -20,11 +20,11 @@ class AccountError(PlatformioException): MESSAGE = "{0}" -class AccountNotAuthenticated(AccountError): +class AccountNotAuthorized(AccountError): - MESSAGE = "You are not authenticated! Please login to PIO Account." + MESSAGE = "You are not authorized! Please log in to PIO Account." -class AccountAlreadyAuthenticated(AccountError): +class AccountAlreadyAuthorized(AccountError): - MESSAGE = "You are already authenticated with {0} account." + MESSAGE = "You are already authorized with {0} account." diff --git a/platformio/commands/home/command.py b/platformio/commands/home/command.py index 1eaad046..32d28063 100644 --- a/platformio/commands/home/command.py +++ b/platformio/commands/home/command.py @@ -22,11 +22,7 @@ import click from platformio import exception from platformio.compat import WINDOWS -from platformio.managers.core import ( - build_contrib_pysite_deps, - get_core_package_dir, - inject_contrib_pysite, -) +from platformio.managers.core import get_core_package_dir, inject_contrib_pysite @click.command("home", short_help="PIO Home") @@ -55,12 +51,7 @@ def cli(port, host, no_open, shutdown_timeout): # import contrib modules inject_contrib_pysite() - try: - from autobahn.twisted.resource import WebSocketResource - except: # pylint: disable=bare-except - build_contrib_pysite_deps(get_core_package_dir("contrib-pysite")) - from autobahn.twisted.resource import WebSocketResource - + from autobahn.twisted.resource import WebSocketResource from twisted.internet import reactor from twisted.web import server from twisted.internet.error import CannotListenError diff --git a/platformio/commands/home/rpc/handlers/os.py b/platformio/commands/home/rpc/handlers/os.py index 745ae817..2997e8aa 100644 --- a/platformio/commands/home/rpc/handlers/os.py +++ b/platformio/commands/home/rpc/handlers/os.py @@ -107,7 +107,7 @@ class OSRPC(object): @staticmethod def copy(src, dst): - return shutil.copytree(src, dst) + return shutil.copytree(src, dst, symlinks=True) @staticmethod def glob(pathnames, root=None): diff --git a/platformio/commands/home/rpc/handlers/piocore.py b/platformio/commands/home/rpc/handlers/piocore.py index 9bcef9e5..00adf5b6 100644 --- a/platformio/commands/home/rpc/handlers/piocore.py +++ b/platformio/commands/home/rpc/handlers/piocore.py @@ -51,6 +51,8 @@ class MultiThreadingStdStream(object): def write(self, value): thread_id = thread_get_ident() self._ensure_thread_buffer(thread_id) + if PY2 and isinstance(value, unicode): # pylint: disable=undefined-variable + value = value.encode() return self._buffers[thread_id].write( value.decode() if is_bytes(value) else value ) @@ -59,8 +61,8 @@ class MultiThreadingStdStream(object): result = "" try: result = self.getvalue() - self.truncate(0) self.seek(0) + self.truncate(0) except AttributeError: pass return result diff --git a/platformio/commands/home/rpc/handlers/project.py b/platformio/commands/home/rpc/handlers/project.py index 77a04646..3f4cdc88 100644 --- a/platformio/commands/home/rpc/handlers/project.py +++ b/platformio/commands/home/rpc/handlers/project.py @@ -300,7 +300,7 @@ class ProjectRPC(object): src_dir = config.get_optional_dir("src") if os.path.isdir(src_dir): fs.rmtree(src_dir) - shutil.copytree(arduino_project_dir, src_dir) + shutil.copytree(arduino_project_dir, src_dir, symlinks=True) return project_dir @staticmethod @@ -313,7 +313,7 @@ class ProjectRPC(object): AppRPC.load_state()["storage"]["projectsDir"], time.strftime("%y%m%d-%H%M%S-") + os.path.basename(project_dir), ) - shutil.copytree(project_dir, new_project_dir) + shutil.copytree(project_dir, new_project_dir, symlinks=True) state = AppRPC.load_state() args = ["init"] diff --git a/platformio/commands/remote/command.py b/platformio/commands/remote/command.py index d4a1cc34..f9e24c29 100644 --- a/platformio/commands/remote/command.py +++ b/platformio/commands/remote/command.py @@ -44,7 +44,7 @@ def cli(ctx, agent): "https://docs.platformio.org/page/core/installation.html" ) ctx.obj = agent - inject_contrib_pysite() + inject_contrib_pysite(verify_openssl=True) @cli.group("agent", short_help="Start a new agent or list active") diff --git a/platformio/commands/remote/factory/client.py b/platformio/commands/remote/factory/client.py index 202c7da6..26abe080 100644 --- a/platformio/commands/remote/factory/client.py +++ b/platformio/commands/remote/factory/client.py @@ -13,7 +13,7 @@ # limitations under the License. from twisted.cred import credentials # pylint: disable=import-error -from twisted.internet import protocol, reactor # pylint: disable=import-error +from twisted.internet import defer, protocol, reactor # pylint: disable=import-error from twisted.spread import pb # pylint: disable=import-error from platformio.app import get_host_id @@ -35,17 +35,27 @@ class RemoteClientFactory(pb.PBClientFactory, protocol.ReconnectingClientFactory self.remote_client.log.info("Successfully connected") self.remote_client.log.info("Authenticating") + auth_token = None + try: + auth_token = AccountClient().fetch_authentication_token() + except Exception as e: # pylint:disable=broad-except + d = defer.Deferred() + d.addErrback(self.clientAuthorizationFailed) + d.errback(pb.Error(e)) + return d + d = self.login( - credentials.UsernamePassword( - AccountClient().fetch_authentication_token().encode(), - get_host_id().encode(), - ), + credentials.UsernamePassword(auth_token.encode(), get_host_id().encode(),), client=self.remote_client, ) d.addCallback(self.remote_client.cb_client_authorization_made) - d.addErrback(self.remote_client.cb_client_authorization_failed) + d.addErrback(self.clientAuthorizationFailed) return d + def clientAuthorizationFailed(self, err): + AccountClient.delete_local_session() + self.remote_client.cb_client_authorization_failed(err) + def clientConnectionFailed(self, connector, reason): self.remote_client.log.warn( "Could not connect to PIO Remote Cloud. Reconnecting..." diff --git a/platformio/commands/system/__init__.py b/platformio/commands/system/__init__.py new file mode 100644 index 00000000..b0514903 --- /dev/null +++ b/platformio/commands/system/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/commands/system/command.py b/platformio/commands/system/command.py new file mode 100644 index 00000000..48336bfd --- /dev/null +++ b/platformio/commands/system/command.py @@ -0,0 +1,96 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import subprocess + +import click + +from platformio import proc +from platformio.commands.system.completion import ( + get_completion_install_path, + install_completion_code, + uninstall_completion_code, +) + + +@click.group("system", short_help="Miscellaneous system commands") +def cli(): + pass + + +@cli.group("completion", short_help="Shell completion support") +def completion(): + # pylint: disable=import-error,import-outside-toplevel + try: + import click_completion # pylint: disable=unused-import,unused-variable + except ImportError: + click.echo("Installing dependent packages...") + subprocess.check_call( + [proc.get_pythonexe_path(), "-m", "pip", "install", "click-completion"], + ) + + +@completion.command("install", short_help="Install shell completion files/code") +@click.option( + "--shell", + default=None, + type=click.Choice(["fish", "bash", "zsh", "powershell", "auto"]), + help="The shell type, default=auto", +) +@click.option( + "--path", + type=click.Path(file_okay=True, dir_okay=False, readable=True, resolve_path=True), + help="Custom installation path of the code to be evaluated by the shell. " + "The standard installation path is used by default.", +) +def completion_install(shell, path): + + import click_completion # pylint: disable=import-outside-toplevel,import-error + + shell = shell or click_completion.get_auto_shell() + path = path or get_completion_install_path(shell) + install_completion_code(shell, path) + click.echo( + "PlatformIO CLI completion has been installed for %s shell to %s \n" + "Please restart a current shell session." + % (click.style(shell, fg="cyan"), click.style(path, fg="blue")) + ) + + +@completion.command("uninstall", short_help="Uninstall shell completion files/code") +@click.option( + "--shell", + default=None, + type=click.Choice(["fish", "bash", "zsh", "powershell", "auto"]), + help="The shell type, default=auto", +) +@click.option( + "--path", + type=click.Path(file_okay=True, dir_okay=False, readable=True, resolve_path=True), + help="Custom installation path of the code to be evaluated by the shell. " + "The standard installation path is used by default.", +) +def completion_uninstall(shell, path): + + import click_completion # pylint: disable=import-outside-toplevel,import-error + + shell = shell or click_completion.get_auto_shell() + path = path or get_completion_install_path(shell) + uninstall_completion_code(shell, path) + click.echo( + "PlatformIO CLI completion has been uninstalled for %s shell from %s \n" + "Please restart a current shell session." + % (click.style(shell, fg="cyan"), click.style(path, fg="blue")) + ) diff --git a/platformio/commands/system/completion.py b/platformio/commands/system/completion.py new file mode 100644 index 00000000..1a969203 --- /dev/null +++ b/platformio/commands/system/completion.py @@ -0,0 +1,73 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import subprocess + +import click + + +def get_completion_install_path(shell): + home_dir = os.path.expanduser("~") + prog_name = click.get_current_context().find_root().info_name + if shell == "fish": + return os.path.join( + home_dir, ".config", "fish", "completions", "%s.fish" % prog_name + ) + if shell == "bash": + return os.path.join(home_dir, ".bash_completion") + if shell == "zsh": + return os.path.join(home_dir, ".zshrc") + if shell == "powershell": + return subprocess.check_output( + ["powershell", "-NoProfile", "echo $profile"] + ).strip() + raise click.ClickException("%s is not supported." % shell) + + +def is_completion_code_installed(shell, path): + if shell == "fish" or not os.path.exists(path): + return False + + import click_completion # pylint: disable=import-error,import-outside-toplevel + + with open(path) as fp: + return click_completion.get_code(shell=shell) in fp.read() + + +def install_completion_code(shell, path): + import click_completion # pylint: disable=import-error,import-outside-toplevel + + if is_completion_code_installed(shell, path): + return None + + return click_completion.install(shell=shell, path=path, append=shell != "fish") + + +def uninstall_completion_code(shell, path): + if not os.path.exists(path): + return True + if shell == "fish": + os.remove(path) + return True + + import click_completion # pylint: disable=import-error,import-outside-toplevel + + with open(path, "r+") as fp: + contents = fp.read() + fp.seek(0) + fp.truncate() + fp.write(contents.replace(click_completion.get_code(shell=shell), "")) + + return True diff --git a/platformio/exception.py b/platformio/exception.py index 905c5ab1..d291ad7f 100644 --- a/platformio/exception.py +++ b/platformio/exception.py @@ -232,8 +232,8 @@ class InternetIsOffline(UserSideException): MESSAGE = ( "You are not connected to the Internet.\n" - "If you build a project first time, we need Internet connection " - "to install all dependencies and toolchains." + "PlatformIO needs the Internet connection to" + " download dependent packages or to work with PIO Account." ) diff --git a/platformio/managers/core.py b/platformio/managers/core.py index cb6ad31c..53f435fd 100644 --- a/platformio/managers/core.py +++ b/platformio/managers/core.py @@ -24,7 +24,7 @@ from platformio.proc import get_pythonexe_path from platformio.project.config import ProjectConfig CORE_PACKAGES = { - "contrib-piohome": "~3.2.0", + "contrib-piohome": "~3.2.1", "contrib-pysite": "~2.%d%d.0" % (sys.version_info.major, sys.version_info.minor), "tool-unity": "~1.20500.0", "tool-scons": "~2.20501.7" if PY2 else "~3.30102.0", @@ -100,15 +100,27 @@ def update_core_packages(only_check=False, silent=False): return True -def inject_contrib_pysite(): - from site import addsitedir # pylint: disable=import-outside-toplevel +def inject_contrib_pysite(verify_openssl=False): + # pylint: disable=import-outside-toplevel + from site import addsitedir contrib_pysite_dir = get_core_package_dir("contrib-pysite") if contrib_pysite_dir in sys.path: - return + return True addsitedir(contrib_pysite_dir) sys.path.insert(0, contrib_pysite_dir) + if not verify_openssl: + return True + + try: + # pylint: disable=import-error,unused-import,unused-variable + from OpenSSL import SSL + except: # pylint: disable=bare-except + build_contrib_pysite_deps(get_core_package_dir("contrib-pysite")) + + return True + def build_contrib_pysite_deps(target_dir): if os.path.isdir(target_dir): @@ -126,13 +138,13 @@ def build_contrib_pysite_deps(target_dir): pythonexe = get_pythonexe_path() for dep in get_contrib_pysite_deps(): - subprocess.call( + subprocess.check_call( [ pythonexe, "-m", "pip", "install", - "--no-cache-dir", + # "--no-cache-dir", "--no-compile", "-t", target_dir, @@ -146,11 +158,11 @@ def get_contrib_pysite_deps(): sys_type = util.get_systype() py_version = "%d%d" % (sys.version_info.major, sys.version_info.minor) - twisted_version = "19.7.0" + twisted_version = "19.10.0" if PY2 else "20.3.0" result = [ "twisted == %s" % twisted_version, - "autobahn == 19.10.1", - "json-rpc == 1.12.1", + "autobahn == 20.4.3", + "json-rpc == 1.13.0", ] # twisted[tls], see setup.py for %twisted_version% @@ -159,12 +171,12 @@ def get_contrib_pysite_deps(): ) # zeroconf - if sys.version_info.major < 3: + if PY2: result.append( "https://github.com/ivankravets/python-zeroconf/" "archive/pio-py27.zip" ) else: - result.append("zeroconf == 0.23.0") + result.append("zeroconf == 0.26.0") if "windows" in sys_type: result.append("pypiwin32 == 223") diff --git a/platformio/managers/package.py b/platformio/managers/package.py index 07f2a23f..92ba4515 100644 --- a/platformio/managers/package.py +++ b/platformio/managers/package.py @@ -475,7 +475,7 @@ class PkgInstallerMixin(object): self.unpack(_url, tmp_dir) else: fs.rmtree(tmp_dir) - shutil.copytree(_url, tmp_dir) + shutil.copytree(_url, tmp_dir, symlinks=True) elif url.startswith(("http://", "https://")): dlpath = self.download(url, tmp_dir, sha1) assert isfile(dlpath) @@ -582,7 +582,11 @@ class PkgInstallerMixin(object): # remove previous/not-satisfied package if isdir(pkg_dir): fs.rmtree(pkg_dir) - shutil.move(tmp_dir, pkg_dir) + shutil.copytree(tmp_dir, pkg_dir, symlinks=True) + try: + shutil.rmtree(tmp_dir) + except: # pylint: disable=bare-except + pass assert isdir(pkg_dir) self.cache_reset() return pkg_dir diff --git a/platformio/package/manifest/schema.py b/platformio/package/manifest/schema.py index be49b3ee..11d3f902 100644 --- a/platformio/package/manifest/schema.py +++ b/platformio/package/manifest/schema.py @@ -242,7 +242,7 @@ class ManifestSchema(BaseSchema): def load_spdx_licenses(): r = requests.get( "https://raw.githubusercontent.com/spdx/license-list-data" - "/v3.8/json/licenses.json" + "/v3.9/json/licenses.json" ) r.raise_for_status() return r.json() diff --git a/tests/commands/test_account.py b/tests/commands/test_account.py index 20b02734..ef7ffbad 100644 --- a/tests/commands/test_account.py +++ b/tests/commands/test_account.py @@ -20,6 +20,14 @@ import pytest from platformio.commands.account.command import cli as cmd_account +pytestmark = pytest.mark.skipif( + not ( + os.environ.get("PLATFORMIO_TEST_ACCOUNT_LOGIN") + and os.environ.get("PLATFORMIO_TEST_ACCOUNT_PASSWORD") + ), + reason="requires PLATFORMIO_TEST_ACCOUNT_LOGIN, PLATFORMIO_TEST_ACCOUNT_PASSWORD environ variables", +) + @pytest.fixture(scope="session") def credentials(): @@ -93,7 +101,7 @@ def test_account_login(clirunner, credentials, validate_cliresult, isolated_pio_ ) assert result.exit_code > 0 assert result.exception - assert "You are already authenticated with" in str(result.exception) + assert "You are already authorized with" in str(result.exception) finally: clirunner.invoke(cmd_account, ["logout"]) @@ -113,7 +121,7 @@ def test_account_logout(clirunner, credentials, validate_cliresult, isolated_pio result = clirunner.invoke(cmd_account, ["logout"]) assert result.exit_code > 0 assert result.exception - assert "You are not authenticated! Please login to PIO Account" in str( + assert "You are not authorized! Please log in to PIO Account" in str( result.exception ) finally: @@ -192,7 +200,7 @@ def test_account_password_change( ) assert result.exit_code > 0 assert result.exception - assert "You are not authenticated! Please login to PIO Account" in str( + assert "You are not authorized! Please log in to PIO Account" in str( result.exception ) @@ -250,7 +258,7 @@ def test_account_token_with_invalid_password( ) assert result.exit_code > 0 assert result.exception - assert "You are not authenticated! Please login to PIO Account" in str( + assert "You are not authorized! Please log in to PIO Account" in str( result.exception ) @@ -302,7 +310,7 @@ def test_account_token(clirunner, credentials, validate_cliresult, isolated_pio_ ) assert result.exit_code > 0 assert result.exception - assert "You are not authenticated! Please login to PIO Account" in str( + assert "You are not authorized! Please log in to PIO Account" in str( result.exception ) @@ -371,7 +379,7 @@ def test_account_summary(clirunner, credentials, validate_cliresult, isolated_pi result = clirunner.invoke(cmd_account, ["show"],) assert result.exit_code > 0 assert result.exception - assert "You are not authenticated! Please login to PIO Account" in str( + assert "You are not authorized! Please log in to PIO Account" in str( result.exception ) @@ -381,6 +389,16 @@ def test_account_summary(clirunner, credentials, validate_cliresult, isolated_pi ) validate_cliresult(result) + result = clirunner.invoke(cmd_account, ["show", "--json-output", "--offline"]) + validate_cliresult(result) + json_result = json.loads(result.output.strip()) + assert not json_result.get("user_id") + assert json_result.get("profile") + assert json_result.get("profile").get("username") + assert json_result.get("profile").get("email") + assert not json_result.get("packages") + assert not json_result.get("subscriptions") + result = clirunner.invoke(cmd_account, ["show"]) validate_cliresult(result) assert credentials["login"] in result.output @@ -407,12 +425,19 @@ def test_account_summary(clirunner, credentials, validate_cliresult, isolated_pi result = clirunner.invoke(cmd_account, ["show", "--json-output", "--offline"]) validate_cliresult(result) json_result = json.loads(result.output.strip()) - assert not json_result.get("user_id") + assert json_result.get("user_id") assert json_result.get("profile") assert json_result.get("profile").get("username") assert json_result.get("profile").get("email") - assert not json_result.get("packages") - assert not json_result.get("subscriptions") + assert credentials["login"] == json_result.get("profile").get( + "username" + ) or credentials["login"] == json_result.get("profile").get("email") + assert json_result.get("profile").get("firstname") + assert json_result.get("profile").get("lastname") + assert json_result.get("packages") + assert json_result.get("packages")[0].get("name") + assert json_result.get("packages")[0].get("path") + assert json_result.get("subscriptions") is not None finally: clirunner.invoke(cmd_account, ["logout"]) @@ -427,7 +452,7 @@ def test_account_profile_update_with_invalid_password( ) assert result.exit_code > 0 assert result.exception - assert "You are not authenticated! Please login to PIO Account" in str( + assert "You are not authorized! Please log in to PIO Account" in str( result.exception ) @@ -460,7 +485,7 @@ def test_account_profile_update_only_firstname_and_lastname( ) assert result.exit_code > 0 assert result.exception - assert "You are not authenticated! Please login to PIO Account" in str( + assert "You are not authorized! Please log in to PIO Account" in str( result.exception ) @@ -508,7 +533,7 @@ def test_account_profile_update( ) assert result.exit_code > 0 assert result.exception - assert "You are not authenticated! Please login to PIO Account" in str( + assert "You are not authorized! Please log in to PIO Account" in str( result.exception ) @@ -549,7 +574,7 @@ def test_account_profile_update( result = clirunner.invoke(cmd_account, ["show"],) assert result.exit_code > 0 assert result.exception - assert "You are not authenticated! Please login to PIO Account" in str( + assert "You are not authorized! Please log in to PIO Account" in str( result.exception ) diff --git a/tox.ini b/tox.ini index 48792b86..3db3a8ef 100644 --- a/tox.ini +++ b/tox.ini @@ -13,13 +13,13 @@ # limitations under the License. [tox] -envlist = py27,py37 +envlist = py27,py37,py38 [testenv] passenv = * usedevelop = True deps = - py36,py37: black + py36,py37,py38: black isort pylint pytest