From 5de86a641672a4b790f5478534f08754e18bd5d8 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 30 Dec 2020 14:29:19 +0200 Subject: [PATCH 01/50] Check for debug server's "ready_pattern" in "stderr" --- HISTORY.rst | 5 +++++ platformio/commands/debug/process/server.py | 18 +++++++++++++----- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 9e5bff4e..be0f00d5 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -8,6 +8,11 @@ PlatformIO Core 5 **A professional collaborative platform for embedded development** +5.0.4 (2020-12-??) +~~~~~~~~~~~~~~~~~~ + +* Check for debug server's "ready_pattern" in "stderr" + 5.0.4 (2020-12-30) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/commands/debug/process/server.py b/platformio/commands/debug/process/server.py index 7bd5d485..050feac6 100644 --- a/platformio/commands/debug/process/server.py +++ b/platformio/commands/debug/process/server.py @@ -134,6 +134,14 @@ class DebugServer(BaseProcess): self._ready = self._last_activity < (time.time() - auto_ready_delay) elapsed += delay + def _check_ready_by_pattern(self, data): + if self._ready: + return self._ready + ready_pattern = self.debug_options.get("server", {}).get("ready_pattern") + if ready_pattern: + self._ready = ready_pattern.encode() in data + return self._ready + @staticmethod def async_sleep(secs): d = defer.Deferred() @@ -147,11 +155,11 @@ class DebugServer(BaseProcess): super(DebugServer, self).outReceived( escape_gdbmi_stream("@", data) if is_gdbmi_mode() else data ) - if self._ready: - return - ready_pattern = self.debug_options.get("server", {}).get("ready_pattern") - if ready_pattern: - self._ready = ready_pattern.encode() in data + self._check_ready_by_pattern(data) + + def errReceived(self, data): + super(DebugServer, self).errReceived(data) + self._check_ready_by_pattern(data) def processEnded(self, reason): self._process_ended = True From e82443a30233514dca762690be66e35a5fbca62e Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 30 Dec 2020 14:29:41 +0200 Subject: [PATCH 02/50] Bump version to 5.0.5a1 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index a540b3ff..3cd7e445 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (5, 0, 4) +VERSION = (5, 0, "5a1") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 76b49ebc956970f198fafd902fec6e654faf8a27 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 30 Dec 2020 14:38:18 +0200 Subject: [PATCH 03/50] Increase timeout to 60sec when starting debug server and "ready_pattern" is used --- platformio/commands/debug/process/server.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/platformio/commands/debug/process/server.py b/platformio/commands/debug/process/server.py index 050feac6..7a302c9b 100644 --- a/platformio/commands/debug/process/server.py +++ b/platformio/commands/debug/process/server.py @@ -124,13 +124,14 @@ class DebugServer(BaseProcess): @defer.inlineCallbacks def _wait_until_ready(self): - timeout = 10 + ready_pattern = self.debug_options.get("server", {}).get("ready_pattern") + timeout = 60 if ready_pattern else 10 elapsed = 0 delay = 0.5 auto_ready_delay = 0.5 while not self._ready and not self._process_ended and elapsed < timeout: yield self.async_sleep(delay) - if not self.debug_options.get("server", {}).get("ready_pattern"): + if not ready_pattern: self._ready = self._last_activity < (time.time() - auto_ready_delay) elapsed += delay From 556eb3f8c1150cab247ff5bb018328775dac33a5 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 31 Dec 2020 13:47:05 +0200 Subject: [PATCH 04/50] Docs: Update "Wiring Connections" section for ST-Link debugging probe --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 9db46dcc..515ac0e6 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 9db46dccef0a765770e71b64bf29b6e1d91df403 +Subproject commit 515ac0e6b9021d5d1cb6754db37d418e34e6de33 From 1ec2e55322d02ce331a2ad33c22c8b618db2a83d Mon Sep 17 00:00:00 2001 From: sephalon Date: Mon, 4 Jan 2021 12:46:09 +0100 Subject: [PATCH 05/50] Add udev rule for Atmel AVR Dragon (#3786) --- scripts/99-platformio-udev.rules | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/99-platformio-udev.rules b/scripts/99-platformio-udev.rules index b04b946f..86edd5b0 100644 --- a/scripts/99-platformio-udev.rules +++ b/scripts/99-platformio-udev.rules @@ -167,3 +167,6 @@ ATTRS{idVendor}=="c251", ATTRS{idProduct}=="2710", MODE="0666", ENV{ID_MM_DEVICE # CMSIS-DAP compatible adapters ATTRS{product}=="*CMSIS-DAP*", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" + +# Atmel AVR Dragon +ATTRS{idVendor}=="03eb", ATTRS{idProduct}=="2107", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" From 4e637ae58a3845c1968d9e7539a640e5517b64fe Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 18 Jan 2021 18:15:15 +0200 Subject: [PATCH 06/50] Drop Python 2 from PIO Core test --- .github/workflows/core.yml | 2 +- .pylintrc | 1 - platformio/builder/tools/compilation_db.py | 4 ++-- platformio/compat.py | 6 +++--- platformio/maintenance.py | 2 +- 5 files changed, 7 insertions(+), 8 deletions(-) diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml index e0aed6a9..afc59002 100644 --- a/.github/workflows/core.yml +++ b/.github/workflows/core.yml @@ -8,7 +8,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, windows-latest, macos-latest] - python-version: [2.7, 3.7, 3.8] + python-version: [3.6, 3.7, 3.8, 3.9] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v2 diff --git a/.pylintrc b/.pylintrc index e21dfef9..bb68f8a0 100644 --- a/.pylintrc +++ b/.pylintrc @@ -14,7 +14,6 @@ disable= too-few-public-methods, useless-object-inheritance, useless-import-alias, - fixme, bad-option-value, ; PY2 Compat diff --git a/platformio/builder/tools/compilation_db.py b/platformio/builder/tools/compilation_db.py index 150a832e..90b6517a 100644 --- a/platformio/builder/tools/compilation_db.py +++ b/platformio/builder/tools/compilation_db.py @@ -41,7 +41,7 @@ from platformio.proc import where_is_program # should hold the compilation database, otherwise, the file defaults to compile_commands.json, # which is the name that most clang tools search for by default. -# TODO: Is there a better way to do this than this global? Right now this exists so that the +# Is there a better way to do this than this global? Right now this exists so that the # emitter we add can record all of the things it emits, so that the scanner for the top level # compilation database can access the complete list, and also so that the writer has easy # access to write all of the files. But it seems clunky. How can the emitter and the scanner @@ -104,7 +104,7 @@ def makeEmitCompilationDbEntry(comstr): __COMPILATIONDB_ENV=env, ) - # TODO: Technically, these next two lines should not be required: it should be fine to + # Technically, these next two lines should not be required: it should be fine to # cache the entries. However, they don't seem to update properly. Since they are quick # to re-generate disable caching and sidestep this problem. env.AlwaysBuild(entry) diff --git a/platformio/compat.py b/platformio/compat.py index 974bdf2f..e25fbb6a 100644 --- a/platformio/compat.py +++ b/platformio/compat.py @@ -67,9 +67,9 @@ def ensure_python3(raise_exception=True): return compatible raise UserSideException( "Python 3.6 or later is required for this operation. \n" - "Please install the latest Python 3 and reinstall PlatformIO Core using " - "installation script:\n" - "https://docs.platformio.org/page/core/installation.html" + "Please check a migration guide:\n" + "https://docs.platformio.org/en/latest/core/migration.html" + "#drop-support-for-python-2-and-3-5" ) diff --git a/platformio/maintenance.py b/platformio/maintenance.py index 9d2f8245..4c252df8 100644 --- a/platformio/maintenance.py +++ b/platformio/maintenance.py @@ -53,7 +53,7 @@ def on_platformio_start(ctx, force, caller): click.secho( """ Python 2 and Python 3.5 are not compatible with PlatformIO Core 5.0. -Please check the migration guide on how to fix this warning message: +Please check a migration guide on how to fix this warning message: """, fg="yellow", ) From dd7d282d17e5b89e230faf93c51f34c2641caeef Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 18 Jan 2021 18:17:10 +0200 Subject: [PATCH 07/50] Improved listing of multicast DNS services --- HISTORY.rst | 3 ++- platformio/util.py | 29 +++++++++++++---------------- setup.py | 1 + 3 files changed, 16 insertions(+), 17 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index be0f00d5..0b16b2e8 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -8,9 +8,10 @@ PlatformIO Core 5 **A professional collaborative platform for embedded development** -5.0.4 (2020-12-??) +5.0.5 (2021-??-??) ~~~~~~~~~~~~~~~~~~ +* Improved listing of `multicast DNS services `_ * Check for debug server's "ready_pattern" in "stderr" 5.0.4 (2020-12-30) diff --git a/platformio/util.py b/platformio/util.py index e777d10a..c3d0da3c 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -24,8 +24,9 @@ from functools import wraps from glob import glob import click +import zeroconf -from platformio import __version__, exception, proc +from platformio import __version__, compat, exception, proc from platformio.compat import PY2, WINDOWS from platformio.fs import cd, load_json # pylint: disable=unused-import from platformio.proc import exec_command # pylint: disable=unused-import @@ -162,14 +163,7 @@ def get_logical_devices(): def get_mdns_services(): - # pylint: disable=import-outside-toplevel - try: - import zeroconf - except ImportError: - from platformio.package.manager.core import inject_contrib_pysite - - inject_contrib_pysite() - import zeroconf # pylint: disable=import-outside-toplevel + compat.ensure_python3() class mDNSListener(object): def __init__(self): @@ -178,7 +172,15 @@ def get_mdns_services(): self._found_services = [] def __enter__(self): - zeroconf.ServiceBrowser(self._zc, "_services._dns-sd._udp.local.", self) + zeroconf.ServiceBrowser( + self._zc, + [ + "_http._tcp.local.", + "_hap._tcp.local.", + "_services._dns-sd._udp.local.", + ], + self, + ) return self def __exit__(self, etype, value, traceback): @@ -225,12 +227,7 @@ def get_mdns_services(): { "type": service.type, "name": service.name, - "ip": ".".join( - [ - str(c if isinstance(c, int) else ord(c)) - for c in service.address - ] - ), + "ip": ", ".join(service.parsed_addresses()), "port": service.port, "properties": properties, } diff --git a/setup.py b/setup.py index 6472eadb..21d3465c 100644 --- a/setup.py +++ b/setup.py @@ -36,6 +36,7 @@ install_requires = [ "tabulate>=0.8.3,<1", "pyelftools>=0.25,<1", "marshmallow%s" % (">=2,<3" if PY2 else ">=2"), + "zeroconf==%s" % ("0.19.*" if PY2 else "0.28.*"), ] From 6ff67aeadfc63b421c0f652c70ac0df5b8bb4098 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 18 Jan 2021 18:20:26 +0200 Subject: [PATCH 08/50] Significantly speedup PlatformIO Home loading time by migrating to native Python 3 Asynchronous I/O --- HISTORY.rst | 1 + platformio/commands/home/command.py | 123 +++++++------- platformio/commands/home/helpers.py | 24 +-- .../commands/home/rpc/handlers/account.py | 4 +- platformio/commands/home/rpc/handlers/app.py | 2 +- platformio/commands/home/rpc/handlers/ide.py | 11 +- platformio/commands/home/rpc/handlers/misc.py | 21 ++- platformio/commands/home/rpc/handlers/os.py | 30 ++-- .../commands/home/rpc/handlers/piocore.py | 82 ++++------ .../commands/home/rpc/handlers/project.py | 7 +- platformio/commands/home/rpc/server.py | 152 ++++++++++-------- platformio/compat.py | 12 ++ setup.py | 25 +-- 13 files changed, 253 insertions(+), 241 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 0b16b2e8..0bca88c8 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -11,6 +11,7 @@ PlatformIO Core 5 5.0.5 (2021-??-??) ~~~~~~~~~~~~~~~~~~ +* Significantly speedup PlatformIO Home loading time by migrating to native Python 3 Asynchronous I/O * Improved listing of `multicast DNS services `_ * Check for debug server's "ready_pattern" in "stderr" diff --git a/platformio/commands/home/command.py b/platformio/commands/home/command.py index 6cb26ed9..abb13074 100644 --- a/platformio/commands/home/command.py +++ b/platformio/commands/home/command.py @@ -15,17 +15,17 @@ # pylint: disable=too-many-locals,too-many-statements import mimetypes +import os import socket -from os.path import isdir import click from platformio import exception -from platformio.compat import WINDOWS -from platformio.package.manager.core import get_core_package_dir, inject_contrib_pysite +from platformio.compat import WINDOWS, ensure_python3 +from platformio.package.manager.core import get_core_package_dir -@click.command("home", short_help="UI to manage PlatformIO") +@click.command("home", short_help="GUI to manage PlatformIO") @click.option("--port", type=int, default=8008, help="HTTP port, default=8008") @click.option( "--host", @@ -46,60 +46,16 @@ from platformio.package.manager.core import get_core_package_dir, inject_contrib ), ) def cli(port, host, no_open, shutdown_timeout): - # pylint: disable=import-error, import-outside-toplevel - - # import contrib modules - inject_contrib_pysite() - - from autobahn.twisted.resource import WebSocketResource - from twisted.internet import reactor - from twisted.web import server - from twisted.internet.error import CannotListenError - - from platformio.commands.home.rpc.handlers.app import AppRPC - from platformio.commands.home.rpc.handlers.ide import IDERPC - from platformio.commands.home.rpc.handlers.misc import MiscRPC - from platformio.commands.home.rpc.handlers.os import OSRPC - from platformio.commands.home.rpc.handlers.piocore import PIOCoreRPC - from platformio.commands.home.rpc.handlers.project import ProjectRPC - from platformio.commands.home.rpc.handlers.account import AccountRPC - from platformio.commands.home.rpc.server import JSONRPCServerFactory - from platformio.commands.home.web import WebRoot - - factory = JSONRPCServerFactory(shutdown_timeout) - factory.addHandler(AppRPC(), namespace="app") - factory.addHandler(IDERPC(), namespace="ide") - factory.addHandler(MiscRPC(), namespace="misc") - factory.addHandler(OSRPC(), namespace="os") - factory.addHandler(PIOCoreRPC(), namespace="core") - factory.addHandler(ProjectRPC(), namespace="project") - factory.addHandler(AccountRPC(), namespace="account") - - contrib_dir = get_core_package_dir("contrib-piohome") - if not isdir(contrib_dir): - raise exception.PlatformioException("Invalid path to PIO Home Contrib") - # Ensure PIO Home mimetypes are known mimetypes.add_type("text/html", ".html") mimetypes.add_type("text/css", ".css") mimetypes.add_type("application/javascript", ".js") - root = WebRoot(contrib_dir) - root.putChild(b"wsrpc", WebSocketResource(factory)) - site = server.Site(root) - # hook for `platformio-node-helpers` if host == "__do_not_start__": return - already_started = is_port_used(host, port) home_url = "http://%s:%d" % (host, port) - if not no_open: - if already_started: - click.launch(home_url) - else: - reactor.callLater(1, lambda: click.launch(home_url)) - click.echo( "\n".join( [ @@ -115,21 +71,21 @@ def cli(port, host, no_open, shutdown_timeout): click.echo("") click.echo("Open PlatformIO Home in your browser by this URL => %s" % home_url) - try: - reactor.listenTCP(port, site, interface=host) - except CannotListenError as e: - click.secho(str(e), fg="red", err=True) - already_started = True - - if already_started: + if is_port_used(host, port): click.secho( "PlatformIO Home server is already started in another process.", fg="yellow" ) + if not no_open: + click.launch(home_url) return - click.echo("PIO Home has been started. Press Ctrl+C to shutdown.") - - reactor.run() + run_server( + host=host, + port=port, + no_open=no_open, + shutdown_timeout=shutdown_timeout, + home_url=home_url, + ) def is_port_used(host, port): @@ -150,3 +106,54 @@ def is_port_used(host, port): return False return True + + +def run_server(host, port, no_open, shutdown_timeout, home_url): + # pylint: disable=import-error, import-outside-toplevel + + ensure_python3() + + import uvicorn + from starlette.applications import Starlette + from starlette.routing import Mount, WebSocketRoute + from starlette.staticfiles import StaticFiles + + from platformio.commands.home.rpc.handlers.account import AccountRPC + from platformio.commands.home.rpc.handlers.app import AppRPC + from platformio.commands.home.rpc.handlers.ide import IDERPC + from platformio.commands.home.rpc.handlers.misc import MiscRPC + from platformio.commands.home.rpc.handlers.os import OSRPC + from platformio.commands.home.rpc.handlers.piocore import PIOCoreRPC + from platformio.commands.home.rpc.handlers.project import ProjectRPC + from platformio.commands.home.rpc.server import WebSocketJSONRPCServerFactory + + contrib_dir = get_core_package_dir("contrib-piohome") + if not os.path.isdir(contrib_dir): + raise exception.PlatformioException("Invalid path to PIO Home Contrib") + + ws_rpc_factory = WebSocketJSONRPCServerFactory(shutdown_timeout) + ws_rpc_factory.addHandler(AccountRPC(), namespace="account") + ws_rpc_factory.addHandler(AppRPC(), namespace="app") + ws_rpc_factory.addHandler(IDERPC(), namespace="ide") + ws_rpc_factory.addHandler(MiscRPC(), namespace="misc") + ws_rpc_factory.addHandler(OSRPC(), namespace="os") + ws_rpc_factory.addHandler(PIOCoreRPC(), namespace="core") + ws_rpc_factory.addHandler(ProjectRPC(), namespace="project") + + uvicorn.run( + Starlette( + routes=[ + WebSocketRoute("/wsrpc", ws_rpc_factory, name="wsrpc"), + Mount("/", StaticFiles(directory=contrib_dir, html=True)), + ], + on_startup=[ + lambda: click.echo( + "PIO Home has been started. Press Ctrl+C to shutdown." + ), + lambda: None if no_open else click.launch(home_url), + ], + ), + host=host, + port=port, + log_level="warning", + ) diff --git a/platformio/commands/home/helpers.py b/platformio/commands/home/helpers.py index aff92281..dfd57c25 100644 --- a/platformio/commands/home/helpers.py +++ b/platformio/commands/home/helpers.py @@ -12,36 +12,24 @@ # See the License for the specific language governing permissions and # limitations under the License. -# pylint: disable=keyword-arg-before-vararg,arguments-differ,signature-differs - import requests -from twisted.internet import defer # pylint: disable=import-error -from twisted.internet import reactor # pylint: disable=import-error -from twisted.internet import threads # pylint: disable=import-error +from starlette.concurrency import run_in_threadpool from platformio import util from platformio.proc import where_is_program class AsyncSession(requests.Session): - def __init__(self, n=None, *args, **kwargs): - if n: - pool = reactor.getThreadPool() - pool.adjustPoolsize(0, n) - - super(AsyncSession, self).__init__(*args, **kwargs) - - def request(self, *args, **kwargs): + async def request( # pylint: disable=signature-differs,invalid-overridden-method + self, *args, **kwargs + ): func = super(AsyncSession, self).request - return threads.deferToThread(func, *args, **kwargs) - - def wrap(self, *args, **kwargs): # pylint: disable=no-self-use - return defer.ensureDeferred(*args, **kwargs) + return await run_in_threadpool(func, *args, **kwargs) @util.memoized(expire="60s") def requests_session(): - return AsyncSession(n=5) + return AsyncSession() @util.memoized(expire="60s") diff --git a/platformio/commands/home/rpc/handlers/account.py b/platformio/commands/home/rpc/handlers/account.py index d28379f8..337d780a 100644 --- a/platformio/commands/home/rpc/handlers/account.py +++ b/platformio/commands/home/rpc/handlers/account.py @@ -12,12 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -import jsonrpc # pylint: disable=import-error +import jsonrpc from platformio.clients.account import AccountClient -class AccountRPC(object): +class AccountRPC: @staticmethod def call_client(method, *args, **kwargs): try: diff --git a/platformio/commands/home/rpc/handlers/app.py b/platformio/commands/home/rpc/handlers/app.py index 1fd49e22..3c0ce465 100644 --- a/platformio/commands/home/rpc/handlers/app.py +++ b/platformio/commands/home/rpc/handlers/app.py @@ -20,7 +20,7 @@ from platformio import __version__, app, fs, util from platformio.project.helpers import get_project_core_dir, is_platformio_project -class AppRPC(object): +class AppRPC: APPSTATE_PATH = join(get_project_core_dir(), "homestate.json") diff --git a/platformio/commands/home/rpc/handlers/ide.py b/platformio/commands/home/rpc/handlers/ide.py index e3ad75f3..ed95b738 100644 --- a/platformio/commands/home/rpc/handlers/ide.py +++ b/platformio/commands/home/rpc/handlers/ide.py @@ -14,11 +14,12 @@ import time -import jsonrpc # pylint: disable=import-error -from twisted.internet import defer # pylint: disable=import-error +import jsonrpc + +from platformio.compat import get_running_loop -class IDERPC(object): +class IDERPC: def __init__(self): self._queue = {} @@ -28,14 +29,14 @@ class IDERPC(object): code=4005, message="PIO Home IDE agent is not started" ) while self._queue[sid]: - self._queue[sid].pop().callback( + self._queue[sid].pop().set_result( {"id": time.time(), "method": command, "params": params} ) def listen_commands(self, sid=0): if sid not in self._queue: self._queue[sid] = [] - self._queue[sid].append(defer.Deferred()) + self._queue[sid].append(get_running_loop().create_future()) return self._queue[sid][-1] def open_project(self, sid, project_dir): diff --git a/platformio/commands/home/rpc/handlers/misc.py b/platformio/commands/home/rpc/handlers/misc.py index a4bdc652..c16a6cc9 100644 --- a/platformio/commands/home/rpc/handlers/misc.py +++ b/platformio/commands/home/rpc/handlers/misc.py @@ -15,14 +15,13 @@ import json import time -from twisted.internet import defer, reactor # pylint: disable=import-error - from platformio.cache import ContentCache from platformio.commands.home.rpc.handlers.os import OSRPC +from platformio.compat import create_task -class MiscRPC(object): - def load_latest_tweets(self, data_url): +class MiscRPC: + async def load_latest_tweets(self, data_url): cache_key = ContentCache.key_from_args(data_url, "tweets") cache_valid = "180d" with ContentCache() as cc: @@ -31,22 +30,20 @@ class MiscRPC(object): cache_data = json.loads(cache_data) # automatically update cache in background every 12 hours if cache_data["time"] < (time.time() - (3600 * 12)): - reactor.callLater( - 5, self._preload_latest_tweets, data_url, cache_key, cache_valid + create_task( + self._preload_latest_tweets(data_url, cache_key, cache_valid) ) return cache_data["result"] - result = self._preload_latest_tweets(data_url, cache_key, cache_valid) - return result + return await self._preload_latest_tweets(data_url, cache_key, cache_valid) @staticmethod - @defer.inlineCallbacks - def _preload_latest_tweets(data_url, cache_key, cache_valid): - result = json.loads((yield OSRPC.fetch_content(data_url))) + async def _preload_latest_tweets(data_url, cache_key, cache_valid): + result = json.loads((await OSRPC.fetch_content(data_url))) with ContentCache() as cc: cc.set( cache_key, json.dumps({"time": int(time.time()), "result": result}), cache_valid, ) - defer.returnValue(result) + return result diff --git a/platformio/commands/home/rpc/handlers/os.py b/platformio/commands/home/rpc/handlers/os.py index 448c633a..f1042978 100644 --- a/platformio/commands/home/rpc/handlers/os.py +++ b/platformio/commands/home/rpc/handlers/os.py @@ -14,25 +14,23 @@ from __future__ import absolute_import +import glob import io import os import shutil from functools import cmp_to_key import click -from twisted.internet import defer # pylint: disable=import-error from platformio import __default_requests_timeout__, fs, util from platformio.cache import ContentCache from platformio.clients.http import ensure_internet_on from platformio.commands.home import helpers -from platformio.compat import PY2, get_filesystem_encoding, glob_recursive -class OSRPC(object): +class OSRPC: @staticmethod - @defer.inlineCallbacks - def fetch_content(uri, data=None, headers=None, cache_valid=None): + async def fetch_content(uri, data=None, headers=None, cache_valid=None): if not headers: headers = { "User-Agent": ( @@ -46,18 +44,18 @@ class OSRPC(object): if cache_key: result = cc.get(cache_key) if result is not None: - defer.returnValue(result) + return result # check internet before and resolve issue with 60 seconds timeout ensure_internet_on(raise_exception=True) session = helpers.requests_session() if data: - r = yield session.post( + r = await session.post( uri, data=data, headers=headers, timeout=__default_requests_timeout__ ) else: - r = yield session.get( + r = await session.get( uri, headers=headers, timeout=__default_requests_timeout__ ) @@ -66,11 +64,11 @@ class OSRPC(object): if cache_valid: with ContentCache() as cc: cc.set(cache_key, result, cache_valid) - defer.returnValue(result) + return result - def request_content(self, uri, data=None, headers=None, cache_valid=None): + async def request_content(self, uri, data=None, headers=None, cache_valid=None): if uri.startswith("http"): - return self.fetch_content(uri, data, headers, cache_valid) + return await self.fetch_content(uri, data, headers, cache_valid) if os.path.isfile(uri): with io.open(uri, encoding="utf-8") as fp: return fp.read() @@ -82,13 +80,11 @@ class OSRPC(object): @staticmethod def reveal_file(path): - return click.launch( - path.encode(get_filesystem_encoding()) if PY2 else path, locate=True - ) + return click.launch(path, locate=True) @staticmethod def open_file(path): - return click.launch(path.encode(get_filesystem_encoding()) if PY2 else path) + return click.launch(path) @staticmethod def is_file(path): @@ -121,7 +117,9 @@ class OSRPC(object): result = set() for pathname in pathnames: result |= set( - glob_recursive(os.path.join(root, pathname) if root else pathname) + glob.glob( + os.path.join(root, pathname) if root else pathname, recursive=True + ) ) return list(result) diff --git a/platformio/commands/home/rpc/handlers/piocore.py b/platformio/commands/home/rpc/handlers/piocore.py index 7a16f9c6..d74095ab 100644 --- a/platformio/commands/home/rpc/handlers/piocore.py +++ b/platformio/commands/home/rpc/handlers/piocore.py @@ -17,23 +17,15 @@ from __future__ import absolute_import import json import os import sys -from io import BytesIO, StringIO +from io import StringIO import click -import jsonrpc # pylint: disable=import-error -from twisted.internet import defer # pylint: disable=import-error -from twisted.internet import threads # pylint: disable=import-error -from twisted.internet import utils # pylint: disable=import-error +import jsonrpc +from starlette.concurrency import run_in_threadpool -from platformio import __main__, __version__, fs +from platformio import __main__, __version__, fs, proc from platformio.commands.home import helpers -from platformio.compat import ( - PY2, - get_filesystem_encoding, - get_locale_encoding, - is_bytes, - string_types, -) +from platformio.compat import get_locale_encoding, is_bytes try: from thread import get_ident as thread_get_ident @@ -52,13 +44,11 @@ class MultiThreadingStdStream(object): def _ensure_thread_buffer(self, thread_id): if thread_id not in self._buffers: - self._buffers[thread_id] = BytesIO() if PY2 else StringIO() + self._buffers[thread_id] = StringIO() def write(self, value): thread_id = thread_get_ident() self._ensure_thread_buffer(thread_id) - if PY2 and isinstance(value, unicode): # pylint: disable=undefined-variable - value = value.encode() return self._buffers[thread_id].write( value.decode() if is_bytes(value) else value ) @@ -74,7 +64,7 @@ class MultiThreadingStdStream(object): return result -class PIOCoreRPC(object): +class PIOCoreRPC: @staticmethod def version(): return __version__ @@ -89,16 +79,9 @@ class PIOCoreRPC(object): sys.stderr = PIOCoreRPC.thread_stderr @staticmethod - def call(args, options=None): - return defer.maybeDeferred(PIOCoreRPC._call_generator, args, options) - - @staticmethod - @defer.inlineCallbacks - def _call_generator(args, options=None): + async def call(args, options=None): for i, arg in enumerate(args): - if isinstance(arg, string_types): - args[i] = arg.encode(get_filesystem_encoding()) if PY2 else arg - else: + if not isinstance(arg, str): args[i] = str(arg) options = options or {} @@ -106,27 +89,34 @@ class PIOCoreRPC(object): try: if options.get("force_subprocess"): - result = yield PIOCoreRPC._call_subprocess(args, options) - defer.returnValue(PIOCoreRPC._process_result(result, to_json)) - else: - result = yield PIOCoreRPC._call_inline(args, options) - try: - defer.returnValue(PIOCoreRPC._process_result(result, to_json)) - except ValueError: - # fall-back to subprocess method - result = yield PIOCoreRPC._call_subprocess(args, options) - defer.returnValue(PIOCoreRPC._process_result(result, to_json)) + result = await PIOCoreRPC._call_subprocess(args, options) + return PIOCoreRPC._process_result(result, to_json) + result = await PIOCoreRPC._call_inline(args, options) + try: + return PIOCoreRPC._process_result(result, to_json) + except ValueError: + # fall-back to subprocess method + result = await PIOCoreRPC._call_subprocess(args, options) + return PIOCoreRPC._process_result(result, to_json) except Exception as e: # pylint: disable=bare-except raise jsonrpc.exceptions.JSONRPCDispatchException( code=4003, message="PIO Core Call Error", data=str(e) ) @staticmethod - def _call_inline(args, options): - PIOCoreRPC.setup_multithreading_std_streams() - cwd = options.get("cwd") or os.getcwd() + async def _call_subprocess(args, options): + result = await run_in_threadpool( + proc.exec_command, + [helpers.get_core_fullpath()] + args, + cwd=options.get("cwd") or os.getcwd(), + ) + return (result["out"], result["err"], result["returncode"]) - def _thread_task(): + @staticmethod + async def _call_inline(args, options): + PIOCoreRPC.setup_multithreading_std_streams() + + def _thread_safe_call(args, cwd): with fs.cd(cwd): exit_code = __main__.main(["-c"] + args) return ( @@ -135,16 +125,8 @@ class PIOCoreRPC(object): exit_code, ) - return threads.deferToThread(_thread_task) - - @staticmethod - def _call_subprocess(args, options): - cwd = (options or {}).get("cwd") or os.getcwd() - return utils.getProcessOutputAndValue( - helpers.get_core_fullpath(), - args, - path=cwd, - env={k: v for k, v in os.environ.items() if "%" not in k}, + return await run_in_threadpool( + _thread_safe_call, args=args, cwd=options.get("cwd") or os.getcwd() ) @staticmethod diff --git a/platformio/commands/home/rpc/handlers/project.py b/platformio/commands/home/rpc/handlers/project.py index eb9cd237..20b1dcc8 100644 --- a/platformio/commands/home/rpc/handlers/project.py +++ b/platformio/commands/home/rpc/handlers/project.py @@ -18,12 +18,11 @@ import os import shutil import time -import jsonrpc # pylint: disable=import-error +import jsonrpc from platformio import exception, fs from platformio.commands.home.rpc.handlers.app import AppRPC from platformio.commands.home.rpc.handlers.piocore import PIOCoreRPC -from platformio.compat import PY2, get_filesystem_encoding from platformio.ide.projectgenerator import ProjectGenerator from platformio.package.manager.platform import PlatformPackageManager from platformio.project.config import ProjectConfig @@ -32,7 +31,7 @@ from platformio.project.helpers import get_project_dir, is_platformio_project from platformio.project.options import get_config_options_schema -class ProjectRPC(object): +class ProjectRPC: @staticmethod def config_call(init_kwargs, method, *args): assert isinstance(init_kwargs, dict) @@ -254,8 +253,6 @@ class ProjectRPC(object): def import_arduino(self, board, use_arduino_libs, arduino_project_dir): board = str(board) - if arduino_project_dir and PY2: - arduino_project_dir = arduino_project_dir.encode(get_filesystem_encoding()) # don't import PIO Project if is_platformio_project(arduino_project_dir): return arduino_project_dir diff --git a/platformio/commands/home/rpc/server.py b/platformio/commands/home/rpc/server.py index 1924754f..a1448566 100644 --- a/platformio/commands/home/rpc/server.py +++ b/platformio/commands/home/rpc/server.py @@ -12,90 +12,112 @@ # See the License for the specific language governing permissions and # limitations under the License. -# pylint: disable=import-error +import inspect +import json +import sys import click import jsonrpc -from autobahn.twisted.websocket import WebSocketServerFactory, WebSocketServerProtocol -from jsonrpc.exceptions import JSONRPCDispatchException -from twisted.internet import defer, reactor +from starlette.endpoints import WebSocketEndpoint -from platformio.compat import PY2, dump_json_to_unicode, is_bytes +from platformio.compat import create_task, get_running_loop, is_bytes -class JSONRPCServerProtocol(WebSocketServerProtocol): - def onOpen(self): - self.factory.connection_nums += 1 - if self.factory.shutdown_timer: - self.factory.shutdown_timer.cancel() - self.factory.shutdown_timer = None +class JSONRPCServerFactoryBase: - def onClose(self, wasClean, code, reason): # pylint: disable=unused-argument - self.factory.connection_nums -= 1 - if self.factory.connection_nums == 0: - self.factory.shutdownByTimeout() - - def onMessage(self, payload, isBinary): # pylint: disable=unused-argument - # click.echo("> %s" % payload) - response = jsonrpc.JSONRPCResponseManager.handle( - payload, self.factory.dispatcher - ).data - # if error - if "result" not in response: - self.sendJSONResponse(response) - return None - - d = defer.maybeDeferred(lambda: response["result"]) - d.addCallback(self._callback, response) - d.addErrback(self._errback, response) - - return None - - def _callback(self, result, response): - response["result"] = result - self.sendJSONResponse(response) - - def _errback(self, failure, response): - if isinstance(failure.value, JSONRPCDispatchException): - e = failure.value - else: - e = JSONRPCDispatchException(code=4999, message=failure.getErrorMessage()) - del response["result"] - response["error"] = e.error._data # pylint: disable=protected-access - self.sendJSONResponse(response) - - def sendJSONResponse(self, response): - # click.echo("< %s" % response) - if "error" in response: - click.secho("Error: %s" % response["error"], fg="red", err=True) - response = dump_json_to_unicode(response) - if not PY2 and not is_bytes(response): - response = response.encode("utf-8") - self.sendMessage(response) - - -class JSONRPCServerFactory(WebSocketServerFactory): - - protocol = JSONRPCServerProtocol connection_nums = 0 - shutdown_timer = 0 + shutdown_timer = None def __init__(self, shutdown_timeout=0): - super(JSONRPCServerFactory, self).__init__() self.shutdown_timeout = shutdown_timeout self.dispatcher = jsonrpc.Dispatcher() - def shutdownByTimeout(self): + def __call__(self, *args, **kwargs): + raise NotImplementedError + + def addHandler(self, handler, namespace): + self.dispatcher.build_method_map(handler, prefix="%s." % namespace) + + def on_client_connect(self): + self.connection_nums += 1 + if self.shutdown_timer: + self.shutdown_timer.cancel() + self.shutdown_timer = None + + def on_client_disconnect(self): + self.connection_nums -= 1 + if self.connection_nums < 1: + self.connection_nums = 0 + + if self.connection_nums == 0: + self.shutdown_by_timeout() + + async def on_shutdown(self): + pass + + def shutdown_by_timeout(self): if self.shutdown_timeout < 1: return def _auto_shutdown_server(): click.echo("Automatically shutdown server on timeout") - reactor.stop() + try: + get_running_loop().stop() + except: # pylint: disable=bare-except + pass + finally: + sys.exit(0) - self.shutdown_timer = reactor.callLater( + self.shutdown_timer = get_running_loop().call_later( self.shutdown_timeout, _auto_shutdown_server ) - def addHandler(self, handler, namespace): - self.dispatcher.build_method_map(handler, prefix="%s." % namespace) + +class WebSocketJSONRPCServerFactory(JSONRPCServerFactoryBase): + def __call__(self, *args, **kwargs): + ws = WebSocketJSONRPCServer(*args, **kwargs) + ws.factory = self + return ws + + +class WebSocketJSONRPCServer(WebSocketEndpoint): + encoding = "text" + factory: WebSocketJSONRPCServerFactory = None + + async def on_connect(self, websocket): + await websocket.accept() + self.factory.on_client_connect() # pylint: disable=no-member + + async def on_receive(self, websocket, data): + create_task(self._handle_rpc(websocket, data)) + + async def on_disconnect(self, websocket, close_code): + self.factory.on_client_disconnect() # pylint: disable=no-member + + async def _handle_rpc(self, websocket, data): + response = jsonrpc.JSONRPCResponseManager.handle( + data, self.factory.dispatcher # pylint: disable=no-member + ) + if response.result and inspect.isawaitable(response.result): + try: + response.result = await response.result + response.data["result"] = response.result + response.error = None + except Exception as exc: # pylint: disable=broad-except + if not isinstance(exc, jsonrpc.exceptions.JSONRPCDispatchException): + exc = jsonrpc.exceptions.JSONRPCDispatchException( + code=4999, message=str(exc) + ) + response.result = None + response.error = exc.error._data # pylint: disable=protected-access + new_data = response.data.copy() + new_data["error"] = response.error + del new_data["result"] + response.data = new_data + + if response.error: + click.secho("Error: %s" % response.error, fg="red", err=True) + if "result" in response.data and is_bytes(response.data["result"]): + response.data["result"] = response.data["result"].decode("utf-8") + + await websocket.send_text(json.dumps(response.data)) diff --git a/platformio/compat.py b/platformio/compat.py index e25fbb6a..53c1507c 100644 --- a/platformio/compat.py +++ b/platformio/compat.py @@ -78,6 +78,12 @@ if PY2: string_types = (str, unicode) + def create_task(coro, name=None): + raise NotImplementedError + + def get_running_loop(): + raise NotImplementedError + def is_bytes(x): return isinstance(x, (buffer, bytearray)) @@ -129,6 +135,12 @@ else: import importlib.util from glob import escape as glob_escape + if sys.version_info >= (3, 7): + from asyncio import create_task, get_running_loop + else: + from asyncio import ensure_future as create_task + from asyncio import get_event_loop as get_running_loop + string_types = (str,) def is_bytes(x): diff --git a/setup.py b/setup.py index 21d3465c..577c2951 100644 --- a/setup.py +++ b/setup.py @@ -26,19 +26,26 @@ from platformio import ( from platformio.compat import PY2, WINDOWS -install_requires = [ - "bottle<0.13", +minimal_requirements = [ + "bottle==0.12.*", "click>=5,<8%s" % (",!=7.1,!=7.1.1" if WINDOWS else ""), "colorama", - "pyserial>=3,<4,!=3.3", - "requests>=2.4.0,<3", - "semantic_version>=2.8.1,<3", - "tabulate>=0.8.3,<1", - "pyelftools>=0.25,<1", - "marshmallow%s" % (">=2,<3" if PY2 else ">=2"), + "marshmallow%s" % (">=2,<3" if PY2 else ">=2,<4"), + "pyelftools>=0.27,<1", + "pyserial==3.*", + "requests==2.*", + "semantic_version==2.8.*", + "tabulate==0.8.*", "zeroconf==%s" % ("0.19.*" if PY2 else "0.28.*"), ] +home_requirements = [ + "aiofiles==0.6.*", + "json-rpc==1.13.*", + "starlette==0.14.*", + "uvicorn==0.13.*", + "wsproto==1.0.*", +] setup( name=__title__, @@ -52,7 +59,7 @@ setup( python_requires=", ".join( [">=2.7", "!=3.0.*", "!=3.1.*", "!=3.2.*", "!=3.3.*", "!=3.4.*"] ), - install_requires=install_requires, + install_requires=minimal_requirements + ([] if PY2 else home_requirements), packages=find_packages(exclude=["tests.*", "tests"]) + ["scripts"], package_data={ "platformio": [ From db97a7d9d335143e4a189f29ce21971915a44764 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 18 Jan 2021 18:21:27 +0200 Subject: [PATCH 09/50] Bump version to 5.0.5b1 --- docs | 2 +- platformio/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs b/docs index 515ac0e6..b33bf7a2 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 515ac0e6b9021d5d1cb6754db37d418e34e6de33 +Subproject commit b33bf7a282e244f8a4851b6e7ec5a2ecc1e3e307 diff --git a/platformio/__init__.py b/platformio/__init__.py index 3cd7e445..1448ecad 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (5, 0, "5a1") +VERSION = (5, 0, "5b1") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From b90734f1e22ba89dd5f335666e1829a81158f58c Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 18 Jan 2021 20:51:50 +0200 Subject: [PATCH 10/50] List multicast DNS services only when PY3 --- platformio/util.py | 13 +++++++++---- setup.py | 4 +++- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/platformio/util.py b/platformio/util.py index c3d0da3c..6b1af886 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -24,7 +24,6 @@ from functools import wraps from glob import glob import click -import zeroconf from platformio import __version__, compat, exception, proc from platformio.compat import PY2, WINDOWS @@ -165,6 +164,9 @@ def get_logical_devices(): def get_mdns_services(): compat.ensure_python3() + # pylint: disable=import-outside-toplevel + import zeroconf + class mDNSListener(object): def __init__(self): self._zc = zeroconf.Zeroconf(interfaces=zeroconf.InterfaceChoice.All) @@ -186,9 +188,6 @@ def get_mdns_services(): def __exit__(self, etype, value, traceback): self._zc.close() - def remove_service(self, zc, type_, name): - pass - def add_service(self, zc, type_, name): try: assert zeroconf.service_type_name(name) @@ -203,6 +202,12 @@ def get_mdns_services(): if s: self._found_services.append(s) + def remove_service(self, zc, type_, name): + pass + + def update_service(self, zc, type_, name): + pass + def get_services(self): return self._found_services diff --git a/setup.py b/setup.py index 577c2951..2aab992a 100644 --- a/setup.py +++ b/setup.py @@ -36,9 +36,11 @@ minimal_requirements = [ "requests==2.*", "semantic_version==2.8.*", "tabulate==0.8.*", - "zeroconf==%s" % ("0.19.*" if PY2 else "0.28.*"), ] +if not PY2: + minimal_requirements.append("zeroconf==0.28.*") + home_requirements = [ "aiofiles==0.6.*", "json-rpc==1.13.*", From 429065d2b94396ef002a8d2d8debef77f73e237b Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 18 Jan 2021 20:53:19 +0200 Subject: [PATCH 11/50] Legacy support for PIO Home "__shutdown__" query request --- platformio/commands/home/command.py | 85 +++----------------------- platformio/commands/home/helpers.py | 23 +++++++ platformio/commands/home/rpc/server.py | 9 +-- platformio/commands/home/run.py | 79 ++++++++++++++++++++++++ platformio/commands/home/web.py | 28 --------- platformio/proc.py | 10 +++ 6 files changed, 121 insertions(+), 113 deletions(-) create mode 100644 platformio/commands/home/run.py delete mode 100644 platformio/commands/home/web.py diff --git a/platformio/commands/home/command.py b/platformio/commands/home/command.py index abb13074..28cbfef4 100644 --- a/platformio/commands/home/command.py +++ b/platformio/commands/home/command.py @@ -12,17 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -# pylint: disable=too-many-locals,too-many-statements - import mimetypes -import os -import socket import click -from platformio import exception -from platformio.compat import WINDOWS, ensure_python3 -from platformio.package.manager.core import get_core_package_dir +from platformio.commands.home.helpers import is_port_used +from platformio.compat import ensure_python3 @click.command("home", short_help="GUI to manage PlatformIO") @@ -46,6 +41,8 @@ from platformio.package.manager.core import get_core_package_dir ), ) def cli(port, host, no_open, shutdown_timeout): + ensure_python3() + # Ensure PIO Home mimetypes are known mimetypes.add_type("text/html", ".html") mimetypes.add_type("text/css", ".css") @@ -79,6 +76,9 @@ def cli(port, host, no_open, shutdown_timeout): click.launch(home_url) return + # pylint: disable=import-outside-toplevel + from platformio.commands.home.run import run_server + run_server( host=host, port=port, @@ -86,74 +86,3 @@ def cli(port, host, no_open, shutdown_timeout): shutdown_timeout=shutdown_timeout, home_url=home_url, ) - - -def is_port_used(host, port): - socket.setdefaulttimeout(1) - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - if WINDOWS: - try: - s.bind((host, port)) - s.close() - return False - except (OSError, socket.error): - pass - else: - try: - s.connect((host, port)) - s.close() - except socket.error: - return False - - return True - - -def run_server(host, port, no_open, shutdown_timeout, home_url): - # pylint: disable=import-error, import-outside-toplevel - - ensure_python3() - - import uvicorn - from starlette.applications import Starlette - from starlette.routing import Mount, WebSocketRoute - from starlette.staticfiles import StaticFiles - - from platformio.commands.home.rpc.handlers.account import AccountRPC - from platformio.commands.home.rpc.handlers.app import AppRPC - from platformio.commands.home.rpc.handlers.ide import IDERPC - from platformio.commands.home.rpc.handlers.misc import MiscRPC - from platformio.commands.home.rpc.handlers.os import OSRPC - from platformio.commands.home.rpc.handlers.piocore import PIOCoreRPC - from platformio.commands.home.rpc.handlers.project import ProjectRPC - from platformio.commands.home.rpc.server import WebSocketJSONRPCServerFactory - - contrib_dir = get_core_package_dir("contrib-piohome") - if not os.path.isdir(contrib_dir): - raise exception.PlatformioException("Invalid path to PIO Home Contrib") - - ws_rpc_factory = WebSocketJSONRPCServerFactory(shutdown_timeout) - ws_rpc_factory.addHandler(AccountRPC(), namespace="account") - ws_rpc_factory.addHandler(AppRPC(), namespace="app") - ws_rpc_factory.addHandler(IDERPC(), namespace="ide") - ws_rpc_factory.addHandler(MiscRPC(), namespace="misc") - ws_rpc_factory.addHandler(OSRPC(), namespace="os") - ws_rpc_factory.addHandler(PIOCoreRPC(), namespace="core") - ws_rpc_factory.addHandler(ProjectRPC(), namespace="project") - - uvicorn.run( - Starlette( - routes=[ - WebSocketRoute("/wsrpc", ws_rpc_factory, name="wsrpc"), - Mount("/", StaticFiles(directory=contrib_dir, html=True)), - ], - on_startup=[ - lambda: click.echo( - "PIO Home has been started. Press Ctrl+C to shutdown." - ), - lambda: None if no_open else click.launch(home_url), - ], - ), - host=host, - port=port, - log_level="warning", - ) diff --git a/platformio/commands/home/helpers.py b/platformio/commands/home/helpers.py index dfd57c25..5c6e0c88 100644 --- a/platformio/commands/home/helpers.py +++ b/platformio/commands/home/helpers.py @@ -12,10 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +import socket + import requests from starlette.concurrency import run_in_threadpool from platformio import util +from platformio.compat import WINDOWS from platformio.proc import where_is_program @@ -37,3 +40,23 @@ def get_core_fullpath(): return where_is_program( "platformio" + (".exe" if "windows" in util.get_systype() else "") ) + + +def is_port_used(host, port): + socket.setdefaulttimeout(1) + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + if WINDOWS: + try: + s.bind((host, port)) + s.close() + return False + except (OSError, socket.error): + pass + else: + try: + s.connect((host, port)) + s.close() + except socket.error: + return False + + return True diff --git a/platformio/commands/home/rpc/server.py b/platformio/commands/home/rpc/server.py index a1448566..8ba41da9 100644 --- a/platformio/commands/home/rpc/server.py +++ b/platformio/commands/home/rpc/server.py @@ -14,13 +14,13 @@ import inspect import json -import sys import click import jsonrpc from starlette.endpoints import WebSocketEndpoint from platformio.compat import create_task, get_running_loop, is_bytes +from platformio.proc import force_exit class JSONRPCServerFactoryBase: @@ -61,12 +61,7 @@ class JSONRPCServerFactoryBase: def _auto_shutdown_server(): click.echo("Automatically shutdown server on timeout") - try: - get_running_loop().stop() - except: # pylint: disable=bare-except - pass - finally: - sys.exit(0) + force_exit() self.shutdown_timer = get_running_loop().call_later( self.shutdown_timeout, _auto_shutdown_server diff --git a/platformio/commands/home/run.py b/platformio/commands/home/run.py new file mode 100644 index 00000000..33096233 --- /dev/null +++ b/platformio/commands/home/run.py @@ -0,0 +1,79 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import click +import uvicorn +from starlette.applications import Starlette +from starlette.middleware import Middleware +from starlette.routing import Mount, WebSocketRoute +from starlette.staticfiles import StaticFiles + +from platformio.commands.home.rpc.handlers.account import AccountRPC +from platformio.commands.home.rpc.handlers.app import AppRPC +from platformio.commands.home.rpc.handlers.ide import IDERPC +from platformio.commands.home.rpc.handlers.misc import MiscRPC +from platformio.commands.home.rpc.handlers.os import OSRPC +from platformio.commands.home.rpc.handlers.piocore import PIOCoreRPC +from platformio.commands.home.rpc.handlers.project import ProjectRPC +from platformio.commands.home.rpc.server import WebSocketJSONRPCServerFactory +from platformio.compat import get_running_loop +from platformio.exception import PlatformioException +from platformio.package.manager.core import get_core_package_dir +from platformio.proc import force_exit + + +class ShutdownMiddleware: + def __init__(self, app): + self.app = app + + async def __call__(self, scope, receive, send): + if scope["type"] == "http" and b"__shutdown__" in scope.get("query_string", {}): + get_running_loop().call_later(0.5, force_exit) + await self.app(scope, receive, send) + + +def run_server(host, port, no_open, shutdown_timeout, home_url): + contrib_dir = get_core_package_dir("contrib-piohome") + if not os.path.isdir(contrib_dir): + raise PlatformioException("Invalid path to PIO Home Contrib") + + ws_rpc_factory = WebSocketJSONRPCServerFactory(shutdown_timeout) + ws_rpc_factory.addHandler(AccountRPC(), namespace="account") + ws_rpc_factory.addHandler(AppRPC(), namespace="app") + ws_rpc_factory.addHandler(IDERPC(), namespace="ide") + ws_rpc_factory.addHandler(MiscRPC(), namespace="misc") + ws_rpc_factory.addHandler(OSRPC(), namespace="os") + ws_rpc_factory.addHandler(PIOCoreRPC(), namespace="core") + ws_rpc_factory.addHandler(ProjectRPC(), namespace="project") + + uvicorn.run( + Starlette( + middleware=[Middleware(ShutdownMiddleware)], + routes=[ + WebSocketRoute("/wsrpc", ws_rpc_factory, name="wsrpc"), + Mount("/", StaticFiles(directory=contrib_dir, html=True)), + ], + on_startup=[ + lambda: click.echo( + "PIO Home has been started. Press Ctrl+C to shutdown." + ), + lambda: None if no_open else click.launch(home_url), + ], + ), + host=host, + port=port, + log_level="warning", + ) diff --git a/platformio/commands/home/web.py b/platformio/commands/home/web.py deleted file mode 100644 index 32bf0692..00000000 --- a/platformio/commands/home/web.py +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright (c) 2014-present PlatformIO -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from twisted.internet import reactor # pylint: disable=import-error -from twisted.web import static # pylint: disable=import-error - - -class WebRoot(static.File): - def render_GET(self, request): - if request.args.get(b"__shutdown__", False): - reactor.stop() - return "Server has been stopped" - - request.setHeader("cache-control", "no-cache, no-store, must-revalidate") - request.setHeader("pragma", "no-cache") - request.setHeader("expires", "0") - return static.File.render_GET(self, request) diff --git a/platformio/proc.py b/platformio/proc.py index 8db7153e..d9df0a3b 100644 --- a/platformio/proc.py +++ b/platformio/proc.py @@ -24,6 +24,7 @@ from platformio.compat import ( WINDOWS, get_filesystem_encoding, get_locale_encoding, + get_running_loop, string_types, ) @@ -214,3 +215,12 @@ def append_env_path(name, value): return cur_value os.environ[name] = os.pathsep.join([cur_value, value]) return os.environ[name] + + +def force_exit(code=0): + try: + get_running_loop().stop() + except: # pylint: disable=bare-except + pass + finally: + sys.exit(code) From bd897d780b25f1155ced5222f549165b9af5634e Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 18 Jan 2021 21:19:15 +0200 Subject: [PATCH 12/50] Implement "__shutdown__" endpoint for PIO Home server --- platformio/commands/home/run.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/platformio/commands/home/run.py b/platformio/commands/home/run.py index 33096233..63c48b48 100644 --- a/platformio/commands/home/run.py +++ b/platformio/commands/home/run.py @@ -12,13 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. +import asyncio import os import click import uvicorn from starlette.applications import Starlette from starlette.middleware import Middleware -from starlette.routing import Mount, WebSocketRoute +from starlette.responses import PlainTextResponse +from starlette.routing import Mount, Route, WebSocketRoute from starlette.staticfiles import StaticFiles from platformio.commands.home.rpc.handlers.account import AccountRPC @@ -29,7 +31,6 @@ from platformio.commands.home.rpc.handlers.os import OSRPC from platformio.commands.home.rpc.handlers.piocore import PIOCoreRPC from platformio.commands.home.rpc.handlers.project import ProjectRPC from platformio.commands.home.rpc.server import WebSocketJSONRPCServerFactory -from platformio.compat import get_running_loop from platformio.exception import PlatformioException from platformio.package.manager.core import get_core_package_dir from platformio.proc import force_exit @@ -41,10 +42,15 @@ class ShutdownMiddleware: async def __call__(self, scope, receive, send): if scope["type"] == "http" and b"__shutdown__" in scope.get("query_string", {}): - get_running_loop().call_later(0.5, force_exit) + await shutdown_server() await self.app(scope, receive, send) +async def shutdown_server(_=None): + asyncio.get_event_loop().call_later(0.5, force_exit) + return PlainTextResponse("Server has been shutdown!") + + def run_server(host, port, no_open, shutdown_timeout, home_url): contrib_dir = get_core_package_dir("contrib-piohome") if not os.path.isdir(contrib_dir): @@ -64,6 +70,7 @@ def run_server(host, port, no_open, shutdown_timeout, home_url): middleware=[Middleware(ShutdownMiddleware)], routes=[ WebSocketRoute("/wsrpc", ws_rpc_factory, name="wsrpc"), + Route("/__shutdown__", shutdown_server, methods=["POST"]), Mount("/", StaticFiles(directory=contrib_dir, html=True)), ], on_startup=[ From 733ca5174bc2b28f23bf6ff77290425b3aabeff1 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 18 Jan 2021 21:19:57 +0200 Subject: [PATCH 13/50] Bump version to 5.0.5b2 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 1448ecad..093fd05f 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (5, 0, "5b1") +VERSION = (5, 0, "5b2") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 9b93fcd947e244a541bb5dc20d1258a52d34b3c0 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 20 Jan 2021 14:27:03 +0200 Subject: [PATCH 14/50] Do not install tool-unity for even non-test proejct --- platformio/builder/tools/pioide.py | 2 +- platformio/package/manager/core.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/platformio/builder/tools/pioide.py b/platformio/builder/tools/pioide.py index 8248c06a..37cd55f0 100644 --- a/platformio/builder/tools/pioide.py +++ b/platformio/builder/tools/pioide.py @@ -59,7 +59,7 @@ def _dump_includes(env): includes["toolchain"].extend([os.path.realpath(inc) for inc in glob(g)]) includes["unity"] = [] - unity_dir = get_core_package_dir("tool-unity") + unity_dir = get_core_package_dir("tool-unity", auto_install=False) if unity_dir: includes["unity"].append(unity_dir) diff --git a/platformio/package/manager/core.py b/platformio/package/manager/core.py index a324beb4..f31a8767 100644 --- a/platformio/package/manager/core.py +++ b/platformio/package/manager/core.py @@ -27,7 +27,7 @@ from platformio.package.meta import PackageItem, PackageSpec from platformio.proc import get_pythonexe_path -def get_core_package_dir(name): +def get_core_package_dir(name, auto_install=True): if name not in __core_packages__: raise exception.PlatformioException("Please upgrade PlatformIO Core") pm = ToolPackageManager() @@ -37,6 +37,8 @@ def get_core_package_dir(name): pkg = pm.get_package(spec) if pkg: return pkg.path + if not auto_install: + return None assert pm.install(spec) _remove_unnecessary_packages() return pm.get_package(spec).path From 7f26c11c9db84fb511a1c58aedb4ef15ed51844e Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 20 Jan 2021 14:36:45 +0200 Subject: [PATCH 15/50] Fix an issue with "coroutine' object has no attribute 'addCallback'" --- .../commands/home/rpc/handlers/project.py | 25 +++++++------------ platformio/commands/home/run.py | 4 +-- 2 files changed, 11 insertions(+), 18 deletions(-) diff --git a/platformio/commands/home/rpc/handlers/project.py b/platformio/commands/home/rpc/handlers/project.py index 20b1dcc8..c8ec6f54 100644 --- a/platformio/commands/home/rpc/handlers/project.py +++ b/platformio/commands/home/rpc/handlers/project.py @@ -184,7 +184,7 @@ class ProjectRPC: ) return sorted(result, key=lambda data: data["platform"]["title"]) - def init(self, board, framework, project_dir): + async def init(self, board, framework, project_dir): assert project_dir state = AppRPC.load_state() if not os.path.isdir(project_dir): @@ -197,14 +197,13 @@ class ProjectRPC: and state["storage"]["coreCaller"] in ProjectGenerator.get_supported_ides() ): args.extend(["--ide", state["storage"]["coreCaller"]]) - d = PIOCoreRPC.call( + await PIOCoreRPC.call( args, options={"cwd": project_dir, "force_subprocess": True} ) - d.addCallback(self._generate_project_main, project_dir, framework) - return d + return self._generate_project_main(project_dir, framework) @staticmethod - def _generate_project_main(_, project_dir, framework): + def _generate_project_main(project_dir, framework): main_content = None if framework == "arduino": main_content = "\n".join( @@ -251,7 +250,7 @@ class ProjectRPC: fp.write(main_content.strip()) return project_dir - def import_arduino(self, board, use_arduino_libs, arduino_project_dir): + async def import_arduino(self, board, use_arduino_libs, arduino_project_dir): board = str(board) # don't import PIO Project if is_platformio_project(arduino_project_dir): @@ -290,14 +289,9 @@ class ProjectRPC: and state["storage"]["coreCaller"] in ProjectGenerator.get_supported_ides() ): args.extend(["--ide", state["storage"]["coreCaller"]]) - d = PIOCoreRPC.call( + await PIOCoreRPC.call( args, options={"cwd": project_dir, "force_subprocess": True} ) - d.addCallback(self._finalize_arduino_import, project_dir, arduino_project_dir) - return d - - @staticmethod - def _finalize_arduino_import(_, project_dir, arduino_project_dir): with fs.cd(project_dir): config = ProjectConfig() src_dir = config.get_optional_dir("src") @@ -307,7 +301,7 @@ class ProjectRPC: return project_dir @staticmethod - def import_pio(project_dir): + async def import_pio(project_dir): if not project_dir or not is_platformio_project(project_dir): raise jsonrpc.exceptions.JSONRPCDispatchException( code=4001, message="Not an PlatformIO project: %s" % project_dir @@ -325,8 +319,7 @@ class ProjectRPC: and state["storage"]["coreCaller"] in ProjectGenerator.get_supported_ides() ): args.extend(["--ide", state["storage"]["coreCaller"]]) - d = PIOCoreRPC.call( + await PIOCoreRPC.call( args, options={"cwd": new_project_dir, "force_subprocess": True} ) - d.addCallback(lambda _: new_project_dir) - return d + return new_project_dir diff --git a/platformio/commands/home/run.py b/platformio/commands/home/run.py index 63c48b48..585c9268 100644 --- a/platformio/commands/home/run.py +++ b/platformio/commands/home/run.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import asyncio import os import click @@ -31,6 +30,7 @@ from platformio.commands.home.rpc.handlers.os import OSRPC from platformio.commands.home.rpc.handlers.piocore import PIOCoreRPC from platformio.commands.home.rpc.handlers.project import ProjectRPC from platformio.commands.home.rpc.server import WebSocketJSONRPCServerFactory +from platformio.compat import get_running_loop from platformio.exception import PlatformioException from platformio.package.manager.core import get_core_package_dir from platformio.proc import force_exit @@ -47,7 +47,7 @@ class ShutdownMiddleware: async def shutdown_server(_=None): - asyncio.get_event_loop().call_later(0.5, force_exit) + get_running_loop().call_later(0.5, force_exit) return PlainTextResponse("Server has been shutdown!") From 11a71b7fbb3cf9e5623851f05c3cac89792d71e8 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 20 Jan 2021 14:37:19 +0200 Subject: [PATCH 16/50] Bump version to 5.0.5b3 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 093fd05f..99a13111 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (5, 0, "5b2") +VERSION = (5, 0, "5b3") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 985f31877cb9381d2d47d638e4da4d5b0d355ac0 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 20 Jan 2021 15:14:45 +0200 Subject: [PATCH 17/50] Automatically install tool-unity when there are tests and "idedata" target is called --- platformio/builder/tools/pioide.py | 5 ++++- tests/commands/test_check.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/platformio/builder/tools/pioide.py b/platformio/builder/tools/pioide.py index 37cd55f0..7c3ec208 100644 --- a/platformio/builder/tools/pioide.py +++ b/platformio/builder/tools/pioide.py @@ -59,7 +59,10 @@ def _dump_includes(env): includes["toolchain"].extend([os.path.realpath(inc) for inc in glob(g)]) includes["unity"] = [] - unity_dir = get_core_package_dir("tool-unity", auto_install=False) + unity_dir = get_core_package_dir( + "tool-unity", + auto_install=os.path.isdir(env.GetProjectConfig().get_optional_dir("test")), + ) if unity_dir: includes["unity"].append(unity_dir) diff --git a/tests/commands/test_check.py b/tests/commands/test_check.py index b8c8e65a..c631a613 100644 --- a/tests/commands/test_check.py +++ b/tests/commands/test_check.py @@ -154,7 +154,7 @@ def test_check_includes_passed(clirunner, check_dir): inc_count = l.count("-I") # at least 1 include path for default mode - assert inc_count > 1 + assert inc_count > 0 def test_check_silent_mode(clirunner, validate_cliresult, check_dir): From e79de0108cb722ad070ddfe3da174084c9575774 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 20 Jan 2021 16:15:05 +0200 Subject: [PATCH 18/50] Upgraded build engine to the SCons 4.1 --- HISTORY.rst | 1 + platformio/__init__.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index 0bca88c8..858f102c 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -14,6 +14,7 @@ PlatformIO Core 5 * Significantly speedup PlatformIO Home loading time by migrating to native Python 3 Asynchronous I/O * Improved listing of `multicast DNS services `_ * Check for debug server's "ready_pattern" in "stderr" +* Upgraded build engine to the SCons 4.1 (`release notes `_) 5.0.4 (2020-12-30) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/__init__.py b/platformio/__init__.py index 99a13111..21edc5e7 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -50,7 +50,7 @@ __core_packages__ = { "contrib-piohome": "~3.3.1", "contrib-pysite": "~2.%d%d.0" % (sys.version_info.major, sys.version_info.minor), "tool-unity": "~1.20500.0", - "tool-scons": "~2.20501.7" if sys.version_info.major == 2 else "~4.40001.0", + "tool-scons": "~2.20501.7" if sys.version_info.major == 2 else "~4.40100.0", "tool-cppcheck": "~1.230.0", "tool-clangtidy": "~1.100000.0", "tool-pvs-studio": "~7.11.0", From 5a356140d6c5532f5a26f931618baaf0302a0459 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 20 Jan 2021 20:44:43 +0200 Subject: [PATCH 19/50] Sync examples and docs --- docs | 2 +- examples | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs b/docs index b33bf7a2..15fabfbd 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit b33bf7a282e244f8a4851b6e7ec5a2ecc1e3e307 +Subproject commit 15fabfbdffeab014b76ecbeb3e65a930d3fdff2b diff --git a/examples b/examples index 161ae730..ced25363 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit 161ae7302b3508cadb10f5552de9f996731976ac +Subproject commit ced253632eac141595967b08ad6eb258c0bc0878 From 52b22b578439cecde029aec72e719b125dfad420 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 20 Jan 2021 20:45:23 +0200 Subject: [PATCH 20/50] Fixed a "UnicodeDecodeError: 'utf-8' codec can't decode byte" // Resolve #3804 , Resolve #3417 --- HISTORY.rst | 2 ++ platformio/proc.py | 19 +++++++++++-------- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 858f102c..9addc2bf 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -15,6 +15,8 @@ PlatformIO Core 5 * Improved listing of `multicast DNS services `_ * Check for debug server's "ready_pattern" in "stderr" * Upgraded build engine to the SCons 4.1 (`release notes `_) +* Fixed a "UnicodeDecodeError: 'utf-8' codec can't decode byte" when using J-Link for a firmware uploading on Linux (`issue #3804 `_) +* Fixed an issue with Python 3.8+ on Windows when network drive is used (`issue #3417 `_) 5.0.4 (2020-12-30) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/proc.py b/platformio/proc.py index d9df0a3b..24640c38 100644 --- a/platformio/proc.py +++ b/platformio/proc.py @@ -32,7 +32,10 @@ from platformio.compat import ( class AsyncPipeBase(object): def __init__(self): self._fd_read, self._fd_write = os.pipe() - self._pipe_reader = os.fdopen(self._fd_read) + if PY2: + self._pipe_reader = os.fdopen(self._fd_read) + else: + self._pipe_reader = os.fdopen(self._fd_read, errors="backslashreplace") self._buffer = "" self._thread = Thread(target=self.run) self._thread.start() @@ -68,10 +71,10 @@ class BuildAsyncPipe(AsyncPipeBase): line = "" print_immediately = False - for byte in iter(lambda: self._pipe_reader.read(1), ""): - self._buffer += byte + for char in iter(lambda: self._pipe_reader.read(1), ""): + self._buffer += char - if line and byte.strip() and line[-3:] == (byte * 3): + if line and char.strip() and line[-3:] == (char * 3): print_immediately = True if print_immediately: @@ -79,12 +82,12 @@ class BuildAsyncPipe(AsyncPipeBase): if line: self.data_callback(line) line = "" - self.data_callback(byte) - if byte == "\n": + self.data_callback(char) + if char == "\n": print_immediately = False else: - line += byte - if byte != "\n": + line += char + if char != "\n": continue self.line_callback(line) line = "" From 4488f25ce0d48bafe65c325609043fe11f73b97b Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 20 Jan 2021 23:26:22 +0200 Subject: [PATCH 21/50] Bump version to 5.0.5b4 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 21edc5e7..4666701e 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (5, 0, "5b3") +VERSION = (5, 0, "5b4") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From f9384ded27de2ce0342a479a8ae314931cd78009 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 22 Jan 2021 22:45:36 +0200 Subject: [PATCH 22/50] =?UTF-8?q?Fixed=20an=20issue=20when=20=E2=80=9Cstri?= =?UTF-8?q?ct=E2=80=9D=20compatibility=20mode=20was=20not=20used=20for=20a?= =?UTF-8?q?=20library=20with=20custom=20=E2=80=9Cplatforms=E2=80=9D=20fiel?= =?UTF-8?q?d=20in=20library.json=20manifest=20//=20Resolve=20#3806?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- HISTORY.rst | 11 ++++++----- platformio/builder/tools/piolib.py | 9 ++++++--- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 9addc2bf..e9b4990f 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -13,10 +13,11 @@ PlatformIO Core 5 * Significantly speedup PlatformIO Home loading time by migrating to native Python 3 Asynchronous I/O * Improved listing of `multicast DNS services `_ -* Check for debug server's "ready_pattern" in "stderr" +* Check for debugging server's "ready_pattern" in "stderr" * Upgraded build engine to the SCons 4.1 (`release notes `_) -* Fixed a "UnicodeDecodeError: 'utf-8' codec can't decode byte" when using J-Link for a firmware uploading on Linux (`issue #3804 `_) -* Fixed an issue with Python 3.8+ on Windows when network drive is used (`issue #3417 `_) +* Fixed a "UnicodeDecodeError: 'utf-8' codec can't decode byte" when using J-Link for firmware uploading on Linux (`issue #3804 `_) +* Fixed an issue with Python 3.8+ on Windows when a network drive is used (`issue #3417 `_) +* Fixed an issue when "strict" compatibility mode was not used for a library with custom "platforms" field in `library.json `__ manifest (`issue #3806 `_) 5.0.4 (2020-12-30) ~~~~~~~~~~~~~~~~~~ @@ -25,8 +26,8 @@ PlatformIO Core 5 - Improved ``.ccls`` configuration file for Emacs, Vim, and Sublime Text integrations - Updated analysis tools: - * `Cppcheck `__ v2.3 with improved C++ parser and several new MISRA rules - * `PVS-Studio `__ v7.11 with new diagnostics and updated mass suppression mechanism + * `Cppcheck `__ v2.3 with improved C++ parser and several new MISRA rules + * `PVS-Studio `__ v7.11 with new diagnostics and updated mass suppression mechanism - Show a warning message about deprecated support for Python 2 and Python 3.5 - Do not provide "intelliSenseMode" option when generating configuration for VSCode C/C++ extension diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index f6b9824b..97e2637a 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -773,11 +773,14 @@ class PlatformIOLibBuilder(LibBuilderBase): @property def lib_compat_mode(self): + mode = self._manifest.get("build", {}).get( + "libCompatMode", + ) + if not mode and self._manifest.get("platforms"): + mode = "strict" # pylint: disable=no-member return self.validate_compat_mode( - self._manifest.get("build", {}).get( - "libCompatMode", LibBuilderBase.lib_compat_mode.fget(self) - ) + mode or LibBuilderBase.lib_compat_mode.fget(self) ) def is_platforms_compatible(self, platforms): From b2c0e6a8c2d9884443e9752504ce2ca5b2b306c0 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 22 Jan 2021 22:46:09 +0200 Subject: [PATCH 23/50] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 15fabfbd..04a05d7f 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 15fabfbdffeab014b76ecbeb3e65a930d3fdff2b +Subproject commit 04a05d7f347764e24817e37de3fe26f701660645 From ddbe339541f9ccfc64e93f2cba2d847ef797fb01 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 22 Jan 2021 22:55:02 +0200 Subject: [PATCH 24/50] Update to iSort 5.0 --- .isort.cfg | 2 +- Makefile | 4 ++-- platformio/platform/factory.py | 6 +++--- tox.ini | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.isort.cfg b/.isort.cfg index de9bf40e..7d21b117 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -1,3 +1,3 @@ [settings] line_length=88 -known_third_party=OpenSSL, SCons, autobahn, jsonrpc, twisted, zope +known_third_party=OpenSSL, SCons, jsonrpc, twisted, zope diff --git a/Makefile b/Makefile index 3ddd1e27..a73d9cba 100644 --- a/Makefile +++ b/Makefile @@ -3,8 +3,8 @@ lint: pylint -j 6 --rcfile=./.pylintrc ./tests isort: - isort -rc ./platformio - isort -rc ./tests + isort ./platformio + isort ./tests format: black --target-version py27 ./platformio diff --git a/platformio/platform/factory.py b/platformio/platform/factory.py index 0f2bd15f..1aff6709 100644 --- a/platformio/platform/factory.py +++ b/platformio/platform/factory.py @@ -37,6 +37,8 @@ class PlatformFactory(object): @classmethod def new(cls, pkg_or_spec): + # pylint: disable=import-outside-toplevel + platform_dir = None platform_name = None if isinstance(pkg_or_spec, PackageItem): @@ -45,9 +47,7 @@ class PlatformFactory(object): elif os.path.isdir(pkg_or_spec): platform_dir = pkg_or_spec else: - from platformio.package.manager.platform import ( # pylint: disable=import-outside-toplevel - PlatformPackageManager, - ) + from platformio.package.manager.platform import PlatformPackageManager pkg = PlatformPackageManager().get_package(pkg_or_spec) if not pkg: diff --git a/tox.ini b/tox.ini index bb41a67b..ecaab77d 100644 --- a/tox.ini +++ b/tox.ini @@ -20,7 +20,7 @@ passenv = * usedevelop = True deps = py36,py37,py38,py39: black - isort<5 + isort pylint pytest pytest-xdist From 65e67b64bd8d2e06e7fd9b922bb815789f46dd74 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 22 Jan 2021 22:55:45 +0200 Subject: [PATCH 25/50] Remove unnecessary dependencies from contrib-pysite --- platformio/package/manager/core.py | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/platformio/package/manager/core.py b/platformio/package/manager/core.py index f31a8767..af51f91a 100644 --- a/platformio/package/manager/core.py +++ b/platformio/package/manager/core.py @@ -162,7 +162,6 @@ def build_contrib_pysite_package(target_dir, with_metadata=True): pkg.dump_meta() # remove unused files - shutil.rmtree(os.path.join(target_dir, "autobahn", "xbr", "contracts")) for root, dirs, files in os.walk(target_dir): for t in ("_test", "test", "tests"): if t in dirs: @@ -171,19 +170,6 @@ def build_contrib_pysite_package(target_dir, with_metadata=True): if name.endswith((".chm", ".pyc")): os.remove(os.path.join(root, name)) - # apply patches - with open( - os.path.join(target_dir, "autobahn", "twisted", "__init__.py"), "r+" - ) as fp: - contents = fp.read() - contents = contents.replace( - "from autobahn.twisted.wamp import ApplicationSession", - "# from autobahn.twisted.wamp import ApplicationSession", - ) - fp.seek(0) - fp.truncate() - fp.write(contents) - return target_dir @@ -194,8 +180,6 @@ def get_contrib_pysite_deps(): twisted_version = "19.10.0" if PY2 else "20.3.0" result = [ "twisted == %s" % twisted_version, - "autobahn == %s" % ("19.11.2" if PY2 else "20.7.1"), - "json-rpc == 1.13.0", ] # twisted[tls], see setup.py for %twisted_version% @@ -203,14 +187,6 @@ def get_contrib_pysite_deps(): ["pyopenssl >= 16.0.0", "service_identity >= 18.1.0", "idna >= 0.6, != 2.3"] ) - # zeroconf - if PY2: - result.append( - "https://github.com/ivankravets/python-zeroconf/" "archive/pio-py27.zip" - ) - else: - result.append("zeroconf == 0.26.0") - if "windows" in sys_type: result.append("pypiwin32 == 223") # workaround for twisted wheels From e695e30a9b313fdcd0036207cf4e46012a1dcd00 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 23 Jan 2021 14:44:53 +0200 Subject: [PATCH 26/50] Fixed an issue with compiler driver for ".ccls" language server // Resolve #3808 --- HISTORY.rst | 3 ++- platformio/ide/tpls/emacs/.ccls.tpl | 2 +- platformio/ide/tpls/sublimetext/.ccls.tpl | 2 +- platformio/ide/tpls/vim/.ccls.tpl | 2 +- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index e9b4990f..8499b973 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -18,12 +18,13 @@ PlatformIO Core 5 * Fixed a "UnicodeDecodeError: 'utf-8' codec can't decode byte" when using J-Link for firmware uploading on Linux (`issue #3804 `_) * Fixed an issue with Python 3.8+ on Windows when a network drive is used (`issue #3417 `_) * Fixed an issue when "strict" compatibility mode was not used for a library with custom "platforms" field in `library.json `__ manifest (`issue #3806 `_) +* Fixed an issue with compiler driver for ".ccls" language server (`issue #3808 `_) 5.0.4 (2020-12-30) ~~~~~~~~~~~~~~~~~~ - Added "Core" suffix when showing PlatformIO Core version using ``pio --version`` command -- Improved ``.ccls`` configuration file for Emacs, Vim, and Sublime Text integrations +- Improved ".ccls" configuration file for Emacs, Vim, and Sublime Text integrations - Updated analysis tools: * `Cppcheck `__ v2.3 with improved C++ parser and several new MISRA rules diff --git a/platformio/ide/tpls/emacs/.ccls.tpl b/platformio/ide/tpls/emacs/.ccls.tpl index 53c4afeb..a747bb61 100644 --- a/platformio/ide/tpls/emacs/.ccls.tpl +++ b/platformio/ide/tpls/emacs/.ccls.tpl @@ -1,4 +1,4 @@ -{{ cxx_path }} +clang {{"%c"}} {{ !cc_flags }} {{"%cpp"}} {{ !cxx_flags }} diff --git a/platformio/ide/tpls/sublimetext/.ccls.tpl b/platformio/ide/tpls/sublimetext/.ccls.tpl index 53c4afeb..a747bb61 100644 --- a/platformio/ide/tpls/sublimetext/.ccls.tpl +++ b/platformio/ide/tpls/sublimetext/.ccls.tpl @@ -1,4 +1,4 @@ -{{ cxx_path }} +clang {{"%c"}} {{ !cc_flags }} {{"%cpp"}} {{ !cxx_flags }} diff --git a/platformio/ide/tpls/vim/.ccls.tpl b/platformio/ide/tpls/vim/.ccls.tpl index 53c4afeb..a747bb61 100644 --- a/platformio/ide/tpls/vim/.ccls.tpl +++ b/platformio/ide/tpls/vim/.ccls.tpl @@ -1,4 +1,4 @@ -{{ cxx_path }} +clang {{"%c"}} {{ !cc_flags }} {{"%cpp"}} {{ !cxx_flags }} From ef6e70a38b03fab62cb9f299bf0e747783387174 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 23 Jan 2021 15:24:32 +0200 Subject: [PATCH 27/50] Fixed an issue when unnecessary packages were removed in ``update --dry-run`` mode // Resolve #3809 --- HISTORY.rst | 1 + platformio/package/manager/platform.py | 37 ++++++++++++++------------ 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 8499b973..826419bd 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -19,6 +19,7 @@ PlatformIO Core 5 * Fixed an issue with Python 3.8+ on Windows when a network drive is used (`issue #3417 `_) * Fixed an issue when "strict" compatibility mode was not used for a library with custom "platforms" field in `library.json `__ manifest (`issue #3806 `_) * Fixed an issue with compiler driver for ".ccls" language server (`issue #3808 `_) +* Fixed an issue when unnecessary packages were removed in ``update --dry-run`` mode (`issue #3809 `_) 5.0.4 (2020-12-30) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/package/manager/platform.py b/platformio/package/manager/platform.py index 71e8c5fb..7626d4d9 100644 --- a/platformio/package/manager/platform.py +++ b/platformio/package/manager/platform.py @@ -69,7 +69,7 @@ class PlatformPackageManager(BasePackageManager): # pylint: disable=too-many-an ) p.install_python_packages() p.on_installed() - self.cleanup_packages(list(p.packages)) + self.autoremove_packages(list(p.packages)) return pkg def uninstall(self, spec, silent=False, skip_dependencies=False): @@ -83,7 +83,7 @@ class PlatformPackageManager(BasePackageManager): # pylint: disable=too-many-an if not skip_dependencies: p.uninstall_python_packages() p.on_uninstalled() - self.cleanup_packages(list(p.packages)) + self.autoremove_packages(list(p.packages)) return pkg def update( # pylint: disable=arguments-differ, too-many-arguments @@ -118,7 +118,8 @@ class PlatformPackageManager(BasePackageManager): # pylint: disable=too-many-an ) p.update_packages(only_check) - self.cleanup_packages(list(p.packages)) + if not only_check: + self.autoremove_packages(list(p.packages)) if missed_pkgs: p.install_packages( @@ -127,28 +128,30 @@ class PlatformPackageManager(BasePackageManager): # pylint: disable=too-many-an return new_pkg or pkg - def cleanup_packages(self, names): + def autoremove_packages(self, names): self.memcache_reset() - deppkgs = {} + required = {} for platform in PlatformPackageManager().get_installed(): p = PlatformFactory.new(platform) for pkg in p.get_installed_packages(): - if pkg.metadata.name not in deppkgs: - deppkgs[pkg.metadata.name] = set() - deppkgs[pkg.metadata.name].add(pkg.metadata.version) + if pkg.metadata.name not in required: + required[pkg.metadata.name] = set() + required[pkg.metadata.name].add(pkg.metadata.version) pm = ToolPackageManager() for pkg in pm.get_installed(): - if pkg.metadata.name not in names: + skip_conds = [ + pkg.metadata.name not in names, + pkg.metadata.spec.url, + pkg.metadata.name in required + and pkg.metadata.version in required[pkg.metadata.name], + ] + if any(skip_conds): continue - if ( - pkg.metadata.name not in deppkgs - or pkg.metadata.version not in deppkgs[pkg.metadata.name] - ): - try: - pm.uninstall(pkg.metadata.spec) - except UnknownPackageError: - pass + try: + pm.uninstall(pkg.metadata.spec) + except UnknownPackageError: + pass self.memcache_reset() return True From 484567f2421e8e99d6e68ac014afa6c8e78845ce Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 23 Jan 2021 15:54:52 +0200 Subject: [PATCH 28/50] Project's "lib_compat_mode" has higher priority than "library.json" --- platformio/builder/tools/piolib.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index 97e2637a..792d6fab 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -773,9 +773,14 @@ class PlatformIOLibBuilder(LibBuilderBase): @property def lib_compat_mode(self): - mode = self._manifest.get("build", {}).get( - "libCompatMode", + missing = object() + mode = self.env.GetProjectConfig().getraw( + "env:" + self.env["PIOENV"], "lib_compat_mode", missing ) + if mode == missing: + mode = self._manifest.get("build", {}).get( + "libCompatMode", + ) if not mode and self._manifest.get("platforms"): mode = "strict" # pylint: disable=no-member From 92655c30c18fdf75acc28650ad9d14c80c87e778 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 23 Jan 2021 22:34:48 +0200 Subject: [PATCH 29/50] Disabled automatic removal of unnecessary development platform packages // Resolve #3708 , Resolve #/3770 --- HISTORY.rst | 1 + platformio/package/manager/platform.py | 32 -------------------------- 2 files changed, 1 insertion(+), 32 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 826419bd..b60d4a2e 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -15,6 +15,7 @@ PlatformIO Core 5 * Improved listing of `multicast DNS services `_ * Check for debugging server's "ready_pattern" in "stderr" * Upgraded build engine to the SCons 4.1 (`release notes `_) +* Disabled automatic removal of unnecessary development platform packages (`issue #3708 `_, `issue #3770 `_) * Fixed a "UnicodeDecodeError: 'utf-8' codec can't decode byte" when using J-Link for firmware uploading on Linux (`issue #3804 `_) * Fixed an issue with Python 3.8+ on Windows when a network drive is used (`issue #3417 `_) * Fixed an issue when "strict" compatibility mode was not used for a library with custom "platforms" field in `library.json `__ manifest (`issue #3806 `_) diff --git a/platformio/package/manager/platform.py b/platformio/package/manager/platform.py index 7626d4d9..5c52630d 100644 --- a/platformio/package/manager/platform.py +++ b/platformio/package/manager/platform.py @@ -69,7 +69,6 @@ class PlatformPackageManager(BasePackageManager): # pylint: disable=too-many-an ) p.install_python_packages() p.on_installed() - self.autoremove_packages(list(p.packages)) return pkg def uninstall(self, spec, silent=False, skip_dependencies=False): @@ -83,7 +82,6 @@ class PlatformPackageManager(BasePackageManager): # pylint: disable=too-many-an if not skip_dependencies: p.uninstall_python_packages() p.on_uninstalled() - self.autoremove_packages(list(p.packages)) return pkg def update( # pylint: disable=arguments-differ, too-many-arguments @@ -118,8 +116,6 @@ class PlatformPackageManager(BasePackageManager): # pylint: disable=too-many-an ) p.update_packages(only_check) - if not only_check: - self.autoremove_packages(list(p.packages)) if missed_pkgs: p.install_packages( @@ -128,34 +124,6 @@ class PlatformPackageManager(BasePackageManager): # pylint: disable=too-many-an return new_pkg or pkg - def autoremove_packages(self, names): - self.memcache_reset() - required = {} - for platform in PlatformPackageManager().get_installed(): - p = PlatformFactory.new(platform) - for pkg in p.get_installed_packages(): - if pkg.metadata.name not in required: - required[pkg.metadata.name] = set() - required[pkg.metadata.name].add(pkg.metadata.version) - - pm = ToolPackageManager() - for pkg in pm.get_installed(): - skip_conds = [ - pkg.metadata.name not in names, - pkg.metadata.spec.url, - pkg.metadata.name in required - and pkg.metadata.version in required[pkg.metadata.name], - ] - if any(skip_conds): - continue - try: - pm.uninstall(pkg.metadata.spec) - except UnknownPackageError: - pass - - self.memcache_reset() - return True - @util.memoized(expire="5s") def get_installed_boards(self): boards = [] From 59b02120b648a41965974f27ac0b6c5d44dd11d3 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 23 Jan 2021 23:20:53 +0200 Subject: [PATCH 30/50] New options for system prune command: remove unnecessary core and development platform packages // Resolve #923 --- HISTORY.rst | 34 ++++++++--- docs | 2 +- platformio/commands/system/command.py | 60 +++++++++++++----- platformio/commands/system/prune.py | 84 ++++++++++++++++++++++++++ platformio/package/manager/core.py | 41 ++++++++++--- platformio/package/manager/platform.py | 37 ++++++++++++ platformio/package/meta.py | 7 ++- platformio/platform/_packages.py | 26 +++++--- 8 files changed, 247 insertions(+), 44 deletions(-) create mode 100644 platformio/commands/system/prune.py diff --git a/HISTORY.rst b/HISTORY.rst index b60d4a2e..c8949179 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -11,16 +11,30 @@ PlatformIO Core 5 5.0.5 (2021-??-??) ~~~~~~~~~~~~~~~~~~ -* Significantly speedup PlatformIO Home loading time by migrating to native Python 3 Asynchronous I/O -* Improved listing of `multicast DNS services `_ -* Check for debugging server's "ready_pattern" in "stderr" -* Upgraded build engine to the SCons 4.1 (`release notes `_) -* Disabled automatic removal of unnecessary development platform packages (`issue #3708 `_, `issue #3770 `_) -* Fixed a "UnicodeDecodeError: 'utf-8' codec can't decode byte" when using J-Link for firmware uploading on Linux (`issue #3804 `_) -* Fixed an issue with Python 3.8+ on Windows when a network drive is used (`issue #3417 `_) -* Fixed an issue when "strict" compatibility mode was not used for a library with custom "platforms" field in `library.json `__ manifest (`issue #3806 `_) -* Fixed an issue with compiler driver for ".ccls" language server (`issue #3808 `_) -* Fixed an issue when unnecessary packages were removed in ``update --dry-run`` mode (`issue #3809 `_) +* **Build System** + + - Upgraded build engine to the SCons 4.1 (`release notes `_) + - Fixed an issue with Python 3.8+ on Windows when a network drive is used (`issue #3417 `_) + - Fixed an issue when "strict" compatibility mode was not used for a library with custom "platforms" field in `library.json `__ manifest (`issue #3806 `_) + +* **Package Management System** + + - New options for `system prune `__ command: + + + ``--dry-run`` option to show data that will be removed + + ``--core-packages`` option to remove unnecessary core packages + + ``--platform-packages`` option to remove unnecessary development platform packages (`issue #923 `_) + + - Disabled automatic removal of unnecessary development platform packages (`issue #3708 `_, `issue #3770 `_) + - Fixed an issue when unnecessary packages were removed in ``update --dry-run`` mode (`issue #3809 `_) + +* **Miscellaneous** + + - Significantly speedup PlatformIO Home loading time by migrating to native Python 3 Asynchronous I/O + - Improved listing of `multicast DNS services `_ + - Check for debugging server's "ready_pattern" in "stderr" + - Fixed a "UnicodeDecodeError: 'utf-8' codec can't decode byte" when using J-Link for firmware uploading on Linux (`issue #3804 `_) + - Fixed an issue with a compiler driver for ".ccls" language server (`issue #3808 `_) 5.0.4 (2020-12-30) ~~~~~~~~~~~~~~~~~~ diff --git a/docs b/docs index 04a05d7f..23165e88 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 04a05d7f347764e24817e37de3fe26f701660645 +Subproject commit 23165e88d7ffa4a28ee8496d374df07b9e8d9910 diff --git a/platformio/commands/system/command.py b/platformio/commands/system/command.py index cb311205..d0c87d84 100644 --- a/platformio/commands/system/command.py +++ b/platformio/commands/system/command.py @@ -13,7 +13,6 @@ # limitations under the License. import json -import os import platform import subprocess import sys @@ -27,11 +26,15 @@ from platformio.commands.system.completion import ( install_completion_code, uninstall_completion_code, ) +from platformio.commands.system.prune import ( + prune_cached_data, + prune_core_packages, + prune_platform_packages, +) from platformio.package.manager.library import LibraryPackageManager from platformio.package.manager.platform import PlatformPackageManager from platformio.package.manager.tool import ToolPackageManager from platformio.project.config import ProjectConfig -from platformio.project.helpers import get_project_cache_dir @click.group("system", short_help="Miscellaneous system commands") @@ -99,22 +102,49 @@ def system_info(json_output): @cli.command("prune", short_help="Remove unused data") @click.option("--force", "-f", is_flag=True, help="Do not prompt for confirmation") -def system_prune(force): - click.secho("WARNING! This will remove:", fg="yellow") - click.echo(" - cached API requests") - click.echo(" - cached package downloads") - click.echo(" - temporary data") - if not force: - click.confirm("Do you want to continue?", abort=True) +@click.option( + "--dry-run", is_flag=True, help="Do not prune, only show data that will be removed" +) +@click.option("--cache", is_flag=True, help="Prune only cached data") +@click.option( + "--core-packages", is_flag=True, help="Prune only unnecessary core packages" +) +@click.option( + "--platform-packages", + is_flag=True, + help="Prune only unnecessary development platform packages", +) +def system_prune(force, dry_run, cache, core_packages, platform_packages): + if dry_run: + click.secho( + "Dry run mode (do not prune, only show data that will be removed)", + fg="yellow", + ) + click.echo() - reclaimed_total = 0 - cache_dir = get_project_cache_dir() - if os.path.isdir(cache_dir): - reclaimed_total += fs.calculate_folder_size(cache_dir) - fs.rmtree(cache_dir) + reclaimed_cache = 0 + reclaimed_core_packages = 0 + reclaimed_platform_packages = 0 + prune_all = not any([cache, core_packages, platform_packages]) + + if cache or prune_all: + reclaimed_cache = prune_cached_data(force, dry_run) + click.echo() + + if core_packages or prune_all: + reclaimed_core_packages = prune_core_packages(force, dry_run) + click.echo() + + if platform_packages or prune_all: + reclaimed_platform_packages = prune_platform_packages(force, dry_run) + click.echo() click.secho( - "Total reclaimed space: %s" % fs.humanize_file_size(reclaimed_total), fg="green" + "Total reclaimed space: %s" + % fs.humanize_file_size( + reclaimed_cache + reclaimed_core_packages + reclaimed_platform_packages + ), + fg="green", ) diff --git a/platformio/commands/system/prune.py b/platformio/commands/system/prune.py new file mode 100644 index 00000000..a6bfa1d6 --- /dev/null +++ b/platformio/commands/system/prune.py @@ -0,0 +1,84 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from operator import itemgetter + +import click +from tabulate import tabulate + +from platformio import fs +from platformio.package.manager.core import remove_unnecessary_core_packages +from platformio.package.manager.platform import remove_unnecessary_platform_packages +from platformio.project.helpers import get_project_cache_dir + + +def prune_cached_data(force, dry_run): + reclaimed_space = 0 + click.secho("Prune cached data:", bold=True) + click.echo(" - cached API requests") + click.echo(" - cached package downloads") + click.echo(" - temporary data") + cache_dir = get_project_cache_dir() + if os.path.isdir(cache_dir): + reclaimed_space += fs.calculate_folder_size(cache_dir) + if not dry_run: + if not force: + click.confirm("Do you want to continue?", abort=True) + fs.rmtree(cache_dir) + click.secho("Space on disk: %s" % fs.humanize_file_size(reclaimed_space)) + return reclaimed_space + + +def prune_core_packages(force, dry_run): + click.secho("Prune unnecessary core packages:", bold=True) + return _prune_packages(force, dry_run, remove_unnecessary_core_packages) + + +def prune_platform_packages(force, dry_run): + click.secho("Prune unnecessary development platform packages:", bold=True) + return _prune_packages(force, dry_run, remove_unnecessary_platform_packages) + + +def _prune_packages(force, dry_run, handler): + click.echo("Calculating...") + items = [ + ( + pkg, + fs.calculate_folder_size(pkg.path), + ) + for pkg in handler(dry_run=True) + ] + items = sorted(items, key=itemgetter(1), reverse=True) + reclaimed_space = sum([item[1] for item in items]) + if items: + click.echo( + tabulate( + [ + ( + pkg.metadata.spec.humanize(), + str(pkg.metadata.version), + fs.humanize_file_size(size), + ) + for (pkg, size) in items + ], + headers=["Package", "Version", "Size"], + ) + ) + if not dry_run: + if not force: + click.confirm("Do you want to continue?", abort=True) + handler(dry_run=False) + click.secho("Space on disk: %s" % fs.humanize_file_size(reclaimed_space)) + return reclaimed_space diff --git a/platformio/package/manager/core.py b/platformio/package/manager/core.py index af51f91a..24d494b3 100644 --- a/platformio/package/manager/core.py +++ b/platformio/package/manager/core.py @@ -27,6 +27,17 @@ from platformio.package.meta import PackageItem, PackageSpec from platformio.proc import get_pythonexe_path +def get_installed_core_packages(): + result = [] + pm = ToolPackageManager() + for name, requirements in __core_packages__.items(): + spec = PackageSpec(owner="platformio", name=name, requirements=requirements) + pkg = pm.get_package(spec) + if pkg: + result.append(pkg) + return result + + def get_core_package_dir(name, auto_install=True): if name not in __core_packages__: raise exception.PlatformioException("Please upgrade PlatformIO Core") @@ -40,7 +51,7 @@ def get_core_package_dir(name, auto_install=True): if not auto_install: return None assert pm.install(spec) - _remove_unnecessary_packages() + remove_unnecessary_core_packages() return pm.get_package(spec).path @@ -54,24 +65,40 @@ def update_core_packages(only_check=False, silent=False): if not silent or pm.outdated(pkg, spec).is_outdated(): pm.update(pkg, spec, only_check=only_check) if not only_check: - _remove_unnecessary_packages() + remove_unnecessary_core_packages() return True -def _remove_unnecessary_packages(): +def remove_unnecessary_core_packages(dry_run=False): + candidates = [] pm = ToolPackageManager() best_pkg_versions = {} + for name, requirements in __core_packages__.items(): spec = PackageSpec(owner="platformio", name=name, requirements=requirements) pkg = pm.get_package(spec) if not pkg: continue best_pkg_versions[pkg.metadata.name] = pkg.metadata.version + for pkg in pm.get_installed(): - if pkg.metadata.name not in best_pkg_versions: - continue - if pkg.metadata.version != best_pkg_versions[pkg.metadata.name]: - pm.uninstall(pkg) + skip_conds = [ + os.path.isfile(os.path.join(pkg.path, ".piokeep")), + pkg.metadata.spec.owner != "platformio", + pkg.metadata.name not in best_pkg_versions, + pkg.metadata.name in best_pkg_versions + and pkg.metadata.version == best_pkg_versions[pkg.metadata.name], + ] + if not any(skip_conds): + candidates.append(pkg) + + if dry_run: + return candidates + + for pkg in candidates: + pm.uninstall(pkg) + + return candidates def inject_contrib_pysite(verify_openssl=False): diff --git a/platformio/package/manager/platform.py b/platformio/package/manager/platform.py index 5c52630d..efe8a361 100644 --- a/platformio/package/manager/platform.py +++ b/platformio/package/manager/platform.py @@ -12,10 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os + from platformio import util from platformio.clients.http import HTTPClientError, InternetIsOffline from platformio.package.exception import UnknownPackageError from platformio.package.manager.base import BasePackageManager +from platformio.package.manager.core import get_installed_core_packages from platformio.package.manager.tool import ToolPackageManager from platformio.package.meta import PackageType from platformio.platform.exception import IncompatiblePlatform, UnknownBoard @@ -164,3 +167,37 @@ class PlatformPackageManager(BasePackageManager): # pylint: disable=too-many-an ): return manifest raise UnknownBoard(id_) + + +# +# Helpers +# + + +def remove_unnecessary_platform_packages(dry_run=False): + candidates = [] + required = set() + core_packages = get_installed_core_packages() + for platform in PlatformPackageManager().get_installed(): + p = PlatformFactory.new(platform) + for pkg in p.get_installed_packages(with_optional=True): + required.add(pkg) + + pm = ToolPackageManager() + for pkg in pm.get_installed(): + skip_conds = [ + pkg.metadata.spec.url, + os.path.isfile(os.path.join(pkg.path, ".piokeep")), + pkg in required, + pkg in core_packages, + ] + if not any(skip_conds): + candidates.append(pkg) + + if dry_run: + return candidates + + for pkg in candidates: + pm.uninstall(pkg) + + return candidates diff --git a/platformio/package/meta.py b/platformio/package/meta.py index edc5d0ff..156445d3 100644 --- a/platformio/package/meta.py +++ b/platformio/package/meta.py @@ -405,7 +405,12 @@ class PackageItem(object): ) def __eq__(self, other): - return all([self.path == other.path, self.metadata == other.metadata]) + if not self.path or not other.path: + return self.path == other.path + return os.path.realpath(self.path) == os.path.realpath(other.path) + + def __hash__(self): + return hash(os.path.realpath(self.path)) def exists(self): return os.path.isdir(self.path) diff --git a/platformio/platform/_packages.py b/platformio/platform/_packages.py index ac495b48..08c40c9b 100644 --- a/platformio/platform/_packages.py +++ b/platformio/platform/_packages.py @@ -17,18 +17,18 @@ from platformio.package.meta import PackageSpec class PlatformPackagesMixin(object): - def get_package_spec(self, name): - version = self.packages[name].get("version", "") - if any(c in version for c in (":", "/", "@")): + def get_package_spec(self, name, version=None): + version = version or self.packages[name].get("version") + if version and any(c in version for c in (":", "/", "@")): return PackageSpec("%s=%s" % (name, version)) return PackageSpec( owner=self.packages[name].get("owner"), name=name, requirements=version ) - def get_package(self, name): + def get_package(self, name, spec=None): if not name: return None - return self.pm.get_package(self.get_package_spec(name)) + return self.pm.get_package(spec or self.get_package_spec(name)) def get_package_dir(self, name): pkg = self.get_package(name) @@ -38,12 +38,18 @@ class PlatformPackagesMixin(object): pkg = self.get_package(name) return str(pkg.metadata.version) if pkg else None - def get_installed_packages(self): + def get_installed_packages(self, with_optional=False): result = [] - for name in self.packages: - pkg = self.get_package(name) - if pkg: - result.append(pkg) + for name, options in self.packages.items(): + versions = [options.get("version")] + if with_optional: + versions.extend(options.get("optionalVersions", [])) + for version in versions: + if not version: + continue + pkg = self.get_package(name, self.get_package_spec(name, version)) + if pkg: + result.append(pkg) return result def dump_used_packages(self): From bd4d3b914b8f22bf76fabcc49f2a2f8e6173a249 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sun, 24 Jan 2021 15:49:56 +0200 Subject: [PATCH 31/50] Revert "lib_compat_mode" changes // Resolve #3811 Resolve #3806 --- HISTORY.rst | 1 - platformio/builder/tools/piolib.py | 14 +++----------- 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index c8949179..0d914a36 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -15,7 +15,6 @@ PlatformIO Core 5 - Upgraded build engine to the SCons 4.1 (`release notes `_) - Fixed an issue with Python 3.8+ on Windows when a network drive is used (`issue #3417 `_) - - Fixed an issue when "strict" compatibility mode was not used for a library with custom "platforms" field in `library.json `__ manifest (`issue #3806 `_) * **Package Management System** diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index 792d6fab..f6b9824b 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -773,19 +773,11 @@ class PlatformIOLibBuilder(LibBuilderBase): @property def lib_compat_mode(self): - missing = object() - mode = self.env.GetProjectConfig().getraw( - "env:" + self.env["PIOENV"], "lib_compat_mode", missing - ) - if mode == missing: - mode = self._manifest.get("build", {}).get( - "libCompatMode", - ) - if not mode and self._manifest.get("platforms"): - mode = "strict" # pylint: disable=no-member return self.validate_compat_mode( - mode or LibBuilderBase.lib_compat_mode.fget(self) + self._manifest.get("build", {}).get( + "libCompatMode", LibBuilderBase.lib_compat_mode.fget(self) + ) ) def is_platforms_compatible(self, platforms): From 15ff8f9d2ab13f492bd165998cfc924f1a4913f0 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sun, 24 Jan 2021 15:58:07 +0200 Subject: [PATCH 32/50] Bump version to 5.0.5b5 --- docs | 2 +- platformio/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs b/docs index 23165e88..89ee78e1 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 23165e88d7ffa4a28ee8496d374df07b9e8d9910 +Subproject commit 89ee78e13f83b0f3347423e1494c6e139eb6b29b diff --git a/platformio/__init__.py b/platformio/__init__.py index 4666701e..26485ade 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (5, 0, "5b4") +VERSION = (5, 0, "5b5") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 2c389ae11e53071a31a1b3bc8d3f94e4cc7435c8 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sun, 24 Jan 2021 17:21:22 +0200 Subject: [PATCH 33/50] Added new `check_prune_system_threshold` setting --- HISTORY.rst | 1 + docs | 2 +- platformio/app.py | 4 +++ platformio/commands/system/prune.py | 46 +++++++++++++++++++---------- platformio/maintenance.py | 30 +++++++++++++++++++ 5 files changed, 66 insertions(+), 17 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 0d914a36..171ab8d5 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -24,6 +24,7 @@ PlatformIO Core 5 + ``--core-packages`` option to remove unnecessary core packages + ``--platform-packages`` option to remove unnecessary development platform packages (`issue #923 `_) + - Added new `check_prune_system_threshold `__ setting - Disabled automatic removal of unnecessary development platform packages (`issue #3708 `_, `issue #3770 `_) - Fixed an issue when unnecessary packages were removed in ``update --dry-run`` mode (`issue #3809 `_) diff --git a/docs b/docs index 89ee78e1..ee815b1b 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 89ee78e13f83b0f3347423e1494c6e139eb6b29b +Subproject commit ee815b1b4214d10f10635f9c9116e4017751184f diff --git a/platformio/app.py b/platformio/app.py index 9b22b638..04d02c39 100644 --- a/platformio/app.py +++ b/platformio/app.py @@ -55,6 +55,10 @@ DEFAULT_SETTINGS = { "description": "Check for the platform updates interval (days)", "value": 7, }, + "check_prune_system_threshold": { + "description": "Check for pruning unnecessary data threshold (megabytes)", + "value": 1024, + }, "enable_cache": { "description": "Enable caching for HTTP API requests", "value": True, diff --git a/platformio/commands/system/prune.py b/platformio/commands/system/prune.py index a6bfa1d6..e0ef2dd8 100644 --- a/platformio/commands/system/prune.py +++ b/platformio/commands/system/prune.py @@ -24,12 +24,13 @@ from platformio.package.manager.platform import remove_unnecessary_platform_pack from platformio.project.helpers import get_project_cache_dir -def prune_cached_data(force, dry_run): +def prune_cached_data(force=False, dry_run=False, silent=False): reclaimed_space = 0 - click.secho("Prune cached data:", bold=True) - click.echo(" - cached API requests") - click.echo(" - cached package downloads") - click.echo(" - temporary data") + if not silent: + click.secho("Prune cached data:", bold=True) + click.echo(" - cached API requests") + click.echo(" - cached package downloads") + click.echo(" - temporary data") cache_dir = get_project_cache_dir() if os.path.isdir(cache_dir): reclaimed_space += fs.calculate_folder_size(cache_dir) @@ -37,22 +38,26 @@ def prune_cached_data(force, dry_run): if not force: click.confirm("Do you want to continue?", abort=True) fs.rmtree(cache_dir) - click.secho("Space on disk: %s" % fs.humanize_file_size(reclaimed_space)) + if not silent: + click.secho("Space on disk: %s" % fs.humanize_file_size(reclaimed_space)) return reclaimed_space -def prune_core_packages(force, dry_run): - click.secho("Prune unnecessary core packages:", bold=True) - return _prune_packages(force, dry_run, remove_unnecessary_core_packages) +def prune_core_packages(force=False, dry_run=False, silent=False): + if not silent: + click.secho("Prune unnecessary core packages:", bold=True) + return _prune_packages(force, dry_run, silent, remove_unnecessary_core_packages) -def prune_platform_packages(force, dry_run): - click.secho("Prune unnecessary development platform packages:", bold=True) - return _prune_packages(force, dry_run, remove_unnecessary_platform_packages) +def prune_platform_packages(force=False, dry_run=False, silent=False): + if not silent: + click.secho("Prune unnecessary development platform packages:", bold=True) + return _prune_packages(force, dry_run, silent, remove_unnecessary_platform_packages) -def _prune_packages(force, dry_run, handler): - click.echo("Calculating...") +def _prune_packages(force, dry_run, silent, handler): + if not silent: + click.echo("Calculating...") items = [ ( pkg, @@ -62,7 +67,7 @@ def _prune_packages(force, dry_run, handler): ] items = sorted(items, key=itemgetter(1), reverse=True) reclaimed_space = sum([item[1] for item in items]) - if items: + if items and not silent: click.echo( tabulate( [ @@ -80,5 +85,14 @@ def _prune_packages(force, dry_run, handler): if not force: click.confirm("Do you want to continue?", abort=True) handler(dry_run=False) - click.secho("Space on disk: %s" % fs.humanize_file_size(reclaimed_space)) + if not silent: + click.secho("Space on disk: %s" % fs.humanize_file_size(reclaimed_space)) return reclaimed_space + + +def calculate_unnecessary_system_data(): + return ( + prune_cached_data(force=True, dry_run=True, silent=True) + + prune_core_packages(force=True, dry_run=True, silent=True) + + prune_platform_packages(force=True, dry_run=True, silent=True) + ) diff --git a/platformio/maintenance.py b/platformio/maintenance.py index 4c252df8..472a82a5 100644 --- a/platformio/maintenance.py +++ b/platformio/maintenance.py @@ -26,6 +26,7 @@ from platformio.commands import PlatformioCLI from platformio.commands.lib.command import CTX_META_STORAGE_DIRS_KEY from platformio.commands.lib.command import lib_update as cmd_lib_update from platformio.commands.platform import platform_update as cmd_platform_update +from platformio.commands.system.prune import calculate_unnecessary_system_data from platformio.commands.upgrade import get_latest_version from platformio.compat import ensure_python3 from platformio.package.manager.core import update_core_packages @@ -73,6 +74,7 @@ def on_platformio_end(ctx, result): # pylint: disable=unused-argument check_platformio_upgrade() check_internal_updates(ctx, "platforms") check_internal_updates(ctx, "libraries") + check_prune_system() except ( http.HTTPClientError, http.InternetIsOffline, @@ -347,3 +349,31 @@ def check_internal_updates(ctx, what): # pylint: disable=too-many-branches click.echo("*" * terminal_width) click.echo("") + + +def check_prune_system(): + last_check = app.get_state_item("last_check", {}) + interval = 30 * 3600 * 24 # 1 time per month + if (time() - interval) < last_check.get("prune_system", 0): + return + + last_check["prune_system"] = int(time()) + app.set_state_item("last_check", last_check) + threshold_mb = int(app.get_setting("check_prune_system_threshold") or 0) + if threshold_mb <= 0: + return + + unnecessary_mb = calculate_unnecessary_system_data() / 1024 + if unnecessary_mb < threshold_mb: + return + + terminal_width, _ = click.get_terminal_size() + click.echo() + click.echo("*" * terminal_width) + click.secho( + "We found %s of unnecessary PlatformIO system data (temporary files, " + "unnecessary packages, etc.).\nUse `pio system prune --dry-run` to list " + "them or `pio system prune` to save disk space." + % fs.humanize_file_size(unnecessary_mb), + fg="yellow", + ) From 0ed99b7687e494eee810b3c471c1f9ea1a18361a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 25 Jan 2021 23:44:26 +0200 Subject: [PATCH 34/50] Added a new ``--session-id`` option to `pio home` // Resolve #3397 --- HISTORY.rst | 6 +++++- docs | 2 +- platformio/__init__.py | 2 +- platformio/commands/home/command.py | 17 ++++++++++++++--- platformio/commands/home/run.py | 23 ++++++++++++++++++----- 5 files changed, 39 insertions(+), 11 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 171ab8d5..58b77160 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -28,9 +28,13 @@ PlatformIO Core 5 - Disabled automatic removal of unnecessary development platform packages (`issue #3708 `_, `issue #3770 `_) - Fixed an issue when unnecessary packages were removed in ``update --dry-run`` mode (`issue #3809 `_) -* **Miscellaneous** +* **PlatformIO Home** - Significantly speedup PlatformIO Home loading time by migrating to native Python 3 Asynchronous I/O + - Added a new ``--session-id`` option to `pio home `__ command that helps to keep PlatformIO Home isolated from other instances and protect from 3rd party access (`issue #3397 `_) + +* **Miscellaneous** + - Improved listing of `multicast DNS services `_ - Check for debugging server's "ready_pattern" in "stderr" - Fixed a "UnicodeDecodeError: 'utf-8' codec can't decode byte" when using J-Link for firmware uploading on Linux (`issue #3804 `_) diff --git a/docs b/docs index ee815b1b..24f57666 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit ee815b1b4214d10f10635f9c9116e4017751184f +Subproject commit 24f57666602e68e53904ec3600e5f011d55b3aba diff --git a/platformio/__init__.py b/platformio/__init__.py index 26485ade..7b9eaeb6 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -47,7 +47,7 @@ __pioremote_endpoint__ = "ssl:host=remote.platformio.org:port=4413" __default_requests_timeout__ = (10, None) # (connect, read) __core_packages__ = { - "contrib-piohome": "~3.3.1", + "contrib-piohome": "~3.3.2", "contrib-pysite": "~2.%d%d.0" % (sys.version_info.major, sys.version_info.minor), "tool-unity": "~1.20500.0", "tool-scons": "~2.20501.7" if sys.version_info.major == 2 else "~4.40100.0", diff --git a/platformio/commands/home/command.py b/platformio/commands/home/command.py index 28cbfef4..2973bdd2 100644 --- a/platformio/commands/home/command.py +++ b/platformio/commands/home/command.py @@ -40,7 +40,14 @@ from platformio.compat import ensure_python3 "are connected. Default is 0 which means never auto shutdown" ), ) -def cli(port, host, no_open, shutdown_timeout): +@click.option( + "--session-id", + help=( + "A unique session identifier to keep PIO Home isolated from other instances " + "and protect from 3rd party access" + ), +) +def cli(port, host, no_open, shutdown_timeout, session_id): ensure_python3() # Ensure PIO Home mimetypes are known @@ -52,7 +59,11 @@ def cli(port, host, no_open, shutdown_timeout): if host == "__do_not_start__": return - home_url = "http://%s:%d" % (host, port) + home_url = "http://%s:%d%s" % ( + host, + port, + ("/session/%s/" % session_id) if session_id else "/", + ) click.echo( "\n".join( [ @@ -61,7 +72,7 @@ def cli(port, host, no_open, shutdown_timeout): " /\\-_--\\ PlatformIO Home", "/ \\_-__\\", "|[]| [] | %s" % home_url, - "|__|____|______________%s" % ("_" * len(host)), + "|__|____|__%s" % ("_" * len(home_url)), ] ) ) diff --git a/platformio/commands/home/run.py b/platformio/commands/home/run.py index 585c9268..6e93cc2b 100644 --- a/platformio/commands/home/run.py +++ b/platformio/commands/home/run.py @@ -13,6 +13,7 @@ # limitations under the License. import os +from urllib.parse import urlparse import click import uvicorn @@ -21,6 +22,7 @@ from starlette.middleware import Middleware from starlette.responses import PlainTextResponse from starlette.routing import Mount, Route, WebSocketRoute from starlette.staticfiles import StaticFiles +from starlette.status import HTTP_403_FORBIDDEN from platformio.commands.home.rpc.handlers.account import AccountRPC from platformio.commands.home.rpc.handlers.app import AppRPC @@ -51,6 +53,12 @@ async def shutdown_server(_=None): return PlainTextResponse("Server has been shutdown!") +async def protected_page(_): + return PlainTextResponse( + "Protected PlatformIO Home session", status_code=HTTP_403_FORBIDDEN + ) + + def run_server(host, port, no_open, shutdown_timeout, home_url): contrib_dir = get_core_package_dir("contrib-piohome") if not os.path.isdir(contrib_dir): @@ -65,14 +73,19 @@ def run_server(host, port, no_open, shutdown_timeout, home_url): ws_rpc_factory.addHandler(PIOCoreRPC(), namespace="core") ws_rpc_factory.addHandler(ProjectRPC(), namespace="project") + path = urlparse(home_url).path + routes = [ + WebSocketRoute(path + "wsrpc", ws_rpc_factory, name="wsrpc"), + Route(path + "__shutdown__", shutdown_server, methods=["POST"]), + Mount(path, StaticFiles(directory=contrib_dir, html=True), name="static"), + ] + if path != "/": + routes.append(Route("/", protected_page)) + uvicorn.run( Starlette( middleware=[Middleware(ShutdownMiddleware)], - routes=[ - WebSocketRoute("/wsrpc", ws_rpc_factory, name="wsrpc"), - Route("/__shutdown__", shutdown_server, methods=["POST"]), - Mount("/", StaticFiles(directory=contrib_dir, html=True)), - ], + routes=routes, on_startup=[ lambda: click.echo( "PIO Home has been started. Press Ctrl+C to shutdown." From dd4fff3a79152894be59590bb6c6dec1a5a561c7 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 25 Jan 2021 23:50:41 +0200 Subject: [PATCH 35/50] Bump version to 5.1.0rc1 --- HISTORY.rst | 2 +- platformio/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 58b77160..e437f689 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -8,7 +8,7 @@ PlatformIO Core 5 **A professional collaborative platform for embedded development** -5.0.5 (2021-??-??) +5.1.0 (2021-??-??) ~~~~~~~~~~~~~~~~~~ * **Build System** diff --git a/platformio/__init__.py b/platformio/__init__.py index 7b9eaeb6..f049e6fd 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (5, 0, "5b5") +VERSION = (5, 1, "0rc1") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 4012a86cac806d40bfd1238bdc941df3deb4347a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 26 Jan 2021 16:15:11 +0200 Subject: [PATCH 36/50] Fixed a "ValueError: Invalid simple block" when uninstalling a package with a custom name and external source // Resolve #3816 --- HISTORY.rst | 1 + platformio/package/manager/library.py | 24 ++++++++++-------------- platformio/package/meta.py | 11 ++++++++--- platformio/platform/_packages.py | 7 +++---- tests/package/test_meta.py | 9 +++++++++ 5 files changed, 31 insertions(+), 21 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index e437f689..5bccb302 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -27,6 +27,7 @@ PlatformIO Core 5 - Added new `check_prune_system_threshold `__ setting - Disabled automatic removal of unnecessary development platform packages (`issue #3708 `_, `issue #3770 `_) - Fixed an issue when unnecessary packages were removed in ``update --dry-run`` mode (`issue #3809 `_) + - Fixed a "ValueError: Invalid simple block" when uninstalling a package with a custom name and external source (`issue #3816 `_) * **PlatformIO Home** diff --git a/platformio/package/manager/library.py b/platformio/package/manager/library.py index 9f8bd28a..3f77846b 100644 --- a/platformio/package/manager/library.py +++ b/platformio/package/manager/library.py @@ -112,16 +112,11 @@ class LibraryPackageManager(BasePackageManager): # pylint: disable=too-many-anc ) def _install_dependency(self, dependency, silent=False): - if set(["name", "version"]) <= set(dependency.keys()) and any( - c in dependency["version"] for c in (":", "/", "@") - ): - spec = PackageSpec("%s=%s" % (dependency["name"], dependency["version"])) - else: - spec = PackageSpec( - owner=dependency.get("owner"), - name=dependency.get("name"), - requirements=dependency.get("version"), - ) + spec = PackageSpec( + owner=dependency.get("owner"), + name=dependency.get("name"), + requirements=dependency.get("version"), + ) search_filters = { key: value for key, value in dependency.items() @@ -143,11 +138,12 @@ class LibraryPackageManager(BasePackageManager): # pylint: disable=too-many-anc if not silent: self.print_message("Removing dependencies...", fg="yellow") for dependency in manifest.get("dependencies"): - pkg = self.get_package( - PackageSpec( - name=dependency.get("name"), requirements=dependency.get("version") - ) + spec = PackageSpec( + owner=dependency.get("owner"), + name=dependency.get("name"), + requirements=dependency.get("version"), ) + pkg = self.get_package(spec) if not pkg: continue self._uninstall(pkg, silent=silent) diff --git a/platformio/package/meta.py b/platformio/package/meta.py index 156445d3..74af1916 100644 --- a/platformio/package/meta.py +++ b/platformio/package/meta.py @@ -107,16 +107,21 @@ class PackageSpec(object): # pylint: disable=too-many-instance-attributes def __init__( # pylint: disable=redefined-builtin,too-many-arguments self, raw=None, owner=None, id=None, name=None, requirements=None, url=None ): + self._requirements = None self.owner = owner self.id = id self.name = name - self._requirements = None self.url = url self.raw = raw if requirements: - self.requirements = requirements + try: + self.requirements = requirements + except ValueError as exc: + if not self.name or self.url or self.raw: + raise exc + self.raw = "%s=%s" % (self.name, requirements) self._name_is_custom = False - self._parse(raw) + self._parse(self.raw) def __eq__(self, other): return all( diff --git a/platformio/platform/_packages.py b/platformio/platform/_packages.py index 08c40c9b..786f1efc 100644 --- a/platformio/platform/_packages.py +++ b/platformio/platform/_packages.py @@ -18,11 +18,10 @@ from platformio.package.meta import PackageSpec class PlatformPackagesMixin(object): def get_package_spec(self, name, version=None): - version = version or self.packages[name].get("version") - if version and any(c in version for c in (":", "/", "@")): - return PackageSpec("%s=%s" % (name, version)) return PackageSpec( - owner=self.packages[name].get("owner"), name=name, requirements=version + owner=self.packages[name].get("owner"), + name=name, + requirements=version or self.packages[name].get("version"), ) def get_package(self, name, spec=None): diff --git a/tests/package/test_meta.py b/tests/package/test_meta.py index 32b3d56a..1cda6409 100644 --- a/tests/package/test_meta.py +++ b/tests/package/test_meta.py @@ -169,6 +169,15 @@ def test_spec_vcs_urls(): url="git+git@github.com:platformio/platformio-core.git", requirements="^1.2.3,!=5", ) + assert PackageSpec( + owner="platformio", + name="external-repo", + requirements="https://github.com/platformio/platformio-core", + ) == PackageSpec( + owner="platformio", + name="external-repo", + url="git+https://github.com/platformio/platformio-core", + ) def test_spec_as_dict(): From 8ff270c5f714121f80421b0f885c9242bb730493 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 26 Jan 2021 17:05:37 +0200 Subject: [PATCH 37/50] Skip non-existing package when checking for update// Resolve #3818 --- platformio/package/manager/_update.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/platformio/package/manager/_update.py b/platformio/package/manager/_update.py index 1487d0bf..c81e7186 100644 --- a/platformio/package/manager/_update.py +++ b/platformio/package/manager/_update.py @@ -26,7 +26,10 @@ class PackageManagerUpdateMixin(object): def outdated(self, pkg, spec=None): assert isinstance(pkg, PackageItem) assert not spec or isinstance(spec, PackageSpec) - assert os.path.isdir(pkg.path) and pkg.metadata + assert pkg.metadata + + if not os.path.isdir(pkg.path): + return PackageOutdatedResult(current=pkg.metadata.version) # skip detached package to a specific version detached_conditions = [ From 0a8b66ee95e0dc5f7a7e72eafb7d2aee05dd626c Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 26 Jan 2021 21:21:41 +0200 Subject: [PATCH 38/50] Configure a custom debug adapter speed using a new `debug_speed` option // Resolve #3799 --- HISTORY.rst | 6 +++++- docs | 2 +- platformio/__init__.py | 2 +- platformio/project/options.py | 5 +++++ 4 files changed, 12 insertions(+), 3 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 5bccb302..9b5218b9 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -34,10 +34,14 @@ PlatformIO Core 5 - Significantly speedup PlatformIO Home loading time by migrating to native Python 3 Asynchronous I/O - Added a new ``--session-id`` option to `pio home `__ command that helps to keep PlatformIO Home isolated from other instances and protect from 3rd party access (`issue #3397 `_) +* **Debugging** + + - Configure a custom debug adapter speed using a new `debug_speed `__ option (`issue #3799 `_) + - Handle debugging server's "ready_pattern" in "stderr" output + * **Miscellaneous** - Improved listing of `multicast DNS services `_ - - Check for debugging server's "ready_pattern" in "stderr" - Fixed a "UnicodeDecodeError: 'utf-8' codec can't decode byte" when using J-Link for firmware uploading on Linux (`issue #3804 `_) - Fixed an issue with a compiler driver for ".ccls" language server (`issue #3808 `_) diff --git a/docs b/docs index 24f57666..d50d1e2c 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 24f57666602e68e53904ec3600e5f011d55b3aba +Subproject commit d50d1e2c75a44aac26dda85f58787a5210125927 diff --git a/platformio/__init__.py b/platformio/__init__.py index f049e6fd..cebcd1ed 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -47,7 +47,7 @@ __pioremote_endpoint__ = "ssl:host=remote.platformio.org:port=4413" __default_requests_timeout__ = (10, None) # (connect, read) __core_packages__ = { - "contrib-piohome": "~3.3.2", + "contrib-piohome": "~3.3.3", "contrib-pysite": "~2.%d%d.0" % (sys.version_info.major, sys.version_info.minor), "tool-unity": "~1.20500.0", "tool-scons": "~2.20501.7" if sys.version_info.major == 2 else "~4.40100.0", diff --git a/platformio/project/options.py b/platformio/project/options.py index b5eaf337..213a2104 100644 --- a/platformio/project/options.py +++ b/platformio/project/options.py @@ -681,6 +681,11 @@ ProjectOptions = OrderedDict( "network address)" ), ), + ConfigEnvOption( + group="debug", + name="debug_speed", + description="A debug adapter speed (JTAG speed)", + ), ConfigEnvOption( group="debug", name="debug_svd_path", From e2906e3be5700c920797553dfd911ffc36a74376 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 27 Jan 2021 16:10:13 +0200 Subject: [PATCH 39/50] Refactored a workaround for a maximum command line character limitation // Resolve #3792 --- HISTORY.rst | 1 + platformio/builder/main.py | 17 +++++-- platformio/builder/tools/piomaxlen.py | 66 +++++++++++---------------- 3 files changed, 40 insertions(+), 44 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 9b5218b9..f2a5bc61 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -14,6 +14,7 @@ PlatformIO Core 5 * **Build System** - Upgraded build engine to the SCons 4.1 (`release notes `_) + - Refactored a workaround for a maximum command line character limitation (`issue #3792 `_) - Fixed an issue with Python 3.8+ on Windows when a network drive is used (`issue #3417 `_) * **Package Management System** diff --git a/platformio/builder/main.py b/platformio/builder/main.py index b1e8ffbd..6a060dd1 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -81,12 +81,19 @@ DEFAULT_ENV_OPTIONS = dict( IDE_EXTRA_DATA={}, ) +# Declare command verbose messages +command_strings = dict( + ARCOM="Archiving", + LINKCOM="Linking", + RANLIBCOM="Indexing", + ASCOM="Compiling", + ASPPCOM="Compiling", + CCCOM="Compiling", + CXXCOM="Compiling", +) if not int(ARGUMENTS.get("PIOVERBOSE", 0)): - DEFAULT_ENV_OPTIONS["ARCOMSTR"] = "Archiving $TARGET" - DEFAULT_ENV_OPTIONS["LINKCOMSTR"] = "Linking $TARGET" - DEFAULT_ENV_OPTIONS["RANLIBCOMSTR"] = "Indexing $TARGET" - for k in ("ASCOMSTR", "ASPPCOMSTR", "CCCOMSTR", "CXXCOMSTR"): - DEFAULT_ENV_OPTIONS[k] = "Compiling $TARGET" + for name, value in command_strings.items(): + DEFAULT_ENV_OPTIONS["%sSTR" % name] = "%s $TARGET" % (value) env = DefaultEnvironment(**DEFAULT_ENV_OPTIONS) diff --git a/platformio/builder/tools/piomaxlen.py b/platformio/builder/tools/piomaxlen.py index 04f1304f..4077273c 100644 --- a/platformio/builder/tools/piomaxlen.py +++ b/platformio/builder/tools/piomaxlen.py @@ -14,15 +14,18 @@ from __future__ import absolute_import -from hashlib import md5 -from os import makedirs -from os.path import isdir, isfile, join +import hashlib +import os + +from SCons.Platform import TempFileMunge # pylint: disable=import-error from platformio.compat import WINDOWS, hashlib_encode_data -# Windows CLI has limit with command length to 8192 -# Leave 2000 chars for flags and other options -MAX_LINE_LENGTH = 6000 if WINDOWS else 128072 +# There are the next limits depending on a platform: +# - Windows = 8192 +# - Unix = 131072 +# We need ~256 characters for a temporary file path +MAX_LINE_LENGTH = (8192 if WINDOWS else 131072) - 256 def long_sources_hook(env, sources): @@ -41,30 +44,14 @@ def long_sources_hook(env, sources): return '@"%s"' % _file_long_data(env, " ".join(data)) -def long_incflags_hook(env, incflags): - _incflags = env.subst(incflags).replace("\\", "/") - if len(_incflags) < MAX_LINE_LENGTH: - return incflags - - # fix space in paths - data = [] - for line in _incflags.split(" -I"): - line = line.strip() - if not line.startswith("-I"): - line = "-I" + line - data.append('-I"%s"' % line[2:]) - - return '@"%s"' % _file_long_data(env, " ".join(data)) - - def _file_long_data(env, data): build_dir = env.subst("$BUILD_DIR") - if not isdir(build_dir): - makedirs(build_dir) - tmp_file = join( - build_dir, "longcmd-%s" % md5(hashlib_encode_data(data)).hexdigest() + if not os.path.isdir(build_dir): + os.makedirs(build_dir) + tmp_file = os.path.join( + build_dir, "longcmd-%s" % hashlib.md5(hashlib_encode_data(data)).hexdigest() ) - if isfile(tmp_file): + if os.path.isfile(tmp_file): return tmp_file with open(tmp_file, "w") as fp: fp.write(data) @@ -76,17 +63,18 @@ def exists(_): def generate(env): - env.Replace(_long_sources_hook=long_sources_hook) - env.Replace(_long_incflags_hook=long_incflags_hook) - coms = {} - for key in ("ARCOM", "LINKCOM"): - coms[key] = env.get(key, "").replace( - "$SOURCES", "${_long_sources_hook(__env__, SOURCES)}" - ) - for key in ("_CCCOMCOM", "ASPPCOM"): - coms[key] = env.get(key, "").replace( - "$_CPPINCFLAGS", "${_long_incflags_hook(__env__, _CPPINCFLAGS)}" - ) - env.Replace(**coms) + kwargs = dict( + _long_sources_hook=long_sources_hook, + TEMPFILE=TempFileMunge, + MAXLINELENGTH=MAX_LINE_LENGTH, + ) + + for name in ("LINKCOM", "ASCOM", "ASPPCOM", "CCCOM", "CXXCOM"): + kwargs[name] = "${TEMPFILE('%s','$%sSTR')}" % (env.get(name), name) + + kwargs["ARCOM"] = env.get("ARCOM", "").replace( + "$SOURCES", "${_long_sources_hook(__env__, SOURCES)}" + ) + env.Replace(**kwargs) return env From 7810946484330cb62dee02bcde86af6672cdfd09 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 27 Jan 2021 18:47:54 +0200 Subject: [PATCH 40/50] Use project build folder for tempfile workaround with command maxlen --- platformio/__init__.py | 2 +- platformio/builder/tools/piomaxlen.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index cebcd1ed..b3ff0e42 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -50,7 +50,7 @@ __core_packages__ = { "contrib-piohome": "~3.3.3", "contrib-pysite": "~2.%d%d.0" % (sys.version_info.major, sys.version_info.minor), "tool-unity": "~1.20500.0", - "tool-scons": "~2.20501.7" if sys.version_info.major == 2 else "~4.40100.0", + "tool-scons": "~2.20501.7" if sys.version_info.major == 2 else "~4.40100.1", "tool-cppcheck": "~1.230.0", "tool-clangtidy": "~1.100000.0", "tool-pvs-studio": "~7.11.0", diff --git a/platformio/builder/tools/piomaxlen.py b/platformio/builder/tools/piomaxlen.py index 4077273c..c05ae821 100644 --- a/platformio/builder/tools/piomaxlen.py +++ b/platformio/builder/tools/piomaxlen.py @@ -67,6 +67,8 @@ def generate(env): _long_sources_hook=long_sources_hook, TEMPFILE=TempFileMunge, MAXLINELENGTH=MAX_LINE_LENGTH, + TEMPFILESUFFIX=".tmp", + TEMPFILEDIR="$BUILD_DIR", ) for name in ("LINKCOM", "ASCOM", "ASPPCOM", "CCCOM", "CXXCOM"): From d77dbb2cca7d245eaeb64c8b3c83db1ccfcf9450 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 27 Jan 2021 20:30:28 +0200 Subject: [PATCH 41/50] Use "TEMPFILEARGESCFUNC" for GCC workaround on Windows --- platformio/__init__.py | 2 +- platformio/builder/tools/piomaxlen.py | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index b3ff0e42..fa2c7125 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -50,7 +50,7 @@ __core_packages__ = { "contrib-piohome": "~3.3.3", "contrib-pysite": "~2.%d%d.0" % (sys.version_info.major, sys.version_info.minor), "tool-unity": "~1.20500.0", - "tool-scons": "~2.20501.7" if sys.version_info.major == 2 else "~4.40100.1", + "tool-scons": "~2.20501.7" if sys.version_info.major == 2 else "~4.40100.2", "tool-cppcheck": "~1.230.0", "tool-clangtidy": "~1.100000.0", "tool-pvs-studio": "~7.11.0", diff --git a/platformio/builder/tools/piomaxlen.py b/platformio/builder/tools/piomaxlen.py index c05ae821..d386bd14 100644 --- a/platformio/builder/tools/piomaxlen.py +++ b/platformio/builder/tools/piomaxlen.py @@ -16,8 +16,10 @@ from __future__ import absolute_import import hashlib import os +import re from SCons.Platform import TempFileMunge # pylint: disable=import-error +from SCons.Subst import quote_spaces # pylint: disable=import-error from platformio.compat import WINDOWS, hashlib_encode_data @@ -27,6 +29,16 @@ from platformio.compat import WINDOWS, hashlib_encode_data # We need ~256 characters for a temporary file path MAX_LINE_LENGTH = (8192 if WINDOWS else 131072) - 256 +WINPATHSEP_RE = re.compile(r"\\([^\"'\\]|$)") + + +def tempfile_arg_esc_func(arg): + arg = quote_spaces(arg) + if not WINDOWS: + return arg + # GCC requires double Windows slashes, let's use UNIX separator + return WINPATHSEP_RE.sub(r"/\1", arg) + def long_sources_hook(env, sources): _sources = str(sources).replace("\\", "/") @@ -67,6 +79,7 @@ def generate(env): _long_sources_hook=long_sources_hook, TEMPFILE=TempFileMunge, MAXLINELENGTH=MAX_LINE_LENGTH, + TEMPFILEARGESCFUNC=tempfile_arg_esc_func, TEMPFILESUFFIX=".tmp", TEMPFILEDIR="$BUILD_DIR", ) From 9d2fd4982ff0eb5c9a25f72ca4466f09e660156a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 27 Jan 2021 20:40:25 +0200 Subject: [PATCH 42/50] Cleanup code --- platformio/builder/tools/pioide.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/platformio/builder/tools/pioide.py b/platformio/builder/tools/pioide.py index 7c3ec208..88bee2ba 100644 --- a/platformio/builder/tools/pioide.py +++ b/platformio/builder/tools/pioide.py @@ -151,15 +151,6 @@ def _escape_build_flag(flags): def DumpIDEData(env, globalenv): """ env here is `projenv`""" - env["__escape_build_flag"] = _escape_build_flag - - LINTCCOM = ( - "${__escape_build_flag(CFLAGS)} ${__escape_build_flag(CCFLAGS)} $CPPFLAGS" - ) - LINTCXXCOM = ( - "${__escape_build_flag(CXXFLAGS)} ${__escape_build_flag(CCFLAGS)} $CPPFLAGS" - ) - data = { "env_name": env["PIOENV"], "libsource_dirs": [env.subst(l) for l in env.GetLibSourceDirs()], @@ -192,7 +183,17 @@ def DumpIDEData(env, globalenv): _new_defines.append(item) env_.Replace(CPPDEFINES=_new_defines) - data.update({"cc_flags": env_.subst(LINTCCOM), "cxx_flags": env_.subst(LINTCXXCOM)}) + # export C/C++ build flags + env_["__escape_build_flag"] = _escape_build_flag + CFLAGSCOM = ( + "${__escape_build_flag(CFLAGS)} ${__escape_build_flag(CCFLAGS)} $CPPFLAGS" + ) + CXXFLAGSCOM = ( + "${__escape_build_flag(CXXFLAGS)} ${__escape_build_flag(CCFLAGS)} $CPPFLAGS" + ) + data.update( + {"cc_flags": env_.subst(CFLAGSCOM), "cxx_flags": env_.subst(CXXFLAGSCOM)} + ) return data From cb9e72a879932a9c4333e109a6f4496c8d38cce7 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 27 Jan 2021 20:57:53 +0200 Subject: [PATCH 43/50] Dump build flags using SCons.Subst.SUBST_CMD --- platformio/builder/tools/pioide.py | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/platformio/builder/tools/pioide.py b/platformio/builder/tools/pioide.py index 88bee2ba..16b5c229 100644 --- a/platformio/builder/tools/pioide.py +++ b/platformio/builder/tools/pioide.py @@ -17,7 +17,8 @@ from __future__ import absolute_import import os from glob import glob -from SCons.Defaults import processDefines # pylint: disable=import-error +import SCons.Defaults # pylint: disable=import-error +import SCons.Subst # pylint: disable=import-error from platformio.compat import glob_escape from platformio.package.manager.core import get_core_package_dir @@ -95,7 +96,7 @@ def _get_gcc_defines(env): def _dump_defines(env): defines = [] # global symbols - for item in processDefines(env.get("CPPDEFINES", [])): + for item in SCons.Defaults.processDefines(env.get("CPPDEFINES", [])): item = item.strip() if item: defines.append(env.subst(item).replace("\\", "")) @@ -144,8 +145,9 @@ def _get_svd_path(env): return None -def _escape_build_flag(flags): - return [flag if " " not in flag else '"%s"' % flag for flag in flags] +def _subst_cmd(env, cmd): + args = env.subst_list(cmd, SCons.Subst.SUBST_CMD)[0] + return " ".join([SCons.Subst.quote_spaces(arg) for arg in args]) def DumpIDEData(env, globalenv): @@ -153,7 +155,7 @@ def DumpIDEData(env, globalenv): data = { "env_name": env["PIOENV"], - "libsource_dirs": [env.subst(l) for l in env.GetLibSourceDirs()], + "libsource_dirs": [env.subst(item) for item in env.GetLibSourceDirs()], "defines": _dump_defines(env), "includes": _dump_includes(env), "cc_path": where_is_program(env.subst("$CC"), env.subst("${ENV['PATH']}")), @@ -175,7 +177,7 @@ def DumpIDEData(env, globalenv): env_ = env.Clone() # https://github.com/platformio/platformio-atom-ide/issues/34 _new_defines = [] - for item in processDefines(env_.get("CPPDEFINES", [])): + for item in SCons.Defaults.processDefines(env_.get("CPPDEFINES", [])): item = item.replace('\\"', '"') if " " in item: _new_defines.append(item.replace(" ", "\\\\ ")) @@ -184,15 +186,11 @@ def DumpIDEData(env, globalenv): env_.Replace(CPPDEFINES=_new_defines) # export C/C++ build flags - env_["__escape_build_flag"] = _escape_build_flag - CFLAGSCOM = ( - "${__escape_build_flag(CFLAGS)} ${__escape_build_flag(CCFLAGS)} $CPPFLAGS" - ) - CXXFLAGSCOM = ( - "${__escape_build_flag(CXXFLAGS)} ${__escape_build_flag(CCFLAGS)} $CPPFLAGS" - ) data.update( - {"cc_flags": env_.subst(CFLAGSCOM), "cxx_flags": env_.subst(CXXFLAGSCOM)} + { + "cc_flags": _subst_cmd(env_, "$CFLAGS $CCFLAGS $CPPFLAGS"), + "cxx_flags": _subst_cmd(env_, "$CXXFLAGS $CCFLAGS $CPPFLAGS"), + } ) return data From bd75c3e559d7a00a86923abc5bf3032606a33bce Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 27 Jan 2021 20:58:13 +0200 Subject: [PATCH 44/50] Bump version to 5.1.0rc2 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index fa2c7125..9f634ac1 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (5, 1, "0rc1") +VERSION = (5, 1, "0rc2") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 493a33e75434c09d143ce5578ab17f7bd0286353 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 27 Jan 2021 22:25:42 +0200 Subject: [PATCH 45/50] Drop support for Python 2 --- .github/workflows/examples.yml | 2 +- platformio/maintenance.py | 18 ++---------------- setup.py | 4 ---- tox.ini | 4 ++-- 4 files changed, 5 insertions(+), 23 deletions(-) diff --git a/.github/workflows/examples.yml b/.github/workflows/examples.yml index 5825fd81..34d261d4 100644 --- a/.github/workflows/examples.yml +++ b/.github/workflows/examples.yml @@ -8,7 +8,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-16.04, windows-latest, macos-latest] - python-version: [2.7, 3.7] + python-version: [3.7] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v2 diff --git a/platformio/maintenance.py b/platformio/maintenance.py index 472a82a5..0e101339 100644 --- a/platformio/maintenance.py +++ b/platformio/maintenance.py @@ -40,6 +40,8 @@ from platformio.proc import is_container def on_platformio_start(ctx, force, caller): + ensure_python3(raise_exception=True) + app.set_session_var("command_ctx", ctx) app.set_session_var("force_option", force) set_caller(caller) @@ -47,24 +49,8 @@ def on_platformio_start(ctx, force, caller): if PlatformioCLI.in_silence(): return - after_upgrade(ctx) - if not ensure_python3(raise_exception=False): - click.secho( - """ -Python 2 and Python 3.5 are not compatible with PlatformIO Core 5.0. -Please check a migration guide on how to fix this warning message: -""", - fg="yellow", - ) - click.secho( - "https://docs.platformio.org/en/latest/core/migration.html" - "#drop-support-for-python-2-and-3-5", - fg="blue", - ) - click.echo("") - def on_platformio_end(ctx, result): # pylint: disable=unused-argument if PlatformioCLI.in_silence(): diff --git a/setup.py b/setup.py index 2aab992a..ad251a89 100644 --- a/setup.py +++ b/setup.py @@ -58,9 +58,6 @@ setup( author_email=__email__, url=__url__, license=__license__, - python_requires=", ".join( - [">=2.7", "!=3.0.*", "!=3.1.*", "!=3.2.*", "!=3.3.*", "!=3.4.*"] - ), install_requires=minimal_requirements + ([] if PY2 else home_requirements), packages=find_packages(exclude=["tests.*", "tests"]) + ["scripts"], package_data={ @@ -87,7 +84,6 @@ setup( "Operating System :: OS Independent", "Programming Language :: C", "Programming Language :: Python", - "Programming Language :: Python :: 2", "Programming Language :: Python :: 3", "Topic :: Software Development", "Topic :: Software Development :: Build Tools", diff --git a/tox.ini b/tox.ini index ecaab77d..8a1f0b04 100644 --- a/tox.ini +++ b/tox.ini @@ -13,13 +13,13 @@ # limitations under the License. [tox] -envlist = py27,py37,py38,py39 +envlist = py36,py37,py38,py39 [testenv] passenv = * usedevelop = True deps = - py36,py37,py38,py39: black + black isort pylint pytest From 61d70fa688b30a913db2a580767ab6b4fe62efd3 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 27 Jan 2021 22:40:19 +0200 Subject: [PATCH 46/50] Include Unity framework for IDE data only if there are tests in project --- platformio/builder/tools/pioide.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/platformio/builder/tools/pioide.py b/platformio/builder/tools/pioide.py index 16b5c229..de7e0cc8 100644 --- a/platformio/builder/tools/pioide.py +++ b/platformio/builder/tools/pioide.py @@ -59,10 +59,15 @@ def _dump_includes(env): for g in toolchain_incglobs: includes["toolchain"].extend([os.path.realpath(inc) for inc in glob(g)]) + # include Unity framework if there are tests in project includes["unity"] = [] + auto_install_unity = False + test_dir = env.GetProjectConfig().get_optional_dir("test") + if os.path.isdir(test_dir) and os.listdir(test_dir) != ["README"]: + auto_install_unity = True unity_dir = get_core_package_dir( "tool-unity", - auto_install=os.path.isdir(env.GetProjectConfig().get_optional_dir("test")), + auto_install=auto_install_unity, ) if unity_dir: includes["unity"].append(unity_dir) From 808ba603c54889b7e37840a150e069e07ed3e0e8 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 27 Jan 2021 23:06:18 +0200 Subject: [PATCH 47/50] =?UTF-8?q?Fixed=20an=20issue=20when=20"pio=20device?= =?UTF-8?q?=20monitor=20=E2=80=93eol"=20and=20=E2=80=9Csend=5Fon=5Fenter?= =?UTF-8?q?=E2=80=9D=20filter=20do=20not=20work=20properly=20//=20Resolve?= =?UTF-8?q?=20#3787?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- HISTORY.rst | 1 + platformio/commands/device/command.py | 8 ++++---- platformio/commands/device/filters/base.py | 7 ++++--- platformio/commands/device/filters/send_on_enter.py | 11 +++++++++-- platformio/commands/device/helpers.py | 8 ++++---- 5 files changed, 22 insertions(+), 13 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index f2a5bc61..2c24371a 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -45,6 +45,7 @@ PlatformIO Core 5 - Improved listing of `multicast DNS services `_ - Fixed a "UnicodeDecodeError: 'utf-8' codec can't decode byte" when using J-Link for firmware uploading on Linux (`issue #3804 `_) - Fixed an issue with a compiler driver for ".ccls" language server (`issue #3808 `_) + - Fixed an issue when `pio device monitor --eol `__ and "send_on_enter" filter do not work properly (`issue #3787 `_) 5.0.4 (2020-12-30) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/commands/device/command.py b/platformio/commands/device/command.py index 2e254742..fd385a46 100644 --- a/platformio/commands/device/command.py +++ b/platformio/commands/device/command.py @@ -179,7 +179,9 @@ def device_monitor(**kwargs): # pylint: disable=too-many-branches for name in os.listdir(filters_dir): if not name.endswith(".py"): continue - device_helpers.load_monitor_filter(os.path.join(filters_dir, name)) + device_helpers.load_monitor_filter( + os.path.join(filters_dir, name), options=kwargs + ) project_options = {} try: @@ -193,9 +195,7 @@ def device_monitor(**kwargs): # pylint: disable=too-many-branches if "platform" in project_options: with fs.cd(kwargs["project_dir"]): platform = PlatformFactory.new(project_options["platform"]) - device_helpers.register_platform_filters( - platform, kwargs["project_dir"], kwargs["environment"] - ) + device_helpers.register_platform_filters(platform, options=kwargs) if not kwargs["port"]: ports = util.get_serial_ports(filter_hwid=True) diff --git a/platformio/commands/device/filters/base.py b/platformio/commands/device/filters/base.py index 5c6d0400..6745a626 100644 --- a/platformio/commands/device/filters/base.py +++ b/platformio/commands/device/filters/base.py @@ -18,12 +18,13 @@ from platformio.project.config import ProjectConfig class DeviceMonitorFilter(miniterm.Transform): - def __init__(self, project_dir=None, environment=None): + def __init__(self, options=None): """ Called by PlatformIO to pass context """ miniterm.Transform.__init__(self) - self.project_dir = project_dir - self.environment = environment + self.options = options or {} + self.project_dir = self.options.get("project_dir") + self.environment = self.options.get("environment") self.config = ProjectConfig.get_instance() if not self.environment: diff --git a/platformio/commands/device/filters/send_on_enter.py b/platformio/commands/device/filters/send_on_enter.py index 10ca2103..8300c980 100644 --- a/platformio/commands/device/filters/send_on_enter.py +++ b/platformio/commands/device/filters/send_on_enter.py @@ -22,10 +22,17 @@ class SendOnEnter(DeviceMonitorFilter): super(SendOnEnter, self).__init__(*args, **kwargs) self._buffer = "" + if self.options.get("eol") == "CR": + self._eol = "\r" + elif self.options.get("eol") == "LF": + self._eol = "\n" + else: + self._eol = "\r\n" + def tx(self, text): self._buffer += text - if self._buffer.endswith("\r\n"): - text = self._buffer[:-2] + if self._buffer.endswith(self._eol): + text = self._buffer[: len(self._eol) * -1] self._buffer = "" return text return "" diff --git a/platformio/commands/device/helpers.py b/platformio/commands/device/helpers.py index 3bfe8fc6..a65b4895 100644 --- a/platformio/commands/device/helpers.py +++ b/platformio/commands/device/helpers.py @@ -76,7 +76,7 @@ def get_board_hwids(project_dir, platform, board): return platform.board_config(board).get("build.hwids", []) -def load_monitor_filter(path, project_dir=None, environment=None): +def load_monitor_filter(path, options=None): name = os.path.basename(path) name = name[: name.find(".")] module = load_python_module("platformio.commands.device.filters.%s" % name, path) @@ -87,12 +87,12 @@ def load_monitor_filter(path, project_dir=None, environment=None): or cls == DeviceMonitorFilter ): continue - obj = cls(project_dir, environment) + obj = cls(options) miniterm.TRANSFORMATIONS[obj.NAME] = obj return True -def register_platform_filters(platform, project_dir, environment): +def register_platform_filters(platform, options=None): monitor_dir = os.path.join(platform.get_dir(), "monitor") if not os.path.isdir(monitor_dir): return @@ -103,4 +103,4 @@ def register_platform_filters(platform, project_dir, environment): path = os.path.join(monitor_dir, name) if not os.path.isfile(path): continue - load_monitor_filter(path, project_dir, environment) + load_monitor_filter(path, options) From f8193b24195be2fe68d8e8f8f5f0007bb4da5ce0 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 27 Jan 2021 23:06:42 +0200 Subject: [PATCH 48/50] Bump version to 5.1.0rc3 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 9f634ac1..bca2f0d5 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (5, 1, "0rc2") +VERSION = (5, 1, "0rc3") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 078a024931c17e1289df0326c0b6e067825fe4d9 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 28 Jan 2021 13:52:11 +0200 Subject: [PATCH 49/50] Configure default `debug_speed` --- platformio/commands/debug/helpers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/platformio/commands/debug/helpers.py b/platformio/commands/debug/helpers.py index 62ce4f73..7561c338 100644 --- a/platformio/commands/debug/helpers.py +++ b/platformio/commands/debug/helpers.py @@ -176,6 +176,7 @@ def configure_initial_debug_options(platform, env_options): tool_name, tool_settings, ), + speed=env_options.get("debug_speed", tool_settings.get("speed")), server=server_options, ) return result From fd540148f3141de54fd5a87a472977a6fe0403c3 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 28 Jan 2021 19:23:06 +0200 Subject: [PATCH 50/50] Bump version to 5.1.0 --- HISTORY.rst | 16 ++++++++-------- docs | 2 +- examples | 2 +- platformio/__init__.py | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 2c24371a..5687765d 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -8,18 +8,23 @@ PlatformIO Core 5 **A professional collaborative platform for embedded development** -5.1.0 (2021-??-??) +5.1.0 (2021-01-28) ~~~~~~~~~~~~~~~~~~ +* **PlatformIO Home** + + - Boosted PlatformIO Home performance thanks to migrating the codebase to the pure Python 3 Asynchronous I/O stack + - Added a new ``--session-id`` option to `pio home `__ command that helps to keep PlatformIO Home isolated from other instances and protect from 3rd party access (`issue #3397 `_) + * **Build System** - Upgraded build engine to the SCons 4.1 (`release notes `_) - Refactored a workaround for a maximum command line character limitation (`issue #3792 `_) - Fixed an issue with Python 3.8+ on Windows when a network drive is used (`issue #3417 `_) -* **Package Management System** +* **Package Management** - - New options for `system prune `__ command: + - New options for `pio system prune `__ command: + ``--dry-run`` option to show data that will be removed + ``--core-packages`` option to remove unnecessary core packages @@ -30,11 +35,6 @@ PlatformIO Core 5 - Fixed an issue when unnecessary packages were removed in ``update --dry-run`` mode (`issue #3809 `_) - Fixed a "ValueError: Invalid simple block" when uninstalling a package with a custom name and external source (`issue #3816 `_) -* **PlatformIO Home** - - - Significantly speedup PlatformIO Home loading time by migrating to native Python 3 Asynchronous I/O - - Added a new ``--session-id`` option to `pio home `__ command that helps to keep PlatformIO Home isolated from other instances and protect from 3rd party access (`issue #3397 `_) - * **Debugging** - Configure a custom debug adapter speed using a new `debug_speed `__ option (`issue #3799 `_) diff --git a/docs b/docs index d50d1e2c..25edd66d 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit d50d1e2c75a44aac26dda85f58787a5210125927 +Subproject commit 25edd66d5514cc5a0c8d5a01f4752de3e98a03d0 diff --git a/examples b/examples index ced25363..8a6e639b 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit ced253632eac141595967b08ad6eb258c0bc0878 +Subproject commit 8a6e639b2bcb18dec63bc010f359b49f85084b45 diff --git a/platformio/__init__.py b/platformio/__init__.py index bca2f0d5..5ed64a4e 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (5, 1, "0rc3") +VERSION = (5, 1, 0) __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio"