diff --git a/.isort.cfg b/.isort.cfg index 2270008c..de9bf40e 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -1,3 +1,3 @@ [settings] line_length=88 -known_third_party=SCons, twisted, autobahn, jsonrpc +known_third_party=OpenSSL, SCons, autobahn, jsonrpc, twisted, zope diff --git a/HISTORY.rst b/HISTORY.rst index a7664799..7d5de273 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -9,6 +9,7 @@ PlatformIO Core 4 4.3.2 (2020-??-??) ~~~~~~~~~~~~~~~~~~ +* Open source `PIO Remote `__ client * Fixed PIO Unit Testing for Zephyr RTOS * Fixed UnicodeDecodeError on Windows when network drive (NAS) is used (`issue #3417 `_) * Fixed an issue when saving libraries in new project results in error "No option 'lib_deps' in section" (`issue #3442 `_) diff --git a/Makefile b/Makefile index 548f96f3..36b5d396 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,7 @@ format: test: py.test --verbose --capture=no --exitfirst -n 6 --dist=loadscope tests --ignore tests/test_examples.py -before-commit: isort format lint test +before-commit: isort format lint clean-docs: rm -rf docs/_build diff --git a/docs b/docs index 2bf2daaa..ae721948 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 2bf2daaa0a9b33d88d809d6d529059304e72f855 +Subproject commit ae721948ba4640e64073d2357b9c2bd47d3430c1 diff --git a/platformio/__init__.py b/platformio/__init__.py index 6096e53a..8573627e 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -34,3 +34,4 @@ __license__ = "Apache Software License" __copyright__ = "Copyright 2014-present PlatformIO" __apiurl__ = "https://api.platformio.org" +__pioremote_endpoint__ = "ssl:remote.platformio.org:4413" diff --git a/platformio/app.py b/platformio/app.py index e32004ab..6c7c7b1a 100644 --- a/platformio/app.py +++ b/platformio/app.py @@ -13,9 +13,11 @@ # limitations under the License. import codecs +import getpass import hashlib import os import platform +import socket import uuid from os import environ, getenv, listdir, remove from os.path import dirname, isdir, isfile, join, realpath @@ -426,3 +428,17 @@ def get_user_agent(): data.append("Python/%s" % platform.python_version()) data.append("Platform/%s" % platform.platform()) return " ".join(data) + + +def get_host_id(): + h = hashlib.sha1(hashlib_encode_data(get_cid())) + try: + username = getpass.getuser() + h.update(hashlib_encode_data(username)) + except: # pylint: disable=bare-except + pass + return h.hexdigest() + + +def get_host_name(): + return str(socket.gethostname())[:255] diff --git a/platformio/commands/remote/__init__.py b/platformio/commands/remote/__init__.py new file mode 100644 index 00000000..b0514903 --- /dev/null +++ b/platformio/commands/remote/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/commands/remote/ac/__init__.py b/platformio/commands/remote/ac/__init__.py new file mode 100644 index 00000000..b0514903 --- /dev/null +++ b/platformio/commands/remote/ac/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/commands/remote/ac/base.py b/platformio/commands/remote/ac/base.py new file mode 100644 index 00000000..7b76a327 --- /dev/null +++ b/platformio/commands/remote/ac/base.py @@ -0,0 +1,91 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.internet import defer # pylint: disable=import-error +from twisted.spread import pb # pylint: disable=import-error + + +class AsyncCommandBase(object): + + MAX_BUFFER_SIZE = 1024 * 1024 # 1Mb + + def __init__(self, options=None, on_end_callback=None): + self.options = options or {} + self.on_end_callback = on_end_callback + self._buffer = b"" + self._return_code = None + self._d = None + self._paused = False + + try: + self.start() + except Exception as e: + raise pb.Error(str(e)) + + @property + def id(self): + return id(self) + + def pause(self): + self._paused = True + self.stop() + + def unpause(self): + self._paused = False + self.start() + + def start(self): + raise NotImplementedError + + def stop(self): + self.transport.loseConnection() # pylint: disable=no-member + + def _ac_ended(self): + if self.on_end_callback: + self.on_end_callback() + if not self._d or self._d.called: + self._d = None + return + if self._buffer: + self._d.callback(self._buffer) + else: + self._d.callback(None) + + def _ac_ondata(self, data): + self._buffer += data + if len(self._buffer) > self.MAX_BUFFER_SIZE: + self._buffer = self._buffer[-1 * self.MAX_BUFFER_SIZE :] + if self._paused: + return + if self._d and not self._d.called: + self._d.callback(self._buffer) + self._buffer = b"" + + def ac_read(self): + if self._buffer: + result = self._buffer + self._buffer = b"" + return result + if self._return_code is None: + self._d = defer.Deferred() + return self._d + return None + + def ac_write(self, data): + self.transport.write(data) # pylint: disable=no-member + return len(data) + + def ac_close(self): + self.stop() + return self._return_code diff --git a/platformio/commands/remote/ac/process.py b/platformio/commands/remote/ac/process.py new file mode 100644 index 00000000..9e4f6989 --- /dev/null +++ b/platformio/commands/remote/ac/process.py @@ -0,0 +1,42 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from twisted.internet import protocol, reactor # pylint: disable=import-error + +from platformio.commands.remote.ac.base import AsyncCommandBase + + +class ProcessAsyncCmd(protocol.ProcessProtocol, AsyncCommandBase): + def start(self): + env = dict(os.environ).copy() + env.update({"PLATFORMIO_FORCE_ANSI": "true"}) + reactor.spawnProcess( + self, self.options["executable"], self.options["args"], env + ) + + def outReceived(self, data): + self._ac_ondata(data) + + def errReceived(self, data): + self._ac_ondata(data) + + def processExited(self, reason): + self._return_code = reason.value.exitCode + + def processEnded(self, reason): + if self._return_code is None: + self._return_code = reason.value.exitCode + self._ac_ended() diff --git a/platformio/commands/remote/ac/psync.py b/platformio/commands/remote/ac/psync.py new file mode 100644 index 00000000..6773615c --- /dev/null +++ b/platformio/commands/remote/ac/psync.py @@ -0,0 +1,66 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import os +import zlib +from io import BytesIO + +from platformio.commands.remote.ac.base import AsyncCommandBase +from platformio.commands.remote.projectsync import PROJECT_SYNC_STAGE, ProjectSync + + +class ProjectSyncAsyncCmd(AsyncCommandBase): + def __init__(self, *args, **kwargs): + self.psync = None + self._upstream = None + super(ProjectSyncAsyncCmd, self).__init__(*args, **kwargs) + + def start(self): + project_dir = os.path.join( + self.options["agent_working_dir"], "projects", self.options["id"] + ) + self.psync = ProjectSync(project_dir) + for name in self.options["items"]: + self.psync.add_item(os.path.join(project_dir, name), name) + + def stop(self): + self.psync = None + self._upstream = None + self._return_code = PROJECT_SYNC_STAGE.COMPLETED.value + + def ac_write(self, data): + stage = PROJECT_SYNC_STAGE.lookupByValue(data.get("stage")) + + if stage is PROJECT_SYNC_STAGE.DBINDEX: + self.psync.rebuild_dbindex() + return zlib.compress(json.dumps(self.psync.get_dbindex()).encode()) + + if stage is PROJECT_SYNC_STAGE.DELETE: + return self.psync.delete_dbindex( + json.loads(zlib.decompress(data["dbindex"])) + ) + + if stage is PROJECT_SYNC_STAGE.UPLOAD: + if not self._upstream: + self._upstream = BytesIO() + self._upstream.write(data["chunk"]) + if self._upstream.tell() == data["total"]: + self.psync.decompress_items(self._upstream) + self._upstream = None + return PROJECT_SYNC_STAGE.EXTRACTED.value + + return PROJECT_SYNC_STAGE.UPLOAD.value + + return None diff --git a/platformio/commands/remote/ac/serial.py b/platformio/commands/remote/ac/serial.py new file mode 100644 index 00000000..d0181f9c --- /dev/null +++ b/platformio/commands/remote/ac/serial.py @@ -0,0 +1,60 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from time import sleep + +from twisted.internet import protocol, reactor # pylint: disable=import-error +from twisted.internet.serialport import SerialPort # pylint: disable=import-error + +from platformio.commands.remote.ac.base import AsyncCommandBase + + +class SerialPortAsyncCmd(protocol.Protocol, AsyncCommandBase): + def start(self): + SerialPort( + self, + reactor=reactor, + **{ + "deviceNameOrPortNumber": self.options["port"], + "baudrate": self.options["baud"], + "parity": self.options["parity"], + "rtscts": 1 if self.options["rtscts"] else 0, + "xonxoff": 1 if self.options["xonxoff"] else 0, + } + ) + + def connectionMade(self): + self.reset_device() + if self.options.get("rts", None) is not None: + self.transport.setRTS(self.options.get("rts")) + if self.options.get("dtr", None) is not None: + self.transport.setDTR(self.options.get("dtr")) + + def reset_device(self): + self.transport.flushInput() + self.transport.setDTR(False) + self.transport.setRTS(False) + sleep(0.1) + self.transport.setDTR(True) + self.transport.setRTS(True) + sleep(0.1) + + def dataReceived(self, data): + self._ac_ondata(data) + + def connectionLost(self, reason): # pylint: disable=unused-argument + if self._paused: + return + self._return_code = 0 + self._ac_ended() diff --git a/platformio/commands/remote/client/__init__.py b/platformio/commands/remote/client/__init__.py new file mode 100644 index 00000000..b0514903 --- /dev/null +++ b/platformio/commands/remote/client/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/commands/remote/client/agent_list.py b/platformio/commands/remote/client/agent_list.py new file mode 100644 index 00000000..df1de28b --- /dev/null +++ b/platformio/commands/remote/client/agent_list.py @@ -0,0 +1,38 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from datetime import datetime + +import click + +from platformio.commands.remote.client.base import RemoteClientBase + + +class AgentListClient(RemoteClientBase): + def agent_pool_ready(self): + d = self.agentpool.callRemote("list", True) + d.addCallback(self._cbResult) + d.addErrback(self.cb_global_error) + + def _cbResult(self, result): + for item in result: + click.secho(item["name"], fg="cyan") + click.echo("-" * len(item["name"])) + click.echo("ID: %s" % item["id"]) + click.echo( + "Started: %s" + % datetime.fromtimestamp(item["started"]).strftime("%Y-%m-%d %H:%M:%S") + ) + click.echo("") + self.disconnect() diff --git a/platformio/commands/remote/client/agent_service.py b/platformio/commands/remote/client/agent_service.py new file mode 100644 index 00000000..5918d205 --- /dev/null +++ b/platformio/commands/remote/client/agent_service.py @@ -0,0 +1,222 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from os.path import getatime, getmtime, isdir, isfile, join + +from twisted.logger import LogLevel # pylint: disable=import-error +from twisted.spread import pb # pylint: disable=import-error + +from platformio import proc, util +from platformio.commands.remote.ac.process import ProcessAsyncCmd +from platformio.commands.remote.ac.psync import ProjectSyncAsyncCmd +from platformio.commands.remote.ac.serial import SerialPortAsyncCmd +from platformio.commands.remote.client.base import RemoteClientBase +from platformio.project.config import ProjectConfig +from platformio.project.exception import NotPlatformIOProjectError +from platformio.project.helpers import get_project_core_dir + + +class RemoteAgentService(RemoteClientBase): + def __init__(self, name, share, working_dir=None): + RemoteClientBase.__init__(self) + self.log_level = LogLevel.info + self.working_dir = working_dir or join(get_project_core_dir(), "remote") + if not isdir(self.working_dir): + os.makedirs(self.working_dir) + if name: + self.name = str(name)[:50] + self.join_options.update( + {"agent": True, "share": [s.lower().strip()[:50] for s in share]} + ) + + self._acs = {} + + def agent_pool_ready(self): + pass + + def cb_disconnected(self, reason): + for ac in self._acs.values(): + ac.ac_close() + RemoteClientBase.cb_disconnected(self, reason) + + def remote_acread(self, ac_id): + self.log.debug("Async Read: {id}", id=ac_id) + if ac_id not in self._acs: + raise pb.Error("Invalid Async Identifier") + return self._acs[ac_id].ac_read() + + def remote_acwrite(self, ac_id, data): + self.log.debug("Async Write: {id}", id=ac_id) + if ac_id not in self._acs: + raise pb.Error("Invalid Async Identifier") + return self._acs[ac_id].ac_write(data) + + def remote_acclose(self, ac_id): + self.log.debug("Async Close: {id}", id=ac_id) + if ac_id not in self._acs: + raise pb.Error("Invalid Async Identifier") + return_code = self._acs[ac_id].ac_close() + del self._acs[ac_id] + return return_code + + def remote_cmd(self, cmd, options): + self.log.info("Remote command received: {cmd}", cmd=cmd) + self.log.debug("Command options: {options!r}", options=options) + callback = "_process_cmd_%s" % cmd.replace(".", "_") + return getattr(self, callback)(options) + + def _defer_async_cmd(self, ac, pass_agent_name=True): + self._acs[ac.id] = ac + if pass_agent_name: + return (self.id, ac.id, self.name) + return (self.id, ac.id) + + def _process_cmd_device_list(self, _): + return (self.name, util.get_serialports()) + + def _process_cmd_device_monitor(self, options): + if not options["port"]: + for item in util.get_serialports(): + if "VID:PID" in item["hwid"]: + options["port"] = item["port"] + break + + # terminate opened monitors + if options["port"]: + for ac in list(self._acs.values()): + if ( + isinstance(ac, SerialPortAsyncCmd) + and ac.options["port"] == options["port"] + ): + self.log.info( + "Terminate previously opened monitor at {port}", + port=options["port"], + ) + ac.ac_close() + del self._acs[ac.id] + + if not options["port"]: + raise pb.Error("Please specify serial port using `--port` option") + self.log.info("Starting serial monitor at {port}", port=options["port"]) + + return self._defer_async_cmd(SerialPortAsyncCmd(options), pass_agent_name=False) + + def _process_cmd_psync(self, options): + for ac in list(self._acs.values()): + if ( + isinstance(ac, ProjectSyncAsyncCmd) + and ac.options["id"] == options["id"] + ): + self.log.info("Terminate previous Project Sync process") + ac.ac_close() + del self._acs[ac.id] + + options["agent_working_dir"] = self.working_dir + return self._defer_async_cmd( + ProjectSyncAsyncCmd(options), pass_agent_name=False + ) + + def _process_cmd_run(self, options): + return self._process_cmd_run_or_test("run", options) + + def _process_cmd_test(self, options): + return self._process_cmd_run_or_test("test", options) + + def _process_cmd_run_or_test( # pylint: disable=too-many-locals,too-many-branches + self, command, options + ): + assert options and "project_id" in options + project_dir = join(self.working_dir, "projects", options["project_id"]) + origin_pio_ini = join(project_dir, "platformio.ini") + back_pio_ini = join(project_dir, "platformio.ini.bak") + + # remove insecure project options + try: + conf = ProjectConfig(origin_pio_ini) + if isfile(back_pio_ini): + os.remove(back_pio_ini) + os.rename(origin_pio_ini, back_pio_ini) + # cleanup + if conf.has_section("platformio"): + for opt in conf.options("platformio"): + if opt.endswith("_dir"): + conf.remove_option("platformio", opt) + else: + conf.add_section("platformio") + conf.set("platformio", "build_dir", ".pio/build") + conf.save(origin_pio_ini) + + # restore A/M times + os.utime(origin_pio_ini, (getatime(back_pio_ini), getmtime(back_pio_ini))) + except NotPlatformIOProjectError as e: + raise pb.Error(str(e)) + + cmd_args = ["platformio", "--force", command, "-d", project_dir] + for env in options.get("environment", []): + cmd_args.extend(["-e", env]) + for target in options.get("target", []): + cmd_args.extend(["-t", target]) + for ignore in options.get("ignore", []): + cmd_args.extend(["-i", ignore]) + if options.get("upload_port", False): + cmd_args.extend(["--upload-port", options.get("upload_port")]) + if options.get("test_port", False): + cmd_args.extend(["--test-port", options.get("test_port")]) + if options.get("disable_auto_clean", False): + cmd_args.append("--disable-auto-clean") + if options.get("without_building", False): + cmd_args.append("--without-building") + if options.get("without_uploading", False): + cmd_args.append("--without-uploading") + if options.get("silent", False): + cmd_args.append("-s") + if options.get("verbose", False): + cmd_args.append("-v") + + paused_acs = [] + for ac in self._acs.values(): + if not isinstance(ac, SerialPortAsyncCmd): + continue + self.log.info("Pause active monitor at {port}", port=ac.options["port"]) + ac.pause() + paused_acs.append(ac) + + def _cb_on_end(): + if isfile(back_pio_ini): + if isfile(origin_pio_ini): + os.remove(origin_pio_ini) + os.rename(back_pio_ini, origin_pio_ini) + for ac in paused_acs: + ac.unpause() + self.log.info( + "Unpause active monitor at {port}", port=ac.options["port"] + ) + + return self._defer_async_cmd( + ProcessAsyncCmd( + {"executable": proc.where_is_program("platformio"), "args": cmd_args}, + on_end_callback=_cb_on_end, + ) + ) + + def _process_cmd_update(self, options): + cmd_args = ["platformio", "--force", "update"] + if options.get("only_check"): + cmd_args.append("--only-check") + return self._defer_async_cmd( + ProcessAsyncCmd( + {"executable": proc.where_is_program("platformio"), "args": cmd_args} + ) + ) diff --git a/platformio/commands/remote/client/async_base.py b/platformio/commands/remote/client/async_base.py new file mode 100644 index 00000000..a07e110b --- /dev/null +++ b/platformio/commands/remote/client/async_base.py @@ -0,0 +1,65 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import click +from twisted.spread import pb # pylint: disable=import-error + +from platformio.commands.remote.client.base import RemoteClientBase + + +class AsyncClientBase(RemoteClientBase): + def __init__(self, command, agents, options): + RemoteClientBase.__init__(self) + self.command = command + self.agents = agents + self.options = options + + self._acs_total = 0 + self._acs_ended = 0 + + def agent_pool_ready(self): + pass + + def cb_async_result(self, result): + if self._acs_total == 0: + self._acs_total = len(result) + for (success, value) in result: + if not success: + raise pb.Error(value) + self.acread_data(*value) + + def acread_data(self, agent_id, ac_id, agent_name=None): + d = self.agentpool.callRemote("acread", agent_id, ac_id) + d.addCallback(self.cb_acread_result, agent_id, ac_id, agent_name) + d.addErrback(self.cb_global_error) + + def cb_acread_result(self, result, agent_id, ac_id, agent_name): + if result is None: + self.acclose(agent_id, ac_id) + else: + if self._acs_total > 1 and agent_name: + click.echo("[%s] " % agent_name, nl=False) + click.echo(result, nl=False) + self.acread_data(agent_id, ac_id, agent_name) + + def acclose(self, agent_id, ac_id): + d = self.agentpool.callRemote("acclose", agent_id, ac_id) + d.addCallback(self.cb_acclose_result) + d.addErrback(self.cb_global_error) + + def cb_acclose_result(self, exit_code): + self._acs_ended += 1 + if self._acs_ended != self._acs_total: + return + self.disconnect(exit_code) diff --git a/platformio/commands/remote/client/base.py b/platformio/commands/remote/client/base.py new file mode 100644 index 00000000..5e0591eb --- /dev/null +++ b/platformio/commands/remote/client/base.py @@ -0,0 +1,182 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from datetime import datetime +from time import time + +import click +from twisted.internet import defer, endpoints, reactor # pylint: disable=import-error +from twisted.logger import ILogObserver # pylint: disable=import-error +from twisted.logger import Logger # pylint: disable=import-error +from twisted.logger import LogLevel # pylint: disable=import-error +from twisted.logger import formatEvent # pylint: disable=import-error +from twisted.python import failure # pylint: disable=import-error +from twisted.spread import pb # pylint: disable=import-error +from zope.interface import provider # pylint: disable=import-error + +from platformio import __pioremote_endpoint__, __version__, app, exception, maintenance +from platformio.commands.remote.factory.client import RemoteClientFactory +from platformio.commands.remote.factory.ssl import SSLContextFactory + + +class RemoteClientBase( # pylint: disable=too-many-instance-attributes + pb.Referenceable +): + + PING_DELAY = 60 + PING_MAX_FAILURES = 3 + DEBUG = False + + def __init__(self): + self.log_level = LogLevel.warn + self.log = Logger(namespace="remote", observer=self._log_observer) + self.id = app.get_host_id() + self.name = app.get_host_name() + self.join_options = {"corever": __version__} + self.perspective = None + self.agentpool = None + + self._ping_id = 0 + self._ping_caller = None + self._ping_counter = 0 + self._reactor_stopped = False + self._exit_code = 0 + + @provider(ILogObserver) + def _log_observer(self, event): + if not self.DEBUG and ( + event["log_namespace"] != self.log.namespace + or self.log_level > event["log_level"] + ): + return + msg = formatEvent(event) + click.echo( + "%s [%s] %s" + % ( + datetime.fromtimestamp(event["log_time"]).strftime("%Y-%m-%d %H:%M:%S"), + event["log_level"].name, + msg, + ) + ) + + def connect(self): + self.log.info("Name: {name}", name=self.name) + self.log.info("Connecting to PIO Remote Cloud") + endpoint = endpoints.clientFromString(reactor, __pioremote_endpoint__) + factory = RemoteClientFactory() + factory.remote_client = self + factory.sslContextFactory = None + if __pioremote_endpoint__.startswith("ssl:"): + # pylint: disable=protected-access + factory.sslContextFactory = SSLContextFactory(endpoint._host) + endpoint._sslContextFactory = factory.sslContextFactory + endpoint.connect(factory) + reactor.run() + + if self._exit_code != 0: + raise exception.ReturnErrorCode(self._exit_code) + + def cb_client_authorization_failed(self, err): + msg = "Bad account credentials" + if err.check(pb.Error): + msg = err.getErrorMessage() + self.log.error(msg) + self.disconnect(exit_code=1) + + def cb_client_authorization_made(self, perspective): + self.log.info("Successfully authorized") + self.perspective = perspective + d = perspective.callRemote("join", self.id, self.name, self.join_options) + d.addCallback(self._cb_client_join_made) + d.addErrback(self.cb_global_error) + + def _cb_client_join_made(self, result): + code = result[0] + if code == 1: + self.agentpool = result[1] + self.agent_pool_ready() + self.restart_ping() + elif code == 2: + self.remote_service(*result[1:]) + + def remote_service(self, command, options): + if command == "disconnect": + self.log.error( + "PIO Remote Cloud disconnected: {msg}", msg=options.get("message") + ) + self.disconnect() + + def restart_ping(self, reset_counter=True): + # stop previous ping callers + self.stop_ping(reset_counter) + self._ping_caller = reactor.callLater(self.PING_DELAY, self._do_ping) + + def _do_ping(self): + self._ping_counter += 1 + self._ping_id = int(time()) + d = self.perspective.callRemote("service", "ping", {"id": self._ping_id}) + d.addCallback(self._cb_pong) + d.addErrback(self._cb_pong) + + def stop_ping(self, reset_counter=True): + if reset_counter: + self._ping_counter = 0 + if not self._ping_caller or not self._ping_caller.active(): + return + self._ping_caller.cancel() + self._ping_caller = None + + def _cb_pong(self, result): + if not isinstance(result, failure.Failure) and self._ping_id == result: + self.restart_ping() + return + if self._ping_counter >= self.PING_MAX_FAILURES: + self.stop_ping() + self.perspective.broker.transport.loseConnection() + else: + self.restart_ping(reset_counter=False) + + def agent_pool_ready(self): + raise NotImplementedError + + def disconnect(self, exit_code=None): + self.stop_ping() + if exit_code is not None: + self._exit_code = exit_code + if reactor.running and not self._reactor_stopped: + self._reactor_stopped = True + reactor.stop() + + def cb_disconnected(self, _): + self.stop_ping() + self.perspective = None + self.agentpool = None + + def cb_global_error(self, err): + if err.check(pb.PBConnectionLost, defer.CancelledError): + return + + msg = err.getErrorMessage() + if err.check(pb.DeadReferenceError): + msg = "Remote Client has been terminated" + elif "PioAgentNotStartedError" in str(err.type): + msg = ( + "Could not find active agents. Please start it before on " + "a remote machine using `pio remote agent start` command.\n" + "See http://docs.platformio.org/page/plus/pio-remote.html" + ) + else: + maintenance.on_platformio_exception(Exception(err.type)) + click.secho(msg, fg="red", err=True) + self.disconnect(exit_code=1) diff --git a/platformio/commands/remote/client/device_list.py b/platformio/commands/remote/client/device_list.py new file mode 100644 index 00000000..dba1729f --- /dev/null +++ b/platformio/commands/remote/client/device_list.py @@ -0,0 +1,54 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json + +import click + +from platformio.commands.remote.client.base import RemoteClientBase + + +class DeviceListClient(RemoteClientBase): + def __init__(self, agents, json_output): + RemoteClientBase.__init__(self) + self.agents = agents + self.json_output = json_output + + def agent_pool_ready(self): + d = self.agentpool.callRemote("cmd", self.agents, "device.list") + d.addCallback(self._cbResult) + d.addErrback(self.cb_global_error) + + def _cbResult(self, result): + data = {} + for (success, value) in result: + if not success: + click.secho(value, fg="red", err=True) + continue + (agent_name, devlist) = value + data[agent_name] = devlist + + if self.json_output: + click.echo(json.dumps(data)) + else: + for agent_name, devlist in data.items(): + click.echo("Agent %s" % click.style(agent_name, fg="cyan", bold=True)) + click.echo("=" * (6 + len(agent_name))) + for item in devlist: + click.secho(item["port"], fg="cyan") + click.echo("-" * len(item["port"])) + click.echo("Hardware ID: %s" % item["hwid"]) + click.echo("Description: %s" % item["description"]) + click.echo("") + self.disconnect() diff --git a/platformio/commands/remote/client/device_monitor.py b/platformio/commands/remote/client/device_monitor.py new file mode 100644 index 00000000..990bb433 --- /dev/null +++ b/platformio/commands/remote/client/device_monitor.py @@ -0,0 +1,236 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from fnmatch import fnmatch + +import click +from twisted.internet import protocol, reactor, task # pylint: disable=import-error +from twisted.spread import pb # pylint: disable=import-error + +from platformio.commands.remote.client.base import RemoteClientBase + + +class SMBridgeProtocol(protocol.Protocol): # pylint: disable=no-init + def connectionMade(self): + self.factory.add_client(self) + + def connectionLost(self, reason): # pylint: disable=unused-argument + self.factory.remove_client(self) + + def dataReceived(self, data): + self.factory.send_to_server(data) + + +class SMBridgeFactory(protocol.ServerFactory): + def __init__(self, cdm): + self.cdm = cdm + self._clients = [] + + def buildProtocol(self, addr): # pylint: disable=unused-argument + p = SMBridgeProtocol() + p.factory = self # pylint: disable=attribute-defined-outside-init + return p + + def add_client(self, client): + self.cdm.log.debug("SMBridge: Client connected") + self._clients.append(client) + self.cdm.acread_data() + + def remove_client(self, client): + self.cdm.log.debug("SMBridge: Client disconnected") + self._clients.remove(client) + if not self._clients: + self.cdm.client_terminal_stopped() + + def has_clients(self): + return len(self._clients) + + def send_to_clients(self, data): + if not self._clients: + return None + for client in self._clients: + client.transport.write(data) + return len(data) + + def send_to_server(self, data): + self.cdm.acwrite_data(data) + + +class DeviceMonitorClient( # pylint: disable=too-many-instance-attributes + RemoteClientBase +): + + MAX_BUFFER_SIZE = 1024 * 1024 + + def __init__(self, agents, **kwargs): + RemoteClientBase.__init__(self) + self.agents = agents + self.cmd_options = kwargs + + self._bridge_factory = SMBridgeFactory(self) + self._agent_id = None + self._ac_id = None + self._d_acread = None + self._d_acwrite = None + self._acwrite_buffer = "" + + def agent_pool_ready(self): + d = task.deferLater( + reactor, 1, self.agentpool.callRemote, "cmd", self.agents, "device.list" + ) + d.addCallback(self._cb_device_list) + d.addErrback(self.cb_global_error) + + def _cb_device_list(self, result): + devices = [] + hwid_devindexes = [] + for (success, value) in result: + if not success: + click.secho(value, fg="red", err=True) + continue + (agent_name, ports) = value + for item in ports: + if "VID:PID" in item["hwid"]: + hwid_devindexes.append(len(devices)) + devices.append((agent_name, item)) + + if len(result) == 1 and self.cmd_options["port"]: + if set(["*", "?", "[", "]"]) & set(self.cmd_options["port"]): + for agent, item in devices: + if fnmatch(item["port"], self.cmd_options["port"]): + return self.start_remote_monitor(agent, item["port"]) + return self.start_remote_monitor(result[0][1][0], self.cmd_options["port"]) + + device = None + if len(hwid_devindexes) == 1: + device = devices[hwid_devindexes[0]] + else: + click.echo("Available ports:") + for i, device in enumerate(devices): + click.echo( + "{index}. {host}{port} \t{description}".format( + index=i + 1, + host=device[0] + ":" if len(result) > 1 else "", + port=device[1]["port"], + description=device[1]["description"] + if device[1]["description"] != "n/a" + else "", + ) + ) + device_index = click.prompt( + "Please choose a port (number in the list above)", + type=click.Choice([str(i + 1) for i, _ in enumerate(devices)]), + ) + device = devices[int(device_index) - 1] + + self.start_remote_monitor(device[0], device[1]["port"]) + + return None + + def start_remote_monitor(self, agent, port): + options = {"port": port} + for key in ("baud", "parity", "rtscts", "xonxoff", "rts", "dtr"): + options[key] = self.cmd_options[key] + + click.echo( + "Starting Serial Monitor on {host}:{port}".format( + host=agent, port=options["port"] + ) + ) + d = self.agentpool.callRemote("cmd", [agent], "device.monitor", options) + d.addCallback(self.cb_async_result) + d.addErrback(self.cb_global_error) + + def cb_async_result(self, result): + if len(result) != 1: + raise pb.Error("Invalid response from Remote Cloud") + success, value = result[0] + if not success: + raise pb.Error(value) + + reconnected = self._agent_id is not None + self._agent_id, self._ac_id = value + + if reconnected: + self.acread_data(force=True) + self.acwrite_data("", force=True) + return + + # start bridge + port = reactor.listenTCP(0, self._bridge_factory) + address = port.getHost() + self.log.debug("Serial Bridge is started on {address!r}", address=address) + if "sock" in self.cmd_options: + with open(os.path.join(self.cmd_options["sock"], "sock"), "w") as fp: + fp.write("socket://localhost:%d" % address.port) + + def client_terminal_stopped(self): + try: + d = self.agentpool.callRemote("acclose", self._agent_id, self._ac_id) + d.addCallback(lambda r: self.disconnect()) + d.addErrback(self.cb_global_error) + except (AttributeError, pb.DeadReferenceError): + self.disconnect(exit_code=1) + + def acread_data(self, force=False): + if force and self._d_acread: + self._d_acread.cancel() + self._d_acread = None + + if ( + self._d_acread and not self._d_acread.called + ) or not self._bridge_factory.has_clients(): + return + + try: + self._d_acread = self.agentpool.callRemote( + "acread", self._agent_id, self._ac_id + ) + self._d_acread.addCallback(self.cb_acread_result) + self._d_acread.addErrback(self.cb_global_error) + except (AttributeError, pb.DeadReferenceError): + self.disconnect(exit_code=1) + + def cb_acread_result(self, result): + if result is None: + self.disconnect(exit_code=1) + else: + self._bridge_factory.send_to_clients(result) + self.acread_data() + + def acwrite_data(self, data, force=False): + if force and self._d_acwrite: + self._d_acwrite.cancel() + self._d_acwrite = None + + self._acwrite_buffer += data + if len(self._acwrite_buffer) > self.MAX_BUFFER_SIZE: + self._acwrite_buffer = self._acwrite_buffer[-1 * self.MAX_BUFFER_SIZE :] + if (self._d_acwrite and not self._d_acwrite.called) or not self._acwrite_buffer: + return + + data = self._acwrite_buffer + self._acwrite_buffer = "" + try: + d = self.agentpool.callRemote("acwrite", self._agent_id, self._ac_id, data) + d.addCallback(self.cb_acwrite_result) + d.addErrback(self.cb_global_error) + except (AttributeError, pb.DeadReferenceError): + self.disconnect(exit_code=1) + + def cb_acwrite_result(self, result): + assert result > 0 + if self._acwrite_buffer: + self.acwrite_data("") diff --git a/platformio/commands/remote/client/run_or_test.py b/platformio/commands/remote/client/run_or_test.py new file mode 100644 index 00000000..c986ad0a --- /dev/null +++ b/platformio/commands/remote/client/run_or_test.py @@ -0,0 +1,272 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import hashlib +import json +import os +import zlib +from io import BytesIO + +from twisted.spread import pb # pylint: disable=import-error + +from platformio import util +from platformio.commands.remote.client.async_base import AsyncClientBase +from platformio.commands.remote.projectsync import PROJECT_SYNC_STAGE, ProjectSync +from platformio.compat import hashlib_encode_data +from platformio.project.config import ProjectConfig + + +class RunOrTestClient(AsyncClientBase): + + MAX_ARCHIVE_SIZE = 50 * 1024 * 1024 # 50Mb + UPLOAD_CHUNK_SIZE = 256 * 1024 # 256Kb + + PSYNC_SRC_EXTS = [ + "c", + "cpp", + "S", + "spp", + "SPP", + "sx", + "s", + "asm", + "ASM", + "h", + "hpp", + "ipp", + "ino", + "pde", + "json", + "properties", + ] + + PSYNC_SKIP_DIRS = (".git", ".svn", ".hg", "example", "examples", "test", "tests") + + def __init__(self, *args, **kwargs): + AsyncClientBase.__init__(self, *args, **kwargs) + self.project_id = self.generate_project_id(self.options["project_dir"]) + self.psync = ProjectSync(self.options["project_dir"]) + + def generate_project_id(self, path): + h = hashlib.sha1(hashlib_encode_data(self.id)) + h.update(hashlib_encode_data(path)) + return "%s-%s" % (os.path.basename(path), h.hexdigest()) + + def add_project_items(self, psync): + with util.cd(self.options["project_dir"]): + cfg = ProjectConfig.get_instance( + os.path.join(self.options["project_dir"], "platformio.ini") + ) + psync.add_item(cfg.path, "platformio.ini") + psync.add_item(cfg.get_optional_dir("shared"), "shared") + psync.add_item(cfg.get_optional_dir("boards"), "boards") + + if self.options["force_remote"]: + self._add_project_source_items(cfg, psync) + else: + self._add_project_binary_items(cfg, psync) + + if self.command == "test": + psync.add_item(cfg.get_optional_dir("test"), "test") + + def _add_project_source_items(self, cfg, psync): + psync.add_item(cfg.get_optional_dir("lib"), "lib") + psync.add_item( + cfg.get_optional_dir("include"), + "include", + cb_filter=self._cb_tarfile_filter, + ) + psync.add_item( + cfg.get_optional_dir("src"), "src", cb_filter=self._cb_tarfile_filter + ) + if set(["buildfs", "uploadfs", "uploadfsota"]) & set( + self.options.get("target", []) + ): + psync.add_item(cfg.get_optional_dir("data"), "data") + + @staticmethod + def _add_project_binary_items(cfg, psync): + build_dir = cfg.get_optional_dir("build") + for env_name in os.listdir(build_dir): + env_dir = os.path.join(build_dir, env_name) + if not os.path.isdir(env_dir): + continue + for fname in os.listdir(env_dir): + bin_file = os.path.join(env_dir, fname) + bin_exts = (".elf", ".bin", ".hex", ".eep", "program") + if os.path.isfile(bin_file) and fname.endswith(bin_exts): + psync.add_item( + bin_file, os.path.join(".pio", "build", env_name, fname) + ) + + def _cb_tarfile_filter(self, path): + if ( + os.path.isdir(path) + and os.path.basename(path).lower() in self.PSYNC_SKIP_DIRS + ): + return None + if os.path.isfile(path) and not self.is_file_with_exts( + path, self.PSYNC_SRC_EXTS + ): + return None + return path + + @staticmethod + def is_file_with_exts(path, exts): + if path.endswith(tuple(".%s" % e for e in exts)): + return True + return False + + def agent_pool_ready(self): + self.psync_init() + + def psync_init(self): + self.add_project_items(self.psync) + d = self.agentpool.callRemote( + "cmd", + self.agents, + "psync", + dict(id=self.project_id, items=[i[1] for i in self.psync.get_items()]), + ) + d.addCallback(self.cb_psync_init_result) + d.addErrback(self.cb_global_error) + + # build db index while wait for result from agent + self.psync.rebuild_dbindex() + + def cb_psync_init_result(self, result): + self._acs_total = len(result) + for (success, value) in result: + if not success: + raise pb.Error(value) + agent_id, ac_id = value + try: + d = self.agentpool.callRemote( + "acwrite", + agent_id, + ac_id, + dict(stage=PROJECT_SYNC_STAGE.DBINDEX.value), + ) + d.addCallback(self.cb_psync_dbindex_result, agent_id, ac_id) + d.addErrback(self.cb_global_error) + except (AttributeError, pb.DeadReferenceError): + self.disconnect(exit_code=1) + + def cb_psync_dbindex_result(self, result, agent_id, ac_id): + result = set(json.loads(zlib.decompress(result))) + dbindex = set(self.psync.get_dbindex()) + delete = list(result - dbindex) + delta = list(dbindex - result) + + self.log.debug( + "PSync: stats, total={total}, delete={delete}, delta={delta}", + total=len(dbindex), + delete=len(delete), + delta=len(delta), + ) + + if not delete and not delta: + return self.psync_finalize(agent_id, ac_id) + if not delete: + return self.psync_upload(agent_id, ac_id, delta) + + try: + d = self.agentpool.callRemote( + "acwrite", + agent_id, + ac_id, + dict( + stage=PROJECT_SYNC_STAGE.DELETE.value, + dbindex=zlib.compress(json.dumps(delete).encode()), + ), + ) + d.addCallback(self.cb_psync_delete_result, agent_id, ac_id, delta) + d.addErrback(self.cb_global_error) + except (AttributeError, pb.DeadReferenceError): + self.disconnect(exit_code=1) + + return None + + def cb_psync_delete_result(self, result, agent_id, ac_id, dbindex): + assert result + self.psync_upload(agent_id, ac_id, dbindex) + + def psync_upload(self, agent_id, ac_id, dbindex): + assert dbindex + fileobj = BytesIO() + compressed = self.psync.compress_items(fileobj, dbindex, self.MAX_ARCHIVE_SIZE) + fileobj.seek(0) + self.log.debug( + "PSync: upload project, size={size}", size=len(fileobj.getvalue()) + ) + self.psync_upload_chunk( + agent_id, ac_id, list(set(dbindex) - set(compressed)), fileobj + ) + + def psync_upload_chunk(self, agent_id, ac_id, dbindex, fileobj): + offset = fileobj.tell() + total = fileobj.seek(0, os.SEEK_END) + # unwind + fileobj.seek(offset) + chunk = fileobj.read(self.UPLOAD_CHUNK_SIZE) + assert chunk + try: + d = self.agentpool.callRemote( + "acwrite", + agent_id, + ac_id, + dict( + stage=PROJECT_SYNC_STAGE.UPLOAD.value, + chunk=chunk, + length=len(chunk), + total=total, + ), + ) + d.addCallback( + self.cb_psync_upload_chunk_result, agent_id, ac_id, dbindex, fileobj + ) + d.addErrback(self.cb_global_error) + except (AttributeError, pb.DeadReferenceError): + self.disconnect(exit_code=1) + + def cb_psync_upload_chunk_result( # pylint: disable=too-many-arguments + self, result, agent_id, ac_id, dbindex, fileobj + ): + result = PROJECT_SYNC_STAGE.lookupByValue(result) + self.log.debug("PSync: upload chunk result {r}", r=str(result)) + assert result & (PROJECT_SYNC_STAGE.UPLOAD | PROJECT_SYNC_STAGE.EXTRACTED) + if result is PROJECT_SYNC_STAGE.EXTRACTED: + if dbindex: + self.psync_upload(agent_id, ac_id, dbindex) + else: + self.psync_finalize(agent_id, ac_id) + else: + self.psync_upload_chunk(agent_id, ac_id, dbindex, fileobj) + + def psync_finalize(self, agent_id, ac_id): + try: + d = self.agentpool.callRemote("acclose", agent_id, ac_id) + d.addCallback(self.cb_psync_completed_result, agent_id) + d.addErrback(self.cb_global_error) + except (AttributeError, pb.DeadReferenceError): + self.disconnect(exit_code=1) + + def cb_psync_completed_result(self, result, agent_id): + assert PROJECT_SYNC_STAGE.lookupByValue(result) + options = self.options.copy() + del options["project_dir"] + options["project_id"] = self.project_id + d = self.agentpool.callRemote("cmd", [agent_id], self.command, options) + d.addCallback(self.cb_async_result) + d.addErrback(self.cb_global_error) diff --git a/platformio/commands/remote/client/update_core.py b/platformio/commands/remote/client/update_core.py new file mode 100644 index 00000000..49e4488c --- /dev/null +++ b/platformio/commands/remote/client/update_core.py @@ -0,0 +1,22 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from platformio.commands.remote.client.async_base import AsyncClientBase + + +class UpdateCoreClient(AsyncClientBase): + def agent_pool_ready(self): + d = self.agentpool.callRemote("cmd", self.agents, self.command, self.options) + d.addCallback(self.cb_async_result) + d.addErrback(self.cb_global_error) diff --git a/platformio/commands/remote.py b/platformio/commands/remote/command.py similarity index 57% rename from platformio/commands/remote.py rename to platformio/commands/remote/command.py index ca296b69..869890a8 100644 --- a/platformio/commands/remote.py +++ b/platformio/commands/remote/command.py @@ -12,30 +12,42 @@ # See the License for the specific language governing permissions and # limitations under the License. +# pylint: disable=too-many-arguments, import-outside-toplevel +# pylint: disable=inconsistent-return-statements + import os -import sys +import subprocess import threading from tempfile import mkdtemp from time import sleep import click -from platformio import exception, fs +from platformio import exception, fs, proc from platformio.commands.device import helpers as device_helpers from platformio.commands.device.command import device_monitor as cmd_device_monitor -from platformio.managers.core import pioplus_call +from platformio.commands.run.command import cli as cmd_run +from platformio.commands.test.command import cli as cmd_test +from platformio.compat import PY2 +from platformio.managers.core import inject_contrib_pysite from platformio.project.exception import NotPlatformIOProjectError -# pylint: disable=unused-argument - @click.group("remote", short_help="PIO Remote") @click.option("-a", "--agent", multiple=True) -def cli(**kwargs): - pass +@click.pass_context +def cli(ctx, agent): + if PY2: + raise exception.UserSideException( + "PIO Remote requires Python 3.5 or above. \nPlease install the latest " + "Python 3 and reinstall PlatformIO Core using installation script:\n" + "https://docs.platformio.org/page/core/installation.html" + ) + ctx.obj = agent + inject_contrib_pysite() -@cli.group("agent", short_help="Start new agent or list active") +@cli.group("agent", short_help="Start a new agent or list active") def remote_agent(): pass @@ -49,18 +61,17 @@ def remote_agent(): envvar="PLATFORMIO_REMOTE_AGENT_DIR", type=click.Path(file_okay=False, dir_okay=True, writable=True, resolve_path=True), ) -def remote_agent_start(**kwargs): - pioplus_call(sys.argv[1:]) +def remote_agent_start(name, share, working_dir): + from platformio.commands.remote.client.agent_service import RemoteAgentService - -@remote_agent.command("reload", short_help="Reload agents") -def remote_agent_reload(): - pioplus_call(sys.argv[1:]) + RemoteAgentService(name, share, working_dir).connect() @remote_agent.command("list", short_help="List active agents") def remote_agent_list(): - pioplus_call(sys.argv[1:]) + from platformio.commands.remote.client.agent_list import AgentListClient + + AgentListClient().connect() @cli.command("update", short_help="Update installed Platforms, Packages and Libraries") @@ -73,8 +84,11 @@ def remote_agent_list(): @click.option( "--dry-run", is_flag=True, help="Do not update, only check for the new versions" ) -def remote_update(only_check, dry_run): - pioplus_call(sys.argv[1:]) +@click.pass_obj +def remote_update(agents, only_check, dry_run): + from platformio.commands.remote.client.update_core import UpdateCoreClient + + UpdateCoreClient("update", agents, dict(only_check=only_check or dry_run)).connect() @cli.command("run", short_help="Process project environments remotely") @@ -93,8 +107,65 @@ def remote_update(only_check, dry_run): @click.option("-r", "--force-remote", is_flag=True) @click.option("-s", "--silent", is_flag=True) @click.option("-v", "--verbose", is_flag=True) -def remote_run(**kwargs): - pioplus_call(sys.argv[1:]) +@click.pass_obj +@click.pass_context +def remote_run( + ctx, + agents, + environment, + target, + upload_port, + project_dir, + disable_auto_clean, + force_remote, + silent, + verbose, +): + + from platformio.commands.remote.client.run_or_test import RunOrTestClient + + cr = RunOrTestClient( + "run", + agents, + dict( + environment=environment, + target=target, + upload_port=upload_port, + project_dir=project_dir, + disable_auto_clean=disable_auto_clean, + force_remote=force_remote, + silent=silent, + verbose=verbose, + ), + ) + if force_remote: + return cr.connect() + + click.secho("Building project locally", bold=True) + local_targets = [] + if "clean" in target: + local_targets = ["clean"] + elif set(["buildfs", "uploadfs", "uploadfsota"]) & set(target): + local_targets = ["buildfs"] + else: + local_targets = ["checkprogsize", "buildprog"] + ctx.invoke( + cmd_run, + environment=environment, + target=local_targets, + project_dir=project_dir, + # disable_auto_clean=True, + silent=silent, + verbose=verbose, + ) + + if any(["upload" in t for t in target] + ["program" in target]): + click.secho("Uploading firmware remotely", bold=True) + cr.options["target"] += ("nobuild",) + cr.options["disable_auto_clean"] = True + cr.connect() + + return True @cli.command("test", short_help="Remote Unit Testing") @@ -114,8 +185,59 @@ def remote_run(**kwargs): @click.option("--without-building", is_flag=True) @click.option("--without-uploading", is_flag=True) @click.option("--verbose", "-v", is_flag=True) -def remote_test(**kwargs): - pioplus_call(sys.argv[1:]) +@click.pass_obj +@click.pass_context +def remote_test( + ctx, + agents, + environment, + ignore, + upload_port, + test_port, + project_dir, + force_remote, + without_building, + without_uploading, + verbose, +): + + from platformio.commands.remote.client.run_or_test import RunOrTestClient + + cr = RunOrTestClient( + "test", + agents, + dict( + environment=environment, + ignore=ignore, + upload_port=upload_port, + test_port=test_port, + project_dir=project_dir, + force_remote=force_remote, + without_building=without_building, + without_uploading=without_uploading, + verbose=verbose, + ), + ) + if force_remote: + return cr.connect() + + click.secho("Building project locally", bold=True) + + ctx.invoke( + cmd_test, + environment=environment, + ignore=ignore, + project_dir=project_dir, + without_uploading=True, + without_testing=True, + verbose=verbose, + ) + + click.secho("Testing project remotely", bold=True) + cr.options["without_building"] = True + cr.connect() + + return True @cli.group("device", short_help="Monitor remote device or list existing") @@ -125,8 +247,11 @@ def remote_device(): @remote_device.command("list", short_help="List remote devices") @click.option("--json-output", is_flag=True) -def device_list(json_output): - pioplus_call(sys.argv[1:]) +@click.pass_obj +def device_list(agents, json_output): + from platformio.commands.remote.client.device_list import DeviceListClient + + DeviceListClient(agents, json_output).connect() @remote_device.command("monitor", short_help="Monitor remote device") @@ -193,8 +318,20 @@ def device_list(json_output): "--environment", help="Load configuration from `platformio.ini` and specified environment", ) +@click.option( + "--sock", + type=click.Path( + exists=True, file_okay=False, dir_okay=True, writable=True, resolve_path=True + ), +) +@click.pass_obj @click.pass_context -def device_monitor(ctx, **kwargs): +def device_monitor(ctx, agents, **kwargs): + from platformio.commands.remote.client.device_monitor import DeviceMonitorClient + + if kwargs["sock"]: + return DeviceMonitorClient(agents, **kwargs).connect() + project_options = {} try: with fs.cd(kwargs["project_dir"]): @@ -209,12 +346,9 @@ def device_monitor(ctx, **kwargs): pioplus_argv = ["remote", "device", "monitor"] pioplus_argv.extend(device_helpers.options_to_argv(kwargs, project_options)) pioplus_argv.extend(["--sock", sock_dir]) - try: - pioplus_call(pioplus_argv) - except exception.ReturnErrorCode: - pass + subprocess.call([proc.where_is_program("platformio")] + pioplus_argv) - sock_dir = mkdtemp(suffix="pioplus") + sock_dir = mkdtemp(suffix="pio") sock_file = os.path.join(sock_dir, "sock") try: t = threading.Thread(target=_tx_target, args=(sock_dir,)) @@ -229,3 +363,5 @@ def device_monitor(ctx, **kwargs): t.join(2) finally: fs.rmtree(sock_dir) + + return True diff --git a/platformio/commands/remote/factory/__init__.py b/platformio/commands/remote/factory/__init__.py new file mode 100644 index 00000000..b0514903 --- /dev/null +++ b/platformio/commands/remote/factory/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/commands/remote/factory/client.py b/platformio/commands/remote/factory/client.py new file mode 100644 index 00000000..202c7da6 --- /dev/null +++ b/platformio/commands/remote/factory/client.py @@ -0,0 +1,73 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.cred import credentials # pylint: disable=import-error +from twisted.internet import protocol, reactor # pylint: disable=import-error +from twisted.spread import pb # pylint: disable=import-error + +from platformio.app import get_host_id +from platformio.commands.account.client import AccountClient + + +class RemoteClientFactory(pb.PBClientFactory, protocol.ReconnectingClientFactory): + def clientConnectionMade(self, broker): + if self.sslContextFactory and not self.sslContextFactory.certificate_verified: + self.remote_client.log.error( + "A remote cloud could not prove that its security certificate is " + "from {host}. This may cause a misconfiguration or an attacker " + "intercepting your connection.", + host=self.sslContextFactory.host, + ) + return self.remote_client.disconnect() + pb.PBClientFactory.clientConnectionMade(self, broker) + protocol.ReconnectingClientFactory.resetDelay(self) + self.remote_client.log.info("Successfully connected") + self.remote_client.log.info("Authenticating") + + d = self.login( + credentials.UsernamePassword( + AccountClient().fetch_authentication_token().encode(), + get_host_id().encode(), + ), + client=self.remote_client, + ) + d.addCallback(self.remote_client.cb_client_authorization_made) + d.addErrback(self.remote_client.cb_client_authorization_failed) + return d + + def clientConnectionFailed(self, connector, reason): + self.remote_client.log.warn( + "Could not connect to PIO Remote Cloud. Reconnecting..." + ) + self.remote_client.cb_disconnected(reason) + protocol.ReconnectingClientFactory.clientConnectionFailed( + self, connector, reason + ) + + def clientConnectionLost( # pylint: disable=arguments-differ + self, connector, unused_reason + ): + if not reactor.running: + self.remote_client.log.info("Successfully disconnected") + return + self.remote_client.log.warn( + "Connection is lost to PIO Remote Cloud. Reconnecting" + ) + pb.PBClientFactory.clientConnectionLost( + self, connector, unused_reason, reconnecting=1 + ) + self.remote_client.cb_disconnected(unused_reason) + protocol.ReconnectingClientFactory.clientConnectionLost( + self, connector, unused_reason + ) diff --git a/platformio/commands/remote/factory/ssl.py b/platformio/commands/remote/factory/ssl.py new file mode 100644 index 00000000..a4233a69 --- /dev/null +++ b/platformio/commands/remote/factory/ssl.py @@ -0,0 +1,41 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import certifi +from OpenSSL import SSL # pylint: disable=import-error +from twisted.internet import ssl # pylint: disable=import-error + + +class SSLContextFactory(ssl.ClientContextFactory): + def __init__(self, host): + self.host = host + self.certificate_verified = False + + def getContext(self): + ctx = super(SSLContextFactory, self).getContext() + ctx.set_verify( + SSL.VERIFY_PEER | SSL.VERIFY_FAIL_IF_NO_PEER_CERT, self.verifyHostname + ) + ctx.load_verify_locations(certifi.where()) + return ctx + + def verifyHostname( # pylint: disable=unused-argument,too-many-arguments + self, connection, x509, errno, depth, status + ): + cn = x509.get_subject().commonName + if cn.startswith("*"): + cn = cn[1:] + if self.host.endswith(cn): + self.certificate_verified = True + return status diff --git a/platformio/commands/remote/projectsync.py b/platformio/commands/remote/projectsync.py new file mode 100644 index 00000000..867922bd --- /dev/null +++ b/platformio/commands/remote/projectsync.py @@ -0,0 +1,117 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import tarfile +from binascii import crc32 +from os.path import getmtime, getsize, isdir, isfile, join + +from twisted.python import constants # pylint: disable=import-error + +from platformio.compat import hashlib_encode_data + + +class PROJECT_SYNC_STAGE(constants.Flags): + INIT = constants.FlagConstant() + DBINDEX = constants.FlagConstant() + DELETE = constants.FlagConstant() + UPLOAD = constants.FlagConstant() + EXTRACTED = constants.FlagConstant() + COMPLETED = constants.FlagConstant() + + +class ProjectSync(object): + def __init__(self, path): + self.path = path + if not isdir(self.path): + os.makedirs(self.path) + self.items = [] + self._db = {} + + def add_item(self, path, relpath, cb_filter=None): + self.items.append((path, relpath, cb_filter)) + + def get_items(self): + return self.items + + def rebuild_dbindex(self): + self._db = {} + for (path, relpath, cb_filter) in self.items: + if cb_filter and not cb_filter(path): + continue + self._insert_to_db(path, relpath) + if not isdir(path): + continue + for (root, _, files) in os.walk(path, followlinks=True): + for name in files: + self._insert_to_db( + join(root, name), join(relpath, root[len(path) + 1 :], name) + ) + + def _insert_to_db(self, path, relpath): + if not isfile(path): + return + index_hash = "%s-%s-%s" % (relpath, getmtime(path), getsize(path)) + index = crc32(hashlib_encode_data(index_hash)) + self._db[index] = (path, relpath) + + def get_dbindex(self): + return list(self._db.keys()) + + def delete_dbindex(self, dbindex): + for index in dbindex: + if index not in self._db: + continue + path = self._db[index][0] + if isfile(path): + os.remove(path) + del self._db[index] + self.delete_empty_folders() + return True + + def delete_empty_folders(self): + deleted = False + for item in self.items: + if not isdir(item[0]): + continue + for root, dirs, files in os.walk(item[0]): + if not dirs and not files and root != item[0]: + deleted = True + os.rmdir(root) + if deleted: + return self.delete_empty_folders() + + return True + + def compress_items(self, fileobj, dbindex, max_size): + compressed = [] + total_size = 0 + tar_opts = dict(fileobj=fileobj, mode="w:gz", bufsize=0, dereference=True) + with tarfile.open(**tar_opts) as tgz: + for index in dbindex: + compressed.append(index) + if index not in self._db: + continue + path, relpath = self._db[index] + tgz.add(path, relpath) + total_size += getsize(path) + if total_size > max_size: + break + return compressed + + def decompress_items(self, fileobj): + fileobj.seek(0) + with tarfile.open(fileobj=fileobj, mode="r:gz") as tgz: + tgz.extractall(self.path) + return True diff --git a/platformio/managers/core.py b/platformio/managers/core.py index 64d48085..55af5fe1 100644 --- a/platformio/managers/core.py +++ b/platformio/managers/core.py @@ -16,12 +16,11 @@ import json import os import subprocess import sys -from os.path import dirname, join -from platformio import __version__, exception, fs, util -from platformio.compat import PY2, WINDOWS +from platformio import exception, util +from platformio.compat import PY2 from platformio.managers.package import PackageManager -from platformio.proc import copy_pythonpath_to_osenv, get_pythonexe_path +from platformio.proc import get_pythonexe_path from platformio.project.config import ProjectConfig CORE_PACKAGES = { @@ -185,47 +184,3 @@ def get_contrib_pysite_deps(): ) result[0] = twisted_wheel return result - - -def pioplus_call(args, **kwargs): - if WINDOWS and sys.version_info < (2, 7, 6): - raise exception.PlatformioException( - "PlatformIO Remote v%s does not run under Python version %s.\n" - "Minimum supported version is 2.7.6, please upgrade Python.\n" - "Python 3 is not yet supported.\n" % (__version__, sys.version) - ) - - pioplus_path = join(get_core_package_dir("tool-pioplus"), "pioplus") - pythonexe_path = get_pythonexe_path() - os.environ["PYTHONEXEPATH"] = pythonexe_path - os.environ["PYTHONPYSITEDIR"] = get_core_package_dir("contrib-pysite") - os.environ["PIOCOREPYSITEDIR"] = dirname(fs.get_source_dir() or "") - if dirname(pythonexe_path) not in os.environ["PATH"].split(os.pathsep): - os.environ["PATH"] = (os.pathsep).join( - [dirname(pythonexe_path), os.environ["PATH"]] - ) - copy_pythonpath_to_osenv() - code = subprocess.call([pioplus_path] + args, **kwargs) - - # handle remote update request - if code == 13: - count_attr = "_update_count" - try: - count_value = getattr(pioplus_call, count_attr) - except AttributeError: - count_value = 0 - setattr(pioplus_call, count_attr, 1) - count_value += 1 - setattr(pioplus_call, count_attr, count_value) - if count_value < PIOPLUS_AUTO_UPDATES_MAX: - update_core_packages() - return pioplus_call(args, **kwargs) - - # handle reload request - elif code == 14: - return pioplus_call(args, **kwargs) - - if code != 0: - raise exception.ReturnErrorCode(1) - - return True