Move "remote" command to the root

platformio/remote/client/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

platformio/remote/client/agent_list.py

@@ -0,0 +1,38 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from datetime import datetime
+
+import click
+
+from platformio.remote.client.base import RemoteClientBase
+
+
+class AgentListClient(RemoteClientBase):
+    def agent_pool_ready(self):
+        d = self.agentpool.callRemote("list", True)
+        d.addCallback(self._cbResult)
+        d.addErrback(self.cb_global_error)
+
+    def _cbResult(self, result):
+        for item in result:
+            click.secho(item["name"], fg="cyan")
+            click.echo("-" * len(item["name"]))
+            click.echo("ID: %s" % item["id"])
+            click.echo(
+                "Started: %s"
+                % datetime.fromtimestamp(item["started"]).strftime("%Y-%m-%d %H:%M:%S")
+            )
+            click.echo("")
+        self.disconnect()

platformio/remote/client/agent_service.py

@@ -0,0 +1,228 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+from twisted.logger import LogLevel  # pylint: disable=import-error
+from twisted.spread import pb  # pylint: disable=import-error
+
+from platformio import proc
+from platformio.device.list import list_serial_ports
+from platformio.project.config import ProjectConfig
+from platformio.project.exception import NotPlatformIOProjectError
+from platformio.remote.ac.process import ProcessAsyncCmd
+from platformio.remote.ac.psync import ProjectSyncAsyncCmd
+from platformio.remote.ac.serial import SerialPortAsyncCmd
+from platformio.remote.client.base import RemoteClientBase
+
+
+class RemoteAgentService(RemoteClientBase):
+    def __init__(self, name, share, working_dir=None):
+        RemoteClientBase.__init__(self)
+        self.log_level = LogLevel.info
+        self.working_dir = working_dir or os.path.join(
+            ProjectConfig.get_instance().get("platformio", "core_dir"), "remote"
+        )
+        if not os.path.isdir(self.working_dir):
+            os.makedirs(self.working_dir)
+        if name:
+            self.name = str(name)[:50]
+        self.join_options.update(
+            {"agent": True, "share": [s.lower().strip()[:50] for s in share]}
+        )
+
+        self._acs = {}
+
+    def agent_pool_ready(self):
+        pass
+
+    def cb_disconnected(self, reason):
+        for ac in self._acs.values():
+            ac.ac_close()
+        RemoteClientBase.cb_disconnected(self, reason)
+
+    def remote_acread(self, ac_id):
+        self.log.debug("Async Read: {id}", id=ac_id)
+        if ac_id not in self._acs:
+            raise pb.Error("Invalid Async Identifier")
+        return self._acs[ac_id].ac_read()
+
+    def remote_acwrite(self, ac_id, data):
+        self.log.debug("Async Write: {id}", id=ac_id)
+        if ac_id not in self._acs:
+            raise pb.Error("Invalid Async Identifier")
+        return self._acs[ac_id].ac_write(data)
+
+    def remote_acclose(self, ac_id):
+        self.log.debug("Async Close: {id}", id=ac_id)
+        if ac_id not in self._acs:
+            raise pb.Error("Invalid Async Identifier")
+        return_code = self._acs[ac_id].ac_close()
+        del self._acs[ac_id]
+        return return_code
+
+    def remote_cmd(self, cmd, options):
+        self.log.info("Remote command received: {cmd}", cmd=cmd)
+        self.log.debug("Command options: {options!r}", options=options)
+        callback = "_process_cmd_%s" % cmd.replace(".", "_")
+        return getattr(self, callback)(options)
+
+    def _defer_async_cmd(self, ac, pass_agent_name=True):
+        self._acs[ac.id] = ac
+        if pass_agent_name:
+            return (self.id, ac.id, self.name)
+        return (self.id, ac.id)
+
+    def _process_cmd_device_list(self, _):
+        return (self.name, list_serial_ports())
+
+    def _process_cmd_device_monitor(self, options):
+        if not options["port"]:
+            for item in list_serial_ports():
+                if "VID:PID" in item["hwid"]:
+                    options["port"] = item["port"]
+                    break
+
+        # terminate opened monitors
+        if options["port"]:
+            for ac in list(self._acs.values()):
+                if (
+                    isinstance(ac, SerialPortAsyncCmd)
+                    and ac.options["port"] == options["port"]
+                ):
+                    self.log.info(
+                        "Terminate previously opened monitor at {port}",
+                        port=options["port"],
+                    )
+                    ac.ac_close()
+                    del self._acs[ac.id]
+
+        if not options["port"]:
+            raise pb.Error("Please specify serial port using `--port` option")
+        self.log.info("Starting serial monitor at {port}", port=options["port"])
+
+        return self._defer_async_cmd(SerialPortAsyncCmd(options), pass_agent_name=False)
+
+    def _process_cmd_psync(self, options):
+        for ac in list(self._acs.values()):
+            if (
+                isinstance(ac, ProjectSyncAsyncCmd)
+                and ac.options["id"] == options["id"]
+            ):
+                self.log.info("Terminate previous Project Sync process")
+                ac.ac_close()
+                del self._acs[ac.id]
+
+        options["agent_working_dir"] = self.working_dir
+        return self._defer_async_cmd(
+            ProjectSyncAsyncCmd(options), pass_agent_name=False
+        )
+
+    def _process_cmd_run(self, options):
+        return self._process_cmd_run_or_test("run", options)
+
+    def _process_cmd_test(self, options):
+        return self._process_cmd_run_or_test("test", options)
+
+    def _process_cmd_run_or_test(  # pylint: disable=too-many-locals,too-many-branches,too-many-statements
+        self, command, options
+    ):
+        assert options and "project_id" in options
+        project_dir = os.path.join(self.working_dir, "projects", options["project_id"])
+        origin_pio_ini = os.path.join(project_dir, "platformio.ini")
+        back_pio_ini = os.path.join(project_dir, "platformio.ini.bak")
+
+        # remove insecure project options
+        try:
+            conf = ProjectConfig(origin_pio_ini)
+            if os.path.isfile(back_pio_ini):
+                os.remove(back_pio_ini)
+            os.rename(origin_pio_ini, back_pio_ini)
+            # cleanup
+            if conf.has_section("platformio"):
+                for opt in conf.options("platformio"):
+                    if opt.endswith("_dir"):
+                        conf.remove_option("platformio", opt)
+            else:
+                conf.add_section("platformio")
+            conf.set("platformio", "build_dir", ".pio/build")
+            conf.save(origin_pio_ini)
+
+            # restore A/M times
+            os.utime(
+                origin_pio_ini,
+                (os.path.getatime(back_pio_ini), os.path.getmtime(back_pio_ini)),
+            )
+        except NotPlatformIOProjectError as e:
+            raise pb.Error(str(e))
+
+        cmd_args = ["platformio", "--force", command, "-d", project_dir]
+        for env in options.get("environment", []):
+            cmd_args.extend(["-e", env])
+        for target in options.get("target", []):
+            cmd_args.extend(["-t", target])
+        for filter_ in options.get("filter", []):
+            cmd_args.extend(["-f", filter_])
+        for ignore in options.get("ignore", []):
+            cmd_args.extend(["-i", ignore])
+        if options.get("upload_port", False):
+            cmd_args.extend(["--upload-port", options.get("upload_port")])
+        if options.get("test_port", False):
+            cmd_args.extend(["--test-port", options.get("test_port")])
+        if options.get("disable_auto_clean", False):
+            cmd_args.append("--disable-auto-clean")
+        if options.get("without_building", False):
+            cmd_args.append("--without-building")
+        if options.get("without_uploading", False):
+            cmd_args.append("--without-uploading")
+        if options.get("silent", False):
+            cmd_args.append("-s")
+        if options.get("verbose", False):
+            cmd_args.append("-v")
+
+        paused_acs = []
+        for ac in self._acs.values():
+            if not isinstance(ac, SerialPortAsyncCmd):
+                continue
+            self.log.info("Pause active monitor at {port}", port=ac.options["port"])
+            ac.pause()
+            paused_acs.append(ac)
+
+        def _cb_on_end():
+            if os.path.isfile(back_pio_ini):
+                if os.path.isfile(origin_pio_ini):
+                    os.remove(origin_pio_ini)
+                os.rename(back_pio_ini, origin_pio_ini)
+            for ac in paused_acs:
+                ac.unpause()
+                self.log.info(
+                    "Unpause active monitor at {port}", port=ac.options["port"]
+                )
+
+        return self._defer_async_cmd(
+            ProcessAsyncCmd(
+                {"executable": proc.where_is_program("platformio"), "args": cmd_args},
+                on_end_callback=_cb_on_end,
+            )
+        )
+
+    def _process_cmd_update(self, options):
+        cmd_args = ["platformio", "--force", "update"]
+        if options.get("only_check"):
+            cmd_args.append("--only-check")
+        return self._defer_async_cmd(
+            ProcessAsyncCmd(
+                {"executable": proc.where_is_program("platformio"), "args": cmd_args}
+            )
+        )

platformio/remote/client/async_base.py

@@ -0,0 +1,65 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+from twisted.spread import pb  # pylint: disable=import-error
+
+from platformio.remote.client.base import RemoteClientBase
+
+
+class AsyncClientBase(RemoteClientBase):
+    def __init__(self, command, agents, options):
+        RemoteClientBase.__init__(self)
+        self.command = command
+        self.agents = agents
+        self.options = options
+
+        self._acs_total = 0
+        self._acs_ended = 0
+
+    def agent_pool_ready(self):
+        pass
+
+    def cb_async_result(self, result):
+        if self._acs_total == 0:
+            self._acs_total = len(result)
+        for (success, value) in result:
+            if not success:
+                raise pb.Error(value)
+            self.acread_data(*value)
+
+    def acread_data(self, agent_id, ac_id, agent_name=None):
+        d = self.agentpool.callRemote("acread", agent_id, ac_id)
+        d.addCallback(self.cb_acread_result, agent_id, ac_id, agent_name)
+        d.addErrback(self.cb_global_error)
+
+    def cb_acread_result(self, result, agent_id, ac_id, agent_name):
+        if result is None:
+            self.acclose(agent_id, ac_id)
+        else:
+            if self._acs_total > 1 and agent_name:
+                click.echo("[%s] " % agent_name, nl=False)
+            click.echo(result, nl=False)
+            self.acread_data(agent_id, ac_id, agent_name)
+
+    def acclose(self, agent_id, ac_id):
+        d = self.agentpool.callRemote("acclose", agent_id, ac_id)
+        d.addCallback(self.cb_acclose_result)
+        d.addErrback(self.cb_global_error)
+
+    def cb_acclose_result(self, exit_code):
+        self._acs_ended += 1
+        if self._acs_ended != self._acs_total:
+            return
+        self.disconnect(exit_code)

platformio/remote/client/base.py

@@ -0,0 +1,193 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from datetime import datetime
+from time import time
+
+import click
+from twisted.internet import defer, endpoints, reactor  # pylint: disable=import-error
+from twisted.logger import ILogObserver  # pylint: disable=import-error
+from twisted.logger import Logger  # pylint: disable=import-error
+from twisted.logger import LogLevel  # pylint: disable=import-error
+from twisted.logger import formatEvent  # pylint: disable=import-error
+from twisted.python import failure  # pylint: disable=import-error
+from twisted.spread import pb  # pylint: disable=import-error
+from zope.interface import provider  # pylint: disable=import-error
+
+from platformio import __pioremote_endpoint__, __version__, app, exception, maintenance
+from platformio.remote.factory.client import RemoteClientFactory
+from platformio.remote.factory.ssl import SSLContextFactory
+
+
+class RemoteClientBase(  # pylint: disable=too-many-instance-attributes
+    pb.Referenceable
+):
+
+    PING_DELAY = 60
+    PING_MAX_FAILURES = 3
+    DEBUG = False
+
+    def __init__(self):
+        self.log_level = LogLevel.warn
+        self.log = Logger(namespace="remote", observer=self._log_observer)
+        self.id = app.get_host_id()
+        self.name = app.get_host_name()
+        self.join_options = {"corever": __version__}
+        self.perspective = None
+        self.agentpool = None
+
+        self._ping_id = 0
+        self._ping_caller = None
+        self._ping_counter = 0
+        self._reactor_stopped = False
+        self._exit_code = 0
+
+    @provider(ILogObserver)
+    def _log_observer(self, event):
+        if not self.DEBUG and (
+            event["log_namespace"] != self.log.namespace
+            or self.log_level > event["log_level"]
+        ):
+            return
+        msg = formatEvent(event)
+        click.echo(
+            "%s [%s] %s"
+            % (
+                datetime.fromtimestamp(event["log_time"]).strftime("%Y-%m-%d %H:%M:%S"),
+                event["log_level"].name,
+                msg,
+            )
+        )
+
+    def connect(self):
+        self.log.info("Name: {name}", name=self.name)
+        self.log.info("Connecting to PlatformIO Remote Development Cloud")
+
+        # pylint: disable=protected-access
+        proto, options = endpoints._parse(__pioremote_endpoint__)
+        proto = proto[0]
+
+        factory = RemoteClientFactory()
+        factory.remote_client = self
+        factory.sslContextFactory = None
+        if proto == "ssl":
+            factory.sslContextFactory = SSLContextFactory(options["host"])
+            reactor.connectSSL(
+                options["host"],
+                int(options["port"]),
+                factory,
+                factory.sslContextFactory,
+            )
+        elif proto == "tcp":
+            reactor.connectTCP(options["host"], int(options["port"]), factory)
+        else:
+            raise exception.PlatformioException("Unknown PIO Remote Cloud protocol")
+        reactor.run()
+
+        if self._exit_code != 0:
+            raise exception.ReturnErrorCode(self._exit_code)
+
+    def cb_client_authorization_failed(self, err):
+        msg = "Bad account credentials"
+        if err.check(pb.Error):
+            msg = err.getErrorMessage()
+        self.log.error(msg)
+        self.disconnect(exit_code=1)
+
+    def cb_client_authorization_made(self, perspective):
+        self.log.info("Successfully authorized")
+        self.perspective = perspective
+        d = perspective.callRemote("join", self.id, self.name, self.join_options)
+        d.addCallback(self._cb_client_join_made)
+        d.addErrback(self.cb_global_error)
+
+    def _cb_client_join_made(self, result):
+        code = result[0]
+        if code == 1:
+            self.agentpool = result[1]
+            self.agent_pool_ready()
+            self.restart_ping()
+        elif code == 2:
+            self.remote_service(*result[1:])
+
+    def remote_service(self, command, options):
+        if command == "disconnect":
+            self.log.error(
+                "PIO Remote Cloud disconnected: {msg}", msg=options.get("message")
+            )
+            self.disconnect()
+
+    def restart_ping(self, reset_counter=True):
+        # stop previous ping callers
+        self.stop_ping(reset_counter)
+        self._ping_caller = reactor.callLater(self.PING_DELAY, self._do_ping)
+
+    def _do_ping(self):
+        self._ping_counter += 1
+        self._ping_id = int(time())
+        d = self.perspective.callRemote("service", "ping", {"id": self._ping_id})
+        d.addCallback(self._cb_pong)
+        d.addErrback(self._cb_pong)
+
+    def stop_ping(self, reset_counter=True):
+        if reset_counter:
+            self._ping_counter = 0
+        if not self._ping_caller or not self._ping_caller.active():
+            return
+        self._ping_caller.cancel()
+        self._ping_caller = None
+
+    def _cb_pong(self, result):
+        if not isinstance(result, failure.Failure) and self._ping_id == result:
+            self.restart_ping()
+            return
+        if self._ping_counter >= self.PING_MAX_FAILURES:
+            self.stop_ping()
+            self.perspective.broker.transport.loseConnection()
+        else:
+            self.restart_ping(reset_counter=False)
+
+    def agent_pool_ready(self):
+        raise NotImplementedError
+
+    def disconnect(self, exit_code=None):
+        self.stop_ping()
+        if exit_code is not None:
+            self._exit_code = exit_code
+        if reactor.running and not self._reactor_stopped:
+            self._reactor_stopped = True
+            reactor.stop()
+
+    def cb_disconnected(self, _):
+        self.stop_ping()
+        self.perspective = None
+        self.agentpool = None
+
+    def cb_global_error(self, err):
+        if err.check(pb.PBConnectionLost, defer.CancelledError):
+            return
+
+        msg = err.getErrorMessage()
+        if err.check(pb.DeadReferenceError):
+            msg = "Remote Client has been terminated"
+        elif "PioAgentNotStartedError" in str(err.type):
+            msg = (
+                "Could not find active agents. Please start it before on "
+                "a remote machine using `pio remote agent start` command.\n"
+                "See http://docs.platformio.org/page/plus/pio-remote.html"
+            )
+        else:
+            maintenance.on_platformio_exception(Exception(err.type))
+        click.secho(msg, fg="red", err=True)
+        self.disconnect(exit_code=1)

platformio/remote/client/device_list.py

@@ -0,0 +1,54 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+
+import click
+
+from platformio.remote.client.base import RemoteClientBase
+
+
+class DeviceListClient(RemoteClientBase):
+    def __init__(self, agents, json_output):
+        RemoteClientBase.__init__(self)
+        self.agents = agents
+        self.json_output = json_output
+
+    def agent_pool_ready(self):
+        d = self.agentpool.callRemote("cmd", self.agents, "device.list")
+        d.addCallback(self._cbResult)
+        d.addErrback(self.cb_global_error)
+
+    def _cbResult(self, result):
+        data = {}
+        for (success, value) in result:
+            if not success:
+                click.secho(value, fg="red", err=True)
+                continue
+            (agent_name, devlist) = value
+            data[agent_name] = devlist
+
+        if self.json_output:
+            click.echo(json.dumps(data))
+        else:
+            for agent_name, devlist in data.items():
+                click.echo("Agent %s" % click.style(agent_name, fg="cyan", bold=True))
+                click.echo("=" * (6 + len(agent_name)))
+                for item in devlist:
+                    click.secho(item["port"], fg="cyan")
+                    click.echo("-" * len(item["port"]))
+                    click.echo("Hardware ID: %s" % item["hwid"])
+                    click.echo("Description: %s" % item["description"])
+                    click.echo("")
+        self.disconnect()

platformio/remote/client/device_monitor.py

@@ -0,0 +1,240 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+from fnmatch import fnmatch
+
+import click
+from twisted.internet import protocol, reactor, task  # pylint: disable=import-error
+from twisted.spread import pb  # pylint: disable=import-error
+
+from platformio.remote.client.base import RemoteClientBase
+
+
+class SMBridgeProtocol(protocol.Protocol):  # pylint: disable=no-init
+    def connectionMade(self):
+        self.factory.add_client(self)
+
+    def connectionLost(self, reason):  # pylint: disable=unused-argument
+        self.factory.remove_client(self)
+
+    def dataReceived(self, data):
+        self.factory.send_to_server(data)
+
+
+class SMBridgeFactory(protocol.ServerFactory):
+    def __init__(self, cdm):
+        self.cdm = cdm
+        self._clients = []
+
+    def buildProtocol(self, addr):  # pylint: disable=unused-argument
+        p = SMBridgeProtocol()
+        p.factory = self  # pylint: disable=attribute-defined-outside-init
+        return p
+
+    def add_client(self, client):
+        self.cdm.log.debug("SMBridge: Client connected")
+        self._clients.append(client)
+        self.cdm.acread_data()
+
+    def remove_client(self, client):
+        self.cdm.log.debug("SMBridge: Client disconnected")
+        self._clients.remove(client)
+        if not self._clients:
+            self.cdm.client_terminal_stopped()
+
+    def has_clients(self):
+        return len(self._clients)
+
+    def send_to_clients(self, data):
+        if not self._clients:
+            return None
+        for client in self._clients:
+            client.transport.write(data)
+        return len(data)
+
+    def send_to_server(self, data):
+        self.cdm.acwrite_data(data)
+
+
+class DeviceMonitorClient(  # pylint: disable=too-many-instance-attributes
+    RemoteClientBase
+):
+
+    MAX_BUFFER_SIZE = 1024 * 1024
+
+    def __init__(self, agents, **kwargs):
+        RemoteClientBase.__init__(self)
+        self.agents = agents
+        self.cmd_options = kwargs
+
+        self._bridge_factory = SMBridgeFactory(self)
+        self._agent_id = None
+        self._ac_id = None
+        self._d_acread = None
+        self._d_acwrite = None
+        self._acwrite_buffer = b""
+
+    def agent_pool_ready(self):
+        d = task.deferLater(
+            reactor, 1, self.agentpool.callRemote, "cmd", self.agents, "device.list"
+        )
+        d.addCallback(self._cb_device_list)
+        d.addErrback(self.cb_global_error)
+
+    def _cb_device_list(self, result):
+        devices = []
+        hwid_devindexes = []
+        for (success, value) in result:
+            if not success:
+                click.secho(value, fg="red", err=True)
+                continue
+            (agent_name, ports) = value
+            for item in ports:
+                if "VID:PID" in item["hwid"]:
+                    hwid_devindexes.append(len(devices))
+                devices.append((agent_name, item))
+
+        if len(result) == 1 and self.cmd_options["port"]:
+            if set(["*", "?", "[", "]"]) & set(self.cmd_options["port"]):
+                for agent, item in devices:
+                    if fnmatch(item["port"], self.cmd_options["port"]):
+                        return self.start_remote_monitor(agent, item["port"])
+            return self.start_remote_monitor(result[0][1][0], self.cmd_options["port"])
+
+        device = None
+        if len(hwid_devindexes) == 1:
+            device = devices[hwid_devindexes[0]]
+        else:
+            click.echo("Available ports:")
+            for i, device in enumerate(devices):
+                click.echo(
+                    "{index}. {host}{port} \t{description}".format(
+                        index=i + 1,
+                        host=device[0] + ":" if len(result) > 1 else "",
+                        port=device[1]["port"],
+                        description=device[1]["description"]
+                        if device[1]["description"] != "n/a"
+                        else "",
+                    )
+                )
+            device_index = click.prompt(
+                "Please choose a port (number in the list above)",
+                type=click.Choice([str(i + 1) for i, _ in enumerate(devices)]),
+            )
+            device = devices[int(device_index) - 1]
+
+        self.start_remote_monitor(device[0], device[1]["port"])
+
+        return None
+
+    def start_remote_monitor(self, agent, port):
+        options = {"port": port}
+        for key in ("baud", "parity", "rtscts", "xonxoff", "rts", "dtr"):
+            options[key] = self.cmd_options[key]
+
+        click.echo(
+            "Starting Serial Monitor on {host}:{port}".format(
+                host=agent, port=options["port"]
+            )
+        )
+        d = self.agentpool.callRemote("cmd", [agent], "device.monitor", options)
+        d.addCallback(self.cb_async_result)
+        d.addErrback(self.cb_global_error)
+
+    def cb_async_result(self, result):
+        if len(result) != 1:
+            raise pb.Error("Invalid response from Remote Cloud")
+        success, value = result[0]
+        if not success:
+            raise pb.Error(value)
+
+        reconnected = self._agent_id is not None
+        self._agent_id, self._ac_id = value
+
+        if reconnected:
+            self.acread_data(force=True)
+            self.acwrite_data("", force=True)
+            return
+
+        # start bridge
+        port = reactor.listenTCP(0, self._bridge_factory)
+        address = port.getHost()
+        self.log.debug("Serial Bridge is started on {address!r}", address=address)
+        if "sock" in self.cmd_options:
+            with open(
+                os.path.join(self.cmd_options["sock"], "sock"),
+                mode="w",
+                encoding="utf8",
+            ) as fp:
+                fp.write("socket://localhost:%d" % address.port)
+
+    def client_terminal_stopped(self):
+        try:
+            d = self.agentpool.callRemote("acclose", self._agent_id, self._ac_id)
+            d.addCallback(lambda r: self.disconnect())
+            d.addErrback(self.cb_global_error)
+        except (AttributeError, pb.DeadReferenceError):
+            self.disconnect(exit_code=1)
+
+    def acread_data(self, force=False):
+        if force and self._d_acread:
+            self._d_acread.cancel()
+            self._d_acread = None
+
+        if (
+            self._d_acread and not self._d_acread.called
+        ) or not self._bridge_factory.has_clients():
+            return
+
+        try:
+            self._d_acread = self.agentpool.callRemote(
+                "acread", self._agent_id, self._ac_id
+            )
+            self._d_acread.addCallback(self.cb_acread_result)
+            self._d_acread.addErrback(self.cb_global_error)
+        except (AttributeError, pb.DeadReferenceError):
+            self.disconnect(exit_code=1)
+
+    def cb_acread_result(self, result):
+        if result is None:
+            self.disconnect(exit_code=1)
+        else:
+            self._bridge_factory.send_to_clients(result)
+            self.acread_data()
+
+    def acwrite_data(self, data, force=False):
+        if force and self._d_acwrite:
+            self._d_acwrite.cancel()
+            self._d_acwrite = None
+
+        self._acwrite_buffer += data
+        if len(self._acwrite_buffer) > self.MAX_BUFFER_SIZE:
+            self._acwrite_buffer = self._acwrite_buffer[-1 * self.MAX_BUFFER_SIZE :]
+        if (self._d_acwrite and not self._d_acwrite.called) or not self._acwrite_buffer:
+            return
+
+        data = self._acwrite_buffer
+        self._acwrite_buffer = b""
+        try:
+            d = self.agentpool.callRemote("acwrite", self._agent_id, self._ac_id, data)
+            d.addCallback(self.cb_acwrite_result)
+            d.addErrback(self.cb_global_error)
+        except (AttributeError, pb.DeadReferenceError):
+            self.disconnect(exit_code=1)
+
+    def cb_acwrite_result(self, result):
+        assert result > 0
+        if self._acwrite_buffer:
+            self.acwrite_data(b"")

platformio/remote/client/run_or_test.py

@@ -0,0 +1,272 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import hashlib
+import json
+import os
+import zlib
+from io import BytesIO
+
+from twisted.spread import pb  # pylint: disable=import-error
+
+from platformio import fs
+from platformio.compat import hashlib_encode_data
+from platformio.project.config import ProjectConfig
+from platformio.remote.client.async_base import AsyncClientBase
+from platformio.remote.projectsync import PROJECT_SYNC_STAGE, ProjectSync
+
+
+class RunOrTestClient(AsyncClientBase):
+
+    MAX_ARCHIVE_SIZE = 50 * 1024 * 1024  # 50Mb
+    UPLOAD_CHUNK_SIZE = 256 * 1024  # 256Kb
+
+    PSYNC_SRC_EXTS = [
+        "c",
+        "cpp",
+        "S",
+        "spp",
+        "SPP",
+        "sx",
+        "s",
+        "asm",
+        "ASM",
+        "h",
+        "hpp",
+        "ipp",
+        "ino",
+        "pde",
+        "json",
+        "properties",
+    ]
+
+    PSYNC_SKIP_DIRS = (".git", ".svn", ".hg", "example", "examples", "test", "tests")
+
+    def __init__(self, *args, **kwargs):
+        AsyncClientBase.__init__(self, *args, **kwargs)
+        self.project_id = self.generate_project_id(self.options["project_dir"])
+        self.psync = ProjectSync(self.options["project_dir"])
+
+    def generate_project_id(self, path):
+        h = hashlib.sha1(hashlib_encode_data(self.id))
+        h.update(hashlib_encode_data(path))
+        return "%s-%s" % (os.path.basename(path), h.hexdigest())
+
+    def add_project_items(self, psync):
+        with fs.cd(self.options["project_dir"]):
+            cfg = ProjectConfig.get_instance(
+                os.path.join(self.options["project_dir"], "platformio.ini")
+            )
+            psync.add_item(cfg.path, "platformio.ini")
+            psync.add_item(cfg.get("platformio", "shared_dir"), "shared")
+            psync.add_item(cfg.get("platformio", "boards_dir"), "boards")
+
+            if self.options["force_remote"]:
+                self._add_project_source_items(cfg, psync)
+            else:
+                self._add_project_binary_items(cfg, psync)
+
+            if self.command == "test":
+                psync.add_item(cfg.get("platformio", "test_dir"), "test")
+
+    def _add_project_source_items(self, cfg, psync):
+        psync.add_item(cfg.get("platformio", "lib_dir"), "lib")
+        psync.add_item(
+            cfg.get("platformio", "include_dir"),
+            "include",
+            cb_filter=self._cb_tarfile_filter,
+        )
+        psync.add_item(
+            cfg.get("platformio", "src_dir"), "src", cb_filter=self._cb_tarfile_filter
+        )
+        if set(["buildfs", "uploadfs", "uploadfsota"]) & set(
+            self.options.get("target", [])
+        ):
+            psync.add_item(cfg.get("platformio", "data_dir"), "data")
+
+    @staticmethod
+    def _add_project_binary_items(cfg, psync):
+        build_dir = cfg.get("platformio", "build_dir")
+        for env_name in os.listdir(build_dir):
+            env_dir = os.path.join(build_dir, env_name)
+            if not os.path.isdir(env_dir):
+                continue
+            for fname in os.listdir(env_dir):
+                bin_file = os.path.join(env_dir, fname)
+                bin_exts = (".elf", ".bin", ".hex", ".eep", "program")
+                if os.path.isfile(bin_file) and fname.endswith(bin_exts):
+                    psync.add_item(
+                        bin_file, os.path.join(".pio", "build", env_name, fname)
+                    )
+
+    def _cb_tarfile_filter(self, path):
+        if (
+            os.path.isdir(path)
+            and os.path.basename(path).lower() in self.PSYNC_SKIP_DIRS
+        ):
+            return None
+        if os.path.isfile(path) and not self.is_file_with_exts(
+            path, self.PSYNC_SRC_EXTS
+        ):
+            return None
+        return path
+
+    @staticmethod
+    def is_file_with_exts(path, exts):
+        if path.endswith(tuple(".%s" % e for e in exts)):
+            return True
+        return False
+
+    def agent_pool_ready(self):
+        self.psync_init()
+
+    def psync_init(self):
+        self.add_project_items(self.psync)
+        d = self.agentpool.callRemote(
+            "cmd",
+            self.agents,
+            "psync",
+            dict(id=self.project_id, items=[i[1] for i in self.psync.get_items()]),
+        )
+        d.addCallback(self.cb_psync_init_result)
+        d.addErrback(self.cb_global_error)
+
+        # build db index while wait for result from agent
+        self.psync.rebuild_dbindex()
+
+    def cb_psync_init_result(self, result):
+        self._acs_total = len(result)
+        for (success, value) in result:
+            if not success:
+                raise pb.Error(value)
+            agent_id, ac_id = value
+            try:
+                d = self.agentpool.callRemote(
+                    "acwrite",
+                    agent_id,
+                    ac_id,
+                    dict(stage=PROJECT_SYNC_STAGE.DBINDEX.value),
+                )
+                d.addCallback(self.cb_psync_dbindex_result, agent_id, ac_id)
+                d.addErrback(self.cb_global_error)
+            except (AttributeError, pb.DeadReferenceError):
+                self.disconnect(exit_code=1)
+
+    def cb_psync_dbindex_result(self, result, agent_id, ac_id):
+        result = set(json.loads(zlib.decompress(result)))
+        dbindex = set(self.psync.get_dbindex())
+        delete = list(result - dbindex)
+        delta = list(dbindex - result)
+
+        self.log.debug(
+            "PSync: stats, total={total}, delete={delete}, delta={delta}",
+            total=len(dbindex),
+            delete=len(delete),
+            delta=len(delta),
+        )
+
+        if not delete and not delta:
+            return self.psync_finalize(agent_id, ac_id)
+        if not delete:
+            return self.psync_upload(agent_id, ac_id, delta)
+
+        try:
+            d = self.agentpool.callRemote(
+                "acwrite",
+                agent_id,
+                ac_id,
+                dict(
+                    stage=PROJECT_SYNC_STAGE.DELETE.value,
+                    dbindex=zlib.compress(json.dumps(delete).encode()),
+                ),
+            )
+            d.addCallback(self.cb_psync_delete_result, agent_id, ac_id, delta)
+            d.addErrback(self.cb_global_error)
+        except (AttributeError, pb.DeadReferenceError):
+            self.disconnect(exit_code=1)
+
+        return None
+
+    def cb_psync_delete_result(self, result, agent_id, ac_id, dbindex):
+        assert result
+        self.psync_upload(agent_id, ac_id, dbindex)
+
+    def psync_upload(self, agent_id, ac_id, dbindex):
+        assert dbindex
+        fileobj = BytesIO()
+        compressed = self.psync.compress_items(fileobj, dbindex, self.MAX_ARCHIVE_SIZE)
+        fileobj.seek(0)
+        self.log.debug(
+            "PSync: upload project, size={size}", size=len(fileobj.getvalue())
+        )
+        self.psync_upload_chunk(
+            agent_id, ac_id, list(set(dbindex) - set(compressed)), fileobj
+        )
+
+    def psync_upload_chunk(self, agent_id, ac_id, dbindex, fileobj):
+        offset = fileobj.tell()
+        total = fileobj.seek(0, os.SEEK_END)
+        # unwind
+        fileobj.seek(offset)
+        chunk = fileobj.read(self.UPLOAD_CHUNK_SIZE)
+        assert chunk
+        try:
+            d = self.agentpool.callRemote(
+                "acwrite",
+                agent_id,
+                ac_id,
+                dict(
+                    stage=PROJECT_SYNC_STAGE.UPLOAD.value,
+                    chunk=chunk,
+                    length=len(chunk),
+                    total=total,
+                ),
+            )
+            d.addCallback(
+                self.cb_psync_upload_chunk_result, agent_id, ac_id, dbindex, fileobj
+            )
+            d.addErrback(self.cb_global_error)
+        except (AttributeError, pb.DeadReferenceError):
+            self.disconnect(exit_code=1)
+
+    def cb_psync_upload_chunk_result(  # pylint: disable=too-many-arguments
+        self, result, agent_id, ac_id, dbindex, fileobj
+    ):
+        result = PROJECT_SYNC_STAGE.lookupByValue(result)
+        self.log.debug("PSync: upload chunk result {r}", r=str(result))
+        assert result & (PROJECT_SYNC_STAGE.UPLOAD | PROJECT_SYNC_STAGE.EXTRACTED)
+        if result is PROJECT_SYNC_STAGE.EXTRACTED:
+            if dbindex:
+                self.psync_upload(agent_id, ac_id, dbindex)
+            else:
+                self.psync_finalize(agent_id, ac_id)
+        else:
+            self.psync_upload_chunk(agent_id, ac_id, dbindex, fileobj)
+
+    def psync_finalize(self, agent_id, ac_id):
+        try:
+            d = self.agentpool.callRemote("acclose", agent_id, ac_id)
+            d.addCallback(self.cb_psync_completed_result, agent_id)
+            d.addErrback(self.cb_global_error)
+        except (AttributeError, pb.DeadReferenceError):
+            self.disconnect(exit_code=1)
+
+    def cb_psync_completed_result(self, result, agent_id):
+        assert PROJECT_SYNC_STAGE.lookupByValue(result)
+        options = self.options.copy()
+        del options["project_dir"]
+        options["project_id"] = self.project_id
+        d = self.agentpool.callRemote("cmd", [agent_id], self.command, options)
+        d.addCallback(self.cb_async_result)
+        d.addErrback(self.cb_global_error)

platformio/remote/client/update_core.py

@@ -0,0 +1,22 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from platformio.remote.client.async_base import AsyncClientBase
+
+
+class UpdateCoreClient(AsyncClientBase):
+    def agent_pool_ready(self):
+        d = self.agentpool.callRemote("cmd", self.agents, self.command, self.options)
+        d.addCallback(self.cb_async_result)
+        d.addErrback(self.cb_global_error)