Initial commit of PIO Home

.isort.cfg

@@ -1,3 +1,3 @@
 [settings]
 line_length=79
-known_third_party=bottle,click,pytest,requests,SCons,semantic_version,serial, twisted
+known_third_party=bottle,click,pytest,requests,SCons,semantic_version,serial,twisted,autobahn,bs4,jsonrpc

platformio/commands/home/__init__.py

@@ -0,0 +1,15 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from platformio.commands.home.command import cli

platformio/commands/home/command.py

@@ -0,0 +1,100 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import socket
+from os.path import isdir
+
+import click
+
+from platformio import exception
+from platformio.managers.core import (get_core_package_dir,
+                                      inject_contrib_pysite)
+
+
+@click.command("home", short_help="PIO Home")
+@click.option("--port", type=int, default=8008, help="HTTP port, default=8008")
+@click.option(
+    "--host",
+    default="127.0.0.1",
+    help="HTTP host, default=127.0.0.1. "
+    "You can open PIO Home for inbound connections with --host=0.0.0.0")
+@click.option("--no-open", is_flag=True)  # pylint: disable=too-many-locals
+def cli(port, host, no_open):
+    # import contrib modules
+    inject_contrib_pysite()
+    from autobahn.twisted.resource import WebSocketResource
+    from twisted.internet import reactor
+    from twisted.web import server
+    from platformio.commands.home.rpc.handlers.app import AppRPC
+    from platformio.commands.home.rpc.handlers.ide import IDERPC
+    from platformio.commands.home.rpc.handlers.misc import MiscRPC
+    from platformio.commands.home.rpc.handlers.os import OSRPC
+    from platformio.commands.home.rpc.handlers.piocore import PIOCoreRPC
+    from platformio.commands.home.rpc.handlers.project import ProjectRPC
+    from platformio.commands.home.rpc.server import JSONRPCServerFactory
+    from platformio.commands.home.web import WebRoot
+
+    factory = JSONRPCServerFactory()
+    factory.addHandler(AppRPC(), namespace="app")
+    factory.addHandler(IDERPC(), namespace="ide")
+    factory.addHandler(MiscRPC(), namespace="misc")
+    factory.addHandler(OSRPC(), namespace="os")
+    factory.addHandler(PIOCoreRPC(), namespace="core")
+    factory.addHandler(ProjectRPC(), namespace="project")
+
+    contrib_dir = get_core_package_dir("contrib-piohome")
+    if not isdir(contrib_dir):
+        raise exception.PlatformioException("Invalid path to PIO Home Contrib")
+    root = WebRoot(contrib_dir)
+    root.putChild(b"wsrpc", WebSocketResource(factory))
+    site = server.Site(root)
+
+    # hook for `platformio-node-helpers`
+    if host == "__do_not_start__":
+        return
+
+    # if already started
+    already_started = False
+    socket.setdefaulttimeout(1)
+    try:
+        socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect((host, port))
+        already_started = True
+    except:  # pylint: disable=bare-except
+        pass
+
+    home_url = "http://%s:%d" % (host, port)
+    if not no_open:
+        if already_started:
+            click.launch(home_url)
+        else:
+            reactor.callLater(1, lambda: click.launch(home_url))
+
+    click.echo("\n".join([
+        "",
+        "  ___I_",
+        " /\\-_--\\   PlatformIO Home",
+        "/  \\_-__\\",
+        "|[]| [] |  %s" % home_url,
+        "|__|____|______________%s" % ("_" * len(host)),
+    ]))
+    click.echo("")
+    click.echo("Open PIO Home in your browser by this URL => %s" % home_url)
+
+    if already_started:
+        return
+
+    click.echo("PIO Home has been started. Press Ctrl+C to shutdown.")
+
+    reactor.listenTCP(port, site, interface=host)
+    reactor.run()

platformio/commands/home/helpers.py

@@ -0,0 +1,69 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# pylint: disable=keyword-arg-before-vararg, arguments-differ
+
+import os
+import socket
+
+import requests
+from twisted.internet import threads
+from twisted.internet.defer import ensureDeferred
+
+from platformio import util
+
+
+class AsyncSession(requests.Session):
+
+    def __init__(self, n=None, *args, **kwargs):
+        if n:
+            from twisted.internet import reactor
+            pool = reactor.getThreadPool()
+            pool.adjustPoolsize(0, n)
+
+        super(AsyncSession, self).__init__(*args, **kwargs)
+
+    def request(self, *args, **kwargs):
+        func = super(AsyncSession, self).request
+        return threads.deferToThread(func, *args, **kwargs)
+
+    def wrap(self, *args, **kwargs):  # pylint: disable=no-self-use
+        return ensureDeferred(*args, **kwargs)
+
+
+@util.memoized(expire=5000)
+def requests_session():
+    return AsyncSession(n=5)
+
+
+@util.memoized()
+def get_core_fullpath():
+    return util.where_is_program(
+        "platformio" + (".exe" if "windows" in util.get_systype() else ""))
+
+
+@util.memoized(expire=10000)
+def is_twitter_blocked():
+    ip = "104.244.42.1"
+    timeout = 2
+    try:
+        if os.getenv("HTTP_PROXY", os.getenv("HTTPS_PROXY")):
+            requests.get(
+                "http://%s" % ip, allow_redirects=False, timeout=timeout)
+        else:
+            socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect((ip, 80))
+        return False
+    except:  # pylint: disable=bare-except
+        pass
+    return True

platformio/commands/home/rpc/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

platformio/commands/home/rpc/handlers/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

platformio/commands/home/rpc/handlers/app.py

@@ -0,0 +1,83 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+
+import json
+from os.path import expanduser, isfile, join
+
+from platformio import __version__, app, exception, util
+
+
+class AppRPC(object):
+
+    APPSTATE_PATH = join(util.get_home_dir(), "homestate.json")
+
+    @staticmethod
+    def load_state():
+        state = None
+        try:
+            if isfile(AppRPC.APPSTATE_PATH):
+                state = util.load_json(AppRPC.APPSTATE_PATH)
+        except exception.PlatformioException:
+            pass
+        if not isinstance(state, dict):
+            state = {}
+        storage = state.get("storage", {})
+
+        # base data
+        caller_id = app.get_session_var("caller_id")
+        storage['cid'] = app.get_cid()
+        storage['coreVersion'] = __version__
+        storage['coreSystype'] = util.get_systype()
+        storage['coreCaller'] = (str(caller_id).lower() if caller_id else None)
+        storage['coreSettings'] = {
+            name: {
+                "description": data['description'],
+                "default_value": data['value'],
+                "value": app.get_setting(name)
+            }
+            for name, data in app.DEFAULT_SETTINGS.items()
+        }
+
+        # encode to UTF-8
+        for key in storage['coreSettings']:
+            if not key.endswith("dir"):
+                continue
+            storage['coreSettings'][key][
+                'default_value'] = util.path_to_unicode(
+                    storage['coreSettings'][key]['default_value'])
+            storage['coreSettings'][key]['value'] = util.path_to_unicode(
+                storage['coreSettings'][key]['value'])
+        storage['homeDir'] = util.path_to_unicode(expanduser("~"))
+        storage['projectsDir'] = storage['coreSettings']['projects_dir'][
+            'value']
+
+        # skip non-existing recent projects
+        storage['recentProjects'] = [
+            p for p in storage.get("recentProjects", [])
+            if util.is_platformio_project(p)
+        ]
+
+        state['storage'] = storage
+        return state
+
+    @staticmethod
+    def get_state():
+        return AppRPC.load_state()
+
+    def save_state(self, state):
+        with open(self.APPSTATE_PATH, "w") as fp:
+            json.dump(state, fp)
+        return True

platformio/commands/home/rpc/handlers/ide.py

@@ -0,0 +1,42 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import time
+
+from jsonrpc.exceptions import JSONRPCDispatchException
+from twisted.internet import defer
+
+
+class IDERPC(object):
+
+    def __init__(self):
+        self._queue = []
+
+    def send_command(self, command, params):
+        if not self._queue:
+            raise JSONRPCDispatchException(
+                code=4005, message="PIO Home IDE agent is not started")
+        while self._queue:
+            self._queue.pop().callback({
+                "id": time.time(),
+                "method": command,
+                "params": params
+            })
+
+    def listen_commands(self):
+        self._queue.append(defer.Deferred())
+        return self._queue[-1]
+
+    def open_project(self, project_dir):
+        return self.send_command("open_project", project_dir)

platformio/commands/home/rpc/handlers/misc.py

@@ -0,0 +1,194 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import re
+import time
+
+from bs4 import BeautifulSoup
+from twisted.internet import defer, reactor
+
+from platformio import app
+from platformio.commands.home import helpers
+from platformio.commands.home.rpc.handlers.os import OSRPC
+
+
+class MiscRPC(object):
+
+    def load_latest_tweets(self, username):
+        cache_key = "piohome_latest_tweets_%s" % username
+        cache_valid = "7d"
+        with app.ContentCache() as cc:
+            cache_data = cc.get(cache_key)
+            if cache_data:
+                cache_data = json.loads(cache_data)
+                # automatically update cache in background every 12 hours
+                if cache_data['time'] < (time.time() - (3600 * 12)):
+                    reactor.callLater(5, self._preload_latest_tweets, username,
+                                      cache_key, cache_valid)
+                return cache_data['result']
+
+        result = self._preload_latest_tweets(username, cache_key, cache_valid)
+        return result
+
+    @defer.inlineCallbacks
+    def _preload_latest_tweets(self, username, cache_key, cache_valid):
+        result = yield self._fetch_tweets(username)
+        with app.ContentCache() as cc:
+            cc.set(cache_key,
+                   json.dumps({
+                       "time": int(time.time()),
+                       "result": result
+                   }), cache_valid)
+        defer.returnValue(result)
+
+    @defer.inlineCallbacks
+    def _fetch_tweets(self, username):
+        api_url = ("https://twitter.com/i/profiles/show/%s/timeline/tweets?"
+                   "include_available_features=1&include_entities=1&"
+                   "include_new_items_bar=true") % username
+        if helpers.is_twitter_blocked():
+            api_url = self._get_proxed_uri(api_url)
+        html_or_json = yield OSRPC.fetch_content(
+            api_url, headers=self._get_twitter_headers(username))
+        # issue with PIO Core < 3.5.3 and ContentCache
+        if not isinstance(html_or_json, dict):
+            html_or_json = json.loads(html_or_json)
+        assert "items_html" in html_or_json
+        soup = BeautifulSoup(html_or_json['items_html'], "html.parser")
+        tweet_nodes = soup.find_all(
+            "div", attrs={
+                "class": "tweet",
+                "data-tweet-id": True
+            })
+        result = yield defer.DeferredList(
+            [self._parse_tweet_node(node, username) for node in tweet_nodes],
+            consumeErrors=True)
+        defer.returnValue([r[1] for r in result if r[0]])
+
+    @defer.inlineCallbacks
+    def _parse_tweet_node(self, tweet, username):
+        # remove non-visible items
+        for node in tweet.find_all(class_=["invisible", "u-hidden"]):
+            node.decompose()
+        twitter_url = "https://twitter.com"
+        time_node = tweet.find("span", attrs={"data-time": True})
+        text_node = tweet.find(class_="tweet-text")
+        quote_text_node = tweet.find(class_="QuoteTweet-text")
+        if quote_text_node and not text_node.get_text().strip():
+            text_node = quote_text_node
+        photos = [
+            node.get("data-image-url") for node in (tweet.find_all(class_=[
+                "AdaptiveMedia-photoContainer", "QuoteMedia-photoContainer"
+            ]) or [])
+        ]
+        urls = [
+            node.get("data-expanded-url")
+            for node in (quote_text_node or text_node).find_all(
+                class_="twitter-timeline-link",
+                attrs={"data-expanded-url": True})
+        ]
+
+        # fetch data from iframe card
+        if (not photos or not urls) and tweet.get("data-card2-type"):
+            iframe_node = tweet.find(
+                "div", attrs={"data-full-card-iframe-url": True})
+            if iframe_node:
+                iframe_card = yield self._fetch_iframe_card(
+                    twitter_url + iframe_node.get("data-full-card-iframe-url"),
+                    username)
+                if not photos and iframe_card['photo']:
+                    photos.append(iframe_card['photo'])
+                if not urls and iframe_card['url']:
+                    urls.append(iframe_card['url'])
+                if iframe_card['text_node']:
+                    text_node = iframe_card['text_node']
+
+        if not photos:
+            photos.append(tweet.find("img", class_="avatar").get("src"))
+
+        def _fetch_text(text_node):
+            text = text_node.decode_contents(formatter="html").strip()
+            text = re.sub(r'href="/', 'href="%s/' % twitter_url, text)
+            if "</p>" not in text and "<br" not in text:
+                text = re.sub(r"\n+", "<br />", text)
+            return text
+
+        defer.returnValue({
+            "tweetId":
+            tweet.get("data-tweet-id"),
+            "tweetUrl":
+            twitter_url + tweet.get("data-permalink-path"),
+            "author":
+            tweet.get("data-name"),
+            "time":
+            int(time_node.get("data-time")),
+            "timeFormatted":
+            time_node.string,
+            "text":
+            _fetch_text(text_node),
+            "entries": {
+                "urls":
+                urls,
+                "photos": [
+                    self._get_proxed_uri(uri)
+                    if helpers.is_twitter_blocked() else uri for uri in photos
+                ]
+            },
+            "isPinned":
+            "user-pinned" in tweet.get("class")
+        })
+
+    @defer.inlineCallbacks
+    def _fetch_iframe_card(self, url, username):
+        if helpers.is_twitter_blocked():
+            url = self._get_proxed_uri(url)
+        html = yield OSRPC.fetch_content(
+            url, headers=self._get_twitter_headers(username), cache_valid="7d")
+        soup = BeautifulSoup(html, "html.parser")
+        photo_node = soup.find("img", attrs={"data-src": True})
+        url_node = soup.find("a", class_="TwitterCard-container")
+        text_node = soup.find("div", class_="SummaryCard-content")
+        if text_node:
+            text_node.find(
+                "span", class_="SummaryCard-destination").decompose()
+        defer.returnValue({
+            "photo":
+            photo_node.get("data-src") if photo_node else None,
+            "text_node":
+            text_node,
+            "url":
+            url_node.get("href") if url_node else None
+        })
+
+    @staticmethod
+    def _get_proxed_uri(uri):
+        index = uri.index("://")
+        return "https://dl.platformio.org/__prx__/" + uri[index + 3:]
+
+    @staticmethod
+    def _get_twitter_headers(username):
+        return {
+            "Accept":
+            "application/json, text/javascript, */*; q=0.01",
+            "Referer":
+            "https://twitter.com/%s" % username,
+            "User-Agent":
+            ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit"
+             "/603.3.8 (KHTML, like Gecko) Version/10.1.2 Safari/603.3.8"),
+            "X-Twitter-Active-User":
+            "yes",
+            "X-Requested-With":
+            "XMLHttpRequest"
+        }

platformio/commands/home/rpc/handlers/os.py

@@ -0,0 +1,153 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+
+import glob
+import os
+import shutil
+import sys
+from functools import cmp_to_key
+from os.path import expanduser, isdir, isfile, join
+
+import click
+from twisted.internet import defer
+
+from platformio import app, util
+from platformio.commands.home import helpers
+
+
+class OSRPC(object):
+
+    @staticmethod
+    @defer.inlineCallbacks
+    def fetch_content(uri, data=None, headers=None, cache_valid=None):
+        timeout = 2
+        if not headers:
+            headers = {
+                "User-Agent":
+                ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) "
+                 "AppleWebKit/603.3.8 (KHTML, like Gecko) Version/10.1.2 "
+                 "Safari/603.3.8")
+            }
+        cache_key = (app.ContentCache.key_from_args(uri, data)
+                     if cache_valid else None)
+        with app.ContentCache() as cc:
+            if cache_key:
+                result = cc.get(cache_key)
+                if result is not None:
+                    defer.returnValue(result)
+
+        # check internet before and resolve issue with 60 seconds timeout
+        util.internet_on(raise_exception=True)
+
+        session = helpers.requests_session()
+        if data:
+            r = yield session.post(
+                uri, data=data, headers=headers, timeout=timeout)
+        else:
+            r = yield session.get(uri, headers=headers, timeout=timeout)
+
+        r.raise_for_status()
+        result = r.text
+        if cache_valid:
+            with app.ContentCache() as cc:
+                cc.set(cache_key, result, cache_valid)
+        defer.returnValue(result)
+
+    def request_content(self, uri, data=None, headers=None, cache_valid=None):
+        if uri.startswith('http'):
+            return self.fetch_content(uri, data, headers, cache_valid)
+        if isfile(uri):
+            with open(uri) as fp:
+                return fp.read()
+        return None
+
+    @staticmethod
+    def open_url(url):
+        return click.launch(url)
+
+    @staticmethod
+    def reveal_file(path):
+        return click.launch(
+            path.encode(sys.getfilesystemencoding()) if util.PY2 else path,
+            locate=True)
+
+    @staticmethod
+    def is_file(path):
+        return isfile(path)
+
+    @staticmethod
+    def is_dir(path):
+        return isdir(path)
+
+    @staticmethod
+    def make_dirs(path):
+        return os.makedirs(path)
+
+    @staticmethod
+    def rename(src, dst):
+        return os.rename(src, dst)
+
+    @staticmethod
+    def copy(src, dst):
+        return shutil.copytree(src, dst)
+
+    @staticmethod
+    def glob(pathnames, root=None):
+        if not isinstance(pathnames, list):
+            pathnames = [pathnames]
+        result = set()
+        for pathname in pathnames:
+            result |= set(
+                glob.glob(join(root, pathname) if root else pathname))
+        return list(result)
+
+    @staticmethod
+    def list_dir(path):
+
+        def _cmp(x, y):
+            if x[1] and not y[1]:
+                return -1
+            if not x[1] and y[1]:
+                return 1
+            if x[0].lower() > y[0].lower():
+                return 1
+            if x[0].lower() < y[0].lower():
+                return -1
+            return 0
+
+        items = []
+        if path.startswith("~"):
+            path = expanduser(path)
+        if not isdir(path):
+            return items
+        for item in os.listdir(path):
+            try:
+                item_is_dir = isdir(join(path, item))
+                if item_is_dir:
+                    os.listdir(join(path, item))
+                items.append((item, item_is_dir))
+            except OSError:
+                pass
+        return sorted(items, key=cmp_to_key(_cmp))
+
+    @staticmethod
+    def get_logical_devices():
+        items = []
+        for item in util.get_logical_devices():
+            if item['name']:
+                item['name'] = util.path_to_unicode(item['name'])
+            items.append(item)
+        return items

platformio/commands/home/rpc/handlers/piocore.py

@@ -0,0 +1,83 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+
+import json
+import os
+import re
+import sys
+
+from jsonrpc.exceptions import JSONRPCDispatchException
+from twisted.internet.utils import getProcessOutputAndValue
+
+from platformio import __version__, util
+from platformio.commands.home import helpers
+
+
+class PIOCoreRPC(object):
+
+    @staticmethod
+    def call(args, options=None):
+        json_output = "--json-output" in args
+        try:
+            args = [
+                arg.encode(sys.getfilesystemencoding()) if isinstance(
+                    arg, util.string_types) else str(arg) for arg in args
+            ]
+        except UnicodeError:
+            raise JSONRPCDispatchException(
+                code=4002, message="PIO Core: non-ASCII chars in arguments")
+        d = getProcessOutputAndValue(
+            helpers.get_core_fullpath(),
+            args,
+            path=(options or {}).get("cwd"),
+            env={k: v
+                 for k, v in os.environ.items() if "%" not in k})
+        d.addCallback(PIOCoreRPC._call_callback, json_output)
+        d.addErrback(PIOCoreRPC._call_errback)
+        return d
+
+    @staticmethod
+    def _call_callback(result, json_output=False):
+        result = list(result)
+        assert len(result) == 3
+        for i in (0, 1):
+            result[i] = result[i].decode(sys.getfilesystemencoding()).strip()
+        out, err, code = result
+        text = ("%s\n\n%s" % (out, err)).strip()
+        if code != 0:
+            raise Exception(text)
+
+        if not json_output:
+            return text
+
+        try:
+            return json.loads(out)
+        except ValueError as e:
+            if "sh: " in out:
+                return json.loads(
+                    re.sub(r"^sh: [^\n]+$", "", out, flags=re.M).strip())
+            raise e
+
+    @staticmethod
+    def _call_errback(failure):
+        raise JSONRPCDispatchException(
+            code=4003,
+            message="PIO Core Call Error",
+            data=failure.getErrorMessage())
+
+    @staticmethod
+    def version():
+        return __version__

platformio/commands/home/rpc/handlers/project.py

@@ -0,0 +1,283 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+
+import os
+import shutil
+import sys
+import time
+from os.path import (basename, expanduser, getmtime, isdir, isfile, join,
+                     realpath, sep)
+
+from jsonrpc.exceptions import JSONRPCDispatchException
+
+from platformio import exception, util
+from platformio.commands.home.rpc.handlers.app import AppRPC
+from platformio.commands.home.rpc.handlers.piocore import PIOCoreRPC
+from platformio.ide.projectgenerator import ProjectGenerator
+from platformio.managers.platform import PlatformManager
+
+try:
+    from configparser import Error as ConfigParserError
+except ImportError:
+    from ConfigParser import Error as ConfigParserError
+
+
+class ProjectRPC(object):
+
+    @staticmethod
+    def _get_projects(project_dirs=None):
+
+        def _get_project_data(project_dir):
+            data = {"boards": [], "libExtraDirs": []}
+            config = util.load_project_config(project_dir)
+
+            if config.has_section("platformio") and \
+                    config.has_option("platformio", "lib_extra_dirs"):
+                data['libExtraDirs'].extend(
+                    util.parse_conf_multi_values(
+                        config.get("platformio", "lib_extra_dirs")))
+
+            for section in config.sections():
+                if not section.startswith("env:"):
+                    continue
+                if config.has_option(section, "board"):
+                    data['boards'].append(config.get(section, "board"))
+                if config.has_option(section, "lib_extra_dirs"):
+                    data['libExtraDirs'].extend(
+                        util.parse_conf_multi_values(
+                            config.get(section, "lib_extra_dirs")))
+
+            # resolve libExtraDirs paths
+            with util.cd(project_dir):
+                data['libExtraDirs'] = [
+                    expanduser(d) if d.startswith("~") else realpath(d)
+                    for d in data['libExtraDirs']
+                ]
+
+            # skip non existing folders
+            data['libExtraDirs'] = [
+                d for d in data['libExtraDirs'] if isdir(d)
+            ]
+
+            return data
+
+        def _path_to_name(path):
+            return (sep).join(path.split(sep)[-2:])
+
+        if not project_dirs:
+            project_dirs = AppRPC.load_state()['storage']['recentProjects']
+
+        result = []
+        pm = PlatformManager()
+        for project_dir in project_dirs:
+            data = {}
+            boards = []
+            try:
+                data = _get_project_data(project_dir)
+            except exception.NotPlatformIOProject:
+                continue
+            except ConfigParserError:
+                pass
+
+            for board_id in data.get("boards", []):
+                name = board_id
+                try:
+                    name = pm.board_config(board_id)['name']
+                except (exception.UnknownBoard, exception.UnknownPlatform):
+                    pass
+                boards.append({"id": board_id, "name": name})
+
+            result.append({
+                "path":
+                project_dir,
+                "name":
+                _path_to_name(project_dir),
+                "modified":
+                int(getmtime(project_dir)),
+                "boards":
+                boards,
+                "extraLibStorages": [{
+                    "name": _path_to_name(d),
+                    "path": d
+                } for d in data.get("libExtraDirs", [])]
+            })
+        return result
+
+    def get_projects(self, project_dirs=None):
+        return self._get_projects(project_dirs)
+
+    def init(self, board, framework, project_dir):
+        assert project_dir
+        state = AppRPC.load_state()
+        if not isdir(project_dir):
+            os.makedirs(project_dir)
+        args = ["init", "--project-dir", project_dir, "--board", board]
+        if framework:
+            args.extend(["--project-option", "framework = %s" % framework])
+        if (state['storage']['coreCaller'] and state['storage']['coreCaller']
+                in ProjectGenerator.get_supported_ides()):
+            args.extend(["--ide", state['storage']['coreCaller']])
+        d = PIOCoreRPC.call(args)
+        d.addCallback(self._generate_project_main, project_dir, framework)
+        return d
+
+    @staticmethod
+    def _generate_project_main(_, project_dir, framework):
+        main_content = None
+        if framework == "arduino":
+            main_content = "\n".join([
+                "#include <Arduino.h>",
+                "",
+                "void setup() {",
+                "  // put your setup code here, to run once:",
+                "}",
+                "",
+                "void loop() {",
+                "  // put your main code here, to run repeatedly:",
+                "}"
+                ""
+            ])   # yapf: disable
+        elif framework == "mbed":
+            main_content = "\n".join([
+                "#include <mbed.h>",
+                "",
+                "int main() {",
+                "",
+                "  // put your setup code here, to run once:",
+                "",
+                "  while(1) {",
+                "    // put your main code here, to run repeatedly:",
+                "  }",
+                "}",
+                ""
+            ])   # yapf: disable
+        if not main_content:
+            return project_dir
+        with util.cd(project_dir):
+            src_dir = util.get_projectsrc_dir()
+            main_path = join(src_dir, "main.cpp")
+            if isfile(main_path):
+                return project_dir
+            if not isdir(src_dir):
+                os.makedirs(src_dir)
+            with open(main_path, "w") as f:
+                f.write(main_content.strip())
+        return project_dir
+
+    def import_arduino(self, board, use_arduino_libs, arduino_project_dir):
+        # don't import PIO Project
+        if util.is_platformio_project(arduino_project_dir):
+            return arduino_project_dir
+
+        is_arduino_project = any([
+            isfile(
+                join(arduino_project_dir,
+                     "%s.%s" % (basename(arduino_project_dir), ext)))
+            for ext in ("ino", "pde")
+        ])
+        if not is_arduino_project:
+            raise JSONRPCDispatchException(
+                code=4000,
+                message="Not an Arduino project: %s" % arduino_project_dir)
+
+        state = AppRPC.load_state()
+        project_dir = join(state['storage']['projectsDir'].decode("utf-8"),
+                           time.strftime("%y%m%d-%H%M%S-") + board)
+        if not isdir(project_dir):
+            os.makedirs(project_dir)
+        args = ["init", "--project-dir", project_dir, "--board", board]
+        args.extend(["--project-option", "framework = arduino"])
+        if use_arduino_libs:
+            args.extend([
+                "--project-option",
+                "lib_extra_dirs = ~/Documents/Arduino/libraries"
+            ])
+        if (state['storage']['coreCaller'] and state['storage']['coreCaller']
+                in ProjectGenerator.get_supported_ides()):
+            args.extend(["--ide", state['storage']['coreCaller']])
+        d = PIOCoreRPC.call(args)
+        d.addCallback(self._finalize_arduino_import, project_dir,
+                      arduino_project_dir)
+        return d
+
+    @staticmethod
+    def _finalize_arduino_import(_, project_dir, arduino_project_dir):
+        with util.cd(project_dir):
+            src_dir = util.get_projectsrc_dir()
+            if isdir(src_dir):
+                util.rmtree_(src_dir)
+            shutil.copytree(
+                arduino_project_dir.encode(sys.getfilesystemencoding()),
+                src_dir)
+        return project_dir
+
+    @staticmethod
+    def get_project_examples():
+        result = []
+        for manifest in PlatformManager().get_installed():
+            examples_dir = join(manifest['__pkg_dir'], "examples")
+            if not isdir(examples_dir):
+                continue
+            items = []
+            for project_dir, _, __ in os.walk(examples_dir):
+                project_description = None
+                try:
+                    config = util.load_project_config(project_dir)
+                    if config.has_section("platformio") and \
+                            config.has_option("platformio", "description"):
+                        project_description = config.get(
+                            "platformio", "description")
+                except (exception.NotPlatformIOProject,
+                        exception.InvalidProjectConf):
+                    continue
+
+                path_tokens = project_dir.split(sep)
+                items.append({
+                    "name":
+                    "/".join(path_tokens[path_tokens.index("examples") + 1:]),
+                    "path":
+                    project_dir,
+                    "description":
+                    project_description
+                })
+            result.append({
+                "platform": {
+                    "title": manifest['title'],
+                    "version": manifest['version']
+                },
+                "items": sorted(items, key=lambda item: item['name'])
+            })
+        return sorted(result, key=lambda data: data['platform']['title'])
+
+    @staticmethod
+    def import_pio(project_dir):
+        if not project_dir or not util.is_platformio_project(project_dir):
+            raise JSONRPCDispatchException(
+                code=4001,
+                message="Not an PlatformIO project: %s" % project_dir)
+        new_project_dir = join(
+            AppRPC.load_state()['storage']['projectsDir'].decode("utf-8"),
+            time.strftime("%y%m%d-%H%M%S-") + basename(project_dir))
+        shutil.copytree(project_dir, new_project_dir)
+
+        state = AppRPC.load_state()
+        args = ["init", "--project-dir", new_project_dir]
+        if (state['storage']['coreCaller'] and state['storage']['coreCaller']
+                in ProjectGenerator.get_supported_ides()):
+            args.extend(["--ide", state['storage']['coreCaller']])
+        d = PIOCoreRPC.call(args)
+        d.addCallback(lambda _: new_project_dir)
+        return d

platformio/commands/home/rpc/server.py

@@ -0,0 +1,70 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+
+import jsonrpc
+from autobahn.twisted.websocket import (WebSocketServerFactory,
+                                        WebSocketServerProtocol)
+from jsonrpc.exceptions import JSONRPCDispatchException
+from twisted.internet import defer
+
+
+class JSONRPCServerProtocol(WebSocketServerProtocol):
+
+    def onMessage(self, payload, isBinary):  # pylint: disable=unused-argument
+        # print("> %s" % payload)
+        response = jsonrpc.JSONRPCResponseManager.handle(
+            payload, self.factory.dispatcher).data
+        # if error
+        if "result" not in response:
+            self.sendJSONResponse(response)
+            return None
+
+        d = defer.maybeDeferred(lambda: response['result'])
+        d.addCallback(self._callback, response)
+        d.addErrback(self._errback, response)
+
+        return None
+
+    def _callback(self, result, response):
+        response['result'] = result
+        self.sendJSONResponse(response)
+
+    def _errback(self, failure, response):
+        if isinstance(failure.value, JSONRPCDispatchException):
+            e = failure.value
+        else:
+            e = JSONRPCDispatchException(
+                code=4999, message=failure.getErrorMessage())
+        del response["result"]
+        response['error'] = e.error._data  # pylint: disable=protected-access
+        print(response['error'])
+        self.sendJSONResponse(response)
+
+    def sendJSONResponse(self, response):
+        # print("< %s" % response)
+        self.sendMessage(json.dumps(response).encode("utf8"))
+
+
+class JSONRPCServerFactory(WebSocketServerFactory):
+
+    protocol = JSONRPCServerProtocol
+
+    def __init__(self):
+        super(JSONRPCServerFactory, self).__init__()
+        self.dispatcher = jsonrpc.Dispatcher()
+
+    def addHandler(self, handler, namespace):
+        self.dispatcher.build_method_map(handler, prefix="%s." % namespace)

platformio/commands/home/web.py

@@ -12,20 +12,19 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import sys
+from twisted.internet import reactor
-
+from twisted.web import static
-import click
-
-from platformio.managers.core import pioplus_call
 
 
-@click.command("home", short_help="PIO Home")
+class WebRoot(static.File):
-@click.option("--port", type=int, default=8008, help="HTTP port, default=8008")
+
-@click.option(
+    def render_GET(self, request):
-    "--host",
+        if request.args.get("__shutdown__", False):
-    default="127.0.0.1",
+            reactor.stop()
-    help="HTTP host, default=127.0.0.1. "
+            return "Server has been stopped"
-    "You can open PIO Home for inbound connections with --host=0.0.0.0")
+
-@click.option("--no-open", is_flag=True)
+        request.setHeader("cache-control",
-def cli(*args, **kwargs):  # pylint: disable=unused-argument
+                          "no-cache, no-store, must-revalidate")
-    pioplus_call(sys.argv[1:])
+        request.setHeader("pragma", "no-cache")
+        request.setHeader("expires", "0")
+        return static.File.render_GET(self, request)