diff --git a/.isort.cfg b/.isort.cfg index 1ee0d412..5f6d6207 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -1,3 +1,3 @@ [settings] line_length=79 -known_third_party=bottle,click,pytest,requests,SCons,semantic_version,serial, twisted +known_third_party=bottle,click,pytest,requests,SCons,semantic_version,serial,twisted,autobahn,bs4,jsonrpc diff --git a/platformio/commands/home/__init__.py b/platformio/commands/home/__init__.py new file mode 100644 index 00000000..a889291e --- /dev/null +++ b/platformio/commands/home/__init__.py @@ -0,0 +1,15 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from platformio.commands.home.command import cli diff --git a/platformio/commands/home/command.py b/platformio/commands/home/command.py new file mode 100644 index 00000000..0685670a --- /dev/null +++ b/platformio/commands/home/command.py @@ -0,0 +1,100 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import socket +from os.path import isdir + +import click + +from platformio import exception +from platformio.managers.core import (get_core_package_dir, + inject_contrib_pysite) + + +@click.command("home", short_help="PIO Home") +@click.option("--port", type=int, default=8008, help="HTTP port, default=8008") +@click.option( + "--host", + default="127.0.0.1", + help="HTTP host, default=127.0.0.1. " + "You can open PIO Home for inbound connections with --host=0.0.0.0") +@click.option("--no-open", is_flag=True) # pylint: disable=too-many-locals +def cli(port, host, no_open): + # import contrib modules + inject_contrib_pysite() + from autobahn.twisted.resource import WebSocketResource + from twisted.internet import reactor + from twisted.web import server + from platformio.commands.home.rpc.handlers.app import AppRPC + from platformio.commands.home.rpc.handlers.ide import IDERPC + from platformio.commands.home.rpc.handlers.misc import MiscRPC + from platformio.commands.home.rpc.handlers.os import OSRPC + from platformio.commands.home.rpc.handlers.piocore import PIOCoreRPC + from platformio.commands.home.rpc.handlers.project import ProjectRPC + from platformio.commands.home.rpc.server import JSONRPCServerFactory + from platformio.commands.home.web import WebRoot + + factory = JSONRPCServerFactory() + factory.addHandler(AppRPC(), namespace="app") + factory.addHandler(IDERPC(), namespace="ide") + factory.addHandler(MiscRPC(), namespace="misc") + factory.addHandler(OSRPC(), namespace="os") + factory.addHandler(PIOCoreRPC(), namespace="core") + factory.addHandler(ProjectRPC(), namespace="project") + + contrib_dir = get_core_package_dir("contrib-piohome") + if not isdir(contrib_dir): + raise exception.PlatformioException("Invalid path to PIO Home Contrib") + root = WebRoot(contrib_dir) + root.putChild(b"wsrpc", WebSocketResource(factory)) + site = server.Site(root) + + # hook for `platformio-node-helpers` + if host == "__do_not_start__": + return + + # if already started + already_started = False + socket.setdefaulttimeout(1) + try: + socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect((host, port)) + already_started = True + except: # pylint: disable=bare-except + pass + + home_url = "http://%s:%d" % (host, port) + if not no_open: + if already_started: + click.launch(home_url) + else: + reactor.callLater(1, lambda: click.launch(home_url)) + + click.echo("\n".join([ + "", + " ___I_", + " /\\-_--\\ PlatformIO Home", + "/ \\_-__\\", + "|[]| [] | %s" % home_url, + "|__|____|______________%s" % ("_" * len(host)), + ])) + click.echo("") + click.echo("Open PIO Home in your browser by this URL => %s" % home_url) + + if already_started: + return + + click.echo("PIO Home has been started. Press Ctrl+C to shutdown.") + + reactor.listenTCP(port, site, interface=host) + reactor.run() diff --git a/platformio/commands/home/helpers.py b/platformio/commands/home/helpers.py new file mode 100644 index 00000000..abac9416 --- /dev/null +++ b/platformio/commands/home/helpers.py @@ -0,0 +1,69 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=keyword-arg-before-vararg, arguments-differ + +import os +import socket + +import requests +from twisted.internet import threads +from twisted.internet.defer import ensureDeferred + +from platformio import util + + +class AsyncSession(requests.Session): + + def __init__(self, n=None, *args, **kwargs): + if n: + from twisted.internet import reactor + pool = reactor.getThreadPool() + pool.adjustPoolsize(0, n) + + super(AsyncSession, self).__init__(*args, **kwargs) + + def request(self, *args, **kwargs): + func = super(AsyncSession, self).request + return threads.deferToThread(func, *args, **kwargs) + + def wrap(self, *args, **kwargs): # pylint: disable=no-self-use + return ensureDeferred(*args, **kwargs) + + +@util.memoized(expire=5000) +def requests_session(): + return AsyncSession(n=5) + + +@util.memoized() +def get_core_fullpath(): + return util.where_is_program( + "platformio" + (".exe" if "windows" in util.get_systype() else "")) + + +@util.memoized(expire=10000) +def is_twitter_blocked(): + ip = "104.244.42.1" + timeout = 2 + try: + if os.getenv("HTTP_PROXY", os.getenv("HTTPS_PROXY")): + requests.get( + "http://%s" % ip, allow_redirects=False, timeout=timeout) + else: + socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect((ip, 80)) + return False + except: # pylint: disable=bare-except + pass + return True diff --git a/platformio/commands/home/rpc/__init__.py b/platformio/commands/home/rpc/__init__.py new file mode 100644 index 00000000..b0514903 --- /dev/null +++ b/platformio/commands/home/rpc/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/commands/home/rpc/handlers/__init__.py b/platformio/commands/home/rpc/handlers/__init__.py new file mode 100644 index 00000000..b0514903 --- /dev/null +++ b/platformio/commands/home/rpc/handlers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/commands/home/rpc/handlers/app.py b/platformio/commands/home/rpc/handlers/app.py new file mode 100644 index 00000000..ce95a218 --- /dev/null +++ b/platformio/commands/home/rpc/handlers/app.py @@ -0,0 +1,83 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import + +import json +from os.path import expanduser, isfile, join + +from platformio import __version__, app, exception, util + + +class AppRPC(object): + + APPSTATE_PATH = join(util.get_home_dir(), "homestate.json") + + @staticmethod + def load_state(): + state = None + try: + if isfile(AppRPC.APPSTATE_PATH): + state = util.load_json(AppRPC.APPSTATE_PATH) + except exception.PlatformioException: + pass + if not isinstance(state, dict): + state = {} + storage = state.get("storage", {}) + + # base data + caller_id = app.get_session_var("caller_id") + storage['cid'] = app.get_cid() + storage['coreVersion'] = __version__ + storage['coreSystype'] = util.get_systype() + storage['coreCaller'] = (str(caller_id).lower() if caller_id else None) + storage['coreSettings'] = { + name: { + "description": data['description'], + "default_value": data['value'], + "value": app.get_setting(name) + } + for name, data in app.DEFAULT_SETTINGS.items() + } + + # encode to UTF-8 + for key in storage['coreSettings']: + if not key.endswith("dir"): + continue + storage['coreSettings'][key][ + 'default_value'] = util.path_to_unicode( + storage['coreSettings'][key]['default_value']) + storage['coreSettings'][key]['value'] = util.path_to_unicode( + storage['coreSettings'][key]['value']) + storage['homeDir'] = util.path_to_unicode(expanduser("~")) + storage['projectsDir'] = storage['coreSettings']['projects_dir'][ + 'value'] + + # skip non-existing recent projects + storage['recentProjects'] = [ + p for p in storage.get("recentProjects", []) + if util.is_platformio_project(p) + ] + + state['storage'] = storage + return state + + @staticmethod + def get_state(): + return AppRPC.load_state() + + def save_state(self, state): + with open(self.APPSTATE_PATH, "w") as fp: + json.dump(state, fp) + return True diff --git a/platformio/commands/home/rpc/handlers/ide.py b/platformio/commands/home/rpc/handlers/ide.py new file mode 100644 index 00000000..d5c2bf74 --- /dev/null +++ b/platformio/commands/home/rpc/handlers/ide.py @@ -0,0 +1,42 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import time + +from jsonrpc.exceptions import JSONRPCDispatchException +from twisted.internet import defer + + +class IDERPC(object): + + def __init__(self): + self._queue = [] + + def send_command(self, command, params): + if not self._queue: + raise JSONRPCDispatchException( + code=4005, message="PIO Home IDE agent is not started") + while self._queue: + self._queue.pop().callback({ + "id": time.time(), + "method": command, + "params": params + }) + + def listen_commands(self): + self._queue.append(defer.Deferred()) + return self._queue[-1] + + def open_project(self, project_dir): + return self.send_command("open_project", project_dir) diff --git a/platformio/commands/home/rpc/handlers/misc.py b/platformio/commands/home/rpc/handlers/misc.py new file mode 100644 index 00000000..d7deff19 --- /dev/null +++ b/platformio/commands/home/rpc/handlers/misc.py @@ -0,0 +1,194 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import re +import time + +from bs4 import BeautifulSoup +from twisted.internet import defer, reactor + +from platformio import app +from platformio.commands.home import helpers +from platformio.commands.home.rpc.handlers.os import OSRPC + + +class MiscRPC(object): + + def load_latest_tweets(self, username): + cache_key = "piohome_latest_tweets_%s" % username + cache_valid = "7d" + with app.ContentCache() as cc: + cache_data = cc.get(cache_key) + if cache_data: + cache_data = json.loads(cache_data) + # automatically update cache in background every 12 hours + if cache_data['time'] < (time.time() - (3600 * 12)): + reactor.callLater(5, self._preload_latest_tweets, username, + cache_key, cache_valid) + return cache_data['result'] + + result = self._preload_latest_tweets(username, cache_key, cache_valid) + return result + + @defer.inlineCallbacks + def _preload_latest_tweets(self, username, cache_key, cache_valid): + result = yield self._fetch_tweets(username) + with app.ContentCache() as cc: + cc.set(cache_key, + json.dumps({ + "time": int(time.time()), + "result": result + }), cache_valid) + defer.returnValue(result) + + @defer.inlineCallbacks + def _fetch_tweets(self, username): + api_url = ("https://twitter.com/i/profiles/show/%s/timeline/tweets?" + "include_available_features=1&include_entities=1&" + "include_new_items_bar=true") % username + if helpers.is_twitter_blocked(): + api_url = self._get_proxed_uri(api_url) + html_or_json = yield OSRPC.fetch_content( + api_url, headers=self._get_twitter_headers(username)) + # issue with PIO Core < 3.5.3 and ContentCache + if not isinstance(html_or_json, dict): + html_or_json = json.loads(html_or_json) + assert "items_html" in html_or_json + soup = BeautifulSoup(html_or_json['items_html'], "html.parser") + tweet_nodes = soup.find_all( + "div", attrs={ + "class": "tweet", + "data-tweet-id": True + }) + result = yield defer.DeferredList( + [self._parse_tweet_node(node, username) for node in tweet_nodes], + consumeErrors=True) + defer.returnValue([r[1] for r in result if r[0]]) + + @defer.inlineCallbacks + def _parse_tweet_node(self, tweet, username): + # remove non-visible items + for node in tweet.find_all(class_=["invisible", "u-hidden"]): + node.decompose() + twitter_url = "https://twitter.com" + time_node = tweet.find("span", attrs={"data-time": True}) + text_node = tweet.find(class_="tweet-text") + quote_text_node = tweet.find(class_="QuoteTweet-text") + if quote_text_node and not text_node.get_text().strip(): + text_node = quote_text_node + photos = [ + node.get("data-image-url") for node in (tweet.find_all(class_=[ + "AdaptiveMedia-photoContainer", "QuoteMedia-photoContainer" + ]) or []) + ] + urls = [ + node.get("data-expanded-url") + for node in (quote_text_node or text_node).find_all( + class_="twitter-timeline-link", + attrs={"data-expanded-url": True}) + ] + + # fetch data from iframe card + if (not photos or not urls) and tweet.get("data-card2-type"): + iframe_node = tweet.find( + "div", attrs={"data-full-card-iframe-url": True}) + if iframe_node: + iframe_card = yield self._fetch_iframe_card( + twitter_url + iframe_node.get("data-full-card-iframe-url"), + username) + if not photos and iframe_card['photo']: + photos.append(iframe_card['photo']) + if not urls and iframe_card['url']: + urls.append(iframe_card['url']) + if iframe_card['text_node']: + text_node = iframe_card['text_node'] + + if not photos: + photos.append(tweet.find("img", class_="avatar").get("src")) + + def _fetch_text(text_node): + text = text_node.decode_contents(formatter="html").strip() + text = re.sub(r'href="/', 'href="%s/' % twitter_url, text) + if "

" not in text and "", text) + return text + + defer.returnValue({ + "tweetId": + tweet.get("data-tweet-id"), + "tweetUrl": + twitter_url + tweet.get("data-permalink-path"), + "author": + tweet.get("data-name"), + "time": + int(time_node.get("data-time")), + "timeFormatted": + time_node.string, + "text": + _fetch_text(text_node), + "entries": { + "urls": + urls, + "photos": [ + self._get_proxed_uri(uri) + if helpers.is_twitter_blocked() else uri for uri in photos + ] + }, + "isPinned": + "user-pinned" in tweet.get("class") + }) + + @defer.inlineCallbacks + def _fetch_iframe_card(self, url, username): + if helpers.is_twitter_blocked(): + url = self._get_proxed_uri(url) + html = yield OSRPC.fetch_content( + url, headers=self._get_twitter_headers(username), cache_valid="7d") + soup = BeautifulSoup(html, "html.parser") + photo_node = soup.find("img", attrs={"data-src": True}) + url_node = soup.find("a", class_="TwitterCard-container") + text_node = soup.find("div", class_="SummaryCard-content") + if text_node: + text_node.find( + "span", class_="SummaryCard-destination").decompose() + defer.returnValue({ + "photo": + photo_node.get("data-src") if photo_node else None, + "text_node": + text_node, + "url": + url_node.get("href") if url_node else None + }) + + @staticmethod + def _get_proxed_uri(uri): + index = uri.index("://") + return "https://dl.platformio.org/__prx__/" + uri[index + 3:] + + @staticmethod + def _get_twitter_headers(username): + return { + "Accept": + "application/json, text/javascript, */*; q=0.01", + "Referer": + "https://twitter.com/%s" % username, + "User-Agent": + ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit" + "/603.3.8 (KHTML, like Gecko) Version/10.1.2 Safari/603.3.8"), + "X-Twitter-Active-User": + "yes", + "X-Requested-With": + "XMLHttpRequest" + } diff --git a/platformio/commands/home/rpc/handlers/os.py b/platformio/commands/home/rpc/handlers/os.py new file mode 100644 index 00000000..f581c469 --- /dev/null +++ b/platformio/commands/home/rpc/handlers/os.py @@ -0,0 +1,153 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import + +import glob +import os +import shutil +import sys +from functools import cmp_to_key +from os.path import expanduser, isdir, isfile, join + +import click +from twisted.internet import defer + +from platformio import app, util +from platformio.commands.home import helpers + + +class OSRPC(object): + + @staticmethod + @defer.inlineCallbacks + def fetch_content(uri, data=None, headers=None, cache_valid=None): + timeout = 2 + if not headers: + headers = { + "User-Agent": + ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) " + "AppleWebKit/603.3.8 (KHTML, like Gecko) Version/10.1.2 " + "Safari/603.3.8") + } + cache_key = (app.ContentCache.key_from_args(uri, data) + if cache_valid else None) + with app.ContentCache() as cc: + if cache_key: + result = cc.get(cache_key) + if result is not None: + defer.returnValue(result) + + # check internet before and resolve issue with 60 seconds timeout + util.internet_on(raise_exception=True) + + session = helpers.requests_session() + if data: + r = yield session.post( + uri, data=data, headers=headers, timeout=timeout) + else: + r = yield session.get(uri, headers=headers, timeout=timeout) + + r.raise_for_status() + result = r.text + if cache_valid: + with app.ContentCache() as cc: + cc.set(cache_key, result, cache_valid) + defer.returnValue(result) + + def request_content(self, uri, data=None, headers=None, cache_valid=None): + if uri.startswith('http'): + return self.fetch_content(uri, data, headers, cache_valid) + if isfile(uri): + with open(uri) as fp: + return fp.read() + return None + + @staticmethod + def open_url(url): + return click.launch(url) + + @staticmethod + def reveal_file(path): + return click.launch( + path.encode(sys.getfilesystemencoding()) if util.PY2 else path, + locate=True) + + @staticmethod + def is_file(path): + return isfile(path) + + @staticmethod + def is_dir(path): + return isdir(path) + + @staticmethod + def make_dirs(path): + return os.makedirs(path) + + @staticmethod + def rename(src, dst): + return os.rename(src, dst) + + @staticmethod + def copy(src, dst): + return shutil.copytree(src, dst) + + @staticmethod + def glob(pathnames, root=None): + if not isinstance(pathnames, list): + pathnames = [pathnames] + result = set() + for pathname in pathnames: + result |= set( + glob.glob(join(root, pathname) if root else pathname)) + return list(result) + + @staticmethod + def list_dir(path): + + def _cmp(x, y): + if x[1] and not y[1]: + return -1 + if not x[1] and y[1]: + return 1 + if x[0].lower() > y[0].lower(): + return 1 + if x[0].lower() < y[0].lower(): + return -1 + return 0 + + items = [] + if path.startswith("~"): + path = expanduser(path) + if not isdir(path): + return items + for item in os.listdir(path): + try: + item_is_dir = isdir(join(path, item)) + if item_is_dir: + os.listdir(join(path, item)) + items.append((item, item_is_dir)) + except OSError: + pass + return sorted(items, key=cmp_to_key(_cmp)) + + @staticmethod + def get_logical_devices(): + items = [] + for item in util.get_logical_devices(): + if item['name']: + item['name'] = util.path_to_unicode(item['name']) + items.append(item) + return items diff --git a/platformio/commands/home/rpc/handlers/piocore.py b/platformio/commands/home/rpc/handlers/piocore.py new file mode 100644 index 00000000..85255a9c --- /dev/null +++ b/platformio/commands/home/rpc/handlers/piocore.py @@ -0,0 +1,83 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import + +import json +import os +import re +import sys + +from jsonrpc.exceptions import JSONRPCDispatchException +from twisted.internet.utils import getProcessOutputAndValue + +from platformio import __version__, util +from platformio.commands.home import helpers + + +class PIOCoreRPC(object): + + @staticmethod + def call(args, options=None): + json_output = "--json-output" in args + try: + args = [ + arg.encode(sys.getfilesystemencoding()) if isinstance( + arg, util.string_types) else str(arg) for arg in args + ] + except UnicodeError: + raise JSONRPCDispatchException( + code=4002, message="PIO Core: non-ASCII chars in arguments") + d = getProcessOutputAndValue( + helpers.get_core_fullpath(), + args, + path=(options or {}).get("cwd"), + env={k: v + for k, v in os.environ.items() if "%" not in k}) + d.addCallback(PIOCoreRPC._call_callback, json_output) + d.addErrback(PIOCoreRPC._call_errback) + return d + + @staticmethod + def _call_callback(result, json_output=False): + result = list(result) + assert len(result) == 3 + for i in (0, 1): + result[i] = result[i].decode(sys.getfilesystemencoding()).strip() + out, err, code = result + text = ("%s\n\n%s" % (out, err)).strip() + if code != 0: + raise Exception(text) + + if not json_output: + return text + + try: + return json.loads(out) + except ValueError as e: + if "sh: " in out: + return json.loads( + re.sub(r"^sh: [^\n]+$", "", out, flags=re.M).strip()) + raise e + + @staticmethod + def _call_errback(failure): + raise JSONRPCDispatchException( + code=4003, + message="PIO Core Call Error", + data=failure.getErrorMessage()) + + @staticmethod + def version(): + return __version__ diff --git a/platformio/commands/home/rpc/handlers/project.py b/platformio/commands/home/rpc/handlers/project.py new file mode 100644 index 00000000..02961ed8 --- /dev/null +++ b/platformio/commands/home/rpc/handlers/project.py @@ -0,0 +1,283 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import + +import os +import shutil +import sys +import time +from os.path import (basename, expanduser, getmtime, isdir, isfile, join, + realpath, sep) + +from jsonrpc.exceptions import JSONRPCDispatchException + +from platformio import exception, util +from platformio.commands.home.rpc.handlers.app import AppRPC +from platformio.commands.home.rpc.handlers.piocore import PIOCoreRPC +from platformio.ide.projectgenerator import ProjectGenerator +from platformio.managers.platform import PlatformManager + +try: + from configparser import Error as ConfigParserError +except ImportError: + from ConfigParser import Error as ConfigParserError + + +class ProjectRPC(object): + + @staticmethod + def _get_projects(project_dirs=None): + + def _get_project_data(project_dir): + data = {"boards": [], "libExtraDirs": []} + config = util.load_project_config(project_dir) + + if config.has_section("platformio") and \ + config.has_option("platformio", "lib_extra_dirs"): + data['libExtraDirs'].extend( + util.parse_conf_multi_values( + config.get("platformio", "lib_extra_dirs"))) + + for section in config.sections(): + if not section.startswith("env:"): + continue + if config.has_option(section, "board"): + data['boards'].append(config.get(section, "board")) + if config.has_option(section, "lib_extra_dirs"): + data['libExtraDirs'].extend( + util.parse_conf_multi_values( + config.get(section, "lib_extra_dirs"))) + + # resolve libExtraDirs paths + with util.cd(project_dir): + data['libExtraDirs'] = [ + expanduser(d) if d.startswith("~") else realpath(d) + for d in data['libExtraDirs'] + ] + + # skip non existing folders + data['libExtraDirs'] = [ + d for d in data['libExtraDirs'] if isdir(d) + ] + + return data + + def _path_to_name(path): + return (sep).join(path.split(sep)[-2:]) + + if not project_dirs: + project_dirs = AppRPC.load_state()['storage']['recentProjects'] + + result = [] + pm = PlatformManager() + for project_dir in project_dirs: + data = {} + boards = [] + try: + data = _get_project_data(project_dir) + except exception.NotPlatformIOProject: + continue + except ConfigParserError: + pass + + for board_id in data.get("boards", []): + name = board_id + try: + name = pm.board_config(board_id)['name'] + except (exception.UnknownBoard, exception.UnknownPlatform): + pass + boards.append({"id": board_id, "name": name}) + + result.append({ + "path": + project_dir, + "name": + _path_to_name(project_dir), + "modified": + int(getmtime(project_dir)), + "boards": + boards, + "extraLibStorages": [{ + "name": _path_to_name(d), + "path": d + } for d in data.get("libExtraDirs", [])] + }) + return result + + def get_projects(self, project_dirs=None): + return self._get_projects(project_dirs) + + def init(self, board, framework, project_dir): + assert project_dir + state = AppRPC.load_state() + if not isdir(project_dir): + os.makedirs(project_dir) + args = ["init", "--project-dir", project_dir, "--board", board] + if framework: + args.extend(["--project-option", "framework = %s" % framework]) + if (state['storage']['coreCaller'] and state['storage']['coreCaller'] + in ProjectGenerator.get_supported_ides()): + args.extend(["--ide", state['storage']['coreCaller']]) + d = PIOCoreRPC.call(args) + d.addCallback(self._generate_project_main, project_dir, framework) + return d + + @staticmethod + def _generate_project_main(_, project_dir, framework): + main_content = None + if framework == "arduino": + main_content = "\n".join([ + "#include ", + "", + "void setup() {", + " // put your setup code here, to run once:", + "}", + "", + "void loop() {", + " // put your main code here, to run repeatedly:", + "}" + "" + ]) # yapf: disable + elif framework == "mbed": + main_content = "\n".join([ + "#include ", + "", + "int main() {", + "", + " // put your setup code here, to run once:", + "", + " while(1) {", + " // put your main code here, to run repeatedly:", + " }", + "}", + "" + ]) # yapf: disable + if not main_content: + return project_dir + with util.cd(project_dir): + src_dir = util.get_projectsrc_dir() + main_path = join(src_dir, "main.cpp") + if isfile(main_path): + return project_dir + if not isdir(src_dir): + os.makedirs(src_dir) + with open(main_path, "w") as f: + f.write(main_content.strip()) + return project_dir + + def import_arduino(self, board, use_arduino_libs, arduino_project_dir): + # don't import PIO Project + if util.is_platformio_project(arduino_project_dir): + return arduino_project_dir + + is_arduino_project = any([ + isfile( + join(arduino_project_dir, + "%s.%s" % (basename(arduino_project_dir), ext))) + for ext in ("ino", "pde") + ]) + if not is_arduino_project: + raise JSONRPCDispatchException( + code=4000, + message="Not an Arduino project: %s" % arduino_project_dir) + + state = AppRPC.load_state() + project_dir = join(state['storage']['projectsDir'].decode("utf-8"), + time.strftime("%y%m%d-%H%M%S-") + board) + if not isdir(project_dir): + os.makedirs(project_dir) + args = ["init", "--project-dir", project_dir, "--board", board] + args.extend(["--project-option", "framework = arduino"]) + if use_arduino_libs: + args.extend([ + "--project-option", + "lib_extra_dirs = ~/Documents/Arduino/libraries" + ]) + if (state['storage']['coreCaller'] and state['storage']['coreCaller'] + in ProjectGenerator.get_supported_ides()): + args.extend(["--ide", state['storage']['coreCaller']]) + d = PIOCoreRPC.call(args) + d.addCallback(self._finalize_arduino_import, project_dir, + arduino_project_dir) + return d + + @staticmethod + def _finalize_arduino_import(_, project_dir, arduino_project_dir): + with util.cd(project_dir): + src_dir = util.get_projectsrc_dir() + if isdir(src_dir): + util.rmtree_(src_dir) + shutil.copytree( + arduino_project_dir.encode(sys.getfilesystemencoding()), + src_dir) + return project_dir + + @staticmethod + def get_project_examples(): + result = [] + for manifest in PlatformManager().get_installed(): + examples_dir = join(manifest['__pkg_dir'], "examples") + if not isdir(examples_dir): + continue + items = [] + for project_dir, _, __ in os.walk(examples_dir): + project_description = None + try: + config = util.load_project_config(project_dir) + if config.has_section("platformio") and \ + config.has_option("platformio", "description"): + project_description = config.get( + "platformio", "description") + except (exception.NotPlatformIOProject, + exception.InvalidProjectConf): + continue + + path_tokens = project_dir.split(sep) + items.append({ + "name": + "/".join(path_tokens[path_tokens.index("examples") + 1:]), + "path": + project_dir, + "description": + project_description + }) + result.append({ + "platform": { + "title": manifest['title'], + "version": manifest['version'] + }, + "items": sorted(items, key=lambda item: item['name']) + }) + return sorted(result, key=lambda data: data['platform']['title']) + + @staticmethod + def import_pio(project_dir): + if not project_dir or not util.is_platformio_project(project_dir): + raise JSONRPCDispatchException( + code=4001, + message="Not an PlatformIO project: %s" % project_dir) + new_project_dir = join( + AppRPC.load_state()['storage']['projectsDir'].decode("utf-8"), + time.strftime("%y%m%d-%H%M%S-") + basename(project_dir)) + shutil.copytree(project_dir, new_project_dir) + + state = AppRPC.load_state() + args = ["init", "--project-dir", new_project_dir] + if (state['storage']['coreCaller'] and state['storage']['coreCaller'] + in ProjectGenerator.get_supported_ides()): + args.extend(["--ide", state['storage']['coreCaller']]) + d = PIOCoreRPC.call(args) + d.addCallback(lambda _: new_project_dir) + return d diff --git a/platformio/commands/home/rpc/server.py b/platformio/commands/home/rpc/server.py new file mode 100644 index 00000000..cd4f05b9 --- /dev/null +++ b/platformio/commands/home/rpc/server.py @@ -0,0 +1,70 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json + +import jsonrpc +from autobahn.twisted.websocket import (WebSocketServerFactory, + WebSocketServerProtocol) +from jsonrpc.exceptions import JSONRPCDispatchException +from twisted.internet import defer + + +class JSONRPCServerProtocol(WebSocketServerProtocol): + + def onMessage(self, payload, isBinary): # pylint: disable=unused-argument + # print("> %s" % payload) + response = jsonrpc.JSONRPCResponseManager.handle( + payload, self.factory.dispatcher).data + # if error + if "result" not in response: + self.sendJSONResponse(response) + return None + + d = defer.maybeDeferred(lambda: response['result']) + d.addCallback(self._callback, response) + d.addErrback(self._errback, response) + + return None + + def _callback(self, result, response): + response['result'] = result + self.sendJSONResponse(response) + + def _errback(self, failure, response): + if isinstance(failure.value, JSONRPCDispatchException): + e = failure.value + else: + e = JSONRPCDispatchException( + code=4999, message=failure.getErrorMessage()) + del response["result"] + response['error'] = e.error._data # pylint: disable=protected-access + print(response['error']) + self.sendJSONResponse(response) + + def sendJSONResponse(self, response): + # print("< %s" % response) + self.sendMessage(json.dumps(response).encode("utf8")) + + +class JSONRPCServerFactory(WebSocketServerFactory): + + protocol = JSONRPCServerProtocol + + def __init__(self): + super(JSONRPCServerFactory, self).__init__() + self.dispatcher = jsonrpc.Dispatcher() + + def addHandler(self, handler, namespace): + self.dispatcher.build_method_map(handler, prefix="%s." % namespace) diff --git a/platformio/commands/home.py b/platformio/commands/home/web.py similarity index 54% rename from platformio/commands/home.py rename to platformio/commands/home/web.py index cd6b86f6..313ce084 100644 --- a/platformio/commands/home.py +++ b/platformio/commands/home/web.py @@ -12,20 +12,19 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys - -import click - -from platformio.managers.core import pioplus_call +from twisted.internet import reactor +from twisted.web import static -@click.command("home", short_help="PIO Home") -@click.option("--port", type=int, default=8008, help="HTTP port, default=8008") -@click.option( - "--host", - default="127.0.0.1", - help="HTTP host, default=127.0.0.1. " - "You can open PIO Home for inbound connections with --host=0.0.0.0") -@click.option("--no-open", is_flag=True) -def cli(*args, **kwargs): # pylint: disable=unused-argument - pioplus_call(sys.argv[1:]) +class WebRoot(static.File): + + def render_GET(self, request): + if request.args.get("__shutdown__", False): + reactor.stop() + return "Server has been stopped" + + request.setHeader("cache-control", + "no-cache, no-store, must-revalidate") + request.setHeader("pragma", "no-cache") + request.setHeader("expires", "0") + return static.File.render_GET(self, request)