Added platforms frame logic; implemented "install" and "run" commands

platformio/__init__.py

@@ -14,3 +14,5 @@ __email__ = "me@ikravets.com"
 
 __license__ = "MIT Licence"
 __copyright__ = "Copyright (C) 2014 Ivan Kravets"
+
+__pkgmanifesturl__ = "http://192.168.0.13/packages/manifest.json"

platformio/__main__.py

@@ -3,11 +3,13 @@
 
 from os import listdir
 from os.path import join
-from sys import exit
+from sys import exit as sys_exit
+from traceback import format_exc
 
-from click import command, MultiCommand, version_option
+from click import command, MultiCommand, style, version_option
 
 from platformio import __version__
+from platformio.exception import PlatformioException, UnknownCLICommand
 from platformio.util import get_source_dir
 
 
@@ -24,7 +26,11 @@ class PlatformioCLI(MultiCommand):
         return cmds
 
     def get_command(self, ctx, name):
-        mod = __import__("platformio.commands." + name, None, None, ["cli"])
+        try:
+            mod = __import__("platformio.commands." + name,
+                             None, None, ["cli"])
+        except ImportError:
+            raise UnknownCLICommand(name)
         return mod.cli
 
 
@@ -35,8 +41,14 @@ def cli():
 
 
 def main():
+    try:
         cli()
+    except Exception as e:  # pylint: disable=W0703
+        if isinstance(e, PlatformioException):
+            sys_exit(style("Error: ", fg="red") + str(e))
+        else:
+            print format_exc()
 
 
 if __name__ == "__main__":
-    exit(main())
+    sys_exit(main())

platformio/commands/install.py

@@ -0,0 +1,19 @@
+# Copyright (C) Ivan Kravets <me@ikravets.com>
+# See LICENSE for details.
+
+from click import argument, command, option, secho
+
+from platformio.platforms.base import PlatformFactory
+
+
+@command("run", short_help="Install new platforms")
+@argument("platform")
+@option('--with-package', multiple=True, metavar="<package>")
+@option('--without-package', multiple=True, metavar="<package>")
+def cli(platform, with_package, without_package):
+
+    p = PlatformFactory().newPlatform(platform)
+
+    if p.install(with_package, without_package):
+        secho("The platform '%s' has been successfully installed!" % platform,
+              fg="green")

platformio/commands/run.py

@@ -1,21 +1,24 @@
 # Copyright (C) Ivan Kravets <me@ikravets.com>
 # See LICENSE for details.
 
-from click import command, echo, option, style
+from click import command, echo, option, secho, style
 
-from platformio.util import get_project_config, run_builder, textindent
+from platformio.exception import UndefinedEnvPlatform
+from platformio.platforms.base import PlatformFactory
+from platformio.util import get_project_config
 
 
 @command("run", short_help="Process project environments")
 @option("--environment", "-e", multiple=True)
 @option("--target", "-t", multiple=True)
 def cli(environment, target):
+
     config = get_project_config()
     for section in config.sections():
         if section[:4] != "env:":
             continue
-        envname = section[4:]
 
+        envname = section[4:]
         if environment and envname not in environment:
             echo("Skipped %s environment" % style(envname, fg="yellow"))
             continue
@@ -31,6 +34,10 @@ def cli(environment, target):
         elif config.has_option(section, "targets"):
             envtargets = config.get(section, "targets").split()
 
-        result = run_builder(variables, envtargets)
+        if not config.has_option(section, "platform"):
-        echo(textindent(style(result['out'], fg="green"), ".    "))
+            raise UndefinedEnvPlatform(envname)
-        echo(textindent(style(result['err'], fg="red"), ".    "))
+
+        p = PlatformFactory().newPlatform(config.get(section, "platform"))
+        result = p.run(variables, envtargets)
+        secho(result['out'], fg="green")
+        secho(result['err'], fg="red")

platformio/downloader.py

@@ -0,0 +1,82 @@
+# Copyright (C) Ivan Kravets <me@ikravets.com>
+# See LICENSE for details.
+
+from email.utils import parsedate_tz
+from math import ceil
+from os.path import getsize, join
+from subprocess import check_output
+from time import mktime
+
+from click import progressbar
+from requests import get
+
+from platformio.exception import (FDSHASumMismatch, FDSizeMismatch,
+                                  FDUnrecognizedStatusCode)
+from platformio.util import change_filemtime
+
+
+class FileDownloader(object):
+
+    CHUNK_SIZE = 1024
+
+    def __init__(self, url, dest_dir=None):
+        self._url = url
+        self._fname = url.split("/")[-1]
+
+        self._destination = self._fname
+        if dest_dir:
+            self.set_destination(join(dest_dir, self._fname))
+        self._progressbar = None
+
+        self._request = get(url, stream=True)
+        if self._request.status_code != 200:
+            raise FDUnrecognizedStatusCode(self._request.status_code, url)
+
+    def set_destination(self, destination):
+        self._destination = destination
+
+    def get_filepath(self):
+        return self._destination
+
+    def get_lmtime(self):
+        return self._request.headers['last-modified']
+
+    def get_size(self):
+        return int(self._request.headers['content-length'])
+
+    def start(self):
+        itercontent = self._request.iter_content(chunk_size=self.CHUNK_SIZE)
+        f = open(self._destination, "wb")
+        chunks = int(ceil(self.get_size() / float(self.CHUNK_SIZE)))
+
+        with progressbar(length=chunks, label="Downloading") as pb:
+            for _ in pb:
+                f.write(next(itercontent))
+        f.close()
+        self._request.close()
+
+        self._preserve_filemtime(self.get_lmtime())
+
+    def verify(self, sha1=None):
+        _dlsize = getsize(self._destination)
+        if _dlsize != self.get_size():
+            raise FDSizeMismatch(_dlsize, self._fname, self.get_size())
+
+        if not sha1:
+            return
+
+        try:
+            res = check_output(["shasum", self._destination])
+            dlsha1 = res[:40]
+            if sha1 != dlsha1:
+                raise FDSHASumMismatch(dlsha1, self._fname, sha1)
+        except OSError:
+            pass
+
+    def _preserve_filemtime(self, lmdate):
+        timedata = parsedate_tz(lmdate)
+        lmtime = mktime(timedata[:9])
+        change_filemtime(self._destination, lmtime)
+
+    def __del__(self):
+        self._request.close()

platformio/exception.py

@@ -0,0 +1,75 @@
+# Copyright (C) Ivan Kravets <me@ikravets.com>
+# See LICENSE for details.
+
+
+class PlatformioException(Exception):
+
+    MESSAGE = None
+
+    def __str__(self):  # pragma: no cover
+        if self.MESSAGE:
+            return self.MESSAGE % self.args
+        else:
+            return Exception.__str__(self)
+
+
+class UnknownPlatform(PlatformioException):
+
+    MESSAGE = "Unknown platform '%s'"
+
+
+class UnknownCLICommand(PlatformioException):
+
+    MESSAGE = "Unknown command '%s'"
+
+
+class UnknownPackage(PlatformioException):
+
+    MESSAGE = "Detected unknown package '%s'"
+
+
+class InvalidPackageVersion(PlatformioException):
+
+    MESSAGE = "The package '%s' with version '%d' does not exist"
+
+
+class PackageInstalled(PlatformioException):
+
+    MESSAGE = "The package '%s' is installed already"
+
+
+class NonSystemPackage(PlatformioException):
+
+    MESSAGE = "The package '%s' is not available for your system '%s'"
+
+
+class FDUnrecognizedStatusCode(PlatformioException):
+
+    MESSAGE = "Got an unrecognized status code '%s' when downloaded %s"
+
+
+class FDSizeMismatch(PlatformioException):
+
+    MESSAGE = ("The size (%d bytes) of downloaded file '%s' "
+               "is not equal to remote size (%d bytes)")
+
+
+class FDSHASumMismatch(PlatformioException):
+
+    MESSAGE = ("The 'sha1' sum '%s' of downloaded file '%s' "
+               "is not equal to remote '%s'")
+
+
+class NotPlatformProject(PlatformioException):
+
+    MESSAGE = "Not a platformio project. Use `platformio init` command"
+
+
+class UndefinedEnvPlatform(PlatformioException):
+
+    MESSAGE = "Please specify platform for '%s' environment"
+
+
+class UnsupportedArchiveType(PlatformioException):
+
+    MESSAGE = "Can not unpack file '%s'"

platformio/pkgmanager.py

@@ -0,0 +1,97 @@
+# Copyright (C) Ivan Kravets <me@ikravets.com>
+# See LICENSE for details.
+
+import json
+from os import makedirs, remove
+from os.path import isdir, isfile, join
+
+from requests import get
+
+from platformio import __pkgmanifesturl__
+from platformio.downloader import FileDownloader
+from platformio.exception import (InvalidPackageVersion, NonSystemPackage,
+                                  PackageInstalled, UnknownPackage)
+from platformio.unpacker import FileUnpacker
+from platformio.util import get_home_dir, get_system
+
+
+class PackageManager(object):
+
+    def __init__(self, platform_name):
+        self._platform_name = platform_name
+        self._platforms_dir = get_home_dir()
+        self._dbfile = join(self._platforms_dir, "installed.json")
+
+    @staticmethod
+    def get_manifest():
+        return get(__pkgmanifesturl__).json()
+
+    @staticmethod
+    def download(url, dest_dir, sha1=None):
+        fd = FileDownloader(url, dest_dir)
+        fd.start()
+        fd.verify(sha1)
+        return fd.get_filepath()
+
+    @staticmethod
+    def unpack(pkgpath, dest_dir):
+        fu = FileUnpacker(pkgpath, dest_dir)
+        return fu.start()
+
+    def get_installed(self):
+        data = {}
+        if isfile(self._dbfile):
+            with open(self._dbfile) as fp:
+                data = json.load(fp)
+        return data
+
+    def is_installed(self, name):
+        installed = self.get_installed()
+        return (self._platform_name in installed and name in
+                installed[self._platform_name])
+
+    def get_info(self, name, version=None):
+        if self.is_installed(name):
+            raise PackageInstalled(name)
+
+        manifest = self.get_manifest()
+        if name not in manifest:
+            raise UnknownPackage(name)
+
+        # check system platform
+        system = get_system()
+        builds = ([b for b in manifest[name] if b['system'] == "all" or system
+                   in b['system']])
+        if not builds:
+            raise NonSystemPackage(name, system)
+
+        if version:
+            for b in builds:
+                if b['version'] == version:
+                    return b
+            raise InvalidPackageVersion(name, version)
+        else:
+            return sorted(builds, key=lambda s: s['version'])[-1]
+
+    def install(self, name, path):
+        info = self.get_info(name)
+        pkg_dir = join(self._platforms_dir, self._platform_name, path)
+        if not isdir(pkg_dir):
+            makedirs(pkg_dir)
+
+        dlpath = self.download(info['url'], pkg_dir, info['sha1'])
+        if self.unpack(dlpath, pkg_dir):
+            self._register(name, info['version'], path)
+        # remove archive
+        remove(dlpath)
+
+    def _register(self, name, version, path):
+        data = self.get_installed()
+        if self._platform_name not in data:
+            data[self._platform_name] = {}
+        data[self._platform_name][name] = {
+            "version": version,
+            "path": path
+        }
+        with open(self._dbfile, "w") as fp:
+            json.dump(data, fp)

platformio/platforms/__init__.py

@@ -0,0 +1,2 @@
+# Copyright (C) Ivan Kravets <me@ikravets.com>
+# See LICENSE for details.

platformio/platforms/atmelavr.py

@@ -0,0 +1,37 @@
+# Copyright (C) Ivan Kravets <me@ikravets.com>
+# See LICENSE for details.
+
+from os.path import join
+
+from platformio.platforms.base import BasePlatform
+
+
+class AtmelavrPlatform(BasePlatform):
+
+    PACKAGES = {
+
+        "toolchain-atmelavr": {
+            "path": join("tools", "toolchain"),
+            "default": True
+        },
+
+        "tool-avrdude": {
+            "path": join("tools", "avrdude"),
+            "default": True,
+        },
+
+        "framework-arduinoavr": {
+            "path": join("frameworks", "arduino"),
+            "default": False
+        }
+    }
+
+    def get_name(self):
+        return "atmelavr"
+
+    def after_run(self, result):
+        # fix STDERR "flash written" for avrdude
+        if "flash written" in result['err']:
+            result['out'] += "\n" + result['err']
+            result['err'] = ""
+        return result

platformio/platforms/base.py

@@ -0,0 +1,78 @@
+# Copyright (C) Ivan Kravets <me@ikravets.com>
+# See LICENSE for details.
+
+from os.path import join
+
+from click import echo, secho, style
+
+from platformio.exception import (PackageInstalled, UnknownPackage,
+                                  UnknownPlatform)
+from platformio.pkgmanager import PackageManager
+from platformio.util import exec_command, get_source_dir
+
+
+class PlatformFactory(object):
+
+    @staticmethod
+    def newPlatform(name):
+        clsname = "%sPlatform" % name.title()
+        try:
+            mod = __import__("platformio.platforms." + name.lower(),
+                             None, None, [clsname])
+        except ImportError:
+            raise UnknownPlatform(name)
+
+        obj = getattr(mod, clsname)()
+        assert isinstance(obj, BasePlatform)
+        return obj
+
+
+class BasePlatform(object):
+
+    PACKAGES = {}
+
+    def get_name(self):
+        raise NotImplementedError()
+
+    def install(self, with_packages, without_packages):
+        requirements = []
+        pm = PackageManager(self.get_name())
+
+        upkgs = set(with_packages + without_packages)
+        ppkgs = set(self.PACKAGES.keys())
+        if not upkgs.issubset(ppkgs):
+            raise UnknownPackage(", ".join(upkgs - ppkgs))
+
+        for name, opts in self.PACKAGES.iteritems():
+            if name in without_packages:
+                continue
+            elif name in with_packages or opts["default"]:
+                requirements.append((name, opts["path"]))
+
+        for (name, path) in requirements:
+            echo("Installing %s package:" % style(name, fg="cyan"))
+            try:
+                pm.install(name, path)
+            except PackageInstalled:
+                secho("Already installed", fg="yellow")
+
+        return True
+
+    def after_run(self, result):  # pylint: disable=R0201
+        return result
+
+    def run(self, variables, targets):
+        assert isinstance(variables, list)
+        assert isinstance(targets, list)
+
+        if "clean" in targets:
+            targets.remove("clean")
+            targets.append("-c")
+
+        result = exec_command([
+            "scons",
+            "-Q",
+            "-f", join(get_source_dir(), "builder", "main.py")
+        ] + variables + targets)
+
+        return self.after_run(result)

platformio/platforms/timsp430.py

@@ -0,0 +1,30 @@
+# Copyright (C) Ivan Kravets <me@ikravets.com>
+# See LICENSE for details.
+
+from os.path import join
+
+from platformio.platforms.base import BasePlatform
+
+
+class Timsp430Platform(BasePlatform):
+
+    PACKAGES = {
+
+        "toolchain-timsp430": {
+            "path": join("tools", "toolchain"),
+            "default": True
+        },
+
+        "tool-mspdebug": {
+            "path": join("tools", "mspdebug"),
+            "default": True,
+        },
+
+        "framework-energiamsp430": {
+            "path": join("frameworks", "energia"),
+            "default": False
+        }
+    }
+
+    def get_name(self):
+        return "timsp430"

platformio/platforms/titiva.py

@@ -0,0 +1,30 @@
+# Copyright (C) Ivan Kravets <me@ikravets.com>
+# See LICENSE for details.
+
+from os.path import join
+
+from platformio.platforms.base import BasePlatform
+
+
+class TitivaPlatform(BasePlatform):
+
+    PACKAGES = {
+
+        "toolchain-titiva": {
+            "path": join("tools", "toolchain"),
+            "default": True
+        },
+
+        "tool-lm4flash": {
+            "path": join("tools", "lm4flash"),
+            "default": True,
+        },
+
+        "framework-energiativa": {
+            "path": join("frameworks", "energia"),
+            "default": False
+        }
+    }
+
+    def get_name(self):
+        return "titiva"

platformio/unpacker.py

@@ -0,0 +1,87 @@
+# Copyright (C) Ivan Kravets <me@ikravets.com>
+# See LICENSE for details.
+
+from os import chmod
+from os.path import join, splitext
+from tarfile import open as tarfile_open
+from time import mktime
+from zipfile import ZipFile
+
+from click import progressbar
+
+from platformio.exception import UnsupportedArchiveType
+from platformio.util import change_filemtime
+
+
+class ArchiveBase(object):
+
+    def __init__(self, arhfileobj):
+        self._afo = arhfileobj
+
+    def get_items(self):
+        raise NotImplementedError()
+
+    def extract_item(self, item, dest_dir):
+        self._afo.extract(item, dest_dir)
+        self.after_extract(item, dest_dir)
+
+    def after_extract(self, item, dest_dir):
+        pass
+
+
+class TARArchive(ArchiveBase):
+
+    def __init__(self, archpath):
+        ArchiveBase.__init__(self, tarfile_open(archpath))
+
+    def get_items(self):
+        return self._afo.getmembers()
+
+
+class ZIPArchive(ArchiveBase):
+
+    def __init__(self, archpath):
+        ArchiveBase.__init__(self, ZipFile(archpath))
+
+    @staticmethod
+    def preserve_permissions(item, dest_dir):
+        attrs = item.external_attr >> 16L
+        if attrs:
+            chmod(join(dest_dir, item.filename), attrs)
+
+    @staticmethod
+    def preserve_mtime(item, dest_dir):
+        change_filemtime(
+            join(dest_dir, item.filename),
+            mktime(list(item.date_time) + [0]*3)
+        )
+
+    def get_items(self):
+        return self._afo.infolist()
+
+    def after_extract(self, item, dest_dir):
+        self.preserve_permissions(item, dest_dir)
+        self.preserve_mtime(item, dest_dir)
+
+
+class FileUnpacker(object):
+
+    def __init__(self, archpath, dest_dir="."):
+        self._archpath = archpath
+        self._dest_dir = dest_dir
+        self._unpacker = None
+
+        _, archext = splitext(archpath.lower())
+        if archext in (".gz", ".bz2"):
+            self._unpacker = TARArchive(archpath)
+        elif archext == ".zip":
+            self._unpacker = ZIPArchive(archpath)
+
+        if not self._unpacker:
+            raise UnsupportedArchiveType(archpath)
+
+    def start(self):
+        with progressbar(self._unpacker.get_items(), label="Unpacking") as pb:
+            for item in pb:
+                self._unpacker.extract_item(item, self._dest_dir)
+        return True

platformio/util.py

@@ -1,11 +1,12 @@
 # Copyright (C) Ivan Kravets <me@ikravets.com>
 # See LICENSE for details.
 
-from os import getcwd
+from os import getcwd, utime
 from os.path import dirname, expanduser, isfile, join, realpath
+from platform import architecture, system
 from subprocess import PIPE, Popen
-from sys import exit
+
-from textwrap import fill
+from platformio.exception import NotPlatformProject
 
 try:
     from configparser import ConfigParser
@@ -26,48 +27,23 @@ def get_project_dir():
 
 
 def get_project_config():
-    try:
-        return getattr(get_project_config, "_cache")
-    except AttributeError:
-        pass
-
     path = join(get_project_dir(), "platformio.ini")
     if not isfile(path):
-        exit("Not a platformio project. Use `platformio init` command")
+        raise NotPlatformProject()
-
+    cp = ConfigParser()
-    get_project_config._cache = ConfigParser()
+    cp.read(path)
-    get_project_config._cache.read(path)
+    return cp
-    return get_project_config._cache
 
 
-def textindent(text, quote):
+def get_system():
-    return fill(text, initial_indent=quote,
+    return (system() + architecture()[0][:-3]).lower()
-                subsequent_indent=quote, width=120)
+
+
+def change_filemtime(path, time):
+    utime(path, (time, time))
 
 
 def exec_command(args):
     p = Popen(args, stdout=PIPE, stderr=PIPE)
     out, err = p.communicate()
-    result = dict(out=out.strip(), err=err.strip())
+    return dict(out=out.strip(), err=err.strip())
-
-    # fix STDERR "flash written"
-    if "flash written" in result['err']:
-        result['out'] += "\n" + result['err']
-        result['err'] = ""
-
-    return result
-
-
-def run_builder(variables, targets):
-    assert isinstance(variables, list)
-    assert isinstance(targets, list)
-
-    if "clean" in targets:
-        targets.remove("clean")
-        targets.append("-c")
-
-    return exec_command([
-        "scons",
-        "-Q",
-        "-f", join(get_source_dir(), "builder", "main.py")
-    ] + variables + targets)