From 937c38a7b7c4a2e420ca60038bcc9cf2f90fec62 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 7 Jun 2014 13:34:31 +0300 Subject: [PATCH] Added platforms frame logic; implemented "install" and "run" commands --- platformio/__init__.py | 2 + platformio/__main__.py | 22 ++++++-- platformio/commands/install.py | 19 +++++++ platformio/commands/run.py | 19 +++++-- platformio/downloader.py | 82 +++++++++++++++++++++++++++ platformio/exception.py | 75 ++++++++++++++++++++++++ platformio/pkgmanager.py | 97 ++++++++++++++++++++++++++++++++ platformio/platforms/__init__.py | 2 + platformio/platforms/atmelavr.py | 37 ++++++++++++ platformio/platforms/base.py | 78 +++++++++++++++++++++++++ platformio/platforms/timsp430.py | 30 ++++++++++ platformio/platforms/titiva.py | 30 ++++++++++ platformio/unpacker.py | 87 ++++++++++++++++++++++++++++ platformio/util.py | 56 ++++++------------ 14 files changed, 585 insertions(+), 51 deletions(-) create mode 100644 platformio/commands/install.py create mode 100644 platformio/downloader.py create mode 100644 platformio/exception.py create mode 100644 platformio/pkgmanager.py create mode 100644 platformio/platforms/__init__.py create mode 100644 platformio/platforms/atmelavr.py create mode 100644 platformio/platforms/base.py create mode 100644 platformio/platforms/timsp430.py create mode 100644 platformio/platforms/titiva.py create mode 100644 platformio/unpacker.py diff --git a/platformio/__init__.py b/platformio/__init__.py index db04a2b1..f1a9c450 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,3 +14,5 @@ __email__ = "me@ikravets.com" __license__ = "MIT Licence" __copyright__ = "Copyright (C) 2014 Ivan Kravets" + +__pkgmanifesturl__ = "http://192.168.0.13/packages/manifest.json" diff --git a/platformio/__main__.py b/platformio/__main__.py index d1cc6852..a7a74f2a 100644 --- a/platformio/__main__.py +++ b/platformio/__main__.py @@ -3,11 +3,13 @@ from os import listdir from os.path import join -from sys import exit +from sys import exit as sys_exit +from traceback import format_exc -from click import command, MultiCommand, version_option +from click import command, MultiCommand, style, version_option from platformio import __version__ +from platformio.exception import PlatformioException, UnknownCLICommand from platformio.util import get_source_dir @@ -24,7 +26,11 @@ class PlatformioCLI(MultiCommand): return cmds def get_command(self, ctx, name): - mod = __import__("platformio.commands." + name, None, None, ["cli"]) + try: + mod = __import__("platformio.commands." + name, + None, None, ["cli"]) + except ImportError: + raise UnknownCLICommand(name) return mod.cli @@ -35,8 +41,14 @@ def cli(): def main(): - cli() + try: + cli() + except Exception as e: # pylint: disable=W0703 + if isinstance(e, PlatformioException): + sys_exit(style("Error: ", fg="red") + str(e)) + else: + print format_exc() if __name__ == "__main__": - exit(main()) + sys_exit(main()) diff --git a/platformio/commands/install.py b/platformio/commands/install.py new file mode 100644 index 00000000..52f1ac89 --- /dev/null +++ b/platformio/commands/install.py @@ -0,0 +1,19 @@ +# Copyright (C) Ivan Kravets +# See LICENSE for details. + +from click import argument, command, option, secho + +from platformio.platforms.base import PlatformFactory + + +@command("run", short_help="Install new platforms") +@argument("platform") +@option('--with-package', multiple=True, metavar="") +@option('--without-package', multiple=True, metavar="") +def cli(platform, with_package, without_package): + + p = PlatformFactory().newPlatform(platform) + + if p.install(with_package, without_package): + secho("The platform '%s' has been successfully installed!" % platform, + fg="green") diff --git a/platformio/commands/run.py b/platformio/commands/run.py index b626ad0b..9ba9a408 100644 --- a/platformio/commands/run.py +++ b/platformio/commands/run.py @@ -1,21 +1,24 @@ # Copyright (C) Ivan Kravets # See LICENSE for details. -from click import command, echo, option, style +from click import command, echo, option, secho, style -from platformio.util import get_project_config, run_builder, textindent +from platformio.exception import UndefinedEnvPlatform +from platformio.platforms.base import PlatformFactory +from platformio.util import get_project_config @command("run", short_help="Process project environments") @option("--environment", "-e", multiple=True) @option("--target", "-t", multiple=True) def cli(environment, target): + config = get_project_config() for section in config.sections(): if section[:4] != "env:": continue - envname = section[4:] + envname = section[4:] if environment and envname not in environment: echo("Skipped %s environment" % style(envname, fg="yellow")) continue @@ -31,6 +34,10 @@ def cli(environment, target): elif config.has_option(section, "targets"): envtargets = config.get(section, "targets").split() - result = run_builder(variables, envtargets) - echo(textindent(style(result['out'], fg="green"), ". ")) - echo(textindent(style(result['err'], fg="red"), ". ")) + if not config.has_option(section, "platform"): + raise UndefinedEnvPlatform(envname) + + p = PlatformFactory().newPlatform(config.get(section, "platform")) + result = p.run(variables, envtargets) + secho(result['out'], fg="green") + secho(result['err'], fg="red") diff --git a/platformio/downloader.py b/platformio/downloader.py new file mode 100644 index 00000000..106e45bd --- /dev/null +++ b/platformio/downloader.py @@ -0,0 +1,82 @@ +# Copyright (C) Ivan Kravets +# See LICENSE for details. + +from email.utils import parsedate_tz +from math import ceil +from os.path import getsize, join +from subprocess import check_output +from time import mktime + +from click import progressbar +from requests import get + +from platformio.exception import (FDSHASumMismatch, FDSizeMismatch, + FDUnrecognizedStatusCode) +from platformio.util import change_filemtime + + +class FileDownloader(object): + + CHUNK_SIZE = 1024 + + def __init__(self, url, dest_dir=None): + self._url = url + self._fname = url.split("/")[-1] + + self._destination = self._fname + if dest_dir: + self.set_destination(join(dest_dir, self._fname)) + self._progressbar = None + + self._request = get(url, stream=True) + if self._request.status_code != 200: + raise FDUnrecognizedStatusCode(self._request.status_code, url) + + def set_destination(self, destination): + self._destination = destination + + def get_filepath(self): + return self._destination + + def get_lmtime(self): + return self._request.headers['last-modified'] + + def get_size(self): + return int(self._request.headers['content-length']) + + def start(self): + itercontent = self._request.iter_content(chunk_size=self.CHUNK_SIZE) + f = open(self._destination, "wb") + chunks = int(ceil(self.get_size() / float(self.CHUNK_SIZE))) + + with progressbar(length=chunks, label="Downloading") as pb: + for _ in pb: + f.write(next(itercontent)) + f.close() + self._request.close() + + self._preserve_filemtime(self.get_lmtime()) + + def verify(self, sha1=None): + _dlsize = getsize(self._destination) + if _dlsize != self.get_size(): + raise FDSizeMismatch(_dlsize, self._fname, self.get_size()) + + if not sha1: + return + + try: + res = check_output(["shasum", self._destination]) + dlsha1 = res[:40] + if sha1 != dlsha1: + raise FDSHASumMismatch(dlsha1, self._fname, sha1) + except OSError: + pass + + def _preserve_filemtime(self, lmdate): + timedata = parsedate_tz(lmdate) + lmtime = mktime(timedata[:9]) + change_filemtime(self._destination, lmtime) + + def __del__(self): + self._request.close() diff --git a/platformio/exception.py b/platformio/exception.py new file mode 100644 index 00000000..a068a915 --- /dev/null +++ b/platformio/exception.py @@ -0,0 +1,75 @@ +# Copyright (C) Ivan Kravets +# See LICENSE for details. + + +class PlatformioException(Exception): + + MESSAGE = None + + def __str__(self): # pragma: no cover + if self.MESSAGE: + return self.MESSAGE % self.args + else: + return Exception.__str__(self) + + +class UnknownPlatform(PlatformioException): + + MESSAGE = "Unknown platform '%s'" + + +class UnknownCLICommand(PlatformioException): + + MESSAGE = "Unknown command '%s'" + + +class UnknownPackage(PlatformioException): + + MESSAGE = "Detected unknown package '%s'" + + +class InvalidPackageVersion(PlatformioException): + + MESSAGE = "The package '%s' with version '%d' does not exist" + + +class PackageInstalled(PlatformioException): + + MESSAGE = "The package '%s' is installed already" + + +class NonSystemPackage(PlatformioException): + + MESSAGE = "The package '%s' is not available for your system '%s'" + + +class FDUnrecognizedStatusCode(PlatformioException): + + MESSAGE = "Got an unrecognized status code '%s' when downloaded %s" + + +class FDSizeMismatch(PlatformioException): + + MESSAGE = ("The size (%d bytes) of downloaded file '%s' " + "is not equal to remote size (%d bytes)") + + +class FDSHASumMismatch(PlatformioException): + + MESSAGE = ("The 'sha1' sum '%s' of downloaded file '%s' " + "is not equal to remote '%s'") + + +class NotPlatformProject(PlatformioException): + + MESSAGE = "Not a platformio project. Use `platformio init` command" + + +class UndefinedEnvPlatform(PlatformioException): + + MESSAGE = "Please specify platform for '%s' environment" + + +class UnsupportedArchiveType(PlatformioException): + + MESSAGE = "Can not unpack file '%s'" diff --git a/platformio/pkgmanager.py b/platformio/pkgmanager.py new file mode 100644 index 00000000..c281faf9 --- /dev/null +++ b/platformio/pkgmanager.py @@ -0,0 +1,97 @@ +# Copyright (C) Ivan Kravets +# See LICENSE for details. + +import json +from os import makedirs, remove +from os.path import isdir, isfile, join + +from requests import get + +from platformio import __pkgmanifesturl__ +from platformio.downloader import FileDownloader +from platformio.exception import (InvalidPackageVersion, NonSystemPackage, + PackageInstalled, UnknownPackage) +from platformio.unpacker import FileUnpacker +from platformio.util import get_home_dir, get_system + + +class PackageManager(object): + + def __init__(self, platform_name): + self._platform_name = platform_name + self._platforms_dir = get_home_dir() + self._dbfile = join(self._platforms_dir, "installed.json") + + @staticmethod + def get_manifest(): + return get(__pkgmanifesturl__).json() + + @staticmethod + def download(url, dest_dir, sha1=None): + fd = FileDownloader(url, dest_dir) + fd.start() + fd.verify(sha1) + return fd.get_filepath() + + @staticmethod + def unpack(pkgpath, dest_dir): + fu = FileUnpacker(pkgpath, dest_dir) + return fu.start() + + def get_installed(self): + data = {} + if isfile(self._dbfile): + with open(self._dbfile) as fp: + data = json.load(fp) + return data + + def is_installed(self, name): + installed = self.get_installed() + return (self._platform_name in installed and name in + installed[self._platform_name]) + + def get_info(self, name, version=None): + if self.is_installed(name): + raise PackageInstalled(name) + + manifest = self.get_manifest() + if name not in manifest: + raise UnknownPackage(name) + + # check system platform + system = get_system() + builds = ([b for b in manifest[name] if b['system'] == "all" or system + in b['system']]) + if not builds: + raise NonSystemPackage(name, system) + + if version: + for b in builds: + if b['version'] == version: + return b + raise InvalidPackageVersion(name, version) + else: + return sorted(builds, key=lambda s: s['version'])[-1] + + def install(self, name, path): + info = self.get_info(name) + pkg_dir = join(self._platforms_dir, self._platform_name, path) + if not isdir(pkg_dir): + makedirs(pkg_dir) + + dlpath = self.download(info['url'], pkg_dir, info['sha1']) + if self.unpack(dlpath, pkg_dir): + self._register(name, info['version'], path) + # remove archive + remove(dlpath) + + def _register(self, name, version, path): + data = self.get_installed() + if self._platform_name not in data: + data[self._platform_name] = {} + data[self._platform_name][name] = { + "version": version, + "path": path + } + with open(self._dbfile, "w") as fp: + json.dump(data, fp) diff --git a/platformio/platforms/__init__.py b/platformio/platforms/__init__.py new file mode 100644 index 00000000..ca6f0304 --- /dev/null +++ b/platformio/platforms/__init__.py @@ -0,0 +1,2 @@ +# Copyright (C) Ivan Kravets +# See LICENSE for details. diff --git a/platformio/platforms/atmelavr.py b/platformio/platforms/atmelavr.py new file mode 100644 index 00000000..b8c0a919 --- /dev/null +++ b/platformio/platforms/atmelavr.py @@ -0,0 +1,37 @@ +# Copyright (C) Ivan Kravets +# See LICENSE for details. + +from os.path import join + +from platformio.platforms.base import BasePlatform + + +class AtmelavrPlatform(BasePlatform): + + PACKAGES = { + + "toolchain-atmelavr": { + "path": join("tools", "toolchain"), + "default": True + }, + + "tool-avrdude": { + "path": join("tools", "avrdude"), + "default": True, + }, + + "framework-arduinoavr": { + "path": join("frameworks", "arduino"), + "default": False + } + } + + def get_name(self): + return "atmelavr" + + def after_run(self, result): + # fix STDERR "flash written" for avrdude + if "flash written" in result['err']: + result['out'] += "\n" + result['err'] + result['err'] = "" + return result diff --git a/platformio/platforms/base.py b/platformio/platforms/base.py new file mode 100644 index 00000000..b387c3a9 --- /dev/null +++ b/platformio/platforms/base.py @@ -0,0 +1,78 @@ +# Copyright (C) Ivan Kravets +# See LICENSE for details. + +from os.path import join + +from click import echo, secho, style + +from platformio.exception import (PackageInstalled, UnknownPackage, + UnknownPlatform) +from platformio.pkgmanager import PackageManager +from platformio.util import exec_command, get_source_dir + + +class PlatformFactory(object): + + @staticmethod + def newPlatform(name): + clsname = "%sPlatform" % name.title() + try: + mod = __import__("platformio.platforms." + name.lower(), + None, None, [clsname]) + except ImportError: + raise UnknownPlatform(name) + + obj = getattr(mod, clsname)() + assert isinstance(obj, BasePlatform) + return obj + + +class BasePlatform(object): + + PACKAGES = {} + + def get_name(self): + raise NotImplementedError() + + def install(self, with_packages, without_packages): + requirements = [] + pm = PackageManager(self.get_name()) + + upkgs = set(with_packages + without_packages) + ppkgs = set(self.PACKAGES.keys()) + if not upkgs.issubset(ppkgs): + raise UnknownPackage(", ".join(upkgs - ppkgs)) + + for name, opts in self.PACKAGES.iteritems(): + if name in without_packages: + continue + elif name in with_packages or opts["default"]: + requirements.append((name, opts["path"])) + + for (name, path) in requirements: + echo("Installing %s package:" % style(name, fg="cyan")) + try: + pm.install(name, path) + except PackageInstalled: + secho("Already installed", fg="yellow") + + return True + + def after_run(self, result): # pylint: disable=R0201 + return result + + def run(self, variables, targets): + assert isinstance(variables, list) + assert isinstance(targets, list) + + if "clean" in targets: + targets.remove("clean") + targets.append("-c") + + result = exec_command([ + "scons", + "-Q", + "-f", join(get_source_dir(), "builder", "main.py") + ] + variables + targets) + + return self.after_run(result) diff --git a/platformio/platforms/timsp430.py b/platformio/platforms/timsp430.py new file mode 100644 index 00000000..daa8ed8e --- /dev/null +++ b/platformio/platforms/timsp430.py @@ -0,0 +1,30 @@ +# Copyright (C) Ivan Kravets +# See LICENSE for details. + +from os.path import join + +from platformio.platforms.base import BasePlatform + + +class Timsp430Platform(BasePlatform): + + PACKAGES = { + + "toolchain-timsp430": { + "path": join("tools", "toolchain"), + "default": True + }, + + "tool-mspdebug": { + "path": join("tools", "mspdebug"), + "default": True, + }, + + "framework-energiamsp430": { + "path": join("frameworks", "energia"), + "default": False + } + } + + def get_name(self): + return "timsp430" diff --git a/platformio/platforms/titiva.py b/platformio/platforms/titiva.py new file mode 100644 index 00000000..0a0e7b84 --- /dev/null +++ b/platformio/platforms/titiva.py @@ -0,0 +1,30 @@ +# Copyright (C) Ivan Kravets +# See LICENSE for details. + +from os.path import join + +from platformio.platforms.base import BasePlatform + + +class TitivaPlatform(BasePlatform): + + PACKAGES = { + + "toolchain-titiva": { + "path": join("tools", "toolchain"), + "default": True + }, + + "tool-lm4flash": { + "path": join("tools", "lm4flash"), + "default": True, + }, + + "framework-energiativa": { + "path": join("frameworks", "energia"), + "default": False + } + } + + def get_name(self): + return "titiva" diff --git a/platformio/unpacker.py b/platformio/unpacker.py new file mode 100644 index 00000000..f80081dc --- /dev/null +++ b/platformio/unpacker.py @@ -0,0 +1,87 @@ +# Copyright (C) Ivan Kravets +# See LICENSE for details. + +from os import chmod +from os.path import join, splitext +from tarfile import open as tarfile_open +from time import mktime +from zipfile import ZipFile + +from click import progressbar + +from platformio.exception import UnsupportedArchiveType +from platformio.util import change_filemtime + + +class ArchiveBase(object): + + def __init__(self, arhfileobj): + self._afo = arhfileobj + + def get_items(self): + raise NotImplementedError() + + def extract_item(self, item, dest_dir): + self._afo.extract(item, dest_dir) + self.after_extract(item, dest_dir) + + def after_extract(self, item, dest_dir): + pass + + +class TARArchive(ArchiveBase): + + def __init__(self, archpath): + ArchiveBase.__init__(self, tarfile_open(archpath)) + + def get_items(self): + return self._afo.getmembers() + + +class ZIPArchive(ArchiveBase): + + def __init__(self, archpath): + ArchiveBase.__init__(self, ZipFile(archpath)) + + @staticmethod + def preserve_permissions(item, dest_dir): + attrs = item.external_attr >> 16L + if attrs: + chmod(join(dest_dir, item.filename), attrs) + + @staticmethod + def preserve_mtime(item, dest_dir): + change_filemtime( + join(dest_dir, item.filename), + mktime(list(item.date_time) + [0]*3) + ) + + def get_items(self): + return self._afo.infolist() + + def after_extract(self, item, dest_dir): + self.preserve_permissions(item, dest_dir) + self.preserve_mtime(item, dest_dir) + + +class FileUnpacker(object): + + def __init__(self, archpath, dest_dir="."): + self._archpath = archpath + self._dest_dir = dest_dir + self._unpacker = None + + _, archext = splitext(archpath.lower()) + if archext in (".gz", ".bz2"): + self._unpacker = TARArchive(archpath) + elif archext == ".zip": + self._unpacker = ZIPArchive(archpath) + + if not self._unpacker: + raise UnsupportedArchiveType(archpath) + + def start(self): + with progressbar(self._unpacker.get_items(), label="Unpacking") as pb: + for item in pb: + self._unpacker.extract_item(item, self._dest_dir) + return True diff --git a/platformio/util.py b/platformio/util.py index 6e6e9803..fe7cdb50 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -1,16 +1,17 @@ # Copyright (C) Ivan Kravets # See LICENSE for details. -from os import getcwd +from os import getcwd, utime from os.path import dirname, expanduser, isfile, join, realpath +from platform import architecture, system from subprocess import PIPE, Popen -from sys import exit -from textwrap import fill + +from platformio.exception import NotPlatformProject try: from configparser import ConfigParser except ImportError: - from ConfigParser import ConfigParser + from ConfigParser import ConfigParser def get_home_dir(): @@ -26,48 +27,23 @@ def get_project_dir(): def get_project_config(): - try: - return getattr(get_project_config, "_cache") - except AttributeError: - pass - path = join(get_project_dir(), "platformio.ini") if not isfile(path): - exit("Not a platformio project. Use `platformio init` command") - - get_project_config._cache = ConfigParser() - get_project_config._cache.read(path) - return get_project_config._cache + raise NotPlatformProject() + cp = ConfigParser() + cp.read(path) + return cp -def textindent(text, quote): - return fill(text, initial_indent=quote, - subsequent_indent=quote, width=120) +def get_system(): + return (system() + architecture()[0][:-3]).lower() + + +def change_filemtime(path, time): + utime(path, (time, time)) def exec_command(args): p = Popen(args, stdout=PIPE, stderr=PIPE) out, err = p.communicate() - result = dict(out=out.strip(), err=err.strip()) - - # fix STDERR "flash written" - if "flash written" in result['err']: - result['out'] += "\n" + result['err'] - result['err'] = "" - - return result - - -def run_builder(variables, targets): - assert isinstance(variables, list) - assert isinstance(targets, list) - - if "clean" in targets: - targets.remove("clean") - targets.append("-c") - - return exec_command([ - "scons", - "-Q", - "-f", join(get_source_dir(), "builder", "main.py") - ] + variables + targets) + return dict(out=out.strip(), err=err.strip())