Decentralized architecture for development platforms // Issue #479

This commit is contained in:
Ivan Kravets committed 2016-05-26 19:43:36 +03:00
1 parent 349b159a7b
commit d68eb28629
40 files changed
+1269 -1042

No files matched your search

+217 -101
View File
@@ -1,4 +1,4 @@
# Copyright 2014-2016 Ivan Kravets <me@ikravets.com>
# Copyright 2014-present Ivan Kravets <me@ikravets.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -12,38 +12,41 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from os import makedirs, remove
from os.path import basename, isdir, isfile, join
import os
from os.path import dirname, isdir, isfile, join
from shutil import rmtree
from time import time
import click
import requests
import semantic_version
from platformio import exception, telemetry, util
from platformio.app import get_state_item, set_state_item
from platformio.downloader import FileDownloader
from platformio.unpacker import FileUnpacker
class PackageManager(object):
def __init__(self):
self._package_dir = join(util.get_home_dir(), "packages")
if not isdir(self._package_dir):
makedirs(self._package_dir)
assert isdir(self._package_dir)
_INSTALLED_CACHE = {}
@classmethod
@util.memoized
def get_manifest(cls):
return util.get_api_result("/packages/manifest")
def __init__(self, package_dir=None, repositories=None):
self._INSTALLED_CACHE = {}
self.repositories = repositories
self.package_dir = package_dir or join(util.get_home_dir(), "packages")
if not isdir(self.package_dir):
os.makedirs(self.package_dir)
assert isdir(self.package_dir)
@staticmethod
def reset_cache():
PackageManager._INSTALLED_CACHE = {}
@staticmethod
def download(url, dest_dir, sha1=None):
fd = FileDownloader(url, dest_dir)
fd.start()
fd.verify(sha1)
if sha1:
fd.verify(sha1)
return fd.get_filepath()
@staticmethod
@@ -52,98 +55,180 @@ class PackageManager(object):
return fu.start()
@staticmethod
def get_installed():
return get_state_item("installed_packages", {})
def get_manifest_name():
return "package.json"
def get_outdated(self):
outdated = []
for name, data in self.get_installed().items():
if data['version'] != self.get_info(name)['version']:
outdated.append(name)
return outdated
def is_installed(self, name):
return name in self.get_installed()
def get_info(self, name, version=None):
manifest = self.get_manifest()
if name not in manifest:
raise exception.UnknownPackage(name)
# check system platform
@staticmethod
def max_satisfying_version(versions, requirements=None):
item = None
systype = util.get_systype()
builds = ([b for b in manifest[name] if b['system'] == "all" or systype
in b['system']])
if not builds:
raise exception.NonSystemPackage(name, systype)
if requirements is not None:
requirements = str(requirements)
for v in versions:
if isinstance(v['version'], int):
continue
if v['system'] not in ("all", "*") and systype not in v['system']:
continue
if requirements and not semantic_version.match(
requirements, v['version']):
continue
if item is None or semantic_version.compare(
v['version'], item['version']) == 1:
item = v
return item
if version:
for b in builds:
if b['version'] == version:
return b
raise exception.InvalidPackageVersion(name, version)
else:
return sorted(builds, key=lambda s: s['version'])[-1]
def get_installed(self):
if self.package_dir in PackageManager._INSTALLED_CACHE:
return PackageManager._INSTALLED_CACHE[self.package_dir]
items = []
for p in sorted(os.listdir(self.package_dir)):
manifest_path = join(self.package_dir, p, self.get_manifest_name())
if not isfile(manifest_path):
continue
manifest = util.load_json(manifest_path)
manifest['_manifest_path'] = manifest_path
assert set(["name", "version"]) <= set(manifest.keys())
items.append(manifest)
PackageManager._INSTALLED_CACHE[self.package_dir] = items
return items
def install(self, name):
click.echo("Installing %s package:" % click.style(name, fg="cyan"))
def is_installed(self, name, requirements=None):
installed = self.get_installed()
if requirements is None:
return any([p['name'] == name for p in installed])
if self.is_installed(name):
click.secho("Already installed", fg="yellow")
return False
for p in installed:
if p['name'] != name:
continue
elif semantic_version.match(requirements, p['version']):
return True
return None
info = self.get_info(name)
pkg_dir = join(self._package_dir, name)
if not isdir(pkg_dir):
makedirs(pkg_dir)
def get_latest_version(self, name, requirements):
for versions in PackageRepoIterator(name, self.repositories):
pkgdata = self.max_satisfying_version(versions, requirements)
if pkgdata:
return pkgdata['version']
return None
dlpath = None
try:
dlpath = self.download(info['url'], pkg_dir, info['sha1'])
except (requests.exceptions.ConnectionError,
requests.exceptions.ChunkedEncodingError,
requests.exceptions.SSLError,
exception.FDUnrecognizedStatusCode, StopIteration):
if not info['url'].startswith("http://dl.platformio.org"):
def install(self, name, requirements, silent=False, trigger_event=True):
installed = self.is_installed(name, requirements)
if not installed or not silent:
click.echo("Installing package %s @ %s:" % (
click.style(name, fg="cyan"),
requirements if requirements else "latest"))
if installed:
if not silent:
click.secho("Already installed", fg="yellow")
return
if not self._install_from_piorepo(name, requirements):
raise exception.PackageInstallError(
name, requirements, util.get_systype())
self.reset_cache()
if trigger_event:
telemetry.on_event(
category="PackageManager", action="Install", label=name)
def _install_from_piorepo(self, name, requirements):
pkg_dir = None
success = False
pkgdata = None
versions = None
for versions in PackageRepoIterator(name, self.repositories):
dlpath = None
pkgdata = self.max_satisfying_version(versions, requirements)
if not pkgdata:
continue
pkg_dir = join(self.package_dir, name)
if isfile(join(pkg_dir, self.get_manifest_name())):
pkg_dir = join(
self.package_dir, "%s@%s" % (name, pkgdata['version']))
# remove previous/not-satisfied package
if isdir(pkg_dir):
rmtree(pkg_dir)
os.makedirs(pkg_dir)
try:
dlpath = self.download(
"http://dl.platformio.org/packages/%s" %
basename(info['url']), pkg_dir, info['sha1'])
pkgdata['url'], pkg_dir, pkgdata.get("sha1"))
assert isfile(dlpath)
self.unpack(dlpath, pkg_dir)
success = True
break
except Exception as e: # pylint: disable=broad-except
click.secho("Warning! Package Mirror: %s" % e, fg="yellow")
click.secho("Looking for other Package Mirror...", fg="yellow")
finally:
if dlpath and isfile(dlpath):
os.remove(dlpath)
assert isfile(dlpath)
if versions is None:
raise exception.UnknownPackage(name)
elif not pkgdata:
raise exception.UndefinedPackageVersion(
name, requirements, util.get_systype())
if self.unpack(dlpath, pkg_dir):
self._register(name, info['version'])
# remove archive
remove(dlpath)
return success
telemetry.on_event(
category="PackageManager", action="Install", label=name)
def uninstall(self, name, requirements=None, trigger_event=True):
click.echo("Uninstalling package %s @ %s: \t" % (
click.style(name, fg="cyan"),
requirements if requirements else "latest"), nl=False)
found = False
for manifest in self.get_installed():
if manifest['name'] != name:
continue
if (requirements and not semantic_version.match(
requirements, manifest['version'])):
continue
found = True
if isfile(manifest['_manifest_path']):
rmtree(dirname(manifest['_manifest_path']))
def uninstall(self, name):
click.echo("Uninstalling %s package: \t" %
click.style(name, fg="cyan"), nl=False)
if not self.is_installed(name):
if not found:
click.secho("Not installed", fg="yellow")
return False
else:
click.echo("[%s]" % click.style("OK", fg="green"))
if isdir(join(self._package_dir, name)):
rmtree(join(self._package_dir, name))
self._unregister(name)
click.echo("[%s]" % click.style("OK", fg="green"))
self.reset_cache()
if trigger_event:
telemetry.on_event(
category="PackageManager", action="Uninstall", label=name)
# report usage
telemetry.on_event(
category="PackageManager", action="Uninstall", label=name)
def update(self, name, requirements=None, keep_versions=None):
click.echo("Updating package %s @ %s:" % (
click.style(name, fg="yellow"),
requirements if requirements else "latest"))
def update(self, name):
click.echo("Updating %s package:" % click.style(name, fg="yellow"))
latest_version = self.get_latest_version(name, requirements)
if latest_version is None:
click.secho("Ignored! '%s' is not listed in "
"Package Repository" % name, fg="yellow")
return
installed = self.get_installed()
current_version = installed[name]['version']
latest_version = self.get_info(name)['version']
current = None
other_versions = []
for manifest in self.get_installed():
if manifest['name'] != name:
continue
other_versions.append(manifest['version'])
if (requirements and not semantic_version.match(
requirements, manifest['version'])):
continue
if (not current or semantic_version.compare(
manifest['version'], current['version']) == 1):
current = manifest
click.echo("Versions: Current=%d, Latest=%d \t " %
if current is None:
return
current_version = current['version']
click.echo("Versions: Current=%s, Latest=%s \t " %
(current_version, latest_version), nl=False)
if current_version == latest_version:
@@ -152,21 +237,52 @@ class PackageManager(object):
else:
click.echo("[%s]" % (click.style("Out-of-date", fg="red")))
self.uninstall(name)
self.install(name)
for v in other_versions:
if not keep_versions or v not in keep_versions:
self.uninstall(name, v, trigger_event=False)
self.install(name, latest_version, trigger_event=False)
telemetry.on_event(
category="PackageManager", action="Update", label=name)
return True
def _register(self, name, version):
data = self.get_installed()
data[name] = {
"version": version,
"time": int(time())
}
set_state_item("installed_packages", data)
def _unregister(self, name):
data = self.get_installed()
del data[name]
set_state_item("installed_packages", data)
class PackageRepoIterator(object):
_MANIFEST_CACHE = {}
def __init__(self, package, repositories):
assert isinstance(repositories, list)
self.package = package
self.repositories = iter(repositories)
def __iter__(self):
return self
def __next__(self):
return self.next()
def next(self):
manifest = {}
repo = next(self.repositories)
if isinstance(repo, dict):
manifest = repo
elif repo in PackageRepoIterator._MANIFEST_CACHE:
manifest = PackageRepoIterator._MANIFEST_CACHE[repo]
else:
r = None
try:
r = requests.get(repo, headers=util.get_request_defheaders())
r.raise_for_status()
manifest = r.json()
except: # pylint: disable=bare-except
pass
finally:
if r:
r.close()
PackageRepoIterator._MANIFEST_CACHE[repo] = manifest
if self.package in manifest:
return manifest[self.package]
else:
return self.next()