Implement packages updating; other improvements to Package Manager // Issue #479

platformio/builder/tools/devplatform.py

@@ -49,9 +49,8 @@ def BoardConfig(env, board=None):
 
 def GetFrameworkScript(env, framework):
     p = env.DevPlatform()
-    frameworks = p.get_frameworks()
+    assert p.frameworks and framework in p.frameworks
-    assert frameworks and framework in frameworks
+    script_path = env.subst(p.frameworks[framework]['script'])
-    script_path = env.subst(frameworks[framework]['script'])
     if not isfile(script_path):
         script_path = join(p.get_dir(), script_path)
     return script_path

platformio/commands/platform.py

@@ -97,10 +97,10 @@ def platform_list(json_output):
     for manifest in PlatformManager().get_installed():
         p = PlatformFactory.newPlatform(manifest['_manifest_path'])
         platforms.append({
-            "name": p.get_name(),
+            "name": p.name,
-            "title": p.get_title(),
+            "title": p.title,
-            "description": p.get_description(),
+            "description": p.description,
-            "version": p.get_version(),
+            "version": p.version,
             "packages": p.get_installed_packages().keys()
         })
 
@@ -125,24 +125,23 @@ def platform_show(ctx, platform):
             raise exception.PlatformNotInstalledYet(platform)
 
     click.echo("{name} ~ {title}".format(
-        name=click.style(p.get_name(), fg="cyan"), title=p.get_title()))
+        name=click.style(p.name, fg="cyan"), title=p.title))
-    click.echo("=" * (3 + len(p.get_name() + p.get_title())))
+    click.echo("=" * (3 + len(p.name + p.title)))
-    click.echo(p.get_manifest().get("description"))
+    click.echo(p.description)
     click.echo()
-    click.echo("Version: %s" % p.get_version())
+    click.echo("Version: %s" % p.version)
-    if "homepage" in p.get_manifest():
+    if p.homepage:
-        click.echo("Home: %s" % p.get_manifest().get("homepage"))
+        click.echo("Home: %s" % p.homepage)
-    if "license" in p.get_manifest():
+    if p.license:
-        click.echo("License: %s" % p.get_manifest().get("license").get("type"))
+        click.echo("License: %s" % p.license.get("type"))
-    if "frameworks" in p.get_manifest():
+    if p.frameworks:
-        click.echo("Frameworks: %s" %
+        click.echo("Frameworks: %s" % ", ".join(p.frameworks.keys()))
-                   ", ".join(p.get_manifest().get("frameworks").keys()))
 
-    if not p.get_packages():
+    if not p.packages:
         return
 
     installed_pkgs = p.get_installed_packages()
-    for name, opts in p.get_packages().items():
+    for name, opts in p.packages.items():
         click.echo()
         click.echo("Package %s" % click.style(name, fg="yellow"))
         click.echo("-" * (8 + len(name)))
@@ -173,15 +172,11 @@ def platform_uninstall(platforms):
 @cli.command("update", short_help="Update installed Platforms")
 @click.option("--only-packages", is_flag=True)
 def platform_update(only_packages):
-    for manifest in PlatformManager().get_installed():
+    pm = PlatformManager()
+    for manifest in pm.get_installed():
         click.echo("Platform %s @ %s" % (
             click.style(manifest['name'], fg="cyan"), manifest['version']))
         click.echo("--------")
-        if only_packages:
+        pm.update(manifest['name'], manifest['version'],
-            status = PlatformFactory.newPlatform(
+                  only_packages=only_packages)
-                manifest['name'], manifest['version']).update_packages()
-            if status is None:
-                click.secho("Packages are up-to-date", fg="green")
-        else:
-            PlatformManager().update(manifest['name'], manifest['version'])
         click.echo()

platformio/exception.py

@@ -81,6 +81,11 @@ class UndefinedPackageVersion(PlatformioException):
         " for your system '{2}'"
 
 
+class UndefinedPlatformVersion(PlatformioException):
+
+    MESSAGE = "Can not find platform '{0}' with version requirements '{1}'"
+
+
 class PackageInstallError(PlatformioException):
 
     MESSAGE = "Can not install package '{0}' with version requirements '{1}' "\

platformio/managers/package.py

@@ -41,8 +41,8 @@ class PackageManager(object):
     def reset_cache():
         PackageManager._INSTALLED_CACHE = {}
 
-    @staticmethod
+    @property
-    def get_manifest_name():
+    def manifest_name(self):
         return "package.json"
 
     @staticmethod
@@ -59,11 +59,11 @@ class PackageManager(object):
         return fu.start()
 
     def check_structure(self, pkg_dir):
-        if isfile(join(pkg_dir, self.get_manifest_name())):
+        if isfile(join(pkg_dir, self.manifest_name)):
             return True
 
         for root, _, files in os.walk(pkg_dir):
-            if self.get_manifest_name() not in files:
+            if self.manifest_name not in files:
                 continue
             # copy contents to the root of package directory
             for item in os.listdir(root):
@@ -80,20 +80,27 @@ class PackageManager(object):
                     break
             break
 
-        if isfile(join(pkg_dir, self.get_manifest_name())):
+        if isfile(join(pkg_dir, self.manifest_name)):
             return True
 
         raise exception.PlatformioException(
             "Could not find '%s' manifest file in the package" %
-            self.get_manifest_name())
+            self.manifest_name)
 
     def make_pkg_dir(self, name, version):
         pkg_dir = join(self.package_dir, name)
-        if isfile(join(pkg_dir, self.get_manifest_name())):
+        if isfile(join(pkg_dir, self.manifest_name)):
-            _manifest = util.load_json(
+            manifest = util.load_json(
-                join(pkg_dir, self.get_manifest_name()))
+                join(pkg_dir, self.manifest_name))
-            if (_manifest['name'] == name and
+
-                    _manifest['version'] != version):
+            cmp_result = semantic_version.compare(version, manifest['version'])
+            if cmp_result == 1:
+                # if main package version < new package, backup it
+                print pkg_dir, join(
+                    self.package_dir, "%s@%s" % (name, manifest['version']))
+                os.rename(pkg_dir, join(
+                    self.package_dir, "%s@%s" % (name, manifest['version'])))
+            elif cmp_result == -1:
                 pkg_dir = join(
                     self.package_dir, "%s@%s" % (name, version))
 
@@ -103,6 +110,7 @@ class PackageManager(object):
         os.makedirs(pkg_dir)
         assert isdir(pkg_dir)
 
+        self.reset_cache()
         return pkg_dir
 
     @staticmethod
@@ -124,6 +132,17 @@ class PackageManager(object):
                 item = v
         return item
 
+    def get_latest_repo_version(self, name, requirements):
+        version = None
+        for versions in PackageRepoIterator(name, self.repositories):
+            pkgdata = self.max_satisfying_repo_version(versions, requirements)
+            if not pkgdata:
+                continue
+            if (not version or semantic_version.compare(
+                    pkgdata['version'], version) == 1):
+                version = pkgdata['version']
+        return version
+
     def max_satisfying_version(self, name, requirements=None):
         best = None
         for manifest in self.get_installed():
@@ -142,7 +161,7 @@ class PackageManager(object):
             return PackageManager._INSTALLED_CACHE[self.package_dir]
         items = []
         for p in sorted(os.listdir(self.package_dir)):
-            manifest_path = join(self.package_dir, p, self.get_manifest_name())
+            manifest_path = join(self.package_dir, p, self.manifest_name)
             if not isfile(manifest_path):
                 continue
             manifest = util.load_json(manifest_path)
@@ -164,18 +183,11 @@ class PackageManager(object):
                 return True
         return None
 
-    def get_latest_version(self, name, requirements):
-        for versions in PackageRepoIterator(name, self.repositories):
-            pkgdata = self.max_satisfying_repo_version(versions, requirements)
-            if pkgdata:
-                return pkgdata['version']
-        return None
-
     def install(self, name, requirements, silent=False, trigger_event=True):
         installed = self.is_installed(name, requirements)
         if not installed or not silent:
             click.echo("Installing %s %s @ %s:" % (
-                self.get_manifest_name().split(".")[0],
+                self.manifest_name.split(".")[0],
                 click.style(name, fg="cyan"),
                 requirements if requirements else "latest"))
         if installed:
@@ -228,27 +240,31 @@ class PackageManager(object):
         if versions is None:
             raise exception.UnknownPackage(name)
         elif not pkgdata:
-            raise exception.UndefinedPackageVersion(
+            if "platform" in self.manifest_name:
-                name, requirements or "latest", util.get_systype())
+                raise exception.UndefinedPlatformVersion(
+                    name, requirements or "latest")
+            else:
+                raise exception.UndefinedPackageVersion(
+                    name, requirements or "latest", util.get_systype())
 
-        return join(pkg_dir, self.get_manifest_name())
+        return join(pkg_dir, self.manifest_name)
 
     def _install_from_local_dir(self, local_dir):
-        if not isfile(join(local_dir, self.get_manifest_name())):
+        if not isfile(join(local_dir, self.manifest_name)):
             raise exception.InvalidLocalPackage(
-                local_dir, self.get_manifest_name())
+                local_dir, self.manifest_name)
 
-        manifest = util.load_json(join(local_dir, self.get_manifest_name()))
+        manifest = util.load_json(join(local_dir, self.manifest_name))
         assert set(["name", "version"]) <= set(manifest.keys())
         pkg_dir = self.make_pkg_dir(manifest['name'], manifest['version'])
         rmtree(pkg_dir)
         copytree(local_dir, pkg_dir, symlinks=True)
 
-        return join(pkg_dir, self.get_manifest_name())
+        return join(pkg_dir, self.manifest_name)
 
     def uninstall(self, name, requirements=None, trigger_event=True):
         click.echo("Uninstalling %s %s @ %s: \t" % (
-            self.get_manifest_name().split(".")[0],
+            self.manifest_name.split(".")[0],
             click.style(name, fg="cyan"),
             requirements if requirements else "latest"), nl=False)
         found = False
@@ -277,24 +293,22 @@ class PackageManager(object):
             telemetry.on_event(
                 category="PackageManager", action="Uninstall", label=name)
 
-    def update(self, name, requirements=None, keep_versions=None):
+    def update(self, name, requirements=None):
         click.echo("Updating %s %s @ %s:" % (
-            self.get_manifest_name().split(".")[0],
+            self.manifest_name.split(".")[0],
             click.style(name, fg="yellow"),
             requirements if requirements else "latest"))
 
-        latest_version = self.get_latest_version(name, requirements)
+        latest_version = self.get_latest_repo_version(name, requirements)
         if latest_version is None:
             click.secho("Ignored! '%s' is not listed in "
                         "Package Repository" % name, fg="yellow")
             return
 
         current = None
-        other_versions = []
         for manifest in self.get_installed():
             if manifest['name'] != name:
                 continue
-            other_versions.append(manifest['version'])
             if (requirements and not semantic_version.match(
                     requirements, manifest['version'])):
                 continue
@@ -315,9 +329,6 @@ class PackageManager(object):
         else:
             click.echo("[%s]" % (click.style("Out-of-date", fg="red")))
 
-        for v in other_versions:
-            if not keep_versions or v not in keep_versions:
-                self.uninstall(name, v, trigger_event=False)
         self.install(name, latest_version, trigger_event=False)
 
         telemetry.on_event(

platformio/managers/platform.py

@@ -36,33 +36,61 @@ class PlatformManager(PackageManager):
             ["http://dl.platformio.org/platforms/manifest.json"]
         )
 
-    @staticmethod
+    @property
-    def get_manifest_name():
+    def manifest_name(self):
         return "platform.json"
 
     def install(self,  # pylint: disable=too-many-arguments,arguments-differ
                 name, requirements=None, with_packages=None,
                 without_packages=None, skip_default_packages=False):
         manifest_path = PackageManager.install(self, name, requirements)
-        return PlatformFactory.newPlatform(
+        PlatformFactory.newPlatform(
             manifest_path, requirements).install_packages(
                 with_packages, without_packages, skip_default_packages)
+        self.cleanup_packages()
+        return True
 
     def uninstall(self,  # pylint: disable=arguments-differ
                   name, requirements=None):
-        if PlatformFactory.newPlatform(
+        PackageManager.uninstall(self, name, requirements)
-                name, requirements).uninstall_packages():
+        self.cleanup_packages()
-            return PackageManager.uninstall(self, name, requirements)
+        return True
-        return False
 
     def update(self,  # pylint: disable=arguments-differ
-               name, version):
+               name, requirements=None, only_packages=False):
-        raise NotImplementedError()
+        if not only_packages:
+            PackageManager.update(self, name)
+        PlatformFactory.newPlatform(
+            name, requirements).update_packages()
+        self.cleanup_packages()
+        return True
 
-    def is_outdated(self, name, version):
+    def is_outdated(self, name, requirements=None):
-        # @TODO disable auto-update temporary
+        p = PlatformFactory.newPlatform(name, requirements)
-        return False
+        return (p.are_outdated_packages() or
-        raise NotImplementedError()
+                p.version != self.get_latest_repo_version(name, requirements))
+
+    def cleanup_packages(self):
+        self.reset_cache()
+        deppkgs = {}
+        for manifest in PlatformManager().get_installed():
+            p = PlatformFactory.newPlatform(
+                manifest['name'], manifest['version'])
+            for pkgname, pkgmanifest in p.get_installed_packages().items():
+                if pkgname not in deppkgs:
+                    deppkgs[pkgname] = set()
+                deppkgs[pkgname].add(pkgmanifest['version'])
+
+        pm = PackageManager()
+        for manifest in pm.get_installed():
+            if manifest['name'] not in deppkgs:
+                continue
+            if manifest['version'] not in deppkgs[manifest['name']]:
+                pm.uninstall(
+                    manifest['name'], manifest['version'], trigger_event=False)
+
+        self.reset_cache()
+        return True
 
     def get_installed_boards(self):
         boards = []
@@ -71,7 +99,7 @@ class PlatformManager(PackageManager):
             for id_, config in p.get_boards().items():
                 manifest = config.get_manifest().copy()
                 manifest['id'] = id_
-                manifest['platform'] = p.get_name()
+                manifest['platform'] = p.name
                 boards.append(manifest)
         return boards
 
@@ -138,7 +166,7 @@ class PlatformPackagesMixin(object):
     def get_installed_packages(self):
         items = {}
         installed = self.pm.get_installed()
-        for name, opts in self.get_packages().items():
+        for name, opts in self.packages.items():
             manifest = None
             for p in installed:
                 if (p['name'] != name or not semantic_version.match(
@@ -159,11 +187,11 @@ class PlatformPackagesMixin(object):
             self.pkg_types_to_names(without_packages or []))
 
         upkgs = with_packages | without_packages
-        ppkgs = set(self.get_packages().keys())
+        ppkgs = set(self.packages.keys())
         if not upkgs.issubset(ppkgs):
             raise exception.UnknownPackage(", ".join(upkgs - ppkgs))
 
-        for name, opts in self.get_packages().items():
+        for name, opts in self.packages.items():
             if name in without_packages:
                 continue
             elif (name in with_packages or
@@ -172,53 +200,14 @@ class PlatformPackagesMixin(object):
 
         return True
 
-    def uninstall_packages(self):
-        deppkgs = set()
-        for manifest in PlatformManager().get_installed():
-            if manifest['name'] == self.get_name():
-                continue
-            p = PlatformFactory.newPlatform(
-                manifest['name'], manifest['version'])
-            for pkgname, pkgmanifest in p.get_installed_packages().items():
-                deppkgs.add((pkgname, pkgmanifest['version']))
-
-        for manifest in self.pm.get_installed():
-            if manifest['name'] not in self.get_packages().keys():
-                continue
-            if (manifest['name'], manifest['version']) not in deppkgs:
-                self.pm.uninstall(manifest['name'], manifest['version'])
-        return True
-
     def update_packages(self):
-        outdated = None
+        for name in self.get_installed_packages().keys():
-        for pkgname, pkgmanifest in self.get_installed_packages().items():
+            self.pm.update(name, self.packages[name]['version'])
-            requirements = self.get_packages()[pkgname]['version']
-            latest_version = self.pm.get_latest_version(
-                pkgname, requirements)
-            if (not latest_version or
-                    pkgmanifest['version'] == latest_version):
-                continue
-
-            # check other platforms
-            keep_versions = set([latest_version])
-            for pfmanifest in PlatformManager().get_installed():
-                if pfmanifest['name'] == self.get_name():
-                    continue
-                p = PlatformFactory.newPlatform(
-                    pfmanifest['name'], pfmanifest['version'])
-                if pkgname not in p.get_packages():
-                    continue
-                keep_versions.add(p.pm.get_latest_version(
-                    pkgname, p.get_packages()[pkgname]['version']))
-
-            outdated = self.pm.update(
-                pkgname, requirements, keep_versions)
-        return outdated
 
     def are_outdated_packages(self):
         for name, opts in self.get_installed_packages().items():
-            if (opts['version'] != self.pm.get_latest_version(
+            if (opts['version'] != self.pm.get_latest_repo_version(
-                    name, self.get_packages()[name].get("version"))):
+                    name, self.packages[name].get("version"))):
                 return True
         return False
 
@@ -331,10 +320,10 @@ class BasePlatform(PlatformPackagesMixin, PlatformRunMixin):
     def __init__(self, manifest_path):
         self._BOARDS_CACHE = {}
         self.manifest_path = manifest_path
-        self.manifest = util.load_json(manifest_path)
+        self._manifest = util.load_json(manifest_path)
 
         self.pm = PackageManager(
-            repositories=self.manifest.get("packageRepositories"))
+            repositories=self._manifest.get("packageRepositories"))
 
         self._found_error = False
         self._last_echo_line = None
@@ -344,20 +333,48 @@ class BasePlatform(PlatformPackagesMixin, PlatformRunMixin):
         # 3 = 2 + others
         self._verbose_level = 3
 
-    def get_name(self):
+    @property
-        return self.manifest['name']
+    def name(self):
+        return self._manifest['name']
 
-    def get_title(self):
+    @property
-        return self.manifest['title']
+    def title(self):
+        return self._manifest['title']
 
-    def get_description(self):
+    @property
-        return self.manifest['description']
+    def description(self):
+        return self._manifest['description']
 
-    def get_version(self):
+    @property
-        return self.manifest['version']
+    def version(self):
+        return self._manifest['version']
 
-    def get_manifest(self):
+    @property
-        return self.manifest
+    def homepage(self):
+        return self._manifest.get("homepage")
+
+    @property
+    def license(self):
+        return self._manifest.get("license")
+
+    @property
+    def frameworks(self):
+        return self._manifest.get("frameworks")
+
+    @property
+    def manifest(self):
+        return self._manifest
+
+    @property
+    def packages(self):
+        packages = self._manifest.get("packages", {})
+        if "tool-scons" not in packages:
+            packages['tool-scons'] = {
+                "version": self._manifest.get("engines", {}).get(
+                    "scons", ">=2.3.0,<2.6.0"),
+                "optional": False
+            }
+        return packages
 
     def get_dir(self):
         return dirname(self.manifest_path)
@@ -369,7 +386,7 @@ class BasePlatform(PlatformPackagesMixin, PlatformRunMixin):
         raise NotImplementedError()
 
     def is_embedded(self):
-        for opts in self.get_packages().values():
+        for opts in self.packages.values():
             if opts.get("type") == "uploader":
                 return True
         return False
@@ -396,19 +413,6 @@ class BasePlatform(PlatformPackagesMixin, PlatformRunMixin):
     def board_config(self, id_):
         return self.get_boards(id_)
 
-    def get_packages(self):
-        packages = self.manifest.get("packages", {})
-        if "tool-scons" not in packages:
-            packages['tool-scons'] = {
-                "version": self.manifest.get("engines", {}).get(
-                    "scons", ">=2.3.0,<2.6.0"),
-                "optional": False
-            }
-        return packages
-
-    def get_frameworks(self):
-        return self.get_manifest().get("frameworks")
-
     def get_package_dir(self, name):
         packages = self.get_installed_packages()
         if name not in packages:
@@ -422,14 +426,14 @@ class BasePlatform(PlatformPackagesMixin, PlatformRunMixin):
         return packages[name]['version']
 
     def get_package_type(self, name):
-        return self.get_packages()[name].get("type")
+        return self.packages[name].get("type")
 
     def pkg_types_to_names(self, types):
         names = []
         for type_ in types:
             name = type_
             # lookup by package types
-            for _name, _opts in self.get_packages().items():
+            for _name, _opts in self.packages.items():
                 if _opts.get("type") == type_:
                     name = None
                     names.append(_name)
@@ -440,22 +444,21 @@ class BasePlatform(PlatformPackagesMixin, PlatformRunMixin):
 
     def configure_default_packages(self, variables, targets):
         # enbale used frameworks
-        frameworks = self.get_frameworks()
         for framework in variables.get("framework", "").split(","):
             framework = framework.lower().strip()
-            if not framework or framework not in frameworks:
+            if not framework or framework not in self.frameworks:
                 continue
-            _pkg_name = frameworks[framework]['package']
+            _pkg_name = self.frameworks[framework]['package']
-            self.get_packages()[_pkg_name]['optional'] = False
+            self.packages[_pkg_name]['optional'] = False
 
         # enable upload tools for upload targets
         if any(["upload" in t for t in targets] + ["program" in targets]):
-            for _name, _opts in self.get_packages().iteritems():
+            for _name, _opts in self.packages.iteritems():
                 if _opts.get("type") == "uploader":
-                    self.get_packages()[_name]['optional'] = False
+                    self.packages[_name]['optional'] = False
                 elif "uploadlazy" in targets:
                     # skip all packages, allow only upload tools
-                    self.get_packages()[_name]['optional'] = True
+                    self.packages[_name]['optional'] = True
 
 
 class PlatformBoardConfig(object):
@@ -464,11 +467,11 @@ class PlatformBoardConfig(object):
         if not isfile(manifest_path):
             raise exception.UnknownBoard(basename(manifest_path[:-5]))
         self.manifest_path = manifest_path
-        self.manifest = util.load_json(manifest_path)
+        self._manifest = util.load_json(manifest_path)
 
     def get(self, path, default=None):
         try:
-            value = self.manifest
+            value = self._manifest
             for k in path.split("."):
                 value = value[k]
             return value
@@ -486,4 +489,4 @@ class PlatformBoardConfig(object):
             return False
 
     def get_manifest(self):
-        return self.manifest
+        return self._manifest