Install development platform from local dir and VCS(git, hg, svn) // Issue #479

platformio/commands/run.py

@@ -205,7 +205,8 @@ class EnvironmentProcessor(object):
         try:
             p = PlatformFactory.newPlatform(platform, version)
         except exception.UnknownPlatform:
-            self.cmd_ctx.invoke(cmd_platform_install, platforms=[platform])
+            self.cmd_ctx.invoke(
+                cmd_platform_install, platforms=[self.options['platform']])
             p = PlatformFactory.newPlatform(platform, version)
 
         return p.run(build_vars, build_targets, self.verbose_level)

platformio/exception.py

@@ -70,11 +70,6 @@ class UnknownPackage(PlatformioException):
     MESSAGE = "Detected unknown package '{0}'"
 
 
-class InvalidLocalPackage(PlatformioException):
-
-    MESSAGE = "Invalid local package '{0}'. Can not find manifest '{1}'"
-
-
 class UndefinedPackageVersion(PlatformioException):
 
     MESSAGE = "Can not find package '{0}' with version requirements '{1}'"\

platformio/managers/package.py

@@ -15,6 +15,7 @@
 import os
 from os.path import dirname, isdir, isfile, islink, join
 from shutil import copyfile, copytree, rmtree
+from tempfile import mkdtemp
 
 import click
 import requests
@@ -23,6 +24,7 @@ import semantic_version
 from platformio import exception, telemetry, util
 from platformio.downloader import FileDownloader
 from platformio.unpacker import FileUnpacker
+from platformio.vcsclient import VCSClientFactory
 
 
 class PackageManager(object):
@@ -87,32 +89,6 @@ class PackageManager(object):
             "Could not find '%s' manifest file in the package" %
             self.manifest_name)
 
-    def make_pkg_dir(self, name, version):
-        pkg_dir = join(self.package_dir, name)
-        if isfile(join(pkg_dir, self.manifest_name)):
-            manifest = util.load_json(
-                join(pkg_dir, self.manifest_name))
-
-            cmp_result = semantic_version.compare(version, manifest['version'])
-            if cmp_result == 1:
-                # if main package version < new package, backup it
-                print pkg_dir, join(
-                    self.package_dir, "%s@%s" % (name, manifest['version']))
-                os.rename(pkg_dir, join(
-                    self.package_dir, "%s@%s" % (name, manifest['version'])))
-            elif cmp_result == -1:
-                pkg_dir = join(
-                    self.package_dir, "%s@%s" % (name, version))
-
-        # remove previous/not-satisfied package
-        if isdir(pkg_dir):
-            rmtree(pkg_dir)
-        os.makedirs(pkg_dir)
-        assert isdir(pkg_dir)
-
-        self.reset_cache()
-        return pkg_dir
-
     @staticmethod
     def max_satisfying_repo_version(versions, requirements=None):
         item = None
@@ -196,12 +172,11 @@ class PackageManager(object):
             return self.max_satisfying_version(
                 name, requirements).get("_manifest_path")
 
-        manifest_path = None
-        if name.startswith("file://"):
-            manifest_path = self._install_from_local_dir(name[7:])
+        if "://" in name:
+            pkg_dir = self._install_from_url(name, requirements)
         else:
-            manifest_path = self._install_from_piorepo(name, requirements)
-        if not isfile(manifest_path):
+            pkg_dir = self._install_from_piorepo(name, requirements)
+        if not pkg_dir or not isfile(join(pkg_dir, self.manifest_name)):
             raise exception.PackageInstallError(
                 name, requirements or "latest", util.get_systype())
 
@@ -210,32 +185,23 @@ class PackageManager(object):
             telemetry.on_event(
                 category="PackageManager", action="Install", label=name)
 
-        return manifest_path
+        return join(pkg_dir, self.manifest_name)
 
     def _install_from_piorepo(self, name, requirements):
         pkg_dir = None
         pkgdata = None
         versions = None
         for versions in PackageRepoIterator(name, self.repositories):
-            dlpath = None
             pkgdata = self.max_satisfying_repo_version(versions, requirements)
             if not pkgdata:
                 continue
-
-            pkg_dir = self.make_pkg_dir(name, pkgdata['version'])
             try:
-                dlpath = self.download(
-                    pkgdata['url'], pkg_dir, pkgdata.get("sha1"))
-                assert isfile(dlpath)
-                self.unpack(dlpath, pkg_dir)
-                self.check_structure(pkg_dir)
+                pkg_dir = self._install_from_url(
+                    pkgdata['url'], requirements, pkgdata.get("sha1"))
                 break
             except Exception as e:  # pylint: disable=broad-except
                 click.secho("Warning! Package Mirror: %s" % e, fg="yellow")
                 click.secho("Looking for another mirror...", fg="yellow")
-            finally:
-                if dlpath and isfile(dlpath):
-                    os.remove(dlpath)
 
         if versions is None:
             raise exception.UnknownPackage(name)
@@ -246,21 +212,64 @@ class PackageManager(object):
             else:
                 raise exception.UndefinedPackageVersion(
                     name, requirements or "latest", util.get_systype())
+        return pkg_dir
 
-        return join(pkg_dir, self.manifest_name)
+    def _install_from_url(self, url, requirements=None, sha1=None):
+        pkg_dir = None
+        tmp_dir = mkdtemp("-package", "installing-", self.package_dir)
 
-    def _install_from_local_dir(self, local_dir):
-        if not isfile(join(local_dir, self.manifest_name)):
-            raise exception.InvalidLocalPackage(
-                local_dir, self.manifest_name)
+        # Handle GitHub URL (https://github.com/user/repo.git)
+        if url.endswith(".git") and not url.startswith("git"):
+            url = "git+" + url
 
-        manifest = util.load_json(join(local_dir, self.manifest_name))
-        assert set(["name", "version"]) <= set(manifest.keys())
-        pkg_dir = self.make_pkg_dir(manifest['name'], manifest['version'])
-        rmtree(pkg_dir)
-        copytree(local_dir, pkg_dir, symlinks=True)
+        try:
+            if url.startswith("file://"):
+                rmtree(tmp_dir)
+                copytree(url[7:], tmp_dir)
+            elif url.startswith(("http://", "https://", "ftp://")):
+                dlpath = self.download(url, tmp_dir, sha1)
+                assert isfile(dlpath)
+                self.unpack(dlpath, tmp_dir)
+                os.remove(dlpath)
+            else:
+                repo = VCSClientFactory.newClient(url)
+                repo.export(tmp_dir)
 
-        return join(pkg_dir, self.manifest_name)
+            self.check_structure(tmp_dir)
+            pkg_dir = self._install_from_tmp_dir(tmp_dir, requirements)
+        finally:
+            if isdir(tmp_dir):
+                rmtree(tmp_dir)
+        return pkg_dir
+
+    def _install_from_tmp_dir(self, tmp_dir, requirements=None):
+        tmpmanifest = util.load_json(join(tmp_dir, self.manifest_name))
+        assert set(["name", "version"]) <= set(tmpmanifest.keys())
+        name = tmpmanifest['name']
+
+        # package should satisfy requirements
+        if requirements:
+            assert semantic_version.match(requirements, tmpmanifest['version'])
+
+        pkg_dir = join(self.package_dir, name)
+        if isfile(join(pkg_dir, self.manifest_name)):
+            manifest = util.load_json(join(pkg_dir, self.manifest_name))
+            cmp_result = semantic_version.compare(
+                tmpmanifest['version'], manifest['version'])
+            if cmp_result == 1:
+                # if main package version < new package, backup it
+                os.rename(pkg_dir, join(
+                    self.package_dir, "%s@%s" % (name, manifest['version'])))
+            elif cmp_result == -1:
+                pkg_dir = join(
+                    self.package_dir, "%s@%s" % (name, tmpmanifest['version']))
+
+        # remove previous/not-satisfied package
+        if isdir(pkg_dir):
+            rmtree(pkg_dir)
+        os.rename(tmp_dir, pkg_dir)
+        assert isdir(pkg_dir)
+        return pkg_dir
 
     def uninstall(self, name, requirements=None, trigger_event=True):
         click.echo("Uninstalling %s %s @ %s: \t" % (

platformio/managers/platform.py

@@ -44,25 +44,26 @@ class PlatformManager(PackageManager):
                 name, requirements=None, with_packages=None,
                 without_packages=None, skip_default_packages=False):
         manifest_path = PackageManager.install(self, name, requirements)
-        PlatformFactory.newPlatform(
-            manifest_path, requirements).install_packages(
-                with_packages, without_packages, skip_default_packages)
-        self.cleanup_packages()
+        p = PlatformFactory.newPlatform(manifest_path, requirements)
+        p.install_packages(
+            with_packages, without_packages, skip_default_packages)
+        self.cleanup_packages(p.packages.keys())
         return True
 
     def uninstall(self,  # pylint: disable=arguments-differ
                   name, requirements=None):
+        p = PlatformFactory.newPlatform(name, requirements)
         PackageManager.uninstall(self, name, requirements)
-        self.cleanup_packages()
+        self.cleanup_packages(p.packages.keys())
         return True
 
     def update(self,  # pylint: disable=arguments-differ
                name, requirements=None, only_packages=False):
         if not only_packages:
             PackageManager.update(self, name)
-        PlatformFactory.newPlatform(
-            name, requirements).update_packages()
-        self.cleanup_packages()
+        p = PlatformFactory.newPlatform(name, requirements)
+        p.update_packages()
+        self.cleanup_packages(p.packages.keys())
         return True
 
     def is_outdated(self, name, requirements=None):
@@ -70,7 +71,7 @@ class PlatformManager(PackageManager):
         return (p.are_outdated_packages() or
                 p.version != self.get_latest_repo_version(name, requirements))
 
-    def cleanup_packages(self):
+    def cleanup_packages(self, names):
         self.reset_cache()
         deppkgs = {}
         for manifest in PlatformManager().get_installed():
@@ -83,9 +84,10 @@ class PlatformManager(PackageManager):
 
         pm = PackageManager()
         for manifest in pm.get_installed():
-            if manifest['name'] not in deppkgs:
+            if manifest['name'] not in names:
                 continue
-            if manifest['version'] not in deppkgs[manifest['name']]:
+            if (manifest['name'] not in deppkgs or
+                    manifest['version'] not in deppkgs[manifest['name']]):
                 pm.uninstall(
                     manifest['name'], manifest['version'], trigger_event=False)
 

platformio/vcsclient.py

@@ -0,0 +1,97 @@
+# Copyright 2014-present Ivan Kravets <me@ikravets.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from platform import system
+from subprocess import check_call
+from sys import modules
+from urlparse import urlsplit, urlunsplit
+
+from platformio.exception import PlatformioException
+
+
+class VCSClientFactory(object):
+
+    @staticmethod
+    def newClient(url):
+        scheme, netloc, path, query, fragment = urlsplit(url)
+        type_ = scheme
+        if "+" in type_:
+            type_, scheme = type_.split("+", 1)
+        url = urlunsplit((scheme, netloc, path, query, None))
+        clsname = "%sClient" % type_.title()
+        obj = getattr(modules[__name__], clsname)(url, fragment)
+        assert isinstance(obj, VCSClientBase)
+        return obj
+
+
+class VCSClientBase(object):
+
+    command = None
+
+    def __init__(self, url, branch=None):
+        self.url = url
+        self.branch = branch
+        self.check_client()
+
+    def check_client(self):
+        try:
+            assert self.command
+            assert self.run_cmd(["--version"]) == 0
+        except (AssertionError, OSError):
+            raise PlatformioException(
+                "VCS: `%s` client is not installed in your system" %
+                self.command)
+        return True
+
+    def export(self, dst_dir):
+        raise NotImplementedError
+
+    def run_cmd(self, args):
+        return check_call([self.command] + args, shell=system() == "Windows")
+
+
+class GitClient(VCSClientBase):
+
+    command = "git"
+
+    def export(self, dst_dir):
+        args = ["clone", "--recursive", "--depth", "1"]
+        if self.branch:
+            args.extend(["--branch", self.branch])
+        args.extend([self.url, dst_dir])
+        self.run_cmd(args)
+
+
+class HgClient(VCSClientBase):
+
+    command = "hg"
+
+    def export(self, dst_dir):
+        args = ["clone"]
+        if self.branch:
+            args.extend(["--updaterev", self.branch])
+        args.extend([self.url, dst_dir])
+        self.run_cmd(args)
+
+
+class SvnClient(VCSClientBase):
+
+    command = "svn"
+
+    def export(self, dst_dir):
+        args = ["export", "--force"]
+        if self.branch:
+            args.extend(["--revision", self.branch])
+        args.extend([self.url, dst_dir])
+        self.run_cmd(args)