Implement installing packages/libraries from different archive (with/without manifests) // Resolve #767

platformio/__init__.py

@@ -14,7 +14,7 @@
 
 import sys
 
-VERSION = (3, 0, "0b11")
+VERSION = (3, 0, "0b12")
 __version__ = ".".join([str(s) for s in VERSION])
 
 __title__ = "platformio"

platformio/commands/lib.py

@@ -143,7 +143,7 @@ def echo_liblist_item(item):
     click.echo(
         LIBLIST_TPL.format(
             id=click.style(
-                str(item.get("id", "VCS")), fg="green"),
+                str(item.get("id", "-")), fg="green"),
             name=click.style(
                 item['name'], fg="cyan"),
             compatibility=click.style(

platformio/downloader.py

@@ -30,16 +30,6 @@ class FileDownloader(object):
     CHUNK_SIZE = 1024
 
     def __init__(self, url, dest_dir=None):
-        self._url = url
-        self._fname = url.split("/")[-1]
-
-        self._destination = self._fname
-        if dest_dir:
-            self.set_destination(join(dest_dir, self._fname))
-
-        self._progressbar = None
-        self._request = None
-
         # make connection
         self._request = requests.get(url,
                                      stream=True,
@@ -47,6 +37,17 @@ class FileDownloader(object):
         if self._request.status_code != 200:
             raise FDUnrecognizedStatusCode(self._request.status_code, url)
 
+        disposition = self._request.headers.get("content-disposition")
+        if disposition and "filename=" in disposition:
+            self._fname = disposition[disposition.index("filename=") + 9:]
+        else:
+            self._fname = url.split("/")[-1]
+
+        self._progressbar = None
+        self._destination = self._fname
+        if dest_dir:
+            self.set_destination(join(dest_dir, self._fname))
+
     def set_destination(self, destination):
         self._destination = destination
 

platformio/exception.py

@@ -75,6 +75,11 @@ class UnknownPackage(PlatformioException):
     MESSAGE = "Detected unknown package '{0}'"
 
 
+class MissingPackageManifest(PlatformioException):
+
+    MESSAGE = "Could not find '{0}' manifest file in the package"
+
+
 class UndefinedPackageVersion(PlatformioException):
 
     MESSAGE = "Could not find a version that satisfies the requirement '{0}'"\

platformio/managers/lib.py

@@ -13,7 +13,9 @@
 # limitations under the License.
 
 import json
-from os.path import join
+import os
+from hashlib import md5
+from os.path import dirname, join
 
 import click
 import semantic_version
@@ -33,6 +35,69 @@ class LibraryManager(BasePkgManager):
     def manifest_name(self):
         return ".library.json"
 
+    def check_pkg_structure(self, pkg_dir):
+        try:
+            return BasePkgManager.check_pkg_structure(self, pkg_dir)
+        except exception.MissingPackageManifest:
+            # we will generate manifest automatically
+            pass
+
+        manifest = {
+            "name": "Library_" + md5(pkg_dir).hexdigest()[:5],
+            "version": "0.0.0"
+        }
+        manifest_path = self._find_any_manifest(pkg_dir)
+        if manifest_path:
+            _manifest = self._parse_manifest(manifest_path)
+            pkg_dir = dirname(manifest_path)
+            for key in ("name", "version"):
+                if key not in _manifest:
+                    _manifest[key] = manifest[key]
+            manifest = _manifest
+        else:
+            for root, dirs, files in os.walk(pkg_dir):
+                if len(dirs) == 1 and not files:
+                    manifest['name'] = dirs[0]
+                    continue
+                if dirs or files:
+                    pkg_dir = root
+                    break
+
+        with open(join(pkg_dir, self.manifest_name), "w") as fp:
+            json.dump(manifest, fp)
+
+        return pkg_dir
+
+    @staticmethod
+    def _find_any_manifest(pkg_dir):
+        manifests = ("library.json", "library.properties", "module.json")
+        for root, _, files in os.walk(pkg_dir):
+            for manifest in manifests:
+                if manifest in files:
+                    return join(root, manifest)
+        return None
+
+    @staticmethod
+    def _parse_manifest(path):
+        manifest = {}
+        if path.endswith(".json"):
+            return util.load_json(path)
+        elif path.endswith("library.properties"):
+            with open(path) as fp:
+                for line in fp.readlines():
+                    if "=" not in line:
+                        continue
+                    key, value = line.split("=", 1)
+                    manifest[key.strip()] = value.strip()
+            manifest['frameworks'] = ["arduino"]
+            if "author" in manifest:
+                manifest['authors'] = [{"name": manifest['author']}]
+                del manifest['author']
+            if "sentence" in manifest:
+                manifest['description'] = manifest['sentence']
+                del manifest['sentence']
+        return manifest
+
     @staticmethod
     def normalize_dependencies(dependencies):
         if not dependencies:

platformio/managers/package.py

@@ -15,7 +15,7 @@
 import json
 import os
 from os.path import basename, dirname, isdir, isfile, islink, join
-from shutil import copyfile, copytree
+from shutil import copytree
 from tempfile import mkdtemp
 
 import click
@@ -147,32 +147,13 @@ class PkgInstallerMixin(object):
 
     def check_pkg_structure(self, pkg_dir):
         if self.manifest_exists(pkg_dir):
-            return True
+            return pkg_dir
 
         for root, _, _ in os.walk(pkg_dir):
-            if not self.manifest_exists(root):
-                continue
-            # copy contents to the root of package directory
-            for item in os.listdir(root):
-                item_path = join(root, item)
-                if isfile(item_path):
-                    copyfile(item_path, join(pkg_dir, item))
-                elif isdir(item_path):
-                    copytree(item_path, join(pkg_dir, item), symlinks=True)
-            # remove not used contents
-            while True:
-                util.rmtree_(root)
-                root = dirname(root)
-                if root == pkg_dir:
-                    break
-            break
+            if self.manifest_exists(root):
+                return root
 
-        if self.manifest_exists(pkg_dir):
-            return True
-
-        raise exception.PlatformioException(
-            "Could not find '%s' manifest file in the package" %
-            self.manifest_name)
+        raise exception.MissingPackageManifest(self.manifest_name)
 
     def _install_from_piorepo(self, name, requirements):
         pkg_dir = None
@@ -226,8 +207,8 @@ class PkgInstallerMixin(object):
                         "requirements": requirements
                     }, fp)
 
-            self.check_pkg_structure(tmp_dir)
-            pkg_dir = self._install_from_tmp_dir(tmp_dir, requirements)
+            pkg_dir = self.check_pkg_structure(tmp_dir)
+            pkg_dir = self._install_from_tmp_dir(pkg_dir, requirements)
         finally:
             if isdir(tmp_dir):
                 util.rmtree_(tmp_dir)
@@ -554,7 +535,12 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin):
                 manifest['version'] = vcs.get_current_revision()
                 json.dump(manifest, fp)
         else:
-            latest_version = self.get_latest_repo_version(name, requirements)
+            latest_version = None
+            try:
+                latest_version = self.get_latest_repo_version(name,
+                                                              requirements)
+            except exception.PlatformioException:
+                pass
             if not latest_version:
                 click.echo("[%s]" % (click.style("Unknown", fg="yellow")))
                 return