Implement installing packages/libraries from different archive (with/without manifests) // Resolve #767

docs/librarymanager/index.rst

@@ -50,6 +50,7 @@ You can use library ID, Name or even repository URL. For example,
     Json@~5.6,!=5.4
     https://github.com/gioblu/PJON.git@v2.0
     https://github.com/me-no-dev/ESPAsyncTCP.git
+    https://github.com/adafruit/DHT-sensor-library/archive/master.zip
 
 Please follow to :ref:`cmd_lib_install` for detailed documentation about
 possible values.

docs/userguide/lib/cmd_install.rst

@@ -76,7 +76,7 @@ The ``version`` supports `Semantic Versioning <http://semver.org>`_ (
 * ``>0.1.0,!=0.2.0,<0.3.0`` - any version greater than ``0.1.0``, not equal to
   ``0.2.0`` and less than ``0.3.0``
 
-Also, PlatformIO supports installing from local directory or archive. Need
+PlatformIO supports installing from local directory or archive. Need
 to use ``file://`` prefix before local path. Also, directory or
 archive should contain ``.library.json`` manifest (see :ref:`library_config`).
 
@@ -243,3 +243,15 @@ Examples
     updating to branch default
     2 files updated, 0 files merged, 0 files removed, 0 files unresolved
     TextLCD @ 308d188a2d3a has been successfully installed!
+
+5. Install from archive using URL
+
+.. code::
+
+    > platformio lib -g install  https://github.com/adafruit/DHT-sensor-library/archive/master.zip
+
+    Library Storage: /storage/dir/...
+    LibraryManager: Installing master
+    Downloading  [####################################]  100%
+    Unpacking  [####################################]  100%
+    DHT sensor library @ 1.2.3 has been successfully installed!

platformio/__init__.py

@@ -14,7 +14,7 @@
 
 import sys
 
-VERSION = (3, 0, "0b11")
+VERSION = (3, 0, "0b12")
 __version__ = ".".join([str(s) for s in VERSION])
 
 __title__ = "platformio"

platformio/commands/lib.py

@@ -143,7 +143,7 @@ def echo_liblist_item(item):
     click.echo(
         LIBLIST_TPL.format(
             id=click.style(
-                str(item.get("id", "VCS")), fg="green"),
+                str(item.get("id", "-")), fg="green"),
             name=click.style(
                 item['name'], fg="cyan"),
             compatibility=click.style(

platformio/downloader.py

@@ -30,16 +30,6 @@ class FileDownloader(object):
     CHUNK_SIZE = 1024
 
     def __init__(self, url, dest_dir=None):
-        self._url = url
-        self._fname = url.split("/")[-1]
-
-        self._destination = self._fname
-        if dest_dir:
-            self.set_destination(join(dest_dir, self._fname))
-
-        self._progressbar = None
-        self._request = None
-
         # make connection
         self._request = requests.get(url,
                                      stream=True,
@@ -47,6 +37,17 @@ class FileDownloader(object):
         if self._request.status_code != 200:
             raise FDUnrecognizedStatusCode(self._request.status_code, url)
 
+        disposition = self._request.headers.get("content-disposition")
+        if disposition and "filename=" in disposition:
+            self._fname = disposition[disposition.index("filename=") + 9:]
+        else:
+            self._fname = url.split("/")[-1]
+
+        self._progressbar = None
+        self._destination = self._fname
+        if dest_dir:
+            self.set_destination(join(dest_dir, self._fname))
+
     def set_destination(self, destination):
         self._destination = destination
 

platformio/exception.py

@@ -75,6 +75,11 @@ class UnknownPackage(PlatformioException):
     MESSAGE = "Detected unknown package '{0}'"
 
 
+class MissingPackageManifest(PlatformioException):
+
+    MESSAGE = "Could not find '{0}' manifest file in the package"
+
+
 class UndefinedPackageVersion(PlatformioException):
 
     MESSAGE = "Could not find a version that satisfies the requirement '{0}'"\

platformio/managers/lib.py

@@ -13,7 +13,9 @@
 # limitations under the License.
 
 import json
-from os.path import join
+import os
+from hashlib import md5
+from os.path import dirname, join
 
 import click
 import semantic_version
@@ -33,6 +35,69 @@ class LibraryManager(BasePkgManager):
     def manifest_name(self):
         return ".library.json"
 
+    def check_pkg_structure(self, pkg_dir):
+        try:
+            return BasePkgManager.check_pkg_structure(self, pkg_dir)
+        except exception.MissingPackageManifest:
+            # we will generate manifest automatically
+            pass
+
+        manifest = {
+            "name": "Library_" + md5(pkg_dir).hexdigest()[:5],
+            "version": "0.0.0"
+        }
+        manifest_path = self._find_any_manifest(pkg_dir)
+        if manifest_path:
+            _manifest = self._parse_manifest(manifest_path)
+            pkg_dir = dirname(manifest_path)
+            for key in ("name", "version"):
+                if key not in _manifest:
+                    _manifest[key] = manifest[key]
+            manifest = _manifest
+        else:
+            for root, dirs, files in os.walk(pkg_dir):
+                if len(dirs) == 1 and not files:
+                    manifest['name'] = dirs[0]
+                    continue
+                if dirs or files:
+                    pkg_dir = root
+                    break
+
+        with open(join(pkg_dir, self.manifest_name), "w") as fp:
+            json.dump(manifest, fp)
+
+        return pkg_dir
+
+    @staticmethod
+    def _find_any_manifest(pkg_dir):
+        manifests = ("library.json", "library.properties", "module.json")
+        for root, _, files in os.walk(pkg_dir):
+            for manifest in manifests:
+                if manifest in files:
+                    return join(root, manifest)
+        return None
+
+    @staticmethod
+    def _parse_manifest(path):
+        manifest = {}
+        if path.endswith(".json"):
+            return util.load_json(path)
+        elif path.endswith("library.properties"):
+            with open(path) as fp:
+                for line in fp.readlines():
+                    if "=" not in line:
+                        continue
+                    key, value = line.split("=", 1)
+                    manifest[key.strip()] = value.strip()
+            manifest['frameworks'] = ["arduino"]
+            if "author" in manifest:
+                manifest['authors'] = [{"name": manifest['author']}]
+                del manifest['author']
+            if "sentence" in manifest:
+                manifest['description'] = manifest['sentence']
+                del manifest['sentence']
+        return manifest
+
     @staticmethod
     def normalize_dependencies(dependencies):
         if not dependencies:

platformio/managers/package.py

@@ -15,7 +15,7 @@
 import json
 import os
 from os.path import basename, dirname, isdir, isfile, islink, join
-from shutil import copyfile, copytree
+from shutil import copytree
 from tempfile import mkdtemp
 
 import click
@@ -147,32 +147,13 @@ class PkgInstallerMixin(object):
 
     def check_pkg_structure(self, pkg_dir):
         if self.manifest_exists(pkg_dir):
-            return True
+            return pkg_dir
 
         for root, _, _ in os.walk(pkg_dir):
-            if not self.manifest_exists(root):
+            if self.manifest_exists(root):
-                continue
+                return root
-            # copy contents to the root of package directory
-            for item in os.listdir(root):
-                item_path = join(root, item)
-                if isfile(item_path):
-                    copyfile(item_path, join(pkg_dir, item))
-                elif isdir(item_path):
-                    copytree(item_path, join(pkg_dir, item), symlinks=True)
-            # remove not used contents
-            while True:
-                util.rmtree_(root)
-                root = dirname(root)
-                if root == pkg_dir:
-                    break
-            break
 
-        if self.manifest_exists(pkg_dir):
+        raise exception.MissingPackageManifest(self.manifest_name)
-            return True
-
-        raise exception.PlatformioException(
-            "Could not find '%s' manifest file in the package" %
-            self.manifest_name)
 
     def _install_from_piorepo(self, name, requirements):
         pkg_dir = None
@@ -226,8 +207,8 @@ class PkgInstallerMixin(object):
                         "requirements": requirements
                     }, fp)
 
-            self.check_pkg_structure(tmp_dir)
+            pkg_dir = self.check_pkg_structure(tmp_dir)
-            pkg_dir = self._install_from_tmp_dir(tmp_dir, requirements)
+            pkg_dir = self._install_from_tmp_dir(pkg_dir, requirements)
         finally:
             if isdir(tmp_dir):
                 util.rmtree_(tmp_dir)
@@ -554,7 +535,12 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin):
                 manifest['version'] = vcs.get_current_revision()
                 json.dump(manifest, fp)
         else:
-            latest_version = self.get_latest_repo_version(name, requirements)
+            latest_version = None
+            try:
+                latest_version = self.get_latest_repo_version(name,
+                                                              requirements)
+            except exception.PlatformioException:
+                pass
             if not latest_version:
                 click.echo("[%s]" % (click.style("Unknown", fg="yellow")))
                 return

tests/commands/test_lib.py

@@ -36,15 +36,31 @@ def test_search(clirunner, validate_cliresult):
 
 def test_global_install_registry(clirunner, validate_cliresult,
                                  isolated_pio_home):
-    result = clirunner.invoke(cmd_lib,
+    result = clirunner.invoke(cmd_lib, [
-                              ["-g", "install", "58", "OneWire",
+        "-g", "install", "58", "OneWire",
-                               "ArduinoJson@5.4.0", "ArduinoJson@>5.4"])
+        "http://dl.platformio.org/libraries/archives/3/3756.tar.gz",
+        "ArduinoJson@5.4.0", "ArduinoJson@>5.4"
+    ])
     validate_cliresult(result)
     items1 = [d.basename for d in isolated_pio_home.join("lib").listdir()]
-    items2 = ["DHT22_ID58", "ArduinoJson_ID64", "Json_ID64", "OneWire_ID1"]
+    items2 = ["DHT22_ID58", "ArduinoJson_ID64", "Json_ID64", "OneWire_ID1",
+              "ESPAsyncTCP_ID305"]
     assert set(items1) == set(items2)
 
 
+def test_global_install_archive(clirunner, validate_cliresult,
+                                isolated_pio_home):
+    result = clirunner.invoke(cmd_lib, [
+        "-g", "install", "https://github.com/adafruit/Adafruit-ST7735-Library/"
+        "archive/master.zip",
+        "http://www.airspayce.com/mikem/arduino/RadioHead/RadioHead-1.62.zip"
+    ])
+    validate_cliresult(result)
+    items1 = [d.basename for d in isolated_pio_home.join("lib").listdir()]
+    items2 = ["Adafruit ST7735 Library", "RadioHead"]
+    assert set(items1) >= set(items2)
+
+
 def test_global_install_repository(clirunner, validate_cliresult,
                                    isolated_pio_home):
     result = clirunner.invoke(
@@ -54,7 +70,6 @@ def test_global_install_repository(clirunner, validate_cliresult,
          "https://github.com/gioblu/PJON.git#3.0",
          "https://gitlab.com/ivankravets/rs485-nodeproto.git",
          # "https://developer.mbed.org/users/simon/code/TextLCD/",
-         "http://dl.platformio.org/libraries/archives/3/3756.tar.gz",
          "knolleary/pubsubclient"])
     validate_cliresult(result)
     items1 = [d.basename for d in isolated_pio_home.join("lib").listdir()]
@@ -73,7 +88,8 @@ def test_global_lib_list(clirunner, validate_cliresult, isolated_pio_home):
          for n in ("PJON", "git+https://github.com/knolleary/pubsubclient")])
     items1 = [i['name'] for i in json.loads(result.output)]
     items2 = ["OneWire", "DHT22", "PJON", "ESPAsyncTCP", "Json", "ArduinoJson",
-              "pubsubclient", "rs485-nodeproto"]
+              "pubsubclient", "rs485-nodeproto", "Adafruit ST7735 Library",
+              "RadioHead"]
     assert set(items1) == set(items2)
 
 
@@ -102,12 +118,13 @@ def test_global_lib_update(clirunner, validate_cliresult, isolated_pio_home):
 
 def test_global_lib_uninstall(clirunner, validate_cliresult,
                               isolated_pio_home):
-    result = clirunner.invoke(
+    result = clirunner.invoke(cmd_lib,
-        cmd_lib, ["-g", "uninstall", "1", "ArduinoJson@!=5.4.0", "TextLCD"])
+                              ["-g", "uninstall", "1", "ArduinoJson@!=5.4.0",
+                               "TextLCD", "Adafruit ST7735 Library"])
     validate_cliresult(result)
     items1 = [d.basename for d in isolated_pio_home.join("lib").listdir()]
     items2 = ["DHT22_ID58", "Json_ID64", "ESPAsyncTCP_ID305", "pubsubclient",
-              "PJON", "rs485-nodeproto"]
+              "PJON", "rs485-nodeproto", "RadioHead_ID124"]
     assert set(items1) == set(items2)
 
 
@@ -121,8 +138,8 @@ def test_project_lib_complex(clirunner, validate_cliresult, tmpdir):
         result = clirunner.invoke(cmd_lib, ["install", "54", "ArduinoJson"])
         validate_cliresult(result)
         items1 = [d.basename
-                  for d in tmpdir.join(basename(util.get_projectlibdeps_dir(
+                  for d in tmpdir.join(basename(util.get_projectlibdeps_dir()))
-                  ))).listdir()]
+                  .listdir()]
         items2 = ["DallasTemperature_ID54", "OneWire_ID1", "ArduinoJson_ID64"]
         assert set(items1) == set(items2)