From a05d192bebe96fd1be37ba8325af752b9052ab55 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sun, 4 Sep 2016 00:35:47 +0300 Subject: [PATCH] Implement installing packages/libraries from different archive (with/without manifests) // Resolve #767 --- docs/librarymanager/index.rst | 1 + docs/userguide/lib/cmd_install.rst | 14 ++++++- platformio/__init__.py | 2 +- platformio/commands/lib.py | 2 +- platformio/downloader.py | 21 +++++----- platformio/exception.py | 5 +++ platformio/managers/lib.py | 67 +++++++++++++++++++++++++++++- platformio/managers/package.py | 40 ++++++------------ tests/commands/test_lib.py | 39 ++++++++++++----- 9 files changed, 139 insertions(+), 52 deletions(-) diff --git a/docs/librarymanager/index.rst b/docs/librarymanager/index.rst index 8291dd38..52d8ec87 100644 --- a/docs/librarymanager/index.rst +++ b/docs/librarymanager/index.rst @@ -50,6 +50,7 @@ You can use library ID, Name or even repository URL. For example, Json@~5.6,!=5.4 https://github.com/gioblu/PJON.git@v2.0 https://github.com/me-no-dev/ESPAsyncTCP.git + https://github.com/adafruit/DHT-sensor-library/archive/master.zip Please follow to :ref:`cmd_lib_install` for detailed documentation about possible values. diff --git a/docs/userguide/lib/cmd_install.rst b/docs/userguide/lib/cmd_install.rst index ae130e38..60a846b3 100644 --- a/docs/userguide/lib/cmd_install.rst +++ b/docs/userguide/lib/cmd_install.rst @@ -76,7 +76,7 @@ The ``version`` supports `Semantic Versioning `_ ( * ``>0.1.0,!=0.2.0,<0.3.0`` - any version greater than ``0.1.0``, not equal to ``0.2.0`` and less than ``0.3.0`` -Also, PlatformIO supports installing from local directory or archive. Need +PlatformIO supports installing from local directory or archive. Need to use ``file://`` prefix before local path. Also, directory or archive should contain ``.library.json`` manifest (see :ref:`library_config`). @@ -243,3 +243,15 @@ Examples updating to branch default 2 files updated, 0 files merged, 0 files removed, 0 files unresolved TextLCD @ 308d188a2d3a has been successfully installed! + +5. Install from archive using URL + +.. code:: + + > platformio lib -g install https://github.com/adafruit/DHT-sensor-library/archive/master.zip + + Library Storage: /storage/dir/... + LibraryManager: Installing master + Downloading [####################################] 100% + Unpacking [####################################] 100% + DHT sensor library @ 1.2.3 has been successfully installed! diff --git a/platformio/__init__.py b/platformio/__init__.py index d1a55985..7e0bed98 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (3, 0, "0b11") +VERSION = (3, 0, "0b12") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" diff --git a/platformio/commands/lib.py b/platformio/commands/lib.py index 5b72b57d..bbb196b7 100644 --- a/platformio/commands/lib.py +++ b/platformio/commands/lib.py @@ -143,7 +143,7 @@ def echo_liblist_item(item): click.echo( LIBLIST_TPL.format( id=click.style( - str(item.get("id", "VCS")), fg="green"), + str(item.get("id", "-")), fg="green"), name=click.style( item['name'], fg="cyan"), compatibility=click.style( diff --git a/platformio/downloader.py b/platformio/downloader.py index e91f75db..29db6a05 100644 --- a/platformio/downloader.py +++ b/platformio/downloader.py @@ -30,16 +30,6 @@ class FileDownloader(object): CHUNK_SIZE = 1024 def __init__(self, url, dest_dir=None): - self._url = url - self._fname = url.split("/")[-1] - - self._destination = self._fname - if dest_dir: - self.set_destination(join(dest_dir, self._fname)) - - self._progressbar = None - self._request = None - # make connection self._request = requests.get(url, stream=True, @@ -47,6 +37,17 @@ class FileDownloader(object): if self._request.status_code != 200: raise FDUnrecognizedStatusCode(self._request.status_code, url) + disposition = self._request.headers.get("content-disposition") + if disposition and "filename=" in disposition: + self._fname = disposition[disposition.index("filename=") + 9:] + else: + self._fname = url.split("/")[-1] + + self._progressbar = None + self._destination = self._fname + if dest_dir: + self.set_destination(join(dest_dir, self._fname)) + def set_destination(self, destination): self._destination = destination diff --git a/platformio/exception.py b/platformio/exception.py index 4baf827f..5d8f3b76 100644 --- a/platformio/exception.py +++ b/platformio/exception.py @@ -75,6 +75,11 @@ class UnknownPackage(PlatformioException): MESSAGE = "Detected unknown package '{0}'" +class MissingPackageManifest(PlatformioException): + + MESSAGE = "Could not find '{0}' manifest file in the package" + + class UndefinedPackageVersion(PlatformioException): MESSAGE = "Could not find a version that satisfies the requirement '{0}'"\ diff --git a/platformio/managers/lib.py b/platformio/managers/lib.py index 2ebb7c07..2f8e6ad3 100644 --- a/platformio/managers/lib.py +++ b/platformio/managers/lib.py @@ -13,7 +13,9 @@ # limitations under the License. import json -from os.path import join +import os +from hashlib import md5 +from os.path import dirname, join import click import semantic_version @@ -33,6 +35,69 @@ class LibraryManager(BasePkgManager): def manifest_name(self): return ".library.json" + def check_pkg_structure(self, pkg_dir): + try: + return BasePkgManager.check_pkg_structure(self, pkg_dir) + except exception.MissingPackageManifest: + # we will generate manifest automatically + pass + + manifest = { + "name": "Library_" + md5(pkg_dir).hexdigest()[:5], + "version": "0.0.0" + } + manifest_path = self._find_any_manifest(pkg_dir) + if manifest_path: + _manifest = self._parse_manifest(manifest_path) + pkg_dir = dirname(manifest_path) + for key in ("name", "version"): + if key not in _manifest: + _manifest[key] = manifest[key] + manifest = _manifest + else: + for root, dirs, files in os.walk(pkg_dir): + if len(dirs) == 1 and not files: + manifest['name'] = dirs[0] + continue + if dirs or files: + pkg_dir = root + break + + with open(join(pkg_dir, self.manifest_name), "w") as fp: + json.dump(manifest, fp) + + return pkg_dir + + @staticmethod + def _find_any_manifest(pkg_dir): + manifests = ("library.json", "library.properties", "module.json") + for root, _, files in os.walk(pkg_dir): + for manifest in manifests: + if manifest in files: + return join(root, manifest) + return None + + @staticmethod + def _parse_manifest(path): + manifest = {} + if path.endswith(".json"): + return util.load_json(path) + elif path.endswith("library.properties"): + with open(path) as fp: + for line in fp.readlines(): + if "=" not in line: + continue + key, value = line.split("=", 1) + manifest[key.strip()] = value.strip() + manifest['frameworks'] = ["arduino"] + if "author" in manifest: + manifest['authors'] = [{"name": manifest['author']}] + del manifest['author'] + if "sentence" in manifest: + manifest['description'] = manifest['sentence'] + del manifest['sentence'] + return manifest + @staticmethod def normalize_dependencies(dependencies): if not dependencies: diff --git a/platformio/managers/package.py b/platformio/managers/package.py index 14e9644f..c536a9e6 100644 --- a/platformio/managers/package.py +++ b/platformio/managers/package.py @@ -15,7 +15,7 @@ import json import os from os.path import basename, dirname, isdir, isfile, islink, join -from shutil import copyfile, copytree +from shutil import copytree from tempfile import mkdtemp import click @@ -147,32 +147,13 @@ class PkgInstallerMixin(object): def check_pkg_structure(self, pkg_dir): if self.manifest_exists(pkg_dir): - return True + return pkg_dir for root, _, _ in os.walk(pkg_dir): - if not self.manifest_exists(root): - continue - # copy contents to the root of package directory - for item in os.listdir(root): - item_path = join(root, item) - if isfile(item_path): - copyfile(item_path, join(pkg_dir, item)) - elif isdir(item_path): - copytree(item_path, join(pkg_dir, item), symlinks=True) - # remove not used contents - while True: - util.rmtree_(root) - root = dirname(root) - if root == pkg_dir: - break - break + if self.manifest_exists(root): + return root - if self.manifest_exists(pkg_dir): - return True - - raise exception.PlatformioException( - "Could not find '%s' manifest file in the package" % - self.manifest_name) + raise exception.MissingPackageManifest(self.manifest_name) def _install_from_piorepo(self, name, requirements): pkg_dir = None @@ -226,8 +207,8 @@ class PkgInstallerMixin(object): "requirements": requirements }, fp) - self.check_pkg_structure(tmp_dir) - pkg_dir = self._install_from_tmp_dir(tmp_dir, requirements) + pkg_dir = self.check_pkg_structure(tmp_dir) + pkg_dir = self._install_from_tmp_dir(pkg_dir, requirements) finally: if isdir(tmp_dir): util.rmtree_(tmp_dir) @@ -554,7 +535,12 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin): manifest['version'] = vcs.get_current_revision() json.dump(manifest, fp) else: - latest_version = self.get_latest_repo_version(name, requirements) + latest_version = None + try: + latest_version = self.get_latest_repo_version(name, + requirements) + except exception.PlatformioException: + pass if not latest_version: click.echo("[%s]" % (click.style("Unknown", fg="yellow"))) return diff --git a/tests/commands/test_lib.py b/tests/commands/test_lib.py index 0daae542..9e8f1c26 100644 --- a/tests/commands/test_lib.py +++ b/tests/commands/test_lib.py @@ -36,15 +36,31 @@ def test_search(clirunner, validate_cliresult): def test_global_install_registry(clirunner, validate_cliresult, isolated_pio_home): - result = clirunner.invoke(cmd_lib, - ["-g", "install", "58", "OneWire", - "ArduinoJson@5.4.0", "ArduinoJson@>5.4"]) + result = clirunner.invoke(cmd_lib, [ + "-g", "install", "58", "OneWire", + "http://dl.platformio.org/libraries/archives/3/3756.tar.gz", + "ArduinoJson@5.4.0", "ArduinoJson@>5.4" + ]) validate_cliresult(result) items1 = [d.basename for d in isolated_pio_home.join("lib").listdir()] - items2 = ["DHT22_ID58", "ArduinoJson_ID64", "Json_ID64", "OneWire_ID1"] + items2 = ["DHT22_ID58", "ArduinoJson_ID64", "Json_ID64", "OneWire_ID1", + "ESPAsyncTCP_ID305"] assert set(items1) == set(items2) +def test_global_install_archive(clirunner, validate_cliresult, + isolated_pio_home): + result = clirunner.invoke(cmd_lib, [ + "-g", "install", "https://github.com/adafruit/Adafruit-ST7735-Library/" + "archive/master.zip", + "http://www.airspayce.com/mikem/arduino/RadioHead/RadioHead-1.62.zip" + ]) + validate_cliresult(result) + items1 = [d.basename for d in isolated_pio_home.join("lib").listdir()] + items2 = ["Adafruit ST7735 Library", "RadioHead"] + assert set(items1) >= set(items2) + + def test_global_install_repository(clirunner, validate_cliresult, isolated_pio_home): result = clirunner.invoke( @@ -54,7 +70,6 @@ def test_global_install_repository(clirunner, validate_cliresult, "https://github.com/gioblu/PJON.git#3.0", "https://gitlab.com/ivankravets/rs485-nodeproto.git", # "https://developer.mbed.org/users/simon/code/TextLCD/", - "http://dl.platformio.org/libraries/archives/3/3756.tar.gz", "knolleary/pubsubclient"]) validate_cliresult(result) items1 = [d.basename for d in isolated_pio_home.join("lib").listdir()] @@ -73,7 +88,8 @@ def test_global_lib_list(clirunner, validate_cliresult, isolated_pio_home): for n in ("PJON", "git+https://github.com/knolleary/pubsubclient")]) items1 = [i['name'] for i in json.loads(result.output)] items2 = ["OneWire", "DHT22", "PJON", "ESPAsyncTCP", "Json", "ArduinoJson", - "pubsubclient", "rs485-nodeproto"] + "pubsubclient", "rs485-nodeproto", "Adafruit ST7735 Library", + "RadioHead"] assert set(items1) == set(items2) @@ -102,12 +118,13 @@ def test_global_lib_update(clirunner, validate_cliresult, isolated_pio_home): def test_global_lib_uninstall(clirunner, validate_cliresult, isolated_pio_home): - result = clirunner.invoke( - cmd_lib, ["-g", "uninstall", "1", "ArduinoJson@!=5.4.0", "TextLCD"]) + result = clirunner.invoke(cmd_lib, + ["-g", "uninstall", "1", "ArduinoJson@!=5.4.0", + "TextLCD", "Adafruit ST7735 Library"]) validate_cliresult(result) items1 = [d.basename for d in isolated_pio_home.join("lib").listdir()] items2 = ["DHT22_ID58", "Json_ID64", "ESPAsyncTCP_ID305", "pubsubclient", - "PJON", "rs485-nodeproto"] + "PJON", "rs485-nodeproto", "RadioHead_ID124"] assert set(items1) == set(items2) @@ -121,8 +138,8 @@ def test_project_lib_complex(clirunner, validate_cliresult, tmpdir): result = clirunner.invoke(cmd_lib, ["install", "54", "ArduinoJson"]) validate_cliresult(result) items1 = [d.basename - for d in tmpdir.join(basename(util.get_projectlibdeps_dir( - ))).listdir()] + for d in tmpdir.join(basename(util.get_projectlibdeps_dir())) + .listdir()] items2 = ["DallasTemperature_ID54", "OneWire_ID1", "ArduinoJson_ID64"] assert set(items1) == set(items2)