diff --git a/HISTORY.rst b/HISTORY.rst index fa6f3a30..d535501a 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -38,6 +38,7 @@ PlatformIO 3.0 * Escape project path when Glob matching is used * Do not overwrite project configuration variables when system environment variables are set +* Handle dependencies when installing non-registry package/library (VCS, archive, local folder) * Fixed package installing with VCS branch for Python 2.7.3 (`issue #885 `_) diff --git a/platformio/commands/lib.py b/platformio/commands/lib.py index d418221e..068c96b5 100644 --- a/platformio/commands/lib.py +++ b/platformio/commands/lib.py @@ -15,7 +15,7 @@ # pylint: disable=too-many-branches, too-many-locals import json -from os.path import join +from os.path import isdir, join from time import sleep from urllib import quote @@ -116,22 +116,23 @@ def lib_uninstall(lm, libraries): @click.pass_obj def lib_update(lm, libraries, only_check, json_output): if not libraries: - libraries = [] - for manifest in lm.get_installed(): - if "@vcs-" in manifest['__pkg_dir']: - libraries.append("%s=%s" % (manifest['name'], manifest['url'])) - else: - libraries.append(str(manifest.get("id", manifest['name']))) + libraries = [manifest['__pkg_dir'] for manifest in lm.get_installed()] if only_check and json_output: result = [] for library in libraries: - name, requirements, url = lm.parse_pkg_name(library) - latest = lm.outdated(name, requirements, url) + pkg_dir = library if isdir(library) else None + requirements = None + url = None + if not pkg_dir: + name, requirements, url = lm.parse_pkg_input(library) + pkg_dir = lm.get_package_dir(name, requirements, url) + if not pkg_dir: + continue + latest = lm.outdated(pkg_dir, requirements) if not latest: continue - manifest = lm.load_manifest( - lm.get_package_dir(name, requirements, url)) + manifest = lm.load_manifest(pkg_dir) manifest['versionLatest'] = latest result.append(manifest) return click.echo(json.dumps(result)) @@ -167,6 +168,9 @@ def print_lib_item(item): click.echo("Authors: %s" % ", ".join( item.get("authornames", [a.get("name", "") for a in item.get("authors", [])]))) + + if "__src_url" in item: + click.secho("Source: %s" % item['__src_url']) click.echo() @@ -270,8 +274,7 @@ def get_builtin_libs(storage_names=None): storage_names = storage_names or [] pm = PlatformManager() for manifest in pm.get_installed(): - p = PlatformFactory.newPlatform( - pm.get_manifest_path(manifest['__pkg_dir'])) + p = PlatformFactory.newPlatform(manifest['__pkg_dir']) for storage in p.get_lib_storages(): if storage_names and storage['name'] not in storage_names: continue @@ -308,7 +311,7 @@ def lib_builtin(storage, json_output): @click.option("--json-output", is_flag=True) def lib_show(library, json_output): lm = LibraryManager() - name, requirements, _ = lm.parse_pkg_name(library) + name, requirements, _ = lm.parse_pkg_input(library) lib_id = lm.get_pkg_id_by_name( name, requirements, silent=json_output, interactive=not json_output) lib = get_api_result("/lib/info/%d" % lib_id, cache_valid="1d") diff --git a/platformio/commands/platform.py b/platformio/commands/platform.py index 0d138633..0d01cc6d 100644 --- a/platformio/commands/platform.py +++ b/platformio/commands/platform.py @@ -13,7 +13,7 @@ # limitations under the License. import json -from os.path import dirname, isfile, join +from os.path import dirname, isdir import click @@ -94,10 +94,12 @@ def _get_installed_platform_data(platform, # del data['version'] # return data - data['__pkg_dir'] = dirname(p.manifest_path) - # if VCS cloned platform - if not isfile(join(data['__pkg_dir'], "platform.json")): - data['__pkg_dir'] = dirname(data['__pkg_dir']) + # overwrite VCS version and add extra fields + manifest = PlatformManager().load_manifest(dirname(p.manifest_path)) + assert manifest + for key in manifest: + if key == "version" or key.startswith("__"): + data[key] = manifest[key] if with_boards: data['boards'] = [c.get_brief_data() for c in p.get_boards().values()] @@ -214,7 +216,7 @@ def platform_list(json_output): for manifest in pm.get_installed(): platforms.append( _get_installed_platform_data( - pm.get_manifest_path(manifest['__pkg_dir']), + manifest['__pkg_dir'], with_boards=False, expose_packages=False)) if json_output: @@ -336,22 +338,25 @@ def platform_update(platforms, only_packages, only_check, json_output): pm = PlatformManager() if not platforms: platforms = [] - for manifest in pm.get_installed(): - if "@vcs-" in manifest['__pkg_dir']: - platforms.append("%s=%s" % (manifest['name'], manifest['url'])) - else: - platforms.append(manifest['name']) + platforms = [manifest['__pkg_dir'] for manifest in pm.get_installed()] if only_check and json_output: result = [] for platform in platforms: - name, requirements, url = pm.parse_pkg_name(platform) - latest = pm.outdated(name, requirements, url) + pkg_dir = platform if isdir(platform) else None + requirements = None + url = None + if not pkg_dir: + name, requirements, url = pm.parse_pkg_input(platform) + pkg_dir = pm.get_package_dir(name, requirements, url) + if not pkg_dir: + continue + latest = pm.outdated(pkg_dir, requirements) if not latest: continue data = _get_installed_platform_data( - name, with_boards=False, expose_packages=False) - data['versionLatest'] = latest or "Unknown" + pkg_dir, with_boards=False, expose_packages=False) + data['versionLatest'] = latest result.append(data) return click.echo(json.dumps(result)) else: diff --git a/platformio/maintenance.py b/platformio/maintenance.py index af98280b..2acc8ef1 100644 --- a/platformio/maintenance.py +++ b/platformio/maintenance.py @@ -163,8 +163,7 @@ def after_upgrade(ctx): # update development platforms pm = PlatformManager() for manifest in pm.get_installed(): - # pm.update(manifest['name'], "^" + manifest['version']) - pm.update(manifest['name']) + pm.update(manifest['__pkg_dir']) # update PlatformIO Plus tool if installed pioplus_update() @@ -262,7 +261,7 @@ def check_internal_updates(ctx, what): outdated_items = [] for manifest in pm.get_installed(): if manifest['name'] not in outdated_items and \ - pm.outdated(manifest['name']): + pm.outdated(manifest['__pkg_dir']): outdated_items.append(manifest['name']) if not outdated_items: diff --git a/platformio/managers/lib.py b/platformio/managers/lib.py index 0e9a5457..2b466c62 100644 --- a/platformio/managers/lib.py +++ b/platformio/managers/lib.py @@ -15,10 +15,8 @@ # pylint: disable=too-many-arguments, too-many-locals, too-many-branches import json -import os import re from glob import glob -from hashlib import md5 from os.path import isdir, join import arrow @@ -61,8 +59,8 @@ class LibraryManager(BasePkgManager): return None - def load_manifest(self, path): - manifest = BasePkgManager.load_manifest(self, path) + def load_manifest(self, pkg_dir): + manifest = BasePkgManager.load_manifest(self, pkg_dir) if not manifest: return manifest @@ -76,6 +74,9 @@ class LibraryManager(BasePkgManager): manifest['authors'] = [{"name": manifest['author']}] del manifest['author'] + if "authors" in manifest and not isinstance(manifest['authors'], list): + manifest['authors'] = [manifest['authors']] + if "keywords" not in manifest: keywords = [] for keyword in re.split(r"[\s/]+", @@ -123,31 +124,6 @@ class LibraryManager(BasePkgManager): return manifest - def check_pkg_structure(self, pkg_dir): - try: - return BasePkgManager.check_pkg_structure(self, pkg_dir) - except exception.MissingPackageManifest: - # we will generate manifest automatically - # if library doesn't contain any - pass - - manifest = { - "name": "Library_" + md5(pkg_dir).hexdigest()[:5], - "version": "0.0.0" - } - for root, dirs, files in os.walk(pkg_dir): - if len(dirs) == 1 and not files: - manifest['name'] = dirs[0] - continue - if dirs or files: - pkg_dir = root - break - - with open(join(pkg_dir, self.manifest_names[0]), "w") as fp: - json.dump(manifest, fp) - - return pkg_dir - @staticmethod def normalize_dependencies(dependencies): if not dependencies: @@ -239,7 +215,7 @@ class LibraryManager(BasePkgManager): }, silent, interactive)['id']) def _install_from_piorepo(self, name, requirements): - assert name.startswith("id=") + assert name.startswith("id="), name version = self.get_latest_repo_version(name, requirements) if not version: raise exception.UndefinedPackageVersion(requirements or "latest", @@ -260,28 +236,23 @@ class LibraryManager(BasePkgManager): silent=False, trigger_event=True, interactive=False): - already_installed = False - _name, _requirements, _url = self.parse_pkg_name(name, requirements) - try: + _name, _requirements, _url = self.parse_pkg_input(name, + requirements) if not _url: - _name = "id=%d" % self.get_pkg_id_by_name( + name = "id=%d" % self.get_pkg_id_by_name( _name, _requirements, silent=silent, interactive=interactive) - already_installed = self.get_package(_name, _requirements, _url) - pkg_dir = BasePkgManager.install( - self, _name - if not _url else name, _requirements, silent, trigger_event) + requirements = _requirements + pkg_dir = BasePkgManager.install(self, name, requirements, silent, + trigger_event) except exception.InternetIsOffline as e: if not silent: click.secho(str(e), fg="yellow") return - if already_installed: - return - manifest = self.load_manifest(pkg_dir) if "dependencies" not in manifest: return pkg_dir diff --git a/platformio/managers/package.py b/platformio/managers/package.py index 76cac33c..94e59e87 100644 --- a/platformio/managers/package.py +++ b/platformio/managers/package.py @@ -17,7 +17,7 @@ import hashlib import json import os import shutil -from os.path import basename, dirname, getsize, isdir, isfile, islink, join +from os.path import basename, getsize, isdir, isfile, islink, join from tempfile import mkdtemp import click @@ -127,16 +127,36 @@ class PkgRepoMixin(object): class PkgInstallerMixin(object): - VCS_MANIFEST_NAME = ".piopkgmanager.json" + SRC_MANIFEST_NAME = ".piopkgmanager.json" FILE_CACHE_VALID = "1m" # 1 month FILE_CACHE_MAX_SIZE = 1024 * 1024 - _INSTALLED_CACHE = {} + MEMORY_CACHE = {} - def reset_cache(self): - if self.package_dir in PkgInstallerMixin._INSTALLED_CACHE: - del PkgInstallerMixin._INSTALLED_CACHE[self.package_dir] + @staticmethod + def cache_get(key, default=None): + return PkgInstallerMixin.MEMORY_CACHE.get(key, default) + + @staticmethod + def cache_set(key, value): + PkgInstallerMixin.MEMORY_CACHE[key] = value + + @staticmethod + def cache_reset(): + PkgInstallerMixin.MEMORY_CACHE = {} + + def read_dirs(self, src_dir): + cache_key = "read_dirs-%s" % src_dir + result = self.cache_get(cache_key) + if result: + return result + result = [ + join(src_dir, name) for name in sorted(os.listdir(src_dir)) + if isdir(join(src_dir, name)) + ] + self.cache_set(cache_key, result) + return result def download(self, url, dest_dir, sha1=None): cache_key_fname = app.ContentCache.key_from_args(url, "fname") @@ -171,26 +191,23 @@ class PkgInstallerMixin(object): return fu.start() @staticmethod - def generate_install_dirname(manifest): + def get_install_dirname(manifest): name = manifest['name'] if "id" in manifest: name += "_ID%d" % manifest['id'] return name - def get_vcs_manifest_path(self, pkg_dir): + def get_src_manifest_path(self, pkg_dir): for item in os.listdir(pkg_dir): if not isdir(join(pkg_dir, item)): continue - if isfile(join(pkg_dir, item, self.VCS_MANIFEST_NAME)): - return join(pkg_dir, item, self.VCS_MANIFEST_NAME) + if isfile(join(pkg_dir, item, self.SRC_MANIFEST_NAME)): + return join(pkg_dir, item, self.SRC_MANIFEST_NAME) return None def get_manifest_path(self, pkg_dir): if not isdir(pkg_dir): return None - manifest_path = self.get_vcs_manifest_path(pkg_dir) - if manifest_path: - return manifest_path for name in self.manifest_names: manifest_path = join(pkg_dir, name) if isfile(manifest_path): @@ -198,73 +215,104 @@ class PkgInstallerMixin(object): return None def manifest_exists(self, pkg_dir): - return self.get_manifest_path(pkg_dir) is not None + return self.get_manifest_path(pkg_dir) or \ + self.get_src_manifest_path(pkg_dir) - def load_manifest(self, path): # pylint: disable=too-many-branches - assert path - pkg_dir = path - if isdir(path): - path = self.get_manifest_path(path) - if not path: - return None - else: - pkg_dir = dirname(pkg_dir) + def load_manifest(self, pkg_dir): + cache_key = "load_manifest-%s" % pkg_dir + result = self.cache_get(cache_key) + if result: + return result - is_vcs_pkg = False - if isfile(path) and path.endswith(self.VCS_MANIFEST_NAME): - is_vcs_pkg = True - pkg_dir = dirname(dirname(path)) + manifest_path = self.get_manifest_path(pkg_dir) + if not manifest_path: + return None - # return from cache - if self.package_dir in PkgInstallerMixin._INSTALLED_CACHE: - for manifest in PkgInstallerMixin._INSTALLED_CACHE[ - self.package_dir]: - if not is_vcs_pkg and manifest['__pkg_dir'] == pkg_dir: - return manifest + # if non-registry packages: VCS or archive + src_manifest_path = self.get_src_manifest_path(pkg_dir) + src_manifest = None + if src_manifest_path: + src_manifest = util.load_json(src_manifest_path) manifest = {} - if path.endswith(".json"): - manifest = util.load_json(path) - elif path.endswith(".properties"): - with codecs.open(path, encoding="utf-8") as fp: + if manifest_path.endswith(".json"): + manifest = util.load_json(manifest_path) + elif manifest_path.endswith(".properties"): + with codecs.open(manifest_path, encoding="utf-8") as fp: for line in fp.readlines(): if "=" not in line: continue key, value = line.split("=", 1) manifest[key.strip()] = value.strip() - else: + + if src_manifest: if "name" not in manifest: - manifest['name'] = basename(pkg_dir) - if "version" not in manifest: - manifest['version'] = "0.0.0" + manifest['name'] = src_manifest['name'] + if "version" in src_manifest: + manifest['version'] = src_manifest['version'] + manifest['__src_url'] = src_manifest['url'] + + if "name" not in manifest: + manifest['name'] = basename(pkg_dir) + if "version" not in manifest: + manifest['version'] = "0.0.0" manifest['__pkg_dir'] = pkg_dir + self.cache_set(cache_key, manifest) return manifest def get_installed(self): - if self.package_dir in PkgInstallerMixin._INSTALLED_CACHE: - return PkgInstallerMixin._INSTALLED_CACHE[self.package_dir] items = [] - for p in sorted(os.listdir(self.package_dir)): - pkg_dir = join(self.package_dir, p) - if not isdir(pkg_dir): - continue + for pkg_dir in self.read_dirs(self.package_dir): manifest = self.load_manifest(pkg_dir) if not manifest: continue assert "name" in manifest items.append(manifest) - PkgInstallerMixin._INSTALLED_CACHE[self.package_dir] = items return items - def check_pkg_structure(self, pkg_dir): - if self.manifest_exists(pkg_dir): - return pkg_dir + def get_package(self, name, requirements=None, url=None): + pkg_id = int(name[3:]) if name.startswith("id=") else 0 + best = None + for manifest in self.get_installed(): + if url: + if manifest.get("__src_url") != url: + continue + elif pkg_id and manifest.get("id") != pkg_id: + continue + elif not pkg_id and manifest['name'] != name: + continue - for root, _, _ in os.walk(pkg_dir): + # strict version or VCS HASH + if requirements and requirements == manifest['version']: + return manifest + + try: + if requirements and not semantic_version.Spec( + requirements).match( + semantic_version.Version( + manifest['version'], partial=True)): + continue + elif not best or (semantic_version.Version( + manifest['version'], partial=True) > + semantic_version.Version( + best['version'], partial=True)): + best = manifest + except ValueError: + pass + + return best + + def get_package_dir(self, name, requirements=None, url=None): + manifest = self.get_package(name, requirements, url) + return manifest.get("__pkg_dir") if manifest else None + + def find_pkg_root(self, src_dir): + if self.manifest_exists(src_dir): + return src_dir + for root, _, _ in os.walk(src_dir): if self.manifest_exists(root): return root - raise exception.MissingPackageManifest(", ".join(self.manifest_names)) def _install_from_piorepo(self, name, requirements): @@ -291,18 +339,25 @@ class PkgInstallerMixin(object): util.get_systype()) return pkg_dir - def _install_from_url(self, name, url, requirements=None, sha1=None): + def _install_from_url(self, + name, + url, + requirements=None, + sha1=None, + track=False): pkg_dir = None tmp_dir = mkdtemp("-package", "installing-", self.package_dir) + src_manifest_dir = None + src_manifest = {"name": name, "url": url, "requirements": requirements} try: if url.startswith("file://"): - url = url[7:] - if isfile(url): - self.unpack(url, tmp_dir) + _url = url[7:] + if isfile(_url): + self.unpack(_url, tmp_dir) else: util.rmtree_(tmp_dir) - shutil.copytree(url, tmp_dir) + shutil.copytree(_url, tmp_dir) elif url.startswith(("http://", "https://")): dlpath = self.download(url, tmp_dir, sha1) assert isfile(dlpath) @@ -311,29 +366,52 @@ class PkgInstallerMixin(object): else: vcs = VCSClientFactory.newClient(tmp_dir, url) assert vcs.export() - with open(join(vcs.storage_dir, self.VCS_MANIFEST_NAME), - "w") as fp: - json.dump({ - "name": name, - "version": vcs.get_current_revision(), - "url": url, - "requirements": requirements - }, fp) + src_manifest_dir = vcs.storage_dir + src_manifest['version'] = vcs.get_current_revision() + + pkg_dir = self.find_pkg_root(tmp_dir) + + # write source data to a special manifest + if track: + if not src_manifest_dir: + src_manifest_dir = join(pkg_dir, ".pio") + self._update_src_manifest(src_manifest, src_manifest_dir) - pkg_dir = self.check_pkg_structure(tmp_dir) pkg_dir = self._install_from_tmp_dir(pkg_dir, requirements) finally: if isdir(tmp_dir): util.rmtree_(tmp_dir) return pkg_dir - def _install_from_tmp_dir(self, tmp_dir, requirements=None): - tmp_manifest_path = self.get_manifest_path(tmp_dir) - tmp_manifest = self.load_manifest(tmp_manifest_path) + def _update_src_manifest(self, data, src_dir): + if not isdir(src_dir): + os.makedirs(src_dir) + src_manifest_path = join(src_dir, self.SRC_MANIFEST_NAME) + _data = data + if isfile(src_manifest_path): + _data.update(util.load_json(src_manifest_path)) + with open(src_manifest_path, "w") as fp: + json.dump(_data, fp) + + def _install_from_tmp_dir( # pylint: disable=too-many-branches + self, tmp_dir, requirements=None): + tmp_manifest = self.load_manifest(tmp_dir) assert set(["name", "version"]) <= set(tmp_manifest.keys()) - name = self.generate_install_dirname(tmp_manifest) - pkg_dir = join(self.package_dir, name) + pkg_dirname = self.get_install_dirname(tmp_manifest) + pkg_dir = join(self.package_dir, pkg_dirname) + cur_manifest = self.load_manifest(pkg_dir) + + tmp_semver = None + cur_semver = None + try: + tmp_semver = semantic_version.Version( + tmp_manifest['version'], partial=True) + if cur_manifest: + cur_semver = semantic_version.Version( + cur_manifest['version'], partial=True) + except ValueError: + pass # package should satisfy requirements if requirements: @@ -341,45 +419,51 @@ class PkgInstallerMixin(object): "Package version %s doesn't satisfy requirements %s" % ( tmp_manifest['version'], requirements)) try: - reqspec = semantic_version.Spec(requirements) - tmp_version = semantic_version.Version( - tmp_manifest['version'], partial=True) - assert tmp_version in reqspec, mismatch_error - except ValueError: + assert tmp_semver and tmp_semver in semantic_version.Spec( + requirements), mismatch_error + except (AssertionError, ValueError): assert tmp_manifest['version'] == requirements, mismatch_error - if self.manifest_exists(pkg_dir): - cur_manifest_path = self.get_manifest_path(pkg_dir) - cur_manifest = self.load_manifest(cur_manifest_path) - - if tmp_manifest_path.endswith(self.VCS_MANIFEST_NAME): - if cur_manifest.get("url") != tmp_manifest['url']: - pkg_dir = join(self.package_dir, "%s@vcs-%s" % ( - name, hashlib.md5(tmp_manifest['url']).hexdigest())) + # check if package already exists + if cur_manifest: + # 0-overwrite, 1-rename, 2-fix to a version + action = 0 + if "__src_url" in cur_manifest: + if cur_manifest['__src_url'] != tmp_manifest.get("__src_url"): + action = 1 + elif "__src_url" in tmp_manifest: + action = 2 else: - try: - tmp_version = semantic_version.Version( - tmp_manifest['version'], partial=True) - cur_version = semantic_version.Version( - cur_manifest['version'], partial=True) + if tmp_semver and (not cur_semver or tmp_semver > cur_semver): + action = 1 + elif tmp_semver and cur_semver and tmp_semver != cur_semver: + action = 2 - # if current package version < new package, backup it - if tmp_version > cur_version: - os.rename(pkg_dir, - join(self.package_dir, "%s@%s" % - (name, cur_manifest['version']))) - elif tmp_version < cur_version: - pkg_dir = join(self.package_dir, "%s@%s" % - (name, tmp_manifest['version'])) - except ValueError: - pkg_dir = join(self.package_dir, - "%s@%s" % (name, tmp_manifest['version'])) + # rename + if action == 1: + target_dirname = "%s@%s" % (pkg_dirname, + cur_manifest['version']) + if "__src_url" in cur_manifest: + target_dirname = "%s@src-%s" % ( + pkg_dirname, + hashlib.md5(cur_manifest['__src_url']).hexdigest()) + os.rename(pkg_dir, join(self.package_dir, target_dirname)) + # fix to a version + elif action == 2: + target_dirname = "%s@%s" % (pkg_dirname, + tmp_manifest['version']) + if "__src_url" in tmp_manifest: + target_dirname = "%s@src-%s" % ( + pkg_dirname, + hashlib.md5(tmp_manifest['__src_url']).hexdigest()) + pkg_dir = join(self.package_dir, target_dirname) # remove previous/not-satisfied package if isdir(pkg_dir): util.rmtree_(pkg_dir) os.rename(tmp_dir, pkg_dir) assert isdir(pkg_dir) + self.cache_reset() return pkg_dir @@ -400,14 +484,20 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin): click.echo("%s: %s" % (self.__class__.__name__, message), nl=nl) @staticmethod - def parse_pkg_name( # pylint: disable=too-many-branches + def parse_pkg_input( # pylint: disable=too-many-branches text, requirements=None): text = str(text) - url_marker = "://" - if not any([ - requirements, "@" not in text, text.startswith("git@"), - url_marker in text - ]): + # git@github.com:user/package.git + url_marker = text[:4] + if url_marker not in ("git@", "git+") or ":" not in text: + url_marker = "://" + + req_conditions = [ + not requirements, "@" in text, + (url_marker != "git@" and "://git@" not in text) or + text.count("@") > 1 + ] + if all(req_conditions): text, requirements = text.rsplit("@", 1) if text.isdigit(): text = "id=" + text @@ -423,22 +513,18 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin): url.startswith("http") and (url.split("#", 1)[0] if "#" in url else url).endswith(".git") ] - if any(git_conditions): url = "git+" + url + # Handle Developer Mbed URL # (https://developer.mbed.org/users/user/code/package/) - elif url.startswith("https://developer.mbed.org"): + if url.startswith("https://developer.mbed.org"): url = "hg+" + url - # git@github.com:user/package.git - if url.startswith("git@"): - url_marker = "git@" - if any([s in url for s in ("\\", "/")]) and url_marker not in url: if isfile(url) or isdir(url): url = "file://" + url - elif url.count("/") == 1 and not url.startswith("git@"): + elif url.count("/") == 1 and "git" not in url_marker: url = "git+https://github.com/" + url # determine name @@ -448,55 +534,13 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin): _url = _url[:-1] name = basename(_url) if "." in name and not name.startswith("."): - name = name.split(".", 1)[0] + name = name.rsplit(".", 1)[0] if url_marker not in url: url = None return (name or text, requirements, url) - def get_package(self, name, requirements=None, url=None): - pkg_id = int(name[3:]) if name.startswith("id=") else 0 - best = None - reqspec = None - if requirements: - try: - reqspec = semantic_version.Spec(requirements) - except ValueError: - pass - - for manifest in self.get_installed(): - if pkg_id and manifest.get("id") != pkg_id: - continue - elif not pkg_id and manifest['name'] != name: - continue - elif not reqspec and (requirements or url): - conds = [ - requirements == manifest['version'], url and - url in manifest.get("url", "") - ] - if not best or any(conds): - best = manifest - continue - try: - if reqspec and not reqspec.match( - semantic_version.Version( - manifest['version'], partial=True)): - continue - elif not best or (semantic_version.Version( - manifest['version'], partial=True) > - semantic_version.Version( - best['version'], partial=True)): - best = manifest - except ValueError: - pass - - return best - - def get_package_dir(self, name, requirements=None, url=None): - package = self.get_package(name, requirements, url) - return package.get("__pkg_dir") if package else None - - def outdated(self, name, requirements=None, url=None): + def outdated(self, pkg_dir, requirements=None): """ Has 3 different results: `None` - unknown package, VCS is fixed to commit @@ -504,27 +548,26 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin): `String` - a found latest version """ latest = None - package_dir = self.get_package_dir(name, requirements, url) - if not package_dir or ("@" in package_dir and - "@vcs-" not in package_dir): + manifest = self.load_manifest(pkg_dir) + # skip a fixed package to a specific version + if "@" in pkg_dir and "__src_url" not in manifest: return None - is_vcs_pkg = False - manifest_path = self.get_vcs_manifest_path(package_dir) - if manifest_path: - is_vcs_pkg = True - manifest = self.load_manifest(manifest_path) - else: - manifest = self.load_manifest(package_dir) - if is_vcs_pkg: - vcs = VCSClientFactory.newClient( - package_dir, manifest['url'], silent=True) + if "__src_url" in manifest: + try: + vcs = VCSClientFactory.newClient( + pkg_dir, manifest['__src_url'], silent=True) + except (AttributeError, exception.PlatformioException): + return None if not vcs.can_be_updated: return None latest = vcs.get_latest_revision() else: try: latest = self.get_latest_repo_version( - name, requirements, silent=True) + "id=%d" % manifest['id'] + if "id" in manifest else manifest['name'], + requirements, + silent=True) except (exception.PlatformioException, ValueError): return None if not latest: @@ -543,7 +586,7 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin): silent=False, trigger_event=True, interactive=False): # pylint: disable=unused-argument - name, requirements, url = self.parse_pkg_name(name, requirements) + name, requirements, url = self.parse_pkg_input(name, requirements) package_dir = self.get_package_dir(name, requirements, url) if not package_dir or not silent: @@ -560,15 +603,16 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin): return package_dir if url: - pkg_dir = self._install_from_url(name, url, requirements) + pkg_dir = self._install_from_url( + name, url, requirements, track=True) else: pkg_dir = self._install_from_piorepo(name, requirements) if not pkg_dir or not self.manifest_exists(pkg_dir): raise exception.PackageInstallError(name, requirements or "*", util.get_systype()) - self.reset_cache() manifest = self.load_manifest(pkg_dir) + assert manifest if trigger_event: telemetry.on_event( @@ -576,42 +620,45 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin): action="Install", label=manifest['name']) - click.secho( - "{name} @ {version} has been successfully installed!".format( - **manifest), - fg="green") + if not silent: + click.secho( + "{name} @ {version} has been successfully installed!".format( + **manifest), + fg="green") return pkg_dir - def uninstall(self, name, requirements=None, trigger_event=True): - name, requirements, url = self.parse_pkg_name(name, requirements) - package_dir = self.get_package_dir(name, requirements, url) - if not package_dir: - click.secho( - "%s @ %s is not installed" % (name, requirements or "*"), - fg="yellow") - return + def uninstall(self, package, requirements=None, trigger_event=True): + if isdir(package): + pkg_dir = package + else: + name, requirements, url = self.parse_pkg_input(package, + requirements) + pkg_dir = self.get_package_dir(name, requirements, url) - manifest = self.load_manifest(package_dir) + if not pkg_dir: + raise exception.UnknownPackage("%s @ %s" % + (package, requirements or "*")) + + manifest = self.load_manifest(pkg_dir) click.echo( "Uninstalling %s @ %s: \t" % (click.style( manifest['name'], fg="cyan"), manifest['version']), nl=False) - if isdir(package_dir): - if islink(package_dir): - os.unlink(package_dir) - else: - util.rmtree_(package_dir) - self.reset_cache() + if islink(pkg_dir): + os.unlink(pkg_dir) + else: + util.rmtree_(pkg_dir) + self.cache_reset() # unfix package with the same name - package_dir = self.get_package_dir(manifest['name']) - if package_dir and "@" in package_dir: - os.rename(package_dir, - join(self.package_dir, - self.generate_install_dirname(manifest))) - self.reset_cache() + pkg_dir = self.get_package_dir(manifest['name']) + if pkg_dir and "@" in pkg_dir: + os.rename( + pkg_dir, + join(self.package_dir, self.get_install_dirname(manifest))) + self.cache_reset() click.echo("[%s]" % click.style("OK", fg="green")) @@ -624,25 +671,23 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin): def update( # pylint: disable=too-many-return-statements self, - name, + package, requirements=None, only_check=False): - name, requirements, url = self.parse_pkg_name(name, requirements) - package_dir = self.get_package_dir(name, requirements, url) - if not package_dir: - click.secho( - "%s @ %s is not installed" % (name, requirements or "*"), - fg="yellow") - return - - is_vcs_pkg = False - if self.get_vcs_manifest_path(package_dir): - is_vcs_pkg = True - manifest_path = self.get_vcs_manifest_path(package_dir) + if isdir(package): + pkg_dir = package else: - manifest_path = self.get_manifest_path(package_dir) + name, requirements, url = self.parse_pkg_input(package, + requirements) + pkg_dir = self.get_package_dir(name, requirements, url) + + if not pkg_dir: + raise exception.UnknownPackage("%s @ %s" % + (package, requirements or "*")) + + manifest = self.load_manifest(pkg_dir) + name = manifest['name'] - manifest = self.load_manifest(manifest_path) click.echo( "{} {:<40} @ {:<15}".format( "Checking" if only_check else "Updating", @@ -651,7 +696,8 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin): if not util.internet_on(): click.echo("[%s]" % (click.style("Off-line", fg="yellow"))) return - latest = self.outdated(name, requirements, url) + + latest = self.outdated(pkg_dir, requirements) if latest: click.echo("[%s]" % (click.style(latest, fg="red"))) elif latest is False: @@ -659,26 +705,18 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin): else: click.echo("[%s]" % (click.style("Skip", fg="yellow"))) - if only_check or latest is False or (not is_vcs_pkg and not latest): + if only_check or not latest: return - if is_vcs_pkg: - vcs = VCSClientFactory.newClient(package_dir, manifest['url']) - if not vcs.can_be_updated: - click.secho( - "Skip update because repository is fixed " - "to %s revision" % manifest['version'], - fg="yellow") - return + if "__src_url" in manifest: + vcs = VCSClientFactory.newClient(pkg_dir, manifest['__vcs_url']) assert vcs.update() - with open(manifest_path, "w") as fp: - manifest['version'] = vcs.get_current_revision() - json.dump(manifest, fp) + self._update_src_manifest( + dict(version=vcs.get_current_revision()), vcs.storage_dir) else: - self.uninstall(name, manifest['version'], trigger_event=False) + self.uninstall(pkg_dir, trigger_event=False) self.install(name, latest, trigger_event=False) - self.reset_cache() telemetry.on_event( category=self.__class__.__name__, action="Update", diff --git a/platformio/managers/platform.py b/platformio/managers/platform.py index 78516b73..dddafc45 100644 --- a/platformio/managers/platform.py +++ b/platformio/managers/platform.py @@ -63,7 +63,7 @@ class PlatformManager(BasePkgManager): trigger_event=True, **_): # pylint: disable=too-many-arguments platform_dir = BasePkgManager.install(self, name, requirements) - p = PlatformFactory.newPlatform(self.get_manifest_path(platform_dir)) + p = PlatformFactory.newPlatform(platform_dir) # @Hook: when 'update' operation (trigger_event is False), # don't cleanup packages or install them @@ -75,10 +75,16 @@ class PlatformManager(BasePkgManager): self.cleanup_packages(p.packages.keys()) return True - def uninstall(self, name, requirements=None, trigger_event=True): - name, requirements, _ = self.parse_pkg_name(name, requirements) - p = PlatformFactory.newPlatform(name, requirements) - BasePkgManager.uninstall(self, name, requirements) + def uninstall(self, package, requirements=None, trigger_event=True): + if isdir(package): + pkg_dir = package + else: + name, requirements, url = self.parse_pkg_input(package, + requirements) + pkg_dir = self.get_package_dir(name, requirements, url) + + p = PlatformFactory.newPlatform(pkg_dir) + BasePkgManager.uninstall(self, pkg_dir, requirements) # @Hook: when 'update' operation (trigger_event is False), # don't cleanup packages or install them @@ -90,18 +96,23 @@ class PlatformManager(BasePkgManager): def update( # pylint: disable=arguments-differ self, - name, + package, requirements=None, - only_packages=False, - only_check=False): - name, requirements, _ = self.parse_pkg_name(name, requirements) + only_check=False, + only_packages=False): + if isdir(package): + pkg_dir = package + else: + name, requirements, url = self.parse_pkg_input(package, + requirements) + pkg_dir = self.get_package_dir(name, requirements, url) - p = PlatformFactory.newPlatform(name, requirements) + p = PlatformFactory.newPlatform(pkg_dir) pkgs_before = pkgs_after = p.get_installed_packages().keys() if not only_packages: - BasePkgManager.update(self, name, requirements, only_check) - p = PlatformFactory.newPlatform(name, requirements) + BasePkgManager.update(self, pkg_dir, requirements, only_check) + p = PlatformFactory.newPlatform(pkg_dir) pkgs_after = p.get_installed_packages().keys() p.update_packages(only_check) @@ -115,11 +126,10 @@ class PlatformManager(BasePkgManager): return True def cleanup_packages(self, names): - self.reset_cache() + self.cache_reset() deppkgs = {} for manifest in PlatformManager().get_installed(): - p = PlatformFactory.newPlatform(manifest['name'], - manifest['version']) + p = PlatformFactory.newPlatform(manifest['__pkg_dir']) for pkgname, pkgmanifest in p.get_installed_packages().items(): if pkgname not in deppkgs: deppkgs[pkgname] = set() @@ -131,17 +141,15 @@ class PlatformManager(BasePkgManager): continue if (manifest['name'] not in deppkgs or manifest['version'] not in deppkgs[manifest['name']]): - pm.uninstall( - manifest['name'], manifest['version'], trigger_event=False) + pm.uninstall(manifest['__pkg_dir'], trigger_event=False) - self.reset_cache() + self.cache_reset() return True def get_installed_boards(self): boards = [] for manifest in self.get_installed(): - p = PlatformFactory.newPlatform( - self.get_manifest_path(manifest['__pkg_dir'])) + p = PlatformFactory.newPlatform(manifest['__pkg_dir']) for config in p.get_boards().values(): board = config.get_brief_data() if board not in boards: @@ -183,7 +191,10 @@ class PlatformFactory(object): @classmethod def newPlatform(cls, name, requirements=None): platform_dir = None - if name.endswith("platform.json") and isfile(name): + if isdir(name): + platform_dir = name + name = PlatformManager().load_manifest(platform_dir)['name'] + elif name.endswith("platform.json") and isfile(name): platform_dir = dirname(name) name = util.load_json(name)['name'] else: @@ -249,8 +260,8 @@ class PlatformPackagesMixin(object): if self.is_valid_requirements(version): package = self.pm.get_package(name, version) else: - package = self.pm.get_package(*self._parse_pkg_name(name, - version)) + package = self.pm.get_package(*self._parse_pkg_input(name, + version)) if package: items[name] = package return items @@ -267,46 +278,47 @@ class PlatformPackagesMixin(object): self.pm.update("%s=%s" % (name, version), requirements, only_check) - def are_outdated_packages(self): - latest = None - for name in self.get_installed_packages(): - version = self.packages[name].get("version", "") - if self.is_valid_requirements(version): - latest = self.pm.outdated(name, version) - else: - requirements = None - if "@" in version: - version, requirements = version.rsplit("@", 1) - latest = self.pm.outdated(name, requirements, version) - if latest or latest is None: - return True - return False + # def are_outdated_packages(self): + # latest = None + # for name in self.get_installed_packages(): + # version = self.packages[name].get("version", "") + # if self.is_valid_requirements(version): + # latest = self.pm.outdated(name, version) + # else: + # requirements = None + # if "@" in version: + # version, requirements = version.rsplit("@", 1) + # latest = self.pm.outdated(name, requirements, version) + # if latest or latest is None: + # return True + # return False def get_package_dir(self, name): version = self.packages[name].get("version", "") if self.is_valid_requirements(version): return self.pm.get_package_dir(name, version) else: - return self.pm.get_package_dir(*self._parse_pkg_name(name, - version)) + return self.pm.get_package_dir(*self._parse_pkg_input(name, + version)) def get_package_version(self, name): version = self.packages[name].get("version", "") if self.is_valid_requirements(version): package = self.pm.get_package(name, version) else: - package = self.pm.get_package(*self._parse_pkg_name(name, version)) + package = self.pm.get_package(*self._parse_pkg_input(name, + version)) return package['version'] if package else None @staticmethod def is_valid_requirements(requirements): return requirements and "://" not in requirements - def _parse_pkg_name(self, name, version): + def _parse_pkg_input(self, name, version): requirements = None if "@" in version: version, requirements = version.rsplit("@", 1) - return self.pm.parse_pkg_name("%s=%s" % (name, version), requirements) + return self.pm.parse_pkg_input("%s=%s" % (name, version), requirements) class PlatformRunMixin(object): diff --git a/platformio/util.py b/platformio/util.py index 1e70b471..20d6214a 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -141,7 +141,12 @@ class memoized(object): def __get__(self, obj, objtype): '''Support instance methods.''' - return functools.partial(self.__call__, obj) + fn = functools.partial(self.__call__, obj) + fn.reset = self._reset + return fn + + def _reset(self): + self.cache = {} def singleton(cls): diff --git a/tests/commands/test_lib.py b/tests/commands/test_lib.py index 3b3a213a..d0ae5665 100644 --- a/tests/commands/test_lib.py +++ b/tests/commands/test_lib.py @@ -16,7 +16,7 @@ import json import re from os.path import basename -from platformio import util +from platformio import exception, util from platformio.commands.init import cli as cmd_init from platformio.commands.lib import cli as cmd_lib @@ -37,15 +37,35 @@ def test_search(clirunner, validate_cliresult): def test_global_install_registry(clirunner, validate_cliresult, isolated_pio_home): result = clirunner.invoke(cmd_lib, [ - "-g", "install", "58", "OneWire", + "-g", "install", "58", "547@2.2.4", "DallasTemperature", "http://dl.platformio.org/libraries/archives/3/5174.tar.gz", - "ArduinoJson@5.6.7", "ArduinoJson@~5.7.0" + "ArduinoJson@5.6.7", "ArduinoJson@~5.7.0", "1089@fee16e880b" ]) validate_cliresult(result) + + # check lib with duplicate URL + result = clirunner.invoke(cmd_lib, [ + "-g", "install", + "http://dl.platformio.org/libraries/archives/3/5174.tar.gz" + ]) + validate_cliresult(result) + assert "is already installed" in result.output + + # check lib with duplicate ID + result = clirunner.invoke(cmd_lib, ["-g", "install", "305"]) + validate_cliresult(result) + assert "is already installed" in result.output + + # install unknown library + result = clirunner.invoke(cmd_lib, ["-g", "install", "Unknown"]) + assert result.exit_code != 0 + assert isinstance(result.exception, exception.LibNotFound) + items1 = [d.basename for d in isolated_pio_home.join("lib").listdir()] items2 = [ - "DHT22_ID58", "ArduinoJson_ID64", "ArduinoJson_ID64@5.6.7", - "OneWire_ID1", "ESPAsyncTCP_ID305" + "ArduinoJson_ID64", "ArduinoJson_ID64@5.6.7", "DallasTemperature_ID54", + "DHT22_ID58", "ESPAsyncTCP_ID305", "NeoPixelBus_ID547", "OneWire_ID1", + "IRremoteESP8266_ID1089" ] assert set(items1) == set(items2) @@ -55,11 +75,29 @@ def test_global_install_archive(clirunner, validate_cliresult, result = clirunner.invoke(cmd_lib, [ "-g", "install", "https://github.com/adafruit/Adafruit-ST7735-Library/" "archive/master.zip", + "http://www.airspayce.com/mikem/arduino/RadioHead/RadioHead-1.62.zip", + "https://github.com/bblanchon/ArduinoJson/archive/v5.8.2.zip", + "https://github.com/bblanchon/ArduinoJson/archive/v5.8.2.zip@5.8.2" + ]) + validate_cliresult(result) + + # incorrect requirements + result = clirunner.invoke(cmd_lib, [ + "-g", "install", + "https://github.com/bblanchon/ArduinoJson/archive/v5.8.2.zip@1.2.3" + ]) + assert result.exit_code != 0 + + # check lib with duplicate URL + result = clirunner.invoke(cmd_lib, [ + "-g", "install", "http://www.airspayce.com/mikem/arduino/RadioHead/RadioHead-1.62.zip" ]) validate_cliresult(result) + assert "is already installed" in result.output + items1 = [d.basename for d in isolated_pio_home.join("lib").listdir()] - items2 = ["Adafruit ST7735 Library", "RadioHead"] + items2 = ["Adafruit ST7735 Library", "RadioHead-1.62"] assert set(items1) >= set(items2) @@ -71,14 +109,20 @@ def test_global_install_repository(clirunner, validate_cliresult, "-g", "install", "https://github.com/gioblu/PJON.git#3.0", + "https://github.com/gioblu/PJON.git#6.2", + "https://github.com/bblanchon/ArduinoJson.git", "https://gitlab.com/ivankravets/rs485-nodeproto.git", # "https://developer.mbed.org/users/simon/code/TextLCD/", "knolleary/pubsubclient" ]) validate_cliresult(result) items1 = [d.basename for d in isolated_pio_home.join("lib").listdir()] - items2 = ["PJON", "ESPAsyncTCP", "PubSubClient"] - assert set(items2) & set(items1) + items2 = [ + "PJON", "PJON@src-79de467ebe19de18287becff0a1fb42d", + "ArduinoJson@src-69ebddd821f771debe7ee734d3c7fa81", "rs485-nodeproto", + "PubSubClient" + ] + assert set(items1) >= set(items2) def test_global_lib_list(clirunner, validate_cliresult, isolated_pio_home): @@ -89,13 +133,15 @@ def test_global_lib_list(clirunner, validate_cliresult, isolated_pio_home): result = clirunner.invoke(cmd_lib, ["-g", "list", "--json-output"]) assert all([ n in result.output - for n in ("PJON", "git+https://github.com/knolleary/pubsubclient") + for n in ("PJON", "git+https://github.com/knolleary/pubsubclient", + "https://github.com/bblanchon/ArduinoJson/archive/v5.8.2.zip" + ) ]) items1 = [i['name'] for i in json.loads(result.output)] items2 = [ "OneWire", "DHT22", "PJON", "ESPAsyncTCP", "ArduinoJson", - "pubsubclient", "rs485-nodeproto", "Adafruit ST7735 Library", - "RadioHead" + "PubSubClient", "rs485-nodeproto", "Adafruit ST7735 Library", + "RadioHead-1.62", "DallasTemperature", "NeoPixelBus", "IRremoteESP8266" ] assert set(items1) == set(items2) @@ -106,31 +152,71 @@ def test_global_lib_update_check(clirunner, validate_cliresult, cmd_lib, ["-g", "update", "--only-check", "--json-output"]) validate_cliresult(result) output = json.loads(result.output) - assert set(["ArduinoJson", "ESPAsyncTCP", "RadioHead"]) == set( + assert set(["ArduinoJson", "IRremoteESP8266", "NeoPixelBus"]) == set( [l['name'] for l in output]) def test_global_lib_update(clirunner, validate_cliresult, isolated_pio_home): + # update library using package directory + result = clirunner.invoke( + cmd_lib, + ["-g", "update", "NeoPixelBus", "--only-check", "--json-output"]) + validate_cliresult(result) + oudated = json.loads(result.output) + assert len(oudated) == 1 + assert "__pkg_dir" in oudated[0] + result = clirunner.invoke(cmd_lib, + ["-g", "update", oudated[0]['__pkg_dir']]) + validate_cliresult(result) + assert "Uninstalling NeoPixelBus @ 2.2.4" in result.output + + # update rest libraries result = clirunner.invoke(cmd_lib, ["-g", "update"]) validate_cliresult(result) - assert "[Up-to-date]" in result.output + validate_cliresult(result) + assert result.output.count("[Skip]") == 5 + assert result.output.count("[Up-to-date]") == 9 assert "Uninstalling ArduinoJson @ 5.7.3" in result.output + assert "Uninstalling IRremoteESP8266 @ fee16e880b" in result.output + + # update unknown library + result = clirunner.invoke(cmd_lib, ["-g", "update", "Unknown"]) + assert result.exit_code != 0 + assert isinstance(result.exception, exception.UnknownPackage) def test_global_lib_uninstall(clirunner, validate_cliresult, isolated_pio_home): + # uninstall using package directory + result = clirunner.invoke(cmd_lib, ["-g", "list", "--json-output"]) + validate_cliresult(result) + items = json.loads(result.output) + result = clirunner.invoke(cmd_lib, + ["-g", "uninstall", items[0]['__pkg_dir']]) + validate_cliresult(result) + assert "Uninstalling Adafruit ST7735 Library" in result.output + + # uninstall the rest libraries result = clirunner.invoke(cmd_lib, [ - "-g", "uninstall", "1", "ArduinoJson@!=5.6.7", "TextLCD", - "Adafruit ST7735 Library" + "-g", "uninstall", "1", "ArduinoJson@!=5.6.7", + "https://github.com/bblanchon/ArduinoJson.git", "IRremoteESP8266@>=0.2" ]) validate_cliresult(result) + items1 = [d.basename for d in isolated_pio_home.join("lib").listdir()] items2 = [ - "DHT22_ID58", "ArduinoJson_ID64", "ESPAsyncTCP_ID305", - "pubsubclient", "PJON", "rs485-nodeproto", "RadioHead_ID124" + "ArduinoJson", "ArduinoJson_ID64@5.6.7", "DallasTemperature_ID54", + "DHT22_ID58", "ESPAsyncTCP_ID305", "NeoPixelBus_ID547", "PJON", + "PJON@src-79de467ebe19de18287becff0a1fb42d", "PubSubClient", + "RadioHead-1.62", "rs485-nodeproto" ] assert set(items1) == set(items2) + # uninstall unknown library + result = clirunner.invoke(cmd_lib, ["-g", "uninstall", "Unknown"]) + assert result.exit_code != 0 + assert isinstance(result.exception, exception.UnknownPackage) + def test_lib_show(clirunner, validate_cliresult, isolated_pio_home): result = clirunner.invoke(cmd_lib, ["show", "64"]) @@ -142,31 +228,24 @@ def test_lib_show(clirunner, validate_cliresult, isolated_pio_home): assert "OneWire" in result.output -def test_project_lib_complex(clirunner, validate_cliresult, tmpdir): - with tmpdir.as_cwd(): - # init - result = clirunner.invoke(cmd_init) - validate_cliresult(result) +def test_lib_builtin(clirunner, validate_cliresult, isolated_pio_home): + result = clirunner.invoke(cmd_lib, ["builtin"]) + validate_cliresult(result) + result = clirunner.invoke(cmd_lib, ["builtin", "--json-output"]) + validate_cliresult(result) - # isntall - result = clirunner.invoke(cmd_lib, ["install", "54", "ArduinoJson"]) - validate_cliresult(result) - items1 = [ - d.basename - for d in tmpdir.join(basename(util.get_projectlibdeps_dir())) - .listdir() - ] - items2 = ["DallasTemperature_ID54", "OneWire_ID1", "ArduinoJson_ID64"] - assert set(items1) == set(items2) - # list - result = clirunner.invoke(cmd_lib, ["list", "--json-output"]) - validate_cliresult(result) - items1 = [i['name'] for i in json.loads(result.output)] - items2 = ["DallasTemperature", "OneWire", "ArduinoJson"] - assert set(items1) == set(items2) +def test_lib_stats(clirunner, validate_cliresult, isolated_pio_home): + result = clirunner.invoke(cmd_lib, ["stats"]) + validate_cliresult(result) + assert all([ + s in result.output + for s in ("UPDATED", "ago", "http://platformio.org/lib/show") + ]) - # update - result = clirunner.invoke(cmd_lib, ["update"]) - validate_cliresult(result) - assert "[Up-to-date]" in result.output + result = clirunner.invoke(cmd_lib, ["stats", "--json-output"]) + validate_cliresult(result) + assert set([ + "dlweek", "added", "updated", "topkeywords", "dlmonth", "dlday", + "lastkeywords" + ]) == set(json.loads(result.output).keys()) diff --git a/tests/test_maintenance.py b/tests/test_maintenance.py index ba4e2796..fd71acf1 100644 --- a/tests/test_maintenance.py +++ b/tests/test_maintenance.py @@ -156,7 +156,7 @@ def test_check_platform_updates(clirunner, validate_cliresult, manifest['version'] = "0.0.0" manifest_path.write(json.dumps(manifest)) # reset cached manifests - PlatformManager().reset_cache() + PlatformManager().cache_reset() # reset check time interval = int(app.get_setting("check_platforms_interval")) * 3600 * 24 diff --git a/tests/test_managers.py b/tests/test_managers.py index 14ee9e38..a9320def 100644 --- a/tests/test_managers.py +++ b/tests/test_managers.py @@ -12,70 +12,118 @@ # See the License for the specific language governing permissions and # limitations under the License. +import json +from os.path import join + from platformio import util -from platformio.managers.package import BasePkgManager +from platformio.managers.package import PackageManager -def test_pkg_name_parser(): +def test_pkg_input_parser(): items = [ ["PkgName", ("PkgName", None, None)], [("PkgName", "!=1.2.3,<2.0"), ("PkgName", "!=1.2.3,<2.0", None)], ["PkgName@1.2.3", ("PkgName", "1.2.3", None)], [("PkgName@1.2.3", "1.2.5"), ("PkgName@1.2.3", "1.2.5", None)], ["id:13", ("id:13", None, None)], - ["id:13@~1.2.3", ("id:13", "~1.2.3", None)], [ + ["id:13@~1.2.3", ("id:13", "~1.2.3", None)], + [ util.get_home_dir(), (".platformio", None, "file://" + util.get_home_dir()) - ], [ + ], + [ "LocalName=" + util.get_home_dir(), ("LocalName", None, "file://" + util.get_home_dir()) - ], [ + ], + [ + "LocalName=%s@>2.3.0" % util.get_home_dir(), + ("LocalName", ">2.3.0", "file://" + util.get_home_dir()) + ], + [ "https://github.com/user/package.git", ("package", None, "git+https://github.com/user/package.git") - ], [ - "https://gitlab.com/user/package.git", - ("package", None, "git+https://gitlab.com/user/package.git") - ], [ + ], + [ + "MyPackage=https://gitlab.com/user/package.git", + ("MyPackage", None, "git+https://gitlab.com/user/package.git") + ], + [ + "MyPackage=https://gitlab.com/user/package.git@3.2.1,!=2", + ("MyPackage", "3.2.1,!=2", + "git+https://gitlab.com/user/package.git") + ], + [ + "https://somedomain.com/path/LibraryName-1.2.3.zip", + ("LibraryName-1.2.3", None, + "https://somedomain.com/path/LibraryName-1.2.3.zip") + ], + [ "https://github.com/user/package/archive/branch.zip", ("branch", None, "https://github.com/user/package/archive/branch.zip") - ], [ + ], + [ + "https://github.com/user/package/archive/branch.zip@~1.2.3", + ("branch", "~1.2.3", + "https://github.com/user/package/archive/branch.zip") + ], + [ "https://github.com/user/package/archive/branch.tar.gz", - ("branch", None, + ("branch.tar", None, "https://github.com/user/package/archive/branch.tar.gz") - ], [ + ], + [ + "https://github.com/user/package/archive/branch.tar.gz@!=5", + ("branch.tar", "!=5", + "https://github.com/user/package/archive/branch.tar.gz") + ], + [ "https://developer.mbed.org/users/user/code/package/", ("package", None, "hg+https://developer.mbed.org/users/user/code/package/") - ], [ + ], + [ "https://github.com/user/package#v1.2.3", ("package", None, "git+https://github.com/user/package#v1.2.3") - ], [ + ], + [ "https://github.com/user/package.git#branch", ("package", None, "git+https://github.com/user/package.git#branch") - ], [ + ], + [ "PkgName=https://github.com/user/package.git#a13d344fg56", ("PkgName", None, "git+https://github.com/user/package.git#a13d344fg56") - ], [ + ], + [ + "user/package", + ("package", None, "git+https://github.com/user/package") + ], + [ "PkgName=user/package", ("PkgName", None, "git+https://github.com/user/package") - ], [ + ], + [ "PkgName=user/package#master", ("PkgName", None, "git+https://github.com/user/package#master") - ], [ + ], + [ "git+https://github.com/user/package", ("package", None, "git+https://github.com/user/package") - ], [ + ], + [ "hg+https://example.com/user/package", ("package", None, "hg+https://example.com/user/package") - ], [ + ], + [ "git@github.com:user/package.git", ("package", None, "git@github.com:user/package.git") - ], [ + ], + [ "git@github.com:user/package.git#v1.2.0", ("package", None, "git@github.com:user/package.git#v1.2.0") - ], [ + ], + [ "git+ssh://git@gitlab.private-server.com/user/package#1.2.0", ("package", None, "git+ssh://git@gitlab.private-server.com/user/package#1.2.0") @@ -83,6 +131,72 @@ def test_pkg_name_parser(): ] for params, result in items: if isinstance(params, tuple): - assert BasePkgManager.parse_pkg_name(*params) == result + assert PackageManager.parse_pkg_input(*params) == result else: - assert BasePkgManager.parse_pkg_name(params) == result + assert PackageManager.parse_pkg_input(params) == result + + +def test_install_packages(isolated_pio_home, tmpdir): + packages = [ + dict(id=1, name="name_1", version="shasum"), + dict(id=1, name="name_1", version="2.0.0"), + dict(id=1, name="name_1", version="2.1.0"), + dict(id=1, name="name_1", version="1.2.0"), + dict(id=1, name="name_1", version="1.0.0"), + dict(name="name_2", version="1.0.0"), + dict(name="name_2", version="2.0.0", + __src_url="git+https://github.com"), + dict(name="name_2", version="3.0.0", + __src_url="git+https://github2.com"), + dict(name="name_2", version="4.0.0", + __src_url="git+https://github2.com") + ] + + pm = PackageManager(join(util.get_home_dir(), "packages")) + for package in packages: + tmp_dir = tmpdir.mkdir("tmp-package") + tmp_dir.join("package.json").write(json.dumps(package)) + pm._install_from_url(package['name'], "file://%s" % str(tmp_dir)) + tmp_dir.remove(rec=1) + + assert len(pm.get_installed()) == len(packages) - 1 + + pkg_dirnames = [ + 'name_1_ID1', 'name_1_ID1@1.0.0', 'name_1_ID1@1.2.0', + 'name_1_ID1@2.0.0', 'name_1_ID1@shasum', 'name_2', + 'name_2@src-177cbce1f0705580d17790fda1cc2ef5', + 'name_2@src-f863b537ab00f4c7b5011fc44b120e1f' + ] + assert set([p.basename for p in isolated_pio_home.join( + "packages").listdir()]) == set(pkg_dirnames) + + +def test_get_package(isolated_pio_home): + tests = [ + [("unknown", ), None], + [("1", ), None], + [("id=1", "shasum"), dict(id=1, name="name_1", version="shasum")], + [("id=1", "*"), dict(id=1, name="name_1", version="2.1.0")], + [("id=1", "^1"), dict(id=1, name="name_1", version="1.2.0")], + [("id=1", "^1"), dict(id=1, name="name_1", version="1.2.0")], + [("name_1", "<2"), dict(id=1, name="name_1", version="1.2.0")], + [("name_1", ">2"), None], + [("name_1", "2-0-0"), dict(id=1, name="name_1", version="2.1.0")], + [("name_1", "2-0-0"), dict(id=1, name="name_1", version="2.1.0")], + [("name_2", ), dict(name="name_2", version="4.0.0")], + [("url_has_higher_priority", None, "git+https://github.com"), + dict(name="name_2", version="2.0.0", + __src_url="git+https://github.com")], + [("name_2", None, "git+https://github.com"), + dict(name="name_2", version="2.0.0", + __src_url="git+https://github.com")], + ] + + pm = PackageManager(join(util.get_home_dir(), "packages")) + for test in tests: + manifest = pm.get_package(*test[0]) + if test[1] is None: + assert manifest is None, test + continue + for key, value in test[1].items(): + assert manifest[key] == value, test diff --git a/tests/test_pkgmanifest.py b/tests/test_pkgmanifest.py index 8793f9ab..f70dccde 100644 --- a/tests/test_pkgmanifest.py +++ b/tests/test_pkgmanifest.py @@ -16,19 +16,6 @@ import pytest import requests -def pytest_generate_tests(metafunc): - if "package_data" not in metafunc.fixturenames: - return - pkgs_manifest = requests.get( - "https://dl.bintray.com/platformio/dl-packages/manifest.json").json() - assert isinstance(pkgs_manifest, dict) - packages = [] - for _, variants in pkgs_manifest.iteritems(): - for item in variants: - packages.append(item) - metafunc.parametrize("package_data", packages) - - def validate_response(req): assert req.status_code == 200 assert int(req.headers['Content-Length']) > 0 @@ -36,13 +23,22 @@ def validate_response(req): "application/octet-stream") -def test_package(package_data): - assert package_data['url'].endswith(".tar.gz") +def test_packages(): + pkgs_manifest = requests.get( + "https://dl.bintray.com/platformio/dl-packages/manifest.json").json() + assert isinstance(pkgs_manifest, dict) + items = [] + for _, variants in pkgs_manifest.iteritems(): + for item in variants: + items.append(item) - r = requests.head(package_data['url'], allow_redirects=True) - validate_response(r) + for item in items: + assert item['url'].endswith(".tar.gz"), item - if "X-Checksum-Sha1" not in r.headers: - return pytest.skip("X-Checksum-Sha1 is not provided") + r = requests.head(item['url'], allow_redirects=True) + validate_response(r) - assert package_data['sha1'] == r.headers.get("X-Checksum-Sha1") + if "X-Checksum-Sha1" not in r.headers: + return pytest.skip("X-Checksum-Sha1 is not provided") + + assert item['sha1'] == r.headers.get("X-Checksum-Sha1"), item diff --git a/tox.ini b/tox.ini index 1f19bf1f..0c09acce 100644 --- a/tox.ini +++ b/tox.ini @@ -24,6 +24,7 @@ deps = yapf pylint pytest + show commands = python --version [testenv:docs]