platformio/platformio-core
1
0
Fork 1
mirror of https://github.com/platformio/platformio-core.git synced 2025-07-30 01:57:13 +02:00
Code Issues Packages Projects Releases Wiki Activity

Switch library manager to the new package manager

Browse Source
This commit is contained in:
Ivan Kravets
2020-08-12 13:27:05 +03:00
parent 2dd69e21c0
commit 893ca1b328
30 changed files with 1318 additions and 915 deletions
Download Patch File Download Diff File Expand all files Collapse all files

58
platformio/builder/tools/piolib.py
View File

@ -34,11 +34,13 @@ from SCons.Script import DefaultEnvironment # pylint: disable=import-error
from platformio import exception, fs, util
from platformio.builder.tools import platformio as piotool
from platformio.compat import WINDOWS, hashlib_encode_data, string_types
from platformio.managers.lib import LibraryManager
from platformio.package.exception import UnknownPackageError
from platformio.package.manager.library import LibraryPackageManager
from platformio.package.manifest.parser import (
ManifestParserError,
ManifestParserFactory,
)
from platformio.package.meta import PackageSourceItem
from platformio.project.options import ProjectOptions
@ -851,34 +853,36 @@ class ProjectAsLibBuilder(LibBuilderBase):
pass
def install_dependencies(self):
def _is_builtin(uri):
def _is_builtin(spec):
for lb in self.env.GetLibBuilders():
if lb.name == uri:
if lb.name == spec:
return True
return False
not_found_uri = []
for uri in self.dependencies:
not_found_specs = []
for spec in self.dependencies:
# check if built-in library
if _is_builtin(uri):
if _is_builtin(spec):
continue
found = False
for storage_dir in self.env.GetLibSourceDirs():
lm = LibraryManager(storage_dir)
if lm.get_package_dir(*lm.parse_pkg_uri(uri)):
lm = LibraryPackageManager(storage_dir)
if lm.get_package(spec):
found = True
break
if not found:
not_found_uri.append(uri)
not_found_specs.append(spec)
did_install = False
lm = LibraryManager(self.env.subst(join("$PROJECT_LIBDEPS_DIR", "$PIOENV")))
for uri in not_found_uri:
lm = LibraryPackageManager(
self.env.subst(join("$PROJECT_LIBDEPS_DIR", "$PIOENV"))
)
for spec in not_found_specs:
try:
lm.install(uri)
lm.install(spec)
did_install = True
except (exception.LibNotFound, exception.InternetIsOffline) as e:
except (UnknownPackageError, exception.InternetIsOffline) as e:
click.secho("Warning! %s" % e, fg="yellow")
# reset cache
@ -886,17 +890,17 @@ class ProjectAsLibBuilder(LibBuilderBase):
DefaultEnvironment().Replace(__PIO_LIB_BUILDERS=None)
def process_dependencies(self): # pylint: disable=too-many-branches
for uri in self.dependencies:
for spec in self.dependencies:
found = False
for storage_dir in self.env.GetLibSourceDirs():
if found:
break
lm = LibraryManager(storage_dir)
lib_dir = lm.get_package_dir(*lm.parse_pkg_uri(uri))
if not lib_dir:
lm = LibraryPackageManager(storage_dir)
pkg = lm.get_package(spec)
if not pkg:
continue
for lb in self.env.GetLibBuilders():
if lib_dir != lb.path:
if pkg.path != lb.path:
continue
if lb not in self.depbuilders:
self.depend_recursive(lb)
@ -908,7 +912,7 @@ class ProjectAsLibBuilder(LibBuilderBase):
# look for built-in libraries by a name
# which don't have package manifest
for lb in self.env.GetLibBuilders():
if lb.name != uri:
if lb.name != spec:
continue
if lb not in self.depbuilders:
self.depend_recursive(lb)
@ -1000,10 +1004,6 @@ def GetLibBuilders(env): # pylint: disable=too-many-branches
def ConfigureProjectLibBuilder(env):
def _get_vcs_info(lb):
path = LibraryManager.get_src_manifest_path(lb.path)
return fs.load_json(path) if path else None
def _correct_found_libs(lib_builders):
# build full dependency graph
found_lbs = [lb for lb in lib_builders if lb.dependent]
@ -1019,15 +1019,13 @@ def ConfigureProjectLibBuilder(env):
margin = "| " * (level)
for lb in root.depbuilders:
title = "<%s>" % lb.name
vcs_info = _get_vcs_info(lb)
if lb.version:
title += " %s" % lb.version
if vcs_info and vcs_info.get("version"):
title += " #%s" % vcs_info.get("version")
pkg = PackageSourceItem(lb.path)
if pkg.metadata:
title += " %s" % pkg.metadata.version
click.echo("%s|-- %s" % (margin, title), nl=False)
if int(ARGUMENTS.get("PIOVERBOSE", 0)):
if vcs_info:
click.echo(" [%s]" % vcs_info.get("url"), nl=False)
if pkg.metadata and pkg.metadata.spec.external:
click.echo(" [%s]" % pkg.metadata.spec.url, nl=False)
click.echo(" (", nl=False)
click.echo(lb.path, nl=False)
click.echo(")", nl=False)

5
platformio/clients/http.py
View File

@ -30,7 +30,9 @@ class HTTPClientError(PlatformioException):
class HTTPClient(object):
def __init__(self, base_url):
def __init__(
self, base_url,
):
if base_url.endswith("/"):
base_url = base_url[:-1]
self.base_url = base_url
@ -51,6 +53,7 @@ class HTTPClient(object):
self._session.close()
self._session = None
@util.throttle(500)
def send_request(self, method, path, **kwargs):
# check Internet before and resolve issue with 60 seconds timeout
# print(self, method, path, kwargs)

13
platformio/commands/lib/__init__.py Normal file
View File

@ -0,0 +1,13 @@
# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

178
platformio/commands/lib.py → platformio/commands/lib/command.py
View File

@ -18,16 +18,21 @@ import os
import time
import click
import semantic_version
from tabulate import tabulate
from platformio import exception, fs, util
from platformio.commands import PlatformioCLI
from platformio.commands.lib.helpers import (
get_builtin_libs,
is_builtin_lib,
save_project_libdeps,
)
from platformio.compat import dump_json_to_unicode
from platformio.managers.lib import LibraryManager, get_builtin_libs, is_builtin_lib
from platformio.package.exception import UnknownPackageError
from platformio.package.manager.library import LibraryPackageManager
from platformio.package.meta import PackageSourceItem, PackageSpec
from platformio.proc import is_ci
from platformio.project.config import ProjectConfig
from platformio.project.exception import InvalidProjectConfError
from platformio.project.helpers import get_project_dir, is_platformio_project
try:
@ -124,89 +129,106 @@ def cli(ctx, **options):
@cli.command("install", short_help="Install library")
@click.argument("libraries", required=False, nargs=-1, metavar="[LIBRARY...]")
@click.option(
"--save",
"--save/--no-save",
is_flag=True,
help="Save installed libraries into the `platformio.ini` dependency list",
default=True,
help="Save installed libraries into the `platformio.ini` dependency list"
" (enabled by default)",
)
@click.option("-s", "--silent", is_flag=True, help="Suppress progress reporting")
@click.option(
"--interactive", is_flag=True, help="Allow to make a choice for all prompts"
"--interactive",
is_flag=True,
help="Deprecated! Please use a strict dependency specification (owner/libname)",
)
@click.option(
"-f", "--force", is_flag=True, help="Reinstall/redownload library if exists"
)
@click.pass_context
def lib_install( # pylint: disable=too-many-arguments
def lib_install( # pylint: disable=too-many-arguments,unused-argument
ctx, libraries, save, silent, interactive, force
):
storage_dirs = ctx.meta[CTX_META_STORAGE_DIRS_KEY]
storage_libdeps = ctx.meta.get(CTX_META_STORAGE_LIBDEPS_KEY, [])
installed_manifests = {}
installed_pkgs = {}
for storage_dir in storage_dirs:
if not silent and (libraries or storage_dir in storage_libdeps):
print_storage_header(storage_dirs, storage_dir)
lm = LibraryManager(storage_dir)
lm = LibraryPackageManager(storage_dir)
if libraries:
for library in libraries:
pkg_dir = lm.install(
library, silent=silent, interactive=interactive, force=force
)
installed_manifests[library] = lm.load_manifest(pkg_dir)
installed_pkgs = {
library: lm.install(library, silent=silent, force=force)
for library in libraries
}
elif storage_dir in storage_libdeps:
builtin_lib_storages = None
for library in storage_libdeps[storage_dir]:
try:
pkg_dir = lm.install(
library, silent=silent, interactive=interactive, force=force
)
installed_manifests[library] = lm.load_manifest(pkg_dir)
except exception.LibNotFound as e:
lm.install(library, silent=silent, force=force)
except UnknownPackageError as e:
if builtin_lib_storages is None:
builtin_lib_storages = get_builtin_libs()
if not silent or not is_builtin_lib(builtin_lib_storages, library):
click.secho("Warning! %s" % e, fg="yellow")
if not save or not libraries:
return
if save and installed_pkgs:
_save_deps(ctx, installed_pkgs)
def _save_deps(ctx, pkgs, action="add"):
specs = []
for library, pkg in pkgs.items():
spec = PackageSpec(library)
if spec.external:
specs.append(spec)
else:
specs.append(
PackageSpec(
owner=pkg.metadata.spec.owner,
name=pkg.metadata.spec.name,
requirements=spec.requirements
or (
("^%s" % pkg.metadata.version)
if not pkg.metadata.version.build
else pkg.metadata.version
),
)
)
input_dirs = ctx.meta.get(CTX_META_INPUT_DIRS_KEY, [])
project_environments = ctx.meta[CTX_META_PROJECT_ENVIRONMENTS_KEY]
for input_dir in input_dirs:
config = ProjectConfig.get_instance(os.path.join(input_dir, "platformio.ini"))
config.validate(project_environments)
for env in config.envs():
if project_environments and env not in project_environments:
continue
config.expand_interpolations = False
try:
lib_deps = config.get("env:" + env, "lib_deps")
except InvalidProjectConfError:
lib_deps = []
for library in libraries:
if library in lib_deps:
continue
manifest = installed_manifests[library]
try:
assert library.lower() == manifest["name"].lower()
assert semantic_version.Version(manifest["version"])
lib_deps.append("{name}@^{version}".format(**manifest))
except (AssertionError, ValueError):
lib_deps.append(library)
config.set("env:" + env, "lib_deps", lib_deps)
config.save()
if not is_platformio_project(input_dir):
continue
save_project_libdeps(input_dir, specs, project_environments, action=action)
@cli.command("uninstall", short_help="Uninstall libraries")
@cli.command("uninstall", short_help="Remove libraries")
@click.argument("libraries", nargs=-1, metavar="[LIBRARY...]")
@click.option(
"--save/--no-save",
is_flag=True,
default=True,
help="Remove libraries from the `platformio.ini` dependency list and save changes"
" (enabled by default)",
)
@click.option("-s", "--silent", is_flag=True, help="Suppress progress reporting")
@click.pass_context
def lib_uninstall(ctx, libraries):
def lib_uninstall(ctx, libraries, save, silent):
storage_dirs = ctx.meta[CTX_META_STORAGE_DIRS_KEY]
uninstalled_pkgs = {}
for storage_dir in storage_dirs:
print_storage_header(storage_dirs, storage_dir)
lm = LibraryManager(storage_dir)
for library in libraries:
lm.uninstall(library)
lm = LibraryPackageManager(storage_dir)
uninstalled_pkgs = {
library: lm.uninstall(library, silent=silent) for library in libraries
}
if save and uninstalled_pkgs:
_save_deps(ctx, uninstalled_pkgs, action="remove")
@cli.command("update", short_help="Update installed libraries")
@ -220,42 +242,53 @@ def lib_uninstall(ctx, libraries):
@click.option(
"--dry-run", is_flag=True, help="Do not update, only check for the new versions"
)
@click.option("-s", "--silent", is_flag=True, help="Suppress progress reporting")
@click.option("--json-output", is_flag=True)
@click.pass_context
def lib_update(ctx, libraries, only_check, dry_run, json_output):
def lib_update( # pylint: disable=too-many-arguments
ctx, libraries, only_check, dry_run, silent, json_output
):
storage_dirs = ctx.meta[CTX_META_STORAGE_DIRS_KEY]
only_check = dry_run or only_check
json_result = {}
for storage_dir in storage_dirs:
if not json_output:
print_storage_header(storage_dirs, storage_dir)
lm = LibraryManager(storage_dir)
_libraries = libraries
if not _libraries:
_libraries = [manifest["__pkg_dir"] for manifest in lm.get_installed()]
lm = LibraryPackageManager(storage_dir)
_libraries = libraries or lm.get_installed()
if only_check and json_output:
result = []
for library in _libraries:
pkg_dir = library if os.path.isdir(library) else None
requirements = None
url = None
if not pkg_dir:
name, requirements, url = lm.parse_pkg_uri(library)
pkg_dir = lm.get_package_dir(name, requirements, url)
if not pkg_dir:
spec = None
pkg = None
if isinstance(library, PackageSourceItem):
pkg = library
else:
spec = PackageSpec(library)
pkg = lm.get_package(spec)
if not pkg:
continue
latest = lm.outdated(pkg_dir, requirements)
if not latest:
outdated = lm.outdated(pkg, spec)
if not outdated.is_outdated(allow_incompatible=True):
continue
manifest = lm.load_manifest(pkg_dir)
manifest["versionLatest"] = latest
manifest = lm.legacy_load_manifest(pkg)
manifest["versionWanted"] = (
str(outdated.wanted) if outdated.wanted else None
)
manifest["versionLatest"] = (
str(outdated.latest) if outdated.latest else None
)
result.append(manifest)
json_result[storage_dir] = result
else:
for library in _libraries:
lm.update(library, only_check=only_check)
spec = (
None
if isinstance(library, PackageSourceItem)
else PackageSpec(library)
)
lm.update(library, spec=spec, only_check=only_check, silent=silent)
if json_output:
return click.echo(
@ -276,8 +309,8 @@ def lib_list(ctx, json_output):
for storage_dir in storage_dirs:
if not json_output:
print_storage_header(storage_dirs, storage_dir)
lm = LibraryManager(storage_dir)
items = lm.get_installed()
lm = LibraryPackageManager(storage_dir)
items = lm.legacy_get_installed()
if json_output:
json_result[storage_dir] = items
elif items:
@ -301,6 +334,7 @@ def lib_list(ctx, json_output):
@click.option("--json-output", is_flag=True)
@click.option("--page", type=click.INT, default=1)
@click.option("--id", multiple=True)
@click.option("-o", "--owner", multiple=True)
@click.option("-n", "--name", multiple=True)
@click.option("-a", "--author", multiple=True)
@click.option("-k", "--keyword", multiple=True)
@ -404,12 +438,8 @@ def lib_builtin(storage, json_output):
@click.argument("library", metavar="[LIBRARY]")
@click.option("--json-output", is_flag=True)
def lib_show(library, json_output):
lm = LibraryManager()
name, requirements, _ = lm.parse_pkg_uri(library)
lib_id = lm.search_lib_id(
{"name": name, "requirements": requirements},
silent=json_output,
interactive=not json_output,
lib_id = LibraryPackageManager().reveal_registry_package_id(
library, silent=json_output
)
lib = util.get_api_result("/lib/info/%d" % lib_id, cache_valid="1d")
if json_output:

90
platformio/commands/lib/helpers.py Normal file
View File

@ -0,0 +1,90 @@
# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
from platformio.compat import ci_strings_are_equal
from platformio.managers.platform import PlatformFactory, PlatformManager
from platformio.package.meta import PackageSpec
from platformio.project.config import ProjectConfig
from platformio.project.exception import InvalidProjectConfError
def get_builtin_libs(storage_names=None):
# pylint: disable=import-outside-toplevel
from platformio.package.manager.library import LibraryPackageManager
items = []
storage_names = storage_names or []
pm = PlatformManager()
for manifest in pm.get_installed():
p = PlatformFactory.newPlatform(manifest["__pkg_dir"])
for storage in p.get_lib_storages():
if storage_names and storage["name"] not in storage_names:
continue
lm = LibraryPackageManager(storage["path"])
items.append(
{
"name": storage["name"],
"path": storage["path"],
"items": lm.legacy_get_installed(),
}
)
return items
def is_builtin_lib(storages, name):
for storage in storages or []:
if any(lib.get("name") == name for lib in storage["items"]):
return True
return False
def ignore_deps_by_specs(deps, specs):
result = []
for dep in deps:
depspec = PackageSpec(dep)
if depspec.external:
result.append(dep)
continue
ignore_conditions = []
for spec in specs:
if depspec.owner:
ignore_conditions.append(
ci_strings_are_equal(depspec.owner, spec.owner)
and ci_strings_are_equal(depspec.name, spec.name)
)
else:
ignore_conditions.append(ci_strings_are_equal(depspec.name, spec.name))
if not any(ignore_conditions):
result.append(dep)
return result
def save_project_libdeps(project_dir, specs, environments=None, action="add"):
config = ProjectConfig.get_instance(os.path.join(project_dir, "platformio.ini"))
config.validate(environments)
for env in config.envs():
if environments and env not in environments:
continue
config.expand_interpolations = False
lib_deps = []
try:
lib_deps = ignore_deps_by_specs(config.get("env:" + env, "lib_deps"), specs)
except InvalidProjectConfError:
pass
if action == "add":
lib_deps.extend(spec.as_dependency() for spec in specs)
config.set("env:" + env, "lib_deps", lib_deps)
config.save()

4
platformio/commands/system/command.py
View File

@ -26,9 +26,9 @@ from platformio.commands.system.completion import (
install_completion_code,
uninstall_completion_code,
)
from platformio.managers.lib import LibraryManager
from platformio.managers.package import PackageManager
from platformio.managers.platform import PlatformManager
from platformio.package.manager.library import LibraryPackageManager
from platformio.project.config import ProjectConfig
@ -73,7 +73,7 @@ def system_info(json_output):
}
data["global_lib_nums"] = {
"title": "Global Libraries",
"value": len(LibraryManager().get_installed()),
"value": len(LibraryPackageManager().get_installed()),
}
data["dev_platform_nums"] = {
"title": "Development Platforms",

8
platformio/commands/update.py
View File

@ -15,11 +15,11 @@
import click
from platformio import app
from platformio.commands.lib import CTX_META_STORAGE_DIRS_KEY
from platformio.commands.lib import lib_update as cmd_lib_update
from platformio.commands.lib.command import CTX_META_STORAGE_DIRS_KEY
from platformio.commands.lib.command import lib_update as cmd_lib_update
from platformio.commands.platform import platform_update as cmd_platform_update
from platformio.managers.core import update_core_packages
from platformio.managers.lib import LibraryManager
from platformio.package.manager.library import LibraryPackageManager
@click.command(
@ -55,5 +55,5 @@ def cli(ctx, core_packages, only_check, dry_run):
click.echo()
click.echo("Library Manager")
click.echo("===============")
ctx.meta[CTX_META_STORAGE_DIRS_KEY] = [LibraryManager().package_dir]
ctx.meta[CTX_META_STORAGE_DIRS_KEY] = [LibraryPackageManager().package_dir]
ctx.invoke(cmd_lib_update, only_check=only_check)

8
platformio/compat.py
View File

@ -50,6 +50,14 @@ def get_object_members(obj, ignore_private=True):
}
def ci_strings_are_equal(a, b):
if a == b:
return True
if not a or not b:
return False
return a.strip().lower() == b.strip().lower()
if PY2:
import imp

9
platformio/exception.py
View File

@ -124,15 +124,6 @@ class PackageInstallError(PlatformIOPackageException):
#
class LibNotFound(PlatformioException):
MESSAGE = (
"Library `{0}` has not been found in PlatformIO Registry.\n"
"You can ignore this message, if `{0}` is a built-in library "
"(included in framework, SDK). E.g., SPI, Wire, etc."
)
class NotGlobalLibDir(UserSideException):
MESSAGE = (

41
platformio/maintenance.py
View File

@ -21,13 +21,13 @@ import semantic_version
from platformio import __version__, app, exception, fs, telemetry, util
from platformio.commands import PlatformioCLI
from platformio.commands.lib import CTX_META_STORAGE_DIRS_KEY
from platformio.commands.lib import lib_update as cmd_lib_update
from platformio.commands.lib.command import CTX_META_STORAGE_DIRS_KEY
from platformio.commands.lib.command import lib_update as cmd_lib_update
from platformio.commands.platform import platform_update as cmd_platform_update
from platformio.commands.upgrade import get_latest_version
from platformio.managers.core import update_core_packages
from platformio.managers.lib import LibraryManager
from platformio.managers.platform import PlatformFactory, PlatformManager
from platformio.package.manager.library import LibraryPackageManager
from platformio.proc import is_container
@ -240,7 +240,7 @@ def check_platformio_upgrade():
click.echo("")
def check_internal_updates(ctx, what):
def check_internal_updates(ctx, what): # pylint: disable=too-many-branches
last_check = app.get_state_item("last_check", {})
interval = int(app.get_setting("check_%s_interval" % what)) * 3600 * 24
if (time() - interval) < last_check.get(what + "_update", 0):
@ -251,20 +251,27 @@ def check_internal_updates(ctx, what):
util.internet_on(raise_exception=True)
pm = PlatformManager() if what == "platforms" else LibraryManager()
outdated_items = []
for manifest in pm.get_installed():
if manifest["name"] in outdated_items:
continue
conds = [
pm.outdated(manifest["__pkg_dir"]),
what == "platforms"
and PlatformFactory.newPlatform(
manifest["__pkg_dir"]
).are_outdated_packages(),
]
if any(conds):
outdated_items.append(manifest["name"])
pm = PlatformManager() if what == "platforms" else LibraryPackageManager()
if isinstance(pm, PlatformManager):
for manifest in pm.get_installed():
if manifest["name"] in outdated_items:
continue
conds = [
pm.outdated(manifest["__pkg_dir"]),
what == "platforms"
and PlatformFactory.newPlatform(
manifest["__pkg_dir"]
).are_outdated_packages(),
]
if any(conds):
outdated_items.append(manifest["name"])
else:
for pkg in pm.get_installed():
if pkg.metadata.name in outdated_items:
continue
if pm.outdated(pkg).is_outdated():
outdated_items.append(pkg.metadata.name)
if not outdated_items:
return

374
platformio/managers/lib.py
View File

@ -1,374 +0,0 @@
# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# pylint: disable=too-many-arguments, too-many-locals, too-many-branches
# pylint: disable=too-many-return-statements
import json
from glob import glob
from os.path import isdir, join
import click
import semantic_version
from platformio import app, exception, util
from platformio.compat import glob_escape
from platformio.managers.package import BasePkgManager
from platformio.managers.platform import PlatformFactory, PlatformManager
from platformio.package.exception import ManifestException
from platformio.package.manifest.parser import ManifestParserFactory
from platformio.project.config import ProjectConfig
class LibraryManager(BasePkgManager):
FILE_CACHE_VALID = "30d" # 1 month
def __init__(self, package_dir=None):
self.config = ProjectConfig.get_instance()
super(LibraryManager, self).__init__(
package_dir or self.config.get_optional_dir("globallib")
)
@property
def manifest_names(self):
return [".library.json", "library.json", "library.properties", "module.json"]
def get_manifest_path(self, pkg_dir):
path = BasePkgManager.get_manifest_path(self, pkg_dir)
if path:
return path
# if library without manifest, returns first source file
src_dir = join(glob_escape(pkg_dir))
if isdir(join(pkg_dir, "src")):
src_dir = join(src_dir, "src")
chs_files = glob(join(src_dir, "*.[chS]"))
if chs_files:
return chs_files[0]
cpp_files = glob(join(src_dir, "*.cpp"))
if cpp_files:
return cpp_files[0]
return None
def max_satisfying_repo_version(self, versions, requirements=None):
def _cmp_dates(datestr1, datestr2):
date1 = util.parse_date(datestr1)
date2 = util.parse_date(datestr2)
if date1 == date2:
return 0
return -1 if date1 < date2 else 1
semver_spec = None
try:
semver_spec = (
semantic_version.SimpleSpec(requirements) if requirements else None
)
except ValueError:
pass
item = {}
for v in versions:
semver_new = self.parse_semver_version(v["name"])
if semver_spec:
# pylint: disable=unsupported-membership-test
if not semver_new or semver_new not in semver_spec:
continue
if not item or self.parse_semver_version(item["name"]) < semver_new:
item = v
elif requirements:
if requirements == v["name"]:
return v
else:
if not item or _cmp_dates(item["released"], v["released"]) == -1:
item = v
return item
def get_latest_repo_version(self, name, requirements, silent=False):
item = self.max_satisfying_repo_version(
util.get_api_result(
"/lib/info/%d"
% self.search_lib_id(
{"name": name, "requirements": requirements}, silent=silent
),
cache_valid="1h",
)["versions"],
requirements,
)
return item["name"] if item else None
def _install_from_piorepo(self, name, requirements):
assert name.startswith("id="), name
version = self.get_latest_repo_version(name, requirements)
if not version:
raise exception.UndefinedPackageVersion(
requirements or "latest", util.get_systype()
)
dl_data = util.get_api_result(
"/lib/download/" + str(name[3:]), dict(version=version), cache_valid="30d"
)
assert dl_data
return self._install_from_url(
name,
dl_data["url"].replace("http://", "https://")
if app.get_setting("strict_ssl")
else dl_data["url"],
requirements,
)
def search_lib_id( # pylint: disable=too-many-branches
self, filters, silent=False, interactive=False
):
assert isinstance(filters, dict)
assert "name" in filters
# try to find ID within installed packages
lib_id = self._get_lib_id_from_installed(filters)
if lib_id:
return lib_id
# looking in PIO Library Registry
if not silent:
click.echo(
"Looking for %s library in registry"
% click.style(filters["name"], fg="cyan")
)
query = []
for key in filters:
if key not in ("name", "authors", "frameworks", "platforms"):
continue
values = filters[key]
if not isinstance(values, list):
values = [v.strip() for v in values.split(",") if v]
for value in values:
query.append(
'%s:"%s"' % (key[:-1] if key.endswith("s") else key, value)
)
lib_info = None
result = util.get_api_result(
"/v2/lib/search", dict(query=" ".join(query)), cache_valid="1h"
)
if result["total"] == 1:
lib_info = result["items"][0]
elif result["total"] > 1:
if silent and not interactive:
lib_info = result["items"][0]
else:
click.secho(
"Conflict: More than one library has been found "
"by request %s:" % json.dumps(filters),
fg="yellow",
err=True,
)
# pylint: disable=import-outside-toplevel
from platformio.commands.lib import print_lib_item
for item in result["items"]:
print_lib_item(item)
if not interactive:
click.secho(
"Automatically chose the first available library "
"(use `--interactive` option to make a choice)",
fg="yellow",
err=True,
)
lib_info = result["items"][0]
else:
deplib_id = click.prompt(
"Please choose library ID",
type=click.Choice([str(i["id"]) for i in result["items"]]),
)
for item in result["items"]:
if item["id"] == int(deplib_id):
lib_info = item
break
if not lib_info:
if list(filters) == ["name"]:
raise exception.LibNotFound(filters["name"])
raise exception.LibNotFound(str(filters))
if not silent:
click.echo(
"Found: %s"
% click.style(
"https://platformio.org/lib/show/{id}/{name}".format(**lib_info),
fg="blue",
)
)
return int(lib_info["id"])
def _get_lib_id_from_installed(self, filters):
if filters["name"].startswith("id="):
return int(filters["name"][3:])
package_dir = self.get_package_dir(
filters["name"], filters.get("requirements", filters.get("version"))
)
if not package_dir:
return None
manifest = self.load_manifest(package_dir)
if "id" not in manifest:
return None
for key in ("frameworks", "platforms"):
if key not in filters:
continue
if key not in manifest:
return None
if not util.items_in_list(
util.items_to_list(filters[key]), util.items_to_list(manifest[key])
):
return None
if "authors" in filters:
if "authors" not in manifest:
return None
manifest_authors = manifest["authors"]
if not isinstance(manifest_authors, list):
manifest_authors = [manifest_authors]
manifest_authors = [
a["name"]
for a in manifest_authors
if isinstance(a, dict) and "name" in a
]
filter_authors = filters["authors"]
if not isinstance(filter_authors, list):
filter_authors = [filter_authors]
if not set(filter_authors) <= set(manifest_authors):
return None
return int(manifest["id"])
def install( # pylint: disable=arguments-differ
self,
name,
requirements=None,
silent=False,
after_update=False,
interactive=False,
force=False,
):
_name, _requirements, _url = self.parse_pkg_uri(name, requirements)
if not _url:
name = "id=%d" % self.search_lib_id(
{"name": _name, "requirements": _requirements},
silent=silent,
interactive=interactive,
)
requirements = _requirements
pkg_dir = BasePkgManager.install(
self,
name,
requirements,
silent=silent,
after_update=after_update,
force=force,
)
if not pkg_dir:
return None
manifest = None
try:
manifest = ManifestParserFactory.new_from_dir(pkg_dir).as_dict()
except ManifestException:
pass
if not manifest or not manifest.get("dependencies"):
return pkg_dir
if not silent:
click.secho("Installing dependencies", fg="yellow")
builtin_lib_storages = None
for filters in manifest["dependencies"]:
assert "name" in filters
# avoid circle dependencies
if not self.INSTALL_HISTORY:
self.INSTALL_HISTORY = []
history_key = str(filters)
if history_key in self.INSTALL_HISTORY:
continue
self.INSTALL_HISTORY.append(history_key)
if any(s in filters.get("version", "") for s in ("\\", "/")):
self.install(
"{name}={version}".format(**filters),
silent=silent,
after_update=after_update,
interactive=interactive,
force=force,
)
else:
try:
lib_id = self.search_lib_id(filters, silent, interactive)
except exception.LibNotFound as e:
if builtin_lib_storages is None:
builtin_lib_storages = get_builtin_libs()
if not silent or is_builtin_lib(
builtin_lib_storages, filters["name"]
):
click.secho("Warning! %s" % e, fg="yellow")
continue
if filters.get("version"):
self.install(
lib_id,
filters.get("version"),
silent=silent,
after_update=after_update,
interactive=interactive,
force=force,
)
else:
self.install(
lib_id,
silent=silent,
after_update=after_update,
interactive=interactive,
force=force,
)
return pkg_dir
def get_builtin_libs(storage_names=None):
items = []
storage_names = storage_names or []
pm = PlatformManager()
for manifest in pm.get_installed():
p = PlatformFactory.newPlatform(manifest["__pkg_dir"])
for storage in p.get_lib_storages():
if storage_names and storage["name"] not in storage_names:
continue
lm = LibraryManager(storage["path"])
items.append(
{
"name": storage["name"],
"path": storage["path"],
"items": lm.get_installed(),
}
)
return items
def is_builtin_lib(storages, name):
for storage in storages or []:
if any(l.get("name") == name for l in storage["items"]):
return True
return False

8
platformio/managers/package.py
View File

@ -482,7 +482,7 @@ class PkgInstallerMixin(object):
self.unpack(dlpath, tmp_dir)
os.remove(dlpath)
else:
vcs = VCSClientFactory.newClient(tmp_dir, url)
vcs = VCSClientFactory.new(tmp_dir, url)
assert vcs.export()
src_manifest_dir = vcs.storage_dir
src_manifest["version"] = vcs.get_current_revision()
@ -628,9 +628,7 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin):
if "__src_url" in manifest:
try:
vcs = VCSClientFactory.newClient(
pkg_dir, manifest["__src_url"], silent=True
)
vcs = VCSClientFactory.new(pkg_dir, manifest["__src_url"], silent=True)
except (AttributeError, exception.PlatformioException):
return None
if not vcs.can_be_updated:
@ -800,7 +798,7 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin):
return True
if "__src_url" in manifest:
vcs = VCSClientFactory.newClient(pkg_dir, manifest["__src_url"])
vcs = VCSClientFactory.new(pkg_dir, manifest["__src_url"])
assert vcs.update()
self._update_src_manifest(
dict(version=vcs.get_current_revision()), vcs.storage_dir

39
platformio/package/manager/_install.py
View File

@ -20,7 +20,7 @@ import tempfile
import click
from platformio import app, compat, fs, util
from platformio.package.exception import PackageException
from platformio.package.exception import MissingPackageManifestError, PackageException
from platformio.package.meta import PackageSourceItem, PackageSpec
from platformio.package.unpack import FileUnpacker
from platformio.package.vcsclient import VCSClientFactory
@ -83,7 +83,7 @@ class PackageManagerInstallMixin(object):
msg = "Installing %s" % click.style(spec.humanize(), fg="cyan")
self.print_message(msg)
if spec.url:
if spec.external:
pkg = self.install_from_url(spec.url, spec, silent=silent)
else:
pkg = self.install_from_registry(spec, search_filters, silent=silent)
@ -152,7 +152,7 @@ class PackageManagerInstallMixin(object):
assert os.path.isfile(dl_path)
self.unpack(dl_path, tmp_dir)
else:
vcs = VCSClientFactory.newClient(tmp_dir, url)
vcs = VCSClientFactory.new(tmp_dir, url)
assert vcs.export()
root_dir = self.find_pkg_root(tmp_dir, spec)
@ -189,12 +189,20 @@ class PackageManagerInstallMixin(object):
# what to do with existing package?
action = "overwrite"
if dst_pkg.metadata and dst_pkg.metadata.spec.url:
if tmp_pkg.metadata.spec.has_custom_name():
action = "overwrite"
dst_pkg = PackageSourceItem(
os.path.join(self.package_dir, tmp_pkg.metadata.spec.name)
)
elif dst_pkg.metadata and dst_pkg.metadata.spec.external:
if dst_pkg.metadata.spec.url != tmp_pkg.metadata.spec.url:
action = "detach-existing"
elif tmp_pkg.metadata.spec.url:
elif tmp_pkg.metadata.spec.external:
action = "detach-new"
elif dst_pkg.metadata and dst_pkg.metadata.version != tmp_pkg.metadata.version:
elif dst_pkg.metadata and (
dst_pkg.metadata.version != tmp_pkg.metadata.version
or dst_pkg.metadata.spec.owner != tmp_pkg.metadata.spec.owner
):
action = (
"detach-existing"
if tmp_pkg.metadata.version > dst_pkg.metadata.version
@ -231,7 +239,7 @@ class PackageManagerInstallMixin(object):
tmp_pkg.get_safe_dirname(),
tmp_pkg.metadata.version,
)
if tmp_pkg.metadata.spec.url:
if tmp_pkg.metadata.spec.external:
target_dirname = "%s@src-%s" % (
tmp_pkg.get_safe_dirname(),
hashlib.md5(
@ -247,3 +255,20 @@ class PackageManagerInstallMixin(object):
_cleanup_dir(dst_pkg.path)
shutil.move(tmp_pkg.path, dst_pkg.path)
return PackageSourceItem(dst_pkg.path)
def get_installed(self):
result = []
for name in os.listdir(self.package_dir):
pkg_dir = os.path.join(self.package_dir, name)
if not os.path.isdir(pkg_dir):
continue
pkg = PackageSourceItem(pkg_dir)
if not pkg.metadata:
try:
spec = self.build_legacy_spec(pkg_dir)
pkg.metadata = self.build_metadata(pkg_dir, spec)
except MissingPackageManifestError:
pass
if pkg.metadata:
result.append(pkg)
return result

57
platformio/package/manager/_legacy.py Normal file
View File

@ -0,0 +1,57 @@
# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
from platformio import fs
from platformio.package.meta import PackageSourceItem, PackageSpec
class PackageManagerLegacyMixin(object):
def build_legacy_spec(self, pkg_dir):
# find src manifest
src_manifest_name = ".piopkgmanager.json"
src_manifest_path = None
for name in os.listdir(pkg_dir):
if not os.path.isfile(os.path.join(pkg_dir, name, src_manifest_name)):
continue
src_manifest_path = os.path.join(pkg_dir, name, src_manifest_name)
break
if src_manifest_path:
src_manifest = fs.load_json(src_manifest_path)
return PackageSpec(
name=src_manifest.get("name"),
url=src_manifest.get("url"),
requirements=src_manifest.get("requirements"),
)
# fall back to a package manifest
manifest = self.load_manifest(pkg_dir)
return PackageSpec(name=manifest.get("name"))
def legacy_load_manifest(self, pkg):
assert isinstance(pkg, PackageSourceItem)
manifest = self.load_manifest(pkg)
manifest["__pkg_dir"] = pkg.path
for key in ("name", "version"):
if not manifest.get(key):
manifest[key] = str(getattr(pkg.metadata, key))
if pkg.metadata and pkg.metadata.spec and pkg.metadata.spec.external:
manifest["__src_url"] = pkg.metadata.spec.url
manifest["version"] = str(pkg.metadata.version)
return manifest
def legacy_get_installed(self):
return [self.legacy_load_manifest(pkg) for pkg in self.get_installed()]

50
platformio/package/manager/_registry.py
View File

@ -79,10 +79,10 @@ class RegistryFileMirrorsIterator(object):
class PackageManageRegistryMixin(object):
def install_from_registry(self, spec, search_filters=None, silent=False):
if spec.owner and spec.name and not search_filters:
package = self.fetch_registry_package(spec.owner, spec.name)
package = self.fetch_registry_package(spec)
if not package:
raise UnknownPackageError(spec.humanize())
version = self._pick_best_pkg_version(package["versions"], spec)
version = self.pick_best_registry_version(package["versions"], spec)
else:
packages = self.search_registry_packages(spec, search_filters)
if not packages:
@ -131,10 +131,33 @@ class PackageManageRegistryMixin(object):
"items"
]
def fetch_registry_package(self, owner, name):
return self.get_registry_client_instance().get_package(
self.pkg_type, owner, name
)
def fetch_registry_package(self, spec):
result = None
if spec.owner and spec.name:
result = self.get_registry_client_instance().get_package(
self.pkg_type, spec.owner, spec.name
)
if not result and (spec.id or (spec.name and not spec.owner)):
packages = self.search_registry_packages(spec)
if packages:
result = self.get_registry_client_instance().get_package(
self.pkg_type, packages[0]["owner"]["username"], packages[0]["name"]
)
if not result:
raise UnknownPackageError(spec.humanize())
return result
def reveal_registry_package_id(self, spec, silent=False):
spec = self.ensure_spec(spec)
if spec.id:
return spec.id
packages = self.search_registry_packages(spec)
if not packages:
raise UnknownPackageError(spec.humanize())
if len(packages) > 1 and not silent:
self.print_multi_package_issue(packages, spec)
click.echo("")
return packages[0]["id"]
@staticmethod
def print_multi_package_issue(packages, spec):
@ -160,7 +183,7 @@ class PackageManageRegistryMixin(object):
def find_best_registry_version(self, packages, spec):
# find compatible version within the latest package versions
for package in packages:
version = self._pick_best_pkg_version([package["version"]], spec)
version = self.pick_best_registry_version([package["version"]], spec)
if version:
return (package, version)
@ -169,9 +192,13 @@ class PackageManageRegistryMixin(object):
# if the custom version requirements, check ALL package versions
for package in packages:
version = self._pick_best_pkg_version(
version = self.pick_best_registry_version(
self.fetch_registry_package(
package["owner"]["username"], package["name"]
PackageSpec(
id=package["id"],
owner=package["owner"]["username"],
name=package["name"],
)
).get("versions"),
spec,
)
@ -180,11 +207,12 @@ class PackageManageRegistryMixin(object):
time.sleep(1)
return None
def _pick_best_pkg_version(self, versions, spec):
def pick_best_registry_version(self, versions, spec=None):
assert not spec or isinstance(spec, PackageSpec)
best = None
for version in versions:
semver = PackageMetaData.to_semver(version["name"])
if spec.requirements and semver not in spec.requirements:
if spec and spec.requirements and semver not in spec.requirements:
continue
if not any(
self.is_system_compatible(f.get("system")) for f in version["files"]

7
platformio/package/manager/_uninstall.py
View File

@ -31,10 +31,7 @@ class PackageManagerUninstallMixin(object):
self.unlock()
def _uninstall(self, pkg, silent=False, skip_dependencies=False):
if not isinstance(pkg, PackageSourceItem):
pkg = (
PackageSourceItem(pkg) if os.path.isdir(pkg) else self.get_package(pkg)
)
pkg = self.get_package(pkg)
if not pkg or not pkg.metadata:
raise UnknownPackageError(pkg)
@ -73,7 +70,7 @@ class PackageManagerUninstallMixin(object):
if not silent:
click.echo("[%s]" % click.style("OK", fg="green"))
return True
return pkg
def _uninstall_dependencies(self, pkg, silent=False):
assert isinstance(pkg, PackageSourceItem)

166
platformio/package/manager/_update.py Normal file
View File

@ -0,0 +1,166 @@
# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import click
from platformio import util
from platformio.package.exception import UnknownPackageError
from platformio.package.meta import (
PackageOutdatedResult,
PackageSourceItem,
PackageSpec,
)
from platformio.package.vcsclient import VCSBaseException, VCSClientFactory
class PackageManagerUpdateMixin(object):
def outdated(self, pkg, spec=None):
assert isinstance(pkg, PackageSourceItem)
assert not spec or isinstance(spec, PackageSpec)
assert os.path.isdir(pkg.path) and pkg.metadata
# skip detached package to a specific version
detached_conditions = [
"@" in pkg.path,
pkg.metadata.spec and not pkg.metadata.spec.external,
not spec,
]
if all(detached_conditions):
return PackageOutdatedResult(current=pkg.metadata.version, detached=True)
latest = None
wanted = None
if pkg.metadata.spec.external:
latest = self._fetch_vcs_latest_version(pkg)
else:
try:
reg_pkg = self.fetch_registry_package(pkg.metadata.spec)
latest = (
self.pick_best_registry_version(reg_pkg["versions"]) or {}
).get("name")
if spec:
wanted = (
self.pick_best_registry_version(reg_pkg["versions"], spec) or {}
).get("name")
if not wanted: # wrong library
latest = None
except UnknownPackageError:
pass
return PackageOutdatedResult(
current=pkg.metadata.version, latest=latest, wanted=wanted
)
def _fetch_vcs_latest_version(self, pkg):
vcs = None
try:
vcs = VCSClientFactory.new(pkg.path, pkg.metadata.spec.url, silent=True)
except VCSBaseException:
return None
if not vcs.can_be_updated:
return None
return str(
self.build_metadata(
pkg.path, pkg.metadata.spec, vcs_revision=vcs.get_latest_revision()
).version
)
def update(self, pkg, spec=None, only_check=False, silent=False):
pkg = self.get_package(pkg)
if not pkg or not pkg.metadata:
raise UnknownPackageError(pkg)
if not silent:
click.echo(
"{} {:<45} {:<30}".format(
"Checking" if only_check else "Updating",
click.style(pkg.metadata.spec.humanize(), fg="cyan"),
"%s (%s)" % (pkg.metadata.version, spec.requirements)
if spec and spec.requirements
else str(pkg.metadata.version),
),
nl=False,
)
if not util.internet_on():
if not silent:
click.echo("[%s]" % (click.style("Off-line", fg="yellow")))
return pkg
outdated = self.outdated(pkg, spec)
if not silent:
self.print_outdated_state(outdated)
up_to_date = any(
[
outdated.detached,
not outdated.latest,
outdated.latest and outdated.current == outdated.latest,
outdated.wanted and outdated.current == outdated.wanted,
]
)
if only_check or up_to_date:
return pkg
try:
self.lock()
return self._update(pkg, outdated, silent=silent)
finally:
self.unlock()
@staticmethod
def print_outdated_state(outdated):
if outdated.detached:
return click.echo("[%s]" % (click.style("Detached", fg="yellow")))
if not outdated.latest or outdated.current == outdated.latest:
return click.echo("[%s]" % (click.style("Up-to-date", fg="green")))
if outdated.wanted and outdated.current == outdated.wanted:
return click.echo(
"[%s]"
% (click.style("Incompatible (%s)" % outdated.latest, fg="yellow"))
)
return click.echo(
"[%s]" % (click.style(str(outdated.wanted or outdated.latest), fg="red"))
)
def _update(self, pkg, outdated, silent=False):
if pkg.metadata.spec.external:
vcs = VCSClientFactory.new(pkg.path, pkg.metadata.spec.url)
assert vcs.update()
pkg.metadata.version = self._fetch_vcs_latest_version(pkg)
pkg.dump_meta()
return pkg
new_pkg = self.install(
PackageSpec(
id=pkg.metadata.spec.id,
owner=pkg.metadata.spec.owner,
name=pkg.metadata.spec.name,
requirements=outdated.wanted or outdated.latest,
),
silent=silent,
)
if new_pkg:
old_pkg = self.get_package(
PackageSpec(
id=pkg.metadata.spec.id,
owner=pkg.metadata.spec.owner,
name=pkg.metadata.name,
requirements=pkg.metadata.version,
)
)
if old_pkg:
self.uninstall(old_pkg, silent=silent, skip_dependencies=True)
return new_pkg

76
platformio/package/manager/base.py
View File

@ -18,14 +18,17 @@ from datetime import datetime
import click
import semantic_version
from platformio import fs, util
from platformio import util
from platformio.commands import PlatformioCLI
from platformio.compat import ci_strings_are_equal
from platformio.package.exception import ManifestException, MissingPackageManifestError
from platformio.package.lockfile import LockFile
from platformio.package.manager._download import PackageManagerDownloadMixin
from platformio.package.manager._install import PackageManagerInstallMixin
from platformio.package.manager._legacy import PackageManagerLegacyMixin
from platformio.package.manager._registry import PackageManageRegistryMixin
from platformio.package.manager._uninstall import PackageManagerUninstallMixin
from platformio.package.manager._update import PackageManagerUpdateMixin
from platformio.package.manifest.parser import ManifestParserFactory
from platformio.package.meta import (
PackageMetaData,
@ -41,6 +44,8 @@ class BasePackageManager( # pylint: disable=too-many-public-methods
PackageManageRegistryMixin,
PackageManagerInstallMixin,
PackageManagerUninstallMixin,
PackageManagerUpdateMixin,
PackageManagerLegacyMixin,
):
_MEMORY_CACHE = {}
@ -83,10 +88,6 @@ class BasePackageManager( # pylint: disable=too-many-public-methods
return True
return util.items_in_list(value, util.get_systype())
@staticmethod
def generate_rand_version():
return datetime.now().strftime("0.0.0+%Y%m%d%H%M%S")
@staticmethod
def ensure_dir_exists(path):
if not os.path.isdir(path):
@ -162,27 +163,9 @@ class BasePackageManager( # pylint: disable=too-many-public-methods
click.secho(str(e), fg="yellow")
raise MissingPackageManifestError(", ".join(self.manifest_names))
def build_legacy_spec(self, pkg_dir):
# find src manifest
src_manifest_name = ".piopkgmanager.json"
src_manifest_path = None
for name in os.listdir(pkg_dir):
if not os.path.isfile(os.path.join(pkg_dir, name, src_manifest_name)):
continue
src_manifest_path = os.path.join(pkg_dir, name, src_manifest_name)
break
if src_manifest_path:
src_manifest = fs.load_json(src_manifest_path)
return PackageSpec(
name=src_manifest.get("name"),
url=src_manifest.get("url"),
requirements=src_manifest.get("requirements"),
)
# fall back to a package manifest
manifest = self.load_manifest(pkg_dir)
return PackageSpec(name=manifest.get("name"))
@staticmethod
def generate_rand_version():
return datetime.now().strftime("0.0.0+%Y%m%d%H%M%S")
def build_metadata(self, pkg_dir, spec, vcs_revision=None):
manifest = self.load_manifest(pkg_dir)
@ -192,7 +175,7 @@ class BasePackageManager( # pylint: disable=too-many-public-methods
version=manifest.get("version"),
spec=spec,
)
if not metadata.name or spec.is_custom_name():
if not metadata.name or spec.has_custom_name():
metadata.name = spec.name
if vcs_revision:
metadata.version = "%s+sha.%s" % (
@ -203,42 +186,27 @@ class BasePackageManager( # pylint: disable=too-many-public-methods
metadata.version = self.generate_rand_version()
return metadata
def get_installed(self):
result = []
for name in os.listdir(self.package_dir):
pkg_dir = os.path.join(self.package_dir, name)
if not os.path.isdir(pkg_dir):
continue
pkg = PackageSourceItem(pkg_dir)
if not pkg.metadata:
try:
spec = self.build_legacy_spec(pkg_dir)
pkg.metadata = self.build_metadata(pkg_dir, spec)
except MissingPackageManifestError:
pass
if pkg.metadata:
result.append(pkg)
return result
def get_package(self, spec):
def _ci_strings_are_equal(a, b):
if a == b:
return True
if not a or not b:
return False
return a.strip().lower() == b.strip().lower()
if isinstance(spec, PackageSourceItem):
return spec
if not isinstance(spec, PackageSpec) and os.path.isdir(spec):
for pkg in self.get_installed():
if spec == pkg.path:
return pkg
return None
spec = self.ensure_spec(spec)
best = None
for pkg in self.get_installed():
skip_conditions = [
spec.owner
and not _ci_strings_are_equal(spec.owner, pkg.metadata.spec.owner),
spec.url and spec.url != pkg.metadata.spec.url,
and not ci_strings_are_equal(spec.owner, pkg.metadata.spec.owner),
spec.external and spec.url != pkg.metadata.spec.url,
spec.id and spec.id != pkg.metadata.spec.id,
not spec.id
and not spec.url
and not _ci_strings_are_equal(spec.name, pkg.metadata.name),
and not spec.external
and not ci_strings_are_equal(spec.name, pkg.metadata.name),
]
if any(skip_conditions):
continue

2
platformio/package/manager/library.py
View File

@ -21,7 +21,7 @@ from platformio.package.meta import PackageSpec, PackageType
from platformio.project.helpers import get_project_global_lib_dir
class LibraryPackageManager(BasePackageManager):
class LibraryPackageManager(BasePackageManager): # pylint: disable=too-many-ancestors
def __init__(self, package_dir=None):
super(LibraryPackageManager, self).__init__(
PackageType.LIBRARY, package_dir or get_project_global_lib_dir()

2
platformio/package/manager/platform.py
View File

@ -17,7 +17,7 @@ from platformio.package.meta import PackageType
from platformio.project.config import ProjectConfig
class PlatformPackageManager(BasePackageManager):
class PlatformPackageManager(BasePackageManager): # pylint: disable=too-many-ancestors
def __init__(self, package_dir=None):
self.config = ProjectConfig.get_instance()
super(PlatformPackageManager, self).__init__(

2
platformio/package/manager/tool.py
View File

@ -17,7 +17,7 @@ from platformio.package.meta import PackageType
from platformio.project.config import ProjectConfig
class ToolPackageManager(BasePackageManager):
class ToolPackageManager(BasePackageManager): # pylint: disable=too-many-ancestors
def __init__(self, package_dir=None):
self.config = ProjectConfig.get_instance()
super(ToolPackageManager, self).__init__(

2
platformio/package/manifest/schema.py
View File

@ -250,7 +250,7 @@ class ManifestSchema(BaseSchema):
def load_spdx_licenses():
r = requests.get(
"https://raw.githubusercontent.com/spdx/license-list-data"
"/v3.9/json/licenses.json"
"/v3.10/json/licenses.json"
)
r.raise_for_status()
return r.json()

69
platformio/package/meta.py
View File

@ -65,7 +65,44 @@ class PackageType(object):
return None
class PackageSpec(object):
class PackageOutdatedResult(object):
def __init__(self, current, latest=None, wanted=None, detached=False):
self.current = current
self.latest = latest
self.wanted = wanted
self.detached = detached
def __repr__(self):
return (
"PackageOutdatedResult <current={current} latest={latest} wanted={wanted} "
"detached={detached}>".format(
current=self.current,
latest=self.latest,
wanted=self.wanted,
detached=self.detached,
)
)
def __setattr__(self, name, value):
if (
value
and name in ("current", "latest", "wanted")
and not isinstance(value, semantic_version.Version)
):
value = semantic_version.Version(str(value))
return super(PackageOutdatedResult, self).__setattr__(name, value)
def is_outdated(self, allow_incompatible=False):
if self.detached or not self.latest or self.current == self.latest:
return False
if allow_incompatible:
return self.current != self.latest
if self.wanted:
return self.current != self.wanted
return True
class PackageSpec(object): # pylint: disable=too-many-instance-attributes
def __init__( # pylint: disable=redefined-builtin,too-many-arguments
self, raw=None, owner=None, id=None, name=None, requirements=None, url=None
):
@ -74,6 +111,7 @@ class PackageSpec(object):
self.name = name
self._requirements = None
self.url = url
self.raw = raw
if requirements:
self.requirements = requirements
self._name_is_custom = False
@ -104,6 +142,10 @@ class PackageSpec(object):
"requirements={requirements} url={url}>".format(**self.as_dict())
)
@property
def external(self):
return bool(self.url)
@property
def requirements(self):
return self._requirements
@ -116,24 +158,24 @@ class PackageSpec(object):
self._requirements = (
value
if isinstance(value, semantic_version.SimpleSpec)
else semantic_version.SimpleSpec(value)
else semantic_version.SimpleSpec(str(value))
)
def humanize(self):
result = ""
if self.url:
result = self.url
elif self.id:
result = "id:%d" % self.id
else:
result = ""
elif self.name:
if self.owner:
result = self.owner + "/"
result += self.name
elif self.id:
result = "id:%d" % self.id
if self.requirements:
result += " @ " + str(self.requirements)
return result
def is_custom_name(self):
def has_custom_name(self):
return self._name_is_custom
def as_dict(self):
@ -145,6 +187,19 @@ class PackageSpec(object):
url=self.url,
)
def as_dependency(self):
if self.url:
return self.raw or self.url
result = ""
if self.name:
result = "%s/%s" % (self.owner, self.name) if self.owner else self.name
elif self.id:
result = str(self.id)
assert result
if self.requirements:
result = "%s@%s" % (result, self.requirements)
return result
def _parse(self, raw):
if raw is None:
return

34
platformio/package/vcsclient.py
View File

@ -17,7 +17,11 @@ from os.path import join
from subprocess import CalledProcessError, check_call
from sys import modules
from platformio.exception import PlatformioException, UserSideException
from platformio.package.exception import (
PackageException,
PlatformioException,
UserSideException,
)
from platformio.proc import exec_command
try:
@ -26,9 +30,13 @@ except ImportError:
from urlparse import urlparse
class VCSBaseException(PackageException):
pass
class VCSClientFactory(object):
@staticmethod
def newClient(src_dir, remote_url, silent=False):
def new(src_dir, remote_url, silent=False):
result = urlparse(remote_url)
type_ = result.scheme
tag = None
@ -41,12 +49,15 @@ class VCSClientFactory(object):
if "#" in remote_url:
remote_url, tag = remote_url.rsplit("#", 1)
if not type_:
raise PlatformioException("VCS: Unknown repository type %s" % remote_url)
obj = getattr(modules[__name__], "%sClient" % type_.title())(
src_dir, remote_url, tag, silent
)
assert isinstance(obj, VCSClientBase)
return obj
raise VCSBaseException("VCS: Unknown repository type %s" % remote_url)
try:
obj = getattr(modules[__name__], "%sClient" % type_.title())(
src_dir, remote_url, tag, silent
)
assert isinstance(obj, VCSClientBase)
return obj
except (AttributeError, AssertionError):
raise VCSBaseException("VCS: Unknown repository type %s" % remote_url)
class VCSClientBase(object):
@ -101,7 +112,7 @@ class VCSClientBase(object):
check_call(args, **kwargs)
return True
except CalledProcessError as e:
raise PlatformioException("VCS: Could not process command %s" % e.cmd)
raise VCSBaseException("VCS: Could not process command %s" % e.cmd)
def get_cmd_output(self, args, **kwargs):
args = [self.command] + args
@ -110,7 +121,7 @@ class VCSClientBase(object):
result = exec_command(args, **kwargs)
if result["returncode"] == 0:
return result["out"].strip()
raise PlatformioException(
raise VCSBaseException(
"VCS: Could not receive an output from `%s` command (%s)" % (args, result)
)
@ -227,7 +238,6 @@ class SvnClient(VCSClientBase):
return self.run_cmd(args)
def update(self):
args = ["update"]
return self.run_cmd(args)
@ -239,4 +249,4 @@ class SvnClient(VCSClientBase):
line = line.strip()
if line.startswith("Revision:"):
return line.split(":", 1)[1].strip()
raise PlatformioException("Could not detect current SVN revision")
raise VCSBaseException("Could not detect current SVN revision")

2
tests/commands/test_ci.py
View File

@ -15,7 +15,7 @@
from os.path import isfile, join
from platformio.commands.ci import cli as cmd_ci
from platformio.commands.lib import cli as cmd_lib
from platformio.commands.lib.command import cli as cmd_lib
def test_ci_empty(clirunner):

442
tests/commands/test_lib.py
View File

@ -13,332 +13,184 @@
# limitations under the License.
import json
import re
import os
from platformio import exception
from platformio.commands import PlatformioCLI
from platformio.commands.lib import cli as cmd_lib
import semantic_version
PlatformioCLI.leftover_args = ["--json-output"] # hook for click
from platformio.clients.registry import RegistryClient
from platformio.commands.lib.command import cli as cmd_lib
from platformio.package.meta import PackageType
from platformio.package.vcsclient import VCSClientFactory
from platformio.project.config import ProjectConfig
def test_search(clirunner, validate_cliresult):
result = clirunner.invoke(cmd_lib, ["search", "DHT22"])
def test_saving_deps(clirunner, validate_cliresult, isolated_pio_core, tmpdir_factory):
regclient = RegistryClient()
project_dir = tmpdir_factory.mktemp("project")
project_dir.join("platformio.ini").write(
"""
[env]
lib_deps = ArduinoJson
[env:one]
board = devkit
[env:two]
framework = foo
lib_deps =
CustomLib
ArduinoJson @ 5.10.1
"""
)
result = clirunner.invoke(cmd_lib, ["-d", str(project_dir), "install", "64"])
validate_cliresult(result)
match = re.search(r"Found\s+(\d+)\slibraries:", result.output)
assert int(match.group(1)) > 2
aj_pkg_data = regclient.get_package(PackageType.LIBRARY, "bblanchon", "ArduinoJson")
config = ProjectConfig(os.path.join(str(project_dir), "platformio.ini"))
assert config.get("env:one", "lib_deps") == [
"bblanchon/ArduinoJson@^%s" % aj_pkg_data["version"]["name"]
]
assert config.get("env:two", "lib_deps") == [
"CustomLib",
"bblanchon/ArduinoJson@^%s" % aj_pkg_data["version"]["name"],
]
result = clirunner.invoke(cmd_lib, ["search", "DHT22", "--platform=timsp430"])
# ensure "build" version without NPM spec
result = clirunner.invoke(
cmd_lib,
["-d", str(project_dir), "-e", "one", "install", "mbed-sam-grove/LinkedList"],
)
validate_cliresult(result)
match = re.search(r"Found\s+(\d+)\slibraries:", result.output)
assert int(match.group(1)) > 1
ll_pkg_data = regclient.get_package(
PackageType.LIBRARY, "mbed-sam-grove", "LinkedList"
)
config = ProjectConfig(os.path.join(str(project_dir), "platformio.ini"))
assert config.get("env:one", "lib_deps") == [
"bblanchon/ArduinoJson@^%s" % aj_pkg_data["version"]["name"],
"mbed-sam-grove/LinkedList@%s" % ll_pkg_data["version"]["name"],
]
def test_global_install_registry(clirunner, validate_cliresult, isolated_pio_core):
# check external package via Git repo
result = clirunner.invoke(
cmd_lib,
[
"-g",
"-d",
str(project_dir),
"-e",
"one",
"install",
"64",
"ArduinoJson@~5.10.0",
"547@2.2.4",
"AsyncMqttClient@<=0.8.2",
"Adafruit PN532@1.2.0",
"https://github.com/OttoWinter/async-mqtt-client.git#v0.8.3 @ 0.8.3",
],
)
validate_cliresult(result)
config = ProjectConfig(os.path.join(str(project_dir), "platformio.ini"))
assert len(config.get("env:one", "lib_deps")) == 3
assert config.get("env:one", "lib_deps")[2] == (
"https://github.com/OttoWinter/async-mqtt-client.git#v0.8.3 @ 0.8.3"
)
# install unknown library
result = clirunner.invoke(cmd_lib, ["-g", "install", "Unknown"])
assert result.exit_code != 0
assert isinstance(result.exception, exception.LibNotFound)
items1 = [d.basename for d in isolated_pio_core.join("lib").listdir()]
items2 = [
"ArduinoJson",
"ArduinoJson@5.10.1",
"NeoPixelBus",
"AsyncMqttClient",
"ESPAsyncTCP",
"AsyncTCP",
"Adafruit PN532",
"Adafruit BusIO",
# test uninstalling
result = clirunner.invoke(
cmd_lib, ["-d", str(project_dir), "uninstall", "ArduinoJson"]
)
validate_cliresult(result)
config = ProjectConfig(os.path.join(str(project_dir), "platformio.ini"))
assert len(config.get("env:one", "lib_deps")) == 2
assert len(config.get("env:two", "lib_deps")) == 1
assert config.get("env:one", "lib_deps") == [
"mbed-sam-grove/LinkedList@%s" % ll_pkg_data["version"]["name"],
"https://github.com/OttoWinter/async-mqtt-client.git#v0.8.3 @ 0.8.3",
]
assert set(items1) == set(items2)
# test list
result = clirunner.invoke(cmd_lib, ["-d", str(project_dir), "list"])
validate_cliresult(result)
assert "Version: 0.8.3+sha." in result.stdout
assert (
"Source: git+https://github.com/OttoWinter/async-mqtt-client.git#v0.8.3"
in result.stdout
)
result = clirunner.invoke(
cmd_lib, ["-d", str(project_dir), "list", "--json-output"]
)
validate_cliresult(result)
data = {}
for key, value in json.loads(result.stdout).items():
data[os.path.basename(key)] = value
ame_lib = next(
item for item in data["one"] if item["name"] == "AsyncMqttClient-esphome"
)
ame_vcs = VCSClientFactory.new(ame_lib["__pkg_dir"], ame_lib["__src_url"])
assert data["two"] == []
assert "__pkg_dir" in data["one"][0]
assert (
ame_lib["__src_url"]
== "git+https://github.com/OttoWinter/async-mqtt-client.git#v0.8.3"
)
assert ame_lib["version"] == ("0.8.3+sha.%s" % ame_vcs.get_current_revision())
def test_global_install_archive(clirunner, validate_cliresult, isolated_pio_core):
def test_update(clirunner, validate_cliresult, isolated_pio_core, tmpdir_factory):
storage_dir = tmpdir_factory.mktemp("test-updates")
result = clirunner.invoke(
cmd_lib,
["-d", str(storage_dir), "install", "ArduinoJson @ 5.10.1", "Blynk @ ~0.5.0"],
)
validate_cliresult(result)
result = clirunner.invoke(
cmd_lib, ["-d", str(storage_dir), "update", "--dry-run", "--json-output"]
)
validate_cliresult(result)
outdated = json.loads(result.stdout)
assert len(outdated) == 2
# ArduinoJson
assert outdated[0]["version"] == "5.10.1"
assert outdated[0]["versionWanted"] is None
assert semantic_version.Version(
outdated[0]["versionLatest"]
) > semantic_version.Version("6.16.0")
# Blynk
assert outdated[1]["version"] == "0.5.4"
assert outdated[1]["versionWanted"] is None
assert semantic_version.Version(
outdated[1]["versionLatest"]
) > semantic_version.Version("0.6.0")
# check with spec
result = clirunner.invoke(
cmd_lib,
[
"-g",
"install",
"https://github.com/bblanchon/ArduinoJson/archive/v5.8.2.zip",
"https://github.com/bblanchon/ArduinoJson/archive/v5.8.2.zip@5.8.2",
"SomeLib=http://dl.platformio.org/libraries/archives/0/9540.tar.gz",
"https://github.com/Pedroalbuquerque/ESP32WebServer/archive/master.zip",
"-d",
str(storage_dir),
"update",
"--dry-run",
"--json-output",
"ArduinoJson @ ^5",
],
)
validate_cliresult(result)
# incorrect requirements
outdated = json.loads(result.stdout)
assert outdated[0]["version"] == "5.10.1"
assert outdated[0]["versionWanted"] == "5.13.4"
assert semantic_version.Version(
outdated[0]["versionLatest"]
) > semantic_version.Version("6.16.0")
# update with spec
result = clirunner.invoke(
cmd_lib,
[
"-g",
"install",
"https://github.com/bblanchon/ArduinoJson/archive/v5.8.2.zip@1.2.3",
],
cmd_lib, ["-d", str(storage_dir), "update", "--silent", "ArduinoJson @ ^5.10.1"]
)
assert result.exit_code != 0
items1 = [d.basename for d in isolated_pio_core.join("lib").listdir()]
items2 = ["ArduinoJson", "SomeLib_ID54", "OneWire", "ESP32WebServer"]
assert set(items1) >= set(items2)
def test_global_install_repository(clirunner, validate_cliresult, isolated_pio_core):
validate_cliresult(result)
result = clirunner.invoke(
cmd_lib,
[
"-g",
"install",
"https://github.com/gioblu/PJON.git#3.0",
"https://github.com/gioblu/PJON.git#6.2",
"https://github.com/bblanchon/ArduinoJson.git",
"https://gitlab.com/ivankravets/rs485-nodeproto.git",
"https://github.com/platformio/platformio-libmirror.git",
# "https://developer.mbed.org/users/simon/code/TextLCD/",
"knolleary/pubsubclient#bef58148582f956dfa772687db80c44e2279a163",
],
cmd_lib, ["-d", str(storage_dir), "list", "--json-output"]
)
validate_cliresult(result)
items1 = [d.basename for d in isolated_pio_core.join("lib").listdir()]
items2 = [
"PJON",
"PJON@src-79de467ebe19de18287becff0a1fb42d",
"ArduinoJson@src-69ebddd821f771debe7ee734d3c7fa81",
"rs485-nodeproto",
"platformio-libmirror",
"PubSubClient",
]
assert set(items1) >= set(items2)
items = json.loads(result.stdout)
assert len(items) == 2
assert items[0]["version"] == "5.13.4"
assert items[1]["version"] == "0.5.4"
def test_install_duplicates(clirunner, validate_cliresult, without_internet):
# registry
# Check incompatible
result = clirunner.invoke(
cmd_lib,
["-g", "install", "http://dl.platformio.org/libraries/archives/0/9540.tar.gz"],
cmd_lib, ["-d", str(storage_dir), "update", "--dry-run", "ArduinoJson @ ^5"]
)
validate_cliresult(result)
assert "is already installed" in result.output
# archive
result = clirunner.invoke(
cmd_lib,
[
"-g",
"install",
"https://github.com/Pedroalbuquerque/ESP32WebServer/archive/master.zip",
],
)
validate_cliresult(result)
assert "is already installed" in result.output
# repository
result = clirunner.invoke(
cmd_lib,
["-g", "install", "https://github.com/platformio/platformio-libmirror.git"],
)
validate_cliresult(result)
assert "is already installed" in result.output
def test_global_lib_list(clirunner, validate_cliresult):
result = clirunner.invoke(cmd_lib, ["-g", "list"])
validate_cliresult(result)
assert all(
[
n in result.output
for n in (
"Source: https://github.com/Pedroalbuquerque/ESP32WebServer/archive/master.zip",
"Version: 5.10.1",
"Source: git+https://github.com/gioblu/PJON.git#3.0",
"Version: 1fb26fd",
)
]
)
result = clirunner.invoke(cmd_lib, ["-g", "list", "--json-output"])
assert all(
[
n in result.output
for n in (
"__pkg_dir",
'"__src_url": "git+https://gitlab.com/ivankravets/rs485-nodeproto.git"',
'"version": "5.10.1"',
)
]
)
items1 = [i["name"] for i in json.loads(result.output)]
items2 = [
"ESP32WebServer",
"ArduinoJson",
"ArduinoJson",
"ArduinoJson",
"ArduinoJson",
"AsyncMqttClient",
"AsyncTCP",
"SomeLib",
"ESPAsyncTCP",
"NeoPixelBus",
"OneWire",
"PJON",
"PJON",
"PubSubClient",
"Adafruit PN532",
"Adafruit BusIO",
"platformio-libmirror",
"rs485-nodeproto",
]
assert sorted(items1) == sorted(items2)
versions1 = [
"{name}@{version}".format(**item) for item in json.loads(result.output)
]
versions2 = [
"ArduinoJson@5.8.2",
"ArduinoJson@5.10.1",
"AsyncMqttClient@0.8.2",
"NeoPixelBus@2.2.4",
"PJON@07fe9aa",
"PJON@1fb26fd",
"PubSubClient@bef5814",
"Adafruit PN532@1.2.0",
]
assert set(versions1) >= set(versions2)
def test_global_lib_update_check(clirunner, validate_cliresult):
result = clirunner.invoke(
cmd_lib, ["-g", "update", "--only-check", "--json-output"]
)
validate_cliresult(result)
output = json.loads(result.output)
assert set(["ESPAsyncTCP", "NeoPixelBus"]) == set([l["name"] for l in output])
def test_global_lib_update(clirunner, validate_cliresult):
# update library using package directory
result = clirunner.invoke(
cmd_lib, ["-g", "update", "NeoPixelBus", "--only-check", "--json-output"]
)
validate_cliresult(result)
oudated = json.loads(result.output)
assert len(oudated) == 1
assert "__pkg_dir" in oudated[0]
result = clirunner.invoke(cmd_lib, ["-g", "update", oudated[0]["__pkg_dir"]])
validate_cliresult(result)
assert "Uninstalling NeoPixelBus @ 2.2.4" in result.output
# update rest libraries
result = clirunner.invoke(cmd_lib, ["-g", "update"])
validate_cliresult(result)
assert result.output.count("[Detached]") == 5
assert result.output.count("[Up-to-date]") == 12
# update unknown library
result = clirunner.invoke(cmd_lib, ["-g", "update", "Unknown"])
assert result.exit_code != 0
assert isinstance(result.exception, exception.UnknownPackage)
def test_global_lib_uninstall(clirunner, validate_cliresult, isolated_pio_core):
# uninstall using package directory
result = clirunner.invoke(cmd_lib, ["-g", "list", "--json-output"])
validate_cliresult(result)
items = json.loads(result.output)
items = sorted(items, key=lambda item: item["__pkg_dir"])
result = clirunner.invoke(cmd_lib, ["-g", "uninstall", items[0]["__pkg_dir"]])
validate_cliresult(result)
assert ("Uninstalling %s" % items[0]["name"]) in result.output
# uninstall the rest libraries
result = clirunner.invoke(
cmd_lib,
[
"-g",
"uninstall",
"OneWire",
"https://github.com/bblanchon/ArduinoJson.git",
"ArduinoJson@!=5.6.7",
"Adafruit PN532",
],
)
validate_cliresult(result)
items1 = [d.basename for d in isolated_pio_core.join("lib").listdir()]
items2 = [
"rs485-nodeproto",
"platformio-libmirror",
"PubSubClient",
"ArduinoJson@src-69ebddd821f771debe7ee734d3c7fa81",
"ESPAsyncTCP",
"ESP32WebServer",
"NeoPixelBus",
"PJON",
"AsyncMqttClient",
"ArduinoJson",
"SomeLib_ID54",
"PJON@src-79de467ebe19de18287becff0a1fb42d",
"AsyncTCP",
]
assert set(items1) == set(items2)
# uninstall unknown library
result = clirunner.invoke(cmd_lib, ["-g", "uninstall", "Unknown"])
assert result.exit_code != 0
assert isinstance(result.exception, exception.UnknownPackage)
def test_lib_show(clirunner, validate_cliresult):
result = clirunner.invoke(cmd_lib, ["show", "64"])
validate_cliresult(result)
assert all([s in result.output for s in ("ArduinoJson", "Arduino", "Atmel AVR")])
result = clirunner.invoke(cmd_lib, ["show", "OneWire", "--json-output"])
validate_cliresult(result)
assert "OneWire" in result.output
def test_lib_builtin(clirunner, validate_cliresult):
result = clirunner.invoke(cmd_lib, ["builtin"])
validate_cliresult(result)
result = clirunner.invoke(cmd_lib, ["builtin", "--json-output"])
validate_cliresult(result)
def test_lib_stats(clirunner, validate_cliresult):
result = clirunner.invoke(cmd_lib, ["stats"])
validate_cliresult(result)
assert all(
[
s in result.output
for s in ("UPDATED", "POPULAR", "https://platformio.org/lib/show")
]
)
result = clirunner.invoke(cmd_lib, ["stats", "--json-output"])
validate_cliresult(result)
assert set(
[
"dlweek",
"added",
"updated",
"topkeywords",
"dlmonth",
"dlday",
"lastkeywords",
]
) == set(json.loads(result.output).keys())
assert "Incompatible" in result.stdout

348
tests/commands/test_lib_complex.py Normal file
View File

@ -0,0 +1,348 @@
# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
import re
from platformio import exception
from platformio.commands import PlatformioCLI
from platformio.commands.lib.command import cli as cmd_lib
from platformio.package.exception import UnknownPackageError
PlatformioCLI.leftover_args = ["--json-output"] # hook for click
def test_search(clirunner, validate_cliresult):
result = clirunner.invoke(cmd_lib, ["search", "DHT22"])
validate_cliresult(result)
match = re.search(r"Found\s+(\d+)\slibraries:", result.output)
assert int(match.group(1)) > 2
result = clirunner.invoke(cmd_lib, ["search", "DHT22", "--platform=timsp430"])
validate_cliresult(result)
match = re.search(r"Found\s+(\d+)\slibraries:", result.output)
assert int(match.group(1)) > 1
def test_global_install_registry(clirunner, validate_cliresult, isolated_pio_core):
result = clirunner.invoke(
cmd_lib,
[
"-g",
"install",
"64",
"ArduinoJson@~5.10.0",
"547@2.2.4",
"AsyncMqttClient@<=0.8.2",
"Adafruit PN532@1.2.0",
],
)
validate_cliresult(result)
# install unknown library
result = clirunner.invoke(cmd_lib, ["-g", "install", "Unknown"])
assert result.exit_code != 0
assert isinstance(result.exception, UnknownPackageError)
items1 = [d.basename for d in isolated_pio_core.join("lib").listdir()]
items2 = [
"ArduinoJson",
"ArduinoJson@5.10.1",
"NeoPixelBus",
"AsyncMqttClient",
"ESPAsyncTCP",
"AsyncTCP",
"Adafruit PN532",
"Adafruit BusIO",
]
assert set(items1) == set(items2)
def test_global_install_archive(clirunner, validate_cliresult, isolated_pio_core):
result = clirunner.invoke(
cmd_lib,
[
"-g",
"install",
"https://github.com/bblanchon/ArduinoJson/archive/v5.8.2.zip",
"https://github.com/bblanchon/ArduinoJson/archive/v5.8.2.zip@5.8.2",
"SomeLib=https://dl.registry.platformio.org/download/milesburton/library/DallasTemperature/3.8.1/DallasTemperature-3.8.1.tar.gz",
"https://github.com/Pedroalbuquerque/ESP32WebServer/archive/master.zip",
],
)
validate_cliresult(result)
# incorrect requirements
result = clirunner.invoke(
cmd_lib,
[
"-g",
"install",
"https://github.com/bblanchon/ArduinoJson/archive/v5.8.2.zip@1.2.3",
],
)
assert result.exit_code != 0
items1 = [d.basename for d in isolated_pio_core.join("lib").listdir()]
items2 = [
"ArduinoJson",
"SomeLib",
"OneWire",
"ESP32WebServer@src-a1a3c75631882b35702e71966ea694e8",
]
assert set(items1) >= set(items2)
def test_global_install_repository(clirunner, validate_cliresult, isolated_pio_core):
result = clirunner.invoke(
cmd_lib,
[
"-g",
"install",
"https://github.com/gioblu/PJON.git#3.0",
"https://github.com/gioblu/PJON.git#6.2",
"https://github.com/bblanchon/ArduinoJson.git",
"https://github.com/platformio/platformio-libmirror.git",
# "https://developer.mbed.org/users/simon/code/TextLCD/",
"https://github.com/knolleary/pubsubclient#bef58148582f956dfa772687db80c44e2279a163",
],
)
validate_cliresult(result)
items1 = [d.basename for d in isolated_pio_core.join("lib").listdir()]
items2 = [
"PJON@src-1204e8bbd80de05e54e171b3a07bcc3f",
"PJON@src-79de467ebe19de18287becff0a1fb42d",
"ArduinoJson@src-69ebddd821f771debe7ee734d3c7fa81",
"platformio-libmirror@src-b7e674cad84244c61b436fcea8f78377",
"PubSubClient@src-98ec699a461a31615982e5adaaefadda",
]
assert set(items1) >= set(items2)
def test_install_duplicates(clirunner, validate_cliresult, without_internet):
# registry
result = clirunner.invoke(
cmd_lib,
[
"-g",
"install",
"https://dl.registry.platformio.org/download/milesburton/library/DallasTemperature/3.8.1/DallasTemperature-3.8.1.tar.gz",
],
)
validate_cliresult(result)
assert "is already installed" in result.output
# archive
result = clirunner.invoke(
cmd_lib,
[
"-g",
"install",
"https://github.com/Pedroalbuquerque/ESP32WebServer/archive/master.zip",
],
)
validate_cliresult(result)
assert "is already installed" in result.output
# repository
result = clirunner.invoke(
cmd_lib,
["-g", "install", "https://github.com/platformio/platformio-libmirror.git"],
)
validate_cliresult(result)
assert "is already installed" in result.output
def test_global_lib_list(clirunner, validate_cliresult):
result = clirunner.invoke(cmd_lib, ["-g", "list"])
validate_cliresult(result)
assert all(
[
n in result.output
for n in (
"Source: https://github.com/Pedroalbuquerque/ESP32WebServer/archive/master.zip",
"Version: 5.10.1",
"Source: git+https://github.com/gioblu/PJON.git#3.0",
"Version: 3.0.0+sha.1fb26fd",
)
]
)
result = clirunner.invoke(cmd_lib, ["-g", "list", "--json-output"])
assert all(
[
n in result.output
for n in (
"__pkg_dir",
'"__src_url": "git+https://github.com/gioblu/PJON.git#6.2"',
'"version": "5.10.1"',
)
]
)
items1 = [i["name"] for i in json.loads(result.output)]
items2 = [
"Adafruit BusIO",
"Adafruit PN532",
"ArduinoJson",
"ArduinoJson",
"ArduinoJson",
"ArduinoJson",
"AsyncMqttClient",
"AsyncTCP",
"DallasTemperature",
"ESP32WebServer",
"ESPAsyncTCP",
"NeoPixelBus",
"OneWire",
"PJON",
"PJON",
"platformio-libmirror",
"PubSubClient",
]
assert sorted(items1) == sorted(items2)
versions1 = [
"{name}@{version}".format(**item) for item in json.loads(result.output)
]
versions2 = [
"ArduinoJson@5.8.2",
"ArduinoJson@5.10.1",
"AsyncMqttClient@0.8.2",
"NeoPixelBus@2.2.4",
"PJON@6.2.0+sha.07fe9aa",
"PJON@3.0.0+sha.1fb26fd",
"PubSubClient@2.6.0+sha.bef5814",
"Adafruit PN532@1.2.0",
]
assert set(versions1) >= set(versions2)
def test_global_lib_update_check(clirunner, validate_cliresult):
result = clirunner.invoke(cmd_lib, ["-g", "update", "--dry-run", "--json-output"])
validate_cliresult(result)
output = json.loads(result.output)
assert set(["ESPAsyncTCP", "NeoPixelBus"]) == set([lib["name"] for lib in output])
def test_global_lib_update(clirunner, validate_cliresult):
# update library using package directory
result = clirunner.invoke(
cmd_lib, ["-g", "update", "NeoPixelBus", "--dry-run", "--json-output"]
)
validate_cliresult(result)
oudated = json.loads(result.output)
assert len(oudated) == 1
assert "__pkg_dir" in oudated[0]
result = clirunner.invoke(cmd_lib, ["-g", "update", oudated[0]["__pkg_dir"]])
validate_cliresult(result)
assert "Removing NeoPixelBus @ 2.2.4" in result.output
# update rest libraries
result = clirunner.invoke(cmd_lib, ["-g", "update"])
validate_cliresult(result)
assert result.output.count("[Detached]") == 1
assert result.output.count("[Up-to-date]") == 15
# update unknown library
result = clirunner.invoke(cmd_lib, ["-g", "update", "Unknown"])
assert result.exit_code != 0
assert isinstance(result.exception, UnknownPackageError)
def test_global_lib_uninstall(clirunner, validate_cliresult, isolated_pio_core):
# uninstall using package directory
result = clirunner.invoke(cmd_lib, ["-g", "list", "--json-output"])
validate_cliresult(result)
items = json.loads(result.output)
items = sorted(items, key=lambda item: item["__pkg_dir"])
result = clirunner.invoke(cmd_lib, ["-g", "uninstall", items[0]["__pkg_dir"]])
validate_cliresult(result)
assert ("Removing %s" % items[0]["name"]) in result.output
# uninstall the rest libraries
result = clirunner.invoke(
cmd_lib,
[
"-g",
"uninstall",
"OneWire",
"https://github.com/bblanchon/ArduinoJson.git",
"ArduinoJson@!=5.6.7",
"Adafruit PN532",
],
)
validate_cliresult(result)
items1 = [d.basename for d in isolated_pio_core.join("lib").listdir()]
items2 = [
"ArduinoJson",
"ArduinoJson@src-69ebddd821f771debe7ee734d3c7fa81",
"AsyncMqttClient",
"AsyncTCP",
"ESP32WebServer@src-a1a3c75631882b35702e71966ea694e8",
"ESPAsyncTCP",
"NeoPixelBus",
"PJON@src-1204e8bbd80de05e54e171b3a07bcc3f",
"PJON@src-79de467ebe19de18287becff0a1fb42d",
"platformio-libmirror@src-b7e674cad84244c61b436fcea8f78377",
"PubSubClient@src-98ec699a461a31615982e5adaaefadda",
"SomeLib",
]
assert set(items1) == set(items2)
# uninstall unknown library
result = clirunner.invoke(cmd_lib, ["-g", "uninstall", "Unknown"])
assert result.exit_code != 0
assert isinstance(result.exception, UnknownPackageError)
def test_lib_show(clirunner, validate_cliresult):
result = clirunner.invoke(cmd_lib, ["show", "64"])
validate_cliresult(result)
assert all([s in result.output for s in ("ArduinoJson", "Arduino", "Atmel AVR")])
result = clirunner.invoke(cmd_lib, ["show", "OneWire", "--json-output"])
validate_cliresult(result)
assert "OneWire" in result.output
def test_lib_builtin(clirunner, validate_cliresult):
result = clirunner.invoke(cmd_lib, ["builtin"])
validate_cliresult(result)
result = clirunner.invoke(cmd_lib, ["builtin", "--json-output"])
validate_cliresult(result)
def test_lib_stats(clirunner, validate_cliresult):
result = clirunner.invoke(cmd_lib, ["stats"])
validate_cliresult(result)
assert all(
[
s in result.output
for s in ("UPDATED", "POPULAR", "https://platformio.org/lib/show")
]
)
result = clirunner.invoke(cmd_lib, ["stats", "--json-output"])
validate_cliresult(result)
assert set(
[
"dlweek",
"added",
"updated",
"topkeywords",
"dlmonth",
"dlday",
"lastkeywords",
]
) == set(json.loads(result.output).keys())

85
tests/package/test_manager.py
View File

@ -16,6 +16,7 @@ import os
import time
import pytest
import semantic_version
from platformio import fs, util
from platformio.package.exception import (
@ -201,6 +202,12 @@ def test_install_from_registry(isolated_pio_core, tmpdir_factory):
assert lm.get_package("OneWire").metadata.version.major >= 2
assert len(lm.get_installed()) == 6
# test conflicted names
lm = LibraryPackageManager(str(tmpdir_factory.mktemp("conflicted-storage")))
lm.install("4@2.6.1", silent=True)
lm.install("5357@2.6.1", silent=True)
assert len(lm.get_installed()) == 2
# Tools
tm = ToolPackageManager(str(tmpdir_factory.mktemp("tool-storage")))
pkg = tm.install("platformio/tool-stlink @ ~1.10400.0", silent=True)
@ -340,3 +347,81 @@ def test_uninstall(isolated_pio_core, tmpdir_factory):
assert lm.install("AsyncMqttClient-esphome @ 0.8.4", silent=True)
assert lm.uninstall("AsyncMqttClient-esphome", silent=True)
assert len(lm.get_installed()) == 0
def test_registry(isolated_pio_core):
lm = LibraryPackageManager()
# reveal ID
assert lm.reveal_registry_package_id(PackageSpec(id=13)) == 13
assert lm.reveal_registry_package_id(PackageSpec(name="OneWire"), silent=True) == 1
with pytest.raises(UnknownPackageError):
lm.reveal_registry_package_id(PackageSpec(name="/non-existing-package/"))
# fetch package data
assert lm.fetch_registry_package(PackageSpec(id=1))["name"] == "OneWire"
assert lm.fetch_registry_package(PackageSpec(name="ArduinoJson"))["id"] == 64
assert (
lm.fetch_registry_package(
PackageSpec(id=13, owner="adafruit", name="Renamed library")
)["name"]
== "Adafruit GFX Library"
)
with pytest.raises(UnknownPackageError):
lm.fetch_registry_package(
PackageSpec(owner="unknown<>owner", name="/non-existing-package/")
)
with pytest.raises(UnknownPackageError):
lm.fetch_registry_package(PackageSpec(name="/non-existing-package/"))
def test_update_with_metadata(isolated_pio_core, tmpdir_factory):
storage_dir = tmpdir_factory.mktemp("storage")
lm = LibraryPackageManager(str(storage_dir))
pkg = lm.install("ArduinoJson @ 5.10.1", silent=True)
# tesy latest
outdated = lm.outdated(pkg)
assert str(outdated.current) == "5.10.1"
assert outdated.wanted is None
assert outdated.latest > outdated.current
assert outdated.latest > semantic_version.Version("5.99.99")
# test wanted
outdated = lm.outdated(pkg, PackageSpec("ArduinoJson@~5"))
assert str(outdated.current) == "5.10.1"
assert str(outdated.wanted) == "5.13.4"
assert outdated.latest > semantic_version.Version("6.16.0")
# update to the wanted 5.x
new_pkg = lm.update("ArduinoJson@^5", PackageSpec("ArduinoJson@^5"), silent=True)
assert str(new_pkg.metadata.version) == "5.13.4"
# check that old version is removed
assert len(lm.get_installed()) == 1
# update to the latest
lm = LibraryPackageManager(str(storage_dir))
pkg = lm.update("ArduinoJson", silent=True)
assert pkg.metadata.version == outdated.latest
def test_update_without_metadata(isolated_pio_core, tmpdir_factory):
storage_dir = tmpdir_factory.mktemp("storage")
storage_dir.join("legacy-package").mkdir().join("library.json").write(
'{"name": "AsyncMqttClient-esphome", "version": "0.8.2"}'
)
storage_dir.join("legacy-dep").mkdir().join("library.json").write(
'{"name": "AsyncTCP-esphome", "version": "1.1.1"}'
)
lm = LibraryPackageManager(str(storage_dir))
pkg = lm.get_package("AsyncMqttClient-esphome")
outdated = lm.outdated(pkg)
assert len(lm.get_installed()) == 2
assert str(pkg.metadata.version) == "0.8.2"
assert outdated.latest > semantic_version.Version("0.8.2")
# update
lm = LibraryPackageManager(str(storage_dir))
new_pkg = lm.update(pkg, silent=True)
assert len(lm.get_installed()) == 3
assert new_pkg.metadata.spec.owner == "ottowinter"

50
tests/package/test_meta.py
View File

@ -17,7 +17,27 @@ import os
import jsondiff
import semantic_version
from platformio.package.meta import PackageMetaData, PackageSpec, PackageType
from platformio.package.meta import (
PackageMetaData,
PackageOutdatedResult,
PackageSpec,
PackageType,
)
def test_outdated_result():
result = PackageOutdatedResult(current="1.2.3", latest="2.0.0")
assert result.is_outdated()
assert result.is_outdated(allow_incompatible=True)
result = PackageOutdatedResult(current="1.2.3", latest="2.0.0", wanted="1.5.4")
assert result.is_outdated()
assert result.is_outdated(allow_incompatible=True)
result = PackageOutdatedResult(current="1.2.3", latest="2.0.0", wanted="1.2.3")
assert not result.is_outdated()
assert result.is_outdated(allow_incompatible=True)
result = PackageOutdatedResult(current="1.2.3", latest="2.0.0", detached=True)
assert not result.is_outdated()
assert not result.is_outdated(allow_incompatible=True)
def test_spec_owner():
@ -45,9 +65,16 @@ def test_spec_name():
def test_spec_requirements():
assert PackageSpec("foo@1.2.3") == PackageSpec(name="foo", requirements="1.2.3")
assert PackageSpec(
name="foo", requirements=semantic_version.Version("1.2.3")
) == PackageSpec(name="foo", requirements="1.2.3")
assert PackageSpec("bar @ ^1.2.3") == PackageSpec(name="bar", requirements="^1.2.3")
assert PackageSpec("13 @ ~2.0") == PackageSpec(id=13, requirements="~2.0")
assert PackageSpec(
name="hello", requirements=semantic_version.SimpleSpec("~1.2.3")
) == PackageSpec(name="hello", requirements="~1.2.3")
spec = PackageSpec("id=20 @ !=1.2.3,<2.0")
assert not spec.external
assert isinstance(spec.requirements, semantic_version.SimpleSpec)
assert semantic_version.Version("1.3.0-beta.1") in spec.requirements
assert spec == PackageSpec(id=20, requirements="!=1.2.3,<2.0")
@ -88,7 +115,8 @@ def test_spec_external_urls():
"Custom-Name="
"https://github.com/platformio/platformio-core/archive/develop.tar.gz@4.4.0"
)
assert spec.is_custom_name()
assert spec.external
assert spec.has_custom_name()
assert spec.name == "Custom-Name"
assert spec == PackageSpec(
url="https://github.com/platformio/platformio-core/archive/develop.tar.gz",
@ -163,6 +191,24 @@ def test_spec_as_dict():
)
def test_spec_as_dependency():
assert PackageSpec("owner/pkgname").as_dependency() == "owner/pkgname"
assert PackageSpec(owner="owner", name="pkgname").as_dependency() == "owner/pkgname"
assert PackageSpec("bob/foo @ ^1.2.3").as_dependency() == "bob/foo@^1.2.3"
assert (
PackageSpec(
"https://github.com/o/r/a/develop.zip?param=value @ !=2"
).as_dependency()
== "https://github.com/o/r/a/develop.zip?param=value @ !=2"
)
assert (
PackageSpec(
"wolfSSL=https://os.mbed.com/users/wolfSSL/code/wolfSSL/"
).as_dependency()
== "wolfSSL=https://os.mbed.com/users/wolfSSL/code/wolfSSL/"
)
def test_metadata_as_dict():
metadata = PackageMetaData(PackageType.LIBRARY, "foo", "1.2.3")
# test setter

4
tests/test_maintenance.py
View File

@ -88,7 +88,9 @@ def test_check_and_update_libraries(clirunner, isolated_pio_core, validate_clire
validate_cliresult(result)
assert "There are the new updates for libraries (ArduinoJson)" in result.output
assert "Please wait while updating libraries" in result.output
assert re.search(r"Updating ArduinoJson\s+@ 6.12.0\s+\[[\d\.]+\]", result.output)
assert re.search(
r"Updating bblanchon/ArduinoJson\s+6\.12\.0\s+\[[\d\.]+\]", result.output
)
# check updated version
result = clirunner.invoke(cli_pio, ["lib", "-g", "list", "--json-output"])