diff --git a/platformio/__init__.py b/platformio/__init__.py index 8da7e6e0..0a1bc48c 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -40,3 +40,13 @@ __apiurl__ = "https://api.platformio.org" __accounts_api__ = "https://api.accounts.platformio.org" __registry_api__ = "https://api.registry.platformio.org" __pioremote_endpoint__ = "ssl:host=remote.platformio.org:port=4413" + +__core_packages__ = { + "contrib-piohome": "~3.2.3", + "contrib-pysite": "~2.%d%d.0" % (sys.version_info.major, sys.version_info.minor), + "tool-unity": "~1.20500.0", + "tool-scons": "~2.20501.7" if sys.version_info.major == 2 else "~3.30102.0", + "tool-cppcheck": "~1.190.0", + "tool-clangtidy": "~1.100000.0", + "tool-pvs-studio": "~7.7.0", +} diff --git a/platformio/builder/tools/pioide.py b/platformio/builder/tools/pioide.py index acb36ae4..6a3d343d 100644 --- a/platformio/builder/tools/pioide.py +++ b/platformio/builder/tools/pioide.py @@ -20,7 +20,7 @@ from glob import glob from SCons.Defaults import processDefines # pylint: disable=import-error from platformio.compat import glob_escape -from platformio.managers.core import get_core_package_dir +from platformio.package.manager.core import get_core_package_dir from platformio.proc import exec_command, where_is_program diff --git a/platformio/builder/tools/piomisc.py b/platformio/builder/tools/piomisc.py index aa5158fe..799b192f 100644 --- a/platformio/builder/tools/piomisc.py +++ b/platformio/builder/tools/piomisc.py @@ -25,7 +25,7 @@ import click from platformio import fs, util from platformio.compat import get_filesystem_encoding, get_locale_encoding, glob_escape -from platformio.managers.core import get_core_package_dir +from platformio.package.manager.core import get_core_package_dir from platformio.proc import exec_command diff --git a/platformio/commands/check/tools/clangtidy.py b/platformio/commands/check/tools/clangtidy.py index f1610452..05be67b4 100644 --- a/platformio/commands/check/tools/clangtidy.py +++ b/platformio/commands/check/tools/clangtidy.py @@ -17,7 +17,7 @@ from os.path import join from platformio.commands.check.defect import DefectItem from platformio.commands.check.tools.base import CheckToolBase -from platformio.managers.core import get_core_package_dir +from platformio.package.manager.core import get_core_package_dir class ClangtidyCheckTool(CheckToolBase): diff --git a/platformio/commands/check/tools/cppcheck.py b/platformio/commands/check/tools/cppcheck.py index 34129714..931b16ed 100644 --- a/platformio/commands/check/tools/cppcheck.py +++ b/platformio/commands/check/tools/cppcheck.py @@ -19,7 +19,7 @@ import click from platformio import proc from platformio.commands.check.defect import DefectItem from platformio.commands.check.tools.base import CheckToolBase -from platformio.managers.core import get_core_package_dir +from platformio.package.manager.core import get_core_package_dir class CppcheckCheckTool(CheckToolBase): diff --git a/platformio/commands/check/tools/pvsstudio.py b/platformio/commands/check/tools/pvsstudio.py index 871ec4bc..ce5d93ec 100644 --- a/platformio/commands/check/tools/pvsstudio.py +++ b/platformio/commands/check/tools/pvsstudio.py @@ -22,7 +22,7 @@ import click from platformio import proc, util from platformio.commands.check.defect import DefectItem from platformio.commands.check.tools.base import CheckToolBase -from platformio.managers.core import get_core_package_dir +from platformio.package.manager.core import get_core_package_dir class PvsStudioCheckTool(CheckToolBase): # pylint: disable=too-many-instance-attributes diff --git a/platformio/commands/debug/command.py b/platformio/commands/debug/command.py index 25286111..78a43eef 100644 --- a/platformio/commands/debug/command.py +++ b/platformio/commands/debug/command.py @@ -24,7 +24,7 @@ import click from platformio import app, exception, fs, proc, util from platformio.commands.debug import helpers from platformio.commands.debug.exception import DebugInvalidOptionsError -from platformio.managers.core import inject_contrib_pysite +from platformio.package.manager.core import inject_contrib_pysite from platformio.project.config import ProjectConfig from platformio.project.exception import ProjectEnvsNotAvailableError from platformio.project.helpers import is_platformio_project, load_project_ide_data diff --git a/platformio/commands/home/command.py b/platformio/commands/home/command.py index 32d28063..dd733bb6 100644 --- a/platformio/commands/home/command.py +++ b/platformio/commands/home/command.py @@ -22,7 +22,7 @@ import click from platformio import exception from platformio.compat import WINDOWS -from platformio.managers.core import get_core_package_dir, inject_contrib_pysite +from platformio.package.manager.core import get_core_package_dir, inject_contrib_pysite @click.command("home", short_help="PIO Home") diff --git a/platformio/commands/remote/command.py b/platformio/commands/remote/command.py index f9e24c29..66c10690 100644 --- a/platformio/commands/remote/command.py +++ b/platformio/commands/remote/command.py @@ -29,7 +29,7 @@ from platformio.commands.device.command import device_monitor as cmd_device_moni from platformio.commands.run.command import cli as cmd_run from platformio.commands.test.command import cli as cmd_test from platformio.compat import PY2 -from platformio.managers.core import inject_contrib_pysite +from platformio.package.manager.core import inject_contrib_pysite from platformio.project.exception import NotPlatformIOProjectError diff --git a/platformio/commands/update.py b/platformio/commands/update.py index bf829165..b1e15a43 100644 --- a/platformio/commands/update.py +++ b/platformio/commands/update.py @@ -18,7 +18,7 @@ from platformio import app from platformio.commands.lib.command import CTX_META_STORAGE_DIRS_KEY from platformio.commands.lib.command import lib_update as cmd_lib_update from platformio.commands.platform import platform_update as cmd_platform_update -from platformio.managers.core import update_core_packages +from platformio.package.manager.core import update_core_packages from platformio.package.manager.library import LibraryPackageManager diff --git a/platformio/maintenance.py b/platformio/maintenance.py index b0e64f52..cf2e0698 100644 --- a/platformio/maintenance.py +++ b/platformio/maintenance.py @@ -25,9 +25,11 @@ from platformio.commands.lib.command import CTX_META_STORAGE_DIRS_KEY from platformio.commands.lib.command import lib_update as cmd_lib_update from platformio.commands.platform import platform_update as cmd_platform_update from platformio.commands.upgrade import get_latest_version -from platformio.managers.core import update_core_packages from platformio.managers.platform import PlatformFactory, PlatformManager +from platformio.package.manager.core import update_core_packages from platformio.package.manager.library import LibraryPackageManager +from platformio.package.manager.tool import ToolPackageManager +from platformio.package.meta import PackageSpec from platformio.proc import is_container @@ -90,7 +92,8 @@ class Upgrader(object): ) self._upgraders = [ - (semantic_version.Version("3.5.0-a.2"), self._update_dev_platforms) + (semantic_version.Version("3.5.0-a.2"), self._update_dev_platforms), + (semantic_version.Version("4.4.0-a.8"), self._update_pkg_metadata), ] def run(self, ctx): @@ -110,6 +113,22 @@ class Upgrader(object): ctx.invoke(cmd_platform_update) return True + @staticmethod + def _update_pkg_metadata(_): + pm = ToolPackageManager() + for pkg in pm.get_installed(): + if not pkg.metadata or pkg.metadata.spec.external or pkg.metadata.spec.id: + continue + result = pm.search_registry_packages(PackageSpec(name=pkg.metadata.name)) + if len(result) != 1: + continue + result = result[0] + pkg.metadata.spec = PackageSpec( + id=result["id"], owner=result["owner"]["username"], name=result["name"], + ) + pkg.dump_meta() + return True + def after_upgrade(ctx): terminal_width, _ = click.get_terminal_size() @@ -160,7 +179,6 @@ def after_upgrade(ctx): ) else: raise exception.UpgradeError("Auto upgrading...") - click.echo("") # PlatformIO banner click.echo("*" * terminal_width) diff --git a/platformio/managers/platform.py b/platformio/managers/platform.py index ada4f4ac..8548bba6 100644 --- a/platformio/managers/platform.py +++ b/platformio/managers/platform.py @@ -30,8 +30,8 @@ from platformio.commands.debug.exception import ( DebugSupportError, ) from platformio.compat import PY2, hashlib_encode_data, is_bytes, load_python_module -from platformio.managers.core import get_core_package_dir from platformio.managers.package import BasePkgManager, PackageManager +from platformio.package.manager.core import get_core_package_dir from platformio.project.config import ProjectConfig try: diff --git a/platformio/package/manager/_registry.py b/platformio/package/manager/_registry.py index 0d1e45e1..7dc09964 100644 --- a/platformio/package/manager/_registry.py +++ b/platformio/package/manager/_registry.py @@ -119,6 +119,7 @@ class PackageManageRegistryMixin(object): return self._registry_client def search_registry_packages(self, spec, filters=None): + assert isinstance(spec, PackageSpec) filters = filters or {} if spec.id: filters["ids"] = str(spec.id) @@ -132,6 +133,7 @@ class PackageManageRegistryMixin(object): ] def fetch_registry_package(self, spec): + assert isinstance(spec, PackageSpec) result = None if spec.owner and spec.name: result = self.get_registry_client_instance().get_package( diff --git a/platformio/managers/core.py b/platformio/package/manager/core.py similarity index 57% rename from platformio/managers/core.py rename to platformio/package/manager/core.py index 27bee8c2..2b872ab6 100644 --- a/platformio/managers/core.py +++ b/platformio/package/manager/core.py @@ -17,89 +17,58 @@ import os import subprocess import sys -from platformio import exception, util +from platformio import __core_packages__, exception, util from platformio.compat import PY2 -from platformio.managers.package import PackageManager +from platformio.package.manager.tool import ToolPackageManager +from platformio.package.meta import PackageSpec from platformio.proc import get_pythonexe_path -from platformio.project.config import ProjectConfig - -CORE_PACKAGES = { - "contrib-piohome": "~3.2.3", - "contrib-pysite": "~2.%d%d.0" % (sys.version_info.major, sys.version_info.minor), - "tool-unity": "~1.20500.0", - "tool-scons": "~2.20501.7" if PY2 else "~3.30102.0", - "tool-cppcheck": "~1.190.0", - "tool-clangtidy": "~1.100000.0", - "tool-pvs-studio": "~7.7.0", -} - -# pylint: disable=arguments-differ,signature-differs - - -class CorePackageManager(PackageManager): - def __init__(self): - config = ProjectConfig.get_instance() - packages_dir = config.get_optional_dir("packages") - super(CorePackageManager, self).__init__( - packages_dir, - [ - "https://dl.bintray.com/platformio/dl-packages/manifest.json", - "http%s://dl.platformio.org/packages/manifest.json" - % ("" if sys.version_info < (2, 7, 9) else "s"), - ], - ) - - def install( # pylint: disable=keyword-arg-before-vararg - self, name, requirements=None, *args, **kwargs - ): - PackageManager.install(self, name, requirements, *args, **kwargs) - self.cleanup_packages() - return self.get_package_dir(name, requirements) - - def update(self, *args, **kwargs): - result = PackageManager.update(self, *args, **kwargs) - self.cleanup_packages() - return result - - def cleanup_packages(self): - self.cache_reset() - best_pkg_versions = {} - for name, requirements in CORE_PACKAGES.items(): - pkg_dir = self.get_package_dir(name, requirements) - if not pkg_dir: - continue - best_pkg_versions[name] = self.load_manifest(pkg_dir)["version"] - for manifest in self.get_installed(): - if manifest["name"] not in best_pkg_versions: - continue - if manifest["version"] != best_pkg_versions[manifest["name"]]: - self.uninstall(manifest["__pkg_dir"], after_update=True) - self.cache_reset() - return True def get_core_package_dir(name): - if name not in CORE_PACKAGES: + if name not in __core_packages__: raise exception.PlatformioException("Please upgrade PIO Core") - requirements = CORE_PACKAGES[name] - pm = CorePackageManager() - pkg_dir = pm.get_package_dir(name, requirements) - if pkg_dir: - return pkg_dir - return pm.install(name, requirements) + pm = ToolPackageManager() + spec = PackageSpec( + owner="platformio", name=name, requirements=__core_packages__[name] + ) + pkg = pm.get_package(spec) + if pkg: + return pkg.path + pkg = pm.install(spec).path + _remove_unnecessary_packages() + return pkg def update_core_packages(only_check=False, silent=False): - pm = CorePackageManager() - for name, requirements in CORE_PACKAGES.items(): - pkg_dir = pm.get_package_dir(name) - if not pkg_dir: + pm = ToolPackageManager() + for name, requirements in __core_packages__.items(): + spec = PackageSpec(owner="platformio", name=name, requirements=requirements) + pkg = pm.get_package(spec) + if not pkg: continue - if not silent or pm.outdated(pkg_dir, requirements): - pm.update(name, requirements, only_check=only_check) + if not silent or pm.outdated(pkg, spec).is_outdated(): + pm.update(pkg, spec, only_check=only_check) + if not only_check: + _remove_unnecessary_packages() return True +def _remove_unnecessary_packages(): + pm = ToolPackageManager() + best_pkg_versions = {} + for name, requirements in __core_packages__.items(): + spec = PackageSpec(owner="platformio", name=name, requirements=requirements) + pkg = pm.get_package(spec) + if not pkg: + continue + best_pkg_versions[pkg.metadata.name] = pkg.metadata.version + for pkg in pm.get_installed(): + if pkg.metadata.name not in best_pkg_versions: + continue + if pkg.metadata.version != best_pkg_versions[pkg.metadata.name]: + pm.uninstall(pkg) + + def inject_contrib_pysite(verify_openssl=False): # pylint: disable=import-outside-toplevel from site import addsitedir diff --git a/platformio/util.py b/platformio/util.py index 36254586..5b38909d 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -196,7 +196,7 @@ def get_mdns_services(): import zeroconf except ImportError: from site import addsitedir - from platformio.managers.core import get_core_package_dir + from platformio.package.manager.core import get_core_package_dir contrib_pysite_dir = get_core_package_dir("contrib-pysite") addsitedir(contrib_pysite_dir) diff --git a/tests/test_managers.py b/tests/test_managers.py deleted file mode 100644 index 308523cd..00000000 --- a/tests/test_managers.py +++ /dev/null @@ -1,234 +0,0 @@ -# Copyright (c) 2014-present PlatformIO -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import json -from os.path import join - -from platformio.managers.package import PackageManager -from platformio.project.helpers import get_project_core_dir - - -def test_pkg_input_parser(): - items = [ - ["PkgName", ("PkgName", None, None)], - [("PkgName", "!=1.2.3,<2.0"), ("PkgName", "!=1.2.3,<2.0", None)], - ["PkgName@1.2.3", ("PkgName", "1.2.3", None)], - [("PkgName@1.2.3", "1.2.5"), ("PkgName@1.2.3", "1.2.5", None)], - ["id=13", ("id=13", None, None)], - ["id=13@~1.2.3", ("id=13", "~1.2.3", None)], - [ - get_project_core_dir(), - (".platformio", None, "file://" + get_project_core_dir()), - ], - [ - "LocalName=" + get_project_core_dir(), - ("LocalName", None, "file://" + get_project_core_dir()), - ], - [ - "LocalName=%s@>2.3.0" % get_project_core_dir(), - ("LocalName", ">2.3.0", "file://" + get_project_core_dir()), - ], - [ - "https://github.com/user/package.git", - ("package", None, "git+https://github.com/user/package.git"), - ], - [ - "MyPackage=https://gitlab.com/user/package.git", - ("MyPackage", None, "git+https://gitlab.com/user/package.git"), - ], - [ - "MyPackage=https://gitlab.com/user/package.git@3.2.1,!=2", - ("MyPackage", "3.2.1,!=2", "git+https://gitlab.com/user/package.git"), - ], - [ - "https://somedomain.com/path/LibraryName-1.2.3.zip", - ( - "LibraryName-1.2.3", - None, - "https://somedomain.com/path/LibraryName-1.2.3.zip", - ), - ], - [ - "https://github.com/user/package/archive/branch.zip", - ("branch", None, "https://github.com/user/package/archive/branch.zip"), - ], - [ - "https://github.com/user/package/archive/branch.zip@~1.2.3", - ("branch", "~1.2.3", "https://github.com/user/package/archive/branch.zip"), - ], - [ - "https://github.com/user/package/archive/branch.tar.gz", - ( - "branch.tar", - None, - "https://github.com/user/package/archive/branch.tar.gz", - ), - ], - [ - "https://github.com/user/package/archive/branch.tar.gz@!=5", - ( - "branch.tar", - "!=5", - "https://github.com/user/package/archive/branch.tar.gz", - ), - ], - [ - "https://developer.mbed.org/users/user/code/package/", - ("package", None, "hg+https://developer.mbed.org/users/user/code/package/"), - ], - [ - "https://os.mbed.com/users/user/code/package/", - ("package", None, "hg+https://os.mbed.com/users/user/code/package/"), - ], - [ - "https://github.com/user/package#v1.2.3", - ("package", None, "git+https://github.com/user/package#v1.2.3"), - ], - [ - "https://github.com/user/package.git#branch", - ("package", None, "git+https://github.com/user/package.git#branch"), - ], - [ - "PkgName=https://github.com/user/package.git#a13d344fg56", - ("PkgName", None, "git+https://github.com/user/package.git#a13d344fg56"), - ], - ["user/package", ("package", None, "git+https://github.com/user/package")], - [ - "PkgName=user/package", - ("PkgName", None, "git+https://github.com/user/package"), - ], - [ - "PkgName=user/package#master", - ("PkgName", None, "git+https://github.com/user/package#master"), - ], - [ - "git+https://github.com/user/package", - ("package", None, "git+https://github.com/user/package"), - ], - [ - "hg+https://example.com/user/package", - ("package", None, "hg+https://example.com/user/package"), - ], - [ - "git@github.com:user/package.git", - ("package", None, "git+git@github.com:user/package.git"), - ], - [ - "git@github.com:user/package.git#v1.2.0", - ("package", None, "git+git@github.com:user/package.git#v1.2.0"), - ], - [ - "LocalName=git@github.com:user/package.git#v1.2.0@~1.2.0", - ("LocalName", "~1.2.0", "git+git@github.com:user/package.git#v1.2.0"), - ], - [ - "git+ssh://git@gitlab.private-server.com/user/package#1.2.0", - ( - "package", - None, - "git+ssh://git@gitlab.private-server.com/user/package#1.2.0", - ), - ], - [ - "git+ssh://user@gitlab.private-server.com:1234/package#1.2.0", - ( - "package", - None, - "git+ssh://user@gitlab.private-server.com:1234/package#1.2.0", - ), - ], - [ - "LocalName=git+ssh://user@gitlab.private-server.com:1234" - "/package#1.2.0@!=13", - ( - "LocalName", - "!=13", - "git+ssh://user@gitlab.private-server.com:1234/package#1.2.0", - ), - ], - ] - for params, result in items: - if isinstance(params, tuple): - assert PackageManager.parse_pkg_uri(*params) == result - else: - assert PackageManager.parse_pkg_uri(params) == result - - -def test_install_packages(isolated_pio_core, tmpdir): - packages = [ - dict(id=1, name="name_1", version="shasum"), - dict(id=1, name="name_1", version="2.0.0"), - dict(id=1, name="name_1", version="2.1.0"), - dict(id=1, name="name_1", version="1.2"), - dict(id=1, name="name_1", version="1.0.0"), - dict(name="name_2", version="1.0.0"), - dict(name="name_2", version="2.0.0", __src_url="git+https://github.com"), - dict(name="name_2", version="3.0.0", __src_url="git+https://github2.com"), - dict(name="name_2", version="4.0.0", __src_url="git+https://github2.com"), - ] - - pm = PackageManager(join(get_project_core_dir(), "packages")) - for package in packages: - tmp_dir = tmpdir.mkdir("tmp-package") - tmp_dir.join("package.json").write(json.dumps(package)) - pm._install_from_url(package["name"], "file://%s" % str(tmp_dir)) - tmp_dir.remove(rec=1) - - assert len(pm.get_installed()) == len(packages) - 1 - - pkg_dirnames = [ - "name_1_ID1", - "name_1_ID1@1.0.0", - "name_1_ID1@1.2", - "name_1_ID1@2.0.0", - "name_1_ID1@shasum", - "name_2", - "name_2@src-177cbce1f0705580d17790fda1cc2ef5", - "name_2@src-f863b537ab00f4c7b5011fc44b120e1f", - ] - assert set( - [p.basename for p in isolated_pio_core.join("packages").listdir()] - ) == set(pkg_dirnames) - - -def test_get_package(): - tests = [ - [("unknown",), None], - [("1",), None], - [("id=1", "shasum"), dict(id=1, name="name_1", version="shasum")], - [("id=1", "*"), dict(id=1, name="name_1", version="2.1.0")], - [("id=1", "^1"), dict(id=1, name="name_1", version="1.2")], - [("id=1", "^1"), dict(id=1, name="name_1", version="1.2")], - [("name_1", "<2"), dict(id=1, name="name_1", version="1.2")], - [("name_1", ">2"), None], - [("name_1", "2-0-0"), None], - [("name_2",), dict(name="name_2", version="4.0.0")], - [ - ("url_has_higher_priority", None, "git+https://github.com"), - dict(name="name_2", version="2.0.0", __src_url="git+https://github.com"), - ], - [ - ("name_2", None, "git+https://github.com"), - dict(name="name_2", version="2.0.0", __src_url="git+https://github.com"), - ], - ] - - pm = PackageManager(join(get_project_core_dir(), "packages")) - for test in tests: - manifest = pm.get_package(*test[0]) - if test[1] is None: - assert manifest is None, test - continue - for key, value in test[1].items(): - assert manifest[key] == value, test