Switch legacy core package manager to the new

platformio/__init__.py

@@ -40,3 +40,13 @@ __apiurl__ = "https://api.platformio.org"
 __accounts_api__ = "https://api.accounts.platformio.org"
 __registry_api__ = "https://api.registry.platformio.org"
 __pioremote_endpoint__ = "ssl:host=remote.platformio.org:port=4413"
+
+__core_packages__ = {
+    "contrib-piohome": "~3.2.3",
+    "contrib-pysite": "~2.%d%d.0" % (sys.version_info.major, sys.version_info.minor),
+    "tool-unity": "~1.20500.0",
+    "tool-scons": "~2.20501.7" if sys.version_info.major == 2 else "~3.30102.0",
+    "tool-cppcheck": "~1.190.0",
+    "tool-clangtidy": "~1.100000.0",
+    "tool-pvs-studio": "~7.7.0",
+}

platformio/builder/tools/pioide.py

@@ -20,7 +20,7 @@ from glob import glob
 from SCons.Defaults import processDefines  # pylint: disable=import-error
 
 from platformio.compat import glob_escape
-from platformio.managers.core import get_core_package_dir
+from platformio.package.manager.core import get_core_package_dir
 from platformio.proc import exec_command, where_is_program
 
 

platformio/builder/tools/piomisc.py

@@ -25,7 +25,7 @@ import click
 
 from platformio import fs, util
 from platformio.compat import get_filesystem_encoding, get_locale_encoding, glob_escape
-from platformio.managers.core import get_core_package_dir
+from platformio.package.manager.core import get_core_package_dir
 from platformio.proc import exec_command
 
 

platformio/commands/check/tools/clangtidy.py

@@ -17,7 +17,7 @@ from os.path import join
 
 from platformio.commands.check.defect import DefectItem
 from platformio.commands.check.tools.base import CheckToolBase
-from platformio.managers.core import get_core_package_dir
+from platformio.package.manager.core import get_core_package_dir
 
 
 class ClangtidyCheckTool(CheckToolBase):

platformio/commands/check/tools/cppcheck.py

@@ -19,7 +19,7 @@ import click
 from platformio import proc
 from platformio.commands.check.defect import DefectItem
 from platformio.commands.check.tools.base import CheckToolBase
-from platformio.managers.core import get_core_package_dir
+from platformio.package.manager.core import get_core_package_dir
 
 
 class CppcheckCheckTool(CheckToolBase):

platformio/commands/check/tools/pvsstudio.py

@@ -22,7 +22,7 @@ import click
 from platformio import proc, util
 from platformio.commands.check.defect import DefectItem
 from platformio.commands.check.tools.base import CheckToolBase
-from platformio.managers.core import get_core_package_dir
+from platformio.package.manager.core import get_core_package_dir
 
 
 class PvsStudioCheckTool(CheckToolBase):  # pylint: disable=too-many-instance-attributes

platformio/commands/debug/command.py

@@ -24,7 +24,7 @@ import click
 from platformio import app, exception, fs, proc, util
 from platformio.commands.debug import helpers
 from platformio.commands.debug.exception import DebugInvalidOptionsError
-from platformio.managers.core import inject_contrib_pysite
+from platformio.package.manager.core import inject_contrib_pysite
 from platformio.project.config import ProjectConfig
 from platformio.project.exception import ProjectEnvsNotAvailableError
 from platformio.project.helpers import is_platformio_project, load_project_ide_data

platformio/commands/home/command.py

@@ -22,7 +22,7 @@ import click
 
 from platformio import exception
 from platformio.compat import WINDOWS
-from platformio.managers.core import get_core_package_dir, inject_contrib_pysite
+from platformio.package.manager.core import get_core_package_dir, inject_contrib_pysite
 
 
 @click.command("home", short_help="PIO Home")

platformio/commands/remote/command.py

@@ -29,7 +29,7 @@ from platformio.commands.device.command import device_monitor as cmd_device_moni
 from platformio.commands.run.command import cli as cmd_run
 from platformio.commands.test.command import cli as cmd_test
 from platformio.compat import PY2
-from platformio.managers.core import inject_contrib_pysite
+from platformio.package.manager.core import inject_contrib_pysite
 from platformio.project.exception import NotPlatformIOProjectError
 
 

platformio/commands/update.py

@@ -18,7 +18,7 @@ from platformio import app
 from platformio.commands.lib.command import CTX_META_STORAGE_DIRS_KEY
 from platformio.commands.lib.command import lib_update as cmd_lib_update
 from platformio.commands.platform import platform_update as cmd_platform_update
-from platformio.managers.core import update_core_packages
+from platformio.package.manager.core import update_core_packages
 from platformio.package.manager.library import LibraryPackageManager
 
 

platformio/maintenance.py

@@ -25,9 +25,11 @@ from platformio.commands.lib.command import CTX_META_STORAGE_DIRS_KEY
 from platformio.commands.lib.command import lib_update as cmd_lib_update
 from platformio.commands.platform import platform_update as cmd_platform_update
 from platformio.commands.upgrade import get_latest_version
-from platformio.managers.core import update_core_packages
 from platformio.managers.platform import PlatformFactory, PlatformManager
+from platformio.package.manager.core import update_core_packages
 from platformio.package.manager.library import LibraryPackageManager
+from platformio.package.manager.tool import ToolPackageManager
+from platformio.package.meta import PackageSpec
 from platformio.proc import is_container
 
 
@@ -90,7 +92,8 @@ class Upgrader(object):
         )
 
         self._upgraders = [
-            (semantic_version.Version("3.5.0-a.2"), self._update_dev_platforms)
+            (semantic_version.Version("3.5.0-a.2"), self._update_dev_platforms),
+            (semantic_version.Version("4.4.0-a.8"), self._update_pkg_metadata),
         ]
 
     def run(self, ctx):
@@ -110,6 +113,22 @@ class Upgrader(object):
         ctx.invoke(cmd_platform_update)
         return True
 
+    @staticmethod
+    def _update_pkg_metadata(_):
+        pm = ToolPackageManager()
+        for pkg in pm.get_installed():
+            if not pkg.metadata or pkg.metadata.spec.external or pkg.metadata.spec.id:
+                continue
+            result = pm.search_registry_packages(PackageSpec(name=pkg.metadata.name))
+            if len(result) != 1:
+                continue
+            result = result[0]
+            pkg.metadata.spec = PackageSpec(
+                id=result["id"], owner=result["owner"]["username"], name=result["name"],
+            )
+            pkg.dump_meta()
+        return True
+
 
 def after_upgrade(ctx):
     terminal_width, _ = click.get_terminal_size()
@@ -160,7 +179,6 @@ def after_upgrade(ctx):
             )
         else:
             raise exception.UpgradeError("Auto upgrading...")
-        click.echo("")
 
     # PlatformIO banner
     click.echo("*" * terminal_width)

platformio/managers/platform.py

@@ -30,8 +30,8 @@ from platformio.commands.debug.exception import (
     DebugSupportError,
 )
 from platformio.compat import PY2, hashlib_encode_data, is_bytes, load_python_module
-from platformio.managers.core import get_core_package_dir
 from platformio.managers.package import BasePkgManager, PackageManager
+from platformio.package.manager.core import get_core_package_dir
 from platformio.project.config import ProjectConfig
 
 try:

platformio/package/manager/_registry.py

@@ -119,6 +119,7 @@ class PackageManageRegistryMixin(object):
         return self._registry_client
 
     def search_registry_packages(self, spec, filters=None):
+        assert isinstance(spec, PackageSpec)
         filters = filters or {}
         if spec.id:
             filters["ids"] = str(spec.id)
@@ -132,6 +133,7 @@ class PackageManageRegistryMixin(object):
         ]
 
     def fetch_registry_package(self, spec):
+        assert isinstance(spec, PackageSpec)
         result = None
         if spec.owner and spec.name:
             result = self.get_registry_client_instance().get_package(

platformio/package/manager/core.py

@@ -17,89 +17,58 @@ import os
 import subprocess
 import sys
 
-from platformio import exception, util
+from platformio import __core_packages__, exception, util
 from platformio.compat import PY2
-from platformio.managers.package import PackageManager
+from platformio.package.manager.tool import ToolPackageManager
+from platformio.package.meta import PackageSpec
 from platformio.proc import get_pythonexe_path
-from platformio.project.config import ProjectConfig
-
-CORE_PACKAGES = {
-    "contrib-piohome": "~3.2.3",
-    "contrib-pysite": "~2.%d%d.0" % (sys.version_info.major, sys.version_info.minor),
-    "tool-unity": "~1.20500.0",
-    "tool-scons": "~2.20501.7" if PY2 else "~3.30102.0",
-    "tool-cppcheck": "~1.190.0",
-    "tool-clangtidy": "~1.100000.0",
-    "tool-pvs-studio": "~7.7.0",
-}
-
-# pylint: disable=arguments-differ,signature-differs
-
-
-class CorePackageManager(PackageManager):
-    def __init__(self):
-        config = ProjectConfig.get_instance()
-        packages_dir = config.get_optional_dir("packages")
-        super(CorePackageManager, self).__init__(
-            packages_dir,
-            [
-                "https://dl.bintray.com/platformio/dl-packages/manifest.json",
-                "http%s://dl.platformio.org/packages/manifest.json"
-                % ("" if sys.version_info < (2, 7, 9) else "s"),
-            ],
-        )
-
-    def install(  # pylint: disable=keyword-arg-before-vararg
-        self, name, requirements=None, *args, **kwargs
-    ):
-        PackageManager.install(self, name, requirements, *args, **kwargs)
-        self.cleanup_packages()
-        return self.get_package_dir(name, requirements)
-
-    def update(self, *args, **kwargs):
-        result = PackageManager.update(self, *args, **kwargs)
-        self.cleanup_packages()
-        return result
-
-    def cleanup_packages(self):
-        self.cache_reset()
-        best_pkg_versions = {}
-        for name, requirements in CORE_PACKAGES.items():
-            pkg_dir = self.get_package_dir(name, requirements)
-            if not pkg_dir:
-                continue
-            best_pkg_versions[name] = self.load_manifest(pkg_dir)["version"]
-        for manifest in self.get_installed():
-            if manifest["name"] not in best_pkg_versions:
-                continue
-            if manifest["version"] != best_pkg_versions[manifest["name"]]:
-                self.uninstall(manifest["__pkg_dir"], after_update=True)
-        self.cache_reset()
-        return True
 
 
 def get_core_package_dir(name):
-    if name not in CORE_PACKAGES:
+    if name not in __core_packages__:
         raise exception.PlatformioException("Please upgrade PIO Core")
-    requirements = CORE_PACKAGES[name]
+    pm = ToolPackageManager()
-    pm = CorePackageManager()
+    spec = PackageSpec(
-    pkg_dir = pm.get_package_dir(name, requirements)
+        owner="platformio", name=name, requirements=__core_packages__[name]
-    if pkg_dir:
+    )
-        return pkg_dir
+    pkg = pm.get_package(spec)
-    return pm.install(name, requirements)
+    if pkg:
+        return pkg.path
+    pkg = pm.install(spec).path
+    _remove_unnecessary_packages()
+    return pkg
 
 
 def update_core_packages(only_check=False, silent=False):
-    pm = CorePackageManager()
+    pm = ToolPackageManager()
-    for name, requirements in CORE_PACKAGES.items():
+    for name, requirements in __core_packages__.items():
-        pkg_dir = pm.get_package_dir(name)
+        spec = PackageSpec(owner="platformio", name=name, requirements=requirements)
-        if not pkg_dir:
+        pkg = pm.get_package(spec)
+        if not pkg:
             continue
-        if not silent or pm.outdated(pkg_dir, requirements):
+        if not silent or pm.outdated(pkg, spec).is_outdated():
-            pm.update(name, requirements, only_check=only_check)
+            pm.update(pkg, spec, only_check=only_check)
+    if not only_check:
+        _remove_unnecessary_packages()
     return True
 
 
+def _remove_unnecessary_packages():
+    pm = ToolPackageManager()
+    best_pkg_versions = {}
+    for name, requirements in __core_packages__.items():
+        spec = PackageSpec(owner="platformio", name=name, requirements=requirements)
+        pkg = pm.get_package(spec)
+        if not pkg:
+            continue
+        best_pkg_versions[pkg.metadata.name] = pkg.metadata.version
+    for pkg in pm.get_installed():
+        if pkg.metadata.name not in best_pkg_versions:
+            continue
+        if pkg.metadata.version != best_pkg_versions[pkg.metadata.name]:
+            pm.uninstall(pkg)
+
+
 def inject_contrib_pysite(verify_openssl=False):
     # pylint: disable=import-outside-toplevel
     from site import addsitedir

platformio/util.py

@@ -196,7 +196,7 @@ def get_mdns_services():
         import zeroconf
     except ImportError:
         from site import addsitedir
-        from platformio.managers.core import get_core_package_dir
+        from platformio.package.manager.core import get_core_package_dir
 
         contrib_pysite_dir = get_core_package_dir("contrib-pysite")
         addsitedir(contrib_pysite_dir)

tests/test_managers.py

@@ -1,234 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import json
-from os.path import join
-
-from platformio.managers.package import PackageManager
-from platformio.project.helpers import get_project_core_dir
-
-
-def test_pkg_input_parser():
-    items = [
-        ["PkgName", ("PkgName", None, None)],
-        [("PkgName", "!=1.2.3,<2.0"), ("PkgName", "!=1.2.3,<2.0", None)],
-        ["PkgName@1.2.3", ("PkgName", "1.2.3", None)],
-        [("PkgName@1.2.3", "1.2.5"), ("PkgName@1.2.3", "1.2.5", None)],
-        ["id=13", ("id=13", None, None)],
-        ["id=13@~1.2.3", ("id=13", "~1.2.3", None)],
-        [
-            get_project_core_dir(),
-            (".platformio", None, "file://" + get_project_core_dir()),
-        ],
-        [
-            "LocalName=" + get_project_core_dir(),
-            ("LocalName", None, "file://" + get_project_core_dir()),
-        ],
-        [
-            "LocalName=%s@>2.3.0" % get_project_core_dir(),
-            ("LocalName", ">2.3.0", "file://" + get_project_core_dir()),
-        ],
-        [
-            "https://github.com/user/package.git",
-            ("package", None, "git+https://github.com/user/package.git"),
-        ],
-        [
-            "MyPackage=https://gitlab.com/user/package.git",
-            ("MyPackage", None, "git+https://gitlab.com/user/package.git"),
-        ],
-        [
-            "MyPackage=https://gitlab.com/user/package.git@3.2.1,!=2",
-            ("MyPackage", "3.2.1,!=2", "git+https://gitlab.com/user/package.git"),
-        ],
-        [
-            "https://somedomain.com/path/LibraryName-1.2.3.zip",
-            (
-                "LibraryName-1.2.3",
-                None,
-                "https://somedomain.com/path/LibraryName-1.2.3.zip",
-            ),
-        ],
-        [
-            "https://github.com/user/package/archive/branch.zip",
-            ("branch", None, "https://github.com/user/package/archive/branch.zip"),
-        ],
-        [
-            "https://github.com/user/package/archive/branch.zip@~1.2.3",
-            ("branch", "~1.2.3", "https://github.com/user/package/archive/branch.zip"),
-        ],
-        [
-            "https://github.com/user/package/archive/branch.tar.gz",
-            (
-                "branch.tar",
-                None,
-                "https://github.com/user/package/archive/branch.tar.gz",
-            ),
-        ],
-        [
-            "https://github.com/user/package/archive/branch.tar.gz@!=5",
-            (
-                "branch.tar",
-                "!=5",
-                "https://github.com/user/package/archive/branch.tar.gz",
-            ),
-        ],
-        [
-            "https://developer.mbed.org/users/user/code/package/",
-            ("package", None, "hg+https://developer.mbed.org/users/user/code/package/"),
-        ],
-        [
-            "https://os.mbed.com/users/user/code/package/",
-            ("package", None, "hg+https://os.mbed.com/users/user/code/package/"),
-        ],
-        [
-            "https://github.com/user/package#v1.2.3",
-            ("package", None, "git+https://github.com/user/package#v1.2.3"),
-        ],
-        [
-            "https://github.com/user/package.git#branch",
-            ("package", None, "git+https://github.com/user/package.git#branch"),
-        ],
-        [
-            "PkgName=https://github.com/user/package.git#a13d344fg56",
-            ("PkgName", None, "git+https://github.com/user/package.git#a13d344fg56"),
-        ],
-        ["user/package", ("package", None, "git+https://github.com/user/package")],
-        [
-            "PkgName=user/package",
-            ("PkgName", None, "git+https://github.com/user/package"),
-        ],
-        [
-            "PkgName=user/package#master",
-            ("PkgName", None, "git+https://github.com/user/package#master"),
-        ],
-        [
-            "git+https://github.com/user/package",
-            ("package", None, "git+https://github.com/user/package"),
-        ],
-        [
-            "hg+https://example.com/user/package",
-            ("package", None, "hg+https://example.com/user/package"),
-        ],
-        [
-            "git@github.com:user/package.git",
-            ("package", None, "git+git@github.com:user/package.git"),
-        ],
-        [
-            "git@github.com:user/package.git#v1.2.0",
-            ("package", None, "git+git@github.com:user/package.git#v1.2.0"),
-        ],
-        [
-            "LocalName=git@github.com:user/package.git#v1.2.0@~1.2.0",
-            ("LocalName", "~1.2.0", "git+git@github.com:user/package.git#v1.2.0"),
-        ],
-        [
-            "git+ssh://git@gitlab.private-server.com/user/package#1.2.0",
-            (
-                "package",
-                None,
-                "git+ssh://git@gitlab.private-server.com/user/package#1.2.0",
-            ),
-        ],
-        [
-            "git+ssh://user@gitlab.private-server.com:1234/package#1.2.0",
-            (
-                "package",
-                None,
-                "git+ssh://user@gitlab.private-server.com:1234/package#1.2.0",
-            ),
-        ],
-        [
-            "LocalName=git+ssh://user@gitlab.private-server.com:1234"
-            "/package#1.2.0@!=13",
-            (
-                "LocalName",
-                "!=13",
-                "git+ssh://user@gitlab.private-server.com:1234/package#1.2.0",
-            ),
-        ],
-    ]
-    for params, result in items:
-        if isinstance(params, tuple):
-            assert PackageManager.parse_pkg_uri(*params) == result
-        else:
-            assert PackageManager.parse_pkg_uri(params) == result
-
-
-def test_install_packages(isolated_pio_core, tmpdir):
-    packages = [
-        dict(id=1, name="name_1", version="shasum"),
-        dict(id=1, name="name_1", version="2.0.0"),
-        dict(id=1, name="name_1", version="2.1.0"),
-        dict(id=1, name="name_1", version="1.2"),
-        dict(id=1, name="name_1", version="1.0.0"),
-        dict(name="name_2", version="1.0.0"),
-        dict(name="name_2", version="2.0.0", __src_url="git+https://github.com"),
-        dict(name="name_2", version="3.0.0", __src_url="git+https://github2.com"),
-        dict(name="name_2", version="4.0.0", __src_url="git+https://github2.com"),
-    ]
-
-    pm = PackageManager(join(get_project_core_dir(), "packages"))
-    for package in packages:
-        tmp_dir = tmpdir.mkdir("tmp-package")
-        tmp_dir.join("package.json").write(json.dumps(package))
-        pm._install_from_url(package["name"], "file://%s" % str(tmp_dir))
-        tmp_dir.remove(rec=1)
-
-    assert len(pm.get_installed()) == len(packages) - 1
-
-    pkg_dirnames = [
-        "name_1_ID1",
-        "name_1_ID1@1.0.0",
-        "name_1_ID1@1.2",
-        "name_1_ID1@2.0.0",
-        "name_1_ID1@shasum",
-        "name_2",
-        "name_2@src-177cbce1f0705580d17790fda1cc2ef5",
-        "name_2@src-f863b537ab00f4c7b5011fc44b120e1f",
-    ]
-    assert set(
-        [p.basename for p in isolated_pio_core.join("packages").listdir()]
-    ) == set(pkg_dirnames)
-
-
-def test_get_package():
-    tests = [
-        [("unknown",), None],
-        [("1",), None],
-        [("id=1", "shasum"), dict(id=1, name="name_1", version="shasum")],
-        [("id=1", "*"), dict(id=1, name="name_1", version="2.1.0")],
-        [("id=1", "^1"), dict(id=1, name="name_1", version="1.2")],
-        [("id=1", "^1"), dict(id=1, name="name_1", version="1.2")],
-        [("name_1", "<2"), dict(id=1, name="name_1", version="1.2")],
-        [("name_1", ">2"), None],
-        [("name_1", "2-0-0"), None],
-        [("name_2",), dict(name="name_2", version="4.0.0")],
-        [
-            ("url_has_higher_priority", None, "git+https://github.com"),
-            dict(name="name_2", version="2.0.0", __src_url="git+https://github.com"),
-        ],
-        [
-            ("name_2", None, "git+https://github.com"),
-            dict(name="name_2", version="2.0.0", __src_url="git+https://github.com"),
-        ],
-    ]
-
-    pm = PackageManager(join(get_project_core_dir(), "packages"))
-    for test in tests:
-        manifest = pm.get_package(*test[0])
-        if test[1] is None:
-            assert manifest is None, test
-            continue
-        for key, value in test[1].items():
-            assert manifest[key] == value, test