platformio/platformio-core
1
0
Fork 1
mirror of https://github.com/platformio/platformio-core.git synced 2025-07-29 17:47:14 +02:00
Code Issues Packages Projects Releases Wiki Activity

Switch legacy core package manager to the new

Browse Source
This commit is contained in:
Ivan Kravets
2020-08-13 18:30:04 +03:00
parent fd7dba1d74
commit 64ff6a0ff5
16 changed files with 83 additions and 318 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
platformio/__init__.py
View File

@ -40,3 +40,13 @@ __apiurl__ = "https://api.platformio.org"
__accounts_api__ = "https://api.accounts.platformio.org"
__registry_api__ = "https://api.registry.platformio.org"
__pioremote_endpoint__ = "ssl:host=remote.platformio.org:port=4413"
__core_packages__ = {
"contrib-piohome": "~3.2.3",
"contrib-pysite": "~2.%d%d.0" % (sys.version_info.major, sys.version_info.minor),
"tool-unity": "~1.20500.0",
"tool-scons": "~2.20501.7" if sys.version_info.major == 2 else "~3.30102.0",
"tool-cppcheck": "~1.190.0",
"tool-clangtidy": "~1.100000.0",
"tool-pvs-studio": "~7.7.0",
}

2
platformio/builder/tools/pioide.py
View File

@ -20,7 +20,7 @@ from glob import glob
from SCons.Defaults import processDefines # pylint: disable=import-error
from platformio.compat import glob_escape
from platformio.managers.core import get_core_package_dir
from platformio.package.manager.core import get_core_package_dir
from platformio.proc import exec_command, where_is_program

2
platformio/builder/tools/piomisc.py
View File

@ -25,7 +25,7 @@ import click
from platformio import fs, util
from platformio.compat import get_filesystem_encoding, get_locale_encoding, glob_escape
from platformio.managers.core import get_core_package_dir
from platformio.package.manager.core import get_core_package_dir
from platformio.proc import exec_command

2
platformio/commands/check/tools/clangtidy.py
View File

@ -17,7 +17,7 @@ from os.path import join
from platformio.commands.check.defect import DefectItem
from platformio.commands.check.tools.base import CheckToolBase
from platformio.managers.core import get_core_package_dir
from platformio.package.manager.core import get_core_package_dir
class ClangtidyCheckTool(CheckToolBase):

2
platformio/commands/check/tools/cppcheck.py
View File

@ -19,7 +19,7 @@ import click
from platformio import proc
from platformio.commands.check.defect import DefectItem
from platformio.commands.check.tools.base import CheckToolBase
from platformio.managers.core import get_core_package_dir
from platformio.package.manager.core import get_core_package_dir
class CppcheckCheckTool(CheckToolBase):

2
platformio/commands/check/tools/pvsstudio.py
View File

@ -22,7 +22,7 @@ import click
from platformio import proc, util
from platformio.commands.check.defect import DefectItem
from platformio.commands.check.tools.base import CheckToolBase
from platformio.managers.core import get_core_package_dir
from platformio.package.manager.core import get_core_package_dir
class PvsStudioCheckTool(CheckToolBase): # pylint: disable=too-many-instance-attributes

2
platformio/commands/debug/command.py
View File

@ -24,7 +24,7 @@ import click
from platformio import app, exception, fs, proc, util
from platformio.commands.debug import helpers
from platformio.commands.debug.exception import DebugInvalidOptionsError
from platformio.managers.core import inject_contrib_pysite
from platformio.package.manager.core import inject_contrib_pysite
from platformio.project.config import ProjectConfig
from platformio.project.exception import ProjectEnvsNotAvailableError
from platformio.project.helpers import is_platformio_project, load_project_ide_data

2
platformio/commands/home/command.py
View File

@ -22,7 +22,7 @@ import click
from platformio import exception
from platformio.compat import WINDOWS
from platformio.managers.core import get_core_package_dir, inject_contrib_pysite
from platformio.package.manager.core import get_core_package_dir, inject_contrib_pysite
@click.command("home", short_help="PIO Home")

2
platformio/commands/remote/command.py
View File

@ -29,7 +29,7 @@ from platformio.commands.device.command import device_monitor as cmd_device_moni
from platformio.commands.run.command import cli as cmd_run
from platformio.commands.test.command import cli as cmd_test
from platformio.compat import PY2
from platformio.managers.core import inject_contrib_pysite
from platformio.package.manager.core import inject_contrib_pysite
from platformio.project.exception import NotPlatformIOProjectError

2
platformio/commands/update.py
View File

@ -18,7 +18,7 @@ from platformio import app
from platformio.commands.lib.command import CTX_META_STORAGE_DIRS_KEY
from platformio.commands.lib.command import lib_update as cmd_lib_update
from platformio.commands.platform import platform_update as cmd_platform_update
from platformio.managers.core import update_core_packages
from platformio.package.manager.core import update_core_packages
from platformio.package.manager.library import LibraryPackageManager

24
platformio/maintenance.py
View File

@ -25,9 +25,11 @@ from platformio.commands.lib.command import CTX_META_STORAGE_DIRS_KEY
from platformio.commands.lib.command import lib_update as cmd_lib_update
from platformio.commands.platform import platform_update as cmd_platform_update
from platformio.commands.upgrade import get_latest_version
from platformio.managers.core import update_core_packages
from platformio.managers.platform import PlatformFactory, PlatformManager
from platformio.package.manager.core import update_core_packages
from platformio.package.manager.library import LibraryPackageManager
from platformio.package.manager.tool import ToolPackageManager
from platformio.package.meta import PackageSpec
from platformio.proc import is_container
@ -90,7 +92,8 @@ class Upgrader(object):
)
self._upgraders = [
(semantic_version.Version("3.5.0-a.2"), self._update_dev_platforms)
(semantic_version.Version("3.5.0-a.2"), self._update_dev_platforms),
(semantic_version.Version("4.4.0-a.8"), self._update_pkg_metadata),
]
def run(self, ctx):
@ -110,6 +113,22 @@ class Upgrader(object):
ctx.invoke(cmd_platform_update)
return True
@staticmethod
def _update_pkg_metadata(_):
pm = ToolPackageManager()
for pkg in pm.get_installed():
if not pkg.metadata or pkg.metadata.spec.external or pkg.metadata.spec.id:
continue
result = pm.search_registry_packages(PackageSpec(name=pkg.metadata.name))
if len(result) != 1:
continue
result = result[0]
pkg.metadata.spec = PackageSpec(
id=result["id"], owner=result["owner"]["username"], name=result["name"],
)
pkg.dump_meta()
return True
def after_upgrade(ctx):
terminal_width, _ = click.get_terminal_size()
@ -160,7 +179,6 @@ def after_upgrade(ctx):
)
else:
raise exception.UpgradeError("Auto upgrading...")
click.echo("")
# PlatformIO banner
click.echo("*" * terminal_width)

2
platformio/managers/platform.py
View File

@ -30,8 +30,8 @@ from platformio.commands.debug.exception import (
DebugSupportError,
)
from platformio.compat import PY2, hashlib_encode_data, is_bytes, load_python_module
from platformio.managers.core import get_core_package_dir
from platformio.managers.package import BasePkgManager, PackageManager
from platformio.package.manager.core import get_core_package_dir
from platformio.project.config import ProjectConfig
try:

2
platformio/package/manager/_registry.py
View File

@ -119,6 +119,7 @@ class PackageManageRegistryMixin(object):
return self._registry_client
def search_registry_packages(self, spec, filters=None):
assert isinstance(spec, PackageSpec)
filters = filters or {}
if spec.id:
filters["ids"] = str(spec.id)
@ -132,6 +133,7 @@ class PackageManageRegistryMixin(object):
]
def fetch_registry_package(self, spec):
assert isinstance(spec, PackageSpec)
result = None
if spec.owner and spec.name:
result = self.get_registry_client_instance().get_package(

109
platformio/managers/core.py → platformio/package/manager/core.py
View File

@ -17,89 +17,58 @@ import os
import subprocess
import sys
from platformio import exception, util
from platformio import __core_packages__, exception, util
from platformio.compat import PY2
from platformio.managers.package import PackageManager
from platformio.package.manager.tool import ToolPackageManager
from platformio.package.meta import PackageSpec
from platformio.proc import get_pythonexe_path
from platformio.project.config import ProjectConfig
CORE_PACKAGES = {
"contrib-piohome": "~3.2.3",
"contrib-pysite": "~2.%d%d.0" % (sys.version_info.major, sys.version_info.minor),
"tool-unity": "~1.20500.0",
"tool-scons": "~2.20501.7" if PY2 else "~3.30102.0",
"tool-cppcheck": "~1.190.0",
"tool-clangtidy": "~1.100000.0",
"tool-pvs-studio": "~7.7.0",
}
# pylint: disable=arguments-differ,signature-differs
class CorePackageManager(PackageManager):
def __init__(self):
config = ProjectConfig.get_instance()
packages_dir = config.get_optional_dir("packages")
super(CorePackageManager, self).__init__(
packages_dir,
[
"https://dl.bintray.com/platformio/dl-packages/manifest.json",
"http%s://dl.platformio.org/packages/manifest.json"
% ("" if sys.version_info < (2, 7, 9) else "s"),
],
)
def install( # pylint: disable=keyword-arg-before-vararg
self, name, requirements=None, *args, **kwargs
):
PackageManager.install(self, name, requirements, *args, **kwargs)
self.cleanup_packages()
return self.get_package_dir(name, requirements)
def update(self, *args, **kwargs):
result = PackageManager.update(self, *args, **kwargs)
self.cleanup_packages()
return result
def cleanup_packages(self):
self.cache_reset()
best_pkg_versions = {}
for name, requirements in CORE_PACKAGES.items():
pkg_dir = self.get_package_dir(name, requirements)
if not pkg_dir:
continue
best_pkg_versions[name] = self.load_manifest(pkg_dir)["version"]
for manifest in self.get_installed():
if manifest["name"] not in best_pkg_versions:
continue
if manifest["version"] != best_pkg_versions[manifest["name"]]:
self.uninstall(manifest["__pkg_dir"], after_update=True)
self.cache_reset()
return True
def get_core_package_dir(name):
if name not in CORE_PACKAGES:
if name not in __core_packages__:
raise exception.PlatformioException("Please upgrade PIO Core")
requirements = CORE_PACKAGES[name]
pm = CorePackageManager()
pkg_dir = pm.get_package_dir(name, requirements)
if pkg_dir:
return pkg_dir
return pm.install(name, requirements)
pm = ToolPackageManager()
spec = PackageSpec(
owner="platformio", name=name, requirements=__core_packages__[name]
)
pkg = pm.get_package(spec)
if pkg:
return pkg.path
pkg = pm.install(spec).path
_remove_unnecessary_packages()
return pkg
def update_core_packages(only_check=False, silent=False):
pm = CorePackageManager()
for name, requirements in CORE_PACKAGES.items():
pkg_dir = pm.get_package_dir(name)
if not pkg_dir:
pm = ToolPackageManager()
for name, requirements in __core_packages__.items():
spec = PackageSpec(owner="platformio", name=name, requirements=requirements)
pkg = pm.get_package(spec)
if not pkg:
continue
if not silent or pm.outdated(pkg_dir, requirements):
pm.update(name, requirements, only_check=only_check)
if not silent or pm.outdated(pkg, spec).is_outdated():
pm.update(pkg, spec, only_check=only_check)
if not only_check:
_remove_unnecessary_packages()
return True
def _remove_unnecessary_packages():
pm = ToolPackageManager()
best_pkg_versions = {}
for name, requirements in __core_packages__.items():
spec = PackageSpec(owner="platformio", name=name, requirements=requirements)
pkg = pm.get_package(spec)
if not pkg:
continue
best_pkg_versions[pkg.metadata.name] = pkg.metadata.version
for pkg in pm.get_installed():
if pkg.metadata.name not in best_pkg_versions:
continue
if pkg.metadata.version != best_pkg_versions[pkg.metadata.name]:
pm.uninstall(pkg)
def inject_contrib_pysite(verify_openssl=False):
# pylint: disable=import-outside-toplevel
from site import addsitedir

2
platformio/util.py
View File

@ -196,7 +196,7 @@ def get_mdns_services():
import zeroconf
except ImportError:
from site import addsitedir
from platformio.managers.core import get_core_package_dir
from platformio.package.manager.core import get_core_package_dir
contrib_pysite_dir = get_core_package_dir("contrib-pysite")
addsitedir(contrib_pysite_dir)

234
tests/test_managers.py
View File

@ -1,234 +0,0 @@
# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
from os.path import join
from platformio.managers.package import PackageManager
from platformio.project.helpers import get_project_core_dir
def test_pkg_input_parser():
items = [
["PkgName", ("PkgName", None, None)],
[("PkgName", "!=1.2.3,<2.0"), ("PkgName", "!=1.2.3,<2.0", None)],
["PkgName@1.2.3", ("PkgName", "1.2.3", None)],
[("PkgName@1.2.3", "1.2.5"), ("PkgName@1.2.3", "1.2.5", None)],
["id=13", ("id=13", None, None)],
["id=13@~1.2.3", ("id=13", "~1.2.3", None)],
[
get_project_core_dir(),
(".platformio", None, "file://" + get_project_core_dir()),
],
[
"LocalName=" + get_project_core_dir(),
("LocalName", None, "file://" + get_project_core_dir()),
],
[
"LocalName=%s@>2.3.0" % get_project_core_dir(),
("LocalName", ">2.3.0", "file://" + get_project_core_dir()),
],
[
"https://github.com/user/package.git",
("package", None, "git+https://github.com/user/package.git"),
],
[
"MyPackage=https://gitlab.com/user/package.git",
("MyPackage", None, "git+https://gitlab.com/user/package.git"),
],
[
"MyPackage=https://gitlab.com/user/package.git@3.2.1,!=2",
("MyPackage", "3.2.1,!=2", "git+https://gitlab.com/user/package.git"),
],
[
"https://somedomain.com/path/LibraryName-1.2.3.zip",
(
"LibraryName-1.2.3",
None,
"https://somedomain.com/path/LibraryName-1.2.3.zip",
),
],
[
"https://github.com/user/package/archive/branch.zip",
("branch", None, "https://github.com/user/package/archive/branch.zip"),
],
[
"https://github.com/user/package/archive/branch.zip@~1.2.3",
("branch", "~1.2.3", "https://github.com/user/package/archive/branch.zip"),
],
[
"https://github.com/user/package/archive/branch.tar.gz",
(
"branch.tar",
None,
"https://github.com/user/package/archive/branch.tar.gz",
),
],
[
"https://github.com/user/package/archive/branch.tar.gz@!=5",
(
"branch.tar",
"!=5",
"https://github.com/user/package/archive/branch.tar.gz",
),
],
[
"https://developer.mbed.org/users/user/code/package/",
("package", None, "hg+https://developer.mbed.org/users/user/code/package/"),
],
[
"https://os.mbed.com/users/user/code/package/",
("package", None, "hg+https://os.mbed.com/users/user/code/package/"),
],
[
"https://github.com/user/package#v1.2.3",
("package", None, "git+https://github.com/user/package#v1.2.3"),
],
[
"https://github.com/user/package.git#branch",
("package", None, "git+https://github.com/user/package.git#branch"),
],
[
"PkgName=https://github.com/user/package.git#a13d344fg56",
("PkgName", None, "git+https://github.com/user/package.git#a13d344fg56"),
],
["user/package", ("package", None, "git+https://github.com/user/package")],
[
"PkgName=user/package",
("PkgName", None, "git+https://github.com/user/package"),
],
[
"PkgName=user/package#master",
("PkgName", None, "git+https://github.com/user/package#master"),
],
[
"git+https://github.com/user/package",
("package", None, "git+https://github.com/user/package"),
],
[
"hg+https://example.com/user/package",
("package", None, "hg+https://example.com/user/package"),
],
[
"git@github.com:user/package.git",
("package", None, "git+git@github.com:user/package.git"),
],
[
"git@github.com:user/package.git#v1.2.0",
("package", None, "git+git@github.com:user/package.git#v1.2.0"),
],
[
"LocalName=git@github.com:user/package.git#v1.2.0@~1.2.0",
("LocalName", "~1.2.0", "git+git@github.com:user/package.git#v1.2.0"),
],
[
"git+ssh://git@gitlab.private-server.com/user/package#1.2.0",
(
"package",
None,
"git+ssh://git@gitlab.private-server.com/user/package#1.2.0",
),
],
[
"git+ssh://user@gitlab.private-server.com:1234/package#1.2.0",
(
"package",
None,
"git+ssh://user@gitlab.private-server.com:1234/package#1.2.0",
),
],
[
"LocalName=git+ssh://user@gitlab.private-server.com:1234"
"/package#1.2.0@!=13",
(
"LocalName",
"!=13",
"git+ssh://user@gitlab.private-server.com:1234/package#1.2.0",
),
],
]
for params, result in items:
if isinstance(params, tuple):
assert PackageManager.parse_pkg_uri(*params) == result
else:
assert PackageManager.parse_pkg_uri(params) == result
def test_install_packages(isolated_pio_core, tmpdir):
packages = [
dict(id=1, name="name_1", version="shasum"),
dict(id=1, name="name_1", version="2.0.0"),
dict(id=1, name="name_1", version="2.1.0"),
dict(id=1, name="name_1", version="1.2"),
dict(id=1, name="name_1", version="1.0.0"),
dict(name="name_2", version="1.0.0"),
dict(name="name_2", version="2.0.0", __src_url="git+https://github.com"),
dict(name="name_2", version="3.0.0", __src_url="git+https://github2.com"),
dict(name="name_2", version="4.0.0", __src_url="git+https://github2.com"),
]
pm = PackageManager(join(get_project_core_dir(), "packages"))
for package in packages:
tmp_dir = tmpdir.mkdir("tmp-package")
tmp_dir.join("package.json").write(json.dumps(package))
pm._install_from_url(package["name"], "file://%s" % str(tmp_dir))
tmp_dir.remove(rec=1)
assert len(pm.get_installed()) == len(packages) - 1
pkg_dirnames = [
"name_1_ID1",
"name_1_ID1@1.0.0",
"name_1_ID1@1.2",
"name_1_ID1@2.0.0",
"name_1_ID1@shasum",
"name_2",
"name_2@src-177cbce1f0705580d17790fda1cc2ef5",
"name_2@src-f863b537ab00f4c7b5011fc44b120e1f",
]
assert set(
[p.basename for p in isolated_pio_core.join("packages").listdir()]
) == set(pkg_dirnames)
def test_get_package():
tests = [
[("unknown",), None],
[("1",), None],
[("id=1", "shasum"), dict(id=1, name="name_1", version="shasum")],
[("id=1", "*"), dict(id=1, name="name_1", version="2.1.0")],
[("id=1", "^1"), dict(id=1, name="name_1", version="1.2")],
[("id=1", "^1"), dict(id=1, name="name_1", version="1.2")],
[("name_1", "<2"), dict(id=1, name="name_1", version="1.2")],
[("name_1", ">2"), None],
[("name_1", "2-0-0"), None],
[("name_2",), dict(name="name_2", version="4.0.0")],
[
("url_has_higher_priority", None, "git+https://github.com"),
dict(name="name_2", version="2.0.0", __src_url="git+https://github.com"),
],
[
("name_2", None, "git+https://github.com"),
dict(name="name_2", version="2.0.0", __src_url="git+https://github.com"),
],
]
pm = PackageManager(join(get_project_core_dir(), "packages"))
for test in tests:
manifest = pm.get_package(*test[0])
if test[1] is None:
assert manifest is None, test
continue
for key, value in test[1].items():
assert manifest[key] == value, test