feedc0de/platformio-core
1
0
Fork 0
forked from platformio/platformio-core
Code Pull Requests Activity

Replace package meta URL with URI

Browse Source
This commit is contained in:
Ivan Kravets
2022-04-04 14:18:11 +03:00
parent f555656c92
commit a03d82ff1a
13 changed files with 92 additions and 90 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
docs

Submodule docs updated: a0c38a9138...e24bd4f12d

2
platformio/builder/tools/piolib.py
View File

@ -1102,7 +1102,7 @@ def ConfigureProjectLibBuilder(env):
"(License: %s, " % (_get_lib_license(pkg) or "Unknown"), nl=False
)
if pkg.metadata and pkg.metadata.spec.external:
click.echo("URI: %s, " % pkg.metadata.spec.url, nl=False)
click.echo("URI: %s, " % pkg.metadata.spec.uri, nl=False)
click.echo("Path: %s" % lb.path, nl=False)
click.echo(")", nl=False)
click.echo("")

2
platformio/builder/tools/pioplatform.py
View File

@ -156,7 +156,7 @@ def PrintConfiguration(env): # pylint: disable=too-many-statements
and pkg_metadata
and pkg_metadata.spec.external
):
data.append("(%s)" % pkg_metadata.spec.url)
data.append("(%s)" % pkg_metadata.spec.uri)
if board_config:
data.extend([">", board_config.get("name")])
return data

28
platformio/package/manager/_install.py
View File

@ -92,7 +92,7 @@ class PackageManagerInstallMixin(object):
self.log.info("Installing %s" % click.style(spec.humanize(), fg="cyan"))
if spec.external:
pkg = self.install_from_url(spec.url, spec)
pkg = self.install_from_uri(spec.uri, spec)
else:
pkg = self.install_from_registry(spec, search_qualifiers)
@ -146,24 +146,24 @@ class PackageManagerInstallMixin(object):
}
return self._install(spec, search_qualifiers=search_qualifiers or None)
def install_from_url(self, url, spec, checksum=None):
def install_from_uri(self, uri, spec, checksum=None):
spec = self.ensure_spec(spec)
tmp_dir = tempfile.mkdtemp(prefix="pkg-installing-", dir=self.get_tmp_dir())
vcs = None
try:
if url.startswith("file://"):
_url = url[7:]
if os.path.isfile(_url):
self.unpack(_url, tmp_dir)
if uri.startswith("file://"):
_uri = uri[7:]
if os.path.isfile(_uri):
self.unpack(_uri, tmp_dir)
else:
fs.rmtree(tmp_dir)
shutil.copytree(_url, tmp_dir, symlinks=True)
elif url.startswith(("http://", "https://")):
dl_path = self.download(url, checksum)
shutil.copytree(_uri, tmp_dir, symlinks=True)
elif uri.startswith(("http://", "https://")):
dl_path = self.download(uri, checksum)
assert os.path.isfile(dl_path)
self.unpack(dl_path, tmp_dir)
else:
vcs = VCSClientFactory.new(tmp_dir, url)
vcs = VCSClientFactory.new(tmp_dir, uri)
assert vcs.export()
root_dir = self.find_pkg_root(tmp_dir, spec)
@ -210,7 +210,7 @@ class PackageManagerInstallMixin(object):
)
elif dst_pkg.metadata:
if dst_pkg.metadata.spec.external:
if dst_pkg.metadata.spec.url != tmp_pkg.metadata.spec.url:
if dst_pkg.metadata.spec.uri != tmp_pkg.metadata.spec.uri:
action = "detach-existing"
elif (
dst_pkg.metadata.version != tmp_pkg.metadata.version
@ -231,11 +231,11 @@ class PackageManagerInstallMixin(object):
tmp_pkg.get_safe_dirname(),
dst_pkg.metadata.version,
)
if dst_pkg.metadata.spec.url:
if dst_pkg.metadata.spec.uri:
target_dirname = "%s@src-%s" % (
tmp_pkg.get_safe_dirname(),
hashlib.md5(
compat.hashlib_encode_data(dst_pkg.metadata.spec.url)
compat.hashlib_encode_data(dst_pkg.metadata.spec.uri)
).hexdigest(),
)
# move existing into the new place
@ -256,7 +256,7 @@ class PackageManagerInstallMixin(object):
target_dirname = "%s@src-%s" % (
tmp_pkg.get_safe_dirname(),
hashlib.md5(
compat.hashlib_encode_data(tmp_pkg.metadata.spec.url)
compat.hashlib_encode_data(tmp_pkg.metadata.spec.uri)
).hexdigest(),
)
pkg_dir = os.path.join(self.package_dir, target_dirname)

4
platformio/package/manager/_legacy.py
View File

@ -33,7 +33,7 @@ class PackageManagerLegacyMixin(object):
src_manifest = fs.load_json(src_manifest_path)
return PackageSpec(
name=src_manifest.get("name"),
url=src_manifest.get("url"),
uri=src_manifest.get("url"),
requirements=src_manifest.get("requirements"),
)
@ -51,7 +51,7 @@ class PackageManagerLegacyMixin(object):
if not manifest.get(key):
manifest[key] = str(getattr(pkg.metadata, key))
if pkg.metadata and pkg.metadata.spec and pkg.metadata.spec.external:
manifest["__src_url"] = pkg.metadata.spec.url
manifest["__src_url"] = pkg.metadata.spec.uri
manifest["version"] = str(pkg.metadata.version)
if pkg.metadata and pkg.metadata.spec.owner:
manifest["ownername"] = pkg.metadata.spec.owner

2
platformio/package/manager/_registry.py
View File

@ -102,7 +102,7 @@ class PackageManageRegistryMixin(object):
for url, checksum in RegistryFileMirrorIterator(pkgfile["download_url"]):
try:
return self.install_from_url(
return self.install_from_uri(
url,
PackageSpec(
owner=package["owner"]["username"],

4
platformio/package/manager/_update.py
View File

@ -67,7 +67,7 @@ class PackageManagerUpdateMixin(object):
def _fetch_vcs_latest_version(self, pkg):
vcs = None
try:
vcs = VCSClientFactory.new(pkg.path, pkg.metadata.spec.url, silent=True)
vcs = VCSClientFactory.new(pkg.path, pkg.metadata.spec.uri, silent=True)
except VCSBaseException:
return None
if not vcs.can_be_updated:
@ -112,7 +112,7 @@ class PackageManagerUpdateMixin(object):
def _update(self, pkg, outdated, skip_dependencies=False):
if pkg.metadata.spec.external:
vcs = VCSClientFactory.new(pkg.path, pkg.metadata.spec.url)
vcs = VCSClientFactory.new(pkg.path, pkg.metadata.spec.uri)
assert vcs.update()
pkg.metadata.version = self._fetch_vcs_latest_version(pkg)
pkg.dump_meta()

8
platformio/package/manager/base.py
View File

@ -274,12 +274,12 @@ class BasePackageManager( # pylint: disable=too-many-public-methods,too-many-in
# external "URL" mismatch
if spec.external:
# local folder mismatch
if os.path.abspath(spec.url) == os.path.abspath(pkg.path) or (
spec.url.startswith("file://")
and os.path.abspath(pkg.path) == os.path.abspath(spec.url[7:])
if os.path.abspath(spec.uri) == os.path.abspath(pkg.path) or (
spec.uri.startswith("file://")
and os.path.abspath(pkg.path) == os.path.abspath(spec.uri[7:])
):
return True
if spec.url != pkg.metadata.spec.url:
if spec.uri != pkg.metadata.spec.uri:
return False
# "owner" mismatch

2
platformio/package/manager/platform.py
View File

@ -167,7 +167,7 @@ def remove_unnecessary_platform_packages(dry_run=False):
pm = ToolPackageManager()
for pkg in pm.get_installed():
skip_conds = [
pkg.metadata.spec.url,
pkg.metadata.spec.uri,
os.path.isfile(os.path.join(pkg.path, ".piokeep")),
pkg in required,
pkg in core_packages,

76
platformio/package/meta.py
View File

@ -17,6 +17,7 @@ import os
import re
import tarfile
from binascii import crc32
from urllib.parse import urlparse
import semantic_version
@ -24,11 +25,6 @@ from platformio.compat import get_object_members, hashlib_encode_data, string_ty
from platformio.package.manifest.parser import ManifestFileType
from platformio.package.version import cast_version_to_semver
try:
from urllib.parse import urlparse
except ImportError:
from urlparse import urlparse
class PackageType(object):
LIBRARY = "library"
@ -127,19 +123,19 @@ class PackageOutdatedResult(object):
class PackageSpec(object): # pylint: disable=too-many-instance-attributes
def __init__( # pylint: disable=redefined-builtin,too-many-arguments
self, raw=None, owner=None, id=None, name=None, requirements=None, url=None
self, raw=None, owner=None, id=None, name=None, requirements=None, uri=None
):
self._requirements = None
self.owner = owner
self.id = id
self.name = name
self.url = url
self.uri = uri
self.raw = raw
if requirements:
try:
self.requirements = requirements
except ValueError as exc:
if not self.name or self.url or self.raw:
if not self.name or self.uri or self.raw:
raise exc
self.raw = "%s=%s" % (self.name, requirements)
self._name_is_custom = False
@ -152,7 +148,7 @@ class PackageSpec(object): # pylint: disable=too-many-instance-attributes
self.id == other.id,
self.name == other.name,
self.requirements == other.requirements,
self.url == other.url,
self.uri == other.uri,
]
)
@ -160,19 +156,19 @@ class PackageSpec(object): # pylint: disable=too-many-instance-attributes
return crc32(
hashlib_encode_data(
"%s-%s-%s-%s-%s"
% (self.owner, self.id, self.name, self.requirements, self.url)
% (self.owner, self.id, self.name, self.requirements, self.uri)
)
)
def __repr__(self):
return (
"PackageSpec <owner={owner} id={id} name={name} "
"requirements={requirements} url={url}>".format(**self.as_dict())
"requirements={requirements} uri={uri}>".format(**self.as_dict())
)
@property
def external(self):
return bool(self.url)
return bool(self.uri)
@property
def requirements(self):
@ -191,8 +187,8 @@ class PackageSpec(object): # pylint: disable=too-many-instance-attributes
def humanize(self):
result = ""
if self.url:
result = self.url
if self.uri:
result = self.uri
elif self.name:
if self.owner:
result = self.owner + "/"
@ -212,12 +208,12 @@ class PackageSpec(object): # pylint: disable=too-many-instance-attributes
id=self.id,
name=self.name,
requirements=str(self.requirements) if self.requirements else None,
url=self.url,
uri=self.uri,
)
def as_dependency(self):
if self.url:
return self.raw or self.url
if self.uri:
return self.raw or self.uri
result = ""
if self.name:
result = "%s/%s" % (self.owner, self.name) if self.owner else self.name
@ -241,16 +237,16 @@ class PackageSpec(object): # pylint: disable=too-many-instance-attributes
self._parse_custom_name,
self._parse_id,
self._parse_owner,
self._parse_url,
self._parse_uri,
)
for parser in parsers:
if raw is None:
break
raw = parser(raw)
# if name is not custom, parse it from URL
if not self.name and self.url:
self.name = self._parse_name_from_url(self.url)
# if name is not custom, parse it from URI
if not self.name and self.uri:
self.name = self._parse_name_from_uri(self.uri)
elif raw:
# the leftover is a package name
self.name = raw
@ -298,14 +294,18 @@ class PackageSpec(object): # pylint: disable=too-many-instance-attributes
self.name = tokens[1].strip()
return None
def _parse_url(self, raw):
def _parse_uri(self, raw):
if not any(s in raw for s in ("@", ":", "/")):
return raw
self.url = raw.strip()
parts = urlparse(self.url)
self.uri = raw.strip()
parts = urlparse(self.uri)
# if local file or valid URL with scheme vcs+protocol://
if parts.scheme == "file" or "+" in parts.scheme or self.url.startswith("git+"):
# if local file or valid URI with scheme vcs+protocol://
if (
parts.scheme in ("file", )
or "+" in parts.scheme
or self.uri.startswith("git+")
):
return None
# parse VCS
@ -323,29 +323,29 @@ class PackageSpec(object): # pylint: disable=too-many-instance-attributes
in ("mbed.com", "os.mbed.com", "developer.mbed.org")
]
if any(git_conditions):
self.url = "git+" + self.url
self.uri = "git+" + self.uri
elif any(hg_conditions):
self.url = "hg+" + self.url
self.uri = "hg+" + self.uri
return None
@staticmethod
def _parse_name_from_url(url):
if url.endswith("/"):
url = url[:-1]
def _parse_name_from_uri(uri):
if uri.endswith("/"):
uri = uri[:-1]
stop_chars = ["#", "?"]
if url.startswith("file://"):
if uri.startswith(("file://", )):
stop_chars.append("@") # detached path
for c in stop_chars:
if c in url:
url = url[: url.index(c)]
if c in uri:
uri = uri[: uri.index(c)]
# parse real repository name from Github
parts = urlparse(url)
parts = urlparse(uri)
if parts.netloc == "github.com" and parts.path.count("/") > 2:
return parts.path.split("/")[2]
name = os.path.basename(url)
name = os.path.basename(uri)
if "." in name:
return name.split(".", 1)[0].strip()
return name
@ -412,6 +412,10 @@ class PackageMetaData(object):
with open(path, encoding="utf8") as fp:
data = json.load(fp)
if data["spec"]:
# legacy support for Core<5.3 packages
if "url" in data["spec"]:
data["spec"]["uri"] = data["spec"]["url"]
del data["spec"]["url"]
data["spec"] = PackageSpec(**data["spec"])
return PackageMetaData(**data)

2
platformio/platform/_packages.py
View File

@ -62,7 +62,7 @@ class PlatformPackagesMixin(object):
continue
item = {"name": pkg.metadata.name, "version": str(pkg.metadata.version)}
if pkg.metadata.spec.external:
item["src_url"] = pkg.metadata.spec.url
item["src_url"] = pkg.metadata.spec.uri
result.append(item)
return result

12
tests/package/test_manager.py
View File

@ -23,7 +23,6 @@ import pytest
import semantic_version
from platformio import fs, util
from platformio.compat import PY2
from platformio.package.exception import (
MissingPackageManifestError,
UnknownPackageError,
@ -106,7 +105,7 @@ def test_build_legacy_spec(isolated_pio_core, tmpdir_factory):
)
assert pm.build_legacy_spec(str(pkg1_dir)) == PackageSpec(
name="StreamSpy-0.0.1.tar",
url="https://dl.platformio.org/e8936b7/StreamSpy-0.0.1.tar.gz",
uri="https://dl.platformio.org/e8936b7/StreamSpy-0.0.1.tar.gz",
)
# without src manifest
@ -148,8 +147,7 @@ def test_build_metadata(isolated_pio_core, tmpdir_factory):
assert metadata.version.build[1] == vcs_revision
@pytest.mark.skipif(PY2, reason="Requires Python 3.5 or higher")
def test_install_from_url(isolated_pio_core, tmpdir_factory):
def test_install_from_uri(isolated_pio_core, tmpdir_factory):
tmp_dir = tmpdir_factory.mktemp("tmp")
storage_dir = tmpdir_factory.mktemp("storage")
lm = LibraryPackageManager(str(storage_dir))
@ -189,7 +187,7 @@ version = 5.2.7
"""
)
spec = PackageSpec("company/wifilib @ ^5")
pkg = lm.install_from_url("file://%s" % src_dir, spec)
pkg = lm.install_from_uri("file://%s" % src_dir, spec)
assert str(pkg.metadata.version) == "5.2.7"
# check package folder names
@ -416,11 +414,11 @@ def test_uninstall(isolated_pio_core, tmpdir_factory):
# foo @ 1.0.0
pkg_dir = tmp_dir.join("foo").mkdir()
pkg_dir.join("library.json").write('{"name": "foo", "version": "1.0.0"}')
foo_1_0_0_pkg = lm.install_from_url("file://%s" % pkg_dir, "foo")
foo_1_0_0_pkg = lm.install_from_uri("file://%s" % pkg_dir, "foo")
# foo @ 1.3.0
pkg_dir = tmp_dir.join("foo-1.3.0").mkdir()
pkg_dir.join("library.json").write('{"name": "foo", "version": "1.3.0"}')
lm.install_from_url("file://%s" % pkg_dir, "foo")
lm.install_from_uri("file://%s" % pkg_dir, "foo")
# bar
pkg_dir = tmp_dir.join("bar").mkdir()
pkg_dir.join("library.json").write('{"name": "bar", "version": "1.0.0"}')

38
tests/package/test_meta.py
View File

@ -82,22 +82,22 @@ def test_spec_requirements():
def test_spec_local_urls(tmpdir_factory):
assert PackageSpec("file:///tmp/foo.tar.gz") == PackageSpec(
url="file:///tmp/foo.tar.gz", name="foo"
uri="file:///tmp/foo.tar.gz", name="foo"
)
assert PackageSpec("customName=file:///tmp/bar.zip") == PackageSpec(
url="file:///tmp/bar.zip", name="customName"
uri="file:///tmp/bar.zip", name="customName"
)
assert PackageSpec("file:///tmp/some-lib/") == PackageSpec(
url="file:///tmp/some-lib/", name="some-lib"
uri="file:///tmp/some-lib/", name="some-lib"
)
# detached package
assert PackageSpec("file:///tmp/some-lib@src-67e1043a673d2") == PackageSpec(
url="file:///tmp/some-lib@src-67e1043a673d2", name="some-lib"
uri="file:///tmp/some-lib@src-67e1043a673d2", name="some-lib"
)
# detached folder without scheme
pkg_dir = tmpdir_factory.mktemp("storage").join("detached@1.2.3").mkdir()
assert PackageSpec(str(pkg_dir)) == PackageSpec(
name="detached", url="file://%s" % pkg_dir
name="detached", uri="file://%s" % pkg_dir
)
@ -105,14 +105,14 @@ def test_spec_external_urls():
assert PackageSpec(
"https://github.com/platformio/platformio-core/archive/develop.zip"
) == PackageSpec(
url="https://github.com/platformio/platformio-core/archive/develop.zip",
uri="https://github.com/platformio/platformio-core/archive/develop.zip",
name="platformio-core",
)
assert PackageSpec(
"https://github.com/platformio/platformio-core/archive/develop.zip?param=value"
" @ !=2"
) == PackageSpec(
url="https://github.com/platformio/platformio-core/archive/"
uri="https://github.com/platformio/platformio-core/archive/"
"develop.zip?param=value",
name="platformio-core",
requirements="!=2",
@ -125,7 +125,7 @@ def test_spec_external_urls():
assert spec.has_custom_name()
assert spec.name == "Custom-Name"
assert spec == PackageSpec(
url="https://github.com/platformio/platformio-core/archive/develop.tar.gz",
uri="https://github.com/platformio/platformio-core/archive/develop.tar.gz",
name="Custom-Name",
requirements="4.4.0",
)
@ -133,40 +133,40 @@ def test_spec_external_urls():
def test_spec_vcs_urls():
assert PackageSpec("https://github.com/platformio/platformio-core") == PackageSpec(
name="platformio-core", url="git+https://github.com/platformio/platformio-core"
name="platformio-core", uri="git+https://github.com/platformio/platformio-core"
)
assert PackageSpec("https://gitlab.com/username/reponame") == PackageSpec(
name="reponame", url="git+https://gitlab.com/username/reponame"
name="reponame", uri="git+https://gitlab.com/username/reponame"
)
assert PackageSpec(
"wolfSSL=https://os.mbed.com/users/wolfSSL/code/wolfSSL/"
) == PackageSpec(
name="wolfSSL", url="hg+https://os.mbed.com/users/wolfSSL/code/wolfSSL/"
name="wolfSSL", uri="hg+https://os.mbed.com/users/wolfSSL/code/wolfSSL/"
)
assert PackageSpec(
"https://github.com/platformio/platformio-core.git#master"
) == PackageSpec(
name="platformio-core",
url="git+https://github.com/platformio/platformio-core.git#master",
uri="git+https://github.com/platformio/platformio-core.git#master",
)
assert PackageSpec(
"core=git+ssh://github.com/platformio/platformio-core.git#v4.4.0@4.4.0"
) == PackageSpec(
name="core",
url="git+ssh://github.com/platformio/platformio-core.git#v4.4.0",
uri="git+ssh://github.com/platformio/platformio-core.git#v4.4.0",
requirements="4.4.0",
)
assert PackageSpec(
"username@github.com:platformio/platformio-core.git"
) == PackageSpec(
name="platformio-core",
url="git+username@github.com:platformio/platformio-core.git",
uri="git+username@github.com:platformio/platformio-core.git",
)
assert PackageSpec(
"pkg=git+git@github.com:platformio/platformio-core.git @ ^1.2.3,!=5"
) == PackageSpec(
name="pkg",
url="git+git@github.com:platformio/platformio-core.git",
uri="git+git@github.com:platformio/platformio-core.git",
requirements="^1.2.3,!=5",
)
assert PackageSpec(
@ -176,7 +176,7 @@ def test_spec_vcs_urls():
) == PackageSpec(
owner="platformio",
name="external-repo",
url="git+https://github.com/platformio/platformio-core",
uri="git+https://github.com/platformio/platformio-core",
)
@ -188,7 +188,7 @@ def test_spec_as_dict():
"id": None,
"name": "foo",
"requirements": "1.2.3",
"url": None,
"uri": None,
},
)
assert not jsondiff.diff(
@ -201,7 +201,7 @@ def test_spec_as_dict():
"id": None,
"name": "platformio-core",
"requirements": "!=2",
"url": "https://github.com/platformio/platformio-core/archive/develop.zip?param=value",
"uri": "https://github.com/platformio/platformio-core/archive/develop.zip?param=value",
},
)
@ -255,7 +255,7 @@ def test_metadata_as_dict():
"id": None,
"name": "toolchain",
"requirements": "~2.0.0",
"url": None,
"uri": None,
},
},
)