From a03d82ff1a534e561f0f3230ba8dbd20d9002888 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 4 Apr 2022 14:18:11 +0300 Subject: [PATCH] Replace package meta URL with URI --- docs | 2 +- platformio/builder/tools/piolib.py | 2 +- platformio/builder/tools/pioplatform.py | 2 +- platformio/package/manager/_install.py | 28 ++++----- platformio/package/manager/_legacy.py | 4 +- platformio/package/manager/_registry.py | 2 +- platformio/package/manager/_update.py | 4 +- platformio/package/manager/base.py | 8 +-- platformio/package/manager/platform.py | 2 +- platformio/package/meta.py | 76 +++++++++++++------------ platformio/platform/_packages.py | 2 +- tests/package/test_manager.py | 12 ++-- tests/package/test_meta.py | 38 ++++++------- 13 files changed, 92 insertions(+), 90 deletions(-) diff --git a/docs b/docs index a0c38a91..e24bd4f1 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit a0c38a913845ca86a7d2495d7c9b2524dd948c73 +Subproject commit e24bd4f12de6da6646bdcbf45fafced11032aa3e diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index 2044a46d..cf6f4239 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -1102,7 +1102,7 @@ def ConfigureProjectLibBuilder(env): "(License: %s, " % (_get_lib_license(pkg) or "Unknown"), nl=False ) if pkg.metadata and pkg.metadata.spec.external: - click.echo("URI: %s, " % pkg.metadata.spec.url, nl=False) + click.echo("URI: %s, " % pkg.metadata.spec.uri, nl=False) click.echo("Path: %s" % lb.path, nl=False) click.echo(")", nl=False) click.echo("") diff --git a/platformio/builder/tools/pioplatform.py b/platformio/builder/tools/pioplatform.py index 1f83913b..573b2906 100644 --- a/platformio/builder/tools/pioplatform.py +++ b/platformio/builder/tools/pioplatform.py @@ -156,7 +156,7 @@ def PrintConfiguration(env): # pylint: disable=too-many-statements and pkg_metadata and pkg_metadata.spec.external ): - data.append("(%s)" % pkg_metadata.spec.url) + data.append("(%s)" % pkg_metadata.spec.uri) if board_config: data.extend([">", board_config.get("name")]) return data diff --git a/platformio/package/manager/_install.py b/platformio/package/manager/_install.py index ed0157a5..f3631844 100644 --- a/platformio/package/manager/_install.py +++ b/platformio/package/manager/_install.py @@ -92,7 +92,7 @@ class PackageManagerInstallMixin(object): self.log.info("Installing %s" % click.style(spec.humanize(), fg="cyan")) if spec.external: - pkg = self.install_from_url(spec.url, spec) + pkg = self.install_from_uri(spec.uri, spec) else: pkg = self.install_from_registry(spec, search_qualifiers) @@ -146,24 +146,24 @@ class PackageManagerInstallMixin(object): } return self._install(spec, search_qualifiers=search_qualifiers or None) - def install_from_url(self, url, spec, checksum=None): + def install_from_uri(self, uri, spec, checksum=None): spec = self.ensure_spec(spec) tmp_dir = tempfile.mkdtemp(prefix="pkg-installing-", dir=self.get_tmp_dir()) vcs = None try: - if url.startswith("file://"): - _url = url[7:] - if os.path.isfile(_url): - self.unpack(_url, tmp_dir) + if uri.startswith("file://"): + _uri = uri[7:] + if os.path.isfile(_uri): + self.unpack(_uri, tmp_dir) else: fs.rmtree(tmp_dir) - shutil.copytree(_url, tmp_dir, symlinks=True) - elif url.startswith(("http://", "https://")): - dl_path = self.download(url, checksum) + shutil.copytree(_uri, tmp_dir, symlinks=True) + elif uri.startswith(("http://", "https://")): + dl_path = self.download(uri, checksum) assert os.path.isfile(dl_path) self.unpack(dl_path, tmp_dir) else: - vcs = VCSClientFactory.new(tmp_dir, url) + vcs = VCSClientFactory.new(tmp_dir, uri) assert vcs.export() root_dir = self.find_pkg_root(tmp_dir, spec) @@ -210,7 +210,7 @@ class PackageManagerInstallMixin(object): ) elif dst_pkg.metadata: if dst_pkg.metadata.spec.external: - if dst_pkg.metadata.spec.url != tmp_pkg.metadata.spec.url: + if dst_pkg.metadata.spec.uri != tmp_pkg.metadata.spec.uri: action = "detach-existing" elif ( dst_pkg.metadata.version != tmp_pkg.metadata.version @@ -231,11 +231,11 @@ class PackageManagerInstallMixin(object): tmp_pkg.get_safe_dirname(), dst_pkg.metadata.version, ) - if dst_pkg.metadata.spec.url: + if dst_pkg.metadata.spec.uri: target_dirname = "%s@src-%s" % ( tmp_pkg.get_safe_dirname(), hashlib.md5( - compat.hashlib_encode_data(dst_pkg.metadata.spec.url) + compat.hashlib_encode_data(dst_pkg.metadata.spec.uri) ).hexdigest(), ) # move existing into the new place @@ -256,7 +256,7 @@ class PackageManagerInstallMixin(object): target_dirname = "%s@src-%s" % ( tmp_pkg.get_safe_dirname(), hashlib.md5( - compat.hashlib_encode_data(tmp_pkg.metadata.spec.url) + compat.hashlib_encode_data(tmp_pkg.metadata.spec.uri) ).hexdigest(), ) pkg_dir = os.path.join(self.package_dir, target_dirname) diff --git a/platformio/package/manager/_legacy.py b/platformio/package/manager/_legacy.py index 5c35ebeb..978efc9c 100644 --- a/platformio/package/manager/_legacy.py +++ b/platformio/package/manager/_legacy.py @@ -33,7 +33,7 @@ class PackageManagerLegacyMixin(object): src_manifest = fs.load_json(src_manifest_path) return PackageSpec( name=src_manifest.get("name"), - url=src_manifest.get("url"), + uri=src_manifest.get("url"), requirements=src_manifest.get("requirements"), ) @@ -51,7 +51,7 @@ class PackageManagerLegacyMixin(object): if not manifest.get(key): manifest[key] = str(getattr(pkg.metadata, key)) if pkg.metadata and pkg.metadata.spec and pkg.metadata.spec.external: - manifest["__src_url"] = pkg.metadata.spec.url + manifest["__src_url"] = pkg.metadata.spec.uri manifest["version"] = str(pkg.metadata.version) if pkg.metadata and pkg.metadata.spec.owner: manifest["ownername"] = pkg.metadata.spec.owner diff --git a/platformio/package/manager/_registry.py b/platformio/package/manager/_registry.py index 061f63a8..d96c8bc3 100644 --- a/platformio/package/manager/_registry.py +++ b/platformio/package/manager/_registry.py @@ -102,7 +102,7 @@ class PackageManageRegistryMixin(object): for url, checksum in RegistryFileMirrorIterator(pkgfile["download_url"]): try: - return self.install_from_url( + return self.install_from_uri( url, PackageSpec( owner=package["owner"]["username"], diff --git a/platformio/package/manager/_update.py b/platformio/package/manager/_update.py index 3f71cd7e..c5ae3d8d 100644 --- a/platformio/package/manager/_update.py +++ b/platformio/package/manager/_update.py @@ -67,7 +67,7 @@ class PackageManagerUpdateMixin(object): def _fetch_vcs_latest_version(self, pkg): vcs = None try: - vcs = VCSClientFactory.new(pkg.path, pkg.metadata.spec.url, silent=True) + vcs = VCSClientFactory.new(pkg.path, pkg.metadata.spec.uri, silent=True) except VCSBaseException: return None if not vcs.can_be_updated: @@ -112,7 +112,7 @@ class PackageManagerUpdateMixin(object): def _update(self, pkg, outdated, skip_dependencies=False): if pkg.metadata.spec.external: - vcs = VCSClientFactory.new(pkg.path, pkg.metadata.spec.url) + vcs = VCSClientFactory.new(pkg.path, pkg.metadata.spec.uri) assert vcs.update() pkg.metadata.version = self._fetch_vcs_latest_version(pkg) pkg.dump_meta() diff --git a/platformio/package/manager/base.py b/platformio/package/manager/base.py index b3db0ce0..76e41a36 100644 --- a/platformio/package/manager/base.py +++ b/platformio/package/manager/base.py @@ -274,12 +274,12 @@ class BasePackageManager( # pylint: disable=too-many-public-methods,too-many-in # external "URL" mismatch if spec.external: # local folder mismatch - if os.path.abspath(spec.url) == os.path.abspath(pkg.path) or ( - spec.url.startswith("file://") - and os.path.abspath(pkg.path) == os.path.abspath(spec.url[7:]) + if os.path.abspath(spec.uri) == os.path.abspath(pkg.path) or ( + spec.uri.startswith("file://") + and os.path.abspath(pkg.path) == os.path.abspath(spec.uri[7:]) ): return True - if spec.url != pkg.metadata.spec.url: + if spec.uri != pkg.metadata.spec.uri: return False # "owner" mismatch diff --git a/platformio/package/manager/platform.py b/platformio/package/manager/platform.py index e7322f03..608b9ca0 100644 --- a/platformio/package/manager/platform.py +++ b/platformio/package/manager/platform.py @@ -167,7 +167,7 @@ def remove_unnecessary_platform_packages(dry_run=False): pm = ToolPackageManager() for pkg in pm.get_installed(): skip_conds = [ - pkg.metadata.spec.url, + pkg.metadata.spec.uri, os.path.isfile(os.path.join(pkg.path, ".piokeep")), pkg in required, pkg in core_packages, diff --git a/platformio/package/meta.py b/platformio/package/meta.py index 5458dd71..be494608 100644 --- a/platformio/package/meta.py +++ b/platformio/package/meta.py @@ -17,6 +17,7 @@ import os import re import tarfile from binascii import crc32 +from urllib.parse import urlparse import semantic_version @@ -24,11 +25,6 @@ from platformio.compat import get_object_members, hashlib_encode_data, string_ty from platformio.package.manifest.parser import ManifestFileType from platformio.package.version import cast_version_to_semver -try: - from urllib.parse import urlparse -except ImportError: - from urlparse import urlparse - class PackageType(object): LIBRARY = "library" @@ -127,19 +123,19 @@ class PackageOutdatedResult(object): class PackageSpec(object): # pylint: disable=too-many-instance-attributes def __init__( # pylint: disable=redefined-builtin,too-many-arguments - self, raw=None, owner=None, id=None, name=None, requirements=None, url=None + self, raw=None, owner=None, id=None, name=None, requirements=None, uri=None ): self._requirements = None self.owner = owner self.id = id self.name = name - self.url = url + self.uri = uri self.raw = raw if requirements: try: self.requirements = requirements except ValueError as exc: - if not self.name or self.url or self.raw: + if not self.name or self.uri or self.raw: raise exc self.raw = "%s=%s" % (self.name, requirements) self._name_is_custom = False @@ -152,7 +148,7 @@ class PackageSpec(object): # pylint: disable=too-many-instance-attributes self.id == other.id, self.name == other.name, self.requirements == other.requirements, - self.url == other.url, + self.uri == other.uri, ] ) @@ -160,19 +156,19 @@ class PackageSpec(object): # pylint: disable=too-many-instance-attributes return crc32( hashlib_encode_data( "%s-%s-%s-%s-%s" - % (self.owner, self.id, self.name, self.requirements, self.url) + % (self.owner, self.id, self.name, self.requirements, self.uri) ) ) def __repr__(self): return ( "PackageSpec ".format(**self.as_dict()) + "requirements={requirements} uri={uri}>".format(**self.as_dict()) ) @property def external(self): - return bool(self.url) + return bool(self.uri) @property def requirements(self): @@ -191,8 +187,8 @@ class PackageSpec(object): # pylint: disable=too-many-instance-attributes def humanize(self): result = "" - if self.url: - result = self.url + if self.uri: + result = self.uri elif self.name: if self.owner: result = self.owner + "/" @@ -212,12 +208,12 @@ class PackageSpec(object): # pylint: disable=too-many-instance-attributes id=self.id, name=self.name, requirements=str(self.requirements) if self.requirements else None, - url=self.url, + uri=self.uri, ) def as_dependency(self): - if self.url: - return self.raw or self.url + if self.uri: + return self.raw or self.uri result = "" if self.name: result = "%s/%s" % (self.owner, self.name) if self.owner else self.name @@ -241,16 +237,16 @@ class PackageSpec(object): # pylint: disable=too-many-instance-attributes self._parse_custom_name, self._parse_id, self._parse_owner, - self._parse_url, + self._parse_uri, ) for parser in parsers: if raw is None: break raw = parser(raw) - # if name is not custom, parse it from URL - if not self.name and self.url: - self.name = self._parse_name_from_url(self.url) + # if name is not custom, parse it from URI + if not self.name and self.uri: + self.name = self._parse_name_from_uri(self.uri) elif raw: # the leftover is a package name self.name = raw @@ -298,14 +294,18 @@ class PackageSpec(object): # pylint: disable=too-many-instance-attributes self.name = tokens[1].strip() return None - def _parse_url(self, raw): + def _parse_uri(self, raw): if not any(s in raw for s in ("@", ":", "/")): return raw - self.url = raw.strip() - parts = urlparse(self.url) + self.uri = raw.strip() + parts = urlparse(self.uri) - # if local file or valid URL with scheme vcs+protocol:// - if parts.scheme == "file" or "+" in parts.scheme or self.url.startswith("git+"): + # if local file or valid URI with scheme vcs+protocol:// + if ( + parts.scheme in ("file", ) + or "+" in parts.scheme + or self.uri.startswith("git+") + ): return None # parse VCS @@ -323,29 +323,29 @@ class PackageSpec(object): # pylint: disable=too-many-instance-attributes in ("mbed.com", "os.mbed.com", "developer.mbed.org") ] if any(git_conditions): - self.url = "git+" + self.url + self.uri = "git+" + self.uri elif any(hg_conditions): - self.url = "hg+" + self.url + self.uri = "hg+" + self.uri return None @staticmethod - def _parse_name_from_url(url): - if url.endswith("/"): - url = url[:-1] + def _parse_name_from_uri(uri): + if uri.endswith("/"): + uri = uri[:-1] stop_chars = ["#", "?"] - if url.startswith("file://"): + if uri.startswith(("file://", )): stop_chars.append("@") # detached path for c in stop_chars: - if c in url: - url = url[: url.index(c)] + if c in uri: + uri = uri[: uri.index(c)] # parse real repository name from Github - parts = urlparse(url) + parts = urlparse(uri) if parts.netloc == "github.com" and parts.path.count("/") > 2: return parts.path.split("/")[2] - name = os.path.basename(url) + name = os.path.basename(uri) if "." in name: return name.split(".", 1)[0].strip() return name @@ -412,6 +412,10 @@ class PackageMetaData(object): with open(path, encoding="utf8") as fp: data = json.load(fp) if data["spec"]: + # legacy support for Core<5.3 packages + if "url" in data["spec"]: + data["spec"]["uri"] = data["spec"]["url"] + del data["spec"]["url"] data["spec"] = PackageSpec(**data["spec"]) return PackageMetaData(**data) diff --git a/platformio/platform/_packages.py b/platformio/platform/_packages.py index c36cb956..c741c790 100644 --- a/platformio/platform/_packages.py +++ b/platformio/platform/_packages.py @@ -62,7 +62,7 @@ class PlatformPackagesMixin(object): continue item = {"name": pkg.metadata.name, "version": str(pkg.metadata.version)} if pkg.metadata.spec.external: - item["src_url"] = pkg.metadata.spec.url + item["src_url"] = pkg.metadata.spec.uri result.append(item) return result diff --git a/tests/package/test_manager.py b/tests/package/test_manager.py index ab9aa6c3..12cedd40 100644 --- a/tests/package/test_manager.py +++ b/tests/package/test_manager.py @@ -23,7 +23,6 @@ import pytest import semantic_version from platformio import fs, util -from platformio.compat import PY2 from platformio.package.exception import ( MissingPackageManifestError, UnknownPackageError, @@ -106,7 +105,7 @@ def test_build_legacy_spec(isolated_pio_core, tmpdir_factory): ) assert pm.build_legacy_spec(str(pkg1_dir)) == PackageSpec( name="StreamSpy-0.0.1.tar", - url="https://dl.platformio.org/e8936b7/StreamSpy-0.0.1.tar.gz", + uri="https://dl.platformio.org/e8936b7/StreamSpy-0.0.1.tar.gz", ) # without src manifest @@ -148,8 +147,7 @@ def test_build_metadata(isolated_pio_core, tmpdir_factory): assert metadata.version.build[1] == vcs_revision -@pytest.mark.skipif(PY2, reason="Requires Python 3.5 or higher") -def test_install_from_url(isolated_pio_core, tmpdir_factory): +def test_install_from_uri(isolated_pio_core, tmpdir_factory): tmp_dir = tmpdir_factory.mktemp("tmp") storage_dir = tmpdir_factory.mktemp("storage") lm = LibraryPackageManager(str(storage_dir)) @@ -189,7 +187,7 @@ version = 5.2.7 """ ) spec = PackageSpec("company/wifilib @ ^5") - pkg = lm.install_from_url("file://%s" % src_dir, spec) + pkg = lm.install_from_uri("file://%s" % src_dir, spec) assert str(pkg.metadata.version) == "5.2.7" # check package folder names @@ -416,11 +414,11 @@ def test_uninstall(isolated_pio_core, tmpdir_factory): # foo @ 1.0.0 pkg_dir = tmp_dir.join("foo").mkdir() pkg_dir.join("library.json").write('{"name": "foo", "version": "1.0.0"}') - foo_1_0_0_pkg = lm.install_from_url("file://%s" % pkg_dir, "foo") + foo_1_0_0_pkg = lm.install_from_uri("file://%s" % pkg_dir, "foo") # foo @ 1.3.0 pkg_dir = tmp_dir.join("foo-1.3.0").mkdir() pkg_dir.join("library.json").write('{"name": "foo", "version": "1.3.0"}') - lm.install_from_url("file://%s" % pkg_dir, "foo") + lm.install_from_uri("file://%s" % pkg_dir, "foo") # bar pkg_dir = tmp_dir.join("bar").mkdir() pkg_dir.join("library.json").write('{"name": "bar", "version": "1.0.0"}') diff --git a/tests/package/test_meta.py b/tests/package/test_meta.py index 1cda6409..869f0a89 100644 --- a/tests/package/test_meta.py +++ b/tests/package/test_meta.py @@ -82,22 +82,22 @@ def test_spec_requirements(): def test_spec_local_urls(tmpdir_factory): assert PackageSpec("file:///tmp/foo.tar.gz") == PackageSpec( - url="file:///tmp/foo.tar.gz", name="foo" + uri="file:///tmp/foo.tar.gz", name="foo" ) assert PackageSpec("customName=file:///tmp/bar.zip") == PackageSpec( - url="file:///tmp/bar.zip", name="customName" + uri="file:///tmp/bar.zip", name="customName" ) assert PackageSpec("file:///tmp/some-lib/") == PackageSpec( - url="file:///tmp/some-lib/", name="some-lib" + uri="file:///tmp/some-lib/", name="some-lib" ) # detached package assert PackageSpec("file:///tmp/some-lib@src-67e1043a673d2") == PackageSpec( - url="file:///tmp/some-lib@src-67e1043a673d2", name="some-lib" + uri="file:///tmp/some-lib@src-67e1043a673d2", name="some-lib" ) # detached folder without scheme pkg_dir = tmpdir_factory.mktemp("storage").join("detached@1.2.3").mkdir() assert PackageSpec(str(pkg_dir)) == PackageSpec( - name="detached", url="file://%s" % pkg_dir + name="detached", uri="file://%s" % pkg_dir ) @@ -105,14 +105,14 @@ def test_spec_external_urls(): assert PackageSpec( "https://github.com/platformio/platformio-core/archive/develop.zip" ) == PackageSpec( - url="https://github.com/platformio/platformio-core/archive/develop.zip", + uri="https://github.com/platformio/platformio-core/archive/develop.zip", name="platformio-core", ) assert PackageSpec( "https://github.com/platformio/platformio-core/archive/develop.zip?param=value" " @ !=2" ) == PackageSpec( - url="https://github.com/platformio/platformio-core/archive/" + uri="https://github.com/platformio/platformio-core/archive/" "develop.zip?param=value", name="platformio-core", requirements="!=2", @@ -125,7 +125,7 @@ def test_spec_external_urls(): assert spec.has_custom_name() assert spec.name == "Custom-Name" assert spec == PackageSpec( - url="https://github.com/platformio/platformio-core/archive/develop.tar.gz", + uri="https://github.com/platformio/platformio-core/archive/develop.tar.gz", name="Custom-Name", requirements="4.4.0", ) @@ -133,40 +133,40 @@ def test_spec_external_urls(): def test_spec_vcs_urls(): assert PackageSpec("https://github.com/platformio/platformio-core") == PackageSpec( - name="platformio-core", url="git+https://github.com/platformio/platformio-core" + name="platformio-core", uri="git+https://github.com/platformio/platformio-core" ) assert PackageSpec("https://gitlab.com/username/reponame") == PackageSpec( - name="reponame", url="git+https://gitlab.com/username/reponame" + name="reponame", uri="git+https://gitlab.com/username/reponame" ) assert PackageSpec( "wolfSSL=https://os.mbed.com/users/wolfSSL/code/wolfSSL/" ) == PackageSpec( - name="wolfSSL", url="hg+https://os.mbed.com/users/wolfSSL/code/wolfSSL/" + name="wolfSSL", uri="hg+https://os.mbed.com/users/wolfSSL/code/wolfSSL/" ) assert PackageSpec( "https://github.com/platformio/platformio-core.git#master" ) == PackageSpec( name="platformio-core", - url="git+https://github.com/platformio/platformio-core.git#master", + uri="git+https://github.com/platformio/platformio-core.git#master", ) assert PackageSpec( "core=git+ssh://github.com/platformio/platformio-core.git#v4.4.0@4.4.0" ) == PackageSpec( name="core", - url="git+ssh://github.com/platformio/platformio-core.git#v4.4.0", + uri="git+ssh://github.com/platformio/platformio-core.git#v4.4.0", requirements="4.4.0", ) assert PackageSpec( "username@github.com:platformio/platformio-core.git" ) == PackageSpec( name="platformio-core", - url="git+username@github.com:platformio/platformio-core.git", + uri="git+username@github.com:platformio/platformio-core.git", ) assert PackageSpec( "pkg=git+git@github.com:platformio/platformio-core.git @ ^1.2.3,!=5" ) == PackageSpec( name="pkg", - url="git+git@github.com:platformio/platformio-core.git", + uri="git+git@github.com:platformio/platformio-core.git", requirements="^1.2.3,!=5", ) assert PackageSpec( @@ -176,7 +176,7 @@ def test_spec_vcs_urls(): ) == PackageSpec( owner="platformio", name="external-repo", - url="git+https://github.com/platformio/platformio-core", + uri="git+https://github.com/platformio/platformio-core", ) @@ -188,7 +188,7 @@ def test_spec_as_dict(): "id": None, "name": "foo", "requirements": "1.2.3", - "url": None, + "uri": None, }, ) assert not jsondiff.diff( @@ -201,7 +201,7 @@ def test_spec_as_dict(): "id": None, "name": "platformio-core", "requirements": "!=2", - "url": "https://github.com/platformio/platformio-core/archive/develop.zip?param=value", + "uri": "https://github.com/platformio/platformio-core/archive/develop.zip?param=value", }, ) @@ -255,7 +255,7 @@ def test_metadata_as_dict(): "id": None, "name": "toolchain", "requirements": "~2.0.0", - "url": None, + "uri": None, }, }, )