Replace package meta URL with URI

platformio/builder/tools/piolib.py

@@ -1102,7 +1102,7 @@ def ConfigureProjectLibBuilder(env):
                     "(License: %s, " % (_get_lib_license(pkg) or "Unknown"), nl=False
                 )
                 if pkg.metadata and pkg.metadata.spec.external:
-                    click.echo("URI: %s, " % pkg.metadata.spec.url, nl=False)
+                    click.echo("URI: %s, " % pkg.metadata.spec.uri, nl=False)
                 click.echo("Path: %s" % lb.path, nl=False)
                 click.echo(")", nl=False)
             click.echo("")

platformio/builder/tools/pioplatform.py

@@ -156,7 +156,7 @@ def PrintConfiguration(env):  # pylint: disable=too-many-statements
             and pkg_metadata
             and pkg_metadata.spec.external
         ):
-            data.append("(%s)" % pkg_metadata.spec.url)
+            data.append("(%s)" % pkg_metadata.spec.uri)
         if board_config:
             data.extend([">", board_config.get("name")])
         return data

platformio/package/manager/_install.py

@@ -92,7 +92,7 @@ class PackageManagerInstallMixin(object):
         self.log.info("Installing %s" % click.style(spec.humanize(), fg="cyan"))
 
         if spec.external:
-            pkg = self.install_from_url(spec.url, spec)
+            pkg = self.install_from_uri(spec.uri, spec)
         else:
             pkg = self.install_from_registry(spec, search_qualifiers)
 
@@ -146,24 +146,24 @@ class PackageManagerInstallMixin(object):
         }
         return self._install(spec, search_qualifiers=search_qualifiers or None)
 
-    def install_from_url(self, url, spec, checksum=None):
+    def install_from_uri(self, uri, spec, checksum=None):
         spec = self.ensure_spec(spec)
         tmp_dir = tempfile.mkdtemp(prefix="pkg-installing-", dir=self.get_tmp_dir())
         vcs = None
         try:
-            if url.startswith("file://"):
+            if uri.startswith("file://"):
-                _url = url[7:]
+                _uri = uri[7:]
-                if os.path.isfile(_url):
+                if os.path.isfile(_uri):
-                    self.unpack(_url, tmp_dir)
+                    self.unpack(_uri, tmp_dir)
                 else:
                     fs.rmtree(tmp_dir)
-                    shutil.copytree(_url, tmp_dir, symlinks=True)
+                    shutil.copytree(_uri, tmp_dir, symlinks=True)
-            elif url.startswith(("http://", "https://")):
+            elif uri.startswith(("http://", "https://")):
-                dl_path = self.download(url, checksum)
+                dl_path = self.download(uri, checksum)
                 assert os.path.isfile(dl_path)
                 self.unpack(dl_path, tmp_dir)
             else:
-                vcs = VCSClientFactory.new(tmp_dir, url)
+                vcs = VCSClientFactory.new(tmp_dir, uri)
                 assert vcs.export()
 
             root_dir = self.find_pkg_root(tmp_dir, spec)
@@ -210,7 +210,7 @@ class PackageManagerInstallMixin(object):
             )
         elif dst_pkg.metadata:
             if dst_pkg.metadata.spec.external:
-                if dst_pkg.metadata.spec.url != tmp_pkg.metadata.spec.url:
+                if dst_pkg.metadata.spec.uri != tmp_pkg.metadata.spec.uri:
                     action = "detach-existing"
             elif (
                 dst_pkg.metadata.version != tmp_pkg.metadata.version
@@ -231,11 +231,11 @@ class PackageManagerInstallMixin(object):
                 tmp_pkg.get_safe_dirname(),
                 dst_pkg.metadata.version,
             )
-            if dst_pkg.metadata.spec.url:
+            if dst_pkg.metadata.spec.uri:
                 target_dirname = "%s@src-%s" % (
                     tmp_pkg.get_safe_dirname(),
                     hashlib.md5(
-                        compat.hashlib_encode_data(dst_pkg.metadata.spec.url)
+                        compat.hashlib_encode_data(dst_pkg.metadata.spec.uri)
                     ).hexdigest(),
                 )
             # move existing into the new place
@@ -256,7 +256,7 @@ class PackageManagerInstallMixin(object):
                 target_dirname = "%s@src-%s" % (
                     tmp_pkg.get_safe_dirname(),
                     hashlib.md5(
-                        compat.hashlib_encode_data(tmp_pkg.metadata.spec.url)
+                        compat.hashlib_encode_data(tmp_pkg.metadata.spec.uri)
                     ).hexdigest(),
                 )
             pkg_dir = os.path.join(self.package_dir, target_dirname)

platformio/package/manager/_legacy.py

@@ -33,7 +33,7 @@ class PackageManagerLegacyMixin(object):
             src_manifest = fs.load_json(src_manifest_path)
             return PackageSpec(
                 name=src_manifest.get("name"),
-                url=src_manifest.get("url"),
+                uri=src_manifest.get("url"),
                 requirements=src_manifest.get("requirements"),
             )
 
@@ -51,7 +51,7 @@ class PackageManagerLegacyMixin(object):
             if not manifest.get(key):
                 manifest[key] = str(getattr(pkg.metadata, key))
         if pkg.metadata and pkg.metadata.spec and pkg.metadata.spec.external:
-            manifest["__src_url"] = pkg.metadata.spec.url
+            manifest["__src_url"] = pkg.metadata.spec.uri
             manifest["version"] = str(pkg.metadata.version)
         if pkg.metadata and pkg.metadata.spec.owner:
             manifest["ownername"] = pkg.metadata.spec.owner

platformio/package/manager/_registry.py

@@ -102,7 +102,7 @@ class PackageManageRegistryMixin(object):
 
         for url, checksum in RegistryFileMirrorIterator(pkgfile["download_url"]):
             try:
-                return self.install_from_url(
+                return self.install_from_uri(
                     url,
                     PackageSpec(
                         owner=package["owner"]["username"],

platformio/package/manager/_update.py

@@ -67,7 +67,7 @@ class PackageManagerUpdateMixin(object):
     def _fetch_vcs_latest_version(self, pkg):
         vcs = None
         try:
-            vcs = VCSClientFactory.new(pkg.path, pkg.metadata.spec.url, silent=True)
+            vcs = VCSClientFactory.new(pkg.path, pkg.metadata.spec.uri, silent=True)
         except VCSBaseException:
             return None
         if not vcs.can_be_updated:
@@ -112,7 +112,7 @@ class PackageManagerUpdateMixin(object):
 
     def _update(self, pkg, outdated, skip_dependencies=False):
         if pkg.metadata.spec.external:
-            vcs = VCSClientFactory.new(pkg.path, pkg.metadata.spec.url)
+            vcs = VCSClientFactory.new(pkg.path, pkg.metadata.spec.uri)
             assert vcs.update()
             pkg.metadata.version = self._fetch_vcs_latest_version(pkg)
             pkg.dump_meta()

platformio/package/manager/base.py

@@ -274,12 +274,12 @@ class BasePackageManager(  # pylint: disable=too-many-public-methods,too-many-in
         # external "URL" mismatch
         if spec.external:
             # local folder mismatch
-            if os.path.abspath(spec.url) == os.path.abspath(pkg.path) or (
+            if os.path.abspath(spec.uri) == os.path.abspath(pkg.path) or (
-                spec.url.startswith("file://")
+                spec.uri.startswith("file://")
-                and os.path.abspath(pkg.path) == os.path.abspath(spec.url[7:])
+                and os.path.abspath(pkg.path) == os.path.abspath(spec.uri[7:])
             ):
                 return True
-            if spec.url != pkg.metadata.spec.url:
+            if spec.uri != pkg.metadata.spec.uri:
                 return False
 
         # "owner" mismatch

platformio/package/manager/platform.py

@@ -167,7 +167,7 @@ def remove_unnecessary_platform_packages(dry_run=False):
     pm = ToolPackageManager()
     for pkg in pm.get_installed():
         skip_conds = [
-            pkg.metadata.spec.url,
+            pkg.metadata.spec.uri,
             os.path.isfile(os.path.join(pkg.path, ".piokeep")),
             pkg in required,
             pkg in core_packages,

platformio/package/meta.py

@@ -17,6 +17,7 @@ import os
 import re
 import tarfile
 from binascii import crc32
+from urllib.parse import urlparse
 
 import semantic_version
 
@@ -24,11 +25,6 @@ from platformio.compat import get_object_members, hashlib_encode_data, string_ty
 from platformio.package.manifest.parser import ManifestFileType
 from platformio.package.version import cast_version_to_semver
 
-try:
-    from urllib.parse import urlparse
-except ImportError:
-    from urlparse import urlparse
-
 
 class PackageType(object):
     LIBRARY = "library"
@@ -127,19 +123,19 @@ class PackageOutdatedResult(object):
 
 class PackageSpec(object):  # pylint: disable=too-many-instance-attributes
     def __init__(  # pylint: disable=redefined-builtin,too-many-arguments
-        self, raw=None, owner=None, id=None, name=None, requirements=None, url=None
+        self, raw=None, owner=None, id=None, name=None, requirements=None, uri=None
     ):
         self._requirements = None
         self.owner = owner
         self.id = id
         self.name = name
-        self.url = url
+        self.uri = uri
         self.raw = raw
         if requirements:
             try:
                 self.requirements = requirements
             except ValueError as exc:
-                if not self.name or self.url or self.raw:
+                if not self.name or self.uri or self.raw:
                     raise exc
                 self.raw = "%s=%s" % (self.name, requirements)
         self._name_is_custom = False
@@ -152,7 +148,7 @@ class PackageSpec(object):  # pylint: disable=too-many-instance-attributes
                 self.id == other.id,
                 self.name == other.name,
                 self.requirements == other.requirements,
-                self.url == other.url,
+                self.uri == other.uri,
             ]
         )
 
@@ -160,19 +156,19 @@ class PackageSpec(object):  # pylint: disable=too-many-instance-attributes
         return crc32(
             hashlib_encode_data(
                 "%s-%s-%s-%s-%s"
-                % (self.owner, self.id, self.name, self.requirements, self.url)
+                % (self.owner, self.id, self.name, self.requirements, self.uri)
             )
         )
 
     def __repr__(self):
         return (
             "PackageSpec <owner={owner} id={id} name={name} "
-            "requirements={requirements} url={url}>".format(**self.as_dict())
+            "requirements={requirements} uri={uri}>".format(**self.as_dict())
         )
 
     @property
     def external(self):
-        return bool(self.url)
+        return bool(self.uri)
 
     @property
     def requirements(self):
@@ -191,8 +187,8 @@ class PackageSpec(object):  # pylint: disable=too-many-instance-attributes
 
     def humanize(self):
         result = ""
-        if self.url:
+        if self.uri:
-            result = self.url
+            result = self.uri
         elif self.name:
             if self.owner:
                 result = self.owner + "/"
@@ -212,12 +208,12 @@ class PackageSpec(object):  # pylint: disable=too-many-instance-attributes
             id=self.id,
             name=self.name,
             requirements=str(self.requirements) if self.requirements else None,
-            url=self.url,
+            uri=self.uri,
         )
 
     def as_dependency(self):
-        if self.url:
+        if self.uri:
-            return self.raw or self.url
+            return self.raw or self.uri
         result = ""
         if self.name:
             result = "%s/%s" % (self.owner, self.name) if self.owner else self.name
@@ -241,16 +237,16 @@ class PackageSpec(object):  # pylint: disable=too-many-instance-attributes
             self._parse_custom_name,
             self._parse_id,
             self._parse_owner,
-            self._parse_url,
+            self._parse_uri,
         )
         for parser in parsers:
             if raw is None:
                 break
             raw = parser(raw)
 
-        # if name is not custom, parse it from URL
+        # if name is not custom, parse it from URI
-        if not self.name and self.url:
+        if not self.name and self.uri:
-            self.name = self._parse_name_from_url(self.url)
+            self.name = self._parse_name_from_uri(self.uri)
         elif raw:
             # the leftover is a package name
             self.name = raw
@@ -298,14 +294,18 @@ class PackageSpec(object):  # pylint: disable=too-many-instance-attributes
         self.name = tokens[1].strip()
         return None
 
-    def _parse_url(self, raw):
+    def _parse_uri(self, raw):
         if not any(s in raw for s in ("@", ":", "/")):
             return raw
-        self.url = raw.strip()
+        self.uri = raw.strip()
-        parts = urlparse(self.url)
+        parts = urlparse(self.uri)
 
-        # if local file or valid URL with scheme vcs+protocol://
+        # if local file or valid URI with scheme vcs+protocol://
-        if parts.scheme == "file" or "+" in parts.scheme or self.url.startswith("git+"):
+        if (
+            parts.scheme in ("file", )
+            or "+" in parts.scheme
+            or self.uri.startswith("git+")
+        ):
             return None
 
         # parse VCS
@@ -323,29 +323,29 @@ class PackageSpec(object):  # pylint: disable=too-many-instance-attributes
             in ("mbed.com", "os.mbed.com", "developer.mbed.org")
         ]
         if any(git_conditions):
-            self.url = "git+" + self.url
+            self.uri = "git+" + self.uri
         elif any(hg_conditions):
-            self.url = "hg+" + self.url
+            self.uri = "hg+" + self.uri
 
         return None
 
     @staticmethod
-    def _parse_name_from_url(url):
+    def _parse_name_from_uri(uri):
-        if url.endswith("/"):
+        if uri.endswith("/"):
-            url = url[:-1]
+            uri = uri[:-1]
         stop_chars = ["#", "?"]
-        if url.startswith("file://"):
+        if uri.startswith(("file://", )):
             stop_chars.append("@")  # detached path
         for c in stop_chars:
-            if c in url:
+            if c in uri:
-                url = url[: url.index(c)]
+                uri = uri[: uri.index(c)]
 
         # parse real repository name from Github
-        parts = urlparse(url)
+        parts = urlparse(uri)
         if parts.netloc == "github.com" and parts.path.count("/") > 2:
             return parts.path.split("/")[2]
 
-        name = os.path.basename(url)
+        name = os.path.basename(uri)
         if "." in name:
             return name.split(".", 1)[0].strip()
         return name
@@ -412,6 +412,10 @@ class PackageMetaData(object):
         with open(path, encoding="utf8") as fp:
             data = json.load(fp)
             if data["spec"]:
+                # legacy support for Core<5.3 packages
+                if "url" in data["spec"]:
+                    data["spec"]["uri"] = data["spec"]["url"]
+                    del data["spec"]["url"]
                 data["spec"] = PackageSpec(**data["spec"])
             return PackageMetaData(**data)
 

platformio/platform/_packages.py

@@ -62,7 +62,7 @@ class PlatformPackagesMixin(object):
                 continue
             item = {"name": pkg.metadata.name, "version": str(pkg.metadata.version)}
             if pkg.metadata.spec.external:
-                item["src_url"] = pkg.metadata.spec.url
+                item["src_url"] = pkg.metadata.spec.uri
             result.append(item)
         return result
 

tests/package/test_manager.py

@@ -23,7 +23,6 @@ import pytest
 import semantic_version
 
 from platformio import fs, util
-from platformio.compat import PY2
 from platformio.package.exception import (
     MissingPackageManifestError,
     UnknownPackageError,
@@ -106,7 +105,7 @@ def test_build_legacy_spec(isolated_pio_core, tmpdir_factory):
     )
     assert pm.build_legacy_spec(str(pkg1_dir)) == PackageSpec(
         name="StreamSpy-0.0.1.tar",
-        url="https://dl.platformio.org/e8936b7/StreamSpy-0.0.1.tar.gz",
+        uri="https://dl.platformio.org/e8936b7/StreamSpy-0.0.1.tar.gz",
     )
 
     # without src manifest
@@ -148,8 +147,7 @@ def test_build_metadata(isolated_pio_core, tmpdir_factory):
     assert metadata.version.build[1] == vcs_revision
 
 
-@pytest.mark.skipif(PY2, reason="Requires Python 3.5 or higher")
+def test_install_from_uri(isolated_pio_core, tmpdir_factory):
-def test_install_from_url(isolated_pio_core, tmpdir_factory):
     tmp_dir = tmpdir_factory.mktemp("tmp")
     storage_dir = tmpdir_factory.mktemp("storage")
     lm = LibraryPackageManager(str(storage_dir))
@@ -189,7 +187,7 @@ version = 5.2.7
 """
     )
     spec = PackageSpec("company/wifilib @ ^5")
-    pkg = lm.install_from_url("file://%s" % src_dir, spec)
+    pkg = lm.install_from_uri("file://%s" % src_dir, spec)
     assert str(pkg.metadata.version) == "5.2.7"
 
     # check package folder names
@@ -416,11 +414,11 @@ def test_uninstall(isolated_pio_core, tmpdir_factory):
     # foo @ 1.0.0
     pkg_dir = tmp_dir.join("foo").mkdir()
     pkg_dir.join("library.json").write('{"name": "foo", "version": "1.0.0"}')
-    foo_1_0_0_pkg = lm.install_from_url("file://%s" % pkg_dir, "foo")
+    foo_1_0_0_pkg = lm.install_from_uri("file://%s" % pkg_dir, "foo")
     # foo @ 1.3.0
     pkg_dir = tmp_dir.join("foo-1.3.0").mkdir()
     pkg_dir.join("library.json").write('{"name": "foo", "version": "1.3.0"}')
-    lm.install_from_url("file://%s" % pkg_dir, "foo")
+    lm.install_from_uri("file://%s" % pkg_dir, "foo")
     # bar
     pkg_dir = tmp_dir.join("bar").mkdir()
     pkg_dir.join("library.json").write('{"name": "bar", "version": "1.0.0"}')

tests/package/test_meta.py

@@ -82,22 +82,22 @@ def test_spec_requirements():
 
 def test_spec_local_urls(tmpdir_factory):
     assert PackageSpec("file:///tmp/foo.tar.gz") == PackageSpec(
-        url="file:///tmp/foo.tar.gz", name="foo"
+        uri="file:///tmp/foo.tar.gz", name="foo"
     )
     assert PackageSpec("customName=file:///tmp/bar.zip") == PackageSpec(
-        url="file:///tmp/bar.zip", name="customName"
+        uri="file:///tmp/bar.zip", name="customName"
     )
     assert PackageSpec("file:///tmp/some-lib/") == PackageSpec(
-        url="file:///tmp/some-lib/", name="some-lib"
+        uri="file:///tmp/some-lib/", name="some-lib"
     )
     # detached package
     assert PackageSpec("file:///tmp/some-lib@src-67e1043a673d2") == PackageSpec(
-        url="file:///tmp/some-lib@src-67e1043a673d2", name="some-lib"
+        uri="file:///tmp/some-lib@src-67e1043a673d2", name="some-lib"
     )
     # detached folder without scheme
     pkg_dir = tmpdir_factory.mktemp("storage").join("detached@1.2.3").mkdir()
     assert PackageSpec(str(pkg_dir)) == PackageSpec(
-        name="detached", url="file://%s" % pkg_dir
+        name="detached", uri="file://%s" % pkg_dir
     )
 
 
@@ -105,14 +105,14 @@ def test_spec_external_urls():
     assert PackageSpec(
         "https://github.com/platformio/platformio-core/archive/develop.zip"
     ) == PackageSpec(
-        url="https://github.com/platformio/platformio-core/archive/develop.zip",
+        uri="https://github.com/platformio/platformio-core/archive/develop.zip",
         name="platformio-core",
     )
     assert PackageSpec(
         "https://github.com/platformio/platformio-core/archive/develop.zip?param=value"
         " @ !=2"
     ) == PackageSpec(
-        url="https://github.com/platformio/platformio-core/archive/"
+        uri="https://github.com/platformio/platformio-core/archive/"
         "develop.zip?param=value",
         name="platformio-core",
         requirements="!=2",
@@ -125,7 +125,7 @@ def test_spec_external_urls():
     assert spec.has_custom_name()
     assert spec.name == "Custom-Name"
     assert spec == PackageSpec(
-        url="https://github.com/platformio/platformio-core/archive/develop.tar.gz",
+        uri="https://github.com/platformio/platformio-core/archive/develop.tar.gz",
         name="Custom-Name",
         requirements="4.4.0",
     )
@@ -133,40 +133,40 @@ def test_spec_external_urls():
 
 def test_spec_vcs_urls():
     assert PackageSpec("https://github.com/platformio/platformio-core") == PackageSpec(
-        name="platformio-core", url="git+https://github.com/platformio/platformio-core"
+        name="platformio-core", uri="git+https://github.com/platformio/platformio-core"
     )
     assert PackageSpec("https://gitlab.com/username/reponame") == PackageSpec(
-        name="reponame", url="git+https://gitlab.com/username/reponame"
+        name="reponame", uri="git+https://gitlab.com/username/reponame"
     )
     assert PackageSpec(
         "wolfSSL=https://os.mbed.com/users/wolfSSL/code/wolfSSL/"
     ) == PackageSpec(
-        name="wolfSSL", url="hg+https://os.mbed.com/users/wolfSSL/code/wolfSSL/"
+        name="wolfSSL", uri="hg+https://os.mbed.com/users/wolfSSL/code/wolfSSL/"
     )
     assert PackageSpec(
         "https://github.com/platformio/platformio-core.git#master"
     ) == PackageSpec(
         name="platformio-core",
-        url="git+https://github.com/platformio/platformio-core.git#master",
+        uri="git+https://github.com/platformio/platformio-core.git#master",
     )
     assert PackageSpec(
         "core=git+ssh://github.com/platformio/platformio-core.git#v4.4.0@4.4.0"
     ) == PackageSpec(
         name="core",
-        url="git+ssh://github.com/platformio/platformio-core.git#v4.4.0",
+        uri="git+ssh://github.com/platformio/platformio-core.git#v4.4.0",
         requirements="4.4.0",
     )
     assert PackageSpec(
         "username@github.com:platformio/platformio-core.git"
     ) == PackageSpec(
         name="platformio-core",
-        url="git+username@github.com:platformio/platformio-core.git",
+        uri="git+username@github.com:platformio/platformio-core.git",
     )
     assert PackageSpec(
         "pkg=git+git@github.com:platformio/platformio-core.git @ ^1.2.3,!=5"
     ) == PackageSpec(
         name="pkg",
-        url="git+git@github.com:platformio/platformio-core.git",
+        uri="git+git@github.com:platformio/platformio-core.git",
         requirements="^1.2.3,!=5",
     )
     assert PackageSpec(
@@ -176,7 +176,7 @@ def test_spec_vcs_urls():
     ) == PackageSpec(
         owner="platformio",
         name="external-repo",
-        url="git+https://github.com/platformio/platformio-core",
+        uri="git+https://github.com/platformio/platformio-core",
     )
 
 
@@ -188,7 +188,7 @@ def test_spec_as_dict():
             "id": None,
             "name": "foo",
             "requirements": "1.2.3",
-            "url": None,
+            "uri": None,
         },
     )
     assert not jsondiff.diff(
@@ -201,7 +201,7 @@ def test_spec_as_dict():
             "id": None,
             "name": "platformio-core",
             "requirements": "!=2",
-            "url": "https://github.com/platformio/platformio-core/archive/develop.zip?param=value",
+            "uri": "https://github.com/platformio/platformio-core/archive/develop.zip?param=value",
         },
     )
 
@@ -255,7 +255,7 @@ def test_metadata_as_dict():
                 "id": None,
                 "name": "toolchain",
                 "requirements": "~2.0.0",
-                "url": None,
+                "uri": None,
             },
         },
     )