Initial version of a new package manager

This commit is contained in:
Ivan Kravets committed 2020-07-31 15:42:26 +03:00
1 parent abc0489ac6
commit d329aef876
27 files changed
+2074 -407

No files matched your search

+142
View File
@@ -0,0 +1,142 @@
# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import io
import math
import sys
from email.utils import parsedate_tz
from os.path import getsize, join
from time import mktime
import click
import requests
from platformio import app, fs, util
from platformio.package.exception import PackageException
class FileDownloader(object):
def __init__(self, url, dest_dir=None):
self._request = None
# make connection
self._request = requests.get(
url,
stream=True,
headers={"User-Agent": app.get_user_agent()},
verify=sys.version_info >= (2, 7, 9),
)
if self._request.status_code != 200:
raise PackageException(
"Got the unrecognized status code '{0}' when downloaded {1}".format(
self._request.status_code, url
)
)
disposition = self._request.headers.get("content-disposition")
if disposition and "filename=" in disposition:
self._fname = (
disposition[disposition.index("filename=") + 9 :]
.replace('"', "")
.replace("'", "")
)
else:
self._fname = [p for p in url.split("/") if p][-1]
self._fname = str(self._fname)
self._destination = self._fname
if dest_dir:
self.set_destination(join(dest_dir, self._fname))
def set_destination(self, destination):
self._destination = destination
def get_filepath(self):
return self._destination
def get_lmtime(self):
return self._request.headers.get("last-modified")
def get_size(self):
if "content-length" not in self._request.headers:
return -1
return int(self._request.headers["content-length"])
def start(self, with_progress=True, silent=False):
label = "Downloading"
itercontent = self._request.iter_content(chunk_size=io.DEFAULT_BUFFER_SIZE)
fp = open(self._destination, "wb")
try:
if not with_progress or self.get_size() == -1:
if not silent:
click.echo("%s..." % label)
for chunk in itercontent:
if chunk:
fp.write(chunk)
else:
chunks = int(math.ceil(self.get_size() / float(io.DEFAULT_BUFFER_SIZE)))
with click.progressbar(length=chunks, label=label) as pb:
for _ in pb:
fp.write(next(itercontent))
finally:
fp.close()
self._request.close()
if self.get_lmtime():
self._preserve_filemtime(self.get_lmtime())
return True
def verify(self, checksum=None):
_dlsize = getsize(self._destination)
if self.get_size() != -1 and _dlsize != self.get_size():
raise PackageException(
(
"The size ({0:d} bytes) of downloaded file '{1}' "
"is not equal to remote size ({2:d} bytes)"
).format(_dlsize, self._fname, self.get_size())
)
if not checksum:
return True
checksum_len = len(checksum)
hash_algo = None
if checksum_len == 32:
hash_algo = "md5"
elif checksum_len == 40:
hash_algo = "sha1"
elif checksum_len == 64:
hash_algo = "sha256"
if not hash_algo:
raise PackageException(
"Could not determine checksum algorithm by %s" % checksum
)
dl_checksum = fs.calculate_file_hashsum(hash_algo, self._destination)
if checksum.lower() != dl_checksum.lower():
raise PackageException(
"The checksum '{0}' of the downloaded file '{1}' "
"does not match to the remote '{2}'".format(
dl_checksum, self._fname, checksum
)
)
return True
def _preserve_filemtime(self, lmdate):
timedata = parsedate_tz(lmdate)
lmtime = mktime(timedata[:9])
util.change_filemtime(self._destination, lmtime)
def __del__(self):
if self._request:
self._request.close()
+15 -1
View File
@@ -12,7 +12,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from platformio.exception import PlatformioException
from platformio import util
from platformio.exception import PlatformioException, UserSideException
class PackageException(PlatformioException):
@@ -44,3 +45,16 @@ class ManifestValidationError(ManifestException):
"https://docs.platformio.org/page/librarymanager/config.html"
% self.messages
)
class MissingPackageManifestError(ManifestException):
MESSAGE = "Could not find one of '{0}' manifest files in the package"
class UnknownPackageError(UserSideException):
MESSAGE = (
"Could not find a package with '{0}' requirements for your system '%s'"
% util.get_systype()
)
+108
View File
@@ -0,0 +1,108 @@
# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
from time import sleep, time
from platformio import exception
LOCKFILE_TIMEOUT = 3600 # in seconds, 1 hour
LOCKFILE_DELAY = 0.2
LOCKFILE_INTERFACE_FCNTL = 1
LOCKFILE_INTERFACE_MSVCRT = 2
try:
import fcntl
LOCKFILE_CURRENT_INTERFACE = LOCKFILE_INTERFACE_FCNTL
except ImportError:
try:
import msvcrt
LOCKFILE_CURRENT_INTERFACE = LOCKFILE_INTERFACE_MSVCRT
except ImportError:
LOCKFILE_CURRENT_INTERFACE = None
class LockFileExists(Exception):
pass
class LockFile(object):
def __init__(self, path, timeout=LOCKFILE_TIMEOUT, delay=LOCKFILE_DELAY):
self.timeout = timeout
self.delay = delay
self._lock_path = os.path.realpath(path) + ".lock"
self._fp = None
def _lock(self):
if not LOCKFILE_CURRENT_INTERFACE and os.path.exists(self._lock_path):
# remove stale lock
if time() - os.path.getmtime(self._lock_path) > 10:
try:
os.remove(self._lock_path)
except: # pylint: disable=bare-except
pass
else:
raise LockFileExists
self._fp = open(self._lock_path, "w")
try:
if LOCKFILE_CURRENT_INTERFACE == LOCKFILE_INTERFACE_FCNTL:
fcntl.flock(self._fp.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
elif LOCKFILE_CURRENT_INTERFACE == LOCKFILE_INTERFACE_MSVCRT:
msvcrt.locking(self._fp.fileno(), msvcrt.LK_NBLCK, 1)
except IOError:
self._fp = None
raise LockFileExists
return True
def _unlock(self):
if not self._fp:
return
if LOCKFILE_CURRENT_INTERFACE == LOCKFILE_INTERFACE_FCNTL:
fcntl.flock(self._fp.fileno(), fcntl.LOCK_UN)
elif LOCKFILE_CURRENT_INTERFACE == LOCKFILE_INTERFACE_MSVCRT:
msvcrt.locking(self._fp.fileno(), msvcrt.LK_UNLCK, 1)
self._fp.close()
self._fp = None
def acquire(self):
elapsed = 0
while elapsed < self.timeout:
try:
return self._lock()
except LockFileExists:
sleep(self.delay)
elapsed += self.delay
raise exception.LockFileTimeoutError()
def release(self):
self._unlock()
if os.path.exists(self._lock_path):
try:
os.remove(self._lock_path)
except: # pylint: disable=bare-except
pass
def __enter__(self):
self.acquire()
def __exit__(self, type_, value, traceback):
self.release()
def __del__(self):
self.release()
+13
View File
@@ -0,0 +1,13 @@
# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+95
View File
@@ -0,0 +1,95 @@
# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import hashlib
import os
import tempfile
import time
import click
from platformio import app, compat
from platformio.package.download import FileDownloader
from platformio.package.lockfile import LockFile
class PackageManagerDownloadMixin(object):
DOWNLOAD_CACHE_EXPIRE = 86400 * 30 # keep package in a local cache for 1 month
def compute_download_path(self, *args):
request_hash = hashlib.new("sha256")
for arg in args:
request_hash.update(compat.hashlib_encode_data(arg))
dl_path = os.path.join(self.get_download_dir(), request_hash.hexdigest())
return dl_path
def get_download_usagedb_path(self):
return os.path.join(self.get_download_dir(), "usage.db")
def set_download_utime(self, path, utime=None):
with app.State(self.get_download_usagedb_path(), lock=True) as state:
state[os.path.basename(path)] = int(time.time() if not utime else utime)
def cleanup_expired_downloads(self):
with app.State(self.get_download_usagedb_path(), lock=True) as state:
# remove outdated
for fname in list(state.keys()):
if state[fname] > (time.time() - self.DOWNLOAD_CACHE_EXPIRE):
continue
del state[fname]
dl_path = os.path.join(self.get_download_dir(), fname)
if os.path.isfile(dl_path):
os.remove(dl_path)
def download(self, url, checksum=None, silent=False):
dl_path = self.compute_download_path(url, checksum or "")
if os.path.isfile(dl_path):
self.set_download_utime(dl_path)
return dl_path
with_progress = not silent and not app.is_disabled_progressbar()
tmp_path = tempfile.mkstemp(dir=self.get_download_dir())[1]
try:
with LockFile(dl_path):
try:
fd = FileDownloader(url)
fd.set_destination(tmp_path)
fd.start(with_progress=with_progress, silent=silent)
except IOError as e:
raise_error = not with_progress
if with_progress:
try:
fd = FileDownloader(url)
fd.set_destination(tmp_path)
fd.start(with_progress=False, silent=silent)
except IOError:
raise_error = True
if raise_error:
click.secho(
"Error: Please read http://bit.ly/package-manager-ioerror",
fg="red",
err=True,
)
raise e
if checksum:
fd.verify(checksum)
os.rename(tmp_path, dl_path)
finally:
if os.path.isfile(tmp_path):
os.remove(tmp_path)
assert os.path.isfile(dl_path)
self.set_download_utime(dl_path)
return dl_path
+282
View File
@@ -0,0 +1,282 @@
# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import hashlib
import os
import shutil
import tempfile
import click
from platformio import app, compat, fs, util
from platformio.package.exception import PackageException, UnknownPackageError
from platformio.package.lockfile import LockFile
from platformio.package.meta import PackageSourceItem, PackageSpec
from platformio.package.unpack import FileUnpacker
from platformio.package.vcsclient import VCSClientFactory
class PackageManagerInstallMixin(object):
INSTALL_HISTORY = None # avoid circle dependencies
@staticmethod
def unpack(src, dst):
with_progress = not app.is_disabled_progressbar()
try:
with FileUnpacker(src) as fu:
return fu.unpack(dst, with_progress=with_progress)
except IOError as e:
if not with_progress:
raise e
with FileUnpacker(src) as fu:
return fu.unpack(dst, with_progress=False)
def install(self, spec, silent=False):
with LockFile(self.package_dir):
pkg = self._install(spec, silent=silent)
self.memcache_reset()
self.cleanup_expired_downloads()
return pkg
def _install(self, spec, search_filters=None, silent=False):
spec = self.ensure_spec(spec)
# avoid circle dependencies
if not self.INSTALL_HISTORY:
self.INSTALL_HISTORY = []
if spec in self.INSTALL_HISTORY:
return None
self.INSTALL_HISTORY.append(spec)
# check if package is already installed
pkg = self.get_package(spec)
if pkg:
if not silent:
click.secho(
"{name} @ {version} is already installed".format(
**pkg.metadata.as_dict()
),
fg="yellow",
)
return pkg
if not silent:
msg = "Installing %s" % click.style(spec.humanize(), fg="cyan")
self.print_message(msg)
if spec.url:
pkg = self.install_from_url(spec.url, spec, silent=silent)
else:
pkg = self.install_from_registry(spec, search_filters, silent=silent)
if not pkg or not pkg.metadata:
raise PackageException(
"Could not install package '%s' for '%s' system"
% (spec.humanize(), util.get_systype())
)
if not silent:
self.print_message(
click.style(
"{name} @ {version} has been successfully installed!".format(
**pkg.metadata.as_dict()
),
fg="green",
)
)
self.memcache_reset()
self.install_dependencies(pkg, silent)
return pkg
def install_dependencies(self, pkg, silent=False):
assert isinstance(pkg, PackageSourceItem)
manifest = self.load_manifest(pkg)
if not manifest.get("dependencies"):
return
if not silent:
self.print_message(click.style("Installing dependencies...", fg="yellow"))
for dependency in manifest.get("dependencies"):
if not self.install_dependency(dependency, silent) and not silent:
click.secho(
"Warning! Could not install dependency %s for package '%s'"
% (dependency, pkg.metadata.name),
fg="yellow",
)
def install_dependency(self, dependency, silent=False):
spec = PackageSpec(
name=dependency.get("name"), requirements=dependency.get("version")
)
search_filters = {
key: value
for key, value in dependency.items()
if key in ("authors", "platforms", "frameworks")
}
return self._install(spec, search_filters=search_filters or None, silent=silent)
def install_from_url(self, url, spec, checksum=None, silent=False):
spec = self.ensure_spec(spec)
tmp_dir = tempfile.mkdtemp(prefix="pkg-installing-", dir=self.get_tmp_dir())
vcs = None
try:
if url.startswith("file://"):
_url = url[7:]
if os.path.isfile(_url):
self.unpack(_url, tmp_dir)
else:
fs.rmtree(tmp_dir)
shutil.copytree(_url, tmp_dir, symlinks=True)
elif url.startswith(("http://", "https://")):
dl_path = self.download(url, checksum, silent=silent)
assert os.path.isfile(dl_path)
self.unpack(dl_path, tmp_dir)
else:
vcs = VCSClientFactory.newClient(tmp_dir, url)
assert vcs.export()
root_dir = self.find_pkg_root(tmp_dir, spec)
pkg_item = PackageSourceItem(
root_dir,
self.build_metadata(
root_dir, spec, vcs.get_current_revision() if vcs else None
),
)
pkg_item.dump_meta()
return self._install_tmp_pkg(pkg_item)
finally:
if os.path.isdir(tmp_dir):
fs.rmtree(tmp_dir)
def _install_tmp_pkg(self, tmp_pkg):
assert isinstance(tmp_pkg, PackageSourceItem)
# validate package version and declared requirements
if (
tmp_pkg.metadata.spec.requirements
and tmp_pkg.metadata.version not in tmp_pkg.metadata.spec.requirements
):
raise PackageException(
"Package version %s doesn't satisfy requirements %s based on %s"
% (
tmp_pkg.metadata.version,
tmp_pkg.metadata.spec.requirements,
tmp_pkg.metadata,
)
)
dst_pkg = PackageSourceItem(
os.path.join(self.package_dir, tmp_pkg.get_safe_dirname())
)
# what to do with existing package?
action = "overwrite"
if dst_pkg.metadata and dst_pkg.metadata.spec.url:
if dst_pkg.metadata.spec.url != tmp_pkg.metadata.spec.url:
action = "detach-existing"
elif tmp_pkg.metadata.spec.url:
action = "detach-new"
elif dst_pkg.metadata and dst_pkg.metadata.version != tmp_pkg.metadata.version:
action = (
"detach-existing"
if tmp_pkg.metadata.version > dst_pkg.metadata.version
else "detach-new"
)
def _cleanup_dir(path):
if os.path.isdir(path):
fs.rmtree(path)
if action == "detach-existing":
target_dirname = "%s@%s" % (
tmp_pkg.get_safe_dirname(),
dst_pkg.metadata.version,
)
if dst_pkg.metadata.spec.url:
target_dirname = "%s@src-%s" % (
tmp_pkg.get_safe_dirname(),
hashlib.md5(
compat.hashlib_encode_data(dst_pkg.metadata.spec.url)
).hexdigest(),
)
# move existing into the new place
pkg_dir = os.path.join(self.package_dir, target_dirname)
_cleanup_dir(pkg_dir)
shutil.move(dst_pkg.path, pkg_dir)
# move new source to the destination location
_cleanup_dir(dst_pkg.path)
shutil.move(tmp_pkg.path, dst_pkg.path)
return PackageSourceItem(dst_pkg.path)
if action == "detach-new":
target_dirname = "%s@%s" % (
tmp_pkg.get_safe_dirname(),
tmp_pkg.metadata.version,
)
if tmp_pkg.metadata.spec.url:
target_dirname = "%s@src-%s" % (
tmp_pkg.get_safe_dirname(),
hashlib.md5(
compat.hashlib_encode_data(tmp_pkg.metadata.spec.url)
).hexdigest(),
)
pkg_dir = os.path.join(self.package_dir, target_dirname)
_cleanup_dir(pkg_dir)
shutil.move(tmp_pkg.path, pkg_dir)
return PackageSourceItem(pkg_dir)
# otherwise, overwrite existing
_cleanup_dir(dst_pkg.path)
shutil.move(tmp_pkg.path, dst_pkg.path)
return PackageSourceItem(dst_pkg.path)
def uninstall(self, path_or_spec, silent=False):
with LockFile(self.package_dir):
pkg = (
PackageSourceItem(path_or_spec)
if os.path.isdir(path_or_spec)
else self.get_package(path_or_spec)
)
if not pkg or not pkg.metadata:
raise UnknownPackageError(path_or_spec)
if not silent:
self.print_message(
"Uninstalling %s @ %s: \t"
% (click.style(pkg.metadata.name, fg="cyan"), pkg.metadata.version),
nl=False,
)
if os.path.islink(pkg.path):
os.unlink(pkg.path)
else:
fs.rmtree(pkg.path)
self.memcache_reset()
# unfix detached-package with the same name
detached_pkg = self.get_package(PackageSpec(name=pkg.metadata.name))
if (
detached_pkg
and "@" in detached_pkg.path
and not os.path.isdir(
os.path.join(self.package_dir, detached_pkg.get_safe_dirname())
)
):
shutil.move(
detached_pkg.path,
os.path.join(self.package_dir, detached_pkg.get_safe_dirname()),
)
self.memcache_reset()
if not silent:
click.echo("[%s]" % click.style("OK", fg="green"))
return True
+190
View File
@@ -0,0 +1,190 @@
# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import time
import click
from platformio.clients.http import HTTPClient
from platformio.clients.registry import RegistryClient
from platformio.package.exception import UnknownPackageError
from platformio.package.meta import PackageMetaData, PackageSpec
try:
from urllib.parse import urlparse
except ImportError:
from urlparse import urlparse
class RegistryFileMirrorsIterator(object):
HTTP_CLIENT_INSTANCES = {}
def __init__(self, download_url):
self.download_url = download_url
self._url_parts = urlparse(download_url)
self._base_url = "%s://%s" % (self._url_parts.scheme, self._url_parts.netloc)
self._visited_mirrors = []
def __iter__(self):
return self
def __next__(self):
http = self.get_http_client()
response = http.send_request(
"head",
self._url_parts.path,
allow_redirects=False,
params=dict(bypass=",".join(self._visited_mirrors))
if self._visited_mirrors
else None,
)
stop_conditions = [
response.status_code not in (302, 307),
not response.headers.get("Location"),
not response.headers.get("X-PIO-Mirror"),
response.headers.get("X-PIO-Mirror") in self._visited_mirrors,
]
if any(stop_conditions):
raise StopIteration
self._visited_mirrors.append(response.headers.get("X-PIO-Mirror"))
return (
response.headers.get("Location"),
response.headers.get("X-PIO-Content-SHA256"),
)
def get_http_client(self):
if self._base_url not in RegistryFileMirrorsIterator.HTTP_CLIENT_INSTANCES:
RegistryFileMirrorsIterator.HTTP_CLIENT_INSTANCES[
self._base_url
] = HTTPClient(self._base_url)
return RegistryFileMirrorsIterator.HTTP_CLIENT_INSTANCES[self._base_url]
class PackageManageRegistryMixin(object):
def install_from_registry(self, spec, search_filters=None, silent=False):
packages = self.search_registry_packages(spec, search_filters)
if not packages:
raise UnknownPackageError(spec.humanize())
if len(packages) > 1 and not silent:
self.print_multi_package_issue(packages, spec)
package, version = self.find_best_registry_version(packages, spec)
pkgfile = self._pick_compatible_pkg_file(version["files"]) if version else None
if not pkgfile:
raise UnknownPackageError(spec.humanize())
for url, checksum in RegistryFileMirrorsIterator(pkgfile["download_url"]):
try:
return self.install_from_url(
url,
PackageSpec(
owner=package["owner"]["username"],
id=package["id"],
name=package["name"],
),
checksum or pkgfile["checksum"]["sha256"],
silent=silent,
)
except Exception as e: # pylint: disable=broad-except
click.secho("Warning! Package Mirror: %s" % e, fg="yellow")
click.secho("Looking for another mirror...", fg="yellow")
return None
def get_registry_client_instance(self):
if not self._registry_client:
self._registry_client = RegistryClient()
return self._registry_client
def search_registry_packages(self, spec, filters=None):
filters = filters or {}
if spec.id:
filters["ids"] = str(spec.id)
else:
filters["types"] = self.pkg_type
filters["names"] = '"%s"' % spec.name.lower()
if spec.owner:
filters["owners"] = spec.owner.lower()
return self.get_registry_client_instance().list_packages(filters=filters)[
"items"
]
def fetch_registry_package_versions(self, owner, name):
return self.get_registry_client_instance().get_package(
self.pkg_type, owner, name
)["versions"]
@staticmethod
def print_multi_package_issue(packages, spec):
click.secho(
"Warning! More than one package has been found by ", fg="yellow", nl=False
)
click.secho(spec.humanize(), fg="cyan", nl=False)
click.secho(" requirements:", fg="yellow")
for item in packages:
click.echo(
" - {owner}/{name} @ {version}".format(
owner=click.style(item["owner"]["username"], fg="cyan"),
name=item["name"],
version=item["version"]["name"],
)
)
click.secho(
"Please specify detailed REQUIREMENTS using package owner and version "
"(showed above) to avoid project compatibility issues.",
fg="yellow",
)
def find_best_registry_version(self, packages, spec):
# find compatible version within the latest package versions
for package in packages:
version = self._pick_best_pkg_version([package["version"]], spec)
if version:
return (package, version)
if not spec.requirements:
return None
# if the custom version requirements, check ALL package versions
for package in packages:
version = self._pick_best_pkg_version(
self.fetch_registry_package_versions(
package["owner"]["username"], package["name"]
),
spec,
)
if version:
return (package, version)
time.sleep(1)
return None
def _pick_best_pkg_version(self, versions, spec):
best = None
for version in versions:
semver = PackageMetaData.to_semver(version["name"])
if spec.requirements and semver not in spec.requirements:
continue
if not any(
self.is_system_compatible(f.get("system")) for f in version["files"]
):
continue
if not best or (semver > PackageMetaData.to_semver(best["name"])):
best = version
return best
def _pick_compatible_pkg_file(self, version_files):
for item in version_files:
if self.is_system_compatible(item.get("system")):
return item
return None
+233
View File
@@ -0,0 +1,233 @@
# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
from datetime import datetime
import click
import semantic_version
from platformio import fs, util
from platformio.commands import PlatformioCLI
from platformio.package.exception import ManifestException, MissingPackageManifestError
from platformio.package.manager._download import PackageManagerDownloadMixin
from platformio.package.manager._install import PackageManagerInstallMixin
from platformio.package.manager._registry import PackageManageRegistryMixin
from platformio.package.manifest.parser import ManifestParserFactory
from platformio.package.meta import (
PackageMetaData,
PackageSourceItem,
PackageSpec,
PackageType,
)
from platformio.project.helpers import get_project_cache_dir
class BasePackageManager(
PackageManagerDownloadMixin, PackageManageRegistryMixin, PackageManagerInstallMixin
):
MEMORY_CACHE = {}
def __init__(self, pkg_type, package_dir):
self.pkg_type = pkg_type
self.package_dir = self.ensure_dir_exists(package_dir)
self.MEMORY_CACHE = {}
self._download_dir = None
self._tmp_dir = None
self._registry_client = None
def memcache_get(self, key, default=None):
return self.MEMORY_CACHE.get(key, default)
def memcache_set(self, key, value):
self.MEMORY_CACHE[key] = value
def memcache_reset(self):
self.MEMORY_CACHE.clear()
@staticmethod
def is_system_compatible(value):
if not value or "*" in value:
return True
return util.items_in_list(value, util.get_systype())
@staticmethod
def generate_rand_version():
return datetime.now().strftime("0.0.0+%Y%m%d%H%M%S")
@staticmethod
def ensure_dir_exists(path):
if not os.path.isdir(path):
os.makedirs(path)
assert os.path.isdir(path)
return path
@staticmethod
def ensure_spec(spec):
return spec if isinstance(spec, PackageSpec) else PackageSpec(spec)
@property
def manifest_names(self):
raise NotImplementedError
def print_message(self, message, nl=True):
click.echo("%s: %s" % (self.__class__.__name__, message), nl=nl)
def get_download_dir(self):
if not self._download_dir:
self._download_dir = self.ensure_dir_exists(
os.path.join(get_project_cache_dir(), "downloads")
)
return self._download_dir
def get_tmp_dir(self):
if not self._tmp_dir:
self._tmp_dir = self.ensure_dir_exists(
os.path.join(get_project_cache_dir(), "tmp")
)
return self._tmp_dir
def find_pkg_root(self, path, spec): # pylint: disable=unused-argument
if self.manifest_exists(path):
return path
for root, _, _ in os.walk(path):
if self.manifest_exists(root):
return root
raise MissingPackageManifestError(", ".join(self.manifest_names))
def get_manifest_path(self, pkg_dir):
if not os.path.isdir(pkg_dir):
return None
for name in self.manifest_names:
manifest_path = os.path.join(pkg_dir, name)
if os.path.isfile(manifest_path):
return manifest_path
return None
def manifest_exists(self, pkg_dir):
return self.get_manifest_path(pkg_dir)
def load_manifest(self, src):
path = src.path if isinstance(src, PackageSourceItem) else src
cache_key = "load_manifest-%s" % path
result = self.memcache_get(cache_key)
if result:
return result
candidates = (
[os.path.join(path, name) for name in self.manifest_names]
if os.path.isdir(path)
else [path]
)
for item in candidates:
if not os.path.isfile(item):
continue
try:
result = ManifestParserFactory.new_from_file(item).as_dict()
self.memcache_set(cache_key, result)
return result
except ManifestException as e:
if not PlatformioCLI.in_silence():
click.secho(str(e), fg="yellow")
raise MissingPackageManifestError(", ".join(self.manifest_names))
def build_legacy_spec(self, pkg_dir):
# find src manifest
src_manifest_name = ".piopkgmanager.json"
src_manifest_path = None
for name in os.listdir(pkg_dir):
if not os.path.isfile(os.path.join(pkg_dir, name, src_manifest_name)):
continue
src_manifest_path = os.path.join(pkg_dir, name, src_manifest_name)
break
if src_manifest_path:
src_manifest = fs.load_json(src_manifest_path)
return PackageSpec(
name=src_manifest.get("name"),
url=src_manifest.get("url"),
requirements=src_manifest.get("requirements"),
)
# fall back to a package manifest
manifest = self.load_manifest(pkg_dir)
return PackageSpec(name=manifest.get("name"))
def build_metadata(self, pkg_dir, spec, vcs_revision=None):
manifest = self.load_manifest(pkg_dir)
metadata = PackageMetaData(
type=self.pkg_type,
name=manifest.get("name"),
version=manifest.get("version"),
spec=spec,
)
if not metadata.name or spec.is_custom_name():
metadata.name = spec.name
if vcs_revision:
metadata.version = "%s+sha.%s" % (
metadata.version if metadata.version else "0.0.0",
vcs_revision,
)
if not metadata.version:
metadata.version = self.generate_rand_version()
return metadata
def get_installed(self):
result = []
for name in os.listdir(self.package_dir):
pkg_dir = os.path.join(self.package_dir, name)
if not os.path.isdir(pkg_dir):
continue
pkg = PackageSourceItem(pkg_dir)
if not pkg.metadata:
try:
spec = self.build_legacy_spec(pkg_dir)
pkg.metadata = self.build_metadata(pkg_dir, spec)
except MissingPackageManifestError:
pass
if pkg.metadata:
result.append(pkg)
return result
def get_package(self, spec):
def _ci_strings_are_equal(a, b):
if a == b:
return True
if not a or not b:
return False
return a.strip().lower() == b.strip().lower()
spec = self.ensure_spec(spec)
best = None
for pkg in self.get_installed():
skip_conditions = [
spec.owner
and not _ci_strings_are_equal(spec.owner, pkg.metadata.spec.owner),
spec.url and spec.url != pkg.metadata.spec.url,
spec.id and spec.id != pkg.metadata.spec.id,
not spec.id
and not spec.url
and not _ci_strings_are_equal(spec.name, pkg.metadata.name),
]
if any(skip_conditions):
continue
if self.pkg_type == PackageType.TOOL:
# TODO: check "system" for pkg
pass
assert isinstance(pkg.metadata.version, semantic_version.Version)
if spec.requirements and pkg.metadata.version not in spec.requirements:
continue
if not best or (pkg.metadata.version > best.metadata.version):
best = pkg
return best
+64
View File
@@ -0,0 +1,64 @@
# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
import os
from platformio.package.exception import MissingPackageManifestError
from platformio.package.manager.base import BasePackageManager
from platformio.package.meta import PackageSpec, PackageType
from platformio.project.helpers import get_project_global_lib_dir
class LibraryPackageManager(BasePackageManager):
def __init__(self, package_dir=None):
super(LibraryPackageManager, self).__init__(
PackageType.LIBRARY, package_dir or get_project_global_lib_dir()
)
@property
def manifest_names(self):
return PackageType.get_manifest_map()[PackageType.LIBRARY]
def find_pkg_root(self, path, spec):
try:
return super(LibraryPackageManager, self).find_pkg_root(path, spec)
except MissingPackageManifestError:
pass
assert isinstance(spec, PackageSpec)
root_dir = self.find_library_root(path)
# automatically generate library manifest
with open(os.path.join(root_dir, "library.json"), "w") as fp:
json.dump(
dict(name=spec.name, version=self.generate_rand_version(),),
fp,
indent=2,
)
return root_dir
@staticmethod
def find_library_root(path):
for root, dirs, files in os.walk(path):
if not files and len(dirs) == 1:
continue
for fname in files:
if not fname.endswith((".c", ".cpp", ".h", ".S")):
continue
if os.path.isdir(os.path.join(os.path.dirname(root), "src")):
return os.path.dirname(root)
return root
return path
+30
View File
@@ -0,0 +1,30 @@
# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from platformio.package.manager.base import BasePackageManager
from platformio.package.meta import PackageType
from platformio.project.config import ProjectConfig
class PlatformPackageManager(BasePackageManager):
def __init__(self, package_dir=None):
self.config = ProjectConfig.get_instance()
super(PlatformPackageManager, self).__init__(
PackageType.PLATFORM,
package_dir or self.config.get_optional_dir("platforms"),
)
@property
def manifest_names(self):
return PackageType.get_manifest_map()[PackageType.PLATFORM]
+29
View File
@@ -0,0 +1,29 @@
# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from platformio.package.manager.base import BasePackageManager
from platformio.package.meta import PackageType
from platformio.project.config import ProjectConfig
class ToolPackageManager(BasePackageManager):
def __init__(self, package_dir=None):
self.config = ProjectConfig.get_instance()
super(ToolPackageManager, self).__init__(
PackageType.TOOL, package_dir or self.config.get_optional_dir("packages"),
)
@property
def manifest_names(self):
return PackageType.get_manifest_map()[PackageType.TOOL]
+382
View File
@@ -0,0 +1,382 @@
# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
import os
import re
import tarfile
import semantic_version
from platformio.compat import get_object_members, string_types
from platformio.package.manifest.parser import ManifestFileType
try:
from urllib.parse import urlparse
except ImportError:
from urlparse import urlparse
class PackageType(object):
LIBRARY = "library"
PLATFORM = "platform"
TOOL = "tool"
@classmethod
def items(cls):
return get_object_members(cls)
@classmethod
def get_manifest_map(cls):
return {
cls.PLATFORM: (ManifestFileType.PLATFORM_JSON,),
cls.LIBRARY: (
ManifestFileType.LIBRARY_JSON,
ManifestFileType.LIBRARY_PROPERTIES,
ManifestFileType.MODULE_JSON,
),
cls.TOOL: (ManifestFileType.PACKAGE_JSON,),
}
@classmethod
def from_archive(cls, path):
assert path.endswith("tar.gz")
manifest_map = cls.get_manifest_map()
with tarfile.open(path, mode="r:gz") as tf:
for t in sorted(cls.items().values()):
for manifest in manifest_map[t]:
try:
if tf.getmember(manifest):
return t
except KeyError:
pass
return None
class PackageSpec(object):
def __init__( # pylint: disable=redefined-builtin,too-many-arguments
self, raw=None, owner=None, id=None, name=None, requirements=None, url=None
):
self.owner = owner
self.id = id
self.name = name
self._requirements = None
self.url = url
if requirements:
self.requirements = requirements
self._name_is_custom = False
self._parse(raw)
def __eq__(self, other):
return all(
[
self.owner == other.owner,
self.id == other.id,
self.name == other.name,
self.requirements == other.requirements,
self.url == other.url,
]
)
def __repr__(self):
return (
"PackageSpec <owner={owner} id={id} name={name} "
"requirements={requirements} url={url}>".format(**self.as_dict())
)
@property
def requirements(self):
return self._requirements
@requirements.setter
def requirements(self, value):
if not value:
self._requirements = None
return
self._requirements = (
value
if isinstance(value, semantic_version.SimpleSpec)
else semantic_version.SimpleSpec(value)
)
def humanize(self):
if self.url:
result = self.url
elif self.id:
result = "id:%d" % self.id
else:
result = ""
if self.owner:
result = self.owner + "/"
result += self.name
if self.requirements:
result += " @ " + str(self.requirements)
return result
def is_custom_name(self):
return self._name_is_custom
def as_dict(self):
return dict(
owner=self.owner,
id=self.id,
name=self.name,
requirements=str(self.requirements) if self.requirements else None,
url=self.url,
)
def _parse(self, raw):
if raw is None:
return
if not isinstance(raw, string_types):
raw = str(raw)
raw = raw.strip()
parsers = (
self._parse_requirements,
self._parse_custom_name,
self._parse_id,
self._parse_owner,
self._parse_url,
)
for parser in parsers:
if raw is None:
break
raw = parser(raw)
# if name is not custom, parse it from URL
if not self.name and self.url:
self.name = self._parse_name_from_url(self.url)
elif raw:
# the leftover is a package name
self.name = raw
def _parse_requirements(self, raw):
if "@" not in raw:
return raw
tokens = raw.rsplit("@", 1)
if any(s in tokens[1] for s in (":", "/")):
return raw
self.requirements = tokens[1].strip()
return tokens[0].strip()
def _parse_custom_name(self, raw):
if "=" not in raw or raw.startswith("id="):
return raw
tokens = raw.split("=", 1)
if "/" in tokens[0]:
return raw
self.name = tokens[0].strip()
self._name_is_custom = True
return tokens[1].strip()
def _parse_id(self, raw):
if raw.isdigit():
self.id = int(raw)
return None
if raw.startswith("id="):
return self._parse_id(raw[3:])
return raw
def _parse_owner(self, raw):
if raw.count("/") != 1 or "@" in raw:
return raw
tokens = raw.split("/", 1)
self.owner = tokens[0].strip()
self.name = tokens[1].strip()
return None
def _parse_url(self, raw):
if not any(s in raw for s in ("@", ":", "/")):
return raw
self.url = raw.strip()
parts = urlparse(self.url)
# if local file or valid URL with scheme vcs+protocol://
if parts.scheme == "file" or "+" in parts.scheme or self.url.startswith("git+"):
return None
# parse VCS
git_conditions = [
parts.path.endswith(".git"),
# Handle GitHub URL (https://github.com/user/package)
parts.netloc in ("github.com", "gitlab.com", "bitbucket.com")
and not parts.path.endswith((".zip", ".tar.gz")),
]
hg_conditions = [
# Handle Developer Mbed URL
# (https://developer.mbed.org/users/user/code/package/)
# (https://os.mbed.com/users/user/code/package/)
parts.netloc
in ("mbed.com", "os.mbed.com", "developer.mbed.org")
]
if any(git_conditions):
self.url = "git+" + self.url
elif any(hg_conditions):
self.url = "hg+" + self.url
return None
@staticmethod
def _parse_name_from_url(url):
if url.endswith("/"):
url = url[:-1]
for c in ("#", "?"):
if c in url:
url = url[: url.index(c)]
# parse real repository name from Github
parts = urlparse(url)
if parts.netloc == "github.com" and parts.path.count("/") > 2:
return parts.path.split("/")[2]
name = os.path.basename(url)
if "." in name:
return name.split(".", 1)[0].strip()
return name
class PackageMetaData(object):
def __init__( # pylint: disable=redefined-builtin
self, type, name, version, spec=None
):
assert type in PackageType.items().values()
if spec:
assert isinstance(spec, PackageSpec)
self.type = type
self.name = name
self._version = None
self.version = version
self.spec = spec
def __repr__(self):
return (
"PackageMetaData <type={type} name={name} version={version} "
"spec={spec}".format(**self.as_dict())
)
def __eq__(self, other):
return all(
[
self.type == other.type,
self.name == other.name,
self.version == other.version,
self.spec == other.spec,
]
)
@property
def version(self):
return self._version
@version.setter
def version(self, value):
if not value:
self._version = None
return
self._version = (
value
if isinstance(value, semantic_version.Version)
else self.to_semver(value)
)
@staticmethod
def to_semver(value, force=True, raise_exception=False):
assert value
try:
return semantic_version.Version(value)
except ValueError:
pass
if force:
try:
return semantic_version.Version.coerce(value)
except ValueError:
pass
if raise_exception:
raise ValueError("Invalid SemVer version %s" % value)
# parse commit hash
if re.match(r"^[\da-f]+$", value, flags=re.I):
return semantic_version.Version("0.0.0+sha." + value)
return semantic_version.Version("0.0.0+" + value)
def as_dict(self):
return dict(
type=self.type,
name=self.name,
version=str(self.version),
spec=self.spec.as_dict() if self.spec else None,
)
def dump(self, path):
with open(path, "w") as fp:
return json.dump(self.as_dict(), fp)
@staticmethod
def load(path):
with open(path) as fp:
data = json.load(fp)
if data["spec"]:
data["spec"] = PackageSpec(**data["spec"])
return PackageMetaData(**data)
class PackageSourceItem(object):
METAFILE_NAME = ".piopm"
def __init__(self, path, metadata=None):
self.path = path
self.metadata = metadata
if not self.metadata and self.exists():
self.metadata = self.load_meta()
def __repr__(self):
return "PackageSourceItem <path={path} metadata={metadata}".format(
path=self.path, metadata=self.metadata
)
def __eq__(self, other):
return all([self.path == other.path, self.metadata == other.metadata])
def exists(self):
return os.path.isdir(self.path)
def get_safe_dirname(self):
assert self.metadata
return re.sub(r"[^\da-z\_\-\. ]", "_", self.metadata.name, flags=re.I)
def get_metafile_locations(self):
return [
os.path.join(self.path, ".git"),
os.path.join(self.path, ".hg"),
os.path.join(self.path, ".svn"),
self.path,
]
def load_meta(self):
assert self.exists()
for location in self.get_metafile_locations():
manifest_path = os.path.join(location, self.METAFILE_NAME)
if os.path.isfile(manifest_path):
return PackageMetaData.load(manifest_path)
return None
def dump_meta(self):
assert self.exists()
location = None
for location in self.get_metafile_locations():
if os.path.isdir(location):
break
assert location
return self.metadata.dump(os.path.join(location, self.METAFILE_NAME))
+3 -1
View File
@@ -23,7 +23,8 @@ from platformio import fs
from platformio.package.exception import PackageException
from platformio.package.manifest.parser import ManifestFileType, ManifestParserFactory
from platformio.package.manifest.schema import ManifestSchema
from platformio.unpacker import FileUnpacker
from platformio.package.meta import PackageSourceItem
from platformio.package.unpack import FileUnpacker
class PackagePacker(object):
@@ -36,6 +37,7 @@ class PackagePacker(object):
".svn/",
".pio/",
"**/.pio/",
PackageSourceItem.METAFILE_NAME,
]
INCLUDE_DEFAULT = ManifestFileType.items().values()
-169
View File
@@ -1,169 +0,0 @@
# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import tarfile
from platformio.compat import get_object_members, string_types
from platformio.package.manifest.parser import ManifestFileType
class PackageType(object):
LIBRARY = "library"
PLATFORM = "platform"
TOOL = "tool"
@classmethod
def items(cls):
return get_object_members(cls)
@classmethod
def get_manifest_map(cls):
return {
cls.PLATFORM: (ManifestFileType.PLATFORM_JSON,),
cls.LIBRARY: (
ManifestFileType.LIBRARY_JSON,
ManifestFileType.LIBRARY_PROPERTIES,
ManifestFileType.MODULE_JSON,
),
cls.TOOL: (ManifestFileType.PACKAGE_JSON,),
}
@classmethod
def from_archive(cls, path):
assert path.endswith("tar.gz")
manifest_map = cls.get_manifest_map()
with tarfile.open(path, mode="r:gz") as tf:
for t in sorted(cls.items().values()):
for manifest in manifest_map[t]:
try:
if tf.getmember(manifest):
return t
except KeyError:
pass
return None
class PackageSpec(object):
def __init__( # pylint: disable=redefined-builtin,too-many-arguments
self, raw=None, ownername=None, id=None, name=None, requirements=None, url=None
):
self.ownername = ownername
self.id = id
self.name = name
self.requirements = requirements
self.url = url
self._parse(raw)
def __repr__(self):
return (
"PackageSpec <ownername={ownername} id={id} name={name} "
"requirements={requirements} url={url}>".format(
ownername=self.ownername,
id=self.id,
name=self.name,
requirements=self.requirements,
url=self.url,
)
)
def __eq__(self, other):
return all(
[
self.ownername == other.ownername,
self.id == other.id,
self.name == other.name,
self.requirements == other.requirements,
self.url == other.url,
]
)
def _parse(self, raw):
if raw is None:
return
if not isinstance(raw, string_types):
raw = str(raw)
raw = raw.strip()
parsers = (
self._parse_requirements,
self._parse_fixed_name,
self._parse_id,
self._parse_ownername,
self._parse_url,
)
for parser in parsers:
if raw is None:
break
raw = parser(raw)
# if name is not fixed, parse it from URL
if not self.name and self.url:
self.name = self._parse_name_from_url(self.url)
elif raw:
# the leftover is a package name
self.name = raw
def _parse_requirements(self, raw):
if "@" not in raw:
return raw
tokens = raw.rsplit("@", 1)
if any(s in tokens[1] for s in (":", "/")):
return raw
self.requirements = tokens[1].strip()
return tokens[0].strip()
def _parse_fixed_name(self, raw):
if "=" not in raw or raw.startswith("id="):
return raw
tokens = raw.split("=", 1)
if "/" in tokens[0]:
return raw
self.name = tokens[0].strip()
return tokens[1].strip()
def _parse_id(self, raw):
if raw.isdigit():
self.id = int(raw)
return None
if raw.startswith("id="):
return self._parse_id(raw[3:])
return raw
def _parse_ownername(self, raw):
if raw.count("/") != 1 or "@" in raw:
return raw
tokens = raw.split("/", 1)
self.ownername = tokens[0].strip()
self.name = tokens[1].strip()
return None
def _parse_url(self, raw):
if not any(s in raw for s in ("@", ":", "/")):
return raw
self.url = raw.strip()
return None
@staticmethod
def _parse_name_from_url(url):
if url.endswith("/"):
url = url[:-1]
for c in ("#", "?"):
if c in url:
url = url[: url.index(c)]
name = os.path.basename(url)
if "." in name:
return name.split(".", 1)[0].strip()
return name
+188
View File
@@ -0,0 +1,188 @@
# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
from tarfile import open as tarfile_open
from time import mktime
from zipfile import ZipFile
import click
from platformio import util
from platformio.package.exception import PackageException
class ExtractArchiveItemError(PackageException):
MESSAGE = (
"Could not extract `{0}` to `{1}`. Try to disable antivirus "
"tool or check this solution -> http://bit.ly/faq-package-manager"
)
class BaseArchiver(object):
def __init__(self, arhfileobj):
self._afo = arhfileobj
def get_items(self):
raise NotImplementedError()
def get_item_filename(self, item):
raise NotImplementedError()
def is_link(self, item):
raise NotImplementedError()
def extract_item(self, item, dest_dir):
self._afo.extract(item, dest_dir)
self.after_extract(item, dest_dir)
def after_extract(self, item, dest_dir):
pass
def close(self):
self._afo.close()
class TARArchiver(BaseArchiver):
def __init__(self, archpath):
super(TARArchiver, self).__init__(tarfile_open(archpath))
def get_items(self):
return self._afo.getmembers()
def get_item_filename(self, item):
return item.name
@staticmethod
def is_link(item):
return item.islnk() or item.issym()
@staticmethod
def resolve_path(path):
return os.path.realpath(os.path.abspath(path))
def is_bad_path(self, path, base):
return not self.resolve_path(os.path.join(base, path)).startswith(base)
def is_bad_link(self, item, base):
return not self.resolve_path(
os.path.join(os.path.join(base, os.path.dirname(item.name)), item.linkname)
).startswith(base)
def extract_item(self, item, dest_dir):
dest_dir = self.resolve_path(dest_dir)
bad_conds = [
self.is_bad_path(item.name, dest_dir),
self.is_link(item) and self.is_bad_link(item, dest_dir),
]
if not any(bad_conds):
super(TARArchiver, self).extract_item(item, dest_dir)
else:
click.secho(
"Blocked insecure item `%s` from TAR archive" % item.name,
fg="red",
err=True,
)
class ZIPArchiver(BaseArchiver):
def __init__(self, archpath):
super(ZIPArchiver, self).__init__(ZipFile(archpath))
@staticmethod
def preserve_permissions(item, dest_dir):
attrs = item.external_attr >> 16
if attrs:
os.chmod(os.path.join(dest_dir, item.filename), attrs)
@staticmethod
def preserve_mtime(item, dest_dir):
util.change_filemtime(
os.path.join(dest_dir, item.filename),
mktime(tuple(item.date_time) + tuple([0, 0, 0])),
)
@staticmethod
def is_link(_):
return False
def get_items(self):
return self._afo.infolist()
def get_item_filename(self, item):
return item.filename
def after_extract(self, item, dest_dir):
self.preserve_permissions(item, dest_dir)
self.preserve_mtime(item, dest_dir)
class FileUnpacker(object):
def __init__(self, path):
self.path = path
self._archiver = None
def _init_archiver(self):
magic_map = {
b"\x1f\x8b\x08": TARArchiver,
b"\x42\x5a\x68": TARArchiver,
b"\x50\x4b\x03\x04": ZIPArchiver,
}
magic_len = max(len(k) for k in magic_map)
with open(self.path, "rb") as fp:
data = fp.read(magic_len)
for magic, archiver in magic_map.items():
if data.startswith(magic):
return archiver(self.path)
raise PackageException("Unknown archive type '%s'" % self.path)
def __enter__(self):
self._archiver = self._init_archiver()
return self
def __exit__(self, *args):
if self._archiver:
self._archiver.close()
def unpack(
self, dest_dir=None, with_progress=True, check_unpacked=True, silent=False
):
assert self._archiver
if not dest_dir:
dest_dir = os.getcwd()
if not with_progress or silent:
if not silent:
click.echo("Unpacking...")
for item in self._archiver.get_items():
self._archiver.extract_item(item, dest_dir)
else:
items = self._archiver.get_items()
with click.progressbar(items, label="Unpacking") as pb:
for item in pb:
self._archiver.extract_item(item, dest_dir)
if not check_unpacked:
return True
# check on disk
for item in self._archiver.get_items():
filename = self._archiver.get_item_filename(item)
item_path = os.path.join(dest_dir, filename)
try:
if not self._archiver.is_link(item) and not os.path.exists(item_path):
raise ExtractArchiveItemError(filename, dest_dir)
except NotImplementedError:
pass
return True
+242
View File
@@ -0,0 +1,242 @@
# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import re
from os.path import join
from subprocess import CalledProcessError, check_call
from sys import modules
from platformio.exception import PlatformioException, UserSideException
from platformio.proc import exec_command
try:
from urllib.parse import urlparse
except ImportError:
from urlparse import urlparse
class VCSClientFactory(object):
@staticmethod
def newClient(src_dir, remote_url, silent=False):
result = urlparse(remote_url)
type_ = result.scheme
tag = None
if not type_ and remote_url.startswith("git+"):
type_ = "git"
remote_url = remote_url[4:]
elif "+" in result.scheme:
type_, _ = result.scheme.split("+", 1)
remote_url = remote_url[len(type_) + 1 :]
if "#" in remote_url:
remote_url, tag = remote_url.rsplit("#", 1)
if not type_:
raise PlatformioException("VCS: Unknown repository type %s" % remote_url)
obj = getattr(modules[__name__], "%sClient" % type_.title())(
src_dir, remote_url, tag, silent
)
assert isinstance(obj, VCSClientBase)
return obj
class VCSClientBase(object):
command = None
def __init__(self, src_dir, remote_url=None, tag=None, silent=False):
self.src_dir = src_dir
self.remote_url = remote_url
self.tag = tag
self.silent = silent
self.check_client()
def check_client(self):
try:
assert self.command
if self.silent:
self.get_cmd_output(["--version"])
else:
assert self.run_cmd(["--version"])
except (AssertionError, OSError, PlatformioException):
raise UserSideException(
"VCS: `%s` client is not installed in your system" % self.command
)
return True
@property
def storage_dir(self):
return join(self.src_dir, "." + self.command)
def export(self):
raise NotImplementedError
def update(self):
raise NotImplementedError
@property
def can_be_updated(self):
return not self.tag
def get_current_revision(self):
raise NotImplementedError
def get_latest_revision(self):
return None if self.can_be_updated else self.get_current_revision()
def run_cmd(self, args, **kwargs):
args = [self.command] + args
if "cwd" not in kwargs:
kwargs["cwd"] = self.src_dir
try:
check_call(args, **kwargs)
return True
except CalledProcessError as e:
raise PlatformioException("VCS: Could not process command %s" % e.cmd)
def get_cmd_output(self, args, **kwargs):
args = [self.command] + args
if "cwd" not in kwargs:
kwargs["cwd"] = self.src_dir
result = exec_command(args, **kwargs)
if result["returncode"] == 0:
return result["out"].strip()
raise PlatformioException(
"VCS: Could not receive an output from `%s` command (%s)" % (args, result)
)
class GitClient(VCSClientBase):
command = "git"
def check_client(self):
try:
return VCSClientBase.check_client(self)
except UserSideException:
raise UserSideException(
"Please install Git client from https://git-scm.com/downloads"
)
def get_branches(self):
output = self.get_cmd_output(["branch"])
output = output.replace("*", "") # fix active branch
return [b.strip() for b in output.split("\n")]
def get_current_branch(self):
output = self.get_cmd_output(["branch"])
for line in output.split("\n"):
line = line.strip()
if line.startswith("*"):
branch = line[1:].strip()
if branch != "(no branch)":
return branch
return None
def get_tags(self):
output = self.get_cmd_output(["tag", "-l"])
return [t.strip() for t in output.split("\n")]
@staticmethod
def is_commit_id(text):
return text and re.match(r"[0-9a-f]{7,}$", text) is not None
@property
def can_be_updated(self):
return not self.tag or not self.is_commit_id(self.tag)
def export(self):
is_commit = self.is_commit_id(self.tag)
args = ["clone", "--recursive"]
if not self.tag or not is_commit:
args += ["--depth", "1"]
if self.tag:
args += ["--branch", self.tag]
args += [self.remote_url, self.src_dir]
assert self.run_cmd(args)
if is_commit:
assert self.run_cmd(["reset", "--hard", self.tag])
return self.run_cmd(
["submodule", "update", "--init", "--recursive", "--force"]
)
return True
def update(self):
args = ["pull", "--recurse-submodules"]
return self.run_cmd(args)
def get_current_revision(self):
return self.get_cmd_output(["rev-parse", "--short", "HEAD"])
def get_latest_revision(self):
if not self.can_be_updated:
return self.get_current_revision()
branch = self.get_current_branch()
if not branch:
return self.get_current_revision()
result = self.get_cmd_output(["ls-remote"])
for line in result.split("\n"):
ref_pos = line.strip().find("refs/heads/" + branch)
if ref_pos > 0:
return line[:ref_pos].strip()[:7]
return None
class HgClient(VCSClientBase):
command = "hg"
def export(self):
args = ["clone"]
if self.tag:
args.extend(["--updaterev", self.tag])
args.extend([self.remote_url, self.src_dir])
return self.run_cmd(args)
def update(self):
args = ["pull", "--update"]
return self.run_cmd(args)
def get_current_revision(self):
return self.get_cmd_output(["identify", "--id"])
def get_latest_revision(self):
if not self.can_be_updated:
return self.get_latest_revision()
return self.get_cmd_output(["identify", "--id", self.remote_url])
class SvnClient(VCSClientBase):
command = "svn"
def export(self):
args = ["checkout"]
if self.tag:
args.extend(["--revision", self.tag])
args.extend([self.remote_url, self.src_dir])
return self.run_cmd(args)
def update(self):
args = ["update"]
return self.run_cmd(args)
def get_current_revision(self):
output = self.get_cmd_output(
["info", "--non-interactive", "--trust-server-cert", "-r", "HEAD"]
)
for line in output.split("\n"):
line = line.strip()
if line.startswith("Revision:"):
return line.split(":", 1)[1].strip()
raise PlatformioException("Could not detect current SVN revision")