forked from platformio/platformio-core
Initial version of a new package manager
This commit is contained in:
1 parent
abc0489ac6
commit
d329aef876
27 files changed
+2074
-407
No files matched your search
@@ -0,0 +1,142 @@
|
||||
# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import io
|
||||
import math
|
||||
import sys
|
||||
from email.utils import parsedate_tz
|
||||
from os.path import getsize, join
|
||||
from time import mktime
|
||||
|
||||
import click
|
||||
import requests
|
||||
|
||||
from platformio import app, fs, util
|
||||
from platformio.package.exception import PackageException
|
||||
|
||||
|
||||
class FileDownloader(object):
|
||||
def __init__(self, url, dest_dir=None):
|
||||
self._request = None
|
||||
# make connection
|
||||
self._request = requests.get(
|
||||
url,
|
||||
stream=True,
|
||||
headers={"User-Agent": app.get_user_agent()},
|
||||
verify=sys.version_info >= (2, 7, 9),
|
||||
)
|
||||
if self._request.status_code != 200:
|
||||
raise PackageException(
|
||||
"Got the unrecognized status code '{0}' when downloaded {1}".format(
|
||||
self._request.status_code, url
|
||||
)
|
||||
)
|
||||
|
||||
disposition = self._request.headers.get("content-disposition")
|
||||
if disposition and "filename=" in disposition:
|
||||
self._fname = (
|
||||
disposition[disposition.index("filename=") + 9 :]
|
||||
.replace('"', "")
|
||||
.replace("'", "")
|
||||
)
|
||||
else:
|
||||
self._fname = [p for p in url.split("/") if p][-1]
|
||||
self._fname = str(self._fname)
|
||||
self._destination = self._fname
|
||||
if dest_dir:
|
||||
self.set_destination(join(dest_dir, self._fname))
|
||||
|
||||
def set_destination(self, destination):
|
||||
self._destination = destination
|
||||
|
||||
def get_filepath(self):
|
||||
return self._destination
|
||||
|
||||
def get_lmtime(self):
|
||||
return self._request.headers.get("last-modified")
|
||||
|
||||
def get_size(self):
|
||||
if "content-length" not in self._request.headers:
|
||||
return -1
|
||||
return int(self._request.headers["content-length"])
|
||||
|
||||
def start(self, with_progress=True, silent=False):
|
||||
label = "Downloading"
|
||||
itercontent = self._request.iter_content(chunk_size=io.DEFAULT_BUFFER_SIZE)
|
||||
fp = open(self._destination, "wb")
|
||||
try:
|
||||
if not with_progress or self.get_size() == -1:
|
||||
if not silent:
|
||||
click.echo("%s..." % label)
|
||||
for chunk in itercontent:
|
||||
if chunk:
|
||||
fp.write(chunk)
|
||||
else:
|
||||
chunks = int(math.ceil(self.get_size() / float(io.DEFAULT_BUFFER_SIZE)))
|
||||
with click.progressbar(length=chunks, label=label) as pb:
|
||||
for _ in pb:
|
||||
fp.write(next(itercontent))
|
||||
finally:
|
||||
fp.close()
|
||||
self._request.close()
|
||||
|
||||
if self.get_lmtime():
|
||||
self._preserve_filemtime(self.get_lmtime())
|
||||
|
||||
return True
|
||||
|
||||
def verify(self, checksum=None):
|
||||
_dlsize = getsize(self._destination)
|
||||
if self.get_size() != -1 and _dlsize != self.get_size():
|
||||
raise PackageException(
|
||||
(
|
||||
"The size ({0:d} bytes) of downloaded file '{1}' "
|
||||
"is not equal to remote size ({2:d} bytes)"
|
||||
).format(_dlsize, self._fname, self.get_size())
|
||||
)
|
||||
if not checksum:
|
||||
return True
|
||||
|
||||
checksum_len = len(checksum)
|
||||
hash_algo = None
|
||||
if checksum_len == 32:
|
||||
hash_algo = "md5"
|
||||
elif checksum_len == 40:
|
||||
hash_algo = "sha1"
|
||||
elif checksum_len == 64:
|
||||
hash_algo = "sha256"
|
||||
|
||||
if not hash_algo:
|
||||
raise PackageException(
|
||||
"Could not determine checksum algorithm by %s" % checksum
|
||||
)
|
||||
|
||||
dl_checksum = fs.calculate_file_hashsum(hash_algo, self._destination)
|
||||
if checksum.lower() != dl_checksum.lower():
|
||||
raise PackageException(
|
||||
"The checksum '{0}' of the downloaded file '{1}' "
|
||||
"does not match to the remote '{2}'".format(
|
||||
dl_checksum, self._fname, checksum
|
||||
)
|
||||
)
|
||||
return True
|
||||
|
||||
def _preserve_filemtime(self, lmdate):
|
||||
timedata = parsedate_tz(lmdate)
|
||||
lmtime = mktime(timedata[:9])
|
||||
util.change_filemtime(self._destination, lmtime)
|
||||
|
||||
def __del__(self):
|
||||
if self._request:
|
||||
self._request.close()
|
||||
@@ -12,7 +12,8 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from platformio.exception import PlatformioException
|
||||
from platformio import util
|
||||
from platformio.exception import PlatformioException, UserSideException
|
||||
|
||||
|
||||
class PackageException(PlatformioException):
|
||||
@@ -44,3 +45,16 @@ class ManifestValidationError(ManifestException):
|
||||
"https://docs.platformio.org/page/librarymanager/config.html"
|
||||
% self.messages
|
||||
)
|
||||
|
||||
|
||||
class MissingPackageManifestError(ManifestException):
|
||||
|
||||
MESSAGE = "Could not find one of '{0}' manifest files in the package"
|
||||
|
||||
|
||||
class UnknownPackageError(UserSideException):
|
||||
|
||||
MESSAGE = (
|
||||
"Could not find a package with '{0}' requirements for your system '%s'"
|
||||
% util.get_systype()
|
||||
)
|
||||
@@ -0,0 +1,108 @@
|
||||
# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import os
|
||||
from time import sleep, time
|
||||
|
||||
from platformio import exception
|
||||
|
||||
LOCKFILE_TIMEOUT = 3600 # in seconds, 1 hour
|
||||
LOCKFILE_DELAY = 0.2
|
||||
|
||||
LOCKFILE_INTERFACE_FCNTL = 1
|
||||
LOCKFILE_INTERFACE_MSVCRT = 2
|
||||
|
||||
try:
|
||||
import fcntl
|
||||
|
||||
LOCKFILE_CURRENT_INTERFACE = LOCKFILE_INTERFACE_FCNTL
|
||||
except ImportError:
|
||||
try:
|
||||
import msvcrt
|
||||
|
||||
LOCKFILE_CURRENT_INTERFACE = LOCKFILE_INTERFACE_MSVCRT
|
||||
except ImportError:
|
||||
LOCKFILE_CURRENT_INTERFACE = None
|
||||
|
||||
|
||||
class LockFileExists(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class LockFile(object):
|
||||
def __init__(self, path, timeout=LOCKFILE_TIMEOUT, delay=LOCKFILE_DELAY):
|
||||
self.timeout = timeout
|
||||
self.delay = delay
|
||||
self._lock_path = os.path.realpath(path) + ".lock"
|
||||
self._fp = None
|
||||
|
||||
def _lock(self):
|
||||
if not LOCKFILE_CURRENT_INTERFACE and os.path.exists(self._lock_path):
|
||||
# remove stale lock
|
||||
if time() - os.path.getmtime(self._lock_path) > 10:
|
||||
try:
|
||||
os.remove(self._lock_path)
|
||||
except: # pylint: disable=bare-except
|
||||
pass
|
||||
else:
|
||||
raise LockFileExists
|
||||
|
||||
self._fp = open(self._lock_path, "w")
|
||||
try:
|
||||
if LOCKFILE_CURRENT_INTERFACE == LOCKFILE_INTERFACE_FCNTL:
|
||||
fcntl.flock(self._fp.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
elif LOCKFILE_CURRENT_INTERFACE == LOCKFILE_INTERFACE_MSVCRT:
|
||||
msvcrt.locking(self._fp.fileno(), msvcrt.LK_NBLCK, 1)
|
||||
except IOError:
|
||||
self._fp = None
|
||||
raise LockFileExists
|
||||
return True
|
||||
|
||||
def _unlock(self):
|
||||
if not self._fp:
|
||||
return
|
||||
if LOCKFILE_CURRENT_INTERFACE == LOCKFILE_INTERFACE_FCNTL:
|
||||
fcntl.flock(self._fp.fileno(), fcntl.LOCK_UN)
|
||||
elif LOCKFILE_CURRENT_INTERFACE == LOCKFILE_INTERFACE_MSVCRT:
|
||||
msvcrt.locking(self._fp.fileno(), msvcrt.LK_UNLCK, 1)
|
||||
self._fp.close()
|
||||
self._fp = None
|
||||
|
||||
def acquire(self):
|
||||
elapsed = 0
|
||||
while elapsed < self.timeout:
|
||||
try:
|
||||
return self._lock()
|
||||
except LockFileExists:
|
||||
sleep(self.delay)
|
||||
elapsed += self.delay
|
||||
|
||||
raise exception.LockFileTimeoutError()
|
||||
|
||||
def release(self):
|
||||
self._unlock()
|
||||
if os.path.exists(self._lock_path):
|
||||
try:
|
||||
os.remove(self._lock_path)
|
||||
except: # pylint: disable=bare-except
|
||||
pass
|
||||
|
||||
def __enter__(self):
|
||||
self.acquire()
|
||||
|
||||
def __exit__(self, type_, value, traceback):
|
||||
self.release()
|
||||
|
||||
def __del__(self):
|
||||
self.release()
|
||||
@@ -0,0 +1,13 @@
|
||||
# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
@@ -0,0 +1,95 @@
|
||||
# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
import tempfile
|
||||
import time
|
||||
|
||||
import click
|
||||
|
||||
from platformio import app, compat
|
||||
from platformio.package.download import FileDownloader
|
||||
from platformio.package.lockfile import LockFile
|
||||
|
||||
|
||||
class PackageManagerDownloadMixin(object):
|
||||
|
||||
DOWNLOAD_CACHE_EXPIRE = 86400 * 30 # keep package in a local cache for 1 month
|
||||
|
||||
def compute_download_path(self, *args):
|
||||
request_hash = hashlib.new("sha256")
|
||||
for arg in args:
|
||||
request_hash.update(compat.hashlib_encode_data(arg))
|
||||
dl_path = os.path.join(self.get_download_dir(), request_hash.hexdigest())
|
||||
return dl_path
|
||||
|
||||
def get_download_usagedb_path(self):
|
||||
return os.path.join(self.get_download_dir(), "usage.db")
|
||||
|
||||
def set_download_utime(self, path, utime=None):
|
||||
with app.State(self.get_download_usagedb_path(), lock=True) as state:
|
||||
state[os.path.basename(path)] = int(time.time() if not utime else utime)
|
||||
|
||||
def cleanup_expired_downloads(self):
|
||||
with app.State(self.get_download_usagedb_path(), lock=True) as state:
|
||||
# remove outdated
|
||||
for fname in list(state.keys()):
|
||||
if state[fname] > (time.time() - self.DOWNLOAD_CACHE_EXPIRE):
|
||||
continue
|
||||
del state[fname]
|
||||
dl_path = os.path.join(self.get_download_dir(), fname)
|
||||
if os.path.isfile(dl_path):
|
||||
os.remove(dl_path)
|
||||
|
||||
def download(self, url, checksum=None, silent=False):
|
||||
dl_path = self.compute_download_path(url, checksum or "")
|
||||
if os.path.isfile(dl_path):
|
||||
self.set_download_utime(dl_path)
|
||||
return dl_path
|
||||
|
||||
with_progress = not silent and not app.is_disabled_progressbar()
|
||||
tmp_path = tempfile.mkstemp(dir=self.get_download_dir())[1]
|
||||
try:
|
||||
with LockFile(dl_path):
|
||||
try:
|
||||
fd = FileDownloader(url)
|
||||
fd.set_destination(tmp_path)
|
||||
fd.start(with_progress=with_progress, silent=silent)
|
||||
except IOError as e:
|
||||
raise_error = not with_progress
|
||||
if with_progress:
|
||||
try:
|
||||
fd = FileDownloader(url)
|
||||
fd.set_destination(tmp_path)
|
||||
fd.start(with_progress=False, silent=silent)
|
||||
except IOError:
|
||||
raise_error = True
|
||||
if raise_error:
|
||||
click.secho(
|
||||
"Error: Please read http://bit.ly/package-manager-ioerror",
|
||||
fg="red",
|
||||
err=True,
|
||||
)
|
||||
raise e
|
||||
if checksum:
|
||||
fd.verify(checksum)
|
||||
os.rename(tmp_path, dl_path)
|
||||
finally:
|
||||
if os.path.isfile(tmp_path):
|
||||
os.remove(tmp_path)
|
||||
|
||||
assert os.path.isfile(dl_path)
|
||||
self.set_download_utime(dl_path)
|
||||
return dl_path
|
||||
@@ -0,0 +1,282 @@
|
||||
# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
import shutil
|
||||
import tempfile
|
||||
|
||||
import click
|
||||
|
||||
from platformio import app, compat, fs, util
|
||||
from platformio.package.exception import PackageException, UnknownPackageError
|
||||
from platformio.package.lockfile import LockFile
|
||||
from platformio.package.meta import PackageSourceItem, PackageSpec
|
||||
from platformio.package.unpack import FileUnpacker
|
||||
from platformio.package.vcsclient import VCSClientFactory
|
||||
|
||||
|
||||
class PackageManagerInstallMixin(object):
|
||||
|
||||
INSTALL_HISTORY = None # avoid circle dependencies
|
||||
|
||||
@staticmethod
|
||||
def unpack(src, dst):
|
||||
with_progress = not app.is_disabled_progressbar()
|
||||
try:
|
||||
with FileUnpacker(src) as fu:
|
||||
return fu.unpack(dst, with_progress=with_progress)
|
||||
except IOError as e:
|
||||
if not with_progress:
|
||||
raise e
|
||||
with FileUnpacker(src) as fu:
|
||||
return fu.unpack(dst, with_progress=False)
|
||||
|
||||
def install(self, spec, silent=False):
|
||||
with LockFile(self.package_dir):
|
||||
pkg = self._install(spec, silent=silent)
|
||||
self.memcache_reset()
|
||||
self.cleanup_expired_downloads()
|
||||
return pkg
|
||||
|
||||
def _install(self, spec, search_filters=None, silent=False):
|
||||
spec = self.ensure_spec(spec)
|
||||
|
||||
# avoid circle dependencies
|
||||
if not self.INSTALL_HISTORY:
|
||||
self.INSTALL_HISTORY = []
|
||||
if spec in self.INSTALL_HISTORY:
|
||||
return None
|
||||
self.INSTALL_HISTORY.append(spec)
|
||||
|
||||
# check if package is already installed
|
||||
pkg = self.get_package(spec)
|
||||
if pkg:
|
||||
if not silent:
|
||||
click.secho(
|
||||
"{name} @ {version} is already installed".format(
|
||||
**pkg.metadata.as_dict()
|
||||
),
|
||||
fg="yellow",
|
||||
)
|
||||
return pkg
|
||||
|
||||
if not silent:
|
||||
msg = "Installing %s" % click.style(spec.humanize(), fg="cyan")
|
||||
self.print_message(msg)
|
||||
|
||||
if spec.url:
|
||||
pkg = self.install_from_url(spec.url, spec, silent=silent)
|
||||
else:
|
||||
pkg = self.install_from_registry(spec, search_filters, silent=silent)
|
||||
|
||||
if not pkg or not pkg.metadata:
|
||||
raise PackageException(
|
||||
"Could not install package '%s' for '%s' system"
|
||||
% (spec.humanize(), util.get_systype())
|
||||
)
|
||||
|
||||
if not silent:
|
||||
self.print_message(
|
||||
click.style(
|
||||
"{name} @ {version} has been successfully installed!".format(
|
||||
**pkg.metadata.as_dict()
|
||||
),
|
||||
fg="green",
|
||||
)
|
||||
)
|
||||
|
||||
self.memcache_reset()
|
||||
self.install_dependencies(pkg, silent)
|
||||
return pkg
|
||||
|
||||
def install_dependencies(self, pkg, silent=False):
|
||||
assert isinstance(pkg, PackageSourceItem)
|
||||
manifest = self.load_manifest(pkg)
|
||||
if not manifest.get("dependencies"):
|
||||
return
|
||||
if not silent:
|
||||
self.print_message(click.style("Installing dependencies...", fg="yellow"))
|
||||
for dependency in manifest.get("dependencies"):
|
||||
if not self.install_dependency(dependency, silent) and not silent:
|
||||
click.secho(
|
||||
"Warning! Could not install dependency %s for package '%s'"
|
||||
% (dependency, pkg.metadata.name),
|
||||
fg="yellow",
|
||||
)
|
||||
|
||||
def install_dependency(self, dependency, silent=False):
|
||||
spec = PackageSpec(
|
||||
name=dependency.get("name"), requirements=dependency.get("version")
|
||||
)
|
||||
search_filters = {
|
||||
key: value
|
||||
for key, value in dependency.items()
|
||||
if key in ("authors", "platforms", "frameworks")
|
||||
}
|
||||
return self._install(spec, search_filters=search_filters or None, silent=silent)
|
||||
|
||||
def install_from_url(self, url, spec, checksum=None, silent=False):
|
||||
spec = self.ensure_spec(spec)
|
||||
tmp_dir = tempfile.mkdtemp(prefix="pkg-installing-", dir=self.get_tmp_dir())
|
||||
vcs = None
|
||||
try:
|
||||
if url.startswith("file://"):
|
||||
_url = url[7:]
|
||||
if os.path.isfile(_url):
|
||||
self.unpack(_url, tmp_dir)
|
||||
else:
|
||||
fs.rmtree(tmp_dir)
|
||||
shutil.copytree(_url, tmp_dir, symlinks=True)
|
||||
elif url.startswith(("http://", "https://")):
|
||||
dl_path = self.download(url, checksum, silent=silent)
|
||||
assert os.path.isfile(dl_path)
|
||||
self.unpack(dl_path, tmp_dir)
|
||||
else:
|
||||
vcs = VCSClientFactory.newClient(tmp_dir, url)
|
||||
assert vcs.export()
|
||||
|
||||
root_dir = self.find_pkg_root(tmp_dir, spec)
|
||||
pkg_item = PackageSourceItem(
|
||||
root_dir,
|
||||
self.build_metadata(
|
||||
root_dir, spec, vcs.get_current_revision() if vcs else None
|
||||
),
|
||||
)
|
||||
pkg_item.dump_meta()
|
||||
return self._install_tmp_pkg(pkg_item)
|
||||
finally:
|
||||
if os.path.isdir(tmp_dir):
|
||||
fs.rmtree(tmp_dir)
|
||||
|
||||
def _install_tmp_pkg(self, tmp_pkg):
|
||||
assert isinstance(tmp_pkg, PackageSourceItem)
|
||||
# validate package version and declared requirements
|
||||
if (
|
||||
tmp_pkg.metadata.spec.requirements
|
||||
and tmp_pkg.metadata.version not in tmp_pkg.metadata.spec.requirements
|
||||
):
|
||||
raise PackageException(
|
||||
"Package version %s doesn't satisfy requirements %s based on %s"
|
||||
% (
|
||||
tmp_pkg.metadata.version,
|
||||
tmp_pkg.metadata.spec.requirements,
|
||||
tmp_pkg.metadata,
|
||||
)
|
||||
)
|
||||
dst_pkg = PackageSourceItem(
|
||||
os.path.join(self.package_dir, tmp_pkg.get_safe_dirname())
|
||||
)
|
||||
|
||||
# what to do with existing package?
|
||||
action = "overwrite"
|
||||
if dst_pkg.metadata and dst_pkg.metadata.spec.url:
|
||||
if dst_pkg.metadata.spec.url != tmp_pkg.metadata.spec.url:
|
||||
action = "detach-existing"
|
||||
elif tmp_pkg.metadata.spec.url:
|
||||
action = "detach-new"
|
||||
elif dst_pkg.metadata and dst_pkg.metadata.version != tmp_pkg.metadata.version:
|
||||
action = (
|
||||
"detach-existing"
|
||||
if tmp_pkg.metadata.version > dst_pkg.metadata.version
|
||||
else "detach-new"
|
||||
)
|
||||
|
||||
def _cleanup_dir(path):
|
||||
if os.path.isdir(path):
|
||||
fs.rmtree(path)
|
||||
|
||||
if action == "detach-existing":
|
||||
target_dirname = "%s@%s" % (
|
||||
tmp_pkg.get_safe_dirname(),
|
||||
dst_pkg.metadata.version,
|
||||
)
|
||||
if dst_pkg.metadata.spec.url:
|
||||
target_dirname = "%s@src-%s" % (
|
||||
tmp_pkg.get_safe_dirname(),
|
||||
hashlib.md5(
|
||||
compat.hashlib_encode_data(dst_pkg.metadata.spec.url)
|
||||
).hexdigest(),
|
||||
)
|
||||
# move existing into the new place
|
||||
pkg_dir = os.path.join(self.package_dir, target_dirname)
|
||||
_cleanup_dir(pkg_dir)
|
||||
shutil.move(dst_pkg.path, pkg_dir)
|
||||
# move new source to the destination location
|
||||
_cleanup_dir(dst_pkg.path)
|
||||
shutil.move(tmp_pkg.path, dst_pkg.path)
|
||||
return PackageSourceItem(dst_pkg.path)
|
||||
|
||||
if action == "detach-new":
|
||||
target_dirname = "%s@%s" % (
|
||||
tmp_pkg.get_safe_dirname(),
|
||||
tmp_pkg.metadata.version,
|
||||
)
|
||||
if tmp_pkg.metadata.spec.url:
|
||||
target_dirname = "%s@src-%s" % (
|
||||
tmp_pkg.get_safe_dirname(),
|
||||
hashlib.md5(
|
||||
compat.hashlib_encode_data(tmp_pkg.metadata.spec.url)
|
||||
).hexdigest(),
|
||||
)
|
||||
pkg_dir = os.path.join(self.package_dir, target_dirname)
|
||||
_cleanup_dir(pkg_dir)
|
||||
shutil.move(tmp_pkg.path, pkg_dir)
|
||||
return PackageSourceItem(pkg_dir)
|
||||
|
||||
# otherwise, overwrite existing
|
||||
_cleanup_dir(dst_pkg.path)
|
||||
shutil.move(tmp_pkg.path, dst_pkg.path)
|
||||
return PackageSourceItem(dst_pkg.path)
|
||||
|
||||
def uninstall(self, path_or_spec, silent=False):
|
||||
with LockFile(self.package_dir):
|
||||
pkg = (
|
||||
PackageSourceItem(path_or_spec)
|
||||
if os.path.isdir(path_or_spec)
|
||||
else self.get_package(path_or_spec)
|
||||
)
|
||||
if not pkg or not pkg.metadata:
|
||||
raise UnknownPackageError(path_or_spec)
|
||||
|
||||
if not silent:
|
||||
self.print_message(
|
||||
"Uninstalling %s @ %s: \t"
|
||||
% (click.style(pkg.metadata.name, fg="cyan"), pkg.metadata.version),
|
||||
nl=False,
|
||||
)
|
||||
if os.path.islink(pkg.path):
|
||||
os.unlink(pkg.path)
|
||||
else:
|
||||
fs.rmtree(pkg.path)
|
||||
self.memcache_reset()
|
||||
|
||||
# unfix detached-package with the same name
|
||||
detached_pkg = self.get_package(PackageSpec(name=pkg.metadata.name))
|
||||
if (
|
||||
detached_pkg
|
||||
and "@" in detached_pkg.path
|
||||
and not os.path.isdir(
|
||||
os.path.join(self.package_dir, detached_pkg.get_safe_dirname())
|
||||
)
|
||||
):
|
||||
shutil.move(
|
||||
detached_pkg.path,
|
||||
os.path.join(self.package_dir, detached_pkg.get_safe_dirname()),
|
||||
)
|
||||
self.memcache_reset()
|
||||
|
||||
if not silent:
|
||||
click.echo("[%s]" % click.style("OK", fg="green"))
|
||||
return True
|
||||
@@ -0,0 +1,190 @@
|
||||
# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import time
|
||||
|
||||
import click
|
||||
|
||||
from platformio.clients.http import HTTPClient
|
||||
from platformio.clients.registry import RegistryClient
|
||||
from platformio.package.exception import UnknownPackageError
|
||||
from platformio.package.meta import PackageMetaData, PackageSpec
|
||||
|
||||
try:
|
||||
from urllib.parse import urlparse
|
||||
except ImportError:
|
||||
from urlparse import urlparse
|
||||
|
||||
|
||||
class RegistryFileMirrorsIterator(object):
|
||||
|
||||
HTTP_CLIENT_INSTANCES = {}
|
||||
|
||||
def __init__(self, download_url):
|
||||
self.download_url = download_url
|
||||
self._url_parts = urlparse(download_url)
|
||||
self._base_url = "%s://%s" % (self._url_parts.scheme, self._url_parts.netloc)
|
||||
self._visited_mirrors = []
|
||||
|
||||
def __iter__(self):
|
||||
return self
|
||||
|
||||
def __next__(self):
|
||||
http = self.get_http_client()
|
||||
response = http.send_request(
|
||||
"head",
|
||||
self._url_parts.path,
|
||||
allow_redirects=False,
|
||||
params=dict(bypass=",".join(self._visited_mirrors))
|
||||
if self._visited_mirrors
|
||||
else None,
|
||||
)
|
||||
stop_conditions = [
|
||||
response.status_code not in (302, 307),
|
||||
not response.headers.get("Location"),
|
||||
not response.headers.get("X-PIO-Mirror"),
|
||||
response.headers.get("X-PIO-Mirror") in self._visited_mirrors,
|
||||
]
|
||||
if any(stop_conditions):
|
||||
raise StopIteration
|
||||
self._visited_mirrors.append(response.headers.get("X-PIO-Mirror"))
|
||||
return (
|
||||
response.headers.get("Location"),
|
||||
response.headers.get("X-PIO-Content-SHA256"),
|
||||
)
|
||||
|
||||
def get_http_client(self):
|
||||
if self._base_url not in RegistryFileMirrorsIterator.HTTP_CLIENT_INSTANCES:
|
||||
RegistryFileMirrorsIterator.HTTP_CLIENT_INSTANCES[
|
||||
self._base_url
|
||||
] = HTTPClient(self._base_url)
|
||||
return RegistryFileMirrorsIterator.HTTP_CLIENT_INSTANCES[self._base_url]
|
||||
|
||||
|
||||
class PackageManageRegistryMixin(object):
|
||||
def install_from_registry(self, spec, search_filters=None, silent=False):
|
||||
packages = self.search_registry_packages(spec, search_filters)
|
||||
if not packages:
|
||||
raise UnknownPackageError(spec.humanize())
|
||||
if len(packages) > 1 and not silent:
|
||||
self.print_multi_package_issue(packages, spec)
|
||||
package, version = self.find_best_registry_version(packages, spec)
|
||||
pkgfile = self._pick_compatible_pkg_file(version["files"]) if version else None
|
||||
if not pkgfile:
|
||||
raise UnknownPackageError(spec.humanize())
|
||||
|
||||
for url, checksum in RegistryFileMirrorsIterator(pkgfile["download_url"]):
|
||||
try:
|
||||
return self.install_from_url(
|
||||
url,
|
||||
PackageSpec(
|
||||
owner=package["owner"]["username"],
|
||||
id=package["id"],
|
||||
name=package["name"],
|
||||
),
|
||||
checksum or pkgfile["checksum"]["sha256"],
|
||||
silent=silent,
|
||||
)
|
||||
except Exception as e: # pylint: disable=broad-except
|
||||
click.secho("Warning! Package Mirror: %s" % e, fg="yellow")
|
||||
click.secho("Looking for another mirror...", fg="yellow")
|
||||
|
||||
return None
|
||||
|
||||
def get_registry_client_instance(self):
|
||||
if not self._registry_client:
|
||||
self._registry_client = RegistryClient()
|
||||
return self._registry_client
|
||||
|
||||
def search_registry_packages(self, spec, filters=None):
|
||||
filters = filters or {}
|
||||
if spec.id:
|
||||
filters["ids"] = str(spec.id)
|
||||
else:
|
||||
filters["types"] = self.pkg_type
|
||||
filters["names"] = '"%s"' % spec.name.lower()
|
||||
if spec.owner:
|
||||
filters["owners"] = spec.owner.lower()
|
||||
return self.get_registry_client_instance().list_packages(filters=filters)[
|
||||
"items"
|
||||
]
|
||||
|
||||
def fetch_registry_package_versions(self, owner, name):
|
||||
return self.get_registry_client_instance().get_package(
|
||||
self.pkg_type, owner, name
|
||||
)["versions"]
|
||||
|
||||
@staticmethod
|
||||
def print_multi_package_issue(packages, spec):
|
||||
click.secho(
|
||||
"Warning! More than one package has been found by ", fg="yellow", nl=False
|
||||
)
|
||||
click.secho(spec.humanize(), fg="cyan", nl=False)
|
||||
click.secho(" requirements:", fg="yellow")
|
||||
for item in packages:
|
||||
click.echo(
|
||||
" - {owner}/{name} @ {version}".format(
|
||||
owner=click.style(item["owner"]["username"], fg="cyan"),
|
||||
name=item["name"],
|
||||
version=item["version"]["name"],
|
||||
)
|
||||
)
|
||||
click.secho(
|
||||
"Please specify detailed REQUIREMENTS using package owner and version "
|
||||
"(showed above) to avoid project compatibility issues.",
|
||||
fg="yellow",
|
||||
)
|
||||
|
||||
def find_best_registry_version(self, packages, spec):
|
||||
# find compatible version within the latest package versions
|
||||
for package in packages:
|
||||
version = self._pick_best_pkg_version([package["version"]], spec)
|
||||
if version:
|
||||
return (package, version)
|
||||
|
||||
if not spec.requirements:
|
||||
return None
|
||||
|
||||
# if the custom version requirements, check ALL package versions
|
||||
for package in packages:
|
||||
version = self._pick_best_pkg_version(
|
||||
self.fetch_registry_package_versions(
|
||||
package["owner"]["username"], package["name"]
|
||||
),
|
||||
spec,
|
||||
)
|
||||
if version:
|
||||
return (package, version)
|
||||
time.sleep(1)
|
||||
return None
|
||||
|
||||
def _pick_best_pkg_version(self, versions, spec):
|
||||
best = None
|
||||
for version in versions:
|
||||
semver = PackageMetaData.to_semver(version["name"])
|
||||
if spec.requirements and semver not in spec.requirements:
|
||||
continue
|
||||
if not any(
|
||||
self.is_system_compatible(f.get("system")) for f in version["files"]
|
||||
):
|
||||
continue
|
||||
if not best or (semver > PackageMetaData.to_semver(best["name"])):
|
||||
best = version
|
||||
return best
|
||||
|
||||
def _pick_compatible_pkg_file(self, version_files):
|
||||
for item in version_files:
|
||||
if self.is_system_compatible(item.get("system")):
|
||||
return item
|
||||
return None
|
||||
@@ -0,0 +1,233 @@
|
||||
# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import os
|
||||
from datetime import datetime
|
||||
|
||||
import click
|
||||
import semantic_version
|
||||
|
||||
from platformio import fs, util
|
||||
from platformio.commands import PlatformioCLI
|
||||
from platformio.package.exception import ManifestException, MissingPackageManifestError
|
||||
from platformio.package.manager._download import PackageManagerDownloadMixin
|
||||
from platformio.package.manager._install import PackageManagerInstallMixin
|
||||
from platformio.package.manager._registry import PackageManageRegistryMixin
|
||||
from platformio.package.manifest.parser import ManifestParserFactory
|
||||
from platformio.package.meta import (
|
||||
PackageMetaData,
|
||||
PackageSourceItem,
|
||||
PackageSpec,
|
||||
PackageType,
|
||||
)
|
||||
from platformio.project.helpers import get_project_cache_dir
|
||||
|
||||
|
||||
class BasePackageManager(
|
||||
PackageManagerDownloadMixin, PackageManageRegistryMixin, PackageManagerInstallMixin
|
||||
):
|
||||
MEMORY_CACHE = {}
|
||||
|
||||
def __init__(self, pkg_type, package_dir):
|
||||
self.pkg_type = pkg_type
|
||||
self.package_dir = self.ensure_dir_exists(package_dir)
|
||||
self.MEMORY_CACHE = {}
|
||||
self._download_dir = None
|
||||
self._tmp_dir = None
|
||||
self._registry_client = None
|
||||
|
||||
def memcache_get(self, key, default=None):
|
||||
return self.MEMORY_CACHE.get(key, default)
|
||||
|
||||
def memcache_set(self, key, value):
|
||||
self.MEMORY_CACHE[key] = value
|
||||
|
||||
def memcache_reset(self):
|
||||
self.MEMORY_CACHE.clear()
|
||||
|
||||
@staticmethod
|
||||
def is_system_compatible(value):
|
||||
if not value or "*" in value:
|
||||
return True
|
||||
return util.items_in_list(value, util.get_systype())
|
||||
|
||||
@staticmethod
|
||||
def generate_rand_version():
|
||||
return datetime.now().strftime("0.0.0+%Y%m%d%H%M%S")
|
||||
|
||||
@staticmethod
|
||||
def ensure_dir_exists(path):
|
||||
if not os.path.isdir(path):
|
||||
os.makedirs(path)
|
||||
assert os.path.isdir(path)
|
||||
return path
|
||||
|
||||
@staticmethod
|
||||
def ensure_spec(spec):
|
||||
return spec if isinstance(spec, PackageSpec) else PackageSpec(spec)
|
||||
|
||||
@property
|
||||
def manifest_names(self):
|
||||
raise NotImplementedError
|
||||
|
||||
def print_message(self, message, nl=True):
|
||||
click.echo("%s: %s" % (self.__class__.__name__, message), nl=nl)
|
||||
|
||||
def get_download_dir(self):
|
||||
if not self._download_dir:
|
||||
self._download_dir = self.ensure_dir_exists(
|
||||
os.path.join(get_project_cache_dir(), "downloads")
|
||||
)
|
||||
return self._download_dir
|
||||
|
||||
def get_tmp_dir(self):
|
||||
if not self._tmp_dir:
|
||||
self._tmp_dir = self.ensure_dir_exists(
|
||||
os.path.join(get_project_cache_dir(), "tmp")
|
||||
)
|
||||
return self._tmp_dir
|
||||
|
||||
def find_pkg_root(self, path, spec): # pylint: disable=unused-argument
|
||||
if self.manifest_exists(path):
|
||||
return path
|
||||
for root, _, _ in os.walk(path):
|
||||
if self.manifest_exists(root):
|
||||
return root
|
||||
raise MissingPackageManifestError(", ".join(self.manifest_names))
|
||||
|
||||
def get_manifest_path(self, pkg_dir):
|
||||
if not os.path.isdir(pkg_dir):
|
||||
return None
|
||||
for name in self.manifest_names:
|
||||
manifest_path = os.path.join(pkg_dir, name)
|
||||
if os.path.isfile(manifest_path):
|
||||
return manifest_path
|
||||
return None
|
||||
|
||||
def manifest_exists(self, pkg_dir):
|
||||
return self.get_manifest_path(pkg_dir)
|
||||
|
||||
def load_manifest(self, src):
|
||||
path = src.path if isinstance(src, PackageSourceItem) else src
|
||||
cache_key = "load_manifest-%s" % path
|
||||
result = self.memcache_get(cache_key)
|
||||
if result:
|
||||
return result
|
||||
candidates = (
|
||||
[os.path.join(path, name) for name in self.manifest_names]
|
||||
if os.path.isdir(path)
|
||||
else [path]
|
||||
)
|
||||
for item in candidates:
|
||||
if not os.path.isfile(item):
|
||||
continue
|
||||
try:
|
||||
result = ManifestParserFactory.new_from_file(item).as_dict()
|
||||
self.memcache_set(cache_key, result)
|
||||
return result
|
||||
except ManifestException as e:
|
||||
if not PlatformioCLI.in_silence():
|
||||
click.secho(str(e), fg="yellow")
|
||||
raise MissingPackageManifestError(", ".join(self.manifest_names))
|
||||
|
||||
def build_legacy_spec(self, pkg_dir):
|
||||
# find src manifest
|
||||
src_manifest_name = ".piopkgmanager.json"
|
||||
src_manifest_path = None
|
||||
for name in os.listdir(pkg_dir):
|
||||
if not os.path.isfile(os.path.join(pkg_dir, name, src_manifest_name)):
|
||||
continue
|
||||
src_manifest_path = os.path.join(pkg_dir, name, src_manifest_name)
|
||||
break
|
||||
|
||||
if src_manifest_path:
|
||||
src_manifest = fs.load_json(src_manifest_path)
|
||||
return PackageSpec(
|
||||
name=src_manifest.get("name"),
|
||||
url=src_manifest.get("url"),
|
||||
requirements=src_manifest.get("requirements"),
|
||||
)
|
||||
|
||||
# fall back to a package manifest
|
||||
manifest = self.load_manifest(pkg_dir)
|
||||
return PackageSpec(name=manifest.get("name"))
|
||||
|
||||
def build_metadata(self, pkg_dir, spec, vcs_revision=None):
|
||||
manifest = self.load_manifest(pkg_dir)
|
||||
metadata = PackageMetaData(
|
||||
type=self.pkg_type,
|
||||
name=manifest.get("name"),
|
||||
version=manifest.get("version"),
|
||||
spec=spec,
|
||||
)
|
||||
if not metadata.name or spec.is_custom_name():
|
||||
metadata.name = spec.name
|
||||
if vcs_revision:
|
||||
metadata.version = "%s+sha.%s" % (
|
||||
metadata.version if metadata.version else "0.0.0",
|
||||
vcs_revision,
|
||||
)
|
||||
if not metadata.version:
|
||||
metadata.version = self.generate_rand_version()
|
||||
return metadata
|
||||
|
||||
def get_installed(self):
|
||||
result = []
|
||||
for name in os.listdir(self.package_dir):
|
||||
pkg_dir = os.path.join(self.package_dir, name)
|
||||
if not os.path.isdir(pkg_dir):
|
||||
continue
|
||||
pkg = PackageSourceItem(pkg_dir)
|
||||
if not pkg.metadata:
|
||||
try:
|
||||
spec = self.build_legacy_spec(pkg_dir)
|
||||
pkg.metadata = self.build_metadata(pkg_dir, spec)
|
||||
except MissingPackageManifestError:
|
||||
pass
|
||||
if pkg.metadata:
|
||||
result.append(pkg)
|
||||
return result
|
||||
|
||||
def get_package(self, spec):
|
||||
def _ci_strings_are_equal(a, b):
|
||||
if a == b:
|
||||
return True
|
||||
if not a or not b:
|
||||
return False
|
||||
return a.strip().lower() == b.strip().lower()
|
||||
|
||||
spec = self.ensure_spec(spec)
|
||||
best = None
|
||||
for pkg in self.get_installed():
|
||||
skip_conditions = [
|
||||
spec.owner
|
||||
and not _ci_strings_are_equal(spec.owner, pkg.metadata.spec.owner),
|
||||
spec.url and spec.url != pkg.metadata.spec.url,
|
||||
spec.id and spec.id != pkg.metadata.spec.id,
|
||||
not spec.id
|
||||
and not spec.url
|
||||
and not _ci_strings_are_equal(spec.name, pkg.metadata.name),
|
||||
]
|
||||
if any(skip_conditions):
|
||||
continue
|
||||
if self.pkg_type == PackageType.TOOL:
|
||||
# TODO: check "system" for pkg
|
||||
pass
|
||||
|
||||
assert isinstance(pkg.metadata.version, semantic_version.Version)
|
||||
if spec.requirements and pkg.metadata.version not in spec.requirements:
|
||||
continue
|
||||
if not best or (pkg.metadata.version > best.metadata.version):
|
||||
best = pkg
|
||||
return best
|
||||
@@ -0,0 +1,64 @@
|
||||
# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import json
|
||||
import os
|
||||
|
||||
from platformio.package.exception import MissingPackageManifestError
|
||||
from platformio.package.manager.base import BasePackageManager
|
||||
from platformio.package.meta import PackageSpec, PackageType
|
||||
from platformio.project.helpers import get_project_global_lib_dir
|
||||
|
||||
|
||||
class LibraryPackageManager(BasePackageManager):
|
||||
def __init__(self, package_dir=None):
|
||||
super(LibraryPackageManager, self).__init__(
|
||||
PackageType.LIBRARY, package_dir or get_project_global_lib_dir()
|
||||
)
|
||||
|
||||
@property
|
||||
def manifest_names(self):
|
||||
return PackageType.get_manifest_map()[PackageType.LIBRARY]
|
||||
|
||||
def find_pkg_root(self, path, spec):
|
||||
try:
|
||||
return super(LibraryPackageManager, self).find_pkg_root(path, spec)
|
||||
except MissingPackageManifestError:
|
||||
pass
|
||||
assert isinstance(spec, PackageSpec)
|
||||
|
||||
root_dir = self.find_library_root(path)
|
||||
|
||||
# automatically generate library manifest
|
||||
with open(os.path.join(root_dir, "library.json"), "w") as fp:
|
||||
json.dump(
|
||||
dict(name=spec.name, version=self.generate_rand_version(),),
|
||||
fp,
|
||||
indent=2,
|
||||
)
|
||||
|
||||
return root_dir
|
||||
|
||||
@staticmethod
|
||||
def find_library_root(path):
|
||||
for root, dirs, files in os.walk(path):
|
||||
if not files and len(dirs) == 1:
|
||||
continue
|
||||
for fname in files:
|
||||
if not fname.endswith((".c", ".cpp", ".h", ".S")):
|
||||
continue
|
||||
if os.path.isdir(os.path.join(os.path.dirname(root), "src")):
|
||||
return os.path.dirname(root)
|
||||
return root
|
||||
return path
|
||||
@@ -0,0 +1,30 @@
|
||||
# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from platformio.package.manager.base import BasePackageManager
|
||||
from platformio.package.meta import PackageType
|
||||
from platformio.project.config import ProjectConfig
|
||||
|
||||
|
||||
class PlatformPackageManager(BasePackageManager):
|
||||
def __init__(self, package_dir=None):
|
||||
self.config = ProjectConfig.get_instance()
|
||||
super(PlatformPackageManager, self).__init__(
|
||||
PackageType.PLATFORM,
|
||||
package_dir or self.config.get_optional_dir("platforms"),
|
||||
)
|
||||
|
||||
@property
|
||||
def manifest_names(self):
|
||||
return PackageType.get_manifest_map()[PackageType.PLATFORM]
|
||||
@@ -0,0 +1,29 @@
|
||||
# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from platformio.package.manager.base import BasePackageManager
|
||||
from platformio.package.meta import PackageType
|
||||
from platformio.project.config import ProjectConfig
|
||||
|
||||
|
||||
class ToolPackageManager(BasePackageManager):
|
||||
def __init__(self, package_dir=None):
|
||||
self.config = ProjectConfig.get_instance()
|
||||
super(ToolPackageManager, self).__init__(
|
||||
PackageType.TOOL, package_dir or self.config.get_optional_dir("packages"),
|
||||
)
|
||||
|
||||
@property
|
||||
def manifest_names(self):
|
||||
return PackageType.get_manifest_map()[PackageType.TOOL]
|
||||
@@ -0,0 +1,382 @@
|
||||
# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import tarfile
|
||||
|
||||
import semantic_version
|
||||
|
||||
from platformio.compat import get_object_members, string_types
|
||||
from platformio.package.manifest.parser import ManifestFileType
|
||||
|
||||
try:
|
||||
from urllib.parse import urlparse
|
||||
except ImportError:
|
||||
from urlparse import urlparse
|
||||
|
||||
|
||||
class PackageType(object):
|
||||
LIBRARY = "library"
|
||||
PLATFORM = "platform"
|
||||
TOOL = "tool"
|
||||
|
||||
@classmethod
|
||||
def items(cls):
|
||||
return get_object_members(cls)
|
||||
|
||||
@classmethod
|
||||
def get_manifest_map(cls):
|
||||
return {
|
||||
cls.PLATFORM: (ManifestFileType.PLATFORM_JSON,),
|
||||
cls.LIBRARY: (
|
||||
ManifestFileType.LIBRARY_JSON,
|
||||
ManifestFileType.LIBRARY_PROPERTIES,
|
||||
ManifestFileType.MODULE_JSON,
|
||||
),
|
||||
cls.TOOL: (ManifestFileType.PACKAGE_JSON,),
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def from_archive(cls, path):
|
||||
assert path.endswith("tar.gz")
|
||||
manifest_map = cls.get_manifest_map()
|
||||
with tarfile.open(path, mode="r:gz") as tf:
|
||||
for t in sorted(cls.items().values()):
|
||||
for manifest in manifest_map[t]:
|
||||
try:
|
||||
if tf.getmember(manifest):
|
||||
return t
|
||||
except KeyError:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
class PackageSpec(object):
|
||||
def __init__( # pylint: disable=redefined-builtin,too-many-arguments
|
||||
self, raw=None, owner=None, id=None, name=None, requirements=None, url=None
|
||||
):
|
||||
self.owner = owner
|
||||
self.id = id
|
||||
self.name = name
|
||||
self._requirements = None
|
||||
self.url = url
|
||||
if requirements:
|
||||
self.requirements = requirements
|
||||
self._name_is_custom = False
|
||||
self._parse(raw)
|
||||
|
||||
def __eq__(self, other):
|
||||
return all(
|
||||
[
|
||||
self.owner == other.owner,
|
||||
self.id == other.id,
|
||||
self.name == other.name,
|
||||
self.requirements == other.requirements,
|
||||
self.url == other.url,
|
||||
]
|
||||
)
|
||||
|
||||
def __repr__(self):
|
||||
return (
|
||||
"PackageSpec <owner={owner} id={id} name={name} "
|
||||
"requirements={requirements} url={url}>".format(**self.as_dict())
|
||||
)
|
||||
|
||||
@property
|
||||
def requirements(self):
|
||||
return self._requirements
|
||||
|
||||
@requirements.setter
|
||||
def requirements(self, value):
|
||||
if not value:
|
||||
self._requirements = None
|
||||
return
|
||||
self._requirements = (
|
||||
value
|
||||
if isinstance(value, semantic_version.SimpleSpec)
|
||||
else semantic_version.SimpleSpec(value)
|
||||
)
|
||||
|
||||
def humanize(self):
|
||||
if self.url:
|
||||
result = self.url
|
||||
elif self.id:
|
||||
result = "id:%d" % self.id
|
||||
else:
|
||||
result = ""
|
||||
if self.owner:
|
||||
result = self.owner + "/"
|
||||
result += self.name
|
||||
if self.requirements:
|
||||
result += " @ " + str(self.requirements)
|
||||
return result
|
||||
|
||||
def is_custom_name(self):
|
||||
return self._name_is_custom
|
||||
|
||||
def as_dict(self):
|
||||
return dict(
|
||||
owner=self.owner,
|
||||
id=self.id,
|
||||
name=self.name,
|
||||
requirements=str(self.requirements) if self.requirements else None,
|
||||
url=self.url,
|
||||
)
|
||||
|
||||
def _parse(self, raw):
|
||||
if raw is None:
|
||||
return
|
||||
if not isinstance(raw, string_types):
|
||||
raw = str(raw)
|
||||
raw = raw.strip()
|
||||
|
||||
parsers = (
|
||||
self._parse_requirements,
|
||||
self._parse_custom_name,
|
||||
self._parse_id,
|
||||
self._parse_owner,
|
||||
self._parse_url,
|
||||
)
|
||||
for parser in parsers:
|
||||
if raw is None:
|
||||
break
|
||||
raw = parser(raw)
|
||||
|
||||
# if name is not custom, parse it from URL
|
||||
if not self.name and self.url:
|
||||
self.name = self._parse_name_from_url(self.url)
|
||||
elif raw:
|
||||
# the leftover is a package name
|
||||
self.name = raw
|
||||
|
||||
def _parse_requirements(self, raw):
|
||||
if "@" not in raw:
|
||||
return raw
|
||||
tokens = raw.rsplit("@", 1)
|
||||
if any(s in tokens[1] for s in (":", "/")):
|
||||
return raw
|
||||
self.requirements = tokens[1].strip()
|
||||
return tokens[0].strip()
|
||||
|
||||
def _parse_custom_name(self, raw):
|
||||
if "=" not in raw or raw.startswith("id="):
|
||||
return raw
|
||||
tokens = raw.split("=", 1)
|
||||
if "/" in tokens[0]:
|
||||
return raw
|
||||
self.name = tokens[0].strip()
|
||||
self._name_is_custom = True
|
||||
return tokens[1].strip()
|
||||
|
||||
def _parse_id(self, raw):
|
||||
if raw.isdigit():
|
||||
self.id = int(raw)
|
||||
return None
|
||||
if raw.startswith("id="):
|
||||
return self._parse_id(raw[3:])
|
||||
return raw
|
||||
|
||||
def _parse_owner(self, raw):
|
||||
if raw.count("/") != 1 or "@" in raw:
|
||||
return raw
|
||||
tokens = raw.split("/", 1)
|
||||
self.owner = tokens[0].strip()
|
||||
self.name = tokens[1].strip()
|
||||
return None
|
||||
|
||||
def _parse_url(self, raw):
|
||||
if not any(s in raw for s in ("@", ":", "/")):
|
||||
return raw
|
||||
self.url = raw.strip()
|
||||
parts = urlparse(self.url)
|
||||
|
||||
# if local file or valid URL with scheme vcs+protocol://
|
||||
if parts.scheme == "file" or "+" in parts.scheme or self.url.startswith("git+"):
|
||||
return None
|
||||
|
||||
# parse VCS
|
||||
git_conditions = [
|
||||
parts.path.endswith(".git"),
|
||||
# Handle GitHub URL (https://github.com/user/package)
|
||||
parts.netloc in ("github.com", "gitlab.com", "bitbucket.com")
|
||||
and not parts.path.endswith((".zip", ".tar.gz")),
|
||||
]
|
||||
hg_conditions = [
|
||||
# Handle Developer Mbed URL
|
||||
# (https://developer.mbed.org/users/user/code/package/)
|
||||
# (https://os.mbed.com/users/user/code/package/)
|
||||
parts.netloc
|
||||
in ("mbed.com", "os.mbed.com", "developer.mbed.org")
|
||||
]
|
||||
if any(git_conditions):
|
||||
self.url = "git+" + self.url
|
||||
elif any(hg_conditions):
|
||||
self.url = "hg+" + self.url
|
||||
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _parse_name_from_url(url):
|
||||
if url.endswith("/"):
|
||||
url = url[:-1]
|
||||
for c in ("#", "?"):
|
||||
if c in url:
|
||||
url = url[: url.index(c)]
|
||||
|
||||
# parse real repository name from Github
|
||||
parts = urlparse(url)
|
||||
if parts.netloc == "github.com" and parts.path.count("/") > 2:
|
||||
return parts.path.split("/")[2]
|
||||
|
||||
name = os.path.basename(url)
|
||||
if "." in name:
|
||||
return name.split(".", 1)[0].strip()
|
||||
return name
|
||||
|
||||
|
||||
class PackageMetaData(object):
|
||||
def __init__( # pylint: disable=redefined-builtin
|
||||
self, type, name, version, spec=None
|
||||
):
|
||||
assert type in PackageType.items().values()
|
||||
if spec:
|
||||
assert isinstance(spec, PackageSpec)
|
||||
self.type = type
|
||||
self.name = name
|
||||
self._version = None
|
||||
self.version = version
|
||||
self.spec = spec
|
||||
|
||||
def __repr__(self):
|
||||
return (
|
||||
"PackageMetaData <type={type} name={name} version={version} "
|
||||
"spec={spec}".format(**self.as_dict())
|
||||
)
|
||||
|
||||
def __eq__(self, other):
|
||||
return all(
|
||||
[
|
||||
self.type == other.type,
|
||||
self.name == other.name,
|
||||
self.version == other.version,
|
||||
self.spec == other.spec,
|
||||
]
|
||||
)
|
||||
|
||||
@property
|
||||
def version(self):
|
||||
return self._version
|
||||
|
||||
@version.setter
|
||||
def version(self, value):
|
||||
if not value:
|
||||
self._version = None
|
||||
return
|
||||
self._version = (
|
||||
value
|
||||
if isinstance(value, semantic_version.Version)
|
||||
else self.to_semver(value)
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def to_semver(value, force=True, raise_exception=False):
|
||||
assert value
|
||||
try:
|
||||
return semantic_version.Version(value)
|
||||
except ValueError:
|
||||
pass
|
||||
if force:
|
||||
try:
|
||||
return semantic_version.Version.coerce(value)
|
||||
except ValueError:
|
||||
pass
|
||||
if raise_exception:
|
||||
raise ValueError("Invalid SemVer version %s" % value)
|
||||
# parse commit hash
|
||||
if re.match(r"^[\da-f]+$", value, flags=re.I):
|
||||
return semantic_version.Version("0.0.0+sha." + value)
|
||||
return semantic_version.Version("0.0.0+" + value)
|
||||
|
||||
def as_dict(self):
|
||||
return dict(
|
||||
type=self.type,
|
||||
name=self.name,
|
||||
version=str(self.version),
|
||||
spec=self.spec.as_dict() if self.spec else None,
|
||||
)
|
||||
|
||||
def dump(self, path):
|
||||
with open(path, "w") as fp:
|
||||
return json.dump(self.as_dict(), fp)
|
||||
|
||||
@staticmethod
|
||||
def load(path):
|
||||
with open(path) as fp:
|
||||
data = json.load(fp)
|
||||
if data["spec"]:
|
||||
data["spec"] = PackageSpec(**data["spec"])
|
||||
return PackageMetaData(**data)
|
||||
|
||||
|
||||
class PackageSourceItem(object):
|
||||
|
||||
METAFILE_NAME = ".piopm"
|
||||
|
||||
def __init__(self, path, metadata=None):
|
||||
self.path = path
|
||||
self.metadata = metadata
|
||||
if not self.metadata and self.exists():
|
||||
self.metadata = self.load_meta()
|
||||
|
||||
def __repr__(self):
|
||||
return "PackageSourceItem <path={path} metadata={metadata}".format(
|
||||
path=self.path, metadata=self.metadata
|
||||
)
|
||||
|
||||
def __eq__(self, other):
|
||||
return all([self.path == other.path, self.metadata == other.metadata])
|
||||
|
||||
def exists(self):
|
||||
return os.path.isdir(self.path)
|
||||
|
||||
def get_safe_dirname(self):
|
||||
assert self.metadata
|
||||
return re.sub(r"[^\da-z\_\-\. ]", "_", self.metadata.name, flags=re.I)
|
||||
|
||||
def get_metafile_locations(self):
|
||||
return [
|
||||
os.path.join(self.path, ".git"),
|
||||
os.path.join(self.path, ".hg"),
|
||||
os.path.join(self.path, ".svn"),
|
||||
self.path,
|
||||
]
|
||||
|
||||
def load_meta(self):
|
||||
assert self.exists()
|
||||
for location in self.get_metafile_locations():
|
||||
manifest_path = os.path.join(location, self.METAFILE_NAME)
|
||||
if os.path.isfile(manifest_path):
|
||||
return PackageMetaData.load(manifest_path)
|
||||
return None
|
||||
|
||||
def dump_meta(self):
|
||||
assert self.exists()
|
||||
location = None
|
||||
for location in self.get_metafile_locations():
|
||||
if os.path.isdir(location):
|
||||
break
|
||||
assert location
|
||||
return self.metadata.dump(os.path.join(location, self.METAFILE_NAME))
|
||||
@@ -23,7 +23,8 @@ from platformio import fs
|
||||
from platformio.package.exception import PackageException
|
||||
from platformio.package.manifest.parser import ManifestFileType, ManifestParserFactory
|
||||
from platformio.package.manifest.schema import ManifestSchema
|
||||
from platformio.unpacker import FileUnpacker
|
||||
from platformio.package.meta import PackageSourceItem
|
||||
from platformio.package.unpack import FileUnpacker
|
||||
|
||||
|
||||
class PackagePacker(object):
|
||||
@@ -36,6 +37,7 @@ class PackagePacker(object):
|
||||
".svn/",
|
||||
".pio/",
|
||||
"**/.pio/",
|
||||
PackageSourceItem.METAFILE_NAME,
|
||||
]
|
||||
INCLUDE_DEFAULT = ManifestFileType.items().values()
|
||||
|
||||
|
||||
@@ -1,169 +0,0 @@
|
||||
# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import os
|
||||
import tarfile
|
||||
|
||||
from platformio.compat import get_object_members, string_types
|
||||
from platformio.package.manifest.parser import ManifestFileType
|
||||
|
||||
|
||||
class PackageType(object):
|
||||
LIBRARY = "library"
|
||||
PLATFORM = "platform"
|
||||
TOOL = "tool"
|
||||
|
||||
@classmethod
|
||||
def items(cls):
|
||||
return get_object_members(cls)
|
||||
|
||||
@classmethod
|
||||
def get_manifest_map(cls):
|
||||
return {
|
||||
cls.PLATFORM: (ManifestFileType.PLATFORM_JSON,),
|
||||
cls.LIBRARY: (
|
||||
ManifestFileType.LIBRARY_JSON,
|
||||
ManifestFileType.LIBRARY_PROPERTIES,
|
||||
ManifestFileType.MODULE_JSON,
|
||||
),
|
||||
cls.TOOL: (ManifestFileType.PACKAGE_JSON,),
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def from_archive(cls, path):
|
||||
assert path.endswith("tar.gz")
|
||||
manifest_map = cls.get_manifest_map()
|
||||
with tarfile.open(path, mode="r:gz") as tf:
|
||||
for t in sorted(cls.items().values()):
|
||||
for manifest in manifest_map[t]:
|
||||
try:
|
||||
if tf.getmember(manifest):
|
||||
return t
|
||||
except KeyError:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
class PackageSpec(object):
|
||||
def __init__( # pylint: disable=redefined-builtin,too-many-arguments
|
||||
self, raw=None, ownername=None, id=None, name=None, requirements=None, url=None
|
||||
):
|
||||
self.ownername = ownername
|
||||
self.id = id
|
||||
self.name = name
|
||||
self.requirements = requirements
|
||||
self.url = url
|
||||
|
||||
self._parse(raw)
|
||||
|
||||
def __repr__(self):
|
||||
return (
|
||||
"PackageSpec <ownername={ownername} id={id} name={name} "
|
||||
"requirements={requirements} url={url}>".format(
|
||||
ownername=self.ownername,
|
||||
id=self.id,
|
||||
name=self.name,
|
||||
requirements=self.requirements,
|
||||
url=self.url,
|
||||
)
|
||||
)
|
||||
|
||||
def __eq__(self, other):
|
||||
return all(
|
||||
[
|
||||
self.ownername == other.ownername,
|
||||
self.id == other.id,
|
||||
self.name == other.name,
|
||||
self.requirements == other.requirements,
|
||||
self.url == other.url,
|
||||
]
|
||||
)
|
||||
|
||||
def _parse(self, raw):
|
||||
if raw is None:
|
||||
return
|
||||
if not isinstance(raw, string_types):
|
||||
raw = str(raw)
|
||||
raw = raw.strip()
|
||||
|
||||
parsers = (
|
||||
self._parse_requirements,
|
||||
self._parse_fixed_name,
|
||||
self._parse_id,
|
||||
self._parse_ownername,
|
||||
self._parse_url,
|
||||
)
|
||||
for parser in parsers:
|
||||
if raw is None:
|
||||
break
|
||||
raw = parser(raw)
|
||||
|
||||
# if name is not fixed, parse it from URL
|
||||
if not self.name and self.url:
|
||||
self.name = self._parse_name_from_url(self.url)
|
||||
elif raw:
|
||||
# the leftover is a package name
|
||||
self.name = raw
|
||||
|
||||
def _parse_requirements(self, raw):
|
||||
if "@" not in raw:
|
||||
return raw
|
||||
tokens = raw.rsplit("@", 1)
|
||||
if any(s in tokens[1] for s in (":", "/")):
|
||||
return raw
|
||||
self.requirements = tokens[1].strip()
|
||||
return tokens[0].strip()
|
||||
|
||||
def _parse_fixed_name(self, raw):
|
||||
if "=" not in raw or raw.startswith("id="):
|
||||
return raw
|
||||
tokens = raw.split("=", 1)
|
||||
if "/" in tokens[0]:
|
||||
return raw
|
||||
self.name = tokens[0].strip()
|
||||
return tokens[1].strip()
|
||||
|
||||
def _parse_id(self, raw):
|
||||
if raw.isdigit():
|
||||
self.id = int(raw)
|
||||
return None
|
||||
if raw.startswith("id="):
|
||||
return self._parse_id(raw[3:])
|
||||
return raw
|
||||
|
||||
def _parse_ownername(self, raw):
|
||||
if raw.count("/") != 1 or "@" in raw:
|
||||
return raw
|
||||
tokens = raw.split("/", 1)
|
||||
self.ownername = tokens[0].strip()
|
||||
self.name = tokens[1].strip()
|
||||
return None
|
||||
|
||||
def _parse_url(self, raw):
|
||||
if not any(s in raw for s in ("@", ":", "/")):
|
||||
return raw
|
||||
self.url = raw.strip()
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _parse_name_from_url(url):
|
||||
if url.endswith("/"):
|
||||
url = url[:-1]
|
||||
for c in ("#", "?"):
|
||||
if c in url:
|
||||
url = url[: url.index(c)]
|
||||
name = os.path.basename(url)
|
||||
if "." in name:
|
||||
return name.split(".", 1)[0].strip()
|
||||
return name
|
||||
@@ -0,0 +1,188 @@
|
||||
# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import os
|
||||
from tarfile import open as tarfile_open
|
||||
from time import mktime
|
||||
from zipfile import ZipFile
|
||||
|
||||
import click
|
||||
|
||||
from platformio import util
|
||||
from platformio.package.exception import PackageException
|
||||
|
||||
|
||||
class ExtractArchiveItemError(PackageException):
|
||||
|
||||
MESSAGE = (
|
||||
"Could not extract `{0}` to `{1}`. Try to disable antivirus "
|
||||
"tool or check this solution -> http://bit.ly/faq-package-manager"
|
||||
)
|
||||
|
||||
|
||||
class BaseArchiver(object):
|
||||
def __init__(self, arhfileobj):
|
||||
self._afo = arhfileobj
|
||||
|
||||
def get_items(self):
|
||||
raise NotImplementedError()
|
||||
|
||||
def get_item_filename(self, item):
|
||||
raise NotImplementedError()
|
||||
|
||||
def is_link(self, item):
|
||||
raise NotImplementedError()
|
||||
|
||||
def extract_item(self, item, dest_dir):
|
||||
self._afo.extract(item, dest_dir)
|
||||
self.after_extract(item, dest_dir)
|
||||
|
||||
def after_extract(self, item, dest_dir):
|
||||
pass
|
||||
|
||||
def close(self):
|
||||
self._afo.close()
|
||||
|
||||
|
||||
class TARArchiver(BaseArchiver):
|
||||
def __init__(self, archpath):
|
||||
super(TARArchiver, self).__init__(tarfile_open(archpath))
|
||||
|
||||
def get_items(self):
|
||||
return self._afo.getmembers()
|
||||
|
||||
def get_item_filename(self, item):
|
||||
return item.name
|
||||
|
||||
@staticmethod
|
||||
def is_link(item):
|
||||
return item.islnk() or item.issym()
|
||||
|
||||
@staticmethod
|
||||
def resolve_path(path):
|
||||
return os.path.realpath(os.path.abspath(path))
|
||||
|
||||
def is_bad_path(self, path, base):
|
||||
return not self.resolve_path(os.path.join(base, path)).startswith(base)
|
||||
|
||||
def is_bad_link(self, item, base):
|
||||
return not self.resolve_path(
|
||||
os.path.join(os.path.join(base, os.path.dirname(item.name)), item.linkname)
|
||||
).startswith(base)
|
||||
|
||||
def extract_item(self, item, dest_dir):
|
||||
dest_dir = self.resolve_path(dest_dir)
|
||||
bad_conds = [
|
||||
self.is_bad_path(item.name, dest_dir),
|
||||
self.is_link(item) and self.is_bad_link(item, dest_dir),
|
||||
]
|
||||
if not any(bad_conds):
|
||||
super(TARArchiver, self).extract_item(item, dest_dir)
|
||||
else:
|
||||
click.secho(
|
||||
"Blocked insecure item `%s` from TAR archive" % item.name,
|
||||
fg="red",
|
||||
err=True,
|
||||
)
|
||||
|
||||
|
||||
class ZIPArchiver(BaseArchiver):
|
||||
def __init__(self, archpath):
|
||||
super(ZIPArchiver, self).__init__(ZipFile(archpath))
|
||||
|
||||
@staticmethod
|
||||
def preserve_permissions(item, dest_dir):
|
||||
attrs = item.external_attr >> 16
|
||||
if attrs:
|
||||
os.chmod(os.path.join(dest_dir, item.filename), attrs)
|
||||
|
||||
@staticmethod
|
||||
def preserve_mtime(item, dest_dir):
|
||||
util.change_filemtime(
|
||||
os.path.join(dest_dir, item.filename),
|
||||
mktime(tuple(item.date_time) + tuple([0, 0, 0])),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def is_link(_):
|
||||
return False
|
||||
|
||||
def get_items(self):
|
||||
return self._afo.infolist()
|
||||
|
||||
def get_item_filename(self, item):
|
||||
return item.filename
|
||||
|
||||
def after_extract(self, item, dest_dir):
|
||||
self.preserve_permissions(item, dest_dir)
|
||||
self.preserve_mtime(item, dest_dir)
|
||||
|
||||
|
||||
class FileUnpacker(object):
|
||||
def __init__(self, path):
|
||||
self.path = path
|
||||
self._archiver = None
|
||||
|
||||
def _init_archiver(self):
|
||||
magic_map = {
|
||||
b"\x1f\x8b\x08": TARArchiver,
|
||||
b"\x42\x5a\x68": TARArchiver,
|
||||
b"\x50\x4b\x03\x04": ZIPArchiver,
|
||||
}
|
||||
magic_len = max(len(k) for k in magic_map)
|
||||
with open(self.path, "rb") as fp:
|
||||
data = fp.read(magic_len)
|
||||
for magic, archiver in magic_map.items():
|
||||
if data.startswith(magic):
|
||||
return archiver(self.path)
|
||||
raise PackageException("Unknown archive type '%s'" % self.path)
|
||||
|
||||
def __enter__(self):
|
||||
self._archiver = self._init_archiver()
|
||||
return self
|
||||
|
||||
def __exit__(self, *args):
|
||||
if self._archiver:
|
||||
self._archiver.close()
|
||||
|
||||
def unpack(
|
||||
self, dest_dir=None, with_progress=True, check_unpacked=True, silent=False
|
||||
):
|
||||
assert self._archiver
|
||||
if not dest_dir:
|
||||
dest_dir = os.getcwd()
|
||||
if not with_progress or silent:
|
||||
if not silent:
|
||||
click.echo("Unpacking...")
|
||||
for item in self._archiver.get_items():
|
||||
self._archiver.extract_item(item, dest_dir)
|
||||
else:
|
||||
items = self._archiver.get_items()
|
||||
with click.progressbar(items, label="Unpacking") as pb:
|
||||
for item in pb:
|
||||
self._archiver.extract_item(item, dest_dir)
|
||||
|
||||
if not check_unpacked:
|
||||
return True
|
||||
|
||||
# check on disk
|
||||
for item in self._archiver.get_items():
|
||||
filename = self._archiver.get_item_filename(item)
|
||||
item_path = os.path.join(dest_dir, filename)
|
||||
try:
|
||||
if not self._archiver.is_link(item) and not os.path.exists(item_path):
|
||||
raise ExtractArchiveItemError(filename, dest_dir)
|
||||
except NotImplementedError:
|
||||
pass
|
||||
return True
|
||||
@@ -0,0 +1,242 @@
|
||||
# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import re
|
||||
from os.path import join
|
||||
from subprocess import CalledProcessError, check_call
|
||||
from sys import modules
|
||||
|
||||
from platformio.exception import PlatformioException, UserSideException
|
||||
from platformio.proc import exec_command
|
||||
|
||||
try:
|
||||
from urllib.parse import urlparse
|
||||
except ImportError:
|
||||
from urlparse import urlparse
|
||||
|
||||
|
||||
class VCSClientFactory(object):
|
||||
@staticmethod
|
||||
def newClient(src_dir, remote_url, silent=False):
|
||||
result = urlparse(remote_url)
|
||||
type_ = result.scheme
|
||||
tag = None
|
||||
if not type_ and remote_url.startswith("git+"):
|
||||
type_ = "git"
|
||||
remote_url = remote_url[4:]
|
||||
elif "+" in result.scheme:
|
||||
type_, _ = result.scheme.split("+", 1)
|
||||
remote_url = remote_url[len(type_) + 1 :]
|
||||
if "#" in remote_url:
|
||||
remote_url, tag = remote_url.rsplit("#", 1)
|
||||
if not type_:
|
||||
raise PlatformioException("VCS: Unknown repository type %s" % remote_url)
|
||||
obj = getattr(modules[__name__], "%sClient" % type_.title())(
|
||||
src_dir, remote_url, tag, silent
|
||||
)
|
||||
assert isinstance(obj, VCSClientBase)
|
||||
return obj
|
||||
|
||||
|
||||
class VCSClientBase(object):
|
||||
|
||||
command = None
|
||||
|
||||
def __init__(self, src_dir, remote_url=None, tag=None, silent=False):
|
||||
self.src_dir = src_dir
|
||||
self.remote_url = remote_url
|
||||
self.tag = tag
|
||||
self.silent = silent
|
||||
self.check_client()
|
||||
|
||||
def check_client(self):
|
||||
try:
|
||||
assert self.command
|
||||
if self.silent:
|
||||
self.get_cmd_output(["--version"])
|
||||
else:
|
||||
assert self.run_cmd(["--version"])
|
||||
except (AssertionError, OSError, PlatformioException):
|
||||
raise UserSideException(
|
||||
"VCS: `%s` client is not installed in your system" % self.command
|
||||
)
|
||||
return True
|
||||
|
||||
@property
|
||||
def storage_dir(self):
|
||||
return join(self.src_dir, "." + self.command)
|
||||
|
||||
def export(self):
|
||||
raise NotImplementedError
|
||||
|
||||
def update(self):
|
||||
raise NotImplementedError
|
||||
|
||||
@property
|
||||
def can_be_updated(self):
|
||||
return not self.tag
|
||||
|
||||
def get_current_revision(self):
|
||||
raise NotImplementedError
|
||||
|
||||
def get_latest_revision(self):
|
||||
return None if self.can_be_updated else self.get_current_revision()
|
||||
|
||||
def run_cmd(self, args, **kwargs):
|
||||
args = [self.command] + args
|
||||
if "cwd" not in kwargs:
|
||||
kwargs["cwd"] = self.src_dir
|
||||
try:
|
||||
check_call(args, **kwargs)
|
||||
return True
|
||||
except CalledProcessError as e:
|
||||
raise PlatformioException("VCS: Could not process command %s" % e.cmd)
|
||||
|
||||
def get_cmd_output(self, args, **kwargs):
|
||||
args = [self.command] + args
|
||||
if "cwd" not in kwargs:
|
||||
kwargs["cwd"] = self.src_dir
|
||||
result = exec_command(args, **kwargs)
|
||||
if result["returncode"] == 0:
|
||||
return result["out"].strip()
|
||||
raise PlatformioException(
|
||||
"VCS: Could not receive an output from `%s` command (%s)" % (args, result)
|
||||
)
|
||||
|
||||
|
||||
class GitClient(VCSClientBase):
|
||||
|
||||
command = "git"
|
||||
|
||||
def check_client(self):
|
||||
try:
|
||||
return VCSClientBase.check_client(self)
|
||||
except UserSideException:
|
||||
raise UserSideException(
|
||||
"Please install Git client from https://git-scm.com/downloads"
|
||||
)
|
||||
|
||||
def get_branches(self):
|
||||
output = self.get_cmd_output(["branch"])
|
||||
output = output.replace("*", "") # fix active branch
|
||||
return [b.strip() for b in output.split("\n")]
|
||||
|
||||
def get_current_branch(self):
|
||||
output = self.get_cmd_output(["branch"])
|
||||
for line in output.split("\n"):
|
||||
line = line.strip()
|
||||
if line.startswith("*"):
|
||||
branch = line[1:].strip()
|
||||
if branch != "(no branch)":
|
||||
return branch
|
||||
return None
|
||||
|
||||
def get_tags(self):
|
||||
output = self.get_cmd_output(["tag", "-l"])
|
||||
return [t.strip() for t in output.split("\n")]
|
||||
|
||||
@staticmethod
|
||||
def is_commit_id(text):
|
||||
return text and re.match(r"[0-9a-f]{7,}$", text) is not None
|
||||
|
||||
@property
|
||||
def can_be_updated(self):
|
||||
return not self.tag or not self.is_commit_id(self.tag)
|
||||
|
||||
def export(self):
|
||||
is_commit = self.is_commit_id(self.tag)
|
||||
args = ["clone", "--recursive"]
|
||||
if not self.tag or not is_commit:
|
||||
args += ["--depth", "1"]
|
||||
if self.tag:
|
||||
args += ["--branch", self.tag]
|
||||
args += [self.remote_url, self.src_dir]
|
||||
assert self.run_cmd(args)
|
||||
if is_commit:
|
||||
assert self.run_cmd(["reset", "--hard", self.tag])
|
||||
return self.run_cmd(
|
||||
["submodule", "update", "--init", "--recursive", "--force"]
|
||||
)
|
||||
return True
|
||||
|
||||
def update(self):
|
||||
args = ["pull", "--recurse-submodules"]
|
||||
return self.run_cmd(args)
|
||||
|
||||
def get_current_revision(self):
|
||||
return self.get_cmd_output(["rev-parse", "--short", "HEAD"])
|
||||
|
||||
def get_latest_revision(self):
|
||||
if not self.can_be_updated:
|
||||
return self.get_current_revision()
|
||||
branch = self.get_current_branch()
|
||||
if not branch:
|
||||
return self.get_current_revision()
|
||||
result = self.get_cmd_output(["ls-remote"])
|
||||
for line in result.split("\n"):
|
||||
ref_pos = line.strip().find("refs/heads/" + branch)
|
||||
if ref_pos > 0:
|
||||
return line[:ref_pos].strip()[:7]
|
||||
return None
|
||||
|
||||
|
||||
class HgClient(VCSClientBase):
|
||||
|
||||
command = "hg"
|
||||
|
||||
def export(self):
|
||||
args = ["clone"]
|
||||
if self.tag:
|
||||
args.extend(["--updaterev", self.tag])
|
||||
args.extend([self.remote_url, self.src_dir])
|
||||
return self.run_cmd(args)
|
||||
|
||||
def update(self):
|
||||
args = ["pull", "--update"]
|
||||
return self.run_cmd(args)
|
||||
|
||||
def get_current_revision(self):
|
||||
return self.get_cmd_output(["identify", "--id"])
|
||||
|
||||
def get_latest_revision(self):
|
||||
if not self.can_be_updated:
|
||||
return self.get_latest_revision()
|
||||
return self.get_cmd_output(["identify", "--id", self.remote_url])
|
||||
|
||||
|
||||
class SvnClient(VCSClientBase):
|
||||
|
||||
command = "svn"
|
||||
|
||||
def export(self):
|
||||
args = ["checkout"]
|
||||
if self.tag:
|
||||
args.extend(["--revision", self.tag])
|
||||
args.extend([self.remote_url, self.src_dir])
|
||||
return self.run_cmd(args)
|
||||
|
||||
def update(self):
|
||||
|
||||
args = ["update"]
|
||||
return self.run_cmd(args)
|
||||
|
||||
def get_current_revision(self):
|
||||
output = self.get_cmd_output(
|
||||
["info", "--non-interactive", "--trust-server-cert", "-r", "HEAD"]
|
||||
)
|
||||
for line in output.split("\n"):
|
||||
line = line.strip()
|
||||
if line.startswith("Revision:"):
|
||||
return line.split(":", 1)[1].strip()
|
||||
raise PlatformioException("Could not detect current SVN revision")
|
||||
Reference in new issue
Block a user