platformio/platformio-core
1
0
Fork 1
mirror of https://github.com/platformio/platformio-core.git synced 2025-07-30 10:07:14 +02:00
Code Issues Packages Projects Releases Wiki Activity

Disable buffering file calculating checksum for downloaded file

Browse Source
This commit is contained in:
Ivan Kravets
2020-01-24 21:40:12 +02:00
parent 04ca6621f1
commit 253e4b13b5
1 changed files with 8 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
platformio/downloader.py
View File

@ -13,10 +13,11 @@
# limitations under the License. # limitations under the License.
import hashlib import hashlib
import io
import math
import sys
from email.utils import parsedate_tz from email.utils import parsedate_tz
from math import ceil
from os.path import getsize, join from os.path import getsize, join
from sys import version_info
from time import mktime from time import mktime
import click import click
@ -32,7 +33,7 @@ from platformio.exception import (
class FileDownloader(object): class FileDownloader(object):
CHUNK_SIZE = 1024 CHUNK_SIZE = 1024 * 8
def __init__(self, url, dest_dir=None): def __init__(self, url, dest_dir=None):
self._request = None self._request = None
@ -41,7 +42,7 @@ class FileDownloader(object):
url, url,
stream=True, stream=True,
headers=util.get_request_defheaders(), headers=util.get_request_defheaders(),
verify=version_info >= (2, 7, 9), verify=sys.version_info >= (2, 7, 9),
) )
if self._request.status_code != 200: if self._request.status_code != 200:
raise FDUnrecognizedStatusCode(self._request.status_code, url) raise FDUnrecognizedStatusCode(self._request.status_code, url)
@ -86,7 +87,7 @@ class FileDownloader(object):
if chunk: if chunk:
f.write(chunk) f.write(chunk)
else: else:
chunks = int(ceil(self.get_size() / float(self.CHUNK_SIZE))) chunks = int(math.ceil(self.get_size() / float(self.CHUNK_SIZE)))
with click.progressbar(length=chunks, label=label) as pb: with click.progressbar(length=chunks, label=label) as pb:
for _ in pb: for _ in pb:
f.write(next(itercontent)) f.write(next(itercontent))
@ -107,9 +108,9 @@ class FileDownloader(object):
return None return None
checksum = hashlib.sha1() checksum = hashlib.sha1()
with open(self._destination, "rb") as fp: with io.open(self._destination, "rb", buffering=0) as fp:
while True: while True:
chunk = fp.read(1024 * 64) chunk = fp.read(self.CHUNK_SIZE)
if not chunk: if not chunk:
break break
checksum.update(chunk) checksum.update(chunk)