platformio/platformio-core
1
0
Fork 1
mirror of https://github.com/platformio/platformio-core.git synced 2025-07-30 01:57:13 +02:00
Code Issues Packages Projects Releases Wiki Activity

Migrate from "requests" to the "httpx"

Browse Source
This commit is contained in:
Ivan Kravets
2023-07-31 19:13:05 +03:00
parent 6b2d04b810
commit 1f7bda7136
15 changed files with 241 additions and 193 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
platformio/__init__.py
View File

@ -62,10 +62,10 @@ __install_requires__ = [
"bottle == 0.12.*",
"click >=8.0.4, <=8.2",
"colorama",
"httpx >=0.22.0, <0.25",
"marshmallow == 3.*",
"pyelftools == 0.29",
"pyserial == 3.5.*", # keep in sync "device/monitor/terminal.py"
"requests == 2.*",
"semantic_version == 2.10.*",
"tabulate == 0.*",
] + [

9
platformio/__main__.py
View File

@ -66,15 +66,6 @@ def configure():
if IS_CYGWIN:
raise exception.CygwinEnvDetected()
# https://urllib3.readthedocs.org
# /en/latest/security.html#insecureplatformwarning
try:
import urllib3 # pylint: disable=import-outside-toplevel
urllib3.disable_warnings()
except (AttributeError, ImportError):
pass
# Handle IOError issue with VSCode's Terminal (Windows)
click_echo_origin = [click.echo, click.secho]

6
platformio/account/client.py
View File

@ -17,7 +17,7 @@ import time
from platformio import __accounts_api__, app
from platformio.exception import PlatformioException, UserSideException
from platformio.http import HTTPClient, HTTPClientError
from platformio.http import HttpApiClient, HttpClientApiError
class AccountError(PlatformioException):
@ -32,7 +32,7 @@ class AccountAlreadyAuthorized(AccountError, UserSideException):
MESSAGE = "You are already authorized with {0} account."
class AccountClient(HTTPClient): # pylint:disable=too-many-public-methods
class AccountClient(HttpApiClient): # pylint:disable=too-many-public-methods
SUMMARY_CACHE_TTL = 60 * 60 * 24 * 7
def __init__(self):
@ -60,7 +60,7 @@ class AccountClient(HTTPClient): # pylint:disable=too-many-public-methods
def fetch_json_data(self, *args, **kwargs):
try:
return super().fetch_json_data(*args, **kwargs)
except HTTPClientError as exc:
except HttpClientApiError as exc:
raise AccountError(exc) from exc
def fetch_authentication_token(self):

4
platformio/builder/tools/piolib.py
View File

@ -29,7 +29,7 @@ from SCons.Script import DefaultEnvironment # pylint: disable=import-error
from platformio import exception, fs
from platformio.builder.tools import piobuild
from platformio.compat import IS_WINDOWS, hashlib_encode_data, string_types
from platformio.http import HTTPClientError, InternetConnectionError
from platformio.http import HttpClientApiError, InternetConnectionError
from platformio.package.exception import (
MissingPackageManifestError,
UnknownPackageError,
@ -983,7 +983,7 @@ class ProjectAsLibBuilder(LibBuilderBase):
lm.install(spec)
did_install = True
except (
HTTPClientError,
HttpClientApiError,
UnknownPackageError,
InternetConnectionError,
) as exc:

212
platformio/http.py
View File

@ -12,22 +12,25 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import contextlib
import itertools
import json
import os
import socket
from urllib.parse import urljoin
import time
import requests.adapters
from urllib3.util.retry import Retry
import httpx
from platformio import __check_internet_hosts__, app, util
from platformio.cache import ContentCache, cleanup_content_cache
from platformio.exception import PlatformioException, UserSideException
__default_requests_timeout__ = (10, None) # (connect, read)
RETRIES_BACKOFF_FACTOR = 2 # 0s, 2s, 4s, 8s, etc.
RETRIES_METHOD_WHITELIST = ["GET"]
RETRIES_STATUS_FORCELIST = [429, 500, 502, 503, 504]
class HTTPClientError(UserSideException):
class HttpClientApiError(UserSideException):
def __init__(self, message, response=None):
super().__init__()
self.message = message
@ -40,84 +43,138 @@ class HTTPClientError(UserSideException):
class InternetConnectionError(UserSideException):
MESSAGE = (
"You are not connected to the Internet.\n"
"PlatformIO needs the Internet connection to"
" download dependent packages or to work with PlatformIO Account."
"PlatformIO needs the Internet connection to "
"download dependent packages or to work with PlatformIO Account."
)
class HTTPSession(requests.Session):
def __init__(self, *args, **kwargs):
self._x_base_url = kwargs.pop("x_base_url") if "x_base_url" in kwargs else None
super().__init__(*args, **kwargs)
self.headers.update({"User-Agent": app.get_user_agent()})
try:
self.verify = app.get_setting("enable_proxy_strict_ssl")
except PlatformioException:
self.verify = True
def exponential_backoff(factor):
yield 0
for n in itertools.count(2):
yield factor * (2 ** (n - 2))
def request( # pylint: disable=signature-differs,arguments-differ
self, method, url, *args, **kwargs
def apply_default_kwargs(kwargs=None):
kwargs = kwargs or {}
# enable redirects by default
kwargs["follow_redirects"] = kwargs.get("follow_redirects", True)
try:
kwargs["verify"] = kwargs.get(
"verify", app.get_setting("enable_proxy_strict_ssl")
)
except PlatformioException:
kwargs["verify"] = True
headers = kwargs.pop("headers", {})
if "User-Agent" not in headers:
headers.update({"User-Agent": app.get_user_agent()})
kwargs["headers"] = headers
retry = kwargs.pop("retry", None)
if retry:
kwargs["transport"] = HTTPRetryTransport(verify=kwargs["verify"], **retry)
return kwargs
class HTTPRetryTransport(httpx.HTTPTransport):
def __init__( # pylint: disable=too-many-arguments
self,
verify=True,
retries=1,
backoff_factor=None,
status_forcelist=None,
method_whitelist=None,
):
# print("HTTPSession::request", self._x_base_url, method, url, args, kwargs)
if "timeout" not in kwargs:
kwargs["timeout"] = __default_requests_timeout__
return super().request(
method,
url
if url.startswith("http") or not self._x_base_url
else urljoin(self._x_base_url, url),
*args,
**kwargs
super().__init__(verify=verify)
self._retries = retries
self._backoff_factor = backoff_factor or RETRIES_BACKOFF_FACTOR
self._status_forcelist = status_forcelist or RETRIES_STATUS_FORCELIST
self._method_whitelist = method_whitelist or RETRIES_METHOD_WHITELIST
def handle_request(self, request):
retries_left = self._retries
delays = exponential_backoff(factor=RETRIES_BACKOFF_FACTOR)
while retries_left > 0:
retries_left -= 1
try:
response = super().handle_request(request)
if response.status_code in RETRIES_STATUS_FORCELIST:
if request.method.upper() not in self._method_whitelist:
return response
raise httpx.HTTPStatusError(
f"Server error '{response.status_code} {response.reason_phrase}' "
f"for url '{request.url}'\n",
request=request,
response=response,
)
return response
except httpx.HTTPError:
if retries_left == 0:
raise
time.sleep(next(delays) or 1)
raise httpx.RequestError(
f"Could not process '{request.url}' request", request=request
)
class HTTPSessionIterator:
def __init__(self, endpoints):
class HTTPSession(httpx.Client):
def __init__(self, *args, **kwargs):
super().__init__(*args, **apply_default_kwargs(kwargs))
class HttpEndpointPool:
def __init__(self, endpoints, session_retry=None):
if not isinstance(endpoints, list):
endpoints = [endpoints]
self.endpoints = endpoints
self.endpoints_iter = iter(endpoints)
# https://urllib3.readthedocs.io/en/stable/reference/urllib3.util.html
self.retry = Retry(
total=5,
backoff_factor=1, # [0, 2, 4, 8, 16] secs
# method_whitelist=list(Retry.DEFAULT_METHOD_WHITELIST) + ["POST"],
status_forcelist=[413, 429, 500, 502, 503, 504],
)
self.session_retry = session_retry
def __iter__(self): # pylint: disable=non-iterator-returned
return self
def __next__(self):
base_url = next(self.endpoints_iter)
session = HTTPSession(x_base_url=base_url)
adapter = requests.adapters.HTTPAdapter(max_retries=self.retry)
session.mount(base_url, adapter)
return session
class HTTPClient:
def __init__(self, endpoints):
self._session_iter = HTTPSessionIterator(endpoints)
self._session = None
self._next_session()
def __del__(self):
if not self._session:
return
try:
self._session.close()
except: # pylint: disable=bare-except
pass
self._endpoints_iter = iter(endpoints)
self._session = None
def _next_session(self):
self.next()
def close(self):
if self._session:
self._session.close()
self._session = next(self._session_iter)
def next(self):
if self._session:
self._session.close()
self._session = HTTPSession(
base_url=next(self._endpoints_iter), retry=self.session_retry
)
def request(self, method, *args, **kwargs):
while True:
try:
return self._session.request(method, *args, **kwargs)
except httpx.HTTPError as exc:
try:
self.next()
except StopIteration as exc2:
raise exc from exc2
class HttpApiClient(contextlib.AbstractContextManager):
def __init__(self, endpoints):
self._endpoint = HttpEndpointPool(endpoints, session_retry=dict(retries=5))
def __exit__(self, *excinfo):
self.close()
def __del__(self):
self.close()
def close(self):
if getattr(self, "_endpoint"):
self._endpoint.close()
@util.throttle(500)
def send_request(self, method, path, **kwargs):
def send_request(self, method, *args, **kwargs):
# check Internet before and resolve issue with 60 seconds timeout
ensure_internet_on(raise_exception=True)
@ -131,19 +188,16 @@ class HTTPClient:
# pylint: disable=import-outside-toplevel
from platformio.account.client import AccountClient
headers["Authorization"] = (
"Bearer %s" % AccountClient().fetch_authentication_token()
)
with AccountClient() as client:
headers["Authorization"] = (
"Bearer %s" % client.fetch_authentication_token()
)
kwargs["headers"] = headers
while True:
try:
return getattr(self._session, method)(path, **kwargs)
except requests.exceptions.RequestException as exc:
try:
self._next_session()
except Exception as exc2:
raise HTTPClientError(str(exc2)) from exc
try:
return self._endpoint.request(method, *args, **kwargs)
except httpx.HTTPError as exc:
raise HttpClientApiError(str(exc)) from exc
def fetch_json_data(self, method, path, **kwargs):
if method not in ("get", "head", "options"):
@ -177,7 +231,7 @@ class HTTPClient:
message = response.json()["message"]
except (KeyError, ValueError):
message = response.text
raise HTTPClientError(message, response)
raise HttpClientApiError(message, response)
#
@ -191,10 +245,10 @@ def _internet_on():
socket.setdefaulttimeout(timeout)
for host in __check_internet_hosts__:
try:
for var in ("HTTP_PROXY", "HTTPS_PROXY"):
for var in ("HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY"):
if not os.getenv(var) and not os.getenv(var.lower()):
continue
requests.get("http://%s" % host, allow_redirects=False, timeout=timeout)
httpx.get("http://%s" % host, follow_redirects=False, timeout=timeout)
return True
# try to resolve `host` for both AF_INET and AF_INET6, and then try to connect
# to all possible addresses (IPv4 and IPv6) in turn until a connection succeeds:

8
platformio/maintenance.py
View File

@ -23,7 +23,11 @@ from platformio import __version__, app, exception, fs, telemetry
from platformio.cache import cleanup_content_cache
from platformio.cli import PlatformioCLI
from platformio.commands.upgrade import get_latest_version
from platformio.http import HTTPClientError, InternetConnectionError, ensure_internet_on
from platformio.http import (
HttpClientApiError,
InternetConnectionError,
ensure_internet_on,
)
from platformio.package.manager.core import update_core_packages
from platformio.package.version import pepver_to_semver
from platformio.system.prune import calculate_unnecessary_system_data
@ -46,7 +50,7 @@ def on_cmd_end():
check_platformio_upgrade()
check_prune_system()
except (
HTTPClientError,
HttpClientApiError,
InternetConnectionError,
exception.GetLatestVersionError,
):

123
platformio/package/download.py
View File

@ -12,48 +12,28 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import io
import os
import tempfile
import time
from email.utils import parsedate
from os.path import getsize, join
from time import mktime
from urllib.parse import urlparse
import click
import httpx
from platformio import fs
from platformio.compat import is_terminal
from platformio.http import HTTPSession
from platformio.http import apply_default_kwargs
from platformio.package.exception import PackageException
class FileDownloader:
def __init__(self, url, dest_dir=None):
self._http_session = HTTPSession()
self._http_response = None
# make connection
self._http_response = self._http_session.get(
url,
stream=True,
)
if self._http_response.status_code != 200:
raise PackageException(
"Got the unrecognized status code '{0}' when downloaded {1}".format(
self._http_response.status_code, url
)
)
def __init__(self, url, dst_dir=None):
self.url = url
self.dst_dir = dst_dir
disposition = self._http_response.headers.get("content-disposition")
if disposition and "filename=" in disposition:
self._fname = (
disposition[disposition.index("filename=") + 9 :]
.replace('"', "")
.replace("'", "")
)
else:
self._fname = [p for p in url.split("/") if p][-1]
self._fname = str(self._fname)
self._destination = self._fname
if dest_dir:
self.set_destination(join(dest_dir, self._fname))
self._destination = None
self._http_response = None
def set_destination(self, destination):
self._destination = destination
@ -69,18 +49,34 @@ class FileDownloader:
return -1
return int(self._http_response.headers["content-length"])
def get_disposition_filname(self):
disposition = self._http_response.headers.get("content-disposition")
if disposition and "filename=" in disposition:
return (
disposition[disposition.index("filename=") + 9 :]
.replace('"', "")
.replace("'", "")
)
return [p for p in urlparse(self.url).path.split("/") if p][-1]
def start(self, with_progress=True, silent=False):
label = "Downloading"
file_size = self.get_size()
itercontent = self._http_response.iter_content(
chunk_size=io.DEFAULT_BUFFER_SIZE
)
try:
with httpx.stream("GET", self.url, **apply_default_kwargs()) as response:
if response.status_code != 200:
raise PackageException(
f"Got the unrecognized status code '{response.status_code}' "
"when downloading '{self.url}'"
)
self._http_response = response
total_size = self.get_size()
if not self._destination:
assert self.dst_dir
with open(self._destination, "wb") as fp:
if file_size == -1 or not with_progress or silent:
if total_size == -1 or not with_progress or silent:
if not silent:
click.echo(f"{label}...")
for chunk in itercontent:
for chunk in response.iter_bytes():
fp.write(chunk)
elif not is_terminal():
@ -88,10 +84,10 @@ class FileDownloader:
print_percent_step = 10
printed_percents = 0
downloaded_size = 0
for chunk in itercontent:
for chunk in response.iter_bytes():
fp.write(chunk)
downloaded_size += len(chunk)
if (downloaded_size / file_size * 100) >= (
if (downloaded_size / total_size * 100) >= (
printed_percents + print_percent_step
):
printed_percents += print_percent_step
@ -100,33 +96,39 @@ class FileDownloader:
else:
with click.progressbar(
length=file_size,
iterable=itercontent,
length=total_size,
iterable=response.iter_bytes(),
label=label,
update_min_steps=min(
256 * 1024, file_size / 100
256 * 1024, total_size / 100
), # every 256Kb or less
) as pb:
for chunk in pb:
pb.update(len(chunk))
fp.write(chunk)
finally:
self._http_response.close()
self._http_session.close()
if self.get_lmtime():
self._preserve_filemtime(self.get_lmtime())
last_modified = self.get_lmtime()
if last_modified:
self._preserve_filemtime(last_modified)
return True
def _set_tmp_destination(self):
dst_dir = self.dst_dir or tempfile.mkdtemp()
self.set_destination(os.path.join(dst_dir, self.get_disposition_filname()))
def _preserve_filemtime(self, lmdate):
lmtime = time.mktime(parsedate(lmdate))
fs.change_filemtime(self._destination, lmtime)
def verify(self, checksum=None):
_dlsize = getsize(self._destination)
if self.get_size() != -1 and _dlsize != self.get_size():
remote_size = self.get_size()
downloaded_size = os.path.getsize(self._destination)
if remote_size not in (-1, downloaded_size):
raise PackageException(
(
"The size ({0:d} bytes) of downloaded file '{1}' "
"is not equal to remote size ({2:d} bytes)"
).format(_dlsize, self._fname, self.get_size())
f"The size ({downloaded_size} bytes) of downloaded file "
f"'{self._destination}' is not equal to remote size "
f"({remote_size} bytes)"
)
if not checksum:
return True
@ -142,7 +144,7 @@ class FileDownloader:
if not hash_algo:
raise PackageException(
"Could not determine checksum algorithm by %s" % checksum
f"Could not determine checksum algorithm by {checksum}"
)
dl_checksum = fs.calculate_file_hashsum(hash_algo, self._destination)
@ -150,16 +152,7 @@ class FileDownloader:
raise PackageException(
"The checksum '{0}' of the downloaded file '{1}' "
"does not match to the remote '{2}'".format(
dl_checksum, self._fname, checksum
dl_checksum, self._destination, checksum
)
)
return True
def _preserve_filemtime(self, lmdate):
lmtime = mktime(parsedate(lmdate))
fs.change_filemtime(self._destination, lmtime)
def __del__(self):
self._http_session.close()
if self._http_response:
self._http_response.close()

3
platformio/package/manager/_registry.py
View File

@ -15,6 +15,7 @@
import time
import click
import httpx
from platformio.package.exception import UnknownPackageError
from platformio.package.meta import PackageSpec
@ -57,7 +58,7 @@ class PackageManagerRegistryMixin:
),
checksum or pkgfile["checksum"]["sha256"],
)
except Exception as exc: # pylint: disable=broad-except
except httpx.HTTPError as exc:
self.log.warning(
click.style("Warning! Package Mirror: %s" % exc, fg="yellow")
)

4
platformio/package/manager/platform.py
View File

@ -15,7 +15,7 @@
import os
from platformio import util
from platformio.http import HTTPClientError, InternetConnectionError
from platformio.http import HttpClientApiError, InternetConnectionError
from platformio.package.exception import UnknownPackageError
from platformio.package.manager.base import BasePackageManager
from platformio.package.manager.core import get_installed_core_packages
@ -128,7 +128,7 @@ class PlatformPackageManager(BasePackageManager): # pylint: disable=too-many-an
key = "%s:%s" % (board["platform"], board["id"])
if key not in know_boards:
boards.append(board)
except (HTTPClientError, InternetConnectionError):
except (HttpClientApiError, InternetConnectionError):
pass
return sorted(boards, key=lambda b: b["name"])

4
platformio/package/manifest/schema.py
View File

@ -17,8 +17,8 @@
import json
import re
import httpx
import marshmallow
import requests
import semantic_version
from marshmallow import Schema, ValidationError, fields, validate, validates
@ -252,7 +252,7 @@ class ManifestSchema(BaseSchema):
def validate_license(self, value):
try:
spdx = self.load_spdx_licenses()
except requests.exceptions.RequestException as exc:
except httpx.HTTPError as exc:
raise ValidationError(
"Could not load SPDX licenses for validation"
) from exc

9
platformio/platform/factory.py
View File

@ -16,6 +16,8 @@ import os
import re
import sys
import httpx
from platformio import fs
from platformio.compat import load_python_module
from platformio.package.meta import PackageItem
@ -31,13 +33,16 @@ class PlatformFactory:
name = re.sub(r"[^\da-z\_]+", "", name, flags=re.I)
return "%sPlatform" % name.lower().capitalize()
@staticmethod
def load_platform_module(name, path):
@classmethod
def load_platform_module(cls, name, path):
# backward compatibiility with the legacy dev-platforms
sys.modules["platformio.managers.platform"] = base
try:
return load_python_module("platformio.platform.%s" % name, path)
except ImportError as exc:
if exc.name == "requests" and not sys.modules.get("requests"):
sys.modules["requests"] = httpx
return cls.load_platform_module(name, path)
raise UnknownPlatform(name) from exc
@classmethod

13
platformio/registry/client.py
View File

@ -16,13 +16,14 @@
from platformio import __registry_mirror_hosts__, fs
from platformio.account.client import AccountClient, AccountError
from platformio.http import HTTPClient, HTTPClientError
from platformio.http import HttpApiClient, HttpClientApiError
class RegistryClient(HTTPClient):
def __init__(self):
endpoints = [f"https://api.{host}" for host in __registry_mirror_hosts__]
super().__init__(endpoints)
class RegistryClient(HttpApiClient):
def __init__(self, endpoints=None):
super().__init__(
endpoints or [f"https://api.{host}" for host in __registry_mirror_hosts__]
)
@staticmethod
def allowed_private_packages():
@ -157,7 +158,7 @@ class RegistryClient(HTTPClient):
x_cache_valid="1h",
x_with_authorization=self.allowed_private_packages(),
)
except HTTPClientError as exc:
except HttpClientApiError as exc:
if exc.response is not None and exc.response.status_code == 404:
return None
raise exc

17
platformio/registry/mirror.py
View File

@ -17,7 +17,6 @@ from urllib.parse import urlparse
from platformio import __registry_mirror_hosts__
from platformio.cache import ContentCache
from platformio.http import HTTPClient
from platformio.registry.client import RegistryClient
@ -49,15 +48,15 @@ class RegistryFileMirrorIterator:
except (ValueError, KeyError):
pass
http = self.get_http_client()
response = http.send_request(
registry = self.get_api_client()
response = registry.send_request(
"head",
self._url_parts.path,
allow_redirects=False,
follow_redirects=False,
params=dict(bypass=",".join(self._visited_mirrors))
if self._visited_mirrors
else None,
x_with_authorization=RegistryClient.allowed_private_packages(),
x_with_authorization=registry.allowed_private_packages(),
)
stop_conditions = [
response.status_code not in (302, 307),
@ -85,14 +84,14 @@ class RegistryFileMirrorIterator:
response.headers.get("X-PIO-Content-SHA256"),
)
def get_http_client(self):
def get_api_client(self):
if self._mirror not in RegistryFileMirrorIterator.HTTP_CLIENT_INSTANCES:
endpoints = [self._mirror]
for host in __registry_mirror_hosts__:
endpoint = f"https://dl.{host}"
if endpoint not in endpoints:
endpoints.append(endpoint)
RegistryFileMirrorIterator.HTTP_CLIENT_INSTANCES[self._mirror] = HTTPClient(
endpoints
)
RegistryFileMirrorIterator.HTTP_CLIENT_INSTANCES[
self._mirror
] = RegistryClient(endpoints)
return RegistryFileMirrorIterator.HTTP_CLIENT_INSTANCES[self._mirror]

8
platformio/telemetry.py
View File

@ -22,7 +22,7 @@ import time
import traceback
from collections import deque
import requests
import httpx
from platformio import __title__, __version__, app, exception, fs, util
from platformio.cli import PlatformioCLI
@ -134,13 +134,11 @@ class TelemetryLogger:
# print("_commit_payload", payload)
try:
r = self._http_session.post(
"https://collector.platformio.org/collect",
json=payload,
timeout=(2, 5), # connect, read
"https://collector.platformio.org/collect", json=payload, timeout=2
)
r.raise_for_status()
return True
except requests.exceptions.HTTPError as exc:
except httpx.HTTPStatusError as exc:
# skip Bad Request
if exc.response.status_code >= 400 and exc.response.status_code < 500:
return True

12
setup.py
View File

@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import platform
import os
from setuptools import find_packages, setup
from platformio import (
@ -26,10 +26,12 @@ from platformio import (
__install_requires__,
)
# issue #4702; Broken "requests/charset_normalizer" on macOS ARM
if platform.system() == "Darwin" and "arm" in platform.machine().lower():
__install_requires__.append("chardet>=3.0.2,<4")
# handle extra dependency for SOCKS proxy
if any(
os.getenv(key, "").startswith("socks5://")
for key in ("HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY")
):
__install_requires__.append("socksio")
setup(
name=__title__,