From 1f7bda713691c1bbc51ccb5ac06a04127c7ccb4d Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 31 Jul 2023 19:13:05 +0300 Subject: [PATCH] Migrate from "requests" to the "httpx" --- platformio/__init__.py | 2 +- platformio/__main__.py | 9 - platformio/account/client.py | 6 +- platformio/builder/tools/piolib.py | 4 +- platformio/http.py | 212 +++++++++++++++--------- platformio/maintenance.py | 8 +- platformio/package/download.py | 123 +++++++------- platformio/package/manager/_registry.py | 3 +- platformio/package/manager/platform.py | 4 +- platformio/package/manifest/schema.py | 4 +- platformio/platform/factory.py | 9 +- platformio/registry/client.py | 13 +- platformio/registry/mirror.py | 17 +- platformio/telemetry.py | 8 +- setup.py | 12 +- 15 files changed, 241 insertions(+), 193 deletions(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 643dcc2e..bfea5739 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -62,10 +62,10 @@ __install_requires__ = [ "bottle == 0.12.*", "click >=8.0.4, <=8.2", "colorama", + "httpx >=0.22.0, <0.25", "marshmallow == 3.*", "pyelftools == 0.29", "pyserial == 3.5.*", # keep in sync "device/monitor/terminal.py" - "requests == 2.*", "semantic_version == 2.10.*", "tabulate == 0.*", ] + [ diff --git a/platformio/__main__.py b/platformio/__main__.py index 58cabe8b..896ac6e1 100644 --- a/platformio/__main__.py +++ b/platformio/__main__.py @@ -66,15 +66,6 @@ def configure(): if IS_CYGWIN: raise exception.CygwinEnvDetected() - # https://urllib3.readthedocs.org - # /en/latest/security.html#insecureplatformwarning - try: - import urllib3 # pylint: disable=import-outside-toplevel - - urllib3.disable_warnings() - except (AttributeError, ImportError): - pass - # Handle IOError issue with VSCode's Terminal (Windows) click_echo_origin = [click.echo, click.secho] diff --git a/platformio/account/client.py b/platformio/account/client.py index 06432144..7210ba3d 100644 --- a/platformio/account/client.py +++ b/platformio/account/client.py @@ -17,7 +17,7 @@ import time from platformio import __accounts_api__, app from platformio.exception import PlatformioException, UserSideException -from platformio.http import HTTPClient, HTTPClientError +from platformio.http import HttpApiClient, HttpClientApiError class AccountError(PlatformioException): @@ -32,7 +32,7 @@ class AccountAlreadyAuthorized(AccountError, UserSideException): MESSAGE = "You are already authorized with {0} account." -class AccountClient(HTTPClient): # pylint:disable=too-many-public-methods +class AccountClient(HttpApiClient): # pylint:disable=too-many-public-methods SUMMARY_CACHE_TTL = 60 * 60 * 24 * 7 def __init__(self): @@ -60,7 +60,7 @@ class AccountClient(HTTPClient): # pylint:disable=too-many-public-methods def fetch_json_data(self, *args, **kwargs): try: return super().fetch_json_data(*args, **kwargs) - except HTTPClientError as exc: + except HttpClientApiError as exc: raise AccountError(exc) from exc def fetch_authentication_token(self): diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index dee1d5bc..b7c8d7f3 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -29,7 +29,7 @@ from SCons.Script import DefaultEnvironment # pylint: disable=import-error from platformio import exception, fs from platformio.builder.tools import piobuild from platformio.compat import IS_WINDOWS, hashlib_encode_data, string_types -from platformio.http import HTTPClientError, InternetConnectionError +from platformio.http import HttpClientApiError, InternetConnectionError from platformio.package.exception import ( MissingPackageManifestError, UnknownPackageError, @@ -983,7 +983,7 @@ class ProjectAsLibBuilder(LibBuilderBase): lm.install(spec) did_install = True except ( - HTTPClientError, + HttpClientApiError, UnknownPackageError, InternetConnectionError, ) as exc: diff --git a/platformio/http.py b/platformio/http.py index c325e96b..e7de99e1 100644 --- a/platformio/http.py +++ b/platformio/http.py @@ -12,22 +12,25 @@ # See the License for the specific language governing permissions and # limitations under the License. +import contextlib +import itertools import json import os import socket -from urllib.parse import urljoin +import time -import requests.adapters -from urllib3.util.retry import Retry +import httpx from platformio import __check_internet_hosts__, app, util from platformio.cache import ContentCache, cleanup_content_cache from platformio.exception import PlatformioException, UserSideException -__default_requests_timeout__ = (10, None) # (connect, read) +RETRIES_BACKOFF_FACTOR = 2 # 0s, 2s, 4s, 8s, etc. +RETRIES_METHOD_WHITELIST = ["GET"] +RETRIES_STATUS_FORCELIST = [429, 500, 502, 503, 504] -class HTTPClientError(UserSideException): +class HttpClientApiError(UserSideException): def __init__(self, message, response=None): super().__init__() self.message = message @@ -40,84 +43,138 @@ class HTTPClientError(UserSideException): class InternetConnectionError(UserSideException): MESSAGE = ( "You are not connected to the Internet.\n" - "PlatformIO needs the Internet connection to" - " download dependent packages or to work with PlatformIO Account." + "PlatformIO needs the Internet connection to " + "download dependent packages or to work with PlatformIO Account." ) -class HTTPSession(requests.Session): - def __init__(self, *args, **kwargs): - self._x_base_url = kwargs.pop("x_base_url") if "x_base_url" in kwargs else None - super().__init__(*args, **kwargs) - self.headers.update({"User-Agent": app.get_user_agent()}) - try: - self.verify = app.get_setting("enable_proxy_strict_ssl") - except PlatformioException: - self.verify = True +def exponential_backoff(factor): + yield 0 + for n in itertools.count(2): + yield factor * (2 ** (n - 2)) - def request( # pylint: disable=signature-differs,arguments-differ - self, method, url, *args, **kwargs + +def apply_default_kwargs(kwargs=None): + kwargs = kwargs or {} + # enable redirects by default + kwargs["follow_redirects"] = kwargs.get("follow_redirects", True) + + try: + kwargs["verify"] = kwargs.get( + "verify", app.get_setting("enable_proxy_strict_ssl") + ) + except PlatformioException: + kwargs["verify"] = True + + headers = kwargs.pop("headers", {}) + if "User-Agent" not in headers: + headers.update({"User-Agent": app.get_user_agent()}) + kwargs["headers"] = headers + + retry = kwargs.pop("retry", None) + if retry: + kwargs["transport"] = HTTPRetryTransport(verify=kwargs["verify"], **retry) + + return kwargs + + +class HTTPRetryTransport(httpx.HTTPTransport): + def __init__( # pylint: disable=too-many-arguments + self, + verify=True, + retries=1, + backoff_factor=None, + status_forcelist=None, + method_whitelist=None, ): - # print("HTTPSession::request", self._x_base_url, method, url, args, kwargs) - if "timeout" not in kwargs: - kwargs["timeout"] = __default_requests_timeout__ - return super().request( - method, - url - if url.startswith("http") or not self._x_base_url - else urljoin(self._x_base_url, url), - *args, - **kwargs + super().__init__(verify=verify) + self._retries = retries + self._backoff_factor = backoff_factor or RETRIES_BACKOFF_FACTOR + self._status_forcelist = status_forcelist or RETRIES_STATUS_FORCELIST + self._method_whitelist = method_whitelist or RETRIES_METHOD_WHITELIST + + def handle_request(self, request): + retries_left = self._retries + delays = exponential_backoff(factor=RETRIES_BACKOFF_FACTOR) + while retries_left > 0: + retries_left -= 1 + try: + response = super().handle_request(request) + if response.status_code in RETRIES_STATUS_FORCELIST: + if request.method.upper() not in self._method_whitelist: + return response + raise httpx.HTTPStatusError( + f"Server error '{response.status_code} {response.reason_phrase}' " + f"for url '{request.url}'\n", + request=request, + response=response, + ) + return response + except httpx.HTTPError: + if retries_left == 0: + raise + time.sleep(next(delays) or 1) + + raise httpx.RequestError( + f"Could not process '{request.url}' request", request=request ) -class HTTPSessionIterator: - def __init__(self, endpoints): +class HTTPSession(httpx.Client): + def __init__(self, *args, **kwargs): + super().__init__(*args, **apply_default_kwargs(kwargs)) + + +class HttpEndpointPool: + def __init__(self, endpoints, session_retry=None): if not isinstance(endpoints, list): endpoints = [endpoints] self.endpoints = endpoints - self.endpoints_iter = iter(endpoints) - # https://urllib3.readthedocs.io/en/stable/reference/urllib3.util.html - self.retry = Retry( - total=5, - backoff_factor=1, # [0, 2, 4, 8, 16] secs - # method_whitelist=list(Retry.DEFAULT_METHOD_WHITELIST) + ["POST"], - status_forcelist=[413, 429, 500, 502, 503, 504], - ) + self.session_retry = session_retry - def __iter__(self): # pylint: disable=non-iterator-returned - return self - - def __next__(self): - base_url = next(self.endpoints_iter) - session = HTTPSession(x_base_url=base_url) - adapter = requests.adapters.HTTPAdapter(max_retries=self.retry) - session.mount(base_url, adapter) - return session - - -class HTTPClient: - def __init__(self, endpoints): - self._session_iter = HTTPSessionIterator(endpoints) - self._session = None - self._next_session() - - def __del__(self): - if not self._session: - return - try: - self._session.close() - except: # pylint: disable=bare-except - pass + self._endpoints_iter = iter(endpoints) self._session = None - def _next_session(self): + self.next() + + def close(self): if self._session: self._session.close() - self._session = next(self._session_iter) + + def next(self): + if self._session: + self._session.close() + self._session = HTTPSession( + base_url=next(self._endpoints_iter), retry=self.session_retry + ) + + def request(self, method, *args, **kwargs): + while True: + try: + return self._session.request(method, *args, **kwargs) + except httpx.HTTPError as exc: + try: + self.next() + except StopIteration as exc2: + raise exc from exc2 + + +class HttpApiClient(contextlib.AbstractContextManager): + def __init__(self, endpoints): + self._endpoint = HttpEndpointPool(endpoints, session_retry=dict(retries=5)) + + def __exit__(self, *excinfo): + self.close() + + def __del__(self): + self.close() + + def close(self): + if getattr(self, "_endpoint"): + self._endpoint.close() @util.throttle(500) - def send_request(self, method, path, **kwargs): + def send_request(self, method, *args, **kwargs): # check Internet before and resolve issue with 60 seconds timeout ensure_internet_on(raise_exception=True) @@ -131,19 +188,16 @@ class HTTPClient: # pylint: disable=import-outside-toplevel from platformio.account.client import AccountClient - headers["Authorization"] = ( - "Bearer %s" % AccountClient().fetch_authentication_token() - ) + with AccountClient() as client: + headers["Authorization"] = ( + "Bearer %s" % client.fetch_authentication_token() + ) kwargs["headers"] = headers - while True: - try: - return getattr(self._session, method)(path, **kwargs) - except requests.exceptions.RequestException as exc: - try: - self._next_session() - except Exception as exc2: - raise HTTPClientError(str(exc2)) from exc + try: + return self._endpoint.request(method, *args, **kwargs) + except httpx.HTTPError as exc: + raise HttpClientApiError(str(exc)) from exc def fetch_json_data(self, method, path, **kwargs): if method not in ("get", "head", "options"): @@ -177,7 +231,7 @@ class HTTPClient: message = response.json()["message"] except (KeyError, ValueError): message = response.text - raise HTTPClientError(message, response) + raise HttpClientApiError(message, response) # @@ -191,10 +245,10 @@ def _internet_on(): socket.setdefaulttimeout(timeout) for host in __check_internet_hosts__: try: - for var in ("HTTP_PROXY", "HTTPS_PROXY"): + for var in ("HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY"): if not os.getenv(var) and not os.getenv(var.lower()): continue - requests.get("http://%s" % host, allow_redirects=False, timeout=timeout) + httpx.get("http://%s" % host, follow_redirects=False, timeout=timeout) return True # try to resolve `host` for both AF_INET and AF_INET6, and then try to connect # to all possible addresses (IPv4 and IPv6) in turn until a connection succeeds: diff --git a/platformio/maintenance.py b/platformio/maintenance.py index 45b4e5c9..bd10337c 100644 --- a/platformio/maintenance.py +++ b/platformio/maintenance.py @@ -23,7 +23,11 @@ from platformio import __version__, app, exception, fs, telemetry from platformio.cache import cleanup_content_cache from platformio.cli import PlatformioCLI from platformio.commands.upgrade import get_latest_version -from platformio.http import HTTPClientError, InternetConnectionError, ensure_internet_on +from platformio.http import ( + HttpClientApiError, + InternetConnectionError, + ensure_internet_on, +) from platformio.package.manager.core import update_core_packages from platformio.package.version import pepver_to_semver from platformio.system.prune import calculate_unnecessary_system_data @@ -46,7 +50,7 @@ def on_cmd_end(): check_platformio_upgrade() check_prune_system() except ( - HTTPClientError, + HttpClientApiError, InternetConnectionError, exception.GetLatestVersionError, ): diff --git a/platformio/package/download.py b/platformio/package/download.py index 17cc1f30..bda5ef38 100644 --- a/platformio/package/download.py +++ b/platformio/package/download.py @@ -12,48 +12,28 @@ # See the License for the specific language governing permissions and # limitations under the License. -import io +import os +import tempfile +import time from email.utils import parsedate -from os.path import getsize, join -from time import mktime +from urllib.parse import urlparse import click +import httpx from platformio import fs from platformio.compat import is_terminal -from platformio.http import HTTPSession +from platformio.http import apply_default_kwargs from platformio.package.exception import PackageException class FileDownloader: - def __init__(self, url, dest_dir=None): - self._http_session = HTTPSession() - self._http_response = None - # make connection - self._http_response = self._http_session.get( - url, - stream=True, - ) - if self._http_response.status_code != 200: - raise PackageException( - "Got the unrecognized status code '{0}' when downloaded {1}".format( - self._http_response.status_code, url - ) - ) + def __init__(self, url, dst_dir=None): + self.url = url + self.dst_dir = dst_dir - disposition = self._http_response.headers.get("content-disposition") - if disposition and "filename=" in disposition: - self._fname = ( - disposition[disposition.index("filename=") + 9 :] - .replace('"', "") - .replace("'", "") - ) - else: - self._fname = [p for p in url.split("/") if p][-1] - self._fname = str(self._fname) - self._destination = self._fname - if dest_dir: - self.set_destination(join(dest_dir, self._fname)) + self._destination = None + self._http_response = None def set_destination(self, destination): self._destination = destination @@ -69,18 +49,34 @@ class FileDownloader: return -1 return int(self._http_response.headers["content-length"]) + def get_disposition_filname(self): + disposition = self._http_response.headers.get("content-disposition") + if disposition and "filename=" in disposition: + return ( + disposition[disposition.index("filename=") + 9 :] + .replace('"', "") + .replace("'", "") + ) + return [p for p in urlparse(self.url).path.split("/") if p][-1] + def start(self, with_progress=True, silent=False): label = "Downloading" - file_size = self.get_size() - itercontent = self._http_response.iter_content( - chunk_size=io.DEFAULT_BUFFER_SIZE - ) - try: + with httpx.stream("GET", self.url, **apply_default_kwargs()) as response: + if response.status_code != 200: + raise PackageException( + f"Got the unrecognized status code '{response.status_code}' " + "when downloading '{self.url}'" + ) + self._http_response = response + total_size = self.get_size() + if not self._destination: + assert self.dst_dir + with open(self._destination, "wb") as fp: - if file_size == -1 or not with_progress or silent: + if total_size == -1 or not with_progress or silent: if not silent: click.echo(f"{label}...") - for chunk in itercontent: + for chunk in response.iter_bytes(): fp.write(chunk) elif not is_terminal(): @@ -88,10 +84,10 @@ class FileDownloader: print_percent_step = 10 printed_percents = 0 downloaded_size = 0 - for chunk in itercontent: + for chunk in response.iter_bytes(): fp.write(chunk) downloaded_size += len(chunk) - if (downloaded_size / file_size * 100) >= ( + if (downloaded_size / total_size * 100) >= ( printed_percents + print_percent_step ): printed_percents += print_percent_step @@ -100,33 +96,39 @@ class FileDownloader: else: with click.progressbar( - length=file_size, - iterable=itercontent, + length=total_size, + iterable=response.iter_bytes(), label=label, update_min_steps=min( - 256 * 1024, file_size / 100 + 256 * 1024, total_size / 100 ), # every 256Kb or less ) as pb: for chunk in pb: pb.update(len(chunk)) fp.write(chunk) - finally: - self._http_response.close() - self._http_session.close() - if self.get_lmtime(): - self._preserve_filemtime(self.get_lmtime()) + last_modified = self.get_lmtime() + if last_modified: + self._preserve_filemtime(last_modified) return True + def _set_tmp_destination(self): + dst_dir = self.dst_dir or tempfile.mkdtemp() + self.set_destination(os.path.join(dst_dir, self.get_disposition_filname())) + + def _preserve_filemtime(self, lmdate): + lmtime = time.mktime(parsedate(lmdate)) + fs.change_filemtime(self._destination, lmtime) + def verify(self, checksum=None): - _dlsize = getsize(self._destination) - if self.get_size() != -1 and _dlsize != self.get_size(): + remote_size = self.get_size() + downloaded_size = os.path.getsize(self._destination) + if remote_size not in (-1, downloaded_size): raise PackageException( - ( - "The size ({0:d} bytes) of downloaded file '{1}' " - "is not equal to remote size ({2:d} bytes)" - ).format(_dlsize, self._fname, self.get_size()) + f"The size ({downloaded_size} bytes) of downloaded file " + f"'{self._destination}' is not equal to remote size " + f"({remote_size} bytes)" ) if not checksum: return True @@ -142,7 +144,7 @@ class FileDownloader: if not hash_algo: raise PackageException( - "Could not determine checksum algorithm by %s" % checksum + f"Could not determine checksum algorithm by {checksum}" ) dl_checksum = fs.calculate_file_hashsum(hash_algo, self._destination) @@ -150,16 +152,7 @@ class FileDownloader: raise PackageException( "The checksum '{0}' of the downloaded file '{1}' " "does not match to the remote '{2}'".format( - dl_checksum, self._fname, checksum + dl_checksum, self._destination, checksum ) ) return True - - def _preserve_filemtime(self, lmdate): - lmtime = mktime(parsedate(lmdate)) - fs.change_filemtime(self._destination, lmtime) - - def __del__(self): - self._http_session.close() - if self._http_response: - self._http_response.close() diff --git a/platformio/package/manager/_registry.py b/platformio/package/manager/_registry.py index 4c339be8..d888c96e 100644 --- a/platformio/package/manager/_registry.py +++ b/platformio/package/manager/_registry.py @@ -15,6 +15,7 @@ import time import click +import httpx from platformio.package.exception import UnknownPackageError from platformio.package.meta import PackageSpec @@ -57,7 +58,7 @@ class PackageManagerRegistryMixin: ), checksum or pkgfile["checksum"]["sha256"], ) - except Exception as exc: # pylint: disable=broad-except + except httpx.HTTPError as exc: self.log.warning( click.style("Warning! Package Mirror: %s" % exc, fg="yellow") ) diff --git a/platformio/package/manager/platform.py b/platformio/package/manager/platform.py index 6dbd480c..7640ac8c 100644 --- a/platformio/package/manager/platform.py +++ b/platformio/package/manager/platform.py @@ -15,7 +15,7 @@ import os from platformio import util -from platformio.http import HTTPClientError, InternetConnectionError +from platformio.http import HttpClientApiError, InternetConnectionError from platformio.package.exception import UnknownPackageError from platformio.package.manager.base import BasePackageManager from platformio.package.manager.core import get_installed_core_packages @@ -128,7 +128,7 @@ class PlatformPackageManager(BasePackageManager): # pylint: disable=too-many-an key = "%s:%s" % (board["platform"], board["id"]) if key not in know_boards: boards.append(board) - except (HTTPClientError, InternetConnectionError): + except (HttpClientApiError, InternetConnectionError): pass return sorted(boards, key=lambda b: b["name"]) diff --git a/platformio/package/manifest/schema.py b/platformio/package/manifest/schema.py index 7e0b494e..718a3b67 100644 --- a/platformio/package/manifest/schema.py +++ b/platformio/package/manifest/schema.py @@ -17,8 +17,8 @@ import json import re +import httpx import marshmallow -import requests import semantic_version from marshmallow import Schema, ValidationError, fields, validate, validates @@ -252,7 +252,7 @@ class ManifestSchema(BaseSchema): def validate_license(self, value): try: spdx = self.load_spdx_licenses() - except requests.exceptions.RequestException as exc: + except httpx.HTTPError as exc: raise ValidationError( "Could not load SPDX licenses for validation" ) from exc diff --git a/platformio/platform/factory.py b/platformio/platform/factory.py index 334888b5..6a0de30f 100644 --- a/platformio/platform/factory.py +++ b/platformio/platform/factory.py @@ -16,6 +16,8 @@ import os import re import sys +import httpx + from platformio import fs from platformio.compat import load_python_module from platformio.package.meta import PackageItem @@ -31,13 +33,16 @@ class PlatformFactory: name = re.sub(r"[^\da-z\_]+", "", name, flags=re.I) return "%sPlatform" % name.lower().capitalize() - @staticmethod - def load_platform_module(name, path): + @classmethod + def load_platform_module(cls, name, path): # backward compatibiility with the legacy dev-platforms sys.modules["platformio.managers.platform"] = base try: return load_python_module("platformio.platform.%s" % name, path) except ImportError as exc: + if exc.name == "requests" and not sys.modules.get("requests"): + sys.modules["requests"] = httpx + return cls.load_platform_module(name, path) raise UnknownPlatform(name) from exc @classmethod diff --git a/platformio/registry/client.py b/platformio/registry/client.py index eae58a90..af6c876b 100644 --- a/platformio/registry/client.py +++ b/platformio/registry/client.py @@ -16,13 +16,14 @@ from platformio import __registry_mirror_hosts__, fs from platformio.account.client import AccountClient, AccountError -from platformio.http import HTTPClient, HTTPClientError +from platformio.http import HttpApiClient, HttpClientApiError -class RegistryClient(HTTPClient): - def __init__(self): - endpoints = [f"https://api.{host}" for host in __registry_mirror_hosts__] - super().__init__(endpoints) +class RegistryClient(HttpApiClient): + def __init__(self, endpoints=None): + super().__init__( + endpoints or [f"https://api.{host}" for host in __registry_mirror_hosts__] + ) @staticmethod def allowed_private_packages(): @@ -157,7 +158,7 @@ class RegistryClient(HTTPClient): x_cache_valid="1h", x_with_authorization=self.allowed_private_packages(), ) - except HTTPClientError as exc: + except HttpClientApiError as exc: if exc.response is not None and exc.response.status_code == 404: return None raise exc diff --git a/platformio/registry/mirror.py b/platformio/registry/mirror.py index 4b4508f6..3e8e400f 100644 --- a/platformio/registry/mirror.py +++ b/platformio/registry/mirror.py @@ -17,7 +17,6 @@ from urllib.parse import urlparse from platformio import __registry_mirror_hosts__ from platformio.cache import ContentCache -from platformio.http import HTTPClient from platformio.registry.client import RegistryClient @@ -49,15 +48,15 @@ class RegistryFileMirrorIterator: except (ValueError, KeyError): pass - http = self.get_http_client() - response = http.send_request( + registry = self.get_api_client() + response = registry.send_request( "head", self._url_parts.path, - allow_redirects=False, + follow_redirects=False, params=dict(bypass=",".join(self._visited_mirrors)) if self._visited_mirrors else None, - x_with_authorization=RegistryClient.allowed_private_packages(), + x_with_authorization=registry.allowed_private_packages(), ) stop_conditions = [ response.status_code not in (302, 307), @@ -85,14 +84,14 @@ class RegistryFileMirrorIterator: response.headers.get("X-PIO-Content-SHA256"), ) - def get_http_client(self): + def get_api_client(self): if self._mirror not in RegistryFileMirrorIterator.HTTP_CLIENT_INSTANCES: endpoints = [self._mirror] for host in __registry_mirror_hosts__: endpoint = f"https://dl.{host}" if endpoint not in endpoints: endpoints.append(endpoint) - RegistryFileMirrorIterator.HTTP_CLIENT_INSTANCES[self._mirror] = HTTPClient( - endpoints - ) + RegistryFileMirrorIterator.HTTP_CLIENT_INSTANCES[ + self._mirror + ] = RegistryClient(endpoints) return RegistryFileMirrorIterator.HTTP_CLIENT_INSTANCES[self._mirror] diff --git a/platformio/telemetry.py b/platformio/telemetry.py index 97f17ff5..806eb49f 100644 --- a/platformio/telemetry.py +++ b/platformio/telemetry.py @@ -22,7 +22,7 @@ import time import traceback from collections import deque -import requests +import httpx from platformio import __title__, __version__, app, exception, fs, util from platformio.cli import PlatformioCLI @@ -134,13 +134,11 @@ class TelemetryLogger: # print("_commit_payload", payload) try: r = self._http_session.post( - "https://collector.platformio.org/collect", - json=payload, - timeout=(2, 5), # connect, read + "https://collector.platformio.org/collect", json=payload, timeout=2 ) r.raise_for_status() return True - except requests.exceptions.HTTPError as exc: + except httpx.HTTPStatusError as exc: # skip Bad Request if exc.response.status_code >= 400 and exc.response.status_code < 500: return True diff --git a/setup.py b/setup.py index 46716cc5..7b47eb4e 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import platform +import os from setuptools import find_packages, setup from platformio import ( @@ -26,10 +26,12 @@ from platformio import ( __install_requires__, ) -# issue #4702; Broken "requests/charset_normalizer" on macOS ARM -if platform.system() == "Darwin" and "arm" in platform.machine().lower(): - __install_requires__.append("chardet>=3.0.2,<4") - +# handle extra dependency for SOCKS proxy +if any( + os.getenv(key, "").startswith("socks5://") + for key in ("HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY") +): + __install_requires__.append("socksio") setup( name=__title__,