diff --git a/HISTORY.rst b/HISTORY.rst index adf17a64..44509c13 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -16,6 +16,8 @@ PlatformIO Core 6 6.1.5 (2022-??-??) ~~~~~~~~~~~~~~~~~~ +* Added a new ``enable_proxy_strict_ssl`` setting to disable the proxy server certificate verification (`issue #4432 `_) +* Documented `PlatformIO Core Proxy Configuration `__ * Speeded up device port finder by avoiding loading board HWIDs from development platforms * Improved caching of build metadata in debug mode * Fixed an issue when `pio pkg install --storage-dir `__ command requires PlatformIO project (`issue #4410 `_) diff --git a/docs b/docs index 8637bd52..52e86681 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 8637bd528b09ceba1eac85e8409931ef1abc7829 +Subproject commit 52e866813ce4323a8455f482a0501ad391bac40f diff --git a/platformio/__init__.py b/platformio/__init__.py index b0a9c205..0bb271a6 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -44,8 +44,6 @@ __registry_mirror_hosts__ = [ ] __pioremote_endpoint__ = "ssl:host=remote.platformio.org:port=4413" -__default_requests_timeout__ = (10, None) # (connect, read) - __core_packages__ = { "contrib-piohome": "~3.4.2", "contrib-pysite": "~2.%d%d.0" % (sys.version_info.major, sys.version_info.minor), diff --git a/platformio/app.py b/platformio/app.py index bf7af870..48e90b97 100644 --- a/platformio/app.py +++ b/platformio/app.py @@ -58,6 +58,10 @@ DEFAULT_SETTINGS = { "value": get_default_projects_dir(), "validator": projects_dir_validate, }, + "enable_proxy_strict_ssl": { + "description": "Verify the proxy server certificate against the list of supplied CAs", + "value": True, + }, } SESSION_VARS = { diff --git a/platformio/home/helpers.py b/platformio/home/helpers.py index 494e2709..a777101b 100644 --- a/platformio/home/helpers.py +++ b/platformio/home/helpers.py @@ -14,15 +14,15 @@ import socket -import requests from starlette.concurrency import run_in_threadpool from platformio import util from platformio.compat import IS_WINDOWS +from platformio.http import HTTPSession from platformio.proc import where_is_program -class AsyncSession(requests.Session): +class AsyncSession(HTTPSession): async def request( # pylint: disable=signature-differs,invalid-overridden-method self, *args, **kwargs ): diff --git a/platformio/home/rpc/handlers/os.py b/platformio/home/rpc/handlers/os.py index 9fe0198f..aed62991 100644 --- a/platformio/home/rpc/handlers/os.py +++ b/platformio/home/rpc/handlers/os.py @@ -20,7 +20,7 @@ from functools import cmp_to_key import click -from platformio import __default_requests_timeout__, fs +from platformio import fs from platformio.cache import ContentCache from platformio.device.list.util import list_logical_devices from platformio.home import helpers @@ -50,13 +50,9 @@ class OSRPC: session = helpers.requests_session() if data: - r = await session.post( - uri, data=data, headers=headers, timeout=__default_requests_timeout__ - ) + r = await session.post(uri, data=data, headers=headers) else: - r = await session.get( - uri, headers=headers, timeout=__default_requests_timeout__ - ) + r = await session.get(uri, headers=headers) r.raise_for_status() result = r.text diff --git a/platformio/http.py b/platformio/http.py index f1fd285b..9f4f44dd 100644 --- a/platformio/http.py +++ b/platformio/http.py @@ -20,10 +20,12 @@ from urllib.parse import urljoin import requests.adapters from requests.packages.urllib3.util.retry import Retry # pylint:disable=import-error -from platformio import __check_internet_hosts__, __default_requests_timeout__, app, util +from platformio import __check_internet_hosts__, app, util from platformio.cache import ContentCache, cleanup_content_cache from platformio.exception import PlatformioException, UserSideException +__default_requests_timeout__ = (10, None) # (connect, read) + class HTTPClientError(PlatformioException): def __init__(self, message, response=None): @@ -44,19 +46,30 @@ class InternetIsOffline(UserSideException): ) -class EndpointSession(requests.Session): - def __init__(self, base_url, *args, **kwargs): +class HTTPSession(requests.Session): + def __init__(self, *args, **kwargs): + self._x_base_url = kwargs.pop("x_base_url") if "x_base_url" in kwargs else None super().__init__(*args, **kwargs) - self.base_url = base_url + self.headers.update({"User-Agent": app.get_user_agent()}) + self.verify = app.get_setting("enable_proxy_strict_ssl") def request( # pylint: disable=signature-differs,arguments-differ self, method, url, *args, **kwargs ): - # print(self.base_url, method, url, args, kwargs) - return super().request(method, urljoin(self.base_url, url), *args, **kwargs) + # print("HTTPSession::request", self._x_base_url, method, url, args, kwargs) + if "timeout" not in kwargs: + kwargs["timeout"] = __default_requests_timeout__ + return super().request( + method, + url + if url.startswith("http") or not self._x_base_url + else urljoin(self._x_base_url, url), + *args, + **kwargs + ) -class EndpointSessionIterator: +class HTTPSessionIterator: def __init__(self, endpoints): if not isinstance(endpoints, list): endpoints = [endpoints] @@ -75,8 +88,7 @@ class EndpointSessionIterator: def __next__(self): base_url = next(self.endpoints_iter) - session = EndpointSession(base_url) - session.headers.update({"User-Agent": app.get_user_agent()}) + session = HTTPSession(x_base_url=base_url) adapter = requests.adapters.HTTPAdapter(max_retries=self.retry) session.mount(base_url, adapter) return session @@ -84,7 +96,7 @@ class EndpointSessionIterator: class HTTPClient: def __init__(self, endpoints): - self._session_iter = EndpointSessionIterator(endpoints) + self._session_iter = HTTPSessionIterator(endpoints) self._session = None self._next_session() @@ -122,10 +134,6 @@ class HTTPClient: ) kwargs["headers"] = headers - # set default timeout - if "timeout" not in kwargs: - kwargs["timeout"] = __default_requests_timeout__ - while True: try: return getattr(self._session, method)(path, **kwargs) @@ -201,13 +209,8 @@ def ensure_internet_on(raise_exception=False): def fetch_remote_content(*args, **kwargs): - kwargs["headers"] = kwargs.get("headers", {}) - if "User-Agent" not in kwargs["headers"]: - kwargs["headers"]["User-Agent"] = app.get_user_agent() - - if "timeout" not in kwargs: - kwargs["timeout"] = __default_requests_timeout__ - - r = requests.get(*args, **kwargs) # pylint: disable=missing-timeout - r.raise_for_status() - return r.text + with HTTPSession() as s: + r = s.get(*args, **kwargs) + r.raise_for_status() + r.close() + return r.text diff --git a/platformio/package/download.py b/platformio/package/download.py index b1800e4b..73969dc9 100644 --- a/platformio/package/download.py +++ b/platformio/package/download.py @@ -18,31 +18,30 @@ from os.path import getsize, join from time import mktime import click -import requests -from platformio import __default_requests_timeout__, app, fs +from platformio import fs from platformio.compat import is_terminal +from platformio.http import HTTPSession from platformio.package.exception import PackageException class FileDownloader: def __init__(self, url, dest_dir=None): - self._request = None + self._http_session = HTTPSession() + self._http_response = None # make connection - self._request = requests.get( + self._http_response = self._http_session.get( url, stream=True, - headers={"User-Agent": app.get_user_agent()}, - timeout=__default_requests_timeout__, ) - if self._request.status_code != 200: + if self._http_response.status_code != 200: raise PackageException( "Got the unrecognized status code '{0}' when downloaded {1}".format( - self._request.status_code, url + self._http_response.status_code, url ) ) - disposition = self._request.headers.get("content-disposition") + disposition = self._http_response.headers.get("content-disposition") if disposition and "filename=" in disposition: self._fname = ( disposition[disposition.index("filename=") + 9 :] @@ -63,17 +62,17 @@ class FileDownloader: return self._destination def get_lmtime(self): - return self._request.headers.get("last-modified") + return self._http_response.headers.get("last-modified") def get_size(self): - if "content-length" not in self._request.headers: + if "content-length" not in self._http_response.headers: return -1 - return int(self._request.headers["content-length"]) + return int(self._http_response.headers["content-length"]) def start(self, with_progress=True, silent=False): label = "Downloading" file_size = self.get_size() - itercontent = self._request.iter_content(chunk_size=io.DEFAULT_BUFFER_SIZE) + itercontent = self._http_response.iter_content(chunk_size=io.DEFAULT_BUFFER_SIZE) try: with open(self._destination, "wb") as fp: if file_size == -1 or not with_progress or silent: @@ -110,7 +109,8 @@ class FileDownloader: pb.update(len(chunk)) fp.write(chunk) finally: - self._request.close() + self._http_response.close() + self._http_session.close() if self.get_lmtime(): self._preserve_filemtime(self.get_lmtime()) @@ -158,5 +158,6 @@ class FileDownloader: fs.change_filemtime(self._destination, lmtime) def __del__(self): - if self._request: - self._request.close() + self._http_session.close() + if self._http_response: + self._http_response.close() diff --git a/platformio/telemetry.py b/platformio/telemetry.py index 4eac5eeb..211c0bea 100644 --- a/platformio/telemetry.py +++ b/platformio/telemetry.py @@ -30,6 +30,7 @@ import requests from platformio import __version__, app, exception, util from platformio.cli import PlatformioCLI from platformio.compat import hashlib_encode_data, string_types +from platformio.http import HTTPSession from platformio.proc import is_ci, is_container from platformio.project.helpers import is_platformio_project @@ -206,7 +207,7 @@ class MPDataPusher: def __init__(self): self._queue = queue.LifoQueue() self._failedque = deque() - self._http_session = requests.Session() + self._http_session = HTTPSession() self._http_offline = False self._workers = [] @@ -270,7 +271,6 @@ class MPDataPusher: r = self._http_session.post( "https://ssl.google-analytics.com/collect", data=data, - headers={"User-Agent": app.get_user_agent()}, timeout=1, ) r.raise_for_status()