Added a new `enable_proxy_strict_ssl` setting to disable the proxy server certificate verification // Resolve #4432

HISTORY.rst

@@ -16,6 +16,8 @@ PlatformIO Core 6
 6.1.5 (2022-??-??)
 ~~~~~~~~~~~~~~~~~~
 
+* Added a new ``enable_proxy_strict_ssl`` setting to disable the proxy server certificate verification (`issue #4432 <https://github.com/platformio/platformio-core/issues/4432>`_)
+* Documented `PlatformIO Core Proxy Configuration <https://docs.platformio.org/en/latest/core/installation/proxy-configuration.html>`__
 * Speeded up device port finder by avoiding loading board HWIDs from development platforms
 * Improved caching of build metadata in debug mode
 * Fixed an issue when `pio pkg install --storage-dir <https://docs.platformio.org/en/latest/core/userguide/pkg/cmd_install.html>`__ command requires PlatformIO project (`issue #4410 <https://github.com/platformio/platformio-core/issues/4410>`_)

platformio/__init__.py

@@ -44,8 +44,6 @@ __registry_mirror_hosts__ = [
 ]
 __pioremote_endpoint__ = "ssl:host=remote.platformio.org:port=4413"
 
-__default_requests_timeout__ = (10, None)  # (connect, read)
-
 __core_packages__ = {
     "contrib-piohome": "~3.4.2",
     "contrib-pysite": "~2.%d%d.0" % (sys.version_info.major, sys.version_info.minor),

platformio/app.py

@@ -58,6 +58,10 @@ DEFAULT_SETTINGS = {
         "value": get_default_projects_dir(),
         "validator": projects_dir_validate,
     },
+    "enable_proxy_strict_ssl": {
+        "description": "Verify the proxy server certificate against the list of supplied CAs",
+        "value": True,
+    },
 }
 
 SESSION_VARS = {

platformio/home/helpers.py

@@ -14,15 +14,15 @@
 
 import socket
 
-import requests
 from starlette.concurrency import run_in_threadpool
 
 from platformio import util
 from platformio.compat import IS_WINDOWS
+from platformio.http import HTTPSession
 from platformio.proc import where_is_program
 
 
-class AsyncSession(requests.Session):
+class AsyncSession(HTTPSession):
     async def request(  # pylint: disable=signature-differs,invalid-overridden-method
         self, *args, **kwargs
     ):

platformio/home/rpc/handlers/os.py

@@ -20,7 +20,7 @@ from functools import cmp_to_key
 
 import click
 
-from platformio import __default_requests_timeout__, fs
+from platformio import fs
 from platformio.cache import ContentCache
 from platformio.device.list.util import list_logical_devices
 from platformio.home import helpers
@@ -50,13 +50,9 @@ class OSRPC:
 
         session = helpers.requests_session()
         if data:
-            r = await session.post(
+            r = await session.post(uri, data=data, headers=headers)
-                uri, data=data, headers=headers, timeout=__default_requests_timeout__
-            )
         else:
-            r = await session.get(
+            r = await session.get(uri, headers=headers)
-                uri, headers=headers, timeout=__default_requests_timeout__
-            )
 
         r.raise_for_status()
         result = r.text

platformio/http.py

@@ -20,10 +20,12 @@ from urllib.parse import urljoin
 import requests.adapters
 from requests.packages.urllib3.util.retry import Retry  # pylint:disable=import-error
 
-from platformio import __check_internet_hosts__, __default_requests_timeout__, app, util
+from platformio import __check_internet_hosts__, app, util
 from platformio.cache import ContentCache, cleanup_content_cache
 from platformio.exception import PlatformioException, UserSideException
 
+__default_requests_timeout__ = (10, None)  # (connect, read)
+
 
 class HTTPClientError(PlatformioException):
     def __init__(self, message, response=None):
@@ -44,19 +46,30 @@ class InternetIsOffline(UserSideException):
     )
 
 
-class EndpointSession(requests.Session):
+class HTTPSession(requests.Session):
-    def __init__(self, base_url, *args, **kwargs):
+    def __init__(self, *args, **kwargs):
+        self._x_base_url = kwargs.pop("x_base_url") if "x_base_url" in kwargs else None
         super().__init__(*args, **kwargs)
-        self.base_url = base_url
+        self.headers.update({"User-Agent": app.get_user_agent()})
+        self.verify = app.get_setting("enable_proxy_strict_ssl")
 
     def request(  # pylint: disable=signature-differs,arguments-differ
         self, method, url, *args, **kwargs
     ):
-        # print(self.base_url, method, url, args, kwargs)
+        # print("HTTPSession::request", self._x_base_url, method, url, args, kwargs)
-        return super().request(method, urljoin(self.base_url, url), *args, **kwargs)
+        if "timeout" not in kwargs:
+            kwargs["timeout"] = __default_requests_timeout__
+        return super().request(
+            method,
+            url
+            if url.startswith("http") or not self._x_base_url
+            else urljoin(self._x_base_url, url),
+            *args,
+            **kwargs
+        )
 
 
-class EndpointSessionIterator:
+class HTTPSessionIterator:
     def __init__(self, endpoints):
         if not isinstance(endpoints, list):
             endpoints = [endpoints]
@@ -75,8 +88,7 @@ class EndpointSessionIterator:
 
     def __next__(self):
         base_url = next(self.endpoints_iter)
-        session = EndpointSession(base_url)
+        session = HTTPSession(x_base_url=base_url)
-        session.headers.update({"User-Agent": app.get_user_agent()})
         adapter = requests.adapters.HTTPAdapter(max_retries=self.retry)
         session.mount(base_url, adapter)
         return session
@@ -84,7 +96,7 @@ class EndpointSessionIterator:
 
 class HTTPClient:
     def __init__(self, endpoints):
-        self._session_iter = EndpointSessionIterator(endpoints)
+        self._session_iter = HTTPSessionIterator(endpoints)
         self._session = None
         self._next_session()
 
@@ -122,10 +134,6 @@ class HTTPClient:
             )
         kwargs["headers"] = headers
 
-        # set default timeout
-        if "timeout" not in kwargs:
-            kwargs["timeout"] = __default_requests_timeout__
-
         while True:
             try:
                 return getattr(self._session, method)(path, **kwargs)
@@ -201,13 +209,8 @@ def ensure_internet_on(raise_exception=False):
 
 
 def fetch_remote_content(*args, **kwargs):
-    kwargs["headers"] = kwargs.get("headers", {})
+    with HTTPSession() as s:
-    if "User-Agent" not in kwargs["headers"]:
+        r = s.get(*args, **kwargs)
-        kwargs["headers"]["User-Agent"] = app.get_user_agent()
+        r.raise_for_status()
-
+        r.close()
-    if "timeout" not in kwargs:
+        return r.text
-        kwargs["timeout"] = __default_requests_timeout__
-
-    r = requests.get(*args, **kwargs)  # pylint: disable=missing-timeout
-    r.raise_for_status()
-    return r.text

platformio/package/download.py

@@ -18,31 +18,30 @@ from os.path import getsize, join
 from time import mktime
 
 import click
-import requests
 
-from platformio import __default_requests_timeout__, app, fs
+from platformio import fs
 from platformio.compat import is_terminal
+from platformio.http import HTTPSession
 from platformio.package.exception import PackageException
 
 
 class FileDownloader:
     def __init__(self, url, dest_dir=None):
-        self._request = None
+        self._http_session = HTTPSession()
+        self._http_response = None
         # make connection
-        self._request = requests.get(
+        self._http_response = self._http_session.get(
             url,
             stream=True,
-            headers={"User-Agent": app.get_user_agent()},
-            timeout=__default_requests_timeout__,
         )
-        if self._request.status_code != 200:
+        if self._http_response.status_code != 200:
             raise PackageException(
                 "Got the unrecognized status code '{0}' when downloaded {1}".format(
-                    self._request.status_code, url
+                    self._http_response.status_code, url
                 )
             )
 
-        disposition = self._request.headers.get("content-disposition")
+        disposition = self._http_response.headers.get("content-disposition")
         if disposition and "filename=" in disposition:
             self._fname = (
                 disposition[disposition.index("filename=") + 9 :]
@@ -63,17 +62,17 @@ class FileDownloader:
         return self._destination
 
     def get_lmtime(self):
-        return self._request.headers.get("last-modified")
+        return self._http_response.headers.get("last-modified")
 
     def get_size(self):
-        if "content-length" not in self._request.headers:
+        if "content-length" not in self._http_response.headers:
             return -1
-        return int(self._request.headers["content-length"])
+        return int(self._http_response.headers["content-length"])
 
     def start(self, with_progress=True, silent=False):
         label = "Downloading"
         file_size = self.get_size()
-        itercontent = self._request.iter_content(chunk_size=io.DEFAULT_BUFFER_SIZE)
+        itercontent = self._http_response.iter_content(chunk_size=io.DEFAULT_BUFFER_SIZE)
         try:
             with open(self._destination, "wb") as fp:
                 if file_size == -1 or not with_progress or silent:
@@ -110,7 +109,8 @@ class FileDownloader:
                             pb.update(len(chunk))
                             fp.write(chunk)
         finally:
-            self._request.close()
+            self._http_response.close()
+            self._http_session.close()
 
         if self.get_lmtime():
             self._preserve_filemtime(self.get_lmtime())
@@ -158,5 +158,6 @@ class FileDownloader:
         fs.change_filemtime(self._destination, lmtime)
 
     def __del__(self):
-        if self._request:
+        self._http_session.close()
-            self._request.close()
+        if self._http_response:
+            self._http_response.close()

platformio/telemetry.py

@@ -30,6 +30,7 @@ import requests
 from platformio import __version__, app, exception, util
 from platformio.cli import PlatformioCLI
 from platformio.compat import hashlib_encode_data, string_types
+from platformio.http import HTTPSession
 from platformio.proc import is_ci, is_container
 from platformio.project.helpers import is_platformio_project
 
@@ -206,7 +207,7 @@ class MPDataPusher:
     def __init__(self):
         self._queue = queue.LifoQueue()
         self._failedque = deque()
-        self._http_session = requests.Session()
+        self._http_session = HTTPSession()
         self._http_offline = False
         self._workers = []
 
@@ -270,7 +271,6 @@ class MPDataPusher:
             r = self._http_session.post(
                 "https://ssl.google-analytics.com/collect",
                 data=data,
-                headers={"User-Agent": app.get_user_agent()},
                 timeout=1,
             )
             r.raise_for_status()