Set default timeout for http requests // Resolve #3623

platformio/__init__.py

@@ -14,6 +14,8 @@
 
 import sys
 
+DEFAULT_REQUESTS_TIMEOUT = (10, None)  # (connect, read)
+
 VERSION = (4, 4, "0a8")
 __version__ = ".".join([str(s) for s in VERSION])
 

platformio/app.py

@@ -17,6 +17,7 @@ from __future__ import absolute_import
 import codecs
 import getpass
 import hashlib
+import json
 import os
 import platform
 import socket
@@ -25,9 +26,7 @@ from os import environ, getenv, listdir, remove
 from os.path import dirname, isdir, isfile, join, realpath
 from time import time
 
-import requests
-
-from platformio import __version__, exception, fs, proc
+from platformio import __version__, exception, fs, proc, util
 from platformio.compat import WINDOWS, dump_json_to_unicode, hashlib_encode_data
 from platformio.package.lockfile import LockFile
 from platformio.project.helpers import (
@@ -403,16 +402,14 @@ def get_cid():
         uid = getenv("C9_UID")
     elif getenv("CHE_API", getenv("CHE_API_ENDPOINT")):
         try:
-            uid = (
-                requests.get(
+            uid = json.loads(
+                util.fetch_remote_content(
                     "{api}/user?token={token}".format(
                         api=getenv("CHE_API", getenv("CHE_API_ENDPOINT")),
                         token=getenv("USER_TOKEN"),
                     )
                 )
-                .json()
-                .get("id")
-            )
+            ).get("id")
         except:  # pylint: disable=bare-except
             pass
     if not uid:

platformio/clients/http.py

@@ -15,7 +15,7 @@
 import requests.adapters
 from requests.packages.urllib3.util.retry import Retry  # pylint:disable=import-error
 
-from platformio import app, util
+from platformio import DEFAULT_REQUESTS_TIMEOUT, app, util
 from platformio.exception import PlatformioException
 
 
@@ -58,6 +58,11 @@ class HTTPClient(object):
         # check Internet before and resolve issue with 60 seconds timeout
         # print(self, method, path, kwargs)
         util.internet_on(raise_exception=True)
+
+        # set default timeout
+        if "timeout" not in kwargs:
+            kwargs["timeout"] = DEFAULT_REQUESTS_TIMEOUT
+
         try:
             return getattr(self._session, method)(self.base_url + path, **kwargs)
         except (requests.exceptions.ConnectionError, requests.exceptions.Timeout) as e:

platformio/commands/home/rpc/handlers/os.py

@@ -22,7 +22,7 @@ from functools import cmp_to_key
 import click
 from twisted.internet import defer  # pylint: disable=import-error
 
-from platformio import app, fs, util
+from platformio import DEFAULT_REQUESTS_TIMEOUT, app, fs, util
 from platformio.commands.home import helpers
 from platformio.compat import PY2, get_filesystem_encoding, glob_recursive
 
@@ -51,9 +51,13 @@ class OSRPC(object):
 
         session = helpers.requests_session()
         if data:
-            r = yield session.post(uri, data=data, headers=headers)
+            r = yield session.post(
+                uri, data=data, headers=headers, timeout=DEFAULT_REQUESTS_TIMEOUT
+            )
         else:
-            r = yield session.get(uri, headers=headers)
+            r = yield session.get(
+                uri, headers=headers, timeout=DEFAULT_REQUESTS_TIMEOUT
+            )
 
         r.raise_for_status()
         result = r.text

platformio/commands/upgrade.py

@@ -12,14 +12,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import json
 import os
 import re
 from zipfile import ZipFile
 
 import click
-import requests
 
-from platformio import VERSION, __version__, app, exception
+from platformio import VERSION, __version__, app, exception, util
 from platformio.compat import WINDOWS
 from platformio.proc import exec_command, get_pythonexe_path
 from platformio.project.helpers import get_project_cache_dir
@@ -130,13 +130,11 @@ def get_latest_version():
 
 def get_develop_latest_version():
     version = None
-    r = requests.get(
+    content = util.fetch_remote_content(
         "https://raw.githubusercontent.com/platformio/platformio"
-        "/develop/platformio/__init__.py",
-        headers={"User-Agent": app.get_user_agent()},
+        "/develop/platformio/__init__.py"
     )
-    r.raise_for_status()
-    for line in r.text.split("\n"):
+    for line in content.split("\n"):
         line = line.strip()
         if not line.startswith("VERSION"):
             continue
@@ -152,9 +150,5 @@ def get_develop_latest_version():
 
 
 def get_pypi_latest_version():
-    r = requests.get(
-        "https://pypi.org/pypi/platformio/json",
-        headers={"User-Agent": app.get_user_agent()},
-    )
-    r.raise_for_status()
-    return r.json()["info"]["version"]
+    content = util.fetch_remote_content("https://pypi.org/pypi/platformio/json")
+    return json.loads(content)["info"]["version"]

platformio/package/download.py

@@ -14,7 +14,6 @@
 
 import io
 import math
-import sys
 from email.utils import parsedate_tz
 from os.path import getsize, join
 from time import mktime
@@ -22,7 +21,7 @@ from time import mktime
 import click
 import requests
 
-from platformio import app, fs, util
+from platformio import DEFAULT_REQUESTS_TIMEOUT, app, fs, util
 from platformio.package.exception import PackageException
 
 
@@ -34,7 +33,7 @@ class FileDownloader(object):
             url,
             stream=True,
             headers={"User-Agent": app.get_user_agent()},
-            verify=sys.version_info >= (2, 7, 9),
+            timeout=DEFAULT_REQUESTS_TIMEOUT,
         )
         if self._request.status_code != 200:
             raise PackageException(

platformio/package/manifest/parser.py

@@ -19,8 +19,6 @@ import os
 import re
 import tarfile
 
-import requests
-
 from platformio import util
 from platformio.compat import get_object_members, string_types
 from platformio.package.exception import ManifestParserError, UnknownManifestError
@@ -108,10 +106,9 @@ class ManifestParserFactory(object):
 
     @staticmethod
     def new_from_url(remote_url):
-        r = requests.get(remote_url)
-        r.raise_for_status()
+        content = util.fetch_remote_content(remote_url)
         return ManifestParserFactory.new(
-            r.text,
+            content,
             ManifestFileType.from_uri(remote_url) or ManifestFileType.LIBRARY_JSON,
             remote_url,
         )

platformio/package/manifest/schema.py

@@ -14,11 +14,14 @@
 
 # pylint: disable=too-many-ancestors
 
+import json
+
 import marshmallow
 import requests
 import semantic_version
 from marshmallow import Schema, ValidationError, fields, validate, validates
 
+from platformio import util
 from platformio.package.exception import ManifestValidationError
 from platformio.util import memoized
 
@@ -248,9 +251,9 @@ class ManifestSchema(BaseSchema):
     @staticmethod
     @memoized(expire="1h")
     def load_spdx_licenses():
-        r = requests.get(
+        version = "3.10"
+        spdx_data_url = (
             "https://raw.githubusercontent.com/spdx/license-list-data"
-            "/v3.10/json/licenses.json"
+            "/v%s/json/licenses.json" % version
         )
-        r.raise_for_status()
-        return r.json()
+        return json.loads(util.fetch_remote_content(spdx_data_url))

platformio/util.py

@@ -29,7 +29,7 @@ from glob import glob
 import click
 import requests
 
-from platformio import __apiurl__, __version__, exception
+from platformio import DEFAULT_REQUESTS_TIMEOUT, __apiurl__, __version__, exception
 from platformio.commands import PlatformioCLI
 from platformio.compat import PY2, WINDOWS
 from platformio.fs import cd  # pylint: disable=unused-import
@@ -303,10 +303,16 @@ def _get_api_result(
                 headers=headers,
                 auth=auth,
                 verify=verify_ssl,
+                timeout=DEFAULT_REQUESTS_TIMEOUT,
             )
         else:
             r = _api_request_session().get(
-                url, params=params, headers=headers, auth=auth, verify=verify_ssl
+                url,
+                params=params,
+                headers=headers,
+                auth=auth,
+                verify=verify_ssl,
+                timeout=DEFAULT_REQUESTS_TIMEOUT,
             )
         result = r.json()
         r.raise_for_status()
@@ -398,6 +404,22 @@ def internet_on(raise_exception=False):
     return result
 
 
+def fetch_remote_content(*args, **kwargs):
+    # pylint: disable=import-outside-toplevel
+    from platformio.app import get_user_agent
+
+    kwargs["headers"] = kwargs.get("headers", {})
+    if "User-Agent" not in kwargs["headers"]:
+        kwargs["headers"]["User-Agent"] = get_user_agent()
+
+    if "timeout" not in kwargs:
+        kwargs["timeout"] = DEFAULT_REQUESTS_TIMEOUT
+
+    r = requests.get(*args, **kwargs)
+    r.raise_for_status()
+    return r.text
+
+
 def pepver_to_semver(pepver):
     return re.sub(r"(\.\d+)\.?(dev|a|b|rc|post)", r"\1-\2.", pepver, 1)