diff --git a/platformio/__init__.py b/platformio/__init__.py index ee6d438d..6ef8af69 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,6 +14,8 @@ import sys +DEFAULT_REQUESTS_TIMEOUT = (10, None) # (connect, read) + VERSION = (4, 4, "0a8") __version__ = ".".join([str(s) for s in VERSION]) diff --git a/platformio/app.py b/platformio/app.py index 00a8e89f..21adba1c 100644 --- a/platformio/app.py +++ b/platformio/app.py @@ -17,6 +17,7 @@ from __future__ import absolute_import import codecs import getpass import hashlib +import json import os import platform import socket @@ -25,9 +26,7 @@ from os import environ, getenv, listdir, remove from os.path import dirname, isdir, isfile, join, realpath from time import time -import requests - -from platformio import __version__, exception, fs, proc +from platformio import __version__, exception, fs, proc, util from platformio.compat import WINDOWS, dump_json_to_unicode, hashlib_encode_data from platformio.package.lockfile import LockFile from platformio.project.helpers import ( @@ -403,16 +402,14 @@ def get_cid(): uid = getenv("C9_UID") elif getenv("CHE_API", getenv("CHE_API_ENDPOINT")): try: - uid = ( - requests.get( + uid = json.loads( + util.fetch_remote_content( "{api}/user?token={token}".format( api=getenv("CHE_API", getenv("CHE_API_ENDPOINT")), token=getenv("USER_TOKEN"), ) ) - .json() - .get("id") - ) + ).get("id") except: # pylint: disable=bare-except pass if not uid: diff --git a/platformio/clients/http.py b/platformio/clients/http.py index 974017b7..e18d2eed 100644 --- a/platformio/clients/http.py +++ b/platformio/clients/http.py @@ -15,7 +15,7 @@ import requests.adapters from requests.packages.urllib3.util.retry import Retry # pylint:disable=import-error -from platformio import app, util +from platformio import DEFAULT_REQUESTS_TIMEOUT, app, util from platformio.exception import PlatformioException @@ -58,6 +58,11 @@ class HTTPClient(object): # check Internet before and resolve issue with 60 seconds timeout # print(self, method, path, kwargs) util.internet_on(raise_exception=True) + + # set default timeout + if "timeout" not in kwargs: + kwargs["timeout"] = DEFAULT_REQUESTS_TIMEOUT + try: return getattr(self._session, method)(self.base_url + path, **kwargs) except (requests.exceptions.ConnectionError, requests.exceptions.Timeout) as e: diff --git a/platformio/commands/home/rpc/handlers/os.py b/platformio/commands/home/rpc/handlers/os.py index 3b4bd4e1..2b1662f2 100644 --- a/platformio/commands/home/rpc/handlers/os.py +++ b/platformio/commands/home/rpc/handlers/os.py @@ -22,7 +22,7 @@ from functools import cmp_to_key import click from twisted.internet import defer # pylint: disable=import-error -from platformio import app, fs, util +from platformio import DEFAULT_REQUESTS_TIMEOUT, app, fs, util from platformio.commands.home import helpers from platformio.compat import PY2, get_filesystem_encoding, glob_recursive @@ -51,9 +51,13 @@ class OSRPC(object): session = helpers.requests_session() if data: - r = yield session.post(uri, data=data, headers=headers) + r = yield session.post( + uri, data=data, headers=headers, timeout=DEFAULT_REQUESTS_TIMEOUT + ) else: - r = yield session.get(uri, headers=headers) + r = yield session.get( + uri, headers=headers, timeout=DEFAULT_REQUESTS_TIMEOUT + ) r.raise_for_status() result = r.text diff --git a/platformio/commands/upgrade.py b/platformio/commands/upgrade.py index 6303ea69..c8c8b9fe 100644 --- a/platformio/commands/upgrade.py +++ b/platformio/commands/upgrade.py @@ -12,14 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. +import json import os import re from zipfile import ZipFile import click -import requests -from platformio import VERSION, __version__, app, exception +from platformio import VERSION, __version__, app, exception, util from platformio.compat import WINDOWS from platformio.proc import exec_command, get_pythonexe_path from platformio.project.helpers import get_project_cache_dir @@ -130,13 +130,11 @@ def get_latest_version(): def get_develop_latest_version(): version = None - r = requests.get( + content = util.fetch_remote_content( "https://raw.githubusercontent.com/platformio/platformio" - "/develop/platformio/__init__.py", - headers={"User-Agent": app.get_user_agent()}, + "/develop/platformio/__init__.py" ) - r.raise_for_status() - for line in r.text.split("\n"): + for line in content.split("\n"): line = line.strip() if not line.startswith("VERSION"): continue @@ -152,9 +150,5 @@ def get_develop_latest_version(): def get_pypi_latest_version(): - r = requests.get( - "https://pypi.org/pypi/platformio/json", - headers={"User-Agent": app.get_user_agent()}, - ) - r.raise_for_status() - return r.json()["info"]["version"] + content = util.fetch_remote_content("https://pypi.org/pypi/platformio/json") + return json.loads(content)["info"]["version"] diff --git a/platformio/package/download.py b/platformio/package/download.py index 3c723c4b..7f29e7ac 100644 --- a/platformio/package/download.py +++ b/platformio/package/download.py @@ -14,7 +14,6 @@ import io import math -import sys from email.utils import parsedate_tz from os.path import getsize, join from time import mktime @@ -22,7 +21,7 @@ from time import mktime import click import requests -from platformio import app, fs, util +from platformio import DEFAULT_REQUESTS_TIMEOUT, app, fs, util from platformio.package.exception import PackageException @@ -34,7 +33,7 @@ class FileDownloader(object): url, stream=True, headers={"User-Agent": app.get_user_agent()}, - verify=sys.version_info >= (2, 7, 9), + timeout=DEFAULT_REQUESTS_TIMEOUT, ) if self._request.status_code != 200: raise PackageException( diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py index b4a93d98..d453c83e 100644 --- a/platformio/package/manifest/parser.py +++ b/platformio/package/manifest/parser.py @@ -19,8 +19,6 @@ import os import re import tarfile -import requests - from platformio import util from platformio.compat import get_object_members, string_types from platformio.package.exception import ManifestParserError, UnknownManifestError @@ -108,10 +106,9 @@ class ManifestParserFactory(object): @staticmethod def new_from_url(remote_url): - r = requests.get(remote_url) - r.raise_for_status() + content = util.fetch_remote_content(remote_url) return ManifestParserFactory.new( - r.text, + content, ManifestFileType.from_uri(remote_url) or ManifestFileType.LIBRARY_JSON, remote_url, ) diff --git a/platformio/package/manifest/schema.py b/platformio/package/manifest/schema.py index 8befab52..7dafaa23 100644 --- a/platformio/package/manifest/schema.py +++ b/platformio/package/manifest/schema.py @@ -14,11 +14,14 @@ # pylint: disable=too-many-ancestors +import json + import marshmallow import requests import semantic_version from marshmallow import Schema, ValidationError, fields, validate, validates +from platformio import util from platformio.package.exception import ManifestValidationError from platformio.util import memoized @@ -248,9 +251,9 @@ class ManifestSchema(BaseSchema): @staticmethod @memoized(expire="1h") def load_spdx_licenses(): - r = requests.get( + version = "3.10" + spdx_data_url = ( "https://raw.githubusercontent.com/spdx/license-list-data" - "/v3.10/json/licenses.json" + "/v%s/json/licenses.json" % version ) - r.raise_for_status() - return r.json() + return json.loads(util.fetch_remote_content(spdx_data_url)) diff --git a/platformio/util.py b/platformio/util.py index a1974f17..982b0bab 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -29,7 +29,7 @@ from glob import glob import click import requests -from platformio import __apiurl__, __version__, exception +from platformio import DEFAULT_REQUESTS_TIMEOUT, __apiurl__, __version__, exception from platformio.commands import PlatformioCLI from platformio.compat import PY2, WINDOWS from platformio.fs import cd # pylint: disable=unused-import @@ -303,10 +303,16 @@ def _get_api_result( headers=headers, auth=auth, verify=verify_ssl, + timeout=DEFAULT_REQUESTS_TIMEOUT, ) else: r = _api_request_session().get( - url, params=params, headers=headers, auth=auth, verify=verify_ssl + url, + params=params, + headers=headers, + auth=auth, + verify=verify_ssl, + timeout=DEFAULT_REQUESTS_TIMEOUT, ) result = r.json() r.raise_for_status() @@ -398,6 +404,22 @@ def internet_on(raise_exception=False): return result +def fetch_remote_content(*args, **kwargs): + # pylint: disable=import-outside-toplevel + from platformio.app import get_user_agent + + kwargs["headers"] = kwargs.get("headers", {}) + if "User-Agent" not in kwargs["headers"]: + kwargs["headers"]["User-Agent"] = get_user_agent() + + if "timeout" not in kwargs: + kwargs["timeout"] = DEFAULT_REQUESTS_TIMEOUT + + r = requests.get(*args, **kwargs) + r.raise_for_status() + return r.text + + def pepver_to_semver(pepver): return re.sub(r"(\.\d+)\.?(dev|a|b|rc|post)", r"\1-\2.", pepver, 1)