use urllib3 instead of urllib(2)

This commit is contained in:
Noam Meltzer 2016-05-30 00:26:18 +03:00
parent f8a9722573
commit 3076dfc086
2 changed files with 60 additions and 58 deletions

View file

@ -1 +1,2 @@
future future
urllib3

View file

@ -18,29 +18,31 @@
# along with this program. If not, see [http://www.gnu.org/licenses/]. # along with this program. If not, see [http://www.gnu.org/licenses/].
"""This module contains methods to make POST and GET requests""" """This module contains methods to make POST and GET requests"""
import functools
import json import json
import socket
from ssl import SSLError
from future.moves.http.client import HTTPException import urllib3
from future.moves.urllib.error import HTTPError, URLError
from future.moves.urllib.request import urlopen, urlretrieve, Request
from telegram import (InputFile, TelegramError) from telegram import (InputFile, TelegramError)
from telegram.error import Unauthorized, NetworkError, TimedOut, BadRequest from telegram.error import Unauthorized, NetworkError, TimedOut, BadRequest
_CON_POOL = None
def _get_con_pool():
if _CON_POOL is not None:
return _CON_POOL
global _CON_POOL
_CON_POOL = urllib3.PoolManager(10)
return _CON_POOL
def _parse(json_data): def _parse(json_data):
"""Try and parse the JSON returned from Telegram and return an empty """Try and parse the JSON returned from Telegram.
dictionary if there is any error.
Args:
url:
urllib.urlopen object
Returns: Returns:
A JSON parsed as Python dict with results. dict: A JSON parsed as Python dict with results - on error this dict will be empty.
""" """
decoded_s = json_data.decode('utf-8') decoded_s = json_data.decode('utf-8')
try: try:
@ -54,53 +56,49 @@ def _parse(json_data):
return data['result'] return data['result']
def _try_except_req(func): def _request_wrapper(*args, **kwargs):
"""Decorator for requests to handle known exceptions""" """Wraps urllib3 request for handling known exceptions.
@functools.wraps(func) Args:
def decorator(*args, **kwargs): args: unnamed arguments, passed to urllib3 request.
try: kwargs: keyword arguments, passed tp urllib3 request.
return func(*args, **kwargs)
except HTTPError as error: Returns:
# `HTTPError` inherits from `URLError` so `HTTPError` handling must str: A non-parsed JSON text.
# come first.
errcode = error.getcode()
try: Raises:
message = _parse(error.read()) TelegramError
if errcode in (401, 403): """
raise Unauthorized()
elif errcode == 400:
raise BadRequest(message)
elif errcode == 502:
raise NetworkError('Bad Gateway')
except ValueError:
message = 'Unknown HTTPError {0}'.format(error.getcode())
raise NetworkError('{0} ({1})'.format(message, errcode)) try:
resp = _get_con_pool().request(*args, **kwargs)
except urllib3.exceptions.TimeoutError as error:
raise TimedOut()
except urllib3.exceptions.HTTPError as error:
# HTTPError must come last as its the base urllib3 exception class
# TODO: do something smart here; for now just raise NetowrkError
raise NetworkError('urllib3 HTTPError {0}'.format(error))
except URLError as error: if 200 <= resp.status <= 299:
raise NetworkError('URLError: {0}'.format(error.reason)) # 200-299 range are HTTP success statuses
return resp.data
except (SSLError, socket.timeout) as error: try:
err_s = str(error) message = _parse(resp.data)
if 'operation timed out' in err_s: except ValueError:
raise TimedOut() raise NetworkError('Unknown HTTPError {0}'.format(resp.status))
raise NetworkError(err_s) if resp.status in (401, 403):
raise Unauthorized()
except HTTPException as error: elif resp.status == 400:
raise NetworkError('HTTPException: {0!r}'.format(error)) raise BadRequest(repr(message))
elif resp.status == 502:
except socket.error as error: raise NetworkError('Bad Gateway')
raise NetworkError('socket.error: {0!r}'.format(error)) else:
raise NetworkError('{0} ({1})'.format(message, resp.status))
return decorator
@_try_except_req
def get(url): def get(url):
"""Request an URL. """Request an URL.
Args: Args:
@ -109,13 +107,13 @@ def get(url):
Returns: Returns:
A JSON object. A JSON object.
""" """
result = urlopen(url).read() result = _request_wrapper('GET', url)
return _parse(result) return _parse(result)
@_try_except_req
def post(url, data, timeout=None): def post(url, data, timeout=None):
"""Request an URL. """Request an URL.
Args: Args:
@ -142,16 +140,17 @@ def post(url, data, timeout=None):
if InputFile.is_inputfile(data): if InputFile.is_inputfile(data):
data = InputFile(data) data = InputFile(data)
request = Request(url, data=data.to_form(), headers=data.headers) result = _request_wrapper('POST', url, body=data.to_form(), headers=data.headers)
else: else:
data = json.dumps(data) data = json.dumps(data)
request = Request(url, data=data.encode(), headers={'Content-Type': 'application/json'}) result = _request_wrapper('POST',
url,
body=data.encode(),
headers={'Content-Type': 'application/json'})
result = urlopen(request, **urlopen_kwargs).read()
return _parse(result) return _parse(result)
@_try_except_req
def download(url, filename): def download(url, filename):
"""Download a file by its URL. """Download a file by its URL.
Args: Args:
@ -160,6 +159,8 @@ def download(url, filename):
filename: filename:
The filename within the path to download the file. The filename within the path to download the file.
"""
urlretrieve(url, filename) """
buf = _request_wrapper('GET', url)
with open(filename, 'wb') as fobj:
fobj.write(buf)