def fetch(url, method='GET', payload=None, referer=None, user_agent=None, accept_language=None):
    if isinstance(url, unicode):
        url = url.encode('utf-8')
    r = urlparse.urlparse(url)
    if r.scheme!='http' and r.scheme!='https':
        raise UrlFetchError('Scheme must be http or https')
    if method=='GET' and payload:
        raise UrlFetchError('Cannot set payload for GET')
    logging.info('%s %s' % (method, url))
    opener = urllib2.build_opener(urllib2.HTTPHandler)
    request = urllib2.Request(url, data=payload)
    request.get_method = lambda: method
    if payload:
        request.add_header('Content-Length', len(payload))
    if referer:
        request.add_header('Referer', referer)
    if user_agent:
        request.add_header('User-Agent', _USER_AGENTS.get(user_agent.lower(), user_agent))
    if accept_language:
        request.add_header('Accept-Language', accept_language)
    try:
        response = opener.open(request)
        return _get_content_type(response.headers), response.read()
    except urllib2.URLError, e:
        logging.warn('URLError: %s, reason: %s' % (url, e.reason))
        raise UrlFetchError(e)
    except urllib2.HTTPError, e:
        logging.warn('HTTPError: %s, body: %s' % (e.code, e.read))
        raise UrlFetchError(e)
Fetch url and return its response.
dataset/ETHPy150Open michaelliao/transwarp/transwarp/urlfetch.py/fetch