<s> """<STR_LIT>""" <EOL> from __future__ import absolute_import , division , print_function , with_statement <EOL> import functools <EOL> import time <EOL> import weakref <EOL> from tornado . concurrent import Future <EOL> from tornado . escape import utf8 <EOL> from tornado import httputil , stack_context <EOL> from tornado . ioloop import IOLoop <EOL> from tornado . util import Configurable <EOL> class HTTPClient ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , async_client_class = None , ** kwargs ) : <EOL> self . _io_loop = IOLoop ( ) <EOL> if async_client_class is None : <EOL> async_client_class = AsyncHTTPClient <EOL> self . _async_client = async_client_class ( self . _io_loop , ** kwargs ) <EOL> self . _closed = False <EOL> def __del__ ( self ) : <EOL> self . close ( ) <EOL> def close ( self ) : <EOL> """<STR_LIT>""" <EOL> if not self . _closed : <EOL> self . _async_client . close ( ) <EOL> self . _io_loop . close ( ) <EOL> self . _closed = True <EOL> def fetch ( self , request , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> response = self . _io_loop . run_sync ( functools . partial ( <EOL> self . _async_client . fetch , request , ** kwargs ) ) <EOL> response . rethrow ( ) <EOL> return response <EOL> class AsyncHTTPClient ( Configurable ) : <EOL> """<STR_LIT>""" <EOL> @ classmethod <EOL> def configurable_base ( cls ) : <EOL> return AsyncHTTPClient <EOL> @ classmethod <EOL> def configurable_default ( cls ) : <EOL> from tornado . simple_httpclient import SimpleAsyncHTTPClient <EOL> return SimpleAsyncHTTPClient <EOL> @ classmethod <EOL> def _async_clients ( cls ) : <EOL> attr_name = '<STR_LIT>' + cls . __name__ <EOL> if not hasattr ( cls , attr_name ) : <EOL> setattr ( cls , attr_name , weakref . WeakKeyDictionary ( ) ) <EOL> return getattr ( cls , attr_name ) <EOL> def __new__ ( cls , io_loop = None , force_instance = False , ** kwargs ) : <EOL> io_loop = io_loop or IOLoop . current ( ) <EOL> if io_loop in cls . _async_clients ( ) and not force_instance : <EOL> return cls . _async_clients ( ) [ io_loop ] <EOL> instance = super ( AsyncHTTPClient , cls ) . __new__ ( cls , io_loop = io_loop , <EOL> ** kwargs ) <EOL> if not force_instance : <EOL> cls . _async_clients ( ) [ io_loop ] = instance <EOL> return instance <EOL> def initialize ( self , io_loop , defaults = None ) : <EOL> self . io_loop = io_loop <EOL> self . defaults = dict ( HTTPRequest . _DEFAULTS ) <EOL> if defaults is not None : <EOL> self . defaults . update ( defaults ) <EOL> def close ( self ) : <EOL> """<STR_LIT>""" <EOL> if self . _async_clients ( ) . get ( self . io_loop ) is self : <EOL> del self . _async_clients ( ) [ self . io_loop ] <EOL> def fetch ( self , request , callback = None , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> if not isinstance ( request , HTTPRequest ) : <EOL> request = HTTPRequest ( url = request , ** kwargs ) <EOL> request . headers = httputil . HTTPHeaders ( request . headers ) <EOL> request = _RequestProxy ( request , self . defaults ) <EOL> future = Future ( ) <EOL> if callback is not None : <EOL> callback = stack_context . wrap ( callback ) <EOL> def handle_future ( future ) : <EOL> exc = future . exception ( ) <EOL> if isinstance ( exc , HTTPError ) and exc . response is not None : <EOL> response = exc . response <EOL> elif exc is not None : <EOL> response = HTTPResponse ( <EOL> request , <NUM_LIT> , error = exc , <EOL> request_time = time . time ( ) - request . start_time ) <EOL> else : <EOL> response = future . result ( ) <EOL> self . io_loop . add_callback ( callback , response ) <EOL> future . add_done_callback ( handle_future ) <EOL> def handle_response ( response ) : <EOL> if response . error : <EOL> future . set_exception ( response . error ) <EOL> else : <EOL> future . set_result ( response ) <EOL> self . fetch_impl ( request , handle_response ) <EOL> return future <EOL> def fetch_impl ( self , request , callback ) : <EOL> raise NotImplementedError ( ) <EOL> @ classmethod <EOL> def configure ( cls , impl , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> super ( AsyncHTTPClient , cls ) . configure ( impl , ** kwargs ) <EOL> class HTTPRequest ( object ) : <EOL> """<STR_LIT>""" <EOL> _DEFAULTS = dict ( <EOL> connect_timeout = <NUM_LIT> , <EOL> request_timeout = <NUM_LIT> , <EOL> follow_redirects = True , <EOL> max_redirects = <NUM_LIT:5> , <EOL> use_gzip = True , <EOL> proxy_password = '<STR_LIT>' , <EOL> allow_nonstandard_methods = False , <EOL> validate_cert = True ) <EOL> def __init__ ( self , url , method = "<STR_LIT:GET>" , headers = None , body = None , <EOL> auth_username = None , auth_password = None , auth_mode = None , <EOL> connect_timeout = None , request_timeout = None , <EOL> if_modified_since = None , follow_redirects = None , <EOL> max_redirects = None , user_agent = None , use_gzip = None , <EOL> network_interface = None , streaming_callback = None , <EOL> header_callback = None , prepare_curl_callback = None , <EOL> proxy_host = None , proxy_port = None , proxy_username = None , <EOL> proxy_password = None , allow_nonstandard_methods = None , <EOL> validate_cert = None , ca_certs = None , <EOL> allow_ipv6 = None , <EOL> client_key = None , client_cert = None ) : <EOL> r"""<STR_LIT>""" <EOL> if headers is None : <EOL> headers = httputil . HTTPHeaders ( ) <EOL> if if_modified_since : <EOL> headers [ "<STR_LIT>" ] = httputil . format_timestamp ( <EOL> if_modified_since ) <EOL> self . proxy_host = proxy_host <EOL> self . proxy_port = proxy_port <EOL> self . proxy_username = proxy_username <EOL> self . proxy_password = proxy_password <EOL> self . url = url <EOL> self . method = method <EOL> self . headers = headers <EOL> self . body = utf8 ( body ) <EOL> self . auth_username = auth_username <EOL> self . auth_password = auth_password <EOL> self . auth_mode = auth_mode <EOL> self . connect_timeout = connect_timeout <EOL> self . request_timeout = request_timeout <EOL> self . follow_redirects = follow_redirects <EOL> self . max_redirects = max_redirects <EOL> self . user_agent = user_agent <EOL> self . use_gzip = use_gzip <EOL> self . network_interface = network_interface <EOL> self . streaming_callback = stack_context . wrap ( streaming_callback ) <EOL> self . header_callback = stack_context . wrap ( header_callback ) <EOL> self . prepare_curl_callback = stack_context . wrap ( prepare_curl_callback ) <EOL> self . allow_nonstandard_methods = allow_nonstandard_methods <EOL> self . validate_cert = validate_cert <EOL> self . ca_certs = ca_certs <EOL> self . allow_ipv6 = allow_ipv6 <EOL> self . client_key = client_key <EOL> self . client_cert = client_cert <EOL> self . start_time = time . time ( ) <EOL> class HTTPResponse ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , request , code , headers = None , buffer = None , <EOL> effective_url = None , error = None , request_time = None , <EOL> time_info = None , reason = None ) : <EOL> if isinstance ( request , _RequestProxy ) : <EOL> self . request = request . request <EOL> else : <EOL> self . request = request <EOL> self . code = code <EOL> self . reason = reason or httputil . responses . get ( code , "<STR_LIT>" ) <EOL> if headers is not None : <EOL> self . headers = headers <EOL> else : <EOL> self . headers = httputil . HTTPHeaders ( ) <EOL> self . buffer = buffer <EOL> self . _body = None <EOL> if effective_url is None : <EOL> self . effective_url = request . url <EOL> else : <EOL> self . effective_url = effective_url <EOL> if error is None : <EOL> if self . code < <NUM_LIT:200> or self . code >= <NUM_LIT> : <EOL> self . error = HTTPError ( self . code , response = self ) <EOL> else : <EOL> self . error = None <EOL> else : <EOL> self . error = error <EOL> self . request_time = request_time <EOL> self . time_info = time_info or { } <EOL> def _get_body ( self ) : <EOL> if self . buffer is None : <EOL> return None <EOL> elif self . _body is None : <EOL> self . _body = self . buffer . getvalue ( ) <EOL> return self . _body <EOL> body = property ( _get_body ) <EOL> def rethrow ( self ) : <EOL> """<STR_LIT>""" <EOL> if self . error : <EOL> raise self . error <EOL> def __repr__ ( self ) : <EOL> args = "<STR_LIT:U+002C>" . join ( "<STR_LIT>" % i for i in sorted ( self . __dict__ . items ( ) ) ) <EOL> return "<STR_LIT>" % ( self . __class__ . __name__ , args ) <EOL> class HTTPError ( Exception ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , code , message = None , response = None ) : <EOL> self . code = code <EOL> message = message or httputil . responses . get ( code , "<STR_LIT>" ) <EOL> self . response = response <EOL> Exception . __init__ ( self , "<STR_LIT>" % ( self . code , message ) ) <EOL> class _RequestProxy ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , request , defaults ) : <EOL> self . request = request <EOL> self . defaults = defaults <EOL> def __getattr__ ( self , name ) : <EOL> request_attr = getattr ( self . request , name ) <EOL> if request_attr is not None : <EOL> return request_attr <EOL> elif self . defaults is not None : <EOL> return self . defaults . get ( name , None ) <EOL> else : <EOL> return None <EOL> def main ( ) : <EOL> from tornado . options import define , options , parse_command_line <EOL> define ( "<STR_LIT>" , type = bool , default = False ) <EOL> define ( "<STR_LIT>" , type = bool , default = True ) <EOL> define ( "<STR_LIT>" , type = bool , default = True ) <EOL> define ( "<STR_LIT>" , type = bool , default = True ) <EOL> args = parse_command_line ( ) <EOL> client = HTTPClient ( ) <EOL> for arg in args : <EOL> try : <EOL> response = client . fetch ( arg , <EOL> follow_redirects = options . follow_redirects , <EOL> validate_cert = options . validate_cert , <EOL> ) <EOL> except HTTPError as e : <EOL> if e . response is not None : <EOL> response = e . response <EOL> else : <EOL> raise <EOL> if options . print_headers : <EOL> print ( response . headers ) <EOL> if options . print_body : <EOL> print ( response . body ) <EOL> client . close ( ) <EOL> if __name__ == "<STR_LIT:__main__>" : <EOL> main ( ) </s>
