<s> """<STR_LIT>""" <EOL> from __future__ import absolute_import , division , print_function , with_statement <EOL> import collections <EOL> import logging <EOL> import pycurl <EOL> import threading <EOL> import time <EOL> from tornado import httputil <EOL> from tornado import ioloop <EOL> from tornado . log import gen_log <EOL> from tornado import stack_context <EOL> from tornado . escape import utf8 , native_str <EOL> from tornado . httpclient import HTTPResponse , HTTPError , AsyncHTTPClient , main <EOL> from tornado . util import bytes_type <EOL> try : <EOL> from io import BytesIO <EOL> except ImportError : <EOL> from cStringIO import StringIO as BytesIO <EOL> class CurlAsyncHTTPClient ( AsyncHTTPClient ) : <EOL> def initialize ( self , io_loop , max_clients = <NUM_LIT:10> , defaults = None ) : <EOL> super ( CurlAsyncHTTPClient , self ) . initialize ( io_loop , defaults = defaults ) <EOL> self . _multi = pycurl . CurlMulti ( ) <EOL> self . _multi . setopt ( pycurl . M_TIMERFUNCTION , self . _set_timeout ) <EOL> self . _multi . setopt ( pycurl . M_SOCKETFUNCTION , self . _handle_socket ) <EOL> self . _curls = [ _curl_create ( ) for i in range ( max_clients ) ] <EOL> self . _free_list = self . _curls [ : ] <EOL> self . _requests = collections . deque ( ) <EOL> self . _fds = { } <EOL> self . _timeout = None <EOL> try : <EOL> self . _socket_action = self . _multi . socket_action <EOL> except AttributeError : <EOL> gen_log . warning ( "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" ) <EOL> self . _socket_action = lambda fd , action : self . _multi . socket_all ( ) <EOL> self . _force_timeout_callback = ioloop . PeriodicCallback ( <EOL> self . _handle_force_timeout , <NUM_LIT:1000> , io_loop = io_loop ) <EOL> self . _force_timeout_callback . start ( ) <EOL> dummy_curl_handle = pycurl . Curl ( ) <EOL> self . _multi . add_handle ( dummy_curl_handle ) <EOL> self . _multi . remove_handle ( dummy_curl_handle ) <EOL> def close ( self ) : <EOL> self . _force_timeout_callback . stop ( ) <EOL> if self . _timeout is not None : <EOL> self . io_loop . remove_timeout ( self . _timeout ) <EOL> for curl in self . _curls : <EOL> curl . close ( ) <EOL> self . _multi . close ( ) <EOL> self . _closed = True <EOL> super ( CurlAsyncHTTPClient , self ) . close ( ) <EOL> def fetch_impl ( self , request , callback ) : <EOL> self . _requests . append ( ( request , callback ) ) <EOL> self . _process_queue ( ) <EOL> self . _set_timeout ( <NUM_LIT:0> ) <EOL> def _handle_socket ( self , event , fd , multi , data ) : <EOL> """<STR_LIT>""" <EOL> event_map = { <EOL> pycurl . POLL_NONE : ioloop . IOLoop . NONE , <EOL> pycurl . POLL_IN : ioloop . IOLoop . READ , <EOL> pycurl . POLL_OUT : ioloop . IOLoop . WRITE , <EOL> pycurl . POLL_INOUT : ioloop . IOLoop . READ | ioloop . IOLoop . WRITE <EOL> } <EOL> if event == pycurl . POLL_REMOVE : <EOL> if fd in self . _fds : <EOL> self . io_loop . remove_handler ( fd ) <EOL> del self . _fds [ fd ] <EOL> else : <EOL> ioloop_event = event_map [ event ] <EOL> if fd in self . _fds : <EOL> self . io_loop . remove_handler ( fd ) <EOL> self . io_loop . add_handler ( fd , self . _handle_events , <EOL> ioloop_event ) <EOL> self . _fds [ fd ] = ioloop_event <EOL> def _set_timeout ( self , msecs ) : <EOL> """<STR_LIT>""" <EOL> if self . _timeout is not None : <EOL> self . io_loop . remove_timeout ( self . _timeout ) <EOL> self . _timeout = self . io_loop . add_timeout ( <EOL> self . io_loop . time ( ) + msecs / <NUM_LIT> , self . _handle_timeout ) <EOL> def _handle_events ( self , fd , events ) : <EOL> """<STR_LIT>""" <EOL> action = <NUM_LIT:0> <EOL> if events & ioloop . IOLoop . READ : <EOL> action |= pycurl . CSELECT_IN <EOL> if events & ioloop . IOLoop . WRITE : <EOL> action |= pycurl . CSELECT_OUT <EOL> while True : <EOL> try : <EOL> ret , num_handles = self . _socket_action ( fd , action ) <EOL> except pycurl . error as e : <EOL> ret = e . args [ <NUM_LIT:0> ] <EOL> if ret != pycurl . E_CALL_MULTI_PERFORM : <EOL> break <EOL> self . _finish_pending_requests ( ) <EOL> def _handle_timeout ( self ) : <EOL> """<STR_LIT>""" <EOL> with stack_context . NullContext ( ) : <EOL> self . _timeout = None <EOL> while True : <EOL> try : <EOL> ret , num_handles = self . _socket_action ( <EOL> pycurl . SOCKET_TIMEOUT , <NUM_LIT:0> ) <EOL> except pycurl . error as e : <EOL> ret = e . args [ <NUM_LIT:0> ] <EOL> if ret != pycurl . E_CALL_MULTI_PERFORM : <EOL> break <EOL> self . _finish_pending_requests ( ) <EOL> new_timeout = self . _multi . timeout ( ) <EOL> if new_timeout >= <NUM_LIT:0> : <EOL> self . _set_timeout ( new_timeout ) <EOL> def _handle_force_timeout ( self ) : <EOL> """<STR_LIT>""" <EOL> with stack_context . NullContext ( ) : <EOL> while True : <EOL> try : <EOL> ret , num_handles = self . _multi . socket_all ( ) <EOL> except pycurl . error as e : <EOL> ret = e . args [ <NUM_LIT:0> ] <EOL> if ret != pycurl . E_CALL_MULTI_PERFORM : <EOL> break <EOL> self . _finish_pending_requests ( ) <EOL> def _finish_pending_requests ( self ) : <EOL> """<STR_LIT>""" <EOL> while True : <EOL> num_q , ok_list , err_list = self . _multi . info_read ( ) <EOL> for curl in ok_list : <EOL> self . _finish ( curl ) <EOL> for curl , errnum , errmsg in err_list : <EOL> self . _finish ( curl , errnum , errmsg ) <EOL> if num_q == <NUM_LIT:0> : <EOL> break <EOL> self . _process_queue ( ) <EOL> def _process_queue ( self ) : <EOL> with stack_context . NullContext ( ) : <EOL> while True : <EOL> started = <NUM_LIT:0> <EOL> while self . _free_list and self . _requests : <EOL> started += <NUM_LIT:1> <EOL> curl = self . _free_list . pop ( ) <EOL> ( request , callback ) = self . _requests . popleft ( ) <EOL> curl . info = { <EOL> "<STR_LIT>" : httputil . HTTPHeaders ( ) , <EOL> "<STR_LIT>" : BytesIO ( ) , <EOL> "<STR_LIT>" : request , <EOL> "<STR_LIT>" : callback , <EOL> "<STR_LIT>" : time . time ( ) , <EOL> } <EOL> if pycurl . version_info ( ) [ <NUM_LIT:2> ] <= <NUM_LIT> : <EOL> curl . setopt ( pycurl . IPRESOLVE , pycurl . IPRESOLVE_V4 ) <EOL> _curl_setup_request ( curl , request , curl . info [ "<STR_LIT>" ] , <EOL> curl . info [ "<STR_LIT>" ] ) <EOL> self . _multi . add_handle ( curl ) <EOL> if not started : <EOL> break <EOL> def _finish ( self , curl , curl_error = None , curl_message = None ) : <EOL> info = curl . info <EOL> curl . info = None <EOL> self . _multi . remove_handle ( curl ) <EOL> self . _free_list . append ( curl ) <EOL> buffer = info [ "<STR_LIT>" ] <EOL> if curl_error : <EOL> error = CurlError ( curl_error , curl_message ) <EOL> code = error . code <EOL> effective_url = None <EOL> buffer . close ( ) <EOL> buffer = None <EOL> else : <EOL> error = None <EOL> code = curl . getinfo ( pycurl . HTTP_CODE ) <EOL> effective_url = curl . getinfo ( pycurl . EFFECTIVE_URL ) <EOL> buffer . seek ( <NUM_LIT:0> ) <EOL> time_info = dict ( <EOL> queue = info [ "<STR_LIT>" ] - info [ "<STR_LIT>" ] . start_time , <EOL> namelookup = curl . getinfo ( pycurl . NAMELOOKUP_TIME ) , <EOL> connect = curl . getinfo ( pycurl . CONNECT_TIME ) , <EOL> pretransfer = curl . getinfo ( pycurl . PRETRANSFER_TIME ) , <EOL> starttransfer = curl . getinfo ( pycurl . STARTTRANSFER_TIME ) , <EOL> total = curl . getinfo ( pycurl . TOTAL_TIME ) , <EOL> redirect = curl . getinfo ( pycurl . REDIRECT_TIME ) , <EOL> ) <EOL> try : <EOL> info [ "<STR_LIT>" ] ( HTTPResponse ( <EOL> request = info [ "<STR_LIT>" ] , code = code , headers = info [ "<STR_LIT>" ] , <EOL> buffer = buffer , effective_url = effective_url , error = error , <EOL> request_time = time . time ( ) - info [ "<STR_LIT>" ] , <EOL> time_info = time_info ) ) <EOL> except Exception : <EOL> self . handle_callback_exception ( info [ "<STR_LIT>" ] ) <EOL> def handle_callback_exception ( self , callback ) : <EOL> self . io_loop . handle_callback_exception ( callback ) <EOL> class CurlError ( HTTPError ) : <EOL> def __init__ ( self , errno , message ) : <EOL> HTTPError . __init__ ( self , <NUM_LIT> , message ) <EOL> self . errno = errno <EOL> def _curl_create ( ) : <EOL> curl = pycurl . Curl ( ) <EOL> if gen_log . isEnabledFor ( logging . DEBUG ) : <EOL> curl . setopt ( pycurl . VERBOSE , <NUM_LIT:1> ) <EOL> curl . setopt ( pycurl . DEBUGFUNCTION , _curl_debug ) <EOL> return curl <EOL> def _curl_setup_request ( curl , request , buffer , headers ) : <EOL> curl . setopt ( pycurl . URL , native_str ( request . url ) ) <EOL> if "<STR_LIT>" not in request . headers : <EOL> request . headers [ "<STR_LIT>" ] = "<STR_LIT>" <EOL> if "<STR_LIT>" not in request . headers : <EOL> request . headers [ "<STR_LIT>" ] = "<STR_LIT>" <EOL> if isinstance ( request . headers , httputil . HTTPHeaders ) : <EOL> curl . setopt ( pycurl . HTTPHEADER , <EOL> [ native_str ( "<STR_LIT>" % i ) for i in request . headers . get_all ( ) ] ) <EOL> else : <EOL> curl . setopt ( pycurl . HTTPHEADER , <EOL> [ native_str ( "<STR_LIT>" % i ) for i in request . headers . items ( ) ] ) <EOL> if request . header_callback : <EOL> curl . setopt ( pycurl . HEADERFUNCTION , request . header_callback ) <EOL> else : <EOL> curl . setopt ( pycurl . HEADERFUNCTION , <EOL> lambda line : _curl_header_callback ( headers , line ) ) <EOL> if request . streaming_callback : <EOL> write_function = request . streaming_callback <EOL> else : <EOL> write_function = buffer . write <EOL> if bytes_type is str : <EOL> curl . setopt ( pycurl . WRITEFUNCTION , write_function ) <EOL> else : <EOL> curl . setopt ( pycurl . WRITEFUNCTION , lambda s : write_function ( utf8 ( s ) ) ) <EOL> curl . setopt ( pycurl . FOLLOWLOCATION , request . follow_redirects ) <EOL> curl . setopt ( pycurl . MAXREDIRS , request . max_redirects ) <EOL> curl . setopt ( pycurl . CONNECTTIMEOUT_MS , int ( <NUM_LIT:1000> * request . connect_timeout ) ) <EOL> curl . setopt ( pycurl . TIMEOUT_MS , int ( <NUM_LIT:1000> * request . request_timeout ) ) <EOL> if request . user_agent : <EOL> curl . setopt ( pycurl . USERAGENT , native_str ( request . user_agent ) ) <EOL> else : <EOL> curl . setopt ( pycurl . USERAGENT , "<STR_LIT>" ) <EOL> if request . network_interface : <EOL> curl . setopt ( pycurl . INTERFACE , request . network_interface ) <EOL> if request . use_gzip : <EOL> curl . setopt ( pycurl . ENCODING , "<STR_LIT>" ) <EOL> else : <EOL> curl . setopt ( pycurl . ENCODING , "<STR_LIT:none>" ) <EOL> if request . proxy_host and request . proxy_port : <EOL> curl . setopt ( pycurl . PROXY , request . proxy_host ) <EOL> curl . setopt ( pycurl . PROXYPORT , request . proxy_port ) <EOL> if request . proxy_username : <EOL> credentials = '<STR_LIT>' % ( request . proxy_username , <EOL> request . proxy_password ) <EOL> curl . setopt ( pycurl . PROXYUSERPWD , credentials ) <EOL> else : <EOL> curl . setopt ( pycurl . PROXY , '<STR_LIT>' ) <EOL> if request . validate_cert : <EOL> curl . setopt ( pycurl . SSL_VERIFYPEER , <NUM_LIT:1> ) <EOL> curl . setopt ( pycurl . SSL_VERIFYHOST , <NUM_LIT:2> ) <EOL> else : <EOL> curl . setopt ( pycurl . SSL_VERIFYPEER , <NUM_LIT:0> ) <EOL> curl . setopt ( pycurl . SSL_VERIFYHOST , <NUM_LIT:0> ) <EOL> if request . ca_certs is not None : <EOL> curl . setopt ( pycurl . CAINFO , request . ca_certs ) <EOL> else : <EOL> pass <EOL> if request . allow_ipv6 is False : <EOL> curl . setopt ( pycurl . IPRESOLVE , pycurl . IPRESOLVE_V4 ) <EOL> curl_options = { <EOL> "<STR_LIT:GET>" : pycurl . HTTPGET , <EOL> "<STR_LIT:POST>" : pycurl . POST , <EOL> "<STR_LIT>" : pycurl . UPLOAD , <EOL> "<STR_LIT>" : pycurl . NOBODY , <EOL> } <EOL> custom_methods = set ( [ "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" ] ) <EOL> for o in curl_options . values ( ) : <EOL> curl . setopt ( o , False ) <EOL> if request . method in curl_options : <EOL> curl . unsetopt ( pycurl . CUSTOMREQUEST ) <EOL> curl . setopt ( curl_options [ request . method ] , True ) <EOL> elif request . allow_nonstandard_methods or request . method in custom_methods : <EOL> curl . setopt ( pycurl . CUSTOMREQUEST , request . method ) <EOL> else : <EOL> raise KeyError ( '<STR_LIT>' + request . method ) <EOL> if request . method in ( "<STR_LIT:POST>" , "<STR_LIT>" ) : <EOL> request_buffer = BytesIO ( utf8 ( request . body ) ) <EOL> curl . setopt ( pycurl . READFUNCTION , request_buffer . read ) <EOL> if request . method == "<STR_LIT:POST>" : <EOL> def ioctl ( cmd ) : <EOL> if cmd == curl . IOCMD_RESTARTREAD : <EOL> request_buffer . seek ( <NUM_LIT:0> ) <EOL> curl . setopt ( pycurl . IOCTLFUNCTION , ioctl ) <EOL> curl . setopt ( pycurl . POSTFIELDSIZE , len ( request . body ) ) <EOL> else : <EOL> curl . setopt ( pycurl . INFILESIZE , len ( request . body ) ) <EOL> if request . auth_username is not None : <EOL> userpwd = "<STR_LIT>" % ( request . auth_username , request . auth_password or '<STR_LIT>' ) <EOL> if request . auth_mode is None or request . auth_mode == "<STR_LIT>" : <EOL> curl . setopt ( pycurl . HTTPAUTH , pycurl . HTTPAUTH_BASIC ) <EOL> elif request . auth_mode == "<STR_LIT>" : <EOL> curl . setopt ( pycurl . HTTPAUTH , pycurl . HTTPAUTH_DIGEST ) <EOL> else : <EOL> raise ValueError ( "<STR_LIT>" % request . auth_mode ) <EOL> curl . setopt ( pycurl . USERPWD , native_str ( userpwd ) ) <EOL> gen_log . debug ( "<STR_LIT>" , request . method , request . url , <EOL> request . auth_username ) <EOL> else : <EOL> curl . unsetopt ( pycurl . USERPWD ) <EOL> gen_log . debug ( "<STR_LIT>" , request . method , request . url ) <EOL> if request . client_cert is not None : <EOL> curl . setopt ( pycurl . SSLCERT , request . client_cert ) <EOL> if request . client_key is not None : <EOL> curl . setopt ( pycurl . SSLKEY , request . client_key ) <EOL> if threading . activeCount ( ) > <NUM_LIT:1> : <EOL> curl . setopt ( pycurl . NOSIGNAL , <NUM_LIT:1> ) <EOL> if request . prepare_curl_callback is not None : <EOL> request . prepare_curl_callback ( curl ) <EOL> def _curl_header_callback ( headers , header_line ) : <EOL> header_line = header_line . strip ( ) <EOL> if header_line . startswith ( "<STR_LIT>" ) : <EOL> headers . clear ( ) <EOL> return <EOL> if not header_line : <EOL> return <EOL> headers . parse_line ( header_line ) <EOL> def _curl_debug ( debug_type , debug_msg ) : <EOL> debug_types = ( '<STR_LIT:I>' , '<STR_LIT:<>' , '<STR_LIT:>>' , '<STR_LIT:<>' , '<STR_LIT:>>' ) <EOL> if debug_type == <NUM_LIT:0> : <EOL> gen_log . debug ( '<STR_LIT:%s>' , debug_msg . strip ( ) ) <EOL> elif debug_type in ( <NUM_LIT:1> , <NUM_LIT:2> ) : <EOL> for line in debug_msg . splitlines ( ) : <EOL> gen_log . debug ( '<STR_LIT>' , debug_types [ debug_type ] , line ) <EOL> elif debug_type == <NUM_LIT:4> : <EOL> gen_log . debug ( '<STR_LIT>' , debug_types [ debug_type ] , debug_msg ) <EOL> if __name__ == "<STR_LIT:__main__>" : <EOL> AsyncHTTPClient . configure ( CurlAsyncHTTPClient ) <EOL> main ( ) </s>
