<s> import logging <EOL> import socket <EOL> from httplib import HTTPConnection , HTTPSConnection , HTTPException <EOL> from Queue import Queue , Empty , Full <EOL> from select import select <EOL> from socket import error as SocketError , timeout as SocketTimeout <EOL> try : <EOL> import ssl <EOL> BaseSSLError = ssl . SSLError <EOL> except ImportError : <EOL> ssl = None <EOL> BaseSSLError = None <EOL> from . request import RequestMethods <EOL> from . response import HTTPResponse <EOL> from . exceptions import ( <EOL> SSLError , <EOL> MaxRetryError , <EOL> TimeoutError , <EOL> HostChangedError , <EOL> EmptyPoolError , <EOL> ) <EOL> log = logging . getLogger ( __name__ ) <EOL> _Default = object ( ) <EOL> class VerifiedHTTPSConnection ( HTTPSConnection ) : <EOL> """<STR_LIT>""" <EOL> cert_reqs = None <EOL> ca_certs = None <EOL> def set_cert ( self , key_file = None , cert_file = None , <EOL> cert_reqs = '<STR_LIT>' , ca_certs = None ) : <EOL> ssl_req_scheme = { <EOL> '<STR_LIT>' : ssl . CERT_NONE , <EOL> '<STR_LIT>' : ssl . CERT_OPTIONAL , <EOL> '<STR_LIT>' : ssl . CERT_REQUIRED <EOL> } <EOL> self . key_file = key_file <EOL> self . cert_file = cert_file <EOL> self . cert_reqs = ssl_req_scheme . get ( cert_reqs ) or ssl . CERT_NONE <EOL> self . ca_certs = ca_certs <EOL> def connect ( self ) : <EOL> sock = socket . create_connection ( ( self . host , self . port ) , self . timeout ) <EOL> self . sock = ssl . wrap_socket ( sock , self . key_file , self . cert_file , <EOL> cert_reqs = self . cert_reqs , <EOL> ca_certs = self . ca_certs ) <EOL> class ConnectionPool ( object ) : <EOL> """<STR_LIT>""" <EOL> pass <EOL> class HTTPConnectionPool ( ConnectionPool , RequestMethods ) : <EOL> """<STR_LIT>""" <EOL> scheme = '<STR_LIT:http>' <EOL> def __init__ ( self , host , port = None , strict = False , timeout = None , maxsize = <NUM_LIT:1> , <EOL> block = False , headers = None ) : <EOL> self . host = host <EOL> self . port = port <EOL> self . strict = strict <EOL> self . timeout = timeout <EOL> self . pool = Queue ( maxsize ) <EOL> self . block = block <EOL> self . headers = headers or { } <EOL> for _ in xrange ( maxsize ) : <EOL> self . pool . put ( None ) <EOL> self . num_connections = <NUM_LIT:0> <EOL> self . num_requests = <NUM_LIT:0> <EOL> def _new_conn ( self ) : <EOL> """<STR_LIT>""" <EOL> self . num_connections += <NUM_LIT:1> <EOL> log . info ( "<STR_LIT>" % <EOL> ( self . num_connections , self . host ) ) <EOL> return HTTPConnection ( host = self . host , port = self . port ) <EOL> def _get_conn ( self , timeout = None ) : <EOL> """<STR_LIT>""" <EOL> conn = None <EOL> try : <EOL> conn = self . pool . get ( block = self . block , timeout = timeout ) <EOL> if conn and conn . sock and select ( [ conn . sock ] , [ ] , [ ] , <NUM_LIT:0.0> ) [ <NUM_LIT:0> ] : <EOL> log . info ( "<STR_LIT>" % self . host ) <EOL> conn . close ( ) <EOL> except Empty : <EOL> if self . block : <EOL> raise EmptyPoolError ( "<STR_LIT>" <EOL> "<STR_LIT>" ) <EOL> pass <EOL> return conn or self . _new_conn ( ) <EOL> def _put_conn ( self , conn ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> self . pool . put ( conn , block = False ) <EOL> except Full : <EOL> log . warning ( "<STR_LIT>" <EOL> % self . host ) <EOL> def _make_request ( self , conn , method , url , timeout = _Default , <EOL> ** httplib_request_kw ) : <EOL> """<STR_LIT>""" <EOL> self . num_requests += <NUM_LIT:1> <EOL> if timeout is _Default : <EOL> timeout = self . timeout <EOL> conn . request ( method , url , ** httplib_request_kw ) <EOL> conn . sock . settimeout ( timeout ) <EOL> httplib_response = conn . getresponse ( ) <EOL> log . debug ( "<STR_LIT>" % <EOL> ( method , url , <EOL> conn . _http_vsn_str , <EOL> httplib_response . status , httplib_response . length ) ) <EOL> return httplib_response <EOL> def is_same_host ( self , url ) : <EOL> """<STR_LIT>""" <EOL> return ( url . startswith ( '<STR_LIT:/>' ) or <EOL> get_host ( url ) == ( self . scheme , self . host , self . port ) ) <EOL> def urlopen ( self , method , url , body = None , headers = None , retries = <NUM_LIT:3> , <EOL> redirect = True , assert_same_host = True , timeout = _Default , <EOL> pool_timeout = None , release_conn = None , ** response_kw ) : <EOL> """<STR_LIT>""" <EOL> if headers is None : <EOL> headers = self . headers <EOL> if retries < <NUM_LIT:0> : <EOL> raise MaxRetryError ( "<STR_LIT>" % url ) <EOL> if release_conn is None : <EOL> release_conn = response_kw . get ( '<STR_LIT>' , True ) <EOL> if assert_same_host and not self . is_same_host ( url ) : <EOL> host = "<STR_LIT>" % ( self . scheme , self . host ) <EOL> if self . port : <EOL> host = "<STR_LIT>" % ( host , self . port ) <EOL> raise HostChangedError ( "<STR_LIT>" <EOL> "<STR_LIT>" % ( host , url ) ) <EOL> conn = None <EOL> try : <EOL> conn = self . _get_conn ( timeout = pool_timeout ) <EOL> httplib_response = self . _make_request ( conn , method , url , <EOL> timeout = timeout , <EOL> body = body , headers = headers ) <EOL> response_conn = not release_conn and conn <EOL> response = HTTPResponse . from_httplib ( httplib_response , <EOL> pool = self , <EOL> connection = response_conn , <EOL> ** response_kw ) <EOL> except ( SocketTimeout , Empty ) , e : <EOL> raise TimeoutError ( "<STR_LIT>" % <EOL> self . timeout ) <EOL> except ( BaseSSLError ) , e : <EOL> raise SSLError ( e ) <EOL> except ( HTTPException , SocketError ) , e : <EOL> conn = None <EOL> finally : <EOL> if conn and release_conn : <EOL> self . _put_conn ( conn ) <EOL> if not conn : <EOL> log . warn ( "<STR_LIT>" <EOL> "<STR_LIT>" % ( retries , e , url ) ) <EOL> return self . urlopen ( method , url , body , headers , retries - <NUM_LIT:1> , <EOL> redirect , assert_same_host ) <EOL> if ( redirect and <EOL> response . status in [ <NUM_LIT> , <NUM_LIT> , <NUM_LIT> , <NUM_LIT> ] and <EOL> '<STR_LIT:location>' in response . headers ) : <EOL> log . info ( "<STR_LIT>" % <EOL> ( url , response . headers . get ( '<STR_LIT:location>' ) ) ) <EOL> return self . urlopen ( method , response . headers . get ( '<STR_LIT:location>' ) , body , <EOL> headers , retries - <NUM_LIT:1> , redirect , <EOL> assert_same_host ) <EOL> return response <EOL> class HTTPSConnectionPool ( HTTPConnectionPool ) : <EOL> """<STR_LIT>""" <EOL> scheme = '<STR_LIT>' <EOL> def __init__ ( self , host , port = None , <EOL> strict = False , timeout = None , maxsize = <NUM_LIT:1> , <EOL> block = False , headers = None , <EOL> key_file = None , cert_file = None , <EOL> cert_reqs = '<STR_LIT>' , ca_certs = None ) : <EOL> super ( HTTPSConnectionPool , self ) . __init__ ( host , port , <EOL> strict , timeout , maxsize , <EOL> block , headers ) <EOL> self . key_file = key_file <EOL> self . cert_file = cert_file <EOL> self . cert_reqs = cert_reqs <EOL> self . ca_certs = ca_certs <EOL> def _new_conn ( self ) : <EOL> """<STR_LIT>""" <EOL> self . num_connections += <NUM_LIT:1> <EOL> log . info ( "<STR_LIT>" <EOL> % ( self . num_connections , self . host ) ) <EOL> if not ssl : <EOL> return HTTPSConnection ( host = self . host , port = self . port ) <EOL> connection = VerifiedHTTPSConnection ( host = self . host , port = self . port ) <EOL> connection . set_cert ( key_file = self . key_file , cert_file = self . cert_file , <EOL> cert_reqs = self . cert_reqs , ca_certs = self . ca_certs ) <EOL> return connection <EOL> def make_headers ( keep_alive = None , accept_encoding = None , user_agent = None , <EOL> basic_auth = None ) : <EOL> """<STR_LIT>""" <EOL> headers = { } <EOL> if accept_encoding : <EOL> if isinstance ( accept_encoding , str ) : <EOL> pass <EOL> elif isinstance ( accept_encoding , list ) : <EOL> accept_encoding = '<STR_LIT:U+002C>' . join ( accept_encoding ) <EOL> else : <EOL> accept_encoding = '<STR_LIT>' <EOL> headers [ '<STR_LIT>' ] = accept_encoding <EOL> if user_agent : <EOL> headers [ '<STR_LIT>' ] = user_agent <EOL> if keep_alive : <EOL> headers [ '<STR_LIT>' ] = '<STR_LIT>' <EOL> if basic_auth : <EOL> headers [ '<STR_LIT>' ] = '<STR_LIT>' + basic_auth . encode ( '<STR_LIT>' ) . strip ( ) <EOL> return headers <EOL> def get_host ( url ) : <EOL> """<STR_LIT>""" <EOL> port = None <EOL> scheme = '<STR_LIT:http>' <EOL> if '<STR_LIT>' in url : <EOL> scheme , url = url . split ( '<STR_LIT>' , <NUM_LIT:1> ) <EOL> if '<STR_LIT:/>' in url : <EOL> url , _path = url . split ( '<STR_LIT:/>' , <NUM_LIT:1> ) <EOL> if '<STR_LIT::>' in url : <EOL> url , port = url . split ( '<STR_LIT::>' , <NUM_LIT:1> ) <EOL> port = int ( port ) <EOL> return scheme , url , port <EOL> def connection_from_url ( url , ** kw ) : <EOL> """<STR_LIT>""" <EOL> scheme , host , port = get_host ( url ) <EOL> if scheme == '<STR_LIT>' : <EOL> return HTTPSConnectionPool ( host , port = port , ** kw ) <EOL> else : <EOL> return HTTPConnectionPool ( host , port = port , ** kw ) </s>
