<s> import urllib <EOL> import time <EOL> from twisted . internet . defer import Deferred <EOL> from twisted . internet import reactor , ssl <EOL> from twisted . web . client import HTTPClientFactory , _parse <EOL> import dateutil . parser <EOL> from . unicodeconverter import convertToUTF8 <EOL> from OpenSSL import SSL <EOL> import logging <EOL> LOGGER = logging . getLogger ( "<STR_LIT>" ) <EOL> class AllCipherSSLClientContextFactory ( ssl . ClientContextFactory ) : <EOL> """<STR_LIT>""" <EOL> def getContext ( self ) : <EOL> context = SSL . Context ( self . method ) <EOL> context . set_cipher_list ( "<STR_LIT>" ) <EOL> return context <EOL> class RequestQueuer ( object ) : <EOL> """<STR_LIT>""" <EOL> pending_reqs = { } <EOL> last_req = { } <EOL> active_reqs = { } <EOL> min_req_interval_per_hosts = { } <EOL> max_reqs_per_hosts_per_sec = { } <EOL> max_simul_reqs_per_hosts = { } <EOL> def __init__ ( self , max_simultaneous_requests = <NUM_LIT:50> , <EOL> max_requests_per_host_per_second = <NUM_LIT:1> , <EOL> max_simultaneous_requests_per_host = <NUM_LIT:5> ) : <EOL> """<STR_LIT>""" <EOL> if max_simultaneous_requests == <NUM_LIT:0> : <EOL> self . max_simul_reqs = <NUM_LIT> <EOL> else : <EOL> self . max_simul_reqs = int ( max_simultaneous_requests ) <EOL> self . max_reqs_per_host_per_sec = max_requests_per_host_per_second <EOL> if max_requests_per_host_per_second == <NUM_LIT:0> : <EOL> self . min_req_interval_per_host = <NUM_LIT:0> <EOL> else : <EOL> max_req_per_host_per_sec = float ( max_requests_per_host_per_second ) <EOL> self . min_req_interval_per_host = <NUM_LIT:1.0> / max_req_per_host_per_sec <EOL> if max_simultaneous_requests_per_host == <NUM_LIT:0> : <EOL> self . max_simul_reqs_per_host = self . max_simul_reqs <EOL> else : <EOL> self . max_simul_reqs_per_host = int ( max_simultaneous_requests_per_host ) <EOL> def getPending ( self ) : <EOL> """<STR_LIT>""" <EOL> return sum ( [ len ( x ) for x in self . pending_reqs . values ( ) ] ) <EOL> def getActive ( self ) : <EOL> """<STR_LIT>""" <EOL> return sum ( self . active_reqs . values ( ) ) <EOL> def getActiveRequestsByHost ( self ) : <EOL> """<STR_LIT>""" <EOL> return self . active_reqs <EOL> def getPendingRequestsByHost ( self ) : <EOL> """<STR_LIT>""" <EOL> reqs = [ ( x [ <NUM_LIT:0> ] , len ( x [ <NUM_LIT:1> ] ) ) for x in self . pending_reqs . items ( ) ] <EOL> return dict ( reqs ) <EOL> def setHostMaxRequestsPerSecond ( self , host , max_requests_per_second ) : <EOL> """<STR_LIT>""" <EOL> self . max_reqs_per_hosts_per_sec [ host ] = max_requests_per_second <EOL> if max_requests_per_second == <NUM_LIT:0> : <EOL> self . min_req_interval_per_hosts [ host ] = <NUM_LIT:0> <EOL> else : <EOL> min_req_interval = <NUM_LIT:1.0> / float ( max_requests_per_second ) <EOL> self . min_req_interval_per_hosts [ host ] = min_req_interval <EOL> def getHostMaxRequestsPerSecond ( self , host ) : <EOL> """<STR_LIT>""" <EOL> if host in self . max_reqs_per_hosts_per_sec : <EOL> return self . max_reqs_per_hosts_per_sec [ host ] <EOL> else : <EOL> return self . max_reqs_per_host_per_sec <EOL> def setHostMaxSimultaneousRequests ( self , host , max_simultaneous_requests ) : <EOL> """<STR_LIT>""" <EOL> if max_simultaneous_requests == <NUM_LIT:0> : <EOL> self . max_simul_reqs_per_hosts [ host ] = self . max_simul_reqs <EOL> else : <EOL> self . max_simul_reqs_per_hosts [ host ] = int ( max_simultaneous_requests ) <EOL> def getHostMaxSimultaneousRequests ( self , host ) : <EOL> """<STR_LIT>""" <EOL> if host in self . max_simul_reqs_per_hosts : <EOL> return self . max_simul_reqs_per_hosts [ host ] <EOL> else : <EOL> return self . max_simul_reqs_per_host <EOL> def getPage ( self , <EOL> url , <EOL> last_modified = None , <EOL> etag = None , <EOL> method = '<STR_LIT:GET>' , <EOL> postdata = None , <EOL> headers = None , <EOL> agent = "<STR_LIT>" , <EOL> timeout = <NUM_LIT> , <EOL> cookies = None , <EOL> follow_redirect = True , <EOL> prioritize = False <EOL> ) : <EOL> """<STR_LIT>""" <EOL> if headers is None : <EOL> headers = { } <EOL> if postdata is not None : <EOL> if isinstance ( postdata , dict ) : <EOL> for key in postdata : <EOL> postdata [ key ] = convertToUTF8 ( postdata [ key ] ) <EOL> postdata = urllib . urlencode ( postdata ) <EOL> else : <EOL> convertToUTF8 ( postdata ) <EOL> if method . lower ( ) == "<STR_LIT>" : <EOL> headers [ "<STR_LIT>" ] = "<STR_LIT>" <EOL> if last_modified is not None : <EOL> time_tuple = dateutil . parser . parse ( last_modified ) . timetuple ( ) <EOL> time_string = time . strftime ( "<STR_LIT>" , time_tuple ) <EOL> headers [ '<STR_LIT>' ] = time_string <EOL> if etag is not None : <EOL> headers [ "<STR_LIT>" ] = etag <EOL> req = { <EOL> "<STR_LIT:url>" : convertToUTF8 ( url ) , <EOL> "<STR_LIT>" : method , <EOL> "<STR_LIT>" : postdata , <EOL> "<STR_LIT>" : headers , <EOL> "<STR_LIT>" : agent , <EOL> "<STR_LIT>" : timeout , <EOL> "<STR_LIT>" : cookies , <EOL> "<STR_LIT>" : follow_redirect , <EOL> "<STR_LIT>" : Deferred ( ) <EOL> } <EOL> host = _parse ( req [ "<STR_LIT:url>" ] ) [ <NUM_LIT:1> ] <EOL> if host not in self . pending_reqs : <EOL> self . pending_reqs [ host ] = [ ] <EOL> if prioritize : <EOL> self . pending_reqs [ host ] . insert ( <NUM_LIT:0> , req ) <EOL> else : <EOL> self . pending_reqs [ host ] . append ( req ) <EOL> self . _checkActive ( ) <EOL> return req [ "<STR_LIT>" ] <EOL> def _hostRequestCheck ( self , host ) : <EOL> if host not in self . pending_reqs : <EOL> return False <EOL> if host in self . last_req : <EOL> if host in self . min_req_interval_per_hosts : <EOL> if time . time ( ) - self . last_req [ host ] < self . min_req_interval_per_hosts [ host ] : <EOL> return False <EOL> else : <EOL> if time . time ( ) - self . last_req [ host ] < self . min_req_interval_per_host : <EOL> return False <EOL> if host in self . active_reqs : <EOL> if host in self . max_simul_reqs_per_hosts : <EOL> if self . active_reqs [ host ] > self . max_simul_reqs_per_hosts [ host ] : <EOL> return False <EOL> else : <EOL> if self . active_reqs [ host ] > self . max_simul_reqs_per_host : <EOL> return False <EOL> return True <EOL> def _checkActive ( self ) : <EOL> while self . getActive ( ) < self . max_simul_reqs and self . getPending ( ) > <NUM_LIT:0> : <EOL> hosts = self . pending_reqs . keys ( ) <EOL> dispatched_requests = False <EOL> for host in hosts : <EOL> if len ( self . pending_reqs [ host ] ) == <NUM_LIT:0> : <EOL> del self . pending_reqs [ host ] <EOL> elif self . _hostRequestCheck ( host ) : <EOL> dispatched_requests = True <EOL> req = self . pending_reqs [ host ] . pop ( <NUM_LIT:0> ) <EOL> d = self . _getPage ( req ) <EOL> d . addCallback ( self . _requestComplete , req [ "<STR_LIT>" ] , host ) <EOL> d . addErrback ( self . _requestError , req [ "<STR_LIT>" ] , host ) <EOL> self . last_req [ host ] = time . time ( ) <EOL> self . active_reqs [ host ] = self . active_reqs . get ( host , <NUM_LIT:0> ) + <NUM_LIT:1> <EOL> if not dispatched_requests : <EOL> break <EOL> if self . getPending ( ) > <NUM_LIT:0> : <EOL> reactor . callLater ( <NUM_LIT> , self . _checkActive ) <EOL> def _requestComplete ( self , response , deferred , host ) : <EOL> self . active_reqs [ host ] -= <NUM_LIT:1> <EOL> self . _checkActive ( ) <EOL> deferred . callback ( response ) <EOL> return None <EOL> def _requestError ( self , error , deferred , host ) : <EOL> self . active_reqs [ host ] -= <NUM_LIT:1> <EOL> self . _checkActive ( ) <EOL> deferred . errback ( error ) <EOL> return None <EOL> def _getPage ( self , req ) : <EOL> scheme , host , port = _parse ( req [ '<STR_LIT:url>' ] ) [ <NUM_LIT:0> : <NUM_LIT:3> ] <EOL> factory = HTTPClientFactory ( <EOL> req [ '<STR_LIT:url>' ] , <EOL> method = req [ '<STR_LIT>' ] , <EOL> postdata = req [ '<STR_LIT>' ] , <EOL> headers = req [ '<STR_LIT>' ] , <EOL> agent = req [ '<STR_LIT>' ] , <EOL> timeout = req [ '<STR_LIT>' ] , <EOL> cookies = req [ '<STR_LIT>' ] , <EOL> followRedirect = req [ '<STR_LIT>' ] <EOL> ) <EOL> if scheme == '<STR_LIT>' : <EOL> reactor . connectSSL ( <EOL> host , <EOL> port , <EOL> factory , <EOL> AllCipherSSLClientContextFactory ( ) , <EOL> timeout = req [ '<STR_LIT>' ] <EOL> ) <EOL> else : <EOL> reactor . connectTCP ( host , port , factory , timeout = req [ '<STR_LIT>' ] ) <EOL> factory . deferred . addCallback ( self . _getPageComplete , factory ) <EOL> factory . deferred . addErrback ( self . _getPageError , factory ) <EOL> return factory . deferred <EOL> def _getPageComplete ( self , response , factory ) : <EOL> return { <EOL> "<STR_LIT>" : response , <EOL> "<STR_LIT>" : factory . response_headers , <EOL> "<STR_LIT:status>" : int ( factory . status ) , <EOL> "<STR_LIT:message>" : factory . message <EOL> } <EOL> def _getPageError ( self , error , factory ) : <EOL> if hasattr ( factory , "<STR_LIT>" ) and factory . response_headers is not None : <EOL> error . value . headers = factory . response_headers <EOL> return error </s>
