<s> """<STR_LIT>""" <EOL> import os , types <EOL> from urlparse import urlunparse <EOL> from urllib import splithost , splittype <EOL> import zlib <EOL> from zope . interface import implements <EOL> from twisted . python import log <EOL> from twisted . python . failure import Failure <EOL> from twisted . web import http <EOL> from twisted . internet import defer , protocol , task , reactor <EOL> from twisted . internet . interfaces import IProtocol <EOL> from twisted . internet . endpoints import TCP4ClientEndpoint , SSL4ClientEndpoint <EOL> from twisted . python import failure <EOL> from twisted . python . util import InsensitiveDict <EOL> from twisted . python . components import proxyForInterface <EOL> from twisted . web import error <EOL> from twisted . web . iweb import UNKNOWN_LENGTH , IBodyProducer , IResponse <EOL> from twisted . web . http_headers import Headers <EOL> from twisted . python . compat import set <EOL> class PartialDownloadError ( error . Error ) : <EOL> """<STR_LIT>""" <EOL> class HTTPPageGetter ( http . HTTPClient ) : <EOL> """<STR_LIT>""" <EOL> quietLoss = <NUM_LIT:0> <EOL> followRedirect = True <EOL> failed = <NUM_LIT:0> <EOL> _completelyDone = True <EOL> _specialHeaders = set ( ( '<STR_LIT:host>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ) ) <EOL> def connectionMade ( self ) : <EOL> method = getattr ( self . factory , '<STR_LIT>' , '<STR_LIT:GET>' ) <EOL> self . sendCommand ( method , self . factory . path ) <EOL> if self . factory . scheme == '<STR_LIT:http>' and self . factory . port != <NUM_LIT> : <EOL> host = '<STR_LIT>' % ( self . factory . host , self . factory . port ) <EOL> elif self . factory . scheme == '<STR_LIT>' and self . factory . port != <NUM_LIT> : <EOL> host = '<STR_LIT>' % ( self . factory . host , self . factory . port ) <EOL> else : <EOL> host = self . factory . host <EOL> self . sendHeader ( '<STR_LIT>' , self . factory . headers . get ( "<STR_LIT:host>" , host ) ) <EOL> self . sendHeader ( '<STR_LIT>' , self . factory . agent ) <EOL> data = getattr ( self . factory , '<STR_LIT>' , None ) <EOL> if data is not None : <EOL> self . sendHeader ( "<STR_LIT>" , str ( len ( data ) ) ) <EOL> cookieData = [ ] <EOL> for ( key , value ) in self . factory . headers . items ( ) : <EOL> if key . lower ( ) not in self . _specialHeaders : <EOL> self . sendHeader ( key , value ) <EOL> if key . lower ( ) == '<STR_LIT>' : <EOL> cookieData . append ( value ) <EOL> for cookie , cookval in self . factory . cookies . items ( ) : <EOL> cookieData . append ( '<STR_LIT>' % ( cookie , cookval ) ) <EOL> if cookieData : <EOL> self . sendHeader ( '<STR_LIT>' , '<STR_LIT>' . join ( cookieData ) ) <EOL> self . endHeaders ( ) <EOL> self . headers = { } <EOL> if data is not None : <EOL> self . transport . write ( data ) <EOL> def handleHeader ( self , key , value ) : <EOL> """<STR_LIT>""" <EOL> key = key . lower ( ) <EOL> l = self . headers . setdefault ( key , [ ] ) <EOL> l . append ( value ) <EOL> def handleStatus ( self , version , status , message ) : <EOL> self . version , self . status , self . message = version , status , message <EOL> self . factory . gotStatus ( version , status , message ) <EOL> def handleEndHeaders ( self ) : <EOL> self . factory . gotHeaders ( self . headers ) <EOL> m = getattr ( self , '<STR_LIT>' + self . status , self . handleStatusDefault ) <EOL> m ( ) <EOL> def handleStatus_200 ( self ) : <EOL> pass <EOL> handleStatus_201 = lambda self : self . handleStatus_200 ( ) <EOL> handleStatus_202 = lambda self : self . handleStatus_200 ( ) <EOL> def handleStatusDefault ( self ) : <EOL> self . failed = <NUM_LIT:1> <EOL> def handleStatus_301 ( self ) : <EOL> l = self . headers . get ( '<STR_LIT:location>' ) <EOL> if not l : <EOL> self . handleStatusDefault ( ) <EOL> return <EOL> url = l [ <NUM_LIT:0> ] <EOL> if self . followRedirect : <EOL> scheme , host , port , path = _parse ( url , defaultPort = self . transport . getPeer ( ) . port ) <EOL> self . factory . _redirectCount += <NUM_LIT:1> <EOL> if self . factory . _redirectCount >= self . factory . redirectLimit : <EOL> err = error . InfiniteRedirection ( <EOL> self . status , <EOL> '<STR_LIT>' , <EOL> location = url ) <EOL> self . factory . noPage ( failure . Failure ( err ) ) <EOL> self . quietLoss = True <EOL> self . transport . loseConnection ( ) <EOL> return <EOL> self . _completelyDone = False <EOL> self . factory . setURL ( url ) <EOL> if self . factory . scheme == '<STR_LIT>' : <EOL> from twisted . internet import ssl <EOL> contextFactory = ssl . ClientContextFactory ( ) <EOL> reactor . connectSSL ( self . factory . host , self . factory . port , <EOL> self . factory , contextFactory ) <EOL> else : <EOL> reactor . connectTCP ( self . factory . host , self . factory . port , <EOL> self . factory ) <EOL> else : <EOL> self . handleStatusDefault ( ) <EOL> self . factory . noPage ( <EOL> failure . Failure ( <EOL> error . PageRedirect ( <EOL> self . status , self . message , location = url ) ) ) <EOL> self . quietLoss = True <EOL> self . transport . loseConnection ( ) <EOL> def handleStatus_302 ( self ) : <EOL> if self . afterFoundGet : <EOL> self . handleStatus_303 ( ) <EOL> else : <EOL> self . handleStatus_301 ( ) <EOL> def handleStatus_303 ( self ) : <EOL> self . factory . method = '<STR_LIT:GET>' <EOL> self . handleStatus_301 ( ) <EOL> def connectionLost ( self , reason ) : <EOL> """<STR_LIT>""" <EOL> if not self . quietLoss : <EOL> http . HTTPClient . connectionLost ( self , reason ) <EOL> self . factory . noPage ( reason ) <EOL> if self . _completelyDone : <EOL> self . factory . _disconnectedDeferred . callback ( None ) <EOL> def handleResponse ( self , response ) : <EOL> if self . quietLoss : <EOL> return <EOL> if self . failed : <EOL> self . factory . noPage ( <EOL> failure . Failure ( <EOL> error . Error ( <EOL> self . status , self . message , response ) ) ) <EOL> if self . factory . method == '<STR_LIT>' : <EOL> self . factory . page ( '<STR_LIT>' ) <EOL> elif self . length != None and self . length != <NUM_LIT:0> : <EOL> self . factory . noPage ( failure . Failure ( <EOL> PartialDownloadError ( self . status , self . message , response ) ) ) <EOL> else : <EOL> self . factory . page ( response ) <EOL> self . transport . loseConnection ( ) <EOL> def timeout ( self ) : <EOL> self . quietLoss = True <EOL> self . transport . loseConnection ( ) <EOL> self . factory . noPage ( defer . TimeoutError ( "<STR_LIT>" % ( self . factory . url , self . factory . timeout ) ) ) <EOL> class HTTPPageDownloader ( HTTPPageGetter ) : <EOL> transmittingPage = <NUM_LIT:0> <EOL> def handleStatus_200 ( self , partialContent = <NUM_LIT:0> ) : <EOL> HTTPPageGetter . handleStatus_200 ( self ) <EOL> self . transmittingPage = <NUM_LIT:1> <EOL> self . factory . pageStart ( partialContent ) <EOL> def handleStatus_206 ( self ) : <EOL> self . handleStatus_200 ( partialContent = <NUM_LIT:1> ) <EOL> def handleResponsePart ( self , data ) : <EOL> if self . transmittingPage : <EOL> self . factory . pagePart ( data ) <EOL> def handleResponseEnd ( self ) : <EOL> if self . length : <EOL> self . transmittingPage = <NUM_LIT:0> <EOL> self . factory . noPage ( <EOL> failure . Failure ( <EOL> PartialDownloadError ( self . status ) ) ) <EOL> if self . transmittingPage : <EOL> self . factory . pageEnd ( ) <EOL> self . transmittingPage = <NUM_LIT:0> <EOL> if self . failed : <EOL> self . factory . noPage ( <EOL> failure . Failure ( <EOL> error . Error ( <EOL> self . status , self . message , None ) ) ) <EOL> self . transport . loseConnection ( ) <EOL> class HTTPClientFactory ( protocol . ClientFactory ) : <EOL> """<STR_LIT>""" <EOL> protocol = HTTPPageGetter <EOL> url = None <EOL> scheme = None <EOL> host = '<STR_LIT>' <EOL> port = None <EOL> path = None <EOL> def __init__ ( self , url , method = '<STR_LIT:GET>' , postdata = None , headers = None , <EOL> agent = "<STR_LIT>" , timeout = <NUM_LIT:0> , cookies = None , <EOL> followRedirect = True , redirectLimit = <NUM_LIT:20> , <EOL> afterFoundGet = False ) : <EOL> self . followRedirect = followRedirect <EOL> self . redirectLimit = redirectLimit <EOL> self . _redirectCount = <NUM_LIT:0> <EOL> self . timeout = timeout <EOL> self . agent = agent <EOL> self . afterFoundGet = afterFoundGet <EOL> if cookies is None : <EOL> cookies = { } <EOL> self . cookies = cookies <EOL> if headers is not None : <EOL> self . headers = InsensitiveDict ( headers ) <EOL> else : <EOL> self . headers = InsensitiveDict ( ) <EOL> if postdata is not None : <EOL> self . headers . setdefault ( '<STR_LIT>' , len ( postdata ) ) <EOL> self . headers . setdefault ( "<STR_LIT>" , "<STR_LIT>" ) <EOL> self . postdata = postdata <EOL> self . method = method <EOL> self . setURL ( url ) <EOL> self . waiting = <NUM_LIT:1> <EOL> self . _disconnectedDeferred = defer . Deferred ( ) <EOL> self . deferred = defer . Deferred ( ) <EOL> self . deferred . addBoth ( self . _waitForDisconnect ) <EOL> self . response_headers = None <EOL> def _waitForDisconnect ( self , passthrough ) : <EOL> """<STR_LIT>""" <EOL> self . _disconnectedDeferred . addCallback ( lambda ignored : passthrough ) <EOL> return self . _disconnectedDeferred <EOL> def __repr__ ( self ) : <EOL> return "<STR_LIT>" % ( self . __class__ . __name__ , self . url ) <EOL> def setURL ( self , url ) : <EOL> self . url = url <EOL> scheme , host , port , path = _parse ( url ) <EOL> if scheme and host : <EOL> self . scheme = scheme <EOL> self . host = host <EOL> self . port = port <EOL> self . path = path <EOL> def buildProtocol ( self , addr ) : <EOL> p = protocol . ClientFactory . buildProtocol ( self , addr ) <EOL> p . followRedirect = self . followRedirect <EOL> p . afterFoundGet = self . afterFoundGet <EOL> if self . timeout : <EOL> timeoutCall = reactor . callLater ( self . timeout , p . timeout ) <EOL> self . deferred . addBoth ( self . _cancelTimeout , timeoutCall ) <EOL> return p <EOL> def _cancelTimeout ( self , result , timeoutCall ) : <EOL> if timeoutCall . active ( ) : <EOL> timeoutCall . cancel ( ) <EOL> return result <EOL> def gotHeaders ( self , headers ) : <EOL> self . response_headers = headers <EOL> if '<STR_LIT>' in headers : <EOL> for cookie in headers [ '<STR_LIT>' ] : <EOL> cookparts = cookie . split ( '<STR_LIT:;>' ) <EOL> cook = cookparts [ <NUM_LIT:0> ] <EOL> cook . lstrip ( ) <EOL> k , v = cook . split ( '<STR_LIT:=>' , <NUM_LIT:1> ) <EOL> self . cookies [ k . lstrip ( ) ] = v . lstrip ( ) <EOL> def gotStatus ( self , version , status , message ) : <EOL> self . version , self . status , self . message = version , status , message <EOL> def page ( self , page ) : <EOL> if self . waiting : <EOL> self . waiting = <NUM_LIT:0> <EOL> self . deferred . callback ( page ) <EOL> def noPage ( self , reason ) : <EOL> if self . waiting : <EOL> self . waiting = <NUM_LIT:0> <EOL> self . deferred . errback ( reason ) <EOL> def clientConnectionFailed ( self , _ , reason ) : <EOL> """<STR_LIT>""" <EOL> if self . waiting : <EOL> self . waiting = <NUM_LIT:0> <EOL> self . _disconnectedDeferred . callback ( None ) <EOL> self . deferred . errback ( reason ) <EOL> class HTTPDownloader ( HTTPClientFactory ) : <EOL> """<STR_LIT>""" <EOL> protocol = HTTPPageDownloader <EOL> value = None <EOL> def __init__ ( self , url , fileOrName , <EOL> method = '<STR_LIT:GET>' , postdata = None , headers = None , <EOL> agent = "<STR_LIT>" , supportPartial = <NUM_LIT:0> , <EOL> timeout = <NUM_LIT:0> , cookies = None , followRedirect = <NUM_LIT:1> , <EOL> redirectLimit = <NUM_LIT:20> , afterFoundGet = False ) : <EOL> self . requestedPartial = <NUM_LIT:0> <EOL> if isinstance ( fileOrName , types . StringTypes ) : <EOL> self . fileName = fileOrName <EOL> self . file = None <EOL> if supportPartial and os . path . exists ( self . fileName ) : <EOL> fileLength = os . path . getsize ( self . fileName ) <EOL> if fileLength : <EOL> self . requestedPartial = fileLength <EOL> if headers == None : <EOL> headers = { } <EOL> headers [ "<STR_LIT>" ] = "<STR_LIT>" % fileLength <EOL> else : <EOL> self . file = fileOrName <EOL> HTTPClientFactory . __init__ ( <EOL> self , url , method = method , postdata = postdata , headers = headers , <EOL> agent = agent , timeout = timeout , cookies = cookies , <EOL> followRedirect = followRedirect , redirectLimit = redirectLimit , <EOL> afterFoundGet = afterFoundGet ) <EOL> def gotHeaders ( self , headers ) : <EOL> HTTPClientFactory . gotHeaders ( self , headers ) <EOL> if self . requestedPartial : <EOL> contentRange = headers . get ( "<STR_LIT>" , None ) <EOL> if not contentRange : <EOL> self . requestedPartial = <NUM_LIT:0> <EOL> return <EOL> start , end , realLength = http . parseContentRange ( contentRange [ <NUM_LIT:0> ] ) <EOL> if start != self . requestedPartial : <EOL> self . requestedPartial = <NUM_LIT:0> <EOL> def openFile ( self , partialContent ) : <EOL> if partialContent : <EOL> file = open ( self . fileName , '<STR_LIT>' ) <EOL> file . seek ( <NUM_LIT:0> , <NUM_LIT:2> ) <EOL> else : <EOL> file = open ( self . fileName , '<STR_LIT:wb>' ) <EOL> return file <EOL> def pageStart ( self , partialContent ) : <EOL> """<STR_LIT>""" <EOL> if partialContent and not self . requestedPartial : <EOL> raise ValueError , "<STR_LIT>" <EOL> if self . waiting : <EOL> try : <EOL> if not self . file : <EOL> self . file = self . openFile ( partialContent ) <EOL> except IOError : <EOL> self . deferred . errback ( failure . Failure ( ) ) <EOL> def pagePart ( self , data ) : <EOL> if not self . file : <EOL> return <EOL> try : <EOL> self . file . write ( data ) <EOL> except IOError : <EOL> self . file = None <EOL> self . deferred . errback ( failure . Failure ( ) ) <EOL> def noPage ( self , reason ) : <EOL> """<STR_LIT>""" <EOL> if self . waiting : <EOL> self . waiting = <NUM_LIT:0> <EOL> if self . file : <EOL> try : <EOL> self . file . close ( ) <EOL> except : <EOL> log . err ( None , "<STR_LIT>" ) <EOL> self . deferred . errback ( reason ) <EOL> def pageEnd ( self ) : <EOL> self . waiting = <NUM_LIT:0> <EOL> if not self . file : <EOL> return <EOL> try : <EOL> self . file . close ( ) <EOL> except IOError : <EOL> self . deferred . errback ( failure . Failure ( ) ) <EOL> return <EOL> self . deferred . callback ( self . value ) <EOL> class _URL ( tuple ) : <EOL> """<STR_LIT>""" <EOL> def __new__ ( self , scheme , host , port , path ) : <EOL> return tuple . __new__ ( _URL , ( scheme , host , port , path ) ) <EOL> def __init__ ( self , scheme , host , port , path ) : <EOL> self . scheme = scheme <EOL> self . host = host <EOL> self . port = port <EOL> self . path = path <EOL> def _parse ( url , defaultPort = None ) : <EOL> """<STR_LIT>""" <EOL> url = url . strip ( ) <EOL> parsed = http . urlparse ( url ) <EOL> scheme = parsed [ <NUM_LIT:0> ] <EOL> path = urlunparse ( ( '<STR_LIT>' , '<STR_LIT>' ) + parsed [ <NUM_LIT:2> : ] ) <EOL> if defaultPort is None : <EOL> if scheme == '<STR_LIT>' : <EOL> defaultPort = <NUM_LIT> <EOL> else : <EOL> defaultPort = <NUM_LIT> <EOL> host , port = parsed [ <NUM_LIT:1> ] , defaultPort <EOL> if '<STR_LIT::>' in host : <EOL> host , port = host . split ( '<STR_LIT::>' ) <EOL> try : <EOL> port = int ( port ) <EOL> except ValueError : <EOL> port = defaultPort <EOL> if path == '<STR_LIT>' : <EOL> path = '<STR_LIT:/>' <EOL> return _URL ( scheme , host , port , path ) <EOL> def _makeGetterFactory ( url , factoryFactory , contextFactory = None , <EOL> * args , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> scheme , host , port , path = _parse ( url ) <EOL> factory = factoryFactory ( url , * args , ** kwargs ) <EOL> if scheme == '<STR_LIT>' : <EOL> from twisted . internet import ssl <EOL> if contextFactory is None : <EOL> contextFactory = ssl . ClientContextFactory ( ) <EOL> reactor . connectSSL ( host , port , factory , contextFactory ) <EOL> else : <EOL> reactor . connectTCP ( host , port , factory ) <EOL> return factory <EOL> def getPage ( url , contextFactory = None , * args , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> return _makeGetterFactory ( <EOL> url , <EOL> HTTPClientFactory , <EOL> contextFactory = contextFactory , <EOL> * args , ** kwargs ) . deferred <EOL> def downloadPage ( url , file , contextFactory = None , * args , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> factoryFactory = lambda url , * a , ** kw : HTTPDownloader ( url , file , * a , ** kw ) <EOL> return _makeGetterFactory ( <EOL> url , <EOL> factoryFactory , <EOL> contextFactory = contextFactory , <EOL> * args , ** kwargs ) . deferred <EOL> from twisted . web . error import SchemeNotSupported <EOL> from twisted . web . _newclient import Request , Response , HTTP11ClientProtocol <EOL> from twisted . web . _newclient import ResponseDone , ResponseFailed <EOL> from twisted . web . _newclient import RequestNotSent , RequestTransmissionFailed <EOL> from twisted . web . _newclient import ResponseNeverReceived <EOL> try : <EOL> from twisted . internet . ssl import ClientContextFactory <EOL> except ImportError : <EOL> class WebClientContextFactory ( object ) : <EOL> """<STR_LIT>""" <EOL> def getContext ( self , hostname , port ) : <EOL> raise NotImplementedError ( "<STR_LIT>" ) <EOL> else : <EOL> class WebClientContextFactory ( ClientContextFactory ) : <EOL> """<STR_LIT>""" <EOL> def getContext ( self , hostname , port ) : <EOL> return ClientContextFactory . getContext ( self ) <EOL> class _WebToNormalContextFactory ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , webContext , hostname , port ) : <EOL> self . _webContext = webContext <EOL> self . _hostname = hostname <EOL> self . _port = port <EOL> def getContext ( self ) : <EOL> """<STR_LIT>""" <EOL> return self . _webContext . getContext ( self . _hostname , self . _port ) <EOL> class FileBodyProducer ( object ) : <EOL> """<STR_LIT>""" <EOL> implements ( IBodyProducer ) <EOL> _SEEK_SET = getattr ( os , '<STR_LIT>' , <NUM_LIT:0> ) <EOL> _SEEK_END = getattr ( os , '<STR_LIT>' , <NUM_LIT:2> ) <EOL> def __init__ ( self , inputFile , cooperator = task , readSize = <NUM_LIT:2> ** <NUM_LIT:16> ) : <EOL> self . _inputFile = inputFile <EOL> self . _cooperate = cooperator . cooperate <EOL> self . _readSize = readSize <EOL> self . length = self . _determineLength ( inputFile ) <EOL> def _determineLength ( self , fObj ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> seek = fObj . seek <EOL> tell = fObj . tell <EOL> except AttributeError : <EOL> return UNKNOWN_LENGTH <EOL> originalPosition = tell ( ) <EOL> seek ( <NUM_LIT:0> , self . _SEEK_END ) <EOL> end = tell ( ) <EOL> seek ( originalPosition , self . _SEEK_SET ) <EOL> return end - originalPosition <EOL> def stopProducing ( self ) : <EOL> """<STR_LIT>""" <EOL> self . _inputFile . close ( ) <EOL> self . _task . stop ( ) <EOL> def startProducing ( self , consumer ) : <EOL> """<STR_LIT>""" <EOL> self . _task = self . _cooperate ( self . _writeloop ( consumer ) ) <EOL> d = self . _task . whenDone ( ) <EOL> def maybeStopped ( reason ) : <EOL> reason . trap ( task . TaskStopped ) <EOL> return defer . Deferred ( ) <EOL> d . addCallbacks ( lambda ignored : None , maybeStopped ) <EOL> return d <EOL> def _writeloop ( self , consumer ) : <EOL> """<STR_LIT>""" <EOL> while True : <EOL> bytes = self . _inputFile . read ( self . _readSize ) <EOL> if not bytes : <EOL> self . _inputFile . close ( ) <EOL> break <EOL> consumer . write ( bytes ) <EOL> yield None <EOL> def pauseProducing ( self ) : <EOL> """<STR_LIT>""" <EOL> self . _task . pause ( ) <EOL> def resumeProducing ( self ) : <EOL> """<STR_LIT>""" <EOL> self . _task . resume ( ) <EOL> class _HTTP11ClientFactory ( protocol . Factory ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , quiescentCallback ) : <EOL> self . _quiescentCallback = quiescentCallback <EOL> def buildProtocol ( self , addr ) : <EOL> return HTTP11ClientProtocol ( self . _quiescentCallback ) <EOL> class _RetryingHTTP11ClientProtocol ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , clientProtocol , newConnection ) : <EOL> self . _clientProtocol = clientProtocol <EOL> self . _newConnection = newConnection <EOL> def _shouldRetry ( self , method , exception , bodyProducer ) : <EOL> """<STR_LIT>""" <EOL> if method not in ( "<STR_LIT:GET>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" ) : <EOL> return False <EOL> if not isinstance ( exception , ( RequestNotSent , RequestTransmissionFailed , <EOL> ResponseNeverReceived ) ) : <EOL> return False <EOL> if bodyProducer is not None : <EOL> return False <EOL> return True <EOL> def request ( self , request ) : <EOL> """<STR_LIT>""" <EOL> d = self . _clientProtocol . request ( request ) <EOL> def failed ( reason ) : <EOL> if self . _shouldRetry ( request . method , reason . value , <EOL> request . bodyProducer ) : <EOL> return self . _newConnection ( ) . addCallback ( <EOL> lambda connection : connection . request ( request ) ) <EOL> else : <EOL> return reason <EOL> d . addErrback ( failed ) <EOL> return d <EOL> class HTTPConnectionPool ( object ) : <EOL> """<STR_LIT>""" <EOL> _factory = _HTTP11ClientFactory <EOL> maxPersistentPerHost = <NUM_LIT:2> <EOL> cachedConnectionTimeout = <NUM_LIT> <EOL> retryAutomatically = True <EOL> def __init__ ( self , reactor , persistent = True ) : <EOL> self . _reactor = reactor <EOL> self . persistent = persistent <EOL> self . _connections = { } <EOL> self . _timeouts = { } <EOL> def getConnection ( self , key , endpoint ) : <EOL> """<STR_LIT>""" <EOL> connections = self . _connections . get ( key ) <EOL> while connections : <EOL> connection = connections . pop ( <NUM_LIT:0> ) <EOL> self . _timeouts [ connection ] . cancel ( ) <EOL> del self . _timeouts [ connection ] <EOL> if connection . state == "<STR_LIT>" : <EOL> if self . retryAutomatically : <EOL> newConnection = lambda : self . _newConnection ( key , endpoint ) <EOL> connection = _RetryingHTTP11ClientProtocol ( <EOL> connection , newConnection ) <EOL> return defer . succeed ( connection ) <EOL> return self . _newConnection ( key , endpoint ) <EOL> def _newConnection ( self , key , endpoint ) : <EOL> """<STR_LIT>""" <EOL> def quiescentCallback ( protocol ) : <EOL> self . _putConnection ( key , protocol ) <EOL> factory = self . _factory ( quiescentCallback ) <EOL> return endpoint . connect ( factory ) <EOL> def _removeConnection ( self , key , connection ) : <EOL> """<STR_LIT>""" <EOL> connection . transport . loseConnection ( ) <EOL> self . _connections [ key ] . remove ( connection ) <EOL> del self . _timeouts [ connection ] <EOL> def _putConnection ( self , key , connection ) : <EOL> """<STR_LIT>""" <EOL> if connection . state != "<STR_LIT>" : <EOL> try : <EOL> raise RuntimeError ( <EOL> "<STR_LIT>" ) <EOL> except : <EOL> log . err ( ) <EOL> return <EOL> connections = self . _connections . setdefault ( key , [ ] ) <EOL> if len ( connections ) == self . maxPersistentPerHost : <EOL> dropped = connections . pop ( <NUM_LIT:0> ) <EOL> dropped . transport . loseConnection ( ) <EOL> self . _timeouts [ dropped ] . cancel ( ) <EOL> del self . _timeouts [ dropped ] <EOL> connections . append ( connection ) <EOL> cid = self . _reactor . callLater ( self . cachedConnectionTimeout , <EOL> self . _removeConnection , <EOL> key , connection ) <EOL> self . _timeouts [ connection ] = cid <EOL> def closeCachedConnections ( self ) : <EOL> """<STR_LIT>""" <EOL> results = [ ] <EOL> for protocols in self . _connections . itervalues ( ) : <EOL> for p in protocols : <EOL> results . append ( p . abort ( ) ) <EOL> self . _connections = { } <EOL> for dc in self . _timeouts . values ( ) : <EOL> dc . cancel ( ) <EOL> self . _timeouts = { } <EOL> return defer . gatherResults ( results ) . addCallback ( lambda ign : None ) <EOL> class _AgentBase ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , reactor , pool ) : <EOL> if pool is None : <EOL> pool = HTTPConnectionPool ( reactor , False ) <EOL> self . _reactor = reactor <EOL> self . _pool = pool <EOL> def _computeHostValue ( self , scheme , host , port ) : <EOL> """<STR_LIT>""" <EOL> if ( scheme , port ) in ( ( '<STR_LIT:http>' , <NUM_LIT> ) , ( '<STR_LIT>' , <NUM_LIT> ) ) : <EOL> return host <EOL> return '<STR_LIT>' % ( host , port ) <EOL> def _requestWithEndpoint ( self , key , endpoint , method , parsedURI , <EOL> headers , bodyProducer , requestPath ) : <EOL> """<STR_LIT>""" <EOL> if headers is None : <EOL> headers = Headers ( ) <EOL> if not headers . hasHeader ( '<STR_LIT:host>' ) : <EOL> headers = headers . copy ( ) <EOL> headers . addRawHeader ( <EOL> '<STR_LIT:host>' , self . _computeHostValue ( parsedURI . scheme , parsedURI . host , <EOL> parsedURI . port ) ) <EOL> d = self . _pool . getConnection ( key , endpoint ) <EOL> def cbConnected ( proto ) : <EOL> return proto . request ( <EOL> Request ( method , requestPath , headers , bodyProducer , <EOL> persistent = self . _pool . persistent ) ) <EOL> d . addCallback ( cbConnected ) <EOL> return d <EOL> class Agent ( _AgentBase ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , reactor , contextFactory = WebClientContextFactory ( ) , <EOL> connectTimeout = None , bindAddress = None , <EOL> pool = None ) : <EOL> _AgentBase . __init__ ( self , reactor , pool ) <EOL> self . _contextFactory = contextFactory <EOL> self . _connectTimeout = connectTimeout <EOL> self . _bindAddress = bindAddress <EOL> def _wrapContextFactory ( self , host , port ) : <EOL> """<STR_LIT>""" <EOL> return _WebToNormalContextFactory ( self . _contextFactory , host , port ) <EOL> def _getEndpoint ( self , scheme , host , port ) : <EOL> """<STR_LIT>""" <EOL> kwargs = { } <EOL> if self . _connectTimeout is not None : <EOL> kwargs [ '<STR_LIT>' ] = self . _connectTimeout <EOL> kwargs [ '<STR_LIT>' ] = self . _bindAddress <EOL> if scheme == '<STR_LIT:http>' : <EOL> return TCP4ClientEndpoint ( self . _reactor , host , port , ** kwargs ) <EOL> elif scheme == '<STR_LIT>' : <EOL> return SSL4ClientEndpoint ( self . _reactor , host , port , <EOL> self . _wrapContextFactory ( host , port ) , <EOL> ** kwargs ) <EOL> else : <EOL> raise SchemeNotSupported ( "<STR_LIT>" % ( scheme , ) ) <EOL> def request ( self , method , uri , headers = None , bodyProducer = None ) : <EOL> """<STR_LIT>""" <EOL> parsedURI = _parse ( uri ) <EOL> try : <EOL> endpoint = self . _getEndpoint ( parsedURI . scheme , parsedURI . host , <EOL> parsedURI . port ) <EOL> except SchemeNotSupported : <EOL> return defer . fail ( Failure ( ) ) <EOL> key = ( parsedURI . scheme , parsedURI . host , parsedURI . port ) <EOL> return self . _requestWithEndpoint ( key , endpoint , method , parsedURI , <EOL> headers , bodyProducer , parsedURI . path ) <EOL> class ProxyAgent ( _AgentBase ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , endpoint , reactor = None , pool = None ) : <EOL> if reactor is None : <EOL> from twisted . internet import reactor <EOL> _AgentBase . __init__ ( self , reactor , pool ) <EOL> self . _proxyEndpoint = endpoint <EOL> def request ( self , method , uri , headers = None , bodyProducer = None ) : <EOL> """<STR_LIT>""" <EOL> key = ( "<STR_LIT>" , self . _proxyEndpoint ) <EOL> return self . _requestWithEndpoint ( key , self . _proxyEndpoint , method , <EOL> _parse ( uri ) , headers , bodyProducer , <EOL> uri ) <EOL> class _FakeUrllib2Request ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , uri ) : <EOL> self . uri = uri <EOL> self . headers = Headers ( ) <EOL> self . type , rest = splittype ( self . uri ) <EOL> self . host , rest = splithost ( rest ) <EOL> def has_header ( self , header ) : <EOL> return self . headers . hasHeader ( header ) <EOL> def add_unredirected_header ( self , name , value ) : <EOL> self . headers . addRawHeader ( name , value ) <EOL> def get_full_url ( self ) : <EOL> return self . uri <EOL> def get_header ( self , name , default = None ) : <EOL> headers = self . headers . getRawHeaders ( name , default ) <EOL> if headers is not None : <EOL> return headers [ <NUM_LIT:0> ] <EOL> return None <EOL> def get_host ( self ) : <EOL> return self . host <EOL> def get_type ( self ) : <EOL> return self . type <EOL> def is_unverifiable ( self ) : <EOL> return False <EOL> class _FakeUrllib2Response ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , response ) : <EOL> self . response = response <EOL> def info ( self ) : <EOL> class _Meta ( object ) : <EOL> def getheaders ( zelf , name ) : <EOL> return self . response . headers . getRawHeaders ( name , [ ] ) <EOL> return _Meta ( ) <EOL> class CookieAgent ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , agent , cookieJar ) : <EOL> self . _agent = agent <EOL> self . cookieJar = cookieJar <EOL> def request ( self , method , uri , headers = None , bodyProducer = None ) : <EOL> """<STR_LIT>""" <EOL> if headers is None : <EOL> headers = Headers ( ) <EOL> lastRequest = _FakeUrllib2Request ( uri ) <EOL> if not headers . hasHeader ( '<STR_LIT>' ) : <EOL> self . cookieJar . add_cookie_header ( lastRequest ) <EOL> cookieHeader = lastRequest . get_header ( '<STR_LIT>' , None ) <EOL> if cookieHeader is not None : <EOL> headers = headers . copy ( ) <EOL> headers . addRawHeader ( '<STR_LIT>' , cookieHeader ) <EOL> d = self . _agent . request ( method , uri , headers , bodyProducer ) <EOL> d . addCallback ( self . _extractCookies , lastRequest ) <EOL> return d <EOL> def _extractCookies ( self , response , request ) : <EOL> """<STR_LIT>""" <EOL> resp = _FakeUrllib2Response ( response ) <EOL> self . cookieJar . extract_cookies ( resp , request ) <EOL> return response <EOL> class GzipDecoder ( proxyForInterface ( IResponse ) ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , response ) : <EOL> self . original = response <EOL> self . length = UNKNOWN_LENGTH <EOL> def deliverBody ( self , protocol ) : <EOL> """<STR_LIT>""" <EOL> self . original . deliverBody ( _GzipProtocol ( protocol , self . original ) ) <EOL> class _GzipProtocol ( proxyForInterface ( IProtocol ) ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , protocol , response ) : <EOL> self . original = protocol <EOL> self . _response = response <EOL> self . _zlibDecompress = zlib . decompressobj ( <NUM_LIT:16> + zlib . MAX_WBITS ) <EOL> def dataReceived ( self , data ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> rawData = self . _zlibDecompress . decompress ( data ) <EOL> except zlib . error : <EOL> raise ResponseFailed ( [ failure . Failure ( ) ] , self . _response ) <EOL> if rawData : <EOL> self . original . dataReceived ( rawData ) <EOL> def connectionLost ( self , reason ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> rawData = self . _zlibDecompress . flush ( ) <EOL> except zlib . error : <EOL> raise ResponseFailed ( [ reason , failure . Failure ( ) ] , self . _response ) <EOL> if rawData : <EOL> self . original . dataReceived ( rawData ) <EOL> self . original . connectionLost ( reason ) <EOL> class ContentDecoderAgent ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , agent , decoders ) : <EOL> self . _agent = agent <EOL> self . _decoders = dict ( decoders ) <EOL> self . _supported = '<STR_LIT:U+002C>' . join ( [ decoder [ <NUM_LIT:0> ] for decoder in decoders ] ) <EOL> def request ( self , method , uri , headers = None , bodyProducer = None ) : <EOL> """<STR_LIT>""" <EOL> if headers is None : <EOL> headers = Headers ( ) <EOL> else : <EOL> headers = headers . copy ( ) <EOL> headers . addRawHeader ( '<STR_LIT>' , self . _supported ) <EOL> deferred = self . _agent . request ( method , uri , headers , bodyProducer ) <EOL> return deferred . addCallback ( self . _handleResponse ) <EOL> def _handleResponse ( self , response ) : <EOL> """<STR_LIT>""" <EOL> contentEncodingHeaders = response . headers . getRawHeaders ( <EOL> '<STR_LIT>' , [ ] ) <EOL> contentEncodingHeaders = '<STR_LIT:U+002C>' . join ( contentEncodingHeaders ) . split ( '<STR_LIT:U+002C>' ) <EOL> while contentEncodingHeaders : <EOL> name = contentEncodingHeaders . pop ( ) . strip ( ) <EOL> decoder = self . _decoders . get ( name ) <EOL> if decoder is not None : <EOL> response = decoder ( response ) <EOL> else : <EOL> contentEncodingHeaders . append ( name ) <EOL> break <EOL> if contentEncodingHeaders : <EOL> response . headers . setRawHeaders ( <EOL> '<STR_LIT>' , [ '<STR_LIT:U+002C>' . join ( contentEncodingHeaders ) ] ) <EOL> else : <EOL> response . headers . removeHeader ( '<STR_LIT>' ) <EOL> return response <EOL> class RedirectAgent ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , agent , redirectLimit = <NUM_LIT:20> ) : <EOL> self . _agent = agent <EOL> self . _redirectLimit = redirectLimit <EOL> def request ( self , method , uri , headers = None , bodyProducer = None ) : <EOL> """<STR_LIT>""" <EOL> deferred = self . _agent . request ( method , uri , headers , bodyProducer ) <EOL> return deferred . addCallback ( <EOL> self . _handleResponse , method , uri , headers , <NUM_LIT:0> ) <EOL> def _handleRedirect ( self , response , method , uri , headers , redirectCount ) : <EOL> """<STR_LIT>""" <EOL> if redirectCount >= self . _redirectLimit : <EOL> err = error . InfiniteRedirection ( <EOL> response . code , <EOL> '<STR_LIT>' , <EOL> location = uri ) <EOL> raise ResponseFailed ( [ failure . Failure ( err ) ] , response ) <EOL> locationHeaders = response . headers . getRawHeaders ( '<STR_LIT:location>' , [ ] ) <EOL> if not locationHeaders : <EOL> err = error . RedirectWithNoLocation ( <EOL> response . code , '<STR_LIT>' , uri ) <EOL> raise ResponseFailed ( [ failure . Failure ( err ) ] , response ) <EOL> location = locationHeaders [ <NUM_LIT:0> ] <EOL> deferred = self . _agent . request ( method , location , headers ) <EOL> return deferred . addCallback ( <EOL> self . _handleResponse , method , uri , headers , redirectCount + <NUM_LIT:1> ) <EOL> def _handleResponse ( self , response , method , uri , headers , redirectCount ) : <EOL> """<STR_LIT>""" <EOL> if response . code in ( http . MOVED_PERMANENTLY , http . FOUND , <EOL> http . TEMPORARY_REDIRECT ) : <EOL> if method not in ( '<STR_LIT:GET>' , '<STR_LIT>' ) : <EOL> err = error . PageRedirect ( response . code , location = uri ) <EOL> raise ResponseFailed ( [ failure . Failure ( err ) ] , response ) <EOL> return self . _handleRedirect ( response , method , uri , headers , <EOL> redirectCount ) <EOL> elif response . code == http . SEE_OTHER : <EOL> return self . _handleRedirect ( response , '<STR_LIT:GET>' , uri , headers , <EOL> redirectCount ) <EOL> return response <EOL> __all__ = [ <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' ] </s>
