<s> """<STR_LIT>""" <EOL> import os , urllib2 , bisect , httplib , types , tempfile <EOL> try : <EOL> import threading as _threading <EOL> except ImportError : <EOL> import dummy_threading as _threading <EOL> try : <EOL> set <EOL> except NameError : <EOL> import sets <EOL> set = sets . Set <EOL> from _request import Request <EOL> import _response <EOL> import _rfc3986 <EOL> import _sockettimeout <EOL> import _urllib2_fork <EOL> from _util import isstringlike <EOL> open_file = open <EOL> class ContentTooShortError ( urllib2 . URLError ) : <EOL> def __init__ ( self , reason , result ) : <EOL> urllib2 . URLError . __init__ ( self , reason ) <EOL> self . result = result <EOL> def set_request_attr ( req , name , value , default ) : <EOL> try : <EOL> getattr ( req , name ) <EOL> except AttributeError : <EOL> setattr ( req , name , default ) <EOL> if value is not default : <EOL> setattr ( req , name , value ) <EOL> class OpenerDirector ( _urllib2_fork . OpenerDirector ) : <EOL> def __init__ ( self ) : <EOL> _urllib2_fork . OpenerDirector . __init__ ( self ) <EOL> self . process_response = { } <EOL> self . process_request = { } <EOL> self . _any_request = { } <EOL> self . _any_response = { } <EOL> self . _handler_index_valid = True <EOL> self . _tempfiles = [ ] <EOL> def add_handler ( self , handler ) : <EOL> if not hasattr ( handler , "<STR_LIT>" ) : <EOL> raise TypeError ( "<STR_LIT>" % <EOL> type ( handler ) ) <EOL> if handler in self . handlers : <EOL> return <EOL> bisect . insort ( self . handlers , handler ) <EOL> handler . add_parent ( self ) <EOL> self . _handler_index_valid = False <EOL> def _maybe_reindex_handlers ( self ) : <EOL> if self . _handler_index_valid : <EOL> return <EOL> handle_error = { } <EOL> handle_open = { } <EOL> process_request = { } <EOL> process_response = { } <EOL> any_request = set ( ) <EOL> any_response = set ( ) <EOL> unwanted = [ ] <EOL> for handler in self . handlers : <EOL> added = False <EOL> for meth in dir ( handler ) : <EOL> if meth in [ "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" ] : <EOL> continue <EOL> if meth == "<STR_LIT>" : <EOL> any_request . add ( handler ) <EOL> added = True <EOL> continue <EOL> elif meth == "<STR_LIT>" : <EOL> any_response . add ( handler ) <EOL> added = True <EOL> continue <EOL> ii = meth . find ( "<STR_LIT:_>" ) <EOL> scheme = meth [ : ii ] <EOL> condition = meth [ ii + <NUM_LIT:1> : ] <EOL> if condition . startswith ( "<STR_LIT:error>" ) : <EOL> jj = meth [ ii + <NUM_LIT:1> : ] . find ( "<STR_LIT:_>" ) + ii + <NUM_LIT:1> <EOL> kind = meth [ jj + <NUM_LIT:1> : ] <EOL> try : <EOL> kind = int ( kind ) <EOL> except ValueError : <EOL> pass <EOL> lookup = handle_error . setdefault ( scheme , { } ) <EOL> elif condition == "<STR_LIT>" : <EOL> kind = scheme <EOL> lookup = handle_open <EOL> elif condition == "<STR_LIT>" : <EOL> kind = scheme <EOL> lookup = process_request <EOL> elif condition == "<STR_LIT>" : <EOL> kind = scheme <EOL> lookup = process_response <EOL> else : <EOL> continue <EOL> lookup . setdefault ( kind , set ( ) ) . add ( handler ) <EOL> added = True <EOL> if not added : <EOL> unwanted . append ( handler ) <EOL> for handler in unwanted : <EOL> self . handlers . remove ( handler ) <EOL> for lookup in [ process_request , process_response ] : <EOL> for scheme , handlers in lookup . iteritems ( ) : <EOL> lookup [ scheme ] = handlers <EOL> for scheme , lookup in handle_error . iteritems ( ) : <EOL> for code , handlers in lookup . iteritems ( ) : <EOL> handlers = list ( handlers ) <EOL> handlers . sort ( ) <EOL> lookup [ code ] = handlers <EOL> for scheme , handlers in handle_open . iteritems ( ) : <EOL> handlers = list ( handlers ) <EOL> handlers . sort ( ) <EOL> handle_open [ scheme ] = handlers <EOL> self . handle_error = handle_error <EOL> self . handle_open = handle_open <EOL> self . process_request = process_request <EOL> self . process_response = process_response <EOL> self . _any_request = any_request <EOL> self . _any_response = any_response <EOL> def _request ( self , url_or_req , data , visit , <EOL> timeout = _sockettimeout . _GLOBAL_DEFAULT_TIMEOUT ) : <EOL> if isstringlike ( url_or_req ) : <EOL> req = Request ( url_or_req , data , visit = visit , timeout = timeout ) <EOL> else : <EOL> req = url_or_req <EOL> if data is not None : <EOL> req . add_data ( data ) <EOL> set_request_attr ( req , "<STR_LIT>" , visit , None ) <EOL> set_request_attr ( req , "<STR_LIT>" , timeout , <EOL> _sockettimeout . _GLOBAL_DEFAULT_TIMEOUT ) <EOL> return req <EOL> def open ( self , fullurl , data = None , <EOL> timeout = _sockettimeout . _GLOBAL_DEFAULT_TIMEOUT ) : <EOL> req = self . _request ( fullurl , data , None , timeout ) <EOL> req_scheme = req . get_type ( ) <EOL> self . _maybe_reindex_handlers ( ) <EOL> request_processors = set ( self . process_request . get ( req_scheme , [ ] ) ) <EOL> request_processors . update ( self . _any_request ) <EOL> request_processors = list ( request_processors ) <EOL> request_processors . sort ( ) <EOL> for processor in request_processors : <EOL> for meth_name in [ "<STR_LIT>" , req_scheme + "<STR_LIT>" ] : <EOL> meth = getattr ( processor , meth_name , None ) <EOL> if meth : <EOL> req = meth ( req ) <EOL> urlopen = _urllib2_fork . OpenerDirector . _open <EOL> response = urlopen ( self , req , data ) <EOL> response_processors = set ( self . process_response . get ( req_scheme , [ ] ) ) <EOL> response_processors . update ( self . _any_response ) <EOL> response_processors = list ( response_processors ) <EOL> response_processors . sort ( ) <EOL> for processor in response_processors : <EOL> for meth_name in [ "<STR_LIT>" , req_scheme + "<STR_LIT>" ] : <EOL> meth = getattr ( processor , meth_name , None ) <EOL> if meth : <EOL> response = meth ( req , response ) <EOL> return response <EOL> def error ( self , proto , * args ) : <EOL> if proto in [ '<STR_LIT:http>' , '<STR_LIT>' ] : <EOL> dict = self . handle_error [ '<STR_LIT:http>' ] <EOL> proto = args [ <NUM_LIT:2> ] <EOL> meth_name = '<STR_LIT>' % proto <EOL> http_err = <NUM_LIT:1> <EOL> orig_args = args <EOL> else : <EOL> dict = self . handle_error <EOL> meth_name = proto + '<STR_LIT>' <EOL> http_err = <NUM_LIT:0> <EOL> args = ( dict , proto , meth_name ) + args <EOL> result = apply ( self . _call_chain , args ) <EOL> if result : <EOL> return result <EOL> if http_err : <EOL> args = ( dict , '<STR_LIT:default>' , '<STR_LIT>' ) + orig_args <EOL> return apply ( self . _call_chain , args ) <EOL> BLOCK_SIZE = <NUM_LIT> * <NUM_LIT:8> <EOL> def retrieve ( self , fullurl , filename = None , reporthook = None , data = None , <EOL> timeout = _sockettimeout . _GLOBAL_DEFAULT_TIMEOUT , <EOL> open = open_file ) : <EOL> """<STR_LIT>""" <EOL> req = self . _request ( fullurl , data , False , timeout ) <EOL> scheme = req . get_type ( ) <EOL> fp = self . open ( req ) <EOL> try : <EOL> headers = fp . info ( ) <EOL> if filename is None and scheme == '<STR_LIT:file>' : <EOL> return None , headers <EOL> if filename : <EOL> tfp = open ( filename , '<STR_LIT:wb>' ) <EOL> else : <EOL> path = _rfc3986 . urlsplit ( req . get_full_url ( ) ) [ <NUM_LIT:2> ] <EOL> suffix = os . path . splitext ( path ) [ <NUM_LIT:1> ] <EOL> fd , filename = tempfile . mkstemp ( suffix ) <EOL> self . _tempfiles . append ( filename ) <EOL> tfp = os . fdopen ( fd , '<STR_LIT:wb>' ) <EOL> try : <EOL> result = filename , headers <EOL> bs = self . BLOCK_SIZE <EOL> size = - <NUM_LIT:1> <EOL> read = <NUM_LIT:0> <EOL> blocknum = <NUM_LIT:0> <EOL> if reporthook : <EOL> if "<STR_LIT>" in headers : <EOL> size = int ( headers [ "<STR_LIT>" ] ) <EOL> reporthook ( blocknum , bs , size ) <EOL> while <NUM_LIT:1> : <EOL> block = fp . read ( bs ) <EOL> if block == "<STR_LIT>" : <EOL> break <EOL> read += len ( block ) <EOL> tfp . write ( block ) <EOL> blocknum += <NUM_LIT:1> <EOL> if reporthook : <EOL> reporthook ( blocknum , bs , size ) <EOL> finally : <EOL> tfp . close ( ) <EOL> finally : <EOL> fp . close ( ) <EOL> if size >= <NUM_LIT:0> and read < size : <EOL> raise ContentTooShortError ( <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" % ( read , size ) , <EOL> result <EOL> ) <EOL> return result <EOL> def close ( self ) : <EOL> _urllib2_fork . OpenerDirector . close ( self ) <EOL> self . open = self . error = self . retrieve = self . add_handler = None <EOL> if self . _tempfiles : <EOL> for filename in self . _tempfiles : <EOL> try : <EOL> os . unlink ( filename ) <EOL> except OSError : <EOL> pass <EOL> del self . _tempfiles [ : ] <EOL> def wrapped_open ( urlopen , process_response_object , fullurl , data = None , <EOL> timeout = _sockettimeout . _GLOBAL_DEFAULT_TIMEOUT ) : <EOL> success = True <EOL> try : <EOL> response = urlopen ( fullurl , data , timeout ) <EOL> except urllib2 . HTTPError , error : <EOL> success = False <EOL> if error . fp is None : <EOL> raise <EOL> response = error <EOL> if response is not None : <EOL> response = process_response_object ( response ) <EOL> if not success : <EOL> raise response <EOL> return response <EOL> class ResponseProcessingOpener ( OpenerDirector ) : <EOL> def open ( self , fullurl , data = None , <EOL> timeout = _sockettimeout . _GLOBAL_DEFAULT_TIMEOUT ) : <EOL> def bound_open ( fullurl , data = None , <EOL> timeout = _sockettimeout . _GLOBAL_DEFAULT_TIMEOUT ) : <EOL> return OpenerDirector . open ( self , fullurl , data , timeout ) <EOL> return wrapped_open ( <EOL> bound_open , self . process_response_object , fullurl , data , timeout ) <EOL> def process_response_object ( self , response ) : <EOL> return response <EOL> class SeekableResponseOpener ( ResponseProcessingOpener ) : <EOL> def process_response_object ( self , response ) : <EOL> return _response . seek_wrapped_response ( response ) <EOL> def isclass ( obj ) : <EOL> return isinstance ( obj , ( types . ClassType , type ) ) <EOL> class OpenerFactory : <EOL> """<STR_LIT>""" <EOL> default_classes = [ <EOL> _urllib2_fork . ProxyHandler , <EOL> _urllib2_fork . UnknownHandler , <EOL> _urllib2_fork . HTTPHandler , <EOL> _urllib2_fork . HTTPDefaultErrorHandler , <EOL> _urllib2_fork . HTTPRedirectHandler , <EOL> _urllib2_fork . FTPHandler , <EOL> _urllib2_fork . FileHandler , <EOL> _urllib2_fork . HTTPCookieProcessor , <EOL> _urllib2_fork . HTTPErrorProcessor , <EOL> ] <EOL> if hasattr ( httplib , '<STR_LIT>' ) : <EOL> default_classes . append ( _urllib2_fork . HTTPSHandler ) <EOL> handlers = [ ] <EOL> replacement_handlers = [ ] <EOL> def __init__ ( self , klass = OpenerDirector ) : <EOL> self . klass = klass <EOL> def build_opener ( self , * handlers ) : <EOL> """<STR_LIT>""" <EOL> opener = self . klass ( ) <EOL> default_classes = list ( self . default_classes ) <EOL> skip = set ( ) <EOL> for klass in default_classes : <EOL> for check in handlers : <EOL> if isclass ( check ) : <EOL> if issubclass ( check , klass ) : <EOL> skip . add ( klass ) <EOL> elif isinstance ( check , klass ) : <EOL> skip . add ( klass ) <EOL> for klass in skip : <EOL> default_classes . remove ( klass ) <EOL> for klass in default_classes : <EOL> opener . add_handler ( klass ( ) ) <EOL> for h in handlers : <EOL> if isclass ( h ) : <EOL> h = h ( ) <EOL> opener . add_handler ( h ) <EOL> return opener <EOL> build_opener = OpenerFactory ( ) . build_opener <EOL> _opener = None <EOL> urlopen_lock = _threading . Lock ( ) <EOL> def urlopen ( url , data = None , timeout = _sockettimeout . _GLOBAL_DEFAULT_TIMEOUT ) : <EOL> global _opener <EOL> if _opener is None : <EOL> urlopen_lock . acquire ( ) <EOL> try : <EOL> if _opener is None : <EOL> _opener = build_opener ( ) <EOL> finally : <EOL> urlopen_lock . release ( ) <EOL> return _opener . open ( url , data , timeout ) <EOL> def urlretrieve ( url , filename = None , reporthook = None , data = None , <EOL> timeout = _sockettimeout . _GLOBAL_DEFAULT_TIMEOUT ) : <EOL> global _opener <EOL> if _opener is None : <EOL> urlopen_lock . acquire ( ) <EOL> try : <EOL> if _opener is None : <EOL> _opener = build_opener ( ) <EOL> finally : <EOL> urlopen_lock . release ( ) <EOL> return _opener . retrieve ( url , filename , reporthook , data , timeout ) <EOL> def install_opener ( opener ) : <EOL> global _opener <EOL> _opener = opener </s>
