<s> """<STR_LIT>""" <EOL> import base64 <EOL> import hashlib <EOL> import httplib <EOL> import mimetools <EOL> import os <EOL> import posixpath <EOL> import random <EOL> import re <EOL> import socket <EOL> import sys <EOL> import time <EOL> import urlparse <EOL> import bisect <EOL> import warnings <EOL> try : <EOL> from cStringIO import StringIO <EOL> except ImportError : <EOL> from StringIO import StringIO <EOL> from urllib import ( unwrap , unquote , splittype , splithost , quote , <EOL> addinfourl , splitport , splittag , toBytes , <EOL> splitattr , ftpwrapper , splituser , splitpasswd , splitvalue ) <EOL> from urllib import localhost , url2pathname , getproxies , proxy_bypass <EOL> __version__ = sys . version [ : <NUM_LIT:3> ] <EOL> _opener = None <EOL> def urlopen ( url , data = None , timeout = socket . _GLOBAL_DEFAULT_TIMEOUT ) : <EOL> global _opener <EOL> if _opener is None : <EOL> _opener = build_opener ( ) <EOL> return _opener . open ( url , data , timeout ) <EOL> def install_opener ( opener ) : <EOL> global _opener <EOL> _opener = opener <EOL> class URLError ( IOError ) : <EOL> def __init__ ( self , reason ) : <EOL> self . args = reason , <EOL> self . reason = reason <EOL> def __str__ ( self ) : <EOL> return '<STR_LIT>' % self . reason <EOL> class HTTPError ( URLError , addinfourl ) : <EOL> """<STR_LIT>""" <EOL> __super_init = addinfourl . __init__ <EOL> def __init__ ( self , url , code , msg , hdrs , fp ) : <EOL> self . code = code <EOL> self . msg = msg <EOL> self . hdrs = hdrs <EOL> self . fp = fp <EOL> self . filename = url <EOL> if fp is not None : <EOL> self . __super_init ( fp , hdrs , url , code ) <EOL> def __str__ ( self ) : <EOL> return '<STR_LIT>' % ( self . code , self . msg ) <EOL> @ property <EOL> def reason ( self ) : <EOL> return self . msg <EOL> def info ( self ) : <EOL> return self . hdrs <EOL> _cut_port_re = re . compile ( r"<STR_LIT>" ) <EOL> def request_host ( request ) : <EOL> """<STR_LIT>""" <EOL> url = request . get_full_url ( ) <EOL> host = urlparse . urlparse ( url ) [ <NUM_LIT:1> ] <EOL> if host == "<STR_LIT>" : <EOL> host = request . get_header ( "<STR_LIT>" , "<STR_LIT>" ) <EOL> host = _cut_port_re . sub ( "<STR_LIT>" , host , <NUM_LIT:1> ) <EOL> return host . lower ( ) <EOL> class Request : <EOL> def __init__ ( self , url , data = None , headers = { } , <EOL> origin_req_host = None , unverifiable = False ) : <EOL> self . __original = unwrap ( url ) <EOL> self . __original , self . __fragment = splittag ( self . __original ) <EOL> self . type = None <EOL> self . host = None <EOL> self . port = None <EOL> self . _tunnel_host = None <EOL> self . data = data <EOL> self . headers = { } <EOL> for key , value in headers . items ( ) : <EOL> self . add_header ( key , value ) <EOL> self . unredirected_hdrs = { } <EOL> if origin_req_host is None : <EOL> origin_req_host = request_host ( self ) <EOL> self . origin_req_host = origin_req_host <EOL> self . unverifiable = unverifiable <EOL> def __getattr__ ( self , attr ) : <EOL> if attr [ : <NUM_LIT:12> ] == '<STR_LIT>' : <EOL> name = attr [ <NUM_LIT:12> : ] <EOL> if hasattr ( Request , '<STR_LIT>' + name ) : <EOL> getattr ( self , '<STR_LIT>' + name ) ( ) <EOL> return getattr ( self , attr ) <EOL> raise AttributeError , attr <EOL> def get_method ( self ) : <EOL> if self . has_data ( ) : <EOL> return "<STR_LIT:POST>" <EOL> else : <EOL> return "<STR_LIT:GET>" <EOL> def add_data ( self , data ) : <EOL> self . data = data <EOL> def has_data ( self ) : <EOL> return self . data is not None <EOL> def get_data ( self ) : <EOL> return self . data <EOL> def get_full_url ( self ) : <EOL> if self . __fragment : <EOL> return '<STR_LIT>' % ( self . __original , self . __fragment ) <EOL> else : <EOL> return self . __original <EOL> def get_type ( self ) : <EOL> if self . type is None : <EOL> self . type , self . __r_type = splittype ( self . __original ) <EOL> if self . type is None : <EOL> raise ValueError , "<STR_LIT>" % self . __original <EOL> return self . type <EOL> def get_host ( self ) : <EOL> if self . host is None : <EOL> self . host , self . __r_host = splithost ( self . __r_type ) <EOL> if self . host : <EOL> self . host = unquote ( self . host ) <EOL> return self . host <EOL> def get_selector ( self ) : <EOL> return self . __r_host <EOL> def set_proxy ( self , host , type ) : <EOL> if self . type == '<STR_LIT>' and not self . _tunnel_host : <EOL> self . _tunnel_host = self . host <EOL> else : <EOL> self . type = type <EOL> self . __r_host = self . __original <EOL> self . host = host <EOL> def has_proxy ( self ) : <EOL> return self . __r_host == self . __original <EOL> def get_origin_req_host ( self ) : <EOL> return self . origin_req_host <EOL> def is_unverifiable ( self ) : <EOL> return self . unverifiable <EOL> def add_header ( self , key , val ) : <EOL> self . headers [ key . capitalize ( ) ] = val <EOL> def add_unredirected_header ( self , key , val ) : <EOL> self . unredirected_hdrs [ key . capitalize ( ) ] = val <EOL> def has_header ( self , header_name ) : <EOL> return ( header_name in self . headers or <EOL> header_name in self . unredirected_hdrs ) <EOL> def get_header ( self , header_name , default = None ) : <EOL> return self . headers . get ( <EOL> header_name , <EOL> self . unredirected_hdrs . get ( header_name , default ) ) <EOL> def header_items ( self ) : <EOL> hdrs = self . unredirected_hdrs . copy ( ) <EOL> hdrs . update ( self . headers ) <EOL> return hdrs . items ( ) <EOL> class OpenerDirector : <EOL> def __init__ ( self ) : <EOL> client_version = "<STR_LIT>" % __version__ <EOL> self . addheaders = [ ( '<STR_LIT>' , client_version ) ] <EOL> self . handlers = [ ] <EOL> self . handle_open = { } <EOL> self . handle_error = { } <EOL> self . process_response = { } <EOL> self . process_request = { } <EOL> def add_handler ( self , handler ) : <EOL> if not hasattr ( handler , "<STR_LIT>" ) : <EOL> raise TypeError ( "<STR_LIT>" % <EOL> type ( handler ) ) <EOL> added = False <EOL> for meth in dir ( handler ) : <EOL> if meth in [ "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" ] : <EOL> continue <EOL> i = meth . find ( "<STR_LIT:_>" ) <EOL> protocol = meth [ : i ] <EOL> condition = meth [ i + <NUM_LIT:1> : ] <EOL> if condition . startswith ( "<STR_LIT:error>" ) : <EOL> j = condition . find ( "<STR_LIT:_>" ) + i + <NUM_LIT:1> <EOL> kind = meth [ j + <NUM_LIT:1> : ] <EOL> try : <EOL> kind = int ( kind ) <EOL> except ValueError : <EOL> pass <EOL> lookup = self . handle_error . get ( protocol , { } ) <EOL> self . handle_error [ protocol ] = lookup <EOL> elif condition == "<STR_LIT>" : <EOL> kind = protocol <EOL> lookup = self . handle_open <EOL> elif condition == "<STR_LIT>" : <EOL> kind = protocol <EOL> lookup = self . process_response <EOL> elif condition == "<STR_LIT>" : <EOL> kind = protocol <EOL> lookup = self . process_request <EOL> else : <EOL> continue <EOL> handlers = lookup . setdefault ( kind , [ ] ) <EOL> if handlers : <EOL> bisect . insort ( handlers , handler ) <EOL> else : <EOL> handlers . append ( handler ) <EOL> added = True <EOL> if added : <EOL> bisect . insort ( self . handlers , handler ) <EOL> handler . add_parent ( self ) <EOL> def close ( self ) : <EOL> pass <EOL> def _call_chain ( self , chain , kind , meth_name , * args ) : <EOL> handlers = chain . get ( kind , ( ) ) <EOL> for handler in handlers : <EOL> func = getattr ( handler , meth_name ) <EOL> result = func ( * args ) <EOL> if result is not None : <EOL> return result <EOL> def open ( self , fullurl , data = None , timeout = socket . _GLOBAL_DEFAULT_TIMEOUT ) : <EOL> if isinstance ( fullurl , basestring ) : <EOL> req = Request ( fullurl , data ) <EOL> else : <EOL> req = fullurl <EOL> if data is not None : <EOL> req . add_data ( data ) <EOL> req . timeout = timeout <EOL> protocol = req . get_type ( ) <EOL> meth_name = protocol + "<STR_LIT>" <EOL> for processor in self . process_request . get ( protocol , [ ] ) : <EOL> meth = getattr ( processor , meth_name ) <EOL> req = meth ( req ) <EOL> response = self . _open ( req , data ) <EOL> meth_name = protocol + "<STR_LIT>" <EOL> for processor in self . process_response . get ( protocol , [ ] ) : <EOL> meth = getattr ( processor , meth_name ) <EOL> response = meth ( req , response ) <EOL> return response <EOL> def _open ( self , req , data = None ) : <EOL> result = self . _call_chain ( self . handle_open , '<STR_LIT:default>' , <EOL> '<STR_LIT>' , req ) <EOL> if result : <EOL> return result <EOL> protocol = req . get_type ( ) <EOL> result = self . _call_chain ( self . handle_open , protocol , protocol + <EOL> '<STR_LIT>' , req ) <EOL> if result : <EOL> return result <EOL> return self . _call_chain ( self . handle_open , '<STR_LIT>' , <EOL> '<STR_LIT>' , req ) <EOL> def error ( self , proto , * args ) : <EOL> if proto in ( '<STR_LIT:http>' , '<STR_LIT>' ) : <EOL> dict = self . handle_error [ '<STR_LIT:http>' ] <EOL> proto = args [ <NUM_LIT:2> ] <EOL> meth_name = '<STR_LIT>' % proto <EOL> http_err = <NUM_LIT:1> <EOL> orig_args = args <EOL> else : <EOL> dict = self . handle_error <EOL> meth_name = proto + '<STR_LIT>' <EOL> http_err = <NUM_LIT:0> <EOL> args = ( dict , proto , meth_name ) + args <EOL> result = self . _call_chain ( * args ) <EOL> if result : <EOL> return result <EOL> if http_err : <EOL> args = ( dict , '<STR_LIT:default>' , '<STR_LIT>' ) + orig_args <EOL> return self . _call_chain ( * args ) <EOL> def build_opener ( * handlers ) : <EOL> """<STR_LIT>""" <EOL> import types <EOL> def isclass ( obj ) : <EOL> return isinstance ( obj , ( types . ClassType , type ) ) <EOL> opener = OpenerDirector ( ) <EOL> default_classes = [ ProxyHandler , UnknownHandler , HTTPHandler , <EOL> HTTPDefaultErrorHandler , HTTPRedirectHandler , <EOL> FTPHandler , FileHandler , HTTPErrorProcessor ] <EOL> if hasattr ( httplib , '<STR_LIT>' ) : <EOL> default_classes . append ( HTTPSHandler ) <EOL> skip = set ( ) <EOL> for klass in default_classes : <EOL> for check in handlers : <EOL> if isclass ( check ) : <EOL> if issubclass ( check , klass ) : <EOL> skip . add ( klass ) <EOL> elif isinstance ( check , klass ) : <EOL> skip . add ( klass ) <EOL> for klass in skip : <EOL> default_classes . remove ( klass ) <EOL> for klass in default_classes : <EOL> opener . add_handler ( klass ( ) ) <EOL> for h in handlers : <EOL> if isclass ( h ) : <EOL> h = h ( ) <EOL> opener . add_handler ( h ) <EOL> return opener <EOL> class BaseHandler : <EOL> handler_order = <NUM_LIT> <EOL> def add_parent ( self , parent ) : <EOL> self . parent = parent <EOL> def close ( self ) : <EOL> pass <EOL> def __lt__ ( self , other ) : <EOL> if not hasattr ( other , "<STR_LIT>" ) : <EOL> return True <EOL> return self . handler_order < other . handler_order <EOL> class HTTPErrorProcessor ( BaseHandler ) : <EOL> """<STR_LIT>""" <EOL> handler_order = <NUM_LIT:1000> <EOL> def http_response ( self , request , response ) : <EOL> code , msg , hdrs = response . code , response . msg , response . info ( ) <EOL> if not ( <NUM_LIT:200> <= code < <NUM_LIT> ) : <EOL> response = self . parent . error ( <EOL> '<STR_LIT:http>' , request , response , code , msg , hdrs ) <EOL> return response <EOL> https_response = http_response <EOL> class HTTPDefaultErrorHandler ( BaseHandler ) : <EOL> def http_error_default ( self , req , fp , code , msg , hdrs ) : <EOL> raise HTTPError ( req . get_full_url ( ) , code , msg , hdrs , fp ) <EOL> class HTTPRedirectHandler ( BaseHandler ) : <EOL> max_repeats = <NUM_LIT:4> <EOL> max_redirections = <NUM_LIT:10> <EOL> def redirect_request ( self , req , fp , code , msg , headers , newurl ) : <EOL> """<STR_LIT>""" <EOL> m = req . get_method ( ) <EOL> if ( code in ( <NUM_LIT> , <NUM_LIT> , <NUM_LIT> , <NUM_LIT> ) and m in ( "<STR_LIT:GET>" , "<STR_LIT>" ) <EOL> or code in ( <NUM_LIT> , <NUM_LIT> , <NUM_LIT> ) and m == "<STR_LIT:POST>" ) : <EOL> newurl = newurl . replace ( '<STR_LIT:U+0020>' , '<STR_LIT>' ) <EOL> newheaders = dict ( ( k , v ) for k , v in req . headers . items ( ) <EOL> if k . lower ( ) not in ( "<STR_LIT>" , "<STR_LIT>" ) <EOL> ) <EOL> return Request ( newurl , <EOL> headers = newheaders , <EOL> origin_req_host = req . get_origin_req_host ( ) , <EOL> unverifiable = True ) <EOL> else : <EOL> raise HTTPError ( req . get_full_url ( ) , code , msg , headers , fp ) <EOL> def http_error_302 ( self , req , fp , code , msg , headers ) : <EOL> if '<STR_LIT:location>' in headers : <EOL> newurl = headers . getheaders ( '<STR_LIT:location>' ) [ <NUM_LIT:0> ] <EOL> elif '<STR_LIT>' in headers : <EOL> newurl = headers . getheaders ( '<STR_LIT>' ) [ <NUM_LIT:0> ] <EOL> else : <EOL> return <EOL> urlparts = urlparse . urlparse ( newurl ) <EOL> if not urlparts . path : <EOL> urlparts = list ( urlparts ) <EOL> urlparts [ <NUM_LIT:2> ] = "<STR_LIT:/>" <EOL> newurl = urlparse . urlunparse ( urlparts ) <EOL> newurl = urlparse . urljoin ( req . get_full_url ( ) , newurl ) <EOL> newurl_lower = newurl . lower ( ) <EOL> if not ( newurl_lower . startswith ( '<STR_LIT>' ) or <EOL> newurl_lower . startswith ( '<STR_LIT>' ) or <EOL> newurl_lower . startswith ( '<STR_LIT>' ) ) : <EOL> raise HTTPError ( newurl , code , <EOL> msg + "<STR_LIT>" % <EOL> newurl , <EOL> headers , fp ) <EOL> new = self . redirect_request ( req , fp , code , msg , headers , newurl ) <EOL> if new is None : <EOL> return <EOL> if hasattr ( req , '<STR_LIT>' ) : <EOL> visited = new . redirect_dict = req . redirect_dict <EOL> if ( visited . get ( newurl , <NUM_LIT:0> ) >= self . max_repeats or <EOL> len ( visited ) >= self . max_redirections ) : <EOL> raise HTTPError ( req . get_full_url ( ) , code , <EOL> self . inf_msg + msg , headers , fp ) <EOL> else : <EOL> visited = new . redirect_dict = req . redirect_dict = { } <EOL> visited [ newurl ] = visited . get ( newurl , <NUM_LIT:0> ) + <NUM_LIT:1> <EOL> fp . read ( ) <EOL> fp . close ( ) <EOL> return self . parent . open ( new , timeout = req . timeout ) <EOL> http_error_301 = http_error_303 = http_error_307 = http_error_302 <EOL> inf_msg = "<STR_LIT>" "<STR_LIT>" "<STR_LIT>" <EOL> def _parse_proxy ( proxy ) : <EOL> """<STR_LIT>""" <EOL> scheme , r_scheme = splittype ( proxy ) <EOL> if not r_scheme . startswith ( "<STR_LIT:/>" ) : <EOL> scheme = None <EOL> authority = proxy <EOL> else : <EOL> if not r_scheme . startswith ( "<STR_LIT>" ) : <EOL> raise ValueError ( "<STR_LIT>" % proxy ) <EOL> end = r_scheme . find ( "<STR_LIT:/>" , <NUM_LIT:2> ) <EOL> if end == - <NUM_LIT:1> : <EOL> end = None <EOL> authority = r_scheme [ <NUM_LIT:2> : end ] <EOL> userinfo , hostport = splituser ( authority ) <EOL> if userinfo is not None : <EOL> user , password = splitpasswd ( userinfo ) <EOL> else : <EOL> user = password = None <EOL> return scheme , user , password , hostport <EOL> class ProxyHandler ( BaseHandler ) : <EOL> handler_order = <NUM_LIT:100> <EOL> def __init__ ( self , proxies = None ) : <EOL> if proxies is None : <EOL> proxies = getproxies ( ) <EOL> assert hasattr ( proxies , '<STR_LIT>' ) , "<STR_LIT>" <EOL> self . proxies = proxies <EOL> for type , url in proxies . items ( ) : <EOL> setattr ( self , '<STR_LIT>' % type , <EOL> lambda r , proxy = url , type = type , meth = self . proxy_open : meth ( r , proxy , type ) ) <EOL> def proxy_open ( self , req , proxy , type ) : <EOL> orig_type = req . get_type ( ) <EOL> proxy_type , user , password , hostport = _parse_proxy ( proxy ) <EOL> if proxy_type is None : <EOL> proxy_type = orig_type <EOL> if req . host and proxy_bypass ( req . host ) : <EOL> return None <EOL> if user and password : <EOL> user_pass = '<STR_LIT>' % ( unquote ( user ) , unquote ( password ) ) <EOL> creds = base64 . b64encode ( user_pass ) . strip ( ) <EOL> req . add_header ( '<STR_LIT>' , '<STR_LIT>' + creds ) <EOL> hostport = unquote ( hostport ) <EOL> req . set_proxy ( hostport , proxy_type ) <EOL> if orig_type == proxy_type or orig_type == '<STR_LIT>' : <EOL> return None <EOL> else : <EOL> return self . parent . open ( req , timeout = req . timeout ) <EOL> class HTTPPasswordMgr : <EOL> def __init__ ( self ) : <EOL> self . passwd = { } <EOL> def add_password ( self , realm , uri , user , passwd ) : <EOL> if isinstance ( uri , basestring ) : <EOL> uri = [ uri ] <EOL> if not realm in self . passwd : <EOL> self . passwd [ realm ] = { } <EOL> for default_port in True , False : <EOL> reduced_uri = tuple ( <EOL> [ self . reduce_uri ( u , default_port ) for u in uri ] ) <EOL> self . passwd [ realm ] [ reduced_uri ] = ( user , passwd ) <EOL> def find_user_password ( self , realm , authuri ) : <EOL> domains = self . passwd . get ( realm , { } ) <EOL> for default_port in True , False : <EOL> reduced_authuri = self . reduce_uri ( authuri , default_port ) <EOL> for uris , authinfo in domains . iteritems ( ) : <EOL> for uri in uris : <EOL> if self . is_suburi ( uri , reduced_authuri ) : <EOL> return authinfo <EOL> return None , None <EOL> def reduce_uri ( self , uri , default_port = True ) : <EOL> """<STR_LIT>""" <EOL> parts = urlparse . urlsplit ( uri ) <EOL> if parts [ <NUM_LIT:1> ] : <EOL> scheme = parts [ <NUM_LIT:0> ] <EOL> authority = parts [ <NUM_LIT:1> ] <EOL> path = parts [ <NUM_LIT:2> ] or '<STR_LIT:/>' <EOL> else : <EOL> scheme = None <EOL> authority = uri <EOL> path = '<STR_LIT:/>' <EOL> host , port = splitport ( authority ) <EOL> if default_port and port is None and scheme is not None : <EOL> dport = { "<STR_LIT:http>" : <NUM_LIT> , <EOL> "<STR_LIT>" : <NUM_LIT> , <EOL> } . get ( scheme ) <EOL> if dport is not None : <EOL> authority = "<STR_LIT>" % ( host , dport ) <EOL> return authority , path <EOL> def is_suburi ( self , base , test ) : <EOL> """<STR_LIT>""" <EOL> if base == test : <EOL> return True <EOL> if base [ <NUM_LIT:0> ] != test [ <NUM_LIT:0> ] : <EOL> return False <EOL> common = posixpath . commonprefix ( ( base [ <NUM_LIT:1> ] , test [ <NUM_LIT:1> ] ) ) <EOL> if len ( common ) == len ( base [ <NUM_LIT:1> ] ) : <EOL> return True <EOL> return False <EOL> class HTTPPasswordMgrWithDefaultRealm ( HTTPPasswordMgr ) : <EOL> def find_user_password ( self , realm , authuri ) : <EOL> user , password = HTTPPasswordMgr . find_user_password ( self , realm , <EOL> authuri ) <EOL> if user is not None : <EOL> return user , password <EOL> return HTTPPasswordMgr . find_user_password ( self , None , authuri ) <EOL> class AbstractBasicAuthHandler : <EOL> rx = re . compile ( '<STR_LIT>' <EOL> '<STR_LIT>' , re . I ) <EOL> def __init__ ( self , password_mgr = None ) : <EOL> if password_mgr is None : <EOL> password_mgr = HTTPPasswordMgr ( ) <EOL> self . passwd = password_mgr <EOL> self . add_password = self . passwd . add_password <EOL> self . retried = <NUM_LIT:0> <EOL> def reset_retry_count ( self ) : <EOL> self . retried = <NUM_LIT:0> <EOL> def http_error_auth_reqed ( self , authreq , host , req , headers ) : <EOL> authreq = headers . get ( authreq , None ) <EOL> if self . retried > <NUM_LIT:5> : <EOL> raise HTTPError ( req . get_full_url ( ) , <NUM_LIT> , "<STR_LIT>" , <EOL> headers , None ) <EOL> else : <EOL> self . retried += <NUM_LIT:1> <EOL> if authreq : <EOL> mo = AbstractBasicAuthHandler . rx . search ( authreq ) <EOL> if mo : <EOL> scheme , quote , realm = mo . groups ( ) <EOL> if quote not in [ '<STR_LIT:">' , "<STR_LIT:'>" ] : <EOL> warnings . warn ( "<STR_LIT>" , <EOL> UserWarning , <NUM_LIT:2> ) <EOL> if scheme . lower ( ) == '<STR_LIT>' : <EOL> response = self . retry_http_basic_auth ( host , req , realm ) <EOL> if response and response . code != <NUM_LIT> : <EOL> self . retried = <NUM_LIT:0> <EOL> return response <EOL> def retry_http_basic_auth ( self , host , req , realm ) : <EOL> user , pw = self . passwd . find_user_password ( realm , host ) <EOL> if pw is not None : <EOL> raw = "<STR_LIT>" % ( user , pw ) <EOL> auth = '<STR_LIT>' % base64 . b64encode ( raw ) . strip ( ) <EOL> if req . headers . get ( self . auth_header , None ) == auth : <EOL> return None <EOL> req . add_unredirected_header ( self . auth_header , auth ) <EOL> return self . parent . open ( req , timeout = req . timeout ) <EOL> else : <EOL> return None <EOL> class HTTPBasicAuthHandler ( AbstractBasicAuthHandler , BaseHandler ) : <EOL> auth_header = '<STR_LIT>' <EOL> def http_error_401 ( self , req , fp , code , msg , headers ) : <EOL> url = req . get_full_url ( ) <EOL> response = self . http_error_auth_reqed ( '<STR_LIT>' , <EOL> url , req , headers ) <EOL> self . reset_retry_count ( ) <EOL> return response <EOL> class ProxyBasicAuthHandler ( AbstractBasicAuthHandler , BaseHandler ) : <EOL> auth_header = '<STR_LIT>' <EOL> def http_error_407 ( self , req , fp , code , msg , headers ) : <EOL> authority = req . get_host ( ) <EOL> response = self . http_error_auth_reqed ( '<STR_LIT>' , <EOL> authority , req , headers ) <EOL> self . reset_retry_count ( ) <EOL> return response <EOL> def randombytes ( n ) : <EOL> """<STR_LIT>""" <EOL> if os . path . exists ( "<STR_LIT>" ) : <EOL> f = open ( "<STR_LIT>" ) <EOL> s = f . read ( n ) <EOL> f . close ( ) <EOL> return s <EOL> else : <EOL> L = [ chr ( random . randrange ( <NUM_LIT:0> , <NUM_LIT> ) ) for i in range ( n ) ] <EOL> return "<STR_LIT>" . join ( L ) <EOL> class AbstractDigestAuthHandler : <EOL> def __init__ ( self , passwd = None ) : <EOL> if passwd is None : <EOL> passwd = HTTPPasswordMgr ( ) <EOL> self . passwd = passwd <EOL> self . add_password = self . passwd . add_password <EOL> self . retried = <NUM_LIT:0> <EOL> self . nonce_count = <NUM_LIT:0> <EOL> self . last_nonce = None <EOL> def reset_retry_count ( self ) : <EOL> self . retried = <NUM_LIT:0> <EOL> def http_error_auth_reqed ( self , auth_header , host , req , headers ) : <EOL> authreq = headers . get ( auth_header , None ) <EOL> if self . retried > <NUM_LIT:5> : <EOL> raise HTTPError ( req . get_full_url ( ) , <NUM_LIT> , "<STR_LIT>" , <EOL> headers , None ) <EOL> else : <EOL> self . retried += <NUM_LIT:1> <EOL> if authreq : <EOL> scheme = authreq . split ( ) [ <NUM_LIT:0> ] <EOL> if scheme . lower ( ) == '<STR_LIT>' : <EOL> return self . retry_http_digest_auth ( req , authreq ) <EOL> def retry_http_digest_auth ( self , req , auth ) : <EOL> token , challenge = auth . split ( '<STR_LIT:U+0020>' , <NUM_LIT:1> ) <EOL> chal = parse_keqv_list ( parse_http_list ( challenge ) ) <EOL> auth = self . get_authorization ( req , chal ) <EOL> if auth : <EOL> auth_val = '<STR_LIT>' % auth <EOL> if req . headers . get ( self . auth_header , None ) == auth_val : <EOL> return None <EOL> req . add_unredirected_header ( self . auth_header , auth_val ) <EOL> resp = self . parent . open ( req , timeout = req . timeout ) <EOL> return resp <EOL> def get_cnonce ( self , nonce ) : <EOL> dig = hashlib . sha1 ( "<STR_LIT>" % ( self . nonce_count , nonce , time . ctime ( ) , <EOL> randombytes ( <NUM_LIT:8> ) ) ) . hexdigest ( ) <EOL> return dig [ : <NUM_LIT:16> ] <EOL> def get_authorization ( self , req , chal ) : <EOL> try : <EOL> realm = chal [ '<STR_LIT>' ] <EOL> nonce = chal [ '<STR_LIT>' ] <EOL> qop = chal . get ( '<STR_LIT>' ) <EOL> algorithm = chal . get ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> opaque = chal . get ( '<STR_LIT>' , None ) <EOL> except KeyError : <EOL> return None <EOL> H , KD = self . get_algorithm_impls ( algorithm ) <EOL> if H is None : <EOL> return None <EOL> user , pw = self . passwd . find_user_password ( realm , req . get_full_url ( ) ) <EOL> if user is None : <EOL> return None <EOL> if req . has_data ( ) : <EOL> entdig = self . get_entity_digest ( req . get_data ( ) , chal ) <EOL> else : <EOL> entdig = None <EOL> A1 = "<STR_LIT>" % ( user , realm , pw ) <EOL> A2 = "<STR_LIT>" % ( req . get_method ( ) , <EOL> req . get_selector ( ) ) <EOL> if qop == '<STR_LIT>' : <EOL> if nonce == self . last_nonce : <EOL> self . nonce_count += <NUM_LIT:1> <EOL> else : <EOL> self . nonce_count = <NUM_LIT:1> <EOL> self . last_nonce = nonce <EOL> ncvalue = '<STR_LIT>' % self . nonce_count <EOL> cnonce = self . get_cnonce ( nonce ) <EOL> noncebit = "<STR_LIT>" % ( nonce , ncvalue , cnonce , qop , H ( A2 ) ) <EOL> respdig = KD ( H ( A1 ) , noncebit ) <EOL> elif qop is None : <EOL> respdig = KD ( H ( A1 ) , "<STR_LIT>" % ( nonce , H ( A2 ) ) ) <EOL> else : <EOL> raise URLError ( "<STR_LIT>" % qop ) <EOL> base = '<STR_LIT>' '<STR_LIT>' % ( user , realm , nonce , req . get_selector ( ) , <EOL> respdig ) <EOL> if opaque : <EOL> base += '<STR_LIT>' % opaque <EOL> if entdig : <EOL> base += '<STR_LIT>' % entdig <EOL> base += '<STR_LIT>' % algorithm <EOL> if qop : <EOL> base += '<STR_LIT>' % ( ncvalue , cnonce ) <EOL> return base <EOL> def get_algorithm_impls ( self , algorithm ) : <EOL> algorithm = algorithm . upper ( ) <EOL> if algorithm == '<STR_LIT>' : <EOL> H = lambda x : hashlib . md5 ( x ) . hexdigest ( ) <EOL> elif algorithm == '<STR_LIT>' : <EOL> H = lambda x : hashlib . sha1 ( x ) . hexdigest ( ) <EOL> KD = lambda s , d : H ( "<STR_LIT>" % ( s , d ) ) <EOL> return H , KD <EOL> def get_entity_digest ( self , data , chal ) : <EOL> return None <EOL> class HTTPDigestAuthHandler ( BaseHandler , AbstractDigestAuthHandler ) : <EOL> """<STR_LIT>""" <EOL> auth_header = '<STR_LIT>' <EOL> handler_order = <NUM_LIT> <EOL> def http_error_401 ( self , req , fp , code , msg , headers ) : <EOL> host = urlparse . urlparse ( req . get_full_url ( ) ) [ <NUM_LIT:1> ] <EOL> retry = self . http_error_auth_reqed ( '<STR_LIT>' , <EOL> host , req , headers ) <EOL> self . reset_retry_count ( ) <EOL> return retry <EOL> class ProxyDigestAuthHandler ( BaseHandler , AbstractDigestAuthHandler ) : <EOL> auth_header = '<STR_LIT>' <EOL> handler_order = <NUM_LIT> <EOL> def http_error_407 ( self , req , fp , code , msg , headers ) : <EOL> host = req . get_host ( ) <EOL> retry = self . http_error_auth_reqed ( '<STR_LIT>' , <EOL> host , req , headers ) <EOL> self . reset_retry_count ( ) <EOL> return retry <EOL> class AbstractHTTPHandler ( BaseHandler ) : <EOL> def __init__ ( self , debuglevel = <NUM_LIT:0> ) : <EOL> self . _debuglevel = debuglevel <EOL> def set_http_debuglevel ( self , level ) : <EOL> self . _debuglevel = level <EOL> def do_request_ ( self , request ) : <EOL> host = request . get_host ( ) <EOL> if not host : <EOL> raise URLError ( '<STR_LIT>' ) <EOL> if request . has_data ( ) : <EOL> data = request . get_data ( ) <EOL> if not request . has_header ( '<STR_LIT>' ) : <EOL> request . add_unredirected_header ( <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' ) <EOL> if not request . has_header ( '<STR_LIT>' ) : <EOL> request . add_unredirected_header ( <EOL> '<STR_LIT>' , '<STR_LIT>' % len ( data ) ) <EOL> sel_host = host <EOL> if request . has_proxy ( ) : <EOL> scheme , sel = splittype ( request . get_selector ( ) ) <EOL> sel_host , sel_path = splithost ( sel ) <EOL> if not request . has_header ( '<STR_LIT>' ) : <EOL> request . add_unredirected_header ( '<STR_LIT>' , sel_host ) <EOL> for name , value in self . parent . addheaders : <EOL> name = name . capitalize ( ) <EOL> if not request . has_header ( name ) : <EOL> request . add_unredirected_header ( name , value ) <EOL> return request <EOL> def do_open ( self , http_class , req ) : <EOL> """<STR_LIT>""" <EOL> host = req . get_host ( ) <EOL> if not host : <EOL> raise URLError ( '<STR_LIT>' ) <EOL> h = http_class ( host , timeout = req . timeout ) <EOL> h . set_debuglevel ( self . _debuglevel ) <EOL> headers = dict ( req . unredirected_hdrs ) <EOL> headers . update ( dict ( ( k , v ) for k , v in req . headers . items ( ) <EOL> if k not in headers ) ) <EOL> headers [ "<STR_LIT>" ] = "<STR_LIT>" <EOL> headers = dict ( <EOL> ( name . title ( ) , val ) for name , val in headers . items ( ) ) <EOL> if req . _tunnel_host : <EOL> tunnel_headers = { } <EOL> proxy_auth_hdr = "<STR_LIT>" <EOL> if proxy_auth_hdr in headers : <EOL> tunnel_headers [ proxy_auth_hdr ] = headers [ proxy_auth_hdr ] <EOL> del headers [ proxy_auth_hdr ] <EOL> h . set_tunnel ( req . _tunnel_host , headers = tunnel_headers ) <EOL> try : <EOL> h . request ( req . get_method ( ) , req . get_selector ( ) , req . data , headers ) <EOL> except socket . error , err : <EOL> h . close ( ) <EOL> raise URLError ( err ) <EOL> else : <EOL> try : <EOL> r = h . getresponse ( buffering = True ) <EOL> except TypeError : <EOL> r = h . getresponse ( ) <EOL> r . recv = r . read <EOL> r . _reuse = lambda : None <EOL> r . _drop = lambda : None <EOL> fp = socket . _fileobject ( r , close = True ) <EOL> resp = addinfourl ( fp , r . msg , req . get_full_url ( ) ) <EOL> resp . code = r . status <EOL> resp . msg = r . reason <EOL> return resp <EOL> class HTTPHandler ( AbstractHTTPHandler ) : <EOL> def http_open ( self , req ) : <EOL> return self . do_open ( httplib . HTTPConnection , req ) <EOL> http_request = AbstractHTTPHandler . do_request_ <EOL> if hasattr ( httplib , '<STR_LIT>' ) : <EOL> class HTTPSHandler ( AbstractHTTPHandler ) : <EOL> def https_open ( self , req ) : <EOL> return self . do_open ( httplib . HTTPSConnection , req ) <EOL> https_request = AbstractHTTPHandler . do_request_ <EOL> class HTTPCookieProcessor ( BaseHandler ) : <EOL> def __init__ ( self , cookiejar = None ) : <EOL> import cookielib <EOL> if cookiejar is None : <EOL> cookiejar = cookielib . CookieJar ( ) <EOL> self . cookiejar = cookiejar <EOL> def http_request ( self , request ) : <EOL> self . cookiejar . add_cookie_header ( request ) <EOL> return request <EOL> def http_response ( self , request , response ) : <EOL> self . cookiejar . extract_cookies ( response , request ) <EOL> return response <EOL> https_request = http_request <EOL> https_response = http_response <EOL> class UnknownHandler ( BaseHandler ) : <EOL> def unknown_open ( self , req ) : <EOL> type = req . get_type ( ) <EOL> raise URLError ( '<STR_LIT>' % type ) <EOL> def parse_keqv_list ( l ) : <EOL> """<STR_LIT>""" <EOL> parsed = { } <EOL> for elt in l : <EOL> k , v = elt . split ( '<STR_LIT:=>' , <NUM_LIT:1> ) <EOL> if v [ <NUM_LIT:0> ] == '<STR_LIT:">' and v [ - <NUM_LIT:1> ] == '<STR_LIT:">' : <EOL> v = v [ <NUM_LIT:1> : - <NUM_LIT:1> ] <EOL> parsed [ k ] = v <EOL> return parsed <EOL> def parse_http_list ( s ) : <EOL> """<STR_LIT>""" <EOL> res = [ ] <EOL> part = '<STR_LIT>' <EOL> escape = quote = False <EOL> for cur in s : <EOL> if escape : <EOL> part += cur <EOL> escape = False <EOL> continue <EOL> if quote : <EOL> if cur == '<STR_LIT:\\>' : <EOL> escape = True <EOL> continue <EOL> elif cur == '<STR_LIT:">' : <EOL> quote = False <EOL> part += cur <EOL> continue <EOL> if cur == '<STR_LIT:U+002C>' : <EOL> res . append ( part ) <EOL> part = '<STR_LIT>' <EOL> continue <EOL> if cur == '<STR_LIT:">' : <EOL> quote = True <EOL> part += cur <EOL> if part : <EOL> res . append ( part ) <EOL> return [ part . strip ( ) for part in res ] <EOL> def _safe_gethostbyname ( host ) : <EOL> try : <EOL> return socket . gethostbyname ( host ) <EOL> except socket . gaierror : <EOL> return None <EOL> class FileHandler ( BaseHandler ) : <EOL> def file_open ( self , req ) : <EOL> url = req . get_selector ( ) <EOL> if url [ : <NUM_LIT:2> ] == '<STR_LIT>' and url [ <NUM_LIT:2> : <NUM_LIT:3> ] != '<STR_LIT:/>' and ( req . host and <EOL> req . host != '<STR_LIT:localhost>' ) : <EOL> req . type = '<STR_LIT>' <EOL> return self . parent . open ( req ) <EOL> else : <EOL> return self . open_local_file ( req ) <EOL> names = None <EOL> def get_names ( self ) : <EOL> if FileHandler . names is None : <EOL> try : <EOL> FileHandler . names = tuple ( <EOL> socket . gethostbyname_ex ( '<STR_LIT:localhost>' ) [ <NUM_LIT:2> ] + <EOL> socket . gethostbyname_ex ( socket . gethostname ( ) ) [ <NUM_LIT:2> ] ) <EOL> except socket . gaierror : <EOL> FileHandler . names = ( socket . gethostbyname ( '<STR_LIT:localhost>' ) , ) <EOL> return FileHandler . names <EOL> def open_local_file ( self , req ) : <EOL> import email . utils <EOL> import mimetypes <EOL> host = req . get_host ( ) <EOL> filename = req . get_selector ( ) <EOL> localfile = url2pathname ( filename ) <EOL> try : <EOL> stats = os . stat ( localfile ) <EOL> size = stats . st_size <EOL> modified = email . utils . formatdate ( stats . st_mtime , usegmt = True ) <EOL> mtype = mimetypes . guess_type ( filename ) [ <NUM_LIT:0> ] <EOL> headers = mimetools . Message ( StringIO ( <EOL> '<STR_LIT>' % <EOL> ( mtype or '<STR_LIT>' , size , modified ) ) ) <EOL> if host : <EOL> host , port = splitport ( host ) <EOL> if not host or ( not port and _safe_gethostbyname ( host ) in self . get_names ( ) ) : <EOL> if host : <EOL> origurl = '<STR_LIT>' + host + filename <EOL> else : <EOL> origurl = '<STR_LIT>' + filename <EOL> return addinfourl ( open ( localfile , '<STR_LIT:rb>' ) , headers , origurl ) <EOL> except OSError , msg : <EOL> raise URLError ( msg ) <EOL> raise URLError ( '<STR_LIT>' ) <EOL> class FTPHandler ( BaseHandler ) : <EOL> def ftp_open ( self , req ) : <EOL> import ftplib <EOL> import mimetypes <EOL> host = req . get_host ( ) <EOL> if not host : <EOL> raise URLError ( '<STR_LIT>' ) <EOL> host , port = splitport ( host ) <EOL> if port is None : <EOL> port = ftplib . FTP_PORT <EOL> else : <EOL> port = int ( port ) <EOL> user , host = splituser ( host ) <EOL> if user : <EOL> user , passwd = splitpasswd ( user ) <EOL> else : <EOL> passwd = None <EOL> host = unquote ( host ) <EOL> user = user or '<STR_LIT>' <EOL> passwd = passwd or '<STR_LIT>' <EOL> try : <EOL> host = socket . gethostbyname ( host ) <EOL> except socket . error , msg : <EOL> raise URLError ( msg ) <EOL> path , attrs = splitattr ( req . get_selector ( ) ) <EOL> dirs = path . split ( '<STR_LIT:/>' ) <EOL> dirs = map ( unquote , dirs ) <EOL> dirs , file = dirs [ : - <NUM_LIT:1> ] , dirs [ - <NUM_LIT:1> ] <EOL> if dirs and not dirs [ <NUM_LIT:0> ] : <EOL> dirs = dirs [ <NUM_LIT:1> : ] <EOL> try : <EOL> fw = self . connect_ftp ( user , passwd , host , port , dirs , req . timeout ) <EOL> type = file and '<STR_LIT:I>' or '<STR_LIT:D>' <EOL> for attr in attrs : <EOL> attr , value = splitvalue ( attr ) <EOL> if attr . lower ( ) == '<STR_LIT:type>' and value in ( '<STR_LIT:a>' , '<STR_LIT:A>' , '<STR_LIT:i>' , '<STR_LIT:I>' , '<STR_LIT:d>' , '<STR_LIT:D>' ) : <EOL> type = value . upper ( ) <EOL> fp , retrlen = fw . retrfile ( file , type ) <EOL> headers = "<STR_LIT>" <EOL> mtype = mimetypes . guess_type ( req . get_full_url ( ) ) [ <NUM_LIT:0> ] <EOL> if mtype : <EOL> headers += "<STR_LIT>" % mtype <EOL> if retrlen is not None and retrlen >= <NUM_LIT:0> : <EOL> headers += "<STR_LIT>" % retrlen <EOL> sf = StringIO ( headers ) <EOL> headers = mimetools . Message ( sf ) <EOL> return addinfourl ( fp , headers , req . get_full_url ( ) ) <EOL> except ftplib . all_errors , msg : <EOL> raise URLError , ( '<STR_LIT>' % msg ) , sys . exc_info ( ) [ <NUM_LIT:2> ] <EOL> def connect_ftp ( self , user , passwd , host , port , dirs , timeout ) : <EOL> fw = ftpwrapper ( user , passwd , host , port , dirs , timeout , <EOL> persistent = False ) <EOL> return fw <EOL> class CacheFTPHandler ( FTPHandler ) : <EOL> def __init__ ( self ) : <EOL> self . cache = { } <EOL> self . timeout = { } <EOL> self . soonest = <NUM_LIT:0> <EOL> self . delay = <NUM_LIT> <EOL> self . max_conns = <NUM_LIT:16> <EOL> def setTimeout ( self , t ) : <EOL> self . delay = t <EOL> def setMaxConns ( self , m ) : <EOL> self . max_conns = m <EOL> def connect_ftp ( self , user , passwd , host , port , dirs , timeout ) : <EOL> key = user , host , port , '<STR_LIT:/>' . join ( dirs ) , timeout <EOL> if key in self . cache : <EOL> self . timeout [ key ] = time . time ( ) + self . delay <EOL> else : <EOL> self . cache [ key ] = ftpwrapper ( user , passwd , host , port , dirs , timeout ) <EOL> self . timeout [ key ] = time . time ( ) + self . delay <EOL> self . check_cache ( ) <EOL> return self . cache [ key ] <EOL> def check_cache ( self ) : <EOL> t = time . time ( ) <EOL> if self . soonest <= t : <EOL> for k , v in self . timeout . items ( ) : <EOL> if v < t : <EOL> self . cache [ k ] . close ( ) <EOL> del self . cache [ k ] <EOL> del self . timeout [ k ] <EOL> self . soonest = min ( self . timeout . values ( ) ) <EOL> if len ( self . cache ) == self . max_conns : <EOL> for k , v in self . timeout . items ( ) : <EOL> if v == self . soonest : <EOL> del self . cache [ k ] <EOL> del self . timeout [ k ] <EOL> break <EOL> self . soonest = min ( self . timeout . values ( ) ) <EOL> def clear_cache ( self ) : <EOL> for conn in self . cache . values ( ) : <EOL> conn . close ( ) <EOL> self . cache . clear ( ) <EOL> self . timeout . clear ( ) </s>
