<s> """<STR_LIT>""" <EOL> import urllib2 , copy , re , os , urllib <EOL> from _html import DefaultFactory <EOL> import _response <EOL> import _request <EOL> import _rfc3986 <EOL> import _sockettimeout <EOL> from _useragent import UserAgentBase <EOL> __version__ = ( <NUM_LIT:0> , <NUM_LIT:1> , <NUM_LIT:11> , None , None ) <EOL> class BrowserStateError ( Exception ) : pass <EOL> class LinkNotFoundError ( Exception ) : pass <EOL> class FormNotFoundError ( Exception ) : pass <EOL> def sanepathname2url ( path ) : <EOL> urlpath = urllib . pathname2url ( path ) <EOL> if os . name == "<STR_LIT>" and urlpath . startswith ( "<STR_LIT>" ) : <EOL> urlpath = urlpath [ <NUM_LIT:2> : ] <EOL> return urlpath <EOL> class History : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self ) : <EOL> self . _history = [ ] <EOL> def add ( self , request , response ) : <EOL> self . _history . append ( ( request , response ) ) <EOL> def back ( self , n , _response ) : <EOL> response = _response <EOL> while n > <NUM_LIT:0> or response is None : <EOL> try : <EOL> request , response = self . _history . pop ( ) <EOL> except IndexError : <EOL> raise BrowserStateError ( "<STR_LIT>" ) <EOL> n -= <NUM_LIT:1> <EOL> return request , response <EOL> def clear ( self ) : <EOL> del self . _history [ : ] <EOL> def close ( self ) : <EOL> for request , response in self . _history : <EOL> if response is not None : <EOL> response . close ( ) <EOL> del self . _history [ : ] <EOL> class HTTPRefererProcessor ( urllib2 . BaseHandler ) : <EOL> def http_request ( self , request ) : <EOL> if hasattr ( request , "<STR_LIT>" ) : <EOL> request = self . parent . _add_referer_header ( <EOL> request , origin_request = False ) <EOL> return request <EOL> https_request = http_request <EOL> class Browser ( UserAgentBase ) : <EOL> """<STR_LIT>""" <EOL> handler_classes = copy . copy ( UserAgentBase . handler_classes ) <EOL> handler_classes [ "<STR_LIT>" ] = HTTPRefererProcessor <EOL> default_features = copy . copy ( UserAgentBase . default_features ) <EOL> default_features . append ( "<STR_LIT>" ) <EOL> def __init__ ( self , <EOL> factory = None , <EOL> history = None , <EOL> request_class = None , <EOL> ) : <EOL> """<STR_LIT>""" <EOL> self . _handle_referer = True <EOL> if history is None : <EOL> history = History ( ) <EOL> self . _history = history <EOL> if request_class is None : <EOL> if not hasattr ( urllib2 . Request , "<STR_LIT>" ) : <EOL> request_class = _request . Request <EOL> else : <EOL> request_class = urllib2 . Request <EOL> if factory is None : <EOL> factory = DefaultFactory ( ) <EOL> factory . set_request_class ( request_class ) <EOL> self . _factory = factory <EOL> self . request_class = request_class <EOL> self . request = None <EOL> self . _set_response ( None , False ) <EOL> UserAgentBase . __init__ ( self ) <EOL> def close ( self ) : <EOL> UserAgentBase . close ( self ) <EOL> if self . _response is not None : <EOL> self . _response . close ( ) <EOL> if self . _history is not None : <EOL> self . _history . close ( ) <EOL> self . _history = None <EOL> self . form = None <EOL> self . request = self . _response = None <EOL> self . request = self . response = self . set_response = None <EOL> self . geturl = self . reload = self . back = None <EOL> self . clear_history = self . set_cookie = self . links = self . forms = None <EOL> self . viewing_html = self . encoding = self . title = None <EOL> self . select_form = self . click = self . submit = self . click_link = None <EOL> self . follow_link = self . find_link = None <EOL> def set_handle_referer ( self , handle ) : <EOL> """<STR_LIT>""" <EOL> self . _set_handler ( "<STR_LIT>" , handle ) <EOL> self . _handle_referer = bool ( handle ) <EOL> def _add_referer_header ( self , request , origin_request = True ) : <EOL> if self . request is None : <EOL> return request <EOL> scheme = request . get_type ( ) <EOL> original_scheme = self . request . get_type ( ) <EOL> if scheme not in [ "<STR_LIT:http>" , "<STR_LIT>" ] : <EOL> return request <EOL> if not origin_request and not self . request . has_header ( "<STR_LIT>" ) : <EOL> return request <EOL> if ( self . _handle_referer and <EOL> original_scheme in [ "<STR_LIT:http>" , "<STR_LIT>" ] and <EOL> not ( original_scheme == "<STR_LIT>" and scheme != "<STR_LIT>" ) ) : <EOL> parts = _rfc3986 . urlsplit ( self . request . get_full_url ( ) ) <EOL> parts = parts [ : - <NUM_LIT:1> ] + ( None , ) <EOL> referer = _rfc3986 . urlunsplit ( parts ) <EOL> request . add_unredirected_header ( "<STR_LIT>" , referer ) <EOL> return request <EOL> def open_novisit ( self , url , data = None , <EOL> timeout = _sockettimeout . _GLOBAL_DEFAULT_TIMEOUT ) : <EOL> """<STR_LIT>""" <EOL> return self . _mech_open ( url , data , visit = False , timeout = timeout ) <EOL> def open ( self , url , data = None , <EOL> timeout = _sockettimeout . _GLOBAL_DEFAULT_TIMEOUT ) : <EOL> return self . _mech_open ( url , data , timeout = timeout ) <EOL> def _mech_open ( self , url , data = None , update_history = True , visit = None , <EOL> timeout = _sockettimeout . _GLOBAL_DEFAULT_TIMEOUT ) : <EOL> try : <EOL> url . get_full_url <EOL> except AttributeError : <EOL> scheme , authority = _rfc3986 . urlsplit ( url ) [ : <NUM_LIT:2> ] <EOL> if scheme is None : <EOL> if self . _response is None : <EOL> raise BrowserStateError ( <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" ) <EOL> url = _rfc3986 . urljoin ( self . _response . geturl ( ) , url ) <EOL> request = self . _request ( url , data , visit , timeout ) <EOL> visit = request . visit <EOL> if visit is None : <EOL> visit = True <EOL> if visit : <EOL> self . _visit_request ( request , update_history ) <EOL> success = True <EOL> try : <EOL> response = UserAgentBase . open ( self , request , data ) <EOL> except urllib2 . HTTPError , error : <EOL> success = False <EOL> if error . fp is None : <EOL> raise <EOL> response = error <EOL> if visit : <EOL> self . _set_response ( response , False ) <EOL> response = copy . copy ( self . _response ) <EOL> elif response is not None : <EOL> response = _response . upgrade_response ( response ) <EOL> if not success : <EOL> raise response <EOL> return response <EOL> def __str__ ( self ) : <EOL> text = [ ] <EOL> text . append ( "<STR_LIT>" % self . __class__ . __name__ ) <EOL> if self . _response : <EOL> text . append ( "<STR_LIT>" % self . _response . geturl ( ) ) <EOL> else : <EOL> text . append ( "<STR_LIT>" ) <EOL> if self . form : <EOL> text . append ( "<STR_LIT>" % str ( self . form ) ) <EOL> text . append ( "<STR_LIT:>>" ) <EOL> return "<STR_LIT>" . join ( text ) <EOL> def response ( self ) : <EOL> """<STR_LIT>""" <EOL> return copy . copy ( self . _response ) <EOL> def open_local_file ( self , filename ) : <EOL> path = sanepathname2url ( os . path . abspath ( filename ) ) <EOL> url = '<STR_LIT>' + path <EOL> return self . open ( url ) <EOL> def set_response ( self , response ) : <EOL> """<STR_LIT>""" <EOL> self . _set_response ( response , True ) <EOL> def _set_response ( self , response , close_current ) : <EOL> if not ( response is None or <EOL> ( hasattr ( response , "<STR_LIT:info>" ) and hasattr ( response , "<STR_LIT>" ) and <EOL> hasattr ( response , "<STR_LIT>" ) <EOL> ) <EOL> ) : <EOL> raise ValueError ( "<STR_LIT>" ) <EOL> self . form = None <EOL> if response is not None : <EOL> response = _response . upgrade_response ( response ) <EOL> if close_current and self . _response is not None : <EOL> self . _response . close ( ) <EOL> self . _response = response <EOL> self . _factory . set_response ( response ) <EOL> def visit_response ( self , response , request = None ) : <EOL> """<STR_LIT>""" <EOL> if request is None : <EOL> request = _request . Request ( response . geturl ( ) ) <EOL> self . _visit_request ( request , True ) <EOL> self . _set_response ( response , False ) <EOL> def _visit_request ( self , request , update_history ) : <EOL> if self . _response is not None : <EOL> self . _response . close ( ) <EOL> if self . request is not None and update_history : <EOL> self . _history . add ( self . request , self . _response ) <EOL> self . _response = None <EOL> self . request = request <EOL> def geturl ( self ) : <EOL> """<STR_LIT>""" <EOL> if self . _response is None : <EOL> raise BrowserStateError ( "<STR_LIT>" ) <EOL> return self . _response . geturl ( ) <EOL> def reload ( self ) : <EOL> """<STR_LIT>""" <EOL> if self . request is None : <EOL> raise BrowserStateError ( "<STR_LIT>" ) <EOL> if self . _response is not None : <EOL> self . _response . close ( ) <EOL> return self . _mech_open ( self . request , update_history = False ) <EOL> def back ( self , n = <NUM_LIT:1> ) : <EOL> """<STR_LIT>""" <EOL> if self . _response is not None : <EOL> self . _response . close ( ) <EOL> self . request , response = self . _history . back ( n , self . _response ) <EOL> self . set_response ( response ) <EOL> if not response . read_complete : <EOL> return self . reload ( ) <EOL> return copy . copy ( response ) <EOL> def clear_history ( self ) : <EOL> self . _history . clear ( ) <EOL> def set_cookie ( self , cookie_string ) : <EOL> """<STR_LIT>""" <EOL> if self . _response is None : <EOL> raise BrowserStateError ( "<STR_LIT>" ) <EOL> if self . request . get_type ( ) not in [ "<STR_LIT:http>" , "<STR_LIT>" ] : <EOL> raise BrowserStateError ( "<STR_LIT>" <EOL> "<STR_LIT>" ) <EOL> cookiejar = self . _ua_handlers [ "<STR_LIT>" ] . cookiejar <EOL> response = self . response ( ) <EOL> headers = response . info ( ) <EOL> headers [ "<STR_LIT>" ] = cookie_string <EOL> cookiejar . extract_cookies ( response , self . request ) <EOL> def links ( self , ** kwds ) : <EOL> """<STR_LIT>""" <EOL> if not self . viewing_html ( ) : <EOL> raise BrowserStateError ( "<STR_LIT>" ) <EOL> links = self . _factory . links ( ) <EOL> if kwds : <EOL> return self . _filter_links ( links , ** kwds ) <EOL> else : <EOL> return links <EOL> def forms ( self ) : <EOL> """<STR_LIT>""" <EOL> if not self . viewing_html ( ) : <EOL> raise BrowserStateError ( "<STR_LIT>" ) <EOL> return self . _factory . forms ( ) <EOL> def global_form ( self ) : <EOL> """<STR_LIT>""" <EOL> if not self . viewing_html ( ) : <EOL> raise BrowserStateError ( "<STR_LIT>" ) <EOL> return self . _factory . global_form <EOL> def viewing_html ( self ) : <EOL> """<STR_LIT>""" <EOL> if self . _response is None : <EOL> raise BrowserStateError ( "<STR_LIT>" ) <EOL> return self . _factory . is_html <EOL> def encoding ( self ) : <EOL> if self . _response is None : <EOL> raise BrowserStateError ( "<STR_LIT>" ) <EOL> return self . _factory . encoding <EOL> def title ( self ) : <EOL> r"""<STR_LIT>""" <EOL> if not self . viewing_html ( ) : <EOL> raise BrowserStateError ( "<STR_LIT>" ) <EOL> return self . _factory . title <EOL> def select_form ( self , name = None , predicate = None , nr = None ) : <EOL> """<STR_LIT>""" <EOL> if not self . viewing_html ( ) : <EOL> raise BrowserStateError ( "<STR_LIT>" ) <EOL> if ( name is None ) and ( predicate is None ) and ( nr is None ) : <EOL> raise ValueError ( <EOL> "<STR_LIT>" ) <EOL> global_form = self . _factory . global_form <EOL> if nr is None and name is None and predicate is not None and predicate ( global_form ) : <EOL> self . form = global_form <EOL> return <EOL> orig_nr = nr <EOL> for form in self . forms ( ) : <EOL> if name is not None and name != form . name : <EOL> continue <EOL> if predicate is not None and not predicate ( form ) : <EOL> continue <EOL> if nr : <EOL> nr -= <NUM_LIT:1> <EOL> continue <EOL> self . form = form <EOL> break <EOL> else : <EOL> description = [ ] <EOL> if name is not None : description . append ( "<STR_LIT>" % name ) <EOL> if predicate is not None : <EOL> description . append ( "<STR_LIT>" % predicate ) <EOL> if orig_nr is not None : description . append ( "<STR_LIT>" % orig_nr ) <EOL> description = "<STR_LIT:U+002CU+0020>" . join ( description ) <EOL> raise FormNotFoundError ( "<STR_LIT>" + description ) <EOL> def click ( self , * args , ** kwds ) : <EOL> """<STR_LIT>""" <EOL> if not self . viewing_html ( ) : <EOL> raise BrowserStateError ( "<STR_LIT>" ) <EOL> request = self . form . click ( * args , ** kwds ) <EOL> return self . _add_referer_header ( request ) <EOL> def submit ( self , * args , ** kwds ) : <EOL> """<STR_LIT>""" <EOL> return self . open ( self . click ( * args , ** kwds ) ) <EOL> def click_link ( self , link = None , ** kwds ) : <EOL> """<STR_LIT>""" <EOL> if not self . viewing_html ( ) : <EOL> raise BrowserStateError ( "<STR_LIT>" ) <EOL> if not link : <EOL> link = self . find_link ( ** kwds ) <EOL> else : <EOL> if kwds : <EOL> raise ValueError ( <EOL> "<STR_LIT>" ) <EOL> request = self . request_class ( link . absolute_url ) <EOL> return self . _add_referer_header ( request ) <EOL> def follow_link ( self , link = None , ** kwds ) : <EOL> """<STR_LIT>""" <EOL> return self . open ( self . click_link ( link , ** kwds ) ) <EOL> def find_link ( self , ** kwds ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> return self . _filter_links ( self . _factory . links ( ) , ** kwds ) . next ( ) <EOL> except StopIteration : <EOL> raise LinkNotFoundError ( ) <EOL> def __getattr__ ( self , name ) : <EOL> form = self . __dict__ . get ( "<STR_LIT>" ) <EOL> if form is None : <EOL> raise AttributeError ( <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" % ( self . __class__ , name ) ) <EOL> return getattr ( form , name ) <EOL> def _filter_links ( self , links , <EOL> text = None , text_regex = None , <EOL> name = None , name_regex = None , <EOL> url = None , url_regex = None , <EOL> tag = None , <EOL> predicate = None , <EOL> nr = <NUM_LIT:0> <EOL> ) : <EOL> if not self . viewing_html ( ) : <EOL> raise BrowserStateError ( "<STR_LIT>" ) <EOL> found_links = [ ] <EOL> orig_nr = nr <EOL> for link in links : <EOL> if url is not None and url != link . url : <EOL> continue <EOL> if url_regex is not None and not re . search ( url_regex , link . url ) : <EOL> continue <EOL> if ( text is not None and <EOL> ( link . text is None or text != link . text ) ) : <EOL> continue <EOL> if ( text_regex is not None and <EOL> ( link . text is None or not re . search ( text_regex , link . text ) ) ) : <EOL> continue <EOL> if name is not None and name != dict ( link . attrs ) . get ( "<STR_LIT:name>" ) : <EOL> continue <EOL> if name_regex is not None : <EOL> link_name = dict ( link . attrs ) . get ( "<STR_LIT:name>" ) <EOL> if link_name is None or not re . search ( name_regex , link_name ) : <EOL> continue <EOL> if tag is not None and tag != link . tag : <EOL> continue <EOL> if predicate is not None and not predicate ( link ) : <EOL> continue <EOL> if nr : <EOL> nr -= <NUM_LIT:1> <EOL> continue <EOL> yield link <EOL> nr = orig_nr </s>
