<s> """<STR_LIT>""" <EOL> import codecs <EOL> import copy <EOL> import htmlentitydefs <EOL> import re <EOL> import _sgmllib_copy as sgmllib <EOL> import _beautifulsoup <EOL> import _form <EOL> from _headersutil import split_header_words , is_html as _is_html <EOL> import _request <EOL> import _rfc3986 <EOL> DEFAULT_ENCODING = "<STR_LIT>" <EOL> COMPRESS_RE = re . compile ( r"<STR_LIT>" ) <EOL> class CachingGeneratorFunction ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , iterable ) : <EOL> self . _cache = [ ] <EOL> self . _iterator = iter ( iterable ) <EOL> def __call__ ( self ) : <EOL> cache = self . _cache <EOL> for item in cache : <EOL> yield item <EOL> for item in self . _iterator : <EOL> cache . append ( item ) <EOL> yield item <EOL> class EncodingFinder : <EOL> def __init__ ( self , default_encoding ) : <EOL> self . _default_encoding = default_encoding <EOL> def encoding ( self , response ) : <EOL> for ct in response . info ( ) . getheaders ( "<STR_LIT>" ) : <EOL> for k , v in split_header_words ( [ ct ] ) [ <NUM_LIT:0> ] : <EOL> if k == "<STR_LIT>" : <EOL> encoding = v <EOL> try : <EOL> codecs . lookup ( v ) <EOL> except LookupError : <EOL> continue <EOL> else : <EOL> return encoding <EOL> return self . _default_encoding <EOL> class ResponseTypeFinder : <EOL> def __init__ ( self , allow_xhtml ) : <EOL> self . _allow_xhtml = allow_xhtml <EOL> def is_html ( self , response , encoding ) : <EOL> ct_hdrs = response . info ( ) . getheaders ( "<STR_LIT>" ) <EOL> url = response . geturl ( ) <EOL> return _is_html ( ct_hdrs , url , self . _allow_xhtml ) <EOL> class Args ( object ) : <EOL> def __init__ ( self , args_map ) : <EOL> self . __dict__ [ "<STR_LIT>" ] = dict ( args_map ) <EOL> def __getattr__ ( self , key ) : <EOL> try : <EOL> return self . dictionary [ key ] <EOL> except KeyError : <EOL> return getattr ( self . __class__ , key ) <EOL> def __setattr__ ( self , key , value ) : <EOL> if key == "<STR_LIT>" : <EOL> raise AttributeError ( ) <EOL> self . dictionary [ key ] = value <EOL> def form_parser_args ( <EOL> select_default = False , <EOL> form_parser_class = None , <EOL> request_class = None , <EOL> backwards_compat = False , <EOL> ) : <EOL> return Args ( locals ( ) ) <EOL> class Link : <EOL> def __init__ ( self , base_url , url , text , tag , attrs ) : <EOL> assert None not in [ url , tag , attrs ] <EOL> self . base_url = base_url <EOL> self . absolute_url = _rfc3986 . urljoin ( base_url , url ) <EOL> self . url , self . text , self . tag , self . attrs = url , text , tag , attrs <EOL> def __cmp__ ( self , other ) : <EOL> try : <EOL> for name in "<STR_LIT:url>" , "<STR_LIT:text>" , "<STR_LIT>" , "<STR_LIT>" : <EOL> if getattr ( self , name ) != getattr ( other , name ) : <EOL> return - <NUM_LIT:1> <EOL> except AttributeError : <EOL> return - <NUM_LIT:1> <EOL> return <NUM_LIT:0> <EOL> def __repr__ ( self ) : <EOL> return "<STR_LIT>" % ( <EOL> self . base_url , self . url , self . text , self . tag , self . attrs ) <EOL> class LinksFactory : <EOL> def __init__ ( self , <EOL> link_parser_class = None , <EOL> link_class = Link , <EOL> urltags = None , <EOL> ) : <EOL> import _pullparser <EOL> if link_parser_class is None : <EOL> link_parser_class = _pullparser . TolerantPullParser <EOL> self . link_parser_class = link_parser_class <EOL> self . link_class = link_class <EOL> if urltags is None : <EOL> urltags = { <EOL> "<STR_LIT:a>" : "<STR_LIT>" , <EOL> "<STR_LIT>" : "<STR_LIT>" , <EOL> "<STR_LIT>" : "<STR_LIT:src>" , <EOL> "<STR_LIT>" : "<STR_LIT:src>" , <EOL> } <EOL> self . urltags = urltags <EOL> self . _response = None <EOL> self . _encoding = None <EOL> def set_response ( self , response , base_url , encoding ) : <EOL> self . _response = response <EOL> self . _encoding = encoding <EOL> self . _base_url = base_url <EOL> def links ( self ) : <EOL> """<STR_LIT>""" <EOL> response = self . _response <EOL> encoding = self . _encoding <EOL> base_url = self . _base_url <EOL> p = self . link_parser_class ( response , encoding = encoding ) <EOL> try : <EOL> for token in p . tags ( * ( self . urltags . keys ( ) + [ "<STR_LIT>" ] ) ) : <EOL> if token . type == "<STR_LIT>" : <EOL> continue <EOL> if token . data == "<STR_LIT>" : <EOL> base_href = dict ( token . attrs ) . get ( "<STR_LIT>" ) <EOL> if base_href is not None : <EOL> base_url = base_href <EOL> continue <EOL> attrs = dict ( token . attrs ) <EOL> tag = token . data <EOL> text = None <EOL> url = attrs . get ( self . urltags [ tag ] ) <EOL> if not url : <EOL> continue <EOL> url = _rfc3986 . clean_url ( url , encoding ) <EOL> if tag == "<STR_LIT:a>" : <EOL> if token . type != "<STR_LIT>" : <EOL> text = p . get_compressed_text ( ( "<STR_LIT>" , tag ) ) <EOL> yield Link ( base_url , url , text , tag , token . attrs ) <EOL> except sgmllib . SGMLParseError , exc : <EOL> raise _form . ParseError ( exc ) <EOL> class FormsFactory : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , <EOL> select_default = False , <EOL> form_parser_class = None , <EOL> request_class = None , <EOL> backwards_compat = False , <EOL> ) : <EOL> self . select_default = select_default <EOL> if form_parser_class is None : <EOL> form_parser_class = _form . FormParser <EOL> self . form_parser_class = form_parser_class <EOL> if request_class is None : <EOL> request_class = _request . Request <EOL> self . request_class = request_class <EOL> self . backwards_compat = backwards_compat <EOL> self . _response = None <EOL> self . encoding = None <EOL> self . global_form = None <EOL> def set_response ( self , response , encoding ) : <EOL> self . _response = response <EOL> self . encoding = encoding <EOL> self . global_form = None <EOL> def forms ( self ) : <EOL> encoding = self . encoding <EOL> forms = _form . ParseResponseEx ( <EOL> self . _response , <EOL> select_default = self . select_default , <EOL> form_parser_class = self . form_parser_class , <EOL> request_class = self . request_class , <EOL> encoding = encoding , <EOL> _urljoin = _rfc3986 . urljoin , <EOL> _urlparse = _rfc3986 . urlsplit , <EOL> _urlunparse = _rfc3986 . urlunsplit , <EOL> ) <EOL> self . global_form = forms [ <NUM_LIT:0> ] <EOL> return forms [ <NUM_LIT:1> : ] <EOL> class TitleFactory : <EOL> def __init__ ( self ) : <EOL> self . _response = self . _encoding = None <EOL> def set_response ( self , response , encoding ) : <EOL> self . _response = response <EOL> self . _encoding = encoding <EOL> def _get_title_text ( self , parser ) : <EOL> import _pullparser <EOL> text = [ ] <EOL> tok = None <EOL> while <NUM_LIT:1> : <EOL> try : <EOL> tok = parser . get_token ( ) <EOL> except _pullparser . NoMoreTokensError : <EOL> break <EOL> if tok . type == "<STR_LIT:data>" : <EOL> text . append ( str ( tok ) ) <EOL> elif tok . type == "<STR_LIT>" : <EOL> t = unescape ( "<STR_LIT>" % tok . data , <EOL> parser . _entitydefs , parser . encoding ) <EOL> text . append ( t ) <EOL> elif tok . type == "<STR_LIT>" : <EOL> t = unescape_charref ( tok . data , parser . encoding ) <EOL> text . append ( t ) <EOL> elif tok . type in [ "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" ] : <EOL> tag_name = tok . data <EOL> if tok . type == "<STR_LIT>" and tag_name == "<STR_LIT:title>" : <EOL> break <EOL> text . append ( str ( tok ) ) <EOL> return COMPRESS_RE . sub ( "<STR_LIT:U+0020>" , "<STR_LIT>" . join ( text ) . strip ( ) ) <EOL> def title ( self ) : <EOL> import _pullparser <EOL> p = _pullparser . TolerantPullParser ( <EOL> self . _response , encoding = self . _encoding ) <EOL> try : <EOL> try : <EOL> p . get_tag ( "<STR_LIT:title>" ) <EOL> except _pullparser . NoMoreTokensError : <EOL> return None <EOL> else : <EOL> return self . _get_title_text ( p ) <EOL> except sgmllib . SGMLParseError , exc : <EOL> raise _form . ParseError ( exc ) <EOL> def unescape ( data , entities , encoding ) : <EOL> if data is None or "<STR_LIT:&>" not in data : <EOL> return data <EOL> def replace_entities ( match ) : <EOL> ent = match . group ( ) <EOL> if ent [ <NUM_LIT:1> ] == "<STR_LIT:#>" : <EOL> return unescape_charref ( ent [ <NUM_LIT:2> : - <NUM_LIT:1> ] , encoding ) <EOL> repl = entities . get ( ent [ <NUM_LIT:1> : - <NUM_LIT:1> ] ) <EOL> if repl is not None : <EOL> repl = unichr ( repl ) <EOL> if type ( repl ) != type ( "<STR_LIT>" ) : <EOL> try : <EOL> repl = repl . encode ( encoding ) <EOL> except UnicodeError : <EOL> repl = ent <EOL> else : <EOL> repl = ent <EOL> return repl <EOL> return re . sub ( r"<STR_LIT>" , replace_entities , data ) <EOL> def unescape_charref ( data , encoding ) : <EOL> name , base = data , <NUM_LIT:10> <EOL> if name . startswith ( "<STR_LIT:x>" ) : <EOL> name , base = name [ <NUM_LIT:1> : ] , <NUM_LIT:16> <EOL> uc = unichr ( int ( name , base ) ) <EOL> if encoding is None : <EOL> return uc <EOL> else : <EOL> try : <EOL> repl = uc . encode ( encoding ) <EOL> except UnicodeError : <EOL> repl = "<STR_LIT>" % data <EOL> return repl <EOL> class MechanizeBs ( _beautifulsoup . BeautifulSoup ) : <EOL> _entitydefs = htmlentitydefs . name2codepoint <EOL> PARSER_MASSAGE = [ ( re . compile ( '<STR_LIT>' ) , <EOL> lambda ( x ) : x . group ( <NUM_LIT:1> ) + '<STR_LIT>' ) , <EOL> ( re . compile ( '<STR_LIT>' ) , <EOL> lambda ( x ) : '<STR_LIT>' + x . group ( <NUM_LIT:1> ) + '<STR_LIT:>>' ) <EOL> ] <EOL> def __init__ ( self , encoding , text = None , avoidParserProblems = True , <EOL> initialTextIsEverything = True ) : <EOL> self . _encoding = encoding <EOL> _beautifulsoup . BeautifulSoup . __init__ ( <EOL> self , text , avoidParserProblems , initialTextIsEverything ) <EOL> def handle_charref ( self , ref ) : <EOL> t = unescape ( "<STR_LIT>" % ref , self . _entitydefs , self . _encoding ) <EOL> self . handle_data ( t ) <EOL> def handle_entityref ( self , ref ) : <EOL> t = unescape ( "<STR_LIT>" % ref , self . _entitydefs , self . _encoding ) <EOL> self . handle_data ( t ) <EOL> def unescape_attrs ( self , attrs ) : <EOL> escaped_attrs = [ ] <EOL> for key , val in attrs : <EOL> val = unescape ( val , self . _entitydefs , self . _encoding ) <EOL> escaped_attrs . append ( ( key , val ) ) <EOL> return escaped_attrs <EOL> class RobustLinksFactory : <EOL> compress_re = COMPRESS_RE <EOL> def __init__ ( self , <EOL> link_parser_class = None , <EOL> link_class = Link , <EOL> urltags = None , <EOL> ) : <EOL> if link_parser_class is None : <EOL> link_parser_class = MechanizeBs <EOL> self . link_parser_class = link_parser_class <EOL> self . link_class = link_class <EOL> if urltags is None : <EOL> urltags = { <EOL> "<STR_LIT:a>" : "<STR_LIT>" , <EOL> "<STR_LIT>" : "<STR_LIT>" , <EOL> "<STR_LIT>" : "<STR_LIT:src>" , <EOL> "<STR_LIT>" : "<STR_LIT:src>" , <EOL> } <EOL> self . urltags = urltags <EOL> self . _bs = None <EOL> self . _encoding = None <EOL> self . _base_url = None <EOL> def set_soup ( self , soup , base_url , encoding ) : <EOL> self . _bs = soup <EOL> self . _base_url = base_url <EOL> self . _encoding = encoding <EOL> def links ( self ) : <EOL> bs = self . _bs <EOL> base_url = self . _base_url <EOL> encoding = self . _encoding <EOL> for ch in bs . recursiveChildGenerator ( ) : <EOL> if ( isinstance ( ch , _beautifulsoup . Tag ) and <EOL> ch . name in self . urltags . keys ( ) + [ "<STR_LIT>" ] ) : <EOL> link = ch <EOL> attrs = bs . unescape_attrs ( link . attrs ) <EOL> attrs_dict = dict ( attrs ) <EOL> if link . name == "<STR_LIT>" : <EOL> base_href = attrs_dict . get ( "<STR_LIT>" ) <EOL> if base_href is not None : <EOL> base_url = base_href <EOL> continue <EOL> url_attr = self . urltags [ link . name ] <EOL> url = attrs_dict . get ( url_attr ) <EOL> if not url : <EOL> continue <EOL> url = _rfc3986 . clean_url ( url , encoding ) <EOL> text = link . fetchText ( lambda t : True ) <EOL> if not text : <EOL> if link . name == "<STR_LIT:a>" : <EOL> text = "<STR_LIT>" <EOL> else : <EOL> text = None <EOL> else : <EOL> text = self . compress_re . sub ( "<STR_LIT:U+0020>" , "<STR_LIT:U+0020>" . join ( text ) . strip ( ) ) <EOL> yield Link ( base_url , url , text , link . name , attrs ) <EOL> class RobustFormsFactory ( FormsFactory ) : <EOL> def __init__ ( self , * args , ** kwds ) : <EOL> args = form_parser_args ( * args , ** kwds ) <EOL> if args . form_parser_class is None : <EOL> args . form_parser_class = _form . RobustFormParser <EOL> FormsFactory . __init__ ( self , ** args . dictionary ) <EOL> def set_response ( self , response , encoding ) : <EOL> self . _response = response <EOL> self . encoding = encoding <EOL> class RobustTitleFactory : <EOL> def __init__ ( self ) : <EOL> self . _bs = self . _encoding = None <EOL> def set_soup ( self , soup , encoding ) : <EOL> self . _bs = soup <EOL> self . _encoding = encoding <EOL> def title ( self ) : <EOL> title = self . _bs . first ( "<STR_LIT:title>" ) <EOL> if title == _beautifulsoup . Null : <EOL> return None <EOL> else : <EOL> inner_html = "<STR_LIT>" . join ( [ str ( node ) for node in title . contents ] ) <EOL> return COMPRESS_RE . sub ( "<STR_LIT:U+0020>" , inner_html . strip ( ) ) <EOL> class Factory : <EOL> """<STR_LIT>""" <EOL> LAZY_ATTRS = [ "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT:title>" , "<STR_LIT>" ] <EOL> def __init__ ( self , forms_factory , links_factory , title_factory , <EOL> encoding_finder = EncodingFinder ( DEFAULT_ENCODING ) , <EOL> response_type_finder = ResponseTypeFinder ( allow_xhtml = False ) , <EOL> ) : <EOL> """<STR_LIT>""" <EOL> self . _forms_factory = forms_factory <EOL> self . _links_factory = links_factory <EOL> self . _title_factory = title_factory <EOL> self . _encoding_finder = encoding_finder <EOL> self . _response_type_finder = response_type_finder <EOL> self . set_response ( None ) <EOL> def set_request_class ( self , request_class ) : <EOL> """<STR_LIT>""" <EOL> self . _forms_factory . request_class = request_class <EOL> def set_response ( self , response ) : <EOL> """<STR_LIT>""" <EOL> self . _response = response <EOL> self . _forms_genf = self . _links_genf = None <EOL> self . _get_title = None <EOL> for name in self . LAZY_ATTRS : <EOL> try : <EOL> delattr ( self , name ) <EOL> except AttributeError : <EOL> pass <EOL> def __getattr__ ( self , name ) : <EOL> if name not in self . LAZY_ATTRS : <EOL> return getattr ( self . __class__ , name ) <EOL> if name == "<STR_LIT>" : <EOL> self . encoding = self . _encoding_finder . encoding ( <EOL> copy . copy ( self . _response ) ) <EOL> return self . encoding <EOL> elif name == "<STR_LIT>" : <EOL> self . is_html = self . _response_type_finder . is_html ( <EOL> copy . copy ( self . _response ) , self . encoding ) <EOL> return self . is_html <EOL> elif name == "<STR_LIT:title>" : <EOL> if self . is_html : <EOL> self . title = self . _title_factory . title ( ) <EOL> else : <EOL> self . title = None <EOL> return self . title <EOL> elif name == "<STR_LIT>" : <EOL> self . forms ( ) <EOL> return self . global_form <EOL> def forms ( self ) : <EOL> """<STR_LIT>""" <EOL> if self . _forms_genf is None : <EOL> try : <EOL> self . _forms_genf = CachingGeneratorFunction ( <EOL> self . _forms_factory . forms ( ) ) <EOL> except : <EOL> self . set_response ( self . _response ) <EOL> raise <EOL> self . global_form = getattr ( <EOL> self . _forms_factory , "<STR_LIT>" , None ) <EOL> return self . _forms_genf ( ) <EOL> def links ( self ) : <EOL> """<STR_LIT>""" <EOL> if self . _links_genf is None : <EOL> try : <EOL> self . _links_genf = CachingGeneratorFunction ( <EOL> self . _links_factory . links ( ) ) <EOL> except : <EOL> self . set_response ( self . _response ) <EOL> raise <EOL> return self . _links_genf ( ) <EOL> class DefaultFactory ( Factory ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , i_want_broken_xhtml_support = False ) : <EOL> Factory . __init__ ( <EOL> self , <EOL> forms_factory = FormsFactory ( ) , <EOL> links_factory = LinksFactory ( ) , <EOL> title_factory = TitleFactory ( ) , <EOL> response_type_finder = ResponseTypeFinder ( <EOL> allow_xhtml = i_want_broken_xhtml_support ) , <EOL> ) <EOL> def set_response ( self , response ) : <EOL> Factory . set_response ( self , response ) <EOL> if response is not None : <EOL> self . _forms_factory . set_response ( <EOL> copy . copy ( response ) , self . encoding ) <EOL> self . _links_factory . set_response ( <EOL> copy . copy ( response ) , response . geturl ( ) , self . encoding ) <EOL> self . _title_factory . set_response ( <EOL> copy . copy ( response ) , self . encoding ) <EOL> class RobustFactory ( Factory ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , i_want_broken_xhtml_support = False , <EOL> soup_class = None ) : <EOL> Factory . __init__ ( <EOL> self , <EOL> forms_factory = RobustFormsFactory ( ) , <EOL> links_factory = RobustLinksFactory ( ) , <EOL> title_factory = RobustTitleFactory ( ) , <EOL> response_type_finder = ResponseTypeFinder ( <EOL> allow_xhtml = i_want_broken_xhtml_support ) , <EOL> ) <EOL> if soup_class is None : <EOL> soup_class = MechanizeBs <EOL> self . _soup_class = soup_class <EOL> def set_response ( self , response ) : <EOL> Factory . set_response ( self , response ) <EOL> if response is not None : <EOL> data = response . read ( ) <EOL> soup = self . _soup_class ( self . encoding , data ) <EOL> self . _forms_factory . set_response ( <EOL> copy . copy ( response ) , self . encoding ) <EOL> self . _links_factory . set_soup ( <EOL> soup , response . geturl ( ) , self . encoding ) <EOL> self . _title_factory . set_soup ( soup , self . encoding ) </s>
