<s> import itertools <EOL> import logging <EOL> import re <EOL> import sys <EOL> import urlparse <EOL> import html5lib <EOL> from html5lib . sanitizer import HTMLSanitizer <EOL> from html5lib . serializer . htmlserializer import HTMLSerializer <EOL> from encoding import force_unicode <EOL> from sanitizer import BleachSanitizer <EOL> VERSION = ( <NUM_LIT:1> , <NUM_LIT:1> , <NUM_LIT:1> ) <EOL> __version__ = '<STR_LIT:.>' . join ( map ( str , VERSION ) ) <EOL> __all__ = [ '<STR_LIT>' , '<STR_LIT>' ] <EOL> log = logging . getLogger ( '<STR_LIT>' ) <EOL> ALLOWED_TAGS = [ <EOL> '<STR_LIT:a>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT:b>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT:code>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT:i>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> ] <EOL> ALLOWED_ATTRIBUTES = { <EOL> '<STR_LIT:a>' : [ '<STR_LIT>' , '<STR_LIT:title>' ] , <EOL> '<STR_LIT>' : [ '<STR_LIT:title>' ] , <EOL> '<STR_LIT>' : [ '<STR_LIT:title>' ] , <EOL> } <EOL> ALLOWED_STYLES = [ ] <EOL> TLDS = """<STR_LIT>""" . split ( ) <EOL> TLDS . reverse ( ) <EOL> url_re = re . compile ( <EOL> r"""<STR_LIT>""" % u'<STR_LIT:|>' . join ( TLDS ) , re . VERBOSE | re . UNICODE ) <EOL> proto_re = re . compile ( r'<STR_LIT>' ) <EOL> punct_re = re . compile ( r'<STR_LIT>' ) <EOL> email_re = re . compile ( <EOL> r"""<STR_LIT>""" , <EOL> re . IGNORECASE | re . MULTILINE | re . VERBOSE ) <EOL> NODE_TEXT = <NUM_LIT:4> <EOL> identity = lambda x : x <EOL> def clean ( text , tags = ALLOWED_TAGS , attributes = ALLOWED_ATTRIBUTES , <EOL> styles = ALLOWED_STYLES , strip = False , strip_comments = True ) : <EOL> """<STR_LIT>""" <EOL> if not text : <EOL> return u'<STR_LIT>' <EOL> text = force_unicode ( text ) <EOL> if text . startswith ( u'<STR_LIT>' ) : <EOL> text = u'<STR_LIT:U+0020>' + text <EOL> class s ( BleachSanitizer ) : <EOL> allowed_elements = tags <EOL> allowed_attributes = attributes <EOL> allowed_css_properties = styles <EOL> strip_disallowed_elements = strip <EOL> strip_html_comments = strip_comments <EOL> parser = html5lib . HTMLParser ( tokenizer = s ) <EOL> return _render ( parser . parseFragment ( text ) ) . strip ( ) <EOL> def linkify ( text , nofollow = True , target = None , filter_url = identity , <EOL> filter_text = identity , skip_pre = False , parse_email = False ) : <EOL> """<STR_LIT>""" <EOL> text = force_unicode ( text ) <EOL> if not text : <EOL> return u'<STR_LIT>' <EOL> parser = html5lib . HTMLParser ( tokenizer = HTMLSanitizer ) <EOL> forest = parser . parseFragment ( text ) <EOL> if nofollow : <EOL> rel = u'<STR_LIT>' <EOL> else : <EOL> rel = u'<STR_LIT>' <EOL> def replace_nodes ( tree , new_frag , node ) : <EOL> new_tree = parser . parseFragment ( new_frag ) <EOL> for n in new_tree . childNodes : <EOL> tree . insertBefore ( n , node ) <EOL> tree . removeChild ( node ) <EOL> def strip_wrapping_parentheses ( fragment ) : <EOL> """<STR_LIT>""" <EOL> opening_parentheses = closing_parentheses = <NUM_LIT:0> <EOL> for char in fragment : <EOL> if char == '<STR_LIT:(>' : <EOL> opening_parentheses += <NUM_LIT:1> <EOL> else : <EOL> break <EOL> if opening_parentheses : <EOL> newer_frag = '<STR_LIT>' <EOL> fragment = fragment [ opening_parentheses : ] <EOL> reverse_fragment = fragment [ : : - <NUM_LIT:1> ] <EOL> skip = False <EOL> for char in reverse_fragment : <EOL> if ( char == '<STR_LIT:)>' and <EOL> closing_parentheses < opening_parentheses and <EOL> not skip ) : <EOL> closing_parentheses += <NUM_LIT:1> <EOL> continue <EOL> elif char != '<STR_LIT:)>' : <EOL> skip = True <EOL> newer_frag += char <EOL> fragment = newer_frag [ : : - <NUM_LIT:1> ] <EOL> return fragment , opening_parentheses , closing_parentheses <EOL> def linkify_nodes ( tree , parse_text = True ) : <EOL> for node in tree . childNodes : <EOL> if node . type == NODE_TEXT and parse_text : <EOL> new_frag = node . toxml ( ) <EOL> if parse_email : <EOL> new_frag = re . sub ( email_re , email_repl , new_frag ) <EOL> if new_frag != node . toxml ( ) : <EOL> replace_nodes ( tree , new_frag , node ) <EOL> linkify_nodes ( tree , False ) <EOL> continue <EOL> new_frag = re . sub ( url_re , link_repl , new_frag ) <EOL> replace_nodes ( tree , new_frag , node ) <EOL> elif node . name == '<STR_LIT:a>' : <EOL> if '<STR_LIT>' in node . attributes : <EOL> if nofollow : <EOL> node . attributes [ '<STR_LIT>' ] = '<STR_LIT>' <EOL> if target is not None : <EOL> node . attributes [ '<STR_LIT:target>' ] = target <EOL> href = node . attributes [ '<STR_LIT>' ] <EOL> node . attributes [ '<STR_LIT>' ] = filter_url ( href ) <EOL> elif skip_pre and node . name == '<STR_LIT>' : <EOL> linkify_nodes ( node , False ) <EOL> else : <EOL> linkify_nodes ( node ) <EOL> def email_repl ( match ) : <EOL> repl = u'<STR_LIT>' <EOL> return repl % { '<STR_LIT>' : match . group ( <NUM_LIT:0> ) . replace ( '<STR_LIT:">' , '<STR_LIT>' ) } <EOL> def link_repl ( match ) : <EOL> url = match . group ( <NUM_LIT:0> ) <EOL> open_brackets = close_brackets = <NUM_LIT:0> <EOL> if url . startswith ( '<STR_LIT:(>' ) : <EOL> url , open_brackets , close_brackets = ( <EOL> strip_wrapping_parentheses ( url ) <EOL> ) <EOL> end = u'<STR_LIT>' <EOL> m = re . search ( punct_re , url ) <EOL> if m : <EOL> end = m . group ( <NUM_LIT:0> ) <EOL> url = url [ <NUM_LIT:0> : m . start ( ) ] <EOL> if re . search ( proto_re , url ) : <EOL> href = url <EOL> else : <EOL> href = u'<STR_LIT>' . join ( [ u'<STR_LIT>' , url ] ) <EOL> repl = u'<STR_LIT>' <EOL> attribs = [ rel ] <EOL> if target is not None : <EOL> attribs . append ( '<STR_LIT>' % target ) <EOL> return repl % ( '<STR_LIT:(>' * open_brackets , <EOL> filter_url ( href ) , '<STR_LIT:U+0020>' . join ( attribs ) , filter_text ( url ) , <EOL> end , '<STR_LIT:)>' * close_brackets ) <EOL> linkify_nodes ( forest ) <EOL> return _render ( forest ) <EOL> def delinkify ( text , allow_domains = None , allow_relative = False ) : <EOL> """<STR_LIT>""" <EOL> text = force_unicode ( text ) <EOL> if not text : <EOL> return u'<STR_LIT>' <EOL> parser = html5lib . HTMLParser ( tokenizer = HTMLSanitizer ) <EOL> forest = parser . parseFragment ( text ) <EOL> if allow_domains is None : <EOL> allow_domains = [ ] <EOL> elif isinstance ( allow_domains , basestring ) : <EOL> allow_domains = [ allow_domains ] <EOL> def delinkify_nodes ( tree ) : <EOL> """<STR_LIT>""" <EOL> for node in tree . childNodes : <EOL> if node . name == '<STR_LIT:a>' : <EOL> if '<STR_LIT>' not in node . attributes : <EOL> continue <EOL> parts = urlparse . urlparse ( node . attributes [ '<STR_LIT>' ] ) <EOL> host = parts . hostname <EOL> if any ( _domain_match ( host , d ) for d in allow_domains ) : <EOL> continue <EOL> if host is None and allow_relative : <EOL> continue <EOL> for n in node . childNodes : <EOL> tree . insertBefore ( n , node ) <EOL> tree . removeChild ( node ) <EOL> elif node . type != NODE_TEXT : <EOL> delinkify_nodes ( node ) <EOL> delinkify_nodes ( forest ) <EOL> return _render ( forest ) <EOL> def _domain_match ( test , compare ) : <EOL> test = test . lower ( ) <EOL> compare = compare . lower ( ) <EOL> if '<STR_LIT:*>' not in compare : <EOL> return test == compare <EOL> c = compare . split ( '<STR_LIT:.>' ) [ : : - <NUM_LIT:1> ] <EOL> if '<STR_LIT>' in c and ( c . count ( '<STR_LIT>' ) > <NUM_LIT:1> or not compare . startswith ( '<STR_LIT>' ) ) : <EOL> raise ValidationError ( <EOL> '<STR_LIT>' ) <EOL> t = test . split ( '<STR_LIT:.>' ) [ : : - <NUM_LIT:1> ] <EOL> z = itertools . izip_longest ( c , t ) <EOL> for c , t in z : <EOL> if c == t : <EOL> continue <EOL> elif c == '<STR_LIT:*>' : <EOL> continue <EOL> elif c == '<STR_LIT>' : <EOL> return True <EOL> return False <EOL> return True <EOL> class ValidationError ( ValueError ) : <EOL> pass <EOL> def _render ( tree ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> return force_unicode ( _serialize ( tree ) ) <EOL> except Exception , e : <EOL> log . error ( '<STR_LIT>' % e , exc_info = sys . exc_info ( ) ) <EOL> try : <EOL> return force_unicode ( tree . toxml ( ) ) <EOL> except Exception , e : <EOL> log . error ( '<STR_LIT>' % e , exc_info = sys . exc_info ( ) ) <EOL> return u'<STR_LIT>' <EOL> def _serialize ( domtree ) : <EOL> walker = html5lib . treewalkers . getTreeWalker ( '<STR_LIT>' ) <EOL> stream = walker ( domtree ) <EOL> serializer = HTMLSerializer ( quote_attr_values = True , <EOL> omit_optional_tags = False ) <EOL> return serializer . render ( stream ) </s>
