<s> """<STR_LIT>""" <EOL> __author__ = "<STR_LIT>" <EOL> import os <EOL> import StringIO <EOL> from . pgen2 import driver , literals , token , tokenize , parse , grammar <EOL> from . import pytree <EOL> from . import pygram <EOL> _PATTERN_GRAMMAR_FILE = os . path . join ( os . path . dirname ( __file__ ) , <EOL> "<STR_LIT>" ) <EOL> class PatternSyntaxError ( Exception ) : <EOL> pass <EOL> def tokenize_wrapper ( input ) : <EOL> """<STR_LIT>""" <EOL> skip = set ( ( token . NEWLINE , token . INDENT , token . DEDENT ) ) <EOL> tokens = tokenize . generate_tokens ( StringIO . StringIO ( input ) . readline ) <EOL> for quintuple in tokens : <EOL> type , value , start , end , line_text = quintuple <EOL> if type not in skip : <EOL> yield quintuple <EOL> class PatternCompiler ( object ) : <EOL> def __init__ ( self , grammar_file = _PATTERN_GRAMMAR_FILE ) : <EOL> """<STR_LIT>""" <EOL> self . grammar = driver . load_grammar ( grammar_file ) <EOL> self . syms = pygram . Symbols ( self . grammar ) <EOL> self . pygrammar = pygram . python_grammar <EOL> self . pysyms = pygram . python_symbols <EOL> self . driver = driver . Driver ( self . grammar , convert = pattern_convert ) <EOL> def compile_pattern ( self , input , debug = False , with_tree = False ) : <EOL> """<STR_LIT>""" <EOL> tokens = tokenize_wrapper ( input ) <EOL> try : <EOL> root = self . driver . parse_tokens ( tokens , debug = debug ) <EOL> except parse . ParseError as e : <EOL> raise PatternSyntaxError ( str ( e ) ) <EOL> if with_tree : <EOL> return self . compile_node ( root ) , root <EOL> else : <EOL> return self . compile_node ( root ) <EOL> def compile_node ( self , node ) : <EOL> """<STR_LIT>""" <EOL> if node . type == self . syms . Matcher : <EOL> node = node . children [ <NUM_LIT:0> ] <EOL> if node . type == self . syms . Alternatives : <EOL> alts = [ self . compile_node ( ch ) for ch in node . children [ : : <NUM_LIT:2> ] ] <EOL> if len ( alts ) == <NUM_LIT:1> : <EOL> return alts [ <NUM_LIT:0> ] <EOL> p = pytree . WildcardPattern ( [ [ a ] for a in alts ] , min = <NUM_LIT:1> , max = <NUM_LIT:1> ) <EOL> return p . optimize ( ) <EOL> if node . type == self . syms . Alternative : <EOL> units = [ self . compile_node ( ch ) for ch in node . children ] <EOL> if len ( units ) == <NUM_LIT:1> : <EOL> return units [ <NUM_LIT:0> ] <EOL> p = pytree . WildcardPattern ( [ units ] , min = <NUM_LIT:1> , max = <NUM_LIT:1> ) <EOL> return p . optimize ( ) <EOL> if node . type == self . syms . NegatedUnit : <EOL> pattern = self . compile_basic ( node . children [ <NUM_LIT:1> : ] ) <EOL> p = pytree . NegatedPattern ( pattern ) <EOL> return p . optimize ( ) <EOL> assert node . type == self . syms . Unit <EOL> name = None <EOL> nodes = node . children <EOL> if len ( nodes ) >= <NUM_LIT:3> and nodes [ <NUM_LIT:1> ] . type == token . EQUAL : <EOL> name = nodes [ <NUM_LIT:0> ] . value <EOL> nodes = nodes [ <NUM_LIT:2> : ] <EOL> repeat = None <EOL> if len ( nodes ) >= <NUM_LIT:2> and nodes [ - <NUM_LIT:1> ] . type == self . syms . Repeater : <EOL> repeat = nodes [ - <NUM_LIT:1> ] <EOL> nodes = nodes [ : - <NUM_LIT:1> ] <EOL> pattern = self . compile_basic ( nodes , repeat ) <EOL> if repeat is not None : <EOL> assert repeat . type == self . syms . Repeater <EOL> children = repeat . children <EOL> child = children [ <NUM_LIT:0> ] <EOL> if child . type == token . STAR : <EOL> min = <NUM_LIT:0> <EOL> max = pytree . HUGE <EOL> elif child . type == token . PLUS : <EOL> min = <NUM_LIT:1> <EOL> max = pytree . HUGE <EOL> elif child . type == token . LBRACE : <EOL> assert children [ - <NUM_LIT:1> ] . type == token . RBRACE <EOL> assert len ( children ) in ( <NUM_LIT:3> , <NUM_LIT:5> ) <EOL> min = max = self . get_int ( children [ <NUM_LIT:1> ] ) <EOL> if len ( children ) == <NUM_LIT:5> : <EOL> max = self . get_int ( children [ <NUM_LIT:3> ] ) <EOL> else : <EOL> assert False <EOL> if min != <NUM_LIT:1> or max != <NUM_LIT:1> : <EOL> pattern = pattern . optimize ( ) <EOL> pattern = pytree . WildcardPattern ( [ [ pattern ] ] , min = min , max = max ) <EOL> if name is not None : <EOL> pattern . name = name <EOL> return pattern . optimize ( ) <EOL> def compile_basic ( self , nodes , repeat = None ) : <EOL> assert len ( nodes ) >= <NUM_LIT:1> <EOL> node = nodes [ <NUM_LIT:0> ] <EOL> if node . type == token . STRING : <EOL> value = unicode ( literals . evalString ( node . value ) ) <EOL> return pytree . LeafPattern ( _type_of_literal ( value ) , value ) <EOL> elif node . type == token . NAME : <EOL> value = node . value <EOL> if value . isupper ( ) : <EOL> if value not in TOKEN_MAP : <EOL> raise PatternSyntaxError ( "<STR_LIT>" % value ) <EOL> if nodes [ <NUM_LIT:1> : ] : <EOL> raise PatternSyntaxError ( "<STR_LIT>" ) <EOL> return pytree . LeafPattern ( TOKEN_MAP [ value ] ) <EOL> else : <EOL> if value == "<STR_LIT>" : <EOL> type = None <EOL> elif not value . startswith ( "<STR_LIT:_>" ) : <EOL> type = getattr ( self . pysyms , value , None ) <EOL> if type is None : <EOL> raise PatternSyntaxError ( "<STR_LIT>" % value ) <EOL> if nodes [ <NUM_LIT:1> : ] : <EOL> content = [ self . compile_node ( nodes [ <NUM_LIT:1> ] . children [ <NUM_LIT:1> ] ) ] <EOL> else : <EOL> content = None <EOL> return pytree . NodePattern ( type , content ) <EOL> elif node . value == "<STR_LIT:(>" : <EOL> return self . compile_node ( nodes [ <NUM_LIT:1> ] ) <EOL> elif node . value == "<STR_LIT:[>" : <EOL> assert repeat is None <EOL> subpattern = self . compile_node ( nodes [ <NUM_LIT:1> ] ) <EOL> return pytree . WildcardPattern ( [ [ subpattern ] ] , min = <NUM_LIT:0> , max = <NUM_LIT:1> ) <EOL> assert False , node <EOL> def get_int ( self , node ) : <EOL> assert node . type == token . NUMBER <EOL> return int ( node . value ) <EOL> TOKEN_MAP = { "<STR_LIT>" : token . NAME , <EOL> "<STR_LIT>" : token . STRING , <EOL> "<STR_LIT>" : token . NUMBER , <EOL> "<STR_LIT>" : None } <EOL> def _type_of_literal ( value ) : <EOL> if value [ <NUM_LIT:0> ] . isalpha ( ) : <EOL> return token . NAME <EOL> elif value in grammar . opmap : <EOL> return grammar . opmap [ value ] <EOL> else : <EOL> return None <EOL> def pattern_convert ( grammar , raw_node_info ) : <EOL> """<STR_LIT>""" <EOL> type , value , context , children = raw_node_info <EOL> if children or type in grammar . number2symbol : <EOL> return pytree . Node ( type , children , context = context ) <EOL> else : <EOL> return pytree . Leaf ( type , value , context = context ) <EOL> def compile_pattern ( pattern ) : <EOL> return PatternCompiler ( ) . compile_pattern ( pattern ) </s>
