<s> "<STR_LIT>" <EOL> from . import pytree <EOL> from . pgen2 import grammar , token <EOL> from . pygram import pattern_symbols , python_symbols <EOL> syms = pattern_symbols <EOL> pysyms = python_symbols <EOL> tokens = grammar . opmap <EOL> token_labels = token <EOL> TYPE_ANY = - <NUM_LIT:1> <EOL> TYPE_ALTERNATIVES = - <NUM_LIT:2> <EOL> TYPE_GROUP = - <NUM_LIT:3> <EOL> class MinNode ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , type = None , name = None ) : <EOL> self . type = type <EOL> self . name = name <EOL> self . children = [ ] <EOL> self . leaf = False <EOL> self . parent = None <EOL> self . alternatives = [ ] <EOL> self . group = [ ] <EOL> def __repr__ ( self ) : <EOL> return str ( self . type ) + '<STR_LIT:U+0020>' + str ( self . name ) <EOL> def leaf_to_root ( self ) : <EOL> """<STR_LIT>""" <EOL> node = self <EOL> subp = [ ] <EOL> while node : <EOL> if node . type == TYPE_ALTERNATIVES : <EOL> node . alternatives . append ( subp ) <EOL> if len ( node . alternatives ) == len ( node . children ) : <EOL> subp = [ tuple ( node . alternatives ) ] <EOL> node . alternatives = [ ] <EOL> node = node . parent <EOL> continue <EOL> else : <EOL> node = node . parent <EOL> subp = None <EOL> break <EOL> if node . type == TYPE_GROUP : <EOL> node . group . append ( subp ) <EOL> if len ( node . group ) == len ( node . children ) : <EOL> subp = get_characteristic_subpattern ( node . group ) <EOL> node . group = [ ] <EOL> node = node . parent <EOL> continue <EOL> else : <EOL> node = node . parent <EOL> subp = None <EOL> break <EOL> if node . type == token_labels . NAME and node . name : <EOL> subp . append ( node . name ) <EOL> else : <EOL> subp . append ( node . type ) <EOL> node = node . parent <EOL> return subp <EOL> def get_linear_subpattern ( self ) : <EOL> """<STR_LIT>""" <EOL> for l in self . leaves ( ) : <EOL> subp = l . leaf_to_root ( ) <EOL> if subp : <EOL> return subp <EOL> def leaves ( self ) : <EOL> "<STR_LIT>" <EOL> for child in self . children : <EOL> for x in child . leaves ( ) : <EOL> yield x <EOL> if not self . children : <EOL> yield self <EOL> def reduce_tree ( node , parent = None ) : <EOL> """<STR_LIT>""" <EOL> new_node = None <EOL> if node . type == syms . Matcher : <EOL> node = node . children [ <NUM_LIT:0> ] <EOL> if node . type == syms . Alternatives : <EOL> if len ( node . children ) <= <NUM_LIT:2> : <EOL> new_node = reduce_tree ( node . children [ <NUM_LIT:0> ] , parent ) <EOL> else : <EOL> new_node = MinNode ( type = TYPE_ALTERNATIVES ) <EOL> for child in node . children : <EOL> if node . children . index ( child ) % <NUM_LIT:2> : <EOL> continue <EOL> reduced = reduce_tree ( child , new_node ) <EOL> if reduced is not None : <EOL> new_node . children . append ( reduced ) <EOL> elif node . type == syms . Alternative : <EOL> if len ( node . children ) > <NUM_LIT:1> : <EOL> new_node = MinNode ( type = TYPE_GROUP ) <EOL> for child in node . children : <EOL> reduced = reduce_tree ( child , new_node ) <EOL> if reduced : <EOL> new_node . children . append ( reduced ) <EOL> if not new_node . children : <EOL> new_node = None <EOL> else : <EOL> new_node = reduce_tree ( node . children [ <NUM_LIT:0> ] , parent ) <EOL> elif node . type == syms . Unit : <EOL> if ( isinstance ( node . children [ <NUM_LIT:0> ] , pytree . Leaf ) and <EOL> node . children [ <NUM_LIT:0> ] . value == '<STR_LIT:(>' ) : <EOL> return reduce_tree ( node . children [ <NUM_LIT:1> ] , parent ) <EOL> if ( ( isinstance ( node . children [ <NUM_LIT:0> ] , pytree . Leaf ) and <EOL> node . children [ <NUM_LIT:0> ] . value == '<STR_LIT:[>' ) <EOL> or <EOL> ( len ( node . children ) > <NUM_LIT:1> and <EOL> hasattr ( node . children [ <NUM_LIT:1> ] , "<STR_LIT:value>" ) and <EOL> node . children [ <NUM_LIT:1> ] . value == '<STR_LIT:[>' ) ) : <EOL> return None <EOL> leaf = True <EOL> details_node = None <EOL> alternatives_node = None <EOL> has_repeater = False <EOL> repeater_node = None <EOL> has_variable_name = False <EOL> for child in node . children : <EOL> if child . type == syms . Details : <EOL> leaf = False <EOL> details_node = child <EOL> elif child . type == syms . Repeater : <EOL> has_repeater = True <EOL> repeater_node = child <EOL> elif child . type == syms . Alternatives : <EOL> alternatives_node = child <EOL> if hasattr ( child , '<STR_LIT:value>' ) and child . value == '<STR_LIT:=>' : <EOL> has_variable_name = True <EOL> if has_variable_name : <EOL> name_leaf = node . children [ <NUM_LIT:2> ] <EOL> if hasattr ( name_leaf , '<STR_LIT:value>' ) and name_leaf . value == '<STR_LIT:(>' : <EOL> name_leaf = node . children [ <NUM_LIT:3> ] <EOL> else : <EOL> name_leaf = node . children [ <NUM_LIT:0> ] <EOL> if name_leaf . type == token_labels . NAME : <EOL> if name_leaf . value == '<STR_LIT>' : <EOL> new_node = MinNode ( type = TYPE_ANY ) <EOL> else : <EOL> if hasattr ( token_labels , name_leaf . value ) : <EOL> new_node = MinNode ( type = getattr ( token_labels , name_leaf . value ) ) <EOL> else : <EOL> new_node = MinNode ( type = getattr ( pysyms , name_leaf . value ) ) <EOL> elif name_leaf . type == token_labels . STRING : <EOL> name = name_leaf . value . strip ( "<STR_LIT:'>" ) <EOL> if name in tokens : <EOL> new_node = MinNode ( type = tokens [ name ] ) <EOL> else : <EOL> new_node = MinNode ( type = token_labels . NAME , name = name ) <EOL> elif name_leaf . type == syms . Alternatives : <EOL> new_node = reduce_tree ( alternatives_node , parent ) <EOL> if has_repeater : <EOL> if repeater_node . children [ <NUM_LIT:0> ] . value == '<STR_LIT:*>' : <EOL> new_node = None <EOL> elif repeater_node . children [ <NUM_LIT:0> ] . value == '<STR_LIT:+>' : <EOL> pass <EOL> else : <EOL> raise NotImplementedError <EOL> pass <EOL> if details_node and new_node is not None : <EOL> for child in details_node . children [ <NUM_LIT:1> : - <NUM_LIT:1> ] : <EOL> reduced = reduce_tree ( child , new_node ) <EOL> if reduced is not None : <EOL> new_node . children . append ( reduced ) <EOL> if new_node : <EOL> new_node . parent = parent <EOL> return new_node <EOL> def get_characteristic_subpattern ( subpatterns ) : <EOL> """<STR_LIT>""" <EOL> if not isinstance ( subpatterns , list ) : <EOL> return subpatterns <EOL> if len ( subpatterns ) == <NUM_LIT:1> : <EOL> return subpatterns [ <NUM_LIT:0> ] <EOL> subpatterns_with_names = [ ] <EOL> subpatterns_with_common_names = [ ] <EOL> common_names = [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT:None>' ] <EOL> subpatterns_with_common_chars = [ ] <EOL> common_chars = "<STR_LIT>" <EOL> for subpattern in subpatterns : <EOL> if any ( rec_test ( subpattern , lambda x : type ( x ) is str ) ) : <EOL> if any ( rec_test ( subpattern , <EOL> lambda x : isinstance ( x , str ) and x in common_chars ) ) : <EOL> subpatterns_with_common_chars . append ( subpattern ) <EOL> elif any ( rec_test ( subpattern , <EOL> lambda x : isinstance ( x , str ) and x in common_names ) ) : <EOL> subpatterns_with_common_names . append ( subpattern ) <EOL> else : <EOL> subpatterns_with_names . append ( subpattern ) <EOL> if subpatterns_with_names : <EOL> subpatterns = subpatterns_with_names <EOL> elif subpatterns_with_common_names : <EOL> subpatterns = subpatterns_with_common_names <EOL> elif subpatterns_with_common_chars : <EOL> subpatterns = subpatterns_with_common_chars <EOL> return max ( subpatterns , key = len ) <EOL> def rec_test ( sequence , test_func ) : <EOL> """<STR_LIT>""" <EOL> for x in sequence : <EOL> if isinstance ( x , ( list , tuple ) ) : <EOL> for y in rec_test ( x , test_func ) : <EOL> yield y <EOL> else : <EOL> yield test_func ( x ) </s>
