<s> import py <EOL> from pypy . rlib . parsing . tree import Nonterminal , Symbol , RPythonVisitor <EOL> from pypy . rlib . parsing . parsing import PackratParser , Symbol , ParseError , Rule <EOL> from pypy . rlib . parsing . ebnfparse import parse_ebnf , make_parse_function <EOL> from pypy . rlib . parsing . test . test_parse import EvaluateVisitor <EOL> from sets import Set <EOL> def test_simple ( ) : <EOL> regexs , rules , transformer = parse_ebnf ( """<STR_LIT>""" ) <EOL> class MyEvalVisitor ( EvaluateVisitor ) : <EOL> def visit_primary ( self , node ) : <EOL> if len ( node . children ) == <NUM_LIT:3> : <EOL> return node . children [ <NUM_LIT:1> ] . visit ( self ) <EOL> return int ( node . children [ <NUM_LIT:0> ] . additional_info ) <EOL> parse = make_parse_function ( regexs , rules ) <EOL> tree = parse ( "<STR_LIT>" ) <EOL> assert tree . visit ( MyEvalVisitor ( ) ) == <NUM_LIT:10> + <NUM_LIT> <EOL> tree = parse ( "<STR_LIT>" ) <EOL> r = tree . visit ( MyEvalVisitor ( ) ) <EOL> assert r == <NUM_LIT> * <NUM_LIT:12> + <NUM_LIT> <EOL> tree = parse ( "<STR_LIT>" ) <EOL> assert tree . visit ( MyEvalVisitor ( ) ) == <NUM_LIT:2> * ( <NUM_LIT:3> + <NUM_LIT:5> * <NUM_LIT:2> * ( <NUM_LIT:2> + <NUM_LIT> ) ) <EOL> tree = parse ( "<STR_LIT>" ) <EOL> def test_long_inline_quotes ( ) : <EOL> regexs , rules , transformer = parse_ebnf ( """<STR_LIT>""" ) <EOL> class MyEvalVisitor ( EvaluateVisitor ) : <EOL> def visit_primary ( self , node ) : <EOL> if len ( node . children ) == <NUM_LIT:3> : <EOL> return node . children [ <NUM_LIT:1> ] . visit ( self ) <EOL> return int ( node . children [ <NUM_LIT:0> ] . additional_info ) <EOL> parse = make_parse_function ( regexs , rules ) <EOL> tree = parse ( "<STR_LIT>" ) <EOL> assert tree . visit ( MyEvalVisitor ( ) ) == <NUM_LIT:10> + <NUM_LIT> <EOL> def test_toast ( ) : <EOL> regexs , rules , ToAST = parse_ebnf ( """<STR_LIT>""" ) <EOL> parse = make_parse_function ( regexs , rules ) <EOL> tree = parse ( "<STR_LIT>" ) <EOL> tree = tree . visit ( ToAST ( ) ) [ <NUM_LIT:0> ] <EOL> assert len ( tree . children ) == <NUM_LIT:2> <EOL> assert tree . children [ <NUM_LIT:0> ] . children [ <NUM_LIT:0> ] . symbol == "<STR_LIT>" <EOL> assert tree . children [ <NUM_LIT:1> ] . symbol == "<STR_LIT>" <EOL> def test_eof ( ) : <EOL> grammar = """<STR_LIT>""" <EOL> regexs , rules , ToAST = parse_ebnf ( grammar ) <EOL> class MyEvalVisitor ( EvaluateVisitor ) : <EOL> def visit_primary ( self , node ) : <EOL> if len ( node . children ) == <NUM_LIT:3> : <EOL> return node . children [ <NUM_LIT:1> ] . visit ( self ) <EOL> return int ( node . children [ <NUM_LIT:0> ] . additional_info ) <EOL> def visit_expr ( self , node ) : <EOL> return node . children [ <NUM_LIT:0> ] . visit ( self ) <EOL> visit_additive0_9 = EvaluateVisitor . visit_additive <EOL> parse = make_parse_function ( regexs , rules , eof = True ) <EOL> tree = parse ( "<STR_LIT>" ) <EOL> assert tree . visit ( MyEvalVisitor ( ) ) == <NUM_LIT:10> + <NUM_LIT> <EOL> py . test . raises ( ParseError , parse , "<STR_LIT>" ) <EOL> grammar += "<STR_LIT>" <EOL> py . test . raises ( ParseError , parse_ebnf , grammar ) <EOL> def test_prolog ( ) : <EOL> regexs , rules , ToAST = parse_ebnf ( """<STR_LIT>""" ) <EOL> parse = make_parse_function ( regexs , rules ) <EOL> tree = parse ( "<STR_LIT>" ) <EOL> assert tree is not None <EOL> tree = parse ( """<STR_LIT>""" ) <EOL> assert tree is not None <EOL> def test_toast_bigger ( ) : <EOL> regexs , rules , ToAST = parse_ebnf ( """<STR_LIT>""" ) <EOL> parse = make_parse_function ( regexs , rules ) <EOL> tree = parse ( "<STR_LIT>" ) <EOL> tree = ToAST ( ) . transform ( tree ) <EOL> assert tree . children [ <NUM_LIT:2> ] . symbol == "<STR_LIT>" <EOL> def test_parser_repr_is_evalable ( ) : <EOL> regexs , rules , ToAST = parse_ebnf ( """<STR_LIT>""" ) <EOL> parser = PackratParser ( rules , rules [ <NUM_LIT:0> ] . nonterminal ) <EOL> s = repr ( parser ) <EOL> print s <EOL> newparser = eval ( s ) <EOL> assert repr ( newparser ) == s <EOL> def test_lexer_end_string_corner_case ( ) : <EOL> regexs , rules , ToAST = parse_ebnf ( """<STR_LIT>""" ) <EOL> parse = make_parse_function ( regexs , rules , eof = True ) <EOL> t = parse ( "<STR_LIT>" ) <EOL> assert t . children [ <NUM_LIT:0> ] . additional_info == "<STR_LIT:2>" <EOL> assert t . children [ <NUM_LIT:1> ] . additional_info == "<STR_LIT:.>" <EOL> def test_escape_quotes ( ) : <EOL> regexs , rules , ToAST = parse_ebnf ( """<STR_LIT>""" ) <EOL> parse = make_parse_function ( regexs , rules , eof = True ) <EOL> t = parse ( '<STR_LIT>' ) <EOL> assert t . children [ <NUM_LIT:0> ] . additional_info == '<STR_LIT>' <EOL> assert t . children [ <NUM_LIT:1> ] . additional_info == '<STR_LIT:">' <EOL> def test_leftrecursion ( ) : <EOL> regexs , rules , ToAST = parse_ebnf ( """<STR_LIT>""" ) <EOL> py . test . raises ( AssertionError , make_parse_function , regexs , rules , True ) <EOL> def test_jsonparse ( ) : <EOL> source = """<STR_LIT>""" <EOL> regexs , rules , ToAST = parse_ebnf ( source ) <EOL> parse = make_parse_function ( regexs , rules , eof = True ) <EOL> t = parse ( """<STR_LIT>""" ) <EOL> t = ToAST ( ) . transform ( t ) <EOL> t = parse ( '<STR_LIT>' ) <EOL> print "<STR_LIT>" . join ( list ( t . dot ( ) ) ) <EOL> t = ToAST ( ) . transform ( t ) <EOL> print "<STR_LIT:\n>" . join ( list ( t . dot ( ) ) ) <EOL> def test_starparse ( ) : <EOL> regexs , rules , ToAST = parse_ebnf ( """<STR_LIT>""" ) <EOL> parse = make_parse_function ( regexs , rules , eof = True ) <EOL> t = parse ( """<STR_LIT>""" ) <EOL> t = ToAST ( ) . transform ( t ) <EOL> assert t . symbol == "<STR_LIT:list>" <EOL> assert len ( t . children ) == <NUM_LIT:3> <EOL> assert [ c . symbol for c in t . children ] == [ "<STR_LIT>" ] * <NUM_LIT:3> <EOL> t = parse ( "<STR_LIT>" ) <EOL> t = ToAST ( ) . transform ( t ) <EOL> assert t . symbol == "<STR_LIT:list>" <EOL> assert len ( t . children ) == <NUM_LIT:1> <EOL> assert [ c . symbol for c in t . children ] == [ "<STR_LIT>" ] * <NUM_LIT:1> <EOL> def test_double_star ( ) : <EOL> regexs , rules , ToAST = parse_ebnf ( """<STR_LIT>""" ) <EOL> parse = make_parse_function ( regexs , rules , eof = True ) <EOL> for s in [ "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT:c>" ] : <EOL> t = parse ( s ) <EOL> t = ToAST ( ) . transform ( t ) <EOL> assert [ c . additional_info for c in t . children ] == s . split ( ) <EOL> def test_transform_star ( ) : <EOL> regexs , rules , ToAST = parse_ebnf ( """<STR_LIT>""" ) <EOL> parse = make_parse_function ( regexs , rules ) <EOL> tree = parse ( "<STR_LIT>" ) <EOL> list_expr = tree . visit ( ToAST ( ) ) [ <NUM_LIT:0> ] . children [ <NUM_LIT:0> ] <EOL> assert list_expr . symbol == '<STR_LIT:list>' <EOL> assert len ( list_expr . children ) == <NUM_LIT:2> <EOL> assert list_expr . children [ <NUM_LIT:0> ] . additional_info == '<STR_LIT:(>' <EOL> assert list_expr . children [ <NUM_LIT:1> ] . additional_info == '<STR_LIT:)>' <EOL> tree = parse ( "<STR_LIT>" ) <EOL> list_expr = ToAST ( ) . transform ( tree ) <EOL> def test_quoting ( ) : <EOL> regexs , rules , ToAST = parse_ebnf ( """<STR_LIT>""" ) <EOL> parse = make_parse_function ( regexs , rules , eof = True ) <EOL> t = parse ( """<STR_LIT>""" ) <EOL> assert len ( t . children ) == <NUM_LIT:3> <EOL> assert t . children [ <NUM_LIT:1> ] . additional_info == "<STR_LIT:\n>" <EOL> def test_check_for_missing_names ( ) : <EOL> regexs , rules , ToAST = parse_ebnf ( """<STR_LIT>""" ) <EOL> excinfo = py . test . raises ( ValueError , make_parse_function , regexs , rules ) <EOL> assert "<STR_LIT>" in str ( excinfo . value ) <EOL> def test_starred_star ( ) : <EOL> regexs , rules , ToAST = parse_ebnf ( """<STR_LIT>""" ) <EOL> parse = make_parse_function ( regexs , rules , eof = True ) <EOL> for s in [ "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" ] : <EOL> t = parse ( s ) <EOL> t = ToAST ( ) . transform ( t ) <EOL> assert [ c . additional_info for c in t . children ] == ( s + "<STR_LIT>" ) . split ( ) <EOL> def test_transform_greater_than ( ) : <EOL> regexs , rules , ToAST = parse_ebnf ( """<STR_LIT>""" ) <EOL> parse = make_parse_function ( regexs , rules ) <EOL> t = parse ( "<STR_LIT>" ) <EOL> t = ToAST ( ) . transform ( t ) <EOL> assert len ( t . children ) == <NUM_LIT:3> <EOL> assert t . children [ <NUM_LIT:0> ] . additional_info == "<STR_LIT:A>" <EOL> assert t . children [ <NUM_LIT:1> ] . additional_info == "<STR_LIT:A>" <EOL> assert t . children [ <NUM_LIT:2> ] . additional_info == "<STR_LIT:c>" <EOL> def test_plus ( ) : <EOL> regexs , rules , ToAST = parse_ebnf ( """<STR_LIT>""" ) <EOL> parse = make_parse_function ( regexs , rules ) <EOL> t = parse ( "<STR_LIT>" ) <EOL> t = ToAST ( ) . transform ( t ) <EOL> assert len ( t . children ) == <NUM_LIT:3> <EOL> assert t . children [ <NUM_LIT:0> ] . additional_info == "<STR_LIT:A>" <EOL> assert t . children [ <NUM_LIT:1> ] . additional_info == "<STR_LIT:A>" <EOL> assert t . children [ <NUM_LIT:2> ] . additional_info == "<STR_LIT:B>" <EOL> py . test . raises ( ParseError , parse , "<STR_LIT:B>" ) <EOL> def test_questionmark ( ) : <EOL> regexs , rules , ToAST = parse_ebnf ( """<STR_LIT>""" ) <EOL> parse = make_parse_function ( regexs , rules ) <EOL> t = parse ( "<STR_LIT>" ) <EOL> py . test . raises ( ParseError , parse , "<STR_LIT>" ) <EOL> t = ToAST ( ) . transform ( t ) <EOL> assert len ( t . children ) == <NUM_LIT:3> <EOL> assert t . children [ <NUM_LIT:0> ] . additional_info == "<STR_LIT:B>" <EOL> assert t . children [ <NUM_LIT:1> ] . additional_info == "<STR_LIT:D>" <EOL> assert t . children [ <NUM_LIT:2> ] . additional_info == "<STR_LIT:E>" <EOL> t = parse ( "<STR_LIT>" ) <EOL> t = ToAST ( ) . transform ( t ) <EOL> assert len ( t . children ) == <NUM_LIT:1> <EOL> assert t . children [ <NUM_LIT:0> ] . additional_info == "<STR_LIT:E>" <EOL> def test_grouping_only_parens ( ) : <EOL> regexs , rules , ToAST = parse_ebnf ( """<STR_LIT>""" ) <EOL> parse = make_parse_function ( regexs , rules ) <EOL> t0 = ToAST ( ) . transform ( parse ( "<STR_LIT>" ) ) <EOL> t1 = ToAST ( ) . transform ( parse ( "<STR_LIT>" ) ) <EOL> assert len ( t0 . children ) == len ( t1 . children ) <EOL> def test_mix_star_and_questionmark ( ) : <EOL> regexs , rules , ToAST = parse_ebnf ( """<STR_LIT>""" ) <EOL> parse = make_parse_function ( regexs , rules ) <EOL> t = ToAST ( ) . transform ( parse ( "<STR_LIT>" ) ) <EOL> assert len ( t . children [ <NUM_LIT:0> ] . children ) == <NUM_LIT:3> <EOL> def test_nest_star_and_questionmark ( ) : <EOL> regexs , rules , ToAST = parse_ebnf ( """<STR_LIT>""" ) <EOL> parse = make_parse_function ( regexs , rules ) <EOL> t = ToAST ( ) . transform ( parse ( "<STR_LIT>" ) ) <EOL> t = ToAST ( ) . transform ( parse ( "<STR_LIT>" ) ) <EOL> def test_clash_literal_nonterminal ( ) : <EOL> regexs , rules , ToAST = parse_ebnf ( """<STR_LIT>""" ) <EOL> parse = make_parse_function ( regexs , rules ) <EOL> py . test . raises ( ParseError , parse , "<STR_LIT>" ) <EOL> parse ( "<STR_LIT>" ) <EOL> def test_example1 ( ) : <EOL> regexs , rules , ToAST = parse_ebnf ( """<STR_LIT>""" ) <EOL> parse = make_parse_function ( regexs , rules ) <EOL> t = parse ( "<STR_LIT>" ) <EOL> t = ToAST ( ) . transform ( t ) <EOL> def test_example2 ( ) : <EOL> regexs , rules , ToAST = parse_ebnf ( """<STR_LIT>""" ) <EOL> parse = make_parse_function ( regexs , rules ) <EOL> t = parse ( "<STR_LIT>" ) <EOL> t = ToAST ( ) . transform ( t ) <EOL> def test_bug ( ) : <EOL> py . test . skip ( "<STR_LIT>" ) <EOL> regexs , rules , ToAST = parse_ebnf ( """<STR_LIT>""" ) <EOL> parse = make_parse_function ( regexs , rules ) <EOL> t = parse ( "<STR_LIT>" ) <EOL> t = ToAST ( ) . transform ( t ) <EOL> def test_empty_production ( ) : <EOL> regexs , rules , ToAST = parse_ebnf ( """<STR_LIT>""" ) <EOL> parse = make_parse_function ( regexs , rules , eof = True ) <EOL> t = parse ( "<STR_LIT:U+0020>" ) <EOL> t = ToAST ( ) . transform ( t ) <EOL> assert isinstance ( t , Nonterminal ) <EOL> assert len ( t . children ) == <NUM_LIT:0> <EOL> t = parse ( "<STR_LIT>" ) <EOL> t = ToAST ( ) . transform ( t ) <EOL> assert len ( t . children ) == <NUM_LIT:6> <EOL> excinfo = py . test . raises ( ParseError , parse , "<STR_LIT:a>" ) </s>
