def load_grammar(self, grammar_text, grammar_name='<?>'):
        "Parse grammar_text, verify, and create Grammar object. Display nice messages on error."

        try:
            tree = self.canonize_tree.transform( self.parser.parse(grammar_text+'\n') )
        except UnexpectedCharacters as e:
            context = e.get_context(grammar_text)
            raise GrammarError("Unexpected input at line %d column %d in %s: \n\n%s" %
                               (e.line, e.column, grammar_name, context))
        except UnexpectedToken as e:
            context = e.get_context(grammar_text)
            error = e.match_examples(self.parser.parse, {
                'Unclosed parenthesis': ['a: (\n'],
                'Umatched closing parenthesis': ['a: )\n', 'a: [)\n', 'a: (]\n'],
                'Expecting rule or terminal definition (missing colon)': ['a\n', 'a->\n', 'A->\n', 'a A\n'],
                'Alias expects lowercase name': ['a: -> "a"\n'],
                'Unexpected colon': ['a::\n', 'a: b:\n', 'a: B:\n', 'a: "a":\n'],
                'Misplaced operator': ['a: b??', 'a: b(?)', 'a:+\n', 'a:?\n', 'a:*\n', 'a:|*\n'],
                'Expecting option ("|") or a new rule or terminal definition': ['a:a\n()\n'],
                '%import expects a name': ['%import "a"\n'],
                '%ignore expects a value': ['%ignore %import\n'],
            })
            if error:
                raise GrammarError("%s at line %s column %s\n\n%s" % (error, e.line, e.column, context))
            elif 'STRING' in e.expected:
                raise GrammarError("Expecting a value at line %s column %s\n\n%s" % (e.line, e.column, context))
            raise

        tree = PrepareGrammar().transform(tree)

        # Extract grammar items
        defs = classify(tree.children, lambda c: c.data, lambda c: c.children)
        term_defs = defs.pop('term', [])
        rule_defs = defs.pop('rule', [])
        statements = defs.pop('statement', [])
        assert not defs

        term_defs = [td if len(td)==3 else (td[0], 1, td[1]) for td in term_defs]
        term_defs = [(name.value, (t, int(p))) for name, p, t in term_defs]
        rule_defs = [options_from_rule(*x) for x in rule_defs]

        # Execute statements
        ignore, imports = [], {}
        for (stmt,) in statements:
            if stmt.data == 'ignore':
                t ,= stmt.children
                ignore.append(t)
            elif stmt.data == 'import':
                if len(stmt.children) > 1:
                    path_node, arg1 = stmt.children
                else:
                    path_node, = stmt.children
                    arg1 = None

                if isinstance(arg1, Tree):  # Multi import
                    dotted_path = tuple(path_node.children)
                    names = arg1.children
                    aliases = dict(zip(names, names))  # Can't have aliased multi import, so all aliases will be the same as names
                else:  # Single import
                    dotted_path = tuple(path_node.children[:-1])
                    name = path_node.children[-1]  # Get name from dotted path
                    aliases = {name: arg1 or name}  # Aliases if exist

                if path_node.data == 'import_lib':  # Import from library
                    base_paths = []
                else:  # Relative import
                    if grammar_name == '<string>':  # Import relative to script file path if grammar is coded in script
                        try:
                            base_file = os.path.abspath(sys.modules['__main__'].__file__)
                        except AttributeError:
                            base_file = None
                    else:
                        base_file = grammar_name  # Import relative to grammar file path if external grammar file
                    if base_file:
                        base_paths = [os.path.split(base_file)[0]]
                    else:
                        base_paths = [os.path.abspath(os.path.curdir)]

                try:
                    import_base_paths, import_aliases = imports[dotted_path]
                    assert base_paths == import_base_paths, 'Inconsistent base_paths for %s.' % '.'.join(dotted_path)
                    import_aliases.update(aliases)
                except KeyError:
                    imports[dotted_path] = base_paths, aliases

            elif stmt.data == 'declare':
                for t in stmt.children:
                    term_defs.append([t.value, (None, None)])
            else:
                assert False, stmt

        # import grammars
        for dotted_path, (base_paths, aliases) in imports.items():
            grammar_path = os.path.join(*dotted_path) + EXT
            g = import_grammar(grammar_path, base_paths=base_paths)
            new_td, new_rd = import_from_grammar_into_namespace(g, '__'.join(dotted_path), aliases)

            term_defs += new_td
            rule_defs += new_rd

        # Verify correctness 1
        for name, _ in term_defs:
            if name.startswith('__'):
                raise GrammarError('Names starting with double-underscore are reserved (Error at %s)' % name)

        # Handle ignore tokens
        # XXX A slightly hacky solution. Recognition of %ignore TERMINAL as separate comes from the lexer's
        #     inability to handle duplicate terminals (two names, one value)
        ignore_names = []
        for t in ignore:
            if t.data=='expansions' and len(t.children) == 1:
                t2 ,= t.children
                if t2.data=='expansion' and len(t2.children) == 1:
                    item ,= t2.children
                    if item.data == 'value':
                        item ,= item.children
                        if isinstance(item, Token) and item.type == 'TERMINAL':
                            ignore_names.append(item.value)
                            continue

            name = '__IGNORE_%d'% len(ignore_names)
            ignore_names.append(name)
            term_defs.append((name, (t, 1)))

        # Verify correctness 2
        terminal_names = set()
        for name, _ in term_defs:
            if name in terminal_names:
                raise GrammarError("Terminal '%s' defined more than once" % name)
            terminal_names.add(name)

        if set(ignore_names) > terminal_names:
            raise GrammarError("Terminals %s were marked to ignore but were not defined!" % (set(ignore_names) - terminal_names))

        resolve_term_references(term_defs)

        rules = rule_defs

        rule_names = set()
        for name, _x, _o in rules:
            if name.startswith('__'):
                raise GrammarError('Names starting with double-underscore are reserved (Error at %s)' % name)
            if name in rule_names:
                raise GrammarError("Rule '%s' defined more than once" % name)
            rule_names.add(name)

        for name, expansions, _o in rules:
            used_symbols = {t for x in expansions.find_data('expansion')
                              for t in x.scan_values(lambda t: t.type in ('RULE', 'TERMINAL'))}
            for sym in used_symbols:
                if sym.type == 'TERMINAL':
                    if sym not in terminal_names:
                        raise GrammarError("Token '%s' used but not defined (in rule %s)" % (sym, name))
                else:
                    if sym not in rule_names:
                        raise GrammarError("Rule '%s' used but not defined (in rule %s)" % (sym, name))

        return Grammar(rules, term_defs, ignore_names)
def load_grammar ( self , grammar_text , grammar_name = '<?>' ) : try : tree = self . canonize_tree . transform ( self . parser . parse ( grammar_text + '\n' ) ) except UnexpectedCharacters as e : context = e . get_context ( grammar_text ) raise GrammarError ( "Unexpected input at line %d column %d in %s: \n\n%s" % ( e . line , e . column , grammar_name , context ) ) except UnexpectedToken as e : context = e . get_context ( grammar_text ) error = e . match_examples ( self . parser . parse , { 'Unclosed parenthesis' : [ 'a: (\n' ] , 'Umatched closing parenthesis' : [ 'a: )\n' , 'a: [)\n' , 'a: (]\n' ] , 'Expecting rule or terminal definition (missing colon)' : [ 'a\n' , 'a->\n' , 'A->\n' , 'a A\n' ] , 'Alias expects lowercase name' : [ 'a: -> "a"\n' ] , 'Unexpected colon' : [ 'a::\n' , 'a: b:\n' , 'a: B:\n' , 'a: "a":\n' ] , 'Misplaced operator' : [ 'a: b??' , 'a: b(?)' , 'a:+\n' , 'a:?\n' , 'a:*\n' , 'a:|*\n' ] , 'Expecting option ("|") or a new rule or terminal definition' : [ 'a:a\n()\n' ] , '%import expects a name' : [ '%import "a"\n' ] , '%ignore expects a value' : [ '%ignore %import\n' ] , } ) if error : raise GrammarError ( "%s at line %s column %s\n\n%s" % ( error , e . line , e . column , context ) ) elif 'STRING' in e . expected : raise GrammarError ( "Expecting a value at line %s column %s\n\n%s" % ( e . line , e . column , context ) ) raise tree = PrepareGrammar ( ) . transform ( tree ) defs = classify ( tree . children , lambda c : c . data , lambda c : c . children ) term_defs = defs . pop ( 'term' , [ ] ) rule_defs = defs . pop ( 'rule' , [ ] ) statements = defs . pop ( 'statement' , [ ] ) assert not defs term_defs = [ td if len ( td ) == 3 else ( td [ 0 ] , 1 , td [ 1 ] ) for td in term_defs ] term_defs = [ ( name . value , ( t , int ( p ) ) ) for name , p , t in term_defs ] rule_defs = [ options_from_rule ( * x ) for x in rule_defs ] ignore , imports = [ ] , { } for ( stmt , ) in statements : if stmt . data == 'ignore' : t , = stmt . children ignore . append ( t ) elif stmt . data == 'import' : if len ( stmt . children ) > 1 : path_node , arg1 = stmt . children else : path_node , = stmt . children arg1 = None if isinstance ( arg1 , Tree ) : dotted_path = tuple ( path_node . children ) names = arg1 . children aliases = dict ( zip ( names , names ) ) else : dotted_path = tuple ( path_node . children [ : - 1 ] ) name = path_node . children [ - 1 ] aliases = { name : arg1 or name } if path_node . data == 'import_lib' : base_paths = [ ] else : if grammar_name == '<string>' : try : base_file = os . path . abspath ( sys . modules [ '__main__' ] . __file__ ) except AttributeError : base_file = None else : base_file = grammar_name if base_file : base_paths = [ os . path . split ( base_file ) [ 0 ] ] else : base_paths = [ os . path . abspath ( os . path . curdir ) ] try : import_base_paths , import_aliases = imports [ dotted_path ] assert base_paths == import_base_paths , 'Inconsistent base_paths for %s.' % '.' . join ( dotted_path ) import_aliases . update ( aliases ) except KeyError : imports [ dotted_path ] = base_paths , aliases elif stmt . data == 'declare' : for t in stmt . children : term_defs . append ( [ t . value , ( None , None ) ] ) else : assert False , stmt for dotted_path , ( base_paths , aliases ) in imports . items ( ) : grammar_path = os . path . join ( * dotted_path ) + EXT g = import_grammar ( grammar_path , base_paths = base_paths ) new_td , new_rd = import_from_grammar_into_namespace ( g , '__' . join ( dotted_path ) , aliases ) term_defs += new_td rule_defs += new_rd for name , _ in term_defs : if name . startswith ( '__' ) : raise GrammarError ( 'Names starting with double-underscore are reserved (Error at %s)' % name ) ignore_names = [ ] for t in ignore : if t . data == 'expansions' and len ( t . children ) == 1 : t2 , = t . children if t2 . data == 'expansion' and len ( t2 . children ) == 1 : item , = t2 . children if item . data == 'value' : item , = item . children if isinstance ( item , Token ) and item . type == 'TERMINAL' : ignore_names . append ( item . value ) continue name = '__IGNORE_%d' % len ( ignore_names ) ignore_names . append ( name ) term_defs . append ( ( name , ( t , 1 ) ) ) terminal_names = set ( ) for name , _ in term_defs : if name in terminal_names : raise GrammarError ( "Terminal '%s' defined more than once" % name ) terminal_names . add ( name ) if set ( ignore_names ) > terminal_names : raise GrammarError ( "Terminals %s were marked to ignore but were not defined!" % ( set ( ignore_names ) - terminal_names ) ) resolve_term_references ( term_defs ) rules = rule_defs rule_names = set ( ) for name , _x , _o in rules : if name . startswith ( '__' ) : raise GrammarError ( 'Names starting with double-underscore are reserved (Error at %s)' % name ) if name in rule_names : raise GrammarError ( "Rule '%s' defined more than once" % name ) rule_names . add ( name ) for name , expansions , _o in rules : used_symbols = { t for x in expansions . find_data ( 'expansion' ) for t in x . scan_values ( lambda t : t . type in ( 'RULE' , 'TERMINAL' ) ) } for sym in used_symbols : if sym . type == 'TERMINAL' : if sym not in terminal_names : raise GrammarError ( "Token '%s' used but not defined (in rule %s)" % ( sym , name ) ) else : if sym not in rule_names : raise GrammarError ( "Rule '%s' used but not defined (in rule %s)" % ( sym , name ) ) return Grammar ( rules , term_defs , ignore_names )
Parse grammar_text, verify, and create Grammar object. Display nice messages on error.