class BaronToken(BaseBox):
    """
    Represents a syntactically relevant piece of text.

    :param name: A string describing the kind of text represented.
    :param value: The actual text represented.
    :param source_pos: A :class:`SourcePosition` object representing the
                       position of the first character in the source from which
                       this token was generated.
    """
    def __init__(self, name, value, hidden_tokens_before=None, hidden_tokens_after=None):
        self.name = name
        self.value = value
        self.hidden_tokens_before = list(map(self._translate_tokens_to_ast_node, hidden_tokens_before if hidden_tokens_before else []))
        self.hidden_tokens_after = list(map(self._translate_tokens_to_ast_node, hidden_tokens_after if hidden_tokens_after else []))

    def _translate_tokens_to_ast_node(self, token):
        if token[0] == "ENDL":
            return {
                "type": token[0].lower(),
                "value": token[1],
                "indent": token[3][0][1] if len(token) == 4 and token[3] else "",
                "formatting": list(map(self._translate_tokens_to_ast_node, token[2]) if len(token) >= 3 else []),
            }
        if len(token) >= 3:
            return {
                "type": token[0].lower(),
                "value": token[1],
                "formatting": list(map(self._translate_tokens_to_ast_node, token[2]) if len(token) >= 3 else []),
            }
        if token[0] == "COMMENT":
            return {
                "type": token[0].lower(),
                "value": token[1],
                "formatting": [],
            }
        return {
            "type": token[0].lower(),
            "value": token[1],
        }

    def __repr__(self):
        return "Token(%r, %r, %s, %s)" % (self.name, self.value, self.hidden_tokens_before, self.hidden_tokens_after)

    def __eq__(self, other):
        if not isinstance(other, BaronToken):
            return NotImplemented
        return self.name == other.name and self.value == other.value

    def render(self):
        before = "".join([(x["indent"] if x["type"] == "endl" else "") + x["value"] for x in self.hidden_tokens_before])
        after = "".join([(x["indent"] if x["type"] == "endl" else "") + x["value"] for x in self.hidden_tokens_after])
        # print self.hidden_tokens_before, self.value, self.hidden_tokens_after
        return before + self.value + after

    def gettokentype(self):
        """
        Returns the type or name of the token.
        """
        return self.name

    def getstr(self):
        """
        Returns the string represented by this token.
        """
        return self.value class BaronParserGenerator(ParserGenerator):
    def build(self):
        g = Grammar(self.tokens)

        for level, (assoc, terms) in enumerate(self.precedence, 1):
            for term in terms:
                g.set_precedence(term, assoc, level)

        for prod_name, syms, func, precedence in self.productions:
            g.add_production(prod_name, syms, func, precedence)

        g.set_start()

        for unused_term in g.unused_terminals():
            warnings.warn(
                "Token %r is unused" % unused_term,
                ParserGeneratorWarning,
                stacklevel=2
            )
        for unused_prod in g.unused_productions():
            warnings.warn(
                "Production %r is not reachable" % unused_prod,
                ParserGeneratorWarning,
                stacklevel=2
            )

        g.build_lritems()
        g.compute_first()
        g.compute_follow()

        # win32 temp directories are already per-user
        if os.name == "nt":
            cache_file = os.path.join(
                tempfile.gettempdir(),
                "rply-%s-%s-%s.json" % (self.VERSION, self.cache_id, self.compute_grammar_hash(g))
            )
        else:
            cache_file = os.path.join(
                tempfile.gettempdir(),
                "rply-%s-%s-%s-%s.json" % (self.VERSION, os.getuid(), self.cache_id, self.compute_grammar_hash(g))
            )
        table = None
        if os.path.exists(cache_file):
            with open(cache_file) as f:
                try:
                    data = json.load(f)
                except Exception:
                    os.remove(cache_file)
                    data = None

                if data is not None:
                    stat_result = os.fstat(f.fileno())
                    if (
                        os.name == "nt" or (
                            stat_result.st_uid == os.getuid()
                            and stat.S_IMODE(stat_result.st_mode) == 0o0600
                        )
                    ):
                        if self.data_is_valid(g, data):
                            table = LRTable.from_cache(g, data)

        if table is None:
            table = LRTable.from_grammar(g)
            try:
                fd = os.open(cache_file, os.O_RDWR | os.O_CREAT | os.O_EXCL, 0o0600)
            except OSError as e:
                if e.errno != errno.EEXIST:
                    raise
            else:
                with os.fdopen(fd, "w") as f:
                    json.dump(self.serialize_table(table), f)
        # meh :(
        # if table.sr_conflicts:
            # warnings.warn(
                # "%d shift/reduce conflict%s" % (len(table.sr_conflicts), "s" if len(table.sr_conflicts) > 1 else ""),
                # ParserGeneratorWarning,
                # stacklevel=2,
            # )
        # if table.rr_conflicts:
            # warnings.warn(
                # "%d reduce/reduce conflict%s" % (len(table.rr_conflicts), "s" if len(table.rr_conflicts) > 1 else ""),
                # ParserGeneratorWarning,
                # stacklevel=2,
            # )
        return BaronLRParser(table, self.error_handler) TOKENS = (
    (r'[a-zA-Z_]\w*', 'NAME'),
    (r'0', 'INT'),
    (r'[-+]?\d+[eE][-+]?\d+[jJ]', 'FLOAT_EXPONANT_COMPLEX'),
    (r'[-+]?\d+.\d?[eE][-+]?\d+[jJ]', 'FLOAT_EXPONANT_COMPLEX'),
    (r'[-+]?\d?.\d+[eE][-+]?\d+[jJ]', 'FLOAT_EXPONANT_COMPLEX'),
    (r'\d+[eE][-+]?\d*', 'FLOAT_EXPONANT'),
    (r'\d+\.\d*[eE][-+]?\d*', 'FLOAT_EXPONANT'),
    (r'\.\d+[eE][-+]?\d*', 'FLOAT_EXPONANT'),
    (r'\d*\.\d+[jJ]', 'COMPLEX'),
    (r'\d+\.[jJ]', 'COMPLEX'),
    (r'\d+[jJ]', 'COMPLEX'),
    (r'\d+\.', 'FLOAT'),
    (r'\d*[_\d]*\.[_\d]+[lL]?', 'FLOAT'),
    (r'\d+[_\d]+\.[_\d]*[lL]?', 'FLOAT'),
    (r'\.', 'DOT'),
    (r'[1-9]+[_\d]*[lL]', 'LONG'),
    (r'[1-9]+[_\d]*', 'INT'),
    (r'0[xX][\d_a-fA-F]+[lL]?', 'HEXA'),
    (r'(0[oO][0-7]+)|(0[0-7_]*)[lL]?', 'OCTA'),
    (r'0[bB][01_]+[lL]?', 'BINARY'),
    (r'\(', 'LEFT_PARENTHESIS'),
    (r'\)', 'RIGHT_PARENTHESIS'),
    (r':', 'COLON'),
    (r',', 'COMMA'),
    (r';', 'SEMICOLON'),
    (r'@', 'AT'),
    (r'\+', 'PLUS'),
    (r'-', 'MINUS'),
    (r'\*', 'STAR'),
    (r'/', 'SLASH'),
    (r'\|', 'VBAR'),
    (r'&', 'AMPER'),
    (r'@', 'AT'),
    (r'<', 'LESS'),
    (r'>', 'GREATER'),
    (r'=', 'EQUAL'),
    (r'%', 'PERCENT'),
    (r'\[', 'LEFT_SQUARE_BRACKET'),
    (r'\]', 'RIGHT_SQUARE_BRACKET'),
    (r'\{', 'LEFT_BRACKET'),
    (r'\}', 'RIGHT_BRACKET'),
    (r'`', 'BACKQUOTE'),
    (r'==', 'EQUAL_EQUAL'),
    (r'<>', 'NOT_EQUAL'),
    (r'!=', 'NOT_EQUAL'),
    (r'<=', 'LESS_EQUAL'),
    (r'>=', 'GREATER_EQUAL'),
    (r'~', 'TILDE'),
    (r'\^', 'CIRCUMFLEX'),
    (r'<<', 'LEFT_SHIFT'),
    (r'>>', 'RIGHT_SHIFT'),
    (r'\*\*', 'DOUBLE_STAR'),
    (r'\+=', 'PLUS_EQUAL'),
    (r'-=', 'MINUS_EQUAL'),
    (r'@=', 'AT_EQUAL'),
    (r'\*=', 'STAR_EQUAL'),
    (r'/=', 'SLASH_EQUAL'),
    (r'%=', 'PERCENT_EQUAL'),
    (r'&=', 'AMPER_EQUAL'),
    (r'\|=', 'VBAR_EQUAL'),
    (r'\^=', 'CIRCUMFLEX_EQUAL'),
    (r'<<=', 'LEFT_SHIFT_EQUAL'),
    (r'>>=', 'RIGHT_SHIFT_EQUAL'),
    (r'\.\.\.', 'ELLIPSIS'),
    (r'->', 'RIGHT_ARROW'),
    (r'\*\*=', 'DOUBLE_STAR_EQUAL'),
    (r'//', 'DOUBLE_SLASH'),
    (r'//=', 'DOUBLE_SLASH_EQUAL'),
    (r'\n', 'ENDL'),
    (r'\r\n', 'ENDL'),
    (r'#.*', 'COMMENT'),
    (r'(\s|\\\n|\\\r\n)+', 'SPACE'),
    (r'["\'](.|\n|\r)*["\']', 'STRING'),
    (r'[uU]["\'](.|\n|\r)*["\']', 'UNICODE_STRING'),
    (r'[fF]["\'](.|\n|\r)*["\']', 'INTERPOLATED_STRING'),
    (r'[rR]["\'](.|\n|\r)*["\']', 'RAW_STRING'),
    (r'[bB]["\'](.|\n|\r)*["\']', 'BINARY_STRING'),
    (r'[uU][rR]["\'](.|\n|\r)*["\']', 'UNICODE_RAW_STRING'),
    (r'[bB][rR]["\'](.|\n|\r)*["\']', 'BINARY_RAW_STRING'),
    (r'[fF][rR]["\'](.|\n|\r)*["\']', 'INTERPOLATED_RAW_STRING'),
    (r'[rR][fF]["\'](.|\n|\r)*["\']', 'INTERPOLATED_RAW_STRING'),
) def tokenize(sequence, print_function=False):
    return list(tokenize_generator(sequence, print_function)) def tokenize_current_keywords(print_function=False):
    if print_function is True:
        return [x for x in KEYWORDS if x not in ("print", "exec")]
    else:
        return KEYWORDS def include_imports(pg):
    @pg.production("small_stmt : import")
    @pg.production("small_stmt : from_import")
    def separator(pack):
        (statement,) = pack
        return statement

    @pg.production("import : IMPORT dotted_as_names")
    def importeu(pack):
        (import_, dotted_as_names) = pack
        return {
            "type": "import",
            "value": dotted_as_names,
            "first_formatting": import_.hidden_tokens_before,
            "second_formatting": import_.hidden_tokens_after
        }

    @pg.production("from_import : FROM dotted_name IMPORT from_import_target")
    def from_import_with_space(pack):
        (from_, dotted_name, import_, from_import_target) = pack
        return {
            "type": "from_import",
            "targets": from_import_target,
            "first_formatting": from_.hidden_tokens_after,
            "second_formatting": import_.hidden_tokens_before,
            "third_formatting": import_.hidden_tokens_after,
            "value": dotted_name
        }

    @pg.production("from_import_target : name_as_names")
    def from_import_target_name_as_names(pack):
        (name_as_names,) = pack
        return name_as_names

    @pg.production("from_import_target : LEFT_PARENTHESIS name_as_names RIGHT_PARENTHESIS")
    def from_import_parenthesis(pack):
        (left_parenthesis, name_as_names, right_parenthesis) = pack
        return left_parenthesis.hidden_tokens_before +\
            [{"type": "left_parenthesis", "value": "("}] +\
            left_parenthesis.hidden_tokens_after +\
            name_as_names +\
            right_parenthesis.hidden_tokens_before +\
            [{"type": "right_parenthesis", "value": ")"}] +\
            right_parenthesis.hidden_tokens_after

    @pg.production("from_import_target : STAR")
    def from_import_star(pack):
        (star,) = pack
        return [{
            "type": "star",
            "value": "*",
            "first_formatting": star.hidden_tokens_before,
            "second_formatting": star.hidden_tokens_after
        }]

    @pg.production("name_as_names : name_as_names name_as_name")
    def name_as_names_name_as_name(pack):
        (name_as_names, name_as_name) = pack
        return name_as_names + name_as_name

    @pg.production("name_as_names : name_as_name")
    def name_as_names(pack):
        (name_as_name,) = pack
        return name_as_name

    @pg.production("name_as_name : NAME AS NAME")
    def name_as_name_name_as_name(pack):
        (name, as_, name2) = pack
        return [{
            "type": "name_as_name",
            "value": name.value,
            "first_formatting": as_.hidden_tokens_before,
            "second_formatting": as_.hidden_tokens_after,
            "target": name2.value
        }]

    @pg.production("name_as_name : NAME")
    def name_as_name_name(pack):
        (name,) = pack
        return [{
            "type": "name_as_name",
            "value": name.value,
            "target": "",
            "first_formatting": [],
            "second_formatting": []
        }]

    @pg.production("name_as_name : NAME SPACE")
    def name_as_name_name_space(pack):
        (name, space) = pack
        return [{
            "type": "name_as_name",
            "target": None,
            "value": name.value,
            "first_formatting": [],
            "second_formatting": []
        }] + [create_node_from_token(space)]

    @pg.production("name_as_name : comma")
    def name_as_name_comma_space(pack):
        (comma,) = pack
        return [comma]

    @pg.production("dotted_as_names : dotted_as_names comma dotted_as_name")
    def dotted_as_names_dotted_as_names_dotted_as_name(pack):
        (dotted_as_names, comma, dotted_as_names2) = pack
        return dotted_as_names + [comma] + dotted_as_names2

    @pg.production("dotted_as_names : dotted_as_name")
    def dotted_as_names_dotted_as_name(pack):
        (dotted_as_name,) = pack
        return dotted_as_name

    @pg.production("dotted_as_name : dotted_name AS NAME")
    def dotted_as_name_as(pack):
        (dotted_name, as_, name) = pack
        return [{
            "type": "dotted_as_name",
            "value": dotted_name,
            "first_formatting": as_.hidden_tokens_before,
            "second_formatting": as_.hidden_tokens_after,
            "target": name.value,
        }]

    @pg.production("dotted_as_name : dotted_name")
    def dotted_as_name(pack):
        (dotted_name,) = pack
        return [{
            "type": "dotted_as_name",
            "value": dotted_name,
            "first_formatting": [],
            "second_formatting": [],
            "target": ""
        }]

    @pg.production("dotted_name : dotted_name dotted_name_element")
    def dotted_name_elements_element(pack):
        (dotted_name, dotted_name_element) = pack
        return dotted_name + dotted_name_element

    @pg.production("dotted_name : dotted_name_element")
    def dotted_name_element(pack):
        (dotted_name_element,) = pack
        return dotted_name_element

    @pg.production("dotted_name_element : NAME")
    @pg.production("dotted_name_element : SPACE")
    def dotted_name(pack):
        (token,) = pack
        return [create_node_from_token(token)]

    @pg.production("dotted_name_element : DOT")
    def dotted_name_dot(pack):
        (dot,) = pack
        return [{
            "type": "dot",
            "first_formatting": dot.hidden_tokens_before,
            "second_formatting": dot.hidden_tokens_after,
        }]

    @pg.production("dotted_name_element : ELLIPSIS")
    def dotted_name_dot_dot_dot(pack):
        ellipsis = pack[0]
        return [{
            "type": "ellipsis",
            "first_formatting": ellipsis.hidden_tokens_before,
            "second_formatting": ellipsis.hidden_tokens_after,
        }] def include_control_structures(pg):
    @pg.production("try_stmt : TRY COLON suite excepts")
    def try_excepts_stmt(pack):
        (try_, colon, suite, excepts) = pack
        return [{
            "type": "try",
            "value": suite,
            "first_formatting": colon.hidden_tokens_before,
            "second_formatting": colon.hidden_tokens_after,
            "else": {},
            "finally": {},
            "excepts": excepts,
        }]

    @pg.production("try_stmt : TRY COLON suite excepts else_stmt")
    def try_excepts_else_stmt(pack):
        (try_, colon, suite, excepts, else_stmt) = pack
        return [{
            "type": "try",
            "value": suite,
            "first_formatting": colon.hidden_tokens_before,
            "second_formatting": colon.hidden_tokens_after,
            "else": else_stmt,
            "finally": {},
            "excepts": excepts,
        }]

    @pg.production("try_stmt : TRY COLON suite excepts finally_stmt")
    def try_excepts_finally_stmt(pack):
        (try_, colon, suite, excepts, finally_stmt) = pack
        return [{
            "type": "try",
            "value": suite,
            "first_formatting": colon.hidden_tokens_before,
            "second_formatting": colon.hidden_tokens_after,
            "else": {},
            "finally": finally_stmt,
            "excepts": excepts,
        }]

    @pg.production("try_stmt : TRY COLON suite excepts else_stmt finally_stmt")
    def try_excepts_else_finally_stmt(pack):
        (try_, colon, suite, excepts, else_stmt, finally_stmt) = pack
        return [{
            "type": "try",
            "value": suite,
            "first_formatting": colon.hidden_tokens_before,
            "second_formatting": colon.hidden_tokens_after,
            "else": else_stmt,
            "finally": finally_stmt,
            "excepts": excepts,
        }]

    @pg.production("try_stmt : TRY COLON suite finally_stmt")
    def try_stmt(pack):
        (try_, colon, suite, finally_stmt) = pack
        return [{
            "type": "try",
            "value": suite,
            "first_formatting": colon.hidden_tokens_before,
            "second_formatting": colon.hidden_tokens_after,
            "else": {},
            "finally": finally_stmt,
            "excepts": [],
        }]

    @pg.production("excepts : excepts except_stmt")
    def excepts(pack):
        (excepts_, except_stmt) = pack
        return excepts_ + except_stmt

    @pg.production("excepts : except_stmt")
    def excepts_except_stmt(pack):
        (except_stmt,) = pack
        return except_stmt

    @pg.production("except_stmt : EXCEPT test AS test COLON suite")
    def except_as_stmt(pack):
        (except_, test, as_, test2, colon, suite) = pack
        return [{
            "type": "except",
            "first_formatting": except_.hidden_tokens_after,
            "second_formatting": as_.hidden_tokens_before,
            "third_formatting": as_.hidden_tokens_after,
            "fourth_formatting": colon.hidden_tokens_before,
            "fifth_formatting": colon.hidden_tokens_after,
            "delimiter": "as",
            "target": test2,
            "exception": test,
            "value": suite
        }]

    @pg.production("except_stmt : EXCEPT test COMMA test COLON suite")
    def except_comma_stmt(pack):
        (except_, test, comma, test2, colon, suite) = pack
        return [{
            "type": "except",
            "first_formatting": except_.hidden_tokens_after,
            "second_formatting": comma.hidden_tokens_before,
            "third_formatting": comma.hidden_tokens_after,
            "fourth_formatting": colon.hidden_tokens_before,
            "fifth_formatting": colon.hidden_tokens_after,
            "delimiter": ",",
            "target": test2,
            "exception": test,
            "value": suite
        }]

    @pg.production("except_stmt : EXCEPT COLON suite")
    def except_stmt_empty(pack):
        (except_, colon, suite) = pack
        return [{
            "type": "except",
            "first_formatting": except_.hidden_tokens_after,
            "second_formatting": [],
            "third_formatting": [],
            "fourth_formatting": colon.hidden_tokens_before,
            "fifth_formatting": colon.hidden_tokens_after,
            "delimiter": "",
            "target": {},
            "exception": {},
            "value": suite
        }]

    @pg.production("except_stmt : EXCEPT test COLON suite")
    def except_stmt(pack):
        (except_, test, colon, suite) = pack
        return [{
            "type": "except",
            "first_formatting": except_.hidden_tokens_after,
            "second_formatting": [],
            "third_formatting": [],
            "fourth_formatting": colon.hidden_tokens_before,
            "fifth_formatting": colon.hidden_tokens_after,
            "delimiter": "",
            "target": {},
            "exception": test,
            "value": suite
        }]

    @pg.production("finally_stmt : FINALLY COLON suite")
    def finally_stmt(pack):
        (finally_, colon, suite) = pack
        return {
            "type": "finally",
            "value": suite,
            "first_formatting": colon.hidden_tokens_before,
            "second_formatting": colon.hidden_tokens_after,
        }

    @pg.production("else_stmt : ELSE COLON suite")
    def else_stmt(pack):
        (else_, colon, suite) = pack
        return {
            "type": "else",
            "value": suite,
            "first_formatting": else_.hidden_tokens_after,
            "second_formatting": colon.hidden_tokens_after,
        }

    @pg.production("for_stmt : FOR exprlist IN testlist COLON suite")
    def for_stmt(pack,):
        (for_, exprlist, in_, testlist, colon, suite) = pack
        return [{
            "type": "for",
            "async": False,
            "async_formatting": [] + for_.hidden_tokens_before,
            "value": suite,
            "iterator": exprlist,
            "target": testlist,
            "else": {},
            "first_formatting": for_.hidden_tokens_after,
            "second_formatting": in_.hidden_tokens_before,
            "third_formatting": in_.hidden_tokens_after,
            "fourth_formatting": colon.hidden_tokens_before,
            "fifth_formatting": colon.hidden_tokens_after,
        }]

    @pg.production("for_stmt : FOR exprlist IN testlist COLON suite else_stmt")
    def for_else_stmt(pack,):
        (for_, exprlist, in_, testlist, colon, suite, else_stmt) = pack
        return [{
            "type": "for",
            "value": suite,
            "async": False,
            "async_formatting": [] + for_.hidden_tokens_before,
            "iterator": exprlist,
            "target": testlist,
            "else": else_stmt,
            "first_formatting": for_.hidden_tokens_after,
            "second_formatting": in_.hidden_tokens_before,
            "third_formatting": in_.hidden_tokens_after,
            "fourth_formatting": colon.hidden_tokens_before,
            "fifth_formatting": colon.hidden_tokens_after,
        }]

    @pg.production("while_stmt : WHILE test COLON suite")
    def while_stmt(pack):
        (while_, test, colon, suite) = pack
        return [{
            "type": "while",
            "value": suite,
            "test": test,
            "else": {},
            "first_formatting": while_.hidden_tokens_after,
            "second_formatting": colon.hidden_tokens_before,
            "third_formatting": colon.hidden_tokens_after,
        }]

    @pg.production("while_stmt : WHILE test COLON suite else_stmt")
    def while_stmt_else(pack):
        (while_, test, colon, suite, else_stmt) = pack
        return [{
            "type": "while",
            "value": suite,
            "test": test,
            "else": else_stmt,
            "first_formatting": while_.hidden_tokens_after,
            "second_formatting": colon.hidden_tokens_before,
            "third_formatting": colon.hidden_tokens_after,
        }]

    @pg.production("if_stmt : IF test COLON suite")
    def if_stmt(pack):
        (if_, test, colon, suite) = pack
        return [{
            "type": "ifelseblock",
            "value": [{
                "type": "if",
                "value": suite,
                "test": test,
                "first_formatting": if_.hidden_tokens_after,
                "second_formatting": colon.hidden_tokens_before,
                "third_formatting": colon.hidden_tokens_after,
            }]
        }]

    @pg.production("if_stmt : IF test COLON suite elifs")
    def if_elif_stmt(pack):
        (if_, test, colon, suite, elifs) = pack
        return [{
            "type": "ifelseblock",
            "value": [{
                "type": "if",
                "value": suite,
                "test": test,
                "first_formatting": if_.hidden_tokens_after,
                "second_formatting": colon.hidden_tokens_before,
                "third_formatting": colon.hidden_tokens_after,
            }] + elifs
        }]

    @pg.production("elifs : elifs ELIF test COLON suite")
    def elifs_elif(pack,):
        (elifs, elif_, test, colon, suite) = pack
        return elifs + [{
            "type": "elif",
            "first_formatting": elif_.hidden_tokens_after,
            "second_formatting": colon.hidden_tokens_before,
            "third_formatting": colon.hidden_tokens_after,
            "value": suite,
            "test": test,
        }]

    @pg.production("elifs : ELIF test COLON suite")
    def elif_(pack,):
        (elif_, test, colon, suite) = pack
        return [{
            "type": "elif",
            "first_formatting": elif_.hidden_tokens_after,
            "second_formatting": colon.hidden_tokens_before,
            "third_formatting": colon.hidden_tokens_after,
            "value": suite,
            "test": test,
        }]

    @pg.production("if_stmt : IF test COLON suite else_stmt")
    def if_else_stmt(pack):
        (if_, test, colon, suite, else_stmt) = pack
        return [{
            "type": "ifelseblock",
            "value": [{
                "type": "if",
                "value": suite,
                "test": test,
                "first_formatting": if_.hidden_tokens_after,
                "second_formatting": colon.hidden_tokens_before,
                "third_formatting": colon.hidden_tokens_after,
            }, else_stmt]
        }]

    @pg.production("if_stmt : IF test COLON suite elifs else_stmt")
    def if_elif_else_stmt(pack):
        (if_, test, colon, suite, elifs, else_stmt) = pack
        return [{
            "type": "ifelseblock",
            "value": [{
                "type": "if",
                "value": suite,
                "test": test,
                "first_formatting": if_.hidden_tokens_after,
                "second_formatting": colon.hidden_tokens_before,
                "third_formatting": colon.hidden_tokens_after,
            }] + elifs + [else_stmt]
        }] def include_primivites(pg, print_function):
    if not print_function:
        @pg.production("print_stmt : PRINT")
        def print_stmt_empty(pack):
            (print_,) = pack
            return {
                "type": "print",
                "value": [],
                "destination": None,
                "destination_formatting": [],
                "formatting": [],
            }

        @pg.production("print_stmt : PRINT testlist")
        def print_stmt(pack):
            (print_, testlist) = pack
            return {
                "type": "print",
                "value": testlist["value"] if testlist["type"] == "tuple" and testlist["with_parenthesis"] is False else [testlist],
                "destination": None,
                "destination_formatting": [],
                "formatting": print_.hidden_tokens_after,
            }

        @pg.production("print_stmt : PRINT RIGHT_SHIFT test")
        def print_stmt_redirect(pack):
            (print_, right_shift, test) = pack
            return {
                "type": "print",
                "value": [],
                "destination": test,
                "destination_formatting": right_shift.hidden_tokens_after,
                "formatting": print_.hidden_tokens_after,
            }

        @pg.production("print_stmt : PRINT RIGHT_SHIFT test COMMA testlist")
        def print_stmt_redirect_testlist(pack):
            (print_, right_shift, test, comma, testlist) = pack
            value = [{
                "type": "comma",
                "first_formatting": comma.hidden_tokens_before,
                "second_formatting": comma.hidden_tokens_after,
            }]
            value += testlist["value"] if testlist["type"] == "tuple" else [testlist]
            return {
                "type": "print",
                "value": value,
                "destination": test,
                "destination_formatting": right_shift.hidden_tokens_after,
                "formatting": print_.hidden_tokens_after,
            }

        @pg.production("assert_stmt : EXEC expr")
        def exec_stmt(pack):
            (exec_, expr) = pack
            return {
                "type": "exec",
                "value": expr,
                "globals": None,
                "locals": None,
                "first_formatting": exec_.hidden_tokens_after,
                "second_formatting": [],
                "third_formatting": [],
                "fourth_formatting": [],
                "fifth_formatting": []
            }

        @pg.production("assert_stmt : EXEC expr IN test")
        def exec_stmt_in(pack):
            (exec_, expr, in_, test) = pack
            return {
                "type": "exec",
                "value": expr,
                "globals": test,
                "locals": None,
                "first_formatting": exec_.hidden_tokens_after,
                "second_formatting": in_.hidden_tokens_before,
                "third_formatting": in_.hidden_tokens_after,
                "fourth_formatting": [],
                "fifth_formatting": []
            }

        @pg.production("assert_stmt : EXEC expr IN test COMMA test")
        def exec_stmt_in_comma(pack):
            (exec_, expr, in_, test, comma, test2) = pack
            return {
                "type": "exec",
                "value": expr,
                "globals": test,
                "locals": test2,
                "first_formatting": exec_.hidden_tokens_after,
                "second_formatting": in_.hidden_tokens_before,
                "third_formatting": in_.hidden_tokens_after,
                "fourth_formatting": comma.hidden_tokens_before,
                "fifth_formatting": comma.hidden_tokens_after
            }

    @pg.production("flow_stmt : return_stmt")
    @pg.production("flow_stmt : break_stmt")
    @pg.production("flow_stmt : continue_stmt")
    @pg.production("flow_stmt : yield_stmt")
    @pg.production("yield_stmt : yield_expr")
    def flow(pack):
        (flow_stmt,) = pack
        return flow_stmt

    @pg.production("return_stmt : RETURN")
    def return_empty(pack):
        (token,) = pack
        return {
            "type": token.name.lower(),
            "value": None,
            "formatting": token.hidden_tokens_after,
        }

    @pg.production("yield_expr : YIELD")
    def yield_expr(pack):
        (yield_,) = pack
        return {
            "type": yield_.name.lower(),
            "value": None,
            "formatting": yield_.hidden_tokens_after,
        }

    @pg.production("break_stmt : BREAK")
    @pg.production("continue_stmt : CONTINUE")
    @pg.production("pass_stmt : PASS")
    def break_stmt(pack):
        (token,) = pack
        return {"type": token.name.lower()}

    @pg.production("raise_stmt : RAISE")
    def raise_stmt_empty(pack):
        (raise_,) = pack
        return {
            "type": "raise",
            "value": None,
            "instance": None,
            "traceback": None,
            "first_formatting": raise_.hidden_tokens_after,
            "second_formatting": [],
            "third_formatting": [],
            "fourth_formatting": [],
            "fifth_formatting": [],
            "comma_or_from": None,
        }

    @pg.production("raise_stmt : RAISE test")
    def raise_stmt(pack):
        (raise_, test) = pack
        return {
            "type": "raise",
            "value": test,
            "instance": None,
            "traceback": None,
            "first_formatting": raise_.hidden_tokens_after,
            "second_formatting": [],
            "third_formatting": [],
            "fourth_formatting": [],
            "fifth_formatting": [],
            "comma_or_from": None,
        }

    @pg.production("raise_stmt : RAISE test FROM test")
    def raise_stmt_from(pack):
        (raise_, test, from_, test2) = pack
        return {
            "type": "raise",
            "value": test,
            "instance": test2,
            "traceback": None,
            "first_formatting": raise_.hidden_tokens_after,
            "second_formatting": from_.hidden_tokens_before,
            "third_formatting": from_.hidden_tokens_after,
            "fourth_formatting": [],
            "fifth_formatting": [],
            "comma_or_from": "from",
        }

    @pg.production("raise_stmt : RAISE test COMMA test")
    def raise_stmt_instance(pack):
        (raise_, test, comma, test2) = pack
        return {
            "type": "raise",
            "value": test,
            "instance": test2,
            "traceback": None,
            "first_formatting": raise_.hidden_tokens_after,
            "second_formatting": comma.hidden_tokens_before,
            "third_formatting": comma.hidden_tokens_after,
            "fourth_formatting": [],
            "fifth_formatting": [],
            "comma_or_from": ",",
        }

    @pg.production("raise_stmt : RAISE test COMMA test COMMA test")
    def raise_stmt_instance_traceback(pack):
        (raise_, test, comma, test2, comma2, test3) = pack
        return {
            "type": "raise",
            "value": test,
            "instance": test2,
            "traceback": test3,
            "first_formatting": raise_.hidden_tokens_after,
            "second_formatting": comma.hidden_tokens_before,
            "third_formatting": comma.hidden_tokens_after,
            "fourth_formatting": comma2.hidden_tokens_before,
            "fifth_formatting": comma2.hidden_tokens_after,
            "comma_or_from": ",",
        }

    @pg.production("assert_stmt : ASSERT test")
    def assert_stmt(pack):
        (assert_, test) = pack
        return {
            "type": "assert",
            "value": test,
            "message": None,
            "first_formatting": assert_.hidden_tokens_after,
            "second_formatting": [],
            "third_formatting": []
        }

    @pg.production("assert_stmt : ASSERT test COMMA test")
    def assert_stmt_message(pack):
        (assert_, test, comma, test2) = pack
        return {
            "type": "assert",
            "value": test,
            "message": test2,
            "first_formatting": assert_.hidden_tokens_after,
            "second_formatting": comma.hidden_tokens_before,
            "third_formatting": comma.hidden_tokens_after
        }

    @pg.production("global_stmt : GLOBAL names")
    def global_stmt(pack):
        (global_, names) = pack
        return {
            "type": "global",
            "formatting": global_.hidden_tokens_after,
            "value": names,
        }

    @pg.production("nonlocal_stmt : NONLOCAL names")
    def nonlocal_stmt(pack):
        (token, names) = pack
        return {
            "type": "nonlocal",
            "formatting": token.hidden_tokens_after,
            "value": names,
        }

    @pg.production("names : NAME")
    def names_name(pack):
        (name,) = pack
        return [create_node_from_token(name)]

    @pg.production("names : names comma name")
    def names_names_name(pack):
        (names, comma, name,) = pack
        return names + [comma, name]

    @pg.production("return_stmt : RETURN testlist")
    @pg.production("yield_expr : YIELD testlist")
    @pg.production("del_stmt : DEL exprlist")
    def return_testlist(pack):
        (token, testlist) = pack
        return {
            "type": token.name.lower(),
            "value": testlist,
            "formatting": token.hidden_tokens_after,
        }

    @pg.production("yield_expr : YIELD FROM test")
    def yield_from_expr(pack):
        (yield_, from_, test) = pack
        return {
            "type": "yield_from",
            "first_formatting": from_.hidden_tokens_after,
            "value": test,
            "formatting": yield_.hidden_tokens_after,
        }

    @pg.production("lambdef : LAMBDA COLON test")
    @pg.production("old_lambdef : LAMBDA COLON old_test")
    def lambdef(pack):
        (lambda_, colon, test) = pack
        return {
            "type": "lambda",
            "arguments": [],
            "first_formatting": lambda_.hidden_tokens_after,
            "second_formatting": colon.hidden_tokens_before,
            "third_formatting": colon.hidden_tokens_after,
            "value": test
        }

    @pg.production("lambdef : LAMBDA parameters COLON test")
    @pg.production("old_lambdef : LAMBDA parameters COLON old_test")
    def lambdef_arguments(pack):
        (lambda_, parameters, colon, test) = pack
        return {
            "type": "lambda",
            "arguments": parameters,
            "first_formatting": lambda_.hidden_tokens_after,
            "second_formatting": colon.hidden_tokens_before,
            "third_formatting": colon.hidden_tokens_after,
            "value": test
        } def include_operators(pg):
    @pg.production("old_test : or_test")
    @pg.production("old_test : old_lambdef")
    def old_test(pack):
        (level,) = pack
        return level

    @pg.production("testlist_safe : old_test comma old_test")
    def testlist_safe(pack):
        (old_test, comma, old_test2) = pack
        return [old_test, comma, old_test2]

    @pg.production("testlist_safe : old_test comma testlist_safe")
    def testlist_safe_more(pack):
        (old_test, comma, testlist_safe) = pack
        return [old_test, comma] + testlist_safe

    @pg.production("expr_stmt : test COLON test")
    def alone_annotation(pack):
        target, colon, annotation = pack
        return {
            "type": "standalone_annotation",
            "target": target,
            "annotation": annotation,  # not called "value" in case someone
                                       # wants to work on both assignment and
                                       # standalone annotations
            "first_formatting": colon.hidden_tokens_before,
            "second_formatting": colon.hidden_tokens_after,
        }

    @pg.production("expr_stmt : test COLON test EQUAL test")
    def augmented_assignment_node(pack):
        target, colon, annotation, equal, test = pack
        return {
            "type": "assignment",
            "first_formatting": equal.hidden_tokens_before if equal else [],
            "second_formatting": equal.hidden_tokens_after if equal else [],
            "target": target,
            "value": test,
            "operator": "",
            "annotation": annotation,
            "annotation_first_formatting": colon.hidden_tokens_before,
            "annotation_second_formatting": colon.hidden_tokens_after,
        }

    @pg.production("expr_stmt : testlist_star_expr augassign_operator testlist")
    @pg.production("expr_stmt : testlist_star_expr augassign_operator yield_expr")
    def augmented_assignment_node_2(pack):
        (target, operator, value) = pack
        return {
            "type": "assignment",
            "first_formatting": operator.hidden_tokens_before,
            "second_formatting": operator.hidden_tokens_after,
            "operator": operator.value[:-1],
            "target": target,
            "value": value,
            "annotation": {},
            "annotation_first_formatting": [],
            "annotation_second_formatting": [],
        }

    @pg.production("test_or_star_expr : test")
    @pg.production("test_or_star_expr : star_expr")
    def test_or_star_expr(pack):
        return pack[0]

    @pg.production("star_expr : STAR expr")
    def star_expr(pack):
        star, expr = pack
        return {
            "type": "star_expression",
            "formatting": star.hidden_tokens_after,
            "value": expr,
        }

    @pg.production("augassign_operator : PLUS_EQUAL")
    @pg.production("augassign_operator : MINUS_EQUAL")
    @pg.production("augassign_operator : STAR_EQUAL")
    @pg.production("augassign_operator : SLASH_EQUAL")
    @pg.production("augassign_operator : PERCENT_EQUAL")
    @pg.production("augassign_operator : AMPER_EQUAL")
    @pg.production("augassign_operator : AT_EQUAL")
    @pg.production("augassign_operator : VBAR_EQUAL")
    @pg.production("augassign_operator : CIRCUMFLEX_EQUAL")
    @pg.production("augassign_operator : LEFT_SHIFT_EQUAL")
    @pg.production("augassign_operator : RIGHT_SHIFT_EQUAL")
    @pg.production("augassign_operator : DOUBLE_STAR_EQUAL")
    @pg.production("augassign_operator : DOUBLE_SLASH_EQUAL")
    def augassign_operator(pack):
        (operator,) = pack
        return operator

    @pg.production("expr_stmt : testlist_star_expr EQUAL yield_expr")
    @pg.production("expr_stmt : testlist_star_expr EQUAL expr_stmt")
    def assignment_node(pack):
        (target, equal, value) = pack
        return {
            "type": "assignment",
            "operator": "",
            "value": value,
            "target": target,
            "first_formatting": equal.hidden_tokens_before,
            "second_formatting": equal.hidden_tokens_after,
            "annotation": {},
            "annotation_first_formatting": [],
            "annotation_second_formatting": [],
        }

    @pg.production("test : or_test IF or_test ELSE test")
    def ternary_operator_node(pack):
        (first, if_, second, else_, third) = pack
        return {
            "type": "ternary_operator",
            "first": first,
            "second": third,
            "value": second,
            "first_formatting": if_.hidden_tokens_before,
            "second_formatting": if_.hidden_tokens_after,
            "third_formatting": else_.hidden_tokens_before,
            "fourth_formatting": else_.hidden_tokens_after,
        }

    @pg.production("or_test : and_test OR or_test")
    @pg.production("and_test : not_test AND and_test")
    def and_or_node(pack):
        (first, operator, second) = pack
        return {
            "type": "boolean_operator",
            "value": operator.value,
            "first": first,
            "second": second,
            "first_formatting": operator.hidden_tokens_before,
            "second_formatting": operator.hidden_tokens_after,
        }

    @pg.production("not_test : NOT not_test")
    def not_node(pack):
        (not_, comparison) = pack
        return {
            "type": "unitary_operator",
            "value": "not",
            "target": comparison,
            "formatting": not_.hidden_tokens_after
        }

    @pg.production("comparison : expr LESS comparison")
    @pg.production("comparison : expr GREATER comparison")
    @pg.production("comparison : expr EQUAL_EQUAL comparison")
    @pg.production("comparison : expr LESS_EQUAL comparison")
    @pg.production("comparison : expr GREATER_EQUAL comparison")
    @pg.production("comparison : expr NOT_EQUAL comparison")
    @pg.production("comparison : expr IN comparison")
    @pg.production("comparison : expr IS comparison")
    def comparison_node(pack):
        (expr, comparison_operator, comparison_) = pack
        return {
            "type": "comparison",
            "first": expr,
            "value": {
                "type": "comparison_operator",
                "first": comparison_operator.value,
                "second": "",
                "formatting": [],
            },
            "second": comparison_,
            "first_formatting": comparison_operator.hidden_tokens_before,
            "second_formatting": comparison_operator.hidden_tokens_after
        }

    @pg.production("comparison : expr IS NOT comparison")
    @pg.production("comparison : expr NOT IN comparison")
    def comparison_advanced_node(pack):
        (expr, comparison_operator, comparison_operator2, comparison_) = pack
        return {
            "type": "comparison",
            "value": {
                "type": "comparison_operator",
                "first": comparison_operator.value,
                "second": comparison_operator2.value,
                "formatting": comparison_operator.hidden_tokens_after
            },
            "first": expr,
            "second": comparison_,
            "first_formatting": comparison_operator.hidden_tokens_before,
            "second_formatting": comparison_operator2.hidden_tokens_after,
        }

    @pg.production("expr : xor_expr VBAR expr")
    @pg.production("xor_expr : and_expr CIRCUMFLEX xor_expr")
    @pg.production("and_expr : shift_expr AMPER and_expr")
    @pg.production("shift_expr : arith_expr RIGHT_SHIFT shift_expr")
    @pg.production("shift_expr : arith_expr LEFT_SHIFT shift_expr")
    @pg.production("arith_expr : term PLUS arith_expr")
    @pg.production("arith_expr : term MINUS arith_expr")
    @pg.production("term : factor STAR term")
    @pg.production("term : factor SLASH term")
    @pg.production("term : factor PERCENT term")
    @pg.production("term : factor DOUBLE_SLASH term")
    @pg.production("term : factor AT term")
    @pg.production("power : atom DOUBLE_STAR factor")
    @pg.production("power : atom DOUBLE_STAR power")
    def binary_operator_node(pack):
        (first, operator, second) = pack
        return {
            "type": "binary_operator",
            "value": operator.value,
            "first": first,
            "second": second,
            "first_formatting": operator.hidden_tokens_before,
            "second_formatting": operator.hidden_tokens_after
        }

    @pg.production("factor : PLUS factor")
    @pg.production("factor : MINUS factor")
    @pg.production("factor : TILDE factor")
    def factor_unitary_operator_space(pack):
        (operator, factor,) = pack
        return {
            "type": "unitary_operator",
            "value": operator.value,
            "formatting": operator.hidden_tokens_after,
            "target": factor,
        }

    @pg.production("power : atomtrailers DOUBLE_STAR factor")
    @pg.production("power : atomtrailers DOUBLE_STAR power")
    def power_atomtrailer_power(pack):
        (atomtrailers, double_star, factor) = pack
        return {
            "type": "binary_operator",
            "value": double_star.value,
            "first": {
                "type": "atomtrailers",
                "value": atomtrailers,
            },
            "second": factor,
            "first_formatting": double_star.hidden_tokens_before,
            "second_formatting": double_star.hidden_tokens_after
        }

    @pg.production("power : atomtrailers")
    def power_atomtrailers(pack):
        (atomtrailers,) = pack
        return {
            "type": "atomtrailers",
            "value": atomtrailers
        }

    @pg.production("power : NAME SPACE atomtrailers")
    def power_atomtrailers_await(pack):
        (await_, space, atomtrailers,) = pack

        if await_.value != "await":
            raise ParsingError("The only possible keyword before an atomtrailers is 'await', not '%s'" % await_.value)

        return {
            "type": "await",
            "formatting": [{'type': 'space', 'value': space.value}],
            "value": {
                "type": "atomtrailers",
                "value": atomtrailers,
            }
        }

    @pg.production("atomtrailers : atom")
    def atomtrailers_atom(pack):
        (atom,) = pack
        return [atom]

    @pg.production("atomtrailers : atom trailers")
    def atomtrailer(pack):
        (atom, trailers) = pack
        return [atom] + trailers

    @pg.production("trailers : trailer")
    def trailers(pack):
        (trailer,) = pack
        return trailer

    @pg.production("trailers : trailers trailer")
    def trailers_trailer(pack):
        (trailers, trailer) = pack
        return trailers + trailer

    @pg.production("trailer : DOT NAME")
    def trailer(pack):
        (dot, name,) = pack
        return [{
            "type": "dot",
            "first_formatting": dot.hidden_tokens_before,
            "second_formatting": dot.hidden_tokens_after,
        }, {
            "type": "name",
            "value": name.value,
        }]

    @pg.production("trailer : LEFT_PARENTHESIS argslist RIGHT_PARENTHESIS")
    def trailer_call(pack):
        (left, argslist, right) = pack
        return [{
            "type": "call",
            "value": argslist,
            "first_formatting": left.hidden_tokens_before,
            "second_formatting": left.hidden_tokens_after,
            "third_formatting": right.hidden_tokens_before,
            "fourth_formatting": right.hidden_tokens_after,
        }]

    @pg.production("trailer : LEFT_SQUARE_BRACKET subscript RIGHT_SQUARE_BRACKET")
    @pg.production("trailer : LEFT_SQUARE_BRACKET subscriptlist RIGHT_SQUARE_BRACKET")
    def trailer_getitem_ellipsis(pack):
        (left, subscript, right) = pack
        return [{
            "type": "getitem",
            "value": subscript,
            "first_formatting": left.hidden_tokens_before,
            "second_formatting": left.hidden_tokens_after,
            "third_formatting": right.hidden_tokens_before,
            "fourth_formatting": right.hidden_tokens_after,
        }]

    @pg.production("subscript : ELLIPSIS")
    @pg.production("atom : ELLIPSIS")
    def subscript_ellipsis(pack):
        ellipsis = pack[0]
        return {
            "type": "ellipsis",
            "first_formatting": ellipsis.hidden_tokens_after,
            "second_formatting": ellipsis.hidden_tokens_after,
        }

    @pg.production("subscript : test")
    @pg.production("subscript : slice")
    def subscript_test(pack):
        (test,) = pack
        return test

    @pg.production("slice : COLON")
    def slice(pack):
        (colon,) = pack
        return {
            "type": "slice",
            "lower": {},
            "upper": {},
            "step": {},
            "has_two_colons": False,
            "first_formatting": colon.hidden_tokens_before,
            "second_formatting": colon.hidden_tokens_after,
            "third_formatting": [],
            "fourth_formatting": [],
        }

    @pg.production("slice : COLON COLON")
    def slice_colon(pack):
        (colon, colon2) = pack
        return {
            "type": "slice",
            "lower": {},
            "upper": {},
            "step": {},
            "has_two_colons": True,
            "first_formatting": colon.hidden_tokens_before,
            "second_formatting": colon.hidden_tokens_after,
            "third_formatting": colon2.hidden_tokens_before,
            "fourth_formatting": colon2.hidden_tokens_after,
        }

    @pg.production("slice : test COLON")
    def slice_lower(pack):
        (test, colon,) = pack
        return {
            "type": "slice",
            "lower": test,
            "upper": {},
            "step": {},
            "has_two_colons": False,
            "first_formatting": colon.hidden_tokens_before,
            "second_formatting": colon.hidden_tokens_after,
            "third_formatting": [],
            "fourth_formatting": [],
        }

    @pg.production("slice : test COLON COLON")
    def slice_lower_colon_colon(pack):
        (test, colon, colon2) = pack
        return {
            "type": "slice",
            "lower": test,
            "upper": {},
            "step": {},
            "has_two_colons": True,
            "first_formatting": colon.hidden_tokens_before,
            "second_formatting": colon.hidden_tokens_after,
            "third_formatting": colon2.hidden_tokens_before,
            "fourth_formatting": colon2.hidden_tokens_after,
        }

    @pg.production("slice : COLON test")
    def slice_upper(pack):
        (colon, test,) = pack
        return {
            "type": "slice",
            "lower": {},
            "upper": test,
            "step": {},
            "has_two_colons": False,
            "first_formatting": colon.hidden_tokens_before,
            "second_formatting": colon.hidden_tokens_after,
            "third_formatting": [],
            "fourth_formatting": [],
        }

    @pg.production("slice : COLON test COLON")
    def slice_upper_colon(pack):
        (colon, test, colon2) = pack
        return {
            "type": "slice",
            "lower": {},
            "upper": test,
            "step": {},
            "has_two_colons": True,
            "first_formatting": colon.hidden_tokens_before,
            "second_formatting": colon.hidden_tokens_after,
            "third_formatting": colon2.hidden_tokens_before,
            "fourth_formatting": colon2.hidden_tokens_after,
        }

    @pg.production("slice : COLON COLON test")
    def slice_step(pack):
        (colon, colon2, test) = pack
        return {
            "type": "slice",
            "lower": {},
            "upper": {},
            "step": test,
            "has_two_colons": True,
            "first_formatting": colon.hidden_tokens_before,
            "second_formatting": colon.hidden_tokens_after,
            "third_formatting": colon2.hidden_tokens_before,
            "fourth_formatting": colon2.hidden_tokens_after,
        }

    @pg.production("slice : test COLON test")
    def slice_lower_upper(pack):
        (test, colon, test2,) = pack
        return {
            "type": "slice",
            "lower": test,
            "upper": test2,
            "step": {},
            "has_two_colons": False,
            "first_formatting": colon.hidden_tokens_before,
            "second_formatting": colon.hidden_tokens_after,
            "third_formatting": [],
            "fourth_formatting": [],
        }

    @pg.production("slice : test COLON test COLON")
    def slice_lower_upper_colon(pack):
        (test, colon, test2, colon2) = pack
        return {
            "type": "slice",
            "lower": test,
            "upper": test2,
            "step": {},
            "has_two_colons": True,
            "first_formatting": colon.hidden_tokens_before,
            "second_formatting": colon.hidden_tokens_after,
            "third_formatting": colon2.hidden_tokens_before,
            "fourth_formatting": colon2.hidden_tokens_after,
        }

    @pg.production("slice : test COLON COLON test")
    def slice_lower_step(pack):
        (test, colon, colon2, test2) = pack
        return {
            "type": "slice",
            "lower": test,
            "upper": {},
            "step": test2,
            "has_two_colons": True,
            "first_formatting": colon.hidden_tokens_before,
            "second_formatting": colon.hidden_tokens_after,
            "third_formatting": colon2.hidden_tokens_before,
            "fourth_formatting": colon2.hidden_tokens_after,
        }

    @pg.production("slice : COLON test COLON test")
    def slice_upper_step(pack):
        (colon, test, colon2, test2) = pack
        return {
            "type": "slice",
            "lower": {},
            "upper": test,
            "step": test2,
            "has_two_colons": True,
            "first_formatting": colon.hidden_tokens_before,
            "second_formatting": colon.hidden_tokens_after,
            "third_formatting": colon2.hidden_tokens_before,
            "fourth_formatting": colon2.hidden_tokens_after,
        }

    @pg.production("slice : test COLON test COLON test")
    def slice_lower_upper_step(pack):
        (test, colon, test2, colon2, test3) = pack
        return {
            "type": "slice",
            "lower": test,
            "upper": test2,
            "step": test3,
            "has_two_colons": True,
            "first_formatting": colon.hidden_tokens_before,
            "second_formatting": colon.hidden_tokens_after,
            "third_formatting": colon2.hidden_tokens_before,
            "fourth_formatting": colon2.hidden_tokens_after,
        } def include_data_structures(pg):
    # TODO remove left_parenthesis and use LEFT_PARENTHESIS instead
    @pg.production("atom : left_parenthesis testlist_comp RIGHT_PARENTHESIS")
    def tuple(pack):
        (left_parenthesis, testlist_comp, right_parenthesis,) = pack
        return {
            "type": "tuple",
            "value": testlist_comp,
            "first_formatting": left_parenthesis.hidden_tokens_before,
            "second_formatting": left_parenthesis.hidden_tokens_after,
            "third_formatting": right_parenthesis.hidden_tokens_before,
            "fourth_formatting": right_parenthesis.hidden_tokens_after,
            "with_parenthesis": True,
        }

    @pg.production("atom : left_parenthesis test RIGHT_PARENTHESIS")
    def associative_parenthesis(pack):
        (left_parenthesis, test, right_parenthesis,) = pack
        return {
            "type": "associative_parenthesis",
            "first_formatting": left_parenthesis.hidden_tokens_before,
            "second_formatting": left_parenthesis.hidden_tokens_after,
            "third_formatting": right_parenthesis.hidden_tokens_before,
            "fourth_formatting": right_parenthesis.hidden_tokens_after,
            "value": test
        }

    @pg.production("testlist : test comma")
    @pg.production("testlist_star_expr : test_or_star_expr comma")
    @pg.production("exprlist : expr comma")
    @pg.production("subscriptlist : subscript comma")
    def implicit_tuple_alone(pack):
        (test, comma) = pack
        return {
            "type": "tuple",
            "value": [test, comma],
            "first_formatting": [],
            "second_formatting": [],
            "third_formatting": [],
            "fourth_formatting": [],
            "with_parenthesis": False,
        }

    @pg.production("testlist : test testlist_part")
    @pg.production("testlist_star_expr : test_or_star_expr testlist_star_expr_part")
    @pg.production("exprlist : expr exprlist_part")
    @pg.production("subscriptlist : subscript subscriptlist_part")
    def implicit_tuple(pack):
        (test, testlist_part) = pack
        return {
            "type": "tuple",
            "value": [test] + testlist_part,
            "first_formatting": [],
            "second_formatting": [],
            "third_formatting": [],
            "fourth_formatting": [],
            "with_parenthesis": False,
        }

    @pg.production("testlist_part : COMMA test")
    @pg.production("testlist_star_expr_part : COMMA test_or_star_expr")
    @pg.production("exprlist_part : COMMA expr")
    @pg.production("subscriptlist_part : COMMA subscript")
    def testlist_part(pack):
        (comma, test) = pack
        return [{
            "type": "comma",
            "first_formatting": comma.hidden_tokens_before,
            "second_formatting": comma.hidden_tokens_after,
        }, test]

    @pg.production("testlist_part : COMMA test COMMA")
    @pg.production("testlist_star_expr_part : COMMA test_or_star_expr COMMA")
    @pg.production("exprlist_part : COMMA expr COMMA")
    @pg.production("subscriptlist_part : COMMA subscript COMMA")
    def testlist_part_comma(pack):
        (comma, test, comma2) = pack
        return [{
            "type": "comma",
            "first_formatting": comma.hidden_tokens_before,
            "second_formatting": comma.hidden_tokens_after,
        }, test, {
            "type": "comma",
            "first_formatting": comma2.hidden_tokens_before,
            "second_formatting": comma2.hidden_tokens_after,
        }]

    @pg.production("testlist_part : COMMA test testlist_part")
    @pg.production("testlist_star_expr_part : COMMA test_or_star_expr testlist_star_expr_part")
    @pg.production("exprlist_part : COMMA expr exprlist_part")
    @pg.production("subscriptlist_part : COMMA subscript subscriptlist_part")
    def testlist_part_next(pack):
        (comma, test, testlist_part) = pack
        return [{
            "type": "comma",
            "first_formatting": comma.hidden_tokens_before,
            "second_formatting": comma.hidden_tokens_after,
        }, test] + testlist_part

    @pg.production("testlist_comp :")
    def testlist_comp_empty(empty):
        return []

    @pg.production("testlist_comp : test comma test")
    def testlist_comp_two(pack):
        (test, comma, test2) = pack
        return [test, comma, test2]

    @pg.production("testlist_comp : test comma testlist_comp")
    def testlist_comp_more(pack):
        (test, comma, testlist_comp) = pack
        return [test, comma] + testlist_comp

    @pg.production("atom : LEFT_SQUARE_BRACKET listmaker RIGHT_SQUARE_BRACKET")
    def list_(pack):
        (left_bracket, listmaker, right_bracket,) = pack
        return {
            "type": "list",
            "first_formatting": left_bracket.hidden_tokens_before,
            "second_formatting": left_bracket.hidden_tokens_after,
            "third_formatting": right_bracket.hidden_tokens_before,
            "fourth_formatting": right_bracket.hidden_tokens_after,
            "value": listmaker
        }

    @pg.production("listmaker :")
    def listmaker_empty(empty):
        return []

    @pg.production("listmaker : test")
    def listmaker_one(pack):
        (test,) = pack
        return [test]

    @pg.production("listmaker : test comma listmaker")
    def listmaker_more(pack):
        (test, comma, listmaker) = pack
        return [test, comma] + listmaker

    @pg.production("atom : LEFT_BRACKET dictmaker RIGHT_BRACKET")
    def dict(pack):
        (left_bracket, dictmaker, right_bracket,) = pack
        return {
            "type": "dict",
            "first_formatting": left_bracket.hidden_tokens_before,
            "second_formatting": left_bracket.hidden_tokens_after,
            "third_formatting": right_bracket.hidden_tokens_before,
            "fourth_formatting": right_bracket.hidden_tokens_after,
            "value": dictmaker
        }

    @pg.production("dictmaker : ")
    def dict_empty(empty):
        return []

    @pg.production("dictmaker : test COLON test")
    def dict_one_colon(pack):
        (test, colon, test2) = pack
        return [{
            "first_formatting": colon.hidden_tokens_before,
            "second_formatting": colon.hidden_tokens_after,
            "key": test,
            "value": test2,
            "type": "dictitem"
        }]

    @pg.production("dictmaker : DOUBLE_STAR test")
    def dict_one_double_star(pack):
        (double_star, test) = pack
        return [{
            "type": "dict_argument",
            "annotation": {},
            "annotation_first_formatting": [],
            "annotation_second_formatting": [],
            "formatting": double_star.hidden_tokens_after,
            "value": test,
        }]

    @pg.production("dictmaker : test COLON test comma dictmaker")
    def dict_more_colon(pack):
        (test, colon, test2, comma, dictmaker) = pack
        return [{
            "first_formatting": colon.hidden_tokens_before,
            "second_formatting": colon.hidden_tokens_after,
            "key": test,
            "value": test2,
            "type": "dictitem"
        }, comma] + dictmaker

    @pg.production("dictmaker : DOUBLE_STAR test comma dictmaker")
    def dict_more_double_star(pack):
        (double_star, test, comma, dictmaker) = pack
        return [{
            "type": "dict_argument",
            "annotation": {},
            "annotation_first_formatting": [],
            "annotation_second_formatting": [],
            "formatting": double_star.hidden_tokens_after,
            "value": test,
        }, comma] + dictmaker

    @pg.production("atom : LEFT_BRACKET setmaker RIGHT_BRACKET")
    def set(pack):
        (left_bracket, setmaker, right_bracket,) = pack
        return {
            "type": "set",
            "first_formatting": left_bracket.hidden_tokens_before,
            "second_formatting": left_bracket.hidden_tokens_after,
            "third_formatting": right_bracket.hidden_tokens_before,
            "fourth_formatting": right_bracket.hidden_tokens_after,
            "value": setmaker
        }

    @pg.production("setmaker : ")
    def set_empty(empty):
        return []

    @pg.production("setmaker : test comma setmaker")
    def set_more(pack):
        (test, comma, setmaker) = pack
        return [test, comma] + setmaker

    @pg.production("setmaker : test")
    def set_one(pack):
        (test,) = pack
        return [test]

    @pg.production("atom : left_parenthesis test comp_for RIGHT_PARENTHESIS")
    def generator_comprehension(pack):
        (left_parenthesis, test, comp_for, right_parenthesis,) = pack
        return {
            "type": "generator_comprehension",
            "first_formatting": left_parenthesis.hidden_tokens_before,
            "second_formatting": left_parenthesis.hidden_tokens_after,
            "third_formatting": right_parenthesis.hidden_tokens_before,
            "fourth_formatting": right_parenthesis.hidden_tokens_after,
            "result": test,
            "generators": comp_for,
        }

    @pg.production("atom : LEFT_SQUARE_BRACKET test list_for RIGHT_SQUARE_BRACKET")
    def list_comprehension(pack):
        (left_square_bracket, test, list_for, right_square_bracket) = pack
        return {
            "type": "list_comprehension",
            "first_formatting": left_square_bracket.hidden_tokens_before,
            "second_formatting": left_square_bracket.hidden_tokens_after,
            "third_formatting": right_square_bracket.hidden_tokens_before,
            "fourth_formatting": right_square_bracket.hidden_tokens_after,
            "result": test,
            "generators": list_for,
        }

    @pg.production("atom : LEFT_BRACKET test COLON test comp_for RIGHT_BRACKET")
    def dict_comprehension(pack):
        (left_bracket, test, colon, test2, list_for, right_bracket) = pack
        return {
            "type": "dict_comprehension",
            "first_formatting": left_bracket.hidden_tokens_before,
            "second_formatting": left_bracket.hidden_tokens_after,
            "third_formatting": right_bracket.hidden_tokens_before,
            "fourth_formatting": right_bracket.hidden_tokens_after,
            "result": {
                "key": test,
                "type": "dictitem",
                "value": test2,
                "first_formatting": colon.hidden_tokens_before,
                "second_formatting": colon.hidden_tokens_after,
            },
            "generators": list_for,
        }

    @pg.production("atom : LEFT_BRACKET test comp_for RIGHT_BRACKET")
    def set_comprehension(pack):
        (left_bracket, test, list_for, right_bracket) = pack
        return {
            "type": "set_comprehension",
            "first_formatting": left_bracket.hidden_tokens_before,
            "second_formatting": left_bracket.hidden_tokens_after,
            "third_formatting": right_bracket.hidden_tokens_before,
            "fourth_formatting": right_bracket.hidden_tokens_after,
            "result": test,
            "generators": list_for,
        }

    @pg.production("list_for : FOR exprlist IN old_test")
    @pg.production("comp_for : FOR exprlist IN or_test")
    def comp_for(pack):
        (for_, exprlist, in_, or_test) = pack
        return [{
            "type": "comprehension_loop",
            "first_formatting": for_.hidden_tokens_before,
            "second_formatting": for_.hidden_tokens_after,
            "third_formatting": in_.hidden_tokens_before,
            "fourth_formatting": in_.hidden_tokens_after,
            "target": or_test,
            "iterator": exprlist,
            "ifs": [],
        }]

    @pg.production("list_for : FOR exprlist IN old_test")
    @pg.production("list_for : FOR exprlist IN testlist_safe")
    def comp_for_implicite_tuple(pack):
        (for_, exprlist, in_, testlist_safe) = pack
        return [{
            "type": "comprehension_loop",
            "first_formatting": for_.hidden_tokens_before,
            "second_formatting": for_.hidden_tokens_after,
            "third_formatting": in_.hidden_tokens_before,
            "fourth_formatting": in_.hidden_tokens_after,
            "target": {
                "type": "tuple",
                "value": testlist_safe,
                "with_parenthesis": False,
                "first_formatting": [],
                "second_formatting": [],
                "third_formatting": [],
                "fourth_formatting": [],
            },
            "iterator": exprlist,
            "ifs": [],
        }]

    @pg.production("comp_for : FOR exprlist IN or_test comp_iter")
    @pg.production("list_for : FOR exprlist IN old_test list_iter")
    @pg.production("list_for : FOR exprlist IN testlist_safe list_iter")
    def comp_for_iter(pack):
        (for_, exprlist, in_, or_test, comp_iter) = pack
        my_ifs = []
        for i in comp_iter:
            if i["type"] != "comprehension_if":
                break
            my_ifs.append(i)
            comp_iter = comp_iter[1:]

        return [{
            "type": "comprehension_loop",
            "first_formatting": for_.hidden_tokens_before,
            "second_formatting": for_.hidden_tokens_after,
            "third_formatting": in_.hidden_tokens_before,
            "fourth_formatting": in_.hidden_tokens_after,
            "target": or_test,
            "iterator": exprlist,
            "ifs": my_ifs,
        }] + comp_iter

    @pg.production("list_iter : list_for")
    @pg.production("comp_iter : comp_for")
    def comp_iter_comp_for(pack):
        (comp_for,) = pack
        return comp_for

    @pg.production("list_iter : IF old_test")
    @pg.production("comp_iter : IF old_test")
    def comp_iter_if(pack):
        (if_, old_test) = pack
        return [{
            "type": "comprehension_if",
            "first_formatting": if_.hidden_tokens_before,
            "second_formatting": if_.hidden_tokens_after,
            "value": old_test
        }]

    @pg.production("list_iter : IF old_test list_iter")
    @pg.production("comp_iter : IF old_test comp_iter")
    def comp_iter_if_comp_iter(pack):
        (if_, old_test, comp_iter) = pack
        return [{
            "type": "comprehension_if",
            "first_formatting": if_.hidden_tokens_before,
            "second_formatting": if_.hidden_tokens_after,
            "value": old_test
        }] + comp_iter class ParsingError(BaronError):
    pass
from .token import BaronToken
from .parser import BaronParserGenerator
from .tokenizer import TOKENS, tokenize, tokenize_current_keywords
from .utils import create_node_from_token
from .grammator_imports import include_imports
from .grammator_control_structures import include_control_structures
from .grammator_primitives import include_primivites
from .grammator_operators import include_operators
from .grammator_data_structures import include_data_structures
from .parser import ParsingError
    def yield_atom(pack):
        (left_parenthesis, yield_expr, right_parenthesis) = pack
        return {
            "type": "yield_atom",
            "value": yield_expr["value"],
            "first_formatting": left_parenthesis.hidden_tokens_after,
            "second_formatting": yield_expr["formatting"],
            "third_formatting": right_parenthesis.hidden_tokens_before
        }

    @pg.production("atom : BACKQUOTE testlist1 BACKQUOTE")
    def repr_atom(pack):
        (backquote, testlist1, backquote2) = pack
        return {
            "type": "repr",
            "value": testlist1,
            "first_formatting": backquote.hidden_tokens_after,
            "second_formatting": backquote2.hidden_tokens_before,
        }

    @pg.production("testlist1 : test comma testlist1")
    def testlist1_double(pack):
        (test, comma, test2,) = pack
        return [test, comma] + test2

    @pg.production("testlist1 : test")
    def testlist1(pack):
        (test,) = pack
        return [test]

    # TODO test all the things (except INT)
    @pg.production("atom : INT")
    @pg.production("atom : LONG")
    @pg.production("atom : OCTA")
    @pg.production("atom : HEXA")
    @pg.production("atom : BINARY")
    @pg.production("atom : FLOAT")
    @pg.production("atom : FLOAT_EXPONANT")
    @pg.production("atom : FLOAT_EXPONANT_COMPLEX")
    @pg.production("atom : COMPLEX")
    def int(pack):
        (int_,) = pack
        return create_node_from_token(int_, section="number")

    @pg.production("atom : name")
    def atom_name(pack):
        (name,) = pack
        return name

    @pg.production("atom : strings")
    def strings(pack):
        (string_chain,) = pack
        if len(string_chain) == 1:
            return string_chain[0]
        return {
            "type": "string_chain",
            "value": string_chain
        }

    @pg.production("strings : string strings")
    def strings_string_strings(pack):
        (string_, strings_) = pack
        return string_ + strings_

    @pg.production("strings : string")
    def strings_string(pack):
        (string_,) = pack
        return string_

    # TODO tests those other kind of strings
    @pg.production("string : STRING")
    @pg.production("string : RAW_STRING")
    @pg.production("string : INTERPOLATED_STRING")
    @pg.production("string : UNICODE_STRING")
    @pg.production("string : BINARY_STRING")
    @pg.production("string : UNICODE_RAW_STRING")
    @pg.production("string : BINARY_RAW_STRING")
    @pg.production("string : INTERPOLATED_RAW_STRING")
    def string(pack):
        (string_,) = pack
        return [{
            "type": string_.name.lower(),
            "value": string_.value,
            "first_formatting": string_.hidden_tokens_before,
            "second_formatting": string_.hidden_tokens_after,
        }]

    @pg.production("comma : COMMA")
    def comma(pack):
        (comma,) = pack
        return {
            "type": "comma",
            "first_formatting": comma.hidden_tokens_before,
            "second_formatting": comma.hidden_tokens_after,
        }

    def parse(tokens):
        if print_function:
            new_tokens = []
            for token in tokens:
                if token[0] in ("PRINT", "EXEC"):
                    token = list(token)
                    token[0] = "NAME"
                    token = tuple(token)
                new_tokens.append(token)

            tokens = [BaronToken(*x) if x else x for x in new_tokens] + [None]
        else:
            tokens = [BaronToken(*x) if x else x for x in tokens] + [None]

        return parser.parse(iter(tokens))

    parser = pg.build()
    return parse


def fake_lexer(sequence):
    for i in tokenize(sequence):
        if i is None:
            yield None

        yield BaronToken(*i)