from typing import List, Tuple, NamedTuple
from brah.a_scanner import Source, Token
from brah.f_utils import *
from brah.constants.tokens import *
from brah.constants.scanner import *
from brah.constants.errors import *

__all__ = [
    'TokenError',
    'TokenStream',
    'Lexer',
    'Tkn',
]

# ---------------------------------------------------------
# region CONSTANTS & ENUMS

SM_NONE = 0
SM_NAME = 1
SM_NUMBER = 2
SM_STRING = 3
SM_DELIM = 4
SM_OPERATOR = 5
SM_COMMENT = 6
# 
# TT_NAME = "NAME"
# TT_KW = "KEYWORD"
# TT_INT = "INTEGER"
# TT_FLOAT = "FLOAT"
# TT_STR = "STRING"
# TT_LPAREN = "("
# TT_RPAREN = ")"
# TT_PLUS = "+"
# TT_MINUS = "-"
# TT_MULT = "*"
# TT_DIV = "/"
# TT_MOD = "%"
# TT_COMMA = ","
# TT_SEMI = ";"
# TT_COLON = ":"
# TT_DOT = "."


# endregion (constants)
# ---------------------------------------------------------
# region FUNCTIONS

# endregion (functions)
# ---------------------------------------------------------
# region CLASSES


class Tkn(NamedTuple):
    """Tkn (stands for Token) named tuple class.

    Captures and categorize an single sensible fragment of the source code
    that will later be processed by the parser. Each token holds three very
    basic informations: the kind of token, or its category; the location where
    it occurs in the source code; and the means to retrieve its value in the
    source.

    The categories (or kinds) are broad and does not defines meaning for the
    values, but helps to determine the semantics that apply during parsing.

    **Members:**

    `kind`: the token kind or category as a string.

    `pos`: a 2-int tuple storing the line and column where the token occurs.

    `value`: a slice used to extract the token literal.
    """
    kind: str
    pos: Tuple[int, int]
    value: slice
    location: Location

    def __str__(self):
        return f"[{self.kind}: {self.pos}]"


class TokenError(Exception):
    pass


class TokenStream:
    """TokenStream class.

    Represents the complete collection of tokens captured for a given source
    code file and the means to iterate over it with ease by a parser.

    :param tokens: the list of tokens generated by the Lexer.
    :param src: the object holding the source code
    """

    def __init__(self, tokens: List[Tkn], src: Source):
        self.tokens: List[Tkn] = tokens
        self.src: Source = src
        self.idx: int = 0

    @property
    def token(self) -> Tkn:
        """Gets the current token."""
        return self.tokens[self.idx]

    @property
    def token_val(self) -> str:
        """Gets the current token value."""
        return self.src.src[self.tokens[self.idx].value]

    @property
    def token_loc(self) -> Location:
        """Gets the current token location in source code."""
        return self.token.location

    def next(self) -> Tkn:
        """Advances to the next token.

        The current token is refered by an index. When next() is called, this
        index is incremented.
        
        :returns: The new current token.
        """
        if self.idx < len(self.tokens) - 1:
            self.idx += 1

        return self.token

    def get_val(self) -> str:
        val = self.token_val
        self.next()
        return val

    def get_kind(self) -> str:
        val = self.token.kind
        self.next()
        return val

    def get(self) -> Tkn:
        val = self.token
        self.next()
        return val

    def is_token(self, *token_kinds: str) -> bool:
        """Returns whether the current token kind is one of the given kinds.
        
        :param token_kinds: a variable number of token kind values.
        :returns: True if the token is of given kind, False otherwise.
        """
        return self.token.kind in token_kinds

    def is_keyword(self, keyword: str) -> bool:
        """Returns whether the current value token matches the given string.

        :param keyword: the string representing a keyword to test against.
        :returns: True if the token is of given value, False otherwise.
        """
        return self.token.kind == TT_NAME and self.token_val == keyword

    def is_operator(self, operator: str) -> bool:
        """Returns whether the current value token matches the given string.

        :param operator: the string representing a keyword to test against.
        :returns: True if the token is of given value, False otherwise.
        """
        return self.token_val == operator

    def is_some_keyword(self, *keywords: str) -> bool:
        """Returns whether the current token value matches one of the strings.
        
        :param keywords: the strings representing the keywords.
        :return: True if the token value matches any keyword, False otherwise.
        """
        return self.token.kind == TT_NAME and self.token_val in keywords

    def is_any_operator(self, *operators: str) -> bool:
        """Returns whether the current token value matches one of the strings.

        :param operators: the strings representing the keywords.
        :return: True if the token value matches any keyword, False otherwise.
        """
        return self.token_val in operators

    def match(self, token_value: str) -> bool:
        """Returns whether the current token value matches the given value.

        If the current token matches, the stream advances to the next.

        :param token_value: the token value to match against.
        :returns: True if the token is of given kind, False otherwise.
        """
        if self.token.kind == token_value:
            self.next()
            return True
        else:
            return False

    def match_token(self, *token_kinds: str) -> bool:
        """Returns whether the current token kind is one of the given kinds.

        If the current token matches, the stream advances to the next.

        :param token_kinds: a variable number of token kind values.
        :returns: True if the token is of given kind, False otherwise.
        """
        if self.token.kind in token_kinds:
            self.next()
            return True
        else:
            return False

    def match_keyword(self, keyword: str) -> bool:
        """Returns whether the current token kind is one of the given kinds.

        If the current token value matches, the stream advances to the next.

        :param keyword: the keyword string to test against.
        :returns: True if the token matches the keyword, False otherwise.
        """
        if self.is_keyword(keyword):
            self.next()
            return True
        else:
            return False

    def match_any_keyword(self, *keywords: str) -> bool:
        """Returns whether the current token kind is one of the given kinds.

        If the current token value matches, the stream advances to the next.

        :param keywords: the keyword string to test against.
        :returns: True if the token matches the keyword, False otherwise.
        """
        if self.is_some_keyword(*keywords):
            self.next()
            return True
        else:
            return False

    def match_operator(self, operator: str) -> bool:
        """Returns whether the current token kind is one of the given operator.

        If the current token value matches, the stream advances to the next.

        :param operator: the operator string to test against.
        :returns: True if the token matches the operator, False otherwise.
        """
        if self.is_operator(operator):
            self.next()
            return True
        else:
            return False

    def match_any_operator(self, *operators: str) -> bool:
        """Returns whether the current token kind is one of the given operator.

        If the current token value matches, the stream advances to the next.

        :param operators: the operators strings to test against.
        :returns: True if the token matches any operator, False otherwise.
        """
        if self.token_val in operators:
            self.next()
            return True
        else:
            return False

    def expect(self, *token_kinds: str) -> None:
        """Expects for the current token to match one of given kinds.

        If the current token matches, the stream advances to the next,
        otherwise, an error is raised.

        :param token_kinds: the token kinds to test against.
        :returns: None.
        """
        if self.is_token(*token_kinds):
            self.next()
        else:
            self.unexpected(*token_kinds)

    def expect_keyword(self, keyword: str) -> None:
        """Expects for the current token value to match one of given kinds.

        If the current token matches, the stream advances to the next,
        otherwise, an error is raised.

        :param keyword: the token kinds to test against.
        :returns: None.
        """
        if self.is_keyword(keyword):
            self.next()
        else:
            self.unexpected(keyword)

    def expect_operator(self, operator: str) -> None:
        """Expects for the current token value to match one the given operator.

        If the current token matches, the stream advances to the next,
        otherwise, an error is raised.

        :param operator: the token kinds to test against.
        :returns: None.
        """
        if self.token_val == operator:
            self.next()
        else:
            self.unexpected(operator)

    def unexpected(self, *expected: str):
        """Utility method to raise an error with descriptive information.

        :param expected: the expected value that did not matched.
        :returns: None.
        """
        tokens = ' or '.join([t for t in expected])
        what = self.token.kind if self.token.kind != "\n" else "new line"
        error(self.token.location, EK_SNTX, ERR_UNEXP_TOKEN, tokens, what)
        # raise TokenError(f'Expected {tokens}, got \'{what}\'')


class Lexer:
    """Lexer class.

    The Lexer is responsible for the first part of the compilation process,
    wich is the generation of the tokens that compose the source code.

    The process of generating tokens is done in a single pass with the
    `_scan()` method.
    """

    def __init__(self):
        self.tokens: List[Token] = []

    def gen_tokens(self, src: Source) -> TokenStream:
        """Performs the token generation and returns the stream of tokens

        :param src: the object storing the source code string.
        :returns: the stream of tokens.
        """
        print('Generatin tokens...')
        # self.tokens.append(Token(TK_EOF, '', *src.pos))
        tokens = self._scan(src.src, src.filename)
        stream = TokenStream(tokens, src)
        # for t in tokens:
        #     print(t, src.src[t.value])
        print('Tokens generated')
        self.tokens.clear()
        return stream

    @staticmethod
    def _scan(source: str, fname: str = '<file>') -> List[Tkn]:
        """Traverses the source code string capturing all of its tokens.

        :param source: the source code string.
        :returns: the list of tokens.
        """
        # srclen = len(source)
        tokens: List[Tkn] = []
        line: int = 1
        column: int = 1
        line_start: int = 0
        line_end: int = source.find('\n')

        scan_mode: int = SM_NONE
        start: int = 0

        has_decimal: bool = False
        end_quote: str = ''
        escaped: bool = False
        block: bool = False
        last_char = ''
        oper: str = ''
        suffix = False

        # def error(message: str):
        #     raise TokenError(f"ERROR at line {line}, column {column}: {message}.")

        for idx, char in enumerate(source):
            # nxt = source[idx + 1] if idx < srclen else None
            if scan_mode == SM_NONE:
                if char not in SCN_WHITESPACE:
                    start = idx
                if char in SCN_ALPHA:
                    scan_mode = SM_NAME
                elif char in SCN_DECDIGITS:
                    scan_mode = SM_NUMBER
                    has_decimal = False
                elif char in SCN_QUOTATION:
                    scan_mode = SM_STRING
                    end_quote = char
                    escaped = False
                elif char in SCN_OPERATORS:
                    scan_mode = SM_OPERATOR
                    oper = char
                elif char in SCN_DELIMITERS:
                    scan_mode = SM_NONE
                    loc = Location(line_start, line_end, idx, idx + 1, line, fname, source)
                    tokens.append(Tkn(char, (line, column), slice(idx, idx + 1), loc))
                    start = idx

            elif scan_mode == SM_NAME:
                if char not in SCN_ALPHANUM:
                    loc = Location(line_start, line_end, start, idx, line, fname, source)
                    tokens.append(Tkn(TT_NAME, (line, column), slice(start, idx), loc))
                    start = idx
                if char in SCN_DELIMITERS:
                    scan_mode = SM_NONE
                    loc = Location(line_start, line_end, idx, idx + 1, line, fname, source)
                    tokens.append(Tkn(char, (line, column), slice(idx, idx + 1), loc))
                    start = idx
                elif char in SCN_OPERATORS:
                    scan_mode = SM_OPERATOR
                    oper = char
                elif char in SCN_WHITESPACE:
                    scan_mode = SM_NONE
                elif char in SCN_QUOTATION:
                    loc = Location(line_start, line_end, start, idx + 1, line, fname, source)
                    error(loc, EK_LEXR, ERR_UNEXP_CHAR, char)

            elif scan_mode == SM_NUMBER:
                if char not in SCN_DECDIGITS:
                    if char in SCN_ALPHA:
                        suffix = True
                    elif char != '.':
                        loc = Location(line_start, line_end, start, idx, line, fname, source)
                        tokens.append(Tkn(TT_FLOAT if has_decimal else TT_INT, (line, column), slice(start, idx), loc))
                        start = idx
                        has_decimal = False
                        suffix = False
                elif suffix:
                    loc = Location(line_start, line_end, start, idx + 1, line, fname, source)
                    error(loc, EK_LEXR, ERR_UNEXP_CHAR, char)

                if char in SCN_OPERATORS:
                    if char == '.':
                        if not has_decimal:
                            has_decimal = True
                        else:
                            loc = Location(line_start, line_end, start, idx + 1, line, fname, source)
                            error(loc, EK_LEXR, ERR_UNEXP_CHAR, char)
                    else:
                        scan_mode = SM_OPERATOR
                        oper = char
                elif char in SCN_DELIMITERS:
                    scan_mode = SM_NONE
                    loc = Location(line_start, line_end, start, idx, line, fname, source)
                    tokens.append(Tkn(char, (line, column), slice(idx, idx + 1), loc))
                    start = idx

                elif char in SCN_WHITESPACE:
                    scan_mode = SM_NONE
                elif char in SCN_ALPHA:
                    suffix = True

                elif char in SCN_QUOTATION:
                    loc = Location(line_start, line_end, start, idx + 1, line, fname, source)
                    error(loc, EK_LEXR, ERR_UNEXP_CHAR, char)

            elif scan_mode == SM_STRING:
                if char == end_quote:
                    if not escaped:
                        loc = Location(line_start, line_end, start, idx + 1, line, fname, source)
                        tokens.append(Tkn(TT_STR, (line, column), slice(start, idx + 1), loc))
                        start = idx + 1
                        scan_mode = SM_NONE
                escaped = char == '\\'

            elif scan_mode == SM_DELIM:
                loc = Location(line_start, line_end, start, idx, line, fname, source)
                tokens.append(Tkn(char, (line, column), slice(idx, idx + 1), loc))
                start = idx

            elif scan_mode == SM_COMMENT:
                if block:
                    if last_char + char == '*/':
                        scan_mode = SM_NONE
                        last_char = ''
                        start = idx
                    else:
                        last_char = char
                else:
                    if char == '\n':
                        scan_mode = SM_NONE
                        start = idx

            elif scan_mode == SM_OPERATOR:
                if char not in SCN_OPERATORS:
                    if oper == '/*':
                        block = True
                        scan_mode = SM_COMMENT
                    elif oper == '//':
                        block = False
                        scan_mode = SM_COMMENT
                    else:
                        loc = Location(line_start, line_end, start, idx, line, fname, source)
                        tokens.append(Tkn(oper, (line, column), slice(start, idx), loc))
                        start = idx
                if char in SCN_ALPHA:
                    scan_mode = SM_NAME
                elif char in SCN_DECDIGITS:
                    scan_mode = SM_NUMBER
                    start = idx
                elif char in SCN_DELIMITERS:
                    scan_mode = SM_NONE
                    loc = Location(line_start, line_end, start, idx, line, fname, source)
                    tokens.append(Tkn(char, (line, column), slice(idx, idx + 1), loc))
                    start = idx
                elif char in SCN_WHITESPACE:
                    scan_mode = SM_NONE
                else:
                    oper += char
                    if len(oper) > 3:
                        loc = Location(line_start, line_end, start, idx + 1, line, fname, source)
                        error(loc, EK_LEXR, ERR_OP_TOO_LONG)

            if char == '\n':
                line_start = idx + 1
                line_end = source.find('\n', line_start)
                line += 1
                column = 0
            else:
                column += 1

        # for t in tokens:
        #     print(t.kind, source[t.value])
        return tokens

# endregion (classes)
# ---------------------------------------------------------

from typing import List, Tuple, NamedTuple
from brah.a_scanner import Source, Token
from brah.f_utils import *
from brah.constants.tokens import *
from brah.constants.scanner import *
from brah.constants.errors import *

__all__ = [
    'TokenError',
    'TokenStream',
    'Lexer',
    'Tkn',
]

# ---------------------------------------------------------
# region CONSTANTS & ENUMS

SM_NONE = 0
SM_NAME = 1
SM_NUMBER = 2
SM_STRING = 3
SM_DELIM = 4
SM_OPERATOR = 5
SM_COMMENT = 6
# 
# TT_NAME = "NAME"
# TT_KW = "KEYWORD"
# TT_INT = "INTEGER"
# TT_FLOAT = "FLOAT"
# TT_STR = "STRING"
# TT_LPAREN = "("
# TT_RPAREN = ")"
# TT_PLUS = "+"
# TT_MINUS = "-"
# TT_MULT = "*"
# TT_DIV = "/"
# TT_MOD = "%"
# TT_COMMA = ","
# TT_SEMI = ";"
# TT_COLON = ":"
# TT_DOT = "."


# endregion (constants)
# ---------------------------------------------------------
# region FUNCTIONS

# endregion (functions)
# ---------------------------------------------------------
# region CLASSES


class Tkn(NamedTuple):
    """Tkn (stands for Token) named tuple class.

    Captures and categorize an single sensible fragment of the source code
    that will later be processed by the parser. Each token holds three very
    basic informations: the kind of token, or its category; the location where
    it occurs in the source code; and the means to retrieve its value in the
    source.

    The categories (or kinds) are broad and does not defines meaning for the
    values, but helps to determine the semantics that apply during parsing.

    **Members:**

    `kind`: the token kind or category as a string.

    `pos`: a 2-int tuple storing the line and column where the token occurs.

    `value`: a slice used to extract the token literal.
    """
    kind: str
    pos: Tuple[int, int]
    value: slice
    location: Location

    def __str__(self):
        return f"[{self.kind}: {self.pos}]"


class TokenError(Exception):
    pass


class TokenStream:
    """TokenStream class.

    Represents the complete collection of tokens captured for a given source
    code file and the means to iterate over it with ease by a parser.

    :param tokens: the list of tokens generated by the Lexer.
    :param src: the object holding the source code
    """

    def __init__(self, tokens: List[Tkn], src: Source):
        self.tokens: List[Tkn] = tokens
        self.src: Source = src
        self.idx: int = 0

    @property
    def token(self) -> Tkn:
        """Gets the current token."""
        return self.tokens[self.idx]

    @property
    def token_val(self) -> str:
        """Gets the current token value."""
        return self.src.src[self.tokens[self.idx].value]

    @property
    def token_loc(self) -> Location:
        """Gets the current token location in source code."""
        return self.token.location

    def next(self) -> Tkn:
        """Advances to the next token.

        The current token is refered by an index. When next() is called, this
        index is incremented.
        
        :returns: The new current token.
        """
        if self.idx < len(self.tokens) - 1:
            self.idx += 1

        return self.token

    def get_val(self) -> str:
        val = self.token_val
        self.next()
        return val

    def get_kind(self) -> str:
        val = self.token.kind
        self.next()
        return val

    def get(self) -> Tkn:
        val = self.token
        self.next()
        return val

    def is_token(self, *token_kinds: str) -> bool:
        """Returns whether the current token kind is one of the given kinds.
        
        :param token_kinds: a variable number of token kind values.
        :returns: True if the token is of given kind, False otherwise.
        """
        return self.token.kind in token_kinds

    def is_keyword(self, keyword: str) -> bool:
        """Returns whether the current value token matches the given string.

        :param keyword: the string representing a keyword to test against.
        :returns: True if the token is of given value, False otherwise.
        """
        return self.token.kind == TT_NAME and self.token_val == keyword

    def is_operator(self, operator: str) -> bool:
        """Returns whether the current value token matches the given string.

        :param operator: the string representing a keyword to test against.
        :returns: True if the token is of given value, False otherwise.
        """
        return self.token_val == operator

    def is_some_keyword(self, *keywords: str) -> bool:
        """Returns whether the current token value matches one of the strings.
        
        :param keywords: the strings representing the keywords.
        :return: True if the token value matches any keyword, False otherwise.
        """
        return self.token.kind == TT_NAME and self.token_val in keywords

    def is_any_operator(self, *operators: str) -> bool:
        """Returns whether the current token value matches one of the strings.

        :param operators: the strings representing the keywords.
        :return: True if the token value matches any keyword, False otherwise.
        """
        return self.token_val in operators

    def match(self, token_value: str) -> bool:
        """Returns whether the current token value matches the given value.

        If the current token matches, the stream advances to the next.

        :param token_value: the token value to match against.
        :returns: True if the token is of given kind, False otherwise.
        """
        if self.token.kind == token_value:
            self.next()
            return True
        else:
            return False

    def match_token(self, *token_kinds: str) -> bool:
        """Returns whether the current token kind is one of the given kinds.

        If the current token matches, the stream advances to the next.

        :param token_kinds: a variable number of token kind values.
        :returns: True if the token is of given kind, False otherwise.
        """
        if self.token.kind in token_kinds:
            self.next()
            return True
        else:
            return False

    def match_keyword(self, keyword: str) -> bool:
        """Returns whether the current token kind is one of the given kinds.

        If the current token value matches, the stream advances to the next.

        :param keyword: the keyword string to test against.
        :returns: True if the token matches the keyword, False otherwise.
        """
        if self.is_keyword(keyword):
            self.next()
            return True
        else:
            return False

    def match_any_keyword(self, *keywords: str) -> bool:
        """Returns whether the current token kind is one of the given kinds.

        If the current token value matches, the stream advances to the next.

        :param keywords: the keyword string to test against.
        :returns: True if the token matches the keyword, False otherwise.
        """
        if self.is_some_keyword(*keywords):
            self.next()
            return True
        else:
            return False

    def match_operator(self, operator: str) -> bool:
        """Returns whether the current token kind is one of the given operator.

        If the current token value matches, the stream advances to the next.

        :param operator: the operator string to test against.
        :returns: True if the token matches the operator, False otherwise.
        """
        if self.is_operator(operator):
            self.next()
            return True
        else:
            return False

    def match_any_operator(self, *operators: str) -> bool:
        """Returns whether the current token kind is one of the given operator.

        If the current token value matches, the stream advances to the next.

        :param operators: the operators strings to test against.
        :returns: True if the token matches any operator, False otherwise.
        """
        if self.token_val in operators:
            self.next()
            return True
        else:
            return False

    def expect(self, *token_kinds: str) -> None:
        """Expects for the current token to match one of given kinds.

        If the current token matches, the stream advances to the next,
        otherwise, an error is raised.

        :param token_kinds: the token kinds to test against.
        :returns: None.
        """
        if self.is_token(*token_kinds):
            self.next()
        else:
            self.unexpected(*token_kinds)

    def expect_keyword(self, keyword: str) -> None:
        """Expects for the current token value to match one of given kinds.

        If the current token matches, the stream advances to the next,
        otherwise, an error is raised.

        :param keyword: the token kinds to test against.
        :returns: None.
        """
        if self.is_keyword(keyword):
            self.next()
        else:
            self.unexpected(keyword)

    def expect_operator(self, operator: str) -> None:
        """Expects for the current token value to match one the given operator.

        If the current token matches, the stream advances to the next,
        otherwise, an error is raised.

        :param operator: the token kinds to test against.
        :returns: None.
        """
        if self.token_val == operator:
            self.next()
        else:
            self.unexpected(operator)

    def unexpected(self, *expected: str):
        """Utility method to raise an error with descriptive information.

        :param expected: the expected value that did not matched.
        :returns: None.
        """
        tokens = ' or '.join([t for t in expected])
        what = self.token.kind if self.token.kind != "\n" else "new line"
        error(self.token.location, EK_SNTX, ERR_UNEXP_TOKEN, tokens, what)
        # raise TokenError(f'Expected {tokens}, got \'{what}\'')


class Lexer:
    """Lexer class.

    The Lexer is responsible for the first part of the compilation process,
    wich is the generation of the tokens that compose the source code.

    The process of generating tokens is done in a single pass with the
    `_scan()` method.
    """

    def __init__(self):
        self.tokens: List[Token] = []

    def gen_tokens(self, src: Source) -> TokenStream:
        """Performs the token generation and returns the stream of tokens

        :param src: the object storing the source code string.
        :returns: the stream of tokens.
        """
        print('Generatin tokens...')
        # self.tokens.append(Token(TK_EOF, '', *src.pos))
        tokens = self._scan(src.src, src.filename)
        stream = TokenStream(tokens, src)
        # for t in tokens:
        #     print(t, src.src[t.value])
        print('Tokens generated')
        self.tokens.clear()
        return stream

    @staticmethod
    def _scan(source: str, fname: str = '<file>') -> List[Tkn]:
        """Traverses the source code string capturing all of its tokens.

        :param source: the source code string.
        :returns: the list of tokens.
        """
        # srclen = len(source)
        tokens: List[Tkn] = []
        line: int = 1
        column: int = 1
        line_start: int = 0
        line_end: int = source.find('\n')

        scan_mode: int = SM_NONE
        start: int = 0

        has_decimal: bool = False
        end_quote: str = ''
        escaped: bool = False
        block: bool = False
        last_char = ''
        oper: str = ''
        suffix = False

        # def error(message: str):
        #     raise TokenError(f"ERROR at line {line}, column {column}: {message}.")

        for idx, char in enumerate(source):
            # nxt = source[idx + 1] if idx < srclen else None
            if scan_mode == SM_NONE:
                if char not in SCN_WHITESPACE:
                    start = idx
                if char in SCN_ALPHA:
                    scan_mode = SM_NAME
                elif char in SCN_DECDIGITS:
                    scan_mode = SM_NUMBER
                    has_decimal = False
                elif char in SCN_QUOTATION:
                    scan_mode = SM_STRING
                    end_quote = char
                    escaped = False
                elif char in SCN_OPERATORS:
                    scan_mode = SM_OPERATOR
                    oper = char
                elif char in SCN_DELIMITERS:
                    scan_mode = SM_NONE
                    loc = Location(line_start, line_end, idx, idx + 1, line, fname, source)
                    tokens.append(Tkn(char, (line, column), slice(idx, idx + 1), loc))
                    start = idx

            elif scan_mode == SM_NAME:
                if char not in SCN_ALPHANUM:
                    loc = Location(line_start, line_end, start, idx, line, fname, source)
                    tokens.append(Tkn(TT_NAME, (line, column), slice(start, idx), loc))
                    start = idx
                if char in SCN_DELIMITERS:
                    scan_mode = SM_NONE
                    loc = Location(line_start, line_end, idx, idx + 1, line, fname, source)
                    tokens.append(Tkn(char, (line, column), slice(idx, idx + 1), loc))
                    start = idx
                elif char in SCN_OPERATORS:
                    scan_mode = SM_OPERATOR
                    oper = char
                elif char in SCN_WHITESPACE:
                    scan_mode = SM_NONE
                elif char in SCN_QUOTATION:
                    loc = Location(line_start, line_end, start, idx + 1, line, fname, source)
                    error(loc, EK_LEXR, ERR_UNEXP_CHAR, char)

            elif scan_mode == SM_NUMBER:
                if char not in SCN_DECDIGITS:
                    if char in SCN_ALPHA:
                        suffix = True
                    elif char != '.':
                        loc = Location(line_start, line_end, start, idx, line, fname, source)
                        tokens.append(Tkn(TT_FLOAT if has_decimal else TT_INT, (line, column), slice(start, idx), loc))
                        start = idx
                        has_decimal = False
                        suffix = False
                elif suffix:
                    loc = Location(line_start, line_end, start, idx + 1, line, fname, source)
                    error(loc, EK_LEXR, ERR_UNEXP_CHAR, char)

                if char in SCN_OPERATORS:
                    if char == '.':
                        if not has_decimal:
                            has_decimal = True
                        else:
                            loc = Location(line_start, line_end, start, idx + 1, line, fname, source)
                            error(loc, EK_LEXR, ERR_UNEXP_CHAR, char)
                    else:
                        scan_mode = SM_OPERATOR
                        oper = char
                elif char in SCN_DELIMITERS:
                    scan_mode = SM_NONE
                    loc = Location(line_start, line_end, start, idx, line, fname, source)
                    tokens.append(Tkn(char, (line, column), slice(idx, idx + 1), loc))
                    start = idx

                elif char in SCN_WHITESPACE:
                    scan_mode = SM_NONE
                elif char in SCN_ALPHA:
                    suffix = True

                elif char in SCN_QUOTATION:
                    loc = Location(line_start, line_end, start, idx + 1, line, fname, source)
                    error(loc, EK_LEXR, ERR_UNEXP_CHAR, char)

            elif scan_mode == SM_STRING:
                if char == end_quote:
                    if not escaped:
                        loc = Location(line_start, line_end, start, idx + 1, line, fname, source)
                        tokens.append(Tkn(TT_STR, (line, column), slice(start, idx + 1), loc))
                        start = idx + 1
                        scan_mode = SM_NONE
                escaped = char == '\\'

            elif scan_mode == SM_DELIM:
                loc = Location(line_start, line_end, start, idx, line, fname, source)
                tokens.append(Tkn(char, (line, column), slice(idx, idx + 1), loc))
                start = idx

            elif scan_mode == SM_COMMENT:
                if block:
                    if last_char + char == '*/':
                        scan_mode = SM_NONE
                        last_char = ''
                        start = idx
                    else:
                        last_char = char
                else:
                    if char == '\n':
                        scan_mode = SM_NONE
                        start = idx

            elif scan_mode == SM_OPERATOR:
                if char not in SCN_OPERATORS:
                    if oper == '/*':
                        block = True
                        scan_mode = SM_COMMENT
                    elif oper == '//':
                        block = False
                        scan_mode = SM_COMMENT
                    else:
                        loc = Location(line_start, line_end, start, idx, line, fname, source)
                        tokens.append(Tkn(oper, (line, column), slice(start, idx), loc))
                        start = idx
                if char in SCN_ALPHA:
                    scan_mode = SM_NAME
                elif char in SCN_DECDIGITS:
                    scan_mode = SM_NUMBER
                    start = idx
                elif char in SCN_DELIMITERS:
                    scan_mode = SM_NONE
                    loc = Location(line_start, line_end, start, idx, line, fname, source)
                    tokens.append(Tkn(char, (line, column), slice(idx, idx + 1), loc))
                    start = idx
                elif char in SCN_WHITESPACE:
                    scan_mode = SM_NONE
                else:
                    oper += char
                    if len(oper) > 3:
                        loc = Location(line_start, line_end, start, idx + 1, line, fname, source)
                        error(loc, EK_LEXR, ERR_OP_TOO_LONG)

            if char == '\n':
                line_start = idx + 1
                line_end = source.find('\n', line_start)
                line += 1
                column = 0
            else:
                column += 1

        # for t in tokens:
        #     print(t.kind, source[t.value])
        return tokens

# endregion (classes)
# ---------------------------------------------------------

# --------------------------------------------------------- # region CONSTANTS & ENUMS # # TT_NAME = "NAME" # TT_KW = "KEYWORD" # TT_INT = "INTEGER" # TT_FLOAT = "FLOAT" # TT_STR = "STRING" # TT_LPAREN = "(" # TT_RPAREN = ")" # TT_PLUS = "+" # TT_MINUS = "-" # TT_MULT = "*" # TT_DIV = "/" # TT_MOD = "%" # TT_COMMA = "," # TT_SEMI = ";" # TT_COLON = ":" # TT_DOT = "." # endregion (constants) # --------------------------------------------------------- # region FUNCTIONS # endregion (functions) # --------------------------------------------------------- # region CLASSES Tkn (stands for Token) named tuple class.      Captures and categorize an single sensible fragment of the source code     that will later be processed by the parser. Each token holds three very     basic informations: the kind of token, or its category; the location where     it occurs in the source code; and the means to retrieve its value in the     source.      The categories (or kinds) are broad and does not defines meaning for the     values, but helps to determine the semantics that apply during parsing.      **Members:**      `kind`: the token kind or category as a string.      `pos`: a 2-int tuple storing the line and column where the token occurs.      `value`: a slice used to extract the token literal. TokenStream class.      Represents the complete collection of tokens captured for a given source     code file and the means to iterate over it with ease by a parser.      :param tokens: the list of tokens generated by the Lexer.     :param src: the object holding the source code Gets the current token. Gets the current token value. Gets the current token location in source code. Advances to the next token.          The current token is refered by an index. When next() is called, this         index is incremented.                  :returns: The new current token. Returns whether the current token kind is one of the given kinds.                  :param token_kinds: a variable number of token kind values.         :returns: True if the token is of given kind, False otherwise. Returns whether the current value token matches the given string.          :param keyword: the string representing a keyword to test against.         :returns: True if the token is of given value, False otherwise. Returns whether the current value token matches the given string.          :param operator: the string representing a keyword to test against.         :returns: True if the token is of given value, False otherwise. Returns whether the current token value matches one of the strings.                  :param keywords: the strings representing the keywords.         :return: True if the token value matches any keyword, False otherwise. Returns whether the current token value matches one of the strings.          :param operators: the strings representing the keywords.         :return: True if the token value matches any keyword, False otherwise. Returns whether the current token value matches the given value.          If the current token matches, the stream advances to the next.          :param token_value: the token value to match against.         :returns: True if the token is of given kind, False otherwise. Returns whether the current token kind is one of the given kinds.          If the current token matches, the stream advances to the next.          :param token_kinds: a variable number of token kind values.         :returns: True if the token is of given kind, False otherwise. Returns whether the current token kind is one of the given kinds.          If the current token value matches, the stream advances to the next.          :param keyword: the keyword string to test against.         :returns: True if the token matches the keyword, False otherwise. Returns whether the current token kind is one of the given kinds.          If the current token value matches, the stream advances to the next.          :param keywords: the keyword string to test against.         :returns: True if the token matches the keyword, False otherwise. Returns whether the current token kind is one of the given operator.          If the current token value matches, the stream advances to the next.          :param operator: the operator string to test against.         :returns: True if the token matches the operator, False otherwise. Returns whether the current token kind is one of the given operator.          If the current token value matches, the stream advances to the next.          :param operators: the operators strings to test against.         :returns: True if the token matches any operator, False otherwise. Expects for the current token to match one of given kinds.          If the current token matches, the stream advances to the next,         otherwise, an error is raised.          :param token_kinds: the token kinds to test against.         :returns: None. Expects for the current token value to match one of given kinds.          If the current token matches, the stream advances to the next,         otherwise, an error is raised.          :param keyword: the token kinds to test against.         :returns: None. Expects for the current token value to match one the given operator.          If the current token matches, the stream advances to the next,         otherwise, an error is raised.          :param operator: the token kinds to test against.         :returns: None. Utility method to raise an error with descriptive information.          :param expected: the expected value that did not matched.         :returns: None. # raise TokenError(f'Expected {tokens}, got \'{what}\'') Lexer class.      The Lexer is responsible for the first part of the compilation process,     wich is the generation of the tokens that compose the source code.      The process of generating tokens is done in a single pass with the     `_scan()` method. Performs the token generation and returns the stream of tokens          :param src: the object storing the source code string.         :returns: the stream of tokens. # self.tokens.append(Token(TK_EOF, '', *src.pos)) # for t in tokens: #     print(t, src.src[t.value]) Traverses the source code string capturing all of its tokens.          :param source: the source code string.         :returns: the list of tokens. # srclen = len(source) # def error(message: str): #     raise TokenError(f"ERROR at line {line}, column {column}: {message}.") # nxt = source[idx + 1] if idx < srclen else None # for t in tokens: #     print(t.kind, source[t.value]) # endregion (classes) # ---------------------------------------------------------