from flair.data import Tokenizer
from typing import List


class StatsTokenizer(Tokenizer):

    def __init__(self):
        super(StatsTokenizer, self).__init__()

    def tokenize(self, text: str) -> List[str]:
        return StatsTokenizer.run_tokenize(text)

    @staticmethod
    def run_tokenize(text: str) -> List[str]:
        tokens: List[str] = []
        
        index = -1
        words = text.split()
        for word in words:
            token = ""
            for index, char in enumerate(word):
                if char in [',']:
                    if len(token) > 0:
                        tokens.append(token)
                    
                    token = ""
                elif char in ['(', ')', '<', '=']:
                    if len(token) > 0:
                        tokens.append(token)
                    tokens.append(char)
                    
                    token = ""
                else:
                    token += char
                    
            if len(token) > 0:
                if token.endswith('.'):
                    token = token[:-1]
                tokens.append(token)

        return tokens