|
from flair.data import Tokenizer |
|
from typing import List |
|
|
|
|
|
class StatsTokenizer(Tokenizer): |
|
|
|
def __init__(self): |
|
super(StatsTokenizer, self).__init__() |
|
|
|
def tokenize(self, text: str) -> List[str]: |
|
return StatsTokenizer.run_tokenize(text) |
|
|
|
@staticmethod |
|
def run_tokenize(text: str) -> List[str]: |
|
tokens: List[str] = [] |
|
|
|
index = -1 |
|
words = text.split() |
|
for word in words: |
|
token = "" |
|
for index, char in enumerate(word): |
|
if char in [',']: |
|
if len(token) > 0: |
|
tokens.append(token) |
|
|
|
token = "" |
|
elif char in ['(', ')', '<', '=']: |
|
if len(token) > 0: |
|
tokens.append(token) |
|
tokens.append(char) |
|
|
|
token = "" |
|
else: |
|
token += char |
|
|
|
if len(token) > 0: |
|
if token.endswith('.'): |
|
token = token[:-1] |
|
tokens.append(token) |
|
|
|
return tokens |
