
numerical_tokens = []
fully_numerical_tokens = []
with open("../../data-bin/wikitext-103/dict.txt", "r", encoding='utf-8') as f_r:
    for line in f_r:
        token, id = line.split()
        if any(char.isdigit() for char in token):
            numerical_tokens.append(token)
            if all(char.isdigit() for char in token):
                fully_numerical_tokens.append(token)

print(f"Token numerical tokens: {len(numerical_tokens)}")
print(f"Total fully numerical tokens: {len(fully_numerical_tokens)}")
# print(fully_numerical_tokens)

int_tokens = []
for token in fully_numerical_tokens:
    try:
        int_tokens.append(int(token))
    except:
        pass 
    
print(f"After int operation, total fully numerical tokens: {len(set(int_tokens))}")
final_list = sorted(list(set(int_tokens)))

for i in range(len(final_list)):
    if final_list[i] != i:
        print(final_list[i-1:])
        break


numerical_tokens = []
fully_numerical_tokens = []
with open("../../data-bin/wikitext-103/dict.txt", "r", encoding='utf-8') as f_r:
    for line in f_r:
        token, id = line.split()
        if any(char.isdigit() for char in token):
            numerical_tokens.append(token)
            if all(char.isdigit() for char in token):
                fully_numerical_tokens.append(token)

print(f"Token numerical tokens: {len(numerical_tokens)}")
print(f"Total fully numerical tokens: {len(fully_numerical_tokens)}")
# print(fully_numerical_tokens)

int_tokens = []
for token in fully_numerical_tokens:
    try:
        int_tokens.append(int(token))
    except:
        pass 
    
print(f"After int operation, total fully numerical tokens: {len(set(int_tokens))}")
final_list = sorted(list(set(int_tokens)))

for i in range(len(final_list)):
    if final_list[i] != i:
        print(final_list[i-1:])
        break

# print(fully_numerical_tokens)