# import tiktoken | |
# from typing import Union | |
# def tokenize(text: str, model: str = 'gpt-3.5-turbo') -> Union[int, str]: | |
# encoding = tiktoken.encoding_for_model(model) | |
# encoded = encoding.encode(text) | |
# num_tokens = len(encoded) | |
# return num_tokens, encoded |