Spaces:
Runtime error
Runtime error
Update token_counter.py
Browse files- token_counter.py +1 -1
token_counter.py
CHANGED
@@ -10,6 +10,6 @@ class TokenCounterTool(Tool):
|
|
10 |
def __call__(self, prompt: str):
|
11 |
# token = os.environ['hf']
|
12 |
tokenizer = AutoTokenizer.from_pretrained("lgaalves/gpt2-dolly")
|
13 |
-
tokens = tokenizer(
|
14 |
return f"{len(tokens)}"
|
15 |
|
|
|
10 |
def __call__(self, prompt: str):
|
11 |
# token = os.environ['hf']
|
12 |
tokenizer = AutoTokenizer.from_pretrained("lgaalves/gpt2-dolly")
|
13 |
+
tokens = tokenizer(prompt)["input_ids"]
|
14 |
return f"{len(tokens)}"
|
15 |
|