Spaces:
Sleeping
Sleeping
lex3
Browse files- metrics.py +1 -1
metrics.py
CHANGED
|
@@ -1,6 +1,7 @@
|
|
| 1 |
# Import nltk library for natural language processing
|
| 2 |
import nltk
|
| 3 |
|
|
|
|
| 4 |
|
| 5 |
# Define a function that takes some text as input and returns the number of tokens
|
| 6 |
def token_count(text):
|
|
@@ -19,7 +20,6 @@ def token_count(text):
|
|
| 19 |
return tokens
|
| 20 |
|
| 21 |
def num_tokens(text):
|
| 22 |
-
from transformers import AutoTokenizer
|
| 23 |
|
| 24 |
tokenizer = AutoTokenizer.from_pretrained("gpt2")
|
| 25 |
|
|
|
|
| 1 |
# Import nltk library for natural language processing
|
| 2 |
import nltk
|
| 3 |
|
| 4 |
+
from transformers import AutoTokenizer
|
| 5 |
|
| 6 |
# Define a function that takes some text as input and returns the number of tokens
|
| 7 |
def token_count(text):
|
|
|
|
| 20 |
return tokens
|
| 21 |
|
| 22 |
def num_tokens(text):
|
|
|
|
| 23 |
|
| 24 |
tokenizer = AutoTokenizer.from_pretrained("gpt2")
|
| 25 |
|