Spaces:
Sleeping
Sleeping
setting padding to max length
Browse files- src/utils.py +1 -1
src/utils.py
CHANGED
@@ -26,7 +26,7 @@ def get_datasetdict_object(df_train, df_val, df_test):
|
|
26 |
|
27 |
|
28 |
def tokenize(batch, tokenizer):
|
29 |
-
return tokenizer(batch["tweet"], padding=
|
30 |
|
31 |
|
32 |
def get_dataset(train_path:str, test_path:str, tokenizer):
|
|
|
26 |
|
27 |
|
28 |
def tokenize(batch, tokenizer):
|
29 |
+
return tokenizer(batch["tweet"], padding='max_length')
|
30 |
|
31 |
|
32 |
def get_dataset(train_path:str, test_path:str, tokenizer):
|