zaidmehdi commited on
Commit
9d3bdb3
·
1 Parent(s): 2d22962

setting padding to max length

Browse files
Files changed (1) hide show
  1. src/utils.py +1 -1
src/utils.py CHANGED
@@ -26,7 +26,7 @@ def get_datasetdict_object(df_train, df_val, df_test):
26
 
27
 
28
  def tokenize(batch, tokenizer):
29
- return tokenizer(batch["tweet"], padding=True)
30
 
31
 
32
  def get_dataset(train_path:str, test_path:str, tokenizer):
 
26
 
27
 
28
  def tokenize(batch, tokenizer):
29
+ return tokenizer(batch["tweet"], padding='max_length')
30
 
31
 
32
  def get_dataset(train_path:str, test_path:str, tokenizer):