MatthiasPicard commited on
Commit
184c6e1
·
verified ·
1 Parent(s): 2ae952d

Update tasks/text.py

Browse files
Files changed (1) hide show
  1. tasks/text.py +2 -2
tasks/text.py CHANGED
@@ -94,12 +94,12 @@ async def evaluate_text(request: TextEvaluationRequest):
94
  return tokenizer(df["quote"], truncation=True)
95
  tokenized_test = test_dataset.map(preprocess_function, batched=True)
96
 
97
- data_collator = DataCollatorWithPadding(tokenizer=tokenizer)
98
 
99
  trainer = Trainer(
100
  model=model,
101
  tokenizer=tokenizer,
102
- data_collator=data_collator,
103
  )
104
 
105
  per_device_eval_batch_size=8
 
94
  return tokenizer(df["quote"], truncation=True)
95
  tokenized_test = test_dataset.map(preprocess_function, batched=True)
96
 
97
+ # data_collator = DataCollatorWithPadding(tokenizer=tokenizer)
98
 
99
  trainer = Trainer(
100
  model=model,
101
  tokenizer=tokenizer,
102
+ # data_collator=data_collator,
103
  )
104
 
105
  per_device_eval_batch_size=8