Update tasks/text.py
Browse files- tasks/text.py +2 -2
tasks/text.py
CHANGED
@@ -94,12 +94,12 @@ async def evaluate_text(request: TextEvaluationRequest):
|
|
94 |
return tokenizer(df["quote"], truncation=True)
|
95 |
tokenized_test = test_dataset.map(preprocess_function, batched=True)
|
96 |
|
97 |
-
data_collator = DataCollatorWithPadding(tokenizer=tokenizer)
|
98 |
|
99 |
trainer = Trainer(
|
100 |
model=model,
|
101 |
tokenizer=tokenizer,
|
102 |
-
data_collator=data_collator,
|
103 |
)
|
104 |
|
105 |
per_device_eval_batch_size=8
|
|
|
94 |
return tokenizer(df["quote"], truncation=True)
|
95 |
tokenized_test = test_dataset.map(preprocess_function, batched=True)
|
96 |
|
97 |
+
# data_collator = DataCollatorWithPadding(tokenizer=tokenizer)
|
98 |
|
99 |
trainer = Trainer(
|
100 |
model=model,
|
101 |
tokenizer=tokenizer,
|
102 |
+
# data_collator=data_collator,
|
103 |
)
|
104 |
|
105 |
per_device_eval_batch_size=8
|