Spaces:
Running
Running
Fix bug
Browse files- dataset.py +4 -5
dataset.py
CHANGED
|
@@ -56,11 +56,10 @@ class ParallelTextReader(IterableDataset):
|
|
| 56 |
return pred, [gold]
|
| 57 |
|
| 58 |
def __iter__(self):
|
| 59 |
-
|
| 60 |
-
|
| 61 |
-
|
| 62 |
-
|
| 63 |
-
return mapped_itr
|
| 64 |
|
| 65 |
def __len__(self):
|
| 66 |
return self.num_sentences
|
|
|
|
| 56 |
return pred, [gold]
|
| 57 |
|
| 58 |
def __iter__(self):
|
| 59 |
+
pred_itr = open(self.pred_path, "r", encoding="utf8")
|
| 60 |
+
gold_itr = open(self.gold_path, "r", encoding="utf8")
|
| 61 |
+
mapped_itr = map(self.preprocess, pred_itr, gold_itr)
|
| 62 |
+
return mapped_itr
|
|
|
|
| 63 |
|
| 64 |
def __len__(self):
|
| 65 |
return self.num_sentences
|