| from . import InputExample | |
| import csv | |
| import gzip | |
| import os | |
| class LabelSentenceReader: | |
| """Reads in a file that has at least two columns: a label and a sentence. | |
| This reader can for example be used with the BatchHardTripletLoss. | |
| Maps labels automatically to integers""" | |
| def __init__(self, folder, label_col_idx=0, sentence_col_idx=1, separator='\t'): | |
| self.folder = folder | |
| self.label_map = {} | |
| self.label_col_idx = label_col_idx | |
| self.sentence_col_idx = sentence_col_idx | |
| self.separator = separator | |
| def get_examples(self, filename, max_examples=0): | |
| examples = [] | |
| id = 0 | |
| for line in open(os.path.join(self.folder, filename), encoding="utf-8"): | |
| splits = line.strip().split(self.separator) | |
| label = splits[self.label_col_idx] | |
| sentence = splits[self.sentence_col_idx] | |
| if label not in self.label_map: | |
| self.label_map[label] = len(self.label_map) | |
| label_id = self.label_map[label] | |
| guid = "%s-%d" % (filename, id) | |
| id += 1 | |
| examples.append(InputExample(guid=guid, texts=[sentence], label=label_id)) | |
| if 0 < max_examples <= id: | |
| break | |
| return examples | |