

@factory.post_generation
def init(self, create, extracted, **kwargs):
    tokens = []
    sentences = [0]
    for sent in nltk.sent_tokenize(self.text):
        sent_tokens = nltk.word_tokenize(sent)
        tokens.extend(list(enumerate(sent_tokens)))
        sentences.append((sentences[(- 1)] + len(sent_tokens)))
    self.set_tokenization_result(tokens)
    self.set_sentencer_result(sentences)

dataset/ETHPy150Open machinalis/iepy/tests/factories.py SentencedIEDocFactory.init/original