def convert(document):
    """Convert a document to a Text object"""
    raw_tokens = []
    curpos = 0
    text_spans = []
    all_labels = []
    sent_spans = []
    word_texts = []
    for sentence in document:
        startpos = curpos
        for idx, (text, label) in enumerate(sentence):
            raw_tokens.append(text)
            word_texts.append(text)
            all_labels.append(label)
            text_spans.append((curpos, curpos+len(text)))
            curpos += len(text)
            if idx < len(sentence) - 1:
                raw_tokens.append(' ')
                curpos += 1
        sent_spans.append((startpos, curpos))
        raw_tokens.append('\n')
        curpos += 1
    return {
        TEXT: ''.join(raw_tokens),
        WORDS: [{TEXT: text, START: start, END: end, LABEL: label} for text, (start, end), label in zip(word_texts, text_spans, all_labels)],
        SENTENCES: [{START: start, END:end} for start, end in sent_spans]
    }
def convert ( document ) : raw_tokens = [ ] curpos = 0 text_spans = [ ] all_labels = [ ] sent_spans = [ ] word_texts = [ ] for sentence in document : startpos = curpos for idx , ( text , label ) in enumerate ( sentence ) : raw_tokens . append ( text ) word_texts . append ( text ) all_labels . append ( label ) text_spans . append ( ( curpos , curpos + len ( text ) ) ) curpos += len ( text ) if idx < len ( sentence ) - 1 : raw_tokens . append ( ' ' ) curpos += 1 sent_spans . append ( ( startpos , curpos ) ) raw_tokens . append ( '\n' ) curpos += 1 return { TEXT : '' . join ( raw_tokens ) , WORDS : [ { TEXT : text , START : start , END : end , LABEL : label } for text , ( start , end ) , label in zip ( word_texts , text_spans , all_labels ) ] , SENTENCES : [ { START : start , END : end } for start , end in sent_spans ] }
Convert a document to a Text object