deeppavlov/dataset_readers/morphotagging_dataset_reader.py
def read_infile(infile: Union[Path, str], from_words=False,
                word_column: int = WORD_COLUMN, pos_column: int = POS_COLUMN,
                tag_column: int = TAG_COLUMN, max_sents: int = -1,
                read_only_words: bool = False) -> List[Tuple[List, Union[List, None]]]:
    """Reads input file in CONLL-U format

    Args:
        infile: a path to a file
        word_column: column containing words (default=1)
        pos_column: column containing part-of-speech labels (default=3)
        tag_column: column containing fine-grained tags (default=5)
        max_sents: maximal number of sents to read
        read_only_words: whether to read only words

    Returns:
        a list of sentences. Each item contains a word sequence and a tag sequence, which is ``None``
        in case ``read_only_words = True``
    """
    answer, curr_word_sent, curr_tag_sent = [], [], []
    if from_words:
        word_column, read_only_words = 0, True
    with open(infile, "r", encoding="utf8") as fin:
        for line in fin:
            line = line.strip()
            if line.startswith("#"):
                continue
            if line == "":
                if len(curr_word_sent) > 0:
                    if read_only_words:
                        curr_tag_sent = None
                    answer.append((curr_word_sent, curr_tag_sent))
                curr_tag_sent, curr_word_sent = [], []
                if len(answer) == max_sents:
                    break
                continue
            splitted = line.split("\t")
            index = splitted[0]
            if not from_words and not index.isdigit():
                continue
            curr_word_sent.append(splitted[word_column])
            if not read_only_words:
                pos, tag = splitted[pos_column], splitted[tag_column]
                tag = pos if tag == "_" else "{},{}".format(pos, tag)
                curr_tag_sent.append(tag)
        if len(curr_word_sent) > 0:
            if read_only_words:
                curr_tag_sent = None
            answer.append((curr_word_sent, curr_tag_sent))
    return answer
def read_infile(infile: Union[Path, str], from_words=False,
                word_column: int = WORD_COLUMN, pos_column: int = POS_COLUMN,
                tag_column: int = TAG_COLUMN, max_sents: int = -1,
                read_only_words: bool = False) -> List[Tuple[List, Union[List, None]]]:
    answer, curr_word_sent, curr_tag_sent = [], [], []
    if from_words:
        word_column, read_only_words = 0, True
    with open(infile, "r", encoding="utf8") as fin:
        for line in fin:
            line = line.strip()
            if line.startswith("#"):
                continue
            if line == "":
                if len(curr_word_sent) > 0:
                    if read_only_words:
                        curr_tag_sent = None
                    answer.append((curr_word_sent, curr_tag_sent))
                curr_tag_sent, curr_word_sent = [], []
                if len(answer) == max_sents:
                    break
                continue
            splitted = line.split("\t")
            index = splitted[0]
            if not from_words and not index.isdigit():
                continue
            curr_word_sent.append(splitted[word_column])
            if not read_only_words:
                pos, tag = splitted[pos_column], splitted[tag_column]
                tag = pos if tag == "_" else "{},{}".format(pos, tag)
                curr_tag_sent.append(tag)
        if len(curr_word_sent) > 0:
            if read_only_words:
                curr_tag_sent = None
            answer.append((curr_word_sent, curr_tag_sent))
    return answer
def read_infile ( infile : Union [ Path , str ] , from_words = False , word_column : int = WORD_COLUMN , pos_column : int = POS_COLUMN , tag_column : int = TAG_COLUMN , max_sents : int = - 1 , read_only_words : bool = False ) -> List [ Tuple [ List , Union [ List , None ] ] ] : answer , curr_word_sent , curr_tag_sent = [ ] , [ ] , [ ] if from_words : word_column , read_only_words = 0 , True with open ( infile , "r" , encoding = "utf8" ) as fin : for line in fin : line = line . strip ( ) if line . startswith ( "#" ) : continue if line == "" : if len ( curr_word_sent ) > 0 : if read_only_words : curr_tag_sent = None answer . append ( ( curr_word_sent , curr_tag_sent ) ) curr_tag_sent , curr_word_sent = [ ] , [ ] if len ( answer ) == max_sents : break continue splitted = line . split ( "\t" ) index = splitted [ 0 ] if not from_words and not index . isdigit ( ) : continue curr_word_sent . append ( splitted [ word_column ] ) if not read_only_words : pos , tag = splitted [ pos_column ] , splitted [ tag_column ] tag = pos if tag == "_" else "{},{}" . format ( pos , tag ) curr_tag_sent . append ( tag ) if len ( curr_word_sent ) > 0 : if read_only_words : curr_tag_sent = None answer . append ( ( curr_word_sent , curr_tag_sent ) ) return answer
Reads input file in CONLL-U format

    Args:
        infile: a path to a file
        word_column: column containing words (default=1)
        pos_column: column containing part-of-speech labels (default=3)
        tag_column: column containing fine-grained tags (default=5)
        max_sents: maximal number of sents to read
        read_only_words: whether to read only words

    Returns:
        a list of sentences. Each item contains a word sequence and a tag sequence, which is ``None``
        in case ``read_only_words = True``
Reads input file in CONLL - U format