import nltk


tokenizer = nltk.RegexpTokenizer(r"([A-Z][A-Z0-9.]+|[0-9]+[,.][0-9]+|[cdjlmnst]'|qu'|[\w'-]+|\S)")


class Sentence:
    def __init__(self, text, sent_line):
        self.text = text
        self.sent_line = sent_line

    def __len__(self):
        return len(tokenizer.tokenize(self.text))

    @property
    def id(self):
        return self.sent_line

    def contains_pos(self, postag):
        return False

    def count_pos(self, postag):
        return 0


def read_sentence(stream, sent_line):
    line = next(stream).strip()
    sent_line += 1
    if line:
        return Sentence(line, sent_line), sent_line
    else:
        return None, sent_line


def read_corpus(path):
    sentences = []
    nth = 0
    with open(path) as input_stream:
        try:
            while True:
                sentence, nth = read_sentence(input_stream, nth)
                if sentence:
                    sentences.append(sentence)
        except StopIteration:
            pass
    return sentences

import nltk


tokenizer = nltk.RegexpTokenizer(r"([A-Z][A-Z0-9.]+|[0-9]+[,.][0-9]+|[cdjlmnst]'|qu'|[\w'-]+|\S)")


class Sentence:
    def __init__(self, text, sent_line):
        self.text = text
        self.sent_line = sent_line

    def __len__(self):
        return len(tokenizer.tokenize(self.text))

    @property
    def id(self):
        return self.sent_line

    def contains_pos(self, postag):
        return False

    def count_pos(self, postag):
        return 0


def read_sentence(stream, sent_line):
    line = next(stream).strip()
    sent_line += 1
    if line:
        return Sentence(line, sent_line), sent_line
    else:
        return None, sent_line


def read_corpus(path):
    sentences = []
    nth = 0
    with open(path) as input_stream:
        try:
            while True:
                sentence, nth = read_sentence(input_stream, nth)
                if sentence:
                    sentences.append(sentence)
        except StopIteration:
            pass
    return sentences
