def gen_docs(corpus, lemmatize, rm_stops):
    """Open and process files from a corpus. Return a list of sentences for an author. Each sentence
    is itself a list of tokenized words.
    """

    assert corpus in ['phi5', 'tlg']

    if corpus == 'phi5':
        language = 'latin'
        filepaths = assemble_phi5_author_filepaths()
        jv_replacer = JVReplacer()
        text_cleaner = phi5_plaintext_cleanup
        word_tokenizer = nltk_tokenize_words
        if rm_stops:
            stops = latin_stops
        else:
            stops = None
    elif corpus == 'tlg':
        language = 'greek'
        filepaths = assemble_tlg_author_filepaths()
        text_cleaner = tlg_plaintext_cleanup
        word_tokenizer = nltk_tokenize_words

        if rm_stops:
            stops = latin_stops
        else:
            stops = None

    if lemmatize:
        lemmatizer = LemmaReplacer(language)

    sent_tokenizer = TokenizeSentence(language)

    for filepath in filepaths:
        with open(filepath) as f:
            text = f.read()
        # light first-pass cleanup, before sentence tokenization (which relies on punctuation)
        text = text_cleaner(text, rm_punctuation=False, rm_periods=False)
        sent_tokens = sent_tokenizer.tokenize_sentences(text)
        # doc_sentences = []
        for sentence in sent_tokens:
            # a second cleanup at sentence-level, to rm all punctuation
            sentence = text_cleaner(sentence, rm_punctuation=True, rm_periods=True)
            sentence = word_tokenizer(sentence)
            sentence = [s.lower() for s in sentence]
            sentence = [w for w in sentence if w]
            if language == 'latin':
                sentence = [w[1:] if w.startswith('-') else w for w in sentence]

            if stops:
                sentence = [w for w in sentence if w not in stops]

            sentence = [w for w in sentence if len(w) > 1]  # rm short words

            if sentence:
                sentence = sentence

            if lemmatize:
                sentence = lemmatizer.lemmatize(sentence)
            if sentence and language == 'latin':
                sentence = [jv_replacer.replace(word) for word in sentence]
            if sentence:
                yield sentence
def gen_docs ( corpus , lemmatize , rm_stops ) : assert corpus in [ 'phi5' , 'tlg' ] if corpus == 'phi5' : language = 'latin' filepaths = assemble_phi5_author_filepaths ( ) jv_replacer = JVReplacer ( ) text_cleaner = phi5_plaintext_cleanup word_tokenizer = nltk_tokenize_words if rm_stops : stops = latin_stops else : stops = None elif corpus == 'tlg' : language = 'greek' filepaths = assemble_tlg_author_filepaths ( ) text_cleaner = tlg_plaintext_cleanup word_tokenizer = nltk_tokenize_words if rm_stops : stops = latin_stops else : stops = None if lemmatize : lemmatizer = LemmaReplacer ( language ) sent_tokenizer = TokenizeSentence ( language ) for filepath in filepaths : with open ( filepath ) as f : text = f . read ( ) text = text_cleaner ( text , rm_punctuation = False , rm_periods = False ) sent_tokens = sent_tokenizer . tokenize_sentences ( text ) for sentence in sent_tokens : sentence = text_cleaner ( sentence , rm_punctuation = True , rm_periods = True ) sentence = word_tokenizer ( sentence ) sentence = [ s . lower ( ) for s in sentence ] sentence = [ w for w in sentence if w ] if language == 'latin' : sentence = [ w [ 1 : ] if w . startswith ( '-' ) else w for w in sentence ] if stops : sentence = [ w for w in sentence if w not in stops ] sentence = [ w for w in sentence if len ( w ) > 1 ] if sentence : sentence = sentence if lemmatize : sentence = lemmatizer . lemmatize ( sentence ) if sentence and language == 'latin' : sentence = [ jv_replacer . replace ( word ) for word in sentence ] if sentence : yield sentence
Open and process files from a corpus. Return a list of sentences for an author. Each sentence
    is itself a list of tokenized words.