def dir_corpus(plain_dir, chunk_name='article', encoding='utf8', 
               paragraphs=True, ignore=['.json','.log','.pickle'], 
               nltk_stop=True, stop_freq=1, add_stop=None, decode=False, 
               verbose=1, simple=False, tokenizer=word_tokenize):
    chunks = []
    filenames = os.listdir(plain_dir)
    filenames = filter_by_suffix(filenames, ignore)
    filenames.sort()

    for filename in filenames:
        filename = os.path.join(plain_dir, filename)
        if encoding == 'detect':
            encoding = detect_encoding(filename)
        try:
            if decode:
                with open(filename, mode='r', encoding=encoding) as f:
                    chunks.append(unidecode(f.read()))
            else:
                with open(filename, mode='r', encoding=encoding) as f:
                    chunks.append(f.read())
        except UnicodeDecodeError:
            encoding = detect_encoding(filename)
            if decode:
                with open(filename, mode='r', encoding=encoding) as f:
                    chunks.append(unidecode(f.read()))
            else:
                with open(filename, mode='r', encoding=encoding) as f:
                    chunks.append(f.read())

    words, tok = dir_tokenize(chunks, filenames, chunk_name=chunk_name,
                              paragraphs=paragraphs, verbose=verbose,
                              simple=simple, tokenizer=tokenizer)
    names, data = zip(*tok.items())
    
    c = Corpus(words, context_data=data, context_types=names)
    if nltk_stop or stop_freq or add_stop:
        c = apply_stoplist(c, nltk_stop=nltk_stop,
                              freq=stop_freq, add_stop=add_stop)
    return c
`dir_corpus` is a convenience function for generating Corpus
dataset/ETHPy150Open inpho/vsm/vsm/extensions/corpusbuilders/corpusbuilders.py/dir_corpus