def load_ptb_dataset(path='data/ptb/'):
 print 'Load or Download Penn TreeBank (PTB) dataset > {}'.format(path)
 filename = 'simple-examples.tgz'
 url = 'http://www.fit.vutbr.cz/~imikolov/rnnlm/'
 maybe_download_and_extract(filename, path, url, extract=True)
 data_path = os.path.join(path, 'simple-examples', 'data')
 train_path = os.path.join(data_path, 'ptb.train.txt')
 valid_path = os.path.join(data_path, 'ptb.valid.txt')
 test_path = os.path.join(data_path, 'ptb.test.txt')
 word_to_id = nlp.build_vocab(nlp.read_words(train_path))
 train_data = nlp.words_to_word_ids(nlp.read_words(train_path), word_to_id)
 valid_data = nlp.words_to_word_ids(nlp.read_words(valid_path), word_to_id)
 test_data = nlp.words_to_word_ids(nlp.read_words(test_path), word_to_id)
 vocabulary = len(word_to_id)
 return (train_data, valid_data, test_data, vocabulary)

def load_ptb_dataset ( path = 'data/ptb/' ) : print 'Load or Download Penn TreeBank (PTB) dataset > {}' . format ( path ) filename = 'simple-examples.tgz' url = 'http://www.fit.vutbr.cz/~imikolov/rnnlm/' maybe_download_and_extract ( filename , path , url , extract = True ) data_path = os . path . join ( path , 'simple-examples' , 'data' ) train_path = os . path . join ( data_path , 'ptb.train.txt' ) valid_path = os . path . join ( data_path , 'ptb.valid.txt' ) test_path = os . path . join ( data_path , 'ptb.test.txt' ) word_to_id = nlp . build_vocab ( nlp . read_words ( train_path ) ) train_data = nlp . words_to_word_ids ( nlp . read_words ( train_path ) , word_to_id ) valid_data = nlp . words_to_word_ids ( nlp . read_words ( valid_path ) , word_to_id ) test_data = nlp . words_to_word_ids ( nlp . read_words ( test_path ) , word_to_id ) vocabulary = len ( word_to_id ) return ( train_data , valid_data , test_data , vocabulary )
penn treebank dataset is used in many language modeling papers .