<s> import logging <EOL> import pickle <EOL> from sklearn . datasets import fetch_20newsgroups <EOL> import numpy as np <EOL> from lda2vec import preprocess , Corpus <EOL> logging . basicConfig ( ) <EOL> remove = ( '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ) <EOL> texts = fetch_20newsgroups ( subset = '<STR_LIT:train>' , remove = remove ) . data <EOL> bad = set ( [ "<STR_LIT>" , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] ) <EOL> def clean ( line ) : <EOL> return '<STR_LIT:U+0020>' . join ( w for w in line . split ( ) if not any ( t in w for t in bad ) ) <EOL> max_length = <NUM_LIT> <EOL> texts = [ unicode ( clean ( d ) ) for d in texts ] <EOL> tokens , vocab = preprocess . tokenize ( texts , max_length , merge = True , <EOL> n_threads = <NUM_LIT:4> ) <EOL> corpus = Corpus ( ) <EOL> corpus . update_word_count ( tokens ) <EOL> corpus . finalize ( ) <EOL> compact = corpus . to_compact ( tokens ) <EOL> pruned = corpus . filter_count ( compact , min_count = <NUM_LIT:30> ) <EOL> bow = corpus . compact_to_bow ( pruned ) <EOL> clean = corpus . subsample_frequent ( pruned ) <EOL> doc_ids = np . arange ( pruned . shape [ <NUM_LIT:0> ] ) <EOL> flattened , ( doc_ids , ) = corpus . compact_to_flat ( pruned , doc_ids ) <EOL> assert flattened . min ( ) >= <NUM_LIT:0> <EOL> n_dim = <NUM_LIT> <EOL> vectors = np . random . randn ( ( flattened . max ( ) + <NUM_LIT:1> , n_dim ) ) * <NUM_LIT> <EOL> fn_wordvc = '<STR_LIT>' <EOL> vectors , s , f = corpus . compact_word_vectors ( vocab , filename = fn_wordvc , <EOL> use_spacy = False ) <EOL> pickle . dump ( vocab , open ( '<STR_LIT>' , '<STR_LIT:w>' ) ) <EOL> pickle . dump ( corpus , open ( '<STR_LIT>' , '<STR_LIT:w>' ) ) <EOL> np . save ( "<STR_LIT>" , flattened ) <EOL> np . save ( "<STR_LIT>" , doc_ids ) <EOL> np . save ( "<STR_LIT>" , pruned ) <EOL> np . save ( "<STR_LIT>" , bow ) <EOL> np . save ( "<STR_LIT>" , vectors ) </s>
