<s> from sklearn . cluster import KMeans <EOL> from collections import defaultdict <EOL> from gensim . models . word2vec import Word2Vec <EOL> from Utils . file_utils import load_stop_words <EOL> from Config . generate_cluster_synonyms_config import GenerateClusterSynonymsConfig <EOL> import numpy as np <EOL> import time <EOL> import sys <EOL> def get_vector ( item , model ) : <EOL> vocab = model . vocab [ item ] <EOL> vector = model . syn0 [ vocab . index ] <EOL> return vector <EOL> def get_norm_vector ( item , model ) : <EOL> if item not in model . vocab : <EOL> return None <EOL> vec = get_vector ( item , model ) <EOL> norm = np . linalg . norm ( vec ) <EOL> if norm != <NUM_LIT:0> : <EOL> return vec / norm <EOL> return vec <EOL> def map_keyword ( kw ) : <EOL> return kw . replace ( "<STR_LIT:U+0020>" , "<STR_LIT:_>" ) <EOL> def extract_clusters ( ids , id2kwd ) : <EOL> clusters = defaultdict ( set ) <EOL> for kw_id , label in enumerate ( ids ) : <EOL> kw = id2kwd [ kw_id ] <EOL> clusters [ label ] . add ( kw ) <EOL> return clusters <EOL> def write_most_similar_clusters ( topn , kwd2cluster_sims , synonym_qry_fname , synonyn_index_fname ) : <EOL> kwords = sorted ( kwd2cluster_sims . keys ( ) ) <EOL> cluster_label = lambda lbl : "<STR_LIT>" + str ( lbl ) <EOL> with open ( synonym_qry_fname , "<STR_LIT>" ) as qry_f : <EOL> for kword in kwords : <EOL> cl_sims = kwd2cluster_sims [ kword ] <EOL> top_clusters = cl_sims [ : topn ] <EOL> if len ( top_clusters ) > <NUM_LIT:0> : <EOL> qry_f . write ( "<STR_LIT>" % kword ) <EOL> for lbl , sim in top_clusters : <EOL> qry_f . write ( "<STR_LIT>" % ( cluster_label ( lbl ) , sim ) ) <EOL> qry_f . write ( "<STR_LIT:\n>" ) <EOL> with open ( synonyn_index_fname , "<STR_LIT>" ) as f : <EOL> for kword in kwords : <EOL> lbl , sim = kwd2cluster_sims [ kword ] [ <NUM_LIT:0> ] <EOL> f . write ( "<STR_LIT>" % ( kword , cluster_label ( lbl ) ) ) <EOL> """<STR_LIT>""" <EOL> if len ( sys . argv ) != <NUM_LIT:2> : <EOL> raise Exception ( "<STR_LIT>" ) <EOL> config = GenerateClusterSynonymsConfig ( sys . argv [ <NUM_LIT:1> ] ) <EOL> model = Word2Vec . load ( config . model_file ) <EOL> print ( "<STR_LIT>" ) <EOL> keywords = set ( ) <EOL> for file in config . keywords_files : <EOL> keywords . update ( load_stop_words ( file ) ) <EOL> print ( "<STR_LIT>" % ( len ( keywords ) ) ) <EOL> id2kwd = dict ( ) <EOL> kwd2id = dict ( ) <EOL> vectors = [ ] <EOL> for term in keywords : <EOL> id2kwd [ len ( vectors ) ] = term <EOL> kwd2id [ term ] = len ( vectors ) <EOL> vec = get_norm_vector ( term , model ) <EOL> if vec is not None : <EOL> vectors . append ( vec ) <EOL> start = time . time ( ) <EOL> print ( "<STR_LIT>" % config . num_clusters ) <EOL> km_clusterer = KMeans ( n_clusters = config . num_clusters , n_jobs = <NUM_LIT:1> , verbose = <NUM_LIT:1> , n_init = <NUM_LIT:5> ) <EOL> ids = km_clusterer . fit_predict ( vectors ) <EOL> end = time . time ( ) <EOL> print ( "<STR_LIT>" % ( config . num_clusters , end - start ) ) <EOL> lbl2cluster = extract_clusters ( ids , id2kwd ) <EOL> with open ( config . synonyms_file , "<STR_LIT>" ) as f : <EOL> for lbl , words in lbl2cluster . items ( ) : <EOL> line = "<STR_LIT:U+002C>" . join ( sorted ( words ) ) <EOL> f . write ( line ) <EOL> f . write ( "<STR_LIT>" ) <EOL> f . write ( "<STR_LIT>" + str ( lbl ) + "<STR_LIT:\n>" ) </s>
