<s> import codecs <EOL> import collections <EOL> import itertools <EOL> import math <EOL> import os <EOL> import subprocess <EOL> from xml . dom import minidom <EOL> from east import consts <EOL> from east import exceptions <EOL> from east import utils as common_utils <EOL> from east . synonyms import utils <EOL> class SynonymExtractor ( object ) : <EOL> def __init__ ( self , input_path ) : <EOL> self . current_os = utils . determine_operating_system ( ) <EOL> self . tomita_path , self . tomita_binary = self . _get_tomita_path ( ) <EOL> if self . tomita_binary is None : <EOL> raise exceptions . TomitaNotInstalledException ( ) <EOL> self . text , self . number_of_texts = self . _retrieve_text ( input_path ) <EOL> self . dependency_triples , self . dt_for_r , self . dt_for_w1r , self . dt_for_rw2 = self . _retrieve_dependency_triples ( self . text ) <EOL> self . word_frequencies = self . _calculate_word_frequencies ( self . text ) <EOL> self . frequencies = self . _calculate_dt_frequencies ( self . dependency_triples ) <EOL> self . words = set ( [ dt [ <NUM_LIT:0> ] for dt in self . dependency_triples ] + <EOL> [ dt [ <NUM_LIT:2> ] for dt in self . dependency_triples ] ) <EOL> self . relations = set ( [ dt [ <NUM_LIT:1> ] for dt in self . dependency_triples ] ) <EOL> self . I_memoized = { } <EOL> self . T_memoized = { } <EOL> self . synonyms_memoized = { } <EOL> def _retrieve_text ( self , input_path ) : <EOL> if os . path . isdir ( input_path ) : <EOL> text = "<STR_LIT>" <EOL> number_of_texts = <NUM_LIT:0> <EOL> for file_name in os . listdir ( input_path ) : <EOL> if file_name . endswith ( "<STR_LIT>" ) : <EOL> with open ( os . path . abspath ( input_path ) + "<STR_LIT:/>" + file_name ) as f : <EOL> text += f . read ( ) <EOL> number_of_texts += <NUM_LIT:1> <EOL> else : <EOL> with open ( input_path ) as f : <EOL> text = f . read ( ) <EOL> number_of_texts = <NUM_LIT:1> <EOL> return text , number_of_texts <EOL> def _retrieve_dependency_triples ( self , text ) : <EOL> dependency_triples = [ ] <EOL> dt_for_r = collections . defaultdict ( list ) <EOL> dt_for_w1r = collections . defaultdict ( list ) <EOL> dt_for_rw2 = collections . defaultdict ( list ) <EOL> p = subprocess . Popen ( [ self . tomita_binary , "<STR_LIT>" ] , <EOL> stdin = subprocess . PIPE , stdout = subprocess . PIPE , <EOL> stderr = subprocess . PIPE , cwd = self . tomita_path , <EOL> shell = ( self . current_os == consts . OperatingSystem . WINDOWS ) ) <EOL> out , err = p . communicate ( input = text ) <EOL> xmldoc = minidom . parseString ( out ) <EOL> relations = xmldoc . getElementsByTagName ( '<STR_LIT>' ) <EOL> for rel in relations : <EOL> r = rel . childNodes [ <NUM_LIT:0> ] . nodeName <EOL> w1 , w2 = rel . childNodes [ <NUM_LIT:0> ] . attributes [ '<STR_LIT>' ] . value . split ( '<STR_LIT:U+0020>' , <NUM_LIT:1> ) <EOL> dt = ( w1 , r , w2 ) <EOL> dependency_triples . append ( dt ) <EOL> dt_for_r [ r ] . append ( dt ) <EOL> dt_for_w1r [ ( w1 , r ) ] . append ( dt ) <EOL> dt_for_rw2 [ ( r , w2 ) ] . append ( dt ) <EOL> r_inv = r [ : - <NUM_LIT:3> ] if r . endswith ( "<STR_LIT>" ) else ( r + "<STR_LIT>" ) <EOL> dt_inv = ( w2 , r_inv , w1 ) <EOL> dependency_triples . append ( dt_inv ) <EOL> dt_for_r [ r_inv ] . append ( dt_inv ) <EOL> dt_for_w1r [ ( w2 , r_inv ) ] . append ( dt_inv ) <EOL> dt_for_rw2 [ ( r_inv , w1 ) ] . append ( dt_inv ) <EOL> return dependency_triples , dt_for_r , dt_for_w1r , dt_for_rw2 <EOL> def _get_tomita_path ( self ) : <EOL> tomita_path = ( os . path . dirname ( os . path . abspath ( __file__ ) ) + <EOL> "<STR_LIT>" ) <EOL> if self . current_os == consts . OperatingSystem . LINUX_64 : <EOL> tomita_binary = "<STR_LIT>" <EOL> elif self . current_os == consts . OperatingSystem . LINUX_32 : <EOL> tomita_binary = "<STR_LIT>" <EOL> elif self . current_os == consts . OperatingSystem . WINDOWS : <EOL> tomita_binary = "<STR_LIT>" <EOL> elif self . current_os == consts . OperatingSystem . MACOS : <EOL> tomita_binary = "<STR_LIT>" <EOL> if not os . access ( tomita_path + tomita_binary , os . F_OK ) : <EOL> tomita_binary = None <EOL> return tomita_path , tomita_binary <EOL> def _calculate_word_frequencies ( self , text ) : <EOL> text = common_utils . prepare_text ( text ) <EOL> words = common_utils . tokenize ( text ) <EOL> res = collections . defaultdict ( int ) <EOL> for word in words : <EOL> res [ word ] += <NUM_LIT:1> <EOL> return res <EOL> def _calculate_dt_frequencies ( self , dependency_triples ) : <EOL> res = collections . defaultdict ( int ) <EOL> for dt in dependency_triples : <EOL> res [ dt ] += <NUM_LIT:1> <EOL> return res <EOL> def I ( self , w1 , r , w2 ) : <EOL> if ( w1 , r , w2 ) in self . I_memoized : <EOL> return self . I_memoized [ ( w1 , r , w2 ) ] <EOL> fr_w1rw2 = self . frequencies [ w1 , r , w2 ] <EOL> if not fr_w1rw2 : <EOL> return <NUM_LIT:0.0> <EOL> fr__r_ = sum ( self . frequencies [ triple ] for triple in self . dt_for_r [ r ] ) <EOL> fr_w1r_ = sum ( self . frequencies [ triple ] for triple in self . dt_for_w1r [ ( w1 , r ) ] ) <EOL> fr__rw2 = sum ( self . frequencies [ triple ] for triple in self . dt_for_rw2 [ ( r , w2 ) ] ) <EOL> res = max ( math . log ( float ( fr_w1rw2 ) * fr__r_ / fr_w1r_ / fr__rw2 ) , <NUM_LIT:0.0> ) <EOL> self . I_memoized [ ( w1 , r , w2 ) ] = res <EOL> return res <EOL> def T ( self , w ) : <EOL> if w in self . T_memoized : <EOL> return self . T_memoized [ w ] <EOL> res = set ( filter ( lambda ( r , w_ ) : self . I ( w , r , w_ ) > <NUM_LIT:0> , <EOL> itertools . product ( self . relations , self . words ) ) ) <EOL> self . T_memoized [ w ] = res <EOL> return res <EOL> def similarity ( self , w1 , w2 ) : <EOL> numerator = sum ( self . I ( w1 , r , w ) + self . I ( w2 , r , w ) <EOL> for ( r , w ) in self . T ( w1 ) & self . T ( w2 ) ) <EOL> denominator = ( sum ( self . I ( w1 , r , w ) for ( r , w ) in self . T ( w1 ) ) + <EOL> sum ( self . I ( w2 , r , w ) for ( r , w ) in self . T ( w2 ) ) ) <EOL> if denominator : <EOL> return numerator / denominator <EOL> else : <EOL> return <NUM_LIT:0.0> <EOL> def get_synonyms ( self , threshold = <NUM_LIT> , return_similarity_measure = False ) : <EOL> synonyms = collections . defaultdict ( list ) <EOL> words = filter ( lambda w : len ( w ) > <NUM_LIT:2> and <EOL> self . word_frequencies [ w ] > self . number_of_texts / <NUM_LIT:50> , <EOL> self . words ) <EOL> combs = itertools . combinations ( words , <NUM_LIT:2> ) <EOL> for w1 , w2 in combs : <EOL> sim = self . similarity ( w1 , w2 ) <EOL> if sim > threshold : <EOL> if return_similarity_measure : <EOL> synonyms [ w1 ] . append ( ( w2 , sim ) ) <EOL> synonyms [ w2 ] . append ( ( w1 , sim ) ) <EOL> else : <EOL> synonyms [ w1 ] . append ( w2 ) <EOL> synonyms [ w2 ] . append ( w1 ) <EOL> return synonyms </s>
