<s> """<STR_LIT>""" <EOL> from __future__ import print_function <EOL> import itertools as _itertools <EOL> from nltk . compat import iteritems <EOL> from nltk . probability import FreqDist <EOL> from nltk . util import ngrams <EOL> from nltk . metrics import ContingencyMeasures , BigramAssocMeasures , TrigramAssocMeasures <EOL> from nltk . metrics . spearman import ranks_from_scores , spearman_correlation <EOL> class AbstractCollocationFinder ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , word_fd , ngram_fd ) : <EOL> self . word_fd = word_fd <EOL> self . N = word_fd . N ( ) <EOL> self . ngram_fd = ngram_fd <EOL> @ classmethod <EOL> def _build_new_documents ( cls , documents , window_size , pad_left = False , pad_right = False , pad_symbol = None ) : <EOL> '''<STR_LIT>''' <EOL> padding = ( pad_symbol , ) * ( window_size - <NUM_LIT:1> ) <EOL> if pad_right : <EOL> return _itertools . chain . from_iterable ( _itertools . chain ( doc , padding ) for doc in documents ) <EOL> if pad_left : <EOL> return _itertools . chain . from_iterable ( _itertools . chain ( padding , doc ) for doc in documents ) <EOL> @ classmethod <EOL> def from_documents ( cls , documents ) : <EOL> """<STR_LIT>""" <EOL> return cls . from_words ( cls . _build_new_documents ( documents , cls . default_ws , pad_right = True ) ) <EOL> @ staticmethod <EOL> def _ngram_freqdist ( words , n ) : <EOL> return FreqDist ( tuple ( words [ i : i + n ] ) for i in range ( len ( words ) - <NUM_LIT:1> ) ) <EOL> def _apply_filter ( self , fn = lambda ngram , freq : False ) : <EOL> """<STR_LIT>""" <EOL> tmp_ngram = FreqDist ( ) <EOL> for ngram , freq in iteritems ( self . ngram_fd ) : <EOL> if not fn ( ngram , freq ) : <EOL> tmp_ngram [ ngram ] = freq <EOL> self . ngram_fd = tmp_ngram <EOL> def apply_freq_filter ( self , min_freq ) : <EOL> """<STR_LIT>""" <EOL> self . _apply_filter ( lambda ng , freq : freq < min_freq ) <EOL> def apply_ngram_filter ( self , fn ) : <EOL> """<STR_LIT>""" <EOL> self . _apply_filter ( lambda ng , f : fn ( * ng ) ) <EOL> def apply_word_filter ( self , fn ) : <EOL> """<STR_LIT>""" <EOL> self . _apply_filter ( lambda ng , f : any ( fn ( w ) for w in ng ) ) <EOL> def _score_ngrams ( self , score_fn ) : <EOL> """<STR_LIT>""" <EOL> for tup in self . ngram_fd : <EOL> score = self . score_ngram ( score_fn , * tup ) <EOL> if score is not None : <EOL> yield tup , score <EOL> def score_ngrams ( self , score_fn ) : <EOL> """<STR_LIT>""" <EOL> return sorted ( self . _score_ngrams ( score_fn ) , key = lambda t : ( - t [ <NUM_LIT:1> ] , t [ <NUM_LIT:0> ] ) ) <EOL> def nbest ( self , score_fn , n ) : <EOL> """<STR_LIT>""" <EOL> return [ p for p , s in self . score_ngrams ( score_fn ) [ : n ] ] <EOL> def above_score ( self , score_fn , min_score ) : <EOL> """<STR_LIT>""" <EOL> for ngram , score in self . score_ngrams ( score_fn ) : <EOL> if score > min_score : <EOL> yield ngram <EOL> else : <EOL> break <EOL> class BigramCollocationFinder ( AbstractCollocationFinder ) : <EOL> """<STR_LIT>""" <EOL> default_ws = <NUM_LIT:2> <EOL> def __init__ ( self , word_fd , bigram_fd , window_size = <NUM_LIT:2> ) : <EOL> """<STR_LIT>""" <EOL> AbstractCollocationFinder . __init__ ( self , word_fd , bigram_fd ) <EOL> self . window_size = window_size <EOL> @ classmethod <EOL> def from_words ( cls , words , window_size = <NUM_LIT:2> ) : <EOL> """<STR_LIT>""" <EOL> wfd = FreqDist ( ) <EOL> bfd = FreqDist ( ) <EOL> if window_size < <NUM_LIT:2> : <EOL> raise ValueError ( "<STR_LIT>" ) <EOL> for window in ngrams ( words , window_size , pad_right = True ) : <EOL> w1 = window [ <NUM_LIT:0> ] <EOL> if w1 is None : <EOL> continue <EOL> wfd [ w1 ] += <NUM_LIT:1> <EOL> for w2 in window [ <NUM_LIT:1> : ] : <EOL> if w2 is not None : <EOL> bfd [ ( w1 , w2 ) ] += <NUM_LIT:1> <EOL> return cls ( wfd , bfd , window_size = window_size ) <EOL> def score_ngram ( self , score_fn , w1 , w2 ) : <EOL> """<STR_LIT>""" <EOL> n_all = self . N <EOL> n_ii = self . ngram_fd [ ( w1 , w2 ) ] / ( self . window_size - <NUM_LIT:1.0> ) <EOL> if not n_ii : <EOL> return <EOL> n_ix = self . word_fd [ w1 ] <EOL> n_xi = self . word_fd [ w2 ] <EOL> return score_fn ( n_ii , ( n_ix , n_xi ) , n_all ) <EOL> class TrigramCollocationFinder ( AbstractCollocationFinder ) : <EOL> """<STR_LIT>""" <EOL> default_ws = <NUM_LIT:3> <EOL> def __init__ ( self , word_fd , bigram_fd , wildcard_fd , trigram_fd ) : <EOL> """<STR_LIT>""" <EOL> AbstractCollocationFinder . __init__ ( self , word_fd , trigram_fd ) <EOL> self . wildcard_fd = wildcard_fd <EOL> self . bigram_fd = bigram_fd <EOL> @ classmethod <EOL> def from_words ( cls , words , window_size = <NUM_LIT:3> ) : <EOL> """<STR_LIT>""" <EOL> if window_size < <NUM_LIT:3> : <EOL> raise ValueError ( "<STR_LIT>" ) <EOL> wfd = FreqDist ( ) <EOL> wildfd = FreqDist ( ) <EOL> bfd = FreqDist ( ) <EOL> tfd = FreqDist ( ) <EOL> for window in ngrams ( words , window_size , pad_right = True ) : <EOL> w1 = window [ <NUM_LIT:0> ] <EOL> if w1 is None : <EOL> continue <EOL> for w2 , w3 in _itertools . combinations ( window [ <NUM_LIT:1> : ] , <NUM_LIT:2> ) : <EOL> wfd [ w1 ] += <NUM_LIT:1> <EOL> if w2 is None : <EOL> continue <EOL> bfd [ ( w1 , w2 ) ] += <NUM_LIT:1> <EOL> if w3 is None : <EOL> continue <EOL> wildfd [ ( w1 , w3 ) ] += <NUM_LIT:1> <EOL> tfd [ ( w1 , w2 , w3 ) ] += <NUM_LIT:1> <EOL> return cls ( wfd , bfd , wildfd , tfd ) <EOL> def bigram_finder ( self ) : <EOL> """<STR_LIT>""" <EOL> return BigramCollocationFinder ( self . word_fd , self . bigram_fd ) <EOL> def score_ngram ( self , score_fn , w1 , w2 , w3 ) : <EOL> """<STR_LIT>""" <EOL> n_all = self . N <EOL> n_iii = self . ngram_fd [ ( w1 , w2 , w3 ) ] <EOL> if not n_iii : <EOL> return <EOL> n_iix = self . bigram_fd [ ( w1 , w2 ) ] <EOL> n_ixi = self . wildcard_fd [ ( w1 , w3 ) ] <EOL> n_xii = self . bigram_fd [ ( w2 , w3 ) ] <EOL> n_ixx = self . word_fd [ w1 ] <EOL> n_xix = self . word_fd [ w2 ] <EOL> n_xxi = self . word_fd [ w3 ] <EOL> return score_fn ( n_iii , <EOL> ( n_iix , n_ixi , n_xii ) , <EOL> ( n_ixx , n_xix , n_xxi ) , <EOL> n_all ) <EOL> class QuadgramCollocationFinder ( AbstractCollocationFinder ) : <EOL> """<STR_LIT>""" <EOL> default_ws = <NUM_LIT:4> <EOL> def __init__ ( self , word_fd , quadgram_fd , ii , iii , ixi , ixxi , iixi , ixii ) : <EOL> """<STR_LIT>""" <EOL> AbstractCollocationFinder . __init__ ( self , word_fd , quadgram_fd ) <EOL> self . iii = iii <EOL> self . ii = ii <EOL> self . ixi = ixi <EOL> self . ixxi = ixxi <EOL> self . iixi = iixi <EOL> self . ixii = ixii <EOL> @ classmethod <EOL> def from_words ( cls , words , window_size = <NUM_LIT:4> ) : <EOL> if window_size < <NUM_LIT:4> : <EOL> raise ValueError ( "<STR_LIT>" ) <EOL> ixxx = FreqDist ( ) <EOL> iiii = FreqDist ( ) <EOL> ii = FreqDist ( ) <EOL> iii = FreqDist ( ) <EOL> ixi = FreqDist ( ) <EOL> ixxi = FreqDist ( ) <EOL> iixi = FreqDist ( ) <EOL> ixii = FreqDist ( ) <EOL> for window in ngrams ( words , window_size , pad_right = True ) : <EOL> w1 = window [ <NUM_LIT:0> ] <EOL> if w1 is None : <EOL> continue <EOL> for w2 , w3 , w4 in _itertools . combinations ( window [ <NUM_LIT:1> : ] , <NUM_LIT:3> ) : <EOL> ixxx [ w1 ] += <NUM_LIT:1> <EOL> if w2 is None : <EOL> continue <EOL> ii [ ( w1 , w2 ) ] += <NUM_LIT:1> <EOL> if w3 is None : <EOL> continue <EOL> iii [ ( w1 , w2 , w3 ) ] += <NUM_LIT:1> <EOL> ixi [ ( w1 , w3 ) ] += <NUM_LIT:1> <EOL> if w4 is None : <EOL> continue <EOL> iiii [ ( w1 , w2 , w3 , w4 ) ] += <NUM_LIT:1> <EOL> ixxi [ ( w1 , w4 ) ] += <NUM_LIT:1> <EOL> ixii [ ( w1 , w3 , w4 ) ] += <NUM_LIT:1> <EOL> iixi [ ( w1 , w2 , w4 ) ] += <NUM_LIT:1> <EOL> return cls ( ixxx , iiii , ii , iii , ixi , ixxi , iixi , ixii ) <EOL> def score_ngram ( self , score_fn , w1 , w2 , w3 , w4 ) : <EOL> n_all = self . N <EOL> n_iiii = self . ngram_fd [ ( w1 , w2 , w3 , w4 ) ] <EOL> if not n_iiii : <EOL> return <EOL> n_iiix = self . iii [ ( w1 , w2 , w3 ) ] <EOL> n_xiii = self . iii [ ( w2 , w3 , w4 ) ] <EOL> n_iixi = self . iixi [ ( w1 , w2 , w4 ) ] <EOL> n_ixii = self . ixii [ ( w1 , w3 , w4 ) ] <EOL> n_iixx = self . ii [ ( w1 , w2 ) ] <EOL> n_xxii = self . ii [ ( w3 , w4 ) ] <EOL> n_xiix = self . ii [ ( w2 , w3 ) ] <EOL> n_ixix = self . ixi [ ( w1 , w3 ) ] <EOL> n_ixxi = self . ixxi [ ( w1 , w4 ) ] <EOL> n_xixi = self . ixi [ ( w2 , w4 ) ] <EOL> n_ixxx = self . word_fd [ w1 ] <EOL> n_xixx = self . word_fd [ w2 ] <EOL> n_xxix = self . word_fd [ w3 ] <EOL> n_xxxi = self . word_fd [ w4 ] <EOL> return score_fn ( n_iiii , <EOL> ( n_iiix , n_iixi , n_ixii , n_xiii ) , <EOL> ( n_iixx , n_ixix , n_ixxi , n_xixi , n_xxii , n_xiix ) , <EOL> ( n_ixxx , n_xixx , n_xxix , n_xxxi ) , <EOL> n_all ) <EOL> def demo ( scorer = None , compare_scorer = None ) : <EOL> """<STR_LIT>""" <EOL> from nltk . metrics import BigramAssocMeasures , spearman_correlation , ranks_from_scores <EOL> if scorer is None : <EOL> scorer = BigramAssocMeasures . likelihood_ratio <EOL> if compare_scorer is None : <EOL> compare_scorer = BigramAssocMeasures . raw_freq <EOL> from nltk . corpus import stopwords , webtext <EOL> ignored_words = stopwords . words ( '<STR_LIT>' ) <EOL> word_filter = lambda w : len ( w ) < <NUM_LIT:3> or w . lower ( ) in ignored_words <EOL> for file in webtext . fileids ( ) : <EOL> words = [ word . lower ( ) <EOL> for word in webtext . words ( file ) ] <EOL> cf = BigramCollocationFinder . from_words ( words ) <EOL> cf . apply_freq_filter ( <NUM_LIT:3> ) <EOL> cf . apply_word_filter ( word_filter ) <EOL> corr = spearman_correlation ( ranks_from_scores ( cf . score_ngrams ( scorer ) ) , <EOL> ranks_from_scores ( cf . score_ngrams ( compare_scorer ) ) ) <EOL> print ( file ) <EOL> print ( '<STR_LIT:\t>' , [ '<STR_LIT:U+0020>' . join ( tup ) for tup in cf . nbest ( scorer , <NUM_LIT:15> ) ] ) <EOL> print ( '<STR_LIT>' % ( compare_scorer . __name__ , corr ) ) <EOL> if __name__ == '<STR_LIT:__main__>' : <EOL> import sys <EOL> from nltk . metrics import BigramAssocMeasures <EOL> try : <EOL> scorer = eval ( '<STR_LIT>' + sys . argv [ <NUM_LIT:1> ] ) <EOL> except IndexError : <EOL> scorer = None <EOL> try : <EOL> compare_scorer = eval ( '<STR_LIT>' + sys . argv [ <NUM_LIT:2> ] ) <EOL> except IndexError : <EOL> compare_scorer = None <EOL> demo ( scorer , compare_scorer ) <EOL> __all__ = [ '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' ] </s>
