<s> """<STR_LIT>""" <EOL> from __future__ import division <EOL> import math as _math <EOL> from functools import reduce <EOL> _log2 = lambda x : _math . log ( x , <NUM_LIT> ) <EOL> _ln = _math . log <EOL> _product = lambda s : reduce ( lambda x , y : x * y , s ) <EOL> _SMALL = <NUM_LIT> <EOL> try : <EOL> from scipy . stats import fisher_exact <EOL> except ImportError : <EOL> def fisher_exact ( * _args , ** _kwargs ) : <EOL> raise NotImplementedError <EOL> NGRAM = <NUM_LIT:0> <EOL> """<STR_LIT>""" <EOL> UNIGRAMS = - <NUM_LIT:2> <EOL> """<STR_LIT>""" <EOL> TOTAL = - <NUM_LIT:1> <EOL> """<STR_LIT>""" <EOL> class NgramAssocMeasures ( object ) : <EOL> """<STR_LIT>""" <EOL> _n = <NUM_LIT:0> <EOL> @ staticmethod <EOL> def _contingency ( * marginals ) : <EOL> """<STR_LIT>""" <EOL> raise NotImplementedError ( "<STR_LIT>" <EOL> "<STR_LIT>" ) <EOL> @ staticmethod <EOL> def _marginals ( * contingency ) : <EOL> """<STR_LIT>""" <EOL> raise NotImplementedError ( "<STR_LIT>" <EOL> "<STR_LIT>" ) <EOL> @ classmethod <EOL> def _expected_values ( cls , cont ) : <EOL> """<STR_LIT>""" <EOL> n_all = sum ( cont ) <EOL> bits = [ <NUM_LIT:1> << i for i in range ( cls . _n ) ] <EOL> for i in range ( len ( cont ) ) : <EOL> yield ( _product ( sum ( cont [ x ] for x in range ( <NUM_LIT:2> ** cls . _n ) <EOL> if ( x & j ) == ( i & j ) ) <EOL> for j in bits ) / <EOL> ( n_all ** ( cls . _n - <NUM_LIT:1> ) ) ) <EOL> @ staticmethod <EOL> def raw_freq ( * marginals ) : <EOL> """<STR_LIT>""" <EOL> return marginals [ NGRAM ] / marginals [ TOTAL ] <EOL> @ classmethod <EOL> def student_t ( cls , * marginals ) : <EOL> """<STR_LIT>""" <EOL> return ( ( marginals [ NGRAM ] - <EOL> _product ( marginals [ UNIGRAMS ] ) / <EOL> ( marginals [ TOTAL ] ** ( cls . _n - <NUM_LIT:1> ) ) ) / <EOL> ( marginals [ NGRAM ] + _SMALL ) ** <NUM_LIT> ) <EOL> @ classmethod <EOL> def chi_sq ( cls , * marginals ) : <EOL> """<STR_LIT>""" <EOL> cont = cls . _contingency ( * marginals ) <EOL> exps = cls . _expected_values ( cont ) <EOL> return sum ( ( obs - exp ) ** <NUM_LIT:2> / ( exp + _SMALL ) <EOL> for obs , exp in zip ( cont , exps ) ) <EOL> @ staticmethod <EOL> def mi_like ( * marginals , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> return ( marginals [ NGRAM ] ** kwargs . get ( '<STR_LIT>' , <NUM_LIT:3> ) / <EOL> _product ( marginals [ UNIGRAMS ] ) ) <EOL> @ classmethod <EOL> def pmi ( cls , * marginals ) : <EOL> """<STR_LIT>""" <EOL> return ( _log2 ( marginals [ NGRAM ] * marginals [ TOTAL ] ** ( cls . _n - <NUM_LIT:1> ) ) - <EOL> _log2 ( _product ( marginals [ UNIGRAMS ] ) ) ) <EOL> @ classmethod <EOL> def likelihood_ratio ( cls , * marginals ) : <EOL> """<STR_LIT>""" <EOL> cont = cls . _contingency ( * marginals ) <EOL> return ( cls . _n * <EOL> sum ( obs * _ln ( obs / ( exp + _SMALL ) + _SMALL ) <EOL> for obs , exp in zip ( cont , cls . _expected_values ( cont ) ) ) ) <EOL> @ classmethod <EOL> def poisson_stirling ( cls , * marginals ) : <EOL> """<STR_LIT>""" <EOL> exp = ( _product ( marginals [ UNIGRAMS ] ) / <EOL> ( marginals [ TOTAL ] ** ( cls . _n - <NUM_LIT:1> ) ) ) <EOL> return marginals [ NGRAM ] * ( _log2 ( marginals [ NGRAM ] / exp ) - <NUM_LIT:1> ) <EOL> @ classmethod <EOL> def jaccard ( cls , * marginals ) : <EOL> """<STR_LIT>""" <EOL> cont = cls . _contingency ( * marginals ) <EOL> return cont [ <NUM_LIT:0> ] / sum ( cont [ : - <NUM_LIT:1> ] ) <EOL> class BigramAssocMeasures ( NgramAssocMeasures ) : <EOL> """<STR_LIT>""" <EOL> _n = <NUM_LIT:2> <EOL> @ staticmethod <EOL> def _contingency ( n_ii , n_ix_xi_tuple , n_xx ) : <EOL> """<STR_LIT>""" <EOL> ( n_ix , n_xi ) = n_ix_xi_tuple <EOL> n_oi = n_xi - n_ii <EOL> n_io = n_ix - n_ii <EOL> return ( n_ii , n_oi , n_io , n_xx - n_ii - n_oi - n_io ) <EOL> @ staticmethod <EOL> def _marginals ( n_ii , n_oi , n_io , n_oo ) : <EOL> """<STR_LIT>""" <EOL> return ( n_ii , ( n_oi + n_ii , n_io + n_ii ) , n_oo + n_oi + n_io + n_ii ) <EOL> @ staticmethod <EOL> def _expected_values ( cont ) : <EOL> """<STR_LIT>""" <EOL> n_xx = sum ( cont ) <EOL> for i in range ( <NUM_LIT:4> ) : <EOL> yield ( cont [ i ] + cont [ i ^ <NUM_LIT:1> ] ) * ( cont [ i ] + cont [ i ^ <NUM_LIT:2> ] ) / n_xx <EOL> @ classmethod <EOL> def phi_sq ( cls , * marginals ) : <EOL> """<STR_LIT>""" <EOL> n_ii , n_io , n_oi , n_oo = cls . _contingency ( * marginals ) <EOL> return ( ( n_ii * n_oo - n_io * n_oi ) ** <NUM_LIT:2> / <EOL> ( ( n_ii + n_io ) * ( n_ii + n_oi ) * ( n_io + n_oo ) * ( n_oi + n_oo ) ) ) <EOL> @ classmethod <EOL> def chi_sq ( cls , n_ii , n_ix_xi_tuple , n_xx ) : <EOL> """<STR_LIT>""" <EOL> ( n_ix , n_xi ) = n_ix_xi_tuple <EOL> return n_xx * cls . phi_sq ( n_ii , ( n_ix , n_xi ) , n_xx ) <EOL> @ classmethod <EOL> def fisher ( cls , * marginals ) : <EOL> """<STR_LIT>""" <EOL> n_ii , n_io , n_oi , n_oo = cls . _contingency ( * marginals ) <EOL> ( odds , pvalue ) = fisher_exact ( [ [ n_ii , n_io ] , [ n_oi , n_oo ] ] , alternative = '<STR_LIT>' ) <EOL> return pvalue <EOL> @ staticmethod <EOL> def dice ( n_ii , n_ix_xi_tuple , n_xx ) : <EOL> """<STR_LIT>""" <EOL> ( n_ix , n_xi ) = n_ix_xi_tuple <EOL> return <NUM_LIT:2> * n_ii / ( n_ix + n_xi ) <EOL> class TrigramAssocMeasures ( NgramAssocMeasures ) : <EOL> """<STR_LIT>""" <EOL> _n = <NUM_LIT:3> <EOL> @ staticmethod <EOL> def _contingency ( n_iii , n_iix_tuple , n_ixx_tuple , n_xxx ) : <EOL> """<STR_LIT>""" <EOL> ( n_iix , n_ixi , n_xii ) = n_iix_tuple <EOL> ( n_ixx , n_xix , n_xxi ) = n_ixx_tuple <EOL> n_oii = n_xii - n_iii <EOL> n_ioi = n_ixi - n_iii <EOL> n_iio = n_iix - n_iii <EOL> n_ooi = n_xxi - n_iii - n_oii - n_ioi <EOL> n_oio = n_xix - n_iii - n_oii - n_iio <EOL> n_ioo = n_ixx - n_iii - n_ioi - n_iio <EOL> n_ooo = n_xxx - n_iii - n_oii - n_ioi - n_iio - n_ooi - n_oio - n_ioo <EOL> return ( n_iii , n_oii , n_ioi , n_ooi , <EOL> n_iio , n_oio , n_ioo , n_ooo ) <EOL> @ staticmethod <EOL> def _marginals ( * contingency ) : <EOL> """<STR_LIT>""" <EOL> n_iii , n_oii , n_ioi , n_ooi , n_iio , n_oio , n_ioo , n_ooo = contingency <EOL> return ( n_iii , <EOL> ( n_iii + n_iio , n_iii + n_ioi , n_iii + n_oii ) , <EOL> ( n_iii + n_ioi + n_iio + n_ioo , <EOL> n_iii + n_oii + n_iio + n_oio , <EOL> n_iii + n_oii + n_ioi + n_ooi ) , <EOL> sum ( contingency ) ) <EOL> class QuadgramAssocMeasures ( NgramAssocMeasures ) : <EOL> """<STR_LIT>""" <EOL> _n = <NUM_LIT:4> <EOL> @ staticmethod <EOL> def _contingency ( n_iiii , n_iiix_tuple , n_iixx_tuple , n_ixxx_tuple , n_xxxx ) : <EOL> """<STR_LIT>""" <EOL> ( n_iiix , n_iixi , n_ixii , n_xiii ) = n_iiix_tuple <EOL> ( n_iixx , n_ixix , n_ixxi , n_xixi , n_xxii , n_xiix ) = n_iixx_tuple <EOL> ( n_ixxx , n_xixx , n_xxix , n_xxxi ) = n_ixxx_tuple <EOL> n_oiii = n_xiii - n_iiii <EOL> n_ioii = n_ixii - n_iiii <EOL> n_iioi = n_iixi - n_iiii <EOL> n_ooii = n_xxii - n_iiii - n_oiii - n_ioii <EOL> n_oioi = n_xixi - n_iiii - n_oiii - n_iioi <EOL> n_iooi = n_ixxi - n_iiii - n_ioii - n_iioi <EOL> n_oooi = n_xxxi - n_iiii - n_oiii - n_ioii - n_iioi - n_ooii - n_iooi - n_oioi <EOL> n_iiio = n_iiix - n_iiii <EOL> n_oiio = n_xiix - n_iiii - n_oiii - n_iiio <EOL> n_ioio = n_ixix - n_iiii - n_ioii - n_iiio <EOL> n_ooio = n_xxix - n_iiii - n_oiii - n_ioii - n_iiio - n_ooii - n_ioio - n_oiio <EOL> n_iioo = n_iixx - n_iiii - n_iioi - n_iiio <EOL> n_oioo = n_xixx - n_iiii - n_oiii - n_iioi - n_iiio - n_oioi - n_oiio - n_iioo <EOL> n_iooo = n_ixxx - n_iiii - n_ioii - n_iioi - n_iiio - n_iooi - n_iioo - n_ioio <EOL> n_oooo = n_xxxx - n_iiii - n_oiii - n_ioii - n_iioi - n_ooii - n_oioi - n_iooi - n_oooi - n_iiio - n_oiio - n_ioio - n_ooio - n_iioo - n_oioo - n_iooo <EOL> return ( n_iiii , n_oiii , n_ioii , n_ooii , n_iioi , <EOL> n_oioi , n_iooi , n_oooi , n_iiio , n_oiio , <EOL> n_ioio , n_ooio , n_iioo , n_oioo , n_iooo , n_oooo ) <EOL> @ staticmethod <EOL> def _marginals ( * contingency ) : <EOL> """<STR_LIT>""" <EOL> n_iiii , n_oiii , n_ioii , n_ooii , n_iioi , n_oioi , n_iooi , n_oooi , n_iiio , n_oiio , n_ioio , n_ooio , n_iioo , n_oioo , n_iooo , n_oooo = contingency <EOL> n_iiix = n_iiii + n_iiio <EOL> n_iixi = n_iiii + n_iioi <EOL> n_ixii = n_iiii + n_ioii <EOL> n_xiii = n_iiii + n_oiii <EOL> n_iixx = n_iiii + n_iioi + n_iiio + n_iioo <EOL> n_ixix = n_iiii + n_ioii + n_iiio + n_ioio <EOL> n_ixxi = n_iiii + n_ioii + n_iioi + n_iooi <EOL> n_xixi = n_iiii + n_oiii + n_iioi + n_oioi <EOL> n_xxii = n_iiii + n_oiii + n_ioii + n_ooii <EOL> n_xiix = n_iiii + n_oiii + n_iiio + n_oiio <EOL> n_ixxx = n_iiii + n_ioii + n_iioi + n_iiio + n_iooi + n_iioo + n_ioio + n_iooo <EOL> n_xixx = n_iiii + n_oiii + n_iioi + n_iiio + n_oioi + n_oiio + n_iioo + n_oioo <EOL> n_xxix = n_iiii + n_oiii + n_ioii + n_iiio + n_ooii + n_ioio + n_oiio + n_ooio <EOL> n_xxxi = n_iiii + n_oiii + n_ioii + n_iioi + n_ooii + n_iooi + n_oioi + n_oooi <EOL> n_all = sum ( contingency ) <EOL> return ( n_iiii , <EOL> ( n_iiix , n_iixi , n_ixii , n_xiii ) , <EOL> ( n_iixx , n_ixix , n_ixxi , n_xixi , n_xxii , n_xiix ) , <EOL> ( n_ixxx , n_xixx , n_xxix , n_xxxi ) , <EOL> n_all ) <EOL> class ContingencyMeasures ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , measures ) : <EOL> """<STR_LIT>""" <EOL> self . __class__ . __name__ = '<STR_LIT>' + measures . __class__ . __name__ <EOL> for k in dir ( measures ) : <EOL> if k . startswith ( '<STR_LIT>' ) : <EOL> continue <EOL> v = getattr ( measures , k ) <EOL> if not k . startswith ( '<STR_LIT:_>' ) : <EOL> v = self . _make_contingency_fn ( measures , v ) <EOL> setattr ( self , k , v ) <EOL> @ staticmethod <EOL> def _make_contingency_fn ( measures , old_fn ) : <EOL> """<STR_LIT>""" <EOL> def res ( * contingency ) : <EOL> return old_fn ( * measures . _marginals ( * contingency ) ) <EOL> res . __doc__ = old_fn . __doc__ <EOL> res . __name__ = old_fn . __name__ <EOL> return res </s>
