<s> from __future__ import unicode_literals <EOL> import math <EOL> class BM25 ( object ) : <EOL> def __init__ ( self , docs ) : <EOL> self . D = len ( docs ) <EOL> self . avgdl = sum ( [ len ( doc ) + <NUM_LIT:0.0> for doc in docs ] ) / self . D <EOL> self . docs = docs <EOL> self . f = [ ] <EOL> self . df = { } <EOL> self . idf = { } <EOL> self . k1 = <NUM_LIT> <EOL> self . b = <NUM_LIT> <EOL> self . init ( ) <EOL> def init ( self ) : <EOL> for doc in self . docs : <EOL> tmp = { } <EOL> for word in doc : <EOL> if not word in tmp : <EOL> tmp [ word ] = <NUM_LIT:0> <EOL> tmp [ word ] += <NUM_LIT:1> <EOL> self . f . append ( tmp ) <EOL> for k , v in tmp . items ( ) : <EOL> if k not in self . df : <EOL> self . df [ k ] = <NUM_LIT:0> <EOL> self . df [ k ] += <NUM_LIT:1> <EOL> for k , v in self . df . items ( ) : <EOL> self . idf [ k ] = math . log ( self . D - v + <NUM_LIT:0.5> ) - math . log ( v + <NUM_LIT:0.5> ) <EOL> def sim ( self , doc , index ) : <EOL> score = <NUM_LIT:0> <EOL> for word in doc : <EOL> if word not in self . f [ index ] : <EOL> continue <EOL> d = len ( self . docs [ index ] ) <EOL> score += ( self . idf [ word ] * self . f [ index ] [ word ] * ( self . k1 + <NUM_LIT:1> ) <EOL> / ( self . f [ index ] [ word ] + self . k1 * ( <NUM_LIT:1> - self . b + self . b * d <EOL> / self . avgdl ) ) ) <EOL> return score <EOL> def simall ( self , doc ) : <EOL> scores = [ ] <EOL> for index in range ( self . D ) : <EOL> score = self . sim ( doc , index ) <EOL> scores . append ( score ) <EOL> return scores </s>
