def calculate(self, doc_terms=[], actual_id='', doc_length=-1):
        '''
        Given a set of terms, doc_terms
        We find the doc in training data (calc_id),
        whose LM is more likely to produce those terms
        Then return the data structure calculated back
        doc_length is passed to pr_ngram() and pr_doc()
            it is up to them to use it or not.
            normally, it should be ignored if doc_length == -1
        calculated{
            prob: calculated probability Pr(calc_id/doc_terms)
            calc_id: Document ID in training data.
            actual_id: Just returned back as passed to us.
            seen_unseen_count: Counts for terms seen/unseen in training data
        }  
        ''' 
        calculated = {
            'prob': -1,
            'calc_id': '',
            'actual_id': actual_id,
            'seen_unseen_count': (0,0),
            'all_probs': []
        }
        terms = self.lr_padding(doc_terms)
        ngrams = self.to_ngrams(terms) 
        for doc_id in self.term_count_n:
            #print '\n', doc_id, ':'
            doc_pr = 0
            new_doc = True
            seen_count = 0
            unseen_count = 0
            for ngram in ngrams:
                (ngram_pr, ngram_seen) = self.pr_ngram(doc_id, ngram, 
                        new_doc=new_doc, log=True, logbase=2, doc_length=doc_length)
                doc_pr += ngram_pr
                new_doc = False
                if ngram_seen:
                    seen_count += 1
                else:
                    unseen_count += 1
            doc_pr += self.pr_doc(doc_id, doc_length=doc_length) 
            if self.verbose:            
                print doc_id, actual_id, doc_pr 
            calculated['all_probs'].append(doc_pr) 
            if calculated['prob'] == -1 or doc_pr < calculated['prob']:
                calculated['prob'] = doc_pr
                calculated['calc_id'] = doc_id
                calculated['seen_unseen_count'] = (seen_count, unseen_count)
        self.unseen_counts.per_doc(doc_id=calculated['actual_id'], 
            seen_unseen=calculated['seen_unseen_count'])
        self.unseen_counts.per_cic(calculated_id=calculated['calc_id'], 
            actual_id=calculated['actual_id'],
            seen_unseen=calculated['seen_unseen_count'])
        return calculated
def calculate ( self , doc_terms = [ ] , actual_id = '' , doc_length = - 1 ) : calculated = { 'prob' : - 1 , 'calc_id' : '' , 'actual_id' : actual_id , 'seen_unseen_count' : ( 0 , 0 ) , 'all_probs' : [ ] } terms = self . lr_padding ( doc_terms ) ngrams = self . to_ngrams ( terms ) for doc_id in self . term_count_n : doc_pr = 0 new_doc = True seen_count = 0 unseen_count = 0 for ngram in ngrams : ( ngram_pr , ngram_seen ) = self . pr_ngram ( doc_id , ngram , new_doc = new_doc , log = True , logbase = 2 , doc_length = doc_length ) doc_pr += ngram_pr new_doc = False if ngram_seen : seen_count += 1 else : unseen_count += 1 doc_pr += self . pr_doc ( doc_id , doc_length = doc_length ) if self . verbose : print doc_id , actual_id , doc_pr calculated [ 'all_probs' ] . append ( doc_pr ) if calculated [ 'prob' ] == - 1 or doc_pr < calculated [ 'prob' ] : calculated [ 'prob' ] = doc_pr calculated [ 'calc_id' ] = doc_id calculated [ 'seen_unseen_count' ] = ( seen_count , unseen_count ) self . unseen_counts . per_doc ( doc_id = calculated [ 'actual_id' ] , seen_unseen = calculated [ 'seen_unseen_count' ] ) self . unseen_counts . per_cic ( calculated_id = calculated [ 'calc_id' ] , actual_id = calculated [ 'actual_id' ] , seen_unseen = calculated [ 'seen_unseen_count' ] ) return calculated
Given a set of terms, doc_terms
        We find the doc in training data (calc_id),
        whose LM is more likely to produce those terms
        Then return the data structure calculated back
        doc_length is passed to pr_ngram() and pr_doc()
            it is up to them to use it or not.
            normally, it should be ignored if doc_length == -1
        calculated{
            prob: calculated probability Pr(calc_id/doc_terms)
            calc_id: Document ID in training data.
            actual_id: Just returned back as passed to us.
            seen_unseen_count: Counts for terms seen/unseen in training data
        }