<s> """<STR_LIT>""" <EOL> import logging <EOL> import itertools <EOL> logger = logging . getLogger ( '<STR_LIT>' ) <EOL> import numpy <EOL> from scipy . special import gammaln , digamma , psi <EOL> from scipy . special import gamma as gammafunc <EOL> from scipy . special import polygamma <EOL> try : <EOL> from scipy . maxentropy import logsumexp <EOL> except ImportError : <EOL> from scipy . misc import logsumexp <EOL> from gensim import interfaces , utils <EOL> from six . moves import xrange <EOL> def dirichlet_expectation ( alpha ) : <EOL> """<STR_LIT>""" <EOL> if ( len ( alpha . shape ) == <NUM_LIT:1> ) : <EOL> result = psi ( alpha ) - psi ( numpy . sum ( alpha ) ) <EOL> else : <EOL> result = psi ( alpha ) - psi ( numpy . sum ( alpha , <NUM_LIT:1> ) ) [ : , numpy . newaxis ] <EOL> return result . astype ( alpha . dtype ) <EOL> class LdaState ( utils . SaveLoad ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , eta , shape ) : <EOL> self . eta = eta <EOL> self . sstats = numpy . zeros ( shape ) <EOL> self . numdocs = <NUM_LIT:0> <EOL> def reset ( self ) : <EOL> """<STR_LIT>""" <EOL> self . sstats [ : ] = <NUM_LIT:0.0> <EOL> self . numdocs = <NUM_LIT:0> <EOL> def merge ( self , other ) : <EOL> """<STR_LIT>""" <EOL> assert other is not None <EOL> self . sstats += other . sstats <EOL> self . numdocs += other . numdocs <EOL> def blend ( self , rhot , other , targetsize = None ) : <EOL> """<STR_LIT>""" <EOL> assert other is not None <EOL> if targetsize is None : <EOL> targetsize = self . numdocs <EOL> if self . numdocs == <NUM_LIT:0> or targetsize == self . numdocs : <EOL> scale = <NUM_LIT:1.0> <EOL> else : <EOL> scale = <NUM_LIT:1.0> * targetsize / self . numdocs <EOL> self . sstats *= ( <NUM_LIT:1.0> - rhot ) * scale <EOL> if other . numdocs == <NUM_LIT:0> or targetsize == other . numdocs : <EOL> scale = <NUM_LIT:1.0> <EOL> else : <EOL> logger . info ( "<STR_LIT>" % <EOL> ( other . numdocs , targetsize ) ) <EOL> scale = <NUM_LIT:1.0> * targetsize / other . numdocs <EOL> self . sstats += rhot * scale * other . sstats <EOL> self . numdocs = targetsize <EOL> def blend2 ( self , rhot , other , targetsize = None ) : <EOL> """<STR_LIT>""" <EOL> assert other is not None <EOL> if targetsize is None : <EOL> targetsize = self . numdocs <EOL> self . sstats += other . sstats <EOL> self . numdocs = targetsize <EOL> def get_lambda ( self ) : <EOL> return self . eta + self . sstats <EOL> def get_Elogbeta ( self ) : <EOL> return dirichlet_expectation ( self . get_lambda ( ) ) <EOL> class LdaModel ( interfaces . TransformationABC ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , corpus = None , num_topics = <NUM_LIT:100> , id2word = None , distributed = False , <EOL> chunksize = <NUM_LIT> , passes = <NUM_LIT:1> , update_every = <NUM_LIT:1> , alpha = '<STR_LIT>' , eta = None , decay = <NUM_LIT:0.5> , <EOL> eval_every = <NUM_LIT:10> , iterations = <NUM_LIT:50> , gamma_threshold = <NUM_LIT> ) : <EOL> """<STR_LIT>""" <EOL> self . id2word = id2word <EOL> if corpus is None and self . id2word is None : <EOL> raise ValueError ( '<STR_LIT>' ) <EOL> if self . id2word is None : <EOL> logger . warning ( "<STR_LIT>" ) <EOL> self . id2word = utils . dict_from_corpus ( corpus ) <EOL> self . num_terms = len ( self . id2word ) <EOL> elif len ( self . id2word ) > <NUM_LIT:0> : <EOL> self . num_terms = <NUM_LIT:1> + max ( self . id2word . keys ( ) ) <EOL> else : <EOL> self . num_terms = <NUM_LIT:0> <EOL> if self . num_terms == <NUM_LIT:0> : <EOL> raise ValueError ( "<STR_LIT>" ) <EOL> self . distributed = bool ( distributed ) <EOL> self . num_topics = int ( num_topics ) <EOL> self . chunksize = chunksize <EOL> self . decay = decay <EOL> self . num_updates = <NUM_LIT:0> <EOL> self . passes = passes <EOL> self . update_every = update_every <EOL> self . eval_every = eval_every <EOL> self . optimize_alpha = alpha == '<STR_LIT>' <EOL> if alpha == '<STR_LIT>' or alpha is None : <EOL> logger . info ( "<STR_LIT>" % ( <NUM_LIT:1.0> / num_topics ) ) <EOL> self . alpha = numpy . asarray ( [ <NUM_LIT:1.0> / num_topics for i in xrange ( num_topics ) ] ) <EOL> elif alpha == '<STR_LIT>' : <EOL> self . alpha = numpy . asarray ( [ <NUM_LIT:1.0> / ( i + numpy . sqrt ( num_topics ) ) for i in xrange ( num_topics ) ] ) <EOL> self . alpha /= self . alpha . sum ( ) <EOL> logger . info ( "<STR_LIT>" % list ( self . alpha ) ) <EOL> elif alpha == '<STR_LIT>' : <EOL> self . alpha = numpy . asarray ( [ <NUM_LIT:1.0> / num_topics for i in xrange ( num_topics ) ] ) <EOL> logger . info ( "<STR_LIT>" % list ( self . alpha ) ) <EOL> else : <EOL> self . alpha = alpha if isinstance ( alpha , numpy . ndarray ) else numpy . asarray ( [ alpha ] * num_topics ) <EOL> if len ( self . alpha ) != num_topics : <EOL> raise RuntimeError ( "<STR_LIT>" ) <EOL> if eta is None : <EOL> self . eta = <NUM_LIT:1.0> / num_topics <EOL> else : <EOL> self . eta = eta <EOL> self . iterations = iterations <EOL> self . gamma_threshold = gamma_threshold <EOL> if not distributed : <EOL> logger . info ( "<STR_LIT>" ) <EOL> self . dispatcher = None <EOL> self . numworkers = <NUM_LIT:1> <EOL> else : <EOL> if self . optimize_alpha : <EOL> raise NotImplementedError ( "<STR_LIT>" ) <EOL> try : <EOL> import Pyro4 <EOL> dispatcher = Pyro4 . Proxy ( '<STR_LIT>' ) <EOL> dispatcher . _pyroOneway . add ( "<STR_LIT>" ) <EOL> logger . debug ( "<STR_LIT>" % str ( dispatcher . _pyroUri ) ) <EOL> dispatcher . initialize ( id2word = self . id2word , num_topics = num_topics , <EOL> chunksize = chunksize , alpha = alpha , eta = eta , distributed = False ) <EOL> self . dispatcher = dispatcher <EOL> self . numworkers = len ( dispatcher . getworkers ( ) ) <EOL> logger . info ( "<STR_LIT>" % self . numworkers ) <EOL> except Exception as err : <EOL> logger . error ( "<STR_LIT>" % err ) <EOL> raise RuntimeError ( "<STR_LIT>" % err ) <EOL> self . state = LdaState ( self . eta , ( self . num_topics , self . num_terms ) ) <EOL> self . state . sstats = numpy . random . gamma ( <NUM_LIT> , <NUM_LIT:1.> / <NUM_LIT> , ( self . num_topics , self . num_terms ) ) <EOL> self . sync_state ( ) <EOL> if corpus is not None : <EOL> self . update ( corpus ) <EOL> def __str__ ( self ) : <EOL> return "<STR_LIT>" % ( self . num_terms , self . num_topics , self . decay , self . chunksize , self . alpha ) <EOL> def sync_state ( self ) : <EOL> self . expElogbeta = numpy . exp ( self . state . get_Elogbeta ( ) ) <EOL> def clear ( self ) : <EOL> """<STR_LIT>""" <EOL> self . state = None <EOL> self . Elogbeta = None <EOL> def inference ( self , chunk , collect_sstats = False ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> _ = len ( chunk ) <EOL> except : <EOL> chunk = list ( chunk ) <EOL> if len ( chunk ) > <NUM_LIT:1> : <EOL> logger . debug ( "<STR_LIT>" % len ( chunk ) ) <EOL> gamma = numpy . random . gamma ( <NUM_LIT> , <NUM_LIT:1.> / <NUM_LIT> , ( len ( chunk ) , self . num_topics ) ) <EOL> Elogtheta = dirichlet_expectation ( gamma ) <EOL> expElogtheta = numpy . exp ( Elogtheta ) <EOL> if collect_sstats : <EOL> sstats = numpy . zeros_like ( self . expElogbeta ) <EOL> else : <EOL> sstats = None <EOL> converged = <NUM_LIT:0> <EOL> for d , doc in enumerate ( chunk ) : <EOL> ids = [ id for id , _ in doc ] <EOL> cts = numpy . array ( [ cnt for _ , cnt in doc ] ) <EOL> gammad = gamma [ d , : ] <EOL> Elogthetad = Elogtheta [ d , : ] <EOL> expElogthetad = expElogtheta [ d , : ] <EOL> expElogbetad = self . expElogbeta [ : , ids ] <EOL> phinorm = numpy . dot ( expElogthetad , expElogbetad ) + <NUM_LIT> <EOL> for _ in xrange ( self . iterations ) : <EOL> lastgamma = gammad <EOL> gammad = self . alpha + expElogthetad * numpy . dot ( cts / phinorm , expElogbetad . T ) <EOL> Elogthetad = dirichlet_expectation ( gammad ) <EOL> expElogthetad = numpy . exp ( Elogthetad ) <EOL> phinorm = numpy . dot ( expElogthetad , expElogbetad ) + <NUM_LIT> <EOL> meanchange = numpy . mean ( abs ( gammad - lastgamma ) ) <EOL> if ( meanchange < self . gamma_threshold ) : <EOL> converged += <NUM_LIT:1> <EOL> break <EOL> gamma [ d , : ] = gammad <EOL> if collect_sstats : <EOL> sstats [ : , ids ] += numpy . outer ( expElogthetad . T , cts / phinorm ) <EOL> if len ( chunk ) > <NUM_LIT:1> : <EOL> logger . debug ( "<STR_LIT>" % <EOL> ( converged , len ( chunk ) , self . iterations ) ) <EOL> if collect_sstats : <EOL> sstats *= self . expElogbeta <EOL> return gamma , sstats <EOL> def do_estep ( self , chunk , state = None ) : <EOL> """<STR_LIT>""" <EOL> if state is None : <EOL> state = self . state <EOL> gamma , sstats = self . inference ( chunk , collect_sstats = True ) <EOL> state . sstats += sstats <EOL> state . numdocs += gamma . shape [ <NUM_LIT:0> ] <EOL> return gamma <EOL> def update_alpha ( self , gammat , rho ) : <EOL> """<STR_LIT>""" <EOL> N = float ( len ( gammat ) ) <EOL> logphat = sum ( dirichlet_expectation ( gamma ) for gamma in gammat ) / N <EOL> dalpha = numpy . copy ( self . alpha ) <EOL> gradf = N * ( psi ( numpy . sum ( self . alpha ) ) - psi ( self . alpha ) + logphat ) <EOL> c = N * polygamma ( <NUM_LIT:1> , numpy . sum ( self . alpha ) ) <EOL> q = - N * polygamma ( <NUM_LIT:1> , self . alpha ) <EOL> b = numpy . sum ( gradf / q ) / ( <NUM_LIT:1> / c + numpy . sum ( <NUM_LIT:1> / q ) ) <EOL> dalpha = - ( gradf - b ) / q <EOL> if all ( rho ( ) * dalpha + self . alpha > <NUM_LIT:0> ) : <EOL> self . alpha += rho ( ) * dalpha <EOL> else : <EOL> logger . warning ( "<STR_LIT>" ) <EOL> logger . info ( "<STR_LIT>" % list ( self . alpha ) ) <EOL> return self . alpha <EOL> def log_perplexity ( self , chunk , total_docs = None ) : <EOL> if total_docs is None : <EOL> total_docs = len ( chunk ) <EOL> corpus_words = sum ( cnt for document in chunk for _ , cnt in document ) <EOL> subsample_ratio = <NUM_LIT:1.0> * total_docs / len ( chunk ) <EOL> perwordbound = self . bound ( chunk , subsample_ratio = subsample_ratio ) / ( subsample_ratio * corpus_words ) <EOL> logger . info ( "<STR_LIT>" % <EOL> ( perwordbound , numpy . exp2 ( - perwordbound ) , len ( chunk ) , corpus_words ) ) <EOL> return perwordbound <EOL> def update ( self , corpus , chunksize = None , decay = None , passes = None , update_every = None , eval_every = None , <EOL> iterations = None , gamma_threshold = None ) : <EOL> """<STR_LIT>""" <EOL> if chunksize is None : <EOL> chunksize = self . chunksize <EOL> if decay is None : <EOL> decay = self . decay <EOL> if passes is None : <EOL> passes = self . passes <EOL> if update_every is None : <EOL> update_every = self . update_every <EOL> if eval_every is None : <EOL> eval_every = self . eval_every <EOL> if iterations is None : <EOL> iterations = self . iterations <EOL> if gamma_threshold is None : <EOL> gamma_threshold = self . gamma_threshold <EOL> rho = lambda : pow ( <NUM_LIT:1.0> + self . num_updates , - decay ) <EOL> try : <EOL> lencorpus = len ( corpus ) <EOL> except : <EOL> logger . warning ( "<STR_LIT>" ) <EOL> lencorpus = sum ( <NUM_LIT:1> for _ in corpus ) <EOL> if lencorpus == <NUM_LIT:0> : <EOL> logger . warning ( "<STR_LIT>" ) <EOL> return <EOL> self . state . numdocs += lencorpus <EOL> if update_every : <EOL> updatetype = "<STR_LIT>" <EOL> updateafter = min ( lencorpus , update_every * self . numworkers * chunksize ) <EOL> else : <EOL> updatetype = "<STR_LIT>" <EOL> updateafter = lencorpus <EOL> evalafter = min ( lencorpus , ( eval_every or <NUM_LIT:0> ) * self . numworkers * chunksize ) <EOL> updates_per_pass = max ( <NUM_LIT:1> , lencorpus / updateafter ) <EOL> logger . info ( "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" % <EOL> ( updatetype , self . num_topics , passes , lencorpus , <EOL> updateafter , evalafter , iterations , <EOL> gamma_threshold ) ) <EOL> if updates_per_pass * passes < <NUM_LIT:10> : <EOL> logger . warning ( "<STR_LIT>" <EOL> "<STR_LIT>" ) <EOL> for pass_ in xrange ( passes ) : <EOL> if self . dispatcher : <EOL> logger . info ( '<STR_LIT>' % self . numworkers ) <EOL> self . dispatcher . reset ( self . state ) <EOL> else : <EOL> other = LdaState ( self . eta , self . state . sstats . shape ) <EOL> dirty = False <EOL> reallen = <NUM_LIT:0> <EOL> for chunk_no , chunk in enumerate ( utils . grouper ( corpus , chunksize , as_numpy = True ) ) : <EOL> reallen += len ( chunk ) <EOL> if eval_every and ( ( reallen == lencorpus ) or ( ( chunk_no + <NUM_LIT:1> ) % ( eval_every * self . numworkers ) == <NUM_LIT:0> ) ) : <EOL> self . log_perplexity ( chunk , total_docs = lencorpus ) <EOL> if self . dispatcher : <EOL> logger . info ( '<STR_LIT>' % <EOL> ( pass_ , chunk_no * chunksize + len ( chunk ) , lencorpus ) ) <EOL> self . dispatcher . putjob ( chunk ) <EOL> else : <EOL> logger . info ( '<STR_LIT>' % <EOL> ( pass_ , chunk_no * chunksize + len ( chunk ) , lencorpus ) ) <EOL> gammat = self . do_estep ( chunk , other ) <EOL> if self . optimize_alpha : <EOL> self . update_alpha ( gammat , rho ) <EOL> dirty = True <EOL> del chunk <EOL> if update_every and ( chunk_no + <NUM_LIT:1> ) % ( update_every * self . numworkers ) == <NUM_LIT:0> : <EOL> if self . dispatcher : <EOL> logger . info ( "<STR_LIT>" ) <EOL> other = self . dispatcher . getstate ( ) <EOL> self . do_mstep ( rho ( ) , other ) <EOL> del other <EOL> if self . dispatcher : <EOL> logger . info ( '<STR_LIT>' ) <EOL> self . dispatcher . reset ( self . state ) <EOL> else : <EOL> other = LdaState ( self . eta , self . state . sstats . shape ) <EOL> dirty = False <EOL> if reallen != lencorpus : <EOL> raise RuntimeError ( "<STR_LIT>" ) <EOL> if dirty : <EOL> if self . dispatcher : <EOL> logger . info ( "<STR_LIT>" ) <EOL> other = self . dispatcher . getstate ( ) <EOL> self . do_mstep ( rho ( ) , other ) <EOL> del other <EOL> dirty = False <EOL> def do_mstep ( self , rho , other ) : <EOL> """<STR_LIT>""" <EOL> logger . debug ( "<STR_LIT>" ) <EOL> diff = numpy . log ( self . expElogbeta ) <EOL> self . state . blend ( rho , other ) <EOL> del other <EOL> diff -= self . state . get_Elogbeta ( ) <EOL> self . sync_state ( ) <EOL> self . print_topics ( <NUM_LIT:15> ) <EOL> logger . info ( "<STR_LIT>" % ( numpy . mean ( numpy . abs ( diff ) ) , rho ) ) <EOL> self . num_updates += <NUM_LIT:1> <EOL> def bound ( self , corpus , gamma = None , subsample_ratio = <NUM_LIT:1.0> ) : <EOL> """<STR_LIT>""" <EOL> score = <NUM_LIT:0.0> <EOL> _lambda = self . state . get_lambda ( ) <EOL> Elogbeta = dirichlet_expectation ( _lambda ) <EOL> for d , doc in enumerate ( corpus ) : <EOL> if d % self . chunksize == <NUM_LIT:0> : <EOL> logger . debug ( "<STR_LIT>" % d ) <EOL> if gamma is None : <EOL> gammad , _ = self . inference ( [ doc ] ) <EOL> else : <EOL> gammad = gamma [ d ] <EOL> Elogthetad = dirichlet_expectation ( gammad ) <EOL> score += numpy . sum ( cnt * logsumexp ( Elogthetad + Elogbeta [ : , id ] ) for id , cnt in doc ) <EOL> score += numpy . sum ( ( self . alpha - gammad ) * Elogthetad ) <EOL> score += numpy . sum ( gammaln ( gammad ) - gammaln ( self . alpha ) ) <EOL> score += gammaln ( numpy . sum ( self . alpha ) ) - gammaln ( numpy . sum ( gammad ) ) <EOL> score *= subsample_ratio <EOL> score += numpy . sum ( ( self . eta - _lambda ) * Elogbeta ) <EOL> score += numpy . sum ( gammaln ( _lambda ) - gammaln ( self . eta ) ) <EOL> if numpy . ndim ( self . eta ) == <NUM_LIT:0> : <EOL> sum_eta = self . eta * self . num_terms <EOL> else : <EOL> sum_eta = numpy . sum ( self . eta , <NUM_LIT:1> ) <EOL> score += numpy . sum ( gammaln ( sum_eta ) - gammaln ( numpy . sum ( _lambda , <NUM_LIT:1> ) ) ) <EOL> return score <EOL> def print_topics ( self , num_topics = <NUM_LIT:10> , num_words = <NUM_LIT:10> ) : <EOL> return self . show_topics ( num_topics , num_words , log = True ) <EOL> def show_topics ( self , num_topics = <NUM_LIT:10> , num_words = <NUM_LIT:10> , log = False , formatted = True ) : <EOL> """<STR_LIT>""" <EOL> if num_topics < <NUM_LIT:0> or num_topics >= self . num_topics : <EOL> num_topics = self . num_topics <EOL> chosen_topics = range ( num_topics ) <EOL> else : <EOL> num_topics = min ( num_topics , self . num_topics ) <EOL> sort_alpha = self . alpha + <NUM_LIT> * numpy . random . rand ( len ( self . alpha ) ) <EOL> sorted_topics = list ( numpy . argsort ( sort_alpha ) ) <EOL> chosen_topics = sorted_topics [ : num_topics // <NUM_LIT:2> ] + sorted_topics [ - num_topics // <NUM_LIT:2> : ] <EOL> shown = [ ] <EOL> for i in chosen_topics : <EOL> if formatted : <EOL> topic = self . print_topic ( i , topn = num_words ) <EOL> else : <EOL> topic = self . show_topic ( i , topn = num_words ) <EOL> shown . append ( topic ) <EOL> if log : <EOL> logger . info ( "<STR_LIT>" % ( i , self . alpha [ i ] , topic ) ) <EOL> return shown <EOL> def show_topic ( self , topicid , topn = <NUM_LIT:10> ) : <EOL> topic = self . state . get_lambda ( ) [ topicid ] <EOL> topic = topic / topic . sum ( ) <EOL> bestn = numpy . argsort ( topic ) [ : : - <NUM_LIT:1> ] [ : topn ] <EOL> beststr = [ ( topic [ id ] , self . id2word [ id ] ) for id in bestn ] <EOL> return beststr <EOL> def print_topic ( self , topicid , topn = <NUM_LIT:10> ) : <EOL> return '<STR_LIT>' . join ( [ '<STR_LIT>' % v for v in self . show_topic ( topicid , topn ) ] ) <EOL> def __getitem__ ( self , bow , eps = <NUM_LIT> ) : <EOL> """<STR_LIT>""" <EOL> is_corpus , corpus = utils . is_corpus ( bow ) <EOL> if is_corpus : <EOL> return self . _apply ( corpus ) <EOL> gamma , _ = self . inference ( [ bow ] ) <EOL> topic_dist = gamma [ <NUM_LIT:0> ] / sum ( gamma [ <NUM_LIT:0> ] ) <EOL> return [ ( topicid , topicvalue ) for topicid , topicvalue in enumerate ( topic_dist ) <EOL> if topicvalue >= eps ] <EOL> def save ( self , fname , * args , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> if self . state is not None : <EOL> self . state . save ( fname + '<STR_LIT>' , * args , ** kwargs ) <EOL> super ( LdaModel , self ) . save ( fname , * args , ignore = [ '<STR_LIT:state>' , '<STR_LIT>' ] , ** kwargs ) <EOL> @ classmethod <EOL> def load ( cls , fname , * args , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> kwargs [ '<STR_LIT>' ] = kwargs . get ( '<STR_LIT>' , '<STR_LIT:r>' ) <EOL> result = super ( LdaModel , cls ) . load ( fname , * args , ** kwargs ) <EOL> try : <EOL> result . state = super ( LdaModel , cls ) . load ( fname + '<STR_LIT>' , * args , ** kwargs ) <EOL> except Exception as e : <EOL> logging . warning ( "<STR_LIT>" % ( fname + '<STR_LIT>' , e ) ) <EOL> return result </s>
