<s> import math <EOL> import glob <EOL> import re <EOL> import os <EOL> import subprocess <EOL> from shutil import rmtree <EOL> import logging <EOL> from mrec import load_sparse_matrix , save_recommender <EOL> class ItemSimilarityRunner ( object ) : <EOL> def run ( self , view , model , input_format , trainfile , num_engines , simsdir , overwrite , max_sims , simsfile , modelfile ) : <EOL> logging . info ( '<STR_LIT>' ) <EOL> dataset = load_sparse_matrix ( input_format , trainfile ) <EOL> num_users , num_items = dataset . shape <EOL> del dataset <EOL> logging . info ( '<STR_LIT>' , num_users , num_items ) <EOL> logging . info ( '<STR_LIT>' . format ( simsdir ) ) <EOL> subprocess . check_call ( [ '<STR_LIT>' , '<STR_LIT>' , simsdir ] ) <EOL> done = [ ] <EOL> if not overwrite : <EOL> logging . info ( '<STR_LIT>' ) <EOL> done . extend ( self . find_done ( simsdir ) ) <EOL> if done : <EOL> logging . info ( '<STR_LIT>' . format ( len ( done ) ) ) <EOL> logging . info ( '<STR_LIT>' ) <EOL> tasks = self . create_tasks ( model , input_format , trainfile , simsdir , num_items , num_engines , max_sims , done ) <EOL> if num_engines > <NUM_LIT:0> : <EOL> logging . info ( '<STR_LIT>' <EOL> '<STR_LIT>' , len ( tasks ) ) <EOL> async_job = view . map_async ( process , tasks , retries = <NUM_LIT:2> ) <EOL> results = async_job . get ( ) <EOL> else : <EOL> logging . info ( '<STR_LIT>' ) <EOL> results = [ process ( task ) for task in tasks ] <EOL> logging . info ( '<STR_LIT>' ) <EOL> done = self . find_done ( simsdir ) <EOL> remaining = len ( tasks ) - len ( done ) <EOL> if remaining == <NUM_LIT:0> : <EOL> logging . info ( '<STR_LIT>' ) <EOL> logging . info ( '<STR_LIT>' . format ( len ( done ) ) ) <EOL> paths = [ os . path . join ( simsdir , '<STR_LIT>' . format ( start , end ) ) for start , end in done ] <EOL> cmd = [ '<STR_LIT>' ] + paths <EOL> subprocess . check_call ( cmd , stdout = open ( simsfile , '<STR_LIT:w>' ) ) <EOL> logging . info ( '<STR_LIT>' ) <EOL> rmtree ( simsdir ) <EOL> logging . info ( '<STR_LIT>' , <EOL> num_items , type ( model ) . __name__ , simsfile ) <EOL> model . load_similarity_matrix ( simsfile , num_items ) <EOL> save_recommender ( model , modelfile ) <EOL> logging . info ( '<STR_LIT>' ) <EOL> else : <EOL> logging . error ( '<STR_LIT>' . format ( remaining , len ( tasks ) ) ) <EOL> logging . error ( '<STR_LIT>' ) <EOL> def find_done ( self , outdir ) : <EOL> success_files = glob . glob ( os . path . join ( outdir , '<STR_LIT>' ) ) <EOL> r = re . compile ( '<STR_LIT>' ) <EOL> done = [ ] <EOL> for path in success_files : <EOL> m = r . match ( path ) <EOL> start = int ( m . group ( <NUM_LIT:1> ) ) <EOL> end = int ( m . group ( <NUM_LIT:2> ) ) <EOL> done . append ( ( start , end ) ) <EOL> return done <EOL> def create_tasks ( self , model , input_format , trainfile , outdir , num_items , num_engines , max_similar_items , done ) : <EOL> if num_engines == <NUM_LIT:0> : <EOL> num_engines = <NUM_LIT:1> <EOL> items_per_engine = int ( math . ceil ( float ( num_items ) / num_engines ) ) <EOL> tasks = [ ] <EOL> for start in xrange ( <NUM_LIT:0> , num_items , items_per_engine ) : <EOL> end = min ( num_items , start + items_per_engine ) <EOL> if ( start , end ) not in done : <EOL> tasks . append ( ( model , input_format , trainfile , outdir , start , end , max_similar_items ) ) <EOL> return tasks <EOL> def process ( task ) : <EOL> """<STR_LIT>""" <EOL> import os <EOL> import subprocess <EOL> from mrec import load_fast_sparse_matrix <EOL> model , input_format , trainfile , outdir , start , end , max_similar_items = task <EOL> dataset = load_fast_sparse_matrix ( input_format , trainfile ) <EOL> if hasattr ( model , '<STR_LIT>' ) : <EOL> model . similarity_matrix = None <EOL> outfile = os . path . join ( outdir , '<STR_LIT>' . format ( start , end ) ) <EOL> out = open ( outfile , '<STR_LIT:w>' ) <EOL> for j in xrange ( start , end ) : <EOL> w = model . get_similar_items ( j , max_similar_items = max_similar_items , dataset = dataset ) <EOL> for k , v in w : <EOL> print >> out , '<STR_LIT>' . format ( j + <NUM_LIT:1> , k + <NUM_LIT:1> , v ) <EOL> out . close ( ) <EOL> cmd = [ '<STR_LIT>' , os . path . join ( outdir , '<STR_LIT>' . format ( start , end ) ) ] <EOL> subprocess . check_call ( cmd ) <EOL> return start , end </s>
