<s> """<STR_LIT>""" <EOL> import os <EOL> import os . path as op <EOL> import time <EOL> import logging <EOL> from multiprocessing import Process <EOL> from pbcore . util . Process import backticks <EOL> from pbcore . io import FastaReader <EOL> from pbtools . pbtranscript . Utils import real_ppath , now_str , mkdir <EOL> from pbtools . pbtranscript . io . Summary import ClusterSummary <EOL> class IceFiles ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , prog_name , root_dir , <EOL> bas_fofn = None , ccs_fofn = None , fasta_fofn = None , <EOL> no_log_f = False ) : <EOL> """<STR_LIT>""" <EOL> self . prog_name = str ( prog_name ) <EOL> self . root_dir = real_ppath ( root_dir ) <EOL> self . bas_fofn = real_ppath ( bas_fofn ) <EOL> self . ccs_fofn = real_ppath ( ccs_fofn ) <EOL> self . fasta_fofn = real_ppath ( fasta_fofn ) <EOL> mkdir ( self . root_dir ) <EOL> mkdir ( self . tmp_dir ) <EOL> mkdir ( self . log_dir ) <EOL> mkdir ( self . script_dir ) <EOL> mkdir ( self . out_dir ) <EOL> self . no_log_f = no_log_f <EOL> if not no_log_f : <EOL> self . log_f = open ( self . log_fn , '<STR_LIT:w>' , <NUM_LIT:0> ) <EOL> self . add_log ( msg = "<STR_LIT>" . format ( p = self . prog_name ) ) <EOL> @ property <EOL> def tmp_dir ( self ) : <EOL> """<STR_LIT>""" <EOL> return op . join ( self . root_dir , "<STR_LIT>" ) <EOL> @ property <EOL> def log_dir ( self ) : <EOL> """<STR_LIT>""" <EOL> return op . join ( self . root_dir , "<STR_LIT>" ) <EOL> @ property <EOL> def log_fn ( self ) : <EOL> """<STR_LIT>""" <EOL> return op . join ( self . log_dir , self . prog_name + "<STR_LIT>" ) <EOL> @ property <EOL> def out_dir ( self ) : <EOL> """<STR_LIT>""" <EOL> return op . join ( self . root_dir , "<STR_LIT>" ) <EOL> @ property <EOL> def script_dir ( self ) : <EOL> """<STR_LIT>""" <EOL> return op . join ( self . root_dir , "<STR_LIT>" ) <EOL> @ property <EOL> def nfl_dir ( self ) : <EOL> """<STR_LIT>""" <EOL> return op . join ( self . out_dir , "<STR_LIT>" ) <EOL> @ property <EOL> def quivered_dir ( self ) : <EOL> """<STR_LIT>""" <EOL> return op . join ( self . root_dir , "<STR_LIT>" ) <EOL> @ property <EOL> def quivered_log_dir ( self ) : <EOL> """<STR_LIT>""" <EOL> return op . join ( self . log_dir , "<STR_LIT>" ) <EOL> @ property <EOL> def nfl_all_pickle_fn ( self ) : <EOL> """<STR_LIT>""" <EOL> return op . join ( self . nfl_dir , "<STR_LIT>" ) <EOL> @ property <EOL> def final_consensus_fa ( self ) : <EOL> """<STR_LIT>""" <EOL> return op . join ( self . out_dir , "<STR_LIT>" ) <EOL> @ property <EOL> def final_consensus_sa ( self ) : <EOL> """<STR_LIT>""" <EOL> return op . join ( self . out_dir , "<STR_LIT>" ) <EOL> @ property <EOL> def final_dazz_db ( self ) : <EOL> """<STR_LIT>""" <EOL> return op . join ( self . out_dir , "<STR_LIT>" ) <EOL> @ property <EOL> def final_pickle_fn ( self ) : <EOL> """<STR_LIT>""" <EOL> return op . join ( self . out_dir , "<STR_LIT>" ) <EOL> @ property <EOL> def submitted_quiver_jobs_log ( self ) : <EOL> """<STR_LIT>""" <EOL> return op . join ( self . log_dir , '<STR_LIT>' ) <EOL> def nfl_fa_i ( self , i ) : <EOL> """<STR_LIT>""" <EOL> fa_name = "<STR_LIT>" . format ( i ) <EOL> return op . join ( self . nfl_dir , fa_name ) <EOL> def nfl_pickle_i ( self , i ) : <EOL> """<STR_LIT>""" <EOL> return self . nfl_fa_i ( i ) + "<STR_LIT>" <EOL> def nfl_done_i ( self , i ) : <EOL> """<STR_LIT>""" <EOL> return self . nfl_pickle_i ( i ) + "<STR_LIT>" <EOL> def nfl_script_i ( self , i ) : <EOL> """<STR_LIT>""" <EOL> return op . join ( self . script_dir , <EOL> op . basename ( self . nfl_pickle_i ( i ) ) + "<STR_LIT>" ) <EOL> def cluster_dir ( self , cid ) : <EOL> """<STR_LIT>""" <EOL> return op . join ( self . tmp_dir , str ( int ( cid ) / <NUM_LIT> ) , '<STR_LIT:c>' + str ( cid ) ) <EOL> def raw_fa_of_cluster ( self , cid ) : <EOL> """<STR_LIT>""" <EOL> return op . join ( self . cluster_dir ( cid ) , "<STR_LIT>" ) <EOL> def g_consensus_fa_of_cluster ( self , cid ) : <EOL> """<STR_LIT>""" <EOL> return op . join ( self . cluster_dir ( cid ) , "<STR_LIT>" ) <EOL> def g_consensus_ref_fa_of_cluster ( self , cid ) : <EOL> """<STR_LIT>""" <EOL> return op . join ( self . cluster_dir ( cid ) , "<STR_LIT>" ) <EOL> def first_seq_fa_of_cluster ( self , cid ) : <EOL> """<STR_LIT>""" <EOL> return op . join ( self . cluster_dir ( cid ) , "<STR_LIT>" ) <EOL> def sam_of_cluster ( self , cid ) : <EOL> """<STR_LIT>""" <EOL> return op . join ( self . cluster_dir ( cid ) , "<STR_LIT>" ) <EOL> def add_log ( self , msg , level = logging . DEBUG ) : <EOL> """<STR_LIT>""" <EOL> msg = "<STR_LIT:[>" + str ( self . prog_name ) + "<STR_LIT>" + msg <EOL> level_name = "<STR_LIT>" <EOL> if level == logging . INFO : <EOL> level_name = "<STR_LIT>" <EOL> logging . info ( msg ) <EOL> elif level == logging . ERROR : <EOL> level_name = "<STR_LIT>" <EOL> logging . error ( msg ) <EOL> elif level == logging . WARNING : <EOL> level_name = "<STR_LIT>" <EOL> logging . warn ( msg ) <EOL> else : <EOL> level_name = "<STR_LIT>" <EOL> logging . debug ( msg ) <EOL> if not self . no_log_f : <EOL> self . log_f . write ( "<STR_LIT>" . format ( <EOL> ln = level_name , pn = self . prog_name , t = now_str ( ) , msg = msg ) ) <EOL> def close_log ( self ) : <EOL> """<STR_LIT>""" <EOL> if not self . no_log_f : <EOL> self . add_log ( self . prog_name + "<STR_LIT>" ) <EOL> self . log_f . close ( ) <EOL> def qsub_cmd_and_log ( self , cmd ) : <EOL> """<STR_LIT>""" <EOL> _out , _code , _msg = backticks ( cmd ) <EOL> if _code != <NUM_LIT:0> : <EOL> errMsg = "<STR_LIT>" . format ( cmd = cmd , msg = _msg ) <EOL> self . add_log ( errMsg , level = logging . ERROR ) <EOL> raise RuntimeError ( errMsg ) <EOL> return str ( _out ) . split ( ) [ <NUM_LIT:2> ] <EOL> def run_cmd_and_log ( self , cmd , olog = "<STR_LIT>" , elog = "<STR_LIT>" , description = "<STR_LIT>" ) : <EOL> """<STR_LIT>""" <EOL> _out , _code , _msg = backticks ( cmd ) <EOL> if _code != <NUM_LIT:0> : <EOL> errMsgs = [ "<STR_LIT>" . <EOL> format ( cmd = cmd , msg = _msg ) ] <EOL> if len ( description ) != <NUM_LIT:0> : <EOL> errMsgs . append ( "<STR_LIT>" . format ( description = description ) ) <EOL> if len ( elog ) != <NUM_LIT:0> : <EOL> errMsgs . append ( "<STR_LIT>" . format ( elog = elog ) ) <EOL> if len ( olog ) != <NUM_LIT:0> : <EOL> errMsgs . append ( "<STR_LIT>" . format ( olog = olog ) ) <EOL> errMsg = "<STR_LIT:\n>" . join ( errMsgs ) <EOL> self . add_log ( errMsg , level = logging . ERROR ) <EOL> raise RuntimeError ( errMsg ) <EOL> def write_report ( self , report_fn , uc , partial_uc = None ) : <EOL> """<STR_LIT>""" <EOL> self . add_log ( "<STR_LIT>" . <EOL> format ( nfl = "<STR_LIT>" if partial_uc is not None else "<STR_LIT>" , <EOL> f = report_fn ) , level = logging . INFO ) <EOL> with open ( report_fn , '<STR_LIT:w>' ) as f : <EOL> f . write ( "<STR_LIT>" ) <EOL> for c in uc . keys ( ) : <EOL> for r in uc [ c ] : <EOL> f . write ( "<STR_LIT>" . format ( r = r , c = c ) ) <EOL> if partial_uc is not None : <EOL> for r in partial_uc [ c ] : <EOL> f . write ( "<STR_LIT>" . format ( r = r , c = c ) ) <EOL> def write_summary ( self , summary_fn , isoforms_fa , hq_fa = None , lq_fa = None ) : <EOL> """<STR_LIT>""" <EOL> self . add_log ( "<STR_LIT>" . format ( f = summary_fn ) , <EOL> level = logging . INFO ) <EOL> try : <EOL> summary = ClusterSummary ( ) <EOL> with FastaReader ( isoforms_fa ) as reader : <EOL> for r in reader : <EOL> summary . numConsensusIsoforms += <NUM_LIT:1> <EOL> summary . numTotalBases += len ( r . sequence ) <EOL> if hq_fa is not None : <EOL> summary . num_polished_hq_isoforms = <NUM_LIT:0> <EOL> with FastaReader ( hq_fa ) as reader : <EOL> for r in reader : <EOL> summary . num_polished_hq_isoforms += <NUM_LIT:1> <EOL> if lq_fa is not None : <EOL> summary . num_polished_lq_isoforms = <NUM_LIT:0> <EOL> with FastaReader ( lq_fa ) as reader : <EOL> for r in reader : <EOL> summary . num_polished_lq_isoforms += <NUM_LIT:1> <EOL> summary . write ( summary_fn ) <EOL> except ZeroDivisionError : <EOL> errMsg = "<STR_LIT>" <EOL> self . add_log ( errMsg , level = logging . ERROR ) <EOL> raise RuntimeError ( errMsg ) <EOL> def wait_for_sge_jobs_worker ( cmd ) : <EOL> _out , _code , _msg = backticks ( cmd ) <EOL> if _code != <NUM_LIT:0> : <EOL> errMsg = "<STR_LIT>" . format ( cmd = cmd , msg = _msg ) <EOL> raise RuntimeError ( errMsg ) <EOL> return str ( _out ) . split ( ) [ <NUM_LIT:2> ] <EOL> def wait_for_sge_jobs ( cmd , jids , timeout ) : <EOL> """<STR_LIT>""" <EOL> def get_active_jids ( ) : <EOL> stuff = os . popen ( "<STR_LIT>" ) . read ( ) . strip ( ) . split ( '<STR_LIT:\n>' ) <EOL> for x in stuff [ <NUM_LIT:2> : ] : <EOL> job_id = x . split ( ) [ <NUM_LIT:0> ] <EOL> yield job_id <EOL> p = Process ( target = wait_for_sge_jobs_worker , args = ( cmd , ) ) <EOL> p . start ( ) <EOL> p . join ( timeout ) <EOL> if p . is_alive ( ) : <EOL> active_jids = [ x for x in get_active_jids ( ) ] <EOL> while len ( active_jids ) > <NUM_LIT:0> : <EOL> for jid in active_jids : <EOL> kill_cmd = "<STR_LIT>" + str ( jid ) <EOL> backticks ( kill_cmd ) <EOL> time . sleep ( <NUM_LIT:3> ) <EOL> active_jids = [ x for x in get_active_jids ( ) ] <EOL> return "<STR_LIT>" <EOL> return "<STR_LIT>" </s>
