<s> """<STR_LIT>""" <EOL> import os , sys , re <EOL> import logging <EOL> import os . path as op <EOL> from collections import defaultdict <EOL> from cPickle import load <EOL> from time import sleep <EOL> from pbtools . pbtranscript . __init__ import get_version <EOL> from pbtools . pbtranscript . PBTranscriptOptions import add_polished_isoforms_arguments <EOL> from pbtools . pbtranscript . Utils import phred_to_qv , get_all_files_in_dir , ln <EOL> from pbtools . pbtranscript . ice . IceFiles import IceFiles <EOL> from pbcore . io import FastaWriter , FastqReader , FastqWriter <EOL> class IcePostQuiver ( IceFiles ) : <EOL> """<STR_LIT>""" <EOL> qv_trim_5 = <NUM_LIT:200> <EOL> qv_trim_3 = <NUM_LIT:50> <EOL> qv_max_err = <NUM_LIT:10> <EOL> def __init__ ( self , root_dir , hq_isoforms_fa = None , hq_isoforms_fq = None , <EOL> lq_isoforms_fa = None , lq_isoforms_fq = None , <EOL> use_sge = False , quit_if_not_done = True ) : <EOL> super ( IcePostQuiver , self ) . __init__ ( prog_name = "<STR_LIT>" , <EOL> root_dir = root_dir ) <EOL> self . hq_isoforms_fa = hq_isoforms_fa <EOL> self . hq_isoforms_fq = hq_isoforms_fq <EOL> self . lq_isoforms_fa = lq_isoforms_fa <EOL> self . lq_isoforms_fq = lq_isoforms_fq <EOL> self . use_sge = use_sge <EOL> self . quit_if_not_done = quit_if_not_done <EOL> self . fq_filenames = [ ] <EOL> self . _validate_inputs ( ) <EOL> def get_existing_binned_quivered_fq ( self ) : <EOL> """<STR_LIT>""" <EOL> pattern = r"<STR_LIT>" <EOL> fs = get_all_files_in_dir ( self . quivered_dir , <EOL> extension = "<STR_LIT>" ) <EOL> return [ f for f in fs if re . search ( pattern , f ) is not None ] <EOL> def _validate_inputs ( self ) : <EOL> """<STR_LIT>""" <EOL> errMsg = "<STR_LIT>" <EOL> if not op . exists ( self . nfl_all_pickle_fn ) : <EOL> errMsg = "<STR_LIT>" + "<STR_LIT>" . format ( f = self . nfl_all_pickle_fn ) <EOL> if errMsg != "<STR_LIT>" : <EOL> self . add_log ( errMsg , level = logging . ERROR ) <EOL> raise IOError ( errMsg ) <EOL> def check_quiver_jobs_completion ( self ) : <EOL> """<STR_LIT>""" <EOL> self . add_log ( "<STR_LIT>" ) <EOL> done_flag = True <EOL> bad_sh = [ ] <EOL> self . fq_filenames = [ ] <EOL> submitted = { } <EOL> self . add_log ( "<STR_LIT>" . <EOL> format ( f = self . submitted_quiver_jobs_log ) ) <EOL> with open ( self . submitted_quiver_jobs_log , '<STR_LIT:r>' ) as f : <EOL> for line in f : <EOL> a , b = line . strip ( ) . split ( '<STR_LIT:\t>' ) <EOL> if a == '<STR_LIT>' : <EOL> submitted [ b ] = b <EOL> else : <EOL> submitted [ a ] = b <EOL> stuff = os . popen ( "<STR_LIT>" ) . read ( ) . strip ( ) . split ( '<STR_LIT:\n>' ) <EOL> running_jids = [ ] <EOL> for x in stuff [ <NUM_LIT:2> : ] : <EOL> job_id = x . split ( ) [ <NUM_LIT:0> ] <EOL> running_jids . append ( job_id ) <EOL> if job_id in submitted : <EOL> self . add_log ( "<STR_LIT>" . format ( job_id ) ) <EOL> done_flag = False <EOL> for job_id , sh_name in submitted . iteritems ( ) : <EOL> fq_filename = op . join ( self . quivered_dir , <EOL> op . basename ( sh_name ) . replace ( '<STR_LIT>' , '<STR_LIT>' ) ) <EOL> if not op . exists ( fq_filename ) or os . stat ( fq_filename ) . st_size == <NUM_LIT:0> : <EOL> if job_id in running_jids : <EOL> done_flag = False <EOL> else : <EOL> self . add_log ( "<STR_LIT>" . <EOL> format ( job_id , fq_filename ) ) <EOL> bad_sh . append ( submitted [ job_id ] ) <EOL> else : <EOL> self . add_log ( "<STR_LIT>" . format ( job_id ) ) <EOL> self . fq_filenames . append ( fq_filename ) <EOL> if not done_flag : <EOL> if len ( bad_sh ) == <NUM_LIT:0> : <EOL> return "<STR_LIT>" <EOL> else : <EOL> self . add_log ( "<STR_LIT>" + <EOL> "<STR_LIT>" + <EOL> "<STR_LIT>" . format ( '<STR_LIT:\n>' . join ( bad_sh ) ) ) <EOL> return "<STR_LIT>" <EOL> else : <EOL> return "<STR_LIT>" <EOL> @ property <EOL> def quivered_good_fa ( self ) : <EOL> """<STR_LIT>""" <EOL> return op . join ( self . root_dir , <EOL> "<STR_LIT>" . format ( <EOL> a = self . qv_trim_5 , <EOL> b = self . qv_trim_3 , <EOL> c = self . qv_max_err ) ) <EOL> @ property <EOL> def quivered_good_fq ( self ) : <EOL> """<STR_LIT>""" <EOL> return op . join ( self . root_dir , <EOL> "<STR_LIT>" . format ( <EOL> a = self . qv_trim_5 , <EOL> b = self . qv_trim_3 , <EOL> c = self . qv_max_err ) ) <EOL> @ property <EOL> def quivered_bad_fa ( self ) : <EOL> """<STR_LIT>""" <EOL> return op . join ( self . root_dir , "<STR_LIT>" ) <EOL> @ property <EOL> def quivered_bad_fq ( self ) : <EOL> """<STR_LIT>""" <EOL> return op . join ( self . root_dir , "<STR_LIT>" ) <EOL> def pickup_best_clusters ( self , fq_filenames ) : <EOL> """<STR_LIT>""" <EOL> self . add_log ( "<STR_LIT>" . <EOL> format ( fs = "<STR_LIT:U+002CU+0020>" . join ( fq_filenames ) ) ) <EOL> a = load ( open ( self . final_pickle_fn ) ) <EOL> uc = a [ '<STR_LIT>' ] <EOL> quivered = { } <EOL> for fq in fq_filenames : <EOL> self . add_log ( "<STR_LIT>" . format ( f = fq ) ) <EOL> for r in FastqReader ( fq ) : <EOL> cid = r . name . split ( '<STR_LIT:|>' ) [ <NUM_LIT:0> ] <EOL> if cid . endswith ( '<STR_LIT>' ) : <EOL> cid = cid [ : - <NUM_LIT:4> ] <EOL> cid = int ( cid [ <NUM_LIT:1> : ] ) <EOL> quivered [ cid ] = r <EOL> good = [ ] <EOL> for cid , r in quivered . iteritems ( ) : <EOL> q = [ phred_to_qv ( x ) for x in r . quality ] <EOL> if sum ( q [ self . qv_trim_5 : - self . qv_trim_3 ] ) <= self . qv_max_err : <EOL> good . append ( cid ) <EOL> partial_uc = load ( open ( self . nfl_all_pickle_fn ) ) [ '<STR_LIT>' ] <EOL> partial_uc2 = defaultdict ( lambda : [ ] ) <EOL> partial_uc2 . update ( partial_uc ) <EOL> self . add_log ( "<STR_LIT>" . <EOL> format ( f = self . quivered_good_fa ) ) <EOL> self . add_log ( "<STR_LIT>" . <EOL> format ( f = self . quivered_bad_fa ) ) <EOL> with FastaWriter ( self . quivered_good_fa ) as good_fa_writer , FastaWriter ( self . quivered_bad_fa ) as bad_fa_writer , FastqWriter ( self . quivered_good_fq ) as good_fq_writer , FastqWriter ( self . quivered_bad_fq ) as bad_fq_writer : <EOL> for cid in quivered : <EOL> r = quivered [ cid ] <EOL> newname = "<STR_LIT>" . format ( cid = cid , <EOL> flnc_num = len ( uc [ cid ] ) , <EOL> nfl_num = len ( partial_uc2 [ cid ] ) , <EOL> read_len = len ( r . sequence ) ) <EOL> if cid in good : <EOL> self . add_log ( "<STR_LIT>" . <EOL> format ( c = cid ) ) <EOL> good_fa_writer . writeRecord ( newname , r . sequence ) <EOL> good_fq_writer . writeRecord ( newname , r . sequence , r . quality ) <EOL> else : <EOL> self . add_log ( "<STR_LIT>" . <EOL> format ( c = cid ) ) <EOL> bad_fa_writer . writeRecord ( newname , r . sequence ) <EOL> bad_fq_writer . writeRecord ( newname , r . sequence , r . quality ) <EOL> self . add_log ( "<STR_LIT:->" * <NUM_LIT> , level = logging . INFO ) <EOL> self . add_log ( "<STR_LIT>" + <EOL> "<STR_LIT>" . format ( self . quivered_good_fa , <EOL> self . quivered_good_fq ) ) <EOL> self . add_log ( "<STR_LIT>" + <EOL> "<STR_LIT>" . format ( self . quivered_bad_fa , <EOL> self . quivered_bad_fq ) ) <EOL> self . add_log ( "<STR_LIT:->" * <NUM_LIT> , level = logging . INFO ) <EOL> def run ( self ) : <EOL> """<STR_LIT>""" <EOL> job_stats = self . check_quiver_jobs_completion ( ) <EOL> self . add_log ( "<STR_LIT>" . format ( s = job_stats ) ) <EOL> if self . use_sge is not True and job_stats != "<STR_LIT>" : <EOL> self . add_log ( "<STR_LIT>" + <EOL> "<STR_LIT>" , <EOL> level = logging . ERROR ) <EOL> return - <NUM_LIT:1> <EOL> elif self . use_sge is True : <EOL> while job_stats != "<STR_LIT>" : <EOL> self . add_log ( "<STR_LIT>" ) <EOL> sleep ( <NUM_LIT> ) <EOL> job_stats = self . check_quiver_jobs_completion ( ) <EOL> if job_stats == "<STR_LIT>" : <EOL> break <EOL> elif job_stats == "<STR_LIT>" : <EOL> self . add_log ( "<STR_LIT>" , <EOL> level = logging . ERROR ) <EOL> return <NUM_LIT:1> <EOL> elif job_stats == "<STR_LIT>" : <EOL> self . add_log ( "<STR_LIT>" , <EOL> level = logging . INFO ) <EOL> if self . quit_if_not_done is True : <EOL> return <NUM_LIT:0> <EOL> else : <EOL> msg = "<STR_LIT>" . format ( job_stats ) <EOL> self . add_log ( msg , logging . ERROR ) <EOL> raise ValueError ( msg ) <EOL> self . pickup_best_clusters ( self . fq_filenames ) <EOL> self . add_log ( "<STR_LIT>" ) <EOL> if self . hq_isoforms_fa is not None : <EOL> ln ( self . quivered_good_fa , self . hq_isoforms_fa ) <EOL> if self . hq_isoforms_fq is not None : <EOL> ln ( self . quivered_good_fq , self . hq_isoforms_fq ) <EOL> self . add_log ( "<STR_LIT>" ) <EOL> if self . lq_isoforms_fa is not None : <EOL> ln ( self . quivered_bad_fa , self . lq_isoforms_fa ) <EOL> if self . lq_isoforms_fq is not None : <EOL> ln ( self . quivered_bad_fq , self . lq_isoforms_fq ) <EOL> self . close_log ( ) <EOL> def set_parser ( parser ) : <EOL> """<STR_LIT>""" <EOL> parser . add_argument ( "<STR_LIT>" , <EOL> type = str , <EOL> help = "<STR_LIT>" + "<STR_LIT>" ) <EOL> parser = add_polished_isoforms_arguments ( parser ) <EOL> parser . add_argument ( "<STR_LIT>" , <EOL> default = False , <EOL> dest = "<STR_LIT>" , <EOL> action = "<STR_LIT:store_true>" , <EOL> help = "<STR_LIT>" <EOL> "<STR_LIT>" ) <EOL> parser . add_argument ( "<STR_LIT>" , <EOL> default = False , <EOL> dest = "<STR_LIT>" , <EOL> action = "<STR_LIT:store_true>" , <EOL> help = "<STR_LIT>" ) <EOL> from pbcore . util . ToolRunner import PBToolRunner <EOL> class IcePostQuiverRunner ( PBToolRunner ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self ) : <EOL> desc = "<STR_LIT>" <EOL> PBToolRunner . __init__ ( self , desc ) <EOL> set_parser ( self . parser ) <EOL> def getVersion ( self ) : <EOL> """<STR_LIT>""" <EOL> return get_version ( ) <EOL> def run ( self ) : <EOL> """<STR_LIT>""" <EOL> logging . info ( "<STR_LIT>" . format ( f = op . basename ( __file__ ) , <EOL> v = self . getVersion ( ) ) ) <EOL> args = self . args <EOL> try : <EOL> obj = IcePostQuiver ( root_dir = args . root_dir , <EOL> hq_isoforms_fa = args . hq_isoforms_fa , <EOL> hq_isoforms_fq = args . hq_isoforms_fq , <EOL> lq_isoforms_fa = args . lq_isoforms_fa , <EOL> lq_isoforms_fq = args . lq_isoforms_fq , <EOL> use_sge = args . use_sge , <EOL> quit_if_not_done = args . quit_if_not_done ) <EOL> obj . run ( ) <EOL> except ValueError as e : <EOL> logging . error ( str ( e ) ) <EOL> return <NUM_LIT:1> <EOL> return <NUM_LIT:0> <EOL> def main ( ) : <EOL> """<STR_LIT>""" <EOL> runner = IcePostQuiverRunner ( ) <EOL> return runner . start ( ) <EOL> if __name__ == "<STR_LIT:__main__>" : <EOL> sys . exit ( main ( ) ) </s>
