<s> """<STR_LIT>""" <EOL> import operator <EOL> import os <EOL> import pprint <EOL> import toolz as tz <EOL> import yaml <EOL> def _get_api_client ( config = None ) : <EOL> if not config : config = { } <EOL> if "<STR_LIT>" in config and "<STR_LIT:host>" in config : <EOL> os . environ [ "<STR_LIT>" ] = config [ "<STR_LIT:host>" ] <EOL> os . environ [ "<STR_LIT>" ] = config [ "<STR_LIT>" ] <EOL> assert os . environ . get ( "<STR_LIT>" ) and os . environ . get ( "<STR_LIT>" ) , "<STR_LIT>" <EOL> import arvados <EOL> return arvados . api ( "<STR_LIT>" ) <EOL> def collection_files ( uuid , config = None , add_uuid = False ) : <EOL> """<STR_LIT>""" <EOL> import arvados <EOL> api_client = _get_api_client ( config ) <EOL> cr = arvados . CollectionReader ( uuid , api_client = api_client ) <EOL> cr . normalize ( ) <EOL> out = [ "<STR_LIT>" % ( x . stream_name ( ) , x . name ) for x in cr . all_files ( ) ] <EOL> if add_uuid : <EOL> out = [ "<STR_LIT>" % os . path . normpath ( os . path . join ( uuid , x ) ) for x in out ] <EOL> return out <EOL> def open_remote ( file_ref , config = None ) : <EOL> """<STR_LIT>""" <EOL> import arvados <EOL> api_client = _get_api_client ( config ) <EOL> coll_uuid , coll_ref = file_ref . replace ( "<STR_LIT>" , "<STR_LIT>" ) . split ( "<STR_LIT:/>" , <NUM_LIT:1> ) <EOL> cr = arvados . CollectionReader ( coll_uuid , api_client = api_client ) <EOL> return cr . open ( coll_ref ) <EOL> def file_size ( file_ref , config = None ) : <EOL> """<STR_LIT>""" <EOL> import arvados <EOL> api_client = _get_api_client ( config ) <EOL> coll_uuid , coll_ref = file_ref . replace ( "<STR_LIT>" , "<STR_LIT>" ) . split ( "<STR_LIT:/>" , <NUM_LIT:1> ) <EOL> cr = arvados . CollectionReader ( coll_uuid , api_client = api_client ) <EOL> file = cr [ coll_ref ] <EOL> return file . size ( ) / ( <NUM_LIT> * <NUM_LIT> ) <EOL> def _get_input_ids ( config ) : <EOL> """<STR_LIT>""" <EOL> input_id = config . get ( "<STR_LIT:input>" ) <EOL> if not input_id : <EOL> return [ ] <EOL> elif isinstance ( input_id , basestring ) : <EOL> return [ input_id ] <EOL> else : <EOL> assert isinstance ( input_id , ( list , tuple ) ) , input_id <EOL> return input_id <EOL> def get_files ( target_files , config ) : <EOL> """<STR_LIT>""" <EOL> out = [ ] <EOL> for input_id in _get_input_ids ( config ) : <EOL> for keep_file in collection_files ( input_id , config ) : <EOL> if os . path . basename ( keep_file ) in target_files : <EOL> out . append ( "<STR_LIT>" + os . path . normpath ( os . path . join ( input_id , keep_file ) ) ) <EOL> return out <EOL> def add_remotes ( items , config ) : <EOL> """<STR_LIT>""" <EOL> for k , v in items [ <NUM_LIT:0> ] . get ( "<STR_LIT>" , { } ) . iteritems ( ) : <EOL> config [ k ] = v <EOL> keep_files = reduce ( operator . add , [ collection_files ( x , config , add_uuid = True ) <EOL> for x in _get_input_ids ( config ) ] ) <EOL> if keep_files : <EOL> return _fill_remote ( items , keep_files ) <EOL> else : <EOL> return items <EOL> def _fill_remote ( cur , keep_files ) : <EOL> """<STR_LIT>""" <EOL> if isinstance ( cur , ( list , tuple ) ) : <EOL> return [ _fill_remote ( x , keep_files ) for x in cur ] <EOL> elif isinstance ( cur , dict ) : <EOL> out = { } <EOL> for k , v in cur . items ( ) : <EOL> out [ k ] = _fill_remote ( v , keep_files ) <EOL> return out <EOL> elif isinstance ( cur , basestring ) and os . path . splitext ( cur ) [ - <NUM_LIT:1> ] and not os . path . exists ( cur ) : <EOL> for test_keep in keep_files : <EOL> if test_keep . endswith ( cur ) : <EOL> return test_keep <EOL> return cur <EOL> else : <EOL> return cur <EOL> def get_refs ( genome_build , aligner , config ) : <EOL> """<STR_LIT>""" <EOL> ref_collection = tz . get_in ( [ "<STR_LIT>" , "<STR_LIT>" ] , config ) <EOL> if not ref_collection : <EOL> raise ValueError ( "<STR_LIT>" ) <EOL> cfiles = collection_files ( ref_collection , config [ "<STR_LIT>" ] ) <EOL> ref_prefix = None <EOL> for prefix in [ "<STR_LIT>" , "<STR_LIT>" ] : <EOL> cur_prefix = prefix % genome_build <EOL> if any ( x . startswith ( cur_prefix ) for x in cfiles ) : <EOL> ref_prefix = cur_prefix <EOL> break <EOL> assert ref_prefix , "<STR_LIT>" % ( genome_build , pprint . pformat ( cfiles ) ) <EOL> out = { } <EOL> base_targets = ( "<STR_LIT>" % genome_build , "<STR_LIT>" ) <EOL> for dirname in [ "<STR_LIT>" , "<STR_LIT>" , aligner ] : <EOL> key = { "<STR_LIT>" : "<STR_LIT>" } . get ( dirname , dirname ) <EOL> cur_files = [ x for x in cfiles if x . startswith ( "<STR_LIT>" % ( ref_prefix , dirname ) ) ] <EOL> cur_files = [ "<STR_LIT>" % os . path . normpath ( os . path . join ( ref_collection , x ) ) for x in cur_files ] <EOL> base_files = [ x for x in cur_files if x . endswith ( base_targets ) ] <EOL> if len ( base_files ) > <NUM_LIT:0> : <EOL> assert len ( base_files ) == <NUM_LIT:1> , base_files <EOL> base_file = base_files [ <NUM_LIT:0> ] <EOL> del cur_files [ cur_files . index ( base_file ) ] <EOL> out [ key ] = { "<STR_LIT>" : base_file , "<STR_LIT>" : cur_files } <EOL> else : <EOL> out [ key ] = { "<STR_LIT>" : cur_files } <EOL> return out <EOL> def get_resources ( genome_build , fasta_ref , data ) : <EOL> """<STR_LIT>""" <EOL> aconfig = tz . get_in ( [ "<STR_LIT>" , "<STR_LIT>" ] , data ) <EOL> resources_file = "<STR_LIT>" % ( os . path . splitext ( fasta_ref ) [ <NUM_LIT:0> ] ) <EOL> base_dir = os . path . dirname ( resources_file ) <EOL> with open_remote ( resources_file , aconfig ) as in_handle : <EOL> resources = yaml . safe_load ( in_handle ) <EOL> cfiles = [ os . path . normpath ( os . path . join ( "<STR_LIT>" % aconfig [ "<STR_LIT>" ] , x ) ) <EOL> for x in collection_files ( aconfig [ "<STR_LIT>" ] , aconfig ) ] <EOL> for k1 , v1 in resources . items ( ) : <EOL> if isinstance ( v1 , dict ) : <EOL> for k2 , v2 in v1 . items ( ) : <EOL> if isinstance ( v2 , basestring ) and v2 . startswith ( "<STR_LIT>" ) and os . path . splitext ( v2 ) [ - <NUM_LIT:1> ] : <EOL> test_v2 = os . path . normpath ( os . path . join ( base_dir , v2 ) ) <EOL> if test_v2 in cfiles : <EOL> resources [ k1 ] [ k2 ] = test_v2 <EOL> else : <EOL> del resources [ k1 ] [ k2 ] <EOL> data [ "<STR_LIT>" ] = resources <EOL> return _add_configured_indices ( base_dir , cfiles , data ) <EOL> def _add_configured_indices ( base_dir , cfiles , data ) : <EOL> """<STR_LIT>""" <EOL> snpeff_db = tz . get_in ( [ "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" ] , data ) <EOL> if snpeff_db : <EOL> index_dir = os . path . normpath ( os . path . join ( os . path . dirname ( base_dir ) , "<STR_LIT>" , snpeff_db ) ) <EOL> snpeff_files = [ x for x in cfiles if x . startswith ( index_dir ) ] <EOL> if len ( snpeff_files ) > <NUM_LIT:0> : <EOL> base_files = [ x for x in snpeff_files if x . endswith ( "<STR_LIT>" ) ] <EOL> assert len ( base_files ) == <NUM_LIT:1> , base_files <EOL> del snpeff_files [ snpeff_files . index ( base_files [ <NUM_LIT:0> ] ) ] <EOL> data [ "<STR_LIT>" ] [ "<STR_LIT>" ] = { "<STR_LIT>" : base_files [ <NUM_LIT:0> ] , "<STR_LIT>" : snpeff_files } <EOL> return data </s>
