<s> import os <EOL> import contextlib <EOL> import sys <EOL> import sqlalchemy as sql <EOL> from sqlalchemy . orm import mapper , create_session <EOL> import sqlalchemy <EOL> from ped import get_ped_fields <EOL> def index_variation ( cursor ) : <EOL> cursor . execute ( '''<STR_LIT>''' ) <EOL> cursor . execute ( '''<STR_LIT>''' ) <EOL> cursor . execute ( '''<STR_LIT>''' ) <EOL> cursor . execute ( '''<STR_LIT>''' ) <EOL> cursor . execute ( '''<STR_LIT>''' ) <EOL> cursor . execute ( '''<STR_LIT>''' ) <EOL> cursor . execute ( '''<STR_LIT>''' ) <EOL> cursor . execute ( '''<STR_LIT>''' ) <EOL> cursor . execute ( '''<STR_LIT>''' ) <EOL> cursor . execute ( '''<STR_LIT>''' ) <EOL> cursor . execute ( '''<STR_LIT>''' ) <EOL> cursor . execute ( '''<STR_LIT>''' ) <EOL> cursor . execute ( '''<STR_LIT>''' ) <EOL> cursor . execute ( '''<STR_LIT>''' ) <EOL> cursor . execute ( '''<STR_LIT>''' ) <EOL> cursor . execute ( '''<STR_LIT>''' ) <EOL> cursor . execute ( '''<STR_LIT>''' ) <EOL> cursor . execute ( '''<STR_LIT>''' ) <EOL> cursor . execute ( '''<STR_LIT>''' ) <EOL> cursor . execute ( '''<STR_LIT>''' ) <EOL> cursor . execute ( '''<STR_LIT>''' ) <EOL> cursor . execute ( '''<STR_LIT>''' ) <EOL> cursor . execute ( '''<STR_LIT>''' ) <EOL> cursor . execute ( '''<STR_LIT>''' ) <EOL> cursor . execute ( '''<STR_LIT>''' ) <EOL> cursor . execute ( '''<STR_LIT>''' ) <EOL> cursor . execute ( '''<STR_LIT>''' ) <EOL> cursor . execute ( '<STR_LIT>' ) <EOL> def index_variation_impacts ( cursor ) : <EOL> cursor . execute ( '''<STR_LIT>''' ) <EOL> cursor . execute ( '''<STR_LIT>''' ) <EOL> cursor . execute ( <EOL> '''<STR_LIT>''' ) <EOL> cursor . execute ( '''<STR_LIT>''' ) <EOL> cursor . execute ( '''<STR_LIT>''' ) <EOL> cursor . execute ( '''<STR_LIT>''' ) <EOL> def index_samples ( cursor ) : <EOL> cursor . execute ( '''<STR_LIT>''' ) <EOL> def index_gene_detailed ( cursor ) : <EOL> cursor . execute ( '''<STR_LIT>''' ) <EOL> cursor . execute ( '''<STR_LIT>''' ) <EOL> cursor . execute ( '''<STR_LIT>''' ) <EOL> cursor . execute ( '''<STR_LIT>''' ) <EOL> def index_gene_summary ( cursor ) : <EOL> cursor . execute ( '''<STR_LIT>''' ) <EOL> cursor . execute ( '''<STR_LIT>''' ) <EOL> def create_indices ( cursor ) : <EOL> """<STR_LIT>""" <EOL> index_variation ( cursor ) <EOL> index_variation_impacts ( cursor ) <EOL> index_samples ( cursor ) <EOL> index_gene_detailed ( cursor ) <EOL> index_gene_summary ( cursor ) <EOL> def get_path ( path ) : <EOL> if "<STR_LIT>" not in path : <EOL> if not ( os . path . exists ( path ) or os . access ( os . path . dirname ( os . path . abspath ( path ) ) , os . W_OK ) ) : <EOL> raise Exception ( "<STR_LIT>" % path ) <EOL> path = "<STR_LIT>" % path <EOL> return path <EOL> def create_tables ( path , effect_fields = None ) : <EOL> """<STR_LIT>""" <EOL> if effect_fields : <EOL> effect_string = "<STR_LIT>" . join ( e + "<STR_LIT>" for e in effect_fields ) <EOL> else : <EOL> effect_string = "<STR_LIT>" <EOL> db = dict ( variants = """<STR_LIT>""" % effect_string . rstrip ( "<STR_LIT:U+002C>" ) , <EOL> variant_impacts = """<STR_LIT>""" , <EOL> sample_genotypes = """<STR_LIT>""" , <EOL> sample_genotype_counts = """<STR_LIT>""" , <EOL> resources = """<STR_LIT>""" , <EOL> version = """<STR_LIT>""" , <EOL> gene_detailed = """<STR_LIT>""" , <EOL> gene_summary = """<STR_LIT>""" , <EOL> vcf_header = """<STR_LIT>""" ) <EOL> lookup = { '<STR_LIT>' : sql . Float ( ) , <EOL> '<STR_LIT:float>' : sql . Float ( ) , <EOL> '<STR_LIT:text>' : sql . Text ( ) , <EOL> '<STR_LIT:bool>' : sql . Boolean ( ) , <EOL> '<STR_LIT>' : sql . LargeBinary ( ) , <EOL> '<STR_LIT>' : sql . Float ( ) , <EOL> '<STR_LIT>' : sql . Integer ( ) , <EOL> '<STR_LIT>' : sql . String ( <NUM_LIT:20> ) , <EOL> '<STR_LIT>' : sql . String ( <NUM_LIT> ) , <EOL> '<STR_LIT:int>' : sql . Integer ( ) , <EOL> } <EOL> for table in db : <EOL> db [ table ] = db [ table ] . strip ( ) . strip ( "<STR_LIT:U+002C>" ) . split ( "<STR_LIT>" ) <EOL> db [ table ] = [ x . strip ( ) . split ( ) for x in db [ table ] ] <EOL> cols = [ sql . Column ( c [ <NUM_LIT:0> ] , lookup [ c [ <NUM_LIT:1> ] . lower ( ) ] ) for c in db [ table ] ] <EOL> if table != "<STR_LIT>" : <EOL> for c in cols : <EOL> if c . name in ( "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" ) : <EOL> c . primary_key = True <EOL> if c . name == "<STR_LIT>" and table == "<STR_LIT>" : <EOL> c . autoincrement = False <EOL> db [ table ] = cols <EOL> e = sql . create_engine ( get_path ( path ) , isolation_level = None ) <EOL> e . connect ( ) . connection . connection . text_factory = str <EOL> metadata = sql . MetaData ( bind = e ) <EOL> session = create_session ( bind = e , autocommit = False , autoflush = False ) <EOL> mapped = { } <EOL> tables = [ '<STR_LIT>' ] + [ x for x in sorted ( db ) if x != '<STR_LIT>' ] <EOL> otables = [ sql . Table ( tbl , metadata , * db [ tbl ] ) for tbl in tables ] <EOL> metadata . drop_all ( tables = otables ) <EOL> for t in otables : <EOL> mapped [ t . name ] = t <EOL> session . commit ( ) <EOL> metadata . create_all ( ) <EOL> return session , metadata <EOL> def create_sample_table ( cursor , metadata , args ) : <EOL> NUM_BUILT_IN = <NUM_LIT:6> <EOL> cols = [ sql . Column ( "<STR_LIT>" , sql . Integer , primary_key = True ) ] <EOL> fields = get_ped_fields ( args . ped_file ) <EOL> optional_fields = [ "<STR_LIT>" , "<STR_LIT:name>" , "<STR_LIT>" , "<STR_LIT>" , <EOL> "<STR_LIT>" , "<STR_LIT>" ] <EOL> optional_fields += fields [ NUM_BUILT_IN : ] <EOL> for field in optional_fields : <EOL> if field == "<STR_LIT:name>" : <EOL> cols . append ( sql . Column ( field , sql . String ( <NUM_LIT:50> ) ) ) <EOL> else : <EOL> cols . append ( sql . Column ( field , sql . TEXT ) ) <EOL> t = sql . Table ( "<STR_LIT>" , metadata , * cols ) <EOL> t . drop ( checkfirst = True ) <EOL> metadata . create_all ( tables = [ t ] ) <EOL> def insert_variation ( session , metadata , buffer ) : <EOL> """<STR_LIT>""" <EOL> if len ( buffer ) == <NUM_LIT:0> : return <EOL> tbl = metadata . tables [ '<STR_LIT>' ] <EOL> cols = _get_cols ( tbl ) <EOL> left = set ( cols ) - set ( buffer [ <NUM_LIT:0> ] . keys ( ) ) <EOL> assert len ( left ) == <NUM_LIT:0> , left <EOL> left = set ( buffer [ <NUM_LIT:0> ] . keys ( ) ) - set ( cols ) <EOL> assert len ( left ) == <NUM_LIT:0> , left <EOL> try : <EOL> session . execute ( tbl . insert ( ) , buffer ) <EOL> session . commit ( ) <EOL> except : <EOL> sys . stderr . write ( "<STR_LIT>" ) <EOL> session . rollback ( ) <EOL> stmt = tbl . insert ( ) <EOL> with session . bind . begin ( ) as trans : <EOL> for b in buffer : <EOL> trans . execute ( stmt , b ) <EOL> raise <EOL> def insert_variation_impacts ( session , metadata , buffer ) : <EOL> """<STR_LIT>""" <EOL> if len ( buffer ) == <NUM_LIT:0> : return <EOL> tbl = metadata . tables [ '<STR_LIT>' ] <EOL> session . execute ( tbl . insert ( ) , buffer ) <EOL> session . commit ( ) <EOL> def insert_sample ( session , metadata , sample_list ) : <EOL> """<STR_LIT>""" <EOL> tbl = metadata . tables [ '<STR_LIT>' ] <EOL> cols = [ c . key for c in tbl . c ] <EOL> session . execute ( tbl . insert ( ) , dict ( zip ( cols , sample_list ) ) ) <EOL> def _get_cols ( tbl ) : <EOL> return [ c . key for c in tbl . c ] <EOL> def gen_gene_vals ( cols , table_contents ) : <EOL> for row in table_contents : <EOL> d = dict ( zip ( cols , row ) ) <EOL> d [ '<STR_LIT>' ] = bool ( d [ '<STR_LIT>' ] != '<STR_LIT:0>' ) <EOL> if d [ '<STR_LIT>' ] in ( None , '<STR_LIT:None>' ) : <EOL> d [ '<STR_LIT>' ] = None <EOL> else : <EOL> d [ '<STR_LIT>' ] = float ( d [ '<STR_LIT>' ] ) <EOL> if "<STR_LIT>" in d : <EOL> d [ '<STR_LIT>' ] = bool ( d [ '<STR_LIT>' ] ) <EOL> yield d <EOL> def insert_gene_detailed ( session , metadata , table_contents ) : <EOL> t = metadata . tables [ '<STR_LIT>' ] <EOL> cols = _get_cols ( t ) <EOL> n = <NUM_LIT:1000> <EOL> for chunk in ( table_contents [ i : i + n ] for i in xrange ( <NUM_LIT:0> , len ( table_contents ) , n ) ) : <EOL> session . execute ( t . insert ( ) , list ( gen_gene_vals ( cols , chunk ) ) ) <EOL> def insert_gene_summary ( session , metadata , contents ) : <EOL> t = metadata . tables [ '<STR_LIT>' ] <EOL> cols = _get_cols ( t ) <EOL> session . execute ( t . insert ( ) , list ( gen_gene_vals ( cols , contents ) ) ) <EOL> session . commit ( ) <EOL> def insert_resources ( session , metadata , resources ) : <EOL> """<STR_LIT>""" <EOL> t = metadata . tables [ '<STR_LIT>' ] <EOL> cols = _get_cols ( t ) <EOL> session . execute ( t . insert ( ) , [ dict ( zip ( cols , r ) ) for r in resources ] ) <EOL> session . commit ( ) <EOL> def insert_vcf_header ( session , metadata , vcf_header ) : <EOL> """<STR_LIT>""" <EOL> t = metadata . tables [ '<STR_LIT>' ] <EOL> session . execute ( t . insert ( ) , dict ( vcf_header = vcf_header . rstrip ( "<STR_LIT:\r\n>" ) ) ) <EOL> session . commit ( ) <EOL> def insert_version ( session , metadata , version ) : <EOL> """<STR_LIT>""" <EOL> t = metadata . tables [ '<STR_LIT:version>' ] <EOL> session . execute ( t . insert ( ) , dict ( version = version ) ) <EOL> session . commit ( ) <EOL> def close_and_commit ( session ) : <EOL> """<STR_LIT>""" <EOL> session . commit ( ) <EOL> session . close ( ) <EOL> def empty_tables ( cursor ) : <EOL> cursor . execute ( '''<STR_LIT>''' ) <EOL> cursor . execute ( '''<STR_LIT>''' ) <EOL> def update_gene_summary_w_cancer_census ( session , metadata , genes ) : <EOL> tbl = metadata . tables [ '<STR_LIT>' ] <EOL> stmt = tbl . update ( ) . where ( sql . and_ ( tbl . c . gene == sql . bindparam ( "<STR_LIT>" ) , <EOL> tbl . c . chrom == sql . bindparam ( "<STR_LIT>" ) ) <EOL> ) . values ( in_cosmic_census = sql . bindparam ( "<STR_LIT>" ) ) <EOL> cols = ( "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" ) <EOL> session . execute ( stmt , [ dict ( zip ( cols , g ) ) for g in genes ] ) <EOL> def get_session_metadata ( path ) : <EOL> """<STR_LIT>""" <EOL> engine = sqlalchemy . create_engine ( get_path ( path ) , isolation_level = None ) <EOL> engine . connect ( ) . connection . connection . text_factory = str <EOL> engine . raw_connection ( ) . connection . text_factory = str <EOL> metadata = sql . MetaData ( bind = engine ) <EOL> metadata . reflect ( bind = engine ) <EOL> session = create_session ( bind = engine , autocommit = False , autoflush = False ) <EOL> session . bind . connect ( ) . connection . connection . text_factory = str <EOL> session . bind . raw_connection ( ) . connection . text_factory = str <EOL> return session , metadata <EOL> @ contextlib . contextmanager <EOL> def database_transaction ( db ) : <EOL> session , metadata = get_session_metadata ( db ) <EOL> conn = session . connection ( ) <EOL> if session . bind . name == "<STR_LIT>" : <EOL> conn . execute ( '<STR_LIT>' ) <EOL> conn . execute ( '<STR_LIT>' ) <EOL> yield conn <EOL> session . commit ( ) <EOL> conn . close ( ) </s>
