<s> DEBUG = False <EOL> import codecs <EOL> import sys <EOL> import os <EOL> import urllib <EOL> from string import Template <EOL> import collections <EOL> import MySQLdb <EOL> import pdb <EOL> try : <EOL> import conf <EOL> except : <EOL> raise <EOL> PREFIXES = """<STR_LIT>""" <EOL> ONTOLOGY_HEADER = Template ( """<STR_LIT>""" ) <EOL> STY_URL = "<STR_LIT>" <EOL> HAS_STY = "<STR_LIT>" <EOL> HAS_AUI = "<STR_LIT>" <EOL> HAS_CUI = "<STR_LIT>" <EOL> HAS_TUI = "<STR_LIT>" <EOL> MRCONSO_CODE = <NUM_LIT> <EOL> MRCONSO_AUI = <NUM_LIT:7> <EOL> MRCONSO_STR = <NUM_LIT> <EOL> MRCONSO_STT = <NUM_LIT:4> <EOL> MRCONSO_SCUI = <NUM_LIT:9> <EOL> MRCONSO_ISPREF = <NUM_LIT:6> <EOL> MRCONSO_TTY = <NUM_LIT:12> <EOL> MRCONSO_TS = <NUM_LIT:2> <EOL> MRCONSO_CUI = <NUM_LIT:0> <EOL> MRREL_AUI1 = <NUM_LIT:1> <EOL> MRREL_AUI2 = <NUM_LIT:5> <EOL> MRREL_CUI1 = <NUM_LIT:0> <EOL> MRREL_CUI2 = <NUM_LIT:4> <EOL> MRREL_REL = <NUM_LIT:3> <EOL> MRREL_RELA = <NUM_LIT:7> <EOL> MRDEF_AUI = <NUM_LIT:1> <EOL> MRDEF_DEF = <NUM_LIT:5> <EOL> MRDEF_CUI = <NUM_LIT:0> <EOL> MRSAT_CUI = <NUM_LIT:0> <EOL> MRSAT_CODE = <NUM_LIT:5> <EOL> MRSAT_ATV = <NUM_LIT:10> <EOL> MRSAT_ATN = <NUM_LIT:8> <EOL> MRDOC_DOCKEY = <NUM_LIT:0> <EOL> MRDOC_VALUE = <NUM_LIT:1> <EOL> MRDOC_TYPE = <NUM_LIT:2> <EOL> MRDOC_DESC = <NUM_LIT:3> <EOL> MRRANK_TTY = <NUM_LIT:2> <EOL> MRRANK_RANK = <NUM_LIT:0> <EOL> MRSTY_CUI = <NUM_LIT:0> <EOL> MRSTY_TUI = <NUM_LIT:1> <EOL> MRSAB_LAT = <NUM_LIT> <EOL> def get_umls_url ( code ) : <EOL> return "<STR_LIT>" % ( conf . UMLS_BASE_URI , code ) <EOL> def flatten ( matrix ) : <EOL> return reduce ( lambda x , y : x + y , matrix ) <EOL> def escape ( string ) : <EOL> return string . replace ( "<STR_LIT:\\>" , "<STR_LIT>" ) . replace ( '<STR_LIT:">' , '<STR_LIT>' ) <EOL> def get_url_term ( ns , code ) : <EOL> if ns [ - <NUM_LIT:1> ] == '<STR_LIT:/>' : <EOL> ret = ns + urllib . quote ( code ) <EOL> else : <EOL> ret = "<STR_LIT>" % ( ns , urllib . quote ( code ) ) <EOL> return ret <EOL> def get_rel_fragment ( rel ) : <EOL> return rel [ MRREL_RELA ] if rel [ MRREL_RELA ] else rel [ MRREL_REL ] <EOL> def get_rel_code_source ( rel , on_cuis ) : <EOL> return rel [ - <NUM_LIT:1> ] if not on_cuis else rel [ MRREL_CUI2 ] <EOL> def get_rel_code_target ( rel , on_cuis ) : <EOL> return rel [ - <NUM_LIT:2> ] if not on_cuis else rel [ MRREL_CUI1 ] <EOL> def get_code ( reg , load_on_cuis ) : <EOL> if load_on_cuis : <EOL> return reg [ MRCONSO_CUI ] <EOL> if reg [ MRCONSO_CODE ] : <EOL> return reg [ MRCONSO_CODE ] <EOL> raise AttributeError , "<STR_LIT>" % ( "<STR_LIT:|>" . join ( reg ) ) <EOL> def __get_connection ( ) : <EOL> return MySQLdb . connect ( host = conf . DB_HOST , user = conf . DB_USER , <EOL> passwd = conf . DB_PASS , db = conf . DB_NAME , charset = '<STR_LIT:utf8>' , use_unicode = True ) <EOL> def generate_semantic_types ( con , with_roots = False ) : <EOL> url = get_umls_url ( "<STR_LIT>" ) <EOL> hierarchy = collections . defaultdict ( lambda : list ( ) ) <EOL> all_nodes = list ( ) <EOL> mrsty = UmlsTable ( "<STR_LIT>" , con , <EOL> load_select = "<STR_LIT>" ) <EOL> ont = list ( ) <EOL> for stt in mrsty . scan ( ) : <EOL> hierarchy [ stt [ <NUM_LIT:1> ] ] . append ( stt [ <NUM_LIT:0> ] ) <EOL> sty_term = """<STR_LIT>""" % ( url + stt [ <NUM_LIT:0> ] , stt [ <NUM_LIT:0> ] , stt [ <NUM_LIT:2> ] ) <EOL> ont . append ( sty_term ) <EOL> all_nodes . append ( stt ) <EOL> for node in all_nodes : <EOL> parent = node [ <NUM_LIT:1> ] <EOL> if "<STR_LIT:.>" in parent : <EOL> parent = "<STR_LIT:.>" . join ( node [ <NUM_LIT:1> ] . split ( "<STR_LIT:.>" ) [ <NUM_LIT:0> : - <NUM_LIT:1> ] ) <EOL> else : <EOL> parent = parent [ <NUM_LIT:0> : - <NUM_LIT:1> ] <EOL> rdfs_subclasses = [ ] <EOL> for x in hierarchy [ parent ] : <EOL> if node [ <NUM_LIT:0> ] != x : <EOL> rdfs_subclasses . append ( <EOL> "<STR_LIT>" % ( url + node [ <NUM_LIT:0> ] , url + x ) ) <EOL> if len ( rdfs_subclasses ) == <NUM_LIT:0> and with_roots : <EOL> rdfs_subclasses = [ "<STR_LIT>" % ( url + node [ <NUM_LIT:0> ] ) ] <EOL> for sc in rdfs_subclasses : <EOL> ont . append ( sc ) <EOL> data_ont_ttl = "<STR_LIT:\n>" . join ( ont ) <EOL> return data_ont_ttl <EOL> class UmlsTable ( object ) : <EOL> def __init__ ( self , table_name , conn , load_select = None ) : <EOL> self . table_name = table_name <EOL> self . conn = conn <EOL> self . page_size = <NUM_LIT> <EOL> self . load_select = load_select <EOL> def mesh_tree ( self ) : <EOL> q = """<STR_LIT>""" <EOL> cursor = self . conn . cursor ( ) <EOL> cursor . execute ( q ) <EOL> result = cursor . fetchall ( ) <EOL> edges = collections . defaultdict ( set ) <EOL> for record in result : <EOL> edges [ record [ <NUM_LIT:1> ] ] . add ( record [ <NUM_LIT:0> ] ) <EOL> return edges <EOL> def count ( self ) : <EOL> q = "<STR_LIT>" % self . table_name <EOL> cursor = self . conn . cursor ( ) <EOL> cursor . execute ( q ) <EOL> result = cursor . fetchall ( ) <EOL> for record in result : <EOL> cursor . close ( ) <EOL> return int ( record [ <NUM_LIT:0> ] ) <EOL> def scan ( self , filt = None , limit = None ) : <EOL> i = <NUM_LIT:0> <EOL> page = <NUM_LIT:0> <EOL> cont = True <EOL> cursor = self . conn . cursor ( ) <EOL> while cont : <EOL> if self . load_select : <EOL> q = self . load_select <EOL> else : <EOL> q = "<STR_LIT>" % ( self . table_name , filt , self . page_size , page * self . page_size ) <EOL> if filt == None or len ( filt ) == <NUM_LIT:0> : <EOL> q = "<STR_LIT>" % ( self . table_name , self . page_size , page * self . page_size ) <EOL> sys . stdout . write ( "<STR_LIT>" % q ) <EOL> sys . stdout . flush ( ) <EOL> cursor . execute ( q ) <EOL> result = cursor . fetchall ( ) <EOL> cont = False <EOL> for record in result : <EOL> cont = True <EOL> i += <NUM_LIT:1> <EOL> yield record <EOL> if limit and i >= limit : <EOL> cont = False <EOL> break <EOL> if self . load_select : <EOL> cont = False <EOL> elif not limit and i < self . page_size : <EOL> cont = False <EOL> page += <NUM_LIT:1> <EOL> cursor . close ( ) <EOL> class UmlsClass ( object ) : <EOL> def __init__ ( self , ns , atoms = None , rels = None , <EOL> defs = None , atts = None , rank = None , <EOL> rank_by_tty = None , sty = None , <EOL> sty_by_cui = None , load_on_cuis = False , <EOL> is_root = None ) : <EOL> self . ns = ns <EOL> self . atoms = atoms <EOL> self . rels = rels <EOL> self . defs = defs <EOL> self . atts = atts <EOL> self . rank = rank <EOL> self . rank_by_tty = rank_by_tty <EOL> self . sty = sty <EOL> self . sty_by_cui = sty_by_cui <EOL> self . load_on_cuis = load_on_cuis <EOL> self . is_root = is_root <EOL> self . class_properties = dict ( ) <EOL> def code ( self ) : <EOL> codes = set ( [ get_code ( x , self . load_on_cuis ) for x in self . atoms ] ) <EOL> if len ( codes ) < > <NUM_LIT:1> : <EOL> raise AttributeError , "<STR_LIT>" <EOL> return codes . pop ( ) <EOL> def getAltLabels ( self , prefLabel ) : <EOL> return set ( [ atom [ MRCONSO_STR ] for atom in self . atoms if atom [ MRCONSO_STR ] < > prefLabel ] ) <EOL> def getPrefLabel ( self ) : <EOL> if self . load_on_cuis : <EOL> if len ( self . atoms ) == <NUM_LIT:1> : <EOL> return self . atoms [ <NUM_LIT:0> ] [ MRCONSO_STR ] <EOL> labels = set ( [ x [ MRCONSO_STR ] for x in self . atoms ] ) <EOL> if len ( labels ) == <NUM_LIT:1> : <EOL> return labels . pop ( ) <EOL> is_pref_atoms = filter ( lambda x : x [ MRCONSO_ISPREF ] == '<STR_LIT:Y>' , self . atoms ) <EOL> if len ( is_pref_atoms ) == <NUM_LIT:0> : <EOL> return self . atoms [ <NUM_LIT:0> ] [ MRCONSO_STR ] <EOL> elif len ( is_pref_atoms ) == <NUM_LIT:1> : <EOL> return is_pref_atoms [ <NUM_LIT:0> ] [ MRCONSO_STR ] <EOL> is_pref_atoms = filter ( lambda x : x [ MRCONSO_STT ] == '<STR_LIT>' , is_pref_atoms ) <EOL> if len ( is_pref_atoms ) == <NUM_LIT:0> : <EOL> return self . atoms [ <NUM_LIT:0> ] [ MRCONSO_STR ] <EOL> elif len ( is_pref_atoms ) == <NUM_LIT:1> : <EOL> return is_pref_atoms [ <NUM_LIT:0> ] [ MRCONSO_STR ] <EOL> is_pref_atoms = filter ( lambda x : x [ MRCONSO_TTY ] [ <NUM_LIT:0> ] == '<STR_LIT:P>' , self . atoms ) <EOL> if len ( is_pref_atoms ) == <NUM_LIT:1> : <EOL> return is_pref_atoms [ <NUM_LIT:0> ] [ MRCONSO_STR ] <EOL> return self . atoms [ <NUM_LIT:0> ] [ MRCONSO_STR ] <EOL> else : <EOL> if len ( self . rank ) > <NUM_LIT:0> : <EOL> sort_key = lambda x : int ( self . rank [ self . rank_by_tty [ x [ MRCONSO_TTY ] ] [ <NUM_LIT:0> ] ] [ MRRANK_RANK ] ) <EOL> mmrank_sorted_atoms = sorted ( self . atoms , key = sort_key , reverse = True ) <EOL> return mmrank_sorted_atoms [ <NUM_LIT:0> ] [ MRCONSO_STR ] <EOL> else : <EOL> pref_atom = filter ( lambda x : '<STR_LIT:P>' in x [ MRCONSO_TTY ] , self . atoms ) <EOL> if len ( pref_atom ) == <NUM_LIT:1> : <EOL> return pref_atom [ <NUM_LIT:0> ] [ MRCONSO_STR ] <EOL> raise AttributeError , "<STR_LIT>" <EOL> def getURLTerm ( self , code ) : <EOL> return get_url_term ( self . ns , code ) <EOL> def properties ( self ) : <EOL> return self . class_properties <EOL> def toRDF ( self , fmt = "<STR_LIT>" , hierarchy = True , lang = "<STR_LIT>" , tree = None ) : <EOL> if not fmt == "<STR_LIT>" : <EOL> raise AttributeError , "<STR_LIT>" <EOL> term_code = self . code ( ) <EOL> url_term = self . getURLTerm ( term_code ) <EOL> prefLabel = self . getPrefLabel ( ) <EOL> altLabels = self . getAltLabels ( prefLabel ) <EOL> rdf_term = """<STR_LIT>""" % ( url_term , escape ( prefLabel ) , lang , escape ( term_code ) ) <EOL> if len ( altLabels ) > <NUM_LIT:0> : <EOL> rdf_term += """<STR_LIT>""" % ( "<STR_LIT>" . join ( map ( lambda x : '<STR_LIT>' % ( escape ( x ) , lang ) , <EOL> set ( altLabels ) ) ) ) <EOL> if self . is_root : <EOL> rdf_term += '<STR_LIT>' <EOL> if len ( self . defs ) > <NUM_LIT:0> : <EOL> rdf_term += """<STR_LIT>""" % ( "<STR_LIT>" . join ( map ( lambda x : '<STR_LIT>' % ( escape ( x [ MRDEF_DEF ] ) , lang ) , <EOL> set ( self . defs ) ) ) ) <EOL> count_parents = <NUM_LIT:0> <EOL> if tree : <EOL> if term_code in tree : <EOL> for parent in tree [ term_code ] : <EOL> o = self . getURLTerm ( parent ) <EOL> rdf_term += "<STR_LIT>" % ( o , ) <EOL> for rel in self . rels : <EOL> source_code = get_rel_code_source ( rel , self . load_on_cuis ) <EOL> target_code = get_rel_code_target ( rel , self . load_on_cuis ) <EOL> if source_code < > term_code : <EOL> raise AttributeError , "<STR_LIT>" <EOL> if rel [ MRREL_REL ] == '<STR_LIT>' : <EOL> continue <EOL> if rel [ MRREL_REL ] == '<STR_LIT>' and hierarchy : <EOL> o = self . getURLTerm ( target_code ) <EOL> count_parents += <NUM_LIT:1> <EOL> if target_code == "<STR_LIT>" : <EOL> continue <EOL> if target_code == "<STR_LIT>" : <EOL> continue <EOL> if target_code == "<STR_LIT>" or target_code == "<STR_LIT>" : <EOL> continue <EOL> if not tree : <EOL> rdf_term += "<STR_LIT>" % ( o , ) <EOL> else : <EOL> p = self . getURLTerm ( get_rel_fragment ( rel ) ) <EOL> o = self . getURLTerm ( target_code ) <EOL> rdf_term += "<STR_LIT>" % ( p , o ) <EOL> if p not in self . class_properties : <EOL> self . class_properties [ p ] = UmlsAttribute ( p , get_rel_fragment ( rel ) ) <EOL> for att in self . atts : <EOL> atn = att [ MRSAT_ATN ] <EOL> atv = att [ MRSAT_ATV ] <EOL> if atn == '<STR_LIT>' : <EOL> continue <EOL> if tree and atn == "<STR_LIT>" and term_code . startswith ( "<STR_LIT:D>" ) : <EOL> if len ( atv . split ( "<STR_LIT:.>" ) ) == <NUM_LIT:1> : <EOL> rdf_term += "<STR_LIT>" <EOL> p = self . getURLTerm ( atn ) <EOL> rdf_term += "<STR_LIT>" % ( p , escape ( atv ) ) <EOL> if p not in self . class_properties : <EOL> self . class_properties [ p ] = UmlsAttribute ( p , atn ) <EOL> cuis = set ( [ x [ MRCONSO_CUI ] for x in self . atoms ] ) <EOL> sty_recs = flatten ( [ indexes for indexes in [ self . sty_by_cui [ cui ] for cui in cuis ] ] ) <EOL> types = [ self . sty [ index ] [ MRSTY_TUI ] for index in sty_recs ] <EOL> for t in cuis : <EOL> rdf_term += """<STR_LIT>""" % ( HAS_CUI , t ) <EOL> for t in set ( types ) : <EOL> rdf_term += """<STR_LIT>""" % ( HAS_TUI , t ) <EOL> for t in set ( types ) : <EOL> rdf_term += """<STR_LIT>""" % ( HAS_STY , get_umls_url ( "<STR_LIT>" ) + t ) <EOL> return rdf_term + "<STR_LIT>" <EOL> class UmlsAttribute ( object ) : <EOL> def __init__ ( self , uri , att ) : <EOL> self . uri = uri <EOL> self . att = att <EOL> def getURLTerm ( self , code ) : <EOL> return get_url_term ( self . ns , code ) <EOL> def toRDFWithDesc ( self , label , desc , _type ) : <EOL> uri_rdf = self . uri <EOL> if self . uri . startswith ( "<STR_LIT:http>" ) : <EOL> uri_rdf = "<STR_LIT>" % self . uri <EOL> return """<STR_LIT>""" % ( uri_rdf , _type , label , escape ( desc ) ) <EOL> def toRDF ( self , dockey , desc , fmt = "<STR_LIT>" ) : <EOL> if not fmt == "<STR_LIT>" : <EOL> raise AttributeError , "<STR_LIT>" <EOL> _type = "<STR_LIT>" <EOL> if "<STR_LIT>" in dockey : <EOL> _type = "<STR_LIT>" <EOL> elif dockey == "<STR_LIT>" : <EOL> _type = "<STR_LIT>" <EOL> else : <EOL> raise AttributeError , ( "<STR_LIT>" + dockey ) <EOL> label = self . att <EOL> if len ( desc ) < <NUM_LIT:20> : <EOL> label = desc <EOL> if "<STR_LIT:_>" in label : <EOL> label = "<STR_LIT:U+0020>" . join ( self . att . split ( "<STR_LIT:_>" ) ) <EOL> label = label [ <NUM_LIT:0> ] . upper ( ) + label [ <NUM_LIT:1> : ] <EOL> return """<STR_LIT>""" % ( self . uri , _type , label , escape ( desc ) ) <EOL> class UmlsOntology ( object ) : <EOL> def __init__ ( self , ont_code , ns , con , load_on_cuis = False ) : <EOL> self . loaded = False <EOL> self . ont_code = ont_code <EOL> self . ns = ns <EOL> self . con = con <EOL> self . load_on_cuis = load_on_cuis <EOL> self . atoms = list ( ) <EOL> self . atoms_by_code = collections . defaultdict ( lambda : list ( ) ) <EOL> if not self . load_on_cuis : <EOL> self . atoms_by_aui = collections . defaultdict ( lambda : list ( ) ) <EOL> self . rels = list ( ) <EOL> self . rels_by_aui_src = collections . defaultdict ( lambda : list ( ) ) <EOL> self . defs = list ( ) <EOL> self . defs_by_aui = collections . defaultdict ( lambda : list ( ) ) <EOL> self . atts = list ( ) <EOL> self . atts_by_code = collections . defaultdict ( lambda : list ( ) ) <EOL> self . rank = list ( ) <EOL> self . rank_by_tty = collections . defaultdict ( lambda : list ( ) ) <EOL> self . sty = list ( ) <EOL> self . sty_by_cui = collections . defaultdict ( lambda : list ( ) ) <EOL> self . cui_roots = set ( ) <EOL> self . lang = None <EOL> self . ont_properties = dict ( ) <EOL> def load_tables ( self , limit = None ) : <EOL> mrconso = UmlsTable ( "<STR_LIT>" , self . con ) <EOL> if self . ont_code == '<STR_LIT>' : <EOL> self . tree = mrconso . mesh_tree ( ) <EOL> else : <EOL> self . tree = None <EOL> mrsab = UmlsTable ( "<STR_LIT>" , self . con ) <EOL> for sab_rec in mrsab . scan ( filt = "<STR_LIT>" + self . ont_code + "<STR_LIT:'>" , limit = <NUM_LIT:1> ) : <EOL> self . lang = sab_rec [ MRSAB_LAT ] . lower ( ) <EOL> mrconso_filt = "<STR_LIT>" % ( <EOL> self . ont_code , self . lang ) <EOL> for atom in mrconso . scan ( filt = mrconso_filt , limit = limit ) : <EOL> index = len ( self . atoms ) <EOL> self . atoms_by_code [ get_code ( atom , self . load_on_cuis ) ] . append ( index ) <EOL> if not self . load_on_cuis : <EOL> self . atoms_by_aui [ atom [ MRCONSO_AUI ] ] . append ( index ) <EOL> self . atoms . append ( atom ) <EOL> if DEBUG : <EOL> sys . stderr . write ( "<STR_LIT>" % len ( self . atoms ) ) <EOL> sys . stderr . write ( "<STR_LIT>" % len ( self . atoms_by_aui ) ) <EOL> sys . stderr . write ( "<STR_LIT>" % str ( self . atoms ) ) <EOL> sys . stderr . flush ( ) <EOL> mrconso_filt = "<STR_LIT>" % self . ont_code <EOL> for atom in mrconso . scan ( filt = mrconso_filt , limit = limit ) : <EOL> self . cui_roots . add ( atom [ MRCONSO_CUI ] ) <EOL> if DEBUG : <EOL> sys . stderr . write ( "<STR_LIT>" % len ( self . cui_roots ) ) <EOL> sys . stderr . flush ( ) <EOL> mrrel = UmlsTable ( "<STR_LIT>" , self . con ) <EOL> mrrel_filt = "<STR_LIT>" % self . ont_code <EOL> field = MRREL_AUI2 if not self . load_on_cuis else MRREL_CUI2 <EOL> for rel in mrrel . scan ( filt = mrrel_filt , limit = limit ) : <EOL> index = len ( self . rels ) <EOL> self . rels_by_aui_src [ rel [ field ] ] . append ( index ) <EOL> self . rels . append ( rel ) <EOL> if DEBUG : <EOL> sys . stderr . write ( "<STR_LIT>" % len ( self . rels ) ) <EOL> sys . stderr . flush ( ) <EOL> mrdef = UmlsTable ( "<STR_LIT>" , self . con ) <EOL> mrdef_filt = "<STR_LIT>" % self . ont_code <EOL> field = MRDEF_AUI if not self . load_on_cuis else MRDEF_CUI <EOL> for defi in mrdef . scan ( filt = mrdef_filt ) : <EOL> index = len ( self . defs ) <EOL> self . defs_by_aui [ defi [ field ] ] . append ( index ) <EOL> self . defs . append ( defi ) <EOL> if DEBUG : <EOL> sys . stderr . write ( "<STR_LIT>" % len ( self . defs ) ) <EOL> sys . stderr . flush ( ) <EOL> mrsat = UmlsTable ( "<STR_LIT>" , self . con ) <EOL> mrsat_filt = "<STR_LIT>" % self . ont_code <EOL> field = MRSAT_CODE if not self . load_on_cuis else MRSAT_CUI <EOL> for att in mrsat . scan ( filt = mrsat_filt ) : <EOL> index = len ( self . atts ) <EOL> self . atts_by_code [ att [ field ] ] . append ( index ) <EOL> self . atts . append ( att ) <EOL> if DEBUG : <EOL> sys . stderr . write ( "<STR_LIT>" % len ( self . atts ) ) <EOL> sys . stderr . flush ( ) <EOL> mrrank = UmlsTable ( "<STR_LIT>" , self . con ) <EOL> mrrank_filt = "<STR_LIT>" % self . ont_code <EOL> for rank in mrrank . scan ( filt = mrrank_filt ) : <EOL> index = len ( self . rank ) <EOL> self . rank_by_tty [ rank [ MRRANK_TTY ] ] . append ( index ) <EOL> self . rank . append ( rank ) <EOL> if DEBUG : <EOL> sys . stderr . write ( "<STR_LIT>" % len ( self . rank ) ) <EOL> sys . stderr . flush ( ) <EOL> load_mrsty = "<STR_LIT>" <EOL> load_mrsty %= self . ont_code <EOL> mrsty = UmlsTable ( "<STR_LIT>" , self . con , load_select = load_mrsty ) <EOL> for sty in mrsty . scan ( filt = None ) : <EOL> index = len ( self . sty ) <EOL> self . sty_by_cui [ sty [ MRSTY_CUI ] ] . append ( index ) <EOL> self . sty . append ( sty ) <EOL> if DEBUG : <EOL> sys . stderr . write ( "<STR_LIT>" % len ( self . sty ) ) <EOL> sys . stderr . flush ( ) <EOL> self . loaded = True <EOL> sys . stdout . write ( "<STR_LIT>" % self . ont_code ) <EOL> sys . stdout . flush ( ) <EOL> def terms ( self ) : <EOL> if not self . loaded : <EOL> self . load_tables ( ) <EOL> for code in self . atoms_by_code : <EOL> code_atoms = [ self . atoms [ row ] for row in self . atoms_by_code [ code ] ] <EOL> field = MRCONSO_CUI if self . load_on_cuis else MRCONSO_AUI <EOL> ids = map ( lambda x : x [ field ] , code_atoms ) <EOL> rels = list ( ) <EOL> for _id in ids : <EOL> rels += [ self . rels [ x ] for x in self . rels_by_aui_src [ _id ] ] <EOL> rels_to_class = list ( ) <EOL> is_root = False <EOL> if self . load_on_cuis : <EOL> rels_to_class = rels <EOL> for rel in rels_to_class : <EOL> if rel [ MRREL_CUI1 ] in self . cui_roots : <EOL> is_root = True <EOL> break <EOL> else : <EOL> for rel in rels : <EOL> rel_with_codes = list ( rel ) <EOL> aui_source = rel [ MRREL_AUI2 ] <EOL> aui_target = rel [ MRREL_AUI1 ] <EOL> code_source = [ get_code ( self . atoms [ x ] , self . load_on_cuis ) for x in self . atoms_by_aui [ aui_source ] ] <EOL> code_target = [ get_code ( self . atoms [ x ] , self . load_on_cuis ) for x in self . atoms_by_aui [ aui_target ] ] <EOL> if ( rel [ MRREL_CUI1 ] in self . cui_roots ) and rel [ MRREL_REL ] == "<STR_LIT>" : <EOL> is_root = True <EOL> elif self . ont_code == "<STR_LIT>" : <EOL> if rel [ MRREL_CUI1 ] == "<STR_LIT>" and rel [ MRREL_REL ] == "<STR_LIT>" : <EOL> is_root = True <EOL> if len ( code_source ) < > <NUM_LIT:1> or len ( code_target ) > <NUM_LIT:1> : <EOL> raise AttributeError , "<STR_LIT>" <EOL> if len ( code_source ) == <NUM_LIT:1> and len ( code_target ) == <NUM_LIT:1> and code_source [ <NUM_LIT:0> ] < > code_target [ <NUM_LIT:0> ] : <EOL> code_source = code_source [ <NUM_LIT:0> ] <EOL> code_target = code_target [ <NUM_LIT:0> ] <EOL> rel_with_codes . append ( code_target ) <EOL> rel_with_codes . append ( code_source ) <EOL> rels_to_class . append ( rel_with_codes ) <EOL> aui_codes_def = [ self . defs_by_aui [ _id ] for _id in ids ] <EOL> aui_codes_def = [ item for sublist in aui_codes_def for item in sublist ] <EOL> defs = [ self . defs [ index ] for index in aui_codes_def ] <EOL> atts = [ self . atts [ x ] for x in self . atts_by_code [ code ] ] <EOL> umls_class = UmlsClass ( self . ns , atoms = code_atoms , rels = rels_to_class , <EOL> defs = defs , atts = atts , rank = self . rank , rank_by_tty = self . rank_by_tty , <EOL> sty = self . sty , sty_by_cui = self . sty_by_cui , <EOL> load_on_cuis = self . load_on_cuis , is_root = is_root ) <EOL> yield umls_class <EOL> def write_into ( self , file_path , hierarchy = True ) : <EOL> sys . stdout . write ( "<STR_LIT>" % ( self . ont_code , file_path ) ) <EOL> sys . stdout . flush ( ) <EOL> fout = codecs . open ( file_path , "<STR_LIT:w>" , "<STR_LIT:utf-8>" ) <EOL> fout . write ( PREFIXES ) <EOL> comment = "<STR_LIT>" + "<STR_LIT>" + "<STR_LIT>" + "<STR_LIT>" <EOL> header_values = dict ( <EOL> label = self . ont_code , <EOL> comment = comment % self . ont_code , <EOL> versioninfo = conf . UMLS_VERSION , <EOL> uri = self . ns <EOL> ) <EOL> fout . write ( ONTOLOGY_HEADER . substitute ( header_values ) ) <EOL> for term in self . terms ( ) : <EOL> try : <EOL> rdf_text = term . toRDF ( lang = self . lang , tree = self . tree ) <EOL> fout . write ( rdf_text ) <EOL> except Exception , e : <EOL> print "<STR_LIT>" , e <EOL> for att in term . properties ( ) : <EOL> if att not in self . ont_properties : <EOL> self . ont_properties [ att ] = term . properties ( ) [ att ] <EOL> return fout <EOL> def properties ( self ) : <EOL> return self . ont_properties <EOL> def write_properties ( self , fout , property_docs ) : <EOL> self . ont_properties [ "<STR_LIT>" ] = UmlsAttribute ( HAS_STY , "<STR_LIT>" ) <EOL> for p in self . ont_properties : <EOL> prop = self . ont_properties [ p ] <EOL> if "<STR_LIT>" in p : <EOL> fout . write ( prop . toRDFWithDesc ( <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" ) ) <EOL> continue <EOL> doc = property_docs [ prop . att ] <EOL> if "<STR_LIT>" not in doc : <EOL> raise AttributeError , "<STR_LIT>" + doc <EOL> _desc = doc [ "<STR_LIT>" ] <EOL> if "<STR_LIT>" in doc : <EOL> _desc = "<STR_LIT>" + doc [ "<STR_LIT>" ] <EOL> _dockey = doc [ "<STR_LIT>" ] <EOL> fout . write ( prop . toRDF ( _dockey , _desc ) ) <EOL> def write_semantic_types ( self , sem_types , fout ) : <EOL> fout . write ( sem_types ) <EOL> fout . write ( "<STR_LIT:\n>" ) <EOL> if __name__ == "<STR_LIT:__main__>" : <EOL> con = __get_connection ( ) <EOL> umls_conf = None <EOL> with open ( "<STR_LIT>" , "<STR_LIT:r>" ) as fconf : <EOL> umls_conf = [ line . split ( "<STR_LIT:U+002C>" ) for line in fconf . read ( ) . splitlines ( ) if len ( line ) > <NUM_LIT:0> ] <EOL> umls_conf = filter ( lambda x : not x [ <NUM_LIT:0> ] . startswith ( "<STR_LIT:#>" ) , umls_conf ) <EOL> fconf . close ( ) <EOL> if not os . path . isdir ( conf . OUTPUT_FOLDER ) : <EOL> raise Exception ( "<STR_LIT>" % conf . OUTPUT_FOLDER ) <EOL> sem_types = generate_semantic_types ( con , with_roots = True ) <EOL> output_file = os . path . join ( conf . OUTPUT_FOLDER , "<STR_LIT>" ) <EOL> with codecs . open ( output_file , "<STR_LIT:w>" , "<STR_LIT:utf-8>" ) as semfile : <EOL> semfile . write ( PREFIXES ) <EOL> semfile . write ( sem_types ) <EOL> semfile . flush ( ) <EOL> semfile . close ( ) <EOL> sem_types = generate_semantic_types ( con , with_roots = False ) <EOL> mrdoc = UmlsTable ( "<STR_LIT>" , con ) <EOL> property_docs = dict ( ) <EOL> for doc_record in mrdoc . scan ( ) : <EOL> _type = doc_record [ MRDOC_TYPE ] <EOL> _expl = doc_record [ MRDOC_DESC ] <EOL> _key = doc_record [ MRDOC_VALUE ] <EOL> if _key not in property_docs : <EOL> property_docs [ _key ] = dict ( ) <EOL> property_docs [ _key ] [ "<STR_LIT>" ] = doc_record [ MRDOC_DOCKEY ] <EOL> if "<STR_LIT>" in _type : <EOL> _type = "<STR_LIT>" <EOL> property_docs [ _key ] [ _type ] = _expl <EOL> for ( umls_code , file_out , load_on_field ) in umls_conf : <EOL> alt_uri_code = None <EOL> if "<STR_LIT:;>" in umls_code : <EOL> umls_code , alt_uri_code = umls_code . split ( "<STR_LIT:;>" ) <EOL> if umls_code . startswith ( "<STR_LIT:#>" ) : <EOL> continue <EOL> load_on_cuis = load_on_field == "<STR_LIT>" <EOL> output_file = os . path . join ( conf . OUTPUT_FOLDER , file_out ) <EOL> sys . stdout . write ( "<STR_LIT>" % <EOL> ( umls_code , load_on_field ) ) <EOL> sys . stdout . flush ( ) <EOL> ns = get_umls_url ( umls_code if not alt_uri_code else alt_uri_code ) <EOL> ont = UmlsOntology ( umls_code , ns , con , load_on_cuis = load_on_cuis ) <EOL> ont . load_tables ( ) <EOL> fout = ont . write_into ( output_file , hierarchy = ( ont . ont_code < > "<STR_LIT>" ) ) <EOL> ont . write_properties ( fout , property_docs ) <EOL> ont . write_semantic_types ( sem_types , fout ) <EOL> fout . close ( ) <EOL> sys . stdout . write ( "<STR_LIT>" ) <EOL> sys . stdout . flush ( ) <EOL> sys . stdout . flush ( ) </s>
