@files( ((None, "mgi.import"),))
def importMGIPhenotypesViaReports( infile, outfile ):

    mgi_dir = "mgi"

    if not os.path.exists( mgi_dir ):
        os.mkdir (mgi_dir)

    conversion = \
        (
        # marker/allele information
        { "table": "mgi_marker2allele", "filename" : "MGI_PhenotypicAllele.rpt", 
          "columns" : ( "allele_id", "allele_symbol", "allele_name", "allele_type", "pmid", 
                        "marker_id", None, None, None, None ),
          "separator" : " " },

        # ensembl to marker
        { "table": "mgi_marker2ensembl", "filename" : "MRK_ENSEMBL.rpt", 
          "columns" : ("marker_id", None, None, None, None, "gene_id" ),
          "separator" : " " },

        # map of alleles to phenotypes
        { "table": "mgi_allele2phenotype", "filename" : "MGI_PhenoGenoMP.rpt", 
          "columns" : ( "allele_composition", "allele_symbol", "backgound", "phenotype_id", None, "marker_id" ) },

        # marker/allele information
        { "table": "mgi_marker2allele_qtl", "filename" : "MGI_QTLAllele.rpt", 
          "columns" : ( "allele_id", "allele_symbol", "allele_name", "allele_type", "pmid", 
                        "marker_id", None, None, None, None ) },

        # map of alleles to high-level phenotypes
        { "table": "mgi_allele2hl_phenotype", "filename" : "MGI_PhenotypicAllele.rpt", 
          "columns" : ( "allele_id", None, None, None, None,
                        None, None, None, None, "phenotype_id" ) },

        # phenotype information
        { "table": "mgi_phenotype", "filename" : "VOC_MammalianPhenotype.rpt", 
          "columns" : ( "phenotype_id", "brief", "description", ),
          "separator" : None },
        )

    # collect all filenames
    for filename in [x["filename"] for x in conversion]:
        target = os.path.join( mgi_dir, filename )
        if os.path.exists( target ): continue
        E.info( "downloading %s" % filename )
        statement = "wget -P %(mgi_dir)s ftp://ftp.informatics.jax.org/pub/reports/%(filename)s"
        P.run()
    
    for convert in conversion:
        tablename = convert["table"]

        tmpfile = P.getTempFile(".")
        headers, take = [], []
        for x,c in enumerate(convert["columns"]):
            if c != None:
                headers.append( c )
                take.append( x )

        tmpfile.write( "\t".join(headers) + "\n" )
        for line in open( os.path.join( mgi_dir, convert["filename"])):
            if line.startswith("#"): continue
            data = line[:-1].split("\t")
            tmpfile.write( "\t".join([data[x].strip() for x in take]) + "\n" )
            
        tmpfile.close()
        
        E.info( "creating table %s" % tablename )

        separator = convert.get( "separator", ",")

        tmpfilename = tmpfile.name 
        statement = '''
            cat < %(tmpfilename)s
            | python %(scriptsdir)s/table2table.py 
                     --separator="%(separator)s"
                     --split-fields
            | %(scriptsdir)s/hsort 1
            | uniq
            |python %(scriptsdir)s/csv2db.py %(csv2db_options)s 
                     --table=%(tablename)s
                     --index=marker_id
                     --index=allele_id
                     --index=gene_id
                     --map=allele_name:str
                     --index=phenotype_id
           > %(outfile)s
        '''

        P.run()

        os.unlink( tmpfilename )
MGI phenotype associations can be downloaded via biomart (1) from http://biomart.informatics.jax.org
dataset/ETHPy150Open CGATOxford/cgat/obsolete/pipeline_snps.py/importMGIPhenotypesViaReports